From 6afddcf43b352981996a43195af8243bca7ffeba Mon Sep 17 00:00:00 2001 From: John Coffey Date: Tue, 17 Nov 2026 08:13:05 -0700 Subject: [PATCH] Import upstream v1.1.0, renamed Upstream commit 08ccf5ddfe9bcdd973df6cab8f89374e454af091, renamed by scripts/upstream-import.py. --- CHANGELOG.md | 45 + Cargo.lock | 347 +- Cargo.toml | 47 +- README.md | 4 + src/archive/blobs.rs | 148 + src/archive/defaults.rs | 155 + src/archive/ids/dav.rs | 311 ++ src/archive/ids/ews.rs | 267 ++ src/archive/ids/graph.rs | 105 + src/archive/ids/imap.rs | 167 + src/archive/ids/jmap.rs | 85 + src/archive/ids/maildir.rs | 140 + src/archive/ids/mod.rs | 381 ++ src/archive/ids/sieve.rs | 139 + src/archive/ids/takeout.rs | 163 + src/archive/init.rs | 107 + src/archive/mod.rs | 188 + src/archive/rows/address_books.rs | 98 + src/archive/rows/calendar_events.rs | 115 + src/archive/rows/calendars.rs | 115 + src/archive/rows/contact_cards.rs | 60 + src/archive/rows/emails.rs | 193 + src/archive/rows/file_nodes.rs | 166 + src/archive/rows/identities.rs | 83 + src/archive/rows/mailboxes.rs | 196 ++ src/archive/rows/mod.rs | 217 ++ src/archive/rows/participant_identities.rs | 76 + src/archive/rows/sieve_scripts.rs | 195 ++ src/archive/rows/vacation.rs | 142 + src/{db => archive}/schema.sql | 0 src/archive/sources.rs | 286 ++ src/archive/state/imap.rs | 142 + src/archive/state/jmap.rs | 88 + .../state/mod.rs} | 6 +- src/cli.rs | 1616 --------- src/cli/auth.rs | 164 + src/cli/dav.rs | 156 + src/cli/ews.rs | 346 ++ src/cli/export.rs | 91 + src/cli/global.rs | 89 + src/cli/graph.rs | 277 ++ src/cli/imap.rs | 257 ++ src/cli/inspect.rs | 65 + src/cli/jmap.rs | 90 + src/cli/maildir.rs | 108 + src/cli/managesieve.rs | 105 + src/cli/mod.rs | 290 ++ src/cli/report.rs | 307 ++ src/cli/takeout.rs | 65 + src/console/logo.rs | 80 + src/console/logo/compact.ans | 7 + src/console/logo/compact.txt | 7 + src/console/logo/large.ans | 8 + src/console/logo/large.txt | 8 + src/console/mod.rs | 349 ++ src/console/table.rs | 208 ++ src/console/units.rs | 143 + src/dav/client.rs | 981 ------ src/dav/client/buffered.rs | 187 + src/dav/client/mod.rs | 280 ++ src/dav/client/stream.rs | 255 ++ src/dav/client/wire.rs | 153 + src/dav/discover.rs | 398 +-- src/dav/error.rs | 113 + src/dav/href.rs | 224 +- src/dav/mod.rs | 1 + src/dav/parse.rs | 981 +++--- src/dav/retry.rs | 118 +- src/dav/xml.rs | 341 +- src/db/blobs.rs | 131 - src/db/dav_ids.rs | 336 -- src/db/defaults.rs | 112 - src/db/exchange_ews_ids.rs | 265 -- src/db/exchange_graph_ids.rs | 168 - src/db/ids.rs | 114 - src/db/imap_ids.rs | 292 -- src/db/imap_state.rs | 110 - src/db/init.rs | 107 - src/db/maildir_ids.rs | 269 -- src/db/managesieve_ids.rs | 129 - src/db/mod.rs | 22 - src/db/roles.rs | 92 - src/db/sources.rs | 142 - src/db/sync_state_jmap.rs | 87 - src/db/takeout_ids.rs | 177 - src/db/vacation_response.rs | 66 - src/error.rs | 134 +- src/exchange/address.rs | 205 +- src/exchange/date.rs | 277 +- src/exchange/ews/autodiscover.rs | 421 +++ .../ews}/calendar_map.rs | 1230 ++++--- src/exchange/ews/client.rs | 530 +++ src/exchange/ews/contact_map.rs | 550 +++ src/{exchange_ews => exchange/ews}/error.rs | 29 +- src/{exchange_ews => exchange/ews}/mod.rs | 3 +- src/exchange/ews/oauth.rs | 114 + src/exchange/ews/parse/attachment.rs | 246 ++ src/exchange/ews/parse/calendar.rs | 599 ++++ src/exchange/ews/parse/contact.rs | 427 +++ src/exchange/ews/parse/envelope.rs | 229 ++ src/exchange/ews/parse/folder.rs | 291 ++ src/exchange/ews/parse/item.rs | 376 ++ src/exchange/ews/parse/message.rs | 191 + src/exchange/ews/parse/mod.rs | 198 ++ src/exchange/ews/parse/recurrence.rs | 194 ++ src/exchange/ews/parse/sync.rs | 231 ++ src/exchange/ews/parse/tests.rs | 832 +++++ .../ews}/recurrence.rs | 342 +- src/exchange/ews/retry.rs | 170 + src/{exchange_ews => exchange/ews}/soap.rs | 95 +- src/exchange/ews/srv.rs | 283 ++ src/{exchange_ews => exchange/ews}/types.rs | 184 +- src/exchange/ews/xml.rs | 642 ++++ src/{exchange_graph => exchange/graph}/api.rs | 193 +- src/exchange/graph/calendar_map.rs | 1037 ++++++ src/exchange/graph/client.rs | 384 ++ src/exchange/graph/contact_map.rs | 657 ++++ .../graph}/error.rs | 18 + src/exchange/graph/expand.rs | 647 ++++ src/{exchange_graph => exchange/graph}/mod.rs | 0 src/exchange/graph/oauth.rs | 39 + src/exchange/graph/recurrence.rs | 374 ++ src/exchange/graph/retry.rs | 93 + .../graph}/types.rs | 86 +- src/exchange/jscalendar.rs | 209 +- src/exchange/mailbox.rs | 113 + src/exchange/mod.rs | 5 + src/exchange/oauth/mod.rs | 465 +++ src/exchange/oauth/refresh.rs | 182 + src/exchange/oauth/token.rs | 308 ++ src/exchange/tz.rs | 425 +-- src/exchange/vocab.rs | 387 +++ src/exchange_ews/autodiscover.rs | 578 --- src/exchange_ews/client.rs | 608 ---- src/exchange_ews/contact_map.rs | 563 --- src/exchange_ews/oauth.rs | 386 -- src/exchange_ews/parse.rs | 3093 ----------------- src/exchange_ews/retry.rs | 181 - src/exchange_ews/xml.rs | 519 --- src/exchange_graph/calendar_map.rs | 1017 ------ src/exchange_graph/client.rs | 499 --- src/exchange_graph/contact_map.rs | 669 ---- src/exchange_graph/expand.rs | 637 ---- src/exchange_graph/oauth.rs | 565 --- src/exchange_graph/recurrence.rs | 319 -- src/exchange_graph/retry.rs | 78 - src/{sync => }/export/blobpool.rs | 76 +- src/export/email.rs | 562 +++ src/export/email/import.rs | 138 + src/export/flat.rs | 190 + src/export/keyed.rs | 83 + src/export/mod.rs | 172 + src/export/prune.rs | 153 + src/export/run.rs | 319 ++ src/export/sieve.rs | 162 + src/export/target.rs | 192 + src/export/tree.rs | 419 +++ src/export/uidtype.rs | 189 + src/export/uploader.rs | 88 + src/export/vacation.rs | 40 + src/imap/automap.rs | 203 +- src/imap/client.rs | 749 ++-- src/imap/command.rs | 764 ++-- src/imap/error.rs | 43 +- src/imap/mod.rs | 4 +- src/imap/name.rs | 137 +- src/imap/response.rs | 1207 ------- src/imap/response/lexer.rs | 278 ++ src/imap/response/mod.rs | 642 ++++ src/imap/response/parser.rs | 435 +++ src/imap/retry.rs | 358 +- src/imap/session.rs | 188 + src/imap/transport.rs | 434 --- src/import/dav/collections.rs | 495 +++ src/import/dav/coordinator.rs | 486 +++ src/import/dav/items.rs | 581 ++++ .../import_maildir.rs => import/dav/mod.rs} | 9 +- src/import/dav/replace.rs | 314 ++ src/import/dav/tree.rs | 723 ++++ src/import/ews/attachments.rs | 61 + src/import/ews/calendar.rs | 171 + src/import/ews/contacts.rs | 104 + src/import/ews/coordinator.rs | 602 ++++ src/import/ews/folder_sync.rs | 230 ++ src/import/ews/folders.rs | 777 +++++ src/import/ews/items.rs | 588 ++++ src/import/ews/messages.rs | 210 ++ .../ews/mod.rs} | 4 +- src/import/graph/calendar.rs | 308 ++ src/import/graph/contacts.rs | 264 ++ src/import/graph/coordinator.rs | 515 +++ src/import/graph/files.rs | 302 ++ src/import/graph/folders.rs | 347 ++ src/import/graph/messages.rs | 215 ++ .../graph/mod.rs} | 3 +- src/import/graph/series.rs | 544 +++ src/import/imap/control.rs | 183 + src/import/imap/dry_run.rs | 112 + .../import_imap => import/imap}/fetch.rs | 79 +- .../import_imap => import/imap}/folders.rs | 547 ++- src/import/imap/internaldate.rs | 187 + src/import/imap/keywords.rs | 185 + src/import/imap/messages.rs | 57 + src/import/imap/mod.rs | 51 + src/import/imap/pool.rs | 211 ++ src/import/imap/reconcile.rs | 316 ++ src/import/imap/run.rs | 358 ++ src/import/jmap/blobs.rs | 197 ++ src/import/jmap/kinds.rs | 85 + src/import/jmap/mod.rs | 248 ++ src/import/jmap/reconcile.rs | 599 ++++ src/import/jmap/tree.rs | 224 ++ src/import/maildir/keywords.rs | 220 ++ src/import/maildir/messages.rs | 669 ++++ src/import/maildir/mod.rs | 28 + src/import/maildir/run.rs | 430 +++ .../import_maildir => import/maildir}/tree.rs | 439 ++- src/import/managesieve/control.rs | 101 + src/import/managesieve/mod.rs | 21 + src/import/managesieve/plan.rs | 431 +++ src/import/managesieve/run.rs | 228 ++ src/import/mod.rs | 51 + src/import/takeout/calendar.rs | 273 ++ src/import/takeout/contacts.rs | 215 ++ .../takeout}/labels.rs | 372 +- src/import/takeout/mail.rs | 362 ++ src/import/takeout/mbox.rs | 317 ++ src/import/takeout/mod.rs | 25 + src/import/takeout/run.rs | 388 +++ .../import_takeout => import/takeout}/walk.rs | 170 +- src/inspect/list.rs | 1044 +++--- src/inspect/mod.rs | 406 ++- src/inspect/tree.rs | 551 +-- src/jmap/account.rs | 325 +- src/jmap/blob.rs | 330 +- src/jmap/blobxfer.rs | 49 +- src/jmap/connect.rs | 138 +- src/jmap/error.rs | 34 +- src/jmap/http.rs | 1009 +++--- src/jmap/inflight.rs | 37 +- src/jmap/mod.rs | 1 - src/jmap/request.rs | 1198 +++---- src/jmap/retry.rs | 343 -- src/jmap/session.rs | 288 +- src/jmap/wire/address_book.rs | 7 +- src/jmap/wire/calendar.rs | 323 +- src/jmap/wire/common.rs | 81 +- src/jmap/wire/email.rs | 4 +- src/jmap/wire/file_node.rs | 30 +- src/jmap/wire/mailbox.rs | 7 +- src/lib.rs | 12 +- src/logging.rs | 490 ++- src/main.rs | 120 +- src/managesieve/client.rs | 467 +-- src/managesieve/command.rs | 125 +- src/managesieve/error.rs | 70 +- src/managesieve/mod.rs | 8 +- src/managesieve/name.rs | 78 +- src/managesieve/response.rs | 681 ++-- src/managesieve/retry.rs | 104 +- src/managesieve/session.rs | 205 ++ src/net/auth.rs | 31 + src/net/backoff.rs | 287 ++ src/net/encode.rs | 179 + src/net/http.rs | 454 +++ src/net/mod.rs | 14 + src/net/retry.rs | 435 +++ src/net/sasl.rs | 113 + src/net/stream.rs | 524 +++ src/net/xml.rs | 92 + src/secret.rs | 121 +- src/sync/calcard.rs | 424 +++ src/sync/dates.rs | 79 + src/sync/diff.rs | 123 + src/sync/emailmeta.rs | 280 +- src/sync/export.rs | 605 ---- src/sync/export/email.rs | 584 ---- src/sync/export/flat.rs | 164 - src/sync/export/keyed.rs | 99 - src/sync/export/sieve.rs | 157 - src/sync/export/tree.rs | 474 --- src/sync/export/uidtype.rs | 174 - src/sync/export/vacation.rs | 50 - src/sync/import_dav.rs | 58 - src/sync/import_dav/calcard.rs | 396 --- src/sync/import_dav/collections.rs | 374 -- src/sync/import_dav/coordinator.rs | 632 ---- src/sync/import_dav/items.rs | 653 ---- src/sync/import_dav/replace.rs | 255 -- src/sync/import_dav/tree.rs | 681 ---- src/sync/import_exchange_ews/attachments.rs | 79 - src/sync/import_exchange_ews/calendar.rs | 333 -- src/sync/import_exchange_ews/contacts.rs | 237 -- src/sync/import_exchange_ews/coordinator.rs | 541 --- src/sync/import_exchange_ews/folders.rs | 1024 ------ src/sync/import_exchange_ews/items.rs | 628 ---- src/sync/import_exchange_ews/messages.rs | 386 -- src/sync/import_exchange_graph/calendar.rs | 807 ----- src/sync/import_exchange_graph/contacts.rs | 289 -- src/sync/import_exchange_graph/coordinator.rs | 529 --- src/sync/import_exchange_graph/files.rs | 438 --- src/sync/import_exchange_graph/folders.rs | 581 ---- src/sync/import_exchange_graph/messages.rs | 246 -- src/sync/import_imap.rs | 16 - src/sync/import_imap/coordinator.rs | 1294 ------- src/sync/import_imap/internaldate.rs | 152 - src/sync/import_imap/keywords.rs | 173 - src/sync/import_imap/messages.rs | 158 - src/sync/import_imap/pool.rs | 258 -- src/sync/import_jmap.rs | 1370 -------- src/sync/import_jmap/diff.rs | 88 - src/sync/import_jmap/mapping.rs | 1844 ---------- src/sync/import_jmap/tree.rs | 131 - src/sync/import_maildir/coordinator.rs | 527 --- src/sync/import_maildir/keywords.rs | 200 -- src/sync/import_maildir/messages.rs | 801 ----- src/sync/import_managesieve/coordinator.rs | 639 ---- src/sync/import_managesieve/reconcile.rs | 480 --- src/sync/import_takeout.rs | 16 - src/sync/import_takeout/calendar.rs | 309 -- src/sync/import_takeout/contacts.rs | 253 -- src/sync/import_takeout/coordinator.rs | 420 --- src/sync/import_takeout/mail.rs | 416 --- src/sync/import_takeout/mbox.rs | 278 -- src/sync/import_takeout/tree.rs | 210 -- src/sync/jmap_map/address_book.rs | 46 + src/sync/jmap_map/calendar.rs | 65 + src/sync/jmap_map/calendar_event.rs | 78 + src/sync/jmap_map/contact_card.rs | 75 + src/sync/jmap_map/email.rs | 93 + src/sync/jmap_map/file_node.rs | 136 + src/sync/jmap_map/identity.rs | 54 + src/sync/jmap_map/mailbox.rs | 68 + src/sync/jmap_map/mod.rs | 220 ++ src/sync/jmap_map/participant_identity.rs | 28 + src/sync/jmap_map/sieve_script.rs | 54 + src/sync/jmap_map/tests.rs | 1247 +++++++ src/sync/jmap_map/vacation.rs | 34 + src/sync/keys.rs | 330 +- src/sync/keywords.rs | 196 ++ src/sync/mod.rs | 271 +- src/sync/{import_jmap => }/pool.rs | 33 - src/sync/prune.rs | 124 - src/sync/run.rs | 429 +++ src/sync/tree.rs | 199 ++ src/sync/vanish.rs | 326 ++ src/types.rs | 228 +- tests/common/line_server.rs | 93 + tests/common/mod.rs | 238 ++ tests/integration/baikal.rs | 115 +- tests/integration/cyrus.rs | 210 +- tests/integration/dovecot.rs | 266 +- tests/integration/imap_client.rs | 207 ++ tests/integration/layouts.rs | 555 +++ tests/integration/radicale.rs | 114 +- tests/integration/validate.rs | 15 +- tests/integration_baikal.rs | 268 +- tests/integration_cyrus.rs | 320 +- tests/integration_dovecot.rs | 376 +- tests/integration_radicale.rs | 238 +- tests/integration_webdav.rs | 35 +- tests/live_stalwart.rs | 28 +- tests/mock_dav.rs | 733 ++-- tests/mock_exchange_ews.rs | 571 ++- tests/mock_exchange_graph.rs | 1298 ++++--- tests/mock_imap.rs | 619 ++-- tests/mock_jmap.rs | 332 +- tests/mock_maildir.rs | 298 +- tests/mock_managesieve.rs | 414 +-- tests/mock_sync.rs | 1507 ++++---- tests/sync_caldav.rs | 224 +- tests/sync_carddav.rs | 224 +- tests/sync_exchange_graph.rs | 104 +- tests/sync_imap.rs | 305 +- tests/sync_jmap.rs | 538 +-- tests/sync_maildir.rs | 285 +- tests/sync_managesieve.rs | 218 +- tests/sync_takeout.rs | 266 +- tests/sync_webdav.rs | 190 +- 379 files changed, 60989 insertions(+), 53055 deletions(-) create mode 100644 src/archive/blobs.rs create mode 100644 src/archive/defaults.rs create mode 100644 src/archive/ids/dav.rs create mode 100644 src/archive/ids/ews.rs create mode 100644 src/archive/ids/graph.rs create mode 100644 src/archive/ids/imap.rs create mode 100644 src/archive/ids/jmap.rs create mode 100644 src/archive/ids/maildir.rs create mode 100644 src/archive/ids/mod.rs create mode 100644 src/archive/ids/sieve.rs create mode 100644 src/archive/ids/takeout.rs create mode 100644 src/archive/init.rs create mode 100644 src/archive/mod.rs create mode 100644 src/archive/rows/address_books.rs create mode 100644 src/archive/rows/calendar_events.rs create mode 100644 src/archive/rows/calendars.rs create mode 100644 src/archive/rows/contact_cards.rs create mode 100644 src/archive/rows/emails.rs create mode 100644 src/archive/rows/file_nodes.rs create mode 100644 src/archive/rows/identities.rs create mode 100644 src/archive/rows/mailboxes.rs create mode 100644 src/archive/rows/mod.rs create mode 100644 src/archive/rows/participant_identities.rs create mode 100644 src/archive/rows/sieve_scripts.rs create mode 100644 src/archive/rows/vacation.rs rename src/{db => archive}/schema.sql (100%) create mode 100644 src/archive/sources.rs create mode 100644 src/archive/state/imap.rs create mode 100644 src/archive/state/jmap.rs rename src/{sync/import_managesieve.rs => archive/state/mod.rs} (52%) delete mode 100644 src/cli.rs create mode 100644 src/cli/auth.rs create mode 100644 src/cli/dav.rs create mode 100644 src/cli/ews.rs create mode 100644 src/cli/export.rs create mode 100644 src/cli/global.rs create mode 100644 src/cli/graph.rs create mode 100644 src/cli/imap.rs create mode 100644 src/cli/inspect.rs create mode 100644 src/cli/jmap.rs create mode 100644 src/cli/maildir.rs create mode 100644 src/cli/managesieve.rs create mode 100644 src/cli/mod.rs create mode 100644 src/cli/report.rs create mode 100644 src/cli/takeout.rs create mode 100644 src/console/logo.rs create mode 100644 src/console/logo/compact.ans create mode 100644 src/console/logo/compact.txt create mode 100644 src/console/logo/large.ans create mode 100644 src/console/logo/large.txt create mode 100644 src/console/mod.rs create mode 100644 src/console/table.rs create mode 100644 src/console/units.rs delete mode 100644 src/dav/client.rs create mode 100644 src/dav/client/buffered.rs create mode 100644 src/dav/client/mod.rs create mode 100644 src/dav/client/stream.rs create mode 100644 src/dav/client/wire.rs create mode 100644 src/dav/error.rs delete mode 100644 src/db/blobs.rs delete mode 100644 src/db/dav_ids.rs delete mode 100644 src/db/defaults.rs delete mode 100644 src/db/exchange_ews_ids.rs delete mode 100644 src/db/exchange_graph_ids.rs delete mode 100644 src/db/ids.rs delete mode 100644 src/db/imap_ids.rs delete mode 100644 src/db/imap_state.rs delete mode 100644 src/db/init.rs delete mode 100644 src/db/maildir_ids.rs delete mode 100644 src/db/managesieve_ids.rs delete mode 100644 src/db/mod.rs delete mode 100644 src/db/roles.rs delete mode 100644 src/db/sources.rs delete mode 100644 src/db/sync_state_jmap.rs delete mode 100644 src/db/takeout_ids.rs delete mode 100644 src/db/vacation_response.rs create mode 100644 src/exchange/ews/autodiscover.rs rename src/{exchange_ews => exchange/ews}/calendar_map.rs (55%) create mode 100644 src/exchange/ews/client.rs create mode 100644 src/exchange/ews/contact_map.rs rename src/{exchange_ews => exchange/ews}/error.rs (78%) rename src/{exchange_ews => exchange/ews}/mod.rs (94%) create mode 100644 src/exchange/ews/oauth.rs create mode 100644 src/exchange/ews/parse/attachment.rs create mode 100644 src/exchange/ews/parse/calendar.rs create mode 100644 src/exchange/ews/parse/contact.rs create mode 100644 src/exchange/ews/parse/envelope.rs create mode 100644 src/exchange/ews/parse/folder.rs create mode 100644 src/exchange/ews/parse/item.rs create mode 100644 src/exchange/ews/parse/message.rs create mode 100644 src/exchange/ews/parse/mod.rs create mode 100644 src/exchange/ews/parse/recurrence.rs create mode 100644 src/exchange/ews/parse/sync.rs create mode 100644 src/exchange/ews/parse/tests.rs rename src/{exchange_ews => exchange/ews}/recurrence.rs (55%) create mode 100644 src/exchange/ews/retry.rs rename src/{exchange_ews => exchange/ews}/soap.rs (51%) create mode 100644 src/exchange/ews/srv.rs rename src/{exchange_ews => exchange/ews}/types.rs (80%) create mode 100644 src/exchange/ews/xml.rs rename src/{exchange_graph => exchange/graph}/api.rs (72%) create mode 100644 src/exchange/graph/calendar_map.rs create mode 100644 src/exchange/graph/client.rs create mode 100644 src/exchange/graph/contact_map.rs rename src/{exchange_graph => exchange/graph}/error.rs (76%) create mode 100644 src/exchange/graph/expand.rs rename src/{exchange_graph => exchange/graph}/mod.rs (100%) create mode 100644 src/exchange/graph/oauth.rs create mode 100644 src/exchange/graph/recurrence.rs create mode 100644 src/exchange/graph/retry.rs rename src/{exchange_graph => exchange/graph}/types.rs (73%) create mode 100644 src/exchange/mailbox.rs create mode 100644 src/exchange/oauth/mod.rs create mode 100644 src/exchange/oauth/refresh.rs create mode 100644 src/exchange/oauth/token.rs create mode 100644 src/exchange/vocab.rs delete mode 100644 src/exchange_ews/autodiscover.rs delete mode 100644 src/exchange_ews/client.rs delete mode 100644 src/exchange_ews/contact_map.rs delete mode 100644 src/exchange_ews/oauth.rs delete mode 100644 src/exchange_ews/parse.rs delete mode 100644 src/exchange_ews/retry.rs delete mode 100644 src/exchange_ews/xml.rs delete mode 100644 src/exchange_graph/calendar_map.rs delete mode 100644 src/exchange_graph/client.rs delete mode 100644 src/exchange_graph/contact_map.rs delete mode 100644 src/exchange_graph/expand.rs delete mode 100644 src/exchange_graph/oauth.rs delete mode 100644 src/exchange_graph/recurrence.rs delete mode 100644 src/exchange_graph/retry.rs rename src/{sync => }/export/blobpool.rs (63%) create mode 100644 src/export/email.rs create mode 100644 src/export/email/import.rs create mode 100644 src/export/flat.rs create mode 100644 src/export/keyed.rs create mode 100644 src/export/mod.rs create mode 100644 src/export/prune.rs create mode 100644 src/export/run.rs create mode 100644 src/export/sieve.rs create mode 100644 src/export/target.rs create mode 100644 src/export/tree.rs create mode 100644 src/export/uidtype.rs create mode 100644 src/export/uploader.rs create mode 100644 src/export/vacation.rs delete mode 100644 src/imap/response.rs create mode 100644 src/imap/response/lexer.rs create mode 100644 src/imap/response/mod.rs create mode 100644 src/imap/response/parser.rs create mode 100644 src/imap/session.rs delete mode 100644 src/imap/transport.rs create mode 100644 src/import/dav/collections.rs create mode 100644 src/import/dav/coordinator.rs create mode 100644 src/import/dav/items.rs rename src/{sync/import_maildir.rs => import/dav/mod.rs} (55%) create mode 100644 src/import/dav/replace.rs create mode 100644 src/import/dav/tree.rs create mode 100644 src/import/ews/attachments.rs create mode 100644 src/import/ews/calendar.rs create mode 100644 src/import/ews/contacts.rs create mode 100644 src/import/ews/coordinator.rs create mode 100644 src/import/ews/folder_sync.rs create mode 100644 src/import/ews/folders.rs create mode 100644 src/import/ews/items.rs create mode 100644 src/import/ews/messages.rs rename src/{sync/import_exchange_ews.rs => import/ews/mod.rs} (72%) create mode 100644 src/import/graph/calendar.rs create mode 100644 src/import/graph/contacts.rs create mode 100644 src/import/graph/coordinator.rs create mode 100644 src/import/graph/files.rs create mode 100644 src/import/graph/folders.rs create mode 100644 src/import/graph/messages.rs rename src/{sync/import_exchange_graph.rs => import/graph/mod.rs} (70%) create mode 100644 src/import/graph/series.rs create mode 100644 src/import/imap/control.rs create mode 100644 src/import/imap/dry_run.rs rename src/{sync/import_imap => import/imap}/fetch.rs (56%) rename src/{sync/import_imap => import/imap}/folders.rs (52%) create mode 100644 src/import/imap/internaldate.rs create mode 100644 src/import/imap/keywords.rs create mode 100644 src/import/imap/messages.rs create mode 100644 src/import/imap/mod.rs create mode 100644 src/import/imap/pool.rs create mode 100644 src/import/imap/reconcile.rs create mode 100644 src/import/imap/run.rs create mode 100644 src/import/jmap/blobs.rs create mode 100644 src/import/jmap/kinds.rs create mode 100644 src/import/jmap/mod.rs create mode 100644 src/import/jmap/reconcile.rs create mode 100644 src/import/jmap/tree.rs create mode 100644 src/import/maildir/keywords.rs create mode 100644 src/import/maildir/messages.rs create mode 100644 src/import/maildir/mod.rs create mode 100644 src/import/maildir/run.rs rename src/{sync/import_maildir => import/maildir}/tree.rs (55%) create mode 100644 src/import/managesieve/control.rs create mode 100644 src/import/managesieve/mod.rs create mode 100644 src/import/managesieve/plan.rs create mode 100644 src/import/managesieve/run.rs create mode 100644 src/import/mod.rs create mode 100644 src/import/takeout/calendar.rs create mode 100644 src/import/takeout/contacts.rs rename src/{sync/import_takeout => import/takeout}/labels.rs (51%) create mode 100644 src/import/takeout/mail.rs create mode 100644 src/import/takeout/mbox.rs create mode 100644 src/import/takeout/mod.rs create mode 100644 src/import/takeout/run.rs rename src/{sync/import_takeout => import/takeout}/walk.rs (61%) delete mode 100644 src/jmap/retry.rs create mode 100644 src/managesieve/session.rs create mode 100644 src/net/auth.rs create mode 100644 src/net/backoff.rs create mode 100644 src/net/encode.rs create mode 100644 src/net/http.rs create mode 100644 src/net/mod.rs create mode 100644 src/net/retry.rs create mode 100644 src/net/sasl.rs create mode 100644 src/net/stream.rs create mode 100644 src/net/xml.rs create mode 100644 src/sync/calcard.rs create mode 100644 src/sync/dates.rs create mode 100644 src/sync/diff.rs delete mode 100644 src/sync/export.rs delete mode 100644 src/sync/export/email.rs delete mode 100644 src/sync/export/flat.rs delete mode 100644 src/sync/export/keyed.rs delete mode 100644 src/sync/export/sieve.rs delete mode 100644 src/sync/export/tree.rs delete mode 100644 src/sync/export/uidtype.rs delete mode 100644 src/sync/export/vacation.rs delete mode 100644 src/sync/import_dav.rs delete mode 100644 src/sync/import_dav/calcard.rs delete mode 100644 src/sync/import_dav/collections.rs delete mode 100644 src/sync/import_dav/coordinator.rs delete mode 100644 src/sync/import_dav/items.rs delete mode 100644 src/sync/import_dav/replace.rs delete mode 100644 src/sync/import_dav/tree.rs delete mode 100644 src/sync/import_exchange_ews/attachments.rs delete mode 100644 src/sync/import_exchange_ews/calendar.rs delete mode 100644 src/sync/import_exchange_ews/contacts.rs delete mode 100644 src/sync/import_exchange_ews/coordinator.rs delete mode 100644 src/sync/import_exchange_ews/folders.rs delete mode 100644 src/sync/import_exchange_ews/items.rs delete mode 100644 src/sync/import_exchange_ews/messages.rs delete mode 100644 src/sync/import_exchange_graph/calendar.rs delete mode 100644 src/sync/import_exchange_graph/contacts.rs delete mode 100644 src/sync/import_exchange_graph/coordinator.rs delete mode 100644 src/sync/import_exchange_graph/files.rs delete mode 100644 src/sync/import_exchange_graph/folders.rs delete mode 100644 src/sync/import_exchange_graph/messages.rs delete mode 100644 src/sync/import_imap.rs delete mode 100644 src/sync/import_imap/coordinator.rs delete mode 100644 src/sync/import_imap/internaldate.rs delete mode 100644 src/sync/import_imap/keywords.rs delete mode 100644 src/sync/import_imap/messages.rs delete mode 100644 src/sync/import_imap/pool.rs delete mode 100644 src/sync/import_jmap.rs delete mode 100644 src/sync/import_jmap/diff.rs delete mode 100644 src/sync/import_jmap/mapping.rs delete mode 100644 src/sync/import_jmap/tree.rs delete mode 100644 src/sync/import_maildir/coordinator.rs delete mode 100644 src/sync/import_maildir/keywords.rs delete mode 100644 src/sync/import_maildir/messages.rs delete mode 100644 src/sync/import_managesieve/coordinator.rs delete mode 100644 src/sync/import_managesieve/reconcile.rs delete mode 100644 src/sync/import_takeout.rs delete mode 100644 src/sync/import_takeout/calendar.rs delete mode 100644 src/sync/import_takeout/contacts.rs delete mode 100644 src/sync/import_takeout/coordinator.rs delete mode 100644 src/sync/import_takeout/mail.rs delete mode 100644 src/sync/import_takeout/mbox.rs delete mode 100644 src/sync/import_takeout/tree.rs create mode 100644 src/sync/jmap_map/address_book.rs create mode 100644 src/sync/jmap_map/calendar.rs create mode 100644 src/sync/jmap_map/calendar_event.rs create mode 100644 src/sync/jmap_map/contact_card.rs create mode 100644 src/sync/jmap_map/email.rs create mode 100644 src/sync/jmap_map/file_node.rs create mode 100644 src/sync/jmap_map/identity.rs create mode 100644 src/sync/jmap_map/mailbox.rs create mode 100644 src/sync/jmap_map/mod.rs create mode 100644 src/sync/jmap_map/participant_identity.rs create mode 100644 src/sync/jmap_map/sieve_script.rs create mode 100644 src/sync/jmap_map/tests.rs create mode 100644 src/sync/jmap_map/vacation.rs create mode 100644 src/sync/keywords.rs rename src/sync/{import_jmap => }/pool.rs (77%) delete mode 100644 src/sync/prune.rs create mode 100644 src/sync/run.rs create mode 100644 src/sync/tree.rs create mode 100644 src/sync/vanish.rs create mode 100644 tests/common/line_server.rs create mode 100644 tests/common/mod.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 0709ff4..9fffbb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a ## [Unreleased] +## [27.1-rc.14] + +The fourteenth release candidate of the 27.1 series: POP3, Fastmail, Zoho and +Zimbra sources, a `reconcile` report that a migration was complete, and +upstream v1.0.16's Exchange fixes. + ### Added - `import pop3` imports from a POP3 server: the whole maildrop into one INBOX, keyed by each message's UIDL so a later run adds only what arrived since. @@ -15,6 +21,12 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a kept, since POP3 mailboxes are routinely emptied after download -- and it deletes nothing on the server. `--estimate` reports counts and sizes from STAT/UIDL/LIST before any body is fetched. +- `import zoho` and `import zimbra` import mail over IMAP, as presets for + `import imap`. `zoho` fills in Zoho's server for your region (`--region`, or + `--host` for one not listed); `zimbra` points at your own Zimbra host (IMAPS + on 993, or `--starttls` on 143). Both take only an address and password (or + an app-specific password), reuse the whole IMAP pipeline, and support + `--estimate`. - `import fastmail` imports from a Fastmail account over JMAP. It is `import jmap` with Fastmail's server and bearer-token sign-in filled in, so it needs only an API token; with no `--account-*` it imports the account Fastmail @@ -28,6 +40,39 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a complete; the checksum shows two archives -- an export and a re-import, say -- hold the very same mail. It exits 8 when the archive is not sound. +### Changed +- Upstream v1.0.16 is merged in. +- Outlook categories become valid JMAP keywords, the same way IMAP + `--mime-keywords` does it (`Catégorie Rouge` is `cat_gorie_rouge`), and + duplicates are dropped. Before, only the case was lowered, so `Project + review` gave a keyword JMAP doesn't allow. A read-receipt request no longer + adds `$notified`. This is for EWS and Graph. + +### Fixed +- Exchange EWS and Graph: all-day events and birthdays landed a day early + east of UTC. The date is now taken in the event's time zone. Outlook's + "no date" (year 4500 or later) is no longer imported as a real date. + Archives imported before this keep the wrong dates until the item changes + at the source, so re-import into a fresh archive to correct them. +- Exchange EWS: Exchange 2019 servers were asked for a schema version that + doesn't exist, so every call was refused once and sent again. +- Exchange EWS: hidden system folders (`Recipient Cache`, `GAL Contacts`) + were imported as mailboxes and address books. They are skipped now. +- Exchange EWS: a mail folder under a calendar or contacts folder could stop + a later run with a FOREIGN KEY error. It is filed under its nearest mail + folder on every run. +- Exchange EWS: an attendee's or contact's ItemId could replace the item's + own, giving `UNIQUE constraint failed`. Tasks and posts in a listing shifted + the ids around them. One item that failed to parse or store stopped the rest + of its folder; it is now logged and counted, and the folder goes on. +- Exchange EWS: event attachments, your own response to a meeting and an + event's private flag were never fetched. They are imported now. +- Exchange EWS: a folder with no class (Outlook's `(This computer only)` + contacts) was always taken for mail, and contacts, appointments and tasks + in a mail folder such as Deleted Items were stored as email. A class-less + folder now takes its kind from its items, and mail folders skip non-mail + items, also in `--estimate`. + ## [27.1-rc.13] The thirteenth release candidate of the 27.1 series: shell completions and a diff --git a/Cargo.lock b/Cargo.lock index 90ce205..c3b7897 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -157,7 +157,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -174,9 +174,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.18.2" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" +checksum = "048006790404103c488e6c5616b3e7e702a71292aadda08e8f7698a9e0d2eb53" dependencies = [ "aws-lc-sys", "zeroize", @@ -184,9 +184,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.46.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" +checksum = "8bfbe32ac36b3c428696ce315464181bc4029d689fd58e3f2a4e58c252242f52" dependencies = [ "cc", "cmake", @@ -250,12 +250,6 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bitflags" version = "2.13.2" @@ -283,7 +277,7 @@ checksum = "dbe8358268799ebb3e4df23cb9d47f4c72bbc4f5247e2fa6a1bf7b6c0baea220" dependencies = [ "async-stream", "base64 0.22.1", - "bitflags 2.13.2", + "bitflags", "bollard-buildkit-proto", "bollard-stubs", "bytes", @@ -301,7 +295,7 @@ dependencies = [ "log", "num", "pin-project-lite", - "rand 0.10.3", + "rand 0.10.4", "rustls", "rustls-native-certs", "rustls-pki-types", @@ -359,9 +353,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.20.3" +version = "3.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +checksum = "87a0cf66fd1eb2fb7b822cb78370d82303750809bc6ba855f3f58e517d3238db" [[package]] name = "bytes" @@ -378,10 +372,10 @@ dependencies = [ "ahash", "chrono", "chrono-tz", - "hashify", + "hashify 0.2.9", "jmap-tools", "mail-builder", - "mail-parser", + "mail-parser 0.11.9", "serde", "serde_json", "uuid", @@ -389,9 +383,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.10.0" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +checksum = "33ef9e3e8154a45c7a95cff4c1299702106ad94062d7bb85fdc5d3a5afd91d45" dependencies = [ "find-msvc-tools", "jobserver", @@ -471,7 +465,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -576,9 +570,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.3" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" +checksum = "2b929dd2a135771f40afabd1073f85eed59c800b7c61901e5c254270497dad64" dependencies = [ "cfg-if", ] @@ -618,7 +612,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -629,38 +623,7 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ "darling_core", "quote", - "syn 3.0.9", -] - -[[package]] -name = "defmt" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" -dependencies = [ - "bitflags 1.3.2", - "defmt-macros", -] - -[[package]] -name = "defmt-macros" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" -dependencies = [ - "defmt-parser", - "proc-macro2", - "quote", - "syn 3.0.9", -] - -[[package]] -name = "defmt-parser" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" -dependencies = [ - "thiserror", + "syn 3.0.10", ] [[package]] @@ -680,7 +643,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -739,9 +702,9 @@ checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" [[package]] name = "errno" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +checksum = "e35d6e78f4cebb90e51d543db8fe41528851d6deeef6ef21ebd4bfdc1e784776" dependencies = [ "libc", "windows-sys 0.61.2", @@ -782,7 +745,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee93edf3c501f0035bbeffeccfed0b79e14c311f12195ec0e661e114a0f60da4" dependencies = [ "portable-atomic", - "rand 0.10.3", + "rand 0.10.4", "web-time", ] @@ -794,9 +757,9 @@ checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" [[package]] name = "flate2" -version = "1.1.10" +version = "1.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4" dependencies = [ "crc32fast", "miniz_oxide", @@ -886,7 +849,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -978,15 +941,6 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -[[package]] -name = "hashbrown" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" -dependencies = [ - "foldhash", -] - [[package]] name = "hashbrown" version = "0.17.1" @@ -1008,6 +962,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "hashify" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04a037b1848eada162d6534c206770180e6bfc0eaa97292250c3e7179d8b1f73" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + [[package]] name = "hashlink" version = "0.12.2" @@ -1362,18 +1327,26 @@ dependencies = [ ] [[package]] -name = "itoa" -version = "1.0.18" +name = "itertools" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4" [[package]] name = "jiff" -version = "0.2.38" +version = "0.2.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +checksum = "175dc66334db3b9ab9985bf8b61cfed50dc16d20964bb47f9e479adcae984a7f" dependencies = [ - "defmt", "jiff-core", "jiff-static", "jiff-tzdb-platform", @@ -1386,19 +1359,18 @@ dependencies = [ [[package]] name = "jiff-core" -version = "0.1.1" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +checksum = "e599b80292e597019b8b3e392de3f4bea2959ddfb8c7dff60e035e9e311a2c1b" dependencies = [ - "defmt", "log", ] [[package]] name = "jiff-static" -version = "0.2.38" +version = "0.2.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +checksum = "c7b1af7547d47354bc438d797e3854601bde8f8d185780c5ba4ab27afa54f1ba" dependencies = [ "jiff-core", "proc-macro2", @@ -1408,9 +1380,9 @@ dependencies = [ [[package]] name = "jiff-tzdb" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" +checksum = "0980fca2cc272465bd60e82397607ba9f33add9e48defd77923301bbd03808f2" [[package]] name = "jiff-tzdb-platform" @@ -1427,7 +1399,7 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5bdcd368dcd58a5efaac10c142fb8cddb4d2949af1297b624df3ff0db4cbff54" dependencies = [ - "hashify", + "hashify 0.2.9", "serde", "serde_json", ] @@ -1493,9 +1465,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.106" +version = "0.3.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a" dependencies = [ "cfg-if", "futures-util", @@ -1510,9 +1482,9 @@ checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "libsqlite3-sys" -version = "0.39.1" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14a5f01b87b92705d07b433c06f96cbc764637d81a0bffb5734ce17cf0f9a2b3" +checksum = "b06e69070b7e12262b0cd90832f0a7396f142a2b330181255562decf9f0c6215" dependencies = [ "cc", "pkg-config", @@ -1548,9 +1520,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.34" +version = "0.4.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c" [[package]] name = "mail-builder" @@ -1564,7 +1536,19 @@ version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec00bda90c6e645a54506c630c2820cd6b1890cfd2b0a169b50f74b2b8c7c86" dependencies = [ - "hashify", + "hashify 0.2.9", +] + +[[package]] +name = "mail-parser" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d4ea9311ef3ca6189aa85c5f984bd47b2f12079bcabae28917d900ab26c9ecc" +dependencies = [ + "encodify", + "hashify 0.3.2", + "memchr", + "simdutf8", ] [[package]] @@ -1581,9 +1565,9 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "mime" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +checksum = "d5848f5a9e8117cf94f2f90faac24a87ff81600ff8628c1e3deaab572e76bfa1" [[package]] name = "miniz_oxide" @@ -1597,9 +1581,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.4" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +checksum = "20521a903f02b0a573cd6840e9d0a5a0cc043d361fe924092cd70f8556cc9c4a" dependencies = [ "libc", "wasi", @@ -1733,9 +1717,9 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "openssl-probe" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +checksum = "7e036b411754132fd6b74e94b74e9a13c5cf606fa93d1885c35860d3060820b1" [[package]] name = "parking_lot" @@ -1891,9 +1875,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.4" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +checksum = "3ef0b569d1d907ab3a28596aee62cf1f61dacb0529a4bedcd8e7460e2d5228b4" dependencies = [ "bytes", "prost-derive", @@ -1901,12 +1885,12 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.4" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +checksum = "1ad9c5423459fa2f729b8d9235fab810aa29628b171467744fc86d21460ad70d" dependencies = [ "anyhow", - "itertools", + "itertools 0.15.0", "proc-macro2", "quote", "syn 2.0.119", @@ -1914,18 +1898,18 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.14.4" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +checksum = "f29a7c4c7a84cef41a53d338a2d10d4f1b043ea5c3dde26db22ee2c9c1b1c75b" dependencies = [ "prost", ] [[package]] name = "psl" -version = "2.1.245" +version = "2.1.255" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46cce9bc841c0eeb395275a775d11e3cc91b904e6ab370a9a42369e111b3038e" +checksum = "cb40358efa8d957f3e793e45daedd2fef822c694e1e3a45b827b53de6175e724" dependencies = [ "psl-types", ] @@ -1978,9 +1962,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +checksum = "8f6fa277c67e6ea590466f04c3fe82cb509f0864fe470643e5570de9514a9b11" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -2018,7 +2002,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.2", + "bitflags", ] [[package]] @@ -2038,7 +2022,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -2086,21 +2070,20 @@ dependencies = [ [[package]] name = "rsqlite-vfs" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +checksum = "353b74f9810d5b085b6c42c9d86a5f0fdb795553235cdd48753b32e3809d18a7" dependencies = [ - "hashbrown 0.16.1", "thiserror", ] [[package]] name = "rusqlite" -version = "0.41.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c026b4444579e8d754556c6c75d040d69c4a44283cdb648ab8a8d5965c2cfa8" +checksum = "f172c4812a19b26d1c23cdb0070e69db335503dfa378b910eb857386e5b8beba" dependencies = [ - "bitflags 2.13.2", + "bitflags", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -2126,11 +2109,11 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" dependencies = [ - "bitflags 2.13.2", + "bitflags", "errno", "libc", "linux-raw-sys", @@ -2167,9 +2150,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.1" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +checksum = "102f35f7de7b06d8c61f1cff6592f5253e0dea431c74867aad5dfd70f44663fb" dependencies = [ "zeroize", ] @@ -2203,9 +2186,9 @@ checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-webpki" -version = "0.103.16" +version = "0.103.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" +checksum = "e3a0760a887c14823fca5a1c970b01d47c328c696c0c64829d1df50e8f853fd5" dependencies = [ "aws-lc-rs", "ring", @@ -2279,7 +2262,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.2", + "bitflags", "core-foundation", "core-foundation-sys", "libc", @@ -2329,7 +2312,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -2353,7 +2336,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -2398,7 +2381,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -2465,9 +2448,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.16.4" +version = "1.16.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" +checksum = "149309498a5f7334b12ddae0cad8aba81ce785fe91755068867e017cf98fee46" [[package]] name = "socket2" @@ -2481,14 +2464,12 @@ dependencies = [ [[package]] name = "sqlite-wasm-rs" -version = "0.5.5" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +checksum = "a6cd504cecbfe7eef649988840f9a6cb768d11085efec138f8e99f160632d867" dependencies = [ "cc", - "js-sys", "rsqlite-vfs", - "wasm-bindgen", ] [[package]] @@ -2545,9 +2526,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.9" +version = "3.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79" dependencies = [ "proc-macro2", "quote", @@ -2568,7 +2549,7 @@ checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -2584,6 +2565,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "testcontainers" version = "0.28.0" @@ -2600,7 +2591,7 @@ dependencies = [ "ferroid", "futures", "http", - "itertools", + "itertools 0.14.0", "log", "memchr", "parse-display", @@ -2632,7 +2623,7 @@ checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -2706,7 +2697,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] @@ -2746,9 +2737,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.14.6" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" +checksum = "e2cdfc673aabd30fe8fee92e65837b644c37c5d47f0f97ae6d5e0f1237d15e1b" dependencies = [ "async-trait", "axum", @@ -2767,6 +2758,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-stream", + "tokio-util", "tower", "tower-layer", "tower-service", @@ -2775,9 +2767,9 @@ dependencies = [ [[package]] name = "tonic-prost" -version = "0.14.6" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" +checksum = "eba464054e2226f182a34489b8b079bf4c6e3e7ab5fd7fc30e024f24aa92a484" dependencies = [ "bytes", "prost", @@ -2866,9 +2858,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "3.5.0" +version = "3.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde93ff05758a36a851a398a772b0b79a82fbe3c186a46f2696baaeaf84c966b" +checksum = "3ac5cecf619ca81f66a96482db09687c9f7465af65f4300a8c30cca05bd5a59d" dependencies = [ "base64 0.23.1", "cookie_store", @@ -2886,9 +2878,9 @@ dependencies = [ [[package]] name = "ureq-proto" -version = "0.6.5" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841583fc36089d7d59419568ae157f43a5a03a2e5a81ba87c77d1596ef46e922" +checksum = "622dce33c252115466d83f63e9744a18f12b4406aae51a01c15ad8fcf1185eb3" dependencies = [ "base64 0.23.1", "http", @@ -2917,9 +2909,12 @@ checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" [[package]] name = "utf8_iter" -version = "1.0.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +checksum = "af3bd9d3e571c26fcce9d033b87ea5ab061b8d2ffe9b7a5675d810d68f03edf0" +dependencies = [ + "utf_types", +] [[package]] name = "utf8parse" @@ -2928,17 +2923,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] -name = "uuid" -version = "1.29.0" +name = "utf_types" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" +checksum = "3a3b425286980a367278453f3b6d28cdcb134b574e5a35824a6ff5eebad50bb6" + +[[package]] +name = "uuid" +version = "1.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bed99139f7dac9b5cfe5f17be4d66dc7079a34dee1d74bcc77217119240e267c" dependencies = [ "sha1_smol", ] [[package]] name = "inbuxa-migrate" -version = "1.0.16" +version = "1.1.0" dependencies = [ "blake3", "calcard", @@ -2948,9 +2949,9 @@ dependencies = [ "crossbeam-channel", "encodify", "flate2", - "hashify", + "hashify 0.3.2", "indexmap 2.14.2", - "mail-parser", + "mail-parser 1.0.1", "mockito", "num_cpus", "psl", @@ -2962,6 +2963,7 @@ dependencies = [ "serde", "serde_json", "tempfile", + "terminal_size", "testcontainers", "thiserror", "time", @@ -3017,12 +3019,11 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.129" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c" dependencies = [ "cfg-if", - "once_cell", "rustversion", "wasm-bindgen-macro", "wasm-bindgen-shared", @@ -3030,9 +3031,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.129" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3040,22 +3041,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.129" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.129" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e" dependencies = [ "unicode-ident", ] @@ -3072,9 +3073,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a" +checksum = "293198dcebfb821155b5092ab94b5a82ff180eddd7241df3bd6e66a4ffd1a585" dependencies = [ "rustls-pki-types", ] @@ -3265,9 +3266,9 @@ checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "xattr" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +checksum = "963d6fab92d5504b971dc8bd90ca743c2575e590bc8071ea5bc571598667ae08" dependencies = [ "libc", "rustix", @@ -3292,7 +3293,7 @@ checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", "synstructure", ] @@ -3333,7 +3334,7 @@ checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", "synstructure", ] @@ -3373,7 +3374,7 @@ checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.9", + "syn 3.0.10", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b1a36d6..1e48c8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "inbuxa-migrate" description = "Account migration and backup for inbuxa" -version = "27.1.0-rc.13" +version = "27.1.0-rc.14" authors = ["Coffey Labs LLC "] license = "Apache-2.0 OR MIT" repository = "https://git.coffeylabs.org/inbuxa/inbuxa-migrate" @@ -18,32 +18,33 @@ default = ["bundled-sqlite"] bundled-sqlite = ["rusqlite/bundled"] [dependencies] -serde = { version = "1.0.228", features = ["derive"] } -serde_json = "1.0" -indexmap = { version = "2.14", features = ["serde"] } -rusqlite = { version = "0.41", features = ["blob"] } -blake3 = "1.8.5" -time = { version = "0.3.47", features = ["serde-well-known", "macros"] } +serde = { version = "1.0.229", features = ["derive"] } +serde_json = "1.0.154" +indexmap = { version = "2.14.2", features = ["serde"] } +rusqlite = { version = "0.43", features = ["blob"] } +blake3 = "1.8.7" +time = { version = "0.3.55", features = ["serde-well-known", "macros"] } url= "2.5.8" -thiserror = "2.0.18" -ureq = { version = "3.5.0", default-features = false, features = ["json", "gzip", "rustls-no-provider", "platform-verifier"] } -clap = { version = "4.6.1", features = ["derive"] } -crossbeam-channel = "0.5.15" +thiserror = "2.0.21" +ureq = { version = "3.5.2", default-features = false, features = ["json", "gzip", "rustls-no-provider", "platform-verifier"] } +clap = { version = "4.6.7", features = ["derive"] } +crossbeam-channel = "0.5.17" num_cpus = "1.17.0" -encodify = "1.0" -mail-parser = "0.11.3" -calcard = "0.3" -rustls = { version = "0.23.40", default-features = false, features = ["aws-lc-rs", "std", "tls12", "logging"] } -rustls-platform-verifier = "0.7" -regex = "1.13" -flate2 = "1.1.9" +encodify = "1.0.2" +mail-parser = "1.0.1" +calcard = "0.3.14" +rustls = { version = "0.23.46", default-features = false, features = ["aws-lc-rs", "std", "tls12", "logging"] } +rustls-platform-verifier = "0.7.1" +regex = "1.13.1" +flate2 = "1.1.11" quick-xml = "0.42" -chrono = { version = "0.4", default-features = false, features = ["std"] } -chrono-tz = { version = "0.10", default-features = false } -hashify = "0.2" -psl = "2.1.226" +chrono = { version = "0.4.45", default-features = false, features = ["std"] } +chrono-tz = { version = "0.10.4", default-features = false } +hashify = "0.3.2" +psl = "2.1.255" +terminal_size = "0.4.4" [dev-dependencies] mockito = "1.7.2" -tempfile = "3.13" +tempfile = "3.27.0" testcontainers = { version = "0.28", features = ["blocking"] } diff --git a/README.md b/README.md index 71c4075..374a4e6 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,10 @@ command reference is in [docs/usage.md](docs/usage.md). - **POP3** -- mail only, the whole maildrop into one INBOX, keyed by UIDL. It deletes nothing on the server and keeps a message even once the server no longer has it. +- **Zoho Mail** -- IMAP, with the server for your region filled in: just your + address and password (or an app-specific password). Mail only. +- **Zimbra** -- IMAP against your Zimbra server: the host, your address and + password. Mail only. - **CalDAV, CardDAV** -- calendars and events, address books and contacts. - **WebDAV** -- a plain file collection, kept as a file tree. - **ManageSieve** -- Sieve scripts only, with the active one recorded. diff --git a/src/archive/blobs.rs b/src/archive/blobs.rs new file mode 100644 index 0000000..506e00e --- /dev/null +++ b/src/archive/blobs.rs @@ -0,0 +1,148 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::{Connection, OptionalExtension, params}; + +use super::{BlobId, Store}; + +#[derive(Clone, Copy)] +pub struct Blobs<'a> { + conn: &'a Connection, +} + +impl<'a> Store<'a> { + pub fn blobs(self) -> Blobs<'a> { + Blobs { conn: self.conn } + } +} + +impl Blobs<'_> { + pub fn intern(&self, bytes: &[u8]) -> rusqlite::Result { + let hash = blake3::hash(bytes); + let hash_bytes = hash.as_bytes().as_slice(); + self.conn.execute( + "INSERT OR IGNORE INTO blobs (hash, data) VALUES (?1, ?2)", + params![hash_bytes, bytes], + )?; + self.conn.query_row( + "SELECT id FROM blobs WHERE hash = ?1", + params![hash_bytes], + |row| row.get(0), + ) + } + + pub fn bytes(&self, id: BlobId) -> rusqlite::Result>> { + self.conn + .query_row("SELECT data FROM blobs WHERE id = ?1", [id], |row| { + row.get(0) + }) + .optional() + } + + pub fn len(&self, id: BlobId) -> rusqlite::Result> { + self.conn + .query_row( + "SELECT length(data) FROM blobs WHERE id = ?1", + [id], + |row| { + let len: i64 = row.get(0)?; + Ok(u64::try_from(len).unwrap_or(0)) + }, + ) + .optional() + } + + pub fn gc_orphans(&self) -> rusqlite::Result { + self.conn.execute( + "DELETE FROM blobs WHERE id NOT IN ( + SELECT blob_id FROM emails + UNION SELECT blob_id FROM sieve_scripts + UNION SELECT blob_id FROM file_nodes WHERE blob_id IS NOT NULL + UNION SELECT json_tree.atom FROM contact_cards, json_tree(contact_cards.data) + WHERE json_tree.key = '@blob' + UNION SELECT json_tree.atom FROM calendar_events, json_tree(calendar_events.data) + WHERE json_tree.key = '@blob')", + [], + ) + } +} + +#[cfg(test)] +mod tests { + use rusqlite::params; + + use crate::archive::{Archive, BlobId}; + + #[test] + fn intern_dedups_identical_bytes() { + let archive = Archive::open_in_memory().unwrap(); + let blobs = archive.store().blobs(); + let a = blobs.intern(b"hello world").unwrap(); + let b = blobs.intern(b"hello world").unwrap(); + let other = blobs.intern(b"different").unwrap(); + assert_eq!(a, b); + assert_ne!(a, other); + let n: i64 = archive + .connection() + .query_row("SELECT count(*) FROM blobs", [], |r| r.get(0)) + .unwrap(); + assert_eq!(n, 2); + } + + #[test] + fn bytes_roundtrip() { + let archive = Archive::open_in_memory().unwrap(); + let blobs = archive.store().blobs(); + let id = blobs.intern(b"payload").unwrap(); + assert_eq!(blobs.bytes(id).unwrap().as_deref(), Some(&b"payload"[..])); + assert_eq!(blobs.bytes(BlobId::from_raw(9999)).unwrap(), None); + } + + #[test] + fn len_reports_byte_length() { + let archive = Archive::open_in_memory().unwrap(); + let blobs = archive.store().blobs(); + let id = blobs.intern(b"payload").unwrap(); + assert_eq!(blobs.len(id).unwrap(), Some(7)); + assert_eq!(blobs.len(BlobId::from_raw(9999)).unwrap(), None); + } + + #[test] + fn gc_keeps_referenced_and_reaps_orphans() { + let archive = Archive::open_in_memory().unwrap(); + let blobs = archive.store().blobs(); + let referenced = blobs.intern(b"keep me").unwrap(); + blobs.intern(b"reap me").unwrap(); + archive + .connection() + .execute( + "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords) + VALUES (?1, '2020-01-01T00:00:00Z', '[1]', '[]')", + params![referenced], + ) + .unwrap(); + assert_eq!(blobs.gc_orphans().unwrap(), 1); + assert!(blobs.bytes(referenced).unwrap().is_some()); + } + + #[test] + fn gc_follows_blob_sentinel_in_json_data() { + let archive = Archive::open_in_memory().unwrap(); + let blobs = archive.store().blobs(); + let photo = blobs.intern(b"photo bytes").unwrap(); + blobs.intern(b"orphan").unwrap(); + archive + .connection() + .execute( + "INSERT INTO contact_cards (uid, address_book_ids, data) + VALUES ('u1', '[1]', ?1)", + params![format!(r#"{{"photos":{{"p":{{"@blob":{photo}}}}}}}"#)], + ) + .unwrap(); + assert_eq!(blobs.gc_orphans().unwrap(), 1); + assert!(blobs.bytes(photo).unwrap().is_some()); + } +} diff --git a/src/archive/defaults.rs b/src/archive/defaults.rs new file mode 100644 index 0000000..8991e1b --- /dev/null +++ b/src/archive/defaults.rs @@ -0,0 +1,155 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::archive::{LocalId, Store}; +use crate::types::ObjectType; + +macro_rules! default_taken { + ($table:literal) => { + concat!( + "SELECT EXISTS(SELECT 1 FROM ", + $table, + " WHERE is_default = 1 AND (?1 IS NULL OR id != ?1))" + ) + }; +} + +impl Store<'_> { + pub fn unique_default( + self, + ty: ObjectType, + is_default: bool, + exclude: Option, + ) -> rusqlite::Result { + if !is_default { + return Ok(false); + } + let sql = match ty { + ObjectType::AddressBook => default_taken!("address_books"), + ObjectType::Calendar => default_taken!("calendars"), + ObjectType::ParticipantIdentity => default_taken!("participant_identities"), + _ => return Ok(false), + }; + let taken: bool = self.conn.query_row(sql, [exclude], |row| row.get(0))?; + Ok(!taken) + } +} + +#[cfg(test)] +mod tests { + use rusqlite::params; + + use crate::archive::{Archive, LocalId}; + use crate::types::ObjectType; + + fn insert_book(archive: &Archive, name: &str, is_default: bool) -> LocalId { + let conn = archive.connection(); + conn.execute( + "INSERT INTO address_books (name, sort_order, is_default, is_subscribed) + VALUES (?1, 0, ?2, 1)", + params![name, is_default as i64], + ) + .unwrap(); + LocalId::from_raw(conn.last_insert_rowid()) + } + + #[test] + fn first_claimant_keeps_the_default_second_is_dropped() { + let archive = Archive::open_in_memory().unwrap(); + let store = archive.store(); + let first = store + .unique_default(ObjectType::AddressBook, true, None) + .unwrap(); + assert!(first); + insert_book(&archive, "Contacts", first); + + assert!( + !store + .unique_default(ObjectType::AddressBook, true, None) + .unwrap(), + "an archive must never hold two default address books" + ); + } + + #[test] + fn non_default_stays_non_default() { + let archive = Archive::open_in_memory().unwrap(); + assert!( + !archive + .store() + .unique_default(ObjectType::AddressBook, false, None) + .unwrap() + ); + } + + #[test] + fn each_type_has_its_own_default() { + let archive = Archive::open_in_memory().unwrap(); + insert_book(&archive, "Contacts", true); + assert!( + archive + .store() + .unique_default(ObjectType::Calendar, true, None) + .unwrap(), + "a default address book must not block a default calendar" + ); + } + + #[test] + fn update_excludes_its_own_row() { + let archive = Archive::open_in_memory().unwrap(); + let id = insert_book(&archive, "Contacts", true); + assert!( + archive + .store() + .unique_default(ObjectType::AddressBook, true, Some(id)) + .unwrap() + ); + } + + #[test] + fn update_detects_another_holder() { + let archive = Archive::open_in_memory().unwrap(); + insert_book(&archive, "Contacts", true); + let other = insert_book(&archive, "GAL", false); + assert!( + !archive + .store() + .unique_default(ObjectType::AddressBook, true, Some(other)) + .unwrap() + ); + } + + #[test] + fn types_without_a_default_column_are_never_default() { + let archive = Archive::open_in_memory().unwrap(); + assert!( + !archive + .store() + .unique_default(ObjectType::Mailbox, true, None) + .unwrap() + ); + } + + #[test] + fn participant_identities_have_their_own_default() { + let archive = Archive::open_in_memory().unwrap(); + let store = archive.store(); + archive + .connection() + .execute( + "INSERT INTO participant_identities (name, calendar_address, is_default) + VALUES ('me', 'mailto:me@example.org', 1)", + [], + ) + .unwrap(); + assert!( + !store + .unique_default(ObjectType::ParticipantIdentity, true, None) + .unwrap() + ); + } +} diff --git a/src/archive/ids/dav.rs b/src/archive/ids/dav.rs new file mode 100644 index 0000000..ac6be28 --- /dev/null +++ b/src/archive/ids/dav.rs @@ -0,0 +1,311 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use rusqlite::{Connection, OptionalExtension, params}; + +use crate::archive::{LocalId, SourceId, Store}; +use crate::types::ObjectType; + +const PURGE_CALENDAR_EVENTS: &str = "DELETE FROM calendar_events WHERE id IN + (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)"; +const PURGE_CONTACT_CARDS: &str = "DELETE FROM contact_cards WHERE id IN + (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)"; +const PURGE_CALENDARS: &str = "DELETE FROM calendars WHERE id IN + (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)"; +const PURGE_ADDRESS_BOOKS: &str = "DELETE FROM address_books WHERE id IN + (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)"; +const PURGE_LEAF_FILE_NODES: &str = "DELETE FROM file_nodes WHERE id IN + (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2) + AND NOT EXISTS (SELECT 1 FROM file_nodes AS child WHERE child.parent_id = file_nodes.id)"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DavItem { + pub etag: String, + pub local: LocalId, +} + +#[derive(Clone, Copy)] +pub struct DavIds<'a> { + conn: &'a Connection, + source: SourceId, +} + +impl<'a> Store<'a> { + pub fn dav_ids(self, source: SourceId) -> DavIds<'a> { + DavIds { + conn: self.connection(), + source, + } + } +} + +impl DavIds<'_> { + pub fn insert( + &self, + ty: ObjectType, + collection: &str, + item: &str, + etag: &str, + local: LocalId, + ) -> rusqlite::Result<()> { + self.conn + .prepare_cached( + "INSERT INTO sync_id_dav + (source_id, type_name, collection_href, item_href, etag, local_id) + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + )? + .execute(params![ + self.source, + ty.token(), + collection, + item, + etag, + local + ])?; + Ok(()) + } + + pub fn update_etag(&self, ty: ObjectType, item: &str, etag: &str) -> rusqlite::Result<()> { + self.conn + .prepare_cached( + "UPDATE sync_id_dav SET etag = ?4 + WHERE source_id = ?1 AND type_name = ?2 AND item_href = ?3", + )? + .execute(params![self.source, ty.token(), item, etag])?; + Ok(()) + } + + pub fn local_id(&self, ty: ObjectType, item: &str) -> rusqlite::Result> { + self.conn + .prepare_cached( + "SELECT local_id FROM sync_id_dav + WHERE source_id = ?1 AND type_name = ?2 AND item_href = ?3", + )? + .query_row(params![self.source, ty.token(), item], |row| row.get(0)) + .optional() + } + + pub fn item(&self, ty: ObjectType, item: &str) -> rusqlite::Result> { + self.conn + .prepare_cached( + "SELECT etag, local_id FROM sync_id_dav + WHERE source_id = ?1 AND type_name = ?2 AND item_href = ?3", + )? + .query_row(params![self.source, ty.token(), item], |row| { + Ok(DavItem { + etag: row.get(0)?, + local: row.get(1)?, + }) + }) + .optional() + } + + pub fn items_in( + &self, + ty: ObjectType, + collection: &str, + ) -> rusqlite::Result> { + let mut stmt = self.conn.prepare_cached( + "SELECT item_href, etag, local_id FROM sync_id_dav + WHERE source_id = ?1 AND type_name = ?2 AND collection_href = ?3", + )?; + let rows = stmt.query_map(params![self.source, ty.token(), collection], |row| { + Ok(( + row.get(0)?, + DavItem { + etag: row.get(1)?, + local: row.get(2)?, + }, + )) + })?; + rows.collect() + } + + pub fn all(&self, ty: ObjectType) -> rusqlite::Result> { + let mut stmt = self.conn.prepare_cached( + "SELECT item_href, local_id FROM sync_id_dav + WHERE source_id = ?1 AND type_name = ?2", + )?; + let rows = stmt.query_map(params![self.source, ty.token()], |row| { + Ok((row.get(0)?, row.get(1)?)) + })?; + rows.collect() + } + + pub fn delete(&self, ty: ObjectType, item: &str) -> rusqlite::Result<()> { + self.conn + .prepare_cached( + "DELETE FROM sync_id_dav + WHERE source_id = ?1 AND type_name = ?2 AND item_href = ?3", + )? + .execute(params![self.source, ty.token(), item])?; + Ok(()) + } + + pub fn delete_collection(&self, ty: ObjectType, collection: &str) -> rusqlite::Result { + self.conn.execute( + "DELETE FROM sync_id_dav + WHERE source_id = ?1 AND type_name = ?2 + AND (item_href = ?3 OR collection_href = ?3)", + params![self.source, ty.token(), collection], + ) + } + + pub fn purge(&self, ty: ObjectType) -> rusqlite::Result { + let sql = match ty { + ObjectType::CalendarEvent => PURGE_CALENDAR_EVENTS, + ObjectType::ContactCard => PURGE_CONTACT_CARDS, + ObjectType::Calendar => PURGE_CALENDARS, + ObjectType::AddressBook => PURGE_ADDRESS_BOOKS, + ObjectType::FileNode => return self.purge_file_nodes(), + _ => return Ok(0), + }; + let removed = self.conn.execute(sql, params![self.source, ty.token()])?; + Ok(removed as u64) + } + + fn purge_file_nodes(&self) -> rusqlite::Result { + let mut leaves = self.conn.prepare(PURGE_LEAF_FILE_NODES)?; + let mut removed = 0; + loop { + let n = leaves.execute(params![self.source, ObjectType::FileNode.token()])?; + if n == 0 { + return Ok(removed); + } + removed += n as u64; + } + } +} + +#[cfg(test)] +mod tests { + use super::DavIds; + use crate::archive::sources::{SourceKey, SourceKind}; + use crate::archive::{Archive, LocalId}; + use crate::types::ObjectType::{Calendar, CalendarEvent}; + + fn ids(archive: &Archive) -> DavIds<'_> { + let sid = archive + .store() + .sources() + .upsert( + &SourceKey { + kind: SourceKind::CalDav, + url: "https://x/", + account_id: "u", + }, + Some("u"), + "u", + ) + .unwrap(); + archive.store().dav_ids(sid) + } + + fn id(raw: i64) -> LocalId { + LocalId::from_raw(raw) + } + + #[test] + fn insert_and_lookup_round_trip() { + let archive = Archive::open_in_memory().unwrap(); + let ids = ids(&archive); + ids.insert(CalendarEvent, "/cal/a/", "/cal/a/e.ics", "\"v1\"", id(7)) + .unwrap(); + assert_eq!( + ids.local_id(CalendarEvent, "/cal/a/e.ics").unwrap(), + Some(id(7)) + ); + let item = ids.item(CalendarEvent, "/cal/a/e.ics").unwrap().unwrap(); + assert_eq!((item.etag.as_str(), item.local), ("\"v1\"", id(7))); + assert!(ids.item(Calendar, "/cal/a/e.ics").unwrap().is_none()); + } + + #[test] + fn update_etag_keeps_local_id() { + let archive = Archive::open_in_memory().unwrap(); + let ids = ids(&archive); + ids.insert(CalendarEvent, "/cal/a/", "/cal/a/e.ics", "\"v1\"", id(7)) + .unwrap(); + ids.update_etag(CalendarEvent, "/cal/a/e.ics", "\"v2\"") + .unwrap(); + let items = ids.items_in(CalendarEvent, "/cal/a/").unwrap(); + assert_eq!(items.len(), 1); + let item = &items["/cal/a/e.ics"]; + assert_eq!((item.etag.as_str(), item.local), ("\"v2\"", id(7))); + } + + #[test] + fn items_in_filters_by_collection_href() { + let archive = Archive::open_in_memory().unwrap(); + let ids = ids(&archive); + ids.insert(CalendarEvent, "/cal/a/", "/cal/a/e1.ics", "v1", id(1)) + .unwrap(); + ids.insert(CalendarEvent, "/cal/a/", "/cal/a/e2.ics", "v2", id(2)) + .unwrap(); + ids.insert(CalendarEvent, "/cal/b/", "/cal/b/e3.ics", "v3", id(3)) + .unwrap(); + assert_eq!(ids.items_in(CalendarEvent, "/cal/a/").unwrap().len(), 2); + } + + #[test] + fn delete_removes_only_one_row() { + let archive = Archive::open_in_memory().unwrap(); + let ids = ids(&archive); + ids.insert(CalendarEvent, "/cal/a/", "/cal/a/e1.ics", "v", id(1)) + .unwrap(); + ids.insert(CalendarEvent, "/cal/a/", "/cal/a/e2.ics", "v", id(2)) + .unwrap(); + ids.delete(CalendarEvent, "/cal/a/e1.ics").unwrap(); + let items = ids.items_in(CalendarEvent, "/cal/a/").unwrap(); + assert_eq!(items.len(), 1); + assert!(items.contains_key("/cal/a/e2.ics")); + } + + #[test] + fn delete_collection_removes_collection_row_and_items() { + let archive = Archive::open_in_memory().unwrap(); + let ids = ids(&archive); + ids.insert(Calendar, "/cal/a/", "/cal/a/", "", id(10)) + .unwrap(); + ids.insert(CalendarEvent, "/cal/a/", "/cal/a/e.ics", "v", id(1)) + .unwrap(); + assert_eq!(ids.delete_collection(CalendarEvent, "/cal/a/").unwrap(), 1); + assert!(ids.items_in(CalendarEvent, "/cal/a/").unwrap().is_empty()); + assert_eq!(ids.all(Calendar).unwrap().len(), 1); + } + + #[test] + fn all_lists_every_known_collection() { + let archive = Archive::open_in_memory().unwrap(); + let ids = ids(&archive); + ids.insert(Calendar, "/cal/a/", "/cal/a/", "", id(1)) + .unwrap(); + ids.insert(Calendar, "/cal/b/", "/cal/b/", "", id(2)) + .unwrap(); + let map = ids.all(Calendar).unwrap(); + assert_eq!(map.len(), 2); + assert_eq!(map.get("/cal/a/"), Some(&id(1))); + assert_eq!(map.get("/cal/b/"), Some(&id(2))); + } + + #[test] + fn unique_local_id_per_type_enforced() { + let archive = Archive::open_in_memory().unwrap(); + let ids = ids(&archive); + ids.insert(CalendarEvent, "/cal/a/", "/cal/a/e1.ics", "v", id(1)) + .unwrap(); + let err = ids + .insert(CalendarEvent, "/cal/a/", "/cal/a/e2.ics", "v", id(1)) + .unwrap_err(); + let msg = err.to_string(); + assert!( + msg.contains("UNIQUE"), + "expected UNIQUE violation, got {msg}" + ); + } +} diff --git a/src/archive/ids/ews.rs b/src/archive/ids/ews.rs new file mode 100644 index 0000000..e9778e0 --- /dev/null +++ b/src/archive/ids/ews.rs @@ -0,0 +1,267 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use rusqlite::{Connection, OptionalExtension, params}; + +use crate::archive::{LocalId, SourceId, Store}; +use crate::types::ObjectType; + +#[derive(Debug, Clone)] +pub struct ItemRow { + pub item_id: String, + pub change_key: String, + pub local_id: LocalId, +} + +#[derive(Debug, Clone)] +pub struct FolderRow { + pub item_id: String, + pub folder_id: String, + pub change_key: String, + pub local_id: LocalId, +} + +#[derive(Clone, Copy)] +pub struct EwsIds<'a> { + conn: &'a Connection, + source: SourceId, +} + +impl<'a> Store<'a> { + pub fn ews_ids(self, source: SourceId) -> EwsIds<'a> { + EwsIds { + conn: self.connection(), + source, + } + } +} + +impl EwsIds<'_> { + pub fn insert( + &self, + ty: ObjectType, + folder_id: &str, + item_id: &str, + change_key: &str, + local: LocalId, + ) -> rusqlite::Result<()> { + self.conn + .prepare_cached( + "INSERT INTO sync_id_exchange_ews + (source_id, type_name, folder_id, item_id, change_key, sync_state, local_id) + VALUES (?1, ?2, ?3, ?4, ?5, '', ?6)", + )? + .execute(params![ + self.source, + ty.token(), + folder_id, + item_id, + change_key, + local + ])?; + Ok(()) + } + + pub fn update_change_key( + &self, + ty: ObjectType, + item_id: &str, + change_key: &str, + ) -> rusqlite::Result<()> { + self.conn + .prepare_cached( + "UPDATE sync_id_exchange_ews SET change_key = ?4 + WHERE source_id = ?1 AND type_name = ?2 AND item_id = ?3", + )? + .execute(params![self.source, ty.token(), item_id, change_key])?; + Ok(()) + } + + pub fn delete(&self, ty: ObjectType, item_id: &str) -> rusqlite::Result<()> { + self.conn + .prepare_cached( + "DELETE FROM sync_id_exchange_ews + WHERE source_id = ?1 AND type_name = ?2 AND item_id = ?3", + )? + .execute(params![self.source, ty.token(), item_id])?; + Ok(()) + } + + pub fn local_id(&self, ty: ObjectType, item_id: &str) -> rusqlite::Result> { + self.conn + .prepare_cached( + "SELECT local_id FROM sync_id_exchange_ews + WHERE source_id = ?1 AND type_name = ?2 AND item_id = ?3", + )? + .query_row(params![self.source, ty.token(), item_id], |row| row.get(0)) + .optional() + } + + pub fn items_in(&self, ty: ObjectType, folder_id: &str) -> rusqlite::Result> { + let mut stmt = self.conn.prepare_cached( + "SELECT item_id, change_key, local_id FROM sync_id_exchange_ews + WHERE source_id = ?1 AND type_name = ?2 AND folder_id = ?3", + )?; + let rows = stmt.query_map(params![self.source, ty.token(), folder_id], |row| { + Ok(ItemRow { + item_id: row.get(0)?, + change_key: row.get(1)?, + local_id: row.get(2)?, + }) + })?; + rows.collect() + } + + pub fn all(&self, ty: ObjectType) -> rusqlite::Result> { + let mut stmt = self.conn.prepare_cached( + "SELECT item_id, folder_id, change_key, local_id FROM sync_id_exchange_ews + WHERE source_id = ?1 AND type_name = ?2", + )?; + let rows = stmt.query_map(params![self.source, ty.token()], |row| { + Ok(FolderRow { + item_id: row.get(0)?, + folder_id: row.get(1)?, + change_key: row.get(2)?, + local_id: row.get(3)?, + }) + })?; + rows.map(|row| row.map(|row| (row.item_id.clone(), row))) + .collect() + } + + pub fn sync_state(&self, ty: ObjectType, item_id: &str) -> rusqlite::Result> { + self.conn + .prepare_cached( + "SELECT sync_state FROM sync_id_exchange_ews + WHERE source_id = ?1 AND type_name = ?2 AND item_id = ?3", + )? + .query_row(params![self.source, ty.token(), item_id], |row| row.get(0)) + .optional() + } + + pub fn set_sync_state( + &self, + ty: ObjectType, + item_id: &str, + sync_state: &str, + ) -> rusqlite::Result<()> { + self.conn + .prepare_cached( + "UPDATE sync_id_exchange_ews SET sync_state = ?4 + WHERE source_id = ?1 AND type_name = ?2 AND item_id = ?3", + )? + .execute(params![self.source, ty.token(), item_id, sync_state])?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::Archive; + use crate::archive::sources::{SourceKey, SourceKind}; + + const MAILBOX: ObjectType = ObjectType::Mailbox; + const EMAIL: ObjectType = ObjectType::Email; + + fn setup() -> (Archive, SourceId) { + let archive = Archive::open_in_memory().unwrap(); + let key = SourceKey { + kind: SourceKind::ExchangeEws, + url: "https://x/EWS/Exchange.asmx", + account_id: "u@d", + }; + let sid = archive + .store() + .sources() + .upsert(&key, Some("u"), "u@d") + .unwrap(); + (archive, sid) + } + + fn id(raw: i64) -> LocalId { + LocalId::from_raw(raw) + } + + #[test] + fn insert_and_lookup_round_trip() { + let (archive, sid) = setup(); + let ids = archive.store().ews_ids(sid); + ids.insert(EMAIL, "FOLDER1", "ITEM1", "CK1", id(42)) + .unwrap(); + assert_eq!(ids.local_id(EMAIL, "ITEM1").unwrap(), Some(id(42))); + let rows = ids.items_in(EMAIL, "FOLDER1").unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].change_key, "CK1"); + let token: String = archive + .connection() + .query_row("SELECT type_name FROM sync_id_exchange_ews", [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!(token, "email"); + } + + #[test] + fn update_change_key_preserves_local_id() { + let (archive, sid) = setup(); + let ids = archive.store().ews_ids(sid); + ids.insert(EMAIL, "F1", "I1", "A", id(7)).unwrap(); + ids.update_change_key(EMAIL, "I1", "B").unwrap(); + let rows = ids.items_in(EMAIL, "F1").unwrap(); + assert_eq!(rows[0].change_key, "B"); + assert_eq!(rows[0].local_id, id(7)); + } + + #[test] + fn all_lists_by_item_id() { + let (archive, sid) = setup(); + let ids = archive.store().ews_ids(sid); + ids.insert(MAILBOX, "", "F1", "CK1", id(1)).unwrap(); + ids.insert(MAILBOX, "F1", "F2", "CK2", id(2)).unwrap(); + let map = ids.all(MAILBOX).unwrap(); + assert_eq!(map.len(), 2); + assert_eq!(map.get("F1").map(|r| r.local_id), Some(id(1))); + assert_eq!(map.get("F2").map(|r| r.folder_id.as_str()), Some("F1")); + } + + #[test] + fn sync_state_round_trips() { + let (archive, sid) = setup(); + let ids = archive.store().ews_ids(sid); + ids.insert(MAILBOX, "", "F1", "", id(1)).unwrap(); + assert_eq!(ids.sync_state(MAILBOX, "F1").unwrap().as_deref(), Some("")); + ids.set_sync_state(MAILBOX, "F1", "opaque-state").unwrap(); + assert_eq!( + ids.sync_state(MAILBOX, "F1").unwrap().as_deref(), + Some("opaque-state") + ); + assert_eq!(ids.sync_state(MAILBOX, "missing").unwrap(), None); + } + + #[test] + fn delete_removes_only_one() { + let (archive, sid) = setup(); + let ids = archive.store().ews_ids(sid); + ids.insert(EMAIL, "F1", "I1", "A", id(1)).unwrap(); + ids.insert(EMAIL, "F1", "I2", "A", id(2)).unwrap(); + ids.delete(EMAIL, "I1").unwrap(); + let rows = ids.items_in(EMAIL, "F1").unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].item_id, "I2"); + } + + #[test] + fn unique_local_id_per_type_is_enforced() { + let (archive, sid) = setup(); + let ids = archive.store().ews_ids(sid); + ids.insert(EMAIL, "F1", "I1", "A", id(1)).unwrap(); + let err = ids.insert(EMAIL, "F1", "I2", "A", id(1)).unwrap_err(); + assert!(err.to_string().contains("UNIQUE")); + } +} diff --git a/src/archive/ids/graph.rs b/src/archive/ids/graph.rs new file mode 100644 index 0000000..cc59c2f --- /dev/null +++ b/src/archive/ids/graph.rs @@ -0,0 +1,105 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use super::{TypeColumn, flat_id_table}; + +flat_id_table!( + GraphIds, + graph_ids, + "sync_id_exchange_graph", + "graph_id", + TypeColumn::Token, + "INSERT" +); + +#[cfg(test)] +mod tests { + use crate::archive::sources::{SourceKey, SourceKind}; + use crate::archive::{Archive, LocalId, SourceId}; + use crate::types::ObjectType; + + fn setup() -> (Archive, SourceId) { + let archive = Archive::open_in_memory().unwrap(); + let key = SourceKey { + kind: SourceKind::ExchangeGraph, + url: "https://x|https://graph.microsoft.com/v1.0", + account_id: "u-uuid", + }; + let sid = archive + .store() + .sources() + .upsert(&key, Some("u@d"), "u@d") + .unwrap(); + (archive, sid) + } + + #[test] + fn insert_and_lookup_round_trip() { + let (archive, sid) = setup(); + let ids = archive.store().graph_ids(sid); + ids.insert(ObjectType::Email, "AAkA-base64==", LocalId::from_raw(17)) + .unwrap(); + let id = ids + .local_id(ObjectType::Email, "AAkA-base64==") + .unwrap() + .unwrap(); + assert_eq!(id, LocalId::from_raw(17)); + } + + #[test] + fn graph_id_is_case_sensitive() { + let (archive, sid) = setup(); + let ids = archive.store().graph_ids(sid); + ids.insert(ObjectType::Email, "AaBb", LocalId::from_raw(1)) + .unwrap(); + assert!(ids.local_id(ObjectType::Email, "aabb").unwrap().is_none()); + assert!(ids.local_id(ObjectType::Email, "AaBb").unwrap().is_some()); + } + + #[test] + fn unique_local_id_per_type_is_enforced() { + let (archive, sid) = setup(); + let ids = archive.store().graph_ids(sid); + ids.insert(ObjectType::Email, "ID-A", LocalId::from_raw(1)) + .unwrap(); + let err = ids + .insert(ObjectType::Email, "ID-B", LocalId::from_raw(1)) + .unwrap_err(); + assert!(err.to_string().contains("UNIQUE")); + } + + #[test] + fn all_filters_by_type() { + let (archive, sid) = setup(); + let ids = archive.store().graph_ids(sid); + ids.insert(ObjectType::Email, "M1", LocalId::from_raw(1)) + .unwrap(); + ids.insert(ObjectType::Email, "M2", LocalId::from_raw(2)) + .unwrap(); + ids.insert(ObjectType::Mailbox, "F1", LocalId::from_raw(3)) + .unwrap(); + let emails = ids.all(ObjectType::Email).unwrap(); + let folders = ids.all(ObjectType::Mailbox).unwrap(); + assert_eq!(emails.len(), 2); + assert_eq!(folders.len(), 1); + assert_eq!(emails.get("M1"), Some(&LocalId::from_raw(1))); + assert_eq!(folders.get("F1"), Some(&LocalId::from_raw(3))); + } + + #[test] + fn delete_removes_only_one() { + let (archive, sid) = setup(); + let ids = archive.store().graph_ids(sid); + ids.insert(ObjectType::Email, "M1", LocalId::from_raw(1)) + .unwrap(); + ids.insert(ObjectType::Email, "M2", LocalId::from_raw(2)) + .unwrap(); + ids.delete(ObjectType::Email, "M1").unwrap(); + let remaining = ids.all(ObjectType::Email).unwrap(); + assert_eq!(remaining.len(), 1); + assert!(remaining.contains_key("M2")); + } +} diff --git a/src/archive/ids/imap.rs b/src/archive/ids/imap.rs new file mode 100644 index 0000000..dacce70 --- /dev/null +++ b/src/archive/ids/imap.rs @@ -0,0 +1,167 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use rusqlite::params; + +use super::folder_id_table; +use crate::archive::LocalId; + +folder_id_table!( + ImapIds, + imap_ids, + "sync_id_imap", + "INSERT OR REPLACE", + (u32, u32), + uidvalidity: u32 => u32 = "0", + uid: u32 => u32 = "0" +); + +impl ImapIds<'_> { + pub fn uids_at_validity(&self, folder: &str, uidvalidity: u32) -> rusqlite::Result> { + let mut stmt = self.conn.prepare_cached( + "SELECT uid FROM sync_id_imap + WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3 + AND uidvalidity = ?4 + ORDER BY uid", + )?; + let rows = stmt.query_map( + params![self.source, Self::ITEM, folder, uidvalidity], + |row| row.get(0), + )?; + rows.collect() + } + + pub fn items_at_validity( + &self, + folder: &str, + uidvalidity: u32, + ) -> rusqlite::Result> { + let mut stmt = self.conn.prepare_cached( + "SELECT uid, local_id FROM sync_id_imap + WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3 + AND uidvalidity = ?4", + )?; + let rows = stmt.query_map( + params![self.source, Self::ITEM, folder, uidvalidity], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + rows.collect() + } +} + +#[cfg(test)] +mod tests { + use crate::archive::sources::{SourceKey, SourceKind}; + use crate::archive::{Archive, LocalId}; + + fn setup() -> Archive { + Archive::open_in_memory().unwrap() + } + + fn ids(archive: &Archive) -> super::ImapIds<'_> { + let sid = archive + .store() + .sources() + .upsert( + &SourceKey { + kind: SourceKind::Imap, + url: "imaps://host:993", + account_id: "alice", + }, + None, + "alice", + ) + .unwrap(); + archive.store().imap_ids(sid) + } + + fn id(raw: i64) -> LocalId { + LocalId::from_raw(raw) + } + + #[test] + fn mailbox_roundtrip() { + let archive = setup(); + let ids = ids(&archive); + ids.insert_folder("INBOX", id(7)).unwrap(); + ids.insert_folder("INBOX/Sub", id(8)).unwrap(); + assert_eq!(ids.folder_id("INBOX").unwrap(), Some(id(7))); + let all = ids.folders().unwrap(); + assert_eq!(all.len(), 2); + assert_eq!(all.get("INBOX/Sub"), Some(&id(8))); + ids.delete_folder("INBOX").unwrap(); + assert_eq!(ids.folder_id("INBOX").unwrap(), None); + } + + #[test] + fn email_roundtrip() { + let archive = setup(); + let ids = ids(&archive); + ids.insert_item("INBOX", 12345, 100, id(1)).unwrap(); + ids.insert_item("INBOX", 12345, 101, id(2)).unwrap(); + ids.insert_item("Sent", 7777, 1, id(3)).unwrap(); + assert_eq!(ids.item_id("INBOX", 12345, 100).unwrap(), Some(id(1))); + let inbox = ids.items_in("INBOX").unwrap(); + assert_eq!(inbox.len(), 2); + assert_eq!(inbox.get(&(12345, 100)), Some(&id(1))); + let folders = ids.folders_with_items().unwrap(); + assert!(folders.contains("INBOX")); + assert!(folders.contains("Sent")); + } + + #[test] + fn items_at_validity_ignores_other_validities() { + let archive = setup(); + let ids = ids(&archive); + ids.insert_item("INBOX", 1, 10, id(1)).unwrap(); + ids.insert_item("INBOX", 2, 11, id(2)).unwrap(); + let at = ids.items_at_validity("INBOX", 2).unwrap(); + assert_eq!(at.len(), 1); + assert_eq!(at.get(&11), Some(&id(2))); + } + + #[test] + fn delete_items_in_wipes_only_that_folder() { + let archive = setup(); + let ids = ids(&archive); + ids.insert_item("INBOX", 1, 10, id(1)).unwrap(); + ids.insert_item("INBOX", 1, 11, id(2)).unwrap(); + ids.insert_item("Sent", 1, 10, id(3)).unwrap(); + assert_eq!(ids.delete_items_in("INBOX").unwrap(), 2); + assert!(ids.items_in("INBOX").unwrap().is_empty()); + assert_eq!(ids.items_in("Sent").unwrap().len(), 1); + } + + #[test] + fn re_insert_with_same_local_id_replaces() { + let archive = setup(); + let ids = ids(&archive); + ids.insert_folder("A", id(1)).unwrap(); + ids.insert_folder("B", id(1)).unwrap(); + assert_eq!(ids.folder_id("A").unwrap(), None); + assert_eq!(ids.folder_id("B").unwrap(), Some(id(1))); + assert_eq!(ids.folders().unwrap().len(), 1); + } + + #[test] + fn pk_composite_is_per_type_per_folder_per_uid() { + let archive = setup(); + let ids = ids(&archive); + ids.insert_item("INBOX", 1, 10, id(1)).unwrap(); + ids.insert_item("INBOX", 1, 10, id(1)).unwrap(); + assert_eq!(ids.items_in("INBOX").unwrap().len(), 1); + } + + #[test] + fn mailbox_and_email_namespaces_do_not_collide_on_local_id() { + let archive = setup(); + let ids = ids(&archive); + ids.insert_folder("INBOX", id(42)).unwrap(); + ids.insert_item("INBOX", 1, 1, id(42)).unwrap(); + } +} diff --git a/src/archive/ids/jmap.rs b/src/archive/ids/jmap.rs new file mode 100644 index 0000000..ae0d56a --- /dev/null +++ b/src/archive/ids/jmap.rs @@ -0,0 +1,85 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ +use super::{TypeColumn, flat_id_table}; + +flat_id_table!( + JmapIds, + jmap_ids, + "sync_id_jmap", + "jmap_id", + TypeColumn::JmapName, + "INSERT OR REPLACE" +); + +#[cfg(test)] +mod tests { + use crate::archive::sources::{SourceKey, SourceKind}; + use crate::archive::{Archive, LocalId}; + use crate::types::ObjectType; + + #[test] + fn crud_roundtrip() { + let archive = Archive::open_in_memory().unwrap(); + let store = archive.store(); + let source = store + .sources() + .upsert( + &SourceKey { + kind: SourceKind::Jmap, + url: "u", + account_id: "w", + }, + None, + "alice", + ) + .unwrap(); + let ids = store.jmap_ids(source); + ids.insert(ObjectType::Mailbox, "M1", LocalId(1)).unwrap(); + ids.insert(ObjectType::Mailbox, "M2", LocalId(2)).unwrap(); + assert_eq!( + ids.local_id(ObjectType::Mailbox, "M1").unwrap(), + Some(LocalId(1)) + ); + let map = ids.all(ObjectType::Mailbox).unwrap(); + assert_eq!(map.len(), 2); + assert_eq!(map.get("M2"), Some(&LocalId(2))); + ids.delete(ObjectType::Mailbox, "M1").unwrap(); + assert_eq!(ids.local_id(ObjectType::Mailbox, "M1").unwrap(), None); + } + + #[test] + fn jmap_ids_store_the_jmap_type_name_and_replace_on_conflict() { + let archive = Archive::open_in_memory().unwrap(); + let store = archive.store(); + let source = store + .sources() + .upsert( + &SourceKey { + kind: SourceKind::Jmap, + url: "u", + account_id: "w", + }, + None, + "alice", + ) + .unwrap(); + let ids = store.jmap_ids(source); + ids.insert(ObjectType::CalendarEvent, "E1", LocalId(7)) + .unwrap(); + ids.insert(ObjectType::CalendarEvent, "E1", LocalId(8)) + .unwrap(); + let stored: String = archive + .connection() + .query_row("SELECT type_name FROM sync_id_jmap", [], |r| r.get(0)) + .unwrap(); + assert_eq!(stored, "CalendarEvent"); + assert_eq!( + ids.local_id(ObjectType::CalendarEvent, "E1").unwrap(), + Some(LocalId(8)) + ); + assert_eq!(ids.all(ObjectType::CalendarEvent).unwrap().len(), 1); + } +} diff --git a/src/archive/ids/maildir.rs b/src/archive/ids/maildir.rs new file mode 100644 index 0000000..f2eea06 --- /dev/null +++ b/src/archive/ids/maildir.rs @@ -0,0 +1,140 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use super::folder_id_table; + +folder_id_table!( + MaildirIds, + maildir_ids, + "sync_id_maildir", + "INSERT", + String, + unique_id: &str => String = "''" +); + +#[cfg(test)] +mod tests { + use crate::archive::sources::{SourceKey, SourceKind}; + use crate::archive::{Archive, LocalId}; + + fn setup() -> Archive { + Archive::open_in_memory().unwrap() + } + + fn ids(archive: &Archive) -> super::MaildirIds<'_> { + let sid = archive + .store() + .sources() + .upsert( + &SourceKey { + kind: SourceKind::Maildir, + url: "file:///home/alice/Maildir", + account_id: "/home/alice/Maildir", + }, + Some("Maildir"), + "", + ) + .unwrap(); + archive.store().maildir_ids(sid) + } + + fn id(raw: i64) -> LocalId { + LocalId::from_raw(raw) + } + + #[test] + fn mailbox_roundtrip() { + let archive = setup(); + let ids = ids(&archive); + ids.insert_folder("INBOX", id(7)).unwrap(); + ids.insert_folder("Archive.2025", id(8)).unwrap(); + assert_eq!(ids.folder_id("INBOX").unwrap(), Some(id(7))); + let all = ids.folders().unwrap(); + assert_eq!(all.len(), 2); + assert_eq!(all.get("Archive.2025"), Some(&id(8))); + ids.delete_folder("INBOX").unwrap(); + assert_eq!(ids.folder_id("INBOX").unwrap(), None); + } + + #[test] + fn email_roundtrip() { + let archive = setup(); + let ids = ids(&archive); + ids.insert_item("INBOX", "1739471123.M001.host", id(1)) + .unwrap(); + ids.insert_item("INBOX", "1739471200.M002.host", id(2)) + .unwrap(); + ids.insert_item("Sent", "1739471300.M003.host", id(3)) + .unwrap(); + assert_eq!( + ids.item_id("INBOX", "1739471123.M001.host").unwrap(), + Some(id(1)) + ); + let inbox = ids.items_in("INBOX").unwrap(); + assert_eq!(inbox.len(), 2); + assert_eq!(inbox.get("1739471123.M001.host"), Some(&id(1))); + let folders = ids.folders_with_items().unwrap(); + assert!(folders.contains("INBOX")); + assert!(folders.contains("Sent")); + } + + #[test] + fn delete_email_removes_only_that_row() { + let archive = setup(); + let ids = ids(&archive); + ids.insert_item("INBOX", "u1", id(1)).unwrap(); + ids.insert_item("INBOX", "u2", id(2)).unwrap(); + ids.delete_item("INBOX", "u1").unwrap(); + let inbox = ids.items_in("INBOX").unwrap(); + assert_eq!(inbox.len(), 1); + assert!(inbox.contains_key("u2")); + } + + #[test] + fn delete_all_emails_in_folder_wipes_only_that_folder() { + let archive = setup(); + let ids = ids(&archive); + ids.insert_item("INBOX", "u1", id(1)).unwrap(); + ids.insert_item("INBOX", "u2", id(2)).unwrap(); + ids.insert_item("Sent", "u3", id(3)).unwrap(); + let removed = ids.delete_items_in("INBOX").unwrap(); + assert_eq!(removed, 2); + assert!(ids.items_in("INBOX").unwrap().is_empty()); + assert_eq!(ids.items_in("Sent").unwrap().len(), 1); + } + + #[test] + fn mailbox_and_email_namespaces_do_not_collide_on_local_id() { + let archive = setup(); + let ids = ids(&archive); + ids.insert_folder("INBOX", id(42)).unwrap(); + ids.insert_item("INBOX", "u1", id(42)).unwrap(); + } + + #[test] + fn re_insert_with_same_unique_id_is_rejected_by_pk() { + let archive = setup(); + let ids = ids(&archive); + ids.insert_item("INBOX", "u1", id(1)).unwrap(); + let err = ids + .insert_item("INBOX", "u1", id(2)) + .expect_err("PK conflict"); + assert!(matches!(err, rusqlite::Error::SqliteFailure(_, _))); + assert_eq!(ids.item_id("INBOX", "u1").unwrap(), Some(id(1))); + assert_eq!(ids.items_in("INBOX").unwrap().len(), 1); + } + + #[test] + fn re_insert_with_same_local_id_is_rejected_by_unique_constraint() { + let archive = setup(); + let ids = ids(&archive); + ids.insert_item("INBOX", "u1", id(42)).unwrap(); + let err = ids + .insert_item("INBOX", "u2", id(42)) + .expect_err("UNIQUE conflict"); + assert!(matches!(err, rusqlite::Error::SqliteFailure(_, _))); + } +} diff --git a/src/archive/ids/mod.rs b/src/archive/ids/mod.rs new file mode 100644 index 0000000..4720d0c --- /dev/null +++ b/src/archive/ids/mod.rs @@ -0,0 +1,381 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod dav; +pub mod ews; +pub mod graph; +pub mod imap; +pub mod jmap; +pub mod maildir; +pub mod sieve; +pub mod takeout; + +use crate::types::ObjectType; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TypeColumn { + JmapName, + Token, +} + +impl TypeColumn { + pub fn encode(self, ty: ObjectType) -> &'static str { + match self { + TypeColumn::JmapName => ty.jmap_name(), + TypeColumn::Token => ty.token(), + } + } +} + +macro_rules! flat_id_table { + ($name:ident, $accessor:ident, $table:literal, $ext:literal, $type_column:expr, $insert:literal) => { + #[derive(Clone, Copy)] + pub struct $name<'a> { + conn: &'a rusqlite::Connection, + source: $crate::archive::SourceId, + } + + impl<'a> $crate::archive::Store<'a> { + pub fn $accessor(self, source: $crate::archive::SourceId) -> $name<'a> { + $name { + conn: self.connection(), + source, + } + } + } + + impl $name<'_> { + const TYPE_COLUMN: $crate::archive::ids::TypeColumn = $type_column; + + pub fn insert( + &self, + ty: $crate::types::ObjectType, + ext: &str, + local: $crate::archive::LocalId, + ) -> rusqlite::Result<()> { + self.conn.execute( + concat!( + $insert, + " INTO ", + $table, + " (source_id, type_name, ", + $ext, + ", local_id) VALUES (?1, ?2, ?3, ?4)" + ), + rusqlite::params![self.source, Self::TYPE_COLUMN.encode(ty), ext, local], + )?; + Ok(()) + } + + pub fn local_id( + &self, + ty: $crate::types::ObjectType, + ext: &str, + ) -> rusqlite::Result> { + rusqlite::OptionalExtension::optional(self.conn.query_row( + concat!( + "SELECT local_id FROM ", + $table, + " WHERE source_id = ?1 AND type_name = ?2 AND ", + $ext, + " = ?3" + ), + rusqlite::params![self.source, Self::TYPE_COLUMN.encode(ty), ext], + |row| row.get(0), + )) + } + + pub fn all( + &self, + ty: $crate::types::ObjectType, + ) -> rusqlite::Result> { + let mut stmt = self.conn.prepare_cached(concat!( + "SELECT ", + $ext, + ", local_id FROM ", + $table, + " WHERE source_id = ?1 AND type_name = ?2" + ))?; + let rows = stmt.query_map( + rusqlite::params![self.source, Self::TYPE_COLUMN.encode(ty)], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + rows.collect() + } + + pub fn delete(&self, ty: $crate::types::ObjectType, ext: &str) -> rusqlite::Result<()> { + self.conn.execute( + concat!( + "DELETE FROM ", + $table, + " WHERE source_id = ?1 AND type_name = ?2 AND ", + $ext, + " = ?3" + ), + rusqlite::params![self.source, Self::TYPE_COLUMN.encode(ty), ext], + )?; + Ok(()) + } + } + }; +} + +macro_rules! folder_id_table { + ( + $name:ident, + $accessor:ident, + $table:literal, + $insert:literal, + $key:ty, + $($col:ident: $param:ty => $owned:ty = $placeholder:literal),+ + ) => { + #[derive(Clone, Copy)] + pub struct $name<'a> { + conn: &'a rusqlite::Connection, + source: $crate::archive::SourceId, + } + + impl<'a> $crate::archive::Store<'a> { + pub fn $accessor(self, source: $crate::archive::SourceId) -> $name<'a> { + $name { + conn: self.connection(), + source, + } + } + } + + impl $name<'_> { + const FOLDER: &'static str = $crate::types::ObjectType::Mailbox.token(); + const ITEM: &'static str = $crate::types::ObjectType::Email.token(); + + pub fn insert_folder( + &self, + folder: &str, + local: $crate::archive::LocalId, + ) -> rusqlite::Result<()> { + self.conn.execute( + concat!( + $insert, " INTO ", $table, + " (source_id, type_name, folder, ", $(stringify!($col), ", ",)+ + "local_id) VALUES (?1, ?2, ?3, ", $($placeholder, ", ",)+ "?4)" + ), + rusqlite::params![self.source, Self::FOLDER, folder, local], + )?; + Ok(()) + } + + pub fn insert_item( + &self, + folder: &str, + $($col: $param,)+ + local: $crate::archive::LocalId, + ) -> rusqlite::Result<()> { + self.conn.prepare_cached( + concat!( + $insert, " INTO ", $table, + " (source_id, type_name, folder, ", $(stringify!($col), ", ",)+ + "local_id) VALUES (:source, :type, :folder, ", + $(":", stringify!($col), ", ",)+ ":local)" + ), + )?.execute(&[ + (":source", &self.source as &dyn rusqlite::ToSql), + (":type", &Self::ITEM), + (":folder", &folder), + $((concat!(":", stringify!($col)), &$col),)+ + (":local", &local), + ][..])?; + Ok(()) + } + + pub fn folder_id( + &self, + folder: &str, + ) -> rusqlite::Result> { + rusqlite::OptionalExtension::optional(self.conn.query_row( + concat!( + "SELECT local_id FROM ", $table, + " WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3", + $(" AND ", stringify!($col), " = ", $placeholder,)+ + ), + rusqlite::params![self.source, Self::FOLDER, folder], + |row| row.get(0), + )) + } + + pub fn item_id( + &self, + folder: &str, + $($col: $param,)+ + ) -> rusqlite::Result> { + rusqlite::OptionalExtension::optional(self.conn.prepare_cached( + concat!( + "SELECT local_id FROM ", $table, + " WHERE source_id = :source AND type_name = :type AND folder = :folder", + $(" AND ", stringify!($col), " = :", stringify!($col),)+ + ), + )?.query_row(&[ + (":source", &self.source as &dyn rusqlite::ToSql), + (":type", &Self::ITEM), + (":folder", &folder), + $((concat!(":", stringify!($col)), &$col),)+ + ][..], + |row| row.get(0), + )) + } + + pub fn folders( + &self, + ) -> rusqlite::Result> { + let mut stmt = self.conn.prepare_cached(concat!( + "SELECT folder, local_id FROM ", $table, + " WHERE source_id = ?1 AND type_name = ?2", + $(" AND ", stringify!($col), " = ", $placeholder,)+ + ))?; + let rows = stmt.query_map(rusqlite::params![self.source, Self::FOLDER], |row| { + Ok((row.get(0)?, row.get(1)?)) + })?; + rows.collect() + } + + pub fn items_in( + &self, + folder: &str, + ) -> rusqlite::Result> { + let mut stmt = self.conn.prepare_cached(concat!( + "SELECT ", $(stringify!($col), ", ",)+ "local_id FROM ", $table, + " WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3" + ))?; + let rows = stmt.query_map( + rusqlite::params![self.source, Self::ITEM, folder], + |row| { + Ok(( + ($(row.get::<_, $owned>(stringify!($col))?),+), + row.get("local_id")?, + )) + }, + )?; + rows.collect() + } + + pub fn delete_folder(&self, folder: &str) -> rusqlite::Result<()> { + self.conn.execute( + concat!( + "DELETE FROM ", $table, + " WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3", + $(" AND ", stringify!($col), " = ", $placeholder,)+ + ), + rusqlite::params![self.source, Self::FOLDER, folder], + )?; + Ok(()) + } + + pub fn delete_item(&self, folder: &str, $($col: $param,)+) -> rusqlite::Result<()> { + self.conn.prepare_cached( + concat!( + "DELETE FROM ", $table, + " WHERE source_id = :source AND type_name = :type AND folder = :folder", + $(" AND ", stringify!($col), " = :", stringify!($col),)+ + ), + )?.execute(&[ + (":source", &self.source as &dyn rusqlite::ToSql), + (":type", &Self::ITEM), + (":folder", &folder), + $((concat!(":", stringify!($col)), &$col),)+ + ][..])?; + Ok(()) + } + + pub fn delete_items_in(&self, folder: &str) -> rusqlite::Result { + self.conn.execute( + concat!( + "DELETE FROM ", $table, + " WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3" + ), + rusqlite::params![self.source, Self::ITEM, folder], + ) + } + + pub fn folders_with_items( + &self, + ) -> rusqlite::Result> { + let mut stmt = self.conn.prepare_cached(concat!( + "SELECT DISTINCT folder FROM ", $table, + " WHERE source_id = ?1 AND type_name = ?2" + ))?; + let rows = + stmt.query_map(rusqlite::params![self.source, Self::ITEM], |row| row.get(0))?; + rows.collect() + } + } + }; +} + +pub(crate) use flat_id_table; +pub(crate) use folder_id_table; + +#[cfg(test)] +mod tests { + use crate::archive::sources::{SourceKey, SourceKind}; + use crate::archive::{Archive, LocalId, SourceId}; + + fn source(archive: &Archive, kind: SourceKind) -> SourceId { + archive + .store() + .sources() + .upsert( + &SourceKey { + kind, + url: "u", + account_id: "a", + }, + None, + "a", + ) + .unwrap() + } + + #[test] + fn folder_tables_keep_folders_and_items_apart() { + let archive = Archive::open_in_memory().unwrap(); + let ids = archive.store().imap_ids(source(&archive, SourceKind::Imap)); + ids.insert_folder("INBOX", LocalId::from_raw(1)).unwrap(); + ids.insert_item("INBOX", 9, 4, LocalId::from_raw(10)) + .unwrap(); + ids.insert_item("INBOX", 9, 2, LocalId::from_raw(11)) + .unwrap(); + ids.insert_item("INBOX", 9, 4, LocalId::from_raw(12)) + .unwrap(); + assert_eq!(ids.folder_id("INBOX").unwrap(), Some(LocalId::from_raw(1))); + assert_eq!( + ids.item_id("INBOX", 9, 4).unwrap(), + Some(LocalId::from_raw(12)) + ); + assert_eq!(ids.folders().unwrap().len(), 1); + assert_eq!(ids.items_in("INBOX").unwrap().len(), 2); + assert_eq!(ids.uids_at_validity("INBOX", 9).unwrap(), [2, 4]); + assert_eq!(ids.folders_with_items().unwrap().len(), 1); + ids.delete_item("INBOX", 9, 2).unwrap(); + assert_eq!(ids.delete_items_in("INBOX").unwrap(), 1); + ids.delete_folder("INBOX").unwrap(); + assert!(ids.folders().unwrap().is_empty()); + } + + #[test] + fn maildir_items_fail_on_conflict() { + let archive = Archive::open_in_memory().unwrap(); + let ids = archive + .store() + .maildir_ids(source(&archive, SourceKind::Maildir)); + ids.insert_folder("", LocalId::from_raw(1)).unwrap(); + ids.insert_item("", "uniq", LocalId::from_raw(2)).unwrap(); + assert!(ids.insert_item("", "uniq", LocalId::from_raw(3)).is_err()); + assert_eq!( + ids.items_in("").unwrap().get("uniq"), + Some(&LocalId::from_raw(2)) + ); + assert_eq!(ids.folder_id("").unwrap(), Some(LocalId::from_raw(1))); + } +} diff --git a/src/archive/ids/sieve.rs b/src/archive/ids/sieve.rs new file mode 100644 index 0000000..1704982 --- /dev/null +++ b/src/archive/ids/sieve.rs @@ -0,0 +1,139 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use rusqlite::{Connection, OptionalExtension, params}; + +use crate::archive::{LocalId, SourceId, Store}; + +#[derive(Clone, Copy)] +pub struct SieveIds<'a> { + conn: &'a Connection, + source: SourceId, +} + +impl<'a> Store<'a> { + pub fn sieve_ids(self, source: SourceId) -> SieveIds<'a> { + SieveIds { + conn: self.connection(), + source, + } + } +} + +impl SieveIds<'_> { + pub fn insert(&self, name: &str, local: LocalId) -> rusqlite::Result<()> { + self.conn.execute( + "INSERT OR REPLACE INTO sync_id_managesieve (source_id, name, local_id) + VALUES (?1, ?2, ?3)", + params![self.source, name, local], + )?; + Ok(()) + } + + pub fn local_id(&self, name: &str) -> rusqlite::Result> { + self.conn + .query_row( + "SELECT local_id FROM sync_id_managesieve WHERE source_id = ?1 AND name = ?2", + params![self.source, name], + |row| row.get(0), + ) + .optional() + } + + pub fn all(&self) -> rusqlite::Result> { + let mut stmt = self.conn.prepare_cached( + "SELECT name, local_id FROM sync_id_managesieve WHERE source_id = ?1", + )?; + let rows = stmt.query_map([self.source], |row| Ok((row.get(0)?, row.get(1)?)))?; + rows.collect() + } + + pub fn delete(&self, name: &str) -> rusqlite::Result<()> { + self.conn.execute( + "DELETE FROM sync_id_managesieve WHERE source_id = ?1 AND name = ?2", + params![self.source, name], + )?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use crate::archive::sources::{SourceKey, SourceKind}; + use crate::archive::{Archive, LocalId}; + + fn ids(archive: &Archive) -> super::SieveIds<'_> { + let sid = archive + .store() + .sources() + .upsert( + &SourceKey { + kind: SourceKind::ManageSieve, + url: "sieve://host:4190", + account_id: "alice", + }, + None, + "alice", + ) + .unwrap(); + archive.store().sieve_ids(sid) + } + + fn id(raw: i64) -> LocalId { + LocalId::from_raw(raw) + } + + #[test] + fn insert_and_lookup_by_name() { + let archive = Archive::open_in_memory().unwrap(); + let ids = ids(&archive); + ids.insert("vacation", id(7)).unwrap(); + assert_eq!(ids.local_id("vacation").unwrap(), Some(id(7))); + assert_eq!(ids.local_id("missing").unwrap(), None); + } + + #[test] + fn all_returns_full_map() { + let archive = Archive::open_in_memory().unwrap(); + let ids = ids(&archive); + ids.insert("a", id(1)).unwrap(); + ids.insert("b", id(2)).unwrap(); + let map = ids.all().unwrap(); + assert_eq!(map.len(), 2); + assert_eq!(map.get("a"), Some(&id(1))); + } + + #[test] + fn delete_removes_row() { + let archive = Archive::open_in_memory().unwrap(); + let ids = ids(&archive); + ids.insert("a", id(1)).unwrap(); + ids.delete("a").unwrap(); + assert_eq!(ids.local_id("a").unwrap(), None); + } + + #[test] + fn re_insert_with_same_name_replaces_local_id() { + let archive = Archive::open_in_memory().unwrap(); + let ids = ids(&archive); + ids.insert("a", id(1)).unwrap(); + ids.insert("a", id(2)).unwrap(); + assert_eq!(ids.local_id("a").unwrap(), Some(id(2))); + assert_eq!(ids.all().unwrap().len(), 1); + } + + #[test] + fn insert_or_replace_reassigns_when_local_id_clashes() { + let archive = Archive::open_in_memory().unwrap(); + let ids = ids(&archive); + ids.insert("a", id(1)).unwrap(); + ids.insert("b", id(1)).unwrap(); + assert!(ids.local_id("a").unwrap().is_none()); + assert_eq!(ids.local_id("b").unwrap(), Some(id(1))); + } +} diff --git a/src/archive/ids/takeout.rs b/src/archive/ids/takeout.rs new file mode 100644 index 0000000..bad465c --- /dev/null +++ b/src/archive/ids/takeout.rs @@ -0,0 +1,163 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::params; + +use super::{TypeColumn, flat_id_table}; +use crate::types::ObjectType; + +flat_id_table!( + TakeoutIds, + takeout_ids, + "sync_id_takeout", + "source_obj_id", + TypeColumn::Token, + "INSERT" +); + +impl TakeoutIds<'_> { + pub fn count(&self, ty: ObjectType) -> rusqlite::Result { + let count: i64 = self.conn.query_row( + "SELECT COUNT(*) FROM sync_id_takeout WHERE source_id = ?1 AND type_name = ?2", + params![self.source, Self::TYPE_COLUMN.encode(ty)], + |row| row.get(0), + )?; + Ok(count.unsigned_abs()) + } +} + +#[cfg(test)] +mod tests { + use crate::archive::sources::{SourceKey, SourceKind}; + use crate::archive::{Archive, LocalId}; + use crate::types::ObjectType; + + fn setup() -> Archive { + Archive::open_in_memory().unwrap() + } + + fn ids(archive: &Archive) -> super::TakeoutIds<'_> { + let sid = archive + .store() + .sources() + .upsert( + &SourceKey { + kind: SourceKind::Takeout, + url: "file:///tmp/Takeout", + account_id: "/tmp/Takeout", + }, + Some("Takeout"), + "", + ) + .unwrap(); + archive.store().takeout_ids(sid) + } + + fn id(raw: i64) -> LocalId { + LocalId::from_raw(raw) + } + + #[test] + fn insert_and_lookup_roundtrip() { + let archive = setup(); + let ids = ids(&archive); + ids.insert(ObjectType::Mailbox, "Inbox", id(7)).unwrap(); + ids.insert(ObjectType::Mailbox, "Label/Sub", id(8)).unwrap(); + assert_eq!( + ids.local_id(ObjectType::Mailbox, "Inbox").unwrap(), + Some(id(7)) + ); + assert_eq!( + ids.local_id(ObjectType::Mailbox, "Label/Sub").unwrap(), + Some(id(8)) + ); + assert_eq!(ids.local_id(ObjectType::Mailbox, "Missing").unwrap(), None); + } + + #[test] + fn stored_type_names_are_unchanged() { + let archive = setup(); + let ids = ids(&archive); + let types = [ + (ObjectType::Mailbox, "mailbox"), + (ObjectType::Email, "email"), + (ObjectType::AddressBook, "addressbook"), + (ObjectType::ContactCard, "contactcard"), + (ObjectType::Calendar, "calendar"), + (ObjectType::CalendarEvent, "calendarevent"), + ]; + for (n, (ty, _)) in (1..).zip(types) { + ids.insert(ty, "x", id(n)).unwrap(); + } + for (ty, stored) in types { + let n: i64 = archive + .connection() + .query_row( + "SELECT COUNT(*) FROM sync_id_takeout WHERE type_name = ?1", + [stored], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(n, 1, "{stored}"); + assert_eq!(ids.all(ty).unwrap().len(), 1); + assert_eq!(ids.count(ty).unwrap(), 1); + } + } + + #[test] + fn delete_removes_only_target_row() { + let archive = setup(); + let ids = ids(&archive); + ids.insert(ObjectType::Email, "hash-a", id(1)).unwrap(); + ids.insert(ObjectType::Email, "hash-b", id(2)).unwrap(); + ids.delete(ObjectType::Email, "hash-a").unwrap(); + assert_eq!(ids.local_id(ObjectType::Email, "hash-a").unwrap(), None); + assert_eq!( + ids.local_id(ObjectType::Email, "hash-b").unwrap(), + Some(id(2)) + ); + } + + #[test] + fn pk_rejects_duplicate_source_obj_id() { + let archive = setup(); + let ids = ids(&archive); + ids.insert(ObjectType::Email, "h", id(1)).unwrap(); + let err = ids.insert(ObjectType::Email, "h", id(2)).unwrap_err(); + assert!(matches!(err, rusqlite::Error::SqliteFailure(_, _))); + } + + #[test] + fn unique_constraint_rejects_duplicate_local_id_within_type() { + let archive = setup(); + let ids = ids(&archive); + ids.insert(ObjectType::Email, "h1", id(42)).unwrap(); + let err = ids.insert(ObjectType::Email, "h2", id(42)).unwrap_err(); + assert!(matches!(err, rusqlite::Error::SqliteFailure(_, _))); + } + + #[test] + fn same_local_id_allowed_across_different_types() { + let archive = setup(); + let ids = ids(&archive); + ids.insert(ObjectType::Mailbox, "Inbox", id(42)).unwrap(); + ids.insert(ObjectType::Email, "h1", id(42)).unwrap(); + } + + #[test] + fn all_returns_full_map() { + let archive = setup(); + let ids = ids(&archive); + for (n, hash) in (1..).zip(["h1", "h2", "h3"]) { + ids.insert(ObjectType::Email, hash, id(n)).unwrap(); + } + let all = ids.all(ObjectType::Email).unwrap(); + assert_eq!(all.len(), 3); + assert_eq!(all.get("h1"), Some(&id(1))); + assert_eq!(all.get("h3"), Some(&id(3))); + assert_eq!(ids.count(ObjectType::Email).unwrap(), 3); + } +} diff --git a/src/archive/init.rs b/src/archive/init.rs new file mode 100644 index 0000000..3ffe1a4 --- /dev/null +++ b/src/archive/init.rs @@ -0,0 +1,107 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::path::Path; + +use rusqlite::Connection; + +use crate::archive::{Archive, ArchiveError}; + +const SCHEMA_SQL: &str = include_str!("schema.sql"); + +impl Archive { + pub(super) fn connect(path: &Path) -> Result { + let conn = Connection::open(path)?; + Archive::apply_pragmas(&conn)?; + Archive::apply_schema(&conn)?; + Ok(conn) + } + + pub(super) fn apply_schema(conn: &Connection) -> Result<(), ArchiveError> { + let tx = conn.unchecked_transaction()?; + tx.execute_batch(SCHEMA_SQL)?; + Archive::ensure_calendar_events_data_type(&tx)?; + Archive::ensure_file_nodes_executable(&tx)?; + Archive::ensure_graph_ids_accept_file_nodes(&tx)?; + tx.commit()?; + Ok(()) + } + + fn ensure_calendar_events_data_type(conn: &Connection) -> Result<(), ArchiveError> { + Archive::ensure_column( + conn, + "calendar_events", + "data_type", + "ALTER TABLE calendar_events ADD COLUMN data_type TEXT NOT NULL DEFAULT 'Event'", + ) + } + + fn ensure_file_nodes_executable(conn: &Connection) -> Result<(), ArchiveError> { + Archive::ensure_column( + conn, + "file_nodes", + "executable", + "ALTER TABLE file_nodes ADD COLUMN executable INTEGER NOT NULL DEFAULT 0", + ) + } + + fn ensure_column( + conn: &Connection, + table: &str, + column: &str, + alter: &str, + ) -> Result<(), ArchiveError> { + let mut stmt = conn.prepare(&format!("PRAGMA table_info({table})"))?; + let rows = stmt.query_map([], |row| row.get::<_, String>(1))?; + let has_column = rows.filter_map(|r| r.ok()).any(|name| name == column); + if !has_column { + conn.execute(alter, [])?; + } + Ok(()) + } + + fn ensure_graph_ids_accept_file_nodes(conn: &Connection) -> Result<(), ArchiveError> { + let sql: Option = conn + .query_row( + "SELECT sql FROM sqlite_master WHERE type = 'table' \ + AND name = 'sync_id_exchange_graph'", + [], + |row| row.get(0), + ) + .ok(); + let Some(sql) = sql else { return Ok(()) }; + if sql.contains("'filenode'") { + return Ok(()); + } + conn.execute_batch( + "ALTER TABLE sync_id_exchange_graph RENAME TO sync_id_exchange_graph_old; + CREATE TABLE sync_id_exchange_graph ( + source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + type_name TEXT NOT NULL CHECK (type_name IN ( + 'mailbox','email', + 'calendar','calendarevent', + 'addressbook','contactcard', + 'filenode')), + graph_id TEXT NOT NULL, + local_id INTEGER NOT NULL, + PRIMARY KEY (source_id, type_name, graph_id), + UNIQUE (source_id, type_name, local_id) + ); + INSERT INTO sync_id_exchange_graph SELECT * FROM sync_id_exchange_graph_old; + DROP TABLE sync_id_exchange_graph_old; + CREATE INDEX IF NOT EXISTS sync_id_exchange_graph_type_idx + ON sync_id_exchange_graph (source_id, type_name);", + )?; + Ok(()) + } + + fn apply_pragmas(conn: &Connection) -> Result<(), ArchiveError> { + conn.pragma_update(None, "journal_mode", "WAL")?; + conn.pragma_update(None, "foreign_keys", "ON")?; + conn.pragma_update(None, "locking_mode", "EXCLUSIVE")?; + Ok(()) + } +} diff --git a/src/archive/mod.rs b/src/archive/mod.rs new file mode 100644 index 0000000..6c3e01e --- /dev/null +++ b/src/archive/mod.rs @@ -0,0 +1,188 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod blobs; +mod defaults; +pub mod ids; +mod init; +pub mod rows; +pub mod sources; +pub mod state; + +use std::fmt; +use std::path::Path; + +use rusqlite::types::{FromSql, FromSqlResult, ToSql, ToSqlOutput, ValueRef}; +use rusqlite::{Connection, Transaction}; + +#[derive(Debug, thiserror::Error)] +pub enum ArchiveError { + #[error("sqlite error: {0}")] + Sqlite(#[from] rusqlite::Error), +} + +macro_rules! row_id { + ($($name:ident),+) => { + $( + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] + pub struct $name(i64); + + impl $name { + pub fn get(self) -> i64 { + self.0 + } + } + + impl ToSql for $name { + fn to_sql(&self) -> rusqlite::Result> { + Ok(ToSqlOutput::from(self.0)) + } + } + + impl FromSql for $name { + fn column_result(value: ValueRef<'_>) -> FromSqlResult { + i64::column_result(value).map($name) + } + } + + impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } + } + )+ + }; +} + +row_id!(SourceId, LocalId, BlobId); + +impl LocalId { + pub fn from_raw(raw: i64) -> LocalId { + LocalId(raw) + } +} + +impl BlobId { + pub fn from_raw(raw: i64) -> BlobId { + BlobId(raw) + } +} + +pub struct Archive { + conn: Connection, +} + +impl Archive { + pub fn open(path: impl AsRef) -> Result { + Archive::open_inner(path.as_ref()) + } + + fn open_inner(path: &Path) -> Result { + Ok(Archive { + conn: Archive::connect(path)?, + }) + } + + pub fn open_in_memory() -> Result { + let conn = Connection::open_in_memory()?; + conn.pragma_update(None, "foreign_keys", "ON")?; + Archive::apply_schema(&conn)?; + Ok(Archive { conn }) + } + + pub fn store(&self) -> Store<'_> { + Store { conn: &self.conn } + } + + pub fn transaction(&mut self) -> rusqlite::Result> { + self.conn.transaction().map(Tx) + } + + pub fn connection(&self) -> &Connection { + &self.conn + } +} + +pub struct Tx<'a>(Transaction<'a>); + +impl Tx<'_> { + pub fn store(&self) -> Store<'_> { + Store { conn: &self.0 } + } + + pub fn commit(self) -> rusqlite::Result<()> { + self.0.commit() + } +} + +#[derive(Clone, Copy)] +pub struct Store<'a> { + conn: &'a Connection, +} + +impl<'a> Store<'a> { + pub(crate) fn new(conn: &'a Connection) -> Store<'a> { + Store { conn } + } + + pub fn begin(self) -> rusqlite::Result> { + self.conn.unchecked_transaction().map(Tx) + } + + pub fn connection(self) -> &'a Connection { + self.conn + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn in_memory_archive_has_the_schema() { + let archive = Archive::open_in_memory().unwrap(); + let n: i64 = archive + .connection() + .query_row( + "SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name = 'mailboxes'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(n, 1); + } + + #[test] + fn dropped_transaction_rolls_back() { + let mut archive = Archive::open_in_memory().unwrap(); + { + let tx = archive.transaction().unwrap(); + tx.store() + .connection() + .execute( + "INSERT INTO identities (name, email) VALUES ('a', 'a@example.com')", + [], + ) + .unwrap(); + } + let n: i64 = archive + .connection() + .query_row("SELECT count(*) FROM identities", [], |r| r.get(0)) + .unwrap(); + assert_eq!(n, 0); + } + + #[test] + fn ids_round_trip_through_sql() { + let archive = Archive::open_in_memory().unwrap(); + let id: LocalId = archive + .connection() + .query_row("SELECT ?1", [LocalId::from_raw(42)], |r| r.get(0)) + .unwrap(); + assert_eq!(id, LocalId::from_raw(42)); + assert_eq!(id.to_string(), "42"); + } +} diff --git a/src/archive/rows/address_books.rs b/src/archive/rows/address_books.rs new file mode 100644 index 0000000..f3646d7 --- /dev/null +++ b/src/archive/rows/address_books.rs @@ -0,0 +1,98 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::params; + +use super::{AddressBooks, Subscribed, Write}; +use crate::archive::{LocalId, Store}; +use crate::types::ObjectType; + +#[derive(Debug, Clone, Copy)] +pub struct AddressBookRecord<'r> { + pub name: &'r str, + pub description: Option<&'r str>, + pub sort_order: i64, + pub is_default: bool, + pub subscribed: Subscribed, +} + +impl AddressBooks<'_> { + pub fn insert(&self, a: &AddressBookRecord<'_>) -> rusqlite::Result { + let is_default = + Store::new(self.conn).unique_default(ObjectType::AddressBook, a.is_default, None)?; + self.conn.execute( + "INSERT INTO address_books (name, description, sort_order, is_default, is_subscribed) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + a.name, + a.description, + a.sort_order, + is_default, + a.subscribed + ], + )?; + Ok(LocalId(self.conn.last_insert_rowid())) + } + + pub fn update(&self, id: LocalId, a: &AddressBookRecord<'_>) -> rusqlite::Result { + let is_default = Store::new(self.conn).unique_default( + ObjectType::AddressBook, + a.is_default, + Some(id), + )?; + self.conn + .execute( + "UPDATE address_books SET name = ?1, description = ?2, sort_order = ?3, + is_default = ?4, is_subscribed = ?5 + WHERE id = ?6 AND (name IS NOT ?1 OR description IS NOT ?2 OR sort_order IS NOT ?3 + OR is_default IS NOT ?4 OR is_subscribed IS NOT ?5)", + params![ + a.name, + a.description, + a.sort_order, + is_default, + a.subscribed, + id + ], + ) + .map(Write::from_rows) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::Archive; + + #[test] + fn second_default_is_demoted() { + let archive = Archive::open_in_memory().unwrap(); + let books = archive.store().address_books(); + let record = AddressBookRecord { + name: "Personal", + description: None, + sort_order: 0, + is_default: true, + subscribed: Subscribed::Yes, + }; + let first = books.insert(&record).unwrap(); + let second = books.insert(&record).unwrap(); + let is_default = |id: LocalId| -> bool { + archive + .connection() + .query_row( + "SELECT is_default FROM address_books WHERE id = ?1", + [id], + |r| r.get(0), + ) + .unwrap() + }; + assert!(is_default(first)); + assert!(!is_default(second)); + assert_eq!(books.update(first, &record).unwrap(), Write::Unchanged); + books.delete(second).unwrap(); + } +} diff --git a/src/archive/rows/calendar_events.rs b/src/archive/rows/calendar_events.rs new file mode 100644 index 0000000..bfe8984 --- /dev/null +++ b/src/archive/rows/calendar_events.rs @@ -0,0 +1,115 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::params; +use rusqlite::types::{ToSql, ToSqlOutput}; + +use super::{CalendarEvents, LocalIdArray, Write}; +use crate::archive::LocalId; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EventDataType { + Event, + Task, + Note, + Group, +} + +impl EventDataType { + pub fn as_str(self) -> &'static str { + match self { + EventDataType::Event => "Event", + EventDataType::Task => "Task", + EventDataType::Note => "Note", + EventDataType::Group => "Group", + } + } +} + +impl ToSql for EventDataType { + fn to_sql(&self) -> rusqlite::Result> { + Ok(ToSqlOutput::from(self.as_str())) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct CalendarEventRecord<'r> { + pub calendars: &'r [LocalId], + pub is_draft: bool, + pub use_default_alerts: bool, + pub data_type: EventDataType, + pub data: &'r str, +} + +impl CalendarEvents<'_> { + pub fn insert(&self, e: &CalendarEventRecord<'_>) -> rusqlite::Result { + self.conn.execute( + "INSERT INTO calendar_events (calendar_ids, is_draft, use_default_alerts, data, data_type) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + LocalIdArray(e.calendars), + e.is_draft, + e.use_default_alerts, + e.data, + e.data_type + ], + )?; + Ok(LocalId(self.conn.last_insert_rowid())) + } + + pub fn update(&self, id: LocalId, e: &CalendarEventRecord<'_>) -> rusqlite::Result { + self.conn + .execute( + "UPDATE calendar_events SET calendar_ids = ?1, is_draft = ?2, + use_default_alerts = ?3, data = ?4, data_type = ?5 + WHERE id = ?6 AND (calendar_ids IS NOT ?1 OR is_draft IS NOT ?2 + OR use_default_alerts IS NOT ?3 OR data IS NOT ?4 OR data_type IS NOT ?5)", + params![ + LocalIdArray(e.calendars), + e.is_draft, + e.use_default_alerts, + e.data, + e.data_type, + id + ], + ) + .map(Write::from_rows) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::Archive; + + #[test] + fn insert_update_delete() { + let archive = Archive::open_in_memory().unwrap(); + let events = archive.store().calendar_events(); + let calendars = [LocalId::from_raw(1), LocalId::from_raw(2)]; + let mut record = CalendarEventRecord { + calendars: &calendars, + is_draft: false, + use_default_alerts: false, + data_type: EventDataType::Event, + data: r#"{"uid":"e1"}"#, + }; + let id = events.insert(&record).unwrap(); + assert_eq!(events.update(id, &record).unwrap(), Write::Unchanged); + record.data_type = EventDataType::Task; + assert_eq!(events.update(id, &record).unwrap(), Write::Changed); + let stored: String = archive + .connection() + .query_row( + "SELECT data_type FROM calendar_events WHERE id = ?1", + [id], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(stored, "Task"); + events.delete(id).unwrap(); + } +} diff --git a/src/archive/rows/calendars.rs b/src/archive/rows/calendars.rs new file mode 100644 index 0000000..c45d7f7 --- /dev/null +++ b/src/archive/rows/calendars.rs @@ -0,0 +1,115 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::params; + +use super::{Calendars, Subscribed, Write}; +use crate::archive::{LocalId, Store}; +use crate::types::ObjectType; + +#[derive(Debug, Clone, Copy)] +pub struct CalendarRecord<'r> { + pub name: &'r str, + pub description: Option<&'r str>, + pub color: Option<&'r str>, + pub sort_order: i64, + pub subscribed: Subscribed, + pub is_visible: bool, + pub is_default: bool, + pub include_in_availability: &'r str, + pub default_alerts_with_time: Option<&'r str>, + pub default_alerts_without_time: Option<&'r str>, + pub time_zone: Option<&'r str>, +} + +impl Calendars<'_> { + pub fn insert(&self, c: &CalendarRecord<'_>) -> rusqlite::Result { + let is_default = + Store::new(self.conn).unique_default(ObjectType::Calendar, c.is_default, None)?; + self.conn.execute( + "INSERT INTO calendars (name, description, color, sort_order, is_subscribed, is_visible, + is_default, include_in_availability, default_alerts_with_time, + default_alerts_without_time, time_zone) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", + params![ + c.name, + c.description, + c.color, + c.sort_order, + c.subscribed, + c.is_visible, + is_default, + c.include_in_availability, + c.default_alerts_with_time, + c.default_alerts_without_time, + c.time_zone + ], + )?; + Ok(LocalId(self.conn.last_insert_rowid())) + } + + pub fn update(&self, id: LocalId, c: &CalendarRecord<'_>) -> rusqlite::Result { + let is_default = + Store::new(self.conn).unique_default(ObjectType::Calendar, c.is_default, Some(id))?; + self.conn + .execute( + "UPDATE calendars SET name = ?1, description = ?2, color = ?3, sort_order = ?4, + is_subscribed = ?5, is_visible = ?6, is_default = ?7, + include_in_availability = ?8, default_alerts_with_time = ?9, + default_alerts_without_time = ?10, time_zone = ?11 + WHERE id = ?12 AND (name IS NOT ?1 OR description IS NOT ?2 OR color IS NOT ?3 + OR sort_order IS NOT ?4 OR is_subscribed IS NOT ?5 OR is_visible IS NOT ?6 + OR is_default IS NOT ?7 OR include_in_availability IS NOT ?8 + OR default_alerts_with_time IS NOT ?9 OR default_alerts_without_time IS NOT ?10 + OR time_zone IS NOT ?11)", + params![ + c.name, + c.description, + c.color, + c.sort_order, + c.subscribed, + c.is_visible, + is_default, + c.include_in_availability, + c.default_alerts_with_time, + c.default_alerts_without_time, + c.time_zone, + id + ], + ) + .map(Write::from_rows) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::Archive; + + #[test] + fn insert_update_delete() { + let archive = Archive::open_in_memory().unwrap(); + let calendars = archive.store().calendars(); + let mut record = CalendarRecord { + name: "Work", + description: None, + color: Some("#ff0000"), + sort_order: 1, + subscribed: Subscribed::Yes, + is_visible: true, + is_default: false, + include_in_availability: "all", + default_alerts_with_time: None, + default_alerts_without_time: None, + time_zone: Some("Europe/Paris"), + }; + let id = calendars.insert(&record).unwrap(); + assert_eq!(calendars.update(id, &record).unwrap(), Write::Unchanged); + record.subscribed = Subscribed::No; + assert_eq!(calendars.update(id, &record).unwrap(), Write::Changed); + calendars.delete(id).unwrap(); + } +} diff --git a/src/archive/rows/contact_cards.rs b/src/archive/rows/contact_cards.rs new file mode 100644 index 0000000..bc8c320 --- /dev/null +++ b/src/archive/rows/contact_cards.rs @@ -0,0 +1,60 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::params; + +use super::{ContactCards, LocalIdArray, Write}; +use crate::archive::LocalId; + +#[derive(Debug, Clone, Copy)] +pub struct ContactCardRecord<'r> { + pub uid: &'r str, + pub address_books: &'r [LocalId], + pub data: &'r str, +} + +impl ContactCards<'_> { + pub fn insert(&self, c: &ContactCardRecord<'_>) -> rusqlite::Result { + self.conn.execute( + "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)", + params![c.uid, LocalIdArray(c.address_books), c.data], + )?; + Ok(LocalId(self.conn.last_insert_rowid())) + } + + pub fn update(&self, id: LocalId, c: &ContactCardRecord<'_>) -> rusqlite::Result { + self.conn + .execute( + "UPDATE contact_cards SET uid = ?1, address_book_ids = ?2, data = ?3 + WHERE id = ?4 AND (uid IS NOT ?1 OR address_book_ids IS NOT ?2 OR data IS NOT ?3)", + params![c.uid, LocalIdArray(c.address_books), c.data, id], + ) + .map(Write::from_rows) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::Archive; + + #[test] + fn insert_update_delete() { + let archive = Archive::open_in_memory().unwrap(); + let cards = archive.store().contact_cards(); + let books = [LocalId::from_raw(3)]; + let mut record = ContactCardRecord { + uid: "u1", + address_books: &books, + data: r#"{"name":{"full":"A"}}"#, + }; + let id = cards.insert(&record).unwrap(); + assert_eq!(cards.update(id, &record).unwrap(), Write::Unchanged); + record.data = r#"{"name":{"full":"B"}}"#; + assert_eq!(cards.update(id, &record).unwrap(), Write::Changed); + cards.delete(id).unwrap(); + } +} diff --git a/src/archive/rows/emails.rs b/src/archive/rows/emails.rs new file mode 100644 index 0000000..51afa7e --- /dev/null +++ b/src/archive/rows/emails.rs @@ -0,0 +1,193 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::{OptionalExtension, params}; + +use super::{Emails, LocalIdArray, Write}; +use crate::archive::{BlobId, LocalId}; + +const EMPTY_MATCH: &str = "{}"; + +#[derive(Debug, Clone, Copy)] +pub struct EmailRecord<'r> { + pub blob: BlobId, + pub received_at: &'r str, + pub mailboxes: &'r [LocalId], + pub keywords: &'r str, + pub message_match: Option<&'r str>, +} + +impl Emails<'_> { + pub fn insert(&self, e: &EmailRecord<'_>) -> rusqlite::Result { + self.conn.execute( + "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + e.blob, + e.received_at, + LocalIdArray(e.mailboxes), + e.keywords, + e.message_match.unwrap_or(EMPTY_MATCH) + ], + )?; + Ok(LocalId(self.conn.last_insert_rowid())) + } + + pub fn update(&self, id: LocalId, e: &EmailRecord<'_>) -> rusqlite::Result { + self.conn + .prepare_cached( + "UPDATE emails SET blob_id = ?1, received_at = ?2, mailbox_ids = ?3, keywords = ?4, + message_match = ?5 + WHERE id = ?6 AND (blob_id IS NOT ?1 OR received_at IS NOT ?2 + OR mailbox_ids IS NOT ?3 OR keywords IS NOT ?4 OR message_match IS NOT ?5)", + )? + .execute(params![ + e.blob, + e.received_at, + LocalIdArray(e.mailboxes), + e.keywords, + e.message_match.unwrap_or(EMPTY_MATCH), + id + ]) + .map(Write::from_rows) + } + + pub fn update_placement( + &self, + id: LocalId, + mailboxes: &[LocalId], + keywords: &str, + ) -> rusqlite::Result { + self.conn + .execute( + "UPDATE emails SET mailbox_ids = ?1, keywords = ?2 + WHERE id = ?3 AND (mailbox_ids IS NOT ?1 OR keywords IS NOT ?2)", + params![LocalIdArray(mailboxes), keywords, id], + ) + .map(Write::from_rows) + } +} + +impl Emails<'_> { + pub fn keywords(&self, id: LocalId) -> rusqlite::Result> { + self.conn + .prepare_cached("SELECT blob_id, keywords FROM emails WHERE id = ?1")? + .query_row([id], |r| Ok((r.get(0)?, r.get(1)?))) + .optional() + } + + pub fn update_keywords(&self, id: LocalId, keywords: &str) -> rusqlite::Result { + self.conn + .prepare_cached("UPDATE emails SET keywords = ?1 WHERE id = ?2 AND keywords IS NOT ?1")? + .execute(params![keywords, id]) + .map(Write::from_rows) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::Archive; + + #[test] + fn insert_then_update_placement() { + let archive = Archive::open_in_memory().unwrap(); + let store = archive.store(); + let blob = store.blobs().intern(b"raw message").unwrap(); + let inbox = [LocalId::from_raw(1)]; + let emails = store.emails(); + let id = emails + .insert(&EmailRecord { + blob, + received_at: "2020-01-01T00:00:00Z", + mailboxes: &inbox, + keywords: r#"["$seen"]"#, + message_match: None, + }) + .unwrap(); + let (mailbox_ids, matched): (String, String) = archive + .connection() + .query_row( + "SELECT mailbox_ids, message_match FROM emails WHERE id = ?1", + [id], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + assert_eq!(mailbox_ids, "[1]"); + assert_eq!(matched, "{}"); + assert_eq!( + emails.update_placement(id, &inbox, r#"["$seen"]"#).unwrap(), + Write::Unchanged + ); + assert_eq!( + emails.update_placement(id, &inbox, "[]").unwrap(), + Write::Changed + ); + assert_eq!(emails.keywords(id).unwrap(), Some((blob, "[]".to_owned()))); + assert_eq!( + emails.update_keywords(id, r#"["$flagged"]"#).unwrap(), + Write::Changed + ); + assert_eq!( + emails.update_keywords(id, r#"["$flagged"]"#).unwrap(), + Write::Unchanged + ); + emails.delete(id).unwrap(); + assert_eq!(emails.keywords(id).unwrap(), None); + } + + #[test] + fn update_rewrites_the_whole_row_and_is_guarded() { + let archive = Archive::open_in_memory().unwrap(); + let store = archive.store(); + let old_blob = store.blobs().intern(b"old").unwrap(); + let new_blob = store.blobs().intern(b"new").unwrap(); + let inbox = [LocalId::from_raw(1)]; + let both = [LocalId::from_raw(1), LocalId::from_raw(2)]; + let emails = store.emails(); + let first = EmailRecord { + blob: old_blob, + received_at: "2020-01-01T00:00:00Z", + mailboxes: &inbox, + keywords: "[]", + message_match: None, + }; + let id = emails.insert(&first).unwrap(); + assert_eq!(emails.update(id, &first).unwrap(), Write::Unchanged); + let second = EmailRecord { + blob: new_blob, + received_at: "2021-02-02T00:00:00Z", + mailboxes: &both, + keywords: r#"["$seen"]"#, + message_match: Some(r#"{"f":"00","m":["a@b"]}"#), + }; + assert_eq!(emails.update(id, &second).unwrap(), Write::Changed); + assert_eq!(emails.update(id, &second).unwrap(), Write::Unchanged); + let row: (BlobId, String, String, String, String) = archive + .connection() + .query_row( + "SELECT blob_id, received_at, mailbox_ids, keywords, message_match + FROM emails WHERE id = ?1", + [id], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)), + ) + .unwrap(); + assert_eq!( + row, + ( + new_blob, + "2021-02-02T00:00:00Z".to_owned(), + "[1,2]".to_owned(), + r#"["$seen"]"#.to_owned(), + r#"{"f":"00","m":["a@b"]}"#.to_owned() + ) + ); + assert_eq!( + emails.update(LocalId::from_raw(999), &second).unwrap(), + Write::Unchanged + ); + } +} diff --git a/src/archive/rows/file_nodes.rs b/src/archive/rows/file_nodes.rs new file mode 100644 index 0000000..112ed00 --- /dev/null +++ b/src/archive/rows/file_nodes.rs @@ -0,0 +1,166 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::params; +use rusqlite::types::{FromSql, FromSqlError, FromSqlResult, ToSql, ToSqlOutput, ValueRef}; + +use super::{FileNodes, Subscribed, Write}; +use crate::archive::{BlobId, LocalId}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NodeType { + File, + Directory, + SymLink, +} + +impl NodeType { + pub fn as_str(self) -> &'static str { + match self { + NodeType::File => "file", + NodeType::Directory => "directory", + NodeType::SymLink => "symlink", + } + } + + pub fn parse(name: &str) -> Option { + hashify::map!( + name.as_bytes(), + NodeType, + "file" => NodeType::File, + "directory" => NodeType::Directory, + "symlink" => NodeType::SymLink, + ) + .copied() + } +} + +impl ToSql for NodeType { + fn to_sql(&self) -> rusqlite::Result> { + Ok(ToSqlOutput::from(self.as_str())) + } +} + +impl FromSql for NodeType { + fn column_result(value: ValueRef<'_>) -> FromSqlResult { + NodeType::parse(value.as_str()?).ok_or(FromSqlError::InvalidType) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct FileNodeRecord<'r> { + pub parent: Option, + pub node_type: NodeType, + pub blob: Option, + pub target: Option<&'r str>, + pub name: &'r str, + pub media_type: Option<&'r str>, + pub created: &'r str, + pub modified: Option<&'r str>, + pub subscribed: Subscribed, + pub role: Option<&'r str>, + pub executable: bool, +} + +impl FileNodes<'_> { + pub fn insert(&self, f: &FileNodeRecord<'_>) -> rusqlite::Result { + self.conn.execute( + "INSERT INTO file_nodes (parent_id, node_type, blob_id, target, name, media_type, + created, modified, is_subscribed, role, executable) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", + params![ + f.parent, + f.node_type, + f.blob, + f.target, + f.name, + f.media_type, + f.created, + f.modified, + f.subscribed, + f.role, + f.executable + ], + )?; + Ok(LocalId(self.conn.last_insert_rowid())) + } + + pub fn update(&self, id: LocalId, f: &FileNodeRecord<'_>) -> rusqlite::Result { + self.conn + .execute( + "UPDATE file_nodes SET parent_id = ?1, node_type = ?2, blob_id = ?3, target = ?4, + name = ?5, media_type = ?6, created = ?7, modified = ?8, is_subscribed = ?9, + role = ?10, executable = ?11 + WHERE id = ?12 AND (parent_id IS NOT ?1 OR node_type IS NOT ?2 OR blob_id IS NOT ?3 + OR target IS NOT ?4 OR name IS NOT ?5 OR media_type IS NOT ?6 + OR created IS NOT ?7 OR modified IS NOT ?8 OR is_subscribed IS NOT ?9 + OR role IS NOT ?10 OR executable IS NOT ?11)", + params![ + f.parent, + f.node_type, + f.blob, + f.target, + f.name, + f.media_type, + f.created, + f.modified, + f.subscribed, + f.role, + f.executable, + id + ], + ) + .map(Write::from_rows) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::Archive; + + #[test] + fn insert_update_delete() { + let archive = Archive::open_in_memory().unwrap(); + let store = archive.store(); + let nodes = store.file_nodes(); + let dir = nodes + .insert(&FileNodeRecord { + parent: None, + node_type: NodeType::Directory, + blob: None, + target: None, + name: "docs", + media_type: None, + created: "2020-01-01T00:00:00Z", + modified: None, + subscribed: Subscribed::Yes, + role: None, + executable: false, + }) + .unwrap(); + let blob = store.blobs().intern(b"hello").unwrap(); + let mut file = FileNodeRecord { + parent: Some(dir), + node_type: NodeType::File, + blob: Some(blob), + target: None, + name: "a.txt", + media_type: Some("text/plain"), + created: "2020-01-01T00:00:00Z", + modified: None, + subscribed: Subscribed::Yes, + role: None, + executable: false, + }; + let id = nodes.insert(&file).unwrap(); + assert_eq!(nodes.update(id, &file).unwrap(), Write::Unchanged); + file.executable = true; + assert_eq!(nodes.update(id, &file).unwrap(), Write::Changed); + nodes.delete(id).unwrap(); + nodes.delete(dir).unwrap(); + } +} diff --git a/src/archive/rows/identities.rs b/src/archive/rows/identities.rs new file mode 100644 index 0000000..979dab2 --- /dev/null +++ b/src/archive/rows/identities.rs @@ -0,0 +1,83 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::params; + +use super::{Identities, Write}; +use crate::archive::LocalId; + +#[derive(Debug, Clone, Copy)] +pub struct IdentityRecord<'r> { + pub name: &'r str, + pub email: &'r str, + pub reply_to: Option<&'r str>, + pub bcc: Option<&'r str>, + pub text_signature: &'r str, + pub html_signature: &'r str, +} + +impl Identities<'_> { + pub fn insert(&self, i: &IdentityRecord<'_>) -> rusqlite::Result { + self.conn.execute( + "INSERT INTO identities (name, email, reply_to, bcc, text_signature, html_signature) + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + i.name, + i.email, + i.reply_to, + i.bcc, + i.text_signature, + i.html_signature + ], + )?; + Ok(LocalId(self.conn.last_insert_rowid())) + } + + pub fn update(&self, id: LocalId, i: &IdentityRecord<'_>) -> rusqlite::Result { + self.conn + .execute( + "UPDATE identities SET name = ?1, email = ?2, reply_to = ?3, bcc = ?4, + text_signature = ?5, html_signature = ?6 + WHERE id = ?7 AND (name IS NOT ?1 OR email IS NOT ?2 OR reply_to IS NOT ?3 + OR bcc IS NOT ?4 OR text_signature IS NOT ?5 OR html_signature IS NOT ?6)", + params![ + i.name, + i.email, + i.reply_to, + i.bcc, + i.text_signature, + i.html_signature, + id + ], + ) + .map(Write::from_rows) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::Archive; + + #[test] + fn insert_update_delete() { + let archive = Archive::open_in_memory().unwrap(); + let identities = archive.store().identities(); + let mut record = IdentityRecord { + name: "Alice", + email: "alice@example.com", + reply_to: None, + bcc: Some(r#"[{"email":"b@example.com"}]"#), + text_signature: "", + html_signature: "", + }; + let id = identities.insert(&record).unwrap(); + assert_eq!(identities.update(id, &record).unwrap(), Write::Unchanged); + record.name = "Alice A."; + assert_eq!(identities.update(id, &record).unwrap(), Write::Changed); + identities.delete(id).unwrap(); + } +} diff --git a/src/archive/rows/mailboxes.rs b/src/archive/rows/mailboxes.rs new file mode 100644 index 0000000..d4b26f6 --- /dev/null +++ b/src/archive/rows/mailboxes.rs @@ -0,0 +1,196 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::params; + +use super::{Mailboxes, Subscribed, Write}; +use crate::archive::LocalId; + +#[derive(Debug, Clone, Copy)] +pub struct MailboxRecord<'r> { + pub name: &'r str, + pub parent: Option, + pub role: Option<&'r str>, + pub sort_order: i64, + pub subscribed: Subscribed, +} + +impl Mailboxes<'_> { + pub fn insert(&self, m: &MailboxRecord<'_>) -> rusqlite::Result { + let role = self.unique_role(m.role, None)?; + self.conn.execute( + "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![m.name, m.parent, role, m.sort_order, m.subscribed], + )?; + Ok(LocalId(self.conn.last_insert_rowid())) + } + + pub fn update(&self, id: LocalId, m: &MailboxRecord<'_>) -> rusqlite::Result { + let role = self.unique_role(m.role, Some(id))?; + self.conn + .execute( + "UPDATE mailboxes SET name = ?1, parent_id = ?2, role = ?3, sort_order = ?4, + is_subscribed = ?5 + WHERE id = ?6 AND (name IS NOT ?1 OR parent_id IS NOT ?2 OR role IS NOT ?3 + OR sort_order IS NOT ?4 OR is_subscribed IS NOT ?5)", + params![m.name, m.parent, role, m.sort_order, m.subscribed, id], + ) + .map(Write::from_rows) + } + + pub fn unique_role<'r>( + &self, + role: Option<&'r str>, + exclude: Option, + ) -> rusqlite::Result> { + let Some(r) = role else { + return Ok(None); + }; + let taken: bool = self.conn.query_row( + "SELECT EXISTS(SELECT 1 FROM mailboxes WHERE role = ?1 AND (?2 IS NULL OR id != ?2))", + params![r, exclude], + |row| row.get(0), + )?; + Ok((!taken).then_some(r)) + } + + pub fn detach_children(&self, id: LocalId) -> rusqlite::Result { + self.conn.execute( + "UPDATE mailboxes SET parent_id = NULL WHERE parent_id = ?1", + [id], + ) + } +} + +#[cfg(test)] +mod tests { + use rusqlite::params; + + use super::*; + use crate::archive::Archive; + + fn record<'r>(name: &'r str, role: Option<&'r str>) -> MailboxRecord<'r> { + MailboxRecord { + name, + parent: None, + role, + sort_order: 0, + subscribed: Subscribed::Yes, + } + } + + #[test] + fn insert_drops_a_role_already_taken() { + let archive = Archive::open_in_memory().unwrap(); + let boxes = archive.store().mailboxes(); + let first = boxes.insert(&record("Sent", Some("sent"))).unwrap(); + let second = boxes.insert(&record("Sent 2", Some("sent"))).unwrap(); + let role = |id: LocalId| -> Option { + archive + .connection() + .query_row("SELECT role FROM mailboxes WHERE id = ?1", [id], |r| { + r.get(0) + }) + .unwrap() + }; + assert_eq!(role(first).as_deref(), Some("sent")); + assert_eq!(role(second), None); + } + + #[test] + fn update_reports_whether_the_row_changed() { + let archive = Archive::open_in_memory().unwrap(); + let boxes = archive.store().mailboxes(); + let id = boxes.insert(&record("Inbox", Some("inbox"))).unwrap(); + assert_eq!( + boxes.update(id, &record("Inbox", Some("inbox"))).unwrap(), + Write::Unchanged + ); + assert_eq!( + boxes.update(id, &record("INBOX", Some("inbox"))).unwrap(), + Write::Changed + ); + } + + #[test] + fn detach_children_then_delete_parent() { + let archive = Archive::open_in_memory().unwrap(); + let boxes = archive.store().mailboxes(); + let parent = boxes.insert(&record("Parent", None)).unwrap(); + let child = MailboxRecord { + parent: Some(parent), + ..record("Child", None) + }; + boxes.insert(&child).unwrap(); + assert!(boxes.delete(parent).is_err()); + assert_eq!(boxes.detach_children(parent).unwrap(), 1); + boxes.delete(parent).unwrap(); + } + + fn insert_raw(archive: &Archive, name: &str, role: Option<&str>) -> LocalId { + let conn = archive.connection(); + conn.execute( + "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) + VALUES (?1, NULL, ?2, 0, 1)", + params![name, role], + ) + .unwrap(); + LocalId::from_raw(conn.last_insert_rowid()) + } + + #[test] + fn first_claimant_keeps_role_second_is_dropped() { + let archive = Archive::open_in_memory().unwrap(); + let mailboxes = archive.store().mailboxes(); + let first = mailboxes.unique_role(Some("sent"), None).unwrap(); + assert_eq!(first, Some("sent")); + insert_raw(&archive, "Sent", first); + + assert_eq!(mailboxes.unique_role(Some("sent"), None).unwrap(), None); + } + + #[test] + fn null_role_stays_null() { + let archive = Archive::open_in_memory().unwrap(); + let mailboxes = archive.store().mailboxes(); + assert_eq!(mailboxes.unique_role(None, None).unwrap(), None); + } + + #[test] + fn distinct_roles_are_independent() { + let archive = Archive::open_in_memory().unwrap(); + insert_raw(&archive, "Sent", Some("sent")); + let mailboxes = archive.store().mailboxes(); + assert_eq!( + mailboxes.unique_role(Some("trash"), None).unwrap(), + Some("trash") + ); + } + + #[test] + fn update_excludes_own_row() { + let archive = Archive::open_in_memory().unwrap(); + let id = insert_raw(&archive, "Sent", Some("sent")); + let mailboxes = archive.store().mailboxes(); + assert_eq!( + mailboxes.unique_role(Some("sent"), Some(id)).unwrap(), + Some("sent") + ); + } + + #[test] + fn update_detects_other_holder() { + let archive = Archive::open_in_memory().unwrap(); + insert_raw(&archive, "Sent", Some("sent")); + let other = insert_raw(&archive, "Envoyés", None); + let mailboxes = archive.store().mailboxes(); + assert_eq!( + mailboxes.unique_role(Some("sent"), Some(other)).unwrap(), + None + ); + } +} diff --git a/src/archive/rows/mod.rs b/src/archive/rows/mod.rs new file mode 100644 index 0000000..8a5152b --- /dev/null +++ b/src/archive/rows/mod.rs @@ -0,0 +1,217 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +mod address_books; +mod calendar_events; +mod calendars; +mod contact_cards; +mod emails; +mod file_nodes; +mod identities; +mod mailboxes; +mod participant_identities; +mod sieve_scripts; +mod vacation; + +use std::collections::HashSet; +use std::fmt::Write as _; + +use rusqlite::types::{FromSql, FromSqlResult, ToSql, ToSqlOutput, ValueRef}; +use rusqlite::{Connection, params}; + +use super::{LocalId, Store}; + +pub use self::address_books::AddressBookRecord; +pub use self::calendar_events::{CalendarEventRecord, EventDataType}; +pub use self::calendars::CalendarRecord; +pub use self::contact_cards::ContactCardRecord; +pub use self::emails::EmailRecord; +pub use self::file_nodes::{FileNodeRecord, NodeType}; +pub use self::identities::IdentityRecord; +pub use self::mailboxes::MailboxRecord; +pub use self::participant_identities::ParticipantIdentityRecord; +pub use self::sieve_scripts::{SieveScriptRecord, SieveScriptState}; +pub use self::vacation::{StoredVacation, Vacation, VacationRecord}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Write { + Changed, + Unchanged, +} + +impl Write { + fn from_rows(rows: usize) -> Write { + if rows > 0 { + Write::Changed + } else { + Write::Unchanged + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Subscribed { + Yes, + No, +} + +impl From for Subscribed { + fn from(subscribed: bool) -> Subscribed { + if subscribed { + Subscribed::Yes + } else { + Subscribed::No + } + } +} + +impl ToSql for Subscribed { + fn to_sql(&self) -> rusqlite::Result> { + Ok(ToSqlOutput::from(matches!(self, Subscribed::Yes))) + } +} + +impl FromSql for Subscribed { + fn column_result(value: ValueRef<'_>) -> FromSqlResult { + bool::column_result(value).map(Subscribed::from) + } +} + +struct LocalIdArray<'a>(&'a [LocalId]); + +impl ToSql for LocalIdArray<'_> { + fn to_sql(&self) -> rusqlite::Result> { + let mut out = String::with_capacity(2 + self.0.len() * 4); + out.push('['); + for (i, id) in self.0.iter().enumerate() { + if i > 0 { + out.push(','); + } + let _ = write!(out, "{id}"); + } + out.push(']'); + Ok(ToSqlOutput::from(out)) + } +} + +macro_rules! tables { + ($($handle:ident => $accessor:ident, $table:literal;)+) => { + $( + #[derive(Clone, Copy)] + pub struct $handle<'a> { + conn: &'a Connection, + } + + impl $handle<'_> { + pub fn delete(&self, id: LocalId) -> rusqlite::Result<()> { + self.conn + .execute(concat!("DELETE FROM ", $table, " WHERE id = ?1"), [id])?; + Ok(()) + } + } + )+ + + impl<'a> Store<'a> { + $( + pub fn $accessor(self) -> $handle<'a> { + $handle { conn: self.conn } + } + )+ + } + }; +} + +tables! { + Mailboxes => mailboxes, "mailboxes"; + Emails => emails, "emails"; + Identities => identities, "identities"; + SieveScripts => sieve_scripts, "sieve_scripts"; + AddressBooks => address_books, "address_books"; + ContactCards => contact_cards, "contact_cards"; + Calendars => calendars, "calendars"; + CalendarEvents => calendar_events, "calendar_events"; + ParticipantIdentities => participant_identities, "participant_identities"; + FileNodes => file_nodes, "file_nodes"; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MemberList { + EmailMailboxes, + EventCalendars, + CardAddressBooks, +} + +impl MemberList { + const fn statements(self) -> (&'static str, &'static str) { + match self { + MemberList::EmailMailboxes => ( + "SELECT id, mailbox_ids FROM emails", + "UPDATE emails SET mailbox_ids = ?1 WHERE id = ?2", + ), + MemberList::EventCalendars => ( + "SELECT id, calendar_ids FROM calendar_events", + "UPDATE calendar_events SET calendar_ids = ?1 WHERE id = ?2", + ), + MemberList::CardAddressBooks => ( + "SELECT id, address_book_ids FROM contact_cards", + "UPDATE contact_cards SET address_book_ids = ?1 WHERE id = ?2", + ), + } + } +} + +impl Store<'_> { + pub fn prune_members( + self, + list: MemberList, + removed: &HashSet, + ) -> rusqlite::Result<()> { + let (select, update) = list.statements(); + let rows: Vec<(LocalId, String)> = self + .conn + .prepare(select)? + .query_map([], |r| Ok((r.get(0)?, r.get(1)?)))? + .collect::>()?; + let mut update = self.conn.prepare(update)?; + let mut kept = Vec::new(); + for (id, json) in rows { + let members: Vec = serde_json::from_str(&json).unwrap_or_default(); + if members.iter().any(|&m| removed.contains(&LocalId(m))) { + kept.clear(); + kept.extend( + members + .into_iter() + .map(LocalId) + .filter(|m| !removed.contains(m)), + ); + update.execute(params![LocalIdArray(&kept), id])?; + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::Archive; + + #[test] + fn local_id_array_matches_the_json_encoding() { + let archive = Archive::open_in_memory().unwrap(); + let ids = [LocalId::from_raw(1), LocalId::from_raw(23)]; + let text: String = archive + .connection() + .query_row("SELECT ?1", [LocalIdArray(&ids)], |r| r.get(0)) + .unwrap(); + assert_eq!(text, "[1,23]"); + let empty: String = archive + .connection() + .query_row("SELECT ?1", [LocalIdArray(&[])], |r| r.get(0)) + .unwrap(); + assert_eq!(empty, "[]"); + } +} diff --git a/src/archive/rows/participant_identities.rs b/src/archive/rows/participant_identities.rs new file mode 100644 index 0000000..1404124 --- /dev/null +++ b/src/archive/rows/participant_identities.rs @@ -0,0 +1,76 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::params; + +use super::{ParticipantIdentities, Write}; +use crate::archive::{LocalId, Store}; +use crate::types::ObjectType; + +#[derive(Debug, Clone, Copy)] +pub struct ParticipantIdentityRecord<'r> { + pub name: &'r str, + pub calendar_address: &'r str, + pub is_default: bool, +} + +impl ParticipantIdentities<'_> { + pub fn insert(&self, p: &ParticipantIdentityRecord<'_>) -> rusqlite::Result { + let is_default = Store::new(self.conn).unique_default( + ObjectType::ParticipantIdentity, + p.is_default, + None, + )?; + self.conn.execute( + "INSERT INTO participant_identities (name, calendar_address, is_default) + VALUES (?1, ?2, ?3)", + params![p.name, p.calendar_address, is_default], + )?; + Ok(LocalId(self.conn.last_insert_rowid())) + } + + pub fn update( + &self, + id: LocalId, + p: &ParticipantIdentityRecord<'_>, + ) -> rusqlite::Result { + let is_default = Store::new(self.conn).unique_default( + ObjectType::ParticipantIdentity, + p.is_default, + Some(id), + )?; + self.conn + .execute( + "UPDATE participant_identities SET name = ?1, calendar_address = ?2, is_default = ?3 + WHERE id = ?4 AND (name IS NOT ?1 OR calendar_address IS NOT ?2 + OR is_default IS NOT ?3)", + params![p.name, p.calendar_address, is_default, id], + ) + .map(Write::from_rows) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::Archive; + + #[test] + fn insert_update_delete() { + let archive = Archive::open_in_memory().unwrap(); + let identities = archive.store().participant_identities(); + let mut record = ParticipantIdentityRecord { + name: "Alice", + calendar_address: "mailto:alice@example.com", + is_default: true, + }; + let id = identities.insert(&record).unwrap(); + assert_eq!(identities.update(id, &record).unwrap(), Write::Unchanged); + record.name = "A"; + assert_eq!(identities.update(id, &record).unwrap(), Write::Changed); + identities.delete(id).unwrap(); + } +} diff --git a/src/archive/rows/sieve_scripts.rs b/src/archive/rows/sieve_scripts.rs new file mode 100644 index 0000000..bc240b2 --- /dev/null +++ b/src/archive/rows/sieve_scripts.rs @@ -0,0 +1,195 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::{OptionalExtension, params}; + +use super::{SieveScripts, Write}; +use crate::archive::{BlobId, LocalId, SourceId}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SieveScriptState { + pub id: LocalId, + pub hash: Option<[u8; 32]>, + pub is_active: bool, +} + +#[derive(Debug, Clone, Copy)] +pub struct SieveScriptRecord<'r> { + pub name: Option<&'r str>, + pub is_active: bool, + pub blob: BlobId, +} + +impl SieveScripts<'_> { + pub fn insert(&self, s: &SieveScriptRecord<'_>) -> rusqlite::Result { + self.conn.execute( + "INSERT INTO sieve_scripts (name, is_active, blob_id) VALUES (?1, ?2, ?3)", + params![s.name, s.is_active, s.blob], + )?; + Ok(LocalId(self.conn.last_insert_rowid())) + } + + pub fn update(&self, id: LocalId, s: &SieveScriptRecord<'_>) -> rusqlite::Result { + self.conn + .execute( + "UPDATE sieve_scripts SET name = ?1, is_active = ?2, blob_id = ?3 + WHERE id = ?4 AND (name IS NOT ?1 OR is_active IS NOT ?2 OR blob_id IS NOT ?3)", + params![s.name, s.is_active, s.blob, id], + ) + .map(Write::from_rows) + } +} + +impl SieveScripts<'_> { + pub fn active(&self) -> rusqlite::Result> { + self.conn + .query_row( + "SELECT id FROM sieve_scripts WHERE is_active = 1", + [], + |r| r.get(0), + ) + .optional() + } + + pub fn set_active(&self, active: Option) -> rusqlite::Result<()> { + let current = self.active()?; + if current == active { + return Ok(()); + } + if current.is_some() { + self.conn.execute( + "UPDATE sieve_scripts SET is_active = 0 WHERE is_active = 1", + [], + )?; + } + if let Some(id) = active { + self.conn + .execute("UPDATE sieve_scripts SET is_active = 1 WHERE id = ?1", [id])?; + } + Ok(()) + } + + pub fn managesieve_states(&self, source: SourceId) -> rusqlite::Result> { + self.conn + .prepare( + "SELECT s.id, b.hash, s.is_active + FROM sieve_scripts s JOIN blobs b ON b.id = s.blob_id + JOIN sync_id_managesieve m ON m.local_id = s.id + WHERE m.source_id = ?1", + )? + .query_map([source], |r| { + let hash: Vec = r.get(1)?; + Ok(SieveScriptState { + id: r.get(0)?, + hash: <[u8; 32]>::try_from(hash.as_slice()).ok(), + is_active: r.get(2)?, + }) + })? + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::Archive; + use crate::archive::sources::{SourceKey, SourceKind}; + + #[test] + fn insert_update_delete() { + let archive = Archive::open_in_memory().unwrap(); + let store = archive.store(); + let blob = store.blobs().intern(b"keep;").unwrap(); + let scripts = store.sieve_scripts(); + let mut record = SieveScriptRecord { + name: Some("main"), + is_active: true, + blob, + }; + let id = scripts.insert(&record).unwrap(); + assert_eq!(scripts.update(id, &record).unwrap(), Write::Unchanged); + record.is_active = false; + assert_eq!(scripts.update(id, &record).unwrap(), Write::Changed); + scripts.delete(id).unwrap(); + } + + #[test] + fn set_active_moves_the_single_active_flag() { + let archive = Archive::open_in_memory().unwrap(); + let store = archive.store(); + let blob = store.blobs().intern(b"keep;").unwrap(); + let scripts = store.sieve_scripts(); + let record = |name| SieveScriptRecord { + name: Some(name), + is_active: false, + blob, + }; + let a = scripts.insert(&record("a")).unwrap(); + let b = scripts.insert(&record("b")).unwrap(); + assert_eq!(scripts.active().unwrap(), None); + scripts.set_active(Some(a)).unwrap(); + assert_eq!(scripts.active().unwrap(), Some(a)); + scripts.set_active(Some(b)).unwrap(); + assert_eq!(scripts.active().unwrap(), Some(b)); + scripts.set_active(None).unwrap(); + assert_eq!(scripts.active().unwrap(), None); + } + + // only scripts mapped to the given ManageSieve source are listed, with their blob hash + #[test] + fn managesieve_states_lists_scripts_of_the_source() { + let archive = Archive::open_in_memory().unwrap(); + let store = archive.store(); + let sources = store.sources(); + let key = |account_id| SourceKey { + kind: SourceKind::ManageSieve, + url: "sieve://mail.example.org", + account_id, + }; + let mine = sources.upsert(&key("alice"), None, "alice").unwrap(); + let other = sources.upsert(&key("bob"), None, "bob").unwrap(); + let keep = store.blobs().intern(b"keep;").unwrap(); + let discard = store.blobs().intern(b"discard;").unwrap(); + let scripts = store.sieve_scripts(); + let record = |name, is_active, blob| SieveScriptRecord { + name: Some(name), + is_active, + blob, + }; + let a = scripts.insert(&record("a", true, keep)).unwrap(); + let b = scripts.insert(&record("b", false, discard)).unwrap(); + let c = scripts.insert(&record("c", false, keep)).unwrap(); + scripts.insert(&record("unmapped", false, keep)).unwrap(); + let mut map = archive + .connection() + .prepare( + "INSERT INTO sync_id_managesieve (source_id, name, local_id) VALUES (?1, ?2, ?3)", + ) + .unwrap(); + for (source, name, id) in [(mine, "a", a), (mine, "b", b), (other, "c", c)] { + map.execute(params![source, name, id]).unwrap(); + } + + let mut states = scripts.managesieve_states(mine).unwrap(); + states.sort_unstable_by_key(|s| s.id); + assert_eq!( + states, + [ + SieveScriptState { + id: a, + hash: Some(*blake3::hash(b"keep;").as_bytes()), + is_active: true, + }, + SieveScriptState { + id: b, + hash: Some(*blake3::hash(b"discard;").as_bytes()), + is_active: false, + }, + ] + ); + assert_eq!(scripts.managesieve_states(other).unwrap().len(), 1); + } +} diff --git a/src/archive/rows/vacation.rs b/src/archive/rows/vacation.rs new file mode 100644 index 0000000..0962e63 --- /dev/null +++ b/src/archive/rows/vacation.rs @@ -0,0 +1,142 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::{OptionalExtension, params}; + +use super::Connection; +use crate::archive::{SourceId, Store}; + +#[derive(Debug, Clone, Copy)] +pub struct VacationRecord<'r> { + pub is_enabled: bool, + pub from_date: Option<&'r str>, + pub to_date: Option<&'r str>, + pub subject: Option<&'r str>, + pub text_body: Option<&'r str>, + pub html_body: Option<&'r str>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredVacation { + pub is_enabled: bool, + pub from_date: Option, + pub to_date: Option, + pub subject: Option, + pub text_body: Option, + pub html_body: Option, +} + +#[derive(Clone, Copy)] +pub struct Vacation<'a> { + conn: &'a Connection, +} + +impl<'a> Store<'a> { + pub fn vacation(self) -> Vacation<'a> { + Vacation { conn: self.conn } + } +} + +impl Vacation<'_> { + pub fn store(&self, source: SourceId, v: &VacationRecord<'_>) -> rusqlite::Result<()> { + self.conn.execute( + "INSERT OR REPLACE INTO vacation_response + (id, source_id, is_enabled, from_date, to_date, subject, text_body, html_body) + VALUES (1, ?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![ + source, + v.is_enabled, + v.from_date, + v.to_date, + v.subject, + v.text_body, + v.html_body + ], + )?; + Ok(()) + } + + pub fn clear(&self, source: SourceId) -> rusqlite::Result<()> { + self.conn.execute( + "DELETE FROM vacation_response WHERE source_id = ?1", + [source], + )?; + Ok(()) + } + + pub fn exists(&self) -> rusqlite::Result { + self.conn.query_row( + "SELECT EXISTS(SELECT 1 FROM vacation_response)", + [], + |row| row.get(0), + ) + } + + pub fn load(&self) -> rusqlite::Result> { + self.conn + .query_row( + "SELECT is_enabled, from_date, to_date, subject, text_body, html_body + FROM vacation_response WHERE id = 1", + [], + |row| { + Ok(StoredVacation { + is_enabled: row.get(0)?, + from_date: row.get(1)?, + to_date: row.get(2)?, + subject: row.get(3)?, + text_body: row.get(4)?, + html_body: row.get(5)?, + }) + }, + ) + .optional() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::Archive; + use crate::archive::sources::{SourceKey, SourceKind}; + + #[test] + fn store_load_clear() { + let archive = Archive::open_in_memory().unwrap(); + let store = archive.store(); + let source = store + .sources() + .upsert( + &SourceKey { + kind: SourceKind::Jmap, + url: "https://a/jmap", + account_id: "u", + }, + None, + "u", + ) + .unwrap(); + let vacation = store.vacation(); + assert!(!vacation.exists().unwrap()); + vacation + .store( + source, + &VacationRecord { + is_enabled: true, + from_date: None, + to_date: None, + subject: Some("Away"), + text_body: Some("Back soon"), + html_body: None, + }, + ) + .unwrap(); + let loaded = vacation.load().unwrap().unwrap(); + assert!(loaded.is_enabled); + assert_eq!(loaded.subject.as_deref(), Some("Away")); + vacation.clear(source).unwrap(); + assert!(!vacation.exists().unwrap()); + } +} diff --git a/src/db/schema.sql b/src/archive/schema.sql similarity index 100% rename from src/db/schema.sql rename to src/archive/schema.sql diff --git a/src/archive/sources.rs b/src/archive/sources.rs new file mode 100644 index 0000000..8ed87d7 --- /dev/null +++ b/src/archive/sources.rs @@ -0,0 +1,286 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fmt; + +use rusqlite::types::{ToSql, ToSqlOutput}; +use rusqlite::{Connection, OptionalExtension, params}; + +use super::{SourceId, Store}; + +macro_rules! source_kinds { + ($($variant:ident => $token:literal, $label:literal;)+) => { + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum SourceKind { + $($variant,)+ + } + + impl SourceKind { + pub const ALL: [SourceKind; [$(SourceKind::$variant),+].len()] = + [$(SourceKind::$variant),+]; + + pub const fn as_str(self) -> &'static str { + match self { + $(SourceKind::$variant => $token,)+ + } + } + + pub const fn label(self) -> &'static str { + match self { + $(SourceKind::$variant => $label,)+ + } + } + + pub fn parse(token: &str) -> Option { + hashify::map!(token.as_bytes(), SourceKind, $($token => SourceKind::$variant,)+) + .copied() + } + } + }; +} + +source_kinds! { + Jmap => "jmap", "JMAP"; + Imap => "imap", "IMAP"; + CalDav => "caldav", "CalDAV"; + CardDav => "carddav", "CardDAV"; + WebDav => "webdav", "WebDAV"; + ManageSieve => "managesieve", "ManageSieve"; + Maildir => "maildir", "Maildir"; + Takeout => "takeout", "Google Takeout"; + ExchangeEws => "exchange_ews", "Exchange (EWS)"; + ExchangeGraph => "exchange_graph", "Exchange (Graph)"; +} + +impl fmt::Display for SourceKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl ToSql for SourceKind { + fn to_sql(&self) -> rusqlite::Result> { + Ok(ToSqlOutput::from(self.as_str())) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SourceKey<'k> { + pub kind: SourceKind, + pub url: &'k str, + pub account_id: &'k str, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceChange { + Reject, + Allow, + Replace, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OtherSource { + pub id: SourceId, + pub url: String, + pub account_id: String, +} + +#[derive(Clone, Copy)] +pub struct Sources<'a> { + conn: &'a Connection, +} + +impl<'a> Store<'a> { + pub fn sources(self) -> Sources<'a> { + Sources { conn: self.conn } + } +} + +impl Sources<'_> { + pub fn find(&self, key: &SourceKey<'_>) -> rusqlite::Result> { + self.conn + .query_row( + "SELECT id FROM sources WHERE kind = ?1 AND session_url = ?2 AND account_id = ?3", + params![key.kind, key.url, key.account_id], + |row| row.get(0), + ) + .optional() + } + + pub fn conflicting(&self, key: &SourceKey<'_>) -> rusqlite::Result> { + self.conn + .query_row( + "SELECT session_url, account_id FROM sources + WHERE kind = ?1 AND NOT (session_url = ?2 AND account_id = ?3) LIMIT 1", + params![key.kind, key.url, key.account_id], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() + } + + pub fn others(&self, key: &SourceKey<'_>) -> rusqlite::Result> { + let mut stmt = self.conn.prepare( + "SELECT id, session_url, account_id FROM sources + WHERE kind = ?1 AND NOT (session_url = ?2 AND account_id = ?3) ORDER BY id", + )?; + stmt.query_map(params![key.kind, key.url, key.account_id], |row| { + Ok(OtherSource { + id: row.get(0)?, + url: row.get(1)?, + account_id: row.get(2)?, + }) + })? + .collect() + } + + pub fn upsert( + &self, + key: &SourceKey<'_>, + account_name: Option<&str>, + username: &str, + ) -> rusqlite::Result { + self.conn.execute( + "INSERT INTO sources (kind, session_url, account_id, account_name, username) + VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT (kind, session_url, account_id) + DO UPDATE SET account_name = excluded.account_name, username = excluded.username", + params![key.kind, key.url, key.account_id, account_name, username], + )?; + self.find(key) + .map(|found| found.unwrap_or_else(|| SourceId(self.conn.last_insert_rowid()))) + } + + pub fn delete(&self, id: SourceId) -> rusqlite::Result<()> { + self.conn + .execute("DELETE FROM sources WHERE id = ?1", [id])?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::Archive; + + // Every stored token parses back to its kind; labels are display names. + #[test] + fn source_kind_tokens_round_trip() { + for kind in SourceKind::ALL { + assert_eq!(SourceKind::parse(kind.as_str()), Some(kind)); + } + assert_eq!(SourceKind::parse("Jmap"), None); + assert_eq!(SourceKind::Takeout.label(), "Google Takeout"); + assert_eq!(SourceKind::ExchangeGraph.label(), "Exchange (Graph)"); + } + + fn key<'k>(url: &'k str, acc: &'k str) -> SourceKey<'k> { + SourceKey { + kind: SourceKind::Jmap, + url, + account_id: acc, + } + } + + #[test] + fn upsert_is_idempotent_and_returns_stable_id() { + let archive = Archive::open_in_memory().unwrap(); + let sources = archive.store().sources(); + let k = key("https://a/jmap", "w"); + let id1 = sources.upsert(&k, Some("alice"), "alice").unwrap(); + let id2 = sources.upsert(&k, Some("alice2"), "alice").unwrap(); + assert_eq!(id1, id2); + assert_eq!(sources.find(&k).unwrap(), Some(id1)); + } + + #[test] + fn conflicting_source_detects_a_different_account() { + let archive = Archive::open_in_memory().unwrap(); + let sources = archive.store().sources(); + sources + .upsert(&key("https://a/jmap", "w"), None, "alice") + .unwrap(); + assert!( + sources + .conflicting(&key("https://a/jmap", "w")) + .unwrap() + .is_none() + ); + let conflict = sources + .conflicting(&key("https://b/jmap", "v")) + .unwrap() + .unwrap(); + assert_eq!(conflict, ("https://a/jmap".to_owned(), "w".to_owned())); + } + + #[test] + fn other_sources_lists_every_other_source_of_the_kind() { + let archive = Archive::open_in_memory().unwrap(); + let sources = archive.store().sources(); + let a = sources + .upsert(&key("https://a/jmap", "w"), None, "alice") + .unwrap(); + let b = sources + .upsert(&key("https://b/jmap", "w"), None, "alice") + .unwrap(); + sources + .upsert(&key("https://c/jmap", "w"), None, "alice") + .unwrap(); + let ids: Vec = sources + .others(&key("https://c/jmap", "w")) + .unwrap() + .into_iter() + .map(|o| o.id) + .collect(); + assert_eq!(ids, [a, b]); + let imap = SourceKey { + kind: SourceKind::Imap, + ..key("https://c/jmap", "w") + }; + assert!(sources.others(&imap).unwrap().is_empty()); + } + + #[test] + fn stored_kind_strings_are_unchanged() { + let kinds = [ + SourceKind::Jmap, + SourceKind::Imap, + SourceKind::CalDav, + SourceKind::CardDav, + SourceKind::WebDav, + SourceKind::ManageSieve, + SourceKind::Maildir, + SourceKind::Takeout, + SourceKind::ExchangeEws, + SourceKind::ExchangeGraph, + ] + .map(SourceKind::as_str); + assert_eq!( + kinds, + [ + "jmap", + "imap", + "caldav", + "carddav", + "webdav", + "managesieve", + "maildir", + "takeout", + "exchange_ews", + "exchange_graph" + ] + ); + } + + #[test] + fn delete_removes_the_source() { + let archive = Archive::open_in_memory().unwrap(); + let sources = archive.store().sources(); + let k = key("https://a/jmap", "w"); + let id = sources.upsert(&k, None, "alice").unwrap(); + sources.delete(id).unwrap(); + assert_eq!(sources.find(&k).unwrap(), None); + } +} diff --git a/src/archive/state/imap.rs b/src/archive/state/imap.rs new file mode 100644 index 0000000..8438a11 --- /dev/null +++ b/src/archive/state/imap.rs @@ -0,0 +1,142 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::{Connection, OptionalExtension, params}; + +use crate::archive::{SourceId, Store}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FolderState { + pub uidvalidity: u32, + pub uidnext: u32, + pub last_seen: String, +} + +#[derive(Clone, Copy)] +pub struct ImapFolderState<'a> { + conn: &'a Connection, + source: SourceId, +} + +impl<'a> Store<'a> { + pub fn imap_folder_state(self, source: SourceId) -> ImapFolderState<'a> { + ImapFolderState { + conn: self.connection(), + source, + } + } +} + +impl ImapFolderState<'_> { + pub fn get(&self, folder: &str) -> rusqlite::Result> { + self.conn + .prepare_cached( + "SELECT uidvalidity, uidnext, last_seen FROM imap_folder_state + WHERE source_id = ?1 AND folder = ?2", + )? + .query_row(params![self.source, folder], |row| { + Ok(FolderState { + uidvalidity: row.get(0)?, + uidnext: row.get(1)?, + last_seen: row.get(2)?, + }) + }) + .optional() + } + + pub fn upsert(&self, folder: &str, state: &FolderState) -> rusqlite::Result<()> { + self.conn + .prepare_cached( + "INSERT INTO imap_folder_state (source_id, folder, uidvalidity, uidnext, last_seen) + VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT (source_id, folder) + DO UPDATE SET uidvalidity = excluded.uidvalidity, + uidnext = excluded.uidnext, + last_seen = excluded.last_seen", + )? + .execute(params![ + self.source, + folder, + state.uidvalidity, + state.uidnext, + state.last_seen + ])?; + Ok(()) + } + + pub fn delete(&self, folder: &str) -> rusqlite::Result<()> { + self.conn.execute( + "DELETE FROM imap_folder_state WHERE source_id = ?1 AND folder = ?2", + params![self.source, folder], + )?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::Archive; + use crate::archive::sources::{SourceKey, SourceKind}; + + fn setup() -> (Archive, SourceId) { + let archive = Archive::open_in_memory().unwrap(); + let sid = archive + .store() + .sources() + .upsert( + &SourceKey { + kind: SourceKind::Imap, + url: "imaps://host:993", + account_id: "alice", + }, + None, + "alice", + ) + .unwrap(); + (archive, sid) + } + + fn state(uidvalidity: u32, uidnext: u32, last_seen: &str) -> FolderState { + FolderState { + uidvalidity, + uidnext, + last_seen: last_seen.to_owned(), + } + } + + #[test] + fn upsert_then_get_returns_latest() { + let (archive, sid) = setup(); + let st = archive.store().imap_folder_state(sid); + assert!(st.get("INBOX").unwrap().is_none()); + st.upsert("INBOX", &state(100, 50, "2026-05-24T00:00:00Z")) + .unwrap(); + assert_eq!( + st.get("INBOX").unwrap(), + Some(state(100, 50, "2026-05-24T00:00:00Z")) + ); + st.upsert("INBOX", &state(100, 75, "2026-05-24T01:00:00Z")) + .unwrap(); + assert_eq!( + st.get("INBOX").unwrap(), + Some(state(100, 75, "2026-05-24T01:00:00Z")) + ); + } + + #[test] + fn delete_removes_only_named_folder() { + let (archive, sid) = setup(); + let st = archive.store().imap_folder_state(sid); + st.upsert("INBOX", &state(1, 1, "2026-05-24T00:00:00Z")) + .unwrap(); + st.upsert("Sent", &state(2, 1, "2026-05-24T00:00:00Z")) + .unwrap(); + st.delete("INBOX").unwrap(); + assert!(st.get("INBOX").unwrap().is_none()); + assert!(st.get("Sent").unwrap().is_some()); + } +} diff --git a/src/archive/state/jmap.rs b/src/archive/state/jmap.rs new file mode 100644 index 0000000..d3150ed --- /dev/null +++ b/src/archive/state/jmap.rs @@ -0,0 +1,88 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::{Connection, OptionalExtension, params}; + +use crate::archive::{SourceId, Store}; +use crate::types::ObjectType; + +#[derive(Clone, Copy)] +pub struct JmapState<'a> { + conn: &'a Connection, + source: SourceId, +} + +impl<'a> Store<'a> { + pub fn jmap_state(self, source: SourceId) -> JmapState<'a> { + JmapState { + conn: self.connection(), + source, + } + } +} + +impl JmapState<'_> { + pub fn get(&self, ty: ObjectType) -> rusqlite::Result> { + self.conn + .prepare_cached( + "SELECT state FROM sync_state_jmap WHERE source_id = ?1 AND type_name = ?2", + )? + .query_row(params![self.source, ty.jmap_name()], |row| row.get(0)) + .optional() + } + + pub fn upsert(&self, ty: ObjectType, state: &str) -> rusqlite::Result<()> { + self.conn + .prepare_cached( + "INSERT INTO sync_state_jmap (source_id, type_name, state) + VALUES (?1, ?2, ?3) + ON CONFLICT (source_id, type_name) + DO UPDATE SET state = excluded.state", + )? + .execute(params![self.source, ty.jmap_name(), state])?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use crate::archive::Archive; + use crate::archive::sources::{SourceKey, SourceKind}; + use crate::types::ObjectType; + + #[test] + fn upsert_then_get_returns_latest_per_type() { + let archive = Archive::open_in_memory().unwrap(); + let store = archive.store(); + let source = store + .sources() + .upsert( + &SourceKey { + kind: SourceKind::Jmap, + url: "https://host", + account_id: "alice", + }, + None, + "alice", + ) + .unwrap(); + let state = store.jmap_state(source); + assert!(state.get(ObjectType::Email).unwrap().is_none()); + state.upsert(ObjectType::Email, "s1").unwrap(); + state.upsert(ObjectType::Mailbox, "m1").unwrap(); + assert_eq!(state.get(ObjectType::Email).unwrap().as_deref(), Some("s1")); + assert_eq!( + state.get(ObjectType::Mailbox).unwrap().as_deref(), + Some("m1") + ); + state.upsert(ObjectType::Email, "s2").unwrap(); + assert_eq!(state.get(ObjectType::Email).unwrap().as_deref(), Some("s2")); + assert_eq!( + state.get(ObjectType::Mailbox).unwrap().as_deref(), + Some("m1") + ); + } +} diff --git a/src/sync/import_managesieve.rs b/src/archive/state/mod.rs similarity index 52% rename from src/sync/import_managesieve.rs rename to src/archive/state/mod.rs index b51e01d..7225330 100644 --- a/src/sync/import_managesieve.rs +++ b/src/archive/state/mod.rs @@ -4,7 +4,5 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -pub mod coordinator; -pub mod reconcile; - -pub use coordinator::{ManageSieveAuth, ManageSieveImportConfig, run}; +pub mod imap; +pub mod jmap; diff --git a/src/cli.rs b/src/cli.rs deleted file mode 100644 index 027cafb..0000000 --- a/src/cli.rs +++ /dev/null @@ -1,1616 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::path::PathBuf; - -use clap::{ArgAction, Args, Parser, Subcommand}; -use regex::Regex; - -use crate::error::Error; -use crate::exchange_ews::oauth::OAuthFlow; -use crate::exchange_ews::types::MailboxKind; -use crate::exchange_graph::types::{EventBodyFormat, MailboxKind as GraphMailboxKind, Surfaces}; -use crate::inspect::InspectConfig; -use crate::jmap::account::AccountSelector; -use crate::jmap::http::Auth; -use crate::logging::Logger; -use crate::secret; -use crate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange}; -use crate::sync::import_exchange_ews::{EwsAuth, EwsImportConfig}; -use crate::sync::import_exchange_graph::{GraphAuth, GraphImportConfig}; -use crate::sync::import_imap::{ImapAuth, ImapImportConfig}; -use crate::sync::import_maildir::MaildirImportConfig; -use crate::sync::import_managesieve::{ManageSieveAuth, ManageSieveImportConfig}; -use crate::sync::import_takeout::TakeoutImportConfig; -use crate::sync::{CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; -use crate::types::{ObjectType, parse_object_list}; - -#[derive(Parser)] -#[command( - name = "inbuxa-migrate", - version, - about = "Inbuxa: the JMAP importer-exporter" -)] -pub struct Cli { - #[command(subcommand)] - command: Command, -} - -#[derive(Subcommand)] -enum Command { - #[command(about = "Read a source account into a local archive")] - Import { - #[command(subcommand)] - source: Source, - }, - #[command(about = "Push a local archive into a target JMAP server")] - Export(Box), - #[command(about = "Show the contents of a local archive (read-only)")] - Inspect(Box), -} - -#[derive(Subcommand)] -enum Source { - #[command(about = "JMAP server")] - Jmap(Box), - #[command(about = "IMAP server")] - Imap(Box), - #[command(about = "CalDAV server")] - Caldav(Box), - #[command(about = "CardDAV server")] - Carddav(Box), - #[command(about = "WebDAV server (plain files)")] - Webdav(Box), - #[command(about = "ManageSieve server (Sieve scripts only)")] - Managesieve(Box), - #[command(about = "Local Maildir++ tree")] - Maildir(Box), - #[command(about = "Google Takeout (or any mbox/ics/vcf directory tree)")] - Takeout(Box), - #[command(about = "MS Exchange via EWS (SOAP/XML)", name = "exchange-ews")] - ExchangeEws(Box), - #[command( - about = "MS Exchange Online via Microsoft Graph", - name = "exchange-graph" - )] - ExchangeGraph(Box), -} - -#[derive(Args)] -struct GlobalArgs { - #[arg( - short = 'j', - long, - value_name = "N", - help = "Worker pool size (default: logical CPUs)" - )] - threads: Option, - - #[arg(long, help = "Compute and report the full plan; perform no writes")] - dry_run: bool, - - #[arg( - short = 'v', - long, - action = ArgAction::Count, - conflicts_with = "quiet", - help = "Increase log verbosity (repeatable, max -vvv)", - long_help = "Increase log verbosity (repeatable, max -vvv).\n \ - (default) per-type start/finish lines and a final summary\n \ - -v per-chunk progress, account resolution, retry notices\n \ - -vv every protocol call (method/command, target, status, timing)\n \ - -vvv full request/response bodies and backoff delays (stderr)" - )] - verbose: u8, - - #[arg( - short = 'q', - long, - help = "Quiet: warnings and errors only (verbosity level 0)" - )] - quiet: bool, - - #[arg( - long, - value_name = "N", - default_value_t = 5, - help = "Max retry attempts per request on transient failures" - )] - max_retries: u32, - - #[arg(long, help = "Accept self-signed / invalid TLS certificates")] - allow_invalid_certs: bool, -} - -#[derive(Args)] -#[command(group(clap::ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer"])))] -#[command(group(clap::ArgGroup::new("account").required(true).args(["account_id", "account_name"])))] -struct JmapImportArgs { - #[arg(long, value_name = "URL", help = "Base URL or full JMAP session URL")] - url: String, - - #[arg(long, value_name = "USER", help = "HTTP Basic user")] - auth_basic: Option, - - #[arg( - long, - value_name = "PASS", - help = "Basic password (prefer $INBUXA_MIGRATE_PASSWORD or prompt)" - )] - auth_password: Option, - - #[arg( - long, - value_name = "TOKEN", - num_args = 0..=1, - help = "Bearer token (value optional: falls back to $INBUXA_MIGRATE_TOKEN or prompt)" - )] - auth_bearer: Option>, - - #[arg(long, value_name = "ID", help = "Source account id")] - account_id: Option, - - #[arg(long, value_name = "NAME", help = "Source account name")] - account_name: Option, - - #[arg( - long, - value_name = "LIST", - help = "Comma-separated type list (default: all supported)" - )] - objects: Option, - - #[arg( - long, - help = "Permit importing when the archive records a different JMAP source" - )] - allow_source_change: bool, - - #[command(flatten)] - global: GlobalArgs, - - #[arg( - value_name = "ARCHIVE", - help = "Local SQLite archive (created if absent)" - )] - archive: PathBuf, -} - -#[derive(Args)] -#[command(group(clap::ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer"])))] -struct ImapImportArgs { - #[arg( - long, - value_name = "URL", - help = "imap://host[:port] or imaps://host[:port]" - )] - url: String, - - #[arg( - long, - value_name = "USER", - help = "PLAIN (preferred) with LOGIN fallback" - )] - auth_basic: Option, - - #[arg( - long, - value_name = "PASS", - help = "Basic password (prefer $INBUXA_MIGRATE_PASSWORD or prompt)" - )] - auth_password: Option, - - #[arg( - long, - value_name = "TOKEN", - num_args = 0..=1, - help = "Bearer token for XOAUTH2 (value optional: falls back to $INBUXA_MIGRATE_TOKEN or prompt)" - )] - auth_bearer: Option>, - - #[arg( - long, - value_name = "USER", - help = "XOAUTH2 identity (required with --auth-bearer)" - )] - auth_user: Option, - - #[arg(long, help = "Permit credentials on imap:// without STARTTLS")] - allow_cleartext: bool, - - #[arg(long, help = "Enable COMPRESS=DEFLATE post-auth (off by default)")] - compress: bool, - - #[arg( - long, - value_name = "REGEX", - action = ArgAction::Append, - help = "Folder include filter (repeatable)" - )] - include: Vec, - - #[arg( - long, - value_name = "REGEX", - action = ArgAction::Append, - help = "Folder exclude filter (repeatable)" - )] - exclude: Vec, - - #[arg( - long, - value_name = "ROLE", - action = ArgAction::Append, - help = "Drop folders matching a role (repeatable)" - )] - exclude_special: Vec, - - #[arg( - long, - value_name = "NAME", - action = ArgAction::Append, - help = "Exact include; mutually exclusive with --include/--exclude" - )] - folder: Vec, - - #[arg(long, help = "Keep only folders returned by LSUB")] - subscribed_only: bool, - - #[arg(long, help = "Disable SPECIAL-USE name-heuristic fallback")] - noautomap: bool, - - #[arg(long, help = "Import \\Deleted messages with the $deleted keyword")] - include_deleted: bool, - - #[arg( - long, - help = "Add the phrases of each message's Keywords: header (Exchange categories) as JMAP keywords" - )] - mime_keywords: bool, - - #[arg( - long, - value_name = "N", - default_value_t = 256, - help = "UIDs per metadata FETCH chunk" - )] - fetch_batch: usize, - - #[arg( - long, - value_name = "N", - help = "Parallel IMAP connections for body fetch (1..=8). Default: min(--threads, 8)" - )] - imap_connections: Option, - - #[arg( - long, - help = "Permit importing when the archive records a different IMAP source" - )] - allow_source_change: bool, - - #[command(flatten)] - global: GlobalArgs, - - #[arg( - value_name = "ARCHIVE", - help = "Local SQLite archive (created if absent)" - )] - archive: PathBuf, -} - -#[derive(Args)] -#[command(group(clap::ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer"])))] -#[command(group(clap::ArgGroup::new("account").required(true).args(["account_id", "account_name"])))] -struct ExportArgs { - #[arg(long, value_name = "URL", help = "Base URL or full JMAP session URL")] - url: String, - - #[arg(long, value_name = "USER", help = "HTTP Basic user")] - auth_basic: Option, - - #[arg( - long, - value_name = "PASS", - help = "Basic password (prefer $INBUXA_MIGRATE_PASSWORD or prompt)" - )] - auth_password: Option, - - #[arg( - long, - value_name = "TOKEN", - num_args = 0..=1, - help = "Bearer token (value optional: falls back to $INBUXA_MIGRATE_TOKEN or prompt)" - )] - auth_bearer: Option>, - - #[arg(long, value_name = "ID", help = "Target account id")] - account_id: Option, - - #[arg(long, value_name = "NAME", help = "Target account name")] - account_name: Option, - - #[arg( - long, - value_name = "LIST", - help = "Types to export (default: all present in archive)" - )] - objects: Option, - - #[arg(long, help = "Delete target objects that match nothing in the archive")] - prune: bool, - - #[arg(long, help = "Skip the interactive --prune confirmation")] - yes: bool, - - #[command(flatten)] - global: GlobalArgs, - - #[arg(value_name = "ARCHIVE", help = "Local SQLite archive")] - archive: PathBuf, -} - -#[derive(Args)] -struct InspectArgs { - #[arg(value_name = "ARCHIVE", help = "Local SQLite archive")] - archive: PathBuf, - - #[arg( - value_name = "TYPE", - help = "Object type to dump (omit for a summary of all types)" - )] - target: Option, - - #[arg( - long, - value_name = "N", - help = "Maximum rows to show (omit to dump every row)" - )] - limit: Option, - - #[arg( - long, - value_name = "N", - default_value_t = 0, - help = "Skip the first N rows" - )] - offset: usize, -} - -pub enum Action { - Import(CommonConfig, ImportConfig), - ImportImap(CommonConfig, ImapImportConfig), - ImportDav(CommonConfig, DavImportConfig), - ImportManageSieve(CommonConfig, ManageSieveImportConfig), - ImportMaildir(CommonConfig, MaildirImportConfig), - ImportTakeout(CommonConfig, TakeoutImportConfig), - ImportExchangeEws(CommonConfig, EwsImportConfig), - ImportExchangeGraph(CommonConfig, GraphImportConfig), - Export(CommonConfig, ExportConfig), - Inspect(InspectConfig), -} - -#[derive(Args)] -#[command(group(clap::ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer"])))] -pub struct ManageSieveImportArgs { - #[arg( - long, - value_name = "URL", - help = "sieve://host[:port] or sieves://host[:port]" - )] - url: String, - - #[arg( - long, - value_name = "USER", - help = "SASL PLAIN (preferred) with LOGIN fallback" - )] - auth_basic: Option, - - #[arg( - long, - value_name = "PASS", - help = "Basic password (prefer $INBUXA_MIGRATE_PASSWORD or prompt)" - )] - auth_password: Option, - - #[arg( - long, - value_name = "TOKEN", - num_args = 0..=1, - help = "Bearer token for OAUTHBEARER (value optional: $INBUXA_MIGRATE_TOKEN or prompt)" - )] - auth_bearer: Option>, - - #[arg( - long, - value_name = "USER", - help = "OAUTHBEARER identity (required with --auth-bearer)" - )] - auth_user: Option, - - #[arg(long, help = "Permit credentials on sieve:// without STARTTLS")] - allow_cleartext: bool, - - #[arg( - long, - help = "Permit importing when the archive records a different ManageSieve source" - )] - allow_source_change: bool, - - #[command(flatten)] - global: GlobalArgs, - - #[arg( - value_name = "ARCHIVE", - help = "Local SQLite archive (created if absent)" - )] - archive: PathBuf, -} - -#[derive(Args)] -#[command(group(clap::ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer"])))] -pub struct DavImportArgs { - #[arg(long, value_name = "URL", help = "http(s)://host[:port][/path]")] - url: String, - - #[arg(long, value_name = "USER", help = "HTTP Basic user")] - auth_basic: Option, - - #[arg( - long, - value_name = "PASS", - help = "Basic password (prefer $INBUXA_MIGRATE_PASSWORD or prompt)" - )] - auth_password: Option, - - #[arg( - long, - value_name = "TOKEN", - num_args = 0..=1, - help = "Bearer token (value optional: $INBUXA_MIGRATE_TOKEN or prompt)" - )] - auth_bearer: Option>, - - #[arg(long, help = "Permit credentials on http:// (no TLS)")] - allow_cleartext: bool, - - #[arg( - long, - value_name = "N", - default_value_t = 4, - help = "Parallel DAV worker connections (hard cap 8)" - )] - dav_connections: usize, - - #[arg( - long, - value_name = "N", - default_value_t = 50, - help = "Hrefs per calendar-/addressbook-multiget REPORT batch" - )] - multiget_batch: usize, - - #[arg( - long, - help = "Permit importing when the archive records a different DAV source" - )] - allow_source_change: bool, - - #[arg( - long, - conflicts_with = "allow_source_change", - help = "Remove other recorded sources of this kind, and their items, before importing" - )] - replace_source: bool, - - #[command(flatten)] - global: GlobalArgs, - - #[arg( - value_name = "ARCHIVE", - help = "Local SQLite archive (created if absent)" - )] - archive: PathBuf, -} - -impl Cli { - pub fn resolve(self) -> Result { - match self.command { - Command::Import { source } => resolve_import(source), - Command::Export(args) => resolve_export(*args), - Command::Inspect(args) => resolve_inspect(*args), - } - } -} - -fn resolve_import(source: Source) -> Result { - match source { - Source::Jmap(args) => resolve_jmap_import(*args), - Source::Imap(args) => resolve_imap_import(*args), - Source::Caldav(args) => resolve_dav_import(*args, DavKindArg::Caldav), - Source::Carddav(args) => resolve_dav_import(*args, DavKindArg::Carddav), - Source::Webdav(args) => resolve_dav_import(*args, DavKindArg::Webdav), - Source::Managesieve(args) => resolve_managesieve_import(*args), - Source::Maildir(args) => resolve_maildir_import(*args), - Source::Takeout(args) => resolve_takeout_import(*args), - Source::ExchangeEws(args) => resolve_exchange_ews_import(*args), - Source::ExchangeGraph(args) => resolve_exchange_graph_import(*args), - } -} - -fn resolve_managesieve_import(args: ManageSieveImportArgs) -> Result { - let common = common_config(&args.global, args.archive); - let auth = resolve_managesieve_auth( - args.auth_basic.as_deref(), - args.auth_password.as_deref(), - args.auth_bearer.as_ref(), - args.auth_user.as_deref(), - )?; - Ok(Action::ImportManageSieve( - common, - ManageSieveImportConfig { - url: args.url, - auth, - allow_cleartext: args.allow_cleartext, - allow_source_change: args.allow_source_change, - }, - )) -} - -fn resolve_managesieve_auth( - auth_basic: Option<&str>, - auth_password: Option<&str>, - auth_bearer: Option<&Option>, - auth_user: Option<&str>, -) -> Result { - if let Some(user) = auth_basic { - if auth_user.is_some() { - return Err(Error::Usage( - "--auth-user is only valid with --auth-bearer".to_owned(), - )); - } - let password = secret::resolve(auth_password, "INBUXA_MIGRATE_PASSWORD", "password")?; - return Ok(ManageSieveAuth::Basic { - user: user.to_owned(), - password, - }); - } - if auth_password.is_some() { - return Err(Error::Usage( - "--auth-password is only valid together with --auth-basic".to_owned(), - )); - } - let bearer = auth_bearer.ok_or_else(|| { - Error::Usage("exactly one of --auth-basic / --auth-bearer is required".to_owned()) - })?; - let user = auth_user - .ok_or_else(|| Error::Usage("--auth-user is required with --auth-bearer".to_owned()))? - .to_owned(); - let token = secret::resolve(bearer.as_deref(), "INBUXA_MIGRATE_TOKEN", "bearer token")?; - Ok(ManageSieveAuth::Bearer { user, token }) -} - -#[derive(Args)] -pub struct MaildirImportArgs { - #[arg( - value_name = "MAILDIR", - help = "Path to the Maildir++ root (a directory containing cur/ new/ tmp/)" - )] - maildir: PathBuf, - - #[arg( - value_name = "ARCHIVE", - help = "Local SQLite archive (created if absent)" - )] - archive: PathBuf, - - #[arg( - long, - value_name = "REGEX", - action = ArgAction::Append, - help = "Folder include filter against canonical name (repeatable)" - )] - include: Vec, - - #[arg( - long, - value_name = "REGEX", - action = ArgAction::Append, - help = "Folder exclude filter against canonical name (repeatable)" - )] - exclude: Vec, - - #[arg( - long, - value_name = "NAME", - action = ArgAction::Append, - help = "Exact include; mutually exclusive with --include/--exclude" - )] - folder: Vec, - - #[arg(long, help = "Disable the name-heuristic role assignment")] - noautomap: bool, - - #[arg(long, help = "Import T-flagged messages with the $deleted keyword")] - include_deleted: bool, - - #[arg( - long, - help = "Permit importing when the archive records a different maildir path" - )] - allow_source_change: bool, - - #[command(flatten)] - global: GlobalArgs, -} - -fn resolve_maildir_import(args: MaildirImportArgs) -> Result { - let common = common_config(&args.global, args.archive); - if !args.folder.is_empty() && (!args.include.is_empty() || !args.exclude.is_empty()) { - return Err(Error::Usage( - "--folder is mutually exclusive with --include/--exclude".to_owned(), - )); - } - let include = compile_regexes(&args.include, "--include")?; - let exclude = compile_regexes(&args.exclude, "--exclude")?; - Ok(Action::ImportMaildir( - common, - MaildirImportConfig { - maildir: args.maildir, - include, - exclude, - folder: args.folder, - automap: !args.noautomap, - include_deleted: args.include_deleted, - allow_source_change: args.allow_source_change, - }, - )) -} - -#[derive(Args)] -pub struct TakeoutImportArgs { - #[arg( - value_name = "PATH", - help = "Directory tree to scan recursively for .mbox / .ics / .vcf files" - )] - path: PathBuf, - - #[arg( - value_name = "ARCHIVE", - help = "Local SQLite archive (created if absent)" - )] - archive: PathBuf, - - #[arg(long, help = "Disable the system-label role assignment")] - noautomap: bool, - - #[arg( - long, - help = "Permit importing when the archive records a takeout source at a different path" - )] - allow_source_change: bool, - - #[command(flatten)] - global: GlobalArgs, -} - -fn resolve_takeout_import(args: TakeoutImportArgs) -> Result { - let common = common_config(&args.global, args.archive); - Ok(Action::ImportTakeout( - common, - TakeoutImportConfig { - takeout_root: args.path, - allow_source_change: args.allow_source_change, - automap: !args.noautomap, - }, - )) -} - -fn resolve_dav_import(args: DavImportArgs, kind: DavKindArg) -> Result { - let common = common_config(&args.global, args.archive); - let auth = resolve_dav_auth( - args.auth_basic.as_deref(), - args.auth_password.as_deref(), - args.auth_bearer.as_ref(), - )?; - let dav_connections = args.dav_connections.clamp(1, 8).min(common.threads.max(1)); - let multiget_batch = args.multiget_batch.max(1); - Ok(Action::ImportDav( - common, - DavImportConfig { - kind, - url: args.url, - auth, - allow_cleartext: args.allow_cleartext, - dav_connections, - multiget_batch, - source_change: match (args.replace_source, args.allow_source_change) { - (true, _) => SourceChange::Replace, - (false, true) => SourceChange::Allow, - (false, false) => SourceChange::Reject, - }, - }, - )) -} - -fn resolve_dav_auth( - auth_basic: Option<&str>, - auth_password: Option<&str>, - auth_bearer: Option<&Option>, -) -> Result { - if let Some(user) = auth_basic { - let password = secret::resolve(auth_password, "INBUXA_MIGRATE_PASSWORD", "password")?; - return Ok(DavAuth::Basic { - user: user.to_owned(), - password, - }); - } - if auth_password.is_some() { - return Err(Error::Usage( - "--auth-password is only valid together with --auth-basic".to_owned(), - )); - } - let bearer = auth_bearer.ok_or_else(|| { - Error::Usage("exactly one of --auth-basic / --auth-bearer is required".to_owned()) - })?; - let token = secret::resolve(bearer.as_deref(), "INBUXA_MIGRATE_TOKEN", "bearer token")?; - Ok(DavAuth::Bearer { token }) -} - -fn resolve_jmap_import(args: JmapImportArgs) -> Result { - let common = common_config(&args.global, args.archive); - let auth = resolve_auth( - args.auth_basic.as_deref(), - args.auth_password.as_deref(), - args.auth_bearer.as_ref(), - )?; - let account = resolve_account(args.account_id, args.account_name)?; - let objects = args.objects.as_deref().map(parse_object_list).transpose()?; - - Ok(Action::Import( - common, - ImportConfig { - connect: ConnectConfig { - url: args.url, - auth, - account, - }, - objects, - allow_source_change: args.allow_source_change, - }, - )) -} - -fn resolve_imap_import(args: ImapImportArgs) -> Result { - let common = common_config(&args.global, args.archive); - let auth = resolve_imap_auth( - args.auth_basic.as_deref(), - args.auth_password.as_deref(), - args.auth_bearer.as_ref(), - args.auth_user.as_deref(), - )?; - if !args.folder.is_empty() && (!args.include.is_empty() || !args.exclude.is_empty()) { - return Err(Error::Usage( - "--folder is mutually exclusive with --include/--exclude".to_owned(), - )); - } - let include = compile_regexes(&args.include, "--include")?; - let exclude = compile_regexes(&args.exclude, "--exclude")?; - let imap_connections = args.imap_connections.unwrap_or(common.threads).clamp(1, 8); - - Ok(Action::ImportImap( - common, - ImapImportConfig { - url: args.url, - auth, - allow_cleartext: args.allow_cleartext, - compress: args.compress, - include, - exclude, - exclude_special: args.exclude_special, - folder: args.folder, - subscribed_only: args.subscribed_only, - automap: !args.noautomap, - include_deleted: args.include_deleted, - mime_keywords: args.mime_keywords, - fetch_batch: args.fetch_batch, - imap_connections, - allow_source_change: args.allow_source_change, - }, - )) -} - -fn compile_regexes(raw: &[String], flag: &str) -> Result, Error> { - let mut out = Vec::with_capacity(raw.len()); - for r in raw { - let re = - Regex::new(r).map_err(|e| Error::Usage(format!("{flag} {r:?}: invalid regex: {e}")))?; - out.push(re); - } - Ok(out) -} - -fn resolve_imap_auth( - auth_basic: Option<&str>, - auth_password: Option<&str>, - auth_bearer: Option<&Option>, - auth_user: Option<&str>, -) -> Result { - if let Some(user) = auth_basic { - if auth_user.is_some() { - return Err(Error::Usage( - "--auth-user is only valid with --auth-bearer".to_owned(), - )); - } - let password = secret::resolve(auth_password, "INBUXA_MIGRATE_PASSWORD", "password")?; - return Ok(ImapAuth::Basic { - user: user.to_owned(), - password, - }); - } - if auth_password.is_some() { - return Err(Error::Usage( - "--auth-password is only valid together with --auth-basic".to_owned(), - )); - } - let bearer = auth_bearer.ok_or_else(|| { - Error::Usage("exactly one of --auth-basic / --auth-bearer is required".to_owned()) - })?; - let user = auth_user - .ok_or_else(|| Error::Usage("--auth-user is required with --auth-bearer".to_owned()))? - .to_owned(); - let token = secret::resolve(bearer.as_deref(), "INBUXA_MIGRATE_TOKEN", "bearer token")?; - Ok(ImapAuth::Bearer { user, token }) -} - -fn resolve_export(args: ExportArgs) -> Result { - let common = common_config(&args.global, args.archive); - let auth = resolve_auth( - args.auth_basic.as_deref(), - args.auth_password.as_deref(), - args.auth_bearer.as_ref(), - )?; - let account = resolve_account(args.account_id, args.account_name)?; - let objects = args.objects.as_deref().map(parse_object_list).transpose()?; - - Ok(Action::Export( - common, - ExportConfig { - connect: ConnectConfig { - url: args.url, - auth, - account, - }, - objects, - prune: args.prune, - yes: args.yes, - }, - )) -} - -fn resolve_inspect(args: InspectArgs) -> Result { - let target = match args.target.as_deref() { - Some(token) => Some(ObjectType::parse(token)?), - None => None, - }; - if let Some(limit) = args.limit - && limit == 0 - { - return Err(Error::Usage("--limit must be greater than zero".to_owned())); - } - Ok(Action::Inspect(InspectConfig { - archive: args.archive, - target, - limit: args.limit, - offset: args.offset, - })) -} - -#[derive(Args)] -#[command(group(clap::ArgGroup::new("ews_auth").required(true).args([ - "auth_basic", "auth_bearer", -])))] -pub struct ExchangeEwsImportArgs { - #[arg(long, value_name = "URL", help = "Fully-qualified EWS endpoint URL")] - url: Option, - - #[arg( - long, - value_name = "EMAIL", - help = "Target mailbox SMTP address (required for autodiscover and app-only OAuth)" - )] - mailbox: Option, - - #[arg( - long, - value_name = "KIND", - default_value = "primary", - help = "Mailbox surface: primary | archive | public-folders" - )] - mailbox_kind: String, - - #[arg(long, value_name = "USER", help = "Basic auth user (on-prem only)")] - auth_basic: Option, - - #[arg( - long, - value_name = "PASS", - help = "Basic password (prefer $INBUXA_MIGRATE_PASSWORD or prompt)" - )] - auth_password: Option, - - #[arg( - long, - value_name = "TOKEN", - num_args = 0..=1, - help = "Bearer token (value optional: $INBUXA_MIGRATE_TOKEN or device-code flow)" - )] - auth_bearer: Option>, - - #[arg( - long, - value_name = "TENANT", - help = "Entra tenant id or .onmicrosoft.com short name" - )] - ews_tenant: Option, - - #[arg(long, value_name = "ID", help = "Entra app (client) id")] - ews_client_id: Option, - - #[arg( - long, - value_name = "SECRET", - help = "OAuth client secret (app-only flow; prefer $INBUXA_MIGRATE_EWS_CLIENT_SECRET)" - )] - ews_client_secret: Option, - - #[arg(long, help = "Use the OAuth device-code interactive flow")] - ews_device_code: bool, - - #[arg( - long, - value_name = "N", - default_value_t = 4, - help = "Parallel EWS worker connections (hard cap 8)" - )] - ews_connections: usize, - - #[arg( - long, - value_name = "N", - default_value_t = 20, - help = "ItemIds per GetItem batch" - )] - ews_getitem_batch: usize, - - #[arg( - long, - value_name = "N", - default_value_t = 5, - help = "AttachmentIds per GetAttachment batch" - )] - ews_attachment_batch: usize, - - #[arg(long, help = "Force FindItem even when SyncFolderItems would work")] - ews_no_syncfolderitems: bool, - - #[arg( - long, - help = "Permit importing when the archive records a different EWS source" - )] - allow_source_change: bool, - - #[command(flatten)] - global: GlobalArgs, - - #[arg( - value_name = "ARCHIVE", - help = "Local SQLite archive (created if absent)" - )] - archive: PathBuf, -} - -fn resolve_exchange_ews_import(args: ExchangeEwsImportArgs) -> Result { - let archive = args.archive.clone(); - let common = common_config(&args.global, archive); - let mailbox_kind = match args.mailbox_kind.as_str() { - "primary" => MailboxKind::Primary, - "archive" => MailboxKind::Archive, - "public-folders" => MailboxKind::PublicFolders, - other => { - return Err(Error::Usage(format!( - "--mailbox-kind must be primary | archive | public-folders, got {other:?}" - ))); - } - }; - let auth = resolve_exchange_ews_auth(&args)?; - let ews_connections = args.ews_connections.clamp(1, 8).min(common.threads.max(1)); - let config = EwsImportConfig { - url: args.url, - mailbox: args.mailbox, - mailbox_kind, - auth, - ews_connections, - getitem_batch: args.ews_getitem_batch.max(1), - attachment_batch: args.ews_attachment_batch.max(1), - use_syncfolderitems: !args.ews_no_syncfolderitems, - allow_source_change: args.allow_source_change, - }; - Ok(Action::ImportExchangeEws(common, config)) -} - -fn resolve_exchange_ews_auth(args: &ExchangeEwsImportArgs) -> Result { - if let Some(user) = args.auth_basic.as_deref() { - let password = secret::resolve( - args.auth_password.as_deref(), - "INBUXA_MIGRATE_PASSWORD", - "password", - )?; - return Ok(EwsAuth::Basic { - user: user.to_owned(), - password, - }); - } - if args.auth_password.is_some() { - return Err(Error::Usage( - "--auth-password is only valid with --auth-basic".to_owned(), - )); - } - let Some(bearer) = args.auth_bearer.as_ref() else { - return Err(Error::Usage( - "exactly one of --auth-basic / --auth-bearer is required".to_owned(), - )); - }; - if let Some(token) = bearer.clone() { - return Ok(EwsAuth::Bearer { token }); - } - let tenant = || { - args.ews_tenant - .clone() - .unwrap_or_else(|| "common".to_owned()) - }; - let required_client_id = || { - args.ews_client_id.clone().ok_or_else(|| { - Error::Usage( - "--ews-client-id is required with --ews-device-code or --ews-client-secret" - .to_owned(), - ) - }) - }; - if args.ews_device_code { - return Ok(EwsAuth::OAuth(OAuthFlow::DeviceCode { - tenant: tenant(), - client_id: required_client_id()?, - })); - } - if let Some(secret) = args.ews_client_secret.as_deref() { - return Ok(EwsAuth::OAuth(OAuthFlow::ClientCredentials { - tenant: tenant(), - client_id: required_client_id()?, - client_secret: secret.to_owned(), - })); - } - if let Ok(token) = std::env::var("INBUXA_MIGRATE_TOKEN") - && !token.is_empty() - { - return Ok(EwsAuth::Bearer { token }); - } - if let Some(client_id) = args.ews_client_id.clone() - && let Ok(client_secret) = std::env::var("INBUXA_MIGRATE_EWS_CLIENT_SECRET") - && !client_secret.is_empty() - { - return Ok(EwsAuth::OAuth(OAuthFlow::ClientCredentials { - tenant: tenant(), - client_id, - client_secret, - })); - } - let token = secret::resolve(None, "INBUXA_MIGRATE_TOKEN", "bearer token")?; - Ok(EwsAuth::Bearer { token }) -} - -#[derive(Args)] -pub struct ExchangeGraphImportArgs { - #[arg( - long, - value_name = "UUID", - help = "Entra app (client) id (required unless --access-token is given)" - )] - client_id: Option, - - #[arg( - long, - value_name = "ID", - default_value = "common", - help = "Entra authority tenant" - )] - tenant: String, - - #[arg( - long, - value_name = "UPN|UUID", - help = "Target a different mailbox the signed-in account can read" - )] - user: Option, - - #[arg( - long, - value_name = "KIND", - default_value = "primary", - value_parser = parse_graph_mailbox_kind, - help = "Mailbox surface: primary | archive (public-folders is rejected: use exchange-ews)" - )] - mailbox_kind: GraphMailboxKind, - - #[arg( - long, - value_name = "TOKEN", - num_args = 0..=1, - help = "Pre-acquired bearer token (skip device-code flow). Resolves: flag value -> $INBUXA_MIGRATE_GRAPH_TOKEN -> prompt." - )] - access_token: Option>, - - #[arg( - long, - value_name = "LIST", - help = "Comma-separated surface list: mail | calendar | contacts | files (default: all)" - )] - objects: Option, - - #[arg( - long, - value_name = "FMT", - default_value = "text", - value_parser = parse_event_body_format, - help = "outlook.body-content-type for event GETs: text | html" - )] - event_body_format: EventBodyFormat, - - #[arg( - long, - value_name = "N", - default_value_t = 4, - help = "Parallel Graph worker connections (hard cap 16)" - )] - graph_connections: usize, - - #[arg( - long, - value_name = "N", - default_value_t = 100, - help = "Per-page size ($top query parameter, range [1, 1000])" - )] - top: usize, - - #[arg( - long, - value_name = "N", - default_value_t = 5, - help = "Years either side of today to scan for edited recurrence occurrences (max 5)", - long_help = "Years either side of today to scan for edited recurrence occurrences.\n \ - Graph reports a series exception only inside an expanded calendar view, and caps\n \ - any single view at five years, so inbuxa-migrate scans [today - N, today] and\n \ - [today, today + N]. An occurrence edited outside that span imports with the\n \ - series default instead of its edit." - )] - exception_window_years: i32, - - #[arg( - long, - help = "Skip contact photos (saves one request per contact on large address books)", - long_help = "Skip contact photos.\n Graph will not say which contacts have a photo: $expand=photo comes back empty\n even when one exists, so the only way to find out is to ask for the bytes.\n inbuxa-migrate therefore spends one extra request per contact. Pass this to skip\n photos entirely on a large address book." - )] - skip_contact_photos: bool, - - #[arg( - long, - help = "Skip event file attachments (they are stored as JSCalendar enclosure links)" - )] - skip_event_attachments: bool, - - #[arg( - long, - value_name = "URL", - default_value = "https://graph.microsoft.com/v1.0", - help = "Microsoft Graph API base URL (override for national clouds or tests)", - hide = true - )] - api_base: String, - - #[arg( - long, - help = "Permit importing when the archive records a different exchange_graph source" - )] - allow_source_change: bool, - - #[command(flatten)] - global: GlobalArgs, - - #[arg( - value_name = "ARCHIVE", - help = "Local SQLite archive (created if absent)" - )] - archive: PathBuf, -} - -fn parse_graph_mailbox_kind(s: &str) -> Result { - GraphMailboxKind::parse(s).map_err(|e| e.to_string()) -} - -fn parse_event_body_format(s: &str) -> Result { - EventBodyFormat::parse(s).map_err(|e| e.to_string()) -} - -fn resolve_exchange_graph_import(args: ExchangeGraphImportArgs) -> Result { - let common = common_config(&args.global, args.archive.clone()); - let mailbox_kind = args.mailbox_kind; - let event_body_format = args.event_body_format; - let auth = resolve_graph_auth( - args.access_token.as_ref(), - args.client_id.as_deref(), - &args.tenant, - )?; - let surfaces = args - .objects - .as_deref() - .map(Surfaces::parse_list) - .transpose()? - .unwrap_or_default(); - let top = args.top.clamp(1, 1000); - let graph_connections = args - .graph_connections - .clamp(1, 16) - .min(common.threads.max(1)); - Ok(Action::ImportExchangeGraph( - common, - GraphImportConfig { - auth, - api_base: args.api_base, - user_target: args.user, - mailbox_kind, - surfaces, - event_body_format, - graph_connections, - top, - exception_window_years: args.exception_window_years, - contact_photos: !args.skip_contact_photos, - event_attachments: !args.skip_event_attachments, - allow_source_change: args.allow_source_change, - }, - )) -} - -fn resolve_graph_auth( - access_token: Option<&Option>, - client_id: Option<&str>, - tenant: &str, -) -> Result { - if let Some(slot) = access_token { - let token = secret::resolve( - slot.as_deref(), - "INBUXA_MIGRATE_GRAPH_TOKEN", - "bearer token", - )?; - return Ok(GraphAuth::PreAcquired { token }); - } - let client_id = client_id - .ok_or_else(|| { - Error::Usage( - "--client-id is required for exchange-graph (unless --access-token is given)" - .to_owned(), - ) - })? - .to_owned(); - let authority = crate::exchange_graph::oauth::default_authority(tenant); - Ok(GraphAuth::DeviceCode { - authority, - client_id, - }) -} - -fn common_config(global: &GlobalArgs, archive: PathBuf) -> CommonConfig { - let threads = global - .threads - .filter(|n| *n > 0) - .unwrap_or_else(num_cpus::get) - .max(1); - CommonConfig { - archive, - threads, - dry_run: global.dry_run, - max_retries: global.max_retries, - allow_invalid_certs: global.allow_invalid_certs, - logger: Logger::from_flags(global.quiet, global.verbose), - } -} - -fn resolve_auth( - auth_basic: Option<&str>, - auth_password: Option<&str>, - auth_bearer: Option<&Option>, -) -> Result { - if let Some(user) = auth_basic { - let password = secret::resolve(auth_password, "INBUXA_MIGRATE_PASSWORD", "password")?; - return Ok(Auth::Basic { - user: user.to_owned(), - password, - }); - } - if auth_password.is_some() { - return Err(Error::Usage( - "--auth-password is only valid together with --auth-basic".to_owned(), - )); - } - let bearer = auth_bearer.ok_or_else(|| { - Error::Usage("exactly one of --auth-basic / --auth-bearer is required".to_owned()) - })?; - let token = secret::resolve(bearer.as_deref(), "INBUXA_MIGRATE_TOKEN", "bearer token")?; - Ok(Auth::Bearer { token }) -} - -fn resolve_account( - account_id: Option, - account_name: Option, -) -> Result { - match (account_id, account_name) { - (Some(id), None) => Ok(AccountSelector::Id(id)), - (None, Some(name)) => Ok(AccountSelector::Name(name)), - _ => Err(Error::Usage( - "exactly one of --account-id / --account-name is required".to_owned(), - )), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::env; - - fn lock() -> std::sync::MutexGuard<'static, ()> { - static M: std::sync::Mutex<()> = std::sync::Mutex::new(()); - M.lock().unwrap_or_else(|e| e.into_inner()) - } - - #[test] - fn auth_password_without_basic_is_usage_error() { - let _g = lock(); - unsafe { - env::remove_var("INBUXA_MIGRATE_TOKEN"); - env::remove_var("INBUXA_MIGRATE_PASSWORD"); - } - let bearer: Option = Some("t".to_owned()); - let r = resolve_auth(None, Some("hunter2"), Some(&bearer)); - match r { - Err(Error::Usage(m)) => { - assert!(m.contains("auth-password"), "msg was: {m}"); - } - other => panic!("expected Usage error, got {other:?}"), - } - } - - #[test] - fn auth_basic_with_password_resolves() { - let _g = lock(); - unsafe { - env::remove_var("INBUXA_MIGRATE_TOKEN"); - env::remove_var("INBUXA_MIGRATE_PASSWORD"); - } - let auth = resolve_auth(Some("alice"), Some("pw"), None).unwrap(); - match auth { - Auth::Basic { user, password } => { - assert_eq!(user, "alice"); - assert_eq!(password, "pw"); - } - _ => panic!("expected Basic"), - } - } - - #[test] - fn auth_bearer_inline_token_resolves() { - let _g = lock(); - unsafe { - env::remove_var("INBUXA_MIGRATE_TOKEN"); - env::remove_var("INBUXA_MIGRATE_PASSWORD"); - } - let bearer: Option = Some("t".to_owned()); - let auth = resolve_auth(None, None, Some(&bearer)).unwrap(); - assert!(matches!(auth, Auth::Bearer { token } if token == "t")); - } - - fn ews_auth(extra: &[&str]) -> Result { - let mut argv = vec!["inbuxa-migrate", "import", "exchange-ews", "archive.sqlite"]; - argv.extend_from_slice(extra); - let cli = Cli::try_parse_from(argv).unwrap_or_else(|e| panic!("clap rejected args: {e}")); - match cli.resolve()? { - Action::ImportExchangeEws(_, config) => Ok(config.auth), - _ => panic!("expected ImportExchangeEws"), - } - } - - fn set_ews_env(token: Option<&str>, client_secret: Option<&str>) { - unsafe { - match token { - Some(t) => env::set_var("INBUXA_MIGRATE_TOKEN", t), - None => env::remove_var("INBUXA_MIGRATE_TOKEN"), - } - match client_secret { - Some(s) => env::set_var("INBUXA_MIGRATE_EWS_CLIENT_SECRET", s), - None => env::remove_var("INBUXA_MIGRATE_EWS_CLIENT_SECRET"), - } - } - } - - // A bare --auth-bearer takes INBUXA_MIGRATE_TOKEN without needing --ews-client-id. - #[test] - fn ews_bare_bearer_uses_env_token_without_client_id() { - let _g = lock(); - set_ews_env(Some("envtok"), None); - let auth = ews_auth(&["--auth-bearer"]); - set_ews_env(None, None); - assert!(matches!(auth, Ok(EwsAuth::Bearer { token }) if token == "envtok")); - } - - // INBUXA_MIGRATE_TOKEN wins over client credentials taken from the environment. - #[test] - fn ews_env_token_beats_env_client_secret() { - let _g = lock(); - set_ews_env(Some("envtok"), Some("sec")); - let auth = ews_auth(&["--auth-bearer", "--ews-client-id", "cid"]); - set_ews_env(None, None); - assert!(matches!(auth, Ok(EwsAuth::Bearer { token }) if token == "envtok")); - } - - // The env client secret is used when no token is exported. - #[test] - fn ews_env_client_secret_without_token_uses_client_credentials() { - let _g = lock(); - set_ews_env(None, Some("sec")); - let auth = ews_auth(&["--auth-bearer", "--ews-client-id", "cid"]); - set_ews_env(None, None); - assert!(matches!( - auth, - Ok(EwsAuth::OAuth(OAuthFlow::ClientCredentials { client_id, client_secret, .. })) - if client_id == "cid" && client_secret == "sec" - )); - } - - // An explicit --ews-device-code is not overridden by an exported token. - #[test] - fn ews_device_code_flag_beats_env_token() { - let _g = lock(); - set_ews_env(Some("envtok"), None); - let auth = ews_auth(&[ - "--auth-bearer", - "--ews-client-id", - "cid", - "--ews-device-code", - ]); - set_ews_env(None, None); - assert!(matches!( - auth, - Ok(EwsAuth::OAuth(OAuthFlow::DeviceCode { tenant, client_id })) - if tenant == "common" && client_id == "cid" - )); - } - - // --ews-device-code still needs a client id. - #[test] - fn ews_device_code_without_client_id_is_usage_error() { - let _g = lock(); - set_ews_env(Some("envtok"), None); - let auth = ews_auth(&["--auth-bearer", "--ews-device-code"]); - set_ews_env(None, None); - match auth { - Err(Error::Usage(m)) => assert!(m.contains("--ews-client-id"), "msg was: {m}"), - other => panic!("expected Usage error, got {other:?}"), - } - } - - // An inline token wins over everything else. - #[test] - fn ews_inline_token_beats_env_and_flows() { - let _g = lock(); - set_ews_env(Some("envtok"), None); - let auth = ews_auth(&[ - "--auth-bearer=inline", - "--ews-client-id", - "cid", - "--ews-device-code", - ]); - set_ews_env(None, None); - assert!(matches!(auth, Ok(EwsAuth::Bearer { token }) if token == "inline")); - } - - fn graph_config(objects: &[&str]) -> Result { - let mut argv = vec![ - "inbuxa-migrate", - "import", - "exchange-graph", - "--access-token", - "tok", - ]; - argv.extend_from_slice(objects); - argv.push("archive.sqlite"); - let cli = Cli::try_parse_from(argv).unwrap_or_else(|e| panic!("clap rejected args: {e}")); - match cli.resolve()? { - Action::ImportExchangeGraph(_, config) => Ok(config), - _ => panic!("expected ImportExchangeGraph"), - } - } - - #[test] - fn graph_objects_contacts_selects_only_contacts() { - let _g = lock(); - let config = graph_config(&["--objects", "contacts"]).unwrap(); - assert_eq!( - config.surfaces, - Surfaces { - mail: false, - calendar: false, - contacts: true, - files: false - } - ); - } - - #[test] - fn graph_objects_absent_selects_every_surface() { - let _g = lock(); - let config = graph_config(&[]).unwrap(); - assert_eq!(config.surfaces, Surfaces::ALL); - } - - #[test] - fn graph_objects_rejects_jmap_type_names() { - let _g = lock(); - match graph_config(&["--objects", "contactcard"]) { - Err(Error::Usage(m)) => { - assert!(m.contains("unknown surface: contactcard"), "msg was: {m}"); - assert!( - m.contains("valid: mail, calendar, contacts"), - "msg was: {m}" - ); - } - other => panic!("expected Usage error, got {:?}", other.map(|_| ())), - } - } - - fn webdav_source_change(extra: &[&str]) -> Result { - let mut argv = vec![ - "inbuxa-migrate", - "import", - "webdav", - "--url", - "https://dav.example/files/u/", - "--auth-bearer", - "tok", - ]; - argv.extend_from_slice(extra); - argv.push("archive.sqlite"); - let cli = Cli::try_parse_from(argv)?; - match cli.resolve() { - Ok(Action::ImportDav(_, config)) => Ok(config.source_change), - _ => panic!("expected ImportDav"), - } - } - - // --replace-source maps to Replace and cannot be combined with --allow-source-change. - #[test] - fn dav_source_change_flags_resolve() { - let _g = lock(); - assert_eq!(webdav_source_change(&[]).unwrap(), SourceChange::Reject); - assert_eq!( - webdav_source_change(&["--allow-source-change"]).unwrap(), - SourceChange::Allow - ); - assert_eq!( - webdav_source_change(&["--replace-source"]).unwrap(), - SourceChange::Replace - ); - assert!(webdav_source_change(&["--replace-source", "--allow-source-change"]).is_err()); - } -} diff --git a/src/cli/auth.rs b/src/cli/auth.rs new file mode 100644 index 0000000..e2c08b9 --- /dev/null +++ b/src/cli/auth.rs @@ -0,0 +1,164 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use clap::{ArgGroup, Args}; + +use crate::error::Error; +use crate::jmap::account::AccountSelector; +use crate::net::auth::Auth; +use crate::secret::Secret; + +#[derive(Args)] +#[command(group(ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer"])))] +pub(super) struct AuthArgs { + #[arg(long, value_name = "USER", help = "HTTP Basic user")] + auth_basic: Option, + + #[arg( + long, + value_name = "PASS", + help = "Basic password (prefer $INBUXA_MIGRATE_PASSWORD or prompt)" + )] + auth_password: Option, + + #[arg( + long, + value_name = "TOKEN", + num_args = 0..=1, + help = "Bearer token (value optional: falls back to $INBUXA_MIGRATE_TOKEN or prompt)" + )] + auth_bearer: Option>, +} + +impl AuthArgs { + pub(super) fn resolve(&self) -> Result { + let flags = AuthFlags { + basic: self.auth_basic.as_deref(), + password: self.auth_password.as_deref(), + bearer: self.auth_bearer.as_ref().map(Option::as_deref), + }; + flags.resolve() + } +} + +#[derive(Clone, Copy)] +pub(super) struct AuthFlags<'a> { + pub(super) basic: Option<&'a str>, + pub(super) password: Option<&'a str>, + pub(super) bearer: Option>, +} + +impl AuthFlags<'_> { + pub(super) fn resolve(self) -> Result { + if let Some(user) = self.basic { + let password = Secret::resolve(self.password, "INBUXA_MIGRATE_PASSWORD", "password")?; + return Ok(Auth::Basic { + user: user.to_owned(), + password: password.into_inner(), + }); + } + if self.password.is_some() { + return Err(Error::Usage( + "--auth-password is only valid together with --auth-basic".to_owned(), + )); + } + let bearer = self.bearer.ok_or_else(|| { + Error::Usage("exactly one of --auth-basic / --auth-bearer is required".to_owned()) + })?; + let token = Secret::resolve(bearer, "INBUXA_MIGRATE_TOKEN", "bearer token")?; + Ok(Auth::Bearer { + token: token.into_inner(), + }) + } +} + +pub(super) struct AccountFlags { + pub(super) id: Option, + pub(super) name: Option, +} + +impl AccountFlags { + pub(super) fn resolve(self) -> Result { + match (self.id, self.name) { + (Some(id), None) => Ok(AccountSelector::Id(id)), + (None, Some(name)) => Ok(AccountSelector::Name(name)), + _ => Err(Error::Usage( + "exactly one of --account-id / --account-name is required".to_owned(), + )), + } + } +} + +#[cfg(test)] +mod tests { + use std::env; + + use super::*; + use crate::cli::env_lock as lock; + + #[test] + fn auth_password_without_basic_is_usage_error() { + let _g = lock(); + unsafe { + env::remove_var("INBUXA_MIGRATE_TOKEN"); + env::remove_var("INBUXA_MIGRATE_PASSWORD"); + } + let bearer: Option = Some("t".to_owned()); + let r = AuthFlags { + basic: None, + password: Some("hunter2"), + bearer: Some(bearer.as_deref()), + } + .resolve(); + match r { + Err(Error::Usage(m)) => { + assert!(m.contains("auth-password"), "msg was: {m}"); + } + other => panic!("expected Usage error, got {other:?}"), + } + } + + #[test] + fn auth_basic_with_password_resolves() { + let _g = lock(); + unsafe { + env::remove_var("INBUXA_MIGRATE_TOKEN"); + env::remove_var("INBUXA_MIGRATE_PASSWORD"); + } + let auth = AuthFlags { + basic: Some("alice"), + password: Some("pw"), + bearer: None, + } + .resolve() + .unwrap(); + match auth { + Auth::Basic { user, password } => { + assert_eq!(user, "alice"); + assert_eq!(password, "pw"); + } + _ => panic!("expected Basic"), + } + } + + #[test] + fn auth_bearer_inline_token_resolves() { + let _g = lock(); + unsafe { + env::remove_var("INBUXA_MIGRATE_TOKEN"); + env::remove_var("INBUXA_MIGRATE_PASSWORD"); + } + let bearer: Option = Some("t".to_owned()); + let auth = AuthFlags { + basic: None, + password: None, + bearer: Some(bearer.as_deref()), + } + .resolve() + .unwrap(); + assert!(matches!(auth, Auth::Bearer { token } if token == "t")); + } +} diff --git a/src/cli/dav.rs b/src/cli/dav.rs new file mode 100644 index 0000000..9d89405 --- /dev/null +++ b/src/cli/dav.rs @@ -0,0 +1,156 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::path::PathBuf; + +use clap::Args; + +use super::Action; +use super::auth::AuthFlags; +use super::global::GlobalArgs; +use crate::archive::sources::SourceChange; +use crate::error::Error; +use crate::import::dav::{DavImport, DavKindArg}; + +#[derive(Args)] +#[command(group(clap::ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer"])))] +pub struct DavImportArgs { + #[arg(long, value_name = "URL", help = "http(s)://host[:port][/path]")] + url: String, + + #[arg(long, value_name = "USER", help = "HTTP Basic user")] + auth_basic: Option, + + #[arg( + long, + value_name = "PASS", + help = "Basic password (prefer $INBUXA_MIGRATE_PASSWORD or prompt)" + )] + auth_password: Option, + + #[arg( + long, + value_name = "TOKEN", + num_args = 0..=1, + help = "Bearer token (value optional: $INBUXA_MIGRATE_TOKEN or prompt)" + )] + auth_bearer: Option>, + + #[arg(long, help = "Permit credentials on http:// (no TLS)")] + allow_cleartext: bool, + + #[arg( + long, + value_name = "N", + default_value_t = 4, + help = "Parallel DAV worker connections (hard cap 8)" + )] + dav_connections: usize, + + #[arg( + long, + value_name = "N", + default_value_t = 50, + help = "Hrefs per calendar-/addressbook-multiget REPORT batch" + )] + multiget_batch: usize, + + #[arg( + long, + help = "Permit importing when the archive records a different DAV source" + )] + allow_source_change: bool, + + #[arg( + long, + conflicts_with = "allow_source_change", + help = "Remove other recorded sources of this kind, and their items, before importing" + )] + replace_source: bool, + + #[command(flatten)] + global: GlobalArgs, + + #[arg( + value_name = "ARCHIVE", + help = "Local SQLite archive (created if absent)" + )] + archive: PathBuf, +} + +impl DavImportArgs { + pub(super) fn resolve(self, kind: DavKindArg) -> Result { + let common = self.global.common_config(self.archive); + let auth = AuthFlags { + basic: self.auth_basic.as_deref(), + password: self.auth_password.as_deref(), + bearer: self.auth_bearer.as_ref().map(Option::as_deref), + } + .resolve()?; + let dav_connections = self.dav_connections.clamp(1, 8).min(common.threads.max(1)); + let multiget_batch = self.multiget_batch.max(1); + Ok(Action::import( + common, + DavImport { + kind, + url: self.url, + auth, + allow_cleartext: self.allow_cleartext, + dav_connections, + multiget_batch, + source_change: match (self.replace_source, self.allow_source_change) { + (true, _) => SourceChange::Replace, + (false, true) => SourceChange::Allow, + (false, false) => SourceChange::Reject, + }, + }, + )) + } +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + use super::*; + use crate::cli::{Cli, env_lock as lock}; + use crate::import::Import; + + fn webdav_source_change(extra: &[&str]) -> Result { + let mut argv = vec![ + "inbuxa-migrate", + "import", + "webdav", + "--url", + "https://dav.example/files/u/", + "--auth-bearer", + "tok", + ]; + argv.extend_from_slice(extra); + argv.push("archive.sqlite"); + let cli = Cli::try_parse_from(argv)?; + match cli.resolve() { + Ok(Action::Import(_, Import::Dav(config))) => Ok(config.source_change), + _ => panic!("expected ImportDav"), + } + } + + // --replace-source maps to Replace and cannot be combined with --allow-source-change. + #[test] + fn dav_source_change_flags_resolve() { + let _g = lock(); + assert_eq!(webdav_source_change(&[]).unwrap(), SourceChange::Reject); + assert_eq!( + webdav_source_change(&["--allow-source-change"]).unwrap(), + SourceChange::Allow + ); + assert_eq!( + webdav_source_change(&["--replace-source"]).unwrap(), + SourceChange::Replace + ); + assert!(webdav_source_change(&["--replace-source", "--allow-source-change"]).is_err()); + } +} diff --git a/src/cli/ews.rs b/src/cli/ews.rs new file mode 100644 index 0000000..50c6114 --- /dev/null +++ b/src/cli/ews.rs @@ -0,0 +1,346 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::env; +use std::path::PathBuf; + +use clap::Args; + +use super::Action; +use super::global::GlobalArgs; +use crate::archive::sources::SourceChange; +use crate::error::Error; +use crate::exchange::ews::oauth::OAuthFlow; +use crate::exchange::ews::types::MailboxKind; +use crate::import::ews::items::MAX_EWS_CONNECTIONS; +use crate::import::ews::{EwsAuth, EwsImport, ItemEnumeration}; +use crate::secret::Secret; + +#[derive(Args)] +#[command(group(clap::ArgGroup::new("ews_auth").required(true).args([ + "auth_basic", "auth_bearer", +])))] +pub struct ExchangeEwsImportArgs { + #[arg(long, value_name = "URL", help = "Fully-qualified EWS endpoint URL")] + url: Option, + + #[arg( + long, + value_name = "EMAIL", + help = "Target mailbox SMTP address (required for autodiscover and app-only OAuth)" + )] + mailbox: Option, + + #[arg( + long, + value_name = "KIND", + default_value = "primary", + help = "Mailbox surface: primary | archive | public-folders" + )] + mailbox_kind: String, + + #[arg(long, value_name = "USER", help = "Basic auth user (on-prem only)")] + auth_basic: Option, + + #[arg( + long, + value_name = "PASS", + help = "Basic password (prefer $INBUXA_MIGRATE_PASSWORD or prompt)" + )] + auth_password: Option, + + #[arg( + long, + value_name = "TOKEN", + num_args = 0..=1, + help = "Bearer token (value optional: $INBUXA_MIGRATE_TOKEN or device-code flow)" + )] + auth_bearer: Option>, + + #[arg( + long, + value_name = "TENANT", + help = "Entra tenant id or .onmicrosoft.com short name" + )] + ews_tenant: Option, + + #[arg(long, value_name = "ID", help = "Entra app (client) id")] + ews_client_id: Option, + + #[arg( + long, + value_name = "SECRET", + help = "OAuth client secret (app-only flow; prefer $INBUXA_MIGRATE_EWS_CLIENT_SECRET)" + )] + ews_client_secret: Option, + + #[arg(long, help = "Use the OAuth device-code interactive flow")] + ews_device_code: bool, + + #[arg( + long, + value_name = "N", + default_value_t = 4, + help = "Parallel EWS worker connections (hard cap 8)" + )] + ews_connections: usize, + + #[arg( + long, + value_name = "N", + default_value_t = 20, + help = "ItemIds per GetItem batch" + )] + ews_getitem_batch: usize, + + #[arg( + long, + value_name = "N", + default_value_t = 5, + help = "AttachmentIds per GetAttachment batch" + )] + ews_attachment_batch: usize, + + #[arg(long, help = "Force FindItem even when SyncFolderItems would work")] + ews_no_syncfolderitems: bool, + + #[arg( + long, + help = "Permit importing when the archive records a different EWS source" + )] + allow_source_change: bool, + + #[command(flatten)] + global: GlobalArgs, + + #[arg( + value_name = "ARCHIVE", + help = "Local SQLite archive (created if absent)" + )] + archive: PathBuf, +} + +impl ExchangeEwsImportArgs { + pub(super) fn resolve(self) -> Result { + let archive = self.archive.clone(); + let common = self.global.common_config(archive); + let mailbox_kind = MailboxKind::parse(&self.mailbox_kind).ok_or_else(|| { + Error::Usage(format!( + "--mailbox-kind must be primary | archive | public-folders, got {:?}", + self.mailbox_kind + )) + })?; + let auth = self.auth()?; + let ews_connections = self + .ews_connections + .clamp(1, MAX_EWS_CONNECTIONS) + .min(common.threads.max(1)); + let config = EwsImport { + url: self.url, + mailbox: self.mailbox, + mailbox_kind, + auth, + ews_connections, + getitem_batch: self.ews_getitem_batch.max(1), + attachment_batch: self.ews_attachment_batch.max(1), + enumeration: if self.ews_no_syncfolderitems { + ItemEnumeration::FindItem + } else { + ItemEnumeration::SyncFolderItems + }, + source_change: if self.allow_source_change { + SourceChange::Allow + } else { + SourceChange::Reject + }, + }; + Ok(Action::import(common, config)) + } + + fn auth(&self) -> Result { + if let Some(user) = self.auth_basic.as_deref() { + let password = Secret::resolve( + self.auth_password.as_deref(), + "INBUXA_MIGRATE_PASSWORD", + "password", + )? + .into_inner(); + return Ok(EwsAuth::Basic { + user: user.to_owned(), + password, + }); + } + if self.auth_password.is_some() { + return Err(Error::Usage( + "--auth-password is only valid with --auth-basic".to_owned(), + )); + } + let Some(bearer) = self.auth_bearer.as_ref() else { + return Err(Error::Usage( + "exactly one of --auth-basic / --auth-bearer is required".to_owned(), + )); + }; + if let Some(token) = bearer.clone() { + return Ok(EwsAuth::Bearer { token }); + } + let tenant = || { + self.ews_tenant + .clone() + .unwrap_or_else(|| "common".to_owned()) + }; + let required_client_id = || { + self.ews_client_id.clone().ok_or_else(|| { + Error::Usage( + "--ews-client-id is required with --ews-device-code or --ews-client-secret" + .to_owned(), + ) + }) + }; + if self.ews_device_code { + return Ok(EwsAuth::OAuth(OAuthFlow::DeviceCode { + tenant: tenant(), + client_id: required_client_id()?, + })); + } + if let Some(secret) = self.ews_client_secret.as_deref() { + return Ok(EwsAuth::OAuth(OAuthFlow::ClientCredentials { + tenant: tenant(), + client_id: required_client_id()?, + client_secret: secret.to_owned(), + })); + } + if let Ok(token) = env::var("INBUXA_MIGRATE_TOKEN") + && !token.is_empty() + { + return Ok(EwsAuth::Bearer { token }); + } + if let Some(client_id) = self.ews_client_id.clone() + && let Ok(client_secret) = env::var("INBUXA_MIGRATE_EWS_CLIENT_SECRET") + && !client_secret.is_empty() + { + return Ok(EwsAuth::OAuth(OAuthFlow::ClientCredentials { + tenant: tenant(), + client_id, + client_secret, + })); + } + let token = Secret::resolve(None, "INBUXA_MIGRATE_TOKEN", "bearer token")?.into_inner(); + Ok(EwsAuth::Bearer { token }) + } +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + use super::*; + use crate::cli::{Cli, env_lock as lock}; + use crate::import::Import; + + fn ews_auth(extra: &[&str]) -> Result { + let mut argv = vec!["inbuxa-migrate", "import", "exchange-ews", "archive.sqlite"]; + argv.extend_from_slice(extra); + let cli = Cli::try_parse_from(argv).unwrap_or_else(|e| panic!("clap rejected args: {e}")); + match cli.resolve()? { + Action::Import(_, Import::Ews(config)) => Ok(config.auth), + _ => panic!("expected ImportExchangeEws"), + } + } + + fn set_ews_env(token: Option<&str>, client_secret: Option<&str>) { + unsafe { + match token { + Some(t) => env::set_var("INBUXA_MIGRATE_TOKEN", t), + None => env::remove_var("INBUXA_MIGRATE_TOKEN"), + } + match client_secret { + Some(s) => env::set_var("INBUXA_MIGRATE_EWS_CLIENT_SECRET", s), + None => env::remove_var("INBUXA_MIGRATE_EWS_CLIENT_SECRET"), + } + } + } + + // A bare --auth-bearer takes INBUXA_MIGRATE_TOKEN without needing --ews-client-id. + #[test] + fn ews_bare_bearer_uses_env_token_without_client_id() { + let _g = lock(); + set_ews_env(Some("envtok"), None); + let auth = ews_auth(&["--auth-bearer"]); + set_ews_env(None, None); + assert!(matches!(auth, Ok(EwsAuth::Bearer { token }) if token == "envtok")); + } + + // INBUXA_MIGRATE_TOKEN wins over client credentials taken from the environment. + #[test] + fn ews_env_token_beats_env_client_secret() { + let _g = lock(); + set_ews_env(Some("envtok"), Some("sec")); + let auth = ews_auth(&["--auth-bearer", "--ews-client-id", "cid"]); + set_ews_env(None, None); + assert!(matches!(auth, Ok(EwsAuth::Bearer { token }) if token == "envtok")); + } + + // The env client secret is used when no token is exported. + #[test] + fn ews_env_client_secret_without_token_uses_client_credentials() { + let _g = lock(); + set_ews_env(None, Some("sec")); + let auth = ews_auth(&["--auth-bearer", "--ews-client-id", "cid"]); + set_ews_env(None, None); + assert!(matches!( + auth, + Ok(EwsAuth::OAuth(OAuthFlow::ClientCredentials { client_id, client_secret, .. })) + if client_id == "cid" && client_secret == "sec" + )); + } + + // An explicit --ews-device-code is not overridden by an exported token. + #[test] + fn ews_device_code_flag_beats_env_token() { + let _g = lock(); + set_ews_env(Some("envtok"), None); + let auth = ews_auth(&[ + "--auth-bearer", + "--ews-client-id", + "cid", + "--ews-device-code", + ]); + set_ews_env(None, None); + assert!(matches!( + auth, + Ok(EwsAuth::OAuth(OAuthFlow::DeviceCode { tenant, client_id })) + if tenant == "common" && client_id == "cid" + )); + } + + // --ews-device-code still needs a client id. + #[test] + fn ews_device_code_without_client_id_is_usage_error() { + let _g = lock(); + set_ews_env(Some("envtok"), None); + let auth = ews_auth(&["--auth-bearer", "--ews-device-code"]); + set_ews_env(None, None); + match auth { + Err(Error::Usage(m)) => assert!(m.contains("--ews-client-id"), "msg was: {m}"), + other => panic!("expected Usage error, got {other:?}"), + } + } + + // An inline token wins over everything else. + #[test] + fn ews_inline_token_beats_env_and_flows() { + let _g = lock(); + set_ews_env(Some("envtok"), None); + let auth = ews_auth(&[ + "--auth-bearer=inline", + "--ews-client-id", + "cid", + "--ews-device-code", + ]); + set_ews_env(None, None); + assert!(matches!(auth, Ok(EwsAuth::Bearer { token }) if token == "inline")); + } +} diff --git a/src/cli/export.rs b/src/cli/export.rs new file mode 100644 index 0000000..2222fa2 --- /dev/null +++ b/src/cli/export.rs @@ -0,0 +1,91 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::path::PathBuf; + +use clap::{ArgGroup, Args}; + +use super::Action; +use super::auth::{AccountFlags, AuthArgs}; +use super::global::GlobalArgs; +use crate::error::Error; +use crate::export::{Confirm, Export, Prune}; +use crate::jmap::connect::ConnectConfig; +use crate::types::ObjectTypes; + +#[derive(Args)] +#[command(group(ArgGroup::new("account").required(true).args(["account_id", "account_name"])))] +pub(super) struct ExportArgs { + #[arg(long, value_name = "URL", help = "Base URL or full JMAP session URL")] + url: String, + + #[command(flatten)] + auth: AuthArgs, + + #[arg(long, value_name = "ID", help = "Target account id")] + account_id: Option, + + #[arg(long, value_name = "NAME", help = "Target account name")] + account_name: Option, + + #[arg( + long, + value_name = "LIST", + help = "Types to export (default: all present in archive)" + )] + objects: Option, + + #[arg(long, help = "Delete target objects that match nothing in the archive")] + prune: bool, + + #[arg(long, help = "Skip the interactive --prune confirmation")] + yes: bool, + + #[command(flatten)] + global: GlobalArgs, + + #[arg(value_name = "ARCHIVE", help = "Local SQLite archive")] + archive: PathBuf, +} + +impl ExportArgs { + pub(super) fn resolve(self) -> Result { + let common = self.global.common_config(self.archive); + let auth = self.auth.resolve()?; + let account = AccountFlags { + id: self.account_id, + name: self.account_name, + } + .resolve()?; + let objects = self + .objects + .as_deref() + .map(str::parse::) + .transpose()?; + + Ok(Action::Export( + common, + Export { + connect: ConnectConfig { + url: self.url, + auth, + account, + }, + objects, + prune: if self.prune { + Prune::Destroy + } else { + Prune::Keep + }, + confirm: if self.yes { + Confirm::Assume + } else { + Confirm::Prompt + }, + }, + )) + } +} diff --git a/src/cli/global.rs b/src/cli/global.rs new file mode 100644 index 0000000..3c339fa --- /dev/null +++ b/src/cli/global.rs @@ -0,0 +1,89 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::path::PathBuf; + +use clap::{ArgAction, Args}; + +use crate::logging::{Logger, Verbosity}; +use crate::net::http::TlsPolicy; +use crate::sync::{RunMode, RunOptions}; + +#[derive(Args)] +pub(super) struct GlobalArgs { + #[arg( + short = 'j', + long, + value_name = "N", + help = "Worker pool size (default: logical CPUs)" + )] + threads: Option, + + #[arg(long, help = "Compute and report the full plan; perform no writes")] + dry_run: bool, + + #[arg( + short = 'v', + long, + action = ArgAction::Count, + conflicts_with = "quiet", + help = "Increase log verbosity (repeatable, max -vvv)", + long_help = "Increase log verbosity (repeatable, max -vvv).\n \ + (default) per-type start/finish lines and a final summary\n \ + -v per-chunk progress, account resolution, retry notices\n \ + -vv every protocol call (method/command, target, status, timing)\n \ + -vvv full request/response bodies and backoff delays (stderr)" + )] + verbose: u8, + + #[arg( + short = 'q', + long, + help = "Quiet: warnings and errors only (verbosity level 0)" + )] + quiet: bool, + + #[arg( + long, + value_name = "N", + default_value_t = 5, + help = "Max retry attempts per request on transient failures" + )] + max_retries: u32, + + #[arg(long, help = "Accept self-signed / invalid TLS certificates")] + allow_invalid_certs: bool, +} + +impl GlobalArgs { + pub(super) fn common_config(&self, archive: PathBuf) -> RunOptions { + let threads = self + .threads + .filter(|n| *n > 0) + .unwrap_or_else(num_cpus::get) + .max(1); + RunOptions { + archive, + threads, + mode: if self.dry_run { + RunMode::DryRun + } else { + RunMode::Apply + }, + max_retries: self.max_retries, + tls: if self.allow_invalid_certs { + TlsPolicy::AcceptInvalid + } else { + TlsPolicy::Verify + }, + logger: Logger::from_verbosity(if self.quiet { + Verbosity::Quiet + } else { + Verbosity::Verbose(self.verbose) + }), + } + } +} diff --git a/src/cli/graph.rs b/src/cli/graph.rs new file mode 100644 index 0000000..decead7 --- /dev/null +++ b/src/cli/graph.rs @@ -0,0 +1,277 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::path::PathBuf; + +use clap::Args; + +use super::Action; +use super::global::GlobalArgs; +use crate::archive::sources::SourceChange; +use crate::error::Error; +use crate::exchange::graph::types::{EventBodyFormat, MailboxKind as GraphMailboxKind, Surfaces}; +use crate::exchange::oauth::Authority; +use crate::import::graph::coordinator::{MAX_GRAPH_CONNECTIONS, MAX_PAGE_SIZE}; +use crate::import::graph::{ContactPhotos, EventAttachments, GraphAuth, GraphImport}; +use crate::secret::Secret; + +#[derive(Args)] +pub struct ExchangeGraphImportArgs { + #[arg( + long, + value_name = "UUID", + help = "Entra app (client) id (required unless --access-token is given)" + )] + client_id: Option, + + #[arg( + long, + value_name = "ID", + default_value = "common", + help = "Entra authority tenant" + )] + tenant: String, + + #[arg( + long, + value_name = "UPN|UUID", + help = "Target a different mailbox the signed-in account can read" + )] + user: Option, + + #[arg( + long, + value_name = "KIND", + default_value = "primary", + value_parser = GraphMailboxKind::parse, + help = "Mailbox surface: primary | archive (public-folders is rejected: use exchange-ews)" + )] + mailbox_kind: GraphMailboxKind, + + #[arg( + long, + value_name = "TOKEN", + num_args = 0..=1, + help = "Pre-acquired bearer token (skip device-code flow). Resolves: flag value -> $INBUXA_MIGRATE_GRAPH_TOKEN -> prompt." + )] + access_token: Option>, + + #[arg( + long, + value_name = "LIST", + help = "Comma-separated surface list: mail | calendar | contacts | files (default: all)" + )] + objects: Option, + + #[arg( + long, + value_name = "FMT", + default_value = "text", + value_parser = EventBodyFormat::parse, + help = "outlook.body-content-type for event GETs: text | html" + )] + event_body_format: EventBodyFormat, + + #[arg( + long, + value_name = "N", + default_value_t = 4, + help = "Parallel Graph worker connections (hard cap 16)" + )] + graph_connections: usize, + + #[arg( + long, + value_name = "N", + default_value_t = 100, + help = "Per-page size ($top query parameter, range [1, 1000])" + )] + top: usize, + + #[arg( + long, + value_name = "N", + default_value_t = 5, + help = "Years either side of today to scan for edited recurrence occurrences (max 5)", + long_help = "Years either side of today to scan for edited recurrence occurrences.\n \ + Graph reports a series exception only inside an expanded calendar view, and caps\n \ + any single view at five years, so inbuxa-migrate scans [today - N, today] and\n \ + [today, today + N]. An occurrence edited outside that span imports with the\n \ + series default instead of its edit." + )] + exception_window_years: i32, + + #[arg( + long, + help = "Skip contact photos (saves one request per contact on large address books)", + long_help = "Skip contact photos.\n Graph will not say which contacts have a photo: $expand=photo comes back empty\n even when one exists, so the only way to find out is to ask for the bytes.\n inbuxa-migrate therefore spends one extra request per contact. Pass this to skip\n photos entirely on a large address book." + )] + skip_contact_photos: bool, + + #[arg( + long, + help = "Skip event file attachments (they are stored as JSCalendar enclosure links)" + )] + skip_event_attachments: bool, + + #[arg( + long, + value_name = "URL", + default_value = "https://graph.microsoft.com/v1.0", + help = "Microsoft Graph API base URL (override for national clouds or tests)", + hide = true + )] + api_base: String, + + #[arg( + long, + help = "Permit importing when the archive records a different exchange_graph source" + )] + allow_source_change: bool, + + #[command(flatten)] + global: GlobalArgs, + + #[arg( + value_name = "ARCHIVE", + help = "Local SQLite archive (created if absent)" + )] + archive: PathBuf, +} + +impl ExchangeGraphImportArgs { + pub(super) fn resolve(self) -> Result { + let common = self.global.common_config(self.archive.clone()); + let mailbox_kind = self.mailbox_kind; + let event_body_format = self.event_body_format; + let auth = self.auth()?; + let surfaces = self + .objects + .as_deref() + .map(Surfaces::parse_list) + .transpose()? + .unwrap_or_default(); + let top = self.top.clamp(1, MAX_PAGE_SIZE); + let graph_connections = self + .graph_connections + .clamp(1, MAX_GRAPH_CONNECTIONS) + .min(common.threads.max(1)); + Ok(Action::import( + common, + GraphImport { + auth, + api_base: self.api_base, + user_target: self.user, + mailbox_kind, + surfaces, + event_body_format, + graph_connections, + top, + exception_window_years: self.exception_window_years, + contact_photos: if self.skip_contact_photos { + ContactPhotos::Skip + } else { + ContactPhotos::Fetch + }, + event_attachments: if self.skip_event_attachments { + EventAttachments::Skip + } else { + EventAttachments::Fetch + }, + source_change: if self.allow_source_change { + SourceChange::Allow + } else { + SourceChange::Reject + }, + }, + )) + } + + fn auth(&self) -> Result { + if let Some(slot) = &self.access_token { + let token = Secret::resolve( + slot.as_deref(), + "INBUXA_MIGRATE_GRAPH_TOKEN", + "bearer token", + )? + .into_inner(); + return Ok(GraphAuth::PreAcquired { token }); + } + let client_id = self.client_id.clone().ok_or_else(|| { + Error::Usage( + "--client-id is required for exchange-graph (unless --access-token is given)" + .to_owned(), + ) + })?; + Ok(GraphAuth::DeviceCode { + authority: Authority::for_tenant(&self.tenant).into_string(), + client_id, + }) + } +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + use super::*; + use crate::cli::{Cli, env_lock as lock}; + use crate::import::Import; + + fn graph_config(objects: &[&str]) -> Result { + let mut argv = vec![ + "inbuxa-migrate", + "import", + "exchange-graph", + "--access-token", + "tok", + ]; + argv.extend_from_slice(objects); + argv.push("archive.sqlite"); + let cli = Cli::try_parse_from(argv).unwrap_or_else(|e| panic!("clap rejected args: {e}")); + match cli.resolve()? { + Action::Import(_, Import::Graph(config)) => Ok(config), + _ => panic!("expected ImportExchangeGraph"), + } + } + + #[test] + fn graph_objects_contacts_selects_only_contacts() { + let _g = lock(); + let config = graph_config(&["--objects", "contacts"]).unwrap(); + assert_eq!( + config.surfaces, + Surfaces { + mail: false, + calendar: false, + contacts: true, + files: false + } + ); + } + + #[test] + fn graph_objects_absent_selects_every_surface() { + let _g = lock(); + let config = graph_config(&[]).unwrap(); + assert_eq!(config.surfaces, Surfaces::ALL); + } + + #[test] + fn graph_objects_rejects_jmap_type_names() { + let _g = lock(); + match graph_config(&["--objects", "contactcard"]) { + Err(Error::Usage(m)) => { + assert!(m.contains("unknown surface: contactcard"), "msg was: {m}"); + assert!( + m.contains("valid: mail, calendar, contacts"), + "msg was: {m}" + ); + } + other => panic!("expected Usage error, got {:?}", other.map(|_| ())), + } + } +} diff --git a/src/cli/imap.rs b/src/cli/imap.rs new file mode 100644 index 0000000..7cb3b56 --- /dev/null +++ b/src/cli/imap.rs @@ -0,0 +1,257 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::path::PathBuf; + +use clap::{ArgAction, Args}; + +use super::global::GlobalArgs; +use super::{Action, FolderFilter}; +use crate::archive::sources::SourceChange; +use crate::error::Error; +use crate::imap::automap::Automap; +use crate::imap::session::Compress; +use crate::import::imap::pool::MAX_IMAP_CONNECTIONS; +use crate::import::imap::{DeletedMessages, FolderScope, ImapImport, MimeKeywords}; +use crate::net::sasl::SaslAuth; +use crate::net::stream::Cleartext; +use crate::secret::Secret; + +#[derive(Args)] +#[command(group(clap::ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer"])))] +pub(super) struct ImapImportArgs { + #[arg( + long, + value_name = "URL", + help = "imap://host[:port] or imaps://host[:port]" + )] + url: String, + + #[arg( + long, + value_name = "USER", + help = "PLAIN (preferred) with LOGIN fallback" + )] + auth_basic: Option, + + #[arg( + long, + value_name = "PASS", + help = "Basic password (prefer $INBUXA_MIGRATE_PASSWORD or prompt)" + )] + auth_password: Option, + + #[arg( + long, + value_name = "TOKEN", + num_args = 0..=1, + help = "Bearer token for XOAUTH2 (value optional: falls back to $INBUXA_MIGRATE_TOKEN or prompt)" + )] + auth_bearer: Option>, + + #[arg( + long, + value_name = "USER", + help = "XOAUTH2 identity (required with --auth-bearer)" + )] + auth_user: Option, + + #[arg(long, help = "Permit credentials on imap:// without STARTTLS")] + allow_cleartext: bool, + + #[arg(long, help = "Enable COMPRESS=DEFLATE post-auth (off by default)")] + compress: bool, + + #[arg( + long, + value_name = "REGEX", + action = ArgAction::Append, + help = "Folder include filter (repeatable)" + )] + include: Vec, + + #[arg( + long, + value_name = "REGEX", + action = ArgAction::Append, + help = "Folder exclude filter (repeatable)" + )] + exclude: Vec, + + #[arg( + long, + value_name = "ROLE", + action = ArgAction::Append, + help = "Drop folders matching a role (repeatable)" + )] + exclude_special: Vec, + + #[arg( + long, + value_name = "NAME", + action = ArgAction::Append, + help = "Exact include; mutually exclusive with --include/--exclude" + )] + folder: Vec, + + #[arg(long, help = "Keep only folders returned by LSUB")] + subscribed_only: bool, + + #[arg(long, help = "Disable SPECIAL-USE name-heuristic fallback")] + noautomap: bool, + + #[arg(long, help = "Import \\Deleted messages with the $deleted keyword")] + include_deleted: bool, + + #[arg( + long, + help = "Add the phrases of each message's Keywords: header (Exchange categories) as JMAP keywords" + )] + mime_keywords: bool, + + #[arg( + long, + value_name = "N", + default_value_t = 256, + help = "UIDs per metadata FETCH chunk" + )] + fetch_batch: usize, + + #[arg( + long, + value_name = "N", + help = "Parallel IMAP connections for body fetch (1..=8). Default: min(--threads, 8)" + )] + imap_connections: Option, + + #[arg( + long, + help = "Permit importing when the archive records a different IMAP source" + )] + allow_source_change: bool, + + #[command(flatten)] + global: GlobalArgs, + + #[arg( + value_name = "ARCHIVE", + help = "Local SQLite archive (created if absent)" + )] + archive: PathBuf, +} + +impl ImapImportArgs { + pub(super) fn resolve(self) -> Result { + let common = self.global.common_config(self.archive); + let auth = SaslAuthFlags { + basic: self.auth_basic.as_deref(), + password: self.auth_password.as_deref(), + bearer: self.auth_bearer.as_ref().map(Option::as_deref), + user: self.auth_user.as_deref(), + } + .resolve()?; + if !self.folder.is_empty() && (!self.include.is_empty() || !self.exclude.is_empty()) { + return Err(Error::Usage( + "--folder is mutually exclusive with --include/--exclude".to_owned(), + )); + } + let include = FolderFilter::Include.compile(&self.include)?; + let exclude = FolderFilter::Exclude.compile(&self.exclude)?; + let imap_connections = self + .imap_connections + .unwrap_or(common.threads) + .clamp(1, MAX_IMAP_CONNECTIONS); + + Ok(Action::import( + common, + ImapImport { + url: self.url, + auth, + cleartext: if self.allow_cleartext { + Cleartext::Allow + } else { + Cleartext::Forbid + }, + compress: if self.compress { + Compress::Requested + } else { + Compress::Off + }, + include, + exclude, + exclude_special: self.exclude_special, + folder: self.folder, + scope: if self.subscribed_only { + FolderScope::Subscribed + } else { + FolderScope::All + }, + automap: if self.noautomap { + Automap::Disabled + } else { + Automap::Enabled + }, + deleted: if self.include_deleted { + DeletedMessages::Keep + } else { + DeletedMessages::Skip + }, + mime_keywords: if self.mime_keywords { + MimeKeywords::Extract + } else { + MimeKeywords::Skip + }, + fetch_batch: self.fetch_batch, + imap_connections, + source_change: if self.allow_source_change { + SourceChange::Allow + } else { + SourceChange::Reject + }, + }, + )) + } +} + +#[derive(Clone, Copy)] +pub(super) struct SaslAuthFlags<'a> { + pub(super) basic: Option<&'a str>, + pub(super) password: Option<&'a str>, + pub(super) bearer: Option>, + pub(super) user: Option<&'a str>, +} + +impl SaslAuthFlags<'_> { + pub(super) fn resolve(self) -> Result { + if let Some(user) = self.basic { + if self.user.is_some() { + return Err(Error::Usage( + "--auth-user is only valid with --auth-bearer".to_owned(), + )); + } + let password = + Secret::resolve(self.password, "INBUXA_MIGRATE_PASSWORD", "password")?.into_inner(); + return Ok(SaslAuth::Basic { + user: user.to_owned(), + password, + }); + } + if self.password.is_some() { + return Err(Error::Usage( + "--auth-password is only valid together with --auth-basic".to_owned(), + )); + } + let bearer = self.bearer.ok_or_else(|| { + Error::Usage("exactly one of --auth-basic / --auth-bearer is required".to_owned()) + })?; + let user = self + .user + .ok_or_else(|| Error::Usage("--auth-user is required with --auth-bearer".to_owned()))? + .to_owned(); + let token = Secret::resolve(bearer, "INBUXA_MIGRATE_TOKEN", "bearer token")?.into_inner(); + Ok(SaslAuth::Bearer { user, token }) + } +} diff --git a/src/cli/inspect.rs b/src/cli/inspect.rs new file mode 100644 index 0000000..7609902 --- /dev/null +++ b/src/cli/inspect.rs @@ -0,0 +1,65 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::path::PathBuf; + +use clap::Args; + +use super::Action; +use crate::console::Console; +use crate::error::Error; +use crate::inspect::{Inspect, Page}; +use crate::types::ObjectType; + +#[derive(Args)] +pub(super) struct InspectArgs { + #[arg(value_name = "ARCHIVE", help = "Local SQLite archive")] + archive: PathBuf, + + #[arg( + value_name = "TYPE", + help = "Object type to dump (omit for a summary of all types)" + )] + target: Option, + + #[arg( + long, + value_name = "N", + help = "Maximum rows to show (omit to dump every row)" + )] + limit: Option, + + #[arg( + long, + value_name = "N", + default_value_t = 0, + help = "Skip the first N rows" + )] + offset: usize, +} + +impl InspectArgs { + pub(super) fn resolve(self) -> Result { + let target = match self.target.as_deref() { + Some(token) => Some(token.parse::()?), + None => None, + }; + if let Some(limit) = self.limit + && limit == 0 + { + return Err(Error::Usage("--limit must be greater than zero".to_owned())); + } + Ok(Action::Inspect(Inspect { + archive: self.archive, + target, + page: Page { + limit: self.limit, + offset: self.offset, + }, + console: Console::default(), + })) + } +} diff --git a/src/cli/jmap.rs b/src/cli/jmap.rs new file mode 100644 index 0000000..c62987c --- /dev/null +++ b/src/cli/jmap.rs @@ -0,0 +1,90 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::path::PathBuf; + +use clap::{ArgGroup, Args}; + +use super::Action; +use super::auth::{AccountFlags, AuthArgs}; +use super::global::GlobalArgs; +use crate::archive::sources::SourceChange; +use crate::error::Error; +use crate::import::jmap::JmapImport; +use crate::jmap::connect::ConnectConfig; +use crate::types::ObjectTypes; + +#[derive(Args)] +#[command(group(ArgGroup::new("account").required(true).args(["account_id", "account_name"])))] +pub(super) struct JmapImportArgs { + #[arg(long, value_name = "URL", help = "Base URL or full JMAP session URL")] + url: String, + + #[command(flatten)] + auth: AuthArgs, + + #[arg(long, value_name = "ID", help = "Source account id")] + account_id: Option, + + #[arg(long, value_name = "NAME", help = "Source account name")] + account_name: Option, + + #[arg( + long, + value_name = "LIST", + help = "Comma-separated type list (default: all supported)" + )] + objects: Option, + + #[arg( + long, + help = "Permit importing when the archive records a different JMAP source" + )] + allow_source_change: bool, + + #[command(flatten)] + global: GlobalArgs, + + #[arg( + value_name = "ARCHIVE", + help = "Local SQLite archive (created if absent)" + )] + archive: PathBuf, +} + +impl JmapImportArgs { + pub(super) fn resolve(self) -> Result { + let common = self.global.common_config(self.archive); + let auth = self.auth.resolve()?; + let account = AccountFlags { + id: self.account_id, + name: self.account_name, + } + .resolve()?; + let objects = self + .objects + .as_deref() + .map(str::parse::) + .transpose()?; + + Ok(Action::import( + common, + JmapImport { + connect: ConnectConfig { + url: self.url, + auth, + account, + }, + objects, + source_change: if self.allow_source_change { + SourceChange::Allow + } else { + SourceChange::Reject + }, + }, + )) + } +} diff --git a/src/cli/maildir.rs b/src/cli/maildir.rs new file mode 100644 index 0000000..e9bc3bc --- /dev/null +++ b/src/cli/maildir.rs @@ -0,0 +1,108 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::path::PathBuf; + +use clap::{ArgAction, Args}; + +use super::global::GlobalArgs; +use super::{Action, FolderFilter}; +use crate::archive::sources::SourceChange; +use crate::error::Error; +use crate::imap::automap::Automap; +use crate::import::imap::DeletedMessages; +use crate::import::maildir::MaildirImport; + +#[derive(Args)] +pub struct MaildirImportArgs { + #[arg( + value_name = "MAILDIR", + help = "Path to the Maildir++ root (a directory containing cur/ new/ tmp/)" + )] + maildir: PathBuf, + + #[arg( + value_name = "ARCHIVE", + help = "Local SQLite archive (created if absent)" + )] + archive: PathBuf, + + #[arg( + long, + value_name = "REGEX", + action = ArgAction::Append, + help = "Folder include filter against canonical name (repeatable)" + )] + include: Vec, + + #[arg( + long, + value_name = "REGEX", + action = ArgAction::Append, + help = "Folder exclude filter against canonical name (repeatable)" + )] + exclude: Vec, + + #[arg( + long, + value_name = "NAME", + action = ArgAction::Append, + help = "Exact include; mutually exclusive with --include/--exclude" + )] + folder: Vec, + + #[arg(long, help = "Disable the name-heuristic role assignment")] + noautomap: bool, + + #[arg(long, help = "Import T-flagged messages with the $deleted keyword")] + include_deleted: bool, + + #[arg( + long, + help = "Permit importing when the archive records a different maildir path" + )] + allow_source_change: bool, + + #[command(flatten)] + global: GlobalArgs, +} + +impl MaildirImportArgs { + pub(super) fn resolve(self) -> Result { + let common = self.global.common_config(self.archive); + if !self.folder.is_empty() && (!self.include.is_empty() || !self.exclude.is_empty()) { + return Err(Error::Usage( + "--folder is mutually exclusive with --include/--exclude".to_owned(), + )); + } + let include = FolderFilter::Include.compile(&self.include)?; + let exclude = FolderFilter::Exclude.compile(&self.exclude)?; + Ok(Action::import( + common, + MaildirImport { + maildir: self.maildir, + include, + exclude, + folder: self.folder, + automap: if self.noautomap { + Automap::Disabled + } else { + Automap::Enabled + }, + deleted: if self.include_deleted { + DeletedMessages::Keep + } else { + DeletedMessages::Skip + }, + source_change: if self.allow_source_change { + SourceChange::Allow + } else { + SourceChange::Reject + }, + }, + )) + } +} diff --git a/src/cli/managesieve.rs b/src/cli/managesieve.rs new file mode 100644 index 0000000..6cedfd5 --- /dev/null +++ b/src/cli/managesieve.rs @@ -0,0 +1,105 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::path::PathBuf; + +use clap::Args; + +use super::Action; +use super::global::GlobalArgs; +use super::imap::SaslAuthFlags; +use crate::archive::sources::SourceChange; +use crate::error::Error; +use crate::import::managesieve::ManageSieveImport; +use crate::net::stream::Cleartext; + +#[derive(Args)] +#[command(group(clap::ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer"])))] +pub struct ManageSieveImportArgs { + #[arg( + long, + value_name = "URL", + help = "sieve://host[:port] or sieves://host[:port]" + )] + url: String, + + #[arg( + long, + value_name = "USER", + help = "SASL PLAIN (preferred) with LOGIN fallback" + )] + auth_basic: Option, + + #[arg( + long, + value_name = "PASS", + help = "Basic password (prefer $INBUXA_MIGRATE_PASSWORD or prompt)" + )] + auth_password: Option, + + #[arg( + long, + value_name = "TOKEN", + num_args = 0..=1, + help = "Bearer token for OAUTHBEARER (value optional: $INBUXA_MIGRATE_TOKEN or prompt)" + )] + auth_bearer: Option>, + + #[arg( + long, + value_name = "USER", + help = "OAUTHBEARER identity (required with --auth-bearer)" + )] + auth_user: Option, + + #[arg(long, help = "Permit credentials on sieve:// without STARTTLS")] + allow_cleartext: bool, + + #[arg( + long, + help = "Permit importing when the archive records a different ManageSieve source" + )] + allow_source_change: bool, + + #[command(flatten)] + global: GlobalArgs, + + #[arg( + value_name = "ARCHIVE", + help = "Local SQLite archive (created if absent)" + )] + archive: PathBuf, +} + +impl ManageSieveImportArgs { + pub(super) fn resolve(self) -> Result { + let common = self.global.common_config(self.archive); + let auth = SaslAuthFlags { + basic: self.auth_basic.as_deref(), + password: self.auth_password.as_deref(), + bearer: self.auth_bearer.as_ref().map(Option::as_deref), + user: self.auth_user.as_deref(), + } + .resolve()?; + Ok(Action::import( + common, + ManageSieveImport { + url: self.url, + auth, + cleartext: if self.allow_cleartext { + Cleartext::Allow + } else { + Cleartext::Forbid + }, + source_change: if self.allow_source_change { + SourceChange::Allow + } else { + SourceChange::Reject + }, + }, + )) + } +} diff --git a/src/cli/mod.rs b/src/cli/mod.rs new file mode 100644 index 0000000..af5e189 --- /dev/null +++ b/src/cli/mod.rs @@ -0,0 +1,290 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +mod auth; +mod dav; +mod ews; +mod export; +mod global; +mod graph; +mod imap; +mod inspect; +mod jmap; +mod maildir; +mod managesieve; +mod report; +mod takeout; + +use std::io::{self, IsTerminal}; +use std::path::PathBuf; + +use clap::{CommandFactory, FromArgMatches, Parser, Subcommand}; +use regex::Regex; + +use self::dav::DavImportArgs; +use self::ews::ExchangeEwsImportArgs; +use self::export::ExportArgs; +use self::graph::ExchangeGraphImportArgs; +use self::imap::ImapImportArgs; +use self::inspect::InspectArgs; +use self::jmap::JmapImportArgs; +use self::maildir::MaildirImportArgs; +use self::managesieve::ManageSieveImportArgs; +use self::report::{Report, RunKind}; +use self::takeout::TakeoutImportArgs; +use crate::console::{ColorChoice, Console, Counted, Logo, Severity}; +use crate::error::{Error, ExitStatus}; +use crate::export::Export; +use crate::import::Import; +use crate::import::dav::DavKindArg; +use crate::inspect::Inspect; +use crate::logging::{Level, Logger}; +use crate::sync::{RunMode, RunOptions}; + +#[derive(Parser)] +#[command( + name = "inbuxa-migrate", + version, + about = "Inbuxa: the JMAP importer-exporter" +)] +pub struct Cli { + #[command(subcommand)] + command: Command, + + #[arg( + long, + global = true, + value_enum, + value_name = "WHEN", + default_value = "auto", + help = "Colour output: auto (terminal only, honours NO_COLOR), always or never" + )] + color: ColorChoice, +} + +#[derive(Subcommand)] +enum Command { + #[command(about = "Read a source account into a local archive")] + Import { + #[command(subcommand)] + source: Source, + }, + #[command(about = "Push a local archive into a target JMAP server")] + Export(Box), + #[command(about = "Show the contents of a local archive (read-only)")] + Inspect(Box), +} + +#[derive(Subcommand)] +enum Source { + #[command(about = "JMAP server")] + Jmap(Box), + #[command(about = "IMAP server")] + Imap(Box), + #[command(about = "CalDAV server")] + Caldav(Box), + #[command(about = "CardDAV server")] + Carddav(Box), + #[command(about = "WebDAV server (plain files)")] + Webdav(Box), + #[command(about = "ManageSieve server (Sieve scripts only)")] + Managesieve(Box), + #[command(about = "Local Maildir++ tree")] + Maildir(Box), + #[command(about = "Google Takeout (or any mbox/ics/vcf directory tree)")] + Takeout(Box), + #[command(about = "MS Exchange via EWS (SOAP/XML)", name = "exchange-ews")] + ExchangeEws(Box), + #[command( + about = "MS Exchange Online via Microsoft Graph", + name = "exchange-graph" + )] + ExchangeGraph(Box), +} + +pub enum Action { + Import(RunOptions, Import), + Export(RunOptions, Export), + Inspect(Inspect), +} + +impl Action { + pub fn import(options: RunOptions, import: impl Into) -> Action { + Action::Import(options, import.into()) + } + + pub fn with_console(self, console: Console) -> Action { + match self { + Action::Import(mut options, import) => { + options.logger = options.logger.with_console(console); + Action::Import(options, import) + } + Action::Export(mut options, export) => { + options.logger = options.logger.with_console(console); + Action::Export(options, export) + } + Action::Inspect(inspect) => Action::Inspect(Inspect { console, ..inspect }), + } + } + + pub fn run(self) -> ExitStatus { + let (kind, options, outcome) = match self { + Action::Import(options, import) => { + let run = RunHeader::start(&options); + (RunKind::Import, run, import.run(options)) + } + Action::Export(options, export) => { + let run = RunHeader::start(&options); + (RunKind::Export(export.prune), run, export.run(options)) + } + Action::Inspect(inspect) => { + let console = inspect.console; + return match inspect.run() { + Ok(()) => ExitStatus::Ok, + Err(err) => Action::fail(console, &err), + }; + } + }; + let RunHeader { + logger, + archive, + mode, + } = options; + let console = logger.console(); + let _ = Report { + kind, + archive: &archive, + mode, + summary: &outcome.summary, + } + .write(console.out, &mut io::stdout().lock()); + let failed = outcome.summary.failed(); + match outcome.error { + Some(err) => Action::fail(console, &err), + None if failed > 0 => { + logger.error(format_args!( + "{} failed; the archive is consistent and a re-run resumes", + Counted::new(failed, "object", "objects") + )); + ExitStatus::Partial + } + None => ExitStatus::Ok, + } + } + + pub fn fail(console: Console, err: &Error) -> ExitStatus { + eprintln!("{} {err}", console.err.severity(Severity::Error)); + err.exit_status() + } +} + +struct RunHeader { + logger: Logger, + archive: PathBuf, + mode: RunMode, +} + +impl RunHeader { + fn start(options: &RunOptions) -> RunHeader { + let logger = options.logger; + if logger.enabled(Level::Default) && io::stderr().is_terminal() { + eprintln!("{}\n", Logo::for_stderr().text(logger.console().err)); + } + RunHeader { + logger, + archive: options.archive.clone(), + mode: options.mode, + } + } +} + +impl Cli { + pub fn try_parse_styled(choice: ColorChoice, console: Console) -> Result { + let logo = Logo::for_stdout().text(console.out); + let matches = Cli::command() + .color(choice.clap()) + .before_help(logo) + .before_long_help(logo) + .try_get_matches()?; + Cli::from_arg_matches(&matches) + } + + pub fn resolve(self) -> Result { + match self.command { + Command::Import { source } => source.resolve(), + Command::Export(args) => args.resolve(), + Command::Inspect(args) => args.resolve(), + } + } +} + +impl Source { + fn resolve(self) -> Result { + match self { + Source::Jmap(args) => args.resolve(), + Source::Imap(args) => args.resolve(), + Source::Caldav(args) => args.resolve(DavKindArg::Caldav), + Source::Carddav(args) => args.resolve(DavKindArg::Carddav), + Source::Webdav(args) => args.resolve(DavKindArg::Webdav), + Source::Managesieve(args) => args.resolve(), + Source::Maildir(args) => args.resolve(), + Source::Takeout(args) => args.resolve(), + Source::ExchangeEws(args) => args.resolve(), + Source::ExchangeGraph(args) => args.resolve(), + } + } +} + +#[derive(Clone, Copy)] +enum FolderFilter { + Include, + Exclude, +} + +impl FolderFilter { + fn flag(self) -> &'static str { + match self { + FolderFilter::Include => "--include", + FolderFilter::Exclude => "--exclude", + } + } + + fn compile(self, raw: &[String]) -> Result, Error> { + raw.iter() + .map(|r| { + Regex::new(r) + .map_err(|e| Error::Usage(format!("{} {r:?}: invalid regex: {e}", self.flag()))) + }) + .collect() + } +} + +#[cfg(test)] +pub(crate) fn env_lock() -> std::sync::MutexGuard<'static, ()> { + static M: std::sync::Mutex<()> = std::sync::Mutex::new(()); + M.lock().unwrap_or_else(|e| e.into_inner()) +} + +#[cfg(test)] +mod tests { + use super::*; + + // an invalid pattern is a usage error naming its flag; valid ones compile in order + #[test] + fn folder_filter_names_the_flag_of_an_invalid_pattern() { + let err = FolderFilter::Exclude + .compile(&["^a".to_owned(), "(".to_owned()]) + .unwrap_err(); + assert!( + matches!(&err, Error::Usage(m) if m.starts_with("--exclude \"(\": invalid regex: ")), + "{err}" + ); + let compiled = FolderFilter::Include + .compile(&["^a".to_owned(), "b$".to_owned()]) + .unwrap(); + assert!(compiled.iter().map(Regex::as_str).eq(["^a", "b$"])); + } +} diff --git a/src/cli/report.rs b/src/cli/report.rs new file mode 100644 index 0000000..3f7f8e4 --- /dev/null +++ b/src/cli/report.rs @@ -0,0 +1,307 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fmt; +use std::io::{self, Write}; +use std::path::Path; + +use crate::console::{Column, Grouped, Styled, Table, Tone}; +use crate::export::Prune; +use crate::sync::{RunMode, Summary, TypeCounts}; + +const TOTAL: &str = "Total"; +const IMPORT_PLAN_TITLE: &str = "Import plan (dry run, nothing written)"; +const EXPORT_PLAN_TITLE: &str = "Export plan (dry run, nothing written)"; +const IMPORT_COLUMNS: [Column; 7] = [ + Column::left("Type"), + Column::right("Created"), + Column::right("Fetched"), + Column::right("Updated"), + Column::right("Deleted"), + Column::right("Skipped"), + Column::right("Failed"), +]; +const EXPORT_COLUMNS: [Column; 6] = [ + Column::left("Type"), + Column::right("Created"), + Column::right("Matched"), + Column::right("Updated"), + Column::right("Destroyed"), + Column::right("Failed"), +]; +const FAILURE_COLUMNS: [Column; 2] = [Column::left("Type"), Column::right("Failed")]; +const IMPORT_PLAN_COLUMNS: [Column; 4] = [ + Column::left("Type"), + Column::right("New"), + Column::right("Vanished"), + Column::right("Present"), +]; +const EXPORT_PLAN_COLUMNS: [Column; 4] = [ + Column::left("Type"), + Column::right("Create"), + Column::right("Matched"), + Column::right("Destroy"), +]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum RunKind { + Import, + Export(Prune), +} + +pub(super) struct Report<'a> { + pub kind: RunKind, + pub archive: &'a Path, + pub mode: RunMode, + pub summary: &'a Summary, +} + +impl Report<'_> { + pub(super) fn write(&self, styled: Styled, out: &mut dyn Write) -> io::Result<()> { + self.write_plan(styled, out)?; + self.write_counts(styled, out) + } + + fn write_plan(&self, styled: Styled, out: &mut dyn Write) -> io::Result<()> { + let rows = &self.summary.plan; + if rows.is_empty() { + return Ok(()); + } + let (title, columns) = match self.kind { + RunKind::Import => (IMPORT_PLAN_TITLE, &IMPORT_PLAN_COLUMNS[..]), + RunKind::Export(Prune::Destroy) => (EXPORT_PLAN_TITLE, &EXPORT_PLAN_COLUMNS[..]), + RunKind::Export(Prune::Keep) => ( + EXPORT_PLAN_TITLE, + &EXPORT_PLAN_COLUMNS[..EXPORT_PLAN_COLUMNS.len() - 1], + ), + }; + writeln!(out, "\n{}\n", styled.paint(Tone::Title, title))?; + let mut table = Table::new(columns); + for (ty, plan) in rows { + table + .cell(ty.label(), None) + .count(plan.new, Some(Tone::Good)); + match self.kind { + RunKind::Import => { + table + .count(plan.vanished, Some(Tone::Bad)) + .count(plan.present, None); + } + RunKind::Export(prune) => { + table.count(plan.present, None); + if prune == Prune::Destroy { + table.count(plan.vanished, Some(Tone::Bad)); + } + } + } + } + table.write(styled, out) + } + + fn write_counts(&self, styled: Styled, out: &mut dyn Write) -> io::Result<()> { + if self.mode == RunMode::DryRun { + return self.write_failures(styled, out); + } + let rows = &self.summary.per_type; + if rows.is_empty() { + return Ok(()); + } + writeln!(out, "\n{}\n", styled.paint(Tone::Title, Title(self)))?; + let mut table = Table::new(match self.kind { + RunKind::Import => &IMPORT_COLUMNS[..], + RunKind::Export(_) => &EXPORT_COLUMNS[..], + }); + let mut total = TypeCounts::default(); + for (ty, counts) in rows { + table.cell(ty.label(), None); + self.counts(&mut table, counts, None); + total.add(counts); + } + if rows.len() > 1 { + table.cell(TOTAL, Some(Tone::Heading)); + self.counts(&mut table, &total, Some(Tone::Heading)); + } + table.write(styled, out)?; + self.write_retries(out) + } + + fn write_failures(&self, styled: Styled, out: &mut dyn Write) -> io::Result<()> { + let mut failed = self + .summary + .per_type + .iter() + .filter(|(_, counts)| counts.failed > 0) + .peekable(); + if failed.peek().is_none() { + return Ok(()); + } + writeln!(out, "\n{}\n", styled.paint(Tone::Title, Title(self)))?; + let mut table = Table::new(&FAILURE_COLUMNS); + for (ty, counts) in failed { + table + .cell(ty.label(), None) + .count(counts.failed, Some(Tone::Bad)); + } + table.write(styled, out)?; + self.write_retries(out) + } + + fn write_retries(&self, out: &mut dyn Write) -> io::Result<()> { + let summary = self.summary; + if summary.retries_observed > 0 { + writeln!( + out, + "\n Retries: {} ({} honoured Retry-After)", + Grouped(summary.retries_observed), + Grouped(summary.retry_after_sleeps) + )?; + } + Ok(()) + } + + fn counts(&self, table: &mut Table<'_>, counts: &TypeCounts, tone: Option) { + match self.kind { + RunKind::Import => table + .count(counts.created, tone) + .count(counts.fetched, tone) + .count(counts.updated, tone) + .count(counts.deleted, tone) + .count(counts.skipped, tone), + RunKind::Export(_) => table + .count(counts.created, tone) + .count(counts.skipped, tone) + .count(counts.updated, tone) + .count(counts.deleted, tone), + } + .count(counts.failed, Some(Tone::Bad)); + } +} + +struct Title<'r, 'a>(&'r Report<'a>); + +impl fmt::Display for Title<'_, '_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let report = self.0; + let verb = match (report.kind, report.mode) { + (RunKind::Import, RunMode::Apply) => "Imported into", + (RunKind::Import, RunMode::DryRun) => "Dry run: import into", + (RunKind::Export(_), RunMode::Apply) => "Exported from", + (RunKind::Export(_), RunMode::DryRun) => "Dry run: export from", + }; + write!(f, "{verb} {}", report.archive.display()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sync::{PlanCounts, SummaryType}; + use crate::types::ObjectType; + + fn render(kind: RunKind, mode: RunMode, summary: &Summary) -> String { + let mut out = Vec::new(); + Report { + kind, + archive: Path::new("a.sqlite"), + mode, + summary, + } + .write(Styled::Plain, &mut out) + .unwrap(); + String::from_utf8(out).unwrap() + } + + // An import summary has a title, one row per type in run order and a total row. + #[test] + fn import_summary_lists_types_and_total() { + let mut summary = Summary::default(); + *summary.counts(ObjectType::Mailbox) = TypeCounts { + created: 12, + fetched: 12, + ..TypeCounts::default() + }; + *summary.counts(ObjectType::Email) = TypeCounts { + created: 4210, + fetched: 4210, + failed: 2, + ..TypeCounts::default() + }; + summary.retries_observed = 3; + summary.retry_after_sleeps = 2; + assert_eq!( + render(RunKind::Import, RunMode::Apply, &summary), + "\nImported into a.sqlite\n\n\ + \x20 Type Created Fetched Updated Deleted Skipped Failed\n\ + \x20 Mailboxes 12 12 0 0 0 0\n\ + \x20 Emails 4,210 4,210 0 0 0 2\n\ + \x20 Total 4,222 4,222 0 0 0 2\n\ + \n Retries: 3 (2 honoured Retry-After)\n" + ); + } + + // Export maps matches to skipped and destroys to deleted; one type has no total row. + #[test] + fn export_summary_uses_export_columns() { + let mut summary = Summary::default(); + *summary.counts(SummaryType::VacationResponse) = TypeCounts { + updated: 1, + skipped: 4, + deleted: 5, + ..TypeCounts::default() + }; + assert_eq!( + render(RunKind::Export(Prune::Keep), RunMode::Apply, &summary), + "\nExported from a.sqlite\n\n\ + \x20 Type Created Matched Updated Destroyed Failed\n\ + \x20 Vacation response 0 4 1 5 0\n" + ); + } + + // Dry-run plans use the import or export columns; Destroy needs --prune. + #[test] + fn plan_columns_follow_direction_and_prune() { + let mut summary = Summary::default(); + *summary.plan(ObjectType::Calendar) = PlanCounts { + new: 1, + vanished: 2, + present: 3, + }; + assert_eq!( + render(RunKind::Import, RunMode::DryRun, &summary), + "\nImport plan (dry run, nothing written)\n\n\ + \x20 Type New Vanished Present\n\ + \x20 Calendars 1 2 3\n" + ); + assert_eq!( + render(RunKind::Export(Prune::Destroy), RunMode::DryRun, &summary), + "\nExport plan (dry run, nothing written)\n\n\ + \x20 Type Create Matched Destroy\n\ + \x20 Calendars 1 3 2\n" + ); + assert_eq!( + render(RunKind::Export(Prune::Keep), RunMode::DryRun, &summary), + "\nExport plan (dry run, nothing written)\n\n\ + \x20 Type Create Matched\n\ + \x20 Calendars 1 3\n" + ); + } + + // A dry run lists only the types that failed, after the plan. + #[test] + fn dry_run_counts_show_failures_only() { + let mut summary = Summary::default(); + summary.counts(ObjectType::Mailbox).created = 3; + assert!(render(RunKind::Import, RunMode::DryRun, &summary).is_empty()); + summary.counts(ObjectType::Email).failed = 2; + assert_eq!( + render(RunKind::Import, RunMode::DryRun, &summary), + "\nDry run: import into a.sqlite\n\n\ + \x20 Type Failed\n\ + \x20 Emails 2\n" + ); + assert!(render(RunKind::Import, RunMode::Apply, &Summary::default()).is_empty()); + } +} diff --git a/src/cli/takeout.rs b/src/cli/takeout.rs new file mode 100644 index 0000000..9ebe282 --- /dev/null +++ b/src/cli/takeout.rs @@ -0,0 +1,65 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::path::PathBuf; + +use clap::Args; + +use super::Action; +use super::global::GlobalArgs; +use crate::archive::sources::SourceChange; +use crate::error::Error; +use crate::imap::automap::Automap; +use crate::import::takeout::TakeoutImport; + +#[derive(Args)] +pub struct TakeoutImportArgs { + #[arg( + value_name = "PATH", + help = "Directory tree to scan recursively for .mbox / .ics / .vcf files" + )] + path: PathBuf, + + #[arg( + value_name = "ARCHIVE", + help = "Local SQLite archive (created if absent)" + )] + archive: PathBuf, + + #[arg(long, help = "Disable the system-label role assignment")] + noautomap: bool, + + #[arg( + long, + help = "Permit importing when the archive records a takeout source at a different path" + )] + allow_source_change: bool, + + #[command(flatten)] + global: GlobalArgs, +} + +impl TakeoutImportArgs { + pub(super) fn resolve(self) -> Result { + let common = self.global.common_config(self.archive); + Ok(Action::import( + common, + TakeoutImport { + takeout_root: self.path, + source_change: if self.allow_source_change { + SourceChange::Allow + } else { + SourceChange::Reject + }, + automap: if self.noautomap { + Automap::Disabled + } else { + Automap::Enabled + }, + }, + )) + } +} diff --git a/src/console/logo.rs b/src/console/logo.rs new file mode 100644 index 0000000..45d9c17 --- /dev/null +++ b/src/console/logo.rs @@ -0,0 +1,80 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io; + +use terminal_size::{Width, terminal_size_of}; + +use super::Styled; + +const LARGE_LOGO_COLUMNS: u16 = 91; +const LARGE_ANS: &str = include_str!("logo/large.ans").trim_ascii_end(); +const LARGE_TXT: &str = include_str!("logo/large.txt").trim_ascii_end(); +const COMPACT_ANS: &str = include_str!("logo/compact.ans").trim_ascii_end(); +const COMPACT_TXT: &str = include_str!("logo/compact.txt").trim_ascii_end(); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Logo { + Large, + Compact, +} + +impl Logo { + pub fn for_stdout() -> Logo { + Logo::fitting(terminal_size_of(io::stdout()).map(|(Width(columns), _)| columns)) + } + + pub fn for_stderr() -> Logo { + Logo::fitting(terminal_size_of(io::stderr()).map(|(Width(columns), _)| columns)) + } + + fn fitting(columns: Option) -> Logo { + match columns { + Some(columns) if columns >= LARGE_LOGO_COLUMNS => Logo::Large, + _ => Logo::Compact, + } + } + + pub const fn text(self, styled: Styled) -> &'static str { + match (self, styled) { + (Logo::Large, Styled::Ansi) => LARGE_ANS, + (Logo::Large, Styled::Plain) => LARGE_TXT, + (Logo::Compact, Styled::Ansi) => COMPACT_ANS, + (Logo::Compact, Styled::Plain) => COMPACT_TXT, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // The large logo needs 91 columns; unknown widths get the compact one. + #[test] + fn logo_fits_the_terminal_width() { + assert_eq!(Logo::fitting(Some(LARGE_LOGO_COLUMNS)), Logo::Large); + assert_eq!(Logo::fitting(Some(LARGE_LOGO_COLUMNS - 1)), Logo::Compact); + assert_eq!(Logo::fitting(None), Logo::Compact); + } + + // Plain selects the escape-free variant, and no variant ends in a newline. + #[test] + fn logo_variant_follows_style() { + for logo in [Logo::Large, Logo::Compact] { + let plain = logo.text(Styled::Plain); + let colored = logo.text(Styled::Ansi); + assert!(!plain.contains('\x1b')); + assert!(colored.contains('\x1b')); + assert!(!plain.ends_with('\n') && !colored.ends_with('\n')); + } + assert!( + Logo::Large + .text(Styled::Plain) + .lines() + .all(|line| line.chars().count() <= usize::from(LARGE_LOGO_COLUMNS)) + ); + } +} diff --git a/src/console/logo/compact.ans b/src/console/logo/compact.ans new file mode 100644 index 0000000..4b62507 --- /dev/null +++ b/src/console/logo/compact.ans @@ -0,0 +1,7 @@ + ▄▄██▄▄  + ▄██▀▀█ ▀▀██▄   ▄█████▄ ▄▄ █ ▄▄ +███ █▄▄▄▄▄███ ██ ▀█▀ ██▄▄ ▄▄▄ ▄ █ ▄▄ ▄ ▄▄ ▄▄▄ ▄▄ ▄ ▄▄ ██▄▄ + ▀▀█▄▄ ▀▀███▀▀  ▀██▄▄▄▄ ██▀▀ ▄█▀▀▀▀██ █ ▀█▄ ███ █▀ ██▀▀▀███ ███▀▀ ██▀▀ +█▄▄ ▀████▀ ▄▄█  ▀▀▀▀█▄ ██ ▄█ ██ █ ██ █▀█▄ ██ ██ ██ ██ ██ + ▀██▄▄ ▄▄██▀  ██▄ ██ ██ █▄ ██ █ █▄█▀ ▀█▄█ ▀█▄ ██ ██ ██ +██▄ ▀▀██▀▀ ▄██  ▀█████▀ ▀███ ▀▀███▀██ █ ▀██ ██▀ ▀████▀██ ██ ▀███ diff --git a/src/console/logo/compact.txt b/src/console/logo/compact.txt new file mode 100644 index 0000000..508bffb --- /dev/null +++ b/src/console/logo/compact.txt @@ -0,0 +1,7 @@ + ▄▄██▄▄ + ▄██▀▀█ ▀▀██▄ ▄█████▄ ▄▄ █ ▄▄ +███ █▄▄▄▄▄███ ██ ▀█▀ ██▄▄ ▄▄▄ ▄ █ ▄▄ ▄ ▄▄ ▄▄▄ ▄▄ ▄ ▄▄ ██▄▄ + ▀▀█▄▄ ▀▀███▀▀ ▀██▄▄▄▄ ██▀▀ ▄█▀▀▀▀██ █ ▀█▄ ███ █▀ ██▀▀▀███ ███▀▀ ██▀▀ +█▄▄ ▀████▀ ▄▄█ ▀▀▀▀█▄ ██ ▄█ ██ █ ██ █▀█▄ ██ ██ ██ ██ ██ + ▀██▄▄ ▄▄██▀ ██▄ ██ ██ █▄ ██ █ █▄█▀ ▀█▄█ ▀█▄ ██ ██ ██ +██▄ ▀▀██▀▀ ▄██ ▀█████▀ ▀███ ▀▀███▀██ █ ▀██ ██▀ ▀████▀██ ██ ▀███ diff --git a/src/console/logo/large.ans b/src/console/logo/large.ans new file mode 100644 index 0000000..5a80ad8 --- /dev/null +++ b/src/console/logo/large.ans @@ -0,0 +1,8 @@ + ▄▄██▄▄  + ▄██▀██ ▀▀██▄   ▄▄█████▄ ▄▄ ██ ▄▄ +▄██▀▀ ██ ▀▀██▄ ▄██ ▀██ ██ ██ ██▄ +▀██▄▄ ▀▀▀▀▀█████▀ ▀██▄ ████ ▄█████▄██ ██ ██ ██▄ ▄█▀ ▄█████▄██ ██▄██ ████▀ +▄ ▀▀██▄ ▄██▀▀ ▄  ▀▀█████▄▄ ██ ██▀ ▀██ ██ ▀█▄ ████ ██ ██▀ ▀██ ██▀ ██ +▀██▄▄ ▀▀██▀▀ ▄▄██▀ ▄▄ ▀██▄ ██ ██ ██ ██ ██ █▀ ██ ██ ██ ██ ██ ██ +▄ ▀██▄▄ ▄▄██▀ ▄ ▀██ ██▀ ██ ██▄ ▄██ ██ ████ ▀█▄█▀ ▀█▄ ▄██ ██ ██▄ +██▄▄ ▀▀██▀▀ ▄▄██  ▀▀█████▀ ▀███▀ ▀█████▀██ ██ ▀██ ███ ▀▀████▀██ ██ ▀████ diff --git a/src/console/logo/large.txt b/src/console/logo/large.txt new file mode 100644 index 0000000..b0dcc56 --- /dev/null +++ b/src/console/logo/large.txt @@ -0,0 +1,8 @@ + ▄▄██▄▄ + ▄██▀██ ▀▀██▄ ▄▄█████▄ ▄▄ ██ ▄▄ +▄██▀▀ ██ ▀▀██▄ ▄██ ▀██ ██ ██ ██▄ +▀██▄▄ ▀▀▀▀▀█████▀ ▀██▄ ████ ▄█████▄██ ██ ██ ██▄ ▄█▀ ▄█████▄██ ██▄██ ████▀ +▄ ▀▀██▄ ▄██▀▀ ▄ ▀▀█████▄▄ ██ ██▀ ▀██ ██ ▀█▄ ████ ██ ██▀ ▀██ ██▀ ██ +▀██▄▄ ▀▀██▀▀ ▄▄██▀ ▄▄ ▀██▄ ██ ██ ██ ██ ██ █▀ ██ ██ ██ ██ ██ ██ +▄ ▀██▄▄ ▄▄██▀ ▄ ▀██ ██▀ ██ ██▄ ▄██ ██ ████ ▀█▄█▀ ▀█▄ ▄██ ██ ██▄ +██▄▄ ▀▀██▀▀ ▄▄██ ▀▀█████▀ ▀███▀ ▀█████▀██ ██ ▀██ ███ ▀▀████▀██ ██ ▀████ diff --git a/src/console/mod.rs b/src/console/mod.rs new file mode 100644 index 0000000..662bc0d --- /dev/null +++ b/src/console/mod.rs @@ -0,0 +1,349 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +mod logo; +mod table; +mod units; + +pub use logo::Logo; +pub use table::{Align, Column, Table}; +pub use units::{ByteSize, Grouped}; + +use std::env; +use std::ffi::{OsStr, OsString}; +use std::fmt::{self, Display}; +use std::io::{self, IsTerminal}; + +use clap::ValueEnum; + +const NO_COLOR: &str = "NO_COLOR"; +const TERM: &str = "TERM"; +const DUMB_TERM: &str = "dumb"; +const COLOR_FLAG: &str = "--color"; +const COLOR_FLAG_EQ: &str = "--color="; +const END_OF_FLAGS: &str = "--"; +const RESET: &str = "\x1b[0m"; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, ValueEnum)] +pub enum ColorChoice { + #[default] + Auto, + Always, + Never, +} + +impl ColorChoice { + pub fn from_args(args: impl IntoIterator) -> ColorChoice { + let mut choice = ColorChoice::Auto; + let mut args = args.into_iter(); + while let Some(arg) = args.next() { + let Some(arg) = arg.to_str() else { + continue; + }; + let value = if arg == END_OF_FLAGS { + break; + } else if arg == COLOR_FLAG { + args.next() + .and_then(|v| v.to_str().and_then(ColorChoice::parse)) + } else { + arg.strip_prefix(COLOR_FLAG_EQ).and_then(ColorChoice::parse) + }; + if let Some(value) = value { + choice = value; + } + } + choice + } + + fn parse(value: &str) -> Option { + ColorChoice::from_str(value, true).ok() + } + + fn styled(self, is_terminal: bool) -> Styled { + match self { + ColorChoice::Always => Styled::Ansi, + ColorChoice::Never => Styled::Plain, + ColorChoice::Auto if is_terminal => Styled::Ansi, + ColorChoice::Auto => Styled::Plain, + } + } + + fn effective(self, no_color: Option<&OsStr>, term: Option<&OsStr>) -> ColorChoice { + let plain_env = no_color.is_some_and(|v| !v.is_empty()) + || term.map_or(!cfg!(windows), |t| t == DUMB_TERM); + match self { + ColorChoice::Auto if plain_env => ColorChoice::Never, + choice => choice, + } + } + + pub const fn clap(self) -> clap::ColorChoice { + match self { + ColorChoice::Auto => clap::ColorChoice::Auto, + ColorChoice::Always => clap::ColorChoice::Always, + ColorChoice::Never => clap::ColorChoice::Never, + } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Styled { + #[default] + Plain, + Ansi, +} + +impl Styled { + pub fn paint(self, tone: Tone, value: T) -> Paint { + Paint { + styled: self, + tone, + value, + } + } + + pub fn severity(self, severity: Severity) -> Paint<&'static str> { + self.paint(severity.tone(), severity.prefix()) + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Console { + pub out: Styled, + pub err: Styled, +} + +impl Console { + pub fn detect(choice: ColorChoice) -> Console { + let choice = choice.effective( + env::var_os(NO_COLOR).as_deref(), + env::var_os(TERM).as_deref(), + ); + Console { + out: choice.styled(io::stdout().is_terminal()), + err: choice.styled(io::stderr().is_terminal()), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Tone { + Error, + Warning, + Note, + Heading, + Title, + Dim, + Good, + Bad, +} + +impl Tone { + const fn sgr(self) -> &'static str { + match self { + Tone::Error => "\x1b[1;31m", + Tone::Warning => "\x1b[1;33m", + Tone::Note => "\x1b[1;36m", + Tone::Heading => "\x1b[1m", + Tone::Title => "\x1b[1;38;2;219;45;84m", + Tone::Dim => "\x1b[2m", + Tone::Good => "\x1b[32m", + Tone::Bad => "\x1b[31m", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Severity { + Error, + Warning, + Note, +} + +impl Severity { + const fn prefix(self) -> &'static str { + match self { + Severity::Error => "error:", + Severity::Warning => "warning:", + Severity::Note => "note:", + } + } + + const fn tone(self) -> Tone { + match self { + Severity::Error => Tone::Error, + Severity::Warning => Tone::Warning, + Severity::Note => Tone::Note, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Initial<'a> { + Upper(&'a str), + Lower(&'a str), +} + +impl Display for Initial<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let (text, upper) = match *self { + Initial::Upper(text) => (text, true), + Initial::Lower(text) => (text, false), + }; + let mut chars = text.chars(); + let Some(first) = chars.next() else { + return Ok(()); + }; + let first = if upper { + first.to_ascii_uppercase() + } else { + first.to_ascii_lowercase() + }; + write!(f, "{first}{}", chars.as_str()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Counted { + n: u64, + one: &'static str, + many: &'static str, +} + +impl Counted { + pub const fn new(n: u64, one: &'static str, many: &'static str) -> Counted { + Counted { n, one, many } + } +} + +impl Display for Counted { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let unit = if self.n == 1 { self.one } else { self.many }; + write!(f, "{} {unit}", Grouped(self.n)) + } +} + +pub struct Paint { + styled: Styled, + tone: Tone, + value: T, +} + +impl Display for Paint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.styled { + Styled::Plain => self.value.fmt(f), + Styled::Ansi => { + f.write_str(self.tone.sgr())?; + self.value.fmt(f)?; + f.write_str(RESET) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn args(list: &[&str]) -> Vec { + list.iter().map(OsString::from).collect() + } + + // The colour flag is found before clap parses, in both spellings, last one wins. + #[test] + fn color_flag_is_read_before_parsing() { + assert_eq!( + ColorChoice::from_args(args(&["inbuxa-migrate", "--help"])), + ColorChoice::Auto + ); + assert_eq!( + ColorChoice::from_args(args(&["inbuxa-migrate", "--color", "never", "--help"])), + ColorChoice::Never + ); + assert_eq!( + ColorChoice::from_args(args(&["inbuxa-migrate", "--color=always", "--color=NEVER"])), + ColorChoice::Never + ); + assert_eq!( + ColorChoice::from_args(args(&["inbuxa-migrate", "--", "--color=never"])), + ColorChoice::Auto + ); + } + + // Explicit choices ignore the terminal; auto needs one. + #[test] + fn explicit_choice_overrides_terminal_detection() { + assert_eq!(ColorChoice::Always.styled(false), Styled::Ansi); + assert_eq!(ColorChoice::Never.styled(true), Styled::Plain); + assert_eq!(ColorChoice::Auto.styled(false), Styled::Plain); + } + + // Under auto, NO_COLOR, TERM=dumb and an unset TERM (outside Windows) turn colour off, as clap does. + #[test] + fn auto_follows_no_color_and_term() { + let xterm = Some(OsStr::new("xterm-256color")); + let dumb = Some(OsStr::new("dumb")); + let set = Some(OsStr::new("1")); + let empty = Some(OsStr::new("")); + assert_eq!(ColorChoice::Auto.effective(None, xterm), ColorChoice::Auto); + assert_eq!(ColorChoice::Auto.effective(empty, xterm), ColorChoice::Auto); + assert_eq!(ColorChoice::Auto.effective(set, xterm), ColorChoice::Never); + assert_eq!(ColorChoice::Auto.effective(None, dumb), ColorChoice::Never); + assert_eq!( + ColorChoice::Auto.effective(None, None), + if cfg!(windows) { + ColorChoice::Auto + } else { + ColorChoice::Never + } + ); + assert_eq!( + ColorChoice::Always.effective(set, dumb), + ColorChoice::Always + ); + assert_eq!( + ColorChoice::Never.effective(None, xterm), + ColorChoice::Never + ); + } + + // Only the first letter changes case. + #[test] + fn initial_changes_the_first_letter_only() { + assert_eq!(Initial::Lower("Sieve scripts").to_string(), "sieve scripts"); + assert_eq!( + Initial::Upper("calendar event").to_string(), + "Calendar event" + ); + assert_eq!(Initial::Upper("").to_string(), ""); + } + + // Units follow the count, with thousands grouped. + #[test] + fn counted_pluralises() { + assert_eq!(Counted::new(1, "object", "objects").to_string(), "1 object"); + assert_eq!( + Counted::new(1234, "object", "objects").to_string(), + "1,234 objects" + ); + assert_eq!(Counted::new(0, "item", "items").to_string(), "0 items"); + } + + // Plain painting is the bare value with padding applied; ANSI wraps the padded value. + #[test] + fn paint_keeps_padding_inside_the_escape() { + assert_eq!(format!("{:>4}", Styled::Plain.paint(Tone::Bad, 7)), " 7"); + assert_eq!( + format!("{:>4}", Styled::Ansi.paint(Tone::Bad, 7)), + "\x1b[31m 7\x1b[0m" + ); + assert_eq!( + Styled::Plain.severity(Severity::Warning).to_string(), + "warning:" + ); + } +} diff --git a/src/console/table.rs b/src/console/table.rs new file mode 100644 index 0000000..ff95066 --- /dev/null +++ b/src/console/table.rs @@ -0,0 +1,208 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fmt::{Display, Write as _}; +use std::io::{self, Write}; + +use super::{Grouped, Styled, Tone}; + +const GUTTER: &str = " "; +const INDENT: &str = " "; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Align { + Left, + Right, +} + +#[derive(Clone, Copy, Debug)] +pub struct Column { + pub title: &'static str, + pub align: Align, +} + +impl Column { + pub const fn left(title: &'static str) -> Column { + Column { + title, + align: Align::Left, + } + } + + pub const fn right(title: &'static str) -> Column { + Column { + title, + align: Align::Right, + } + } +} + +struct Cell { + end: usize, + width: usize, + tone: Option, +} + +pub struct Table<'c> { + columns: &'c [Column], + text: String, + cells: Vec, +} + +impl<'c> Table<'c> { + pub fn new(columns: &'c [Column]) -> Table<'c> { + Table { + columns, + text: String::new(), + cells: Vec::new(), + } + } + + pub fn cell(&mut self, value: impl Display, tone: Option) -> &mut Self { + let start = self.text.len(); + let _ = write!(self.text, "{value}"); + let width = self.text.get(start..).map_or(0, |s| s.chars().count()); + self.cells.push(Cell { + end: self.text.len(), + width, + tone, + }); + self + } + + pub fn rows(&self) -> usize { + self.cells.len() / self.columns.len().max(1) + } + + pub fn clear(&mut self) { + self.text.clear(); + self.cells.clear(); + } + + pub fn count(&mut self, n: u64, nonzero: Option) -> &mut Self { + let tone = if n == 0 { Some(Tone::Dim) } else { nonzero }; + self.cell(Grouped(n), tone) + } + + pub fn write(&self, styled: Styled, out: &mut dyn Write) -> io::Result<()> { + let n = self.columns.len().max(1); + let mut widths: Vec = self.columns.iter().map(|c| c.title.len()).collect(); + for row in self.cells.chunks(n) { + for (width, cell) in widths.iter_mut().zip(row) { + *width = (*width).max(cell.width); + } + } + out.write_all(INDENT.as_bytes())?; + for (i, (column, width)) in self.columns.iter().zip(&widths).enumerate() { + Slot { + align: column.align, + fill: width.saturating_sub(column.title.len()), + first: i == 0, + last: i + 1 == n, + } + .write(out, styled, column.title, Some(Tone::Heading))?; + } + out.write_all(b"\n")?; + let mut start = 0; + for row in self.cells.chunks(n) { + out.write_all(INDENT.as_bytes())?; + for (i, ((column, width), cell)) in + self.columns.iter().zip(&widths).zip(row).enumerate() + { + let text = self.text.get(start..cell.end).unwrap_or_default(); + start = cell.end; + Slot { + align: column.align, + fill: width.saturating_sub(cell.width), + first: i == 0, + last: i + 1 == row.len(), + } + .write(out, styled, text, cell.tone)?; + } + out.write_all(b"\n")?; + } + Ok(()) + } +} + +struct Slot { + align: Align, + fill: usize, + first: bool, + last: bool, +} + +impl Slot { + fn write( + self, + out: &mut dyn Write, + styled: Styled, + text: &str, + tone: Option, + ) -> io::Result<()> { + let fill = self.fill; + if !self.first { + out.write_all(GUTTER.as_bytes())?; + } + if self.align == Align::Right { + write!(out, "{:fill$}", "")?; + } + match tone { + Some(tone) => write!(out, "{}", styled.paint(tone, text))?, + None => out.write_all(text.as_bytes())?, + } + if self.align == Align::Left && !self.last { + write!(out, "{:fill$}", "")?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const COLUMNS: [Column; 3] = [ + Column::left("Type"), + Column::right("Created"), + Column::right("Failed"), + ]; + + // Text is left aligned, numbers right aligned, two-space gutter, no trailing spaces. + #[test] + fn plain_table_aligns_columns() { + let mut table = Table::new(&COLUMNS); + table + .cell("Mailboxes", None) + .count(12, None) + .count(0, Some(Tone::Bad)); + table + .cell("Emails", None) + .count(4210, None) + .count(2, Some(Tone::Bad)); + let mut out = Vec::new(); + table.write(Styled::Plain, &mut out).unwrap(); + assert_eq!( + String::from_utf8(out).unwrap(), + " Type Created Failed\n Mailboxes 12 0\n Emails 4,210 2\n" + ); + } + + // Escapes wrap the text only, so styled columns keep the plain alignment. + #[test] + fn styled_table_keeps_alignment() { + let mut table = Table::new(&COLUMNS); + table + .cell("Emails", None) + .count(0, None) + .count(3, Some(Tone::Bad)); + let mut out = Vec::new(); + table.write(Styled::Ansi, &mut out).unwrap(); + let text = String::from_utf8(out).unwrap(); + assert!(text.contains("\x1b[2m0\x1b[0m")); + assert!(text.ends_with(" \x1b[31m3\x1b[0m\n")); + } +} diff --git a/src/console/units.rs b/src/console/units.rs new file mode 100644 index 0000000..52db792 --- /dev/null +++ b/src/console/units.rs @@ -0,0 +1,143 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::{fmt, str}; + +const MAX_GROUPED_LEN: usize = 26; +const BYTE_UNITS: [&str; 5] = ["B", "KB", "MB", "GB", "TB"]; +const KIB: u64 = 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Grouped(pub u64); + +impl Grouped { + pub fn width(self) -> usize { + let digits = self.0.checked_ilog10().map_or(1, |l| l as usize + 1); + digits + (digits - 1) / 3 + } + + fn render(self, buf: &mut [u8; MAX_GROUPED_LEN]) -> &str { + let mut n = self.0; + let mut start = MAX_GROUPED_LEN; + let mut digits = 0; + for slot in buf.iter_mut().rev() { + start -= 1; + if digits == 3 { + *slot = b','; + digits = 0; + continue; + } + *slot = b'0' + (n % 10) as u8; + digits += 1; + n /= 10; + if n == 0 { + break; + } + } + buf.get(start..) + .and_then(|s| str::from_utf8(s).ok()) + .unwrap_or_default() + } +} + +impl fmt::Display for Grouped { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut buf = [0u8; MAX_GROUPED_LEN]; + f.pad(self.render(&mut buf)) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ByteSize(pub u64); + +impl fmt::Display for ByteSize { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.0 < KIB { + return write!(f, "{} B", self.0); + } + let [mut unit, larger @ ..] = BYTE_UNITS; + let mut value = self.0 as f64; + for next in larger { + if value < KIB as f64 { + break; + } + value /= KIB as f64; + unit = next; + } + write!(f, "{value:.1} {unit}") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn reference_count(n: u64) -> String { + let s = n.to_string(); + let bytes = s.as_bytes(); + let len = bytes.len(); + let mut out = String::with_capacity(len + len / 3); + for (i, ch) in bytes.iter().enumerate() { + if i > 0 && (len - i).is_multiple_of(3) { + out.push(','); + } + out.push(*ch as char); + } + out + } + + fn reference_bytes(n: u64) -> String { + const UNITS: &[&str] = &["B", "KB", "MB", "GB", "TB"]; + if n < 1024 { + return format!("{n} B"); + } + let mut value = n as f64; + let mut i = 0; + while value >= 1024.0 && i + 1 < UNITS.len() { + value /= 1024.0; + i += 1; + } + format!("{value:.1} {}", UNITS[i]) + } + + fn samples() -> impl Iterator { + (0..64u32) + .flat_map(|shift| { + let p = 1u64 << shift; + [p - 1, p, p + 1, p.saturating_mul(10), p / 3] + }) + .chain([0, 9, 10, 99, 100, 999, 1_000, 1_234_567_890, u64::MAX]) + .chain((0..20).map(|e| 10u64.pow(e))) + } + + #[test] + fn grouped_inserts_commas() { + assert_eq!(Grouped(0).to_string(), "0"); + assert_eq!(Grouped(999).to_string(), "999"); + assert_eq!(Grouped(1_000).to_string(), "1,000"); + assert_eq!(Grouped(1_234_567_890).to_string(), "1,234,567,890"); + assert_eq!(Grouped(u64::MAX).to_string(), "18,446,744,073,709,551,615"); + } + + #[test] + fn grouped_matches_the_reference_and_pads() { + for n in samples() { + let expected = reference_count(n); + assert_eq!(Grouped(n).to_string(), expected, "{n}"); + assert_eq!(Grouped(n).width(), expected.len(), "{n}"); + assert_eq!(format!("{:>30}", Grouped(n)), format!("{expected:>30}")); + } + } + + #[test] + fn byte_size_matches_the_reference() { + assert_eq!(ByteSize(512).to_string(), "512 B"); + assert_eq!(ByteSize(1_536).to_string(), "1.5 KB"); + for n in samples() { + assert_eq!(ByteSize(n).to_string(), reference_bytes(n), "{n}"); + } + } +} diff --git a/src/dav/client.rs b/src/dav/client.rs deleted file mode 100644 index 666b84a..0000000 --- a/src/dav/client.rs +++ /dev/null @@ -1,981 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::io::{BufReader, Read}; -use std::sync::Arc; -use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; -use std::time::{Duration, Instant}; - -use ureq::Agent; -use ureq::Body; -use ureq::config::{Config, RedirectAuthHeaders}; -use ureq::http::{Method, Request, Response}; -use ureq::tls::{RootCerts, TlsConfig}; - -use crate::dav::parse::{ControlStrippingReader, DavResponse, parse_multistatus}; -use crate::dav::retry::{DavOutcome, classify}; -use crate::jmap::error::JmapError; -use crate::jmap::http::{Auth, RetryPolicy, retry_after_header}; -use crate::jmap::retry::{self, RateLimitState}; -use crate::logging::{HttpCall, LEVEL_BODIES, LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; - -const MAX_BODY: u64 = 512 * 1024 * 1024; -const LONG_RETRY_THRESHOLD: Duration = Duration::from_secs(10); -const MAX_DAV_REDIRECTS: u32 = 5; - -const ACCEPT_DAV: &str = "application/xml, text/xml;q=0.9, */*;q=0.5"; -const ACCEPT_BINARY: &str = "*/*"; - -#[derive(Debug)] -pub struct DavHttpResponse { - pub status: u16, - pub body: Vec, - pub etag: Option, - pub content_type: Option, - pub last_modified: Option, -} - -#[derive(Debug)] -pub struct MultiStatus { - pub status: u16, - pub responses: Vec, - pub final_url: String, -} - -struct Inner { - agent: Agent, - auth: Auth, - retry: RetryPolicy, - rate_limit: RateLimitState, - log_level: AtomicU8, - retries_total: AtomicU64, - retry_after_sleeps: AtomicU64, - user_agent: String, -} - -#[derive(Clone)] -pub struct DavClient { - inner: Arc, -} - -impl DavClient { - pub fn new(auth: Auth, retry: RetryPolicy, allow_invalid_certs: bool) -> Self { - let config: Config = retry - .apply_timeouts(Config::builder()) - .http_status_as_error(false) - .allow_non_standard_methods(true) - .max_redirects(0) - .redirect_auth_headers(RedirectAuthHeaders::SameHost) - .tls_config( - TlsConfig::builder() - .unversioned_rustls_crypto_provider(std::sync::Arc::new( - rustls::crypto::aws_lc_rs::default_provider(), - )) - .root_certs(RootCerts::PlatformVerifier) - .disable_verification(allow_invalid_certs) - .build(), - ) - .build(); - DavClient { - inner: Arc::new(Inner { - agent: config.new_agent(), - auth, - retry, - rate_limit: RateLimitState::new(), - log_level: AtomicU8::new(LEVEL_DEFAULT), - retries_total: AtomicU64::new(0), - retry_after_sleeps: AtomicU64::new(0), - user_agent: format!("inbuxa-migrate/{}", env!("CARGO_PKG_VERSION")), - }), - } - } - - pub fn set_logger(&self, logger: Logger) { - self.inner - .log_level - .store(logger.level(), Ordering::Relaxed); - } - - pub fn logger(&self) -> Logger { - Logger::new(self.inner.log_level.load(Ordering::Relaxed)) - } - - pub fn rate_limit(&self) -> &RateLimitState { - &self.inner.rate_limit - } - - pub fn retries_observed(&self) -> u64 { - self.inner.retries_total.load(Ordering::Relaxed) - } - - pub fn retry_after_sleeps(&self) -> u64 { - self.inner.retry_after_sleeps.load(Ordering::Relaxed) - } - - pub fn propfind(&self, url: &str, depth: u8, body: &str) -> Result { - self.execute( - "PROPFIND", - url, - Some(body.as_bytes()), - Some("application/xml; charset=utf-8"), - Some(depth), - ACCEPT_DAV, - ) - } - - pub fn report(&self, url: &str, depth: u8, body: &str) -> Result { - self.execute( - "REPORT", - url, - Some(body.as_bytes()), - Some("application/xml; charset=utf-8"), - Some(depth), - ACCEPT_DAV, - ) - } - - pub fn get(&self, url: &str) -> Result { - self.execute("GET", url, None, None, None, ACCEPT_BINARY) - } - - pub fn propfind_responses( - &self, - url: &str, - depth: u8, - body: &str, - base_url: &str, - ) -> Result { - self.stream_multistatus("PROPFIND", url, Some(body.as_bytes()), depth, base_url) - } - - pub fn report_responses( - &self, - url: &str, - depth: u8, - body: &str, - base_url: &str, - ) -> Result { - self.stream_multistatus("REPORT", url, Some(body.as_bytes()), depth, base_url) - } - - pub fn get_stream(&self, url: &str) -> Result { - let logger = self.logger(); - let policy = self.inner.retry; - let mut attempt: u32 = 0; - let original_url = url.to_owned(); - let mut current_url = url.to_owned(); - let mut redirects: u32 = 0; - let mut include_auth = true; - loop { - self.inner.rate_limit.cooldown().wait(); - let outcome = self.one_attempt_stream(¤t_url, include_auth); - match outcome { - AttemptStream::Ok { - status, - body_reader, - etag, - content_type, - last_modified, - .. - } if (200..300).contains(&status) => { - self.inner.rate_limit.on_success(); - return Ok(DavStream { - status, - body_reader, - etag, - content_type, - last_modified, - }); - } - AttemptStream::Ok { status, .. } if status == 404 || status == 410 => { - self.inner.rate_limit.on_success(); - return Ok(DavStream { - status, - body_reader: Box::new(std::io::empty()), - etag: None, - content_type: None, - last_modified: None, - }); - } - AttemptStream::Ok { - status, - location: Some(loc), - .. - } if is_redirect_status(status) => { - if redirects >= MAX_DAV_REDIRECTS { - return Err(JmapError::Connect(format!( - "too many redirects ({redirects}) starting from {url}" - ))); - } - redirects += 1; - let next = resolve_redirect(¤t_url, &loc)?; - if include_auth && !same_host(&original_url, &next) { - include_auth = false; - if logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "DAV redirect leaves origin host {original_url}; dropping Authorization for {next}" - ); - } - } - if logger.enabled(LEVEL_PROGRESS) { - eprintln!("DAV {status}: {current_url} -> {next}"); - } - current_url = next; - } - AttemptStream::Ok { - status, - body_reader, - retry_after, - .. - } => { - let mut body_bytes = Vec::new(); - let mut limited = body_reader.take(64 * 1024); - let _ = limited.read_to_end(&mut body_bytes); - self.handle_status( - &logger, - status, - &body_bytes, - retry_after, - &mut attempt, - &policy, - )?; - } - AttemptStream::Transport(err) => { - self.handle_transport(&logger, err, &mut attempt, &policy)?; - } - } - } - } - - fn execute( - &self, - method: &str, - url: &str, - body: Option<&[u8]>, - content_type: Option<&str>, - depth: Option, - accept: &str, - ) -> Result { - let logger = self.logger(); - let policy = self.inner.retry; - let mut attempt: u32 = 0; - let original_url = url.to_owned(); - let mut current_url = url.to_owned(); - let mut redirects: u32 = 0; - let mut include_auth = true; - loop { - self.inner.rate_limit.cooldown().wait(); - let outcome = self.one_attempt(WireRequest { - method, - url: ¤t_url, - body, - content_type, - depth, - accept, - include_auth, - }); - match outcome { - Attempt::Ok { - status, - body, - etag, - content_type, - last_modified, - .. - } if (200..300).contains(&status) => { - self.inner.rate_limit.on_success(); - return Ok(DavHttpResponse { - status, - body, - etag, - content_type, - last_modified, - }); - } - Attempt::Ok { - status, - location: Some(loc), - .. - } if is_redirect_status(status) => { - if redirects >= MAX_DAV_REDIRECTS { - return Err(JmapError::Connect(format!( - "too many redirects ({redirects}) starting from {url}" - ))); - } - redirects += 1; - let next = resolve_redirect(¤t_url, &loc)?; - if include_auth && !same_host(&original_url, &next) { - include_auth = false; - if logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "DAV redirect leaves origin host {original_url}; dropping Authorization for {next}" - ); - } - } - if logger.enabled(LEVEL_PROGRESS) { - eprintln!("DAV {status}: {current_url} -> {next}"); - } - current_url = next; - } - Attempt::Ok { status, body, .. } if is_redirect_status(status) => { - return Err(JmapError::Connect(format!( - "{method} {current_url} returned {status} without Location: {}", - truncate(&body) - ))); - } - Attempt::Ok { - status, - body, - retry_after, - etag: _, - content_type: _, - last_modified: _, - location: _, - } => { - let disposition = classify(status, &body); - match disposition { - DavOutcome::Vanished => { - return Ok(DavHttpResponse { - status, - body, - etag: None, - content_type: None, - last_modified: None, - }); - } - DavOutcome::Auth => { - return Err(JmapError::Auth(format!( - "server returned {status}: {}", - truncate(&body) - ))); - } - DavOutcome::Fatal => { - return Err(JmapError::HttpStatus { - status, - body: truncate(&body), - }); - } - DavOutcome::Retryable => { - attempt += 1; - self.inner.retries_total.fetch_add(1, Ordering::Relaxed); - if attempt > policy.max_retries { - return Err(JmapError::RetriesExhausted(format!( - "{method} {current_url} kept returning {status}" - ))); - } - if retry_after.is_some() { - self.inner - .retry_after_sleeps - .fetch_add(1, Ordering::Relaxed); - } - let delay = self.inner.rate_limit.on_throttle(&policy, retry_after); - if delay >= LONG_RETRY_THRESHOLD { - logger.warn(&format!( - "DAV {method} rate-limited (http {status}); waiting {}s before retry {}/{}", - delay.as_secs(), - attempt, - policy.max_retries, - )); - } - if logger.enabled(LEVEL_BODIES) { - eprintln!( - "retry {}/{} {method} {current_url} after {:?} (http {status}) body={}", - attempt, - policy.max_retries, - delay, - truncate(&body), - ); - } else if logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "retry {}/{} (http {status})", - attempt, policy.max_retries - ); - } - std::thread::sleep(delay); - } - DavOutcome::Success => unreachable!(), - } - } - Attempt::Transport(err) => { - self.handle_transport(&logger, err, &mut attempt, &policy)?; - } - } - } - } - - fn stream_multistatus( - &self, - method: &str, - url: &str, - body: Option<&[u8]>, - depth: u8, - base_url: &str, - ) -> Result { - let logger = self.logger(); - let policy = self.inner.retry; - let mut attempt: u32 = 0; - let original_url = url.to_owned(); - let mut current_url = url.to_owned(); - let mut redirects: u32 = 0; - let mut include_auth = true; - loop { - self.inner.rate_limit.cooldown().wait(); - let started = Instant::now(); - let send = self.build_and_send(WireRequest { - method, - url: ¤t_url, - body, - content_type: Some("application/xml; charset=utf-8"), - depth: Some(depth), - accept: ACCEPT_DAV, - include_auth, - }); - match send { - Ok(mut resp) => { - let status = resp.status().as_u16(); - let retry_after = resp - .headers() - .get("retry-after") - .and_then(|v| v.to_str().ok()) - .and_then(retry_after_header); - let location = resp - .headers() - .get("location") - .and_then(|v| v.to_str().ok()) - .map(str::to_owned); - if is_redirect_status(status) { - let Some(loc) = location else { - return Err(JmapError::Connect(format!( - "{method} {current_url} returned {status} without Location" - ))); - }; - if redirects >= MAX_DAV_REDIRECTS { - return Err(JmapError::Connect(format!( - "too many redirects ({redirects}) starting from {url}" - ))); - } - redirects += 1; - let next = resolve_redirect(¤t_url, &loc)?; - if include_auth && !same_host(&original_url, &next) { - include_auth = false; - if logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "DAV redirect leaves origin host {original_url}; dropping Authorization for {next}" - ); - } - } - if logger.enabled(LEVEL_PROGRESS) { - eprintln!("DAV {status}: {current_url} -> {next}"); - } - current_url = next; - continue; - } - if (200..300).contains(&status) { - self.inner.rate_limit.on_success(); - logger.trace_http(&HttpCall { - proto: "DAV", - method, - url: ¤t_url, - status, - elapsed: started.elapsed(), - note: Some("streamed multistatus"), - request: body, - request_type: Some("application/xml"), - response: b"", - response_type: None, - }); - let parse_base = if current_url == url { - base_url - } else { - ¤t_url - }; - let raw = resp.into_body().into_reader(); - let stripped = ControlStrippingReader::new(raw); - let reader = BufReader::new(stripped); - let responses = parse_multistatus(reader, parse_base) - .map_err(|e| JmapError::Malformed(format!("multistatus parse: {e}")))?; - return Ok(MultiStatus { - status, - responses, - final_url: current_url.clone(), - }); - } - let body_bytes = - match resp.body_mut().with_config().limit(64 * 1024).read_to_vec() { - Ok(b) => b, - Err(e) => { - return Err(JmapError::Transport(format!( - "reading error body: {e}" - ))); - } - }; - logger.trace_http(&HttpCall { - proto: "DAV", - method, - url: ¤t_url, - status, - elapsed: started.elapsed(), - note: None, - request: body, - request_type: Some("application/xml"), - response: &body_bytes, - response_type: None, - }); - let disposition = classify(status, &body_bytes); - match disposition { - DavOutcome::Vanished => { - return Ok(MultiStatus { - status, - responses: Vec::new(), - final_url: current_url.clone(), - }); - } - DavOutcome::Auth => { - return Err(JmapError::Auth(format!( - "server returned {status}: {}", - truncate(&body_bytes) - ))); - } - DavOutcome::Fatal => { - return Err(JmapError::HttpStatus { - status, - body: truncate(&body_bytes), - }); - } - DavOutcome::Retryable => { - attempt += 1; - self.inner.retries_total.fetch_add(1, Ordering::Relaxed); - if attempt > policy.max_retries { - return Err(JmapError::RetriesExhausted(format!( - "{method} {url} kept returning {status}" - ))); - } - if retry_after.is_some() { - self.inner - .retry_after_sleeps - .fetch_add(1, Ordering::Relaxed); - } - let delay = self.inner.rate_limit.on_throttle(&policy, retry_after); - if delay >= LONG_RETRY_THRESHOLD { - logger.warn(&format!( - "DAV {method} rate-limited (http {status}); waiting {}s before retry {}/{}", - delay.as_secs(), - attempt, - policy.max_retries, - )); - } - std::thread::sleep(delay); - } - DavOutcome::Success => unreachable!(), - } - } - Err(e) => { - let err = map_ureq_error(e); - self.handle_transport(&logger, err, &mut attempt, &policy)?; - } - } - } - } - - fn handle_status( - &self, - logger: &Logger, - status: u16, - body: &[u8], - retry_after: Option, - attempt: &mut u32, - policy: &RetryPolicy, - ) -> Result<(), JmapError> { - let disposition = classify(status, body); - match disposition { - DavOutcome::Vanished => Err(JmapError::HttpStatus { - status, - body: truncate(body), - }), - DavOutcome::Auth => Err(JmapError::Auth(format!("server returned {status}"))), - DavOutcome::Fatal => Err(JmapError::HttpStatus { - status, - body: truncate(body), - }), - DavOutcome::Retryable => { - *attempt += 1; - self.inner.retries_total.fetch_add(1, Ordering::Relaxed); - if *attempt > policy.max_retries { - return Err(JmapError::RetriesExhausted(format!( - "GET stream kept returning {status}" - ))); - } - if retry_after.is_some() { - self.inner - .retry_after_sleeps - .fetch_add(1, Ordering::Relaxed); - } - let delay = self.inner.rate_limit.on_throttle(policy, retry_after); - if delay >= LONG_RETRY_THRESHOLD { - logger.warn(&format!( - "DAV stream rate-limited (http {status}); waiting {}s", - delay.as_secs() - )); - } - std::thread::sleep(delay); - Ok(()) - } - DavOutcome::Success => Ok(()), - } - } - - fn handle_transport( - &self, - logger: &Logger, - err: JmapError, - attempt: &mut u32, - policy: &RetryPolicy, - ) -> Result<(), JmapError> { - match transport_disposition(&err) { - retry::Disposition::Fatal => Err(err), - retry::Disposition::Retryable => { - *attempt += 1; - self.inner.retries_total.fetch_add(1, Ordering::Relaxed); - if *attempt > policy.max_retries { - return Err(JmapError::RetriesExhausted(format!("{err}"))); - } - let delay = retry::backoff_delay(policy, *attempt); - if delay >= LONG_RETRY_THRESHOLD { - logger.warn(&format!( - "transient transport failure ({err}); waiting {}s before retry {}/{}", - delay.as_secs(), - attempt, - policy.max_retries, - )); - } - std::thread::sleep(delay); - Ok(()) - } - } - } - - fn one_attempt(&self, req: WireRequest<'_>) -> Attempt { - let logger = self.logger(); - let method = req.method; - let url = req.url; - let request = req.body; - let request_type = req.content_type; - let started = Instant::now(); - match self.build_and_send(req) { - Ok(mut resp) => { - let status = resp.status().as_u16(); - let etag = header_str(&resp, "etag"); - let content_type_header = header_str(&resp, "content-type"); - let last_modified = header_str(&resp, "last-modified"); - let location = header_str(&resp, "location"); - let retry_after = resp - .headers() - .get("retry-after") - .and_then(|v| v.to_str().ok()) - .and_then(retry_after_header); - if is_redirect_status(status) && location.is_some() { - logger.trace_http(&HttpCall { - proto: "DAV", - method, - url, - status, - elapsed: started.elapsed(), - note: location.as_deref(), - request, - request_type, - response: b"", - response_type: None, - }); - return Attempt::Ok { - status, - body: Vec::new(), - retry_after, - etag, - content_type: content_type_header, - last_modified, - location, - }; - } - let body_limit = if (200..300).contains(&status) { - MAX_BODY - } else { - 64 * 1024 - }; - match resp - .body_mut() - .with_config() - .limit(body_limit) - .read_to_vec() - { - Ok(bytes) => { - logger.trace_http(&HttpCall { - proto: "DAV", - method, - url, - status, - elapsed: started.elapsed(), - note: None, - request, - request_type, - response: &bytes, - response_type: content_type_header.as_deref(), - }); - Attempt::Ok { - status, - body: bytes, - retry_after, - etag, - content_type: content_type_header, - last_modified, - location, - } - } - Err(e) => Attempt::Transport(JmapError::Transport(format!( - "reading response body: {e}" - ))), - } - } - Err(e) => { - let err = map_ureq_error(e); - logger.trace_http_error("DAV", method, url, &err.to_string(), started.elapsed()); - Attempt::Transport(err) - } - } - } - - fn one_attempt_stream(&self, url: &str, include_auth: bool) -> AttemptStream { - let req = WireRequest { - method: "GET", - url, - body: None, - content_type: None, - depth: None, - accept: ACCEPT_BINARY, - include_auth, - }; - let logger = self.logger(); - let started = Instant::now(); - match self.build_and_send(req) { - Ok(resp) => { - let status = resp.status().as_u16(); - let etag = header_str(&resp, "etag"); - let content_type = header_str(&resp, "content-type"); - let last_modified = header_str(&resp, "last-modified"); - let location = header_str(&resp, "location"); - let retry_after = resp - .headers() - .get("retry-after") - .and_then(|v| v.to_str().ok()) - .and_then(retry_after_header); - logger.trace_http(&HttpCall { - proto: "DAV", - method: "GET", - url, - status, - elapsed: started.elapsed(), - note: Some("streamed download"), - request: None, - request_type: None, - response: b"", - response_type: None, - }); - let body_reader = resp.into_body().into_reader(); - AttemptStream::Ok { - status, - body_reader: Box::new(body_reader), - etag, - content_type, - last_modified, - location, - retry_after, - } - } - Err(e) => { - let err = map_ureq_error(e); - logger.trace_http_error("DAV", "GET", url, &err.to_string(), started.elapsed()); - AttemptStream::Transport(err) - } - } - } - - fn build_and_send(&self, req: WireRequest<'_>) -> Result, ureq::Error> { - let method_parsed = Method::from_bytes(req.method.as_bytes()) - .map_err(|e| ureq::Error::Other(Box::new(std::io::Error::other(e))))?; - let mut builder = Request::builder() - .method(method_parsed) - .uri(req.url) - .header("Accept", req.accept) - .header("User-Agent", self.inner.user_agent.as_str()); - if req.include_auth { - builder = builder.header("Authorization", self.inner.auth.header_value()); - } - if let Some(d) = req.depth { - builder = builder.header("Depth", d.to_string()); - } - if let Some(ct) = req.content_type { - builder = builder.header("Content-Type", ct); - } - let payload: &[u8] = req.body.unwrap_or(&[]); - let request = builder - .body(payload) - .map_err(|e| ureq::Error::Other(Box::new(std::io::Error::other(e))))?; - self.inner.agent.run(request) - } -} - -struct WireRequest<'a> { - method: &'a str, - url: &'a str, - body: Option<&'a [u8]>, - content_type: Option<&'a str>, - depth: Option, - accept: &'a str, - include_auth: bool, -} - -pub struct DavStream { - pub status: u16, - pub body_reader: Box, - pub etag: Option, - pub content_type: Option, - pub last_modified: Option, -} - -impl Read for DavStream { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - self.body_reader.read(buf) - } -} - -enum Attempt { - Ok { - status: u16, - body: Vec, - retry_after: Option, - etag: Option, - content_type: Option, - last_modified: Option, - location: Option, - }, - Transport(JmapError), -} - -enum AttemptStream { - Ok { - status: u16, - body_reader: Box, - etag: Option, - content_type: Option, - last_modified: Option, - location: Option, - retry_after: Option, - }, - Transport(JmapError), -} - -fn header_str(resp: &Response, name: &str) -> Option { - resp.headers() - .get(name) - .and_then(|v| v.to_str().ok()) - .map(str::to_owned) -} - -fn is_redirect_status(status: u16) -> bool { - matches!(status, 301 | 302 | 303 | 307 | 308) -} - -fn resolve_redirect(current: &str, location: &str) -> Result { - crate::dav::href::join_absolute(current, location) - .map_err(|e| JmapError::Connect(format!("redirect: {e}"))) -} - -fn url_host_and_port(url: &str) -> Option<(String, Option)> { - let parsed = url::Url::parse(url).ok()?; - let host = parsed.host_str()?.to_owned(); - Some((host, parsed.port_or_known_default())) -} - -fn same_host(a: &str, b: &str) -> bool { - match (url_host_and_port(a), url_host_and_port(b)) { - (Some(x), Some(y)) => x == y, - _ => false, - } -} - -fn transport_disposition(err: &JmapError) -> retry::Disposition { - match err { - JmapError::Connect(_) => retry::Disposition::Fatal, - _ => retry::Disposition::Retryable, - } -} - -fn map_ureq_error(err: ureq::Error) -> JmapError { - match err { - ureq::Error::Io(e) => JmapError::Transport(format!("io: {e}")), - ureq::Error::Timeout(t) => JmapError::Transport(format!("timeout: {t}")), - ureq::Error::HostNotFound => JmapError::Transport("host not found".to_owned()), - ureq::Error::ConnectionFailed => JmapError::Transport("connection failed".to_owned()), - ureq::Error::BodyStalled => JmapError::Transport("body stalled".to_owned()), - ureq::Error::Tls(m) => JmapError::Transport(format!("tls: {m}")), - ureq::Error::TooManyRedirects => JmapError::Connect("too many redirects".to_owned()), - ureq::Error::RedirectFailed => JmapError::Connect("redirect failed".to_owned()), - ureq::Error::TlsRequired => { - JmapError::Connect("server requires TLS but transport is unsecured".to_owned()) - } - other => JmapError::Connect(other.to_string()), - } -} - -fn truncate(body: &[u8]) -> String { - let text = String::from_utf8_lossy(body); - if text.len() <= 512 { - return text.into_owned(); - } - let end = text.floor_char_boundary(512); - format!("{}...", &text[..end]) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn client_constructs_cleanly() { - let c = DavClient::new( - Auth::Basic { - user: "u".into(), - password: "p".into(), - }, - RetryPolicy::new(3), - false, - ); - assert_eq!(c.retries_observed(), 0); - assert_eq!(c.retry_after_sleeps(), 0); - } - - #[test] - fn auth_header_appears_via_builder() { - let c = DavClient::new( - Auth::Bearer { - token: "abc".into(), - }, - RetryPolicy::new(0), - false, - ); - let logger = c.logger(); - assert_eq!(logger.level(), LEVEL_DEFAULT); - } - - #[test] - fn truncate_preserves_char_boundaries() { - let mut body = vec![b'a'; 510]; - body.extend_from_slice("\u{1F4A9}".as_bytes()); - body.extend_from_slice(&[b'b'; 600]); - let s = truncate(&body); - assert!(s.ends_with("...")); - assert!(s.is_char_boundary(s.len())); - } -} diff --git a/src/dav/client/buffered.rs b/src/dav/client/buffered.rs new file mode 100644 index 0000000..937f1cd --- /dev/null +++ b/src/dav/client/buffered.rs @@ -0,0 +1,187 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::thread; +use std::time::Instant; + +use super::wire::{RedirectChain, ResponseHeaders, Verb, WireRequest}; +use super::{ + ACCEPT_BINARY, ACCEPT_DAV, DavClient, DavHttpResponse, ERROR_BODY_LIMIT, LONG_RETRY_THRESHOLD, + MAX_BODY, PROTO, XML_BODY_TYPE, +}; +use crate::dav::error::DavError; +use crate::dav::retry::DavOutcome; +use crate::logging::{HttpCall, Level}; +use crate::net::http::BodyExcerpt; + +impl DavClient { + pub(super) fn execute( + &self, + verb: Verb, + url: &str, + xml: Option<(&str, u8)>, + ) -> Result { + let logger = self.logger(); + let method = verb.as_str(); + let max_retries = self.inner.core.retry().max_retries; + let mut attempt: u32 = 0; + let mut chain = RedirectChain::new(url); + loop { + self.inner.core.rate_limit().cooldown().wait(); + let request = WireRequest { + verb, + url: &chain.current, + body: xml.map(|(body, _)| body.as_bytes()), + content_type: xml.map(|_| XML_BODY_TYPE), + depth: xml.map(|(_, depth)| depth), + accept: if xml.is_some() { + ACCEPT_DAV + } else { + ACCEPT_BINARY + }, + credentials: chain.credentials, + }; + let (status, body, headers) = match self.buffered_attempt(&request) { + Ok(landed) => landed, + Err(err) => { + self.transport_retry(logger, err, &mut attempt)?; + continue; + } + }; + if ResponseHeaders::is_redirect(status) { + let Some(location) = headers.location.as_deref() else { + return Err(DavError::Connect(format!( + "{method} {} returned {status} without Location: {}", + chain.current, + BodyExcerpt(&body) + ))); + }; + chain.follow(status, location, logger)?; + continue; + } + match DavOutcome::classify(status, &body) { + DavOutcome::Success => { + self.inner.core.rate_limit().on_success(); + return Ok(DavHttpResponse { + status, + body, + etag: headers.etag, + content_type: headers.content_type, + last_modified: headers.last_modified, + }); + } + DavOutcome::Vanished => { + return Ok(DavHttpResponse { + status, + body, + etag: None, + content_type: None, + last_modified: None, + }); + } + DavOutcome::Auth => { + return Err(DavError::Auth(format!( + "server returned {status}: {}", + BodyExcerpt(&body) + ))); + } + DavOutcome::Fatal => { + return Err(DavError::HttpStatus { + status, + body: BodyExcerpt(&body).to_string(), + }); + } + DavOutcome::Retryable => { + let delay = self.status_retry(&mut attempt, headers.retry_after, || { + format!("{method} {} kept returning {status}", chain.current) + })?; + if delay >= LONG_RETRY_THRESHOLD { + logger.warn(format_args!( + "DAV {method} rate-limited (HTTP {status}); waiting {}s before retry {attempt}/{max_retries}", + delay.as_secs(), + )); + } + if logger.enabled(Level::Bodies) { + logger.progress(format_args!( + "Retry {attempt}/{max_retries}: {method} {} after {delay:?} (HTTP {status}), body {}", + chain.current, + BodyExcerpt(&body), + )); + } else { + logger.progress(format_args!( + "Retry {attempt}/{max_retries} (HTTP {status})" + )); + } + thread::sleep(delay); + } + } + } + } + + fn buffered_attempt( + &self, + request: &WireRequest<'_>, + ) -> Result<(u16, Vec, ResponseHeaders), DavError> { + let logger = self.logger(); + let method = request.verb.as_str(); + let started = Instant::now(); + let mut resp = match self.send(request) { + Ok(resp) => resp, + Err(e) => { + let err = DavError::from(e); + logger.trace_http_error( + PROTO, + method, + request.url, + &err.to_string(), + started.elapsed(), + ); + return Err(err); + } + }; + let status = resp.status().as_u16(); + let headers = ResponseHeaders::of(&resp); + if ResponseHeaders::is_redirect(status) && headers.location.is_some() { + logger.trace_http(&HttpCall { + proto: PROTO, + method, + url: request.url, + status, + elapsed: started.elapsed(), + note: headers.location.as_deref(), + request: request.body, + request_type: request.content_type, + response: b"", + response_type: None, + }); + return Ok((status, Vec::new(), headers)); + } + let limit = if (200..300).contains(&status) { + MAX_BODY + } else { + ERROR_BODY_LIMIT + }; + let body = resp + .body_mut() + .with_config() + .limit(limit) + .read_to_vec() + .map_err(|e| DavError::Transport(format!("reading response body: {e}")))?; + logger.trace_http(&HttpCall { + proto: PROTO, + method, + url: request.url, + status, + elapsed: started.elapsed(), + note: None, + request: request.body, + request_type: request.content_type, + response: &body, + response_type: headers.content_type.as_deref(), + }); + Ok((status, body, headers)) + } +} diff --git a/src/dav/client/mod.rs b/src/dav/client/mod.rs new file mode 100644 index 0000000..fa972a9 --- /dev/null +++ b/src/dav/client/mod.rs @@ -0,0 +1,280 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::error::Error; +use std::io::{self, Read}; +use std::sync::Arc; +use std::thread; +use std::time::Duration; + +use ureq::Body; +use ureq::BodyReader; +use ureq::http::{Method, Request, Response}; + +use crate::dav::error::DavError; +use crate::dav::parse::DavResponse; +use crate::logging::Logger; +use crate::net::auth::Auth; +use crate::net::http::{ + AgentProfile, ClientOptions, HttpCore, Methods, Redirects, RetryStats, TransportError, +}; + +mod buffered; +mod stream; +mod wire; + +use wire::{Credentials, Verb, WireRequest}; + +const MAX_BODY: u64 = 512 * 1024 * 1024; +const ERROR_BODY_LIMIT: u64 = 64 * 1024; +const LONG_RETRY_THRESHOLD: Duration = Duration::from_secs(10); +const MAX_DAV_REDIRECTS: u32 = 5; +const PROTO: &str = "DAV"; + +const ACCEPT_DAV: &str = "application/xml, text/xml;q=0.9, */*;q=0.5"; +const ACCEPT_BINARY: &str = "*/*"; +const XML_BODY_TYPE: &str = "application/xml; charset=utf-8"; +const XML_TRACE_TYPE: &str = "application/xml"; + +const MOVED_PERMANENTLY: u16 = 301; +const FOUND: u16 = 302; +const SEE_OTHER: u16 = 303; +const TEMPORARY_REDIRECT: u16 = 307; +const PERMANENT_REDIRECT: u16 = 308; +const NOT_FOUND: u16 = 404; +const GONE: u16 = 410; + +#[derive(Debug)] +pub struct DavHttpResponse { + pub status: u16, + pub body: Vec, + pub etag: Option, + pub content_type: Option, + pub last_modified: Option, +} + +#[derive(Debug)] +pub struct MultiStatus { + pub status: u16, + pub responses: Vec, + pub final_url: String, +} + +pub struct DavStream { + pub status: u16, + pub etag: Option, + pub content_type: Option, + pub last_modified: Option, + body: Option>, +} + +impl Read for DavStream { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match &mut self.body { + Some(body) => body.read(buf), + None => Ok(0), + } + } +} + +struct Inner { + core: HttpCore, + authorization: String, +} + +#[derive(Clone)] +pub struct DavClient { + inner: Arc, +} + +impl DavClient { + pub fn new(auth: Auth, options: ClientOptions) -> DavClient { + let profile = AgentProfile { + redirects: Redirects::Manual, + methods: Methods::Extended, + max_body: MAX_BODY, + }; + DavClient { + inner: Arc::new(Inner { + core: HttpCore::new(options, profile), + authorization: auth.header_value(), + }), + } + } + + pub fn logger(&self) -> Logger { + self.inner.core.logger() + } + + pub fn stats(&self) -> RetryStats { + self.inner.core.stats().snapshot() + } + + pub fn propfind(&self, url: &str, depth: u8, body: &str) -> Result { + self.execute(Verb::Propfind, url, Some((body, depth))) + } + + pub fn report(&self, url: &str, depth: u8, body: &str) -> Result { + self.execute(Verb::Report, url, Some((body, depth))) + } + + pub fn get(&self, url: &str) -> Result { + self.execute(Verb::Get, url, None) + } + + pub fn propfind_responses( + &self, + url: &str, + depth: u8, + body: &str, + base_url: &str, + ) -> Result { + self.stream_multistatus(Verb::Propfind, url, body.as_bytes(), depth, base_url) + } + + pub fn report_responses( + &self, + url: &str, + depth: u8, + body: &str, + base_url: &str, + ) -> Result { + self.stream_multistatus(Verb::Report, url, body.as_bytes(), depth, base_url) + } + + fn status_retry( + &self, + attempt: &mut u32, + retry_after: Option, + exhausted: impl FnOnce() -> String, + ) -> Result { + let core = &self.inner.core; + *attempt += 1; + core.stats().note_retry(); + if *attempt > core.retry().max_retries { + return Err(DavError::RetriesExhausted(exhausted())); + } + if retry_after.is_some() { + core.stats().note_retry_after(); + } + Ok(core.throttle(retry_after)) + } + + fn transport_retry( + &self, + logger: Logger, + err: DavError, + attempt: &mut u32, + ) -> Result<(), DavError> { + if err.is_connect() { + return Err(err); + } + let core = &self.inner.core; + let max_retries = core.retry().max_retries; + *attempt += 1; + core.stats().note_retry(); + if *attempt > max_retries { + return Err(DavError::RetriesExhausted(err.to_string())); + } + let delay = core.backoff(*attempt); + if delay >= LONG_RETRY_THRESHOLD { + logger.warn(format_args!( + "transient transport failure ({err}); waiting {}s before retry {attempt}/{max_retries}", + delay.as_secs(), + )); + } + thread::sleep(delay); + Ok(()) + } + + fn send(&self, req: &WireRequest<'_>) -> Result, TransportError> { + let mut builder = Request::builder() + .method(Method::from_bytes(req.verb.as_str().as_bytes()).map_err(DavClient::invalid)?) + .uri(req.url) + .header("Accept", req.accept) + .header("User-Agent", self.inner.core.user_agent()); + if req.credentials == Credentials::Send { + builder = builder.header("Authorization", self.inner.authorization.as_str()); + } + if let Some(depth) = req.depth { + builder = builder.header("Depth", u16::from(depth)); + } + if let Some(content_type) = req.content_type { + builder = builder.header("Content-Type", content_type); + } + let request = builder + .body(req.body.unwrap_or_default()) + .map_err(DavClient::invalid)?; + self.inner + .core + .agent() + .run(request) + .map_err(TransportError::from) + } + + fn invalid(e: impl Into>) -> TransportError { + TransportError::Other(ureq::Error::Other(Box::new(io::Error::other(e)))) + } +} + +#[cfg(test)] +mod tests { + use super::wire::RedirectChain; + use super::*; + use crate::logging::Level; + use crate::net::http::TlsPolicy; + use crate::net::retry::RetryPolicy; + + fn options(retries: u32) -> ClientOptions { + ClientOptions { + retry: RetryPolicy::new(retries), + tls: TlsPolicy::Verify, + logger: Logger::new(Level::Default), + } + } + + #[test] + fn client_constructs_cleanly() { + let c = DavClient::new( + Auth::Basic { + user: "u".into(), + password: "p".into(), + }, + options(3), + ); + assert_eq!(c.stats(), RetryStats::default()); + assert_eq!(c.logger().level(), Level::Default); + let c = DavClient::new( + Auth::Bearer { + token: "abc".into(), + }, + options(0), + ); + assert_eq!(c.inner.authorization, "Bearer abc"); + assert_eq!(c.logger().level(), Level::Default); + } + + #[test] + fn redirect_chain_drops_credentials_off_origin_and_caps_hops() { + let logger = Logger::new(Level::Quiet); + let mut chain = RedirectChain::new("https://a.example/dav/"); + chain.follow(FOUND, "/moved/", logger).expect("same host"); + assert_eq!(chain.current, "https://a.example/moved/"); + assert_eq!(chain.credentials, Credentials::Send); + chain + .follow(FOUND, "https://b.example/x/", logger) + .expect("cross host"); + assert_eq!(chain.credentials, Credentials::Withhold); + for _ in 2..MAX_DAV_REDIRECTS { + chain.follow(FOUND, "/y/", logger).expect("within cap"); + } + let err = chain.follow(FOUND, "/z/", logger).expect_err("cap"); + assert_eq!( + err.to_string(), + "connection failure: too many redirects (5) starting from https://a.example/dav/" + ); + } +} diff --git a/src/dav/client/stream.rs b/src/dav/client/stream.rs new file mode 100644 index 0000000..bedfcd7 --- /dev/null +++ b/src/dav/client/stream.rs @@ -0,0 +1,255 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::{BufReader, Read}; +use std::thread; +use std::time::Instant; + +use super::wire::{RedirectChain, ResponseHeaders, Verb, WireRequest}; +use super::{ + ACCEPT_BINARY, ACCEPT_DAV, DavClient, DavStream, ERROR_BODY_LIMIT, GONE, LONG_RETRY_THRESHOLD, + MultiStatus, NOT_FOUND, PROTO, XML_BODY_TYPE, XML_TRACE_TYPE, +}; +use crate::dav::error::DavError; +use crate::dav::parse::{ControlStrippingReader, MultistatusParser}; +use crate::dav::retry::DavOutcome; +use crate::logging::HttpCall; +use crate::net::http::BodyExcerpt; + +impl DavClient { + pub fn get_stream(&self, url: &str) -> Result { + let logger = self.logger(); + let mut attempt: u32 = 0; + let mut chain = RedirectChain::new(url); + loop { + self.inner.core.rate_limit().cooldown().wait(); + let request = WireRequest { + verb: Verb::Get, + url: &chain.current, + body: None, + content_type: None, + depth: None, + accept: ACCEPT_BINARY, + credentials: chain.credentials, + }; + let started = Instant::now(); + let resp = match self.send(&request) { + Ok(resp) => resp, + Err(e) => { + let err = DavError::from(e); + logger.trace_http_error( + PROTO, + "GET", + &chain.current, + &err.to_string(), + started.elapsed(), + ); + self.transport_retry(logger, err, &mut attempt)?; + continue; + } + }; + let status = resp.status().as_u16(); + let headers = ResponseHeaders::of(&resp); + logger.trace_http(&HttpCall { + proto: PROTO, + method: "GET", + url: &chain.current, + status, + elapsed: started.elapsed(), + note: Some("streamed download"), + request: None, + request_type: None, + response: b"", + response_type: None, + }); + match status { + 200..=299 => { + self.inner.core.rate_limit().on_success(); + return Ok(DavStream { + status, + etag: headers.etag, + content_type: headers.content_type, + last_modified: headers.last_modified, + body: Some(resp.into_body().into_reader()), + }); + } + NOT_FOUND | GONE => { + self.inner.core.rate_limit().on_success(); + return Ok(DavStream { + status, + etag: None, + content_type: None, + last_modified: None, + body: None, + }); + } + _ => {} + } + if let Some(location) = headers.location.as_deref() + && ResponseHeaders::is_redirect(status) + { + chain.follow(status, location, logger)?; + continue; + } + let mut body = Vec::new(); + let _ = resp + .into_body() + .into_reader() + .take(ERROR_BODY_LIMIT) + .read_to_end(&mut body); + match DavOutcome::classify(status, &body) { + DavOutcome::Success => {} + DavOutcome::Auth => { + return Err(DavError::Auth(format!("server returned {status}"))); + } + DavOutcome::Vanished | DavOutcome::Fatal => { + return Err(DavError::HttpStatus { + status, + body: BodyExcerpt(&body).to_string(), + }); + } + DavOutcome::Retryable => { + let delay = self.status_retry(&mut attempt, headers.retry_after, || { + format!("GET stream kept returning {status}") + })?; + if delay >= LONG_RETRY_THRESHOLD { + logger.warn(format_args!( + "DAV stream rate-limited (HTTP {status}); waiting {}s", + delay.as_secs() + )); + } + thread::sleep(delay); + } + } + } + } + + pub(super) fn stream_multistatus( + &self, + verb: Verb, + url: &str, + body: &[u8], + depth: u8, + base_url: &str, + ) -> Result { + let logger = self.logger(); + let method = verb.as_str(); + let mut attempt: u32 = 0; + let mut chain = RedirectChain::new(url); + loop { + self.inner.core.rate_limit().cooldown().wait(); + let started = Instant::now(); + let request = WireRequest { + verb, + url: &chain.current, + body: Some(body), + content_type: Some(XML_BODY_TYPE), + depth: Some(depth), + accept: ACCEPT_DAV, + credentials: chain.credentials, + }; + let mut resp = match self.send(&request) { + Ok(resp) => resp, + Err(e) => { + self.transport_retry(logger, DavError::from(e), &mut attempt)?; + continue; + } + }; + let status = resp.status().as_u16(); + if ResponseHeaders::is_redirect(status) { + let Some(location) = ResponseHeaders::text(&resp, "location") else { + return Err(DavError::Connect(format!( + "{method} {} returned {status} without Location", + chain.current + ))); + }; + chain.follow(status, &location, logger)?; + continue; + } + let retry_after = ResponseHeaders::retry_after(&resp); + if (200..300).contains(&status) { + self.inner.core.rate_limit().on_success(); + logger.trace_http(&HttpCall { + proto: PROTO, + method, + url: &chain.current, + status, + elapsed: started.elapsed(), + note: Some("streamed multistatus"), + request: Some(body), + request_type: Some(XML_TRACE_TYPE), + response: b"", + response_type: None, + }); + let parse_base = if chain.current == url { + base_url + } else { + &chain.current + }; + let reader = + BufReader::new(ControlStrippingReader::new(resp.into_body().into_reader())); + let responses = MultistatusParser::parse(reader, parse_base)?; + return Ok(MultiStatus { + status, + responses, + final_url: chain.current, + }); + } + let error_body = resp + .body_mut() + .with_config() + .limit(ERROR_BODY_LIMIT) + .read_to_vec() + .map_err(|e| DavError::Transport(format!("reading error body: {e}")))?; + logger.trace_http(&HttpCall { + proto: PROTO, + method, + url: &chain.current, + status, + elapsed: started.elapsed(), + note: None, + request: Some(body), + request_type: Some(XML_TRACE_TYPE), + response: &error_body, + response_type: None, + }); + match DavOutcome::classify(status, &error_body) { + DavOutcome::Success | DavOutcome::Vanished => { + return Ok(MultiStatus { + status, + responses: Vec::new(), + final_url: chain.current, + }); + } + DavOutcome::Auth => { + return Err(DavError::Auth(format!( + "server returned {status}: {}", + BodyExcerpt(&error_body) + ))); + } + DavOutcome::Fatal => { + return Err(DavError::HttpStatus { + status, + body: BodyExcerpt(&error_body).to_string(), + }); + } + DavOutcome::Retryable => { + let delay = self.status_retry(&mut attempt, retry_after, || { + format!("{method} {url} kept returning {status}") + })?; + if delay >= LONG_RETRY_THRESHOLD { + logger.warn(format_args!( + "DAV {method} rate-limited (HTTP {status}); waiting {}s before retry {attempt}/{}", + delay.as_secs(), + self.inner.core.retry().max_retries, + )); + } + thread::sleep(delay); + } + } + } + } +} diff --git a/src/dav/client/wire.rs b/src/dav/client/wire.rs new file mode 100644 index 0000000..795611a --- /dev/null +++ b/src/dav/client/wire.rs @@ -0,0 +1,153 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::time::Duration; + +use ureq::Body; +use ureq::http::Response; +use url::Url; + +use super::{ + FOUND, MAX_DAV_REDIRECTS, MOVED_PERMANENTLY, PERMANENT_REDIRECT, SEE_OTHER, TEMPORARY_REDIRECT, +}; +use crate::dav::error::DavError; +use crate::dav::href::Href; +use crate::logging::Logger; +use crate::net::retry::RetryAfter; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Credentials { + Send, + Withhold, +} + +#[derive(Debug, Clone, Copy)] +pub(super) enum Verb { + Propfind, + Report, + Get, +} + +impl Verb { + pub(super) fn as_str(self) -> &'static str { + match self { + Verb::Propfind => "PROPFIND", + Verb::Report => "REPORT", + Verb::Get => "GET", + } + } +} + +pub(super) struct WireRequest<'a> { + pub(super) verb: Verb, + pub(super) url: &'a str, + pub(super) body: Option<&'a [u8]>, + pub(super) content_type: Option<&'a str>, + pub(super) depth: Option, + pub(super) accept: &'a str, + pub(super) credentials: Credentials, +} + +pub(super) struct ResponseHeaders { + pub(super) etag: Option, + pub(super) content_type: Option, + pub(super) last_modified: Option, + pub(super) location: Option, + pub(super) retry_after: Option, +} + +impl ResponseHeaders { + pub(super) fn of(resp: &Response) -> ResponseHeaders { + ResponseHeaders { + etag: ResponseHeaders::text(resp, "etag"), + content_type: ResponseHeaders::text(resp, "content-type"), + last_modified: ResponseHeaders::text(resp, "last-modified"), + location: ResponseHeaders::text(resp, "location"), + retry_after: ResponseHeaders::retry_after(resp), + } + } + + pub(super) fn text(resp: &Response, name: &str) -> Option { + resp.headers() + .get(name) + .and_then(|v| v.to_str().ok()) + .map(str::to_owned) + } + + pub(super) fn retry_after(resp: &Response) -> Option { + resp.headers() + .get("retry-after") + .and_then(|v| v.to_str().ok()) + .and_then(RetryAfter::parse_now) + } + + pub(super) fn is_redirect(status: u16) -> bool { + matches!( + status, + MOVED_PERMANENTLY | FOUND | SEE_OTHER | TEMPORARY_REDIRECT | PERMANENT_REDIRECT + ) + } +} + +pub(super) struct RedirectChain<'a> { + pub(super) origin: &'a str, + pub(super) current: String, + pub(super) hops: u32, + pub(super) credentials: Credentials, +} + +impl<'a> RedirectChain<'a> { + pub(super) fn new(origin: &'a str) -> RedirectChain<'a> { + RedirectChain { + origin, + current: origin.to_owned(), + hops: 0, + credentials: Credentials::Send, + } + } + + pub(super) fn follow( + &mut self, + status: u16, + location: &str, + logger: Logger, + ) -> Result<(), DavError> { + if self.hops >= MAX_DAV_REDIRECTS { + return Err(DavError::Connect(format!( + "too many redirects ({}) starting from {}", + self.hops, self.origin + ))); + } + self.hops += 1; + let next = Href::join_absolute(&self.current, location) + .map_err(|e| DavError::Connect(format!("redirect: {e}")))?; + if self.credentials == Credentials::Send && !self.same_host(&next) { + self.credentials = Credentials::Withhold; + logger.progress(format_args!( + "DAV redirect leaves origin host {}; dropping Authorization for {next}", + self.origin + )); + } + logger.progress(format_args!( + "DAV redirect {status}: {} -> {next}", + self.current + )); + self.current = next; + Ok(()) + } + + pub(super) fn same_host(&self, next: &str) -> bool { + match (Url::parse(self.origin), Url::parse(next)) { + (Ok(a), Ok(b)) => match (a.host_str(), b.host_str()) { + (Some(x), Some(y)) => { + x == y && a.port_or_known_default() == b.port_or_known_default() + } + _ => false, + }, + _ => false, + } + } +} diff --git a/src/dav/discover.rs b/src/dav/discover.rs index bbc4721..e4efe2f 100644 --- a/src/dav/discover.rs +++ b/src/dav/discover.rs @@ -4,11 +4,11 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use crate::dav::client::DavClient; -use crate::dav::href::{Href, absolute_url, normalise}; -use crate::dav::parse::DavResponse; -use crate::dav::xml; -use crate::jmap::error::JmapError; +use crate::dav::client::{DavClient, MultiStatus}; +use crate::dav::error::DavError; +use crate::dav::href::{Href, HrefError}; +use crate::dav::parse::ResourceProps; +use crate::dav::xml::Propfind; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DavKind { @@ -26,31 +26,31 @@ impl DavKind { } } - pub fn home_set_body(self) -> Option { + pub fn home_set_body(self) -> Option<&'static str> { match self { - DavKind::Caldav => Some(xml::propfind_calendar_home_set()), - DavKind::Carddav => Some(xml::propfind_addressbook_home_set()), + DavKind::Caldav => Some(Propfind::CalendarHomeSet.body()), + DavKind::Carddav => Some(Propfind::AddressbookHomeSet.body()), DavKind::Webdav => None, } } - pub fn principal_and_home_set_body(self) -> Option { + pub fn principal_and_home_set_body(self) -> Option<&'static str> { match self { - DavKind::Caldav => Some(xml::propfind_principal_and_calendar_home_set()), - DavKind::Carddav => Some(xml::propfind_principal_and_addressbook_home_set()), + DavKind::Caldav => Some(Propfind::PrincipalAndCalendarHomeSet.body()), + DavKind::Carddav => Some(Propfind::PrincipalAndAddressbookHomeSet.body()), DavKind::Webdav => None, } } - pub fn collection_listing_body(self) -> String { + pub fn collection_listing_body(self) -> &'static str { match self { - DavKind::Caldav => xml::propfind_calendar_collections(), - DavKind::Carddav => xml::propfind_addressbook_collections(), - DavKind::Webdav => xml::propfind_webdav_listing(), + DavKind::Caldav => Propfind::CalendarCollections.body(), + DavKind::Carddav => Propfind::AddressbookCollections.body(), + DavKind::Webdav => Propfind::WebdavListing.body(), } } - pub fn extract_home_set(self, props: &crate::dav::parse::ResourceProps) -> Option<&str> { + pub fn extract_home_set(self, props: &ResourceProps) -> Option<&str> { match self { DavKind::Caldav => props.calendar_home_set.as_deref(), DavKind::Carddav => props.addressbook_home_set.as_deref(), @@ -58,7 +58,7 @@ impl DavKind { } } - pub fn item_resourcetype_matches(self, props: &crate::dav::parse::ResourceProps) -> bool { + pub fn item_resourcetype_matches(self, props: &ResourceProps) -> bool { match self { DavKind::Caldav => props.is_calendar, DavKind::Carddav => props.is_addressbook, @@ -71,17 +71,15 @@ impl DavKind { pub struct DiscoveredCollection { pub url: String, pub href: Href, - pub props: crate::dav::parse::ResourceProps, + pub props: ResourceProps, } #[derive(Debug, thiserror::Error)] pub enum DiscoveryError { #[error("transport: {0}")] - Transport(#[from] JmapError), - #[error("xml: {0}")] - Parse(#[from] crate::dav::parse::ParseError), + Transport(#[from] DavError), #[error("href: {0}")] - Href(#[from] crate::dav::href::HrefError), + Href(#[from] HrefError), #[error("no DAV collections found under {url}")] NotFound { url: String }, } @@ -93,229 +91,189 @@ pub struct Discovery { pub collections: Vec, } -pub fn discover( - client: &DavClient, - kind: DavKind, - user_url: &str, -) -> Result { - if let Some(disc) = try_treat_url_as_home_or_collection(client, kind, user_url)? { - return Ok(disc); - } - if let Some(disc) = try_via_principal(client, kind, user_url)? { - return Ok(disc); - } - if let Some(well_known) = kind.well_known_uri() { - let absolute = absolute_url(user_url, &Href::from_normalised(well_known.to_owned()))?; - if let Some(disc) = try_via_principal(client, kind, &absolute)? { +impl Discovery { + pub fn run( + client: &DavClient, + kind: DavKind, + user_url: &str, + ) -> Result { + let probe = Probe { client, kind }; + if let Some(disc) = probe.home_or_collection(user_url)? { return Ok(disc); } - if let Some(disc) = try_treat_url_as_home_or_collection(client, kind, &absolute)? { + if let Some(disc) = probe.via_principal(user_url)? { return Ok(disc); } + if let Some(well_known) = kind.well_known_uri() { + let absolute = Href::join_absolute(user_url, well_known)?; + if let Some(disc) = probe.via_principal(&absolute)? { + return Ok(disc); + } + if let Some(disc) = probe.home_or_collection(&absolute)? { + return Ok(disc); + } + } + Err(DiscoveryError::NotFound { + url: user_url.to_owned(), + }) } - Err(DiscoveryError::NotFound { - url: user_url.to_owned(), - }) } -fn try_treat_url_as_home_or_collection( - client: &DavClient, +#[derive(Clone, Copy)] +struct Probe<'a> { + client: &'a DavClient, kind: DavKind, - url: &str, -) -> Result, DiscoveryError> { - let body = kind.collection_listing_body(); - let collections = match propfind_collections(client, kind, url, &body) { - Ok(c) => c, - Err(DiscoveryError::Transport(JmapError::HttpStatus { status, .. })) - if (400..600).contains(&status) => - { - return Ok(None); - } - Err(DiscoveryError::Transport(JmapError::Malformed(_))) - | Err(DiscoveryError::Transport(JmapError::RetriesExhausted(_))) => { - return Ok(None); - } - Err(e) => return Err(e), - }; - if collections.is_empty() { - return Ok(None); - } - let principal_url = resolve_principal_url(client, kind, url)?; - Ok(Some(Discovery { - principal_url, - home_set_url: url.to_owned(), - collections, - })) } -fn resolve_principal_url( - client: &DavClient, - kind: DavKind, - url: &str, -) -> Result, DiscoveryError> { - if matches!(kind, DavKind::Webdav) { - return Ok(None); +impl Probe<'_> { + fn is_dead_end(err: &DavError) -> bool { + match err { + DavError::HttpStatus { status, .. } => (400..600).contains(status), + DavError::RetriesExhausted(_) | DavError::Multistatus(_) => true, + _ => false, + } } - let body = xml::propfind_current_user_principal(); - let ms = match client.propfind_responses(url, 0, &body, url) { - Ok(ms) => ms, - Err(JmapError::HttpStatus { status, .. }) if (400..600).contains(&status) => { - return Ok(None); - } - Err(JmapError::RetriesExhausted(_)) | Err(JmapError::Malformed(_)) => return Ok(None), - Err(e) => return Err(DiscoveryError::Transport(e)), - }; - let final_url = ms.final_url.clone(); - let Some(principal_href) = ms - .responses - .iter() - .find_map(|r| r.props.current_user_principal.as_deref()) - else { - return Ok(None); - }; - let resolved = absolute_url(&final_url, &normalise(&final_url, principal_href)?)?; - Ok(Some(resolved)) -} -fn try_via_principal( - client: &DavClient, - kind: DavKind, - url: &str, -) -> Result, DiscoveryError> { - let Some(combined_body) = kind.principal_and_home_set_body() else { - return Ok(None); - }; - let first_ms = match client.propfind_responses(url, 0, &combined_body, url) { - Ok(ms) => ms, - Err(JmapError::HttpStatus { status, .. }) if (400..600).contains(&status) => { + fn propfind(&self, url: &str, body: &str) -> Result, DiscoveryError> { + match self.client.propfind_responses(url, 0, body, url) { + Ok(ms) => Ok(Some(ms)), + Err(e) if Probe::is_dead_end(&e) => Ok(None), + Err(e) => Err(DiscoveryError::Transport(e)), + } + } + + fn home_or_collection(&self, url: &str) -> Result, DiscoveryError> { + let collections = match self.collections(url) { + Ok(c) => c, + Err(DiscoveryError::Transport(e)) if Probe::is_dead_end(&e) => return Ok(None), + Err(e) => return Err(e), + }; + if collections.is_empty() { return Ok(None); } - Err(JmapError::RetriesExhausted(_)) | Err(JmapError::Malformed(_)) => return Ok(None), - Err(e) => return Err(DiscoveryError::Transport(e)), - }; - let first_final_url = first_ms.final_url.clone(); - let direct_home_set = first_ms - .responses - .iter() - .find_map(|r| kind.extract_home_set(&r.props)) - .map(|h| h.to_owned()); - let (principal_url, home_set_url) = if let Some(home) = direct_home_set { - let resolved = absolute_url(&first_final_url, &normalise(&first_final_url, &home)?)?; - (first_final_url.clone(), resolved) - } else { - let principal_href = first_ms + let principal_url = self.principal_url(url)?; + Ok(Some(Discovery { + principal_url, + home_set_url: url.to_owned(), + collections, + })) + } + + fn principal_url(&self, url: &str) -> Result, DiscoveryError> { + if self.kind == DavKind::Webdav { + return Ok(None); + } + let Some(ms) = self.propfind(url, Propfind::CurrentUserPrincipal.body())? else { + return Ok(None); + }; + let Some(principal_href) = ms .responses .iter() .find_map(|r| r.props.current_user_principal.as_deref()) - .map(|h| h.to_owned()); - let principal_url = match principal_href { - Some(rel) => absolute_url(&first_final_url, &normalise(&first_final_url, &rel)?)?, - None => first_final_url.clone(), - }; - let Some(home_set_body) = kind.home_set_body() else { + else { return Ok(None); }; - let home_ms = - match client.propfind_responses(&principal_url, 0, &home_set_body, &principal_url) { - Ok(ms) => ms, - Err(JmapError::HttpStatus { status, .. }) if (400..600).contains(&status) => { - return Ok(None); - } - Err(JmapError::RetriesExhausted(_)) | Err(JmapError::Malformed(_)) => { - return Ok(None); - } - Err(e) => return Err(DiscoveryError::Transport(e)), - }; - let principal_after_redirect = home_ms.final_url.clone(); - let home_set_href = home_ms + let resolved = Href::parse(&ms.final_url, principal_href)?.to_absolute(&ms.final_url)?; + Ok(Some(resolved)) + } + + fn via_principal(&self, url: &str) -> Result, DiscoveryError> { + let kind = self.kind; + let Some(combined_body) = kind.principal_and_home_set_body() else { + return Ok(None); + }; + let Some(first_ms) = self.propfind(url, combined_body)? else { + return Ok(None); + }; + let first_final_url = first_ms.final_url.as_str(); + let direct_home_set = first_ms .responses .iter() - .find_map(|r| kind.extract_home_set(&r.props)) - .map(|h| h.to_owned()); - let Some(home_set_rel) = home_set_href else { - return Ok(None); - }; - let resolved = absolute_url( - &principal_after_redirect, - &normalise(&principal_after_redirect, &home_set_rel)?, - )?; - (principal_url, resolved) - }; - let listing_body = kind.collection_listing_body(); - let collections = propfind_collections(client, kind, &home_set_url, &listing_body)?; - if collections.is_empty() { - return Ok(None); - } - Ok(Some(Discovery { - principal_url: Some(principal_url), - home_set_url, - collections, - })) -} - -fn propfind_collections( - client: &DavClient, - kind: DavKind, - url: &str, - body: &str, -) -> Result, DiscoveryError> { - let responses = propfind_depth(client, url, 1, body)?; - let self_href = normalise(url, "").map(|h| h.into_string()).ok(); - let mut out: Vec = Vec::new(); - for r in responses { - let is_self = self_href.as_deref() == Some(r.href.as_str()); - if matches!(kind, DavKind::Webdav) { - if !r.props.is_collection { - continue; - } - if !is_self { - continue; - } + .find_map(|r| kind.extract_home_set(&r.props)); + let (principal_url, home_set_url) = if let Some(home) = direct_home_set { + let resolved = Href::parse(first_final_url, home)?.to_absolute(first_final_url)?; + (first_final_url.to_owned(), resolved) } else { - if !kind.item_resourcetype_matches(&r.props) { - continue; - } - if calendar_supports_only_vfreebusy(&r.props) { - client.logger().warn(&format!( - "calendar {} supports only {:?}; skipping", - r.href.as_str(), - r.props.supported_components, - )); - continue; - } + let principal_href = first_ms + .responses + .iter() + .find_map(|r| r.props.current_user_principal.as_deref()); + let principal_url = match principal_href { + Some(rel) => Href::parse(first_final_url, rel)?.to_absolute(first_final_url)?, + None => first_final_url.to_owned(), + }; + let Some(home_set_body) = kind.home_set_body() else { + return Ok(None); + }; + let Some(home_ms) = self.propfind(&principal_url, home_set_body)? else { + return Ok(None); + }; + let principal_after_redirect = home_ms.final_url.as_str(); + let Some(home_set_rel) = home_ms + .responses + .iter() + .find_map(|r| kind.extract_home_set(&r.props)) + else { + return Ok(None); + }; + let resolved = Href::parse(principal_after_redirect, home_set_rel)? + .to_absolute(principal_after_redirect)?; + (principal_url, resolved) + }; + let collections = self.collections(&home_set_url)?; + if collections.is_empty() { + return Ok(None); } - let abs = absolute_url(url, &r.href)?; - out.push(DiscoveredCollection { - url: abs, - href: r.href.clone(), - props: r.props, - }); + Ok(Some(Discovery { + principal_url: Some(principal_url), + home_set_url, + collections, + })) } - Ok(out) -} -fn calendar_supports_only_vfreebusy(props: &crate::dav::parse::ResourceProps) -> bool { - if !props.is_calendar { - return false; + fn collections(&self, url: &str) -> Result, DiscoveryError> { + let kind = self.kind; + let responses = self + .client + .propfind_responses(url, 1, kind.collection_listing_body(), url)? + .responses; + let self_href = Href::parse(url, "").map(Href::into_string).ok(); + let mut out: Vec = Vec::new(); + for r in responses { + if kind == DavKind::Webdav { + let is_self = self_href.as_deref() == Some(r.href.as_str()); + if !r.props.is_collection || !is_self { + continue; + } + } else { + if !kind.item_resourcetype_matches(&r.props) { + continue; + } + if Probe::supports_only_vfreebusy(&r.props) { + self.client.logger().warn(format_args!( + "calendar {} supports only VFREEBUSY; skipped", + r.href.as_str(), + )); + continue; + } + } + out.push(DiscoveredCollection { + url: r.href.to_absolute(url)?, + href: r.href, + props: r.props, + }); + } + Ok(out) } - if props.supported_components.is_empty() { - return false; - } - let has_useful = props - .supported_components - .iter() - .any(|c| matches!(c.as_str(), "VEVENT" | "VTODO" | "VJOURNAL")); - !has_useful -} -fn propfind_depth( - client: &DavClient, - url: &str, - depth: u8, - body: &str, -) -> Result, DiscoveryError> { - let ms = client.propfind_responses(url, depth, body, url)?; - Ok(ms.responses) + fn supports_only_vfreebusy(props: &ResourceProps) -> bool { + props.is_calendar + && !props.supported_components.is_empty() + && !props + .supported_components + .iter() + .any(|c| hashify::set!(c.as_bytes(), "VEVENT", "VTODO", "VJOURNAL")) + } } #[cfg(test)] @@ -344,7 +302,7 @@ mod tests { #[test] fn item_resourcetype_matches_calendar_for_caldav() { - let p = crate::dav::parse::ResourceProps { + let p = ResourceProps { is_calendar: true, ..Default::default() }; @@ -354,7 +312,7 @@ mod tests { #[test] fn item_resourcetype_matches_collection_for_webdav() { - let p = crate::dav::parse::ResourceProps { + let p = ResourceProps { is_collection: true, ..Default::default() }; diff --git a/src/dav/error.rs b/src/dav/error.rs new file mode 100644 index 0000000..3dcddb1 --- /dev/null +++ b/src/dav/error.rs @@ -0,0 +1,113 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::dav::parse::ParseError; +use crate::error::Error; +use crate::net::http::TransportError; + +#[derive(Debug, thiserror::Error)] +pub enum DavError { + #[error("transport failure: {0}")] + Transport(String), + #[error("connection failure: {0}")] + Connect(String), + #[error("authentication rejected: {0}")] + Auth(String), + #[error("http status {status}: {body}")] + HttpStatus { status: u16, body: String }, + #[error("retries exhausted: {0}")] + RetriesExhausted(String), + #[error("malformed jmap response: multistatus parse: {0}")] + Multistatus(#[from] ParseError), +} + +impl DavError { + pub fn is_connect(&self) -> bool { + matches!(self, DavError::Connect(_)) + } +} + +impl From for DavError { + fn from(err: TransportError) -> DavError { + if err.is_connect() { + DavError::Connect(err.to_string()) + } else { + DavError::Transport(err.to_string()) + } + } +} + +impl From for Error { + fn from(value: DavError) -> Error { + match value { + DavError::Connect(m) | DavError::Transport(m) => Error::Connection(m), + reached @ (DavError::Auth(_) + | DavError::HttpStatus { .. } + | DavError::RetriesExhausted(_)) => Error::Connection(reached.to_string()), + per_unit @ DavError::Multistatus(_) => Error::Partial(per_unit.to_string()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::error::ExitStatus; + use crate::jmap::error::JmapError; + + fn samples() -> [(DavError, JmapError); 6] { + [ + ( + DavError::Connect("refused".to_owned()), + JmapError::Connect("refused".to_owned()), + ), + ( + DavError::Transport("io: broken pipe".to_owned()), + JmapError::Transport("io: broken pipe".to_owned()), + ), + ( + DavError::Auth("server returned 401: no".to_owned()), + JmapError::Auth("server returned 401: no".to_owned()), + ), + ( + DavError::HttpStatus { + status: 404, + body: "".to_owned(), + }, + JmapError::HttpStatus { + status: 404, + body: "".to_owned(), + }, + ), + ( + DavError::RetriesExhausted("gave up".to_owned()), + JmapError::RetriesExhausted("gave up".to_owned()), + ), + ( + DavError::Multistatus(ParseError::MissingHref), + JmapError::Malformed("multistatus parse: missing href in response".to_owned()), + ), + ] + } + + #[test] + fn texts_and_exit_codes_match_the_former_jmap_error() { + for (dav, jmap) in samples() { + assert_eq!(dav.to_string(), jmap.to_string()); + let (dav, jmap) = (Error::from(dav), Error::from(jmap)); + assert_eq!(dav.to_string(), jmap.to_string()); + assert_eq!(dav.exit_status(), jmap.exit_status()); + assert_eq!(dav.aborts_run(), jmap.aborts_run()); + } + } + + #[test] + fn a_malformed_multistatus_stays_per_unit() { + let mapped = Error::from(DavError::Multistatus(ParseError::MissingHref)); + assert!(!mapped.aborts_run()); + assert_eq!(mapped.exit_status(), ExitStatus::Partial); + } +} diff --git a/src/dav/href.rs b/src/dav/href.rs index 6a1c357..d3a3885 100644 --- a/src/dav/href.rs +++ b/src/dav/href.rs @@ -7,6 +7,8 @@ use encodify::hex::decode_pair; use url::Url; +use crate::net::encode::{Percent, SafeSet}; + #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Href(String); @@ -47,131 +49,115 @@ pub enum HrefError { }, } -pub fn normalise(base: &str, raw: &str) -> Result { - let base_url = Url::parse(base).map_err(|e| HrefError::InvalidBase { - url: base.to_owned(), - source: e, - })?; - let resolved = base_url - .join(raw.trim()) - .map_err(|e| HrefError::InvalidHref { - href: raw.to_owned(), - base: base.to_owned(), +impl Href { + pub fn parse(base: &str, raw: &str) -> Result { + let resolved = Href::base(base)? + .join(raw.trim()) + .map_err(|e| HrefError::InvalidHref { + href: raw.to_owned(), + base: base.to_owned(), + source: e, + })?; + Ok(Href::from_url(&resolved)) + } + + pub fn from_url(url: &Url) -> Href { + let path = match url.path() { + "" => "/", + path => path, + }; + let decoded = Href::decode(path); + Href(Percent::encode(&decoded, SafeSet::Path)) + } + + pub fn to_absolute(&self, base: &str) -> Result { + Href::join_absolute(base, &self.0) + } + + pub fn join_absolute(base: &str, href: &str) -> Result { + let resolved = Href::base(base)? + .join(href) + .map_err(|e| HrefError::InvalidHref { + href: href.to_owned(), + base: base.to_owned(), + source: e, + })?; + Ok(resolved.into()) + } + + pub fn last_component(&self) -> String { + let trimmed = self.0.trim_end_matches('/'); + let tail = trimmed.rsplit_once('/').map_or(trimmed, |(_, tail)| tail); + Href::decode(tail) + } + + pub fn parent(&self) -> Href { + let trimmed = self.0.trim_end_matches('/'); + match trimmed.rsplit_once('/') { + Some((parent, _)) => { + let mut out = String::with_capacity(parent.len() + 1); + out.push_str(parent); + out.push('/'); + Href(out) + } + None => Href("/".to_owned()), + } + } + + fn base(base: &str) -> Result { + Url::parse(base).map_err(|e| HrefError::InvalidBase { + url: base.to_owned(), source: e, - })?; - Ok(normalise_path(&resolved)) -} - -pub fn normalise_path(url: &Url) -> Href { - let mut path = url.path().to_owned(); - if path.is_empty() { - path.push('/'); + }) } - let decoded = percent_decode(&path); - let reencoded = percent_encode_safe(&decoded); - Href(reencoded) -} -pub fn absolute_url(base: &str, href: &Href) -> Result { - join_absolute(base, href.as_str()) -} - -pub fn join_absolute(base: &str, href: &str) -> Result { - let base_url = Url::parse(base).map_err(|e| HrefError::InvalidBase { - url: base.to_owned(), - source: e, - })?; - let resolved = base_url.join(href).map_err(|e| HrefError::InvalidHref { - href: href.to_owned(), - base: base.to_owned(), - source: e, - })?; - Ok(resolved.to_string()) -} - -pub fn last_path_component(href: &Href) -> String { - let trimmed = href.as_str().trim_end_matches('/'); - let tail = trimmed.rsplit('/').next().unwrap_or(""); - percent_decode(tail) -} - -pub fn parent_collection(href: &Href) -> Href { - let trimmed = href.as_str().trim_end_matches('/'); - if let Some(slash) = trimmed.rfind('/') { - let parent = &trimmed[..=slash]; - Href(parent.to_owned()) - } else { - Href("/".to_owned()) - } -} - -fn percent_decode(s: &str) -> String { - let bytes = s.as_bytes(); - let mut buf: Vec = Vec::with_capacity(s.len()); - let mut i = 0; - while i < bytes.len() { - if bytes[i] == b'%' - && i + 2 < bytes.len() - && let Some(byte) = decode_pair(bytes[i + 1], bytes[i + 2]) - { - buf.push(byte); - i += 3; - continue; + fn decode(s: &str) -> String { + let mut buf: Vec = Vec::with_capacity(s.len()); + let mut rest = s.as_bytes(); + while let [first, tail @ ..] = rest { + if let (b'%', [high, low, after @ ..]) = (first, tail) + && let Some(byte) = decode_pair(*high, *low) + { + buf.push(byte); + rest = after; + continue; + } + buf.push(*first); + rest = tail; } - buf.push(bytes[i]); - i += 1; - } - match String::from_utf8(buf) { - Ok(decoded) => decoded, - Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(), - } -} - -fn percent_encode_safe(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - for byte in s.bytes() { - if is_safe(byte) { - out.push(byte as char); - } else { - out.push('%'); - out.push(upper_hex(byte >> 4)); - out.push(upper_hex(byte & 0x0f)); + match String::from_utf8(buf) { + Ok(decoded) => decoded, + Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(), } } - out -} - -fn is_safe(b: u8) -> bool { - matches!(b, - b'A'..=b'Z' - | b'a'..=b'z' - | b'0'..=b'9' - | b'-' | b'_' | b'.' | b'~' - | b'/' | b'@' - ) -} - -fn upper_hex(nibble: u8) -> char { - match nibble { - 0..=9 => (b'0' + nibble) as char, - 10..=15 => (b'A' + nibble - 10) as char, - _ => '0', - } } #[cfg(test)] mod tests { use super::*; + #[test] + fn absolute_urls_join_the_base_with_collection_and_item_hrefs() { + assert_eq!( + Href::join_absolute("https://x/dav/", "/dav/cal/u/d/").unwrap(), + "https://x/dav/cal/u/d/" + ); + let item = Href::parse("https://x/", "/dav/cal/u/d/event.ics").unwrap(); + assert_eq!( + item.to_absolute("https://x/dav/cal/u/d/").unwrap(), + "https://x/dav/cal/u/d/event.ics" + ); + } + #[test] fn relative_href_resolves_against_base() { - let h = normalise("https://dav.example.com/", "/dav/cal/user/default/").unwrap(); + let h = Href::parse("https://dav.example.com/", "/dav/cal/user/default/").unwrap(); assert_eq!(h.as_str(), "/dav/cal/user/default/"); } #[test] fn absolute_href_strips_scheme_and_host() { - let h = normalise( + let h = Href::parse( "https://dav.example.com/cal/", "https://dav.example.com/dav/cal/user/default/event-1.ics", ) @@ -181,20 +167,20 @@ mod tests { #[test] fn percent_encoding_canonicalised_to_uppercase_hex() { - let h = normalise("https://x/", "/dav/cal/user%2bwork/event%201.ics").unwrap(); + let h = Href::parse("https://x/", "/dav/cal/user%2bwork/event%201.ics").unwrap(); assert_eq!(h.as_str(), "/dav/cal/user%2Bwork/event%201.ics"); } #[test] fn at_sign_is_preserved_in_collection_paths() { - let h = normalise("https://x/", "/dav/cal/user@host/default/").unwrap(); + let h = Href::parse("https://x/", "/dav/cal/user@host/default/").unwrap(); assert_eq!(h.as_str(), "/dav/cal/user@host/default/"); } #[test] fn trailing_slash_preserved() { - let coll = normalise("https://x/", "/a/b/").unwrap(); - let item = normalise("https://x/", "/a/b").unwrap(); + let coll = Href::parse("https://x/", "/a/b/").unwrap(); + let item = Href::parse("https://x/", "/a/b").unwrap(); assert_eq!(coll.as_str(), "/a/b/"); assert_eq!(item.as_str(), "/a/b"); assert_ne!(coll, item); @@ -202,51 +188,51 @@ mod tests { #[test] fn href_equality_is_byte_exact_after_normalisation() { - let a = normalise("https://x/", "/Dav/Cal/User/Default/").unwrap(); - let b = normalise("https://x/", "/Dav/Cal/User/Default/").unwrap(); + let a = Href::parse("https://x/", "/Dav/Cal/User/Default/").unwrap(); + let b = Href::parse("https://x/", "/Dav/Cal/User/Default/").unwrap(); assert_eq!(a, b); } #[test] fn last_path_component_decodes() { let h = Href("/dav/cal/user/event%201.ics".to_owned()); - assert_eq!(last_path_component(&h), "event 1.ics"); + assert_eq!(h.last_component(), "event 1.ics"); } #[test] fn parent_collection_strips_trailing_slash() { let h = Href("/dav/cal/user/sub/".to_owned()); - assert_eq!(parent_collection(&h).as_str(), "/dav/cal/user/"); + assert_eq!(h.parent().as_str(), "/dav/cal/user/"); } #[test] fn parent_collection_of_file_is_parent_dir() { let h = Href("/dav/cal/user/file.ics".to_owned()); - assert_eq!(parent_collection(&h).as_str(), "/dav/cal/user/"); + assert_eq!(h.parent().as_str(), "/dav/cal/user/"); } #[test] fn parent_collection_of_root_is_root() { let h = Href("/".to_owned()); - assert_eq!(parent_collection(&h).as_str(), "/"); + assert_eq!(h.parent().as_str(), "/"); } #[test] fn empty_path_treated_as_root() { let url = Url::parse("https://dav.example.com").unwrap(); - let h = normalise_path(&url); + let h = Href::from_url(&url); assert_eq!(h.as_str(), "/"); } #[test] fn query_and_fragment_stripped() { - let h = normalise("https://x/", "/dav/cal/?q=1#frag").unwrap(); + let h = Href::parse("https://x/", "/dav/cal/?q=1#frag").unwrap(); assert_eq!(h.as_str(), "/dav/cal/"); } #[test] fn unicode_path_round_trips_through_utf8_percent_encoding() { - let h = normalise("https://x/", "/dav/cal/caf%C3%A9/").unwrap(); + let h = Href::parse("https://x/", "/dav/cal/caf%C3%A9/").unwrap(); assert_eq!(h.as_str(), "/dav/cal/caf%C3%A9/"); } } diff --git a/src/dav/mod.rs b/src/dav/mod.rs index 0981aef..4f0c0e1 100644 --- a/src/dav/mod.rs +++ b/src/dav/mod.rs @@ -6,6 +6,7 @@ pub mod client; pub mod discover; +pub mod error; pub mod href; pub mod parse; pub mod retry; diff --git a/src/dav/parse.rs b/src/dav/parse.rs index 198c976..a622abe 100644 --- a/src/dav/parse.rs +++ b/src/dav/parse.rs @@ -4,14 +4,17 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use std::io::BufRead; +use std::borrow::Cow; +use std::collections::HashSet; +use std::io::{self, BufRead, Read}; use quick_xml::NsReader; -use quick_xml::events::{BytesRef, Event}; +use quick_xml::events::{BytesStart, Event}; use quick_xml::name::ResolveResult; -use crate::dav::href::{Href, normalise}; -use crate::dav::xml::{NS_APPLE_ICAL, NS_CALDAV, NS_CARDDAV, NS_DAV}; +use crate::dav::href::{Href, HrefError}; +use crate::dav::retry::DavOutcome; +use crate::net::xml::Entity; #[derive(Debug, thiserror::Error)] pub enum ParseError { @@ -21,7 +24,7 @@ pub enum ParseError { Href { href: String, #[source] - source: crate::dav::href::HrefError, + source: HrefError, }, #[error("missing href in response")] MissingHref, @@ -57,6 +60,51 @@ pub struct ResourceProps { pub address_data: Option, } +impl ResourceProps { + fn merge(&mut self, src: ResourceProps) { + self.is_collection |= src.is_collection; + self.is_calendar |= src.is_calendar; + self.is_addressbook |= src.is_addressbook; + ResourceProps::overwrite_some(&mut self.displayname, src.displayname); + ResourceProps::overwrite_some(&mut self.current_user_principal, src.current_user_principal); + ResourceProps::overwrite_some(&mut self.calendar_home_set, src.calendar_home_set); + ResourceProps::overwrite_some(&mut self.addressbook_home_set, src.addressbook_home_set); + ResourceProps::overwrite_some(&mut self.etag, src.etag); + ResourceProps::overwrite_some(&mut self.content_type, src.content_type); + ResourceProps::overwrite_some(&mut self.last_modified, src.last_modified); + ResourceProps::overwrite_some(&mut self.creation_date, src.creation_date); + ResourceProps::overwrite_some(&mut self.content_length, src.content_length); + ResourceProps::overwrite_some(&mut self.calendar_description, src.calendar_description); + ResourceProps::overwrite_some( + &mut self.addressbook_description, + src.addressbook_description, + ); + ResourceProps::overwrite_some(&mut self.calendar_color, src.calendar_color); + ResourceProps::overwrite_some(&mut self.calendar_order, src.calendar_order); + ResourceProps::overwrite_some(&mut self.calendar_timezone, src.calendar_timezone); + ResourceProps::overwrite_some(&mut self.calendar_data, src.calendar_data); + ResourceProps::overwrite_some(&mut self.address_data, src.address_data); + if !src.supported_components.is_empty() { + self.supported_components = src.supported_components; + } + } + + fn overwrite_some(target: &mut Option, src: Option) { + if src.is_some() { + *target = src; + } + } + + fn mark_resourcetype(&mut self, element: Element) { + match element { + Element::Collection => self.is_collection = true, + Element::Calendar => self.is_calendar = true, + Element::Addressbook => self.is_addressbook = true, + _ => {} + } + } +} + #[derive(Debug, Clone)] pub struct DavResponse { pub href: Href, @@ -65,8 +113,19 @@ pub struct DavResponse { pub propstat_errors: Vec, } +impl DavResponse { + pub fn is_gone(&self) -> bool { + self.status.is_some_and(DavOutcome::is_gone) + || self + .propstat_errors + .iter() + .copied() + .any(DavOutcome::is_gone) + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum NsKind { +enum Namespace { Dav, Caldav, Carddav, @@ -74,545 +133,438 @@ enum NsKind { Other, } -fn classify_ns(ns: &ResolveResult<'_>) -> NsKind { - match ns { - ResolveResult::Bound(prefix) => { - let p = prefix.as_ref(); - if p == NS_DAV { - NsKind::Dav - } else if p == NS_CALDAV { - NsKind::Caldav - } else if p == NS_CARDDAV { - NsKind::Carddav - } else if p == NS_APPLE_ICAL { - NsKind::Apple - } else { - NsKind::Other - } - } - _ => NsKind::Other, +impl Namespace { + fn of(resolved: &ResolveResult<'_>) -> Namespace { + let ResolveResult::Bound(ns) = resolved else { + return Namespace::Other; + }; + hashify::map!(ns.as_ref().as_bytes(), Namespace, + "DAV:" => Namespace::Dav, + "urn:ietf:params:xml:ns:caldav" => Namespace::Caldav, + "urn:ietf:params:xml:ns:carddav" => Namespace::Carddav, + "http://apple.com/ns/ical/" => Namespace::Apple, + ) + .copied() + .unwrap_or(Namespace::Other) } } -fn is(ns: NsKind, local: &str, expected_ns: NsKind, target: &str) -> bool { - ns == expected_ns && local.eq_ignore_ascii_case(target) +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Element { + Response, + Href, + Status, + Propstat, + Prop, + ResourceType, + Collection, + DisplayName, + CurrentUserPrincipal, + GetEtag, + GetContentType, + GetLastModified, + CreationDate, + GetContentLength, + Calendar, + CalendarHomeSet, + CalendarDescription, + CalendarTimezone, + SupportedCalendarComponentSet, + Comp, + CalendarData, + Addressbook, + AddressbookHomeSet, + AddressbookDescription, + AddressData, + CalendarColor, + CalendarOrder, + Other, +} + +impl Element { + fn of(ns: Namespace, local: &str) -> Element { + let Some(&(expected, element)) = hashify::map_ignore_case!(local.as_bytes(), (Namespace, Element), + "response" => (Namespace::Dav, Element::Response), + "href" => (Namespace::Dav, Element::Href), + "status" => (Namespace::Dav, Element::Status), + "propstat" => (Namespace::Dav, Element::Propstat), + "prop" => (Namespace::Dav, Element::Prop), + "resourcetype" => (Namespace::Dav, Element::ResourceType), + "collection" => (Namespace::Dav, Element::Collection), + "displayname" => (Namespace::Dav, Element::DisplayName), + "current-user-principal" => (Namespace::Dav, Element::CurrentUserPrincipal), + "getetag" => (Namespace::Dav, Element::GetEtag), + "getcontenttype" => (Namespace::Dav, Element::GetContentType), + "getlastmodified" => (Namespace::Dav, Element::GetLastModified), + "creationdate" => (Namespace::Dav, Element::CreationDate), + "getcontentlength" => (Namespace::Dav, Element::GetContentLength), + "calendar" => (Namespace::Caldav, Element::Calendar), + "calendar-home-set" => (Namespace::Caldav, Element::CalendarHomeSet), + "calendar-description" => (Namespace::Caldav, Element::CalendarDescription), + "calendar-timezone" => (Namespace::Caldav, Element::CalendarTimezone), + "supported-calendar-component-set" => (Namespace::Caldav, Element::SupportedCalendarComponentSet), + "comp" => (Namespace::Caldav, Element::Comp), + "calendar-data" => (Namespace::Caldav, Element::CalendarData), + "addressbook" => (Namespace::Carddav, Element::Addressbook), + "addressbook-home-set" => (Namespace::Carddav, Element::AddressbookHomeSet), + "addressbook-description" => (Namespace::Carddav, Element::AddressbookDescription), + "address-data" => (Namespace::Carddav, Element::AddressData), + "calendar-color" => (Namespace::Apple, Element::CalendarColor), + "calendar-order" => (Namespace::Apple, Element::CalendarOrder), + ) else { + return Element::Other; + }; + if expected == ns { + element + } else { + Element::Other + } + } } #[derive(Debug)] enum Step { - StartElement { - ns: NsKind, - local: String, - attrs: Vec<(String, String)>, - }, - EmptyElement { - ns: NsKind, - local: String, - attrs: Vec<(String, String)>, - }, - EndElement { - ns: NsKind, - local: String, - }, + Start(Element, Option), + Empty(Element, Option), + End(Element), Text(String), - CData(String), - Entity(char), + Char(char), Other, Eof, } -fn resolve_entity(g: &BytesRef) -> Option { - if let Ok(Some(c)) = g.resolve_char_ref() { - return Some(c); - } - match g.as_ref() { - "amp" => Some('&'), - "lt" => Some('<'), - "gt" => Some('>'), - "quot" => Some('"'), - "apos" => Some('\''), - _ => None, +impl Step { + fn opening(ns: Namespace, e: &BytesStart<'_>) -> (Element, Option) { + let element = Element::of(ns, e.local_name().as_ref()); + let name = match element { + Element::Comp => e + .attributes() + .flatten() + .find(|a| a.key.as_ref() == "name") + .map(|a| a.value.into_owned()), + _ => None, + }; + (element, name) } } -fn next_step(xml: &mut NsReader, buf: &mut Vec) -> Result { - buf.clear(); - let (ns, ev) = xml.read_resolved_event_into(buf)?; - let ns_kind = classify_ns(&ns); - Ok(match ev { - Event::Start(e) => { - let local = e.local_name().as_ref().to_owned(); - let attrs = e - .attributes() - .flatten() - .map(|a| (a.key.as_ref().to_owned(), a.value.into_owned())) - .collect(); - Step::StartElement { - ns: ns_kind, - local, - attrs, - } - } - Event::Empty(e) => { - let local = e.local_name().as_ref().to_owned(); - let attrs = e - .attributes() - .flatten() - .map(|a| (a.key.as_ref().to_owned(), a.value.into_owned())) - .collect(); - Step::EmptyElement { - ns: ns_kind, - local, - attrs, - } - } - Event::End(e) => Step::EndElement { - ns: ns_kind, - local: e.local_name().as_ref().to_owned(), - }, - Event::Text(t) => Step::Text(t.into_inner().into_owned()), - Event::CData(cd) => Step::CData(cd.into_inner().into_owned()), - Event::GeneralRef(g) => match resolve_entity(&g) { - Some(c) => Step::Entity(c), - None => Step::Other, - }, - Event::Eof => Step::Eof, - _ => Step::Other, - }) +pub struct MultistatusParser<'b, R> { + xml: NsReader, + buf: Vec, + base_url: &'b str, } -pub fn parse_multistatus( - reader: R, - base_url: &str, -) -> Result, ParseError> { - let mut xml = NsReader::from_reader(reader); - let mut buf = Vec::new(); - let mut out: Vec = Vec::new(); - let mut seen: std::collections::HashSet = std::collections::HashSet::new(); - - loop { - let step = next_step(&mut xml, &mut buf)?; - match step { - Step::StartElement { ns, local, .. } if is(ns, &local, NsKind::Dav, "response") => { - if let Some(r) = parse_response(&mut xml, base_url)? - && seen.insert(r.href.as_str().to_owned()) - { - out.push(r); +impl<'b, R: BufRead> MultistatusParser<'b, R> { + pub fn parse(reader: R, base_url: &'b str) -> Result, ParseError> { + let mut parser = MultistatusParser { + xml: NsReader::from_reader(reader), + buf: Vec::new(), + base_url, + }; + let mut out: Vec = Vec::new(); + let mut seen: HashSet = HashSet::new(); + loop { + match parser.next_step()? { + Step::Start(Element::Response, _) => { + let response = parser.response()?; + if seen.insert(response.href.as_str().to_owned()) { + out.push(response); + } } + Step::Eof => break, + _ => {} } - Step::Eof => break, - _ => {} } + Ok(out) } - Ok(out) -} -fn parse_response( - xml: &mut NsReader, - base_url: &str, -) -> Result, ParseError> { - let mut buf = Vec::new(); - let mut href: Option = None; - let mut status: Option = None; - let mut props = ResourceProps::default(); - let mut propstat_errors: Vec = Vec::new(); + fn next_step(&mut self) -> Result { + self.buf.clear(); + let (resolved, event) = self.xml.read_resolved_event_into(&mut self.buf)?; + let ns = Namespace::of(&resolved); + Ok(match event { + Event::Start(e) => { + let (element, name) = Step::opening(ns, &e); + Step::Start(element, name) + } + Event::Empty(e) => { + let (element, name) = Step::opening(ns, &e); + Step::Empty(element, name) + } + Event::End(e) => Step::End(Element::of(ns, e.local_name().as_ref())), + Event::Text(t) => Step::Text(t.into_inner().into_owned()), + Event::CData(cd) => Step::Text(cd.into_inner().into_owned()), + Event::GeneralRef(g) => Entity::from_ref(&g).map_or(Step::Other, Step::Char), + Event::Eof => Step::Eof, + _ => Step::Other, + }) + } - loop { - let step = next_step(xml, &mut buf)?; - match step { - Step::StartElement { ns, local, .. } => { - if is(ns, &local, NsKind::Dav, "href") { - let text = read_token(xml)?; + fn unexpected_eof(context: &str) -> ParseError { + ParseError::Xml(format!("unexpected EOF {context}")) + } + + fn response(&mut self) -> Result { + let mut href: Option = None; + let mut status: Option = None; + let mut props = ResourceProps::default(); + let mut propstat_errors: Vec = Vec::new(); + loop { + match self.next_step()? { + Step::Start(Element::Href, _) => { + let text = self.token()?; if href.is_none() { href = Some(text); } - } else if is(ns, &local, NsKind::Dav, "status") { - let text = read_token(xml)?; - status = parse_http_status(&text); - } else if is(ns, &local, NsKind::Dav, "propstat") { - let (block_status, block_props) = parse_propstat(xml)?; - match block_status { - Some(s) if (200..300).contains(&s) => merge_props(&mut props, block_props), - Some(s) => propstat_errors.push(s), - None => {} - } - } else { - skip_element(xml, &local)?; + } + Step::Start(Element::Status, _) => { + status = MultistatusParser::::http_status(&self.token()?); + } + Step::Start(Element::Propstat, _) => match self.propstat()? { + (Some(s), block) if (200..300).contains(&s) => props.merge(block), + (Some(s), _) => propstat_errors.push(s), + (None, _) => {} + }, + Step::Start(..) => self.skip()?, + Step::End(Element::Response) => break, + Step::Eof => return Err(MultistatusParser::::unexpected_eof("in ")), + _ => {} + } + } + let raw_href = href.ok_or(ParseError::MissingHref)?; + let href = Href::parse(self.base_url, &raw_href).map_err(|source| ParseError::Href { + href: raw_href, + source, + })?; + Ok(DavResponse { + href, + status, + props, + propstat_errors, + }) + } + + fn propstat(&mut self) -> Result<(Option, ResourceProps), ParseError> { + let mut status: Option = None; + let mut props = ResourceProps::default(); + loop { + match self.next_step()? { + Step::Start(Element::Status, _) => { + status = MultistatusParser::::http_status(&self.token()?); + } + Step::Start(Element::Prop, _) => self.prop_block(&mut props)?, + Step::Start(..) => self.skip()?, + Step::End(Element::Propstat) => break, + Step::Eof => return Err(MultistatusParser::::unexpected_eof("in ")), + _ => {} + } + } + Ok((status, props)) + } + + fn prop_block(&mut self, props: &mut ResourceProps) -> Result<(), ParseError> { + loop { + match self.next_step()? { + Step::Start(element, _) => self.prop_element(props, element)?, + Step::End(Element::Prop) => break, + Step::Eof => return Err(MultistatusParser::::unexpected_eof("in ")), + _ => {} + } + } + Ok(()) + } + + fn prop_element( + &mut self, + props: &mut ResourceProps, + element: Element, + ) -> Result<(), ParseError> { + match element { + Element::ResourceType => self.resourcetype(props)?, + Element::DisplayName => props.displayname = Some(self.token()?), + Element::CurrentUserPrincipal => { + props.current_user_principal = self.first_href(element)?; + } + Element::CalendarHomeSet => props.calendar_home_set = self.first_href(element)?, + Element::AddressbookHomeSet => props.addressbook_home_set = self.first_href(element)?, + Element::GetEtag => { + let raw = self.token()?; + if !raw.is_empty() { + props.etag = Some(raw); } } - Step::EndElement { ns, local } if is(ns, &local, NsKind::Dav, "response") => break, - Step::Eof => return Err(ParseError::Xml("unexpected EOF in ".into())), - _ => {} + Element::GetContentType => props.content_type = Some(self.token()?), + Element::GetLastModified => props.last_modified = Some(self.token()?), + Element::CreationDate => props.creation_date = Some(self.token()?), + Element::GetContentLength => props.content_length = self.token()?.parse::().ok(), + Element::CalendarDescription => props.calendar_description = Some(self.token()?), + Element::AddressbookDescription => props.addressbook_description = Some(self.token()?), + Element::CalendarColor => props.calendar_color = Some(self.token()?), + Element::CalendarOrder => props.calendar_order = self.token()?.parse::().ok(), + Element::CalendarTimezone => props.calendar_timezone = Some(self.token()?), + Element::SupportedCalendarComponentSet => self.supported_components(props)?, + Element::CalendarData => props.calendar_data = Some(self.text()?), + Element::AddressData => props.address_data = Some(self.text()?), + _ => self.skip()?, } + Ok(()) } - let raw_href = href.ok_or(ParseError::MissingHref)?; - let normalised = normalise(base_url, &raw_href).map_err(|e| ParseError::Href { - href: raw_href, - source: e, - })?; - Ok(Some(DavResponse { - href: normalised, - status, - props, - propstat_errors, - })) -} - -fn parse_propstat( - xml: &mut NsReader, -) -> Result<(Option, ResourceProps), ParseError> { - let mut buf = Vec::new(); - let mut status: Option = None; - let mut props = ResourceProps::default(); - loop { - let step = next_step(xml, &mut buf)?; - match step { - Step::StartElement { ns, local, .. } => { - if is(ns, &local, NsKind::Dav, "status") { - let text = read_token(xml)?; - status = parse_http_status(&text); - } else if is(ns, &local, NsKind::Dav, "prop") { - parse_prop_block(xml, &mut props)?; - } else { - skip_element(xml, &local)?; + fn resourcetype(&mut self, props: &mut ResourceProps) -> Result<(), ParseError> { + loop { + match self.next_step()? { + Step::Start(element, _) => { + props.mark_resourcetype(element); + self.skip()?; } - } - Step::EndElement { ns, local } if is(ns, &local, NsKind::Dav, "propstat") => break, - Step::Eof => return Err(ParseError::Xml("unexpected EOF in ".into())), - _ => {} - } - } - Ok((status, props)) -} - -fn parse_prop_block( - xml: &mut NsReader, - props: &mut ResourceProps, -) -> Result<(), ParseError> { - let mut buf = Vec::new(); - loop { - let step = next_step(xml, &mut buf)?; - match step { - Step::StartElement { ns, local, .. } => { - handle_prop_element(xml, props, ns, &local)?; - } - Step::EmptyElement { .. } => {} - Step::EndElement { ns, local } if is(ns, &local, NsKind::Dav, "prop") => break, - Step::Eof => return Err(ParseError::Xml("unexpected EOF in ".into())), - _ => {} - } - } - Ok(()) -} - -fn handle_prop_element( - xml: &mut NsReader, - props: &mut ResourceProps, - ns: NsKind, - local: &str, -) -> Result<(), ParseError> { - if is(ns, local, NsKind::Dav, "resourcetype") { - consume_resourcetype(xml, props)?; - } else if is(ns, local, NsKind::Dav, "displayname") { - props.displayname = Some(read_token(xml)?); - } else if is(ns, local, NsKind::Dav, "current-user-principal") { - props.current_user_principal = read_first_href(xml, "current-user-principal")?; - } else if is(ns, local, NsKind::Caldav, "calendar-home-set") { - props.calendar_home_set = read_first_href(xml, "calendar-home-set")?; - } else if is(ns, local, NsKind::Carddav, "addressbook-home-set") { - props.addressbook_home_set = read_first_href(xml, "addressbook-home-set")?; - } else if is(ns, local, NsKind::Dav, "getetag") { - let raw = read_token(xml)?; - if !raw.is_empty() { - props.etag = Some(raw); - } - } else if is(ns, local, NsKind::Dav, "getcontenttype") { - props.content_type = Some(read_token(xml)?); - } else if is(ns, local, NsKind::Dav, "getlastmodified") { - props.last_modified = Some(read_token(xml)?); - } else if is(ns, local, NsKind::Dav, "creationdate") { - props.creation_date = Some(read_token(xml)?); - } else if is(ns, local, NsKind::Dav, "getcontentlength") { - let raw = read_token(xml)?; - props.content_length = raw.parse::().ok(); - } else if is(ns, local, NsKind::Caldav, "calendar-description") { - props.calendar_description = Some(read_token(xml)?); - } else if is(ns, local, NsKind::Carddav, "addressbook-description") { - props.addressbook_description = Some(read_token(xml)?); - } else if is(ns, local, NsKind::Apple, "calendar-color") { - props.calendar_color = Some(read_token(xml)?); - } else if is(ns, local, NsKind::Apple, "calendar-order") { - let raw = read_token(xml)?; - props.calendar_order = raw.parse::().ok(); - } else if is(ns, local, NsKind::Caldav, "calendar-timezone") { - props.calendar_timezone = Some(read_token(xml)?); - } else if is( - ns, - local, - NsKind::Caldav, - "supported-calendar-component-set", - ) { - consume_supported_components(xml, props)?; - } else if is(ns, local, NsKind::Caldav, "calendar-data") { - props.calendar_data = Some(read_text(xml)?); - } else if is(ns, local, NsKind::Carddav, "address-data") { - props.address_data = Some(read_text(xml)?); - } else { - skip_element(xml, local)?; - } - Ok(()) -} - -fn consume_resourcetype( - xml: &mut NsReader, - props: &mut ResourceProps, -) -> Result<(), ParseError> { - let mut buf = Vec::new(); - loop { - let step = next_step(xml, &mut buf)?; - match step { - Step::StartElement { ns, local, .. } => { - mark_resourcetype(ns, &local, props); - skip_element(xml, &local)?; - } - Step::EmptyElement { ns, local, .. } => { - mark_resourcetype(ns, &local, props); - } - Step::EndElement { ns, local } if is(ns, &local, NsKind::Dav, "resourcetype") => break, - Step::Eof => { - return Err(ParseError::Xml("unexpected EOF in ".into())); - } - _ => {} - } - } - Ok(()) -} - -fn mark_resourcetype(ns: NsKind, local: &str, props: &mut ResourceProps) { - if is(ns, local, NsKind::Dav, "collection") { - props.is_collection = true; - } else if is(ns, local, NsKind::Caldav, "calendar") { - props.is_calendar = true; - } else if is(ns, local, NsKind::Carddav, "addressbook") { - props.is_addressbook = true; - } -} - -fn consume_supported_components( - xml: &mut NsReader, - props: &mut ResourceProps, -) -> Result<(), ParseError> { - let mut buf = Vec::new(); - loop { - let step = next_step(xml, &mut buf)?; - match step { - Step::StartElement { - ns, - local, - ref attrs, - } if is(ns, &local, NsKind::Caldav, "comp") => { - if let Some(name) = attr_lookup(attrs, "name") { - props.supported_components.push(name); + Step::Empty(element, _) => props.mark_resourcetype(element), + Step::End(Element::ResourceType) => break, + Step::Eof => { + return Err(MultistatusParser::::unexpected_eof("in ")); } - skip_element(xml, &local)?; + _ => {} } - Step::EmptyElement { - ns, - local, - ref attrs, - } if is(ns, &local, NsKind::Caldav, "comp") => { - if let Some(name) = attr_lookup(attrs, "name") { - props.supported_components.push(name); + } + Ok(()) + } + + fn supported_components(&mut self, props: &mut ResourceProps) -> Result<(), ParseError> { + loop { + match self.next_step()? { + Step::Start(Element::Comp, name) => { + props.supported_components.extend(name); + self.skip()?; } - } - Step::EndElement { ns, local } - if is( - ns, - &local, - NsKind::Caldav, - "supported-calendar-component-set", - ) => - { - break; - } - Step::Eof => { - return Err(ParseError::Xml( - "unexpected EOF in ".into(), - )); - } - _ => {} - } - } - Ok(()) -} - -fn attr_lookup(attrs: &[(String, String)], key: &str) -> Option { - attrs.iter().find(|(k, _)| k == key).map(|(_, v)| v.clone()) -} - -fn read_text(xml: &mut NsReader) -> Result { - let mut buf = Vec::new(); - let mut out = String::new(); - loop { - let step = next_step(xml, &mut buf)?; - match step { - Step::Text(s) | Step::CData(s) => out.push_str(&s), - Step::Entity(c) => out.push(c), - Step::StartElement { local, .. } => skip_element(xml, &local)?, - Step::EndElement { .. } | Step::EmptyElement { .. } => break, - Step::Eof => return Err(ParseError::Xml("unexpected EOF reading text".into())), - _ => {} - } - } - Ok(out) -} - -fn read_token(xml: &mut NsReader) -> Result { - Ok(read_text(xml)?.trim().to_owned()) -} - -fn read_first_href( - xml: &mut NsReader, - closing_local: &str, -) -> Result, ParseError> { - let mut buf = Vec::new(); - let mut out: Option = None; - loop { - let step = next_step(xml, &mut buf)?; - match step { - Step::StartElement { ns, local, .. } => { - if out.is_none() && is(ns, &local, NsKind::Dav, "href") { - out = Some(read_token(xml)?); - } else { - skip_element(xml, &local)?; + Step::Empty(Element::Comp, name) => props.supported_components.extend(name), + Step::End(Element::SupportedCalendarComponentSet) => break, + Step::Eof => { + return Err(MultistatusParser::::unexpected_eof( + "in ", + )); } + _ => {} } - Step::EmptyElement { .. } => {} - Step::EndElement { local, .. } if local.eq_ignore_ascii_case(closing_local) => { - break; + } + Ok(()) + } + + fn text(&mut self) -> Result { + let mut out = String::new(); + loop { + match self.next_step()? { + Step::Text(s) => out.push_str(&s), + Step::Char(c) => out.push(c), + Step::Start(..) => self.skip()?, + Step::End(_) | Step::Empty(..) => break, + Step::Eof => return Err(MultistatusParser::::unexpected_eof("reading text")), + Step::Other => {} } - Step::Eof => return Err(ParseError::Xml("unexpected EOF reading href".into())), - _ => {} + } + Ok(out) + } + + fn token(&mut self) -> Result { + let text = self.text()?; + let trimmed = text.trim(); + Ok(if trimmed.len() == text.len() { + text + } else { + trimmed.to_owned() + }) + } + + fn first_href(&mut self, closing: Element) -> Result, ParseError> { + let mut out: Option = None; + loop { + match self.next_step()? { + Step::Start(Element::Href, _) if out.is_none() => out = Some(self.token()?), + Step::Start(..) => self.skip()?, + Step::End(element) if element == closing => break, + Step::Eof => return Err(MultistatusParser::::unexpected_eof("reading href")), + _ => {} + } + } + Ok(out) + } + + fn skip(&mut self) -> Result<(), ParseError> { + let mut depth: u32 = 1; + while depth > 0 { + match self.next_step()? { + Step::Start(..) => depth += 1, + Step::End(_) => depth -= 1, + Step::Eof => return Err(MultistatusParser::::unexpected_eof("in skip")), + _ => {} + } + } + Ok(()) + } + + fn http_status(text: &str) -> Option { + let mut parts = text.split_ascii_whitespace(); + let _version = parts.next()?; + parts.next()?.parse::().ok() + } +} + +pub struct ControlBytes; + +impl ControlBytes { + pub fn is_illegal(b: u8) -> bool { + matches!(b, 0x00..=0x08 | 0x0b | 0x0c | 0x0e..=0x1f) + } + + pub fn strip(body: &[u8]) -> Cow<'_, [u8]> { + if body.iter().any(|&b| ControlBytes::is_illegal(b)) { + Cow::Owned( + body.iter() + .copied() + .filter(|&b| !ControlBytes::is_illegal(b)) + .collect(), + ) + } else { + Cow::Borrowed(body) } } - Ok(out) -} -fn skip_element(xml: &mut NsReader, target: &str) -> Result<(), ParseError> { - let mut buf = Vec::new(); - let mut depth: i32 = 1; - while depth > 0 { - let step = next_step(xml, &mut buf)?; - match step { - Step::StartElement { .. } => depth += 1, - Step::EndElement { local, .. } => { - depth -= 1; - if depth == 0 && !local.eq_ignore_ascii_case(target) { - return Ok(()); - } + fn compact(filled: &mut [u8]) -> usize { + let mut write = 0; + let mut read = 0; + while let Some(rest) = filled.get(read..) { + let run = rest + .iter() + .position(|&b| ControlBytes::is_illegal(b)) + .unwrap_or(rest.len()); + if write != read { + filled.copy_within(read..read + run, write); } - Step::Eof => return Err(ParseError::Xml("unexpected EOF in skip".into())), - _ => {} + write += run; + read += run + 1; } - } - Ok(()) -} - -fn merge_props(target: &mut ResourceProps, src: ResourceProps) { - if src.is_collection { - target.is_collection = true; - } - if src.is_calendar { - target.is_calendar = true; - } - if src.is_addressbook { - target.is_addressbook = true; - } - overwrite_some(&mut target.displayname, src.displayname); - overwrite_some( - &mut target.current_user_principal, - src.current_user_principal, - ); - overwrite_some(&mut target.calendar_home_set, src.calendar_home_set); - overwrite_some(&mut target.addressbook_home_set, src.addressbook_home_set); - overwrite_some(&mut target.etag, src.etag); - overwrite_some(&mut target.content_type, src.content_type); - overwrite_some(&mut target.last_modified, src.last_modified); - overwrite_some(&mut target.creation_date, src.creation_date); - overwrite_some(&mut target.content_length, src.content_length); - overwrite_some(&mut target.calendar_description, src.calendar_description); - overwrite_some( - &mut target.addressbook_description, - src.addressbook_description, - ); - overwrite_some(&mut target.calendar_color, src.calendar_color); - overwrite_some(&mut target.calendar_order, src.calendar_order); - overwrite_some(&mut target.calendar_timezone, src.calendar_timezone); - overwrite_some(&mut target.calendar_data, src.calendar_data); - overwrite_some(&mut target.address_data, src.address_data); - if !src.supported_components.is_empty() { - target.supported_components = src.supported_components; + write } } -fn overwrite_some(target: &mut Option, src: Option) { - if src.is_some() { - *target = src; - } -} - -fn parse_http_status(text: &str) -> Option { - let mut parts = text.split_ascii_whitespace(); - let _version = parts.next()?; - let code = parts.next()?; - code.parse::().ok() -} - -pub fn strip_ascii_control_chars(body: &[u8]) -> std::borrow::Cow<'_, [u8]> { - if body.iter().any(|&b| is_xml_illegal_control(b)) { - let cleaned: Vec = body - .iter() - .copied() - .filter(|b| !is_xml_illegal_control(*b)) - .collect(); - std::borrow::Cow::Owned(cleaned) - } else { - std::borrow::Cow::Borrowed(body) - } -} - -pub fn is_xml_illegal_control(b: u8) -> bool { - matches!(b, 0x00..=0x08 | 0x0b | 0x0c | 0x0e..=0x1f) -} - -pub struct ControlStrippingReader { +pub struct ControlStrippingReader { inner: R, } -impl ControlStrippingReader { +impl ControlStrippingReader { pub fn new(inner: R) -> Self { - Self { inner } + ControlStrippingReader { inner } } } -impl std::io::Read for ControlStrippingReader { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { +impl Read for ControlStrippingReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { loop { let n = self.inner.read(buf)?; - if n == 0 { + let Some(filled @ [_, ..]) = buf.get_mut(..n) else { return Ok(0); - } - let mut write = 0; - for read in 0..n { - let b = buf[read]; - if !is_xml_illegal_control(b) { - buf[write] = b; - write += 1; - } - } - if write > 0 { - return Ok(write); + }; + let kept = ControlBytes::compact(filled); + if kept > 0 { + return Ok(kept); } } } @@ -623,8 +575,35 @@ mod tests { use super::*; use std::io::Cursor; + fn gone_probe(status: Option, propstat_errors: Vec) -> DavResponse { + DavResponse { + href: Href::from_normalised("/dav/cal/u/d/e.ics".to_owned()), + status, + props: ResourceProps::default(), + propstat_errors, + } + } + + #[test] + fn response_level_404_or_410_is_gone() { + assert!(gone_probe(Some(404), vec![]).is_gone()); + assert!(gone_probe(Some(410), vec![]).is_gone()); + } + + #[test] + fn per_propstat_404_or_410_is_gone() { + assert!(gone_probe(None, vec![404]).is_gone()); + assert!(gone_probe(None, vec![200, 410]).is_gone()); + } + + #[test] + fn other_propstat_errors_are_not_gone() { + assert!(!gone_probe(None, vec![403]).is_gone()); + assert!(!gone_probe(None, vec![]).is_gone()); + } + fn parse(body: &str) -> Vec { - parse_multistatus(Cursor::new(body), "https://x/").expect("parse") + MultistatusParser::parse(Cursor::new(body), "https://x/").expect("parse") } #[test] @@ -808,17 +787,29 @@ END:VCALENDAR #[test] fn strip_control_chars_removes_low_bytes() { let raw = b"\x01"; - let cleaned = strip_ascii_control_chars(raw); + let cleaned = ControlBytes::strip(raw); assert!(!cleaned.contains(&0x01)); } + #[test] + fn stripping_reader_drops_control_bytes_across_reads() { + let raw = b"\x01ab\x02\x03c\x1fd\x01"; + for chunk in [1, 2, 3, 64] { + let mut reader = + ControlStrippingReader::new(io::BufReader::with_capacity(chunk, &raw[..])); + let mut out = Vec::new(); + reader.read_to_end(&mut out).expect("read"); + assert_eq!(out, b"abcd", "chunk {chunk}"); + } + } + #[test] fn strip_control_chars_leaves_tab_newline_cr() { let raw = b"\t\n\r"; - let cleaned = strip_ascii_control_chars(raw); + let cleaned = ControlBytes::strip(raw); match cleaned { - std::borrow::Cow::Borrowed(_) => {} - std::borrow::Cow::Owned(_) => panic!("should not have allocated"), + Cow::Borrowed(_) => {} + Cow::Owned(_) => panic!("should not have allocated"), } } @@ -878,7 +869,7 @@ END:VCALENDAR } #[test] - fn merge_props_last_propstat_wins_for_duplicate_keys() { + fn last_propstat_wins_for_duplicate_keys() { let body = r#" diff --git a/src/dav/retry.rs b/src/dav/retry.rs index b33cc86..437029d 100644 --- a/src/dav/retry.rs +++ b/src/dav/retry.rs @@ -6,7 +6,17 @@ use serde_json::Value; -use crate::jmap::retry::Disposition; +const BAD_REQUEST: u16 = 400; +const UNAUTHORIZED: u16 = 401; +const FORBIDDEN: u16 = 403; +const NOT_FOUND: u16 = 404; +const GONE: u16 = 410; +const LOCKED: u16 = 423; +const TOO_MANY_REQUESTS: u16 = 429; +const BAD_GATEWAY: u16 = 502; +const SERVICE_UNAVAILABLE: u16 = 503; +const GATEWAY_TIMEOUT: u16 = 504; +const INSUFFICIENT_STORAGE: u16 = 507; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DavOutcome { @@ -17,50 +27,42 @@ pub enum DavOutcome { Fatal, } -pub fn classify(status: u16, body: &[u8]) -> DavOutcome { - if (200..300).contains(&status) { - return DavOutcome::Success; - } - if status == 401 { - return DavOutcome::Auth; - } - if status == 403 { - if is_google_usage_limit(body) { - return DavOutcome::Retryable; +impl DavOutcome { + pub fn classify(status: u16, body: &[u8]) -> DavOutcome { + match status { + 200..=299 => DavOutcome::Success, + UNAUTHORIZED => DavOutcome::Auth, + FORBIDDEN if DavOutcome::is_google_usage_limit(body) => DavOutcome::Retryable, + FORBIDDEN => DavOutcome::Auth, + NOT_FOUND | GONE => DavOutcome::Vanished, + LOCKED | TOO_MANY_REQUESTS | BAD_GATEWAY | SERVICE_UNAVAILABLE | GATEWAY_TIMEOUT + | INSUFFICIENT_STORAGE => DavOutcome::Retryable, + _ => DavOutcome::Fatal, } - return DavOutcome::Auth; } - if status == 404 || status == 410 { - return DavOutcome::Vanished; - } - if matches!(status, 423 | 429 | 502 | 503 | 504 | 507) { - return DavOutcome::Retryable; - } - match crate::jmap::retry::classify_http_status(status) { - Disposition::Retryable => DavOutcome::Retryable, - Disposition::Fatal => DavOutcome::Fatal, - } -} -fn is_google_usage_limit(body: &[u8]) -> bool { - let value: Value = match serde_json::from_slice(body) { - Ok(v) => v, - Err(_) => return false, - }; - let errors = value - .get("error") - .and_then(|e| e.get("errors")) - .and_then(Value::as_array); - let Some(errors) = errors else { - return false; - }; - errors.iter().any(|entry| { - entry - .get("domain") - .and_then(Value::as_str) - .map(|d| d == "usageLimits") - .unwrap_or(false) - }) + pub fn is_gone(status: u16) -> bool { + matches!(status, NOT_FOUND | GONE) + } + + pub fn is_error(status: u16) -> bool { + status >= BAD_REQUEST + } + + fn is_google_usage_limit(body: &[u8]) -> bool { + let Ok(value) = serde_json::from_slice::(body) else { + return false; + }; + value + .get("error") + .and_then(|e| e.get("errors")) + .and_then(Value::as_array) + .is_some_and(|errors| { + errors + .iter() + .any(|entry| entry.get("domain").and_then(Value::as_str) == Some("usageLimits")) + }) + } } #[cfg(test)] @@ -69,48 +71,48 @@ mod tests { #[test] fn success_range() { - assert_eq!(classify(200, b""), DavOutcome::Success); - assert_eq!(classify(207, b""), DavOutcome::Success); - assert_eq!(classify(299, b""), DavOutcome::Success); + assert_eq!(DavOutcome::classify(200, b""), DavOutcome::Success); + assert_eq!(DavOutcome::classify(207, b""), DavOutcome::Success); + assert_eq!(DavOutcome::classify(299, b""), DavOutcome::Success); } #[test] fn auth_status() { - assert_eq!(classify(401, b""), DavOutcome::Auth); - assert_eq!(classify(403, b""), DavOutcome::Auth); + assert_eq!(DavOutcome::classify(401, b""), DavOutcome::Auth); + assert_eq!(DavOutcome::classify(403, b""), DavOutcome::Auth); } #[test] fn vanished_status() { - assert_eq!(classify(404, b""), DavOutcome::Vanished); - assert_eq!(classify(410, b""), DavOutcome::Vanished); + assert_eq!(DavOutcome::classify(404, b""), DavOutcome::Vanished); + assert_eq!(DavOutcome::classify(410, b""), DavOutcome::Vanished); } #[test] fn locked_and_quota_are_retryable() { - assert_eq!(classify(423, b""), DavOutcome::Retryable); - assert_eq!(classify(429, b""), DavOutcome::Retryable); - assert_eq!(classify(503, b""), DavOutcome::Retryable); - assert_eq!(classify(507, b""), DavOutcome::Retryable); + assert_eq!(DavOutcome::classify(423, b""), DavOutcome::Retryable); + assert_eq!(DavOutcome::classify(429, b""), DavOutcome::Retryable); + assert_eq!(DavOutcome::classify(503, b""), DavOutcome::Retryable); + assert_eq!(DavOutcome::classify(507, b""), DavOutcome::Retryable); } #[test] fn google_usage_limit_403_is_retryable() { let body = br#"{"error":{"errors":[{"domain":"usageLimits","reason":"rateLimitExceeded"}]}}"#; - assert_eq!(classify(403, body), DavOutcome::Retryable); + assert_eq!(DavOutcome::classify(403, body), DavOutcome::Retryable); } #[test] fn non_google_403_is_auth() { let body = br#"{"error":{"errors":[{"domain":"global","reason":"forbidden"}]}}"#; - assert_eq!(classify(403, body), DavOutcome::Auth); + assert_eq!(DavOutcome::classify(403, body), DavOutcome::Auth); } #[test] fn fatal_4xx_other() { - assert_eq!(classify(400, b""), DavOutcome::Fatal); - assert_eq!(classify(405, b""), DavOutcome::Fatal); - assert_eq!(classify(412, b""), DavOutcome::Fatal); + assert_eq!(DavOutcome::classify(400, b""), DavOutcome::Fatal); + assert_eq!(DavOutcome::classify(405, b""), DavOutcome::Fatal); + assert_eq!(DavOutcome::classify(412, b""), DavOutcome::Fatal); } } diff --git a/src/dav/xml.rs b/src/dav/xml.rs index 791a523..e478e0a 100644 --- a/src/dav/xml.rs +++ b/src/dav/xml.rs @@ -4,187 +4,172 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use crate::dav::href::Href; +use std::fmt::Write; + +use crate::net::xml::XmlEscape; pub const NS_DAV: &str = "DAV:"; pub const NS_CALDAV: &str = "urn:ietf:params:xml:ns:caldav"; pub const NS_CARDDAV: &str = "urn:ietf:params:xml:ns:carddav"; pub const NS_APPLE_ICAL: &str = "http://apple.com/ns/ical/"; -pub fn propfind_current_user_principal() -> String { - String::from( - "\n\ - \ - \ - ", - ) +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Propfind { + CurrentUserPrincipal, + CalendarHomeSet, + AddressbookHomeSet, + PrincipalAndCalendarHomeSet, + PrincipalAndAddressbookHomeSet, + CalendarCollections, + AddressbookCollections, + WebdavListing, + DavItems, } -pub fn propfind_calendar_home_set() -> String { - String::from( - "\n\ - \ - \ - ", - ) -} - -pub fn propfind_addressbook_home_set() -> String { - String::from( - "\n\ - \ - \ - ", - ) -} - -pub fn propfind_principal_and_calendar_home_set() -> String { - String::from( - "\n\ - \ - \ - \ - \ - \ - ", - ) -} - -pub fn propfind_principal_and_addressbook_home_set() -> String { - String::from( - "\n\ - \ - \ - \ - \ - \ - ", - ) -} - -pub fn propfind_calendar_collections() -> String { - String::from( - "\n\ - \ - \ - \ - \ - \ - \ - \ - \ - \ - \ - ", - ) -} - -pub fn propfind_addressbook_collections() -> String { - String::from( - "\n\ - \ - \ - \ - \ - \ - \ - ", - ) -} - -pub fn propfind_webdav_listing() -> String { - String::from( - "\n\ - \ - \ - \ - \ - \ - \ - \ - \ - \ - \ - ", - ) -} - -pub fn propfind_dav_items() -> String { - String::from( - "\n\ - \ - \ - \ - \ - \ - \ - \ - ", - ) -} - -pub fn calendar_multiget(hrefs: &[Href]) -> String { - let mut out = String::with_capacity(256 + hrefs.len() * 64); - out.push_str( - "\n\ - \ - \ - \ - \ - ", - ); - for h in hrefs { - write_href_element(&mut out, h); - } - out.push_str(""); - out -} - -pub fn addressbook_multiget(hrefs: &[Href]) -> String { - let mut out = String::with_capacity(256 + hrefs.len() * 64); - out.push_str( - "\n\ - \ - \ - \ - \ - ", - ); - for h in hrefs { - write_href_element(&mut out, h); - } - out.push_str(""); - out -} - -fn write_href_element(out: &mut String, h: &Href) { - out.push_str(""); - write_escaped(out, h.as_str()); - out.push_str(""); -} - -fn write_escaped(out: &mut String, s: &str) { - for ch in s.chars() { - match ch { - '&' => out.push_str("&"), - '<' => out.push_str("<"), - '>' => out.push_str(">"), - '"' => out.push_str("""), - '\'' => out.push_str("'"), - _ => out.push(ch), +impl Propfind { + pub fn body(self) -> &'static str { + match self { + Propfind::CurrentUserPrincipal => { + "\n\ + \ + \ + " + } + Propfind::CalendarHomeSet => { + "\n\ + \ + \ + " + } + Propfind::AddressbookHomeSet => { + "\n\ + \ + \ + " + } + Propfind::PrincipalAndCalendarHomeSet => { + "\n\ + \ + \ + \ + \ + \ + " + } + Propfind::PrincipalAndAddressbookHomeSet => { + "\n\ + \ + \ + \ + \ + \ + " + } + Propfind::CalendarCollections => { + "\n\ + \ + \ + \ + \ + \ + \ + \ + \ + \ + \ + " + } + Propfind::AddressbookCollections => { + "\n\ + \ + \ + \ + \ + \ + \ + " + } + Propfind::WebdavListing => { + "\n\ + \ + \ + \ + \ + \ + \ + \ + \ + \ + \ + " + } + Propfind::DavItems => { + "\n\ + \ + \ + \ + \ + \ + \ + \ + " + } } } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Multiget { + Calendar, + Addressbook, +} + +impl Multiget { + pub fn body<'h, I>(self, hrefs: I) -> String + where + I: IntoIterator, + I::IntoIter: ExactSizeIterator, + { + let hrefs = hrefs.into_iter(); + let (head, tail) = match self { + Multiget::Calendar => ( + "\n\ + \ + \ + \ + \ + ", + "", + ), + Multiget::Addressbook => ( + "\n\ + \ + \ + \ + \ + ", + "", + ), + }; + let mut out = String::with_capacity(256 + hrefs.len() * 64); + out.push_str(head); + for h in hrefs { + let _ = write!(out, "{}", XmlEscape(h)); + } + out.push_str(tail); + out + } +} + #[cfg(test)] mod tests { use super::*; @@ -192,7 +177,7 @@ mod tests { #[test] fn principal_body_matches_expected_shape() { - let body = propfind_current_user_principal(); + let body = Propfind::CurrentUserPrincipal.body(); assert!(body.starts_with("")); assert!(body.contains("xmlns:d=\"DAV:\"")); @@ -200,25 +185,25 @@ mod tests { #[test] fn calendar_home_set_includes_caldav_namespace() { - let body = propfind_calendar_home_set(); + let body = Propfind::CalendarHomeSet.body(); assert!(body.contains("xmlns:c=\"urn:ietf:params:xml:ns:caldav\"")); assert!(body.contains("")); } #[test] fn addressbook_home_set_includes_carddav_namespace() { - let body = propfind_addressbook_home_set(); + let body = Propfind::AddressbookHomeSet.body(); assert!(body.contains("xmlns:c=\"urn:ietf:params:xml:ns:carddav\"")); assert!(body.contains("")); } #[test] fn calendar_multiget_lists_hrefs_in_order() { - let hrefs = vec![ + let hrefs = [ Href::from_normalised("/dav/cal/u/d/a.ics".to_owned()), Href::from_normalised("/dav/cal/u/d/b.ics".to_owned()), ]; - let body = calendar_multiget(&hrefs); + let body = Multiget::Calendar.body(hrefs.iter().map(Href::as_str)); let a_pos = body.find("/a.ics").unwrap(); let b_pos = body.find("/b.ics").unwrap(); assert!(a_pos < b_pos); @@ -227,8 +212,8 @@ mod tests { #[test] fn addressbook_multiget_emits_address_data_element() { - let hrefs = vec![Href::from_normalised("/dav/card/u/d/a.vcf".to_owned())]; - let body = addressbook_multiget(&hrefs); + let hrefs = [Href::from_normalised("/dav/card/u/d/a.vcf".to_owned())]; + let body = Multiget::Addressbook.body(hrefs.iter().map(Href::as_str)); assert!(body.contains("")); assert!(body.contains("/dav/card/u/d/a.vcf")); } @@ -236,13 +221,13 @@ mod tests { #[test] fn escape_handles_xml_metacharacters() { let h = Href::from_normalised("/dav/cal/u/d/<weird>.ics".to_owned()); - let body = calendar_multiget(&[h]); + let body = Multiget::Calendar.body([h.as_str()]); assert!(body.contains("&lt;weird&gt;.ics")); } #[test] fn webdav_listing_requests_filesystem_props() { - let body = propfind_webdav_listing(); + let body = Propfind::WebdavListing.body(); assert!(body.contains("")); assert!(body.contains("")); assert!(body.contains("")); diff --git a/src/db/blobs.rs b/src/db/blobs.rs deleted file mode 100644 index cc1e4bc..0000000 --- a/src/db/blobs.rs +++ /dev/null @@ -1,131 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use rusqlite::{Connection, OptionalExtension, params}; - -pub fn intern_blob(conn: &Connection, bytes: &[u8]) -> Result { - let hash = blake3::hash(bytes); - let hash_bytes = hash.as_bytes().as_slice(); - conn.execute( - "INSERT OR IGNORE INTO blobs (hash, data) VALUES (?1, ?2)", - params![hash_bytes, bytes], - )?; - conn.query_row( - "SELECT id FROM blobs WHERE hash = ?1", - params![hash_bytes], - |row| row.get(0), - ) -} - -pub fn blob_bytes(conn: &Connection, id: i64) -> Result>, rusqlite::Error> { - conn.query_row("SELECT data FROM blobs WHERE id = ?1", params![id], |row| { - row.get(0) - }) - .optional() -} - -pub fn blob_len(conn: &Connection, id: i64) -> Result, rusqlite::Error> { - conn.query_row( - "SELECT length(data) FROM blobs WHERE id = ?1", - params![id], - |row| { - let len: i64 = row.get(0)?; - Ok(len.max(0) as u64) - }, - ) - .optional() -} - -pub fn gc_orphan_blobs(conn: &Connection) -> Result { - conn.execute( - "DELETE FROM blobs WHERE id NOT IN ( - SELECT blob_id FROM emails - UNION SELECT blob_id FROM sieve_scripts - UNION SELECT blob_id FROM file_nodes WHERE blob_id IS NOT NULL - UNION SELECT json_tree.atom FROM contact_cards, json_tree(contact_cards.data) - WHERE json_tree.key = '@blob' - UNION SELECT json_tree.atom FROM calendar_events, json_tree(calendar_events.data) - WHERE json_tree.key = '@blob')", - [], - ) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::init; - - fn mem() -> Connection { - let c = Connection::open_in_memory().unwrap(); - init::apply_schema(&c).unwrap(); - c - } - - #[test] - fn intern_dedups_identical_bytes() { - let c = mem(); - let a = intern_blob(&c, b"hello world").unwrap(); - let b = intern_blob(&c, b"hello world").unwrap(); - let other = intern_blob(&c, b"different").unwrap(); - assert_eq!(a, b); - assert_ne!(a, other); - let n: i64 = c - .query_row("SELECT count(*) FROM blobs", [], |r| r.get(0)) - .unwrap(); - assert_eq!(n, 2); - } - - #[test] - fn blob_bytes_roundtrip() { - let c = mem(); - let id = intern_blob(&c, b"payload").unwrap(); - assert_eq!( - blob_bytes(&c, id).unwrap().as_deref(), - Some(&b"payload"[..]) - ); - assert_eq!(blob_bytes(&c, 9999).unwrap(), None); - } - - #[test] - fn blob_len_reports_byte_length() { - let c = mem(); - let id = intern_blob(&c, b"payload").unwrap(); - assert_eq!(blob_len(&c, id).unwrap(), Some(7)); - assert_eq!(blob_len(&c, 9999).unwrap(), None); - } - - #[test] - fn gc_keeps_referenced_and_reaps_orphans() { - let c = mem(); - let referenced = intern_blob(&c, b"keep me").unwrap(); - let _orphan = intern_blob(&c, b"reap me").unwrap(); - c.execute( - "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords) - VALUES (?1, '2020-01-01T00:00:00Z', '[1]', '[]')", - params![referenced], - ) - .unwrap(); - let removed = gc_orphan_blobs(&c).unwrap(); - assert_eq!(removed, 1); - assert!(blob_bytes(&c, referenced).unwrap().is_some()); - } - - #[test] - fn gc_follows_blob_sentinel_in_json_data() { - let c = mem(); - let photo = intern_blob(&c, b"photo bytes").unwrap(); - let _orphan = intern_blob(&c, b"orphan").unwrap(); - c.execute( - "INSERT INTO contact_cards (uid, address_book_ids, data) - VALUES ('u1', '[1]', ?1)", - params![format!(r#"{{"photos":{{"p":{{"@blob":{photo}}}}}}}"#)], - ) - .unwrap(); - let removed = gc_orphan_blobs(&c).unwrap(); - assert_eq!(removed, 1); - assert!(blob_bytes(&c, photo).unwrap().is_some()); - } -} diff --git a/src/db/dav_ids.rs b/src/db/dav_ids.rs deleted file mode 100644 index a7cf537..0000000 --- a/src/db/dav_ids.rs +++ /dev/null @@ -1,336 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashMap; - -use rusqlite::{Connection, OptionalExtension, params}; - -pub const CALENDAR: &str = "calendar"; -pub const CALENDAR_EVENT: &str = "calendarevent"; -pub const ADDRESS_BOOK: &str = "addressbook"; -pub const CONTACT_CARD: &str = "contactcard"; -pub const FILE_NODE: &str = "filenode"; - -#[derive(Debug, Clone)] -pub struct ItemRow { - pub item_href: String, - pub etag: String, - pub local_id: i64, -} - -pub fn insert( - conn: &Connection, - source_id: i64, - type_name: &str, - collection_href: &str, - item_href: &str, - etag: &str, - local_id: i64, -) -> Result<(), rusqlite::Error> { - conn.execute( - "INSERT INTO sync_id_dav - (source_id, type_name, collection_href, item_href, etag, local_id) - VALUES (?1, ?2, ?3, ?4, ?5, ?6)", - params![ - source_id, - type_name, - collection_href, - item_href, - etag, - local_id - ], - )?; - Ok(()) -} - -pub fn update_etag( - conn: &Connection, - source_id: i64, - type_name: &str, - item_href: &str, - etag: &str, -) -> Result<(), rusqlite::Error> { - conn.execute( - "UPDATE sync_id_dav - SET etag = ?4 - WHERE source_id = ?1 AND type_name = ?2 AND item_href = ?3", - params![source_id, type_name, item_href, etag], - )?; - Ok(()) -} - -pub fn local_for_item( - conn: &Connection, - source_id: i64, - type_name: &str, - item_href: &str, -) -> Result, rusqlite::Error> { - conn.query_row( - "SELECT local_id FROM sync_id_dav - WHERE source_id = ?1 AND type_name = ?2 AND item_href = ?3", - params![source_id, type_name, item_href], - |row| row.get(0), - ) - .optional() -} - -pub fn items_in_collection( - conn: &Connection, - source_id: i64, - type_name: &str, - collection_href: &str, -) -> Result, rusqlite::Error> { - let mut stmt = conn.prepare( - "SELECT item_href, etag, local_id FROM sync_id_dav - WHERE source_id = ?1 AND type_name = ?2 AND collection_href = ?3", - )?; - let rows = stmt.query_map(params![source_id, type_name, collection_href], |row| { - Ok(ItemRow { - item_href: row.get(0)?, - etag: row.get(1)?, - local_id: row.get(2)?, - }) - })?; - let mut out = Vec::new(); - for r in rows { - out.push(r?); - } - Ok(out) -} - -pub fn collections_of_type( - conn: &Connection, - source_id: i64, - type_name: &str, -) -> Result, rusqlite::Error> { - let mut stmt = conn.prepare( - "SELECT item_href, local_id FROM sync_id_dav - WHERE source_id = ?1 AND type_name = ?2", - )?; - let rows = stmt.query_map(params![source_id, type_name], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) - })?; - let mut map = HashMap::new(); - for r in rows { - let (href, id) = r?; - map.insert(href, id); - } - Ok(map) -} - -pub fn delete_item( - conn: &Connection, - source_id: i64, - type_name: &str, - item_href: &str, -) -> Result<(), rusqlite::Error> { - conn.execute( - "DELETE FROM sync_id_dav - WHERE source_id = ?1 AND type_name = ?2 AND item_href = ?3", - params![source_id, type_name, item_href], - )?; - Ok(()) -} - -pub fn delete_collection( - conn: &Connection, - source_id: i64, - type_name: &str, - collection_href: &str, -) -> Result { - let removed = conn.execute( - "DELETE FROM sync_id_dav - WHERE source_id = ?1 AND type_name = ?2 - AND (item_href = ?3 OR collection_href = ?3)", - params![source_id, type_name, collection_href], - )?; - Ok(removed) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::init; - use crate::db::sources::{SourceKey, upsert_source}; - - fn setup() -> (rusqlite::Connection, i64) { - let conn = rusqlite::Connection::open_in_memory().unwrap(); - init::apply_schema(&conn).unwrap(); - let key = SourceKey { - kind: "caldav".to_owned(), - session_url: "https://x/".to_owned(), - account_id: "u".to_owned(), - }; - let sid = upsert_source(&conn, &key, Some("u"), "u").unwrap(); - (conn, sid) - } - - #[test] - fn insert_and_lookup_round_trip() { - let (conn, sid) = setup(); - insert( - &conn, - sid, - CALENDAR_EVENT, - "/cal/a/", - "/cal/a/e.ics", - "\"v1\"", - 7, - ) - .unwrap(); - let id = local_for_item(&conn, sid, CALENDAR_EVENT, "/cal/a/e.ics") - .unwrap() - .unwrap(); - assert_eq!(id, 7); - } - - #[test] - fn replace_keeps_local_id_on_etag_change() { - let (conn, sid) = setup(); - insert( - &conn, - sid, - CALENDAR_EVENT, - "/cal/a/", - "/cal/a/e.ics", - "\"v1\"", - 7, - ) - .unwrap(); - update_etag(&conn, sid, CALENDAR_EVENT, "/cal/a/e.ics", "\"v2\"").unwrap(); - let rows = items_in_collection(&conn, sid, CALENDAR_EVENT, "/cal/a/").unwrap(); - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].etag, "\"v2\""); - assert_eq!(rows[0].local_id, 7); - } - - #[test] - fn items_in_collection_filters_by_collection_href() { - let (conn, sid) = setup(); - insert( - &conn, - sid, - CALENDAR_EVENT, - "/cal/a/", - "/cal/a/e1.ics", - "v1", - 1, - ) - .unwrap(); - insert( - &conn, - sid, - CALENDAR_EVENT, - "/cal/a/", - "/cal/a/e2.ics", - "v2", - 2, - ) - .unwrap(); - insert( - &conn, - sid, - CALENDAR_EVENT, - "/cal/b/", - "/cal/b/e3.ics", - "v3", - 3, - ) - .unwrap(); - let rows = items_in_collection(&conn, sid, CALENDAR_EVENT, "/cal/a/").unwrap(); - assert_eq!(rows.len(), 2); - } - - #[test] - fn delete_item_removes_only_one_row() { - let (conn, sid) = setup(); - insert( - &conn, - sid, - CALENDAR_EVENT, - "/cal/a/", - "/cal/a/e1.ics", - "v", - 1, - ) - .unwrap(); - insert( - &conn, - sid, - CALENDAR_EVENT, - "/cal/a/", - "/cal/a/e2.ics", - "v", - 2, - ) - .unwrap(); - delete_item(&conn, sid, CALENDAR_EVENT, "/cal/a/e1.ics").unwrap(); - let rows = items_in_collection(&conn, sid, CALENDAR_EVENT, "/cal/a/").unwrap(); - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].item_href, "/cal/a/e2.ics"); - } - - #[test] - fn delete_collection_removes_collection_row_and_items() { - let (conn, sid) = setup(); - insert(&conn, sid, CALENDAR, "/cal/a/", "/cal/a/", "", 10).unwrap(); - insert( - &conn, - sid, - CALENDAR_EVENT, - "/cal/a/", - "/cal/a/e.ics", - "v", - 1, - ) - .unwrap(); - let removed = delete_collection(&conn, sid, CALENDAR_EVENT, "/cal/a/").unwrap(); - assert_eq!(removed, 1); - let rows = items_in_collection(&conn, sid, CALENDAR_EVENT, "/cal/a/").unwrap(); - assert!(rows.is_empty()); - } - - #[test] - fn collections_of_type_lists_all_known_collections() { - let (conn, sid) = setup(); - insert(&conn, sid, CALENDAR, "/cal/a/", "/cal/a/", "", 1).unwrap(); - insert(&conn, sid, CALENDAR, "/cal/b/", "/cal/b/", "", 2).unwrap(); - let map = collections_of_type(&conn, sid, CALENDAR).unwrap(); - assert_eq!(map.len(), 2); - assert_eq!(map.get("/cal/a/"), Some(&1)); - assert_eq!(map.get("/cal/b/"), Some(&2)); - } - - #[test] - fn unique_local_id_per_type_enforced() { - let (conn, sid) = setup(); - insert( - &conn, - sid, - CALENDAR_EVENT, - "/cal/a/", - "/cal/a/e1.ics", - "v", - 1, - ) - .unwrap(); - let err = insert( - &conn, - sid, - CALENDAR_EVENT, - "/cal/a/", - "/cal/a/e2.ics", - "v", - 1, - ) - .unwrap_err(); - let msg = format!("{err}"); - assert!( - msg.contains("UNIQUE"), - "expected UNIQUE violation, got {msg}" - ); - } -} diff --git a/src/db/defaults.rs b/src/db/defaults.rs deleted file mode 100644 index 25c7b0e..0000000 --- a/src/db/defaults.rs +++ /dev/null @@ -1,112 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use crate::types::ObjectType; -use rusqlite::{Connection, params}; - -pub fn unique_default( - conn: &Connection, - ty: ObjectType, - is_default: bool, - exclude_id: Option, -) -> rusqlite::Result { - if !is_default { - return Ok(false); - } - let Some(table) = table_for(ty) else { - return Ok(false); - }; - let taken: bool = conn.query_row( - &format!( - "SELECT EXISTS(SELECT 1 FROM {table} \ - WHERE is_default = 1 AND (?1 IS NULL OR id != ?1))" - ), - params![exclude_id], - |row| row.get(0), - )?; - Ok(!taken) -} - -fn table_for(ty: ObjectType) -> Option<&'static str> { - match ty { - ObjectType::AddressBook => Some("address_books"), - ObjectType::Calendar => Some("calendars"), - ObjectType::ParticipantIdentity => Some("participant_identities"), - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::init; - - fn mem() -> Connection { - let conn = Connection::open_in_memory().unwrap(); - init::apply_schema(&conn).unwrap(); - conn - } - - fn insert_book(conn: &Connection, name: &str, is_default: bool) -> i64 { - conn.execute( - "INSERT INTO address_books (name, sort_order, is_default, is_subscribed) - VALUES (?1, 0, ?2, 1)", - params![name, is_default as i64], - ) - .unwrap(); - conn.last_insert_rowid() - } - - #[test] - fn first_claimant_keeps_the_default_second_is_dropped() { - let conn = mem(); - let first = unique_default(&conn, ObjectType::AddressBook, true, None).unwrap(); - assert!(first); - insert_book(&conn, "Contacts", first); - - assert!( - !unique_default(&conn, ObjectType::AddressBook, true, None).unwrap(), - "an archive must never hold two default address books" - ); - } - - #[test] - fn non_default_stays_non_default() { - let conn = mem(); - assert!(!unique_default(&conn, ObjectType::AddressBook, false, None).unwrap()); - } - - #[test] - fn each_type_has_its_own_default() { - let conn = mem(); - insert_book(&conn, "Contacts", true); - assert!( - unique_default(&conn, ObjectType::Calendar, true, None).unwrap(), - "a default address book must not block a default calendar" - ); - } - - #[test] - fn update_excludes_its_own_row() { - let conn = mem(); - let id = insert_book(&conn, "Contacts", true); - assert!(unique_default(&conn, ObjectType::AddressBook, true, Some(id)).unwrap()); - } - - #[test] - fn update_detects_another_holder() { - let conn = mem(); - insert_book(&conn, "Contacts", true); - let other = insert_book(&conn, "GAL", false); - assert!(!unique_default(&conn, ObjectType::AddressBook, true, Some(other)).unwrap()); - } - - #[test] - fn types_without_a_default_column_are_never_default() { - let conn = mem(); - assert!(!unique_default(&conn, ObjectType::Mailbox, true, None).unwrap()); - } -} diff --git a/src/db/exchange_ews_ids.rs b/src/db/exchange_ews_ids.rs deleted file mode 100644 index 05d68b0..0000000 --- a/src/db/exchange_ews_ids.rs +++ /dev/null @@ -1,265 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashMap; - -use rusqlite::{Connection, OptionalExtension, params}; - -pub const MAILBOX: &str = "mailbox"; -pub const CALENDAR: &str = "calendar"; -pub const ADDRESS_BOOK: &str = "addressbook"; -pub const EMAIL: &str = "email"; -pub const CALENDAR_EVENT: &str = "calendarevent"; -pub const CONTACT_CARD: &str = "contactcard"; - -#[derive(Debug, Clone)] -pub struct ItemRow { - pub item_id: String, - pub change_key: String, - pub local_id: i64, -} - -#[derive(Debug, Clone)] -pub struct FolderRow { - pub item_id: String, - pub folder_id: String, - pub change_key: String, - pub local_id: i64, -} - -pub fn insert( - conn: &Connection, - source_id: i64, - type_name: &str, - folder_id: &str, - item_id: &str, - change_key: &str, - local_id: i64, -) -> Result<(), rusqlite::Error> { - conn.execute( - "INSERT INTO sync_id_exchange_ews - (source_id, type_name, folder_id, item_id, change_key, sync_state, local_id) - VALUES (?1, ?2, ?3, ?4, ?5, '', ?6)", - params![ - source_id, type_name, folder_id, item_id, change_key, local_id - ], - )?; - Ok(()) -} - -pub fn update_change_key( - conn: &Connection, - source_id: i64, - type_name: &str, - item_id: &str, - change_key: &str, -) -> Result<(), rusqlite::Error> { - conn.execute( - "UPDATE sync_id_exchange_ews - SET change_key = ?4 - WHERE source_id = ?1 AND type_name = ?2 AND item_id = ?3", - params![source_id, type_name, item_id, change_key], - )?; - Ok(()) -} - -pub fn delete_item( - conn: &Connection, - source_id: i64, - type_name: &str, - item_id: &str, -) -> Result<(), rusqlite::Error> { - conn.execute( - "DELETE FROM sync_id_exchange_ews - WHERE source_id = ?1 AND type_name = ?2 AND item_id = ?3", - params![source_id, type_name, item_id], - )?; - Ok(()) -} - -pub fn local_for_item( - conn: &Connection, - source_id: i64, - type_name: &str, - item_id: &str, -) -> Result, rusqlite::Error> { - conn.query_row( - "SELECT local_id FROM sync_id_exchange_ews - WHERE source_id = ?1 AND type_name = ?2 AND item_id = ?3", - params![source_id, type_name, item_id], - |row| row.get(0), - ) - .optional() -} - -pub fn items_in_folder( - conn: &Connection, - source_id: i64, - type_name: &str, - folder_id: &str, -) -> Result, rusqlite::Error> { - let mut stmt = conn.prepare( - "SELECT item_id, change_key, local_id FROM sync_id_exchange_ews - WHERE source_id = ?1 AND type_name = ?2 AND folder_id = ?3", - )?; - let rows = stmt.query_map(params![source_id, type_name, folder_id], |row| { - Ok(ItemRow { - item_id: row.get(0)?, - change_key: row.get(1)?, - local_id: row.get(2)?, - }) - })?; - let mut out = Vec::new(); - for r in rows { - out.push(r?); - } - Ok(out) -} - -pub fn folders_of_type( - conn: &Connection, - source_id: i64, - type_name: &str, -) -> Result, rusqlite::Error> { - let mut stmt = conn.prepare( - "SELECT item_id, folder_id, change_key, local_id FROM sync_id_exchange_ews - WHERE source_id = ?1 AND type_name = ?2", - )?; - let rows = stmt.query_map(params![source_id, type_name], |row| { - Ok(FolderRow { - item_id: row.get(0)?, - folder_id: row.get(1)?, - change_key: row.get(2)?, - local_id: row.get(3)?, - }) - })?; - let mut map = HashMap::new(); - for r in rows { - let f = r?; - map.insert(f.item_id.clone(), f); - } - Ok(map) -} - -pub fn get_sync_state( - conn: &Connection, - source_id: i64, - type_name: &str, - item_id: &str, -) -> Result, rusqlite::Error> { - conn.query_row( - "SELECT sync_state FROM sync_id_exchange_ews - WHERE source_id = ?1 AND type_name = ?2 AND item_id = ?3", - params![source_id, type_name, item_id], - |row| row.get::<_, String>(0), - ) - .optional() -} - -pub fn set_sync_state( - conn: &Connection, - source_id: i64, - type_name: &str, - item_id: &str, - sync_state: &str, -) -> Result<(), rusqlite::Error> { - conn.execute( - "UPDATE sync_id_exchange_ews - SET sync_state = ?4 - WHERE source_id = ?1 AND type_name = ?2 AND item_id = ?3", - params![source_id, type_name, item_id, sync_state], - )?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::init; - use crate::db::sources::{SourceKey, upsert_source}; - - fn setup() -> (rusqlite::Connection, i64) { - let conn = rusqlite::Connection::open_in_memory().unwrap(); - init::apply_schema(&conn).unwrap(); - let key = SourceKey { - kind: "exchange_ews".to_owned(), - session_url: "https://x/EWS/Exchange.asmx".to_owned(), - account_id: "u@d".to_owned(), - }; - let sid = upsert_source(&conn, &key, Some("u"), "u@d").unwrap(); - (conn, sid) - } - - #[test] - fn insert_and_lookup_round_trip() { - let (conn, sid) = setup(); - insert(&conn, sid, EMAIL, "FOLDER1", "ITEM1", "CK1", 42).unwrap(); - let id = local_for_item(&conn, sid, EMAIL, "ITEM1").unwrap().unwrap(); - assert_eq!(id, 42); - let rows = items_in_folder(&conn, sid, EMAIL, "FOLDER1").unwrap(); - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].change_key, "CK1"); - } - - #[test] - fn update_change_key_preserves_local_id() { - let (conn, sid) = setup(); - insert(&conn, sid, EMAIL, "F1", "I1", "A", 7).unwrap(); - update_change_key(&conn, sid, EMAIL, "I1", "B").unwrap(); - let rows = items_in_folder(&conn, sid, EMAIL, "F1").unwrap(); - assert_eq!(rows[0].change_key, "B"); - assert_eq!(rows[0].local_id, 7); - } - - #[test] - fn folders_of_type_lists_by_item_id() { - let (conn, sid) = setup(); - insert(&conn, sid, MAILBOX, "", "F1", "CK1", 1).unwrap(); - insert(&conn, sid, MAILBOX, "F1", "F2", "CK2", 2).unwrap(); - let map = folders_of_type(&conn, sid, MAILBOX).unwrap(); - assert_eq!(map.len(), 2); - assert_eq!(map.get("F1").map(|r| r.local_id), Some(1)); - assert_eq!(map.get("F2").map(|r| r.local_id), Some(2)); - } - - #[test] - fn sync_state_round_trips() { - let (conn, sid) = setup(); - insert(&conn, sid, MAILBOX, "", "F1", "", 1).unwrap(); - assert_eq!( - get_sync_state(&conn, sid, MAILBOX, "F1") - .unwrap() - .as_deref(), - Some("") - ); - set_sync_state(&conn, sid, MAILBOX, "F1", "opaque-state").unwrap(); - assert_eq!( - get_sync_state(&conn, sid, MAILBOX, "F1") - .unwrap() - .as_deref(), - Some("opaque-state") - ); - } - - #[test] - fn delete_item_removes_only_one() { - let (conn, sid) = setup(); - insert(&conn, sid, EMAIL, "F1", "I1", "A", 1).unwrap(); - insert(&conn, sid, EMAIL, "F1", "I2", "A", 2).unwrap(); - delete_item(&conn, sid, EMAIL, "I1").unwrap(); - let rows = items_in_folder(&conn, sid, EMAIL, "F1").unwrap(); - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].item_id, "I2"); - } - - #[test] - fn unique_local_id_per_type_is_enforced() { - let (conn, sid) = setup(); - insert(&conn, sid, EMAIL, "F1", "I1", "A", 1).unwrap(); - let err = insert(&conn, sid, EMAIL, "F1", "I2", "A", 1).unwrap_err(); - assert!(err.to_string().contains("UNIQUE")); - } -} diff --git a/src/db/exchange_graph_ids.rs b/src/db/exchange_graph_ids.rs deleted file mode 100644 index 257f181..0000000 --- a/src/db/exchange_graph_ids.rs +++ /dev/null @@ -1,168 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashMap; - -use rusqlite::{Connection, OptionalExtension, params}; - -pub const MAILBOX: &str = "mailbox"; -pub const EMAIL: &str = "email"; -pub const CALENDAR: &str = "calendar"; -pub const CALENDAR_EVENT: &str = "calendarevent"; -pub const ADDRESS_BOOK: &str = "addressbook"; -pub const CONTACT_CARD: &str = "contactcard"; -pub const FILE_NODE: &str = "filenode"; - -#[derive(Debug, Clone)] -pub struct IdRow { - pub graph_id: String, - pub local_id: i64, -} - -pub fn insert( - conn: &Connection, - source_id: i64, - type_name: &str, - graph_id: &str, - local_id: i64, -) -> Result<(), rusqlite::Error> { - conn.execute( - "INSERT INTO sync_id_exchange_graph (source_id, type_name, graph_id, local_id) - VALUES (?1, ?2, ?3, ?4)", - params![source_id, type_name, graph_id, local_id], - )?; - Ok(()) -} - -pub fn delete( - conn: &Connection, - source_id: i64, - type_name: &str, - graph_id: &str, -) -> Result<(), rusqlite::Error> { - conn.execute( - "DELETE FROM sync_id_exchange_graph - WHERE source_id = ?1 AND type_name = ?2 AND graph_id = ?3", - params![source_id, type_name, graph_id], - )?; - Ok(()) -} - -pub fn local_for_graph_id( - conn: &Connection, - source_id: i64, - type_name: &str, - graph_id: &str, -) -> Result, rusqlite::Error> { - conn.query_row( - "SELECT local_id FROM sync_id_exchange_graph - WHERE source_id = ?1 AND type_name = ?2 AND graph_id = ?3", - params![source_id, type_name, graph_id], - |row| row.get(0), - ) - .optional() -} - -pub fn ids_of_type( - conn: &Connection, - source_id: i64, - type_name: &str, -) -> Result, rusqlite::Error> { - let mut stmt = conn.prepare( - "SELECT graph_id, local_id FROM sync_id_exchange_graph - WHERE source_id = ?1 AND type_name = ?2", - )?; - let rows = stmt.query_map(params![source_id, type_name], |row| { - Ok(IdRow { - graph_id: row.get(0)?, - local_id: row.get(1)?, - }) - })?; - let mut map = HashMap::new(); - for r in rows { - let row = r?; - map.insert(row.graph_id, row.local_id); - } - Ok(map) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::init; - use crate::db::sources::{SourceKey, upsert_source}; - - fn setup() -> (Connection, i64) { - let conn = Connection::open_in_memory().unwrap(); - init::apply_schema(&conn).unwrap(); - let key = SourceKey { - kind: "exchange_graph".to_owned(), - session_url: "https://x|https://graph.microsoft.com/v1.0".to_owned(), - account_id: "u-uuid".to_owned(), - }; - let sid = upsert_source(&conn, &key, Some("u@d"), "u@d").unwrap(); - (conn, sid) - } - - #[test] - fn insert_and_lookup_round_trip() { - let (conn, sid) = setup(); - insert(&conn, sid, EMAIL, "AAkA-base64==", 17).unwrap(); - let id = local_for_graph_id(&conn, sid, EMAIL, "AAkA-base64==") - .unwrap() - .unwrap(); - assert_eq!(id, 17); - } - - #[test] - fn graph_id_is_case_sensitive() { - let (conn, sid) = setup(); - insert(&conn, sid, EMAIL, "AaBb", 1).unwrap(); - assert!( - local_for_graph_id(&conn, sid, EMAIL, "aabb") - .unwrap() - .is_none() - ); - assert!( - local_for_graph_id(&conn, sid, EMAIL, "AaBb") - .unwrap() - .is_some() - ); - } - - #[test] - fn unique_local_id_per_type_is_enforced() { - let (conn, sid) = setup(); - insert(&conn, sid, EMAIL, "ID-A", 1).unwrap(); - let err = insert(&conn, sid, EMAIL, "ID-B", 1).unwrap_err(); - assert!(err.to_string().contains("UNIQUE")); - } - - #[test] - fn ids_of_type_filters_by_type() { - let (conn, sid) = setup(); - insert(&conn, sid, EMAIL, "M1", 1).unwrap(); - insert(&conn, sid, EMAIL, "M2", 2).unwrap(); - insert(&conn, sid, MAILBOX, "F1", 3).unwrap(); - let emails = ids_of_type(&conn, sid, EMAIL).unwrap(); - let folders = ids_of_type(&conn, sid, MAILBOX).unwrap(); - assert_eq!(emails.len(), 2); - assert_eq!(folders.len(), 1); - assert_eq!(emails.get("M1"), Some(&1)); - assert_eq!(folders.get("F1"), Some(&3)); - } - - #[test] - fn delete_removes_only_one() { - let (conn, sid) = setup(); - insert(&conn, sid, EMAIL, "M1", 1).unwrap(); - insert(&conn, sid, EMAIL, "M2", 2).unwrap(); - delete(&conn, sid, EMAIL, "M1").unwrap(); - let remaining = ids_of_type(&conn, sid, EMAIL).unwrap(); - assert_eq!(remaining.len(), 1); - assert!(remaining.contains_key("M2")); - } -} diff --git a/src/db/ids.rs b/src/db/ids.rs deleted file mode 100644 index aa7b34d..0000000 --- a/src/db/ids.rs +++ /dev/null @@ -1,114 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashMap; - -use rusqlite::{Connection, OptionalExtension, params}; - -use crate::types::ObjectType; - -pub fn insert( - conn: &Connection, - source_id: i64, - ty: ObjectType, - jmap_id: &str, - local_id: i64, -) -> Result<(), rusqlite::Error> { - conn.execute( - "INSERT OR REPLACE INTO sync_id_jmap (source_id, type_name, jmap_id, local_id) - VALUES (?1, ?2, ?3, ?4)", - params![source_id, ty.jmap_name(), jmap_id, local_id], - )?; - Ok(()) -} - -pub fn local_for( - conn: &Connection, - source_id: i64, - ty: ObjectType, - jmap_id: &str, -) -> Result, rusqlite::Error> { - conn.query_row( - "SELECT local_id FROM sync_id_jmap - WHERE source_id = ?1 AND type_name = ?2 AND jmap_id = ?3", - params![source_id, ty.jmap_name(), jmap_id], - |row| row.get(0), - ) - .optional() -} - -pub fn jmap_to_local( - conn: &Connection, - source_id: i64, - ty: ObjectType, -) -> Result, rusqlite::Error> { - let mut stmt = conn.prepare( - "SELECT jmap_id, local_id FROM sync_id_jmap WHERE source_id = ?1 AND type_name = ?2", - )?; - let rows = stmt.query_map(params![source_id, ty.jmap_name()], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) - })?; - let mut map = HashMap::new(); - for r in rows { - let (j, l) = r?; - map.insert(j, l); - } - Ok(map) -} - -pub fn delete( - conn: &Connection, - source_id: i64, - ty: ObjectType, - jmap_id: &str, -) -> Result<(), rusqlite::Error> { - conn.execute( - "DELETE FROM sync_id_jmap - WHERE source_id = ?1 AND type_name = ?2 AND jmap_id = ?3", - params![source_id, ty.jmap_name(), jmap_id], - )?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::init; - use crate::db::sources::{SourceKey, upsert_source}; - - fn setup() -> (Connection, i64) { - let c = Connection::open_in_memory().unwrap(); - init::apply_schema(&c).unwrap(); - let sid = upsert_source( - &c, - &SourceKey { - kind: "jmap".to_owned(), - session_url: "u".to_owned(), - account_id: "w".to_owned(), - }, - None, - "alice", - ) - .unwrap(); - (c, sid) - } - - #[test] - fn crud_roundtrip() { - let (c, sid) = setup(); - insert(&c, sid, ObjectType::Mailbox, "M1", 1).unwrap(); - insert(&c, sid, ObjectType::Mailbox, "M2", 2).unwrap(); - assert_eq!( - local_for(&c, sid, ObjectType::Mailbox, "M1").unwrap(), - Some(1) - ); - let map = jmap_to_local(&c, sid, ObjectType::Mailbox).unwrap(); - assert_eq!(map.len(), 2); - assert_eq!(map.get("M2"), Some(&2)); - delete(&c, sid, ObjectType::Mailbox, "M1").unwrap(); - assert_eq!(local_for(&c, sid, ObjectType::Mailbox, "M1").unwrap(), None); - } -} diff --git a/src/db/imap_ids.rs b/src/db/imap_ids.rs deleted file mode 100644 index cb784f5..0000000 --- a/src/db/imap_ids.rs +++ /dev/null @@ -1,292 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::{HashMap, HashSet}; - -use rusqlite::{Connection, OptionalExtension, params}; - -const MAILBOX: &str = "mailbox"; -const EMAIL: &str = "email"; - -pub fn insert_mailbox( - conn: &Connection, - source_id: i64, - folder: &str, - local_id: i64, -) -> Result<(), rusqlite::Error> { - conn.execute( - "INSERT OR REPLACE INTO sync_id_imap - (source_id, type_name, folder, uidvalidity, uid, local_id) - VALUES (?1, ?2, ?3, 0, 0, ?4)", - params![source_id, MAILBOX, folder, local_id], - )?; - Ok(()) -} - -pub fn insert_email( - conn: &Connection, - source_id: i64, - folder: &str, - uidvalidity: u32, - uid: u32, - local_id: i64, -) -> Result<(), rusqlite::Error> { - conn.execute( - "INSERT OR REPLACE INTO sync_id_imap - (source_id, type_name, folder, uidvalidity, uid, local_id) - VALUES (?1, ?2, ?3, ?4, ?5, ?6)", - params![source_id, EMAIL, folder, uidvalidity, uid, local_id], - )?; - Ok(()) -} - -pub fn local_for_mailbox( - conn: &Connection, - source_id: i64, - folder: &str, -) -> Result, rusqlite::Error> { - conn.query_row( - "SELECT local_id FROM sync_id_imap - WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3 - AND uidvalidity = 0 AND uid = 0", - params![source_id, MAILBOX, folder], - |row| row.get(0), - ) - .optional() -} - -pub fn local_for_email( - conn: &Connection, - source_id: i64, - folder: &str, - uidvalidity: u32, - uid: u32, -) -> Result, rusqlite::Error> { - conn.query_row( - "SELECT local_id FROM sync_id_imap - WHERE source_id = ?1 AND type_name = ?2 - AND folder = ?3 AND uidvalidity = ?4 AND uid = ?5", - params![source_id, EMAIL, folder, uidvalidity, uid], - |row| row.get(0), - ) - .optional() -} - -pub fn mailbox_folders( - conn: &Connection, - source_id: i64, -) -> Result, rusqlite::Error> { - let mut stmt = conn.prepare( - "SELECT folder, local_id FROM sync_id_imap - WHERE source_id = ?1 AND type_name = ?2", - )?; - let rows = stmt.query_map(params![source_id, MAILBOX], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) - })?; - let mut map = HashMap::new(); - for r in rows { - let (f, l) = r?; - map.insert(f, l); - } - Ok(map) -} - -pub fn email_uids_in_folder( - conn: &Connection, - source_id: i64, - folder: &str, -) -> Result, rusqlite::Error> { - let mut stmt = conn.prepare( - "SELECT uidvalidity, uid, local_id FROM sync_id_imap - WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3", - )?; - let rows = stmt.query_map(params![source_id, EMAIL, folder], |row| { - Ok(( - row.get::<_, u32>(0)?, - row.get::<_, u32>(1)?, - row.get::<_, i64>(2)?, - )) - })?; - let mut map = HashMap::new(); - for r in rows { - let (v, u, l) = r?; - map.insert((v, u), l); - } - Ok(map) -} - -pub fn delete_mailbox( - conn: &Connection, - source_id: i64, - folder: &str, -) -> Result<(), rusqlite::Error> { - conn.execute( - "DELETE FROM sync_id_imap - WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3 - AND uidvalidity = 0 AND uid = 0", - params![source_id, MAILBOX, folder], - )?; - Ok(()) -} - -pub fn delete_email( - conn: &Connection, - source_id: i64, - folder: &str, - uidvalidity: u32, - uid: u32, -) -> Result<(), rusqlite::Error> { - conn.execute( - "DELETE FROM sync_id_imap - WHERE source_id = ?1 AND type_name = ?2 - AND folder = ?3 AND uidvalidity = ?4 AND uid = ?5", - params![source_id, EMAIL, folder, uidvalidity, uid], - )?; - Ok(()) -} - -pub fn delete_all_emails_in_folder( - conn: &Connection, - source_id: i64, - folder: &str, -) -> Result { - conn.execute( - "DELETE FROM sync_id_imap - WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3", - params![source_id, EMAIL, folder], - ) -} - -pub fn email_uids_at_validity( - conn: &Connection, - source_id: i64, - folder: &str, - uidvalidity: u32, -) -> Result, rusqlite::Error> { - let mut stmt = conn.prepare( - "SELECT uid FROM sync_id_imap - WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3 - AND uidvalidity = ?4 - ORDER BY uid", - )?; - let rows = stmt.query_map(params![source_id, EMAIL, folder, uidvalidity], |row| { - row.get::<_, u32>(0) - })?; - let mut out = Vec::new(); - for r in rows { - out.push(r?); - } - Ok(out) -} - -pub fn folders_with_emails( - conn: &Connection, - source_id: i64, -) -> Result, rusqlite::Error> { - let mut stmt = conn.prepare( - "SELECT DISTINCT folder FROM sync_id_imap - WHERE source_id = ?1 AND type_name = ?2", - )?; - let rows = stmt.query_map(params![source_id, EMAIL], |row| row.get::<_, String>(0))?; - let mut out = HashSet::new(); - for r in rows { - out.insert(r?); - } - Ok(out) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::init; - use crate::db::sources::{SourceKey, upsert_source}; - - fn setup() -> (Connection, i64) { - let c = Connection::open_in_memory().unwrap(); - init::apply_schema(&c).unwrap(); - let sid = upsert_source( - &c, - &SourceKey { - kind: "imap".to_owned(), - session_url: "imaps://host:993".to_owned(), - account_id: "alice".to_owned(), - }, - None, - "alice", - ) - .unwrap(); - (c, sid) - } - - #[test] - fn mailbox_roundtrip() { - let (c, sid) = setup(); - insert_mailbox(&c, sid, "INBOX", 7).unwrap(); - insert_mailbox(&c, sid, "INBOX/Sub", 8).unwrap(); - assert_eq!(local_for_mailbox(&c, sid, "INBOX").unwrap(), Some(7)); - let all = mailbox_folders(&c, sid).unwrap(); - assert_eq!(all.len(), 2); - assert_eq!(all.get("INBOX/Sub"), Some(&8)); - delete_mailbox(&c, sid, "INBOX").unwrap(); - assert_eq!(local_for_mailbox(&c, sid, "INBOX").unwrap(), None); - } - - #[test] - fn email_roundtrip() { - let (c, sid) = setup(); - insert_email(&c, sid, "INBOX", 12345, 100, 1).unwrap(); - insert_email(&c, sid, "INBOX", 12345, 101, 2).unwrap(); - insert_email(&c, sid, "Sent", 7777, 1, 3).unwrap(); - assert_eq!( - local_for_email(&c, sid, "INBOX", 12345, 100).unwrap(), - Some(1) - ); - let inbox = email_uids_in_folder(&c, sid, "INBOX").unwrap(); - assert_eq!(inbox.len(), 2); - assert_eq!(inbox.get(&(12345, 100)), Some(&1)); - let folders = folders_with_emails(&c, sid).unwrap(); - assert!(folders.contains("INBOX")); - assert!(folders.contains("Sent")); - } - - #[test] - fn delete_all_emails_in_folder_wipes_only_that_folder() { - let (c, sid) = setup(); - insert_email(&c, sid, "INBOX", 1, 10, 1).unwrap(); - insert_email(&c, sid, "INBOX", 1, 11, 2).unwrap(); - insert_email(&c, sid, "Sent", 1, 10, 3).unwrap(); - let removed = delete_all_emails_in_folder(&c, sid, "INBOX").unwrap(); - assert_eq!(removed, 2); - assert!(email_uids_in_folder(&c, sid, "INBOX").unwrap().is_empty()); - assert_eq!(email_uids_in_folder(&c, sid, "Sent").unwrap().len(), 1); - } - - #[test] - fn re_insert_with_same_local_id_replaces() { - let (c, sid) = setup(); - insert_mailbox(&c, sid, "A", 1).unwrap(); - insert_mailbox(&c, sid, "B", 1).unwrap(); - assert_eq!(local_for_mailbox(&c, sid, "A").unwrap(), None); - assert_eq!(local_for_mailbox(&c, sid, "B").unwrap(), Some(1)); - let all = mailbox_folders(&c, sid).unwrap(); - assert_eq!(all.len(), 1); - } - - #[test] - fn pk_composite_is_per_type_per_folder_per_uid() { - let (c, sid) = setup(); - insert_email(&c, sid, "INBOX", 1, 10, 1).unwrap(); - insert_email(&c, sid, "INBOX", 1, 10, 1).unwrap(); - assert_eq!(email_uids_in_folder(&c, sid, "INBOX").unwrap().len(), 1); - } - - #[test] - fn mailbox_and_email_namespaces_do_not_collide_on_local_id() { - let (c, sid) = setup(); - insert_mailbox(&c, sid, "INBOX", 42).unwrap(); - insert_email(&c, sid, "INBOX", 1, 1, 42).unwrap(); - } -} diff --git a/src/db/imap_state.rs b/src/db/imap_state.rs deleted file mode 100644 index 81a8481..0000000 --- a/src/db/imap_state.rs +++ /dev/null @@ -1,110 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use rusqlite::{Connection, OptionalExtension, params}; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FolderState { - pub uidvalidity: u32, - pub uidnext: u32, - pub last_seen: String, -} - -pub fn get( - conn: &Connection, - source_id: i64, - folder: &str, -) -> Result, rusqlite::Error> { - conn.query_row( - "SELECT uidvalidity, uidnext, last_seen FROM imap_folder_state - WHERE source_id = ?1 AND folder = ?2", - params![source_id, folder], - |row| { - Ok(FolderState { - uidvalidity: row.get(0)?, - uidnext: row.get(1)?, - last_seen: row.get(2)?, - }) - }, - ) - .optional() -} - -pub fn upsert( - conn: &Connection, - source_id: i64, - folder: &str, - uidvalidity: u32, - uidnext: u32, - last_seen_rfc3339: &str, -) -> Result<(), rusqlite::Error> { - conn.execute( - "INSERT INTO imap_folder_state (source_id, folder, uidvalidity, uidnext, last_seen) - VALUES (?1, ?2, ?3, ?4, ?5) - ON CONFLICT (source_id, folder) - DO UPDATE SET uidvalidity = excluded.uidvalidity, - uidnext = excluded.uidnext, - last_seen = excluded.last_seen", - params![source_id, folder, uidvalidity, uidnext, last_seen_rfc3339], - )?; - Ok(()) -} - -pub fn delete(conn: &Connection, source_id: i64, folder: &str) -> Result<(), rusqlite::Error> { - conn.execute( - "DELETE FROM imap_folder_state WHERE source_id = ?1 AND folder = ?2", - params![source_id, folder], - )?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::init; - use crate::db::sources::{SourceKey, upsert_source}; - - fn setup() -> (Connection, i64) { - let c = Connection::open_in_memory().unwrap(); - init::apply_schema(&c).unwrap(); - let sid = upsert_source( - &c, - &SourceKey { - kind: "imap".to_owned(), - session_url: "imaps://host:993".to_owned(), - account_id: "alice".to_owned(), - }, - None, - "alice", - ) - .unwrap(); - (c, sid) - } - - #[test] - fn upsert_then_get_returns_latest() { - let (c, sid) = setup(); - assert!(get(&c, sid, "INBOX").unwrap().is_none()); - upsert(&c, sid, "INBOX", 100, 50, "2026-05-24T00:00:00Z").unwrap(); - let st = get(&c, sid, "INBOX").unwrap().unwrap(); - assert_eq!(st.uidvalidity, 100); - assert_eq!(st.uidnext, 50); - upsert(&c, sid, "INBOX", 100, 75, "2026-05-24T01:00:00Z").unwrap(); - let st = get(&c, sid, "INBOX").unwrap().unwrap(); - assert_eq!(st.uidnext, 75); - assert_eq!(st.last_seen, "2026-05-24T01:00:00Z"); - } - - #[test] - fn delete_removes_only_named_folder() { - let (c, sid) = setup(); - upsert(&c, sid, "INBOX", 1, 1, "2026-05-24T00:00:00Z").unwrap(); - upsert(&c, sid, "Sent", 2, 1, "2026-05-24T00:00:00Z").unwrap(); - delete(&c, sid, "INBOX").unwrap(); - assert!(get(&c, sid, "INBOX").unwrap().is_none()); - assert!(get(&c, sid, "Sent").unwrap().is_some()); - } -} diff --git a/src/db/init.rs b/src/db/init.rs deleted file mode 100644 index 84b8d16..0000000 --- a/src/db/init.rs +++ /dev/null @@ -1,107 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use rusqlite::Connection; - -pub const SCHEMA_SQL: &str = include_str!("schema.sql"); - -pub fn open(path: &std::path::Path) -> Result { - let conn = Connection::open(path)?; - apply_pragmas(&conn)?; - apply_schema(&conn)?; - Ok(conn) -} - -pub fn apply_schema(conn: &Connection) -> Result<(), OpenError> { - let tx = conn.unchecked_transaction()?; - tx.execute_batch(SCHEMA_SQL)?; - ensure_calendar_events_data_type(&tx)?; - ensure_file_nodes_executable(&tx)?; - ensure_graph_ids_accept_file_nodes(&tx)?; - tx.commit()?; - Ok(()) -} - -fn ensure_calendar_events_data_type(conn: &Connection) -> Result<(), OpenError> { - ensure_column( - conn, - "calendar_events", - "data_type", - "ALTER TABLE calendar_events ADD COLUMN data_type TEXT NOT NULL DEFAULT 'Event'", - ) -} - -fn ensure_file_nodes_executable(conn: &Connection) -> Result<(), OpenError> { - ensure_column( - conn, - "file_nodes", - "executable", - "ALTER TABLE file_nodes ADD COLUMN executable INTEGER NOT NULL DEFAULT 0", - ) -} - -fn ensure_column( - conn: &Connection, - table: &str, - column: &str, - alter: &str, -) -> Result<(), OpenError> { - let mut stmt = conn.prepare(&format!("PRAGMA table_info({table})"))?; - let rows = stmt.query_map([], |row| row.get::<_, String>(1))?; - let has_column = rows.filter_map(|r| r.ok()).any(|name| name == column); - if !has_column { - conn.execute(alter, [])?; - } - Ok(()) -} - -fn ensure_graph_ids_accept_file_nodes(conn: &Connection) -> Result<(), OpenError> { - let sql: Option = conn - .query_row( - "SELECT sql FROM sqlite_master WHERE type = 'table' \ - AND name = 'sync_id_exchange_graph'", - [], - |row| row.get(0), - ) - .ok(); - let Some(sql) = sql else { return Ok(()) }; - if sql.contains("'filenode'") { - return Ok(()); - } - conn.execute_batch( - "ALTER TABLE sync_id_exchange_graph RENAME TO sync_id_exchange_graph_old; - CREATE TABLE sync_id_exchange_graph ( - source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE, - type_name TEXT NOT NULL CHECK (type_name IN ( - 'mailbox','email', - 'calendar','calendarevent', - 'addressbook','contactcard', - 'filenode')), - graph_id TEXT NOT NULL, - local_id INTEGER NOT NULL, - PRIMARY KEY (source_id, type_name, graph_id), - UNIQUE (source_id, type_name, local_id) - ); - INSERT INTO sync_id_exchange_graph SELECT * FROM sync_id_exchange_graph_old; - DROP TABLE sync_id_exchange_graph_old; - CREATE INDEX IF NOT EXISTS sync_id_exchange_graph_type_idx - ON sync_id_exchange_graph (source_id, type_name);", - )?; - Ok(()) -} - -fn apply_pragmas(conn: &Connection) -> Result<(), OpenError> { - conn.pragma_update(None, "journal_mode", "WAL")?; - conn.pragma_update(None, "foreign_keys", "ON")?; - conn.pragma_update(None, "locking_mode", "EXCLUSIVE")?; - Ok(()) -} - -#[derive(Debug, thiserror::Error)] -pub enum OpenError { - #[error("sqlite error: {0}")] - Sqlite(#[from] rusqlite::Error), -} diff --git a/src/db/maildir_ids.rs b/src/db/maildir_ids.rs deleted file mode 100644 index a3fd905..0000000 --- a/src/db/maildir_ids.rs +++ /dev/null @@ -1,269 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::{HashMap, HashSet}; - -use rusqlite::{Connection, OptionalExtension, params}; - -const MAILBOX: &str = "mailbox"; -const EMAIL: &str = "email"; - -pub fn insert_mailbox( - conn: &Connection, - source_id: i64, - folder: &str, - local_id: i64, -) -> Result<(), rusqlite::Error> { - conn.execute( - "INSERT INTO sync_id_maildir - (source_id, type_name, folder, unique_id, local_id) - VALUES (?1, ?2, ?3, '', ?4)", - params![source_id, MAILBOX, folder, local_id], - )?; - Ok(()) -} - -pub fn insert_email( - conn: &Connection, - source_id: i64, - folder: &str, - unique_id: &str, - local_id: i64, -) -> Result<(), rusqlite::Error> { - conn.execute( - "INSERT INTO sync_id_maildir - (source_id, type_name, folder, unique_id, local_id) - VALUES (?1, ?2, ?3, ?4, ?5)", - params![source_id, EMAIL, folder, unique_id, local_id], - )?; - Ok(()) -} - -pub fn local_for_mailbox( - conn: &Connection, - source_id: i64, - folder: &str, -) -> Result, rusqlite::Error> { - conn.query_row( - "SELECT local_id FROM sync_id_maildir - WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3 AND unique_id = ''", - params![source_id, MAILBOX, folder], - |row| row.get(0), - ) - .optional() -} - -pub fn local_for_email( - conn: &Connection, - source_id: i64, - folder: &str, - unique_id: &str, -) -> Result, rusqlite::Error> { - conn.query_row( - "SELECT local_id FROM sync_id_maildir - WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3 AND unique_id = ?4", - params![source_id, EMAIL, folder, unique_id], - |row| row.get(0), - ) - .optional() -} - -pub fn mailbox_folders( - conn: &Connection, - source_id: i64, -) -> Result, rusqlite::Error> { - let mut stmt = conn.prepare( - "SELECT folder, local_id FROM sync_id_maildir - WHERE source_id = ?1 AND type_name = ?2 AND unique_id = ''", - )?; - let rows = stmt.query_map(params![source_id, MAILBOX], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) - })?; - let mut map = HashMap::new(); - for r in rows { - let (f, l) = r?; - map.insert(f, l); - } - Ok(map) -} - -pub fn email_ids_in_folder( - conn: &Connection, - source_id: i64, - folder: &str, -) -> Result, rusqlite::Error> { - let mut stmt = conn.prepare( - "SELECT unique_id, local_id FROM sync_id_maildir - WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3", - )?; - let rows = stmt.query_map(params![source_id, EMAIL, folder], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) - })?; - let mut map = HashMap::new(); - for r in rows { - let (u, l) = r?; - map.insert(u, l); - } - Ok(map) -} - -pub fn delete_mailbox( - conn: &Connection, - source_id: i64, - folder: &str, -) -> Result<(), rusqlite::Error> { - conn.execute( - "DELETE FROM sync_id_maildir - WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3 AND unique_id = ''", - params![source_id, MAILBOX, folder], - )?; - Ok(()) -} - -pub fn delete_email( - conn: &Connection, - source_id: i64, - folder: &str, - unique_id: &str, -) -> Result<(), rusqlite::Error> { - conn.execute( - "DELETE FROM sync_id_maildir - WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3 AND unique_id = ?4", - params![source_id, EMAIL, folder, unique_id], - )?; - Ok(()) -} - -pub fn delete_all_emails_in_folder( - conn: &Connection, - source_id: i64, - folder: &str, -) -> Result { - conn.execute( - "DELETE FROM sync_id_maildir - WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3", - params![source_id, EMAIL, folder], - ) -} - -pub fn folders_with_emails( - conn: &Connection, - source_id: i64, -) -> Result, rusqlite::Error> { - let mut stmt = conn.prepare( - "SELECT DISTINCT folder FROM sync_id_maildir - WHERE source_id = ?1 AND type_name = ?2", - )?; - let rows = stmt.query_map(params![source_id, EMAIL], |row| row.get::<_, String>(0))?; - let mut out = HashSet::new(); - for r in rows { - out.insert(r?); - } - Ok(out) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::init; - use crate::db::sources::{SourceKey, upsert_source}; - - fn setup() -> (Connection, i64) { - let c = Connection::open_in_memory().unwrap(); - init::apply_schema(&c).unwrap(); - let sid = upsert_source( - &c, - &SourceKey { - kind: "maildir".to_owned(), - session_url: "file:///home/alice/Maildir".to_owned(), - account_id: "/home/alice/Maildir".to_owned(), - }, - Some("Maildir"), - "", - ) - .unwrap(); - (c, sid) - } - - #[test] - fn mailbox_roundtrip() { - let (c, sid) = setup(); - insert_mailbox(&c, sid, "INBOX", 7).unwrap(); - insert_mailbox(&c, sid, "Archive.2025", 8).unwrap(); - assert_eq!(local_for_mailbox(&c, sid, "INBOX").unwrap(), Some(7)); - let all = mailbox_folders(&c, sid).unwrap(); - assert_eq!(all.len(), 2); - assert_eq!(all.get("Archive.2025"), Some(&8)); - delete_mailbox(&c, sid, "INBOX").unwrap(); - assert_eq!(local_for_mailbox(&c, sid, "INBOX").unwrap(), None); - } - - #[test] - fn email_roundtrip() { - let (c, sid) = setup(); - insert_email(&c, sid, "INBOX", "1739471123.M001.host", 1).unwrap(); - insert_email(&c, sid, "INBOX", "1739471200.M002.host", 2).unwrap(); - insert_email(&c, sid, "Sent", "1739471300.M003.host", 3).unwrap(); - assert_eq!( - local_for_email(&c, sid, "INBOX", "1739471123.M001.host").unwrap(), - Some(1) - ); - let inbox = email_ids_in_folder(&c, sid, "INBOX").unwrap(); - assert_eq!(inbox.len(), 2); - assert_eq!(inbox.get("1739471123.M001.host"), Some(&1)); - let folders = folders_with_emails(&c, sid).unwrap(); - assert!(folders.contains("INBOX")); - assert!(folders.contains("Sent")); - } - - #[test] - fn delete_email_removes_only_that_row() { - let (c, sid) = setup(); - insert_email(&c, sid, "INBOX", "u1", 1).unwrap(); - insert_email(&c, sid, "INBOX", "u2", 2).unwrap(); - delete_email(&c, sid, "INBOX", "u1").unwrap(); - let inbox = email_ids_in_folder(&c, sid, "INBOX").unwrap(); - assert_eq!(inbox.len(), 1); - assert!(inbox.contains_key("u2")); - } - - #[test] - fn delete_all_emails_in_folder_wipes_only_that_folder() { - let (c, sid) = setup(); - insert_email(&c, sid, "INBOX", "u1", 1).unwrap(); - insert_email(&c, sid, "INBOX", "u2", 2).unwrap(); - insert_email(&c, sid, "Sent", "u3", 3).unwrap(); - let removed = delete_all_emails_in_folder(&c, sid, "INBOX").unwrap(); - assert_eq!(removed, 2); - assert!(email_ids_in_folder(&c, sid, "INBOX").unwrap().is_empty()); - assert_eq!(email_ids_in_folder(&c, sid, "Sent").unwrap().len(), 1); - } - - #[test] - fn mailbox_and_email_namespaces_do_not_collide_on_local_id() { - let (c, sid) = setup(); - insert_mailbox(&c, sid, "INBOX", 42).unwrap(); - insert_email(&c, sid, "INBOX", "u1", 42).unwrap(); - } - - #[test] - fn re_insert_with_same_unique_id_is_rejected_by_pk() { - let (c, sid) = setup(); - insert_email(&c, sid, "INBOX", "u1", 1).unwrap(); - let err = insert_email(&c, sid, "INBOX", "u1", 2).expect_err("PK conflict"); - assert!(matches!(err, rusqlite::Error::SqliteFailure(_, _))); - assert_eq!(local_for_email(&c, sid, "INBOX", "u1").unwrap(), Some(1)); - assert_eq!(email_ids_in_folder(&c, sid, "INBOX").unwrap().len(), 1); - } - - #[test] - fn re_insert_with_same_local_id_is_rejected_by_unique_constraint() { - let (c, sid) = setup(); - insert_email(&c, sid, "INBOX", "u1", 42).unwrap(); - let err = insert_email(&c, sid, "INBOX", "u2", 42).expect_err("UNIQUE conflict"); - assert!(matches!(err, rusqlite::Error::SqliteFailure(_, _))); - } -} diff --git a/src/db/managesieve_ids.rs b/src/db/managesieve_ids.rs deleted file mode 100644 index d55d100..0000000 --- a/src/db/managesieve_ids.rs +++ /dev/null @@ -1,129 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashMap; - -use rusqlite::{Connection, OptionalExtension, params}; - -pub fn insert( - conn: &Connection, - source_id: i64, - name: &str, - local_id: i64, -) -> Result<(), rusqlite::Error> { - conn.execute( - "INSERT OR REPLACE INTO sync_id_managesieve (source_id, name, local_id) - VALUES (?1, ?2, ?3)", - params![source_id, name, local_id], - )?; - Ok(()) -} - -pub fn local_for( - conn: &Connection, - source_id: i64, - name: &str, -) -> Result, rusqlite::Error> { - conn.query_row( - "SELECT local_id FROM sync_id_managesieve WHERE source_id = ?1 AND name = ?2", - params![source_id, name], - |row| row.get(0), - ) - .optional() -} - -pub fn all_names( - conn: &Connection, - source_id: i64, -) -> Result, rusqlite::Error> { - let mut stmt = - conn.prepare("SELECT name, local_id FROM sync_id_managesieve WHERE source_id = ?1")?; - let rows = stmt.query_map(params![source_id], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) - })?; - let mut out = HashMap::new(); - for r in rows { - let (n, l) = r?; - out.insert(n, l); - } - Ok(out) -} - -pub fn delete(conn: &Connection, source_id: i64, name: &str) -> Result<(), rusqlite::Error> { - conn.execute( - "DELETE FROM sync_id_managesieve WHERE source_id = ?1 AND name = ?2", - params![source_id, name], - )?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::init; - use crate::db::sources::{SourceKey, upsert_source}; - - fn setup() -> (Connection, i64) { - let c = Connection::open_in_memory().unwrap(); - init::apply_schema(&c).unwrap(); - let sid = upsert_source( - &c, - &SourceKey { - kind: "managesieve".to_owned(), - session_url: "sieve://host:4190".to_owned(), - account_id: "alice".to_owned(), - }, - None, - "alice", - ) - .unwrap(); - (c, sid) - } - - #[test] - fn insert_and_lookup_by_name() { - let (c, sid) = setup(); - insert(&c, sid, "vacation", 7).unwrap(); - assert_eq!(local_for(&c, sid, "vacation").unwrap(), Some(7)); - assert_eq!(local_for(&c, sid, "missing").unwrap(), None); - } - - #[test] - fn all_names_returns_full_map() { - let (c, sid) = setup(); - insert(&c, sid, "a", 1).unwrap(); - insert(&c, sid, "b", 2).unwrap(); - let map = all_names(&c, sid).unwrap(); - assert_eq!(map.len(), 2); - assert_eq!(map.get("a"), Some(&1)); - } - - #[test] - fn delete_removes_row() { - let (c, sid) = setup(); - insert(&c, sid, "a", 1).unwrap(); - delete(&c, sid, "a").unwrap(); - assert_eq!(local_for(&c, sid, "a").unwrap(), None); - } - - #[test] - fn re_insert_with_same_name_replaces_local_id() { - let (c, sid) = setup(); - insert(&c, sid, "a", 1).unwrap(); - insert(&c, sid, "a", 2).unwrap(); - assert_eq!(local_for(&c, sid, "a").unwrap(), Some(2)); - assert_eq!(all_names(&c, sid).unwrap().len(), 1); - } - - #[test] - fn insert_or_replace_reassigns_when_local_id_clashes() { - let (c, sid) = setup(); - insert(&c, sid, "a", 1).unwrap(); - insert(&c, sid, "b", 1).unwrap(); - assert!(local_for(&c, sid, "a").unwrap().is_none()); - assert_eq!(local_for(&c, sid, "b").unwrap(), Some(1)); - } -} diff --git a/src/db/mod.rs b/src/db/mod.rs deleted file mode 100644 index 85dadab..0000000 --- a/src/db/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -pub mod blobs; -pub mod dav_ids; -pub mod defaults; -pub mod exchange_ews_ids; -pub mod exchange_graph_ids; -pub mod ids; -pub mod imap_ids; -pub mod imap_state; -pub mod init; -pub mod maildir_ids; -pub mod managesieve_ids; -pub mod roles; -pub mod sources; -pub mod sync_state_jmap; -pub mod takeout_ids; -pub mod vacation_response; diff --git a/src/db/roles.rs b/src/db/roles.rs deleted file mode 100644 index 45de76d..0000000 --- a/src/db/roles.rs +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use rusqlite::{Connection, params}; - -pub fn unique_role( - conn: &Connection, - role: Option<&str>, - exclude_id: Option, -) -> rusqlite::Result> { - let Some(r) = role else { - return Ok(None); - }; - let taken: bool = conn.query_row( - "SELECT EXISTS(SELECT 1 FROM mailboxes WHERE role = ?1 AND (?2 IS NULL OR id != ?2))", - params![r, exclude_id], - |row| row.get(0), - )?; - Ok((!taken).then(|| r.to_owned())) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::init; - - fn mem() -> Connection { - let conn = Connection::open_in_memory().unwrap(); - init::apply_schema(&conn).unwrap(); - conn - } - - fn insert(conn: &Connection, name: &str, role: Option<&str>) -> i64 { - conn.execute( - "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) - VALUES (?1, NULL, ?2, 0, 1)", - params![name, role], - ) - .unwrap(); - conn.last_insert_rowid() - } - - #[test] - fn first_claimant_keeps_role_second_is_dropped() { - let conn = mem(); - let first = unique_role(&conn, Some("sent"), None).unwrap(); - assert_eq!(first.as_deref(), Some("sent")); - insert(&conn, "Sent", first.as_deref()); - - let second = unique_role(&conn, Some("sent"), None).unwrap(); - assert_eq!(second, None); - } - - #[test] - fn null_role_stays_null() { - let conn = mem(); - assert_eq!(unique_role(&conn, None, None).unwrap(), None); - } - - #[test] - fn distinct_roles_are_independent() { - let conn = mem(); - insert(&conn, "Sent", Some("sent")); - assert_eq!( - unique_role(&conn, Some("trash"), None).unwrap().as_deref(), - Some("trash") - ); - } - - #[test] - fn update_excludes_own_row() { - let conn = mem(); - let id = insert(&conn, "Sent", Some("sent")); - assert_eq!( - unique_role(&conn, Some("sent"), Some(id)) - .unwrap() - .as_deref(), - Some("sent") - ); - } - - #[test] - fn update_detects_other_holder() { - let conn = mem(); - insert(&conn, "Sent", Some("sent")); - let other = insert(&conn, "Envoyés", None); - assert_eq!(unique_role(&conn, Some("sent"), Some(other)).unwrap(), None); - } -} diff --git a/src/db/sources.rs b/src/db/sources.rs deleted file mode 100644 index f55bd9b..0000000 --- a/src/db/sources.rs +++ /dev/null @@ -1,142 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use rusqlite::{Connection, OptionalExtension, params}; - -#[derive(Debug, Clone)] -pub struct SourceKey { - pub kind: String, - pub session_url: String, - pub account_id: String, -} - -pub fn find_source(conn: &Connection, key: &SourceKey) -> Result, rusqlite::Error> { - conn.query_row( - "SELECT id FROM sources WHERE kind = ?1 AND session_url = ?2 AND account_id = ?3", - params![key.kind, key.session_url, key.account_id], - |row| row.get(0), - ) - .optional() -} - -pub fn conflicting_source( - conn: &Connection, - kind: &str, - session_url: &str, - account_id: &str, -) -> Result, rusqlite::Error> { - conn.query_row( - "SELECT session_url, account_id FROM sources - WHERE kind = ?1 AND NOT (session_url = ?2 AND account_id = ?3) LIMIT 1", - params![kind, session_url, account_id], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .optional() -} - -pub fn other_sources( - conn: &Connection, - kind: &str, - session_url: &str, - account_id: &str, -) -> Result, rusqlite::Error> { - let mut stmt = conn.prepare( - "SELECT id, session_url, account_id FROM sources - WHERE kind = ?1 AND NOT (session_url = ?2 AND account_id = ?3) ORDER BY id", - )?; - stmt.query_map(params![kind, session_url, account_id], |row| { - Ok((row.get(0)?, row.get(1)?, row.get(2)?)) - })? - .collect() -} - -pub fn delete_source(conn: &Connection, id: i64) -> Result<(), rusqlite::Error> { - conn.execute("DELETE FROM sources WHERE id = ?1", params![id])?; - Ok(()) -} - -pub fn upsert_source( - conn: &Connection, - key: &SourceKey, - account_name: Option<&str>, - username: &str, -) -> Result { - conn.execute( - "INSERT INTO sources (kind, session_url, account_id, account_name, username) - VALUES (?1, ?2, ?3, ?4, ?5) - ON CONFLICT (kind, session_url, account_id) - DO UPDATE SET account_name = excluded.account_name, username = excluded.username", - params![ - key.kind, - key.session_url, - key.account_id, - account_name, - username - ], - )?; - find_source(conn, key).map(|opt| opt.unwrap_or_else(|| conn.last_insert_rowid())) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::init; - - fn mem() -> Connection { - let c = Connection::open_in_memory().unwrap(); - init::apply_schema(&c).unwrap(); - c - } - - fn key(url: &str, acc: &str) -> SourceKey { - SourceKey { - kind: "jmap".to_owned(), - session_url: url.to_owned(), - account_id: acc.to_owned(), - } - } - - #[test] - fn upsert_is_idempotent_and_returns_stable_id() { - let c = mem(); - let k = key("https://a/jmap", "w"); - let id1 = upsert_source(&c, &k, Some("alice"), "alice").unwrap(); - let id2 = upsert_source(&c, &k, Some("alice2"), "alice").unwrap(); - assert_eq!(id1, id2); - assert!(find_source(&c, &k).unwrap().is_some()); - } - - #[test] - fn conflicting_source_detects_a_different_account() { - let c = mem(); - upsert_source(&c, &key("https://a/jmap", "w"), None, "alice").unwrap(); - assert!( - conflicting_source(&c, "jmap", "https://a/jmap", "w") - .unwrap() - .is_none() - ); - let conflict = conflicting_source(&c, "jmap", "https://b/jmap", "v") - .unwrap() - .unwrap(); - assert_eq!(conflict, ("https://a/jmap".to_owned(), "w".to_owned())); - } - - #[test] - fn other_sources_lists_every_other_source_of_the_kind() { - let c = mem(); - let a = upsert_source(&c, &key("https://a/jmap", "w"), None, "alice").unwrap(); - let b = upsert_source(&c, &key("https://b/jmap", "w"), None, "alice").unwrap(); - upsert_source(&c, &key("https://c/jmap", "w"), None, "alice").unwrap(); - let others = other_sources(&c, "jmap", "https://c/jmap", "w").unwrap(); - let ids: Vec = others.iter().map(|(id, _, _)| *id).collect(); - assert_eq!(ids, [a, b]); - assert!( - other_sources(&c, "imap", "https://c/jmap", "w") - .unwrap() - .is_empty() - ); - } -} diff --git a/src/db/sync_state_jmap.rs b/src/db/sync_state_jmap.rs deleted file mode 100644 index 7cd62db..0000000 --- a/src/db/sync_state_jmap.rs +++ /dev/null @@ -1,87 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use rusqlite::{Connection, OptionalExtension, params}; - -use crate::types::ObjectType; - -pub fn get( - conn: &Connection, - source_id: i64, - ty: ObjectType, -) -> Result, rusqlite::Error> { - conn.query_row( - "SELECT state FROM sync_state_jmap WHERE source_id = ?1 AND type_name = ?2", - params![source_id, ty.jmap_name()], - |row| row.get(0), - ) - .optional() -} - -pub fn upsert( - conn: &Connection, - source_id: i64, - ty: ObjectType, - state: &str, -) -> Result<(), rusqlite::Error> { - conn.execute( - "INSERT INTO sync_state_jmap (source_id, type_name, state) - VALUES (?1, ?2, ?3) - ON CONFLICT (source_id, type_name) - DO UPDATE SET state = excluded.state", - params![source_id, ty.jmap_name(), state], - )?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::init; - use crate::db::sources::{SourceKey, upsert_source}; - - fn setup() -> (Connection, i64) { - let c = Connection::open_in_memory().unwrap(); - init::apply_schema(&c).unwrap(); - let sid = upsert_source( - &c, - &SourceKey { - kind: "jmap".to_owned(), - session_url: "https://host".to_owned(), - account_id: "alice".to_owned(), - }, - None, - "alice", - ) - .unwrap(); - (c, sid) - } - - #[test] - fn upsert_then_get_returns_latest_per_type() { - let (c, sid) = setup(); - assert!(get(&c, sid, ObjectType::Email).unwrap().is_none()); - upsert(&c, sid, ObjectType::Email, "s1").unwrap(); - upsert(&c, sid, ObjectType::Mailbox, "m1").unwrap(); - assert_eq!( - get(&c, sid, ObjectType::Email).unwrap().as_deref(), - Some("s1") - ); - assert_eq!( - get(&c, sid, ObjectType::Mailbox).unwrap().as_deref(), - Some("m1") - ); - upsert(&c, sid, ObjectType::Email, "s2").unwrap(); - assert_eq!( - get(&c, sid, ObjectType::Email).unwrap().as_deref(), - Some("s2") - ); - assert_eq!( - get(&c, sid, ObjectType::Mailbox).unwrap().as_deref(), - Some("m1") - ); - } -} diff --git a/src/db/takeout_ids.rs b/src/db/takeout_ids.rs deleted file mode 100644 index 0fda03f..0000000 --- a/src/db/takeout_ids.rs +++ /dev/null @@ -1,177 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashMap; - -use rusqlite::{Connection, OptionalExtension, params}; - -pub const MAILBOX: &str = "mailbox"; -pub const EMAIL: &str = "email"; -pub const ADDRESS_BOOK: &str = "addressbook"; -pub const CONTACT_CARD: &str = "contactcard"; -pub const CALENDAR: &str = "calendar"; -pub const CALENDAR_EVENT: &str = "calendarevent"; - -pub fn insert( - conn: &Connection, - source_id: i64, - type_name: &str, - source_obj_id: &str, - local_id: i64, -) -> Result<(), rusqlite::Error> { - conn.execute( - "INSERT INTO sync_id_takeout - (source_id, type_name, source_obj_id, local_id) - VALUES (?1, ?2, ?3, ?4)", - params![source_id, type_name, source_obj_id, local_id], - )?; - Ok(()) -} - -pub fn local_for( - conn: &Connection, - source_id: i64, - type_name: &str, - source_obj_id: &str, -) -> Result, rusqlite::Error> { - conn.query_row( - "SELECT local_id FROM sync_id_takeout - WHERE source_id = ?1 AND type_name = ?2 AND source_obj_id = ?3", - params![source_id, type_name, source_obj_id], - |row| row.get(0), - ) - .optional() -} - -pub fn delete( - conn: &Connection, - source_id: i64, - type_name: &str, - source_obj_id: &str, -) -> Result<(), rusqlite::Error> { - conn.execute( - "DELETE FROM sync_id_takeout - WHERE source_id = ?1 AND type_name = ?2 AND source_obj_id = ?3", - params![source_id, type_name, source_obj_id], - )?; - Ok(()) -} - -pub fn all_for_type( - conn: &Connection, - source_id: i64, - type_name: &str, -) -> Result, rusqlite::Error> { - let mut stmt = conn.prepare( - "SELECT source_obj_id, local_id FROM sync_id_takeout - WHERE source_id = ?1 AND type_name = ?2", - )?; - let rows = stmt.query_map(params![source_id, type_name], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) - })?; - let mut map = HashMap::new(); - for r in rows { - let (k, v) = r?; - map.insert(k, v); - } - Ok(map) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::init; - use crate::db::sources::{SourceKey, upsert_source}; - - fn setup() -> (Connection, i64) { - let c = Connection::open_in_memory().unwrap(); - init::apply_schema(&c).unwrap(); - let sid = upsert_source( - &c, - &SourceKey { - kind: "takeout".to_owned(), - session_url: "file:///tmp/Takeout".to_owned(), - account_id: "/tmp/Takeout".to_owned(), - }, - Some("Takeout"), - "", - ) - .unwrap(); - (c, sid) - } - - #[test] - fn insert_and_lookup_roundtrip() { - let (c, sid) = setup(); - insert(&c, sid, MAILBOX, "Inbox", 7).unwrap(); - insert(&c, sid, MAILBOX, "Label/Sub", 8).unwrap(); - assert_eq!(local_for(&c, sid, MAILBOX, "Inbox").unwrap(), Some(7)); - assert_eq!(local_for(&c, sid, MAILBOX, "Label/Sub").unwrap(), Some(8)); - assert_eq!(local_for(&c, sid, MAILBOX, "Missing").unwrap(), None); - } - - #[test] - fn types_are_isolated_namespaces() { - let (c, sid) = setup(); - insert(&c, sid, MAILBOX, "Inbox", 1).unwrap(); - insert(&c, sid, EMAIL, "ab12", 2).unwrap(); - insert(&c, sid, ADDRESS_BOOK, "Imported", 3).unwrap(); - insert(&c, sid, CONTACT_CARD, "uid-1", 4).unwrap(); - insert(&c, sid, CALENDAR, "Imported", 5).unwrap(); - insert(&c, sid, CALENDAR_EVENT, "uid-e1", 6).unwrap(); - assert_eq!(all_for_type(&c, sid, MAILBOX).unwrap().len(), 1); - assert_eq!(all_for_type(&c, sid, EMAIL).unwrap().len(), 1); - assert_eq!(all_for_type(&c, sid, ADDRESS_BOOK).unwrap().len(), 1); - assert_eq!(all_for_type(&c, sid, CONTACT_CARD).unwrap().len(), 1); - assert_eq!(all_for_type(&c, sid, CALENDAR).unwrap().len(), 1); - assert_eq!(all_for_type(&c, sid, CALENDAR_EVENT).unwrap().len(), 1); - } - - #[test] - fn delete_removes_only_target_row() { - let (c, sid) = setup(); - insert(&c, sid, EMAIL, "hash-a", 1).unwrap(); - insert(&c, sid, EMAIL, "hash-b", 2).unwrap(); - delete(&c, sid, EMAIL, "hash-a").unwrap(); - assert_eq!(local_for(&c, sid, EMAIL, "hash-a").unwrap(), None); - assert_eq!(local_for(&c, sid, EMAIL, "hash-b").unwrap(), Some(2)); - } - - #[test] - fn pk_rejects_duplicate_source_obj_id() { - let (c, sid) = setup(); - insert(&c, sid, EMAIL, "h", 1).unwrap(); - let err = insert(&c, sid, EMAIL, "h", 2).expect_err("PK conflict"); - assert!(matches!(err, rusqlite::Error::SqliteFailure(_, _))); - } - - #[test] - fn unique_constraint_rejects_duplicate_local_id_within_type() { - let (c, sid) = setup(); - insert(&c, sid, EMAIL, "h1", 42).unwrap(); - let err = insert(&c, sid, EMAIL, "h2", 42).expect_err("UNIQUE conflict"); - assert!(matches!(err, rusqlite::Error::SqliteFailure(_, _))); - } - - #[test] - fn same_local_id_allowed_across_different_types() { - let (c, sid) = setup(); - insert(&c, sid, MAILBOX, "Inbox", 42).unwrap(); - insert(&c, sid, EMAIL, "h1", 42).unwrap(); - } - - #[test] - fn all_for_type_returns_full_map() { - let (c, sid) = setup(); - insert(&c, sid, EMAIL, "h1", 1).unwrap(); - insert(&c, sid, EMAIL, "h2", 2).unwrap(); - insert(&c, sid, EMAIL, "h3", 3).unwrap(); - let all = all_for_type(&c, sid, EMAIL).unwrap(); - assert_eq!(all.len(), 3); - assert_eq!(all.get("h1"), Some(&1)); - assert_eq!(all.get("h3"), Some(&3)); - } -} diff --git a/src/db/vacation_response.rs b/src/db/vacation_response.rs deleted file mode 100644 index 18a6607..0000000 --- a/src/db/vacation_response.rs +++ /dev/null @@ -1,66 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use rusqlite::{Connection, OptionalExtension, params}; - -use crate::jmap::wire::vacation_response::VacationResponse; - -pub fn store( - conn: &Connection, - source_id: i64, - vacation: &VacationResponse, -) -> Result<(), rusqlite::Error> { - conn.execute( - "INSERT OR REPLACE INTO vacation_response - (id, source_id, is_enabled, from_date, to_date, subject, text_body, html_body) - VALUES (1, ?1, ?2, ?3, ?4, ?5, ?6, ?7)", - params![ - source_id, - vacation.is_enabled, - vacation.from_date, - vacation.to_date, - vacation.subject, - vacation.text_body, - vacation.html_body, - ], - )?; - Ok(()) -} - -pub fn clear(conn: &Connection, source_id: i64) -> Result<(), rusqlite::Error> { - conn.execute( - "DELETE FROM vacation_response WHERE source_id = ?1", - params![source_id], - )?; - Ok(()) -} - -pub fn exists(conn: &Connection) -> Result { - conn.query_row( - "SELECT EXISTS(SELECT 1 FROM vacation_response)", - [], - |row| row.get(0), - ) -} - -pub fn load(conn: &Connection) -> Result, rusqlite::Error> { - conn.query_row( - "SELECT is_enabled, from_date, to_date, subject, text_body, html_body - FROM vacation_response WHERE id = 1", - [], - |row| { - Ok(VacationResponse { - is_enabled: row.get(0)?, - from_date: row.get(1)?, - to_date: row.get(2)?, - subject: row.get(3)?, - text_body: row.get(4)?, - html_body: row.get(5)?, - }) - }, - ) - .optional() -} diff --git a/src/error.rs b/src/error.rs index d45e1f0..9d4fa90 100644 --- a/src/error.rs +++ b/src/error.rs @@ -4,14 +4,17 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use crate::db::init::OpenError; +use std::fmt::Display; +use std::io; + +use crate::archive::ArchiveError; #[derive(Debug, thiserror::Error)] pub enum Error { - #[error("usage error: {0}")] + #[error("{0}")] Usage(String), - #[error("connection error: {0}")] + #[error("connection failed: {0}")] Connection(String), #[error("account resolution failed: {0}")] @@ -29,20 +32,28 @@ pub enum Error { #[error("not yet implemented: {0}")] Unimplemented(&'static str), - #[error("archive error: {0}")] - Db(#[from] OpenError), + #[error("archive: {0}")] + Db(#[from] ArchiveError), - #[error("i/o error: {0}")] - Io(#[from] std::io::Error), + #[error("i/o: {0}")] + Io(#[from] io::Error), } impl From for Error { fn from(e: rusqlite::Error) -> Self { - Error::Db(OpenError::Sqlite(e)) + Error::Db(ArchiveError::Sqlite(e)) } } impl Error { + pub fn partial(e: impl Display) -> Error { + Error::Partial(e.to_string()) + } + + pub fn connection(e: impl Display) -> Error { + Error::Connection(e.to_string()) + } + pub fn aborts_run(&self) -> bool { match self { Error::Partial(_) => false, @@ -57,36 +68,81 @@ impl Error { } } - pub fn exit_code(&self) -> i32 { + pub fn exit_status(&self) -> ExitStatus { match self { - Error::Usage(_) => 1, - Error::Connection(_) => 2, - Error::Account(_) => 3, - Error::SourceChange(_) => 4, - Error::Partial(_) => 5, - Error::PruneAborted => 6, - Error::Unimplemented(_) => 1, - Error::Db(_) => 7, - Error::Io(_) => 7, + Error::Usage(_) | Error::Unimplemented(_) => ExitStatus::Usage, + Error::Connection(_) => ExitStatus::Connection, + Error::Account(_) => ExitStatus::Account, + Error::SourceChange(_) => ExitStatus::SourceChange, + Error::Partial(_) => ExitStatus::Partial, + Error::PruneAborted => ExitStatus::PruneAborted, + Error::Db(_) | Error::Io(_) => ExitStatus::Archive, } } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i32)] +pub enum ExitStatus { + Ok = 0, + Usage = 1, + Connection = 2, + Account = 3, + SourceChange = 4, + Partial = 5, + PruneAborted = 6, + Archive = 7, +} + +impl ExitStatus { + pub fn code(self) -> i32 { + self as i32 + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn exit_codes_match_cli_md_table() { - assert_eq!(Error::Usage("x".into()).exit_code(), 1); - assert_eq!(Error::Unimplemented("x").exit_code(), 1); - assert_eq!(Error::Connection("x".into()).exit_code(), 2); - assert_eq!(Error::Account("x".into()).exit_code(), 3); - assert_eq!(Error::SourceChange("x".into()).exit_code(), 4); - assert_eq!(Error::Partial("x".into()).exit_code(), 5); - assert_eq!(Error::PruneAborted.exit_code(), 6); - let io = Error::Io(std::io::Error::other("disk")); - assert_eq!(io.exit_code(), 7); + assert_eq!(Error::Usage("x".into()).exit_status(), ExitStatus::Usage); + assert_eq!(Error::Unimplemented("x").exit_status(), ExitStatus::Usage); + assert_eq!( + Error::Connection("x".into()).exit_status(), + ExitStatus::Connection + ); + assert_eq!( + Error::Account("x".into()).exit_status(), + ExitStatus::Account + ); + assert_eq!( + Error::SourceChange("x".into()).exit_status(), + ExitStatus::SourceChange + ); + assert_eq!( + Error::Partial("x".into()).exit_status(), + ExitStatus::Partial + ); + assert_eq!(Error::PruneAborted.exit_status(), ExitStatus::PruneAborted); + let io = Error::Io(io::Error::other("disk")); + assert_eq!(io.exit_status(), ExitStatus::Archive); + } + + #[test] + fn exit_status_codes_are_stable() { + let codes = [ + ExitStatus::Ok, + ExitStatus::Usage, + ExitStatus::Connection, + ExitStatus::Account, + ExitStatus::SourceChange, + ExitStatus::Partial, + ExitStatus::PruneAborted, + ExitStatus::Archive, + ] + .map(ExitStatus::code); + assert_eq!(codes, [0, 1, 2, 3, 4, 5, 6, 7]); } #[test] @@ -102,8 +158,8 @@ mod tests { assert!(Error::Usage("bad flag".into()).aborts_run()); assert!(Error::Unimplemented("x").aborts_run()); assert!(Error::PruneAborted.aborts_run()); - assert!(Error::Db(OpenError::Sqlite(rusqlite::Error::QueryReturnedNoRows)).aborts_run()); - assert!(Error::Io(std::io::Error::other("disk")).aborts_run()); + assert!(Error::Db(ArchiveError::Sqlite(rusqlite::Error::QueryReturnedNoRows)).aborts_run()); + assert!(Error::Io(io::Error::other("disk")).aborts_run()); } #[test] @@ -112,13 +168,23 @@ mod tests { Error::Connection("x".into()), Error::Account("x".into()), Error::SourceChange("x".into()), - Error::Db(OpenError::Sqlite(rusqlite::Error::QueryReturnedNoRows)), - Error::Io(std::io::Error::other("disk")), + Error::Db(ArchiveError::Sqlite(rusqlite::Error::QueryReturnedNoRows)), + Error::Io(io::Error::other("disk")), ] { assert!(e.aborts_run(), "{e} must abort the run"); - assert_ne!(e.exit_code(), 5, "{e} must not report a partial failure"); + assert_ne!( + e.exit_status(), + ExitStatus::Partial, + "{e} must not report a partial failure" + ); } - assert_eq!(Error::Connection("x".into()).exit_code(), 2); - assert_eq!(Error::Partial("x".into()).exit_code(), 5); + assert_eq!( + Error::Connection("x".into()).exit_status(), + ExitStatus::Connection + ); + assert_eq!( + Error::Partial("x".into()).exit_status(), + ExitStatus::Partial + ); } } diff --git a/src/exchange/address.rs b/src/exchange/address.rs index c12d43d..6a0aeba 100644 --- a/src/exchange/address.rs +++ b/src/exchange/address.rs @@ -5,79 +5,93 @@ */ const ATEXT_SYMBOLS: &[u8] = b"!#$%&'*+-/=?^_`{|}~"; +const ROUTING_PREFIXES: [&str; 2] = ["mailto:", "smtp:"]; const MAX_LOCAL_PART: usize = 64; const MAX_DOMAIN: usize = 255; const MAX_LABEL: usize = 63; -pub fn as_smtp_address(raw: &str) -> Option { - let candidate = normalise(raw)?; - is_addr_spec(&candidate).then_some(candidate) -} +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SmtpAddress(String); -pub fn extract_smtp_address(raw: &str) -> Option { - let candidate = normalise(raw)?; - if !is_addr_spec(&candidate) { - return None; +impl SmtpAddress { + pub fn parse(raw: &str) -> Option { + let candidate = SmtpAddress::normalise(raw)?; + SmtpAddress::is_addr_spec(candidate).then(|| SmtpAddress(candidate.to_owned())) } - let domain = candidate.split('@').next_back()?; - psl::suffix(domain.as_bytes()) - .is_some_and(|s| s.is_known()) - .then_some(candidate) -} -fn normalise(raw: &str) -> Option { - let mut value = raw.trim(); - if let Some(open) = value.rfind('<') - && let Some(close) = value.rfind('>') - && open < close - { - value = value[open + 1..close].trim(); + pub fn extract(raw: &str) -> Option { + let candidate = SmtpAddress::normalise(raw)?; + let (_, domain) = candidate.rsplit_once('@')?; + (SmtpAddress::is_addr_spec(candidate) + && psl::suffix(domain.as_bytes()).is_some_and(|s| s.is_known())) + .then(|| SmtpAddress(candidate.to_owned())) } - for prefix in ["mailto:", "smtp:"] { - if value.len() > prefix.len() && value[..prefix.len()].eq_ignore_ascii_case(prefix) { - value = value[prefix.len()..].trim(); + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn into_string(self) -> String { + self.0 + } + + fn normalise(raw: &str) -> Option<&str> { + let mut value = raw.trim(); + if let Some((_, after)) = value.rsplit_once('<') + && let Some((inner, _)) = after.rsplit_once('>') + { + value = inner.trim(); } + for prefix in ROUTING_PREFIXES { + if value.len() > prefix.len() + && let Some(head) = value.get(..prefix.len()) + && let Some(rest) = value.get(prefix.len()..) + && head.eq_ignore_ascii_case(prefix) + { + value = rest.trim(); + } + } + (!value.is_empty()).then_some(value) } - (!value.is_empty()).then(|| value.to_owned()) -} -fn is_addr_spec(value: &str) -> bool { - if !value.is_ascii() || value.starts_with('/') { - return false; + fn is_addr_spec(value: &str) -> bool { + if !value.is_ascii() || value.starts_with('/') { + return false; + } + let Some((local, domain)) = value.split_once('@') else { + return false; + }; + !domain.contains('@') && SmtpAddress::is_local_part(local) && SmtpAddress::is_domain(domain) } - let Some((local, domain)) = value.split_once('@') else { - return false; - }; - !domain.contains('@') && is_local_part(local) && is_domain(domain) -} -fn is_local_part(local: &str) -> bool { - if local.is_empty() - || local.len() > MAX_LOCAL_PART - || local.starts_with('.') - || local.ends_with('.') - || local.contains("..") - { - return false; + fn is_local_part(local: &str) -> bool { + if local.is_empty() + || local.len() > MAX_LOCAL_PART + || local.starts_with('.') + || local.ends_with('.') + || local.contains("..") + { + return false; + } + local + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'.' || ATEXT_SYMBOLS.contains(&b)) } - local - .bytes() - .all(|b| b.is_ascii_alphanumeric() || b == b'.' || ATEXT_SYMBOLS.contains(&b)) -} -fn is_domain(domain: &str) -> bool { - if domain.is_empty() || domain.len() > MAX_DOMAIN { - return false; + fn is_domain(domain: &str) -> bool { + if domain.is_empty() || domain.len() > MAX_DOMAIN { + return false; + } + domain.split('.').all(|label| { + !label.is_empty() + && label.len() <= MAX_LABEL + && !label.starts_with('-') + && !label.ends_with('-') + && label + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') + }) } - domain.split('.').all(|label| { - !label.is_empty() - && label.len() <= MAX_LABEL - && !label.starts_with('-') - && !label.ends_with('-') - && label - .bytes() - .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') - }) } #[cfg(test)] @@ -90,11 +104,15 @@ mod tests { #[test] fn plain_addresses_are_accepted() { assert_eq!( - as_smtp_address("user@example.com").as_deref(), + SmtpAddress::parse("user@example.com") + .as_ref() + .map(SmtpAddress::as_str), Some("user@example.com") ); assert_eq!( - as_smtp_address(" First.Last@sub.example.co.uk ").as_deref(), + SmtpAddress::parse(" First.Last@sub.example.co.uk ") + .as_ref() + .map(SmtpAddress::as_str), Some("First.Last@sub.example.co.uk") ); } @@ -102,11 +120,13 @@ mod tests { #[test] fn internal_only_tlds_are_kept_when_the_server_says_smtp() { assert_eq!( - as_smtp_address("user@contoso.local").as_deref(), + SmtpAddress::parse("user@contoso.local") + .as_ref() + .map(SmtpAddress::as_str), Some("user@contoso.local") ); assert!( - extract_smtp_address("user@contoso.local").is_none(), + SmtpAddress::extract("user@contoso.local").is_none(), "an unknown suffix is not enough to override the routing type" ); } @@ -114,19 +134,27 @@ mod tests { #[test] fn routing_prefixes_and_display_names_are_stripped() { assert_eq!( - as_smtp_address("SMTP:user@example.com").as_deref(), + SmtpAddress::parse("SMTP:user@example.com") + .as_ref() + .map(SmtpAddress::as_str), Some("user@example.com") ); assert_eq!( - as_smtp_address("mailto:user@example.com").as_deref(), + SmtpAddress::parse("mailto:user@example.com") + .as_ref() + .map(SmtpAddress::as_str), Some("user@example.com") ); assert_eq!( - as_smtp_address("Jane Doe ").as_deref(), + SmtpAddress::parse("Jane Doe ") + .as_ref() + .map(SmtpAddress::as_str), Some("jane@example.com") ); assert_eq!( - extract_smtp_address("SMTP:jane@example.com").as_deref(), + SmtpAddress::extract("SMTP:jane@example.com") + .as_ref() + .map(SmtpAddress::as_str), Some("jane@example.com") ); } @@ -134,16 +162,24 @@ mod tests { #[test] fn entra_guest_upns_are_accepted() { let guest = "bob_contoso.com#EXT#@fabrikam.onmicrosoft.com"; - assert_eq!(as_smtp_address(guest).as_deref(), Some(guest)); - assert_eq!(extract_smtp_address(guest).as_deref(), Some(guest)); + assert_eq!( + SmtpAddress::parse(guest).as_ref().map(SmtpAddress::as_str), + Some(guest) + ); + assert_eq!( + SmtpAddress::extract(guest) + .as_ref() + .map(SmtpAddress::as_str), + Some(guest) + ); } #[test] fn legacy_exchange_distinguished_names_are_rejected() { - assert!(as_smtp_address(EXCHANGE_DN).is_none()); - assert!(extract_smtp_address(EXCHANGE_DN).is_none()); + assert!(SmtpAddress::parse(EXCHANGE_DN).is_none()); + assert!(SmtpAddress::extract(EXCHANGE_DN).is_none()); assert!( - as_smtp_address( + SmtpAddress::parse( "/O=HOSTING/OU=EXCHANGE ADMINISTRATIVE GROUP (FYDIBOHF23SPDLT)/CN=RECIPIENTS/CN=abc" ) .is_none() @@ -167,34 +203,43 @@ mod tests { "us..er@example.com", "üser@example.com", ] { - assert!(as_smtp_address(bad).is_none(), "must reject {bad:?}"); - assert!(extract_smtp_address(bad).is_none(), "must reject {bad:?}"); + assert!(SmtpAddress::parse(bad).is_none(), "must reject {bad:?}"); + assert!(SmtpAddress::extract(bad).is_none(), "must reject {bad:?}"); } } #[test] fn over_long_parts_are_rejected() { let long_local = format!("{}@example.com", "a".repeat(65)); - assert!(as_smtp_address(&long_local).is_none()); + assert!(SmtpAddress::parse(&long_local).is_none()); let long_label = format!("user@{}.com", "a".repeat(64)); - assert!(as_smtp_address(&long_label).is_none()); + assert!(SmtpAddress::parse(&long_label).is_none()); } #[test] fn dotless_domains_are_kept_when_the_server_says_smtp() { assert_eq!( - as_smtp_address("postmaster@localhost").as_deref(), + SmtpAddress::parse("postmaster@localhost") + .as_ref() + .map(SmtpAddress::as_str), Some("postmaster@localhost"), "RFC 5321 allows a bare domain and old on-prem setups use them" ); - assert!(extract_smtp_address("postmaster@localhost").is_none()); + assert!(SmtpAddress::extract("postmaster@localhost").is_none()); } #[test] fn public_suffix_is_required_only_by_the_strict_extractor() { - assert!(extract_smtp_address("user@example.com").is_some()); - assert!(extract_smtp_address("user@example.co.uk").is_some()); - assert!(extract_smtp_address("user@example.invalidtld").is_none()); - assert!(as_smtp_address("user@example.invalidtld").is_some()); + assert!(SmtpAddress::extract("user@example.com").is_some()); + assert!(SmtpAddress::extract("user@example.co.uk").is_some()); + assert!(SmtpAddress::extract("user@example.invalidtld").is_none()); + assert!(SmtpAddress::parse("user@example.invalidtld").is_some()); + } + + #[test] + fn multibyte_text_at_the_prefix_boundary_does_not_panic() { + // Regression: the routing-prefix check sliced a str at a byte offset. + assert!(SmtpAddress::parse("ab\u{20ac}\u{20ac}x").is_none()); + assert!(SmtpAddress::extract("ab\u{20ac}\u{20ac}x").is_none()); } } diff --git a/src/exchange/date.rs b/src/exchange/date.rs index 077c4ba..42e97f7 100644 --- a/src/exchange/date.rs +++ b/src/exchange/date.rs @@ -4,97 +4,139 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, TimeDelta, TimeZone}; +use std::fmt::Write; + +use chrono::{DateTime, Datelike, NaiveDate, NaiveDateTime, NaiveTime, TimeDelta, TimeZone}; use chrono_tz::Tz; +use serde_json::{Map, Value}; const END_OF_DAY: &str = "23:59:59"; +const MIDNIGHT_FIELD: &str = "00"; const HALF_DAY: TimeDelta = TimeDelta::hours(12); const OUTLOOK_NO_DATE_YEAR: i32 = 4500; +const LOCAL_DATE_TIME: &str = "%Y-%m-%dT%H:%M:%S%.f"; +const DATE_LEN: usize = 10; -pub fn nearest_midnight(value: &str, iana: Option<&str>) -> Option { - let trimmed = value.trim(); - let local = match DateTime::parse_from_rfc3339(trimmed) { - Ok(instant) => match iana.and_then(|tz| tz.parse::().ok()) { - Some(tz) => instant.with_timezone(&tz).naive_local(), - None => instant.naive_local(), - }, - Err(_) => NaiveDateTime::parse_from_str(trimmed, "%Y-%m-%dT%H:%M:%S%.f") - .or_else(|_| { - NaiveDate::parse_from_str(trimmed, "%Y-%m-%d").map(|d| d.and_time(NaiveTime::MIN)) - }) - .ok()?, - }; - local.checked_add_signed(HALF_DAY).map(|dt| dt.date()) -} +#[derive(Debug, Clone, Copy)] +pub struct WireDate<'a>(pub &'a str); -pub fn all_day_date(utc: &str, iana: Option<&str>) -> Option { - let trimmed = utc.trim(); - let instant = DateTime::parse_from_rfc3339(trimmed) - .map(|dt| dt.naive_utc()) - .or_else(|_| NaiveDateTime::parse_from_str(trimmed, "%Y-%m-%dT%H:%M:%S%.f")) - .ok()?; - if instant.time() == NaiveTime::MIN { - return Some(instant.date()); +impl WireDate<'_> { + pub fn nearest_midnight(self, iana: Option<&str>) -> Option { + let trimmed = self.0.trim(); + let local = match DateTime::parse_from_rfc3339(trimmed) { + Ok(instant) => match iana.and_then(|tz| tz.parse::().ok()) { + Some(tz) => instant.with_timezone(&tz).naive_local(), + None => instant.naive_local(), + }, + Err(_) => NaiveDateTime::parse_from_str(trimmed, LOCAL_DATE_TIME) + .or_else(|_| { + NaiveDate::parse_from_str(trimmed, "%Y-%m-%d") + .map(|d| d.and_time(NaiveTime::MIN)) + }) + .ok()?, + }; + local.checked_add_signed(HALF_DAY).map(|dt| dt.date()) } - let local = match iana.and_then(|tz| tz.parse::().ok()) { - Some(tz) => tz.from_utc_datetime(&instant).naive_local(), - None => instant, - }; - local.checked_add_signed(HALF_DAY).map(|dt| dt.date()) -} -pub fn is_outlook_no_date(value: &str) -> bool { - value - .trim() - .split_once('-') - .and_then(|(year, _)| year.parse::().ok()) - .is_some_and(|year| year >= OUTLOOK_NO_DATE_YEAR) -} - -pub fn recurrence_until(raw: &str) -> Option { - let trimmed = raw.trim(); - let date = trimmed.get(..10).filter(|d| is_calendar_date(d))?; - let time = match trimmed.get(10..) { - Some(rest) if rest.starts_with('T') || rest.starts_with('t') => { - clock_time(&rest[1..]).unwrap_or_else(|| END_OF_DAY.to_owned()) + pub fn all_day(self, iana: Option<&str>) -> Option { + let trimmed = self.0.trim(); + let instant = DateTime::parse_from_rfc3339(trimmed) + .map(|dt| dt.naive_utc()) + .or_else(|_| NaiveDateTime::parse_from_str(trimmed, LOCAL_DATE_TIME)) + .ok()?; + if instant.time() == NaiveTime::MIN { + return Some(instant.date()); } - _ => END_OF_DAY.to_owned(), - }; - Some(format!("{date}T{time}")) -} + let local = match iana.and_then(|tz| tz.parse::().ok()) { + Some(tz) => tz.from_utc_datetime(&instant).naive_local(), + None => instant, + }; + local.checked_add_signed(HALF_DAY).map(|dt| dt.date()) + } -fn is_calendar_date(value: &str) -> bool { - let b = value.as_bytes(); - b.len() == 10 - && b[..4].iter().all(u8::is_ascii_digit) - && b[4] == b'-' - && b[5..7].iter().all(u8::is_ascii_digit) - && b[7] == b'-' - && b[8..10].iter().all(u8::is_ascii_digit) -} + pub fn is_outlook_no_date(self) -> bool { + self.0 + .trim() + .split_once('-') + .and_then(|(year, _)| year.parse::().ok()) + .is_some_and(|year| year >= OUTLOOK_NO_DATE_YEAR) + } -fn clock_time(rest: &str) -> Option { - let end = rest - .find(['Z', 'z', '+']) - .or_else(|| rest.find('-')) - .unwrap_or(rest.len()); - let raw = &rest[..end]; - let mut parts = raw.split(':'); - let hour = two_digits(parts.next()?)?; - let minute = two_digits(parts.next().unwrap_or("00"))?; - let second = match parts.next() { - Some(s) => two_digits(s.split('.').next().unwrap_or("00"))?, - None => "00".to_owned(), - }; - Some(format!("{hour}:{minute}:{second}")) -} + pub fn partial_date(self) -> Option { + if self.is_outlook_no_date() { + return None; + } + let date = self.nearest_midnight(None)?; + let mut o = Map::new(); + o.insert("@type".to_owned(), Value::from("PartialDate")); + if date.year() > 0 { + o.insert("year".to_owned(), Value::from(date.year())); + } + o.insert("month".to_owned(), Value::from(date.month())); + o.insert("day".to_owned(), Value::from(date.day())); + Some(Value::Object(o)) + } -fn two_digits(value: &str) -> Option { - let v = value.trim(); - if v.len() == 2 && v.bytes().all(|b| b.is_ascii_digit()) { - Some(v.to_owned()) - } else { - None + pub fn without_fraction(self) -> String { + let raw = self.0; + let trimmed = raw.trim_end_matches('Z'); + let base = trimmed.split_once('.').map_or(trimmed, |(base, _)| base); + let mut out = String::with_capacity(base.len() + 1); + out.push_str(base); + if raw.ends_with('Z') { + out.push('Z'); + } + out + } + + pub fn recurrence_until(self) -> Option { + let trimmed = self.0.trim(); + let date = trimmed + .get(..DATE_LEN) + .filter(|d| WireDate::is_calendar_date(d))?; + let mut out = String::with_capacity(DATE_LEN + 1 + END_OF_DAY.len()); + out.push_str(date); + out.push('T'); + let clock = trimmed + .get(DATE_LEN..) + .and_then(|rest| rest.strip_prefix(['T', 't'])) + .and_then(WireDate::clock_time); + match clock { + Some((hour, minute, second)) => { + let _ = write!(out, "{hour}:{minute}:{second}"); + } + None => out.push_str(END_OF_DAY), + } + Some(out) + } + + fn is_calendar_date(value: &str) -> bool { + let [y0, y1, y2, y3, b'-', m0, m1, b'-', d0, d1] = value.as_bytes() else { + return false; + }; + [y0, y1, y2, y3, m0, m1, d0, d1] + .into_iter() + .all(u8::is_ascii_digit) + } + + fn clock_time(rest: &str) -> Option<(&str, &str, &str)> { + let raw = rest + .split_once(['Z', 'z', '+']) + .or_else(|| rest.split_once('-')) + .map_or(rest, |(clock, _)| clock); + let mut parts = raw.split(':'); + let hour = WireDate::two_digits(parts.next()?)?; + let minute = WireDate::two_digits(parts.next().unwrap_or(MIDNIGHT_FIELD))?; + let second = match parts.next() { + Some(s) => WireDate::two_digits(s.split('.').next().unwrap_or(MIDNIGHT_FIELD))?, + None => MIDNIGHT_FIELD, + }; + Some((hour, minute, second)) + } + + fn two_digits(value: &str) -> Option<&str> { + let v = value.trim(); + (v.len() == 2 && v.bytes().all(|b| b.is_ascii_digit())).then_some(v) } } @@ -109,11 +151,11 @@ mod tests { #[test] fn utc_instant_rounds_to_local_midnight_east_of_utc() { assert_eq!( - nearest_midnight("2026-11-11T23:00:00Z", Some("Europe/Paris")), + WireDate("2026-11-11T23:00:00Z").nearest_midnight(Some("Europe/Paris")), date(2026, 11, 12) ); assert_eq!( - nearest_midnight("2026-07-11T22:00:00Z", Some("Europe/Paris")), + WireDate("2026-07-11T22:00:00Z").nearest_midnight(Some("Europe/Paris")), date(2026, 7, 12) ); } @@ -121,7 +163,7 @@ mod tests { #[test] fn utc_instant_rounds_to_local_midnight_west_of_utc() { assert_eq!( - nearest_midnight("2026-11-12T05:00:00Z", Some("America/New_York")), + WireDate("2026-11-12T05:00:00Z").nearest_midnight(Some("America/New_York")), date(2026, 11, 12) ); } @@ -129,11 +171,11 @@ mod tests { #[test] fn zone_off_by_an_hour_still_gives_the_same_day() { assert_eq!( - nearest_midnight("2026-11-11T23:00:00Z", Some("Europe/London")), + WireDate("2026-11-11T23:00:00Z").nearest_midnight(Some("Europe/London")), date(2026, 11, 12) ); assert_eq!( - nearest_midnight("2026-11-11T23:00:00Z", Some("Europe/Helsinki")), + WireDate("2026-11-11T23:00:00Z").nearest_midnight(Some("Europe/Helsinki")), date(2026, 11, 12) ); } @@ -141,11 +183,11 @@ mod tests { #[test] fn utc_instant_without_zone_rounds_in_utc() { assert_eq!( - nearest_midnight("1990-04-14T22:00:00Z", None), + WireDate("1990-04-14T22:00:00Z").nearest_midnight(None), date(1990, 4, 15) ); assert_eq!( - nearest_midnight("1990-04-15T05:00:00Z", None), + WireDate("1990-04-15T05:00:00Z").nearest_midnight(None), date(1990, 4, 15) ); } @@ -153,21 +195,24 @@ mod tests { #[test] fn floating_and_bare_dates_are_kept() { assert_eq!( - nearest_midnight("2025-12-25T00:00:00", Some("Europe/Paris")), + WireDate("2025-12-25T00:00:00").nearest_midnight(Some("Europe/Paris")), date(2025, 12, 25) ); - assert_eq!(nearest_midnight("2025-12-25", None), date(2025, 12, 25)); - assert_eq!(nearest_midnight("garbage", None), None); + assert_eq!( + WireDate("2025-12-25").nearest_midnight(None), + date(2025, 12, 25) + ); + assert_eq!(WireDate("garbage").nearest_midnight(None), None); } #[test] fn all_day_midnight_is_taken_as_the_date() { assert_eq!( - all_day_date("2026-11-12T00:00:00.0000000", Some("Pacific/Auckland")), + WireDate("2026-11-12T00:00:00.0000000").all_day(Some("Pacific/Auckland")), date(2026, 11, 12) ); assert_eq!( - all_day_date("2026-11-12T00:00:00Z", Some("America/New_York")), + WireDate("2026-11-12T00:00:00Z").all_day(Some("America/New_York")), date(2026, 11, 12) ); } @@ -175,36 +220,36 @@ mod tests { #[test] fn all_day_utc_instant_moves_to_the_local_date() { assert_eq!( - all_day_date("2026-11-11T23:00:00.0000000", Some("Europe/Paris")), + WireDate("2026-11-11T23:00:00.0000000").all_day(Some("Europe/Paris")), date(2026, 11, 12) ); assert_eq!( - all_day_date("2026-11-11T11:00:00Z", Some("Pacific/Auckland")), + WireDate("2026-11-11T11:00:00Z").all_day(Some("Pacific/Auckland")), date(2026, 11, 12) ); assert_eq!( - all_day_date("2026-11-12T05:00:00.0000000", Some("America/New_York")), + WireDate("2026-11-12T05:00:00.0000000").all_day(Some("America/New_York")), date(2026, 11, 12) ); assert_eq!( - all_day_date("2026-11-11T23:00:00", None), + WireDate("2026-11-11T23:00:00").all_day(None), date(2026, 11, 12) ); - assert_eq!(all_day_date("garbage", None), None); + assert_eq!(WireDate("garbage").all_day(None), None); } #[test] fn outlook_no_date_sentinel() { - assert!(is_outlook_no_date("4500-08-31T23:00:00Z")); - assert!(is_outlook_no_date("4501-01-01T00:00:00Z")); - assert!(!is_outlook_no_date("2026-11-12T00:00:00Z")); - assert!(!is_outlook_no_date("garbage")); + assert!(WireDate("4500-08-31T23:00:00Z").is_outlook_no_date()); + assert!(WireDate("4501-01-01T00:00:00Z").is_outlook_no_date()); + assert!(!WireDate("2026-11-12T00:00:00Z").is_outlook_no_date()); + assert!(!WireDate("garbage").is_outlook_no_date()); } #[test] fn bare_date_gets_end_of_day() { assert_eq!( - recurrence_until("2026-08-01").as_deref(), + WireDate("2026-08-01").recurrence_until().as_deref(), Some("2026-08-01T23:59:59") ); } @@ -212,7 +257,7 @@ mod tests { #[test] fn utc_designator_is_dropped() { assert_eq!( - recurrence_until("2026-08-01Z").as_deref(), + WireDate("2026-08-01Z").recurrence_until().as_deref(), Some("2026-08-01T23:59:59") ); } @@ -220,11 +265,11 @@ mod tests { #[test] fn numeric_offset_is_dropped_not_embedded() { assert_eq!( - recurrence_until("2021-09-30-06:00").as_deref(), + WireDate("2021-09-30-06:00").recurrence_until().as_deref(), Some("2021-09-30T23:59:59") ); assert_eq!( - recurrence_until("2021-09-30+02:00").as_deref(), + WireDate("2021-09-30+02:00").recurrence_until().as_deref(), Some("2021-09-30T23:59:59") ); } @@ -232,19 +277,27 @@ mod tests { #[test] fn date_time_keeps_its_clock_time() { assert_eq!( - recurrence_until("2021-09-30T15:15:00").as_deref(), + WireDate("2021-09-30T15:15:00") + .recurrence_until() + .as_deref(), Some("2021-09-30T15:15:00") ); assert_eq!( - recurrence_until("2021-09-30T15:15:00Z").as_deref(), + WireDate("2021-09-30T15:15:00Z") + .recurrence_until() + .as_deref(), Some("2021-09-30T15:15:00") ); assert_eq!( - recurrence_until("2021-09-30T15:15:00-06:00").as_deref(), + WireDate("2021-09-30T15:15:00-06:00") + .recurrence_until() + .as_deref(), Some("2021-09-30T15:15:00") ); assert_eq!( - recurrence_until("2021-09-30T15:15:00.123Z").as_deref(), + WireDate("2021-09-30T15:15:00.123Z") + .recurrence_until() + .as_deref(), Some("2021-09-30T15:15:00") ); } @@ -252,7 +305,7 @@ mod tests { #[test] fn short_clock_time_is_padded_to_seconds() { assert_eq!( - recurrence_until("2021-09-30T15:15").as_deref(), + WireDate("2021-09-30T15:15").recurrence_until().as_deref(), Some("2021-09-30T15:15:00") ); } @@ -260,15 +313,15 @@ mod tests { #[test] fn surrounding_whitespace_is_ignored() { assert_eq!( - recurrence_until(" 2026-08-01Z \n").as_deref(), + WireDate(" 2026-08-01Z \n").recurrence_until().as_deref(), Some("2026-08-01T23:59:59") ); } #[test] fn unusable_input_yields_none() { - assert_eq!(recurrence_until(""), None); - assert_eq!(recurrence_until("not-a-date"), None); - assert_eq!(recurrence_until("2026-8-1"), None); + assert_eq!(WireDate("").recurrence_until(), None); + assert_eq!(WireDate("not-a-date").recurrence_until(), None); + assert_eq!(WireDate("2026-8-1").recurrence_until(), None); } } diff --git a/src/exchange/ews/autodiscover.rs b/src/exchange/ews/autodiscover.rs new file mode 100644 index 0000000..6bfa015 --- /dev/null +++ b/src/exchange/ews/autodiscover.rs @@ -0,0 +1,421 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fmt::{self, Write}; +use std::mem; + +use quick_xml::NsReader; +use quick_xml::events::Event; +use serde_json::Value; +use url::Url; + +use crate::exchange::ews::error::EwsError; +use crate::exchange::ews::srv::SrvRecord; +use crate::net::encode::{Percent, SafeSet}; +use crate::net::http::{AgentProfile, ClientOptions, HttpCore, Methods, Redirects}; +use crate::net::xml::{Entity, XmlEscape}; + +const V2_HOST: &str = "https://outlook.office365.com"; +const EWS_PATH: &str = "/EWS/Exchange.asmx"; +const POX_REQ_NS: &str = + "http://schemas.microsoft.com/exchange/autodiscover/outlook/requestschema/2006"; +const POX_RESP_NS: &str = + "http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a"; +const POX_MAX_BODY: u64 = 2 * 1024 * 1024; +const HTTPS_PORT: u16 = 443; +const MAX_ADDRESS_REDIRECTS: u8 = 4; +const MAX_URL_REDIRECTS: u8 = 1; +const PROFILE: AgentProfile = AgentProfile { + redirects: Redirects::Follow, + methods: Methods::Standard, + max_body: POX_MAX_BODY, +}; + +#[derive(Debug, Clone)] +pub struct DiscoveryResult { + pub ews_url: String, + pub source: DiscoverySource, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DiscoverySource { + V2, + V1, + SuppliedUrl, +} + +impl fmt::Display for DiscoverySource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + DiscoverySource::V2 => "found by Autodiscover v2", + DiscoverySource::V1 => "found by Autodiscover v1", + DiscoverySource::SuppliedUrl => "from --url", + }) + } +} + +#[derive(Debug, Clone)] +enum PoxOutcome { + EwsUrl(String), + RedirectAddr(String), + RedirectUrl(String), + Failed, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PoxElement { + Protocol, + Type, + EwsUrl, + Action, + RedirectAddr, + RedirectUrl, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PoxField { + Type, + EwsUrl, + Action, + RedirectAddr, + RedirectUrl, +} + +pub struct Autodiscover { + core: HttpCore, +} + +impl Autodiscover { + pub fn new(options: ClientOptions) -> Autodiscover { + Autodiscover { + core: HttpCore::new(options, PROFILE), + } + } + + pub fn is_fully_qualified_ews_url(url: &str) -> bool { + Url::parse(url) + .is_ok_and(|u| u.scheme() == "https" && u.path().eq_ignore_ascii_case(EWS_PATH)) + } + + pub fn discover( + &self, + supplied_url: Option<&str>, + email: Option<&str>, + auth_header: Option<&str>, + ) -> Result { + if let Some(url) = supplied_url + && Autodiscover::is_fully_qualified_ews_url(url) + { + return Ok(DiscoveryResult { + ews_url: url.to_owned(), + source: DiscoverySource::SuppliedUrl, + }); + } + let Some(email) = email else { + return Err(EwsError::AutodiscoverFailed( + "either a fully-qualified --url or --mailbox is required".to_owned(), + )); + }; + if let Ok(url) = self.v2(email) { + return Ok(DiscoveryResult { + ews_url: url, + source: DiscoverySource::V2, + }); + } + let mut tried: usize = 0; + let mut redirects: u8 = 0; + let mut url_redirects: u8 = 0; + let mut current_email = email.to_owned(); + 'outer: loop { + let domain = current_email.split('@').nth(1).ok_or_else(|| { + EwsError::AutodiscoverFailed("--mailbox is not an email".to_owned()) + })?; + for candidate in Autodiscover::pox_candidates(domain) { + tried += 1; + match self.v1(&candidate, ¤t_email, auth_header) { + Ok(PoxOutcome::EwsUrl(url)) => { + return Ok(DiscoveryResult { + ews_url: url, + source: DiscoverySource::V1, + }); + } + Ok(PoxOutcome::RedirectAddr(addr)) => { + if redirects >= MAX_ADDRESS_REDIRECTS { + return Err(EwsError::AutodiscoverLoop); + } + redirects += 1; + current_email = addr; + continue 'outer; + } + Ok(PoxOutcome::RedirectUrl(url)) => { + if url_redirects >= MAX_URL_REDIRECTS { + return Err(EwsError::AutodiscoverLoop); + } + url_redirects += 1; + tried += 1; + if let Ok(PoxOutcome::EwsUrl(u)) = + self.v1(&url, ¤t_email, auth_header) + { + return Ok(DiscoveryResult { + ews_url: u, + source: DiscoverySource::V1, + }); + } + } + Err(_) | Ok(PoxOutcome::Failed) => {} + } + } + break; + } + if let Some(url) = supplied_url { + return Ok(DiscoveryResult { + ews_url: url.to_owned(), + source: DiscoverySource::SuppliedUrl, + }); + } + Err(EwsError::AutodiscoverFailed(format!( + "tried {tried} candidate(s), none returned a v1 EwsUrl" + ))) + } + + fn v2(&self, email: &str) -> Result { + let mut url = format!("{V2_HOST}/autodiscover/autodiscover.json?Email="); + Percent::encode_into(&mut url, email, SafeSet::Unreserved); + url.push_str("&Protocol=Ews"); + let mut resp = self + .core + .agent() + .get(&url) + .header("Accept", "application/json") + .call() + .map_err(|e| EwsError::Transport(format!("autodiscover v2: {e}")))?; + let status = resp.status().as_u16(); + if !(200..300).contains(&status) { + return Err(EwsError::AutodiscoverFailed(format!( + "v2 returned http {status}" + ))); + } + let body: Value = resp + .body_mut() + .read_json() + .map_err(|e| EwsError::Malformed(format!("v2 body: {e}")))?; + let protocol = body + .get("Protocol") + .and_then(Value::as_str) + .unwrap_or_default(); + if !protocol.eq_ignore_ascii_case("Ews") { + return Err(EwsError::AutodiscoverFailed(format!( + "v2 returned protocol {protocol:?}" + ))); + } + body.get("Url") + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| EwsError::AutodiscoverFailed("v2 missing Url".to_owned())) + } + + fn pox_candidates(domain: &str) -> Vec { + let mut out = vec![ + format!("https://autodiscover.{domain}/autodiscover/autodiscover.xml"), + format!("https://{domain}/autodiscover/autodiscover.xml"), + ]; + out.extend( + SrvRecord::lookup(&format!("_autodiscover._tcp.{domain}")) + .into_iter() + .map(|SrvRecord { target, port, .. }| { + let mut url = format!("https://{}", target.trim_end_matches('.')); + if port != HTTPS_PORT { + let _ = write!(url, ":{port}"); + } + url.push_str("/autodiscover/autodiscover.xml"); + url + }), + ); + out + } + + fn v1( + &self, + url: &str, + email: &str, + auth_header: Option<&str>, + ) -> Result { + let body = format!( + "\ + \ + {}\ + {POX_RESP_NS}\ + ", + XmlEscape(email) + ); + let mut req = self + .core + .agent() + .post(url) + .header("Content-Type", "text/xml; charset=utf-8"); + if let Some(h) = auth_header { + req = req.header("Authorization", h); + } + let mut resp = req + .send(body.as_bytes()) + .map_err(|e| EwsError::Transport(format!("v1 POST: {e}")))?; + let status = resp.status().as_u16(); + if !(200..300).contains(&status) { + return Err(EwsError::AutodiscoverFailed(format!( + "{url} returned http {status}" + ))); + } + let bytes = resp + .body_mut() + .with_config() + .limit(POX_MAX_BODY) + .read_to_vec() + .map_err(|e| EwsError::Malformed(format!("v1 body: {e}")))?; + PoxOutcome::parse(&bytes) + } +} + +impl PoxElement { + fn of(local: &str) -> Option { + hashify::map_ignore_case!( + local.as_bytes(), + PoxElement, + "Protocol" => PoxElement::Protocol, + "Type" => PoxElement::Type, + "EwsUrl" => PoxElement::EwsUrl, + "Action" => PoxElement::Action, + "RedirectAddr" => PoxElement::RedirectAddr, + "RedirectUrl" => PoxElement::RedirectUrl, + ) + .copied() + } +} + +impl PoxOutcome { + fn parse(body: &[u8]) -> Result { + let mut xml = NsReader::from_reader(body); + xml.config_mut().trim_text(true); + let mut buf = Vec::new(); + let mut current: Option = None; + let mut current_type: Option = None; + let mut ews_url: Option = None; + let mut redirect_addr: Option = None; + let mut redirect_url: Option = None; + let mut action: Option = None; + let mut cur = String::new(); + loop { + buf.clear(); + let (_, ev) = xml.read_resolved_event_into(&mut buf)?; + match ev { + Event::Start(e) => { + cur.clear(); + current = match PoxElement::of(e.local_name().into_inner()) { + Some(PoxElement::Protocol) => { + current_type = None; + None + } + Some(PoxElement::Type) => Some(PoxField::Type), + Some(PoxElement::EwsUrl) => { + matches!(current_type.as_deref(), Some("EXPR" | "EXCH")) + .then_some(PoxField::EwsUrl) + } + Some(PoxElement::Action) => Some(PoxField::Action), + Some(PoxElement::RedirectAddr) => Some(PoxField::RedirectAddr), + Some(PoxElement::RedirectUrl) => Some(PoxField::RedirectUrl), + None => None, + }; + } + Event::End(e) => { + if let Some(field) = current.take() { + let text = Some(mem::take(&mut cur)); + match field { + PoxField::Type => current_type = text, + PoxField::EwsUrl => ews_url = text, + PoxField::Action => action = text, + PoxField::RedirectAddr => redirect_addr = text, + PoxField::RedirectUrl => redirect_url = text, + } + } + cur.clear(); + if PoxElement::of(e.local_name().into_inner()) == Some(PoxElement::Protocol) { + current_type = None; + } + } + Event::Text(t) => cur.push_str(&t), + Event::CData(c) => cur.push_str(&c), + Event::GeneralRef(g) => { + if let Some(c) = Entity::from_ref(&g) { + cur.push(c); + } + } + Event::Eof => break, + _ => {} + } + } + if let Some(addr) = redirect_addr.filter(|_| action.as_deref() == Some("redirectAddr")) { + return Ok(PoxOutcome::RedirectAddr(addr)); + } + if let Some(url) = redirect_url.filter(|_| action.as_deref() == Some("redirectUrl")) { + return Ok(PoxOutcome::RedirectUrl(url)); + } + Ok(ews_url.map_or(PoxOutcome::Failed, PoxOutcome::EwsUrl)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // The discovery source reads as a phrase after the endpoint URL. + #[test] + fn discovery_source_names_how_the_endpoint_was_found() { + assert_eq!(DiscoverySource::V2.to_string(), "found by Autodiscover v2"); + assert_eq!(DiscoverySource::V1.to_string(), "found by Autodiscover v1"); + assert_eq!(DiscoverySource::SuppliedUrl.to_string(), "from --url"); + } + + #[test] + fn detects_fully_qualified_ews_url() { + assert!(Autodiscover::is_fully_qualified_ews_url( + "https://outlook.office365.com/EWS/Exchange.asmx" + )); + assert!(Autodiscover::is_fully_qualified_ews_url( + "https://exchange.example.com/EWS/Exchange.asmx" + )); + assert!(!Autodiscover::is_fully_qualified_ews_url( + "http://outlook.office365.com/EWS/Exchange.asmx" + )); + assert!(!Autodiscover::is_fully_qualified_ews_url( + "https://outlook.office365.com/" + )); + } + + #[test] + fn pox_response_extracts_ews_url() { + let body = format!( + "\ + \ + EXPRhttps://srv/EWS/Exchange.asmx" + ); + match PoxOutcome::parse(body.as_bytes()).unwrap() { + PoxOutcome::EwsUrl(u) => assert_eq!(u, "https://srv/EWS/Exchange.asmx"), + _ => panic!("expected EwsUrl"), + } + } + + #[test] + fn pox_response_handles_redirect_addr() { + let body = format!( + "\ + \ + redirectAddrbob@other.com" + ); + match PoxOutcome::parse(body.as_bytes()).unwrap() { + PoxOutcome::RedirectAddr(a) => assert_eq!(a, "bob@other.com"), + _ => panic!("expected RedirectAddr"), + } + } +} diff --git a/src/exchange_ews/calendar_map.rs b/src/exchange/ews/calendar_map.rs similarity index 55% rename from src/exchange_ews/calendar_map.rs rename to src/exchange/ews/calendar_map.rs index ee3b453..56416c9 100644 --- a/src/exchange_ews/calendar_map.rs +++ b/src/exchange/ews/calendar_map.rs @@ -4,17 +4,25 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; +use chrono_tz::Tz; use serde_json::{Map, Value, json}; +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; -use crate::exchange::address::{as_smtp_address, extract_smtp_address}; -use crate::exchange::date::nearest_midnight; -use crate::exchange::jscalendar::{ - drop_calendar_address_dependents, is_override_ignored, synthetic_attendee_address, -}; -use crate::exchange_ews::parse::{CalendarItemRaw, RawAttendee, RawOccurrence}; -use crate::exchange_ews::recurrence::to_jscalendar_rule; -use crate::exchange_ews::tz::resolve_to_iana; +use crate::exchange::date::WireDate; +use crate::exchange::ews::parse::{CalendarItemRaw, RawAttendee, RawOccurrence}; +use crate::exchange::jscalendar::{OverridePointer, ParticipantMap, ResolvedAddress, Routing}; +use crate::exchange::tz::ZoneId; +use crate::exchange::vocab::{DurationStyle, IsoDuration, Participation, Privacy}; + +const UTC_ZONE: &str = "Etc/UTC"; +const SYNTHETIC_UID_PREFIX: &str = "inbuxa-migrate-ews-event-"; +const SMTP_ROUTING: &str = "SMTP"; +const ORGANIZER: &str = "Organizer"; +const LOCAL_FORMAT: &str = "%Y-%m-%dT%H:%M:%S"; +const MIDNIGHT: &str = "T00:00:00"; +const DATE_LEN: usize = 10; pub struct EventValue { pub data: Value, @@ -22,31 +30,501 @@ pub struct EventValue { pub use_default_alerts: bool, } -pub fn to_jscalendar(raw: &CalendarItemRaw) -> EventValue { - to_jscalendar_with_exceptions(raw, &[], None) -} - -pub fn to_jscalendar_with_exceptions( - raw: &CalendarItemRaw, - modified_full: &[CalendarItemRaw], - owner_smtp: Option<&str>, -) -> EventValue { - let iana = raw - .start_tz - .as_deref() - .map(|tz| resolve_to_iana(tz).unwrap_or_else(|| "Etc/UTC".to_owned())); - let ctx = EventCtx { - iana: iana.as_deref(), - owner_smtp, - }; - let mut event = build_event_map(raw, ctx); - if let Some(overrides) = build_recurrence_overrides(&event, raw, modified_full, ctx) { - event.insert("recurrenceOverrides".to_owned(), overrides); +impl CalendarItemRaw { + pub fn to_jscalendar(&self) -> EventValue { + self.to_jscalendar_with_exceptions(&[], None) } - EventValue { - data: Value::Object(event), - is_draft: false, - use_default_alerts: false, + + pub fn to_jscalendar_with_exceptions( + &self, + modified_full: &[CalendarItemRaw], + owner_smtp: Option<&str>, + ) -> EventValue { + let raw = self; + let iana = raw + .start_tz + .as_deref() + .map(|tz| ZoneId(tz).iana().unwrap_or_else(|| UTC_ZONE.to_owned())); + let ctx = EventCtx { + iana: iana.as_deref(), + owner_smtp, + }; + let mut event = CalendarItemRaw::build_event_map(raw, ctx); + if let Some(overrides) = + CalendarItemRaw::build_recurrence_overrides(&event, raw, modified_full, ctx) + { + event.insert("recurrenceOverrides".to_owned(), overrides); + } + EventValue { + data: Value::Object(event), + is_draft: false, + use_default_alerts: false, + } + } + + fn build_event_map(raw: &CalendarItemRaw, ctx: EventCtx<'_>) -> Map { + let iana = ctx.iana; + let mut event = Map::new(); + event.insert("@type".to_owned(), Value::String("Event".to_owned())); + if let Some(uid) = raw.uid.as_ref() { + event.insert("uid".to_owned(), Value::String(uid.clone())); + } else { + event.insert( + "uid".to_owned(), + Value::String(format!( + "{SYNTHETIC_UID_PREFIX}{}", + blake3::hash(raw.id.id.as_bytes()).to_hex() + )), + ); + } + if let Some(subject) = raw.subject.as_ref() { + event.insert("title".to_owned(), Value::String(subject.clone())); + } + if let Some(loc) = raw.location.as_ref() { + event.insert( + "locations".to_owned(), + json!({"1": {"@type": "Location", "name": loc}}), + ); + } + if let Some(body) = raw.body_text.as_ref() { + event.insert("description".to_owned(), Value::String(body.clone())); + } else if let Some(body) = raw.body_html.as_ref() { + event.insert("description".to_owned(), Value::String(body.clone())); + event.insert( + "descriptionContentType".to_owned(), + Value::String("text/html".to_owned()), + ); + } + if let Some(created) = raw.created.as_ref() { + event.insert( + "created".to_owned(), + Value::String(CalendarItemRaw::normalise_utc_datetime(created)), + ); + } + if let Some(updated) = raw.last_modified.as_ref().or(raw.created.as_ref()) { + event.insert( + "updated".to_owned(), + Value::String(CalendarItemRaw::normalise_utc_datetime(updated)), + ); + } + if !raw.categories.is_empty() { + let mut kw = Map::new(); + for c in &raw.categories { + kw.insert(c.to_ascii_lowercase(), Value::Bool(true)); + } + event.insert("keywords".to_owned(), Value::Object(kw)); + } + if let Some(true) = raw.is_all_day_event { + let start = raw + .start + .as_deref() + .and_then(|s| WireDate(s).nearest_midnight(iana)); + if let Some(start) = start { + event.insert( + "start".to_owned(), + Value::String(CalendarItemRaw::all_day_key(start)), + ); + } + event.insert("showWithoutTime".to_owned(), Value::Bool(true)); + let days = CalendarItemRaw::all_day_span_days(start, raw.end.as_deref(), iana); + event.insert("duration".to_owned(), Value::String(format!("P{days}D"))); + } else { + if let Some(start) = raw.start.as_ref() { + event.insert( + "start".to_owned(), + Value::String(CalendarItemRaw::to_local_datetime_in(start, iana)), + ); + } + if let (Some(start), Some(end)) = (raw.start.as_ref(), raw.end.as_ref()) + && let Some(dur) = CalendarItemRaw::duration_iso8601(start, end) + { + event.insert("duration".to_owned(), Value::String(dur)); + } + if let Some(tz) = iana { + event.insert("timeZone".to_owned(), Value::String(tz.to_owned())); + } + } + if let Some(sensitivity) = raw.sensitivity.as_deref() { + event.insert( + "privacy".to_owned(), + Value::from(Privacy::from_sensitivity(sensitivity).as_str()), + ); + } + if let Some(status) = raw.legacy_free_busy_status.as_ref() { + let mapped = if status == "Free" { "free" } else { "busy" }; + event.insert( + "freeBusyStatus".to_owned(), + Value::String(mapped.to_owned()), + ); + } + let mut participants = Map::new(); + let mut next_id = 1; + if let Some(ResolvedAddress { + uri: cal_addr, + email: smtp, + }) = ResolvedAddress::resolve( + raw.organizer_smtp.as_deref(), + CalendarItemRaw::routing(raw.organizer_routing_type.as_deref()), + raw.organizer_name.as_deref(), + ) { + let key = next_id.to_string(); + next_id += 1; + event.insert( + "organizerCalendarAddress".to_owned(), + Value::String(cal_addr.clone()), + ); + let mut p = Map::new(); + p.insert("@type".to_owned(), Value::String("Participant".to_owned())); + p.insert("calendarAddress".to_owned(), Value::String(cal_addr)); + if let Some(email) = smtp { + p.insert("email".to_owned(), Value::String(email)); + } + let mut roles = Map::new(); + roles.insert("owner".to_owned(), Value::Bool(true)); + roles.insert("chair".to_owned(), Value::Bool(true)); + p.insert("roles".to_owned(), Value::Object(roles)); + if let Some(name) = raw.organizer_name.as_ref() { + p.insert("name".to_owned(), Value::String(name.clone())); + } + participants.insert(key, Value::Object(p)); + } + CalendarItemRaw::add_attendees( + &mut participants, + &mut next_id, + &raw.required_attendees, + AttendeeRole::Required, + ); + CalendarItemRaw::add_attendees( + &mut participants, + &mut next_id, + &raw.optional_attendees, + AttendeeRole::Optional, + ); + CalendarItemRaw::add_attendees( + &mut participants, + &mut next_id, + &raw.resources, + AttendeeRole::Resource, + ); + if let Some(owner) = ctx.owner_smtp + && let Some(status) = raw + .my_response_type + .as_deref() + .filter(|r| *r != ORGANIZER) + .and_then(Participation::from_ews) + { + for p in participants.values_mut().filter_map(Value::as_object_mut) { + let is_owner = p + .get("email") + .and_then(Value::as_str) + .is_some_and(|email| email.eq_ignore_ascii_case(owner)); + if is_owner && p.contains_key("expectReply") { + p.insert( + "participationStatus".to_owned(), + Value::from(status.as_str()), + ); + } + } + } + if !participants.is_empty() { + if !event.contains_key("organizerCalendarAddress") { + ParticipantMap(&mut participants).drop_calendar_address_dependents(); + } + event.insert("participants".to_owned(), Value::Object(participants)); + } + if let Some(rec) = raw.recurrence.as_ref() + && let Some(rule) = rec.to_jscalendar() + { + event.insert("recurrenceRule".to_owned(), rule); + } + if raw.reminder_is_set == Some(true) { + let minutes = raw.reminder_minutes_before_start.unwrap_or(0).max(0); + let offset = if minutes == 0 { + "PT0S".to_owned() + } else { + format!("-PT{minutes}M") + }; + let alert = json!({ + "@type": "Alert", + "trigger": {"@type": "OffsetTrigger", "offset": offset, "relativeTo": "start"}, + "action": "display", + }); + event.insert("alerts".to_owned(), json!({ "1": alert })); + } + if let Some(url) = raw + .join_online_meeting_url + .as_deref() + .or(raw.net_show_url.as_deref()) + .or(raw.meeting_workspace_url.as_deref()) + .filter(|s| !s.is_empty()) + { + let mut vl = Map::new(); + vl.insert( + "@type".to_owned(), + Value::String("VirtualLocation".to_owned()), + ); + vl.insert("uri".to_owned(), Value::String(url.to_owned())); + if raw.is_online_meeting == Some(true) { + vl.insert("features".to_owned(), json!({"video": true})); + } + event.insert("virtualLocations".to_owned(), json!({ "1": vl })); + } + event + } + + fn routing(routing_type: Option<&str>) -> Routing { + if routing_type.is_none_or(|rt| rt.eq_ignore_ascii_case(SMTP_ROUTING)) { + Routing::Smtp + } else { + Routing::Other + } + } + + fn add_attendees( + out: &mut Map, + next_id: &mut u32, + attendees: &[RawAttendee], + role: AttendeeRole, + ) { + for att in attendees { + let key = next_id.to_string(); + *next_id += 1; + let mut p = Map::new(); + p.insert("@type".to_owned(), Value::String("Participant".to_owned())); + if let Some(name) = att.name.as_ref() { + p.insert("name".to_owned(), Value::String(name.clone())); + } + let calendar_address = ResolvedAddress::resolve( + att.email.as_deref(), + CalendarItemRaw::routing(att.routing_type.as_deref()), + att.name.as_deref(), + ); + if let Some(ResolvedAddress { + uri: addr, + email: smtp, + }) = calendar_address + { + p.insert("calendarAddress".to_owned(), Value::String(addr)); + if let Some(email) = smtp { + p.insert("email".to_owned(), Value::String(email)); + } + let mut roles = Map::new(); + match role { + AttendeeRole::Required | AttendeeRole::Resource => { + roles.insert("required".to_owned(), Value::Bool(true)); + } + AttendeeRole::Optional => { + roles.insert("optional".to_owned(), Value::Bool(true)); + } + } + p.insert("roles".to_owned(), Value::Object(roles)); + if role == AttendeeRole::Resource { + p.insert("kind".to_owned(), Value::String("resource".to_owned())); + } else { + p.insert("expectReply".to_owned(), Value::Bool(true)); + } + if let Some(rt) = att.response_type.as_deref() { + let mapped = Participation::from_ews(rt).unwrap_or(Participation::NeedsAction); + p.insert( + "participationStatus".to_owned(), + Value::from(mapped.as_str()), + ); + } + } + out.insert(key, Value::Object(p)); + } + } + + fn is_override_excluded(key: &str) -> bool { + OverridePointer(key).is_ignored() || hashify::set!(key.as_bytes(), "created", "updated") + } + + fn build_recurrence_overrides( + base_event: &Map, + raw: &CalendarItemRaw, + modified_full: &[CalendarItemRaw], + ctx: EventCtx<'_>, + ) -> Option { + let iana = ctx.iana; + if raw.modified_occurrences.is_empty() && raw.deleted_occurrences.is_empty() { + return None; + } + let mut map = Map::new(); + for occ in &raw.modified_occurrences { + let Some(key) = occ + .original_start + .as_deref() + .or(occ.start.as_deref()) + .map(|s| CalendarItemRaw::occurrence_key(raw, s, iana)) + else { + continue; + }; + let full = modified_full + .iter() + .find(|f| !occ.item_id.id.is_empty() && f.id.id == occ.item_id.id); + let patch = match full { + Some(full) => { + let occ_event = CalendarItemRaw::build_event_map(full, ctx); + CalendarItemRaw::override_patch(&occ_event, base_event, &key) + } + None => CalendarItemRaw::time_only_patch(raw, occ, iana), + }; + map.insert(key, Value::Object(patch)); + } + for occ in &raw.deleted_occurrences { + let Some(key) = occ + .start + .as_deref() + .map(|s| CalendarItemRaw::occurrence_key(raw, s, iana)) + else { + continue; + }; + map.insert(key, json!({"excluded": true})); + } + if map.is_empty() { + None + } else { + Some(Value::Object(map)) + } + } + + fn override_patch( + occ_event: &Map, + base_event: &Map, + recurrence_id: &str, + ) -> Map { + let mut patch = Map::new(); + let inherited_start = Value::String(recurrence_id.to_owned()); + for (k, v) in occ_event { + if CalendarItemRaw::is_override_excluded(k) { + continue; + } + let baseline = if k == "start" { + Some(&inherited_start) + } else { + base_event.get(k) + }; + if baseline != Some(v) { + patch.insert(k.clone(), v.clone()); + } + } + for k in base_event.keys() { + if k == "start" || CalendarItemRaw::is_override_excluded(k) { + continue; + } + if !occ_event.contains_key(k) { + patch.insert(k.clone(), Value::Null); + } + } + patch + } + + fn time_only_patch( + series: &CalendarItemRaw, + occ: &RawOccurrence, + iana: Option<&str>, + ) -> Map { + let mut o = Map::new(); + if series.is_all_day_event == Some(true) { + let start = occ + .start + .as_deref() + .and_then(|s| WireDate(s).nearest_midnight(iana)); + if let Some(start) = start { + o.insert( + "start".to_owned(), + Value::String(CalendarItemRaw::all_day_key(start)), + ); + } + if occ.end.is_some() { + let days = CalendarItemRaw::all_day_span_days(start, occ.end.as_deref(), iana); + o.insert("duration".to_owned(), Value::String(format!("P{days}D"))); + } + return o; + } + if let Some(s) = occ.start.as_ref() { + o.insert( + "start".to_owned(), + Value::String(CalendarItemRaw::to_local_datetime_in(s, iana)), + ); + } + if let (Some(s), Some(e)) = (occ.start.as_ref(), occ.end.as_ref()) + && let Some(dur) = CalendarItemRaw::duration_iso8601(s, e) + { + o.insert("duration".to_owned(), Value::String(dur)); + } + o + } + + fn occurrence_key(series: &CalendarItemRaw, start: &str, iana: Option<&str>) -> String { + if series.is_all_day_event == Some(true) + && let Some(date) = WireDate(start).nearest_midnight(iana) + { + return CalendarItemRaw::all_day_key(date); + } + CalendarItemRaw::to_local_datetime_in(start, iana) + } + + fn all_day_key(date: NaiveDate) -> String { + format!("{date}{MIDNIGHT}") + } + + fn split_offset(trimmed: &str) -> Option<&str> { + trimmed.strip_suffix('Z').or_else(|| { + trimmed + .rsplit_once(['+', '-']) + .map(|(base, _)| base) + .filter(|base| base.len() >= DATE_LEN) + }) + } + + fn date_and_clock(stripped: &str) -> Option<(&str, &str)> { + let (date, time) = stripped.split_once('T')?; + Some((date, time.split_once('.').map_or(time, |(clock, _)| clock))) + } + + fn normalise_utc_datetime(s: &str) -> String { + let trimmed = s.trim(); + let stripped = CalendarItemRaw::split_offset(trimmed).unwrap_or(trimmed); + match CalendarItemRaw::date_and_clock(stripped) { + Some((date, clock)) => format!("{date}T{clock}Z"), + None => format!("{stripped}{MIDNIGHT}Z"), + } + } + + fn to_local_datetime_in(s: &str, iana: Option<&str>) -> String { + let trimmed = s.trim(); + let anchored = CalendarItemRaw::split_offset(trimmed); + if anchored.is_some() + && let Some(tz) = iana + && let Some(out) = CalendarItemRaw::convert_utc_to_local(trimmed, tz) + { + return out; + } + let stripped = anchored.unwrap_or(trimmed); + match CalendarItemRaw::date_and_clock(stripped) { + Some((date, clock)) => format!("{date}T{clock}"), + None => format!("{stripped}{MIDNIGHT}"), + } + } + + fn convert_utc_to_local(s: &str, iana: &str) -> Option { + let utc = DateTime::parse_from_rfc3339(s).ok()?.with_timezone(&Utc); + let tz: Tz = iana.parse().ok()?; + Some(utc.with_timezone(&tz).format(LOCAL_FORMAT).to_string()) + } + + fn all_day_span_days(start: Option, end: Option<&str>, iana: Option<&str>) -> i64 { + match (start, end.and_then(|e| WireDate(e).nearest_midnight(iana))) { + (Some(start), Some(end)) => (end - start).num_days().max(1), + _ => 1, + } + } + + fn duration_iso8601(start: &str, end: &str) -> Option { + let s = OffsetDateTime::parse(start, &Rfc3339).ok()?; + let e = OffsetDateTime::parse(end, &Rfc3339).ok()?; + IsoDuration((e - s).whole_seconds()).format(DurationStyle::Ews) } } @@ -56,242 +534,6 @@ struct EventCtx<'a> { owner_smtp: Option<&'a str>, } -fn build_event_map(raw: &CalendarItemRaw, ctx: EventCtx<'_>) -> Map { - let iana = ctx.iana; - let mut event = Map::new(); - event.insert("@type".to_owned(), Value::String("Event".to_owned())); - if let Some(uid) = raw.uid.as_ref() { - event.insert("uid".to_owned(), Value::String(uid.clone())); - } else { - event.insert( - "uid".to_owned(), - Value::String(format!( - "inbuxa-migrate-ews-event-{}", - blake3::hash(raw.id.id.as_bytes()).to_hex() - )), - ); - } - if let Some(subject) = raw.subject.as_ref() { - event.insert("title".to_owned(), Value::String(subject.clone())); - } - if let Some(loc) = raw.location.as_ref() { - event.insert( - "locations".to_owned(), - Value::Object(map_singleton( - "1", - json!({"@type": "Location", "name": loc}), - )), - ); - } - if let Some(body) = raw.body_text.as_ref() { - event.insert("description".to_owned(), Value::String(body.clone())); - } else if let Some(body) = raw.body_html.as_ref() { - event.insert("description".to_owned(), Value::String(body.clone())); - event.insert( - "descriptionContentType".to_owned(), - Value::String("text/html".to_owned()), - ); - } - if let Some(created) = raw.created.as_ref() { - event.insert( - "created".to_owned(), - Value::String(normalise_utc_datetime(created)), - ); - } - if let Some(updated) = raw.last_modified.as_ref().or(raw.created.as_ref()) { - event.insert( - "updated".to_owned(), - Value::String(normalise_utc_datetime(updated)), - ); - } - if !raw.categories.is_empty() { - let mut kw = Map::new(); - for c in &raw.categories { - kw.insert(c.to_ascii_lowercase(), Value::Bool(true)); - } - event.insert("keywords".to_owned(), Value::Object(kw)); - } - if let Some(true) = raw.is_all_day_event { - let start = raw.start.as_deref().and_then(|s| nearest_midnight(s, iana)); - if let Some(start) = start { - event.insert("start".to_owned(), Value::String(all_day_key(start))); - } - event.insert("showWithoutTime".to_owned(), Value::Bool(true)); - let days = all_day_span_days(start, raw.end.as_deref(), iana); - event.insert("duration".to_owned(), Value::String(format!("P{days}D"))); - } else { - if let Some(start) = raw.start.as_ref() { - event.insert( - "start".to_owned(), - Value::String(to_local_datetime_in(start, iana)), - ); - } - if let (Some(start), Some(end)) = (raw.start.as_ref(), raw.end.as_ref()) - && let Some(dur) = duration_iso8601(start, end) - { - event.insert("duration".to_owned(), Value::String(dur)); - } - if let Some(tz) = iana { - event.insert("timeZone".to_owned(), Value::String(tz.to_owned())); - } - } - if let Some(sensitivity) = raw.sensitivity.as_deref() { - event.insert( - "privacy".to_owned(), - Value::String(privacy_for(sensitivity).to_owned()), - ); - } - if let Some(status) = raw.legacy_free_busy_status.as_ref() { - let mapped = match status.as_str() { - "Free" => "free", - _ => "busy", - }; - event.insert( - "freeBusyStatus".to_owned(), - Value::String(mapped.to_owned()), - ); - } - let mut participants = Map::new(); - let mut next_id = 1; - if let Some((cal_addr, smtp)) = resolve_calendar_address( - raw.organizer_smtp.as_deref(), - raw.organizer_routing_type.as_deref(), - raw.organizer_name.as_deref(), - ) { - let key = next_id.to_string(); - next_id += 1; - event.insert( - "organizerCalendarAddress".to_owned(), - Value::String(cal_addr.clone()), - ); - let mut p = Map::new(); - p.insert("@type".to_owned(), Value::String("Participant".to_owned())); - p.insert("calendarAddress".to_owned(), Value::String(cal_addr)); - if let Some(email) = smtp { - p.insert("email".to_owned(), Value::String(email)); - } - let mut roles = Map::new(); - roles.insert("owner".to_owned(), Value::Bool(true)); - roles.insert("chair".to_owned(), Value::Bool(true)); - p.insert("roles".to_owned(), Value::Object(roles)); - if let Some(name) = raw.organizer_name.as_ref() { - p.insert("name".to_owned(), Value::String(name.clone())); - } - participants.insert(key, Value::Object(p)); - } - add_attendees( - &mut participants, - &mut next_id, - &raw.required_attendees, - AttendeeRole::Required, - ); - add_attendees( - &mut participants, - &mut next_id, - &raw.optional_attendees, - AttendeeRole::Optional, - ); - add_attendees( - &mut participants, - &mut next_id, - &raw.resources, - AttendeeRole::Resource, - ); - if let Some(owner) = ctx.owner_smtp - && let Some(status) = raw - .my_response_type - .as_deref() - .and_then(own_response_status) - { - for p in participants.values_mut().filter_map(Value::as_object_mut) { - let is_owner = p - .get("email") - .and_then(Value::as_str) - .is_some_and(|email| email.eq_ignore_ascii_case(owner)); - if is_owner && p.contains_key("expectReply") { - p.insert( - "participationStatus".to_owned(), - Value::String(status.to_owned()), - ); - } - } - } - if !participants.is_empty() { - if !event.contains_key("organizerCalendarAddress") { - drop_calendar_address_dependents(&mut participants); - } - event.insert("participants".to_owned(), Value::Object(participants)); - } - if let Some(rec) = raw.recurrence.as_ref() - && let Some(rule) = to_jscalendar_rule(rec) - { - event.insert("recurrenceRule".to_owned(), rule); - } - if raw.reminder_is_set == Some(true) { - let minutes = raw.reminder_minutes_before_start.unwrap_or(0).max(0); - let offset = if minutes == 0 { - "PT0S".to_owned() - } else { - format!("-PT{minutes}M") - }; - let alert = json!({ - "@type": "Alert", - "trigger": {"@type": "OffsetTrigger", "offset": offset, "relativeTo": "start"}, - "action": "display", - }); - event.insert( - "alerts".to_owned(), - Value::Object(map_singleton("1", alert)), - ); - } - if let Some(url) = raw - .join_online_meeting_url - .as_deref() - .or(raw.net_show_url.as_deref()) - .or(raw.meeting_workspace_url.as_deref()) - .filter(|s| !s.is_empty()) - { - let mut vl = Map::new(); - vl.insert( - "@type".to_owned(), - Value::String("VirtualLocation".to_owned()), - ); - vl.insert("uri".to_owned(), Value::String(url.to_owned())); - if raw.is_online_meeting == Some(true) { - vl.insert("features".to_owned(), json!({"video": true})); - } - event.insert( - "virtualLocations".to_owned(), - Value::Object(map_singleton("1", Value::Object(vl))), - ); - } - event -} - -fn is_smtp_routing(routing_type: Option<&str>) -> bool { - routing_type.is_none_or(|rt| rt.eq_ignore_ascii_case("SMTP")) -} - -fn resolve_calendar_address( - address: Option<&str>, - routing_type: Option<&str>, - name: Option<&str>, -) -> Option<(String, Option)> { - if let Some(addr) = address.filter(|a| !a.trim().is_empty()) { - let smtp = if is_smtp_routing(routing_type) { - as_smtp_address(addr).or_else(|| extract_smtp_address(addr)) - } else { - extract_smtp_address(addr) - }; - if let Some(smtp) = smtp { - return Some((format!("mailto:{smtp}"), Some(smtp))); - } - return Some((synthetic_attendee_address(addr), None)); - } - name.filter(|n| !n.trim().is_empty()) - .map(|n| (synthetic_attendee_address(n), None)) -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum AttendeeRole { Required, @@ -299,322 +541,11 @@ enum AttendeeRole { Resource, } -fn add_attendees( - out: &mut Map, - next_id: &mut u32, - attendees: &[RawAttendee], - role: AttendeeRole, -) { - for att in attendees { - let key = next_id.to_string(); - *next_id += 1; - let mut p = Map::new(); - p.insert("@type".to_owned(), Value::String("Participant".to_owned())); - if let Some(name) = att.name.as_ref() { - p.insert("name".to_owned(), Value::String(name.clone())); - } - let calendar_address = resolve_calendar_address( - att.email.as_deref(), - att.routing_type.as_deref(), - att.name.as_deref(), - ); - if let Some((addr, smtp)) = calendar_address { - p.insert("calendarAddress".to_owned(), Value::String(addr)); - if let Some(email) = smtp { - p.insert("email".to_owned(), Value::String(email)); - } - let mut roles = Map::new(); - match role { - AttendeeRole::Required | AttendeeRole::Resource => { - roles.insert("required".to_owned(), Value::Bool(true)); - } - AttendeeRole::Optional => { - roles.insert("optional".to_owned(), Value::Bool(true)); - } - } - p.insert("roles".to_owned(), Value::Object(roles)); - if role == AttendeeRole::Resource { - p.insert("kind".to_owned(), Value::String("resource".to_owned())); - } else { - p.insert("expectReply".to_owned(), Value::Bool(true)); - } - if let Some(rt) = att.response_type.as_deref() { - let mapped = participation_status(rt).unwrap_or("needs-action"); - p.insert( - "participationStatus".to_owned(), - Value::String(mapped.to_owned()), - ); - } - } - out.insert(key, Value::Object(p)); - } -} - -fn participation_status(response_type: &str) -> Option<&'static str> { - hashify::fnc_map!( - response_type.as_bytes(), - "Accept" => { return Some("accepted"); }, - "Tentative" => { return Some("tentative"); }, - "Decline" => { return Some("declined"); }, - "Organizer" => { return Some("accepted"); }, - _ => {} - ); - None -} - -fn own_response_status(my_response_type: &str) -> Option<&'static str> { - if my_response_type == "Organizer" { - return None; - } - participation_status(my_response_type) -} - -fn privacy_for(sensitivity: &str) -> &'static str { - hashify::fnc_map!( - sensitivity.as_bytes(), - "Private" => { return "private"; }, - "Confidential" => { return "secret"; }, - _ => {} - ); - "public" -} - -const NOT_PATCHED_PER_OCCURRENCE: &[&str] = &["created", "updated"]; - -fn is_override_excluded(key: &str) -> bool { - is_override_ignored(key) || NOT_PATCHED_PER_OCCURRENCE.contains(&key) -} - -fn build_recurrence_overrides( - base_event: &Map, - raw: &CalendarItemRaw, - modified_full: &[CalendarItemRaw], - ctx: EventCtx<'_>, -) -> Option { - let iana = ctx.iana; - if raw.modified_occurrences.is_empty() && raw.deleted_occurrences.is_empty() { - return None; - } - let mut map = Map::new(); - for occ in &raw.modified_occurrences { - let Some(key) = occ - .original_start - .as_deref() - .or(occ.start.as_deref()) - .map(|s| occurrence_key(raw, s, iana)) - else { - continue; - }; - let full = modified_full - .iter() - .find(|f| !occ.item_id.id.is_empty() && f.id.id == occ.item_id.id); - let patch = match full { - Some(full) => { - let occ_event = build_event_map(full, ctx); - override_patch(&occ_event, base_event, &key) - } - None => time_only_patch(raw, occ, iana), - }; - map.insert(key, Value::Object(patch)); - } - for occ in &raw.deleted_occurrences { - let Some(key) = occ.start.as_deref().map(|s| occurrence_key(raw, s, iana)) else { - continue; - }; - map.insert(key, json!({"excluded": true})); - } - if map.is_empty() { - None - } else { - Some(Value::Object(map)) - } -} - -fn override_patch( - occ_event: &Map, - base_event: &Map, - recurrence_id: &str, -) -> Map { - let mut patch = Map::new(); - let inherited_start = Value::String(recurrence_id.to_owned()); - for (k, v) in occ_event { - if is_override_excluded(k) { - continue; - } - let baseline = if k == "start" { - Some(&inherited_start) - } else { - base_event.get(k) - }; - if baseline != Some(v) { - patch.insert(k.clone(), v.clone()); - } - } - for k in base_event.keys() { - if k == "start" || is_override_excluded(k) { - continue; - } - if !occ_event.contains_key(k) { - patch.insert(k.clone(), Value::Null); - } - } - patch -} - -fn time_only_patch( - series: &CalendarItemRaw, - occ: &RawOccurrence, - iana: Option<&str>, -) -> Map { - let mut o = Map::new(); - if series.is_all_day_event == Some(true) { - let start = occ.start.as_deref().and_then(|s| nearest_midnight(s, iana)); - if let Some(start) = start { - o.insert("start".to_owned(), Value::String(all_day_key(start))); - } - if occ.end.is_some() { - let days = all_day_span_days(start, occ.end.as_deref(), iana); - o.insert("duration".to_owned(), Value::String(format!("P{days}D"))); - } - return o; - } - if let Some(s) = occ.start.as_ref() { - o.insert( - "start".to_owned(), - Value::String(to_local_datetime_in(s, iana)), - ); - } - if let (Some(s), Some(e)) = (occ.start.as_ref(), occ.end.as_ref()) - && let Some(dur) = duration_iso8601(s, e) - { - o.insert("duration".to_owned(), Value::String(dur)); - } - o -} - -fn occurrence_key(series: &CalendarItemRaw, start: &str, iana: Option<&str>) -> String { - if series.is_all_day_event == Some(true) - && let Some(date) = nearest_midnight(start, iana) - { - return all_day_key(date); - } - to_local_datetime_in(start, iana) -} - -fn all_day_key(date: NaiveDate) -> String { - format!("{date}T00:00:00") -} - -fn normalise_utc_datetime(s: &str) -> String { - let trimmed = s.trim(); - let stripped = trimmed - .strip_suffix('Z') - .or_else(|| { - trimmed - .rfind(['+', '-']) - .filter(|i| *i >= 10) - .map(|i| &trimmed[..i]) - }) - .unwrap_or(trimmed); - let (date, time) = match stripped.split_once('T') { - Some((d, t)) => (d, t), - None => return format!("{stripped}T00:00:00Z"), - }; - let time_clean = if let Some(dot) = time.find('.') { - &time[..dot] - } else { - time - }; - format!("{date}T{time_clean}Z") -} - -fn to_local_datetime_in(s: &str, iana: Option<&str>) -> String { - let trimmed = s.trim(); - let utc_anchored = - trimmed.ends_with('Z') || trimmed.rfind(['+', '-']).filter(|i| *i >= 10).is_some(); - if utc_anchored - && let Some(tz) = iana - && let Some(out) = convert_utc_to_local(trimmed, tz) - { - return out; - } - let stripped = trimmed - .strip_suffix('Z') - .or_else(|| { - trimmed - .rfind(['+', '-']) - .filter(|i| *i >= 10) - .map(|i| &trimmed[..i]) - }) - .unwrap_or(trimmed); - let (date, time) = match stripped.split_once('T') { - Some((d, t)) => (d, t), - None => return format!("{stripped}T00:00:00"), - }; - let time_clean = if let Some(dot) = time.find('.') { - &time[..dot] - } else { - time - }; - format!("{date}T{time_clean}") -} - -fn convert_utc_to_local(s: &str, iana: &str) -> Option { - use chrono::DateTime; - let utc: DateTime = DateTime::parse_from_rfc3339(s) - .ok()? - .with_timezone(&chrono::Utc); - let tz: chrono_tz::Tz = iana.parse().ok()?; - let local = utc.with_timezone(&tz); - Some(local.format("%Y-%m-%dT%H:%M:%S").to_string()) -} - -fn all_day_span_days(start: Option, end: Option<&str>, iana: Option<&str>) -> i64 { - match (start, end.and_then(|e| nearest_midnight(e, iana))) { - (Some(start), Some(end)) => (end - start).num_days().max(1), - _ => 1, - } -} - -fn duration_iso8601(start: &str, end: &str) -> Option { - use time::OffsetDateTime; - use time::format_description::well_known::Rfc3339; - let s = OffsetDateTime::parse(start, &Rfc3339).ok()?; - let e = OffsetDateTime::parse(end, &Rfc3339).ok()?; - let dur = e - s; - if dur.is_zero() { - return Some("PT0S".to_owned()); - } - let total_seconds = dur.whole_seconds(); - if total_seconds < 0 { - return None; - } - let hours = total_seconds / 3600; - let minutes = (total_seconds % 3600) / 60; - let seconds = total_seconds % 60; - let mut out = String::from("PT"); - if hours > 0 { - out.push_str(&format!("{hours}H")); - } - if minutes > 0 { - out.push_str(&format!("{minutes}M")); - } - if seconds > 0 || (hours == 0 && minutes == 0) { - out.push_str(&format!("{seconds}S")); - } - Some(out) -} - -fn map_singleton(key: &str, value: Value) -> Map { - let mut m = Map::new(); - m.insert(key.to_owned(), value); - m -} - #[cfg(test)] mod tests { use super::*; - use crate::exchange_ews::parse::{RecurrencePattern, RecurrenceRange}; + use crate::exchange::ews::parse::{RecurrencePattern, RecurrenceRange}; + use crate::exchange::jscalendar::SyntheticAttendee; #[test] fn simple_event_round_trip_converts_utc_to_local_wall_clock() { @@ -627,7 +558,7 @@ mod tests { start_tz: Some("Pacific Standard Time".to_owned()), ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; assert_eq!(v["@type"], "Event"); assert_eq!(v["uid"], "uid-1"); assert_eq!(v["title"], "Meeting"); @@ -645,7 +576,7 @@ mod tests { end: Some("2025-06-15T15:00:00".to_owned()), ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; assert_eq!(v["start"], "2025-06-15T14:00:00"); assert!(v.get("timeZone").is_none()); } @@ -659,7 +590,7 @@ mod tests { start_tz: Some("Made Up Time".to_owned()), ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; assert_eq!(v["start"], "2025-06-15T14:00:00"); assert_eq!(v["timeZone"], "Etc/UTC"); } @@ -671,7 +602,7 @@ mod tests { start: Some("2025-06-15T21:00:00Z".to_owned()), end: Some("2025-06-15T22:00:00Z".to_owned()), start_tz: Some("Pacific Standard Time".to_owned()), - recurrence: Some(crate::exchange_ews::parse::RawRecurrence { + recurrence: Some(crate::exchange::ews::parse::RawRecurrence { pattern: Some(RecurrencePattern::Daily { interval: 1 }), range: Some(RecurrenceRange::Numbered { start_date: "2025-06-15".to_owned(), @@ -686,7 +617,7 @@ mod tests { }], ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; let over = v["recurrenceOverrides"].as_object().unwrap(); assert!(over.contains_key("2025-06-16T14:00:00")); } @@ -701,7 +632,7 @@ mod tests { end: Some("2025-12-26T00:00:00".to_owned()), ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; assert_eq!(v["start"], "2025-12-25T00:00:00"); assert_eq!(v["showWithoutTime"], true); assert_eq!(v["duration"], "P1D"); @@ -714,7 +645,7 @@ mod tests { uid: Some("uid-3".to_owned()), start: Some("2025-06-15T14:00:00Z".to_owned()), end: Some("2025-06-15T15:00:00Z".to_owned()), - recurrence: Some(crate::exchange_ews::parse::RawRecurrence { + recurrence: Some(crate::exchange::ews::parse::RawRecurrence { pattern: Some(RecurrencePattern::Daily { interval: 1 }), range: Some(RecurrenceRange::Numbered { start_date: "2025-06-15".to_owned(), @@ -735,7 +666,7 @@ mod tests { }], ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; assert!(v.get("recurrenceRules").is_none()); assert_eq!(v["recurrenceRule"]["frequency"], "daily"); assert_eq!(v["recurrenceRule"]["count"], 3); @@ -754,7 +685,7 @@ mod tests { last_modified: None, ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; assert_eq!( v["updated"], "2025-06-01T09:00:00Z", "updated is mandatory (jscalendarbis 3.1.6); backfill from created" @@ -768,7 +699,7 @@ mod tests { start: Some("2025-06-15T14:00:00Z".to_owned()), end: Some("2025-06-15T15:00:00Z".to_owned()), organizer_smtp: Some("alice@example.com".to_owned()), - required_attendees: vec![crate::exchange_ews::parse::RawAttendee { + required_attendees: vec![crate::exchange::ews::parse::RawAttendee { email: None, routing_type: None, name: Some("No Address Person".to_owned()), @@ -776,7 +707,7 @@ mod tests { }], ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; let p = v["participants"] .as_object() .unwrap() @@ -803,12 +734,12 @@ mod tests { #[test] fn synthetic_attendee_address_is_stable_per_name() { assert_eq!( - synthetic_attendee_address("Jane Doe"), - synthetic_attendee_address("Jane Doe") + SyntheticAttendee("Jane Doe").to_string(), + SyntheticAttendee("Jane Doe").to_string() ); assert_ne!( - synthetic_attendee_address("Jane Doe"), - synthetic_attendee_address("John Doe") + SyntheticAttendee("Jane Doe").to_string(), + SyntheticAttendee("John Doe").to_string() ); } @@ -830,7 +761,7 @@ mod tests { }], ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; let org_addr = v["organizerCalendarAddress"].as_str().unwrap(); assert!( org_addr.starts_with("urn:x-inbuxa-migrate:attendee:") && !org_addr.contains("mailto:"), @@ -863,7 +794,7 @@ mod tests { }], ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; let att = v["participants"] .as_object() .unwrap() @@ -881,7 +812,7 @@ mod tests { subject: Some("Standup".to_owned()), start: Some("2025-06-16T14:00:00Z".to_owned()), end: Some("2025-06-16T14:30:00Z".to_owned()), - recurrence: Some(crate::exchange_ews::parse::RawRecurrence { + recurrence: Some(crate::exchange::ews::parse::RawRecurrence { pattern: Some(RecurrencePattern::Daily { interval: 1 }), range: Some(RecurrenceRange::Numbered { start_date: "2025-06-16".to_owned(), @@ -889,7 +820,7 @@ mod tests { }), }), modified_occurrences: vec![RawOccurrence { - item_id: crate::exchange_ews::types::ItemId::new("EXC1", ""), + item_id: crate::exchange::ews::types::ItemId::new("EXC1", ""), start: Some("2025-06-18T14:00:00Z".to_owned()), end: Some("2025-06-18T14:30:00Z".to_owned()), original_start: Some("2025-06-18T14:00:00Z".to_owned()), @@ -897,7 +828,7 @@ mod tests { ..CalendarItemRaw::default() }; let full_occ = CalendarItemRaw { - id: crate::exchange_ews::types::ItemId::new("EXC1", ""), + id: crate::exchange::ews::types::ItemId::new("EXC1", ""), uid: Some("uid-series".to_owned()), subject: Some("Sprint Retro".to_owned()), location: Some("Big Room".to_owned()), @@ -905,7 +836,9 @@ mod tests { end: Some("2025-06-18T14:30:00Z".to_owned()), ..CalendarItemRaw::default() }; - let v = to_jscalendar_with_exceptions(&master, std::slice::from_ref(&full_occ), None).data; + let v = master + .to_jscalendar_with_exceptions(std::slice::from_ref(&full_occ), None) + .data; let ov = &v["recurrenceOverrides"]["2025-06-18T14:00:00"]; assert_eq!( ov["title"], "Sprint Retro", @@ -931,7 +864,7 @@ mod tests { end: Some("2025-07-07T00:00:00".to_owned()), ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; assert_eq!(v["duration"], "P3D"); assert_eq!(v["showWithoutTime"], true); } @@ -939,13 +872,13 @@ mod tests { #[test] fn ex_routing_with_a_real_address_keeps_the_address() { let raw = CalendarItemRaw { - id: crate::exchange_ews::types::ItemId::new("M", ""), + id: crate::exchange::ews::types::ItemId::new("M", ""), uid: Some("uid-ex".to_owned()), start: Some("2025-06-15T14:00:00Z".to_owned()), end: Some("2025-06-15T15:00:00Z".to_owned()), organizer_smtp: Some("alice@example.com".to_owned()), organizer_routing_type: Some("EX".to_owned()), - required_attendees: vec![crate::exchange_ews::parse::RawAttendee { + required_attendees: vec![crate::exchange::ews::parse::RawAttendee { email: Some("SMTP:bob@example.com".to_owned()), routing_type: Some("EX".to_owned()), name: None, @@ -953,7 +886,7 @@ mod tests { }], ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; assert_eq!( v["organizerCalendarAddress"], "mailto:alice@example.com", "a usable address must not be discarded because RoutingType says EX" @@ -972,7 +905,7 @@ mod tests { let dn = "/o=ExchangeLabs/ou=Exchange Administrative Group \ (FYDIBOHF23SPDLT)/cn=Recipients/cn=bdc77b18152647a29d28ce1188376dc9-kristina"; let raw = CalendarItemRaw { - id: crate::exchange_ews::types::ItemId::new("M", ""), + id: crate::exchange::ews::types::ItemId::new("M", ""), uid: Some("uid-dn".to_owned()), start: Some("2025-06-15T14:00:00Z".to_owned()), end: Some("2025-06-15T15:00:00Z".to_owned()), @@ -980,7 +913,7 @@ mod tests { organizer_routing_type: Some("EX".to_owned()), ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; let addr = v["organizerCalendarAddress"].as_str().unwrap(); assert!( addr.starts_with("urn:x-inbuxa-migrate:attendee:"), @@ -996,7 +929,7 @@ mod tests { #[test] fn smtp_routing_with_an_unusable_address_does_not_produce_a_broken_mailto() { let raw = CalendarItemRaw { - id: crate::exchange_ews::types::ItemId::new("M", ""), + id: crate::exchange::ews::types::ItemId::new("M", ""), uid: Some("uid-bad".to_owned()), start: Some("2025-06-15T14:00:00Z".to_owned()), end: Some("2025-06-15T15:00:00Z".to_owned()), @@ -1004,7 +937,7 @@ mod tests { organizer_routing_type: Some("SMTP".to_owned()), ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; assert!( v["organizerCalendarAddress"] .as_str() @@ -1020,13 +953,13 @@ mod tests { start: Some("2025-06-15T14:00:00Z".to_owned()), end: Some("2025-06-15T15:00:00Z".to_owned()), organizer_smtp: Some("alice@x".to_owned()), - required_attendees: vec![crate::exchange_ews::parse::RawAttendee { + required_attendees: vec![crate::exchange::ews::parse::RawAttendee { email: Some("bob@x".to_owned()), routing_type: None, name: None, response_type: Some("Accept".to_owned()), }], - optional_attendees: vec![crate::exchange_ews::parse::RawAttendee { + optional_attendees: vec![crate::exchange::ews::parse::RawAttendee { email: Some("eve@x".to_owned()), routing_type: None, name: None, @@ -1034,7 +967,7 @@ mod tests { }], ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; assert_eq!(v["organizerCalendarAddress"], "mailto:alice@x"); let participants = v["participants"].as_object().unwrap(); let required = participants @@ -1060,7 +993,7 @@ mod tests { reminder_minutes_before_start: Some(15), ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; let alert = &v["alerts"]["1"]; assert_eq!(alert["@type"], "Alert"); assert_eq!(alert["action"], "display"); @@ -1079,7 +1012,7 @@ mod tests { reminder_minutes_before_start: Some(15), ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; assert!(v.get("alerts").is_none()); } @@ -1093,7 +1026,7 @@ mod tests { net_show_url: Some("https://teams.example/join/abc".to_owned()), ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; let vl = &v["virtualLocations"]["1"]; assert_eq!(vl["@type"], "VirtualLocation"); assert_eq!(vl["uri"], "https://teams.example/join/abc"); @@ -1110,7 +1043,7 @@ mod tests { meeting_workspace_url: Some("https://workspace/secondary".to_owned()), ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; assert_eq!(v["virtualLocations"]["1"]["uri"], "https://join/primary"); } @@ -1121,7 +1054,7 @@ mod tests { start: Some("2025-06-15T14:00:00Z".to_owned()), end: Some("2025-06-15T15:00:00Z".to_owned()), organizer_smtp: Some("alice@example.com".to_owned()), - resources: vec![crate::exchange_ews::parse::RawAttendee { + resources: vec![crate::exchange::ews::parse::RawAttendee { email: Some("room-7@x".to_owned()), routing_type: None, name: Some("Room 7".to_owned()), @@ -1129,7 +1062,7 @@ mod tests { }], ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; let res = v["participants"] .as_object() .unwrap() @@ -1149,7 +1082,7 @@ mod tests { start: Some("2025-06-15T14:00:00Z".to_owned()), end: Some("2025-06-15T15:00:00Z".to_owned()), organizer_smtp: Some("alice@example.com".to_owned()), - required_attendees: vec![crate::exchange_ews::parse::RawAttendee { + required_attendees: vec![crate::exchange::ews::parse::RawAttendee { email: Some("bob@x".to_owned()), routing_type: None, name: None, @@ -1157,7 +1090,7 @@ mod tests { }], ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; let bob = v["participants"] .as_object() .unwrap() @@ -1177,13 +1110,13 @@ mod tests { organizer_smtp: Some("alice@example.com".to_owned()), my_response_type: Some(my_response.to_owned()), required_attendees: vec![ - crate::exchange_ews::parse::RawAttendee { + crate::exchange::ews::parse::RawAttendee { email: Some("Me@Example.com".to_owned()), routing_type: None, name: None, response_type: Some("Unknown".to_owned()), }, - crate::exchange_ews::parse::RawAttendee { + crate::exchange::ews::parse::RawAttendee { email: Some("bob@example.com".to_owned()), routing_type: None, name: None, @@ -1207,13 +1140,17 @@ mod tests { #[test] fn own_response_sets_the_mailbox_owner_participation_status() { let raw = invited_raw("Accept"); - let v = to_jscalendar_with_exceptions(&raw, &[], Some("me@example.com")).data; + let v = raw + .to_jscalendar_with_exceptions(&[], Some("me@example.com")) + .data; assert_eq!(status_of(&v, "Me@Example.com"), "accepted"); assert_eq!(status_of(&v, "bob@example.com"), "needs-action"); - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; assert_eq!(status_of(&v, "Me@Example.com"), "needs-action"); let raw = invited_raw("NoResponseReceived"); - let v = to_jscalendar_with_exceptions(&raw, &[], Some("me@example.com")).data; + let v = raw + .to_jscalendar_with_exceptions(&[], Some("me@example.com")) + .data; assert_eq!(status_of(&v, "Me@Example.com"), "needs-action"); } @@ -1231,7 +1168,7 @@ mod tests { ..CalendarItemRaw::default() }; assert_eq!( - to_jscalendar(&raw).data["privacy"], + raw.to_jscalendar().data["privacy"], privacy, "{sensitivity}" ); @@ -1240,7 +1177,7 @@ mod tests { uid: Some("uid-p".to_owned()), ..CalendarItemRaw::default() }; - assert!(to_jscalendar(&raw).data.get("privacy").is_none()); + assert!(raw.to_jscalendar().data.get("privacy").is_none()); } #[test] @@ -1249,7 +1186,7 @@ mod tests { uid: Some("uid-no-organizer".to_owned()), start: Some("2025-06-15T14:00:00Z".to_owned()), end: Some("2025-06-15T15:00:00Z".to_owned()), - required_attendees: vec![crate::exchange_ews::parse::RawAttendee { + required_attendees: vec![crate::exchange::ews::parse::RawAttendee { email: Some("bob@x".to_owned()), routing_type: None, name: Some("Bob".to_owned()), @@ -1257,7 +1194,7 @@ mod tests { }], ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; assert!(v.get("organizerCalendarAddress").is_none()); let bob = v["participants"] .as_object() @@ -1276,11 +1213,26 @@ mod tests { #[test] fn all_day_span_helper_is_inclusive_of_start() { let start = NaiveDate::from_ymd_opt(2025, 7, 4); - assert_eq!(all_day_span_days(start, Some("2025-07-04"), None), 1); - assert_eq!(all_day_span_days(start, Some("2025-07-05"), None), 1); - assert_eq!(all_day_span_days(start, Some("2025-07-07"), None), 3); - assert_eq!(all_day_span_days(None, Some("2025-07-07"), None), 1); - assert_eq!(all_day_span_days(start, Some("bad"), None), 1); + assert_eq!( + CalendarItemRaw::all_day_span_days(start, Some("2025-07-04"), None), + 1 + ); + assert_eq!( + CalendarItemRaw::all_day_span_days(start, Some("2025-07-05"), None), + 1 + ); + assert_eq!( + CalendarItemRaw::all_day_span_days(start, Some("2025-07-07"), None), + 3 + ); + assert_eq!( + CalendarItemRaw::all_day_span_days(None, Some("2025-07-07"), None), + 1 + ); + assert_eq!( + CalendarItemRaw::all_day_span_days(start, Some("bad"), None), + 1 + ); } #[test] @@ -1293,7 +1245,7 @@ mod tests { start_tz: Some("Romance Standard Time".to_owned()), ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; assert_eq!(v["start"], "2026-11-12T00:00:00"); assert_eq!(v["duration"], "P1D"); assert!(v.get("timeZone").is_none()); @@ -1307,7 +1259,7 @@ mod tests { start: Some("2026-11-11T23:00:00Z".to_owned()), end: Some("2026-11-12T23:00:00Z".to_owned()), start_tz: Some("Romance Standard Time".to_owned()), - recurrence: Some(crate::exchange_ews::parse::RawRecurrence { + recurrence: Some(crate::exchange::ews::parse::RawRecurrence { pattern: Some(RecurrencePattern::Daily { interval: 1 }), range: Some(RecurrenceRange::Numbered { start_date: "2026-11-12".to_owned(), @@ -1328,7 +1280,7 @@ mod tests { }], ..CalendarItemRaw::default() }; - let v = to_jscalendar(&raw).data; + let v = raw.to_jscalendar().data; assert_eq!(v["start"], "2026-11-12T00:00:00"); let over = v["recurrenceOverrides"].as_object().unwrap(); assert_eq!(over.len(), 2); diff --git a/src/exchange/ews/client.rs b/src/exchange/ews/client.rs new file mode 100644 index 0000000..2fc6a21 --- /dev/null +++ b/src/exchange/ews/client.rs @@ -0,0 +1,530 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +use ureq::http::HeaderMap; + +use crate::exchange::ews::error::EwsError; +use crate::exchange::ews::parse::{EnvelopeKind, SoapFault}; +use crate::exchange::ews::retry::FaultDisposition; +use crate::exchange::ews::soap::{EnvelopeOptions, SoapAction}; +use crate::exchange::ews::types::ServerVersion; +use crate::exchange::ews::xml::EwsRequest; +use crate::logging::{HttpCall, Level, Logger}; +use crate::net::auth::Auth; +use crate::net::http::{ + AgentProfile, BodyExcerpt, ClientOptions, HttpCore, Methods, Redirects, RetryStats, + TransportError, +}; +use crate::net::retry::{Disposition, RetryAfter, RetryWait}; + +const MAX_BODY: u64 = 2 * 1024 * 1024 * 1024; +const LONG_RETRY_THRESHOLD: Duration = Duration::from_secs(10); +const UNAUTHORIZED: u16 = 401; +const FORBIDDEN: u16 = 403; +const ACCOUNT_LOCKED: u16 = 456; +const AFFINITY_COOKIE: &str = "X-BackEndOverrideCookie="; +const PROFILE: AgentProfile = AgentProfile { + redirects: Redirects::Follow, + methods: Methods::Standard, + max_body: MAX_BODY, +}; + +struct Inner { + core: HttpCore, + auth: Mutex, + impersonated_smtp: Mutex>, + anchor_mailbox: Mutex>, + affinity_cookie: Mutex>, + version: Mutex, +} + +#[derive(Debug, Clone, Copy)] +struct VersionState { + current: ServerVersion, + accepted_ceiling: Option, +} + +#[derive(Clone)] +pub struct EwsClient { + inner: Arc, +} + +#[derive(Debug, Clone)] +pub struct SoapResponse { + pub body: Vec, + pub server_version: Option, +} + +enum Attempt { + Response { + status: u16, + body: Vec, + retry_after: Option, + }, + Transport(EwsError), +} + +enum Step { + Done(SoapResponse), + Fail(EwsError), + Sleep(Duration), + Downgrade(ServerVersion), +} + +impl EwsClient { + pub fn new(auth: Auth, options: ClientOptions) -> EwsClient { + EwsClient { + inner: Arc::new(Inner { + core: HttpCore::new(options, PROFILE), + auth: Mutex::new(auth), + impersonated_smtp: Mutex::new(None), + anchor_mailbox: Mutex::new(None), + affinity_cookie: Mutex::new(None), + version: Mutex::new(VersionState { + current: ServerVersion::Exchange2013Sp1, + accepted_ceiling: None, + }), + }), + } + } + + pub fn set_auth(&self, auth: Auth) { + if let Ok(mut g) = self.inner.auth.lock() { + *g = auth; + } + } + + pub fn set_anchor_mailbox(&self, smtp: Option) { + if let Ok(mut g) = self.inner.anchor_mailbox.lock() { + *g = smtp; + } + } + + pub fn set_impersonation(&self, smtp: Option) { + if let Ok(mut g) = self.inner.impersonated_smtp.lock() { + *g = smtp; + } + } + + pub fn logger(&self) -> Logger { + self.inner.core.logger() + } + + pub fn set_server_version(&self, version: ServerVersion) { + if let Ok(mut g) = self.inner.version.lock() { + g.current = g.accepted_ceiling.map_or(version, |c| version.min(c)); + } + } + + fn pin_accepted_version(&self, version: ServerVersion) { + if let Ok(mut g) = self.inner.version.lock() { + g.current = version; + g.accepted_ceiling = Some(version); + } + } + + pub fn server_version(&self) -> ServerVersion { + self.inner + .version + .lock() + .map(|g| g.current) + .unwrap_or(ServerVersion::Exchange2013Sp1) + } + + pub fn stats(&self) -> RetryStats { + self.inner.core.stats().snapshot() + } + + pub fn soap_calls(&self) -> u64 { + self.inner.core.stats().requests() + } + + pub fn call(&self, url: &str, request: &EwsRequest<'_>) -> Result { + self.inner.core.stats().note_request(); + let action = SoapAction(request.operation()).to_string(); + let mut attempt: u32 = 0; + let mut downgraded = false; + loop { + self.inner.core.rate_limit().cooldown().wait(); + let sent_version = self.server_version(); + let envelope = self.wrap(request, sent_version); + let step = match self.one_attempt(url, &envelope, &action) { + Attempt::Response { + status, + body, + retry_after, + } => self.on_response( + status, + body, + retry_after, + sent_version, + &action, + &mut attempt, + ), + Attempt::Transport(err) => self.on_transport(err, &mut attempt), + }; + match step { + Step::Done(response) => { + if downgraded { + self.pin_accepted_version(sent_version); + } + if let Some(v) = response.server_version { + self.set_server_version(v); + } + return Ok(response); + } + Step::Fail(e) => return Err(e), + Step::Sleep(d) => thread::sleep(d), + Step::Downgrade(v) => { + self.set_server_version(v); + downgraded = true; + } + } + } + } + + fn wrap(&self, request: &EwsRequest<'_>, version: ServerVersion) -> String { + let smtp = self + .inner + .impersonated_smtp + .lock() + .ok() + .and_then(|g| g.clone()); + EnvelopeOptions { + version, + impersonated_smtp: smtp.as_deref(), + } + .wrap(request) + } + + fn auth_header(&self) -> String { + self.inner + .auth + .lock() + .map(|auth| auth.header_value()) + .unwrap_or_default() + } + + fn anchor_header(&self) -> Option { + self.inner + .anchor_mailbox + .lock() + .ok() + .and_then(|g| g.clone()) + } + + fn affinity_cookie(&self) -> Option { + self.inner + .affinity_cookie + .lock() + .ok() + .and_then(|g| g.clone()) + } + + fn on_response( + &self, + status: u16, + body: Vec, + retry_after: Option, + sent_version: ServerVersion, + action: &str, + attempt: &mut u32, + ) -> Step { + let summary = EnvelopeKind::read(&body); + if (200..300).contains(&status) { + self.inner.core.rate_limit().on_success(); + return match summary { + Ok(EnvelopeKind::Body { version }) => Step::Done(SoapResponse { + body, + server_version: version.to_server_version(), + }), + Ok(EnvelopeKind::Fault { fault, .. }) => { + self.on_fault(&fault, sent_version, action, attempt) + } + Err(e) => Step::Fail(e), + }; + } + if status != UNAUTHORIZED + && status != FORBIDDEN + && let Ok(EnvelopeKind::Fault { fault, .. }) = summary + { + return self.on_fault(&fault, sent_version, action, attempt); + } + match status { + UNAUTHORIZED => Step::Fail(EwsError::Auth(format!( + "server returned 401: {}", + BodyExcerpt(&body) + ))), + ACCOUNT_LOCKED => Step::Fail(EwsError::Auth(format!( + "account is locked (http 456); an administrator must unlock it: {}", + BodyExcerpt(&body) + ))), + FORBIDDEN => Step::Fail(EwsError::HttpStatus { + status, + body: BodyExcerpt(&body).to_string(), + }), + _ => match Disposition::of_status(status) { + Disposition::Fatal => Step::Fail(EwsError::HttpStatus { + status, + body: BodyExcerpt(&body).to_string(), + }), + Disposition::Retry => { + self.on_retryable_status(status, retry_after, action, attempt) + } + }, + } + } + + fn on_retryable_status( + &self, + status: u16, + retry_after: Option, + action: &str, + attempt: &mut u32, + ) -> Step { + let core = &self.inner.core; + let max_retries = core.retry().max_retries; + let logger = core.logger(); + *attempt += 1; + core.stats().note_retry(); + if *attempt > max_retries { + return Step::Fail(EwsError::RetriesExhausted(format!( + "{action} kept returning http {status}" + ))); + } + if retry_after.is_some() { + core.stats().note_retry_after(); + } + let chosen = core.throttle(retry_after); + if chosen >= LONG_RETRY_THRESHOLD { + logger.warn(format_args!( + "EWS {action} rate-limited (http {status}); waiting {}", + RetryWait(chosen) + )); + } + logger.progress(format_args!( + "retry {}/{max_retries} (http {status})", + *attempt + )); + Step::Sleep(chosen) + } + + fn on_transport(&self, err: EwsError, attempt: &mut u32) -> Step { + let core = &self.inner.core; + let max_retries = core.retry().max_retries; + *attempt += 1; + core.stats().note_retry(); + if *attempt > max_retries { + return Step::Fail(err); + } + let delay = core.backoff(*attempt); + if delay >= LONG_RETRY_THRESHOLD { + core.logger().warn(format_args!( + "transient transport failure ({err}); waiting {} before retry {}/{max_retries}", + RetryWait(delay), + *attempt + )); + } + Step::Sleep(delay) + } + + fn on_fault( + &self, + fault: &SoapFault, + sent_version: ServerVersion, + action: &str, + attempt: &mut u32, + ) -> Step { + let core = &self.inner.core; + let logger = core.logger(); + match fault.response_code.fault_disposition() { + FaultDisposition::Fatal => Step::Fail(EwsError::SoapFault { + code: fault.response_code.clone(), + reason: fault.fault_string.clone(), + }), + FaultDisposition::Auth => Step::Fail(EwsError::Auth(fault.fault_string.clone())), + FaultDisposition::VersionError => match sent_version.lower() { + Some(next) => { + logger.warn(format_args!( + "EWS rejected schema version {} ({}); retrying as {}", + sent_version.as_str(), + fault.response_code, + next.as_str() + )); + Step::Downgrade(next) + } + None => Step::Fail(EwsError::SoapFault { + code: fault.response_code.clone(), + reason: format!( + "server rejected every supported EWS schema version (last tried {}): {}", + sent_version.as_str(), + fault.fault_string + ), + }), + }, + FaultDisposition::Retryable { delay } => { + let max_retries = core.retry().max_retries; + *attempt += 1; + core.stats().note_retry(); + if *attempt > max_retries { + return Step::Fail(EwsError::RetriesExhausted(format!( + "{action} kept returning {code}", + code = fault.response_code + ))); + } + let chosen = core.throttle(delay); + if chosen >= LONG_RETRY_THRESHOLD { + logger.warn(format_args!( + "EWS SOAP fault {} ({}); waiting {} before retry {}/{max_retries}", + fault.response_code, + fault.fault_string, + RetryWait(chosen), + *attempt + )); + } + if logger.enabled(Level::Bodies) { + logger.progress(format_args!( + "retry {}/{max_retries} {action} after {chosen:?} ({})", + *attempt, fault.response_code, + )); + } + Step::Sleep(chosen) + } + } + } + + fn one_attempt(&self, url: &str, body: &str, action: &str) -> Attempt { + let core = &self.inner.core; + let mut req = core + .agent() + .post(url) + .header("Authorization", self.auth_header()) + .header("Content-Type", "text/xml; charset=utf-8") + .header("Accept", "text/xml, application/soap+xml, application/xml") + .header("Accept-Encoding", "gzip") + .header("SOAPAction", action) + .header("User-Agent", core.user_agent()) + .header("X-PreferServerAffinity", "True"); + if let Some(anchor) = self.anchor_header() { + req = req.header("X-AnchorMailbox", anchor); + } + if let Some(cookie) = self.affinity_cookie() { + req = req.header("X-BackEndOverrideCookie", cookie); + } + let logger = core.logger(); + let started = Instant::now(); + match req.send(body.as_bytes()) { + Ok(mut resp) => { + let status = resp.status().as_u16(); + let response_type = resp + .headers() + .get("content-type") + .and_then(|v| v.to_str().ok()) + .map(str::to_owned); + let retry_after = resp + .headers() + .get("retry-after") + .and_then(|v| v.to_str().ok()) + .and_then(RetryAfter::parse_now); + if let Some(cookie) = EwsClient::affinity_cookie_of(resp.headers()) + && let Ok(mut g) = self.inner.affinity_cookie.lock() + { + *g = Some(cookie); + } + match resp.body_mut().with_config().limit(MAX_BODY).read_to_vec() { + Ok(bytes) => { + logger.trace_http(&HttpCall { + proto: "EWS", + method: action, + url, + status, + elapsed: started.elapsed(), + note: None, + request: Some(body.as_bytes()), + request_type: Some("text/xml"), + response: &bytes, + response_type: response_type.as_deref(), + }); + Attempt::Response { + status, + body: bytes, + retry_after, + } + } + Err(e) => Attempt::Transport(EwsError::Transport(format!( + "reading response body: {e}" + ))), + } + } + Err(e) => { + let err = EwsError::from(TransportError::from(e)); + logger.trace_http_error("EWS", action, url, &err.to_string(), started.elapsed()); + Attempt::Transport(err) + } + } + } + + fn affinity_cookie_of(headers: &HeaderMap) -> Option { + headers + .get_all("set-cookie") + .iter() + .filter_map(|value| value.to_str().ok()) + .filter_map(|cookie| cookie.trim_start().strip_prefix(AFFINITY_COOKIE)) + .map(|rest| rest.split_once(';').map_or(rest, |(value, _)| value).trim()) + .find(|value| !value.is_empty()) + .map(str::to_owned) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::net::http::TlsPolicy; + use crate::net::retry::RetryPolicy; + + fn options(max_retries: u32) -> ClientOptions { + ClientOptions { + retry: RetryPolicy::new(max_retries), + tls: TlsPolicy::Verify, + logger: Logger::new(Level::Quiet), + } + } + + #[test] + fn client_constructs_with_defaults() { + let c = EwsClient::new(Auth::Bearer { token: "t".into() }, options(3)); + assert_eq!(c.server_version(), ServerVersion::Exchange2013Sp1); + assert_eq!(c.stats(), RetryStats::default()); + } + + #[test] + fn server_version_can_be_pinned() { + let c = EwsClient::new(Auth::Bearer { token: "t".into() }, options(0)); + c.set_server_version(ServerVersion::Exchange2016); + assert_eq!(c.server_version(), ServerVersion::Exchange2016); + } + + #[test] + fn accepted_downgrade_caps_reported_version() { + let c = EwsClient::new(Auth::Bearer { token: "t".into() }, options(0)); + c.pin_accepted_version(ServerVersion::Exchange2013Sp1); + c.set_server_version(ServerVersion::Exchange2016); + assert_eq!(c.server_version(), ServerVersion::Exchange2013Sp1); + c.set_server_version(ServerVersion::Exchange2010Sp2); + assert_eq!(c.server_version(), ServerVersion::Exchange2010Sp2); + } + + #[test] + fn anchor_mailbox_round_trips() { + let c = EwsClient::new(Auth::Bearer { token: "t".into() }, options(0)); + c.set_anchor_mailbox(Some("alice@x".to_owned())); + assert_eq!(c.anchor_header().as_deref(), Some("alice@x")); + } +} diff --git a/src/exchange/ews/contact_map.rs b/src/exchange/ews/contact_map.rs new file mode 100644 index 0000000..63c7339 --- /dev/null +++ b/src/exchange/ews/contact_map.rs @@ -0,0 +1,550 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::{Map, Value, json}; + +use crate::exchange::date::WireDate; +use crate::exchange::ews::parse::{ContactItemRaw, RawContactAddress}; + +const SYNTHETIC_UID_PREFIX: &str = "inbuxa-migrate-ews-syn-"; + +const PREFERRED: u32 = 1; + +type Context = &'static str; + +impl ContactItemRaw { + pub fn synthetic_uid(item_id: &str) -> String { + let hash = blake3::hash(item_id.as_bytes()).to_hex(); + format!("{SYNTHETIC_UID_PREFIX}{hash}") + } + + pub fn to_jscontact(&self) -> Value { + let raw = self; + let mut card = Map::new(); + card.insert("@type".to_owned(), Value::String("Card".to_owned())); + card.insert("version".to_owned(), Value::String("1.0".to_owned())); + if raw.is_group { + card.insert("kind".to_owned(), Value::String("group".to_owned())); + let mut members = Map::new(); + for m in &raw.members { + let key = match m.email.as_ref() { + Some(email) => format!("mailto:{email}"), + None => match m.name.as_ref() { + Some(name) => name.clone(), + None => continue, + }, + }; + members.insert(key, Value::Bool(true)); + } + if !members.is_empty() { + card.insert("members".to_owned(), Value::Object(members)); + } + } + if let Some(full) = raw.display_name.as_ref() { + let mut name = Map::new(); + name.insert("@type".to_owned(), Value::String("Name".to_owned())); + name.insert("full".to_owned(), Value::String(full.clone())); + let components = raw.name_components(); + if !components.is_empty() { + name.insert("components".to_owned(), Value::Array(components)); + } + card.insert("name".to_owned(), Value::Object(name)); + } + if let Some(n) = raw.nickname.as_ref() { + card.insert( + "nicknames".to_owned(), + json!({"1": {"@type": "Nickname", "name": n}}), + ); + } + let mut organizations: Map = Map::new(); + let mut have_org = false; + let mut org_inner: Map = Map::new(); + org_inner.insert("@type".to_owned(), Value::String("Organization".to_owned())); + if let Some(co) = raw.company_name.as_ref() { + org_inner.insert("name".to_owned(), Value::String(co.clone())); + have_org = true; + } + let mut org_units: Vec = Vec::new(); + if let Some(dep) = raw.department.as_ref() { + org_units.push(json!({"@type": "OrgUnit", "name": dep})); + } + if let Some(office) = raw.office_location.as_ref() { + org_units.push(json!({"@type": "OrgUnit", "name": office})); + } + if !org_units.is_empty() { + org_inner.insert("units".to_owned(), Value::Array(org_units)); + have_org = true; + } + if have_org { + organizations.insert("1".to_owned(), Value::Object(org_inner)); + } + let mut next_org_idx = 2u32; + for extra in &raw.companies { + if matches!(raw.company_name.as_deref(), Some(c) if c == extra) { + continue; + } + organizations.insert( + next_org_idx.to_string(), + json!({"@type": "Organization", "name": extra}), + ); + next_org_idx += 1; + } + if !organizations.is_empty() { + card.insert("organizations".to_owned(), Value::Object(organizations)); + } + if let Some(t) = raw.job_title.as_ref() { + card.insert( + "titles".to_owned(), + json!({"1": {"@type": "Title", "name": t}}), + ); + } + { + let mut map = Map::new(); + for (i, (key, addr)) in raw + .emails + .iter() + .filter(|(_, v)| !v.trim().is_empty()) + .enumerate() + { + let mut entry = Map::new(); + entry.insert("@type".to_owned(), Value::String("EmailAddress".to_owned())); + entry.insert("address".to_owned(), Value::String(addr.clone())); + entry.insert("contexts".to_owned(), ContactItemRaw::email_contexts(key)); + map.insert((i + 1).to_string(), Value::Object(entry)); + } + if !map.is_empty() { + card.insert("emails".to_owned(), Value::Object(map)); + } + } + { + let mut map = Map::new(); + for (i, (key, num)) in raw + .phones + .iter() + .filter(|(_, v)| !v.trim().is_empty()) + .enumerate() + { + let mut entry = Map::new(); + entry.insert("@type".to_owned(), Value::String("Phone".to_owned())); + entry.insert("number".to_owned(), Value::String(num.clone())); + entry.insert("contexts".to_owned(), ContactItemRaw::phone_contexts(key)); + entry.insert("features".to_owned(), ContactItemRaw::phone_features(key)); + map.insert((i + 1).to_string(), Value::Object(entry)); + } + if !map.is_empty() { + card.insert("phones".to_owned(), Value::Object(map)); + } + } + { + let mut map = Map::new(); + for (i, (key, addr)) in raw + .ims + .iter() + .filter(|(_, v)| !v.trim().is_empty()) + .enumerate() + { + let entry_id = (i + 1).to_string(); + let mut entry = Map::new(); + entry.insert( + "@type".to_owned(), + Value::String("OnlineService".to_owned()), + ); + entry.insert("user".to_owned(), Value::String(addr.clone())); + if let Some(service) = ContactItemRaw::im_service_from_key(key) { + entry.insert("service".to_owned(), Value::String(service)); + } + map.insert(entry_id, Value::Object(entry)); + } + if !map.is_empty() { + card.insert("onlineServices".to_owned(), Value::Object(map)); + } + } + if !raw.addresses.is_empty() { + let pref_key = raw + .postal_address_index + .as_deref() + .filter(|p| !p.is_empty()); + let mut map = Map::new(); + for (i, addr) in raw.addresses.iter().enumerate() { + let entry_id = (i + 1).to_string(); + let pref = pref_key + .is_some_and(|p| p.eq_ignore_ascii_case(&addr.key)) + .then_some(PREFERRED); + map.insert(entry_id, ContactItemRaw::address_to_jscontact(addr, pref)); + } + card.insert("addresses".to_owned(), Value::Object(map)); + } + if let Some(url) = raw.url.as_ref() { + card.insert( + "links".to_owned(), + json!({"1": {"@type": "Link", "uri": url}}), + ); + } + if let Some(bday) = raw.birthday.as_ref() + && let Some(partial) = WireDate(bday).partial_date() + { + card.insert( + "anniversaries".to_owned(), + json!({"birth": {"@type": "Anniversary", "kind": "birth", "date": partial}}), + ); + } + if let Some(anniv) = raw.wedding_anniversary.as_ref() + && let Some(partial) = WireDate(anniv).partial_date() + { + let key = "wedding"; + let entry = json!({"@type": "Anniversary", "kind": "wedding", "date": partial}); + if let Some(map) = card.get_mut("anniversaries").and_then(Value::as_object_mut) { + map.insert(key.to_owned(), entry); + } else { + card.insert("anniversaries".to_owned(), json!({ key: entry })); + } + } + if !raw.categories.is_empty() { + let mut keywords = Map::new(); + for cat in &raw.categories { + keywords.insert(cat.to_ascii_lowercase(), Value::Bool(true)); + } + card.insert("keywords".to_owned(), Value::Object(keywords)); + } + if let Some(notes) = raw.notes.as_ref().filter(|n| !n.trim().is_empty()) { + card.insert( + "notes".to_owned(), + json!({"1": {"@type": "Note", "note": notes}}), + ); + } + Value::Object(card) + } + + fn name_components(&self) -> Vec { + let raw = self; + let mut out: Vec = Vec::new(); + if let Some(v) = raw.given_name.as_ref() { + out.push(json!({"@type": "NameComponent", "kind": "given", "value": v})); + } + if let Some(v) = raw.middle_name.as_ref() { + out.push(json!({"@type": "NameComponent", "kind": "given2", "value": v})); + } + if let Some(v) = raw.surname.as_ref() { + out.push(json!({"@type": "NameComponent", "kind": "surname", "value": v})); + } + if let Some(v) = raw.generation.as_ref() { + out.push(json!({"@type": "NameComponent", "kind": "generation", "value": v})); + } + out + } + + fn email_contexts(key: &str) -> Value { + let mut o = Map::new(); + let lower = key.to_ascii_lowercase(); + if lower.contains("business") || lower.contains("work") { + o.insert("work".to_owned(), Value::Bool(true)); + } else if lower.contains("home") { + o.insert("private".to_owned(), Value::Bool(true)); + } + Value::Object(o) + } + + fn phone_contexts(key: &str) -> Value { + let mut o = Map::new(); + let lower = key.to_ascii_lowercase(); + if lower.starts_with("business") || lower.starts_with("company") { + o.insert("work".to_owned(), Value::Bool(true)); + } else if lower.starts_with("home") || lower.starts_with("mobile") { + o.insert("private".to_owned(), Value::Bool(true)); + } + Value::Object(o) + } + + fn phone_features(key: &str) -> Value { + let mut o = Map::new(); + let lower = key.to_ascii_lowercase(); + if lower.contains("mobile") { + o.insert("mobile".to_owned(), Value::Bool(true)); + } else if lower.contains("fax") { + o.insert("fax".to_owned(), Value::Bool(true)); + } else if lower.contains("pager") { + o.insert("pager".to_owned(), Value::Bool(true)); + } else { + o.insert("voice".to_owned(), Value::Bool(true)); + } + Value::Object(o) + } + + fn address_to_jscontact(addr: &RawContactAddress, pref: Option) -> Value { + let mut o = Map::new(); + o.insert("@type".to_owned(), Value::String("Address".to_owned())); + let mut components: Vec = Vec::new(); + if let Some(s) = addr.street.as_ref() { + components.push(json!({"@type": "AddressComponent", "kind": "name", "value": s})); + } + if let Some(s) = addr.city.as_ref() { + components.push(json!({"@type": "AddressComponent", "kind": "locality", "value": s})); + } + if let Some(s) = addr.state.as_ref() { + components.push(json!({"@type": "AddressComponent", "kind": "region", "value": s})); + } + if let Some(s) = addr.postal_code.as_ref() { + components.push(json!({"@type": "AddressComponent", "kind": "postcode", "value": s})); + } + if let Some(s) = addr.country.as_ref() { + components.push(json!({"@type": "AddressComponent", "kind": "country", "value": s})); + } + if !components.is_empty() { + o.insert("components".to_owned(), Value::Array(components)); + } + let mut contexts = Map::new(); + if let Some(context) = hashify::map_ignore_case!( + addr.key.as_bytes(), + Context, + "business" => "work", + "work" => "work", + "home" => "private", + ) { + contexts.insert((*context).to_owned(), Value::Bool(true)); + } + if !contexts.is_empty() { + o.insert("contexts".to_owned(), Value::Object(contexts)); + } + if let Some(pref) = pref { + o.insert("pref".to_owned(), Value::from(pref)); + } + Value::Object(o) + } + + fn im_service_from_key(key: &str) -> Option { + let trimmed: String = key.chars().filter(|c| !c.is_ascii_digit()).collect(); + let canonical = trimmed.trim_start_matches("ImAddress"); + if canonical.is_empty() { + return None; + } + Some(canonical.to_owned()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn basic_contact_round_trip() { + let raw = ContactItemRaw { + display_name: Some("Alice Doe".to_owned()), + given_name: Some("Alice".to_owned()), + surname: Some("Doe".to_owned()), + emails: vec![("EmailAddress1".to_owned(), "alice@x".to_owned())], + phones: vec![("BusinessPhone1".to_owned(), "+1".to_owned())], + ..ContactItemRaw::default() + }; + let v = raw.to_jscontact(); + assert_eq!(v["@type"], "Card"); + assert_eq!(v["name"]["full"], "Alice Doe"); + let comps = v["name"]["components"].as_array().unwrap(); + assert_eq!(comps[0]["kind"], "given"); + assert_eq!(comps[0]["value"], "Alice"); + assert_eq!(v["emails"]["1"]["address"], "alice@x"); + assert_eq!(v["phones"]["1"]["contexts"]["work"], true); + } + + #[test] + fn empty_body_and_empty_im_entry_are_omitted() { + let raw = ContactItemRaw { + display_name: Some("Random Person".to_owned()), + notes: Some(String::new()), + ims: vec![("ImAddress1".to_owned(), String::new())], + emails: vec![("EmailAddress1".to_owned(), " ".to_owned())], + phones: vec![("MobilePhone".to_owned(), "+17801234567".to_owned())], + ..ContactItemRaw::default() + }; + let v = raw.to_jscontact(); + assert!(v.get("notes").is_none(), "empty body must not emit notes"); + assert!( + v.get("onlineServices").is_none(), + "empty IM entry must not emit onlineServices" + ); + assert!( + v.get("emails").is_none(), + "whitespace-only email must not emit emails" + ); + assert_eq!(v["phones"]["1"]["number"], "+17801234567"); + } + + #[test] + fn synthetic_uid_is_stable() { + let u1 = ContactItemRaw::synthetic_uid("ITEM-1"); + let u2 = ContactItemRaw::synthetic_uid("ITEM-1"); + assert_eq!(u1, u2); + assert_ne!(u1, ContactItemRaw::synthetic_uid("ITEM-2")); + assert!(u1.starts_with("inbuxa-migrate-ews-syn-")); + } + + #[test] + fn anniversaries_combine_birth_and_wedding() { + let raw = ContactItemRaw { + display_name: Some("X".to_owned()), + birthday: Some("1990-04-15T00:00:00Z".to_owned()), + wedding_anniversary: Some("2015-08-22T00:00:00".to_owned()), + ..ContactItemRaw::default() + }; + let v = raw.to_jscontact(); + assert_eq!(v["anniversaries"]["birth"]["kind"], "birth"); + assert_eq!(v["anniversaries"]["birth"]["date"]["month"], 4); + assert_eq!(v["anniversaries"]["wedding"]["kind"], "wedding"); + } + + #[test] + fn birthday_stored_as_local_midnight_in_utc_keeps_its_day() { + let raw = ContactItemRaw { + display_name: Some("X".to_owned()), + birthday: Some("1990-04-14T22:00:00Z".to_owned()), + ..ContactItemRaw::default() + }; + let v = raw.to_jscontact(); + assert_eq!( + v["anniversaries"]["birth"]["date"], + json!({"@type": "PartialDate", "year": 1990, "month": 4, "day": 15}) + ); + } + + #[test] + fn year_zero_birthday_has_no_year() { + let raw = ContactItemRaw { + display_name: Some("X".to_owned()), + birthday: Some("0000-04-15T00:00:00Z".to_owned()), + ..ContactItemRaw::default() + }; + let v = raw.to_jscontact(); + assert_eq!( + v["anniversaries"]["birth"]["date"], + json!({"@type": "PartialDate", "month": 4, "day": 15}) + ); + } + + #[test] + fn outlook_no_date_anniversary_is_dropped() { + let raw = ContactItemRaw { + display_name: Some("X".to_owned()), + birthday: Some("1990-04-15T00:00:00Z".to_owned()), + wedding_anniversary: Some("4501-01-01T00:00:00Z".to_owned()), + ..ContactItemRaw::default() + }; + let v = raw.to_jscontact(); + assert!(v["anniversaries"].get("wedding").is_none()); + assert_eq!(v["anniversaries"]["birth"]["date"]["day"], 15); + } + + #[test] + fn office_location_is_added_as_org_unit() { + let raw = ContactItemRaw { + display_name: Some("X".to_owned()), + company_name: Some("Initech".to_owned()), + department: Some("R&D".to_owned()), + office_location: Some("Bldg 4".to_owned()), + ..ContactItemRaw::default() + }; + let v = raw.to_jscontact(); + let units = v["organizations"]["1"]["units"].as_array().unwrap(); + assert_eq!(units[0]["name"], "R&D"); + assert_eq!(units[1]["name"], "Bldg 4"); + } + + #[test] + fn extra_companies_become_additional_organizations() { + let raw = ContactItemRaw { + display_name: Some("X".to_owned()), + company_name: Some("Initech".to_owned()), + companies: vec!["Initech".to_owned(), "Initrode".to_owned()], + ..ContactItemRaw::default() + }; + let v = raw.to_jscontact(); + assert_eq!(v["organizations"]["1"]["name"], "Initech"); + assert_eq!(v["organizations"]["2"]["name"], "Initrode"); + } + + #[test] + fn postal_address_index_sets_pref_on_matching_entry() { + let raw = ContactItemRaw { + display_name: Some("X".to_owned()), + addresses: vec![ + RawContactAddress { + key: "Home".to_owned(), + street: Some("1 St".to_owned()), + ..Default::default() + }, + RawContactAddress { + key: "Business".to_owned(), + street: Some("2 Av".to_owned()), + ..Default::default() + }, + ], + postal_address_index: Some("Business".to_owned()), + ..ContactItemRaw::default() + }; + let v = raw.to_jscontact(); + assert!(v["addresses"]["1"].get("pref").is_none()); + assert_eq!(v["addresses"]["2"]["pref"], 1); + } + + #[test] + fn im_service_extracted_from_key() { + let raw = ContactItemRaw { + display_name: Some("X".to_owned()), + ims: vec![("ImAddress1".to_owned(), "alice@msn".to_owned())], + ..ContactItemRaw::default() + }; + let v = raw.to_jscontact(); + assert_eq!(v["onlineServices"]["1"]["user"], "alice@msn"); + assert!(v["onlineServices"]["1"].get("service").is_none()); + } + + #[test] + fn distribution_list_becomes_group_card_with_members() { + let raw = ContactItemRaw { + display_name: Some("The Team".to_owned()), + is_group: true, + members: vec![ + crate::exchange::ews::parse::RawGroupMember { + name: Some("Bob".to_owned()), + email: Some("bob@x".to_owned()), + }, + crate::exchange::ews::parse::RawGroupMember { + name: Some("No Email".to_owned()), + email: None, + }, + ], + ..ContactItemRaw::default() + }; + let v = raw.to_jscontact(); + assert_eq!(v["kind"], "group"); + assert_eq!(v["name"]["full"], "The Team"); + assert_eq!(v["members"]["mailto:bob@x"], true); + assert_eq!(v["members"]["No Email"], true); + } + + #[test] + fn individual_contact_has_no_kind_or_members() { + let raw = ContactItemRaw { + display_name: Some("Alice".to_owned()), + ..ContactItemRaw::default() + }; + let v = raw.to_jscontact(); + assert!(v.get("kind").is_none()); + assert!(v.get("members").is_none()); + } + + #[test] + fn unresolvable_relations_are_dropped() { + let raw = ContactItemRaw { + display_name: Some("X".to_owned()), + manager: Some("Boss".to_owned()), + assistant: Some("Asst".to_owned()), + spouse: Some("Partner".to_owned()), + children: vec!["Kid".to_owned()], + ..ContactItemRaw::default() + }; + let v = raw.to_jscontact(); + assert!(v.get("relatedTo").is_none()); + } +} diff --git a/src/exchange_ews/error.rs b/src/exchange/ews/error.rs similarity index 78% rename from src/exchange_ews/error.rs rename to src/exchange/ews/error.rs index 334617e..0e49d7c 100644 --- a/src/exchange_ews/error.rs +++ b/src/exchange/ews/error.rs @@ -5,8 +5,10 @@ */ use crate::error::Error; -use crate::exchange_ews::types::ResponseCode; +use crate::exchange::ews::types::ResponseCode; +use crate::exchange::oauth::OAuthError; use crate::jmap::error::JmapError; +use crate::net::http::TransportError; #[derive(Debug, thiserror::Error)] pub enum EwsError { @@ -47,12 +49,37 @@ pub enum EwsError { Xml(String), } +impl EwsError { + pub fn is_per_batch(&self) -> bool { + matches!( + self, + EwsError::SoapFault { .. } | EwsError::HttpStatus { .. } | EwsError::Malformed(_) + ) + } +} + +impl From for EwsError { + fn from(value: OAuthError) -> Self { + EwsError::OAuth(value.0) + } +} + impl From for EwsError { fn from(value: quick_xml::Error) -> Self { EwsError::Xml(value.to_string()) } } +impl From for EwsError { + fn from(value: TransportError) -> Self { + if value.is_connect() { + EwsError::Connect(value.to_string()) + } else { + EwsError::Transport(value.to_string()) + } + } +} + impl From for EwsError { fn from(value: JmapError) -> Self { match value { diff --git a/src/exchange_ews/mod.rs b/src/exchange/ews/mod.rs similarity index 94% rename from src/exchange_ews/mod.rs rename to src/exchange/ews/mod.rs index df972b6..58552b3 100644 --- a/src/exchange_ews/mod.rs +++ b/src/exchange/ews/mod.rs @@ -14,11 +14,10 @@ pub mod parse; pub mod recurrence; pub mod retry; pub mod soap; +pub mod srv; pub mod types; pub mod xml; -pub use crate::exchange::tz; - pub use client::EwsClient; pub use error::EwsError; pub use types::{ diff --git a/src/exchange/ews/oauth.rs b/src/exchange/ews/oauth.rs new file mode 100644 index 0000000..43c363d --- /dev/null +++ b/src/exchange/ews/oauth.rs @@ -0,0 +1,114 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::time::Duration; + +use crate::exchange::ews::error::EwsError; +use crate::exchange::oauth::{Authority, Dialect, OAuthClient, OAuthError, RefreshSchedule}; +use crate::net::http::ClientOptions; + +pub use crate::exchange::oauth::AcquiredToken; + +pub const SCOPE_APP_ONLY: &str = "https://outlook.office365.com/.default"; +pub const SCOPE_DELEGATED: &str = + "https://outlook.office365.com/EWS.AccessAsUser.All offline_access"; +pub const REFRESH_SCHEDULE: RefreshSchedule = RefreshSchedule { + lead: Duration::from_secs(5 * 60), + fallback_lifetime: Duration::from_secs(50 * 60), +}; +const REFRESH_ATTEMPTS: u32 = 1; + +#[derive(Debug, Clone)] +pub enum OAuthFlow { + PreAcquired { + token: String, + }, + ClientCredentials { + tenant: String, + client_id: String, + client_secret: String, + }, + DeviceCode { + tenant: String, + client_id: String, + }, +} + +impl OAuthFlow { + pub fn acquire(&self, options: ClientOptions) -> Result { + match self { + OAuthFlow::PreAcquired { token } => Ok(AcquiredToken::from_bearer(token.clone())), + OAuthFlow::ClientCredentials { + tenant, + client_id, + client_secret, + } => { + let authority = Authority::for_tenant(tenant); + Ok( + OAuthClient::new(options, &authority, client_id, SCOPE_APP_ONLY, Dialect::Ews) + .client_credentials(client_secret)?, + ) + } + OAuthFlow::DeviceCode { tenant, client_id } => { + let authority = Authority::for_tenant(tenant); + Ok(OAuthClient::new( + options, + &authority, + client_id, + SCOPE_DELEGATED, + Dialect::Ews, + ) + .device_code()?) + } + } + } + + pub fn renew( + &self, + options: ClientOptions, + refresh_token: Option<&str>, + ) -> Result { + match (refresh_token, self) { + (Some(refresh_token), OAuthFlow::DeviceCode { tenant, client_id }) => { + let authority = Authority::for_tenant(tenant); + OAuthClient::new( + options, + &authority, + client_id, + SCOPE_DELEGATED, + Dialect::Ews, + ) + .refresh(refresh_token, REFRESH_ATTEMPTS) + .map_err(|e| OAuthError(EwsError::from(e).to_string())) + } + _ => self.acquire(options).map_err(|e| OAuthError(e.to_string())), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::logging::{Level, Logger}; + use crate::net::http::TlsPolicy; + use crate::net::retry::RetryPolicy; + + #[test] + fn pre_acquired_flow_keeps_the_token() { + let options = ClientOptions { + retry: RetryPolicy::new(0), + tls: TlsPolicy::Verify, + logger: Logger::new(Level::Quiet), + }; + let acq = OAuthFlow::PreAcquired { + token: "a.e30.c".to_owned(), + } + .acquire(options) + .unwrap(); + assert_eq!(acq.access_token, "a.e30.c"); + assert_eq!(acq.upn, None); + } +} diff --git a/src/exchange/ews/parse/attachment.rs b/src/exchange/ews/parse/attachment.rs new file mode 100644 index 0000000..7a780d2 --- /dev/null +++ b/src/exchange/ews/parse/attachment.rs @@ -0,0 +1,246 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::BufRead; + +use quick_xml::NsReader; +use quick_xml::events::Event; + +use crate::exchange::ews::error::EwsError; +use crate::exchange::ews::parse::{Attrs, Ns, TextBuf, Trim, XmlBool}; +use crate::net::xml::Entity; + +#[derive(Debug, Clone, Default)] +pub struct RawAttachmentRef { + pub attachment_id: String, + pub name: Option, + pub content_type: Option, + pub is_contact_photo: bool, + pub is_item_attachment: bool, +} + +#[derive(Debug, Clone, Default)] +pub struct GetAttachmentInline { + pub attachment_id: String, + pub name: Option, + pub content_type: Option, + pub is_contact_photo: bool, + pub content_base64: String, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum AttachmentElement { + File, + Item, +} + +#[derive(Clone, Copy)] +enum AttachmentField { + Attachment, + AttachmentId, + Name, + ContentType, + IsContactPhoto, + Content, +} + +#[derive(Clone, Copy)] +enum AttachmentText { + Name, + ContentType, + IsContactPhoto, + Content, +} + +impl AttachmentField { + fn parse(local: &str) -> Option { + hashify::map_ignore_case!( + local.as_bytes(), + AttachmentField, + "FileAttachment" => AttachmentField::Attachment, + "ItemAttachment" => AttachmentField::Attachment, + "AttachmentId" => AttachmentField::AttachmentId, + "Name" => AttachmentField::Name, + "ContentType" => AttachmentField::ContentType, + "IsContactPhoto" => AttachmentField::IsContactPhoto, + "Content" => AttachmentField::Content, + ) + .copied() + } +} + +impl RawAttachmentRef { + pub(super) fn read( + xml: &mut NsReader, + element: AttachmentElement, + ) -> Result { + let mut att = RawAttachmentRef { + is_item_attachment: element == AttachmentElement::Item, + ..RawAttachmentRef::default() + }; + let mut buf = Vec::new(); + let mut depth: u32 = 1; + let mut current: Option = None; + let mut cur = TextBuf::default(); + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + let is_empty = matches!(ev, Event::Empty(_)); + if !is_empty { + depth += 1; + } + if Ns::of(&ns) == Ns::Types { + cur.0.clear(); + match AttachmentField::parse(e.local_name().into_inner()) { + Some(AttachmentField::AttachmentId) => { + if let Some(v) = Attrs(e).value("Id") { + att.attachment_id = v; + } + } + Some(AttachmentField::Name) => current = Some(AttachmentText::Name), + Some(AttachmentField::ContentType) => { + current = Some(AttachmentText::ContentType); + } + Some(AttachmentField::IsContactPhoto) => { + current = Some(AttachmentText::IsContactPhoto); + } + _ => current = None, + } + } + if is_empty { + current = None; + } + } + Event::End(_) => { + if let Some(field) = current.take() { + let text = cur.take(); + match field { + AttachmentText::Name => att.name = Some(text), + AttachmentText::ContentType => att.content_type = Some(text), + AttachmentText::IsContactPhoto => { + att.is_contact_photo = XmlBool::parse(&text); + } + AttachmentText::Content => {} + } + } + cur.0.clear(); + if depth == 0 { + break; + } + depth -= 1; + if depth == 0 { + break; + } + } + Event::Text(ref t) => cur.0.push_str(t), + Event::CData(ref c) => cur.0.push_str(c), + Event::GeneralRef(ref g) => cur.push_ref(g), + Event::Eof => break, + _ => {} + } + } + Ok(att) + } +} + +impl GetAttachmentInline { + pub fn parse_all(body: &[u8]) -> Result, EwsError> { + let mut xml = Trim::Text.reader(body); + let mut buf = Vec::new(); + let mut out: Vec = Vec::new(); + let mut current: Option = None; + let mut text_target: Option = None; + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + let is_empty = matches!(ev, Event::Empty(_)); + if Ns::of(&ns) == Ns::Types { + let field = AttachmentField::parse(e.local_name().into_inner()); + if let Some(AttachmentField::Attachment) = field { + current = Some(GetAttachmentInline::default()); + } else if let Some(cur) = current.as_mut() { + text_target = match field { + Some(AttachmentField::AttachmentId) => { + if let Some(v) = Attrs(e).value("Id") { + cur.attachment_id = v; + } + text_target + } + Some(AttachmentField::Name) => Some(AttachmentText::Name), + Some(AttachmentField::ContentType) => { + Some(AttachmentText::ContentType) + } + Some(AttachmentField::IsContactPhoto) => { + Some(AttachmentText::IsContactPhoto) + } + Some(AttachmentField::Content) => Some(AttachmentText::Content), + _ => None, + }; + } + } + if is_empty { + text_target = None; + } + } + Event::End(e) => { + if let Some(AttachmentField::Attachment) = + AttachmentField::parse(e.local_name().into_inner()) + && let Some(att) = current.take() + { + out.push(att); + } + text_target = None; + } + Event::Text(ref t) => GetAttachmentInline::push_text(&mut current, text_target, t), + Event::CData(ref c) => GetAttachmentInline::push_text(&mut current, text_target, c), + Event::GeneralRef(ref g) => { + if let (Some(cur), Some(c)) = (current.as_mut(), Entity::from_ref(g)) { + match text_target { + Some(AttachmentText::Name) => { + cur.name.get_or_insert_with(String::new).push(c); + } + Some(AttachmentText::ContentType) => { + cur.content_type.get_or_insert_with(String::new).push(c); + } + Some(AttachmentText::Content) => cur.content_base64.push(c), + Some(AttachmentText::IsContactPhoto) | None => {} + } + } + } + Event::Eof => break, + _ => {} + } + } + Ok(out) + } + + fn push_text( + current: &mut Option, + target: Option, + text: &str, + ) { + let Some(cur) = current.as_mut() else { + return; + }; + match target { + Some(AttachmentText::Name) => { + cur.name.get_or_insert_with(String::new).push_str(text); + } + Some(AttachmentText::ContentType) => { + cur.content_type + .get_or_insert_with(String::new) + .push_str(text); + } + Some(AttachmentText::IsContactPhoto) => cur.is_contact_photo = XmlBool::parse(text), + Some(AttachmentText::Content) => cur.content_base64.push_str(text), + None => {} + } + } +} diff --git a/src/exchange/ews/parse/calendar.rs b/src/exchange/ews/parse/calendar.rs new file mode 100644 index 0000000..7085656 --- /dev/null +++ b/src/exchange/ews/parse/calendar.rs @@ -0,0 +1,599 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::mem; + +use quick_xml::events::Event; + +use crate::exchange::ews::error::EwsError; +use crate::exchange::ews::parse::attachment::AttachmentElement; +use crate::exchange::ews::parse::recurrence::{ + PendingRecurrence, RecurrenceElement, RecurrenceText, +}; +use crate::exchange::ews::parse::{ + Attrs, ItemDepth, ItemElement, Ns, RawAttachmentRef, RawRecurrence, TextBuf, Trim, XmlBool, +}; +use crate::exchange::ews::types::{CalendarItemType, ItemId}; + +#[derive(Debug, Clone, Default)] +pub struct CalendarItemRaw { + pub element: String, + pub id: ItemId, + pub parent_folder_id: Option, + pub uid: Option, + pub subject: Option, + pub start: Option, + pub end: Option, + pub original_start: Option, + pub is_all_day_event: Option, + pub legacy_free_busy_status: Option, + pub my_response_type: Option, + pub sensitivity: Option, + pub location: Option, + pub calendar_item_type: Option, + pub recurrence_id: Option, + pub start_tz: Option, + pub end_tz: Option, + pub recurrence: Option, + pub modified_occurrences: Vec, + pub deleted_occurrences: Vec, + pub organizer_smtp: Option, + pub organizer_routing_type: Option, + pub organizer_name: Option, + pub required_attendees: Vec, + pub optional_attendees: Vec, + pub resources: Vec, + pub reminder_is_set: Option, + pub reminder_minutes_before_start: Option, + pub is_online_meeting: Option, + pub join_online_meeting_url: Option, + pub net_show_url: Option, + pub meeting_workspace_url: Option, + pub categories: Vec, + pub created: Option, + pub last_modified: Option, + pub body_text: Option, + pub body_html: Option, + pub attachments: Vec, +} + +#[derive(Debug, Clone, Default)] +pub struct RawOccurrence { + pub item_id: ItemId, + pub start: Option, + pub end: Option, + pub original_start: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct RawAttendee { + pub email: Option, + pub routing_type: Option, + pub name: Option, + pub response_type: Option, +} + +#[derive(Clone, Copy)] +enum CalendarField { + ItemId, + ParentFolderId, + Uid, + Subject, + Start, + End, + OriginalStart, + IsAllDayEvent, + LegacyFreeBusyStatus, + MyResponseType, + Sensitivity, + Location, + CalendarItemType, + RecurrenceId, + StartTimeZone, + EndTimeZone, + Recurrence, + ModifiedOccurrences, + DeletedOccurrences, + Occurrence, + DeletedOccurrence, + Organizer, + Attendees(AttendeeKind), + ReminderIsSet, + ReminderMinutesBeforeStart, + IsOnlineMeeting, + JoinOnlineMeetingUrl, + NetShowUrl, + MeetingWorkspaceUrl, + Attendee, + Mailbox, + Name, + EmailAddress, + RoutingType, + ResponseType, + Categories, + String, + DateTimeCreated, + LastModifiedTime, + Body, + Attachment(AttachmentElement), +} + +impl CalendarField { + fn parse(local: &str) -> Option { + hashify::map_ignore_case!( + local.as_bytes(), + CalendarField, + "ItemId" => CalendarField::ItemId, + "ParentFolderId" => CalendarField::ParentFolderId, + "UID" => CalendarField::Uid, + "Subject" => CalendarField::Subject, + "Start" => CalendarField::Start, + "End" => CalendarField::End, + "OriginalStart" => CalendarField::OriginalStart, + "IsAllDayEvent" => CalendarField::IsAllDayEvent, + "LegacyFreeBusyStatus" => CalendarField::LegacyFreeBusyStatus, + "MyResponseType" => CalendarField::MyResponseType, + "Sensitivity" => CalendarField::Sensitivity, + "Location" => CalendarField::Location, + "CalendarItemType" => CalendarField::CalendarItemType, + "RecurrenceId" => CalendarField::RecurrenceId, + "StartTimeZone" => CalendarField::StartTimeZone, + "EndTimeZone" => CalendarField::EndTimeZone, + "Recurrence" => CalendarField::Recurrence, + "ModifiedOccurrences" => CalendarField::ModifiedOccurrences, + "DeletedOccurrences" => CalendarField::DeletedOccurrences, + "Occurrence" => CalendarField::Occurrence, + "DeletedOccurrence" => CalendarField::DeletedOccurrence, + "Organizer" => CalendarField::Organizer, + "RequiredAttendees" => CalendarField::Attendees(AttendeeKind::Required), + "OptionalAttendees" => CalendarField::Attendees(AttendeeKind::Optional), + "Resources" => CalendarField::Attendees(AttendeeKind::Resource), + "ReminderIsSet" => CalendarField::ReminderIsSet, + "ReminderMinutesBeforeStart" => CalendarField::ReminderMinutesBeforeStart, + "IsOnlineMeeting" => CalendarField::IsOnlineMeeting, + "JoinOnlineMeetingUrl" => CalendarField::JoinOnlineMeetingUrl, + "NetShowUrl" => CalendarField::NetShowUrl, + "MeetingWorkspaceUrl" => CalendarField::MeetingWorkspaceUrl, + "Attendee" => CalendarField::Attendee, + "Mailbox" => CalendarField::Mailbox, + "Name" => CalendarField::Name, + "EmailAddress" => CalendarField::EmailAddress, + "RoutingType" => CalendarField::RoutingType, + "ResponseType" => CalendarField::ResponseType, + "Categories" => CalendarField::Categories, + "String" => CalendarField::String, + "DateTimeCreated" => CalendarField::DateTimeCreated, + "LastModifiedTime" => CalendarField::LastModifiedTime, + "Body" => CalendarField::Body, + "FileAttachment" => CalendarField::Attachment(AttachmentElement::File), + "ItemAttachment" => CalendarField::Attachment(AttachmentElement::Item), + ) + .copied() + } +} + +#[derive(Clone, Copy)] +enum AttendeeKind { + Required, + Optional, + Resource, +} + +#[derive(Clone, Copy)] +enum Party { + Organizer, + Attendee, +} + +#[derive(Clone, Copy)] +enum CalendarText { + Uid, + Subject, + Start, + End, + OriginalStart, + OccurrenceStart, + OccurrenceEnd, + OccurrenceOriginalStart, + IsAllDay, + FreeBusy, + MyResponse, + Sensitivity, + Location, + CalendarItemType, + RecurrenceId, + Name(Party), + Email(Party), + Routing(Party), + AttendeeResponse, + ReminderIsSet, + ReminderMinutes, + IsOnlineMeeting, + JoinUrl, + NetShowUrl, + WorkspaceUrl, + Category, + Created, + LastModified, + BodyText, + BodyHtml, +} + +#[derive(Default)] +struct CalendarState { + item: CalendarItemRaw, + text_target: Option, + category_collecting: bool, + attendee_kind: Option, + current_attendee: Option, + in_mailbox: bool, + in_organizer: bool, + occurrence_stack: Vec, + deleted_stack: Vec, + in_modified: bool, + in_deleted: bool, +} + +impl CalendarState { + fn party(&self) -> Party { + if self.in_organizer { + Party::Organizer + } else { + Party::Attendee + } + } + + fn in_occurrences(&self) -> bool { + self.in_modified || self.in_deleted + } + + fn close_text(&mut self, target: CalendarText, text: String) { + let item = &mut self.item; + match target { + CalendarText::Uid => item.uid = Some(text), + CalendarText::Subject => item.subject = Some(text), + CalendarText::Start => item.start = Some(text), + CalendarText::End => item.end = Some(text), + CalendarText::OriginalStart => item.original_start = Some(text), + CalendarText::IsAllDay => item.is_all_day_event = Some(XmlBool::parse(&text)), + CalendarText::FreeBusy => item.legacy_free_busy_status = Some(text), + CalendarText::MyResponse => item.my_response_type = Some(text.trim().to_owned()), + CalendarText::Sensitivity => item.sensitivity = Some(text.trim().to_owned()), + CalendarText::Location => item.location = Some(text), + CalendarText::CalendarItemType => { + item.calendar_item_type = CalendarItemType::parse(text.trim()); + } + CalendarText::RecurrenceId => item.recurrence_id = Some(text), + CalendarText::Name(Party::Organizer) => item.organizer_name = Some(text), + CalendarText::Email(Party::Organizer) => item.organizer_smtp = Some(text), + CalendarText::Routing(Party::Organizer) => item.organizer_routing_type = Some(text), + CalendarText::Name(Party::Attendee) => { + if let Some(att) = self.current_attendee.as_mut() { + att.name = Some(text); + } + } + CalendarText::Email(Party::Attendee) => { + if let Some(att) = self.current_attendee.as_mut() { + att.email = Some(text); + } + } + CalendarText::Routing(Party::Attendee) => { + if let Some(att) = self.current_attendee.as_mut() { + att.routing_type = Some(text); + } + } + CalendarText::AttendeeResponse => { + if let Some(att) = self.current_attendee.as_mut() { + att.response_type = Some(text); + } + } + CalendarText::ReminderIsSet => item.reminder_is_set = Some(XmlBool::parse(&text)), + CalendarText::ReminderMinutes => { + item.reminder_minutes_before_start = text.trim().parse().ok(); + } + CalendarText::IsOnlineMeeting => item.is_online_meeting = Some(XmlBool::parse(&text)), + CalendarText::JoinUrl => item.join_online_meeting_url = Some(text), + CalendarText::NetShowUrl => item.net_show_url = Some(text), + CalendarText::WorkspaceUrl => item.meeting_workspace_url = Some(text), + CalendarText::Category => item.categories.push(text), + CalendarText::Created => item.created = Some(text), + CalendarText::LastModified => item.last_modified = Some(text), + CalendarText::BodyText => item.body_text = Some(text), + CalendarText::BodyHtml => item.body_html = Some(text), + CalendarText::OccurrenceStart => { + match ( + self.occurrence_stack.last_mut(), + self.deleted_stack.last_mut(), + ) { + (Some(occ), Some(deleted)) => { + occ.start = Some(text.clone()); + deleted.start = Some(text); + } + (Some(occ), None) => occ.start = Some(text), + (None, Some(deleted)) => deleted.start = Some(text), + (None, None) => {} + } + } + CalendarText::OccurrenceEnd => { + if let Some(occ) = self.occurrence_stack.last_mut() { + occ.end = Some(text); + } + } + CalendarText::OccurrenceOriginalStart => { + if let Some(occ) = self.occurrence_stack.last_mut() { + occ.original_start = Some(text); + } + } + } + } + + fn close_element(&mut self, field: CalendarField) { + match field { + CalendarField::ModifiedOccurrences => self.in_modified = false, + CalendarField::DeletedOccurrences => self.in_deleted = false, + CalendarField::Occurrence => { + if let Some(occ) = self.occurrence_stack.pop() { + self.item.modified_occurrences.push(occ); + } + } + CalendarField::DeletedOccurrence => { + if let Some(occ) = self.deleted_stack.pop() { + self.item.deleted_occurrences.push(occ); + } + } + CalendarField::Attendee => { + if let Some(att) = self.current_attendee.take() { + match self.attendee_kind { + Some(AttendeeKind::Required) => self.item.required_attendees.push(att), + Some(AttendeeKind::Optional) => self.item.optional_attendees.push(att), + Some(AttendeeKind::Resource) => self.item.resources.push(att), + None => {} + } + } + } + CalendarField::Attendees(_) => self.attendee_kind = None, + CalendarField::Organizer => self.in_organizer = false, + CalendarField::Mailbox => self.in_mailbox = false, + CalendarField::Categories => self.category_collecting = false, + _ => {} + } + } +} + +impl CalendarItemRaw { + pub fn parse(inner_xml: &str) -> Result { + let mut xml = Trim::Keep.reader(inner_xml.as_bytes()); + let mut buf = Vec::new(); + let mut st = CalendarState::default(); + let mut recurrence_depth: usize = 0; + let mut recurrence_text: Option = None; + let mut pending = PendingRecurrence::default(); + let mut depth = ItemDepth::default(); + let mut cur = TextBuf::default(); + + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + let local = e.local_name().into_inner(); + let is_empty = matches!(ev, Event::Empty(_)); + let level = depth.open(is_empty); + if st.item.element.is_empty() && ItemElement::is(local) { + local.clone_into(&mut st.item.element); + depth.enter_item(level); + continue; + } + if Ns::of(&ns) != Ns::Types { + continue; + } + if recurrence_depth > 0 { + match RecurrenceElement::parse(local) { + Some(RecurrenceElement::Text(target)) => { + recurrence_text = Some(target); + } + Some(RecurrenceElement::Pattern(kind)) => pending.pattern = Some(kind), + Some(RecurrenceElement::Range(kind)) => pending.range = Some(kind), + None => recurrence_text = None, + } + if is_empty { + recurrence_text = None; + } else { + recurrence_depth += 1; + } + continue; + } + cur.0.clear(); + let field = CalendarField::parse(local); + let text = match field { + Some(CalendarField::ItemId) => { + let target = if st.in_modified { + st.occurrence_stack.last_mut().map(|occ| &mut occ.item_id) + } else if st.in_deleted { + st.deleted_stack.last_mut().map(|occ| &mut occ.item_id) + } else if depth.is_item_child(level) { + Some(&mut st.item.id) + } else { + None + }; + if let Some(id) = target { + Attrs(e).ids(&mut id.id, &mut id.change_key); + } + st.text_target + } + Some(CalendarField::ParentFolderId) => { + if let Some(pid) = Attrs(e).parent_id() { + st.item.parent_folder_id = Some(pid); + } + st.text_target + } + Some(CalendarField::Uid) => Some(CalendarText::Uid), + Some(CalendarField::Subject) => Some(CalendarText::Subject), + Some(CalendarField::Start) if st.in_occurrences() => { + Some(CalendarText::OccurrenceStart) + } + Some(CalendarField::Start) => Some(CalendarText::Start), + Some(CalendarField::End) if st.in_occurrences() => { + Some(CalendarText::OccurrenceEnd) + } + Some(CalendarField::End) => Some(CalendarText::End), + Some(CalendarField::OriginalStart) if st.in_occurrences() => { + Some(CalendarText::OccurrenceOriginalStart) + } + Some(CalendarField::OriginalStart) => Some(CalendarText::OriginalStart), + Some(CalendarField::IsAllDayEvent) => Some(CalendarText::IsAllDay), + Some(CalendarField::LegacyFreeBusyStatus) => Some(CalendarText::FreeBusy), + Some(CalendarField::MyResponseType) => Some(CalendarText::MyResponse), + Some(CalendarField::Sensitivity) => Some(CalendarText::Sensitivity), + Some(CalendarField::Location) => Some(CalendarText::Location), + Some(CalendarField::CalendarItemType) => { + Some(CalendarText::CalendarItemType) + } + Some(CalendarField::RecurrenceId) => Some(CalendarText::RecurrenceId), + Some(CalendarField::StartTimeZone) => { + if let Some(v) = Attrs(e).value("Id") { + st.item.start_tz = Some(v); + } + st.text_target + } + Some(CalendarField::EndTimeZone) => { + if let Some(v) = Attrs(e).value("Id") { + st.item.end_tz = Some(v); + } + st.text_target + } + Some(CalendarField::Recurrence) => { + recurrence_depth = 1; + st.text_target + } + Some(CalendarField::ModifiedOccurrences) => { + st.in_modified = true; + st.text_target + } + Some(CalendarField::DeletedOccurrences) => { + st.in_deleted = true; + st.text_target + } + Some(CalendarField::Occurrence) => { + let mut occ = RawOccurrence::default(); + Attrs(e).ids(&mut occ.item_id.id, &mut occ.item_id.change_key); + st.occurrence_stack.push(occ); + st.text_target + } + Some(CalendarField::DeletedOccurrence) => { + st.deleted_stack.push(RawOccurrence::default()); + st.text_target + } + Some(CalendarField::Organizer) => { + st.in_organizer = true; + st.text_target + } + Some(CalendarField::Attendees(kind)) => { + st.attendee_kind = Some(kind); + st.text_target + } + Some(CalendarField::ReminderIsSet) => Some(CalendarText::ReminderIsSet), + Some(CalendarField::ReminderMinutesBeforeStart) => { + Some(CalendarText::ReminderMinutes) + } + Some(CalendarField::IsOnlineMeeting) => Some(CalendarText::IsOnlineMeeting), + Some(CalendarField::JoinOnlineMeetingUrl) => Some(CalendarText::JoinUrl), + Some(CalendarField::NetShowUrl) => Some(CalendarText::NetShowUrl), + Some(CalendarField::MeetingWorkspaceUrl) => { + Some(CalendarText::WorkspaceUrl) + } + Some(CalendarField::Attendee) if st.attendee_kind.is_some() => { + st.current_attendee = Some(RawAttendee::default()); + st.text_target + } + Some(CalendarField::Mailbox) => { + st.in_mailbox = true; + st.text_target + } + Some(CalendarField::Name) if st.in_mailbox => { + Some(CalendarText::Name(st.party())) + } + Some(CalendarField::EmailAddress) if st.in_mailbox => { + Some(CalendarText::Email(st.party())) + } + Some(CalendarField::RoutingType) if st.in_mailbox => { + Some(CalendarText::Routing(st.party())) + } + Some(CalendarField::ResponseType) if st.current_attendee.is_some() => { + Some(CalendarText::AttendeeResponse) + } + Some(CalendarField::Categories) => { + st.category_collecting = true; + st.text_target + } + Some(CalendarField::String) if st.category_collecting => { + Some(CalendarText::Category) + } + Some(CalendarField::DateTimeCreated) => Some(CalendarText::Created), + Some(CalendarField::LastModifiedTime) => Some(CalendarText::LastModified), + Some(CalendarField::Body) => { + let html = Attrs(e) + .value("BodyType") + .is_some_and(|t| t.eq_ignore_ascii_case("HTML")); + Some(if html { + CalendarText::BodyHtml + } else { + CalendarText::BodyText + }) + } + Some(CalendarField::Attachment(element)) => { + let att = RawAttachmentRef::read(&mut xml, element)?; + st.item.attachments.push(att); + st.text_target + } + _ => None, + }; + st.text_target = if is_empty { None } else { text }; + } + Event::End(e) => { + depth.close(); + if recurrence_depth > 0 { + recurrence_depth -= 1; + if recurrence_depth == 0 { + st.item.recurrence = Some(mem::take(&mut pending).finish()); + } + recurrence_text = None; + continue; + } + if let Some(target) = st.text_target.take() { + st.close_text(target, cur.take()); + } + cur.0.clear(); + if let Some(field) = CalendarField::parse(e.local_name().into_inner()) { + st.close_element(field); + } + } + Event::Text(ref t) => { + if recurrence_depth > 0 { + if let Some(target) = recurrence_text { + pending.text(target, t); + } + } else { + cur.0.push_str(t); + } + } + Event::CData(ref c) => { + if recurrence_depth == 0 { + cur.0.push_str(c); + } + } + Event::GeneralRef(ref g) => { + if recurrence_depth == 0 { + cur.push_ref(g); + } + } + Event::Eof => break, + _ => {} + } + } + Ok(st.item) + } +} diff --git a/src/exchange/ews/parse/contact.rs b/src/exchange/ews/parse/contact.rs new file mode 100644 index 0000000..da5de7a --- /dev/null +++ b/src/exchange/ews/parse/contact.rs @@ -0,0 +1,427 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use quick_xml::events::Event; + +use crate::exchange::ews::error::EwsError; +use crate::exchange::ews::parse::attachment::AttachmentElement; +use crate::exchange::ews::parse::{Attrs, ItemDepth, Ns, RawAttachmentRef, TextBuf, Trim}; +use crate::exchange::ews::types::ItemId; + +#[derive(Debug, Clone, Default)] +pub struct ContactItemRaw { + pub id: ItemId, + pub parent_folder_id: Option, + pub display_name: Option, + pub given_name: Option, + pub middle_name: Option, + pub surname: Option, + pub initials: Option, + pub nickname: Option, + pub company_name: Option, + pub department: Option, + pub job_title: Option, + pub generation: Option, + pub office_location: Option, + pub url: Option, + pub birthday: Option, + pub wedding_anniversary: Option, + pub notes: Option, + pub categories: Vec, + pub emails: Vec<(String, String)>, + pub phones: Vec<(String, String)>, + pub addresses: Vec, + pub ims: Vec<(String, String)>, + pub manager: Option, + pub spouse: Option, + pub assistant: Option, + pub children: Vec, + pub companies: Vec, + pub profession: Option, + pub postal_address_index: Option, + pub created: Option, + pub last_modified: Option, + pub attachments: Vec, + pub is_group: bool, + pub members: Vec, +} + +#[derive(Debug, Clone, Default)] +pub struct RawGroupMember { + pub name: Option, + pub email: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct RawContactAddress { + pub key: String, + pub street: Option, + pub city: Option, + pub state: Option, + pub country: Option, + pub postal_code: Option, +} + +#[derive(Clone, Copy)] +enum ContactText { + DisplayName, + GivenName, + MiddleName, + Surname, + Initials, + Nickname, + CompanyName, + Department, + JobTitle, + Generation, + OfficeLocation, + Url, + Birthday, + WeddingAnniversary, + Manager, + Spouse, + Assistant, + Profession, + PostalAddressIndex, + Notes, + Created, + LastModified, + MemberName, + MemberEmail, + Category, + Child, + Company, + EntryValue, +} + +#[derive(Clone, Copy)] +enum AddressText { + Street, + City, + State, + Country, + Postal, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum EntryContainer { + Email, + Phone, + Im, + Address, +} + +#[derive(Clone, Copy)] +enum ContactRoot { + Contact, + DistributionList, +} + +#[derive(Clone, Copy)] +enum ContactField { + Root(ContactRoot), + ItemId, + ParentFolderId, + Text(ContactText), + Members, + Member, + Mailbox, + Name, + EmailAddress, + Categories, + String, + Children, + Companies, + Container(EntryContainer), + Entry, + Address(AddressText), + Attachment(AttachmentElement), +} + +impl ContactField { + fn parse(local: &str) -> Option { + hashify::map_ignore_case!( + local.as_bytes(), + ContactField, + "Contact" => ContactField::Root(ContactRoot::Contact), + "DistributionList" => ContactField::Root(ContactRoot::DistributionList), + "ItemId" => ContactField::ItemId, + "ParentFolderId" => ContactField::ParentFolderId, + "DisplayName" => ContactField::Text(ContactText::DisplayName), + "GivenName" => ContactField::Text(ContactText::GivenName), + "MiddleName" => ContactField::Text(ContactText::MiddleName), + "Surname" => ContactField::Text(ContactText::Surname), + "Initials" => ContactField::Text(ContactText::Initials), + "Nickname" => ContactField::Text(ContactText::Nickname), + "CompanyName" => ContactField::Text(ContactText::CompanyName), + "Department" => ContactField::Text(ContactText::Department), + "JobTitle" => ContactField::Text(ContactText::JobTitle), + "Generation" => ContactField::Text(ContactText::Generation), + "OfficeLocation" => ContactField::Text(ContactText::OfficeLocation), + "BusinessHomePage" => ContactField::Text(ContactText::Url), + "Birthday" => ContactField::Text(ContactText::Birthday), + "WeddingAnniversary" => ContactField::Text(ContactText::WeddingAnniversary), + "Manager" => ContactField::Text(ContactText::Manager), + "SpouseName" => ContactField::Text(ContactText::Spouse), + "AssistantName" => ContactField::Text(ContactText::Assistant), + "Profession" => ContactField::Text(ContactText::Profession), + "PostalAddressIndex" => ContactField::Text(ContactText::PostalAddressIndex), + "Body" => ContactField::Text(ContactText::Notes), + "DateTimeCreated" => ContactField::Text(ContactText::Created), + "LastModifiedTime" => ContactField::Text(ContactText::LastModified), + "Members" => ContactField::Members, + "Member" => ContactField::Member, + "Mailbox" => ContactField::Mailbox, + "Name" => ContactField::Name, + "EmailAddress" => ContactField::EmailAddress, + "Categories" => ContactField::Categories, + "String" => ContactField::String, + "Children" => ContactField::Children, + "Companies" => ContactField::Companies, + "EmailAddresses" => ContactField::Container(EntryContainer::Email), + "PhoneNumbers" => ContactField::Container(EntryContainer::Phone), + "ImAddresses" => ContactField::Container(EntryContainer::Im), + "PhysicalAddresses" => ContactField::Container(EntryContainer::Address), + "Entry" => ContactField::Entry, + "Street" => ContactField::Address(AddressText::Street), + "City" => ContactField::Address(AddressText::City), + "State" => ContactField::Address(AddressText::State), + "CountryOrRegion" => ContactField::Address(AddressText::Country), + "PostalCode" => ContactField::Address(AddressText::Postal), + "FileAttachment" => ContactField::Attachment(AttachmentElement::File), + "ItemAttachment" => ContactField::Attachment(AttachmentElement::Item), + ) + .copied() + } +} + +#[derive(Default)] +struct ContactState { + item: ContactItemRaw, + text_target: Option, + address_text: Option, + category_collecting: bool, + children_collecting: bool, + companies_collecting: bool, + entry_container: Option, + entry_key: Option, + current_address: Option, + in_members: bool, + member_mailbox: bool, + current_member: Option, +} + +impl ContactState { + fn close_text(&mut self, target: ContactText, text: String) { + let item = &mut self.item; + let slot = match target { + ContactText::DisplayName => &mut item.display_name, + ContactText::GivenName => &mut item.given_name, + ContactText::MiddleName => &mut item.middle_name, + ContactText::Surname => &mut item.surname, + ContactText::Initials => &mut item.initials, + ContactText::Nickname => &mut item.nickname, + ContactText::CompanyName => &mut item.company_name, + ContactText::Department => &mut item.department, + ContactText::JobTitle => &mut item.job_title, + ContactText::Generation => &mut item.generation, + ContactText::OfficeLocation => &mut item.office_location, + ContactText::Url => &mut item.url, + ContactText::Birthday => &mut item.birthday, + ContactText::WeddingAnniversary => &mut item.wedding_anniversary, + ContactText::Manager => &mut item.manager, + ContactText::Spouse => &mut item.spouse, + ContactText::Assistant => &mut item.assistant, + ContactText::Profession => &mut item.profession, + ContactText::PostalAddressIndex => &mut item.postal_address_index, + ContactText::Notes => &mut item.notes, + ContactText::Created => &mut item.created, + ContactText::LastModified => &mut item.last_modified, + ContactText::MemberName => match self.current_member.as_mut() { + Some(m) => &mut m.name, + None => return, + }, + ContactText::MemberEmail => match self.current_member.as_mut() { + Some(m) => &mut m.email, + None => return, + }, + ContactText::Category => return item.categories.push(text), + ContactText::Child => return item.children.push(text), + ContactText::Company => return item.companies.push(text), + ContactText::EntryValue => { + let key = self.entry_key.clone().unwrap_or_default(); + let list = match self.entry_container { + Some(EntryContainer::Email) => &mut item.emails, + Some(EntryContainer::Phone) => &mut item.phones, + Some(EntryContainer::Im) => &mut item.ims, + Some(EntryContainer::Address) | None => return, + }; + return list.push((key, text)); + } + }; + *slot = Some(text); + } + + fn close_address(&mut self, target: AddressText, text: String) { + let Some(addr) = self.current_address.as_mut() else { + return; + }; + let slot = match target { + AddressText::Street => &mut addr.street, + AddressText::City => &mut addr.city, + AddressText::State => &mut addr.state, + AddressText::Country => &mut addr.country, + AddressText::Postal => &mut addr.postal_code, + }; + *slot = Some(text); + } + + fn close_element(&mut self, field: ContactField) { + match field { + ContactField::Categories => self.category_collecting = false, + ContactField::Children => self.children_collecting = false, + ContactField::Companies => self.companies_collecting = false, + ContactField::Container(_) => self.entry_container = None, + ContactField::Entry => { + if let Some(addr) = self.current_address.take() { + self.item.addresses.push(addr); + } + self.entry_key = None; + } + ContactField::Members => self.in_members = false, + ContactField::Mailbox => self.member_mailbox = false, + ContactField::Member => { + if let Some(m) = self.current_member.take() + && (m.email.is_some() || m.name.is_some()) + { + self.item.members.push(m); + } + } + _ => {} + } + } +} + +impl ContactItemRaw { + pub fn parse(inner_xml: &str) -> Result { + let mut xml = Trim::Keep.reader(inner_xml.as_bytes()); + let mut buf = Vec::new(); + let mut st = ContactState::default(); + let mut seen_root_contact = false; + let mut depth = ItemDepth::default(); + let mut cur = TextBuf::default(); + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + let field = ContactField::parse(e.local_name().into_inner()); + let is_empty = matches!(ev, Event::Empty(_)); + let level = depth.open(is_empty); + if !seen_root_contact && let Some(ContactField::Root(root)) = field { + seen_root_contact = true; + depth.enter_item(level); + if let ContactRoot::DistributionList = root { + st.item.is_group = true; + } + continue; + } + if Ns::of(&ns) != Ns::Types { + continue; + } + cur.0.clear(); + match field { + Some(ContactField::ItemId) => { + if depth.is_item_child(level) { + Attrs(e).ids(&mut st.item.id.id, &mut st.item.id.change_key); + } + } + Some(ContactField::ParentFolderId) => { + if let Some(pid) = Attrs(e).parent_id() { + st.item.parent_folder_id = Some(pid); + } + } + Some(ContactField::Text(target)) => st.text_target = Some(target), + Some(ContactField::Members) => st.in_members = true, + Some(ContactField::Member) if st.in_members => { + st.current_member = Some(RawGroupMember::default()); + } + Some(ContactField::Mailbox) if st.in_members => st.member_mailbox = true, + Some(ContactField::Name) if st.member_mailbox => { + st.text_target = Some(ContactText::MemberName); + } + Some(ContactField::EmailAddress) if st.member_mailbox => { + st.text_target = Some(ContactText::MemberEmail); + } + Some(ContactField::Categories) => st.category_collecting = true, + Some(ContactField::String) if st.category_collecting => { + st.text_target = Some(ContactText::Category); + } + Some(ContactField::Children) => st.children_collecting = true, + Some(ContactField::String) if st.children_collecting => { + st.text_target = Some(ContactText::Child); + } + Some(ContactField::Companies) => st.companies_collecting = true, + Some(ContactField::String) if st.companies_collecting => { + st.text_target = Some(ContactText::Company); + } + Some(ContactField::Container(container)) => { + st.entry_container = Some(container); + } + Some(ContactField::Entry) => { + st.entry_key = Attrs(e).value("Key"); + if st.entry_container == Some(EntryContainer::Address) { + st.current_address = Some(RawContactAddress { + key: st.entry_key.clone().unwrap_or_default(), + ..RawContactAddress::default() + }); + } else { + st.text_target = Some(ContactText::EntryValue); + } + } + _ if st.current_address.is_some() => { + st.address_text = match field { + Some(ContactField::Address(target)) => Some(target), + _ => None, + }; + } + Some(ContactField::Attachment(element)) => { + let att = RawAttachmentRef::read(&mut xml, element)?; + st.item.attachments.push(att); + } + _ => st.text_target = None, + } + if is_empty { + st.text_target = None; + st.address_text = None; + } + } + Event::End(e) => { + depth.close(); + if let Some(target) = st.address_text { + if st.current_address.is_some() { + st.close_address(target, cur.take()); + } + } else if let Some(target) = st.text_target { + st.close_text(target, cur.take()); + } + cur.0.clear(); + st.text_target = None; + st.address_text = None; + if let Some(field) = ContactField::parse(e.local_name().into_inner()) { + st.close_element(field); + } + } + Event::Text(ref t) => cur.0.push_str(t), + Event::CData(ref c) => cur.0.push_str(c), + Event::GeneralRef(ref g) => cur.push_ref(g), + Event::Eof => break, + _ => {} + } + } + Ok(st.item) + } +} diff --git a/src/exchange/ews/parse/envelope.rs b/src/exchange/ews/parse/envelope.rs new file mode 100644 index 0000000..481a66a --- /dev/null +++ b/src/exchange/ews/parse/envelope.rs @@ -0,0 +1,229 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::BufRead; + +use quick_xml::NsReader; +use quick_xml::events::{BytesStart, Event}; + +use crate::exchange::ews::error::EwsError; +use crate::exchange::ews::parse::{Attrs, Ns, Trim}; +use crate::exchange::ews::types::{ResponseCode, ServerVersion}; +use crate::net::xml::Entity; + +#[derive(Debug, Clone, Default)] +pub struct ServerVersionInfo { + pub major_version: Option, + pub minor_version: Option, + pub major_build: Option, + pub minor_build: Option, + pub version: Option, +} + +#[derive(Clone, Copy)] +enum VersionAttr { + MajorVersion, + MinorVersion, + MajorBuildNumber, + MinorBuildNumber, + Version, +} + +impl ServerVersionInfo { + pub fn to_server_version(&self) -> Option { + match (self.major_version, self.minor_version) { + (Some(maj), Some(min)) => Some(ServerVersion::from_build(maj, min)), + _ => None, + } + } + + fn read_attrs(&mut self, e: &BytesStart<'_>) { + for a in e.attributes().flatten() { + let Some(attr) = hashify::map_ignore_case!( + a.key.local_name().into_inner().as_bytes(), + VersionAttr, + "MajorVersion" => VersionAttr::MajorVersion, + "MinorVersion" => VersionAttr::MinorVersion, + "MajorBuildNumber" => VersionAttr::MajorBuildNumber, + "MinorBuildNumber" => VersionAttr::MinorBuildNumber, + "Version" => VersionAttr::Version, + ) else { + continue; + }; + let value = &*a.value; + match attr { + VersionAttr::MajorVersion => self.major_version = value.parse().ok(), + VersionAttr::MinorVersion => self.minor_version = value.parse().ok(), + VersionAttr::MajorBuildNumber => self.major_build = value.parse().ok(), + VersionAttr::MinorBuildNumber => self.minor_build = value.parse().ok(), + VersionAttr::Version => self.version = Some(value.to_owned()), + } + } + } +} + +#[derive(Debug, Clone)] +pub struct SoapFault { + pub fault_string: String, + pub response_code: ResponseCode, + pub back_off_ms: Option, +} + +#[derive(Debug, Clone)] +pub enum EnvelopeKind { + Fault { + version: ServerVersionInfo, + fault: SoapFault, + }, + Body { + version: ServerVersionInfo, + }, +} + +#[derive(Clone, Copy)] +enum EnvelopeElement { + ServerVersionInfo, + Fault, + Body, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum FaultElement { + FaultString, + ResponseCode, + Value, +} + +impl EnvelopeKind { + pub fn read(bytes: &[u8]) -> Result { + let mut xml = Trim::Text.reader(bytes); + let mut buf = Vec::new(); + let mut version = ServerVersionInfo::default(); + let mut seen_body = false; + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + match hashify::map_ignore_case!( + e.local_name().into_inner().as_bytes(), + EnvelopeElement, + "ServerVersionInfo" => EnvelopeElement::ServerVersionInfo, + "Fault" => EnvelopeElement::Fault, + "Body" => EnvelopeElement::Body, + ) { + Some(EnvelopeElement::ServerVersionInfo) if Ns::of(&ns) == Ns::Types => { + version.read_attrs(e); + } + Some(EnvelopeElement::Fault) => { + let fault = SoapFault::read(&mut xml)?; + return Ok(EnvelopeKind::Fault { version, fault }); + } + Some(EnvelopeElement::Body) => seen_body = true, + _ => {} + } + } + Event::Eof => break, + _ => {} + } + } + if seen_body { + Ok(EnvelopeKind::Body { version }) + } else { + Err(EwsError::Malformed("envelope had no soap:Body".to_owned())) + } + } +} + +impl SoapFault { + fn read(xml: &mut NsReader) -> Result { + let mut buf = Vec::new(); + let mut fault_string = String::new(); + let mut code = ResponseCode::Other(String::new()); + let mut back_off: Option = None; + let mut text_target: Option = None; + let mut last_value_name: Option = None; + let mut depth: u32 = 1; + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + match ev { + Event::Start(e) => { + depth += 1; + text_target = match FaultElement::parse(e.local_name().into_inner()) { + Some(FaultElement::Value) if Ns::of(&ns) == Ns::Types => { + last_value_name = Attrs(&e).value("Name"); + Some(FaultElement::Value) + } + Some(FaultElement::Value) | None => None, + other => other, + }; + } + Event::Empty(e) => { + if Ns::of(&ns) == Ns::Types + && FaultElement::parse(e.local_name().into_inner()) + == Some(FaultElement::Value) + { + last_value_name = Attrs(&e).value("Name"); + } + } + Event::End(_) => { + if depth == 0 { + break; + } + depth -= 1; + text_target = None; + if depth == 0 { + break; + } + } + Event::Text(t) => match text_target { + Some(FaultElement::FaultString) => fault_string.push_str(&t), + Some(FaultElement::ResponseCode) => code = ResponseCode::parse(&t), + Some(FaultElement::Value) + if last_value_name.as_deref() == Some("BackOffMilliseconds") => + { + back_off = t.trim().parse().ok(); + } + _ => {} + }, + Event::CData(c) if text_target == Some(FaultElement::FaultString) => { + fault_string.push_str(&c); + } + Event::GeneralRef(ref g) if text_target == Some(FaultElement::FaultString) => { + if let Some(c) = Entity::from_ref(g) { + fault_string.push(c); + } + } + Event::Eof => break, + _ => {} + } + } + if matches!(code, ResponseCode::ServerBusy { .. }) { + code = ResponseCode::ServerBusy { + back_off_ms: back_off, + }; + } + Ok(SoapFault { + fault_string, + response_code: code, + back_off_ms: back_off, + }) + } +} + +impl FaultElement { + fn parse(local: &str) -> Option { + hashify::map_ignore_case!( + local.as_bytes(), + FaultElement, + "faultstring" => FaultElement::FaultString, + "ResponseCode" => FaultElement::ResponseCode, + "Value" => FaultElement::Value, + ) + .copied() + } +} diff --git a/src/exchange/ews/parse/folder.rs b/src/exchange/ews/parse/folder.rs new file mode 100644 index 0000000..ed3025e --- /dev/null +++ b/src/exchange/ews/parse/folder.rs @@ -0,0 +1,291 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::BufRead; + +use quick_xml::NsReader; +use quick_xml::events::{BytesStart, Event}; + +use crate::exchange::ews::error::EwsError; +use crate::exchange::ews::parse::{Attrs, Ns, TextBuf, Trim, XmlBool}; +use crate::exchange::ews::types::FolderId; + +#[derive(Debug, Clone, Default)] +pub struct FindFolderResponse { + pub folders: Vec, + pub more: bool, + pub total_in_view: Option, +} + +#[derive(Debug, Clone)] +pub struct FolderEntry { + pub element: FolderElement, + pub folder_id: FolderId, + pub parent_id: Option, + pub display_name: String, + pub folder_class: String, + pub total_count: Option, + pub child_count: Option, + pub is_hidden: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FolderElement { + Folder, + CalendarFolder, + ContactsFolder, + TasksFolder, + SearchFolder, +} + +#[derive(Clone, Copy)] +enum FolderField { + FolderId, + ParentFolderId, + DisplayName, + FolderClass, + TotalCount, + ChildFolderCount, + ExtendedFieldUri, + Value, +} + +#[derive(Clone, Copy)] +enum FolderText { + DisplayName, + FolderClass, + TotalCount, + ChildCount, + Hidden, +} + +impl FolderElement { + fn from_local(local: &str) -> Option { + hashify::map_ignore_case!( + local.as_bytes(), + FolderElement, + "Folder" => FolderElement::Folder, + "CalendarFolder" => FolderElement::CalendarFolder, + "ContactsFolder" => FolderElement::ContactsFolder, + "TasksFolder" => FolderElement::TasksFolder, + "SearchFolder" => FolderElement::SearchFolder, + ) + .copied() + } + + fn of_types(ns: Ns, e: &BytesStart<'_>) -> Option { + if ns == Ns::Types { + FolderElement::from_local(e.local_name().into_inner()) + } else { + None + } + } +} + +impl FindFolderResponse { + pub fn parse(body: &[u8]) -> Result { + let mut xml = Trim::Text.reader(body); + let mut buf = Vec::new(); + let mut out = FindFolderResponse::default(); + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + let ns = Ns::of(&ns); + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + if ns == Ns::Messages + && e.local_name() + .into_inner() + .eq_ignore_ascii_case("RootFolder") + { + let attrs = Attrs(e); + if let Some(v) = attrs.value("TotalItemsInView") { + out.total_in_view = v.trim().parse().ok(); + } + if let Some(v) = attrs.value("IncludesLastItemInRange") { + out.more = !XmlBool::parse(&v); + } + } else if let Some(elem) = FolderElement::of_types(ns, e) { + out.folders.push(FolderEntry::read(&mut xml, elem)?); + } + } + Event::Eof => break, + _ => {} + } + } + Ok(out) + } +} + +impl Default for FolderEntry { + fn default() -> Self { + FolderEntry::new(FolderElement::Folder) + } +} + +impl FolderEntry { + fn new(element: FolderElement) -> FolderEntry { + FolderEntry { + element, + folder_id: FolderId { + id: String::new(), + change_key: String::new(), + }, + parent_id: None, + display_name: String::new(), + folder_class: String::new(), + total_count: None, + child_count: None, + is_hidden: false, + } + } + + pub fn parse_inner(inner_xml: &str) -> Result, EwsError> { + let mut xml = Trim::Text.reader(inner_xml.as_bytes()); + let mut buf = Vec::new(); + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + if let Some(elem) = FolderElement::of_types(Ns::of(&ns), e) { + return Ok(Some(FolderEntry::read(&mut xml, elem)?)); + } + } + Event::Eof => return Ok(None), + _ => {} + } + } + } + + pub fn parse_get_response(body: &[u8]) -> Result, EwsError> { + let mut xml = Trim::Text.reader(body); + let mut buf = Vec::new(); + let mut out: Vec = Vec::new(); + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + if let Some(elem) = FolderElement::of_types(Ns::of(&ns), e) { + out.push(FolderEntry::read(&mut xml, elem)?); + } + } + Event::Eof => break, + _ => {} + } + } + Ok(out) + } + + fn read( + xml: &mut NsReader, + element: FolderElement, + ) -> Result { + xml.config_mut().trim_text(false); + let mut entry = FolderEntry::new(element); + let mut buf = Vec::new(); + let mut depth: u32 = 1; + let mut current: Option = None; + let mut in_hidden_ext = false; + let mut cur = TextBuf::default(); + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + let is_empty = matches!(ev, Event::Empty(_)); + if !is_empty { + depth += 1; + } + if Ns::of(&ns) == Ns::Types { + entry.open_field(e, &mut in_hidden_ext, &mut current, &mut cur); + } + if is_empty { + current = None; + } + } + Event::End(_) => { + if let Some(field) = current.take() { + let text = cur.take(); + let text = text.trim(); + match field { + FolderText::DisplayName => entry.display_name = text.to_owned(), + FolderText::FolderClass => entry.folder_class = text.to_owned(), + FolderText::TotalCount => entry.total_count = text.parse().ok(), + FolderText::ChildCount => entry.child_count = text.parse().ok(), + FolderText::Hidden => entry.is_hidden = XmlBool::parse(text), + } + } + if depth == 0 { + break; + } + depth -= 1; + if depth == 0 { + break; + } + } + Event::Text(t) => cur.0.push_str(&t), + Event::CData(c) => cur.0.push_str(&c), + Event::GeneralRef(ref g) => cur.push_ref(g), + Event::Eof => break, + _ => {} + } + } + Ok(entry) + } + + fn open_field( + &mut self, + e: &BytesStart<'_>, + in_hidden_ext: &mut bool, + current: &mut Option, + cur: &mut TextBuf, + ) { + let field = hashify::map_ignore_case!( + e.local_name().into_inner().as_bytes(), + FolderField, + "FolderId" => FolderField::FolderId, + "ParentFolderId" => FolderField::ParentFolderId, + "DisplayName" => FolderField::DisplayName, + "FolderClass" => FolderField::FolderClass, + "TotalCount" => FolderField::TotalCount, + "ChildFolderCount" => FolderField::ChildFolderCount, + "ExtendedFieldURI" => FolderField::ExtendedFieldUri, + "Value" => FolderField::Value, + ) + .copied(); + let text = match field { + Some(FolderField::FolderId) => { + return Attrs(e).ids(&mut self.folder_id.id, &mut self.folder_id.change_key); + } + Some(FolderField::ParentFolderId) => { + if let Some(pid) = Attrs(e).parent_id() { + self.parent_id = Some(pid); + } + return; + } + Some(FolderField::ExtendedFieldUri) => { + *in_hidden_ext = Attrs(e).value("PropertyTag").is_some_and(|tag| { + let tag = tag.trim(); + tag.eq_ignore_ascii_case("0x10f4") || tag == "4340" + }); + return; + } + Some(FolderField::DisplayName) => FolderText::DisplayName, + Some(FolderField::FolderClass) => FolderText::FolderClass, + Some(FolderField::TotalCount) => FolderText::TotalCount, + Some(FolderField::ChildFolderCount) => FolderText::ChildCount, + Some(FolderField::Value) if *in_hidden_ext => FolderText::Hidden, + _ => { + *current = None; + return; + } + }; + *current = Some(text); + cur.0.clear(); + } +} diff --git a/src/exchange/ews/parse/item.rs b/src/exchange/ews/parse/item.rs new file mode 100644 index 0000000..2dd3e60 --- /dev/null +++ b/src/exchange/ews/parse/item.rs @@ -0,0 +1,376 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::BufRead; + +use quick_xml::NsReader; +use quick_xml::events::{BytesEnd, BytesStart, Event}; + +use crate::exchange::ews::error::EwsError; +use crate::exchange::ews::parse::{Attrs, ItemDepth, Ns, Trim, XmlBool}; +use crate::exchange::ews::soap::{NS_MESSAGES, NS_TYPES}; +use crate::exchange::ews::types::{ItemId, ResponseCode}; + +#[derive(Debug, Clone, Default)] +pub struct FindItemResponse { + pub items: Vec, + pub total_in_view: Option, + pub more: bool, +} + +#[derive(Debug, Clone)] +pub struct ItemEntry { + pub element: String, + pub id: ItemId, + pub item_class: Option, +} + +#[derive(Clone, Copy)] +enum FindItemElement { + RootFolder, + Items, + ItemId, + ItemClass, +} + +impl FindItemElement { + fn parse(local: &str) -> Option { + hashify::map_ignore_case!( + local.as_bytes(), + FindItemElement, + "RootFolder" => FindItemElement::RootFolder, + "Items" => FindItemElement::Items, + "ItemId" => FindItemElement::ItemId, + "ItemClass" => FindItemElement::ItemClass, + ) + .copied() + } +} + +impl FindItemResponse { + pub fn parse(body: &[u8]) -> Result { + let mut xml = Trim::Text.reader(body); + let mut buf = Vec::new(); + let mut out = FindItemResponse::default(); + let mut in_root = false; + let mut depth = ItemDepth::default(); + let mut items_level: Option = None; + let mut reading_class = false; + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + let ns = Ns::of(&ns); + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + let is_empty = matches!(ev, Event::Empty(_)); + let level = depth.open(is_empty); + let local = e.local_name().into_inner(); + let element = FindItemElement::parse(local); + if ns == Ns::Messages && matches!(element, Some(FindItemElement::RootFolder)) { + in_root = true; + let attrs = Attrs(e); + if let Some(v) = attrs.value("TotalItemsInView") { + out.total_in_view = v.trim().parse().ok(); + } + if let Some(v) = attrs.value("IncludesLastItemInRange") { + out.more = !XmlBool::parse(&v); + } + } else if in_root && ns == Ns::Types { + if items_level.is_none() && matches!(element, Some(FindItemElement::Items)) + { + items_level = Some(level); + } else if items_level.is_some_and(|items| level == items + 1) { + depth.enter_item(level); + out.items.push(ItemEntry { + element: local.to_owned(), + id: ItemId::default(), + item_class: None, + }); + } else if depth.is_item_child(level) { + match element { + Some(FindItemElement::ItemId) => { + if let Some(last) = out.items.last_mut() { + Attrs(e).ids(&mut last.id.id, &mut last.id.change_key); + } + } + Some(FindItemElement::ItemClass) => reading_class = !is_empty, + _ => {} + } + } + } + } + Event::Text(ref t) if reading_class => { + if let Some(last) = out.items.last_mut() { + last.item_class = Some(t.trim().to_owned()); + } + } + Event::End(e) => { + depth.close(); + reading_class = false; + if matches!( + FindItemElement::parse(e.local_name().into_inner()), + Some(FindItemElement::RootFolder) + ) { + in_root = false; + items_level = None; + } + } + Event::Eof => break, + _ => {} + } + } + Ok(out) + } +} + +#[derive(Debug, Clone)] +pub struct ResponseMessage { + pub success: bool, + pub response_code: ResponseCode, + pub message_text: String, + pub inner_xml: String, +} + +#[derive(Clone, Copy)] +enum MessageElement { + ResponseCode, + MessageText, + Capture, +} + +impl MessageElement { + fn parse(local: &str) -> Option { + hashify::map_ignore_case!( + local.as_bytes(), + MessageElement, + "ResponseCode" => MessageElement::ResponseCode, + "MessageText" => MessageElement::MessageText, + "Items" => MessageElement::Capture, + "Attachments" => MessageElement::Capture, + "Folders" => MessageElement::Capture, + ) + .copied() + } +} + +#[derive(Clone, Copy)] +enum MessageText { + ResponseCode, + MessageText, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Escape { + Attribute, + Text, +} + +#[derive(Default)] +struct InnerXml(String); + +impl InnerXml { + const OPEN: &str = " self.0.push_str("&"), + '<' => self.0.push_str("<"), + '>' => self.0.push_str(">"), + '"' if escape == Escape::Attribute => self.0.push_str("""), + _ => self.0.push(ch), + } + } + } + + fn push_tag(&mut self, e: &BytesStart<'_>) { + self.0.push('<'); + self.0.push_str(e.name().into_inner()); + for a in e.attributes().flatten() { + self.0.push(' '); + self.0.push_str(a.key.into_inner()); + self.0.push_str("=\""); + self.push_escaped(&a.value, Escape::Attribute); + self.0.push('"'); + } + } + + fn start(&mut self, e: &BytesStart<'_>) { + self.push_tag(e); + self.0.push('>'); + } + + fn empty(&mut self, e: &BytesStart<'_>) { + self.push_tag(e); + self.0.push_str("/>"); + } + + fn end(&mut self, e: &BytesEnd<'_>) { + self.0.push_str("'); + } + + fn text(&mut self, text: &str) { + self.push_escaped(text, Escape::Text); + } + + fn reference(&mut self, name: &str) { + self.0.push('&'); + self.0.push_str(name); + self.0.push(';'); + } + + fn finish(self) -> String { + if self.0.is_empty() { + return self.0; + } + let parts = [ + InnerXml::OPEN, + NS_TYPES, + "\" xmlns:t=\"", + NS_TYPES, + "\" xmlns:m=\"", + NS_MESSAGES, + "\">", + &self.0, + InnerXml::CLOSE, + ]; + let mut out = String::with_capacity(parts.iter().map(|p| p.len()).sum()); + parts.iter().for_each(|p| out.push_str(p)); + out + } +} + +impl ResponseMessage { + pub fn parse_all( + body: &[u8], + response_message_local: &str, + ) -> Result, EwsError> { + let mut xml = Trim::Keep.reader(body); + let mut buf = Vec::new(); + let mut out: Vec = Vec::new(); + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + match ev { + Event::Start(e) => { + if Ns::of(&ns) == Ns::Messages + && e.local_name() + .into_inner() + .eq_ignore_ascii_case(response_message_local) + { + let success = Attrs(&e) + .value("ResponseClass") + .is_none_or(|class| matches!(class.as_str(), "Success" | "Warning")); + let mut msg = ResponseMessage::read(&mut xml)?; + msg.success = success; + out.push(msg); + } + } + Event::Eof => break, + _ => {} + } + } + Ok(out) + } + + fn read(xml: &mut NsReader) -> Result { + let mut buf = Vec::new(); + let mut response_code = ResponseCode::NoError; + let mut message_text = String::new(); + let mut inner = InnerXml::default(); + let mut depth: u32 = 1; + let mut current: Option = None; + let mut capture_depth: u32 = 0; + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + let in_capture = capture_depth > 0; + match ev { + Event::Start(e) => { + depth += 1; + if in_capture { + inner.start(&e); + capture_depth += 1; + } else { + let element = if Ns::of(&ns) == Ns::Messages { + MessageElement::parse(e.local_name().into_inner()) + } else { + None + }; + current = match element { + Some(MessageElement::ResponseCode) => Some(MessageText::ResponseCode), + Some(MessageElement::MessageText) => Some(MessageText::MessageText), + Some(MessageElement::Capture) => { + capture_depth = 1; + None + } + None => None, + }; + } + } + Event::Empty(e) => { + if in_capture { + inner.empty(&e); + } + } + Event::End(e) => { + if in_capture { + if capture_depth == 1 { + capture_depth = 0; + } else { + inner.end(&e); + capture_depth -= 1; + } + } + if depth == 0 { + break; + } + depth -= 1; + current = None; + if depth == 0 { + break; + } + } + Event::Text(t) => { + if in_capture { + inner.text(&t); + } else { + match current { + Some(MessageText::ResponseCode) => { + response_code = ResponseCode::parse(t.trim()); + } + Some(MessageText::MessageText) => { + message_text = t.into_inner().into_owned(); + } + None => {} + } + } + } + Event::CData(c) => { + if in_capture { + inner.text(&c); + } + } + Event::GeneralRef(ref g) => { + if in_capture { + inner.reference(g); + } + } + Event::Eof => break, + _ => {} + } + } + Ok(ResponseMessage { + success: true, + response_code, + message_text, + inner_xml: inner.finish(), + }) + } +} diff --git a/src/exchange/ews/parse/message.rs b/src/exchange/ews/parse/message.rs new file mode 100644 index 0000000..b62e310 --- /dev/null +++ b/src/exchange/ews/parse/message.rs @@ -0,0 +1,191 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use quick_xml::events::Event; + +use crate::exchange::ews::error::EwsError; +use crate::exchange::ews::parse::{Attrs, ItemDepth, ItemElement, Ns, TextBuf, Trim, XmlBool}; +use crate::exchange::ews::types::ItemId; + +#[derive(Debug, Clone, Default)] +pub struct MessageItem { + pub element: String, + pub id: ItemId, + pub parent_folder_id: Option, + pub mime_content: Option, + pub mime_charset: Option, + pub subject: Option, + pub date_time_received: Option, + pub is_read: Option, + pub is_draft: Option, + pub categories: Vec, + pub flag_status: Option, +} + +#[derive(Clone, Copy)] +enum MessageField { + ItemId, + ParentFolderId, + MimeContent, + Subject, + DateTimeReceived, + IsRead, + IsDraft, + Categories, + String, + Flag, + FlagStatus, + ExtendedFieldUri, + Value, + ExtendedProperty, +} + +#[derive(Clone, Copy)] +enum MessageText { + MimeContent, + Subject, + Received, + IsRead, + IsDraft, + Category, + FlagStatus, + FlagStatusExt, +} + +impl MessageField { + fn parse(local: &str) -> Option { + hashify::map_ignore_case!( + local.as_bytes(), + MessageField, + "ItemId" => MessageField::ItemId, + "ParentFolderId" => MessageField::ParentFolderId, + "MimeContent" => MessageField::MimeContent, + "Subject" => MessageField::Subject, + "DateTimeReceived" => MessageField::DateTimeReceived, + "IsRead" => MessageField::IsRead, + "IsDraft" => MessageField::IsDraft, + "Categories" => MessageField::Categories, + "String" => MessageField::String, + "Flag" => MessageField::Flag, + "FlagStatus" => MessageField::FlagStatus, + "ExtendedFieldURI" => MessageField::ExtendedFieldUri, + "Value" => MessageField::Value, + "ExtendedProperty" => MessageField::ExtendedProperty, + ) + .copied() + } +} + +impl MessageItem { + pub fn parse(inner_xml: &str) -> Result { + let mut xml = Trim::Keep.reader(inner_xml.as_bytes()); + let mut buf = Vec::new(); + let mut item = MessageItem::default(); + let mut current: Option = None; + let mut mime_charset: Option = None; + let mut category_collecting = false; + let mut in_flag = false; + let mut in_flagstatus_ext = false; + let mut depth = ItemDepth::default(); + let mut cur = TextBuf::default(); + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + let local = e.local_name().into_inner(); + let is_empty = matches!(ev, Event::Empty(_)); + let level = depth.open(is_empty); + if item.element.is_empty() && ItemElement::is(local) { + local.clone_into(&mut item.element); + depth.enter_item(level); + } else if Ns::of(&ns) == Ns::Types { + cur.0.clear(); + match MessageField::parse(local) { + Some(MessageField::ItemId) => { + if depth.is_item_child(level) { + Attrs(e).ids(&mut item.id.id, &mut item.id.change_key); + } + } + Some(MessageField::ParentFolderId) => { + if let Some(pid) = Attrs(e).parent_id() { + item.parent_folder_id = Some(pid); + } + } + Some(MessageField::MimeContent) => { + mime_charset = Attrs(e).value("CharacterSet"); + current = Some(MessageText::MimeContent); + } + Some(MessageField::Subject) => current = Some(MessageText::Subject), + Some(MessageField::DateTimeReceived) => { + current = Some(MessageText::Received); + } + Some(MessageField::IsRead) => current = Some(MessageText::IsRead), + Some(MessageField::IsDraft) => current = Some(MessageText::IsDraft), + Some(MessageField::Categories) => category_collecting = true, + Some(MessageField::String) if category_collecting => { + current = Some(MessageText::Category); + } + Some(MessageField::Flag) => in_flag = true, + Some(MessageField::FlagStatus) if in_flag => { + current = Some(MessageText::FlagStatus); + } + Some(MessageField::ExtendedFieldUri) => { + if let Some(tag) = Attrs(e).value("PropertyTag") { + let tag = tag.trim(); + in_flagstatus_ext = + tag.eq_ignore_ascii_case("0x1090") || tag == "4240"; + } + } + Some(MessageField::Value) if in_flagstatus_ext => { + current = Some(MessageText::FlagStatusExt); + } + _ => current = None, + } + } + if is_empty { + current = None; + } + } + Event::End(e) => { + depth.close(); + if let Some(field) = current.take() { + let text = cur.take(); + match field { + MessageText::MimeContent => { + item.mime_content = Some(text); + item.mime_charset = mime_charset.take(); + } + MessageText::Subject => item.subject = Some(text), + MessageText::Received => item.date_time_received = Some(text), + MessageText::IsRead => item.is_read = Some(XmlBool::parse(&text)), + MessageText::IsDraft => item.is_draft = Some(XmlBool::parse(&text)), + MessageText::Category => item.categories.push(text), + MessageText::FlagStatus => item.flag_status = Some(text), + MessageText::FlagStatusExt if text.trim() == "2" => { + item.flag_status = Some("Flagged".to_owned()); + } + MessageText::FlagStatusExt => {} + } + } + cur.0.clear(); + match MessageField::parse(e.local_name().into_inner()) { + Some(MessageField::Categories) => category_collecting = false, + Some(MessageField::Flag) => in_flag = false, + Some(MessageField::ExtendedProperty) => in_flagstatus_ext = false, + _ => {} + } + } + Event::Text(ref t) => cur.0.push_str(t), + Event::CData(ref c) => cur.0.push_str(c), + Event::GeneralRef(ref g) => cur.push_ref(g), + Event::Eof => break, + _ => {} + } + } + Ok(item) + } +} diff --git a/src/exchange/ews/parse/mod.rs b/src/exchange/ews/parse/mod.rs new file mode 100644 index 0000000..b93b7ba --- /dev/null +++ b/src/exchange/ews/parse/mod.rs @@ -0,0 +1,198 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +mod attachment; +mod calendar; +mod contact; +mod envelope; +mod folder; +mod item; +mod message; +mod recurrence; +mod sync; + +#[cfg(test)] +mod tests; + +use std::mem; + +use quick_xml::NsReader; +use quick_xml::XmlVersion; +use quick_xml::events::{BytesRef, BytesStart}; +use quick_xml::name::ResolveResult; + +use crate::exchange::ews::soap::{NS_MESSAGES, NS_TYPES}; +use crate::net::xml::Entity; + +pub use attachment::{GetAttachmentInline, RawAttachmentRef}; +pub use calendar::{CalendarItemRaw, RawAttendee, RawOccurrence}; +pub use contact::{ContactItemRaw, RawContactAddress, RawGroupMember}; +pub use envelope::{EnvelopeKind, ServerVersionInfo, SoapFault}; +pub use folder::{FindFolderResponse, FolderElement, FolderEntry}; +pub use item::{FindItemResponse, ItemEntry, ResponseMessage}; +pub use message::MessageItem; +pub use recurrence::{RawRecurrence, RecurrencePattern, RecurrenceRange}; +pub use sync::{SyncChange, SyncFolderItemsResponse}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Ns { + Messages, + Types, + Other, +} + +impl Ns { + fn of(ns: &ResolveResult<'_>) -> Ns { + match ns { + ResolveResult::Bound(uri) if uri.as_ref() == NS_MESSAGES => Ns::Messages, + ResolveResult::Bound(uri) if uri.as_ref() == NS_TYPES => Ns::Types, + _ => Ns::Other, + } + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Trim { + Text, + Keep, +} + +impl Trim { + fn reader(self, bytes: &[u8]) -> NsReader<&[u8]> { + let mut xml = NsReader::from_reader(bytes); + xml.config_mut().trim_text(self == Trim::Text); + xml + } +} + +#[derive(Clone, Copy)] +struct Attrs<'a, 'b>(&'a BytesStart<'b>); + +impl Attrs<'_, '_> { + fn value(self, name: &str) -> Option { + self.0 + .attributes() + .flatten() + .find(|a| a.key.local_name().into_inner().eq_ignore_ascii_case(name)) + .map(|a| { + a.normalized_value(XmlVersion::Implicit1_0) + .map(|c| c.into_owned()) + .unwrap_or_else(|_| a.value.into_owned()) + }) + } + + fn ids(self, id_out: &mut String, ck_out: &mut String) { + for a in self.0.attributes().flatten() { + let target = match IdAttr::parse(a.key.local_name().into_inner()) { + Some(IdAttr::Id) => &mut *id_out, + Some(IdAttr::ChangeKey) => &mut *ck_out, + None => continue, + }; + *target = a + .normalized_value(XmlVersion::Implicit1_0) + .map(|c| c.into_owned()) + .unwrap_or_else(|_| a.value.into_owned()); + } + } + + fn parent_id(self) -> Option { + let mut id = String::new(); + let mut change_key = String::new(); + self.ids(&mut id, &mut change_key); + (!id.is_empty()).then_some(id) + } +} + +#[derive(Clone, Copy)] +enum IdAttr { + Id, + ChangeKey, +} + +impl IdAttr { + fn parse(name: &str) -> Option { + hashify::map_ignore_case!( + name.as_bytes(), + IdAttr, + "Id" => IdAttr::Id, + "ChangeKey" => IdAttr::ChangeKey, + ) + .copied() + } +} + +struct XmlBool; + +impl XmlBool { + fn parse(text: &str) -> bool { + matches!(text.trim(), "true" | "1") + } +} + +struct ItemElement; + +impl ItemElement { + fn is(local: &str) -> bool { + hashify::set_ignore_case!( + local.as_bytes(), + "Item", + "Message", + "SharingMessage", + "CalendarItem", + "Contact", + "DistributionList", + "MeetingMessage", + "MeetingRequest", + "MeetingResponse", + "MeetingCancellation", + "Task", + "PostItem", + ) + } +} + +#[derive(Default)] +struct TextBuf(String); + +impl TextBuf { + fn push_ref(&mut self, reference: &BytesRef<'_>) { + if let Some(c) = Entity::from_ref(reference) { + self.0.push(c); + } + } + + fn take(&mut self) -> String { + mem::take(&mut self.0) + } +} + +#[derive(Default)] +struct ItemDepth { + depth: usize, + item_level: Option, +} + +impl ItemDepth { + fn open(&mut self, is_empty: bool) -> usize { + let level = self.depth + 1; + if !is_empty { + self.depth = level; + } + level + } + + fn close(&mut self) { + self.depth = self.depth.saturating_sub(1); + } + + fn enter_item(&mut self, level: usize) { + self.item_level = Some(level); + } + + fn is_item_child(&self, level: usize) -> bool { + self.item_level.is_some_and(|item| level == item + 1) + } +} diff --git a/src/exchange/ews/parse/recurrence.rs b/src/exchange/ews/parse/recurrence.rs new file mode 100644 index 0000000..5d7eb1d --- /dev/null +++ b/src/exchange/ews/parse/recurrence.rs @@ -0,0 +1,194 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +#[derive(Debug, Clone, Default)] +pub struct RawRecurrence { + pub pattern: Option, + pub range: Option, +} + +#[derive(Debug, Clone)] +pub enum RecurrencePattern { + Daily { + interval: u32, + }, + Weekly { + interval: u32, + days_of_week: Vec, + }, + AbsoluteMonthly { + interval: u32, + day_of_month: u32, + }, + RelativeMonthly { + interval: u32, + day_of_week_index: String, + days_of_week: Vec, + }, + AbsoluteYearly { + month: String, + day_of_month: u32, + }, + RelativeYearly { + month: String, + day_of_week_index: String, + days_of_week: Vec, + }, +} + +#[derive(Debug, Clone)] +pub enum RecurrenceRange { + NoEnd { + start_date: String, + }, + EndDate { + start_date: String, + end_date: String, + }, + Numbered { + start_date: String, + number_of_occurrences: u32, + }, +} + +#[derive(Clone, Copy)] +pub(super) enum PatternKind { + Daily, + Weekly, + AbsoluteMonthly, + RelativeMonthly, + AbsoluteYearly, + RelativeYearly, +} + +#[derive(Clone, Copy)] +pub(super) enum RangeKind { + NoEnd, + EndDate, + Numbered, +} + +#[derive(Clone, Copy)] +pub(super) enum RecurrenceText { + Interval, + DaysOfWeek, + DayOfMonth, + DayOfWeekIndex, + Month, + StartDate, + EndDate, + NumberOfOccurrences, +} + +#[derive(Clone, Copy)] +pub(super) enum RecurrenceElement { + Text(RecurrenceText), + Pattern(PatternKind), + Range(RangeKind), +} + +impl RecurrenceElement { + pub(super) fn parse(local: &str) -> Option { + hashify::map_ignore_case!( + local.as_bytes(), + RecurrenceElement, + "Interval" => RecurrenceElement::Text(RecurrenceText::Interval), + "DaysOfWeek" => RecurrenceElement::Text(RecurrenceText::DaysOfWeek), + "DayOfMonth" => RecurrenceElement::Text(RecurrenceText::DayOfMonth), + "DayOfWeekIndex" => RecurrenceElement::Text(RecurrenceText::DayOfWeekIndex), + "Month" => RecurrenceElement::Text(RecurrenceText::Month), + "StartDate" => RecurrenceElement::Text(RecurrenceText::StartDate), + "EndDate" => RecurrenceElement::Text(RecurrenceText::EndDate), + "NumberOfOccurrences" => RecurrenceElement::Text(RecurrenceText::NumberOfOccurrences), + "DailyRecurrence" => RecurrenceElement::Pattern(PatternKind::Daily), + "WeeklyRecurrence" => RecurrenceElement::Pattern(PatternKind::Weekly), + "AbsoluteMonthlyRecurrence" => RecurrenceElement::Pattern(PatternKind::AbsoluteMonthly), + "RelativeMonthlyRecurrence" => RecurrenceElement::Pattern(PatternKind::RelativeMonthly), + "AbsoluteYearlyRecurrence" => RecurrenceElement::Pattern(PatternKind::AbsoluteYearly), + "RelativeYearlyRecurrence" => RecurrenceElement::Pattern(PatternKind::RelativeYearly), + "NoEndRecurrence" => RecurrenceElement::Range(RangeKind::NoEnd), + "EndDateRecurrence" => RecurrenceElement::Range(RangeKind::EndDate), + "NumberedRecurrence" => RecurrenceElement::Range(RangeKind::Numbered), + ) + .copied() + } +} + +#[derive(Default)] +pub(super) struct PendingRecurrence { + pub(super) pattern: Option, + pub(super) range: Option, + interval: u32, + days_of_week: Vec, + day_of_month: u32, + day_of_week_index: String, + month: String, + start_date: String, + end_date: String, + number_of_occurrences: u32, +} + +impl PendingRecurrence { + pub(super) fn text(&mut self, target: RecurrenceText, text: &str) { + let trimmed = text.trim(); + match target { + RecurrenceText::Interval => self.interval = trimmed.parse().unwrap_or(1), + RecurrenceText::DaysOfWeek => self + .days_of_week + .extend(text.split_whitespace().map(str::to_owned)), + RecurrenceText::DayOfMonth => self.day_of_month = trimmed.parse().unwrap_or(0), + RecurrenceText::DayOfWeekIndex => trimmed.clone_into(&mut self.day_of_week_index), + RecurrenceText::Month => trimmed.clone_into(&mut self.month), + RecurrenceText::StartDate => trimmed.clone_into(&mut self.start_date), + RecurrenceText::EndDate => trimmed.clone_into(&mut self.end_date), + RecurrenceText::NumberOfOccurrences => { + self.number_of_occurrences = trimmed.parse().unwrap_or(0); + } + } + } + + pub(super) fn finish(self) -> RawRecurrence { + let interval = if self.interval == 0 { 1 } else { self.interval }; + let pattern = self.pattern.map(|kind| match kind { + PatternKind::Daily => RecurrencePattern::Daily { interval }, + PatternKind::Weekly => RecurrencePattern::Weekly { + interval, + days_of_week: self.days_of_week, + }, + PatternKind::AbsoluteMonthly => RecurrencePattern::AbsoluteMonthly { + interval, + day_of_month: self.day_of_month, + }, + PatternKind::RelativeMonthly => RecurrencePattern::RelativeMonthly { + interval, + day_of_week_index: self.day_of_week_index, + days_of_week: self.days_of_week, + }, + PatternKind::AbsoluteYearly => RecurrencePattern::AbsoluteYearly { + month: self.month, + day_of_month: self.day_of_month, + }, + PatternKind::RelativeYearly => RecurrencePattern::RelativeYearly { + month: self.month, + day_of_week_index: self.day_of_week_index, + days_of_week: self.days_of_week, + }, + }); + let start_date = self.start_date; + let range = self.range.map(|kind| match kind { + RangeKind::NoEnd => RecurrenceRange::NoEnd { start_date }, + RangeKind::EndDate => RecurrenceRange::EndDate { + start_date, + end_date: self.end_date, + }, + RangeKind::Numbered => RecurrenceRange::Numbered { + start_date, + number_of_occurrences: self.number_of_occurrences, + }, + }); + RawRecurrence { pattern, range } + } +} diff --git a/src/exchange/ews/parse/sync.rs b/src/exchange/ews/parse/sync.rs new file mode 100644 index 0000000..116d129 --- /dev/null +++ b/src/exchange/ews/parse/sync.rs @@ -0,0 +1,231 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use quick_xml::events::Event; + +use crate::exchange::ews::error::EwsError; +use crate::exchange::ews::parse::{Attrs, ItemElement, Ns, Trim, XmlBool}; +use crate::exchange::ews::types::ItemId; + +#[derive(Debug, Clone)] +pub struct SyncFolderItemsResponse { + pub sync_state: String, + pub more: bool, + pub changes: Vec, +} + +#[derive(Debug, Clone)] +pub enum SyncChange { + Create { + id: ItemId, + element: String, + item_class: Option, + }, + Update { + id: ItemId, + element: String, + item_class: Option, + }, + Delete { + id: ItemId, + }, + ReadFlagChange { + id: ItemId, + is_read: bool, + }, +} + +#[derive(Clone, Copy)] +enum SyncElement { + SyncState, + IncludesLastItemInRange, + Changes, + Create, + Update, + Delete, + ReadFlagChange, + ItemId, + IsRead, + ItemClass, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum SyncText { + SyncState, + IncludesLast, +} + +impl SyncElement { + fn parse(local: &str) -> Option { + hashify::map_ignore_case!( + local.as_bytes(), + SyncElement, + "SyncState" => SyncElement::SyncState, + "IncludesLastItemInRange" => SyncElement::IncludesLastItemInRange, + "Changes" => SyncElement::Changes, + "Create" => SyncElement::Create, + "Update" => SyncElement::Update, + "Delete" => SyncElement::Delete, + "ReadFlagChange" => SyncElement::ReadFlagChange, + "ItemId" => SyncElement::ItemId, + "IsRead" => SyncElement::IsRead, + "ItemClass" => SyncElement::ItemClass, + ) + .copied() + } +} + +impl SyncChange { + fn item_id_mut(&mut self) -> &mut ItemId { + match self { + SyncChange::Create { id, .. } + | SyncChange::Update { id, .. } + | SyncChange::Delete { id } + | SyncChange::ReadFlagChange { id, .. } => id, + } + } +} + +impl SyncFolderItemsResponse { + pub fn parse(body: &[u8]) -> Result { + let mut xml = Trim::Text.reader(body); + let mut buf = Vec::new(); + let mut sync_state = String::new(); + let mut more = false; + let mut changes: Vec = Vec::new(); + let mut current_text: Option = None; + let mut in_changes = false; + let mut current_change: Option = None; + let mut reading_is_read = false; + let mut pending_is_read: Option = None; + let mut reading_item_class = false; + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + let ns = Ns::of(&ns); + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + let local = e.local_name().into_inner(); + let is_empty = matches!(ev, Event::Empty(_)); + let element = SyncElement::parse(local); + match element { + Some(SyncElement::SyncState) if ns == Ns::Messages => { + current_text = Some(SyncText::SyncState); + } + Some(SyncElement::IncludesLastItemInRange) if ns == Ns::Messages => { + current_text = Some(SyncText::IncludesLast); + } + Some(SyncElement::Changes) if ns == Ns::Messages => in_changes = true, + _ if in_changes && ns == Ns::Types => match element { + Some(SyncElement::Create) => { + current_change = Some(SyncChange::Create { + id: ItemId::default(), + element: String::new(), + item_class: None, + }); + } + Some(SyncElement::Update) => { + current_change = Some(SyncChange::Update { + id: ItemId::default(), + element: String::new(), + item_class: None, + }); + } + Some(SyncElement::Delete) => { + current_change = Some(SyncChange::Delete { + id: ItemId::default(), + }); + } + Some(SyncElement::ReadFlagChange) => { + current_change = Some(SyncChange::ReadFlagChange { + id: ItemId::default(), + is_read: false, + }); + pending_is_read = None; + } + Some(SyncElement::ItemId) => { + if let Some(change) = current_change.as_mut() { + let id = change.item_id_mut(); + Attrs(e).ids(&mut id.id, &mut id.change_key); + } + } + Some(SyncElement::IsRead) => reading_is_read = true, + Some(SyncElement::ItemClass) => reading_item_class = !is_empty, + _ if ItemElement::is(local) => { + if let Some( + SyncChange::Create { element, .. } + | SyncChange::Update { element, .. }, + ) = current_change.as_mut() + { + local.clone_into(element); + } + } + _ => {} + }, + _ => {} + } + if is_empty { + current_text = None; + } + } + Event::End(e) => { + match SyncElement::parse(e.local_name().into_inner()) { + Some( + SyncElement::Create + | SyncElement::Update + | SyncElement::Delete + | SyncElement::ReadFlagChange, + ) => { + if let Some(mut change) = current_change.take() { + if let SyncChange::ReadFlagChange { is_read, .. } = &mut change + && let Some(v) = pending_is_read.take() + { + *is_read = v; + } + changes.push(change); + } + } + Some(SyncElement::IsRead) => reading_is_read = false, + Some(SyncElement::ItemClass) => reading_item_class = false, + Some(SyncElement::Changes) => in_changes = false, + _ => {} + } + current_text = None; + } + Event::Text(t) => { + if reading_is_read { + pending_is_read = Some(XmlBool::parse(&t)); + } + if reading_item_class + && let Some( + SyncChange::Create { item_class, .. } + | SyncChange::Update { item_class, .. }, + ) = current_change.as_mut() + { + *item_class = Some(t.trim().to_owned()); + } + match current_text { + Some(SyncText::SyncState) => sync_state = t.into_inner().into_owned(), + Some(SyncText::IncludesLast) => more = !XmlBool::parse(&t), + None => {} + } + } + Event::CData(c) => { + if current_text == Some(SyncText::SyncState) { + sync_state = c.into_inner().into_owned(); + } + } + Event::Eof => break, + _ => {} + } + } + Ok(SyncFolderItemsResponse { + sync_state, + more, + changes, + }) + } +} diff --git a/src/exchange/ews/parse/tests.rs b/src/exchange/ews/parse/tests.rs new file mode 100644 index 0000000..5edcb07 --- /dev/null +++ b/src/exchange/ews/parse/tests.rs @@ -0,0 +1,832 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use super::*; +use crate::exchange::ews::types::{CalendarItemType, ResponseCode, ServerVersion}; + +const NS: &str = " xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\" \ + xmlns:t=\"http://schemas.microsoft.com/exchange/services/2006/types\" \ + xmlns:m=\"http://schemas.microsoft.com/exchange/services/2006/messages\""; + +#[test] +fn server_busy_fault_extracts_backoff_milliseconds() { + let body = format!( + "\ + soap:Serverbusy\ + ErrorServerBusy\ + 297749\ + " + ); + let env = EnvelopeKind::read(body.as_bytes()).unwrap(); + match env { + EnvelopeKind::Fault { fault, version } => { + assert!(matches!( + fault.response_code, + ResponseCode::ServerBusy { + back_off_ms: Some(297749) + } + )); + assert_eq!(fault.back_off_ms, Some(297749)); + assert_eq!( + version.to_server_version(), + Some(ServerVersion::Exchange2016) + ); + } + _ => panic!("expected fault"), + } +} + +#[test] +fn body_envelope_returns_version_only() { + let body = format!( + "\ + " + ); + let env = EnvelopeKind::read(body.as_bytes()).unwrap(); + assert!(matches!(env, EnvelopeKind::Body { .. })); +} + +#[test] +fn find_folder_parses_two_folders() { + let body = format!( + "\ + NoError\ + \ + \ + \ + IPF.NoteInbox\ + 70\ + \ + IPF.AppointmentCalendar\ + \ + " + ); + let r = FindFolderResponse::parse(body.as_bytes()).unwrap(); + assert_eq!(r.folders.len(), 2); + assert_eq!(r.folders[0].folder_id.id, "F1"); + assert_eq!(r.folders[0].folder_id.change_key, "C1"); + assert_eq!(r.folders[0].display_name, "Inbox"); + assert_eq!(r.folders[0].folder_class, "IPF.Note"); + assert_eq!(r.folders[0].parent_id.as_deref(), Some("ROOT")); + assert_eq!(r.folders[1].element, FolderElement::CalendarFolder); + assert!(!r.more); + assert!(!r.folders[0].is_hidden); +} + +#[test] +fn find_folder_reads_pr_attr_hidden() { + let body = format!( + "\ + NoError\ + \ + \ + \ + IPF.ContactRecipient Cache\ + \ + true\ + \ + IPF.NoteInbox\ + \ + false\ + " + ); + let r = FindFolderResponse::parse(body.as_bytes()).unwrap(); + assert_eq!(r.folders.len(), 2); + assert!(r.folders[0].is_hidden); + assert_eq!(r.folders[0].display_name, "Recipient Cache"); + assert!(!r.folders[1].is_hidden); + assert_eq!(r.folders[1].display_name, "Inbox"); +} + +#[test] +fn find_item_reads_item_class() { + let body = format!( + "\ + NoError\ + \ + IPM.Contact\ + \ + " + ); + let r = FindItemResponse::parse(body.as_bytes()).unwrap(); + assert_eq!(r.items[0].item_class.as_deref(), Some("IPM.Contact")); + assert_eq!(r.items[1].item_class, None); +} + +#[test] +fn sync_folder_items_reads_item_class() { + let body = format!( + "\ + NoError\ + AAAA\ + true\ + \ + IPM.Task\ + IPM.Note\ + " + ); + let r = SyncFolderItemsResponse::parse(body.as_bytes()).unwrap(); + let classes: Vec> = r + .changes + .iter() + .map(|c| match c { + SyncChange::Create { item_class, .. } | SyncChange::Update { item_class, .. } => { + item_class.as_deref() + } + _ => None, + }) + .collect(); + assert_eq!(classes, [Some("IPM.Task"), Some("IPM.Note")]); +} + +#[test] +fn find_item_opens_an_entry_for_every_item_element() { + let body = format!( + "\ + NoError\ + \ + \ + \ + \ + \ + " + ); + let r = FindItemResponse::parse(body.as_bytes()).unwrap(); + let ids: Vec<(&str, &str)> = r + .items + .iter() + .map(|i| (i.element.as_str(), i.id.id.as_str())) + .collect(); + assert_eq!( + ids, + [ + ("Message", "M1"), + ("Task", "T1"), + ("PostItem", "P1"), + ("FutureItem", "F1") + ] + ); +} + +#[test] +fn find_item_ignores_item_ids_nested_in_mailboxes() { + let body = format!( + "\ + NoError\ + \ + \ + bob@example.com\ + \ + " + ); + let r = FindItemResponse::parse(body.as_bytes()).unwrap(); + assert_eq!(r.items.len(), 1); + assert_eq!(r.items[0].id.id, "E1"); +} + +#[test] +fn calendar_item_keeps_its_own_id_when_attendees_carry_contact_ids() { + let xml = "\ + \ + alice@example.com\ + Contact\ + \ + bob@example.com\ + Contact\ + "; + let item = CalendarItemRaw::parse(xml).unwrap(); + assert_eq!(item.id.id, "EVENT-A"); + assert_eq!(item.id.change_key, "CK"); + assert_eq!(item.required_attendees.len(), 1); +} + +#[test] +fn message_item_keeps_its_own_id_when_recipients_carry_contact_ids() { + let xml = "\ + \ + bob@example.com\ + "; + let item = MessageItem::parse(xml).unwrap(); + assert_eq!(item.id.id, "MSG-A"); + assert_eq!(item.element, "MeetingRequest"); +} + +#[test] +fn distribution_list_keeps_its_own_id_when_members_carry_contact_ids() { + let xml = "\ + \ + Team\ + Bobbob@example.com\ + Contact\ + "; + let item = ContactItemRaw::parse(xml).unwrap(); + assert_eq!(item.id.id, "DL-A"); + assert!(item.is_group); + assert_eq!(item.members.len(), 1); + assert_eq!(item.members[0].email.as_deref(), Some("bob@example.com")); +} + +#[test] +fn find_item_parses_two_messages_with_pagination_flag() { + let body = format!( + "\ + NoError\ + \ + \ + \ + \ + " + ); + let r = FindItemResponse::parse(body.as_bytes()).unwrap(); + assert_eq!(r.items.len(), 2); + assert_eq!(r.items[0].id.id, "I1"); + assert_eq!(r.items[0].id.change_key, "CK1"); + assert_eq!(r.items[1].id.id, "I2"); + assert!(r.more); + assert_eq!(r.total_in_view, Some(3)); +} + +#[test] +fn parse_response_messages_carries_inner_xml_and_codes() { + let body = format!( + "\ + \ + NoError\ + \ + Hitrue\ + SGVsbG8=\ + \ + \ + ErrorItemNotFound\ + not found\ + \ + " + ); + let r = ResponseMessage::parse_all(body.as_bytes(), "GetItemResponseMessage").unwrap(); + assert_eq!(r.len(), 2); + assert!(r[0].success); + assert!(r[0].inner_xml.contains("\ + \ + NoError\ + {message_xml}\ + " + ); + let r = ResponseMessage::parse_all(body.as_bytes(), "GetItemResponseMessage").unwrap(); + MessageItem::parse(&r[0].inner_xml).unwrap() + } + + let flagged = parse_one( + "\ + Flagged\ + \ + \ + 2\ + \ + true", + ); + assert_eq!(flagged.flag_status.as_deref(), Some("Flagged")); + assert_eq!(flagged.is_read, Some(true)); + + let cleared = parse_one( + "\ + \ + \ + 0\ + ", + ); + assert_eq!(cleared.flag_status, None); +} + +#[test] +fn native_item_flag_still_parses_on_exchange_2013_plus() { + let body = format!( + "\ + \ + NoError\ + \ + Native\ + Flagged\ + false\ + " + ); + let r = ResponseMessage::parse_all(body.as_bytes(), "GetItemResponseMessage").unwrap(); + let p = MessageItem::parse(&r[0].inner_xml).unwrap(); + assert_eq!( + p.flag_status.as_deref(), + Some("Flagged"), + "the 2013+ native item:Flag path must be unaffected by the pre-2013 extended-property fallback" + ); + assert_eq!(p.is_read, Some(false)); +} + +#[test] +fn sync_folder_items_emits_create_delete_readflag() { + let body = format!( + "\ + NoError\ + AAAA\ + true\ + \ + \ + \ + true\ + " + ); + let r = SyncFolderItemsResponse::parse(body.as_bytes()).unwrap(); + assert_eq!(r.sync_state, "AAAA"); + assert!(!r.more); + assert_eq!(r.changes.len(), 3); + match &r.changes[0] { + SyncChange::Create { id, element, .. } => { + assert_eq!(id.id, "I1"); + assert_eq!(element, "Message"); + } + _ => panic!("expected Create"), + } + match &r.changes[2] { + SyncChange::ReadFlagChange { id, is_read } => { + assert_eq!(id.id, "I3"); + assert!(*is_read); + } + _ => panic!("expected ReadFlagChange"), + } +} + +#[test] +fn calendar_master_recurrence_and_occurrences_round_trip() { + let body = format!( + "\ + \ + Daily\ + uid-1\ + 2025-06-15T14:00:00Z\ + 2025-06-15T15:00:00Z\ + RecurringMaster\ + \ + 1\ + 2025-06-153\ + \ + \ + 2025-06-16T15:00:00Z2025-06-16T16:30:00Z2025-06-16T14:00:00Z\ + \ + \ + 2025-06-17T14:00:00Z\ + \ + " + ); + let parsed = CalendarItemRaw::parse(&body).unwrap(); + assert_eq!(parsed.id.id, "M1"); + assert_eq!(parsed.uid.as_deref(), Some("uid-1")); + assert_eq!( + parsed.calendar_item_type, + Some(CalendarItemType::RecurringMaster) + ); + assert_eq!(parsed.modified_occurrences.len(), 1); + assert_eq!( + parsed.modified_occurrences[0].original_start.as_deref(), + Some("2025-06-16T14:00:00Z") + ); + assert_eq!(parsed.deleted_occurrences.len(), 1); + assert_eq!( + parsed.deleted_occurrences[0].start.as_deref(), + Some("2025-06-17T14:00:00Z") + ); + let rec = parsed.recurrence.unwrap(); + assert!(matches!( + rec.pattern, + Some(RecurrencePattern::Daily { interval: 1 }) + )); + assert!(matches!( + rec.range, + Some(RecurrenceRange::Numbered { + number_of_occurrences: 3, + .. + }) + )); +} + +#[test] +fn end_date_recurrence_wire_shape_with_trailing_z_yields_clean_until() { + let body = format!( + "\ + RecDailyEndu\ + 2026-07-01T15:00:00Z2026-07-01T15:30:00Z\ + RecurringMaster\ + \ + 2\ + 2026-07-01Z2026-08-01Z\ + " + ); + let parsed = CalendarItemRaw::parse(&body).unwrap(); + let rule = parsed.recurrence.as_ref().unwrap().to_jscalendar().unwrap(); + assert_eq!(rule["frequency"], "daily"); + assert_eq!(rule["interval"], 2); + assert_eq!( + rule["until"], "2026-08-01T23:59:59", + "trailing Z on EndDate must be stripped, not embedded mid-string" + ); +} + +#[test] +fn absolute_and_relative_recurrence_wire_shapes_round_trip() { + let cases = [ + ( + "1152026-07-01Z", + "monthly", + ), + ( + "1FridayLast2026-07-01Z", + "monthly", + ), + ( + "1January2026-07-01Z", + "yearly", + ), + ( + "MondayFirstSeptember2026-07-01Z", + "yearly", + ), + ]; + for (rec_xml, freq) in cases { + let body = format!( + "u\ + 2026-07-01T15:00:00Z2026-07-01T15:30:00Z\ + {rec_xml}" + ); + let parsed = CalendarItemRaw::parse(&body).unwrap(); + let rule = parsed.recurrence.as_ref().unwrap().to_jscalendar().unwrap(); + assert_eq!(rule["frequency"], freq, "for {rec_xml}"); + } + let body = format!( + "u\ + 2026-07-01T15:00:00Z2026-07-01T15:30:00Z\ + 1\ + FridayLast\ + 2026-07-01Z" + ); + let parsed = CalendarItemRaw::parse(&body).unwrap(); + let rule = parsed.recurrence.as_ref().unwrap().to_jscalendar().unwrap(); + assert_eq!(rule["byDay"][0]["day"], "fr"); + assert_eq!(rule["byDay"][0]["nthOfPeriod"], -1); +} + +#[test] +fn attendee_and_organizer_routing_type_is_parsed() { + let body = format!( + "u\ + Org\ + /o=ExLabs/cn=orgEX\ + Att\ + /o=ExLabs/cn=attEX\ + Accept\ + " + ); + let parsed = CalendarItemRaw::parse(&body).unwrap(); + assert_eq!(parsed.organizer_routing_type.as_deref(), Some("EX")); + assert_eq!(parsed.required_attendees.len(), 1); + assert_eq!( + parsed.required_attendees[0].routing_type.as_deref(), + Some("EX") + ); + assert_eq!( + parsed.required_attendees[0].email.as_deref(), + Some("/o=ExLabs/cn=att") + ); +} + +#[test] +fn calendar_parses_own_response_sensitivity_and_attachments() { + let body = format!( + "\ + \ + Private\ + \ + agenda.pdfapplication/pdf\ + 2020-01-01T00:00:00\ + \ + 2025-06-01T00:00:00Z\ + Accept\ + " + ); + let parsed = CalendarItemRaw::parse(&body).unwrap(); + assert_eq!(parsed.sensitivity.as_deref(), Some("Private")); + assert_eq!(parsed.my_response_type.as_deref(), Some("Accept")); + assert_eq!(parsed.attachments.len(), 1); + assert_eq!(parsed.attachments[0].attachment_id, "ATT1"); + assert_eq!( + parsed.last_modified.as_deref(), + Some("2025-06-01T00:00:00Z") + ); +} + +#[test] +fn calendar_parses_reminder_resources_and_online_meeting() { + let body = format!( + "\ + \ + Standup\ + uid-r\ + true\ + 10\ + 2025-06-15T14:00:00Z\ + 2025-06-15T15:00:00Z\ + true\ + https://meet/abc\ + \ + Bobbob@xAccept\ + \ + \ + Room 7room7@x\ + \ + " + ); + let parsed = CalendarItemRaw::parse(&body).unwrap(); + assert_eq!(parsed.reminder_is_set, Some(true)); + assert_eq!(parsed.reminder_minutes_before_start, Some(10)); + assert_eq!(parsed.is_online_meeting, Some(true)); + assert_eq!(parsed.net_show_url.as_deref(), Some("https://meet/abc")); + assert_eq!(parsed.required_attendees.len(), 1); + assert_eq!(parsed.required_attendees[0].email.as_deref(), Some("bob@x")); + assert_eq!(parsed.resources.len(), 1); + assert_eq!(parsed.resources[0].email.as_deref(), Some("room7@x")); +} + +#[test] +fn distribution_list_parses_members() { + let body = format!( + "\ + \ + The Team\ + \ + Bobbob@xNormal\ + Carolcarol@x\ + \ + " + ); + let parsed = ContactItemRaw::parse(&body).unwrap(); + assert!(parsed.is_group); + assert_eq!(parsed.id.id, "DL1"); + assert_eq!(parsed.display_name.as_deref(), Some("The Team")); + assert_eq!(parsed.members.len(), 2); + assert_eq!(parsed.members[0].email.as_deref(), Some("bob@x")); + assert_eq!(parsed.members[1].name.as_deref(), Some("Carol")); +} + +#[test] +fn contact_decodes_xml_entities() { + let body = format!( + "\ + \ + R&D\ + Sales & Marketing <Team>\ + AT&T\ + R&D\ + a&b@x.com\ + " + ); + let parsed = ContactItemRaw::parse(&body).unwrap(); + assert_eq!( + parsed.display_name.as_deref(), + Some("Sales & Marketing ") + ); + assert_eq!(parsed.company_name.as_deref(), Some("AT&T")); + assert_eq!(parsed.department.as_deref(), Some("R&D")); + assert_eq!(parsed.categories, vec!["R&D".to_owned()]); + assert_eq!(parsed.emails[0].1, "a&b@x.com"); +} + +#[test] +fn calendar_decodes_xml_entities_in_subject_and_location() { + let body = format!( + "\ + \ + Q&A <all hands>\ + Room A & B\ + 2025-06-15T14:00:00Z\ + 2025-06-15T15:00:00Z\ + " + ); + let parsed = CalendarItemRaw::parse(&body).unwrap(); + assert_eq!(parsed.subject.as_deref(), Some("Q&A ")); + assert_eq!(parsed.location.as_deref(), Some("Room A & B")); +} + +#[test] +fn folder_decodes_xml_entities_in_display_name() { + let body = format!( + "\ + Sales & Marketing\ + IPF.Note" + ); + let entry = FolderEntry::parse_inner(&body).unwrap().unwrap(); + assert_eq!(entry.display_name, "Sales & Marketing"); +} + +#[test] +fn response_capture_preserves_entities_for_reparse() { + let body = format!( + "\ + NoError\ + AT&T\ + " + ); + let msgs = ResponseMessage::parse_all(body.as_bytes(), "GetItemResponseMessage").unwrap(); + assert!( + msgs[0].inner_xml.contains("AT&T") || msgs[0].inner_xml.contains("AT&T"), + "capture must preserve the entity for the per-item parser: {}", + msgs[0].inner_xml + ); + let parsed = ContactItemRaw::parse(&msgs[0].inner_xml).unwrap(); + assert_eq!(parsed.company_name.as_deref(), Some("AT&T")); +} + +#[test] +fn contact_parses_emails_phones_addresses() { + let body = format!( + "\ + \ + Alice Doe\ + Alice\ + Doe\ + \ + alice@example.com\ + \ + \ + +15551234\ + \ + \ + 1 MainTownUS\ + \ + " + ); + let parsed = ContactItemRaw::parse(&body).unwrap(); + assert_eq!(parsed.given_name.as_deref(), Some("Alice")); + assert_eq!(parsed.surname.as_deref(), Some("Doe")); + assert_eq!(parsed.emails.len(), 1); + assert_eq!(parsed.emails[0].0, "EmailAddress1"); + assert_eq!(parsed.emails[0].1, "alice@example.com"); + assert_eq!(parsed.phones.len(), 1); + assert_eq!(parsed.phones[0].0, "BusinessPhone1"); + assert_eq!(parsed.addresses.len(), 1); + assert_eq!(parsed.addresses[0].key, "Home"); + assert_eq!(parsed.addresses[0].street.as_deref(), Some("1 Main")); + assert_eq!(parsed.addresses[0].country.as_deref(), Some("US")); +} + +#[test] +fn get_item_with_unprefixed_types_elements_parses() { + let body = format!( + "\ + \ + NoError\ + \ + \ + Hello\ + SGk=\ + \ + " + ); + let msgs = ResponseMessage::parse_all(body.as_bytes(), "GetItemResponseMessage").unwrap(); + assert_eq!(msgs.len(), 1); + let item = MessageItem::parse(&msgs[0].inner_xml).unwrap(); + assert_eq!(item.id.id, "NP1"); + assert_eq!(item.subject.as_deref(), Some("Hello")); + assert_eq!(item.mime_content.as_deref(), Some("SGk=")); +} + +#[test] +fn default_namespace_response_parses_via_resolved_names() { + let body = "\ + \ + \ + \ + NoError\ + \ + \ + IPF.NoteInbox\ + \ + \ + "; + let parsed = FindFolderResponse::parse(body.as_bytes()).unwrap(); + assert_eq!(parsed.folders.len(), 1); + assert_eq!(parsed.folders[0].folder_id.id, "D1"); + assert_eq!(parsed.folders[0].display_name, "Inbox"); +} + +#[test] +fn fault_response_code_in_messages_namespace_is_recognised() { + let body = format!( + "\ + soap:Serverbusy\ + ErrorServerBusy\ + 42\ + " + ); + let env = EnvelopeKind::read(body.as_bytes()).unwrap(); + match env { + EnvelopeKind::Fault { fault, .. } => { + assert!(matches!( + fault.response_code, + ResponseCode::ServerBusy { .. } + )); + assert_eq!(fault.back_off_ms, Some(42)); + } + _ => panic!("expected fault"), + } +} + +#[test] +fn change_key_attribute_with_entities_is_xml_decoded() { + let body = format!( + "\ + NoError\ + \ + \ + X\ + " + ); + let parsed = FindFolderResponse::parse(body.as_bytes()).unwrap(); + assert_eq!(parsed.folders[0].folder_id.change_key, "A&B"); +} + +#[test] +fn warning_response_class_is_treated_as_success() { + let body = format!( + "\ + \ + NoError\ + \ + " + ); + let r = ResponseMessage::parse_all(body.as_bytes(), "GetItemResponseMessage").unwrap(); + assert_eq!(r.len(), 1); + assert!(r[0].success, "Warning should be success-equivalent"); +} + +#[test] +fn get_folder_response_message_captures_folder_inner() { + let body = format!( + "\ + \ + NoError\ + \ + Inbox\ + \ + \ + ErrorAccessDenied\ + " + ); + let msgs = ResponseMessage::parse_all(body.as_bytes(), "GetFolderResponseMessage").unwrap(); + assert_eq!(msgs.len(), 2); + assert!(msgs[0].success); + let f = FolderEntry::parse_inner(&msgs[0].inner_xml) + .unwrap() + .unwrap(); + assert_eq!(f.folder_id.id, "F1"); + assert!(!msgs[1].success); +} + +#[test] +fn get_attachment_inline_decodes_content_field() { + let body = format!( + "\ + \ + NoError\ + \ + p.png\ + image/png\ + true\ + iVBORw0KGgo=\ + " + ); + let items = GetAttachmentInline::parse_all(body.as_bytes()).unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0].attachment_id, "A1"); + assert_eq!(items[0].name.as_deref(), Some("p.png")); + assert_eq!(items[0].content_type.as_deref(), Some("image/png")); + assert!(items[0].is_contact_photo); + assert_eq!(items[0].content_base64, "iVBORw0KGgo="); +} diff --git a/src/exchange_ews/recurrence.rs b/src/exchange/ews/recurrence.rs similarity index 55% rename from src/exchange_ews/recurrence.rs rename to src/exchange/ews/recurrence.rs index 8a365dc..24d0d46 100644 --- a/src/exchange_ews/recurrence.rs +++ b/src/exchange/ews/recurrence.rs @@ -6,193 +6,163 @@ use serde_json::{Map, Value, json}; -use crate::exchange::date::recurrence_until; -use crate::exchange_ews::parse::{RawRecurrence, RecurrencePattern, RecurrenceRange}; +use crate::exchange::date::WireDate; +use crate::exchange::ews::parse::{RawRecurrence, RecurrencePattern, RecurrenceRange}; +use crate::exchange::vocab::{MonthName, Ordinal}; -pub fn to_jscalendar_rule(raw: &RawRecurrence) -> Option { - let mut rule: Map = Map::new(); - match raw.pattern.as_ref()? { - RecurrencePattern::Daily { interval } => { - rule.insert("frequency".to_owned(), Value::String("daily".to_owned())); - if *interval > 1 { - rule.insert("interval".to_owned(), Value::from(*interval)); - } - } - RecurrencePattern::Weekly { - interval, - days_of_week, - } => { - rule.insert("frequency".to_owned(), Value::String("weekly".to_owned())); - if *interval > 1 { - rule.insert("interval".to_owned(), Value::from(*interval)); - } - let days: Vec = days_of_week - .iter() - .filter_map(|d| expand_day(d)) - .flat_map(|slice| slice.iter().map(|t| json!({"day": t}))) - .collect(); - if !days.is_empty() { - rule.insert("byDay".to_owned(), Value::Array(days)); - } - } - RecurrencePattern::AbsoluteMonthly { - interval, - day_of_month, - } => { - rule.insert("frequency".to_owned(), Value::String("monthly".to_owned())); - if *interval > 1 { - rule.insert("interval".to_owned(), Value::from(*interval)); - } - rule.insert( - "byMonthDay".to_owned(), - Value::Array(vec![Value::from(*day_of_month)]), - ); - } - RecurrencePattern::RelativeMonthly { - interval, - day_of_week_index, - days_of_week, - } => { - rule.insert("frequency".to_owned(), Value::String("monthly".to_owned())); - if *interval > 1 { - rule.insert("interval".to_owned(), Value::from(*interval)); - } - insert_relative_by_day(&mut rule, days_of_week, day_of_week_index); - } - RecurrencePattern::AbsoluteYearly { - month, - day_of_month, - } => { - rule.insert("frequency".to_owned(), Value::String("yearly".to_owned())); - if let Some(n) = month_number(month) { - rule.insert( - "byMonth".to_owned(), - Value::Array(vec![Value::String(n.to_string())]), - ); - } - rule.insert( - "byMonthDay".to_owned(), - Value::Array(vec![Value::from(*day_of_month)]), - ); - } - RecurrencePattern::RelativeYearly { - month, - day_of_week_index, - days_of_week, - } => { - rule.insert("frequency".to_owned(), Value::String("yearly".to_owned())); - if let Some(n) = month_number(month) { - rule.insert( - "byMonth".to_owned(), - Value::Array(vec![Value::String(n.to_string())]), - ); - } - insert_relative_by_day(&mut rule, days_of_week, day_of_week_index); - } - } - match raw.range.as_ref() { - Some(RecurrenceRange::NoEnd { .. }) | None => {} - Some(RecurrenceRange::EndDate { end_date, .. }) => { - if let Some(local) = recurrence_until(end_date) { - rule.insert("until".to_owned(), Value::String(local)); - } - } - Some(RecurrenceRange::Numbered { - number_of_occurrences, - .. - }) => { - rule.insert("count".to_owned(), Value::from(*number_of_occurrences)); - } - } - Some(Value::Object(rule)) -} +type DaySet = &'static [&'static str]; -fn expand_day(d: &str) -> Option<&'static [&'static str]> { - Some(match d.to_ascii_lowercase().as_str() { - "monday" | "mo" => &["mo"], - "tuesday" | "tu" => &["tu"], - "wednesday" | "we" => &["we"], - "thursday" | "th" => &["th"], - "friday" | "fr" => &["fr"], - "saturday" | "sa" => &["sa"], - "sunday" | "su" => &["su"], - "day" => &["mo", "tu", "we", "th", "fr", "sa", "su"], - "weekday" => &["mo", "tu", "we", "th", "fr"], - "weekendday" => &["sa", "su"], - _ => return None, - }) -} - -fn insert_relative_by_day(rule: &mut Map, days_of_week: &[String], index: &str) { - let nth = nth_of_period(index); - let mut days: Vec<&'static str> = Vec::new(); - let mut is_set = false; - for d in days_of_week { - if let Some(slice) = expand_day(d) { - if slice.len() > 1 { - is_set = true; +impl RawRecurrence { + pub fn to_jscalendar(&self) -> Option { + let mut rule: Map = Map::new(); + match self.pattern.as_ref()? { + RecurrencePattern::Daily { interval } => { + RawRecurrence::frequency(&mut rule, "daily", Some(*interval)); } - for &t in slice { - if !days.contains(&t) { - days.push(t); + RecurrencePattern::Weekly { + interval, + days_of_week, + } => { + RawRecurrence::frequency(&mut rule, "weekly", Some(*interval)); + let days: Vec = days_of_week + .iter() + .filter_map(|d| RawRecurrence::expand_day(d)) + .flat_map(|slice| slice.iter().map(|t| json!({"day": t}))) + .collect(); + if !days.is_empty() { + rule.insert("byDay".to_owned(), Value::Array(days)); } } + RecurrencePattern::AbsoluteMonthly { + interval, + day_of_month, + } => { + RawRecurrence::frequency(&mut rule, "monthly", Some(*interval)); + rule.insert("byMonthDay".to_owned(), json!([*day_of_month])); + } + RecurrencePattern::RelativeMonthly { + interval, + day_of_week_index, + days_of_week, + } => { + RawRecurrence::frequency(&mut rule, "monthly", Some(*interval)); + RawRecurrence::insert_relative_by_day(&mut rule, days_of_week, day_of_week_index); + } + RecurrencePattern::AbsoluteYearly { + month, + day_of_month, + } => { + RawRecurrence::frequency(&mut rule, "yearly", None); + RawRecurrence::insert_month(&mut rule, month); + rule.insert("byMonthDay".to_owned(), json!([*day_of_month])); + } + RecurrencePattern::RelativeYearly { + month, + day_of_week_index, + days_of_week, + } => { + RawRecurrence::frequency(&mut rule, "yearly", None); + RawRecurrence::insert_month(&mut rule, month); + RawRecurrence::insert_relative_by_day(&mut rule, days_of_week, day_of_week_index); + } + } + match self.range.as_ref() { + Some(RecurrenceRange::NoEnd { .. }) | None => {} + Some(RecurrenceRange::EndDate { end_date, .. }) => { + if let Some(local) = WireDate(end_date).recurrence_until() { + rule.insert("until".to_owned(), Value::String(local)); + } + } + Some(RecurrenceRange::Numbered { + number_of_occurrences, + .. + }) => { + rule.insert("count".to_owned(), Value::from(*number_of_occurrences)); + } + } + Some(Value::Object(rule)) + } + + fn frequency(rule: &mut Map, frequency: &str, interval: Option) { + rule.insert("frequency".to_owned(), Value::from(frequency)); + if let Some(interval) = interval.filter(|i| *i > 1) { + rule.insert("interval".to_owned(), Value::from(interval)); } } - if days.is_empty() { - return; - } - if is_set { - let by_day: Vec = days.iter().map(|t| json!({"day": t})).collect(); - rule.insert("byDay".to_owned(), Value::Array(by_day)); - if let Some(n) = nth { - rule.insert("bySetPosition".to_owned(), json!([n])); + + fn insert_month(rule: &mut Map, month: &str) { + if let Some(n) = MonthName(month).number() { + rule.insert("byMonth".to_owned(), json!([n.to_string()])); } - } else { - let by_day: Vec = days + } + + fn expand_day(day: &str) -> Option { + hashify::map_ignore_case!( + day.as_bytes(), + DaySet, + "monday" => &["mo"], + "mo" => &["mo"], + "tuesday" => &["tu"], + "tu" => &["tu"], + "wednesday" => &["we"], + "we" => &["we"], + "thursday" => &["th"], + "th" => &["th"], + "friday" => &["fr"], + "fr" => &["fr"], + "saturday" => &["sa"], + "sa" => &["sa"], + "sunday" => &["su"], + "su" => &["su"], + "day" => &["mo", "tu", "we", "th", "fr", "sa", "su"], + "weekday" => &["mo", "tu", "we", "th", "fr"], + "weekendday" => &["sa", "su"], + ) + .copied() + } + + fn insert_relative_by_day(rule: &mut Map, days_of_week: &[String], index: &str) { + let nth = Ordinal::parse_exact(index).map(Ordinal::position); + let mut days: Vec<&'static str> = Vec::new(); + let mut is_set = false; + for slice in days_of_week .iter() - .map(|t| { - let mut o = Map::new(); - o.insert("day".to_owned(), Value::String((*t).to_owned())); - if let Some(n) = nth { - o.insert("nthOfPeriod".to_owned(), Value::from(n)); + .filter_map(|d| RawRecurrence::expand_day(d)) + { + is_set |= slice.len() > 1; + for &token in slice { + if !days.contains(&token) { + days.push(token); } - Value::Object(o) - }) - .collect(); + } + } + if days.is_empty() { + return; + } + if is_set { + let by_day = days.iter().map(|t| json!({"day": t})).collect(); + rule.insert("byDay".to_owned(), Value::Array(by_day)); + if let Some(n) = nth { + rule.insert("bySetPosition".to_owned(), json!([n])); + } + return; + } + let by_day = { + days.iter() + .map(|t| { + let mut o = Map::new(); + o.insert("day".to_owned(), Value::from(*t)); + if let Some(n) = nth { + o.insert("nthOfPeriod".to_owned(), Value::from(n)); + } + Value::Object(o) + }) + .collect() + }; rule.insert("byDay".to_owned(), Value::Array(by_day)); } } -fn nth_of_period(index: &str) -> Option { - match index { - "First" => Some(1), - "Second" => Some(2), - "Third" => Some(3), - "Fourth" => Some(4), - "Last" => Some(-1), - _ => None, - } -} - -fn month_number(m: &str) -> Option { - Some(match m { - "January" => 1, - "February" => 2, - "March" => 3, - "April" => 4, - "May" => 5, - "June" => 6, - "July" => 7, - "August" => 8, - "September" => 9, - "October" => 10, - "November" => 11, - "December" => 12, - _ => return None, - }) -} - #[cfg(test)] mod tests { use super::*; @@ -206,7 +176,7 @@ mod tests { number_of_occurrences: 5, }), }; - let rule = to_jscalendar_rule(&raw).unwrap(); + let rule = raw.to_jscalendar().unwrap(); assert_eq!(rule["frequency"], "daily"); assert_eq!(rule["interval"], 3); assert_eq!(rule["count"], 5); @@ -224,7 +194,7 @@ mod tests { start_date: "2025-01-01".to_owned(), }), }; - let rule = to_jscalendar_rule(&raw).unwrap(); + let rule = raw.to_jscalendar().unwrap(); assert_eq!(rule["frequency"], "monthly"); let by_day = rule["byDay"].as_array().unwrap(); assert_eq!(by_day[0]["day"], "mo"); @@ -243,7 +213,7 @@ mod tests { start_date: "2020-01-01".to_owned(), }), }; - let rule = to_jscalendar_rule(&raw).unwrap(); + let rule = raw.to_jscalendar().unwrap(); assert_eq!(rule["frequency"], "yearly"); assert_eq!(rule["byMonth"][0], "6"); assert_eq!(rule["byDay"][0]["day"], "fr"); @@ -261,7 +231,7 @@ mod tests { start_date: "2025-01-15".to_owned(), }), }; - let rule = to_jscalendar_rule(&raw).unwrap(); + let rule = raw.to_jscalendar().unwrap(); assert_eq!(rule["frequency"], "monthly"); assert_eq!(rule["interval"], 2); assert_eq!(rule["byMonthDay"][0], 15); @@ -278,7 +248,7 @@ mod tests { start_date: "2025-01-01".to_owned(), }), }; - let rule = to_jscalendar_rule(&raw).unwrap(); + let rule = raw.to_jscalendar().unwrap(); assert_eq!(rule["frequency"], "yearly"); assert_eq!(rule["byMonth"][0], "1"); assert_eq!(rule["byMonthDay"][0], 1); @@ -296,7 +266,7 @@ mod tests { end_date: "2021-09-30-06:00".to_owned(), }), }; - let rule = to_jscalendar_rule(&raw).expect("rule"); + let rule = raw.to_jscalendar().expect("rule"); assert_eq!(rule["until"], "2021-09-30T23:59:59"); } @@ -309,7 +279,7 @@ mod tests { end_date: "not-a-date".to_owned(), }), }; - let rule = to_jscalendar_rule(&raw).expect("rule"); + let rule = raw.to_jscalendar().expect("rule"); assert!(rule.get("until").is_none()); } @@ -321,7 +291,7 @@ mod tests { start_date: "2025-01-01".to_owned(), }), }; - let rule = to_jscalendar_rule(&raw).unwrap(); + let rule = raw.to_jscalendar().unwrap(); assert!(rule.get("until").is_none()); assert!(rule.get("count").is_none()); } @@ -338,7 +308,7 @@ mod tests { start_date: "2025-01-01".to_owned(), }), }; - let rule = to_jscalendar_rule(&raw).unwrap(); + let rule = raw.to_jscalendar().unwrap(); let by_day = rule["byDay"].as_array().unwrap(); let days: Vec<&str> = by_day.iter().map(|d| d["day"].as_str().unwrap()).collect(); assert_eq!(days, ["mo", "tu", "we", "th", "fr"]); @@ -361,7 +331,7 @@ mod tests { start_date: "2025-01-01".to_owned(), }), }; - let rule = to_jscalendar_rule(&raw).unwrap(); + let rule = raw.to_jscalendar().unwrap(); let days: Vec<&str> = rule["byDay"] .as_array() .unwrap() @@ -381,7 +351,7 @@ mod tests { end_date: "2026-08-01Z".to_owned(), }), }; - let rule = to_jscalendar_rule(&raw).unwrap(); + let rule = raw.to_jscalendar().unwrap(); assert_eq!(rule["until"], "2026-08-01T23:59:59"); } @@ -396,7 +366,7 @@ mod tests { start_date: "2025-01-06".to_owned(), }), }; - let rule = to_jscalendar_rule(&raw).unwrap(); + let rule = raw.to_jscalendar().unwrap(); let days: Vec<&str> = rule["byDay"] .as_array() .unwrap() @@ -419,7 +389,7 @@ mod tests { end_date: "2025-06-30".to_owned(), }), }; - let rule = to_jscalendar_rule(&raw).unwrap(); + let rule = raw.to_jscalendar().unwrap(); assert_eq!(rule["frequency"], "weekly"); assert_eq!(rule["until"], "2025-06-30T23:59:59"); let by_day = rule["byDay"].as_array().unwrap(); diff --git a/src/exchange/ews/retry.rs b/src/exchange/ews/retry.rs new file mode 100644 index 0000000..70a4e26 --- /dev/null +++ b/src/exchange/ews/retry.rs @@ -0,0 +1,170 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::time::Duration; + +use crate::exchange::ews::types::ResponseCode; + +#[derive(Debug, Clone)] +pub enum FaultDisposition { + Fatal, + Auth, + VersionError, + Retryable { delay: Option }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ItemAction { + Success, + Vanished, + Skip, + Retryable, + RetryDiscardSyncState, +} + +impl ResponseCode { + pub fn fault_disposition(&self) -> FaultDisposition { + match self { + ResponseCode::ServerBusy { back_off_ms } => FaultDisposition::Retryable { + delay: back_off_ms.map(Duration::from_millis), + }, + ResponseCode::TimeoutExpired + | ResponseCode::InternalServerTransientError + | ResponseCode::MailboxStoreUnavailable + | ResponseCode::ConnectionFailed + | ResponseCode::AdUnavailable + | ResponseCode::BatchProcessingStopped => FaultDisposition::Retryable { delay: None }, + ResponseCode::AuthenticationRequired => FaultDisposition::Auth, + ResponseCode::InvalidServerVersion + | ResponseCode::IncorrectSchemaVersion + | ResponseCode::InvalidRequest + | ResponseCode::InvalidSchemaVersionForMailboxVersion => FaultDisposition::VersionError, + _ => FaultDisposition::Fatal, + } + } + + pub fn item_action(&self) -> ItemAction { + match self { + ResponseCode::NoError => ItemAction::Success, + ResponseCode::ItemNotFound | ResponseCode::AttachmentNotFound => ItemAction::Vanished, + ResponseCode::ServerBusy { .. } + | ResponseCode::TimeoutExpired + | ResponseCode::InternalServerTransientError + | ResponseCode::MailboxStoreUnavailable + | ResponseCode::ConnectionFailed + | ResponseCode::AdUnavailable + | ResponseCode::BatchProcessingStopped => ItemAction::Retryable, + ResponseCode::InvalidSyncStateData => ItemAction::RetryDiscardSyncState, + ResponseCode::AccessDenied + | ResponseCode::InvalidIdMalformed + | ResponseCode::MimeContentConversionFailed + | ResponseCode::ImpersonationFailed + | ResponseCode::AuthenticationRequired + | ResponseCode::InvalidServerVersion + | ResponseCode::IncorrectSchemaVersion + | ResponseCode::InvalidRequest + | ResponseCode::InvalidSchemaVersionForMailboxVersion + | ResponseCode::Other(_) => ItemAction::Skip, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::net::retry::Disposition; + + #[test] + fn server_busy_with_backoff_is_retryable_with_delay() { + let code = ResponseCode::ServerBusy { + back_off_ms: Some(297749), + }; + match code.fault_disposition() { + FaultDisposition::Retryable { delay: Some(d) } => { + assert_eq!(d, Duration::from_millis(297749)); + } + _ => panic!("expected retryable with delay"), + } + } + + #[test] + fn item_responses_are_classified() { + assert_eq!(ResponseCode::NoError.item_action(), ItemAction::Success); + assert_eq!( + ResponseCode::ItemNotFound.item_action(), + ItemAction::Vanished + ); + assert_eq!( + ResponseCode::AttachmentNotFound.item_action(), + ItemAction::Vanished + ); + assert_eq!(ResponseCode::AccessDenied.item_action(), ItemAction::Skip); + assert_eq!( + ResponseCode::InvalidSyncStateData.item_action(), + ItemAction::RetryDiscardSyncState + ); + assert_eq!( + ResponseCode::ServerBusy { back_off_ms: None }.item_action(), + ItemAction::Retryable + ); + } + + #[test] + fn http_status_table() { + assert_eq!(Disposition::of_status(429), Disposition::Retry); + assert_eq!(Disposition::of_status(503), Disposition::Retry); + assert_eq!(Disposition::of_status(500), Disposition::Fatal); + assert_eq!(Disposition::of_status(401), Disposition::Fatal); + } + + #[test] + fn schema_version_codes_request_a_downgrade() { + for code in [ + ResponseCode::InvalidServerVersion, + ResponseCode::IncorrectSchemaVersion, + ResponseCode::InvalidRequest, + ResponseCode::InvalidSchemaVersionForMailboxVersion, + ] { + assert!(matches!( + code.fault_disposition(), + FaultDisposition::VersionError + )); + } + } + + #[test] + fn transient_codes_are_retryable_at_item_and_fault_level() { + for code in [ + ResponseCode::InternalServerTransientError, + ResponseCode::MailboxStoreUnavailable, + ResponseCode::ConnectionFailed, + ResponseCode::AdUnavailable, + ResponseCode::BatchProcessingStopped, + ] { + assert_eq!(code.item_action(), ItemAction::Retryable); + assert!(matches!( + code.fault_disposition(), + FaultDisposition::Retryable { .. } + )); + } + } + + #[test] + fn mime_content_conversion_failure_skips_item() { + assert_eq!( + ResponseCode::MimeContentConversionFailed.item_action(), + ItemAction::Skip + ); + } + + #[test] + fn other_unknown_codes_skip_at_item_level() { + assert_eq!( + ResponseCode::Other("ErrorWhoKnows".to_owned()).item_action(), + ItemAction::Skip + ); + } +} diff --git a/src/exchange_ews/soap.rs b/src/exchange/ews/soap.rs similarity index 51% rename from src/exchange_ews/soap.rs rename to src/exchange/ews/soap.rs index 765e239..35ecb69 100644 --- a/src/exchange_ews/soap.rs +++ b/src/exchange/ews/soap.rs @@ -4,7 +4,14 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use crate::exchange_ews::types::ServerVersion; +use std::fmt; +use std::fmt::Write; + +use crate::exchange::ews::types::ServerVersion; +use crate::exchange::ews::xml::EwsRequest; +use crate::net::xml::XmlEscape; + +const ENVELOPE_CAPACITY: usize = 512; pub const NS_SOAP: &str = "http://schemas.xmlsoap.org/soap/envelope/"; pub const NS_TYPES: &str = "http://schemas.microsoft.com/exchange/services/2006/types"; @@ -27,54 +34,51 @@ impl Default for EnvelopeOptions<'_> { } } -pub fn wrap_envelope(opts: EnvelopeOptions<'_>, body: &str) -> String { - let mut out = String::with_capacity(body.len() + 512); - out.push_str(""); - out.push_str(""); - out.push_str(""); - if let Some(smtp) = opts.impersonated_smtp { - out.push_str(""); - write_escaped(&mut out, smtp); - out.push_str(""); - } - out.push_str(""); - out.push_str(body); - out.push_str(""); - out -} - -pub fn soap_action(operation: &str) -> String { - format!("\"{SOAP_ACTION_BASE}/{operation}\"") -} - -pub fn write_escaped(out: &mut String, s: &str) { - for ch in s.chars() { - match ch { - '&' => out.push_str("&"), - '<' => out.push_str("<"), - '>' => out.push_str(">"), - '"' => out.push_str("""), - '\'' => out.push_str("'"), - _ => out.push(ch), +impl EnvelopeOptions<'_> { + pub fn wrap(&self, request: &EwsRequest<'_>) -> String { + let mut out = String::with_capacity(request.size_hint() + ENVELOPE_CAPACITY); + let _ = write!( + out, + "", + self.version.as_str() + ); + if let Some(smtp) = self.impersonated_smtp { + let _ = write!( + out, + "{}", + XmlEscape(smtp) + ); } + out.push_str(""); + request.write_into(&mut out); + out.push_str(""); + out + } +} + +#[derive(Debug, Clone, Copy)] +pub struct SoapAction<'a>(pub &'a str); + +impl fmt::Display for SoapAction<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "\"{SOAP_ACTION_BASE}/{}\"", self.0) } } #[cfg(test)] mod tests { use super::*; + use crate::exchange::ews::types::DistinguishedFolderId; + use crate::exchange::ews::xml::{FolderRef, Traversal}; + + const FIND_ROOT: EwsRequest<'static> = EwsRequest::FindFolder { + parent: FolderRef::Distinguished(DistinguishedFolderId::MsgFolderRoot), + traversal: Traversal::Shallow, + }; #[test] fn envelope_has_canonical_namespaces() { - let env = wrap_envelope(EnvelopeOptions::default(), ""); + let env = EnvelopeOptions::default().wrap(&FIND_ROOT); assert!(env.contains("xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\"")); assert!( env.contains("xmlns:t=\"http://schemas.microsoft.com/exchange/services/2006/types\"") @@ -85,7 +89,7 @@ mod tests { ) ); assert!(env.contains("")); - assert!(env.contains("")); + assert!(env.contains("")); } #[test] @@ -94,7 +98,7 @@ mod tests { version: ServerVersion::Exchange2016, impersonated_smtp: Some("alice@contoso.com"), }; - let env = wrap_envelope(opts, ""); + let env = opts.wrap(&FIND_ROOT); assert!(env.contains("")); assert!(env.contains("alice@contoso.com")); assert!(env.contains("Version=\"Exchange2016\"")); @@ -103,15 +107,8 @@ mod tests { #[test] fn soap_action_quotes_operation() { assert_eq!( - soap_action("FindItem"), + SoapAction("FindItem").to_string(), "\"http://schemas.microsoft.com/exchange/services/2006/messages/FindItem\"" ); } - - #[test] - fn write_escaped_handles_metachars() { - let mut s = String::new(); - write_escaped(&mut s, ""); - assert_eq!(s, "<a&b>"); - } } diff --git a/src/exchange/ews/srv.rs b/src/exchange/ews/srv.rs new file mode 100644 index 0000000..fac3c1a --- /dev/null +++ b/src/exchange/ews/srv.rs @@ -0,0 +1,283 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::cmp::Reverse; +use std::fs; +use std::net::UdpSocket; +use std::str; +use std::time::Duration; + +const RESOLV_CONF: &str = "/etc/resolv.conf"; +const DNS_PORT: u16 = 53; +const DNS_TIMEOUT: Duration = Duration::from_secs(3); +const TXN_ID: u16 = 0x1234; +const FLAGS_RECURSION_DESIRED: u16 = 0x0100; +const TYPE_SRV: u16 = 33; +const CLASS_IN: u16 = 1; +const HEADER_LEN: usize = 12; +const QUESTION_TAIL: usize = 4; +const RR_FIXED_LEN: usize = 10; +const SRV_MIN_RDATA: usize = 7; +const UDP_PAYLOAD: usize = 1500; +const POINTER_TAG: u8 = 0xC0; +const POINTER_MASK: u8 = 0x3F; +const MAX_LABEL: usize = 63; +const MAX_NAME: usize = 255; +const MAX_POINTER_HOPS: usize = 8; +const MAX_RECORDS: usize = 64; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SrvRecord { + pub priority: u16, + pub weight: u16, + pub port: u16, + pub target: String, +} + +#[derive(Debug, Clone, Copy)] +struct Message<'a>(&'a [u8]); + +impl SrvRecord { + pub fn lookup(qname: &str) -> Vec { + let Some(query) = SrvRecord::query(qname) else { + return Vec::new(); + }; + let mut out = SrvRecord::nameservers() + .into_iter() + .find_map(|ns| SrvRecord::ask(&ns, &query)) + .unwrap_or_default(); + SrvRecord::sort_by_preference(&mut out); + out + } + + fn sort_by_preference(records: &mut [SrvRecord]) { + records.sort_by_key(|r| (r.priority, Reverse(r.weight))); + } + + fn ask(nameserver: &str, query: &[u8]) -> Option> { + let socket = UdpSocket::bind("0.0.0.0:0").ok()?; + let _ = socket.set_read_timeout(Some(DNS_TIMEOUT)); + socket.send_to(query, (nameserver, DNS_PORT)).ok()?; + let mut buf = [0u8; UDP_PAYLOAD]; + let (n, _) = socket.recv_from(&mut buf).ok()?; + Message(buf.get(..n)?).srv_records() + } + + fn nameservers() -> Vec { + let Ok(text) = fs::read_to_string(RESOLV_CONF) else { + return Vec::new(); + }; + text.lines() + .filter_map(|line| { + let line = line.split('#').next().unwrap_or_default().trim(); + let ns = line.strip_prefix("nameserver")?.trim(); + (!ns.is_empty()).then(|| ns.to_owned()) + }) + .collect() + } + + fn query(qname: &str) -> Option> { + let mut query = Vec::with_capacity(HEADER_LEN + qname.len() + 2 + QUESTION_TAIL); + query.extend_from_slice(&TXN_ID.to_be_bytes()); + query.extend_from_slice(&FLAGS_RECURSION_DESIRED.to_be_bytes()); + query.extend_from_slice(&1u16.to_be_bytes()); + query.extend_from_slice(&[0u8; 6]); + for label in qname.trim_end_matches('.').split('.') { + if label.is_empty() { + continue; + } + if label.len() > MAX_LABEL { + return None; + } + query.push(u8::try_from(label.len()).ok()?); + query.extend_from_slice(label.as_bytes()); + } + query.push(0); + query.extend_from_slice(&TYPE_SRV.to_be_bytes()); + query.extend_from_slice(&CLASS_IN.to_be_bytes()); + Some(query) + } +} + +impl<'a> Message<'a> { + fn u16_at(self, at: usize) -> Option { + let bytes = self.0.get(at..at.checked_add(2)?)?; + let [hi, lo] = bytes else { return None }; + Some(u16::from_be_bytes([*hi, *lo])) + } + + fn skip_name(self, mut cursor: usize) -> Option { + loop { + let len = *self.0.get(cursor)?; + if len == 0 { + return cursor.checked_add(1); + } + if len & POINTER_TAG == POINTER_TAG { + return cursor.checked_add(2); + } + cursor = cursor.checked_add(1 + usize::from(len))?; + } + } + + fn read_name(self, mut cursor: usize) -> Option { + let mut out = String::new(); + let mut hops = 0; + loop { + let len = *self.0.get(cursor)?; + if len == 0 { + return Some(out); + } + if len & POINTER_TAG == POINTER_TAG { + let low = *self.0.get(cursor.checked_add(1)?)?; + hops += 1; + if hops > MAX_POINTER_HOPS { + return None; + } + cursor = usize::from(len & POINTER_MASK) << 8 | usize::from(low); + continue; + } + let start = cursor.checked_add(1)?; + let end = start.checked_add(usize::from(len))?; + let label = str::from_utf8(self.0.get(start..end)?).ok()?; + if out.len() + label.len() + 1 > MAX_NAME { + return None; + } + if !out.is_empty() { + out.push('.'); + } + out.push_str(label); + cursor = end; + } + } + + fn srv_records(self) -> Option> { + let questions = self.u16_at(4)?; + let answers = self.u16_at(6)?; + let mut cursor = HEADER_LEN; + for _ in 0..questions { + cursor = self.skip_name(cursor)?.checked_add(QUESTION_TAIL)?; + if cursor > self.0.len() { + return None; + } + } + let mut out = Vec::new(); + for _ in 0..answers { + cursor = self.skip_name(cursor)?; + let rtype = self.u16_at(cursor)?; + let rdlength = usize::from(self.u16_at(cursor.checked_add(8)?)?); + cursor = cursor.checked_add(RR_FIXED_LEN)?; + let rdata_end = cursor.checked_add(rdlength)?; + if rdata_end > self.0.len() { + return None; + } + if rtype == TYPE_SRV && rdlength >= SRV_MIN_RDATA && out.len() < MAX_RECORDS { + out.push(SrvRecord { + priority: self.u16_at(cursor)?, + weight: self.u16_at(cursor + 2)?, + port: self.u16_at(cursor + 4)?, + target: self.read_name(cursor + 6)?, + }); + } + cursor = rdata_end; + } + Some(out) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn answer(target_labels: &[&str], port: u16) -> Vec { + let mut msg = SrvRecord::query("_autodiscover._tcp.example.com").unwrap(); + msg[6..8].copy_from_slice(&1u16.to_be_bytes()); + msg.extend_from_slice(&[0xC0, 12]); + msg.extend_from_slice(&TYPE_SRV.to_be_bytes()); + msg.extend_from_slice(&CLASS_IN.to_be_bytes()); + msg.extend_from_slice(&300u32.to_be_bytes()); + let mut rdata = Vec::new(); + rdata.extend_from_slice(&10u16.to_be_bytes()); + rdata.extend_from_slice(&5u16.to_be_bytes()); + rdata.extend_from_slice(&port.to_be_bytes()); + for label in target_labels { + rdata.push(u8::try_from(label.len()).unwrap()); + rdata.extend_from_slice(label.as_bytes()); + } + rdata.push(0); + msg.extend_from_slice(&u16::try_from(rdata.len()).unwrap().to_be_bytes()); + msg.extend_from_slice(&rdata); + msg + } + + // Records with equal priority and weight keep their DNS answer order. + #[test] + fn equal_preference_records_keep_answer_order() { + let record = |priority, weight, target: &str| SrvRecord { + priority, + weight, + port: 443, + target: target.to_owned(), + }; + let mut records = vec![ + record(10, 5, "b.example.com"), + record(10, 5, "a.example.com"), + record(0, 1, "first.example.com"), + record(10, 9, "heavy.example.com"), + record(10, 5, "c.example.com"), + ]; + SrvRecord::sort_by_preference(&mut records); + let targets: Vec<&str> = records.iter().map(|r| r.target.as_str()).collect(); + assert_eq!( + targets, + [ + "first.example.com", + "heavy.example.com", + "b.example.com", + "a.example.com", + "c.example.com" + ] + ); + } + + #[test] + fn parses_one_srv_answer() { + let msg = answer(&["mail", "example", "com"], 8443); + assert_eq!( + Message(&msg).srv_records().unwrap(), + vec![SrvRecord { + priority: 10, + weight: 5, + port: 8443, + target: "mail.example.com".to_owned(), + }] + ); + } + + #[test] + fn truncated_answer_is_rejected() { + let msg = answer(&["mail", "example", "com"], 443); + for cut in HEADER_LEN..msg.len() { + assert!(Message(&msg[..cut]).srv_records().is_none(), "cut {cut}"); + } + } + + #[test] + fn pointer_loop_is_bounded() { + let mut msg = answer(&[], 443); + let at = msg.len() - 1; + msg.pop(); + msg.extend_from_slice(&[0xC0, u8::try_from(at).unwrap()]); + msg[at - 8..at - 6].copy_from_slice(&8u16.to_be_bytes()); + assert!(Message(&msg).srv_records().is_none()); + } + + #[test] + fn overlong_label_is_not_queried() { + let label = "a".repeat(MAX_LABEL + 1); + assert!(SrvRecord::query(&format!("{label}.example.com")).is_none()); + assert!(SrvRecord::query("_autodiscover._tcp.example.com").is_some()); + } +} diff --git a/src/exchange_ews/types.rs b/src/exchange/ews/types.rs similarity index 80% rename from src/exchange_ews/types.rs rename to src/exchange/ews/types.rs index 0eb610a..c77deeb 100644 --- a/src/exchange_ews/types.rs +++ b/src/exchange/ews/types.rs @@ -5,6 +5,9 @@ */ use std::fmt; +use std::iter; + +pub use crate::exchange::mailbox::MailboxKind; #[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] pub struct ItemId { @@ -36,22 +39,7 @@ impl FolderId { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum MailboxKind { - Primary, - Archive, - PublicFolders, -} - impl MailboxKind { - pub fn as_str(self) -> &'static str { - match self { - MailboxKind::Primary => "primary", - MailboxKind::Archive => "archive", - MailboxKind::PublicFolders => "public-folders", - } - } - pub fn distinguished_root(self) -> DistinguishedFolderId { match self { MailboxKind::Primary => DistinguishedFolderId::MsgFolderRoot, @@ -142,42 +130,67 @@ impl ServerVersion { } } -const NON_MAIL_ITEM_CLASSES: [&str; 6] = [ - "ipm.contact", - "ipm.distlist", - "ipm.appointment", - "ipm.task", - "ipm.stickynote", - "ipm.activity", -]; +#[derive(Debug, Clone, Copy)] +pub struct ItemClass<'a>(pub &'a str); -pub fn is_non_mail_item_class(class: &str) -> bool { - let class = class.trim(); - NON_MAIL_ITEM_CLASSES.iter().any(|prefix| { - class - .get(..prefix.len()) - .is_some_and(|head| head.eq_ignore_ascii_case(prefix)) - && class - .get(prefix.len()..) - .is_some_and(|rest| rest.is_empty() || rest.starts_with('.')) - }) -} +impl<'a> ItemClass<'a> { + fn prefixes(self) -> impl Iterator { + iter::successors(Some(self.0.trim()), |class| { + class.rsplit_once('.').map(|(head, _)| head) + }) + } -const SKIPPED_CONTAINER_CLASSES: [&str; 8] = [ - "ipf.task", - "ipf.journal", - "ipf.stickynote", - "ipf.configuration", - "ipf.storeitem", - "ipf.skypeteams", - "ipf.files", - "ipf.note.outlookhomepage", -]; + pub fn is_non_mail(self) -> bool { + self.prefixes().any(|prefix| { + hashify::set_ignore_case!( + prefix.as_bytes(), + "ipm.contact", + "ipm.distlist", + "ipm.appointment", + "ipm.task", + "ipm.stickynote", + "ipm.activity", + ) + }) + } -fn class_matches(lower_class: &str, lower_prefix: &str) -> bool { - lower_class == lower_prefix - || (lower_class.starts_with(lower_prefix) - && lower_class[lower_prefix.len()..].starts_with('.')) + fn container(self) -> Option { + self.prefixes().find_map(|prefix| { + hashify::map_ignore_case!( + prefix.as_bytes(), + FolderClass, + "ipf.appointment" => FolderClass::Calendar, + "ipf.contact" => FolderClass::Contacts, + "ipf.task" => FolderClass::Skipped, + "ipf.journal" => FolderClass::Skipped, + "ipf.stickynote" => FolderClass::Skipped, + "ipf.configuration" => FolderClass::Skipped, + "ipf.storeitem" => FolderClass::Skipped, + "ipf.skypeteams" => FolderClass::Skipped, + "ipf.files" => FolderClass::Skipped, + "ipf.note.outlookhomepage" => FolderClass::Skipped, + ) + .copied() + }) + } + + fn item(self) -> Option { + self.prefixes().find_map(|prefix| { + hashify::map_ignore_case!( + prefix.as_bytes(), + FolderClass, + "ipm.contact" => FolderClass::Contacts, + "ipm.distlist" => FolderClass::Contacts, + "ipm.appointment" => FolderClass::Calendar, + ) + .copied() + }) + } + + fn is_note(self) -> bool { + self.prefixes() + .any(|prefix| prefix.eq_ignore_ascii_case("ipf.note")) + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -190,37 +203,19 @@ pub enum FolderClass { impl FolderClass { pub fn from_ipf(class: &str) -> FolderClass { - let lower = class.trim().to_ascii_lowercase(); - if class_matches(&lower, "ipf.appointment") { - FolderClass::Calendar - } else if class_matches(&lower, "ipf.contact") { - FolderClass::Contacts - } else if SKIPPED_CONTAINER_CLASSES - .iter() - .any(|p| class_matches(&lower, p)) - { - FolderClass::Skipped - } else { - FolderClass::Mail - } + ItemClass(class).container().unwrap_or(FolderClass::Mail) } pub fn from_item_classes<'a>(classes: impl IntoIterator) -> FolderClass { - for class in classes { - let lower = class.trim().to_ascii_lowercase(); - if class_matches(&lower, "ipm.contact") || class_matches(&lower, "ipm.distlist") { - return FolderClass::Contacts; - } - if class_matches(&lower, "ipm.appointment") { - return FolderClass::Calendar; - } - } - FolderClass::Mail + classes + .into_iter() + .find_map(|class| ItemClass(class).item()) + .unwrap_or(FolderClass::Mail) } pub fn is_mail_fallback(class: &str) -> bool { - let lower = class.trim().to_ascii_lowercase(); - FolderClass::from_ipf(class) == FolderClass::Mail && !class_matches(&lower, "ipf.note") + let class = ItemClass(class); + class.container().is_none() && !class.is_note() } } @@ -234,13 +229,15 @@ pub enum CalendarItemType { impl CalendarItemType { pub fn parse(value: &str) -> Option { - match value { - "Single" => Some(CalendarItemType::Single), - "RecurringMaster" => Some(CalendarItemType::RecurringMaster), - "Occurrence" => Some(CalendarItemType::Occurrence), - "Exception" => Some(CalendarItemType::Exception), - _ => None, - } + hashify::map!( + value.as_bytes(), + CalendarItemType, + "Single" => CalendarItemType::Single, + "RecurringMaster" => CalendarItemType::RecurringMaster, + "Occurrence" => CalendarItemType::Occurrence, + "Exception" => CalendarItemType::Exception, + ) + .copied() } } @@ -270,8 +267,20 @@ pub enum ResponseCode { } impl ResponseCode { + pub fn restarts_full_sync(&self) -> bool { + matches!( + self, + ResponseCode::InvalidSyncStateData + | ResponseCode::InvalidIdMalformed + | ResponseCode::AccessDenied + | ResponseCode::Other(_) + ) + } + pub fn parse(code: &str) -> ResponseCode { - match code { + hashify::map!( + code.as_bytes(), + ResponseCode, "NoError" => ResponseCode::NoError, "ErrorItemNotFound" => ResponseCode::ItemNotFound, "ErrorAccessDenied" => ResponseCode::AccessDenied, @@ -291,11 +300,10 @@ impl ResponseCode { "ErrorInvalidServerVersion" => ResponseCode::InvalidServerVersion, "ErrorIncorrectSchemaVersion" => ResponseCode::IncorrectSchemaVersion, "ErrorInvalidRequest" => ResponseCode::InvalidRequest, - "ErrorInvalidSchemaVersionForMailboxVersion" => { - ResponseCode::InvalidSchemaVersionForMailboxVersion - } - other => ResponseCode::Other(other.to_owned()), - } + "ErrorInvalidSchemaVersionForMailboxVersion" => ResponseCode::InvalidSchemaVersionForMailboxVersion, + ) + .cloned() + .unwrap_or_else(|| ResponseCode::Other(code.to_owned())) } pub fn as_wire(&self) -> &str { @@ -348,7 +356,7 @@ mod tests { "ipm.stickynote", "IPM.Activity", ] { - assert!(is_non_mail_item_class(class), "{class}"); + assert!(ItemClass(class).is_non_mail(), "{class}"); } for class in [ "IPM.Note", @@ -360,7 +368,7 @@ mod tests { "IPM.Contactless", "", ] { - assert!(!is_non_mail_item_class(class), "{class}"); + assert!(!ItemClass(class).is_non_mail(), "{class}"); } } diff --git a/src/exchange/ews/xml.rs b/src/exchange/ews/xml.rs new file mode 100644 index 0000000..d4a7d55 --- /dev/null +++ b/src/exchange/ews/xml.rs @@ -0,0 +1,642 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fmt::Write; + +use crate::exchange::ews::types::{DistinguishedFolderId, FolderId, ItemId, ServerVersion}; +use crate::net::xml::XmlEscape; + +#[derive(Debug, Clone, Copy)] +pub enum FolderRef<'a> { + Distinguished(DistinguishedFolderId), + DistinguishedIn(DistinguishedFolderId, &'a str), + Concrete(&'a FolderId), +} + +impl<'a> FolderRef<'a> { + pub fn distinguished(id: DistinguishedFolderId, mailbox: Option<&'a str>) -> FolderRef<'a> { + match mailbox { + Some(mailbox) => FolderRef::DistinguishedIn(id, mailbox), + None => FolderRef::Distinguished(id), + } + } +} + +#[derive(Debug, Clone, Copy)] +pub enum FolderShape { + IdOnly, +} + +impl FolderShape { + fn as_str(self) -> &'static str { + match self { + FolderShape::IdOnly => "IdOnly", + } + } +} + +#[derive(Debug, Clone, Copy)] +pub enum Traversal { + Shallow, + Deep, +} + +impl Traversal { + fn as_str(self) -> &'static str { + match self { + Traversal::Shallow => "Shallow", + Traversal::Deep => "Deep", + } + } +} + +const ITEM_CLASS_PROPERTY: &str = + ""; + +const FIND_FOLDER_PROPERTIES: &str = "Default\ + \ + \ + \ + \ + \ + \ + \ + "; + +const MESSAGE_SHAPE_HEAD: &str = "IdOnly\ + true\ + Best\ + \ + \ + \ + \ + \ + "; + +const SORT_BY_CREATION: &str = "\ + "; + +const REQUEST_CAPACITY: usize = 512; +const ITEM_ID_CAPACITY: usize = 128; +const SHORT_ID_CAPACITY: usize = 64; + +#[derive(Debug, Clone, Copy)] +pub enum ItemShape { + Message, + CalendarItem, + Contact, +} + +#[derive(Debug, Clone, Copy)] +pub enum EwsRequest<'a> { + FindFolder { + parent: FolderRef<'a>, + traversal: Traversal, + }, + GetFolder { + folders: &'a [FolderRef<'a>], + shape: FolderShape, + }, + FindItem { + parent: FolderRef<'a>, + traversal: Traversal, + offset: u32, + page_size: u32, + }, + SyncFolderItems { + folder: &'a FolderId, + sync_state: &'a str, + max_changes: u32, + version: ServerVersion, + }, + GetItem { + shape: ItemShape, + ids: &'a [ItemId], + version: ServerVersion, + }, + GetAttachment { + ids: &'a [&'a str], + }, +} + +impl EwsRequest<'_> { + pub fn operation(&self) -> &'static str { + match self { + EwsRequest::FindFolder { .. } => "FindFolder", + EwsRequest::GetFolder { .. } => "GetFolder", + EwsRequest::FindItem { .. } => "FindItem", + EwsRequest::SyncFolderItems { .. } => "SyncFolderItems", + EwsRequest::GetItem { .. } => "GetItem", + EwsRequest::GetAttachment { .. } => "GetAttachment", + } + } + + pub fn size_hint(&self) -> usize { + REQUEST_CAPACITY + + match self { + EwsRequest::GetItem { ids, .. } => ids.len() * ITEM_ID_CAPACITY, + EwsRequest::GetFolder { folders, .. } => folders.len() * SHORT_ID_CAPACITY, + EwsRequest::GetAttachment { ids } => ids.len() * SHORT_ID_CAPACITY, + _ => 0, + } + } + + pub fn body(&self) -> String { + let mut out = String::with_capacity(self.size_hint()); + self.write_into(&mut out); + out + } + + pub fn write_into(&self, out: &mut String) { + match *self { + EwsRequest::FindFolder { parent, traversal } => { + let _ = write!( + out, + "{FIND_FOLDER_PROPERTIES}", + traversal.as_str() + ); + parent.write_into(out); + out.push_str(""); + } + EwsRequest::GetFolder { folders, shape } => { + let _ = write!( + out, + "{}", + shape.as_str() + ); + for folder in folders { + folder.write_into(out); + } + out.push_str(""); + } + EwsRequest::FindItem { + parent, + traversal, + offset, + page_size, + } => { + let _ = write!( + out, + "IdOnly{ITEM_CLASS_PROPERTY}\ + \ + {SORT_BY_CREATION}", + traversal.as_str() + ); + parent.write_into(out); + out.push_str(""); + } + EwsRequest::SyncFolderItems { + folder, + sync_state, + max_changes, + version, + } => { + out.push_str("IdOnly"); + out.push_str(ITEM_CLASS_PROPERTY); + out.push_str(""); + folder.write_into(out); + let _ = write!( + out, + "{}{max_changes}", + XmlEscape(sync_state) + ); + if version >= ServerVersion::Exchange2010 { + out.push_str("NormalItems"); + } + out.push_str(""); + } + EwsRequest::GetItem { + shape, + ids, + version, + } => { + out.push_str(""); + shape.write_into(out, version); + out.push_str(""); + for id in ids { + id.write_into(out); + } + out.push_str(""); + } + EwsRequest::GetAttachment { ids } => { + out.push_str( + "\ + true\ + Best\ + ", + ); + for id in ids { + let _ = write!(out, "", XmlEscape(id)); + } + out.push_str(""); + } + } + } +} + +impl ItemShape { + fn write_into(self, out: &mut String, version: ServerVersion) { + match self { + ItemShape::Message => { + out.push_str(MESSAGE_SHAPE_HEAD); + if version >= ServerVersion::Exchange2013 { + out.push_str(""); + } else { + out.push_str( + "", + ); + } + out.push_str(""); + } + ItemShape::CalendarItem => { + out.push_str( + "IdOnlyBest", + ); + for field in CALENDAR_FIELDS { + let _ = write!(out, ""); + } + if version >= ServerVersion::Exchange2010 { + out.push_str(""); + out.push_str(""); + } + out.push_str(""); + } + ItemShape::Contact => out.push_str("AllProperties"), + } + } +} + +const CALENDAR_FIELDS: &[&str] = &[ + "calendar:UID", + "calendar:RecurrenceId", + "calendar:Start", + "calendar:End", + "calendar:OriginalStart", + "calendar:IsAllDayEvent", + "calendar:LegacyFreeBusyStatus", + "calendar:Location", + "calendar:IsRecurring", + "calendar:CalendarItemType", + "calendar:Organizer", + "calendar:RequiredAttendees", + "calendar:OptionalAttendees", + "calendar:Resources", + "calendar:IsOnlineMeeting", + "calendar:MeetingWorkspaceUrl", + "calendar:NetShowUrl", + "calendar:MyResponseType", + "item:ReminderIsSet", + "item:ReminderMinutesBeforeStart", + "calendar:Recurrence", + "calendar:ModifiedOccurrences", + "calendar:DeletedOccurrences", + "calendar:Duration", + "item:Subject", + "item:Body", + "item:Categories", + "item:Sensitivity", + "item:Attachments", + "item:DateTimeCreated", + "item:LastModifiedTime", + "item:ParentFolderId", +]; + +impl FolderId { + fn write_into(&self, out: &mut String) { + let _ = write!(out, ""); + } +} + +impl ItemId { + fn write_into(&self, out: &mut String) { + let _ = write!(out, ""); + } +} + +impl FolderRef<'_> { + fn write_into(self, out: &mut String) { + match self { + FolderRef::Distinguished(d) => { + let _ = write!(out, "", d.as_str()); + } + FolderRef::DistinguishedIn(d, mailbox) => { + let _ = write!( + out, + "{}", + d.as_str(), + XmlEscape(mailbox) + ); + } + FolderRef::Concrete(f) => f.write_into(out), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn find_folder_uses_deep_traversal_for_msgfolderroot() { + let body = EwsRequest::FindFolder { + parent: FolderRef::Distinguished(DistinguishedFolderId::MsgFolderRoot), + traversal: Traversal::Deep, + } + .body(); + assert!(body.contains("Traversal=\"Deep\"")); + assert!(body.contains("")); + assert!(body.contains("folder:FolderClass")); + assert!(body.contains("folder:DisplayName")); + assert!(body.contains("folder:ParentFolderId")); + assert!(body.contains("PropertyTag=\"0x10F4\" PropertyType=\"Boolean\"")); + } + + #[test] + fn find_folder_archive_uses_archiveroot() { + let body = EwsRequest::FindFolder { + parent: FolderRef::Distinguished(DistinguishedFolderId::ArchiveRoot), + traversal: Traversal::Deep, + } + .body(); + assert!(body.contains("")); + } + + #[test] + fn distinguished_folder_in_named_mailbox_carries_escaped_address() { + let body = EwsRequest::FindFolder { + parent: FolderRef::distinguished( + DistinguishedFolderId::MsgFolderRoot, + Some("b&o@corp.example"), + ), + traversal: Traversal::Deep, + } + .body(); + assert!(body.contains( + "\ + b&o@corp.example\ + " + )); + let body = EwsRequest::FindFolder { + parent: FolderRef::distinguished(DistinguishedFolderId::MsgFolderRoot, None), + traversal: Traversal::Deep, + } + .body(); + assert!(body.contains("")); + } + + #[test] + fn find_item_paginates_with_offset_and_page_size() { + let folder = FolderId::new("FID", "FCK"); + let body = EwsRequest::FindItem { + parent: FolderRef::Concrete(&folder), + traversal: Traversal::Shallow, + offset: 200, + page_size: 50, + } + .body(); + assert!(body.contains("Offset=\"200\"")); + assert!(body.contains("MaxEntriesReturned=\"50\"")); + assert!(body.contains("")); + } + + #[test] + fn find_item_sorts_by_creation_time_for_stable_paging() { + let folder = FolderId::new("FID", "FCK"); + let body = EwsRequest::FindItem { + parent: FolderRef::Concrete(&folder), + traversal: Traversal::Shallow, + offset: 0, + page_size: 50, + } + .body(); + assert!(body.contains("")); + assert!(body.contains("Order=\"Ascending\"")); + assert!(body.contains("item:DateTimeCreated")); + let sort_at = body.find("").unwrap(); + let parents_at = body.find("").unwrap(); + assert!( + sort_at < parents_at, + "SortOrder must precede ParentFolderIds" + ); + } + + #[test] + fn get_item_message_shape_requests_mime() { + let ids = vec![ItemId::new("I1", "CK1"), ItemId::new("I2", "")]; + let body = EwsRequest::GetItem { + shape: ItemShape::Message, + ids: &ids, + version: ServerVersion::Exchange2013Sp1, + } + .body(); + assert!(body.contains("true")); + assert!(body.contains("")); + assert!(body.contains("")); + } + + #[test] + fn item_flag_requested_only_on_exchange_2013_and_later() { + let ids = [ItemId::new("I1", "")]; + let modern = EwsRequest::GetItem { + shape: ItemShape::Message, + ids: &ids, + version: ServerVersion::Exchange2013Sp1, + } + .body(); + assert!(modern.contains("item:Flag")); + let legacy = EwsRequest::GetItem { + shape: ItemShape::Message, + ids: &ids, + version: ServerVersion::Exchange2010Sp2, + } + .body(); + assert!( + !legacy.contains("item:Flag"), + "item:Flag is an Exchange 2013 schema addition; must be omitted on 2010" + ); + assert!( + legacy.contains("PropertyTag=\"0x1090\""), + "on pre-2013 the follow-up flag must be fetched via the PidTagFlagStatus extended property" + ); + assert!( + !modern.contains("0x1090"), + "the extended-property fallback is only used when item:Flag is unavailable" + ); + assert!(legacy.contains("item:DateTimeReceived")); + assert!( + modern.contains("\"message:IsRead\"") && !modern.contains("\"item:IsRead\""), + "IsRead is a MessageType property; its FieldURI is message:IsRead" + ); + } + + #[test] + fn get_item_calendar_shape_lists_calendar_fields() { + let body = EwsRequest::GetItem { + shape: ItemShape::CalendarItem, + ids: &[ItemId::new("X", "")], + version: ServerVersion::Exchange2010Sp2, + } + .body(); + assert!(body.contains("calendar:Recurrence")); + assert!(body.contains("calendar:ModifiedOccurrences")); + assert!(body.contains("calendar:DeletedOccurrences")); + assert!(body.contains("calendar:UID")); + assert!(!body.contains("")); + } + + #[test] + fn get_item_contact_shape_requests_all_properties() { + let body = EwsRequest::GetItem { + shape: ItemShape::Contact, + ids: &[ItemId::new("X", "")], + version: ServerVersion::Exchange2013Sp1, + } + .body(); + assert!(body.contains("AllProperties")); + } + + #[test] + fn sync_folder_items_body_carries_state_and_max() { + let folder = FolderId::new("FID", "FCK"); + let body = EwsRequest::SyncFolderItems { + folder: &folder, + sync_state: "STATE", + max_changes: 512, + version: ServerVersion::Exchange2013Sp1, + } + .body(); + assert!(body.contains("")); + assert!(body.contains("STATE")); + assert!(body.contains("512")); + assert!(body.contains("NormalItems")); + assert!(body.contains("FieldURI=\"item:ItemClass\"")); + } + + #[test] + fn find_item_requests_item_class() { + let folder = FolderId::new("FID", "FCK"); + let body = EwsRequest::FindItem { + parent: FolderRef::Concrete(&folder), + traversal: Traversal::Shallow, + offset: 0, + page_size: 10, + } + .body(); + assert!(body.contains( + "IdOnly" + )); + } + + #[test] + fn sync_scope_omitted_below_exchange_2010() { + let folder = FolderId::new("FID", ""); + let modern = EwsRequest::SyncFolderItems { + folder: &folder, + sync_state: "", + max_changes: 100, + version: ServerVersion::Exchange2010, + } + .body(); + assert!(modern.contains("NormalItems")); + let legacy = EwsRequest::SyncFolderItems { + folder: &folder, + sync_state: "", + max_changes: 100, + version: ServerVersion::Exchange2007, + } + .body(); + assert!( + !legacy.contains("SyncScope"), + "SyncScope is an Exchange 2010 addition" + ); + } + + #[test] + fn calendar_shape_requests_attachments_own_response_and_sensitivity() { + let body = EwsRequest::GetItem { + shape: ItemShape::CalendarItem, + ids: &[ItemId::new("X", "")], + version: ServerVersion::Exchange2007, + } + .body(); + for field in [ + "item:Attachments", + "calendar:MyResponseType", + "item:Sensitivity", + ] { + assert!(body.contains(&format!("\"{field}\"")), "{field}"); + } + } + + #[test] + fn calendar_timezone_fields_gated_on_exchange_2010() { + let modern = EwsRequest::GetItem { + shape: ItemShape::CalendarItem, + ids: &[ItemId::new("X", "")], + version: ServerVersion::Exchange2010, + } + .body(); + assert!(modern.contains("calendar:StartTimeZone")); + assert!(modern.contains("calendar:EndTimeZone")); + let legacy = EwsRequest::GetItem { + shape: ItemShape::CalendarItem, + ids: &[ItemId::new("X", "")], + version: ServerVersion::Exchange2007, + } + .body(); + assert!( + !legacy.contains("TimeZone"), + "StartTimeZone/EndTimeZone are Exchange 2010 additions" + ); + } + + #[test] + fn sync_folder_items_empty_state_round_trips() { + let folder = FolderId::new("FID", ""); + let body = EwsRequest::SyncFolderItems { + folder: &folder, + sync_state: "", + max_changes: 100, + version: ServerVersion::Exchange2013Sp1, + } + .body(); + assert!(body.contains("")); + } + + #[test] + fn get_attachment_lists_ids() { + let body = EwsRequest::GetAttachment { + ids: &["AID1", "AID2"], + } + .body(); + assert!(body.contains("")); + assert!(body.contains("")); + assert!(body.contains("true")); + } + + #[test] + fn get_folder_emits_distinguished_ids() { + let body = EwsRequest::GetFolder { + folders: &[ + FolderRef::Distinguished(DistinguishedFolderId::Inbox), + FolderRef::Distinguished(DistinguishedFolderId::SentItems), + ], + shape: FolderShape::IdOnly, + } + .body(); + assert!(body.contains("")); + assert!(body.contains("")); + assert!(body.contains("IdOnly")); + } +} diff --git a/src/exchange_graph/api.rs b/src/exchange/graph/api.rs similarity index 72% rename from src/exchange_graph/api.rs rename to src/exchange/graph/api.rs index a5bf5be..37a3ff3 100644 --- a/src/exchange_graph/api.rs +++ b/src/exchange/graph/api.rs @@ -4,11 +4,16 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +use std::borrow::Cow; +use std::fmt::{self, Write}; +use std::mem; + use serde_json::Value; -use crate::exchange_graph::client::GraphClient; -use crate::exchange_graph::error::GraphError; -use crate::exchange_graph::types::MailboxKind; +use crate::exchange::graph::client::GraphClient; +use crate::exchange::graph::error::GraphError; +use crate::exchange::graph::types::MailboxKind; +use crate::net::encode::SafeSet; pub const DEFAULT_API_BASE: &str = "https://graph.microsoft.com/v1.0"; @@ -26,31 +31,46 @@ pub const EXCEPTION_WINDOW_MAX_DAYS: i64 = 1825; pub const DRIVE_ITEM_SELECT: &str = "id,name,size,folder,file,package,remoteItem,createdDateTime,lastModifiedDateTime"; +const NEXT_LINK: &str = "@odata.nextLink"; + pub const MESSAGE_STATE_SELECT: &str = "id,isRead,isDraft,flag,categories"; #[derive(Debug, Clone)] pub struct Endpoints { - pub api_base: String, - pub user_path: String, + base: Box, +} + +#[derive(Clone, Copy)] +struct Segment<'a>(&'a str); + +impl fmt::Display for Segment<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for byte in self.0.bytes() { + if SafeSet::Unreserved.contains(byte) { + f.write_char(char::from(byte))?; + } else { + write!(f, "%{byte:02X}")?; + } + } + Ok(()) + } } impl Endpoints { pub fn for_me(api_base: &str) -> Endpoints { Endpoints { - api_base: api_base.trim_end_matches('/').to_owned(), - user_path: "/me".to_owned(), + base: format!("{}/me", api_base.trim_end_matches('/')).into_boxed_str(), } } pub fn for_user(api_base: &str, user_id: &str) -> Endpoints { Endpoints { - api_base: api_base.trim_end_matches('/').to_owned(), - user_path: format!("/users/{user_id}"), + base: format!("{}/users/{user_id}", api_base.trim_end_matches('/')).into_boxed_str(), } } - pub fn me_or_user(&self) -> String { - format!("{}{}", self.api_base, self.user_path) + pub fn me_or_user(&self) -> &str { + &self.base } pub fn mail_folders_root(&self, kind: MailboxKind, top: usize) -> String { @@ -74,7 +94,7 @@ impl Endpoints { format!( "{}/mailFolders/{}/childFolders?$top={top}&includeHiddenFolders=true", self.me_or_user(), - url_escape(folder_id) + Segment(folder_id) ) } @@ -82,7 +102,7 @@ impl Endpoints { format!( "{}/mailFolders/{}/messages?$top={top}&$select={MESSAGE_STATE_SELECT}", self.me_or_user(), - url_escape(folder_id) + Segment(folder_id) ) } @@ -90,7 +110,7 @@ impl Endpoints { format!( "{}/messages/{}/$value", self.me_or_user(), - url_escape(message_id) + Segment(message_id) ) } @@ -110,12 +130,12 @@ impl Endpoints { format!( "{}/calendars/{}/events?$top={top}&$select=id,type,seriesMasterId", self.me_or_user(), - url_escape(calendar_id) + Segment(calendar_id) ) } pub fn event(&self, event_id: &str) -> String { - format!("{}/events/{}", self.me_or_user(), url_escape(event_id)) + format!("{}/events/{}", self.me_or_user(), Segment(event_id)) } pub fn calendar_exceptions( @@ -129,7 +149,7 @@ impl Endpoints { "{}/calendars/{}/calendarView?startDateTime={window_start}&endDateTime={window_end}\ &$filter=type%20eq%20%27exception%27&$top={top}&$select={EXCEPTION_SELECT}", self.me_or_user(), - url_escape(calendar_id) + Segment(calendar_id) ) } @@ -141,7 +161,7 @@ impl Endpoints { format!( "{}/drive/items/{}/children?$top={top}&$select={DRIVE_ITEM_SELECT}", self.me_or_user(), - url_escape(item_id) + Segment(item_id) ) } @@ -149,7 +169,7 @@ impl Endpoints { format!( "{}/drive/items/{}/content", self.me_or_user(), - url_escape(item_id) + Segment(item_id) ) } @@ -165,7 +185,7 @@ impl Endpoints { &$filter=type%20eq%20%27occurrence%27&$top={top}\ &$select=id,type,seriesMasterId,start,originalStartTimeZone", self.me_or_user(), - url_escape(calendar_id) + Segment(calendar_id) ) } @@ -181,7 +201,7 @@ impl Endpoints { format!( "{}/contactFolders/{}", self.me_or_user(), - url_escape(folder_id) + Segment(folder_id) ) } @@ -196,7 +216,7 @@ impl Endpoints { format!( "{}/contactFolders/{}/childFolders?$top={top}", self.me_or_user(), - url_escape(folder_id) + Segment(folder_id) ) } @@ -204,19 +224,19 @@ impl Endpoints { format!( "{}/contactFolders/{}/contacts?$top={top}&$select=id", self.me_or_user(), - url_escape(folder_id) + Segment(folder_id) ) } pub fn contact(&self, contact_id: &str) -> String { - format!("{}/contacts/{}", self.me_or_user(), url_escape(contact_id)) + format!("{}/contacts/{}", self.me_or_user(), Segment(contact_id)) } pub fn contact_photo(&self, contact_id: &str) -> String { format!( "{}/contacts/{}/photo/$value", self.me_or_user(), - url_escape(contact_id) + Segment(contact_id) ) } @@ -224,7 +244,7 @@ impl Endpoints { format!( "{}/events/{}/attachments", self.me_or_user(), - url_escape(event_id) + Segment(event_id) ) } @@ -240,76 +260,67 @@ impl Endpoints { } } -fn url_escape(segment: &str) -> String { - let mut out = String::with_capacity(segment.len()); - for b in segment.bytes() { - match b { - b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { - out.push(b as char); - } - _ => { - use std::fmt::Write; - let _ = write!(out, "%{b:02X}"); +impl GraphClient { + pub fn paged_collect( + &self, + initial_url: &str, + prefer: &[&str], + mut on_page: F, + ) -> Result<(), GraphError> + where + F: FnMut(Value) -> Result<(), GraphError>, + { + let mut url = Cow::Borrowed(initial_url); + loop { + let mut body = self.get_json_with_prefer(&url, prefer)?; + let next = match body.get_mut(NEXT_LINK) { + Some(Value::String(next)) => Some(mem::take(next)), + _ => None, + }; + on_page(body)?; + match next { + Some(next) => url = Cow::Owned(next), + None => return Ok(()), } } } - out -} -pub fn paged_collect( - client: &GraphClient, - initial_url: &str, - prefer: &[&str], - mut on_page: F, -) -> Result<(), GraphError> -where - F: FnMut(&Value) -> Result<(), GraphError>, -{ - let mut url = initial_url.to_owned(); - loop { - let body = client.get_json_with_prefer(&url, prefer)?; - on_page(&body)?; - match body.get("@odata.nextLink").and_then(Value::as_str) { - Some(next) => url = next.to_owned(), - None => return Ok(()), - } + pub fn collect_all_ids( + &self, + initial_url: &str, + prefer: &[&str], + ) -> Result, GraphError> { + let mut out = Vec::new(); + self.paged_collect(initial_url, prefer, |mut page| { + if let Some(Value::Array(items)) = page.get_mut("value") { + out.extend( + items + .iter_mut() + .filter_map(|item| match item.get_mut("id") { + Some(Value::String(id)) => Some(mem::take(id)), + _ => None, + }), + ); + } + Ok(()) + })?; + Ok(out) } -} -pub fn collect_all_ids( - client: &GraphClient, - initial_url: &str, - prefer: &[&str], -) -> Result, GraphError> { - let mut out = Vec::new(); - paged_collect(client, initial_url, prefer, |page| { - if let Some(arr) = page.get("value").and_then(Value::as_array) { - for item in arr { - if let Some(id) = item.get("id").and_then(Value::as_str) { - out.push(id.to_owned()); - } + pub fn collect_all_values( + &self, + initial_url: &str, + prefer: &[&str], + ) -> Result, GraphError> { + let mut out = Vec::new(); + self.paged_collect(initial_url, prefer, |mut page| { + if let Some(Value::Array(items)) = page.get_mut("value") { + out.append(items); } - } - Ok(()) - })?; - Ok(out) -} - -pub fn collect_all_values( - client: &GraphClient, - initial_url: &str, - prefer: &[&str], -) -> Result, GraphError> { - let mut out = Vec::new(); - paged_collect(client, initial_url, prefer, |page| { - if let Some(arr) = page.get("value").and_then(Value::as_array) { - for item in arr { - out.push(item.clone()); - } - } - Ok(()) - })?; - Ok(out) + Ok(()) + })?; + Ok(out) + } } #[cfg(test)] @@ -356,8 +367,8 @@ mod tests { #[test] fn url_escape_encodes_unsafe_bytes_only() { - assert_eq!(url_escape("AAkAAA="), "AAkAAA%3D"); - assert_eq!(url_escape("plain.id_123-x"), "plain.id_123-x"); + assert_eq!(Segment("AAkAAA=").to_string(), "AAkAAA%3D"); + assert_eq!(Segment("plain.id_123-x").to_string(), "plain.id_123-x"); } #[test] diff --git a/src/exchange/graph/calendar_map.rs b/src/exchange/graph/calendar_map.rs new file mode 100644 index 0000000..b4980f3 --- /dev/null +++ b/src/exchange/graph/calendar_map.rs @@ -0,0 +1,1037 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use chrono::{NaiveDate, NaiveDateTime, TimeZone}; +use chrono_tz::Tz; +use serde_json::{Map, Value, json}; + +use crate::exchange::date::WireDate; +use crate::exchange::graph::error::GraphError; +use crate::exchange::graph::recurrence::GraphRecurrence; +use crate::exchange::jscalendar::{ParticipantMap, ResolvedAddress, Routing}; +use crate::exchange::tz::ZoneId; +use crate::exchange::vocab::{DurationStyle, IsoDuration, Participation, Privacy}; + +const UTC_ZONE: &str = "Etc/UTC"; +const LOCAL_FORMAT: &str = "%Y-%m-%dT%H:%M:%S"; +const MIDNIGHT: &str = "T00:00:00"; +const DEFAULT_START_TIME: &str = "00:00:00"; +const SECONDS_PER_DAY: i64 = 86_400; +const SECONDS_PER_HOUR: i64 = 3_600; +const SECONDS_PER_MINUTE: i64 = 60; +const DEFAULT_PRIORITY: i64 = 5; +const ADDRESS_FIELDS: [&str; 5] = ["street", "city", "state", "postalCode", "countryOrRegion"]; + +type Hex = &'static str; +type Role = &'static str; + +#[derive(Debug, Clone, Copy)] +pub struct CalendarColor<'a>(pub &'a str); + +impl CalendarColor<'_> { + pub fn hex(self) -> Option<&'static str> { + hashify::map!( + self.0.as_bytes(), + Hex, + "lightBlue" => "#A6CEE3", + "lightGreen" => "#B2DF8A", + "lightOrange" => "#FDBF6F", + "lightGray" => "#CCCCCC", + "lightYellow" => "#FFFF99", + "lightTeal" => "#A0E7E5", + "lightPink" => "#FB9A99", + "lightBrown" => "#B15928", + "lightRed" => "#FB9A99", + ) + .copied() + } +} + +#[derive(Debug, Clone, Copy)] +pub struct GraphEvent<'a>(pub &'a Value); + +impl GraphEvent<'_> { + pub fn event_type(self) -> EventType { + self.0 + .get("type") + .and_then(Value::as_str) + .and_then(|t| { + hashify::map!( + t.as_bytes(), + EventType, + "seriesMaster" => EventType::SeriesMaster, + "occurrence" => EventType::Occurrence, + "exception" => EventType::Exception, + ) + }) + .copied() + .unwrap_or(EventType::SingleInstance) + } + + pub fn starts_on_outlook_no_date(self) -> bool { + self.0 + .get("start") + .and_then(|s| s.get("dateTime")) + .and_then(Value::as_str) + .is_some_and(|s| WireDate(s).is_outlook_no_date()) + } + + pub fn convert(self, fallback_calendar_tz: Option<&str>) -> Result { + let graph_event = self.0; + let uid = graph_event + .get("iCalUId") + .and_then(Value::as_str) + .or_else(|| graph_event.get("id").and_then(Value::as_str)) + .ok_or_else(|| GraphError::Malformed("event has neither iCalUId nor id".to_owned()))? + .to_owned(); + + let event_type = self.event_type(); + let series_master_id = graph_event + .get("seriesMasterId") + .and_then(Value::as_str) + .map(str::to_owned); + let original_start = graph_event + .get("originalStart") + .and_then(Value::as_str) + .map(str::to_owned); + let is_draft = graph_event + .get("isDraft") + .and_then(Value::as_bool) + .unwrap_or(false); + + let mut card = Map::new(); + card.insert("@type".to_owned(), Value::from("Event")); + card.insert("uid".to_owned(), Value::from(uid.clone())); + + if let Some(subject) = graph_event.get("subject").and_then(Value::as_str) + && !subject.is_empty() + { + card.insert("title".to_owned(), Value::from(subject.to_owned())); + } + + if let Some(body) = graph_event.get("body").and_then(Value::as_object) + && let Some(content) = body.get("content").and_then(Value::as_str) + && !content.is_empty() + { + card.insert("description".to_owned(), Value::from(content.to_owned())); + } + + let start_dt = GraphEvent::local_datetime(graph_event.get("start")); + let end_dt = GraphEvent::local_datetime(graph_event.get("end")); + + let is_all_day = graph_event + .get("isAllDay") + .and_then(Value::as_bool) + .unwrap_or(false); + + let display_tz = graph_event + .get("originalStartTimeZone") + .and_then(Value::as_str) + .and_then(|z| ZoneId(z).iana()) + .or_else(|| { + graph_event + .get("start") + .and_then(|s| s.get("timeZone")) + .and_then(Value::as_str) + .and_then(|z| ZoneId(z).iana()) + }) + .or_else(|| fallback_calendar_tz.and_then(|z| ZoneId(z).iana())) + .filter(|z| z != UTC_ZONE); + + if is_all_day { + let zone = display_tz.as_deref(); + let start = start_dt.as_deref().and_then(|s| WireDate(s).all_day(zone)); + if let Some(start) = start { + card.insert( + "start".to_owned(), + Value::from(GraphEvent::all_day_key(start)), + ); + let days = end_dt + .as_deref() + .and_then(|e| WireDate(e).all_day(zone)) + .map_or(1, |end| (end - start).num_days().max(1)); + card.insert("duration".to_owned(), Value::from(format!("P{days}D"))); + } + card.insert("showWithoutTime".to_owned(), Value::Bool(true)); + } else { + if let Some(start) = start_dt.as_deref() { + let value = display_tz + .as_deref() + .and_then(|tz| GraphEvent::utc_naive_to_local(start, tz)) + .unwrap_or_else(|| start.to_owned()); + card.insert("start".to_owned(), Value::from(value)); + } + if let (Some(start), Some(end)) = (start_dt.as_deref(), end_dt.as_deref()) + && let Some(dur) = GraphEvent::duration_iso8601(start, end) + { + card.insert("duration".to_owned(), Value::from(dur)); + } + let label = display_tz.as_deref().unwrap_or(UTC_ZONE); + card.insert("timeZone".to_owned(), Value::from(label)); + } + + if let Some(true) = graph_event.get("isCancelled").and_then(Value::as_bool) { + card.insert("status".to_owned(), Value::from("cancelled")); + } else { + card.insert("status".to_owned(), Value::from("confirmed")); + } + + if let Some(sens) = graph_event.get("sensitivity").and_then(Value::as_str) { + card.insert( + "privacy".to_owned(), + Value::from(Privacy::from_sensitivity(sens).as_str()), + ); + } + + if let Some(imp) = graph_event.get("importance").and_then(Value::as_str) { + card.insert( + "priority".to_owned(), + Value::from(GraphEvent::priority(imp)), + ); + } + + if let Some(show) = graph_event.get("showAs").and_then(Value::as_str) { + card.insert( + "freeBusyStatus".to_owned(), + Value::from(GraphEvent::free_busy(show)), + ); + } + + if let Some(cats) = graph_event.get("categories").and_then(Value::as_array) + && !cats.is_empty() + { + let mut map = Map::new(); + for cat in cats.iter().filter_map(Value::as_str) { + map.insert(cat.to_owned(), Value::Bool(true)); + } + if !map.is_empty() { + card.insert("keywords".to_owned(), Value::Object(map)); + } + } + + if let Some(locs) = graph_event.get("locations").and_then(Value::as_array) + && !locs.is_empty() + { + let mut locations = Map::new(); + let width = GraphEvent::pad_width(locs.len()); + for (i, loc) in locs.iter().enumerate() { + let key = format!("loc-{:0width$}", i + 1, width = width); + let mut object = Map::new(); + object.insert("@type".to_owned(), Value::from("Location")); + let display_name = loc + .get("displayName") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()); + let address_str = loc + .get("address") + .and_then(Value::as_object) + .map(GraphEvent::joined_address) + .filter(|s| !s.is_empty()); + let name = match (display_name, address_str.as_deref()) { + (Some(dn), Some(addr)) if dn != addr => Some(format!("{dn}, {addr}")), + (Some(dn), _) => Some(dn.to_owned()), + (None, Some(addr)) => Some(addr.to_owned()), + (None, None) => None, + }; + if let Some(n) = name { + object.insert("name".to_owned(), Value::from(n)); + } + locations.insert(key, Value::Object(object)); + } + if !locations.is_empty() { + card.insert("locations".to_owned(), Value::Object(locations)); + } + } + + if let Some(recurrence) = graph_event.get("recurrence") + && !recurrence.is_null() + { + let rule = GraphRecurrence(recurrence).to_rule_map()?; + card.insert("recurrenceRule".to_owned(), Value::Object(rule)); + } + + let mut participants: Vec<(String, Value)> = Vec::new(); + if let Some(org) = graph_event + .get("organizer") + .and_then(|o| o.get("emailAddress")) + { + if let Some(resolved) = GraphEvent::resolve_calendar_address(org) { + card.insert( + "organizerCalendarAddress".to_owned(), + Value::from(resolved.uri), + ); + } + let p = GraphEvent::build_participant(org, &["owner", "chair"], None, None); + participants.push(("organizer".to_owned(), p)); + } + if let Some(attendees) = graph_event.get("attendees").and_then(Value::as_array) { + let width = GraphEvent::pad_width(attendees.len()); + for (i, att) in attendees.iter().enumerate() { + let key = format!("att-{:0width$}", i + 1, width = width); + let Some(email) = att.get("emailAddress") else { + continue; + }; + let role = att + .get("type") + .and_then(Value::as_str) + .map(GraphEvent::attendee_role) + .unwrap_or("required"); + let status = att + .get("status") + .and_then(|s| s.get("response")) + .and_then(Value::as_str) + .and_then(Participation::from_graph); + let p = GraphEvent::build_participant(email, &[role], status, Some(true)); + participants.push((key, p)); + } + } + if !participants.is_empty() { + let mut map: Map = participants.into_iter().collect(); + if !card.contains_key("organizerCalendarAddress") { + ParticipantMap(&mut map).drop_calendar_address_dependents(); + } + card.insert("participants".to_owned(), Value::Object(map)); + } + + if let Some(created) = graph_event.get("createdDateTime").and_then(Value::as_str) { + card.insert( + "created".to_owned(), + Value::from(WireDate(created).without_fraction()), + ); + } + if let Some(updated) = graph_event + .get("lastModifiedDateTime") + .and_then(Value::as_str) + { + card.insert( + "updated".to_owned(), + Value::from(WireDate(updated).without_fraction()), + ); + } + + let use_default_alerts = false; + if let Some(true) = graph_event.get("isReminderOn").and_then(Value::as_bool) + && let Some(mins) = graph_event + .get("reminderMinutesBeforeStart") + .and_then(Value::as_i64) + { + let trigger = format!("-PT{}M", mins.unsigned_abs()); + let alert = json!({ + "@type": "Alert", + "trigger": { + "@type": "OffsetTrigger", + "relativeTo": "start", + "offset": trigger + } + }); + let mut alerts = Map::new(); + alerts.insert("alert-1".to_owned(), alert); + card.insert("alerts".to_owned(), Value::Object(alerts)); + } + + if let Some(cancels) = graph_event + .get("cancelledOccurrences") + .and_then(Value::as_array) + && !cancels.is_empty() + { + let start_time = card + .get("start") + .and_then(Value::as_str) + .and_then(|s| s.split_once('T').map(|(_, time)| time.to_owned())); + let mut overrides = Map::new(); + for entry in cancels.iter().filter_map(Value::as_str) { + if let Some(key) = + GraphEvent::cancelled_occurrence_key(entry, start_time.as_deref()) + { + overrides.insert(key, json!({"excluded": true})); + } + } + if !overrides.is_empty() { + card.insert("recurrenceOverrides".to_owned(), Value::Object(overrides)); + } + } + + Ok(ConvertedEvent { + uid, + data: Value::Object(card), + is_draft, + use_default_alerts, + series_master_id, + event_type, + original_start, + zone: display_tz, + }) + } + + pub fn all_day_key(date: NaiveDate) -> String { + format!("{date}{MIDNIGHT}") + } + + fn utc_naive_to_local(utc_naive: &str, iana: &str) -> Option { + let tz: Tz = iana.parse().ok()?; + let naive = NaiveDateTime::parse_from_str(utc_naive, LOCAL_FORMAT).ok()?; + Some( + tz.from_utc_datetime(&naive) + .naive_local() + .format(LOCAL_FORMAT) + .to_string(), + ) + } + + fn cancelled_occurrence_key(entry: &str, start_time: Option<&str>) -> Option { + let date = entry.rsplit('.').next()?; + let [y0, y1, y2, y3, b'-', m0, m1, b'-', d0, d1] = date.as_bytes() else { + return None; + }; + if ![y0, y1, y2, y3, m0, m1, d0, d1] + .into_iter() + .all(u8::is_ascii_digit) + { + return None; + } + Some(format!( + "{date}T{}", + start_time.unwrap_or(DEFAULT_START_TIME) + )) + } + + fn local_datetime(slot: Option<&Value>) -> Option { + let dt = slot?.get("dateTime").and_then(Value::as_str)?; + Some(dt.split_once('.').map_or(dt, |(base, _)| base).to_owned()) + } + + fn non_empty<'v>(email: &'v Value, key: &str) -> Option<&'v str> { + email + .get(key) + .and_then(Value::as_str) + .map(str::trim) + .filter(|s| !s.is_empty()) + } + + fn resolve_calendar_address(email: &Value) -> Option { + ResolvedAddress::resolve( + GraphEvent::non_empty(email, "address"), + Routing::Smtp, + GraphEvent::non_empty(email, "name"), + ) + } + + fn joined_address(addr: &Map) -> String { + let mut out = String::new(); + for part in ADDRESS_FIELDS + .iter() + .filter_map(|f| addr.get(*f).and_then(Value::as_str)) + .filter(|s| !s.is_empty()) + { + if !out.is_empty() { + out.push_str(", "); + } + out.push_str(part); + } + out + } + + fn build_participant( + email: &Value, + roles: &[&str], + participation: Option, + expect_reply: Option, + ) -> Value { + let mut map = Map::new(); + map.insert("@type".to_owned(), Value::from("Participant")); + if let Some(name) = GraphEvent::non_empty(email, "name") { + map.insert("name".to_owned(), Value::from(name)); + } + if let Some(resolved) = GraphEvent::resolve_calendar_address(email) { + map.insert("calendarAddress".to_owned(), Value::from(resolved.uri)); + if let Some(addr) = resolved.email { + map.insert("email".to_owned(), Value::from(addr)); + } + let role_map: Map = roles + .iter() + .map(|r| ((*r).to_owned(), Value::Bool(true))) + .collect(); + map.insert("roles".to_owned(), Value::Object(role_map)); + if let Some(status) = participation { + map.insert( + "participationStatus".to_owned(), + Value::from(status.as_str()), + ); + } + if let Some(b) = expect_reply { + map.insert("expectReply".to_owned(), Value::Bool(b)); + } + } + Value::Object(map) + } + + fn attendee_role(attendee_type: &str) -> &'static str { + hashify::map!( + attendee_type.as_bytes(), + Role, + "optional" => "optional", + "resource" => "informational", + ) + .copied() + .unwrap_or("required") + } + + fn pad_width(n: usize) -> usize { + n.checked_ilog10().map_or(1, |digits| digits as usize + 1) + } + + fn priority(importance: &str) -> i64 { + hashify::map!( + importance.as_bytes(), + i64, + "high" => 1, + "low" => 9, + ) + .copied() + .unwrap_or(DEFAULT_PRIORITY) + } + + fn free_busy(show_as: &str) -> &'static str { + if show_as == "free" { "free" } else { "busy" } + } + + fn duration_iso8601(start: &str, end: &str) -> Option { + let s = GraphEvent::naive_seconds(start)?; + let e = GraphEvent::naive_seconds(end)?; + IsoDuration(e - s).format(DurationStyle::Graph) + } + + fn naive_seconds(s: &str) -> Option { + let (date, time) = s.split_once('T')?; + let mut date_parts = date.split('-'); + let year: i32 = date_parts.next()?.parse().ok()?; + let month: u32 = date_parts.next()?.parse().ok()?; + let day: u32 = date_parts.next()?.parse().ok()?; + let time = time.split('.').next()?.trim_end_matches('Z'); + let mut time_parts = time.split(':'); + let hh: i64 = time_parts.next()?.parse().ok()?; + let mm: i64 = time_parts.next()?.parse().ok()?; + let ss: i64 = time_parts.next().unwrap_or("0").parse().ok()?; + let days = NaiveDate::from_ymd_opt(year, month, day)? + .signed_duration_since(NaiveDate::default()) + .num_days(); + Some(days * SECONDS_PER_DAY + hh * SECONDS_PER_HOUR + mm * SECONDS_PER_MINUTE + ss) + } +} + +#[derive(Debug, Clone)] +pub struct ConvertedEvent { + pub uid: String, + pub data: Value, + pub is_draft: bool, + pub use_default_alerts: bool, + pub series_master_id: Option, + pub event_type: EventType, + pub original_start: Option, + pub zone: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EventType { + SingleInstance, + SeriesMaster, + Occurrence, + Exception, +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Datelike; + use serde_json::json; + + fn sample() -> Value { + json!({ + "id": "AAA", + "iCalUId": "uid-1", + "type": "singleInstance", + "subject": "Sync up", + "body": {"contentType": "text", "content": "Agenda"}, + "start": {"dateTime": "2026-05-27T10:00:00.0000000", "timeZone": "UTC"}, + "end": {"dateTime": "2026-05-27T11:00:00.0000000", "timeZone": "UTC"}, + "isAllDay": false, + "isCancelled": false, + "sensitivity": "private", + "importance": "high", + "showAs": "busy", + "categories": ["Red", "Internal"], + "createdDateTime": "2026-05-01T08:00:00Z", + "lastModifiedDateTime": "2026-05-02T08:00:00Z", + "isReminderOn": true, + "reminderMinutesBeforeStart": 15, + "isDraft": false, + "organizer": {"emailAddress": {"name": "Alice", "address": "alice@x.com"}}, + "attendees": [ + {"emailAddress": {"name": "Bob", "address": "bob@x.com"}, "type": "required"} + ] + }) + } + + #[test] + fn classify_event_type_dispatch() { + assert_eq!( + GraphEvent(&json!({"type": "seriesMaster"})).event_type(), + EventType::SeriesMaster + ); + assert_eq!( + GraphEvent(&json!({"type": "occurrence"})).event_type(), + EventType::Occurrence + ); + assert_eq!( + GraphEvent(&json!({"type": "exception"})).event_type(), + EventType::Exception + ); + assert_eq!( + GraphEvent(&json!({})).event_type(), + EventType::SingleInstance + ); + } + + #[test] + fn single_instance_converts_to_event_value() { + let conv = GraphEvent(&sample()).convert(Some("UTC")).unwrap(); + assert_eq!(conv.uid, "uid-1"); + assert_eq!(conv.data["@type"], "Event"); + assert_eq!(conv.data["title"], "Sync up"); + assert_eq!(conv.data["description"], "Agenda"); + assert_eq!(conv.data["start"], "2026-05-27T10:00:00"); + assert_eq!(conv.data["duration"], "PT1H"); + assert_eq!(conv.data["status"], "confirmed"); + assert_eq!(conv.data["privacy"], "private"); + assert_eq!(conv.data["priority"], 1); + assert_eq!(conv.data["freeBusyStatus"], "busy"); + let kws = conv.data["keywords"].as_object().unwrap(); + assert!(kws.contains_key("Red")); + assert!(kws.contains_key("Internal")); + assert!(conv.data.get("categories").is_none()); + assert!(conv.data.get("version").is_none()); + assert_eq!(conv.data["timeZone"], "Etc/UTC"); + let alerts = conv.data["alerts"].as_object().unwrap(); + assert_eq!(alerts["alert-1"]["trigger"]["offset"], "-PT15M"); + } + + #[test] + fn cancelled_event_status_is_cancelled() { + let mut v = sample(); + v["isCancelled"] = Value::Bool(true); + let conv = GraphEvent(&v).convert(None).unwrap(); + assert_eq!(conv.data["status"], "cancelled"); + } + + #[test] + fn all_day_sets_show_without_time() { + let mut v = sample(); + v["isAllDay"] = Value::Bool(true); + let conv = GraphEvent(&v).convert(None).unwrap(); + assert_eq!(conv.data["showWithoutTime"], true); + } + + #[test] + fn priority_mapping_covers_three_levels() { + let conv = GraphEvent(&sample()).convert(None).unwrap(); + assert_eq!(conv.data["priority"], 1); + let mut v = sample(); + v["importance"] = Value::from("normal"); + assert_eq!(GraphEvent(&v).convert(None).unwrap().data["priority"], 5); + v["importance"] = Value::from("low"); + assert_eq!(GraphEvent(&v).convert(None).unwrap().data["priority"], 9); + } + + #[test] + fn duration_handles_subhour() { + assert_eq!( + GraphEvent::duration_iso8601("2026-05-27T10:00:00", "2026-05-27T10:30:00").as_deref(), + Some("PT30M") + ); + } + + #[test] + fn duration_handles_multi_day() { + assert_eq!( + GraphEvent::duration_iso8601("2026-05-27T10:00:00", "2026-05-28T11:30:45").as_deref(), + Some("PT25H30M45S") + ); + } + + #[test] + fn windows_tz_fallback_returns_iana() { + assert_eq!( + ZoneId("Pacific Standard Time").iana().as_deref(), + Some("America/Los_Angeles") + ); + assert_eq!( + ZoneId("America/New_York").iana().as_deref(), + Some("America/New_York") + ); + assert!(ZoneId("Made Up Zone").iana().is_none()); + } + + #[test] + fn organizer_and_attendees_become_participants() { + let conv = GraphEvent(&sample()).convert(None).unwrap(); + let participants = conv.data["participants"].as_object().unwrap(); + assert!(participants.contains_key("organizer")); + assert_eq!(participants["organizer"]["email"], "alice@x.com"); + assert_eq!( + participants["organizer"]["calendarAddress"], + "mailto:alice@x.com" + ); + assert_eq!(participants["organizer"]["roles"]["owner"], true); + assert_eq!(participants["organizer"]["roles"]["chair"], true); + assert!(participants.contains_key("att-1")); + assert_eq!(participants["att-1"]["email"], "bob@x.com"); + assert_eq!(participants["att-1"]["roles"]["required"], true); + } + + #[test] + fn a_hybrid_x500_address_is_not_emitted_as_a_mailto() { + let mut v = sample(); + v["attendees"] = json!([ + { + "emailAddress": { + "name": "Kristina Morgental", + "address": "/o=ExchangeLabs/ou=Exchange Administrative Group \ + (FYDIBOHF23SPDLT)/cn=Recipients/cn=bdc77b18152647a29d28ce1188376dc9-kristina" + }, + "type": "required", + "status": {"response": "none"} + } + ]); + let conv = GraphEvent(&v).convert(None).unwrap(); + let p = &conv.data["participants"]["att-1"]; + let addr = p["calendarAddress"].as_str().unwrap(); + assert!( + addr.starts_with("urn:x-inbuxa-migrate:attendee:"), + "a hybrid X500 reference must not become a malformed mailto, got {addr}" + ); + assert!( + p.get("email").is_none(), + "an X500 reference is not an email address" + ); + assert_eq!(p["name"], "Kristina Morgental", "the display name is kept"); + } + + #[test] + fn a_prefixed_or_bracketed_address_is_normalised() { + let mut v = sample(); + v["attendees"] = json!([ + { + "emailAddress": {"name": "Bob", "address": "SMTP:bob@example.com"}, + "type": "required", + "status": {"response": "none"} + }, + { + "emailAddress": {"name": "Eve", "address": "Eve "}, + "type": "required", + "status": {"response": "none"} + } + ]); + let conv = GraphEvent(&v).convert(None).unwrap(); + assert_eq!( + conv.data["participants"]["att-1"]["calendarAddress"], + "mailto:bob@example.com" + ); + assert_eq!( + conv.data["participants"]["att-1"]["email"], + "bob@example.com" + ); + assert_eq!( + conv.data["participants"]["att-2"]["calendarAddress"], + "mailto:eve@example.com" + ); + } + + #[test] + fn a_name_only_attendee_gets_a_stable_synthetic_calendar_address() { + let mut v = sample(); + v["attendees"] = json!([ + { + "emailAddress": {"name": "Room 4"}, + "type": "resource", + "status": {"response": "accepted"} + } + ]); + let conv = GraphEvent(&v).convert(None).unwrap(); + let p = &conv.data["participants"]["att-1"]; + let addr = p["calendarAddress"].as_str().unwrap(); + assert!( + addr.starts_with("urn:x-inbuxa-migrate:attendee:"), + "a name-only attendee needs a synthetic calendarAddress, got {addr}" + ); + assert!( + p.get("email").is_none(), + "there is no SMTP address to report as email" + ); + assert_eq!(p["participationStatus"], "accepted"); + assert_eq!(p["roles"]["informational"], true); + assert_eq!( + addr, + GraphEvent(&v).convert(None).unwrap().data["participants"]["att-1"]["calendarAddress"] + ); + } + + #[test] + fn an_unidentifiable_attendee_keeps_no_calendar_address_dependent_property() { + let mut v = sample(); + v["attendees"] = json!([ + {"emailAddress": {}, "type": "required", "status": {"response": "accepted"}} + ]); + let conv = GraphEvent(&v).convert(None).unwrap(); + let p = &conv.data["participants"]["att-1"]; + for key in [ + "calendarAddress", + "email", + "roles", + "participationStatus", + "expectReply", + ] { + assert!( + p.get(key).is_none(), + "{key} requires calendarAddress (jscalendarbis 3.4.6)" + ); + } + assert_eq!(p["@type"], "Participant"); + } + + #[test] + fn an_organizer_without_an_address_still_yields_a_calendar_address() { + let mut v = sample(); + v["organizer"] = json!({"emailAddress": {"name": "Alice Example"}}); + let conv = GraphEvent(&v).convert(None).unwrap(); + let organizer_address = conv.data["organizerCalendarAddress"].as_str().unwrap(); + assert!(organizer_address.starts_with("urn:x-inbuxa-migrate:attendee:")); + assert_eq!( + conv.data["participants"]["organizer"]["calendarAddress"], + organizer_address + ); + } + + #[test] + fn participants_drop_calendar_addresses_when_no_organizer_is_known() { + let mut v = sample(); + v.as_object_mut().unwrap().remove("organizer"); + let conv = GraphEvent(&v).convert(None).unwrap(); + assert!( + conv.data.get("organizerCalendarAddress").is_none(), + "no organizer is derivable from the source event" + ); + let participants = conv.data["participants"].as_object().unwrap(); + assert!(!participants.is_empty()); + for (key, participant) in participants { + assert!( + participant.get("calendarAddress").is_none(), + "{key} may not carry a calendarAddress without organizerCalendarAddress" + ); + assert!(participant.get("roles").is_none(), "{key} kept roles"); + } + } + + #[test] + fn optional_attendee_keeps_role_and_uses_status_when_present() { + let mut v = sample(); + v["attendees"] = json!([ + { + "emailAddress": {"name": "Carol", "address": "carol@x.com"}, + "type": "optional", + "status": {"response": "tentativelyAccepted"} + } + ]); + let conv = GraphEvent(&v).convert(None).unwrap(); + let p = &conv.data["participants"]["att-1"]; + assert_eq!(p["roles"]["optional"], true); + assert!(p["roles"].get("required").is_none()); + assert_eq!(p["participationStatus"], "tentative"); + } + + #[test] + fn many_attendees_zero_pad_keys_for_stable_sort() { + let mut v = sample(); + let mut atts = Vec::new(); + for i in 1..=11 { + atts.push(json!({ + "emailAddress": {"address": format!("u{i}@x.com")}, + "type": "required" + })); + } + v["attendees"] = Value::Array(atts); + let conv = GraphEvent(&v).convert(None).unwrap(); + let map = conv.data["participants"].as_object().unwrap(); + assert!(map.contains_key("att-01")); + assert!(map.contains_key("att-11")); + } + + #[test] + fn original_start_time_zone_is_preserved() { + let mut v = sample(); + v["originalStartTimeZone"] = Value::from("Pacific Standard Time"); + v["start"]["timeZone"] = Value::from("UTC"); + let conv = GraphEvent(&v).convert(None).unwrap(); + assert_eq!(conv.data["timeZone"], "America/Los_Angeles"); + assert_eq!(conv.data["start"], "2026-05-27T03:00:00"); + } + + #[test] + fn utc_times_are_converted_into_the_original_zone() { + let mut v = sample(); + v["originalStartTimeZone"] = Value::from("Europe/Paris"); + v["start"]["dateTime"] = Value::from("2025-03-12T07:30:00.0000000"); + v["start"]["timeZone"] = Value::from("UTC"); + v["end"]["dateTime"] = Value::from("2025-03-12T08:00:00.0000000"); + v["end"]["timeZone"] = Value::from("UTC"); + let conv = GraphEvent(&v).convert(None).unwrap(); + assert_eq!(conv.data["start"], "2025-03-12T08:30:00"); + assert_eq!(conv.data["timeZone"], "Europe/Paris"); + assert_eq!(conv.data["duration"], "PT30M"); + } + + #[test] + fn summer_event_shifts_by_two_hours_in_paris() { + let mut v = sample(); + v["originalStartTimeZone"] = Value::from("Romance Standard Time"); + v["start"]["dateTime"] = Value::from("2025-07-15T06:30:00.0000000"); + v["start"]["timeZone"] = Value::from("UTC"); + v["end"]["dateTime"] = Value::from("2025-07-15T07:30:00.0000000"); + v["end"]["timeZone"] = Value::from("UTC"); + let conv = GraphEvent(&v).convert(None).unwrap(); + assert_eq!(conv.data["start"], "2025-07-15T08:30:00"); + assert_eq!(conv.data["timeZone"], "Europe/Paris"); + } + + #[test] + fn unresolvable_microsoft_zone_falls_back_to_utc() { + let mut v = sample(); + v["originalStartTimeZone"] = Value::from("tzone://Microsoft/Custom"); + v["start"]["dateTime"] = Value::from("2025-03-12T07:30:00.0000000"); + v["start"]["timeZone"] = Value::from("UTC"); + let conv = GraphEvent(&v).convert(None).unwrap(); + assert_eq!(conv.data["start"], "2025-03-12T07:30:00"); + assert_eq!(conv.data["timeZone"], "Etc/UTC"); + } + + #[test] + fn all_day_keeps_date_and_omits_timezone() { + let mut v = sample(); + v["isAllDay"] = Value::Bool(true); + v["originalStartTimeZone"] = Value::from("Europe/Paris"); + v["start"]["dateTime"] = Value::from("2025-03-12T00:00:00.0000000"); + v["start"]["timeZone"] = Value::from("UTC"); + let conv = GraphEvent(&v).convert(None).unwrap(); + assert_eq!(conv.data["start"], "2025-03-12T00:00:00"); + assert_eq!(conv.data["showWithoutTime"], true); + assert!(conv.data.get("timeZone").is_none()); + } + + #[test] + fn all_day_utc_instant_east_of_utc_keeps_its_local_date() { + let mut v = sample(); + v["isAllDay"] = Value::Bool(true); + v["originalStartTimeZone"] = Value::from("Romance Standard Time"); + v["start"]["dateTime"] = Value::from("2026-10-23T22:00:00.0000000"); + v["end"]["dateTime"] = Value::from("2026-10-25T23:00:00.0000000"); + let conv = GraphEvent(&v).convert(None).unwrap(); + assert_eq!(conv.data["start"], "2026-10-24T00:00:00"); + assert_eq!(conv.data["duration"], "P2D"); + assert_eq!(conv.zone.as_deref(), Some("Europe/Paris")); + } + + #[test] + fn all_day_midnight_is_kept_whatever_the_zone() { + let mut v = sample(); + v["isAllDay"] = Value::Bool(true); + v["originalStartTimeZone"] = Value::from("New Zealand Standard Time"); + v["start"]["dateTime"] = Value::from("2026-11-12T00:00:00.0000000"); + v["end"]["dateTime"] = Value::from("2026-11-13T00:00:00.0000000"); + let conv = GraphEvent(&v).convert(None).unwrap(); + assert_eq!(conv.data["start"], "2026-11-12T00:00:00"); + assert_eq!(conv.data["duration"], "P1D"); + } + + #[test] + fn duration_h_and_s_no_minutes_includes_zero_m() { + assert_eq!( + GraphEvent::duration_iso8601("2026-05-27T10:00:00", "2026-05-27T11:00:05").as_deref(), + Some("PT1H0M5S") + ); + } + + #[test] + fn fractional_seconds_are_stripped_from_timestamps() { + let mut v = sample(); + v["createdDateTime"] = Value::from("2026-05-01T08:00:00.0000000Z"); + v["lastModifiedDateTime"] = Value::from("2026-05-02T08:00:00.123Z"); + let conv = GraphEvent(&v).convert(None).unwrap(); + assert_eq!(conv.data["created"], "2026-05-01T08:00:00Z"); + assert_eq!(conv.data["updated"], "2026-05-02T08:00:00Z"); + } + + #[test] + fn organizer_calendar_address_is_set_alongside_participants() { + let conv = GraphEvent(&sample()).convert(None).unwrap(); + assert_eq!(conv.data["organizerCalendarAddress"], "mailto:alice@x.com"); + } + + #[test] + fn recurrence_emits_singular_recurrence_rule_object() { + let mut v = sample(); + v["recurrence"] = json!({ + "pattern": {"type": "daily", "interval": 1}, + "range": {"type": "noEnd"} + }); + let conv = GraphEvent(&v).convert(None).unwrap(); + assert!(conv.data.get("recurrenceRules").is_none()); + let rule = &conv.data["recurrenceRule"]; + assert!(rule.is_object()); + assert_eq!(rule["@type"], "RecurrenceRule"); + assert_eq!(rule["frequency"], "daily"); + } + + #[test] + fn location_with_address_only_uses_name_property() { + let mut v = sample(); + v["locations"] = json!([{ + "address": {"street": "1 Infinite Loop", "city": "Cupertino"} + }]); + let conv = GraphEvent(&v).convert(None).unwrap(); + let locs = conv.data["locations"].as_object().unwrap(); + let only = locs.values().next().unwrap(); + assert_eq!(only["@type"], "Location"); + assert_eq!(only["name"], "1 Infinite Loop, Cupertino"); + assert!(only.get("description").is_none()); + } + + fn reference_days_from_civil(y: i64, m: i64, d: i64) -> i64 { + let y = if m <= 2 { y - 1 } else { y }; + let era = (if y >= 0 { y } else { y - 399 }) / 400; + let yoe = (y - era * 400) as u64; + let m_adj: i64 = if m > 2 { m - 3 } else { m + 9 }; + let doy = ((153 * m_adj + 2) / 5 + d - 1) as u64; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + era * 146097 + (doe as i64) - 719468 + } + + #[test] + fn naive_seconds_match_the_hand_rolled_civil_day_count() { + // R20: chrono replaces days_from_civil; every valid day from 1600 to 2400 agrees. + let mut day = NaiveDate::from_ymd_opt(1600, 1, 1).unwrap(); + let last = NaiveDate::from_ymd_opt(2400, 12, 31).unwrap(); + while day <= last { + let text = format!("{}T12:34:56.789", day.format("%Y-%m-%d")); + let (y, m, d) = ( + i64::from(day.year()), + i64::from(day.month()), + i64::from(day.day()), + ); + let expected = reference_days_from_civil(y, m, d) * 86_400 + 12 * 3_600 + 34 * 60 + 56; + assert_eq!(GraphEvent::naive_seconds(&text), Some(expected), "{text}"); + day = day.succ_opt().unwrap(); + } + } +} diff --git a/src/exchange/graph/client.rs b/src/exchange/graph/client.rs new file mode 100644 index 0000000..5a6cc5a --- /dev/null +++ b/src/exchange/graph/client.rs @@ -0,0 +1,384 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::str; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +use serde_json::Value; +use ureq::ResponseExt; +use ureq::http::Uri; +use url::Url; + +use crate::exchange::graph::error::GraphError; +use crate::exchange::graph::retry::HttpClass; +use crate::logging::{HttpCall, Level, Logger}; +use crate::net::http::{ + AgentProfile, BodyExcerpt, ClientOptions, HttpCore, Methods, Redirects, RetryStats, + TransportError, +}; +use crate::net::retry::{RetryAfter, RetryWait}; + +const MAX_BODY: u64 = 256 * 1024 * 1024; +const LONG_RETRY_THRESHOLD: Duration = Duration::from_secs(10); +const PROFILE: AgentProfile = AgentProfile { + redirects: Redirects::Follow, + methods: Methods::Standard, + max_body: MAX_BODY, +}; +const IMMUTABLE_ID: &str = "IdType=\"ImmutableId\""; +const DRIVE_CONTENT_SUFFIX: &str = "/content"; + +#[derive(Debug, Clone, Copy)] +pub enum Accept { + Json, + Text, + Binary, +} + +impl Accept { + fn header_value(self) -> &'static str { + match self { + Accept::Json => "application/json", + Accept::Text => "text/plain", + Accept::Binary => "application/octet-stream", + } + } +} + +#[derive(Debug, Clone)] +pub struct GraphResponse { + pub status: u16, + pub body: Vec, + pub content_type: Option, +} + +impl GraphResponse { + pub fn as_str(&self) -> Result<&str, GraphError> { + str::from_utf8(&self.body) + .map_err(|e| GraphError::Malformed(format!("response is not utf-8: {e}"))) + } + + pub fn json(&self) -> Result { + serde_json::from_slice(&self.body) + .map_err(|e| GraphError::Malformed(format!("response is not valid json: {e}"))) + } +} + +struct Inner { + core: HttpCore, + bearer: Mutex, +} + +#[derive(Clone)] +pub struct GraphClient { + inner: Arc, +} + +enum Attempt { + Response { + status: u16, + body: Vec, + retry_after: Option, + content_type: Option, + }, + Transport(GraphError), +} + +struct Request<'a> { + url: &'a str, + accept: Accept, + prefer: &'a [&'a str], +} + +impl GraphClient { + const METHOD: &str = "GET"; + + pub fn new(bearer: String, options: ClientOptions) -> GraphClient { + GraphClient { + inner: Arc::new(Inner { + core: HttpCore::new(options, PROFILE), + bearer: Mutex::new(bearer), + }), + } + } + + pub fn set_bearer(&self, bearer: String) { + let mut g = match self.inner.bearer.lock() { + Ok(g) => g, + Err(p) => p.into_inner(), + }; + *g = bearer; + } + + pub fn logger(&self) -> Logger { + self.inner.core.logger() + } + + fn auth_header(&self) -> String { + let bearer = match self.inner.bearer.lock() { + Ok(g) => g, + Err(p) => p.into_inner(), + }; + format!("Bearer {bearer}") + } + + pub fn stats(&self) -> RetryStats { + self.inner.core.stats().snapshot() + } + + pub fn requests_observed(&self) -> u64 { + self.inner.core.stats().requests() + } + + pub fn get(&self, url: &str, accept: Accept) -> Result { + self.execute(&Request { + url, + accept, + prefer: &[], + }) + } + + pub fn get_with_prefer( + &self, + url: &str, + accept: Accept, + prefer: &[&str], + ) -> Result { + self.execute(&Request { + url, + accept, + prefer, + }) + } + + pub fn get_json(&self, url: &str) -> Result { + self.get(url, Accept::Json).and_then(|r| r.json()) + } + + pub fn get_json_with_prefer(&self, url: &str, prefer: &[&str]) -> Result { + self.get_with_prefer(url, Accept::Json, prefer) + .and_then(|r| r.json()) + } + + fn execute(&self, request: &Request<'_>) -> Result { + let core = &self.inner.core; + let method = GraphClient::METHOD; + let url = request.url; + let max_retries = core.retry().max_retries; + let logger = core.logger(); + let mut attempt: u32 = 0; + loop { + core.rate_limit().cooldown().wait(); + core.stats().note_request(); + let (status, body, retry_after) = match self.one_attempt(request) { + Attempt::Response { + status, + body, + retry_after, + content_type, + } => match HttpClass::of_status(status) { + HttpClass::Success => { + core.rate_limit().on_success(); + return Ok(GraphResponse { + status, + body, + content_type, + }); + } + HttpClass::Vanished => return Err(GraphError::Vanished), + HttpClass::Auth => { + return Err(GraphError::Auth(format!( + "server returned {status}: {}", + BodyExcerpt(&body) + ))); + } + HttpClass::Fatal => { + return Err(GraphError::HttpStatus { + status, + body: BodyExcerpt(&body).to_string(), + }); + } + HttpClass::Retryable => (status, body, retry_after), + }, + Attempt::Transport(err) => { + if matches!(err, GraphError::Connect(_)) { + return Err(err); + } + attempt += 1; + core.stats().note_retry(); + if attempt > max_retries { + return Err(GraphError::RetriesExhausted(format!( + "{method} {url}: {err}" + ))); + } + let delay = core.backoff(attempt); + if delay >= LONG_RETRY_THRESHOLD { + logger.warn(format_args!( + "transient transport failure ({err}); waiting {} before retry {attempt}/{max_retries}", + RetryWait(delay), + )); + } + thread::sleep(delay); + continue; + } + }; + attempt += 1; + core.stats().note_retry(); + if attempt > max_retries { + return Err(GraphError::RetriesExhausted(format!( + "{method} {url} kept returning http {status}" + ))); + } + if retry_after.is_some() { + core.stats().note_retry_after(); + } + let delay = match retry_after { + _ if HttpClass::arms_throttle(status) => core.throttle(retry_after), + Some(d) => d, + None => core.backoff(attempt), + }; + if delay >= LONG_RETRY_THRESHOLD { + logger.warn(format_args!( + "Graph rate-limited (http {status}); waiting {} before retry {attempt}/{max_retries} (shared level {})", + RetryWait(delay), + core.rate_limit().level(), + )); + } + if logger.enabled(Level::Bodies) { + logger.progress(format_args!( + "retry {attempt}/{max_retries} {method} {url} after {delay:?} (http {status}) body={}", + BodyExcerpt(&body) + )); + } else { + logger.progress(format_args!( + "retry {attempt}/{max_retries} (http {status})" + )); + } + thread::sleep(delay); + } + } + + fn one_attempt(&self, request: &Request<'_>) -> Attempt { + let core = &self.inner.core; + let method = GraphClient::METHOD; + let url = request.url; + let mut req = core + .agent() + .get(url) + .header("Authorization", self.auth_header()) + .header("Accept", request.accept.header_value()) + .header("User-Agent", core.user_agent()) + .header("Prefer", IMMUTABLE_ID); + for value in request.prefer { + req = req.header("Prefer", *value); + } + let logger = core.logger(); + let started = Instant::now(); + match req.call() { + Ok(mut resp) => { + let status = resp.status().as_u16(); + let final_uri = resp.get_uri().clone(); + let retry_after = resp + .headers() + .get("retry-after") + .and_then(|v| v.to_str().ok()) + .and_then(RetryAfter::parse_now); + let content_type = resp + .headers() + .get("content-type") + .and_then(|v| v.to_str().ok()) + .map(str::to_owned); + match resp.body_mut().with_config().limit(MAX_BODY).read_to_vec() { + Ok(bytes) => { + GraphClient::warn_on_redirect(logger, url, &final_uri); + logger.trace_http(&HttpCall { + proto: "Graph", + method, + url, + status, + elapsed: started.elapsed(), + note: None, + request: None, + request_type: None, + response: &bytes, + response_type: content_type.as_deref(), + }); + Attempt::Response { + status, + body: bytes, + retry_after, + content_type, + } + } + Err(e) => Attempt::Transport(GraphError::Transport(format!( + "reading response body: {e}" + ))), + } + } + Err(e) => { + let err = GraphError::from(TransportError::from(e)); + logger.trace_http_error("Graph", method, url, &err.to_string(), started.elapsed()); + Attempt::Transport(err) + } + } + } + + fn warn_on_redirect(logger: Logger, requested: &str, final_uri: &Uri) { + if requested.ends_with(DRIVE_CONTENT_SUFFIX) { + return; + } + let landed = final_uri.to_string(); + if HttpCore::is_cross_host(requested, &landed) { + let parsed = Url::parse(&landed).ok(); + let host = parsed + .as_ref() + .and_then(Url::host_str) + .unwrap_or("another host"); + logger.warn(format_args!( + "{} {requested} was redirected across hosts to {host}; verify the Graph endpoint host is reachable directly without redirection", + GraphClient::METHOD + )); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::net::http::TlsPolicy; + use crate::net::retry::RetryPolicy; + + fn options(max_retries: u32) -> ClientOptions { + ClientOptions { + retry: RetryPolicy::new(max_retries), + tls: TlsPolicy::Verify, + logger: Logger::new(Level::Quiet), + } + } + + #[test] + fn defaults_construct() { + let c = GraphClient::new("token".to_owned(), options(3)); + assert_eq!(c.stats(), RetryStats::default()); + assert_eq!(c.requests_observed(), 0); + assert_eq!(c.auth_header(), "Bearer token"); + } + + #[test] + fn bearer_can_be_swapped_at_runtime() { + let c = GraphClient::new("old".to_owned(), options(0)); + c.set_bearer("new".to_owned()); + assert_eq!(c.auth_header(), "Bearer new"); + } + + #[test] + fn accept_header_values_are_what_the_spec_requires() { + assert_eq!(Accept::Json.header_value(), "application/json"); + assert_eq!(Accept::Text.header_value(), "text/plain"); + } +} diff --git a/src/exchange/graph/contact_map.rs b/src/exchange/graph/contact_map.rs new file mode 100644 index 0000000..4fb52d1 --- /dev/null +++ b/src/exchange/graph/contact_map.rs @@ -0,0 +1,657 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::{Map, Value, json}; + +use crate::exchange::date::WireDate; +use crate::exchange::graph::error::GraphError; + +#[derive(Debug, Clone)] +pub struct ConvertedContact { + pub uid: String, + pub data: Value, +} + +const SYNTHETIC_UID_PREFIX: &str = "inbuxa-migrate-graph-"; + +#[derive(Debug, Clone, Copy)] +pub struct GraphContact<'a>(pub &'a Value); + +impl GraphContact<'_> { + pub fn convert(self) -> Result { + let graph_contact = self.0; + let graph_id = graph_contact + .get("id") + .and_then(Value::as_str) + .ok_or_else(|| GraphError::Malformed("contact has no id".to_owned()))?; + let uid = GraphContact::synthetic_uid(graph_id); + + let mut card = Map::new(); + card.insert("@type".to_owned(), Value::from("Card")); + card.insert("version".to_owned(), Value::from("1.0")); + card.insert("uid".to_owned(), Value::from(uid.clone())); + card.insert("kind".to_owned(), Value::from("individual")); + + let mut name_object = Map::new(); + name_object.insert("@type".to_owned(), Value::from("Name")); + if let Some(display) = graph_contact.get("displayName").and_then(Value::as_str) + && !display.is_empty() + { + name_object.insert("full".to_owned(), Value::from(display.to_owned())); + } + let name_slots = [ + ("title", graph_contact.get("title").and_then(Value::as_str)), + ( + "given", + graph_contact.get("givenName").and_then(Value::as_str), + ), + ( + "given2", + graph_contact.get("middleName").and_then(Value::as_str), + ), + ( + "surname", + graph_contact.get("surname").and_then(Value::as_str), + ), + ( + "generation", + graph_contact.get("generation").and_then(Value::as_str), + ), + ]; + let mut components: Vec = Vec::new(); + let mut indices = [None; 5]; + for ((kind, value), index) in name_slots.into_iter().zip(indices.iter_mut()) { + let Some(v) = value else { continue }; + if v.is_empty() { + continue; + } + *index = Some(components.len()); + components.push(json!({"@type": "NameComponent", "kind": kind, "value": v})); + } + let [_, given_index, _, surname_index, _] = indices; + if !components.is_empty() { + name_object.insert("components".to_owned(), Value::Array(components)); + } + if name_object.contains_key("full") || name_object.contains_key("components") { + card.insert("name".to_owned(), Value::Object(name_object)); + } + + if let Some(nick) = graph_contact.get("nickName").and_then(Value::as_str) + && !nick.is_empty() + { + let mut nicks = Map::new(); + nicks.insert( + "nick-1".to_owned(), + json!({"@type": "Nickname", "name": nick}), + ); + card.insert("nicknames".to_owned(), Value::Object(nicks)); + } + + let mut emails = Map::new(); + if let Some(addrs) = graph_contact + .get("emailAddresses") + .and_then(Value::as_array) + { + for (i, e) in addrs.iter().enumerate() { + let Some(addr) = e.get("address").and_then(Value::as_str) else { + continue; + }; + if addr.is_empty() { + continue; + } + let key = format!("email-{}", i + 1); + let mut entry = Map::new(); + entry.insert("@type".to_owned(), Value::from("EmailAddress")); + entry.insert("address".to_owned(), Value::from(addr.to_owned())); + if i == 0 { + entry.insert("pref".to_owned(), Value::from(1)); + } + emails.insert(key, Value::Object(entry)); + } + } + if !emails.is_empty() { + card.insert("emails".to_owned(), Value::Object(emails)); + } + + let mut phones = Map::new(); + GraphContact::push_phones( + &mut phones, + graph_contact + .get("businessPhones") + .and_then(Value::as_array), + "work", + ); + GraphContact::push_phones( + &mut phones, + graph_contact.get("homePhones").and_then(Value::as_array), + "private", + ); + if let Some(mobile) = graph_contact.get("mobilePhone").and_then(Value::as_str) + && !mobile.is_empty() + { + let key = format!("phone-{}", phones.len() + 1); + phones.insert( + key, + json!({ + "@type": "Phone", + "number": mobile, + "features": {"mobile": true, "voice": true} + }), + ); + } + if !phones.is_empty() { + card.insert("phones".to_owned(), Value::Object(phones)); + } + + let mut addresses = Map::new(); + GraphContact::push_address( + &mut addresses, + graph_contact.get("homeAddress"), + Some("private"), + ); + GraphContact::push_address( + &mut addresses, + graph_contact.get("businessAddress"), + Some("work"), + ); + GraphContact::push_address(&mut addresses, graph_contact.get("otherAddress"), None); + if !addresses.is_empty() { + card.insert("addresses".to_owned(), Value::Object(addresses)); + } + + let mut organizations = Map::new(); + let has_company = graph_contact + .get("companyName") + .and_then(Value::as_str) + .is_some_and(|s| !s.is_empty()); + let has_dept = graph_contact + .get("department") + .and_then(Value::as_str) + .is_some_and(|s| !s.is_empty()); + let has_office = graph_contact + .get("officeLocation") + .and_then(Value::as_str) + .is_some_and(|s| !s.is_empty()); + let has_title = graph_contact + .get("jobTitle") + .and_then(Value::as_str) + .is_some_and(|s| !s.is_empty()); + if has_company || has_dept || has_office { + let mut org = Map::new(); + org.insert("@type".to_owned(), Value::from("Organization")); + if let Some(name) = graph_contact.get("companyName").and_then(Value::as_str) + && !name.is_empty() + { + org.insert("name".to_owned(), Value::from(name.to_owned())); + } + let mut units: Vec = Vec::new(); + if let Some(dept) = graph_contact.get("department").and_then(Value::as_str) + && !dept.is_empty() + { + units.push(json!({"@type": "OrgUnit", "name": dept})); + } + if let Some(office) = graph_contact.get("officeLocation").and_then(Value::as_str) + && !office.is_empty() + { + units.push(json!({"@type": "OrgUnit", "name": office})); + } + if !units.is_empty() { + org.insert("units".to_owned(), Value::Array(units)); + } + organizations.insert("org-1".to_owned(), Value::Object(org)); + } + if has_title + && let Some(title) = graph_contact.get("jobTitle").and_then(Value::as_str) + && !title.is_empty() + { + let mut titles = Map::new(); + let mut title_obj = Map::new(); + title_obj.insert("@type".to_owned(), Value::from("Title")); + title_obj.insert("name".to_owned(), Value::from(title.to_owned())); + if organizations.contains_key("org-1") { + title_obj.insert("organizationId".to_owned(), Value::from("org-1")); + } + titles.insert("title-1".to_owned(), Value::Object(title_obj)); + card.insert("titles".to_owned(), Value::Object(titles)); + } + let has_org_one = organizations.contains_key("org-1"); + if !organizations.is_empty() { + card.insert("organizations".to_owned(), Value::Object(organizations)); + } + + if let Some(notes) = graph_contact.get("personalNotes").and_then(Value::as_str) + && !notes.is_empty() + { + let mut map = Map::new(); + map.insert("note-1".to_owned(), json!({"@type": "Note", "note": notes})); + card.insert("notes".to_owned(), Value::Object(map)); + } + + let mut localizations = Map::new(); + let mut ja = Map::new(); + if let Some(name) = graph_contact.get("yomiCompanyName").and_then(Value::as_str) + && !name.is_empty() + && has_org_one + { + ja.insert( + "organizations/org-1/name".to_owned(), + Value::from(name.to_owned()), + ); + } + if let Some(name) = graph_contact.get("yomiGivenName").and_then(Value::as_str) + && !name.is_empty() + && let Some(idx) = given_index + { + ja.insert( + format!("name/components/{idx}/value"), + Value::from(name.to_owned()), + ); + } + if let Some(name) = graph_contact.get("yomiSurname").and_then(Value::as_str) + && !name.is_empty() + && let Some(idx) = surname_index + { + ja.insert( + format!("name/components/{idx}/value"), + Value::from(name.to_owned()), + ); + } + if !ja.is_empty() { + localizations.insert("ja".to_owned(), Value::Object(ja)); + } + if !localizations.is_empty() { + card.insert("localizations".to_owned(), Value::Object(localizations)); + } + + if let Some(birthday) = graph_contact.get("birthday").and_then(Value::as_str) + && let Some(date) = WireDate(birthday).partial_date() + { + let mut anniversaries = Map::new(); + anniversaries.insert( + "anniversary-1".to_owned(), + json!({ + "@type": "Anniversary", + "kind": "birth", + "date": date, + }), + ); + card.insert("anniversaries".to_owned(), Value::Object(anniversaries)); + } + + if let Some(link) = graph_contact + .get("businessHomePage") + .and_then(Value::as_str) + && !link.is_empty() + { + let mut links = Map::new(); + links.insert( + "link-1".to_owned(), + json!({"@type": "Link", "uri": link, "contexts": {"work": true}}), + ); + card.insert("links".to_owned(), Value::Object(links)); + } + + if let Some(cats) = graph_contact.get("categories").and_then(Value::as_array) { + let mut keywords = Map::new(); + for cat in cats.iter().filter_map(Value::as_str) { + if !cat.trim().is_empty() { + keywords.insert(cat.to_ascii_lowercase(), Value::Bool(true)); + } + } + if !keywords.is_empty() { + card.insert("keywords".to_owned(), Value::Object(keywords)); + } + } + + if let Some(ims) = graph_contact.get("imAddresses").and_then(Value::as_array) { + let mut services = Map::new(); + for (i, addr) in ims + .iter() + .filter_map(Value::as_str) + .map(str::trim) + .filter(|a| !a.is_empty()) + .enumerate() + { + let slot = if GraphContact::has_uri_scheme(addr) { + "uri" + } else { + "user" + }; + services.insert( + (i + 1).to_string(), + json!({"@type": "OnlineService", slot: addr}), + ); + } + if !services.is_empty() { + card.insert("onlineServices".to_owned(), Value::Object(services)); + } + } + + if let Some(created) = graph_contact.get("createdDateTime").and_then(Value::as_str) { + card.insert( + "created".to_owned(), + Value::from(WireDate(created).without_fraction()), + ); + } + if let Some(updated) = graph_contact + .get("lastModifiedDateTime") + .and_then(Value::as_str) + { + card.insert( + "updated".to_owned(), + Value::from(WireDate(updated).without_fraction()), + ); + } + + Ok(ConvertedContact { + uid, + data: Value::Object(card), + }) + } + + fn has_uri_scheme(value: &str) -> bool { + let Some((scheme, rest)) = value.split_once(':') else { + return false; + }; + !rest.is_empty() + && !scheme.is_empty() + && scheme.starts_with(|c: char| c.is_ascii_alphabetic()) + && scheme + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.')) + } + + pub fn synthetic_uid(graph_id: &str) -> String { + let hash = blake3::hash(graph_id.as_bytes()); + format!("{SYNTHETIC_UID_PREFIX}{}", hash.to_hex()) + } + + fn push_phones(target: &mut Map, list: Option<&Vec>, context: &str) { + let Some(list) = list else { return }; + for n in list.iter().filter_map(Value::as_str) { + if n.is_empty() { + continue; + } + let key = format!("phone-{}", target.len() + 1); + target.insert( + key, + json!({ + "@type": "Phone", + "number": n, + "contexts": {context: true} + }), + ); + } + } + + fn push_address( + target: &mut Map, + source: Option<&Value>, + context: Option<&str>, + ) { + let Some(source) = source else { return }; + let Some(map) = source.as_object() else { + return; + }; + let street = map + .get("street") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()); + let component_fields: Vec<(&str, &str)> = [ + ("city", "locality"), + ("state", "region"), + ("postalCode", "postcode"), + ("countryOrRegion", "country"), + ] + .into_iter() + .filter_map(|(graph_key, jsc_kind)| { + map.get(graph_key) + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .map(|s| (jsc_kind, s)) + }) + .collect(); + if street.is_none() && component_fields.is_empty() { + return; + } + let mut address = Map::new(); + address.insert("@type".to_owned(), Value::from("Address")); + if let Some(context) = context { + address.insert("contexts".to_owned(), json!({context: true})); + } + if let Some(s) = street { + address.insert("full".to_owned(), Value::from(s.to_owned())); + } + if !component_fields.is_empty() { + let components: Vec = component_fields + .iter() + .map(|(kind, value)| json!({"@type": "AddressComponent", "kind": kind, "value": value})) + .collect(); + address.insert("components".to_owned(), Value::Array(components)); + } + let key = format!("addr-{}", target.len() + 1); + target.insert(key, Value::Object(address)); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn minimal_contact_yields_card_with_synthetic_uid() { + let c = json!({"id": "GRAPH-1", "displayName": "Alice"}); + let conv = GraphContact(&c).convert().unwrap(); + assert!(conv.uid.starts_with("inbuxa-migrate-graph-")); + assert_eq!(conv.data["@type"], "Card"); + assert_eq!(conv.data["kind"], "individual"); + assert_eq!(conv.data["name"]["full"], "Alice"); + } + + #[test] + fn synthetic_uid_is_stable_per_graph_id() { + let a = GraphContact::synthetic_uid("AAA"); + let b = GraphContact::synthetic_uid("AAA"); + let c = GraphContact::synthetic_uid("BBB"); + assert_eq!(a, b); + assert_ne!(a, c); + } + + #[test] + fn emails_map_with_first_as_pref() { + let c = json!({ + "id": "X", + "emailAddresses": [ + {"address": "a@x.com", "name": "Alice"}, + {"address": "b@x.com", "name": "Alice B"} + ] + }); + let conv = GraphContact(&c).convert().unwrap(); + let emails = conv.data["emails"].as_object().unwrap(); + assert_eq!(emails["email-1"]["address"], "a@x.com"); + assert_eq!(emails["email-1"]["pref"], 1); + assert_eq!(emails["email-2"]["address"], "b@x.com"); + assert!(emails["email-2"].get("pref").is_none()); + } + + #[test] + fn phones_carry_context_per_source_field() { + let c = json!({ + "id": "X", + "businessPhones": ["+1-555-2000"], + "homePhones": ["+1-555-1000"], + "mobilePhone": "+1-555-3000" + }); + let conv = GraphContact(&c).convert().unwrap(); + let phones = conv.data["phones"].as_object().unwrap(); + assert_eq!(phones.len(), 3); + let has_work = phones + .values() + .any(|v| v["contexts"]["work"] == Value::Bool(true)); + let has_private = phones + .values() + .any(|v| v["contexts"]["private"] == Value::Bool(true)); + let has_mobile = phones + .values() + .any(|v| v["features"]["mobile"] == Value::Bool(true)); + assert!(has_work && has_private && has_mobile); + } + + #[test] + fn addresses_map_three_contexts() { + let c = json!({ + "id": "X", + "homeAddress": {"city": "Home City"}, + "businessAddress": {"city": "Work City"}, + "otherAddress": {"city": "Other City"} + }); + let conv = GraphContact(&c).convert().unwrap(); + let addresses = conv.data["addresses"].as_object().unwrap(); + assert_eq!(addresses.len(), 3); + let has_other_context = addresses + .values() + .any(|a| a.get("contexts").and_then(|c| c.get("other")).is_some()); + assert!(!has_other_context); + let without_context = addresses + .values() + .filter(|a| a.get("contexts").is_none()) + .count(); + assert_eq!(without_context, 1); + } + + #[test] + fn org_and_title_pair_into_separate_jscontact_objects() { + let c = json!({ + "id": "X", + "companyName": "Acme", + "department": "Engineering", + "officeLocation": "Building 9", + "jobTitle": "Architect" + }); + let conv = GraphContact(&c).convert().unwrap(); + let orgs = conv.data["organizations"].as_object().unwrap(); + assert!(orgs.contains_key("org-1")); + assert_eq!(orgs["org-1"]["name"], "Acme"); + let titles = conv.data["titles"].as_object().unwrap(); + assert_eq!(titles["title-1"]["name"], "Architect"); + assert_eq!(titles["title-1"]["organizationId"], "org-1"); + } + + #[test] + fn yomi_fields_populate_japanese_localization() { + let c = json!({ + "id": "X", + "displayName": "Yamada Taro", + "givenName": "Taro", + "surname": "Yamada", + "companyName": "Acme", + "yomiCompanyName": "アクメ", + "yomiGivenName": "タロウ", + "yomiSurname": "ヤマダ" + }); + let conv = GraphContact(&c).convert().unwrap(); + let ja = conv.data["localizations"]["ja"].as_object().unwrap(); + assert_eq!(ja["organizations/org-1/name"], "アクメ"); + } + + #[test] + fn birthday_stored_as_local_midnight_in_utc_keeps_its_day() { + for raw in ["1990-04-14T22:00:00Z", "1990-04-15T11:59:00Z", "1990-04-15"] { + let conv = GraphContact(&json!({"id": "X", "birthday": raw})) + .convert() + .unwrap(); + assert_eq!( + conv.data["anniversaries"]["anniversary-1"]["date"], + json!({"@type": "PartialDate", "year": 1990, "month": 4, "day": 15}) + ); + } + } + + #[test] + fn outlook_no_date_birthday_is_dropped() { + let c = json!({"id": "X", "birthday": "4501-01-01T00:00:00Z"}); + let conv = GraphContact(&c).convert().unwrap(); + assert!(conv.data.get("anniversaries").is_none()); + } + + #[test] + fn birthday_emits_partialdate_object() { + let c = json!({"id": "X", "birthday": "1980-04-12T00:00:00Z"}); + let conv = GraphContact(&c).convert().unwrap(); + let ann = conv.data["anniversaries"].as_object().unwrap(); + let date = &ann["anniversary-1"]["date"]; + assert_eq!(date["@type"], "PartialDate"); + assert_eq!(date["year"], 1980); + assert_eq!(date["month"], 4); + assert_eq!(date["day"], 12); + assert_eq!(ann["anniversary-1"]["kind"], "birth"); + } + + #[test] + fn mobile_phone_uses_features_not_contexts() { + let c = json!({"id": "X", "mobilePhone": "+1-555-3000"}); + let conv = GraphContact(&c).convert().unwrap(); + let phones = conv.data["phones"].as_object().unwrap(); + let phone = phones.values().next().unwrap(); + assert_eq!(phone["features"]["mobile"], true); + assert!(phone.get("contexts").is_none()); + } + + #[test] + fn generation_maps_to_generation_kind() { + let c = json!({"id": "X", "generation": "Jr."}); + let conv = GraphContact(&c).convert().unwrap(); + let comps = conv.data["name"]["components"].as_array().unwrap(); + assert!( + comps + .iter() + .any(|c| c["kind"] == "generation" && c["value"] == "Jr.") + ); + } + + #[test] + fn only_jobtitle_does_not_emit_empty_organization() { + let c = json!({"id": "X", "jobTitle": "Architect"}); + let conv = GraphContact(&c).convert().unwrap(); + assert!(conv.data.get("organizations").is_none()); + let titles = conv.data["titles"].as_object().unwrap(); + assert_eq!(titles["title-1"]["name"], "Architect"); + assert!(titles["title-1"].get("organizationId").is_none()); + } + + #[test] + fn related_contacts_are_dropped_without_resolvable_uids() { + let c = json!({ + "id": "X", + "manager": "alice@x.com", + "spouseName": "Bob", + "children": ["Kid A", "Kid B"] + }); + let conv = GraphContact(&c).convert().unwrap(); + assert!(conv.data.get("relatedTo").is_none()); + } + + #[test] + fn nameless_contact_omits_name_object() { + let c = json!({"id": "X", "businessPhones": ["+1"]}); + let conv = GraphContact(&c).convert().unwrap(); + assert!(conv.data.get("name").is_none()); + } + + #[test] + fn timestamps_strip_fractional_seconds() { + let c = json!({ + "id": "X", + "displayName": "Alice", + "createdDateTime": "2026-05-01T08:00:00.0000000Z", + "lastModifiedDateTime": "2026-05-02T08:00:00.123Z" + }); + let conv = GraphContact(&c).convert().unwrap(); + assert_eq!(conv.data["created"], "2026-05-01T08:00:00Z"); + assert_eq!(conv.data["updated"], "2026-05-02T08:00:00Z"); + } +} diff --git a/src/exchange_graph/error.rs b/src/exchange/graph/error.rs similarity index 76% rename from src/exchange_graph/error.rs rename to src/exchange/graph/error.rs index 84a6de2..e68de7c 100644 --- a/src/exchange_graph/error.rs +++ b/src/exchange/graph/error.rs @@ -5,6 +5,8 @@ */ use crate::error::Error; +use crate::exchange::oauth::OAuthError; +use crate::net::http::TransportError; #[derive(Debug, thiserror::Error)] pub enum GraphError { @@ -33,6 +35,22 @@ pub enum GraphError { Vanished, } +impl From for GraphError { + fn from(value: OAuthError) -> Self { + GraphError::OAuth(value.0) + } +} + +impl From for GraphError { + fn from(value: TransportError) -> Self { + if value.is_connect() { + GraphError::Connect(value.to_string()) + } else { + GraphError::Transport(value.to_string()) + } + } +} + impl From for Error { fn from(e: GraphError) -> Self { match e { diff --git a/src/exchange/graph/expand.rs b/src/exchange/graph/expand.rs new file mode 100644 index 0000000..de67bf8 --- /dev/null +++ b/src/exchange/graph/expand.rs @@ -0,0 +1,647 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use chrono::{Datelike, Duration, NaiveDate, Weekday}; +use serde_json::{Map, Value, json}; + +use crate::exchange::graph::recurrence::{GraphRecurrence, PatternKind}; +use crate::exchange::vocab::{DayToken, Ordinal, WeekdayName}; + +const MAX_STEPS: usize = 200_000; +const DATE_FORMAT: &str = "%Y-%m-%d"; +const DAYS_PER_WEEK: i64 = 7; +const MONTHS_PER_YEAR: i64 = 12; + +struct Series { + start: NaiveDate, + limit: Option, + window_start: NaiveDate, + window_end: NaiveDate, + emitted: usize, + out: Vec, +} + +impl Series { + fn push(&mut self, date: NaiveDate) { + if date < self.start || self.limit.is_some_and(|end| date > end) { + return; + } + self.emitted += 1; + if date >= self.window_start && date <= self.window_end { + self.out.push(date); + } + } +} + +impl GraphRecurrence<'_> { + pub fn expected_dates( + self, + window_start: NaiveDate, + window_end: NaiveDate, + ) -> Option> { + let pattern = self.0.get("pattern")?; + let range = self.0.get("range")?; + let start = GraphRecurrence::date_of(range.get("startDate"))?; + let count = range + .get("numberOfOccurrences") + .and_then(Value::as_u64) + .filter(|n| *n > 0) + .and_then(|n| usize::try_from(n).ok()); + let interval = pattern + .get("interval") + .and_then(Value::as_u64) + .filter(|i| *i > 0) + .map_or(Some(1), |i| i64::try_from(i).ok())?; + let kind = pattern + .get("type") + .and_then(Value::as_str) + .and_then(PatternKind::parse)?; + let days = GraphRecurrence::weekdays(pattern); + let ordinal = pattern + .get("index") + .and_then(Value::as_str) + .and_then(Ordinal::parse) + .unwrap_or(Ordinal::First); + let day_of_month = || { + pattern + .get("dayOfMonth") + .and_then(Value::as_u64) + .and_then(|d| u32::try_from(d).ok()) + }; + let month = || { + pattern + .get("month") + .and_then(Value::as_u64) + .and_then(|m| u32::try_from(m).ok()) + }; + let years = i32::try_from(interval).ok()?; + + let mut series = Series { + start, + limit: GraphRecurrence::date_of(range.get("endDate")), + window_start, + window_end, + emitted: 0, + out: Vec::new(), + }; + let mut cursor = start; + for _ in 0..MAX_STEPS { + match kind { + PatternKind::Daily => { + series.push(cursor); + cursor = cursor.checked_add_signed(Duration::days(interval))?; + } + PatternKind::Weekly => { + if days.is_empty() { + return None; + } + let first = GraphRecurrence::first_day_of_week(pattern); + let week_start = GraphRecurrence::start_of_week(cursor, first); + for day in &days { + series.push( + week_start + + Duration::days(GraphRecurrence::weekday_offset(first, *day)), + ); + } + cursor = week_start.checked_add_signed(Duration::weeks(interval))?; + } + PatternKind::AbsoluteMonthly => { + let dom = day_of_month()?; + if let Some(date) = + GraphRecurrence::clamped_day(cursor.year(), cursor.month(), dom) + { + series.push(date); + } + cursor = GraphRecurrence::add_months(cursor.with_day(1)?, interval)?; + } + PatternKind::RelativeMonthly => { + if let Some(date) = GraphRecurrence::nth_weekday_of_month( + cursor.year(), + cursor.month(), + &days, + ordinal, + ) { + series.push(date); + } + cursor = GraphRecurrence::add_months(cursor.with_day(1)?, interval)?; + } + PatternKind::AbsoluteYearly => { + let month = month()?; + let dom = day_of_month()?; + if let Some(date) = GraphRecurrence::clamped_day(cursor.year(), month, dom) { + series.push(date); + } + cursor = NaiveDate::from_ymd_opt(cursor.year().checked_add(years)?, 1, 1)?; + } + PatternKind::RelativeYearly => { + let month = month()?; + if let Some(date) = + GraphRecurrence::nth_weekday_of_month(cursor.year(), month, &days, ordinal) + { + series.push(date); + } + cursor = NaiveDate::from_ymd_opt(cursor.year().checked_add(years)?, 1, 1)?; + } + } + if count.is_some_and(|n| series.emitted >= n) + || series.limit.is_some_and(|end| cursor > end) + || cursor > window_end + { + break; + } + } + + series.out.sort_unstable(); + series.out.dedup(); + Some(series.out) + } + + fn date_of(value: Option<&Value>) -> Option { + let raw = value?.as_str()?; + NaiveDate::parse_from_str(raw.split('T').next().unwrap_or(raw), DATE_FORMAT).ok() + } + + fn weekdays(pattern: &Value) -> Vec { + pattern + .get("daysOfWeek") + .and_then(Value::as_array) + .map(|days| { + days.iter() + .filter_map(Value::as_str) + .filter_map(|d| WeekdayName(d).parse()) + .collect() + }) + .unwrap_or_default() + } + + fn first_day_of_week(pattern: &Value) -> Weekday { + pattern + .get("firstDayOfWeek") + .and_then(Value::as_str) + .and_then(|d| WeekdayName(d).parse()) + .unwrap_or(Weekday::Sun) + } + + fn weekday_offset(first: Weekday, day: Weekday) -> i64 { + let from = i64::from(first.num_days_from_sunday()); + let to = i64::from(day.num_days_from_sunday()); + (to - from).rem_euclid(DAYS_PER_WEEK) + } + + fn start_of_week(date: NaiveDate, first: Weekday) -> NaiveDate { + date - Duration::days(GraphRecurrence::weekday_offset(first, date.weekday())) + } + + fn clamped_day(year: i32, month: u32, day: u32) -> Option { + if !(1..=12).contains(&month) { + return None; + } + let last = GraphRecurrence::last_day_of_month(year, month)?; + NaiveDate::from_ymd_opt(year, month, day.min(last)) + } + + fn last_day_of_month(year: i32, month: u32) -> Option { + let first_next = if month == 12 { + NaiveDate::from_ymd_opt(year.checked_add(1)?, 1, 1)? + } else { + NaiveDate::from_ymd_opt(year, month + 1, 1)? + }; + first_next.pred_opt().map(|d| d.day()) + } + + fn add_months(date: NaiveDate, months: i64) -> Option { + let total = i64::from(date.year()) * MONTHS_PER_YEAR + i64::from(date.month0()) + months; + let year = i32::try_from(total.div_euclid(MONTHS_PER_YEAR)).ok()?; + let month = u32::try_from(total.rem_euclid(MONTHS_PER_YEAR) + 1).ok()?; + NaiveDate::from_ymd_opt(year, month, 1) + } + + fn nth_weekday_of_month( + year: i32, + month: u32, + days: &[Weekday], + ordinal: Ordinal, + ) -> Option { + if days.is_empty() || !(1..=12).contains(&month) { + return None; + } + let last = GraphRecurrence::last_day_of_month(year, month)?; + let mut matches: Vec = Vec::new(); + for day in 1..=last { + let date = NaiveDate::from_ymd_opt(year, month, day)?; + if days.contains(&date.weekday()) { + matches.push(date); + } + } + ordinal.pick(&matches) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn day(s: &str) -> NaiveDate { + NaiveDate::parse_from_str(s, "%Y-%m-%d").unwrap() + } + + fn expand(recurrence: Value, from: &str, to: &str) -> Vec { + GraphRecurrence(&recurrence) + .expected_dates(day(from), day(to)) + .expect("pattern is supported") + .into_iter() + .map(|d| d.to_string()) + .collect() + } + + // An interval too large for a week offset yields no dates instead of panicking. + #[test] + fn huge_weekly_interval_is_rejected_without_panicking() { + let recurrence = json!({ + "pattern": {"type": "weekly", "interval": 20000000000u64, "daysOfWeek": ["monday"]}, + "range": {"type": "noEnd", "startDate": "2024-01-01"} + }); + assert_eq!( + GraphRecurrence(&recurrence).expected_dates(day("2024-01-01"), day("2026-01-01")), + None + ); + } + + #[test] + fn daily_numbered_stops_at_the_count() { + let dates = expand( + json!({ + "pattern": {"type": "daily", "interval": 1}, + "range": {"type": "numbered", "startDate": "2026-09-21", + "numberOfOccurrences": 6} + }), + "2026-01-01", + "2027-01-01", + ); + assert_eq!( + dates, + [ + "2026-09-21", + "2026-09-22", + "2026-09-23", + "2026-09-24", + "2026-09-25", + "2026-09-26" + ] + ); + } + + #[test] + fn weekly_interval_two_keeps_all_three_weekdays() { + let dates = expand( + json!({ + "pattern": {"type": "weekly", "interval": 2, + "daysOfWeek": ["monday", "wednesday", "friday"], + "firstDayOfWeek": "monday"}, + "range": {"type": "endDate", "startDate": "2026-09-16", + "endDate": "2026-10-03"} + }), + "2026-01-01", + "2027-01-01", + ); + assert_eq!( + dates, + [ + "2026-09-16", + "2026-09-18", + "2026-09-28", + "2026-09-30", + "2026-10-02" + ], + "interval 2 skips the week of the 21st entirely" + ); + } + + #[test] + fn relative_monthly_third_thursday() { + let dates = expand( + json!({ + "pattern": {"type": "relativeMonthly", "interval": 1, + "daysOfWeek": ["thursday"], "index": "third"}, + "range": {"type": "numbered", "startDate": "2026-09-17", + "numberOfOccurrences": 3} + }), + "2026-01-01", + "2027-06-01", + ); + assert_eq!(dates, ["2026-09-17", "2026-10-15", "2026-11-19"]); + } + + #[test] + fn relative_monthly_last_weekday() { + let dates = expand( + json!({ + "pattern": {"type": "relativeMonthly", "interval": 1, + "daysOfWeek": ["friday"], "index": "last"}, + "range": {"type": "numbered", "startDate": "2026-01-30", + "numberOfOccurrences": 3} + }), + "2026-01-01", + "2027-01-01", + ); + assert_eq!(dates, ["2026-01-30", "2026-02-27", "2026-03-27"]); + } + + #[test] + fn absolute_monthly_clamps_to_a_short_month() { + let dates = expand( + json!({ + "pattern": {"type": "absoluteMonthly", "interval": 1, "dayOfMonth": 31}, + "range": {"type": "numbered", "startDate": "2026-01-31", + "numberOfOccurrences": 3} + }), + "2026-01-01", + "2027-01-01", + ); + assert_eq!( + dates, + ["2026-01-31", "2026-02-28", "2026-03-31"], + "day 31 clamps to the last day of February" + ); + } + + #[test] + fn absolute_yearly_keeps_a_leap_day_series_on_the_28th_in_common_years() { + let dates = expand( + json!({ + "pattern": {"type": "absoluteYearly", "interval": 1, + "month": 2, "dayOfMonth": 29}, + "range": {"type": "numbered", "startDate": "2024-02-29", + "numberOfOccurrences": 3} + }), + "2020-01-01", + "2030-01-01", + ); + assert_eq!(dates, ["2024-02-29", "2025-02-28", "2026-02-28"]); + } + + #[test] + fn window_clips_output_but_the_count_still_tracks_the_whole_series() { + let dates = expand( + json!({ + "pattern": {"type": "daily", "interval": 1}, + "range": {"type": "numbered", "startDate": "2026-09-21", + "numberOfOccurrences": 6} + }), + "2026-09-24", + "2026-09-25", + ); + assert_eq!( + dates, + ["2026-09-24", "2026-09-25"], + "only in-window dates are returned, but occurrences 1-3 still counted \ + toward the numbered range" + ); + } + + #[test] + fn no_end_series_stops_at_the_window() { + let dates = expand( + json!({ + "pattern": {"type": "daily", "interval": 1}, + "range": {"type": "noEnd", "startDate": "2026-09-21"} + }), + "2026-09-21", + "2026-09-24", + ); + assert_eq!( + dates, + ["2026-09-21", "2026-09-22", "2026-09-23", "2026-09-24"] + ); + } + + #[test] + fn unknown_pattern_is_reported_rather_than_guessed() { + assert!( + GraphRecurrence(&json!({ + "pattern": {"type": "somethingNew", "interval": 1}, + "range": {"type": "noEnd", "startDate": "2026-01-01"} + })) + .expected_dates(day("2026-01-01"), day("2027-01-01")) + .is_none() + ); + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Frequency { + Daily, + Weekly, + Monthly, + Yearly, +} + +#[derive(Debug, Clone, Copy)] +pub struct JsRule<'a>(pub &'a Value); + +impl JsRule<'_> { + pub fn to_graph_pattern(self, series_start: NaiveDate) -> Option { + let rule = self.0; + let frequency = rule.get("frequency").and_then(Value::as_str)?; + let interval = rule.get("interval").and_then(Value::as_u64).unwrap_or(1); + let by_day: Vec<&'static str> = rule + .get("byDay") + .and_then(Value::as_array) + .map(|days| { + days.iter() + .filter_map(|d| d.get("day").and_then(Value::as_str)) + .filter_map(DayToken::parse) + .map(|d| WeekdayName::long(d.0)) + .collect() + }) + .unwrap_or_default(); + let first_of = |key: &str| { + rule.get(key) + .and_then(Value::as_array) + .and_then(|a| a.first()) + }; + let by_month_day = first_of("byMonthDay").and_then(Value::as_i64); + let by_month = first_of("byMonth").and_then(|m| { + m.as_str() + .and_then(|s| s.parse::().ok()) + .or_else(|| m.as_i64()) + }); + let set_position = first_of("bySetPosition").and_then(Value::as_i64); + let start_day = i64::from(series_start.day()); + let start_month = i64::from(series_start.month()); + let relative = match set_position { + Some(position) if !by_day.is_empty() => Some(Ordinal::from_position(position)), + _ => None, + }; + + let mut pattern = Map::new(); + pattern.insert("interval".to_owned(), Value::from(interval)); + let frequency = hashify::map!( + frequency.as_bytes(), + Frequency, + "daily" => Frequency::Daily, + "weekly" => Frequency::Weekly, + "monthly" => Frequency::Monthly, + "yearly" => Frequency::Yearly, + ) + .copied()?; + let kind = match (frequency, relative) { + (Frequency::Daily, _) => PatternKind::Daily, + (Frequency::Weekly, _) => { + pattern.insert("daysOfWeek".to_owned(), json!(by_day)); + PatternKind::Weekly + } + (Frequency::Monthly, Some(ordinal)) => { + pattern.insert("daysOfWeek".to_owned(), json!(by_day)); + pattern.insert("index".to_owned(), Value::from(ordinal.name())); + PatternKind::RelativeMonthly + } + (Frequency::Monthly, None) => { + pattern.insert( + "dayOfMonth".to_owned(), + Value::from(by_month_day.unwrap_or(start_day)), + ); + PatternKind::AbsoluteMonthly + } + (Frequency::Yearly, Some(ordinal)) => { + pattern.insert("daysOfWeek".to_owned(), json!(by_day)); + pattern.insert("index".to_owned(), Value::from(ordinal.name())); + pattern.insert( + "month".to_owned(), + Value::from(by_month.unwrap_or(start_month)), + ); + PatternKind::RelativeYearly + } + (Frequency::Yearly, None) => { + pattern.insert( + "month".to_owned(), + Value::from(by_month.unwrap_or(start_month)), + ); + pattern.insert( + "dayOfMonth".to_owned(), + Value::from(by_month_day.unwrap_or(start_day)), + ); + PatternKind::AbsoluteYearly + } + }; + pattern.insert("type".to_owned(), Value::from(kind.as_str())); + if let Some(first) = rule + .get("firstDayOfWeek") + .and_then(Value::as_str) + .and_then(DayToken::parse) + { + pattern.insert( + "firstDayOfWeek".to_owned(), + Value::from(WeekdayName::long(first.0)), + ); + } + + let mut range = Map::new(); + range.insert( + "startDate".to_owned(), + Value::from(series_start.format(DATE_FORMAT).to_string()), + ); + if let Some(count) = rule.get("count").and_then(Value::as_u64) { + range.insert("type".to_owned(), Value::from("numbered")); + range.insert("numberOfOccurrences".to_owned(), Value::from(count)); + } else if let Some(until) = rule.get("until").and_then(Value::as_str) { + range.insert("type".to_owned(), Value::from("endDate")); + range.insert( + "endDate".to_owned(), + Value::from(until.split('T').next().unwrap_or(until)), + ); + } else { + range.insert("type".to_owned(), Value::from("noEnd")); + } + + Some(json!({"pattern": Value::Object(pattern), "range": Value::Object(range)})) + } +} + +#[cfg(test)] +mod roundtrip_tests { + use super::*; + use serde_json::json; + + fn day(s: &str) -> NaiveDate { + NaiveDate::parse_from_str(s, "%Y-%m-%d").unwrap() + } + + fn roundtrip(graph: Value, start: &str, from: &str, to: &str) { + let direct = GraphRecurrence(&graph) + .expected_dates(day(from), day(to)) + .expect("graph pattern expands"); + let rule = GraphRecurrence(&graph) + .to_rule() + .expect("converts to jscalendar"); + let back = JsRule(&rule) + .to_graph_pattern(day(start)) + .expect("jscalendar rule maps back to a graph pattern"); + let viarule = GraphRecurrence(&back) + .expected_dates(day(from), day(to)) + .expect("mapped pattern expands"); + assert_eq!( + direct, viarule, + "a stored rule must expand to the same dates as the Graph pattern it came from" + ); + } + + #[test] + fn stored_rules_expand_identically_to_their_graph_patterns() { + roundtrip( + json!({"pattern": {"type": "daily", "interval": 2}, + "range": {"type": "numbered", "startDate": "2026-09-21", + "numberOfOccurrences": 5}}), + "2026-09-21", + "2026-01-01", + "2027-01-01", + ); + roundtrip( + json!({"pattern": {"type": "weekly", "interval": 2, + "daysOfWeek": ["monday", "wednesday", "friday"], + "firstDayOfWeek": "monday"}, + "range": {"type": "endDate", "startDate": "2026-09-16", + "endDate": "2026-12-31"}}), + "2026-09-16", + "2026-01-01", + "2027-01-01", + ); + roundtrip( + json!({"pattern": {"type": "relativeMonthly", "interval": 1, + "daysOfWeek": ["thursday"], "index": "third"}, + "range": {"type": "numbered", "startDate": "2026-09-17", + "numberOfOccurrences": 6}}), + "2026-09-17", + "2026-01-01", + "2028-01-01", + ); + roundtrip( + json!({"pattern": {"type": "absoluteMonthly", "interval": 1, "dayOfMonth": 31}, + "range": {"type": "numbered", "startDate": "2026-01-31", + "numberOfOccurrences": 4}}), + "2026-01-31", + "2026-01-01", + "2027-01-01", + ); + roundtrip( + json!({"pattern": {"type": "absoluteYearly", "interval": 1, + "month": 11, "dayOfMonth": 29}, + "range": {"type": "noEnd", "startDate": "2026-11-29"}}), + "2026-11-29", + "2026-01-01", + "2031-01-01", + ); + roundtrip( + json!({"pattern": {"type": "relativeYearly", "interval": 1, + "daysOfWeek": ["monday"], "index": "last", "month": 5}, + "range": {"type": "noEnd", "startDate": "2026-05-25"}}), + "2026-05-25", + "2026-01-01", + "2031-01-01", + ); + } +} diff --git a/src/exchange_graph/mod.rs b/src/exchange/graph/mod.rs similarity index 100% rename from src/exchange_graph/mod.rs rename to src/exchange/graph/mod.rs diff --git a/src/exchange/graph/oauth.rs b/src/exchange/graph/oauth.rs new file mode 100644 index 0000000..3f8dd4e --- /dev/null +++ b/src/exchange/graph/oauth.rs @@ -0,0 +1,39 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::time::Duration; + +use crate::exchange::oauth::RefreshSchedule; + +pub const SCOPES: &str = "offline_access User.Read Mail.Read MailboxSettings.Read Calendars.Read Contacts.Read Files.Read"; +pub const REFRESH_SCHEDULE: RefreshSchedule = RefreshSchedule { + lead: Duration::from_secs(60), + fallback_lifetime: Duration::from_secs(50 * 60), +}; +pub const REFRESH_ATTEMPTS: u32 = 4; + +#[cfg(test)] +mod tests { + use super::*; + + // Device code consent covers every --objects surface, including the drive. + #[test] + fn scopes_cover_every_surface() { + for required in [ + "offline_access", + "Mail.Read", + "MailboxSettings.Read", + "Calendars.Read", + "Contacts.Read", + "Files.Read", + ] { + assert!( + SCOPES.split_whitespace().any(|s| s == required), + "missing {required}" + ); + } + } +} diff --git a/src/exchange/graph/recurrence.rs b/src/exchange/graph/recurrence.rs new file mode 100644 index 0000000..3a3d7e2 --- /dev/null +++ b/src/exchange/graph/recurrence.rs @@ -0,0 +1,374 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::{Map, Value, json}; + +use crate::exchange::date::WireDate; +use crate::exchange::graph::error::GraphError; +use crate::exchange::vocab::{DayToken, Ordinal, WeekdayName}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PatternKind { + Daily, + Weekly, + AbsoluteMonthly, + RelativeMonthly, + AbsoluteYearly, + RelativeYearly, +} + +impl PatternKind { + pub fn parse(value: &str) -> Option { + hashify::map!( + value.as_bytes(), + PatternKind, + "daily" => PatternKind::Daily, + "weekly" => PatternKind::Weekly, + "absoluteMonthly" => PatternKind::AbsoluteMonthly, + "relativeMonthly" => PatternKind::RelativeMonthly, + "absoluteYearly" => PatternKind::AbsoluteYearly, + "relativeYearly" => PatternKind::RelativeYearly, + ) + .copied() + } + + pub fn as_str(self) -> &'static str { + match self { + PatternKind::Daily => "daily", + PatternKind::Weekly => "weekly", + PatternKind::AbsoluteMonthly => "absoluteMonthly", + PatternKind::RelativeMonthly => "relativeMonthly", + PatternKind::AbsoluteYearly => "absoluteYearly", + PatternKind::RelativeYearly => "relativeYearly", + } + } + + fn frequency(self) -> &'static str { + match self { + PatternKind::Daily => "daily", + PatternKind::Weekly => "weekly", + PatternKind::AbsoluteMonthly | PatternKind::RelativeMonthly => "monthly", + PatternKind::AbsoluteYearly | PatternKind::RelativeYearly => "yearly", + } + } + + fn is_relative(self) -> bool { + matches!( + self, + PatternKind::RelativeMonthly | PatternKind::RelativeYearly + ) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RangeKind { + EndDate, + Numbered, + NoEnd, +} + +impl RangeKind { + pub fn parse(value: &str) -> RangeKind { + hashify::map!( + value.as_bytes(), + RangeKind, + "endDate" => RangeKind::EndDate, + "numbered" => RangeKind::Numbered, + ) + .copied() + .unwrap_or(RangeKind::NoEnd) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct GraphRecurrence<'a>(pub &'a Value); + +impl GraphRecurrence<'_> { + pub fn to_rule(self) -> Result { + self.to_rule_map().map(Value::Object) + } + + pub fn to_rule_map(self) -> Result, GraphError> { + let pattern = self + .0 + .get("pattern") + .ok_or_else(|| GraphError::Malformed("recurrence.pattern missing".to_owned()))?; + let range = self + .0 + .get("range") + .ok_or_else(|| GraphError::Malformed("recurrence.range missing".to_owned()))?; + let kind = pattern + .get("type") + .and_then(Value::as_str) + .and_then(PatternKind::parse); + + let mut rule = Map::new(); + rule.insert("@type".to_owned(), Value::from("RecurrenceRule")); + rule.insert( + "frequency".to_owned(), + Value::from(kind.map_or("daily", PatternKind::frequency)), + ); + + if let Some(interval) = pattern.get("interval").and_then(Value::as_u64) + && interval != 1 + { + rule.insert("interval".to_owned(), Value::from(interval)); + } + + if let Some(day) = GraphRecurrence::weekday(pattern.get("firstDayOfWeek")) { + rule.insert("firstDayOfWeek".to_owned(), Value::from(day.as_str())); + } + + if let Some(days) = pattern.get("daysOfWeek").and_then(Value::as_array) { + let by_day: Vec = days + .iter() + .filter_map(|d| GraphRecurrence::weekday(Some(d))) + .map(|d| json!({"@type": "NDay", "day": d.as_str()})) + .collect(); + if !by_day.is_empty() { + rule.insert("byDay".to_owned(), Value::Array(by_day)); + } + } + + if let Some(dom) = pattern.get("dayOfMonth").and_then(Value::as_i64) + && dom != 0 + { + rule.insert("byMonthDay".to_owned(), json!([dom])); + } + + if let Some(month) = pattern.get("month").and_then(Value::as_i64) + && month != 0 + { + rule.insert("byMonth".to_owned(), json!([month.to_string()])); + } + + if kind.is_some_and(PatternKind::is_relative) + && let Some(ordinal) = pattern + .get("index") + .and_then(Value::as_str) + .and_then(Ordinal::parse) + { + rule.insert( + "bySetPosition".to_owned(), + json!([i64::from(ordinal.position())]), + ); + } + + let range_kind = range + .get("type") + .and_then(Value::as_str) + .map_or(RangeKind::NoEnd, RangeKind::parse); + match range_kind { + RangeKind::EndDate => { + if let Some(local) = range + .get("endDate") + .and_then(Value::as_str) + .map(WireDate) + .filter(|end| !end.is_outlook_no_date()) + .and_then(WireDate::recurrence_until) + { + rule.insert("until".to_owned(), Value::from(local)); + } + } + RangeKind::Numbered => { + if let Some(count) = range.get("numberOfOccurrences").and_then(Value::as_u64) { + rule.insert("count".to_owned(), Value::from(count)); + } + } + RangeKind::NoEnd => {} + } + + Ok(rule) + } + + fn weekday(name: Option<&Value>) -> Option { + name.and_then(Value::as_str) + .and_then(|n| WeekdayName(n).parse()) + .map(DayToken) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn daily_with_interval() { + let pr = json!({ + "pattern": {"type": "daily", "interval": 3}, + "range": {"type": "noEnd"} + }); + let out = GraphRecurrence(&pr).to_rule().unwrap(); + let rule = &out; + assert_eq!(rule["frequency"], "daily"); + assert_eq!(rule["interval"], 3); + assert!(rule.get("until").is_none()); + assert!(rule.get("count").is_none()); + } + + #[test] + fn weekly_with_days_of_week() { + let pr = json!({ + "pattern": { + "type": "weekly", + "interval": 1, + "daysOfWeek": ["monday", "wednesday", "friday"], + "firstDayOfWeek": "monday" + }, + "range": {"type": "noEnd"} + }); + let out = GraphRecurrence(&pr).to_rule().unwrap(); + let rule = &out; + assert_eq!(rule["frequency"], "weekly"); + assert_eq!(rule["firstDayOfWeek"], "mo"); + let by_day = rule["byDay"].as_array().unwrap(); + assert_eq!(by_day.len(), 3); + assert_eq!(by_day[0]["day"], "mo"); + assert_eq!(by_day[1]["day"], "we"); + assert_eq!(by_day[2]["day"], "fr"); + } + + #[test] + fn absolute_monthly_uses_by_month_day() { + let pr = json!({ + "pattern": {"type": "absoluteMonthly", "interval": 1, "dayOfMonth": 15}, + "range": {"type": "noEnd"} + }); + let out = GraphRecurrence(&pr).to_rule().unwrap(); + let rule = &out; + assert_eq!(rule["frequency"], "monthly"); + assert_eq!(rule["byMonthDay"][0], 15); + } + + #[test] + fn relative_monthly_emits_by_set_position() { + let pr = json!({ + "pattern": { + "type": "relativeMonthly", + "interval": 1, + "daysOfWeek": ["thursday"], + "index": "third" + }, + "range": {"type": "noEnd"} + }); + let out = GraphRecurrence(&pr).to_rule().unwrap(); + let rule = &out; + assert_eq!(rule["frequency"], "monthly"); + assert_eq!(rule["bySetPosition"][0], 3); + assert_eq!(rule["byDay"][0]["day"], "th"); + } + + #[test] + fn index_is_ignored_unless_the_pattern_is_relative() { + for kind in ["daily", "weekly", "absoluteMonthly", "absoluteYearly"] { + let pr = json!({ + "pattern": {"type": kind, "interval": 1, "index": "first", + "daysOfWeek": ["monday"], "dayOfMonth": 1, "month": 1}, + "range": {"type": "noEnd"} + }); + let out = GraphRecurrence(&pr).to_rule().unwrap(); + assert!( + out.get("bySetPosition").is_none(), + "{kind} carries Graph's default index=first, which must not become bySetPosition" + ); + } + } + + #[test] + fn index_survives_for_relative_patterns() { + for kind in ["relativeMonthly", "relativeYearly"] { + let pr = json!({ + "pattern": {"type": kind, "interval": 1, "index": "third", + "daysOfWeek": ["thursday"]}, + "range": {"type": "noEnd"} + }); + let out = GraphRecurrence(&pr).to_rule().unwrap(); + assert_eq!(out["bySetPosition"][0], 3, "{kind} keeps its index"); + } + } + + #[test] + fn absolute_yearly_with_month_and_day() { + let pr = json!({ + "pattern": {"type": "absoluteYearly", "interval": 1, "month": 7, "dayOfMonth": 4}, + "range": {"type": "noEnd"} + }); + let out = GraphRecurrence(&pr).to_rule().unwrap(); + let rule = &out; + assert_eq!(rule["frequency"], "yearly"); + assert_eq!(rule["byMonth"][0], "7"); + assert_eq!(rule["byMonthDay"][0], 4); + } + + #[test] + fn relative_yearly_uses_setpos_and_byday() { + let pr = json!({ + "pattern": { + "type": "relativeYearly", "interval": 1, "daysOfWeek": ["monday"], + "index": "last", "month": 5 + }, + "range": {"type": "noEnd"} + }); + let out = GraphRecurrence(&pr).to_rule().unwrap(); + let rule = &out; + assert_eq!(rule["frequency"], "yearly"); + assert_eq!(rule["bySetPosition"][0], -1); + assert_eq!(rule["byMonth"][0], "5"); + } + + #[test] + fn range_end_date_maps_to_until_local_datetime() { + let pr = json!({ + "pattern": {"type": "daily", "interval": 1}, + "range": {"type": "endDate", "endDate": "2026-12-31"} + }); + let out = GraphRecurrence(&pr).to_rule().unwrap(); + assert_eq!(out["until"], "2026-12-31T23:59:59"); + } + + #[test] + fn range_end_on_outlook_no_date_has_no_until() { + let rule = GraphRecurrence(&json!({ + "pattern": {"type": "daily", "interval": 1}, + "range": {"type": "endDate", "startDate": "2026-01-01", "endDate": "4500-12-31"} + })) + .to_rule() + .unwrap(); + assert!(rule.get("until").is_none()); + } + + #[test] + fn range_end_datetime_preserved() { + let pr = json!({ + "pattern": {"type": "daily", "interval": 1}, + "range": {"type": "endDate", "endDate": "2026-12-31T23:59:59"} + }); + let out = GraphRecurrence(&pr).to_rule().unwrap(); + assert_eq!(out["until"], "2026-12-31T23:59:59"); + } + + #[test] + fn range_numbered_maps_to_count() { + let pr = json!({ + "pattern": {"type": "weekly", "interval": 1, "daysOfWeek": ["monday"]}, + "range": {"type": "numbered", "numberOfOccurrences": 10} + }); + let out = GraphRecurrence(&pr).to_rule().unwrap(); + assert_eq!(out["count"], 10); + } + + #[test] + fn no_end_range_omits_until_and_count() { + let pr = json!({ + "pattern": {"type": "daily", "interval": 1}, + "range": {"type": "noEnd"} + }); + let out = GraphRecurrence(&pr).to_rule().unwrap(); + assert!(out.get("until").is_none()); + assert!(out.get("count").is_none()); + } +} diff --git a/src/exchange/graph/retry.rs b/src/exchange/graph/retry.rs new file mode 100644 index 0000000..74bf22a --- /dev/null +++ b/src/exchange/graph/retry.rs @@ -0,0 +1,93 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HttpClass { + Success, + Retryable, + Auth, + Vanished, + Fatal, +} + +const UNAUTHORIZED: u16 = 401; +const FORBIDDEN: u16 = 403; +const NOT_FOUND: u16 = 404; +const GONE: u16 = 410; +const TOO_MANY_REQUESTS: u16 = 429; +const INTERNAL_SERVER_ERROR: u16 = 500; +const BAD_GATEWAY: u16 = 502; +const SERVICE_UNAVAILABLE: u16 = 503; +const GATEWAY_TIMEOUT: u16 = 504; + +impl HttpClass { + pub fn of_status(status: u16) -> HttpClass { + match status { + 200..=299 => HttpClass::Success, + UNAUTHORIZED | FORBIDDEN => HttpClass::Auth, + NOT_FOUND | GONE => HttpClass::Vanished, + TOO_MANY_REQUESTS + | INTERNAL_SERVER_ERROR + | BAD_GATEWAY + | SERVICE_UNAVAILABLE + | GATEWAY_TIMEOUT => HttpClass::Retryable, + _ => HttpClass::Fatal, + } + } + + pub fn arms_throttle(status: u16) -> bool { + matches!(status, TOO_MANY_REQUESTS | SERVICE_UNAVAILABLE) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rate_limit_status_is_retryable() { + assert_eq!(HttpClass::of_status(429), HttpClass::Retryable); + assert!(HttpClass::arms_throttle(429)); + } + + #[test] + fn service_unavailable_is_retryable_and_throttle_marked() { + assert_eq!(HttpClass::of_status(503), HttpClass::Retryable); + assert!(HttpClass::arms_throttle(503)); + } + + #[test] + fn bad_gateway_and_timeout_retry() { + assert_eq!(HttpClass::of_status(502), HttpClass::Retryable); + assert_eq!(HttpClass::of_status(504), HttpClass::Retryable); + } + + #[test] + fn insufficient_storage_is_per_item_fatal() { + assert_eq!(HttpClass::of_status(507), HttpClass::Fatal); + } + + #[test] + fn auth_failure_is_marked_distinctly() { + assert_eq!(HttpClass::of_status(401), HttpClass::Auth); + assert_eq!(HttpClass::of_status(403), HttpClass::Auth); + } + + #[test] + fn not_found_and_gone_are_vanished() { + assert_eq!(HttpClass::of_status(404), HttpClass::Vanished); + assert_eq!(HttpClass::of_status(410), HttpClass::Vanished); + } + + #[test] + fn unknown_4xx_is_fatal() { + assert_eq!(HttpClass::of_status(400), HttpClass::Fatal); + assert_eq!(HttpClass::of_status(405), HttpClass::Fatal); + assert_eq!(HttpClass::of_status(408), HttpClass::Fatal); + assert_eq!(HttpClass::of_status(422), HttpClass::Fatal); + assert_eq!(HttpClass::of_status(423), HttpClass::Fatal); + } +} diff --git a/src/exchange_graph/types.rs b/src/exchange/graph/types.rs similarity index 73% rename from src/exchange_graph/types.rs rename to src/exchange/graph/types.rs index 25cd61d..08c53b2 100644 --- a/src/exchange_graph/types.rs +++ b/src/exchange/graph/types.rs @@ -5,6 +5,7 @@ */ use crate::error::Error; +use crate::exchange::mailbox::{MailboxKind as ExchangeMailboxKind, SyntheticAccountId}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MailboxKind { @@ -14,22 +15,32 @@ pub enum MailboxKind { impl MailboxKind { pub fn parse(value: &str) -> Result { - match value { - "primary" => Ok(MailboxKind::Primary), - "archive" => Ok(MailboxKind::Archive), - "public-folders" => Err(Error::Usage( + match ExchangeMailboxKind::parse(value) { + Some(ExchangeMailboxKind::Primary) => Ok(MailboxKind::Primary), + Some(ExchangeMailboxKind::Archive) => Ok(MailboxKind::Archive), + Some(ExchangeMailboxKind::PublicFolders) => Err(Error::Usage( "Microsoft Graph does not expose public folders. Run `inbuxa-migrate import exchange-ews --mailbox-kind public-folders` instead.".to_owned(), )), - other => Err(Error::Usage(format!( - "--mailbox-kind must be primary | archive, got {other:?}" + None => Err(Error::Usage(format!( + "--mailbox-kind must be primary | archive, got {value:?}" ))), } } pub fn as_str(self) -> &'static str { - match self { - MailboxKind::Primary => "primary", - MailboxKind::Archive => "archive", + ExchangeMailboxKind::from(self).as_str() + } + + pub fn account_id(self, directory_id: &str) -> SyntheticAccountId<'_> { + ExchangeMailboxKind::from(self).account_id(directory_id) + } +} + +impl From for ExchangeMailboxKind { + fn from(kind: MailboxKind) -> ExchangeMailboxKind { + match kind { + MailboxKind::Primary => ExchangeMailboxKind::Primary, + MailboxKind::Archive => ExchangeMailboxKind::Archive, } } } @@ -42,13 +53,18 @@ pub enum EventBodyFormat { impl EventBodyFormat { pub fn parse(value: &str) -> Result { - match value { - "text" => Ok(EventBodyFormat::Text), - "html" => Ok(EventBodyFormat::Html), - other => Err(Error::Usage(format!( - "--event-body-format must be text | html, got {other:?}" - ))), - } + hashify::map!( + value.as_bytes(), + EventBodyFormat, + "text" => EventBodyFormat::Text, + "html" => EventBodyFormat::Html, + ) + .copied() + .ok_or_else(|| { + Error::Usage(format!( + "--event-body-format must be text | html, got {value:?}" + )) + }) } pub fn prefer_value(self) -> &'static str { @@ -59,6 +75,14 @@ impl EventBodyFormat { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Surface { + Mail, + Calendar, + Contacts, + Files, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Surfaces { pub mail: bool, @@ -95,12 +119,19 @@ impl Surfaces { if token.is_empty() { continue; } - match token.to_ascii_lowercase().as_str() { - "mail" => selected.mail = true, - "calendar" => selected.calendar = true, - "contacts" => selected.contacts = true, - "files" => selected.files = true, - _ => { + match hashify::map_ignore_case!( + token.as_bytes(), + Surface, + "mail" => Surface::Mail, + "calendar" => Surface::Calendar, + "contacts" => Surface::Contacts, + "files" => Surface::Files, + ) { + Some(Surface::Mail) => selected.mail = true, + Some(Surface::Calendar) => selected.calendar = true, + Some(Surface::Contacts) => selected.contacts = true, + Some(Surface::Files) => selected.files = true, + None => { return Err(Error::Usage(format!( "unknown surface: {token} (valid: mail, calendar, contacts, files)" ))); @@ -122,13 +153,6 @@ pub struct ResolvedPrincipal { pub user_principal_name: String, } -pub fn synthetic_account_id(directory_id: &str, kind: MailboxKind) -> String { - match kind { - MailboxKind::Primary => directory_id.to_owned(), - MailboxKind::Archive => format!("{directory_id}#archive"), - } -} - #[cfg(test)] mod tests { use super::*; @@ -150,11 +174,11 @@ mod tests { #[test] fn synthetic_id_matches_ews_pattern() { assert_eq!( - synthetic_account_id("u-uuid", MailboxKind::Primary), + MailboxKind::Primary.account_id("u-uuid").to_string(), "u-uuid" ); assert_eq!( - synthetic_account_id("u-uuid", MailboxKind::Archive), + MailboxKind::Archive.account_id("u-uuid").to_string(), "u-uuid#archive" ); } diff --git a/src/exchange/jscalendar.rs b/src/exchange/jscalendar.rs index cec6020..1aebe20 100644 --- a/src/exchange/jscalendar.rs +++ b/src/exchange/jscalendar.rs @@ -4,21 +4,16 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +use std::fmt; + use serde_json::{Map, Value}; -const IGNORED_EXACT: &[&str] = &[ - "@type", - "method", - "organizerCalendarAddress", - "privacy", - "prodId", - "recurrenceId", - "recurrenceIdTimeZone", - "sentBy", - "uid", -]; +use crate::exchange::address::SmtpAddress; -const IGNORED_FIRST_TOKEN: &[&str] = &["recurrenceOverrides", "recurrenceRule", "relatedTo"]; +const MAILTO: &str = "mailto:"; +const SYNTHETIC_ATTENDEE_PREFIX: &str = "urn:x-inbuxa-migrate:attendee:"; +const PARTICIPANTS: &str = "participants"; +const CALENDAR_ADDRESS: &str = "calendarAddress"; const CALENDAR_ADDRESS_DEPENDENTS: &[&str] = &[ "calendarAddress", @@ -34,50 +29,128 @@ const CALENDAR_ADDRESS_DEPENDENTS: &[&str] = &[ "sentBy", ]; -pub fn synthetic_attendee_address(identifier: &str) -> String { - format!( - "urn:x-inbuxa-migrate:attendee:{}", - blake3::hash(identifier.as_bytes()).to_hex() - ) +#[derive(Debug, Clone, Copy)] +pub struct SyntheticAttendee<'a>(pub &'a str); + +impl fmt::Display for SyntheticAttendee<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{SYNTHETIC_ATTENDEE_PREFIX}{}", + blake3::hash(self.0.as_bytes()).to_hex() + ) + } } -pub fn drop_calendar_address_dependents(participants: &mut Map) { - for participant in participants.values_mut() { - let Some(object) = participant.as_object_mut() else { - continue; - }; - for key in CALENDAR_ADDRESS_DEPENDENTS { - object.remove(*key); +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Routing { + Smtp, + Other, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedAddress { + pub uri: String, + pub email: Option, +} + +impl ResolvedAddress { + pub fn resolve( + address: Option<&str>, + routing: Routing, + name: Option<&str>, + ) -> Option { + if let Some(addr) = address.filter(|a| !a.trim().is_empty()) { + let smtp = match routing { + Routing::Smtp => SmtpAddress::parse(addr), + Routing::Other => SmtpAddress::extract(addr), + }; + return Some(match smtp { + Some(smtp) => ResolvedAddress { + uri: format!("{MAILTO}{}", smtp.as_str()), + email: Some(smtp.into_string()), + }, + None => ResolvedAddress::synthetic(addr), + }); + } + name.filter(|n| !n.trim().is_empty()) + .map(ResolvedAddress::synthetic) + } + + fn synthetic(identifier: &str) -> ResolvedAddress { + ResolvedAddress { + uri: SyntheticAttendee(identifier).to_string(), + email: None, } } } -pub fn is_override_ignored(pointer: &str) -> bool { - if IGNORED_EXACT.contains(&pointer) || is_participant_calendar_address(pointer) { - return true; +pub struct ParticipantMap<'a>(pub &'a mut Map); + +impl ParticipantMap<'_> { + pub fn drop_calendar_address_dependents(self) { + for object in self.0.values_mut().filter_map(Value::as_object_mut) { + for key in CALENDAR_ADDRESS_DEPENDENTS { + object.remove(*key); + } + } } - let first = pointer.split('/').next().unwrap_or(pointer); - IGNORED_FIRST_TOKEN.contains(&first) } -fn is_participant_calendar_address(pointer: &str) -> bool { - let mut tokens = pointer.split('/'); - tokens.next() == Some("participants") - && tokens.next().is_some() - && tokens.next() == Some("calendarAddress") - && tokens.next().is_none() +#[derive(Debug, Clone, Copy)] +pub struct OverridePointer<'a>(pub &'a str); + +impl OverridePointer<'_> { + pub fn is_ignored(self) -> bool { + let pointer = self.0; + if hashify::set!( + pointer.as_bytes(), + "@type", + "method", + "organizerCalendarAddress", + "privacy", + "prodId", + "recurrenceId", + "recurrenceIdTimeZone", + "sentBy", + "uid", + ) || self.is_participant_calendar_address() + { + return true; + } + let first = pointer.split('/').next().unwrap_or(pointer); + hashify::set!( + first.as_bytes(), + "recurrenceOverrides", + "recurrenceRule", + "relatedTo", + ) + } + + fn is_participant_calendar_address(self) -> bool { + let mut tokens = self.0.split('/'); + tokens.next() == Some(PARTICIPANTS) + && tokens.next().is_some() + && tokens.next() == Some(CALENDAR_ADDRESS) + && tokens.next().is_none() + } } -pub fn override_patch_from_event(event: &Value) -> Value { - let Some(object) = event.as_object() else { - return event.clone(); - }; - let patch: Map = object - .iter() - .filter(|(key, _)| !is_override_ignored(key)) - .map(|(key, value)| (key.clone(), value.clone())) - .collect(); - Value::Object(patch) +pub struct OverridePatch; + +impl OverridePatch { + pub fn from_event(event: &Value) -> Value { + let Some(object) = event.as_object() else { + return event.clone(); + }; + Value::Object( + object + .iter() + .filter(|(key, _)| !OverridePointer(key).is_ignored()) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + ) + } } #[cfg(test)] @@ -98,31 +171,28 @@ mod tests { "sentBy", "uid", ] { - assert!(is_override_ignored(pointer), "{pointer} must be ignored"); + assert!( + OverridePointer(pointer).is_ignored(), + "{pointer} must be ignored" + ); } } #[test] fn first_token_pointers_are_ignored_at_any_depth() { - assert!(is_override_ignored("recurrenceRule")); - assert!(is_override_ignored("recurrenceRule/frequency")); - assert!(is_override_ignored( - "recurrenceOverrides/2026-01-01T09:00:00" - )); - assert!(is_override_ignored("relatedTo/uid-1/relation")); + assert!(OverridePointer("recurrenceRule").is_ignored()); + assert!(OverridePointer("recurrenceRule/frequency").is_ignored()); + assert!(OverridePointer("recurrenceOverrides/2026-01-01T09:00:00").is_ignored()); + assert!(OverridePointer("relatedTo/uid-1/relation").is_ignored()); } #[test] fn participant_calendar_address_is_ignored_only_at_its_own_depth() { - assert!(is_override_ignored("participants/att-1/calendarAddress")); - assert!(!is_override_ignored("participants")); - assert!(!is_override_ignored("participants/att-1")); - assert!(!is_override_ignored( - "participants/att-1/participationStatus" - )); - assert!(!is_override_ignored( - "participants/att-1/calendarAddress/extra" - )); + assert!(OverridePointer("participants/att-1/calendarAddress").is_ignored()); + assert!(!OverridePointer("participants").is_ignored()); + assert!(!OverridePointer("participants/att-1").is_ignored()); + assert!(!OverridePointer("participants/att-1/participationStatus").is_ignored()); + assert!(!OverridePointer("participants/att-1/calendarAddress/extra").is_ignored()); } #[test] @@ -137,7 +207,10 @@ mod tests { "updated", "status", ] { - assert!(!is_override_ignored(pointer), "{pointer} must survive"); + assert!( + !OverridePointer(pointer).is_ignored(), + "{pointer} must survive" + ); } } @@ -154,7 +227,7 @@ mod tests { "duration": "PT1H", "title": "Moved" }); - let patch = override_patch_from_event(&event); + let patch = OverridePatch::from_event(&event); let patch = patch.as_object().expect("patch object"); assert_eq!(patch.len(), 3); assert_eq!(patch["start"], "2026-03-04T09:00:00"); @@ -164,15 +237,15 @@ mod tests { #[test] fn an_exclusion_patch_is_preserved() { - let patch = override_patch_from_event(&json!({"excluded": true})); + let patch = OverridePatch::from_event(&json!({"excluded": true})); assert_eq!(patch, json!({"excluded": true})); } #[test] fn a_synthetic_address_is_stable_per_identifier_and_never_a_mailto() { - let addr = synthetic_attendee_address("Jane Doe"); - assert_eq!(addr, synthetic_attendee_address("Jane Doe")); - assert_ne!(addr, synthetic_attendee_address("John Doe")); + let addr = SyntheticAttendee("Jane Doe").to_string(); + assert_eq!(addr, SyntheticAttendee("Jane Doe").to_string()); + assert_ne!(addr, SyntheticAttendee("John Doe").to_string()); assert!(addr.starts_with("urn:x-inbuxa-migrate:attendee:")); assert!( !addr.starts_with("mailto:"), @@ -198,7 +271,7 @@ mod tests { .as_object() .expect("participants object") .clone(); - drop_calendar_address_dependents(&mut participants); + ParticipantMap(&mut participants).drop_calendar_address_dependents(); let p = &participants["1"]; assert_eq!(p["@type"], "Participant"); assert_eq!(p["name"], "Room 4"); diff --git a/src/exchange/mailbox.rs b/src/exchange/mailbox.rs new file mode 100644 index 0000000..852575f --- /dev/null +++ b/src/exchange/mailbox.rs @@ -0,0 +1,113 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fmt; + +const ARCHIVE_SUFFIX: &str = "#archive"; +const PUBLIC_FOLDERS_LOCAL: &str = "__public_folders__@"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MailboxKind { + Primary, + Archive, + PublicFolders, +} + +impl MailboxKind { + pub fn parse(value: &str) -> Option { + hashify::map!( + value.as_bytes(), + MailboxKind, + "primary" => MailboxKind::Primary, + "archive" => MailboxKind::Archive, + "public-folders" => MailboxKind::PublicFolders, + ) + .copied() + } + + pub fn as_str(self) -> &'static str { + match self { + MailboxKind::Primary => "primary", + MailboxKind::Archive => "archive", + MailboxKind::PublicFolders => "public-folders", + } + } + + pub fn account_id(self, owner: &str) -> SyntheticAccountId<'_> { + SyntheticAccountId { owner, kind: self } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct SyntheticAccountId<'a> { + owner: &'a str, + kind: MailboxKind, +} + +impl fmt::Display for SyntheticAccountId<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.kind { + MailboxKind::Primary => f.write_str(self.owner), + MailboxKind::Archive => { + f.write_str(self.owner)?; + f.write_str(ARCHIVE_SUFFIX) + } + MailboxKind::PublicFolders => { + f.write_str(PUBLIC_FOLDERS_LOCAL)?; + f.write_str(self.owner.split('@').nth(1).unwrap_or_default()) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn primary_uses_the_owner_verbatim() { + assert_eq!( + MailboxKind::Primary + .account_id("alice@contoso.com") + .to_string(), + "alice@contoso.com" + ); + } + + #[test] + fn archive_appends_the_suffix() { + assert_eq!( + MailboxKind::Archive.account_id("u-uuid").to_string(), + "u-uuid#archive" + ); + } + + #[test] + fn public_folders_use_the_tenant_domain() { + assert_eq!( + MailboxKind::PublicFolders + .account_id("alice@contoso.com") + .to_string(), + "__public_folders__@contoso.com" + ); + assert_eq!( + MailboxKind::PublicFolders.account_id("alice").to_string(), + "__public_folders__@" + ); + } + + #[test] + fn kinds_parse_exactly() { + for kind in [ + MailboxKind::Primary, + MailboxKind::Archive, + MailboxKind::PublicFolders, + ] { + assert_eq!(MailboxKind::parse(kind.as_str()), Some(kind)); + } + assert_eq!(MailboxKind::parse("Primary"), None); + } +} diff --git a/src/exchange/mod.rs b/src/exchange/mod.rs index 74610f8..99a5df9 100644 --- a/src/exchange/mod.rs +++ b/src/exchange/mod.rs @@ -6,5 +6,10 @@ pub mod address; pub mod date; +pub mod ews; +pub mod graph; pub mod jscalendar; +pub mod mailbox; +pub mod oauth; pub mod tz; +pub mod vocab; diff --git a/src/exchange/oauth/mod.rs b/src/exchange/oauth/mod.rs new file mode 100644 index 0000000..7b50f15 --- /dev/null +++ b/src/exchange/oauth/mod.rs @@ -0,0 +1,465 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod refresh; +pub mod token; + +use std::fmt; +use std::thread; +use std::time::{Duration, Instant}; + +use serde_json::Value; +use ureq::Body; +use ureq::http::Response; + +use crate::net::encode::FormBody; +use crate::net::http::{AgentProfile, ClientOptions, HttpCore, Methods, Redirects}; + +pub use refresh::{RefreshPlan, RefreshSchedule, TokenRefresher}; +pub use token::{AcquiredToken, DeviceCodeResponse, JwtClaims, TokenResponse}; + +const LOGIN_HOST: &str = "https://login.microsoftonline.com"; +const DEVICE_CODE_GRANT: &str = "urn:ietf:params:oauth:grant-type:device_code"; +const FORM_CONTENT_TYPE: &str = "application/x-www-form-urlencoded"; +const MAX_TOKEN_BODY: u64 = 10 * 1024 * 1024; +const SLOW_DOWN_STEP: Duration = Duration::from_secs(5); +const REFRESH_BACKOFF_START: Duration = Duration::from_millis(500); +const REFRESH_BACKOFF_CAP: Duration = Duration::from_secs(8); +const PROFILE: AgentProfile = AgentProfile { + redirects: Redirects::Follow, + methods: Methods::Standard, + max_body: MAX_TOKEN_BODY, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OAuthError(pub String); + +impl fmt::Display for OAuthError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Dialect { + Ews, + Graph, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Authority(String); + +#[derive(Debug, Clone, Copy)] +pub struct Pace { + pub interval: Duration, + pub slow_down: Duration, +} + +#[derive(Debug)] +pub enum PollFailure { + DeadlinePassed, + Transport(String), + Body(String), + Refused { status: u16, payload: Value }, +} + +pub struct DeviceCodeFlow<'a> { + pub response: &'a DeviceCodeResponse, + pub client_id: &'a str, + pub token_endpoint: &'a str, + pub pace: Pace, +} + +pub struct OAuthClient<'a> { + core: HttpCore, + authority: &'a Authority, + client_id: &'a str, + scope: &'static str, + dialect: Dialect, +} + +impl Authority { + pub fn for_tenant(tenant: &str) -> Authority { + Authority(format!("{LOGIN_HOST}/{tenant}")) + } + + pub fn new(authority: String) -> Authority { + Authority(authority) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn into_string(self) -> String { + self.0 + } + + pub fn token_endpoint(&self) -> String { + format!("{}/oauth2/v2.0/token", self.0) + } + + pub fn device_code_endpoint(&self) -> String { + format!("{}/oauth2/v2.0/devicecode", self.0) + } +} + +impl Pace { + pub fn of(response: &DeviceCodeResponse) -> Pace { + Pace { + interval: Duration::from_secs(response.interval), + slow_down: SLOW_DOWN_STEP, + } + } +} + +impl Dialect { + pub fn poll_error(self, failure: PollFailure) -> OAuthError { + let text = match (self, failure) { + (Dialect::Ews, PollFailure::DeadlinePassed) => { + "device code expired before the user completed sign-in".to_owned() + } + (Dialect::Graph, PollFailure::DeadlinePassed) => { + "device code expired before user completed sign-in".to_owned() + } + (_, PollFailure::Transport(e)) => format!("device code polling: {e}"), + (_, PollFailure::Body(e)) => format!("token body: {e}"), + (Dialect::Ews, PollFailure::Refused { status, payload }) => { + Dialect::ews_refusal(status, &payload) + } + (Dialect::Graph, PollFailure::Refused { status, payload }) => { + match TokenResponse::parse(status, &payload) { + TokenResponse::Expired => "device code expired; restart the import".to_owned(), + TokenResponse::Declined => { + "the user declined the device-code sign-in request".to_owned() + } + TokenResponse::BadCode => "device code rejected".to_owned(), + TokenResponse::Other { error, description } => { + format!("device code polling: {error}: {description}") + } + other => format!("device code polling: {other:?}"), + } + } + }; + OAuthError(text) + } + + fn ews_refusal(status: u16, payload: &Value) -> String { + if (200..300).contains(&status) { + return "access_token missing".to_owned(); + } + if let TokenResponse::Expired = TokenResponse::parse(status, payload) { + return "device code expired; restart the import".to_owned(); + } + let error = payload + .get("error") + .and_then(Value::as_str) + .unwrap_or_default(); + if error.is_empty() { + return format!("device code polling: http {status}"); + } + let desc = payload + .get("error_description") + .and_then(Value::as_str) + .unwrap_or_default(); + format!("device code polling: {error}: {desc}") + } +} + +impl DeviceCodeFlow<'_> { + pub fn poll( + &self, + mut step: impl FnMut(&str, &FormBody) -> Result<(u16, Value), PollFailure>, + ) -> Result { + let deadline = Instant::now() + Duration::from_secs(self.response.expires_in); + let form = FormBody::from_pairs(&[ + ("client_id", self.client_id), + ("grant_type", DEVICE_CODE_GRANT), + ("device_code", &self.response.device_code), + ]); + let mut delay = self.pace.interval; + loop { + thread::sleep(delay); + if Instant::now() >= deadline { + return Err(PollFailure::DeadlinePassed); + } + let (status, payload) = step(self.token_endpoint, &form)?; + match TokenResponse::parse(status, &payload) { + TokenResponse::Ok(token) => return Ok(token), + TokenResponse::Pending => {} + TokenResponse::SlowDown => delay = delay.saturating_add(self.pace.slow_down), + _ => return Err(PollFailure::Refused { status, payload }), + } + } + } +} + +impl<'a> OAuthClient<'a> { + pub fn new( + options: ClientOptions, + authority: &'a Authority, + client_id: &'a str, + scope: &'static str, + dialect: Dialect, + ) -> OAuthClient<'a> { + OAuthClient { + core: HttpCore::new(options, PROFILE), + authority, + client_id, + scope, + dialect, + } + } + + fn post(&self, url: &str, form: &FormBody) -> Result, ureq::Error> { + self.core + .agent() + .post(url) + .header("Content-Type", FORM_CONTENT_TYPE) + .send(form.as_str().as_bytes()) + } + + fn read_json(&self, resp: &mut Response) -> Result { + resp.body_mut() + .with_config() + .limit(self.core.max_body()) + .read_json() + } + + fn json(&self, resp: &mut Response) -> Value { + self.read_json(resp).unwrap_or(Value::Null) + } + + pub fn device_code(&self) -> Result { + let form = FormBody::from_pairs(&[("client_id", self.client_id), ("scope", self.scope)]); + let mut resp = self + .post(&self.authority.device_code_endpoint(), &form) + .map_err(|e| OAuthError(format!("device code endpoint: {e}")))?; + let json = self + .read_json(&mut resp) + .map_err(|e| OAuthError(format!("device code body: {e}")))?; + let response = DeviceCodeResponse::parse(&json)?; + response.announce(self.core.logger()); + let token_endpoint = self.authority.token_endpoint(); + DeviceCodeFlow { + response: &response, + client_id: self.client_id, + token_endpoint: &token_endpoint, + pace: Pace::of(&response), + } + .poll(|url, form| { + let mut resp = self + .post(url, form) + .map_err(|e| PollFailure::Transport(e.to_string()))?; + let payload = match self.dialect { + Dialect::Ews if resp.status().is_success() => self + .read_json(&mut resp) + .map_err(|e| PollFailure::Body(e.to_string()))?, + Dialect::Ews | Dialect::Graph => self.json(&mut resp), + }; + Ok((resp.status().as_u16(), payload)) + }) + .map_err(|failure| self.dialect.poll_error(failure)) + } + + pub fn client_credentials(&self, secret: &str) -> Result { + let form = FormBody::from_pairs(&[ + ("client_id", self.client_id), + ("client_secret", secret), + ("scope", self.scope), + ("grant_type", "client_credentials"), + ]); + let resp = self + .post(&self.authority.token_endpoint(), &form) + .map_err(|e| OAuthError(format!("token endpoint: {e}")))?; + self.strict_token(resp) + } + + pub fn refresh(&self, refresh_token: &str, attempts: u32) -> Result { + let form = FormBody::from_pairs(&[ + ("client_id", self.client_id), + ("grant_type", "refresh_token"), + ("refresh_token", refresh_token), + ("scope", self.scope), + ]); + let endpoint = self.authority.token_endpoint(); + match self.dialect { + Dialect::Ews => { + let resp = self + .post(&endpoint, &form) + .map_err(|e| OAuthError(format!("refresh token endpoint: {e}")))?; + self.strict_token(resp) + } + Dialect::Graph => self.refresh_with_retries(&endpoint, &form, attempts), + } + } + + fn refresh_with_retries( + &self, + endpoint: &str, + form: &FormBody, + attempts: u32, + ) -> Result { + let mut delay = REFRESH_BACKOFF_START; + let mut attempt: u32 = 0; + loop { + attempt += 1; + let mut resp = match self.post(endpoint, form) { + Ok(r) => r, + Err(e) if attempt < attempts => { + thread::sleep(delay); + delay = (delay * 2).min(REFRESH_BACKOFF_CAP); + self.core.logger().info(format_args!( + "Retrying the Graph token refresh after a transport error: {e}" + )); + continue; + } + Err(e) => return Err(OAuthError(format!("refresh endpoint: {e}"))), + }; + let status = resp.status().as_u16(); + let payload = self.json(&mut resp); + if resp.status().is_server_error() && attempt < attempts { + thread::sleep(delay); + delay = (delay * 2).min(REFRESH_BACKOFF_CAP); + continue; + } + return match TokenResponse::parse(status, &payload) { + TokenResponse::Ok(t) => Ok(t), + other => Err(OAuthError(format!( + "refresh failed (http {status}): {other:?}" + ))), + }; + } + } + + fn strict_token(&self, mut resp: Response) -> Result { + let status = resp.status().as_u16(); + let json = self + .read_json(&mut resp) + .map_err(|e| OAuthError(format!("token body: {e}")))?; + if !(200..300).contains(&status) { + let desc = json + .get("error_description") + .and_then(Value::as_str) + .unwrap_or("(no error_description)"); + return Err(OAuthError(format!("http {status}: {desc}"))); + } + AcquiredToken::from_json(&json).ok_or_else(|| OAuthError("access_token missing".to_owned())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn response() -> DeviceCodeResponse { + DeviceCodeResponse { + device_code: "DC".to_owned(), + user_code: "U".to_owned(), + verification_uri: "x".to_owned(), + interval: 0, + expires_in: 5, + message: None, + } + } + + fn poll( + reply: impl FnMut(&str, &FormBody) -> Result<(u16, Value), PollFailure>, + ) -> Result { + let response = response(); + DeviceCodeFlow { + response: &response, + client_id: "client-id", + token_endpoint: "https://login.x/common/oauth2/v2.0/token", + pace: Pace { + interval: Duration::from_millis(1), + slow_down: Duration::from_millis(1), + }, + } + .poll(reply) + } + + #[test] + fn authority_endpoints() { + let authority = Authority::for_tenant("common"); + assert_eq!( + authority.as_str(), + "https://login.microsoftonline.com/common" + ); + assert_eq!( + authority.token_endpoint(), + "https://login.microsoftonline.com/common/oauth2/v2.0/token" + ); + assert_eq!( + authority.device_code_endpoint(), + "https://login.microsoftonline.com/common/oauth2/v2.0/devicecode" + ); + } + + #[test] + fn poll_sends_the_device_code_grant() { + let token = poll(|endpoint, form| { + assert!(endpoint.ends_with("/token")); + assert_eq!( + form.as_str(), + "client_id=client-id&grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Adevice_code&device_code=DC" + ); + Ok((200, json!({"access_token": "a.b.c"}))) + }) + .unwrap(); + assert_eq!(token.access_token, "a.b.c"); + } + + #[test] + fn ews_and_graph_keep_their_refusal_texts() { + let declined = || { + poll(|_, _| { + Ok(( + 400, + json!({"error": "authorization_declined", "error_description": "no"}), + )) + }) + .unwrap_err() + }; + assert_eq!( + Dialect::Ews.poll_error(declined()).0, + "device code polling: authorization_declined: no" + ); + assert_eq!( + Dialect::Graph.poll_error(declined()).0, + "the user declined the device-code sign-in request" + ); + let silent = poll(|_, _| Ok((500, Value::Null))).unwrap_err(); + assert_eq!( + Dialect::Ews.poll_error(silent).0, + "device code polling: http 500" + ); + assert_eq!( + Dialect::Ews.poll_error(PollFailure::DeadlinePassed).0, + "device code expired before the user completed sign-in" + ); + assert_eq!( + Dialect::Graph.poll_error(PollFailure::DeadlinePassed).0, + "device code expired before user completed sign-in" + ); + } + + #[test] + fn poll_slows_down_then_succeeds() { + let mut calls = 0; + poll(|_, _| { + calls += 1; + Ok(match calls { + 1 => (400, json!({"error": "slow_down"})), + 2 => (400, json!({"error": "authorization_pending"})), + _ => (200, json!({"access_token": "t"})), + }) + }) + .unwrap(); + assert_eq!(calls, 3); + } +} diff --git a/src/exchange/oauth/refresh.rs b/src/exchange/oauth/refresh.rs new file mode 100644 index 0000000..2f44650 --- /dev/null +++ b/src/exchange/oauth/refresh.rs @@ -0,0 +1,182 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use crate::exchange::oauth::OAuthError; +use crate::exchange::oauth::token::AcquiredToken; +use crate::logging::Logger; + +const SLEEP_CHUNK: Duration = Duration::from_secs(2); +const RETRY_AFTER_FAILURE: Duration = Duration::from_secs(60); + +#[derive(Debug, Clone, Copy)] +pub struct RefreshSchedule { + pub lead: Duration, + pub fallback_lifetime: Duration, +} + +pub struct RefreshPlan { + pub label: &'static str, + pub schedule: RefreshSchedule, + pub deadline_unix: u64, + pub refresh_token: Option, + pub logger: Logger, +} + +pub struct TokenRefresher { + shutdown: Arc, + handle: Option>, +} + +struct Sleeper(Arc); + +impl Sleeper { + fn stopped(&self) -> bool { + self.0.load(Ordering::Relaxed) + } + + fn sleep(&self, total: Duration) -> bool { + let mut left = total; + while !left.is_zero() && !self.stopped() { + let step = SLEEP_CHUNK.min(left); + thread::sleep(step); + left = left.saturating_sub(step); + } + !self.stopped() + } +} + +impl TokenRefresher { + pub fn unix_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) + } + + pub fn spawn(plan: RefreshPlan, mut renew: R, apply: A) -> Option + where + R: FnMut(Option<&str>) -> Result + Send + 'static, + A: Fn(&AcquiredToken) + Send + 'static, + { + let shutdown = Arc::new(AtomicBool::new(false)); + let sleeper = Sleeper(shutdown.clone()); + let handle = thread::Builder::new() + .name(format!( + "inbuxa-migrate-{}-token-refresh", + plan.label.to_ascii_lowercase() + )) + .spawn(move || { + let RefreshPlan { + label, + schedule, + mut deadline_unix, + mut refresh_token, + logger, + } = plan; + while !sleeper.stopped() { + let now = TokenRefresher::unix_now(); + let refresh_at = deadline_unix.saturating_sub(schedule.lead.as_secs()); + if refresh_at > now && !sleeper.sleep(Duration::from_secs(refresh_at - now)) { + break; + } + match renew(refresh_token.as_deref()) { + Ok(token) => { + apply(&token); + let now = TokenRefresher::unix_now(); + deadline_unix = token + .deadline_unix(now) + .filter(|deadline| *deadline > now) + .unwrap_or_else(|| { + now.saturating_add(schedule.fallback_lifetime.as_secs()) + }); + if token.refresh_token.is_some() { + refresh_token = token.refresh_token; + } + } + Err(e) => { + logger.warn(format_args!( + "{label} token refresh failed: {e}; sleeping 60s before retry" + )); + sleeper.sleep(RETRY_AFTER_FAILURE); + } + } + } + }) + .ok()?; + Some(TokenRefresher { + shutdown, + handle: Some(handle), + }) + } +} + +impl Drop for TokenRefresher { + fn drop(&mut self) { + self.shutdown.store(true, Ordering::Relaxed); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + use std::time::Instant; + + use crate::logging::Level; + + const SCHEDULE: RefreshSchedule = RefreshSchedule { + lead: Duration::from_secs(60), + fallback_lifetime: Duration::from_secs(3000), + }; + + // A due token is renewed with the stored refresh token, and drop joins promptly. + #[test] + fn renews_a_due_token_and_joins_on_drop() { + let seen = Arc::new(Mutex::new(Vec::new())); + let applied = Arc::new(Mutex::new(Vec::new())); + let seen_in = seen.clone(); + let applied_in = applied.clone(); + let refresher = TokenRefresher::spawn( + RefreshPlan { + label: "test", + schedule: SCHEDULE, + deadline_unix: TokenRefresher::unix_now(), + refresh_token: Some("R0".to_owned()), + logger: Logger::new(Level::Quiet), + }, + move |refresh| { + seen_in.lock().unwrap().push(refresh.map(str::to_owned)); + Ok(AcquiredToken { + access_token: "fresh".to_owned(), + refresh_token: Some("R1".to_owned()), + expires_in: Some(3600), + tenant_id: None, + upn: None, + name: None, + }) + }, + move |token| applied_in.lock().unwrap().push(token.access_token.clone()), + ) + .unwrap(); + let started = Instant::now(); + while applied.lock().unwrap().is_empty() && started.elapsed() < Duration::from_secs(5) { + thread::sleep(Duration::from_millis(5)); + } + let dropped = Instant::now(); + drop(refresher); + assert!(dropped.elapsed() < Duration::from_secs(5)); + assert_eq!(applied.lock().unwrap().as_slice(), ["fresh"]); + assert_eq!(seen.lock().unwrap().as_slice(), [Some("R0".to_owned())]); + } +} diff --git a/src/exchange/oauth/token.rs b/src/exchange/oauth/token.rs new file mode 100644 index 0000000..bbdb4d7 --- /dev/null +++ b/src/exchange/oauth/token.rs @@ -0,0 +1,308 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use encodify::base64::{Base64, Padding, URL_SAFE}; +use serde_json::Value; + +use crate::exchange::oauth::OAuthError; +use crate::logging::Logger; + +const JWT_SEGMENT: Base64 = URL_SAFE.with_padding(Padding::Optional).any_alphabet(); +const DEFAULT_USER_CODE: &str = "?"; +const DEFAULT_VERIFICATION_URI: &str = "https://microsoft.com/devicelogin"; +const DEFAULT_INTERVAL_SECS: u64 = 5; +const DEFAULT_DEVICE_CODE_LIFETIME_SECS: u64 = 900; + +#[derive(Debug, Clone, Default)] +pub struct JwtClaims { + pub tenant_id: Option, + pub upn: Option, + pub name: Option, + pub exp: Option, +} + +#[derive(Debug, Clone)] +pub struct AcquiredToken { + pub access_token: String, + pub refresh_token: Option, + pub expires_in: Option, + pub tenant_id: Option, + pub upn: Option, + pub name: Option, +} + +#[derive(Debug, Clone)] +pub enum TokenResponse { + Ok(AcquiredToken), + Pending, + SlowDown, + Expired, + Declined, + BadCode, + Other { error: String, description: String }, +} + +#[derive(Debug, Clone)] +pub struct DeviceCodeResponse { + pub device_code: String, + pub user_code: String, + pub verification_uri: String, + pub interval: u64, + pub expires_in: u64, + pub message: Option, +} + +impl JwtClaims { + pub fn decode(token: &str) -> Option { + let mut parts = token.split('.'); + let _header = parts.next()?; + let payload = parts.next()?; + let bytes = JWT_SEGMENT.decode(payload).ok()?; + let value: Value = serde_json::from_slice(&bytes).ok()?; + Some(JwtClaims { + tenant_id: value.get("tid").and_then(Value::as_str).map(str::to_owned), + upn: value + .get("upn") + .and_then(Value::as_str) + .or_else(|| value.get("preferred_username").and_then(Value::as_str)) + .map(str::to_owned), + name: value.get("name").and_then(Value::as_str).map(str::to_owned), + exp: value.get("exp").and_then(Value::as_u64), + }) + } +} + +impl AcquiredToken { + pub fn from_bearer(token: String) -> AcquiredToken { + AcquiredToken::with_claims(token, None, None) + } + + fn with_claims( + access_token: String, + refresh_token: Option, + expires_in: Option, + ) -> AcquiredToken { + let claims = JwtClaims::decode(&access_token).unwrap_or_default(); + AcquiredToken { + access_token, + refresh_token, + expires_in, + tenant_id: claims.tenant_id, + upn: claims.upn, + name: claims.name, + } + } + + pub fn from_json(json: &Value) -> Option { + let access = json.get("access_token").and_then(Value::as_str)?; + Some(AcquiredToken::with_claims( + access.to_owned(), + json.get("refresh_token") + .and_then(Value::as_str) + .map(str::to_owned), + json.get("expires_in").and_then(Value::as_u64), + )) + } + + pub fn deadline_unix(&self, now: u64) -> Option { + match self.expires_in { + Some(expires_in) => Some(now.saturating_add(expires_in)), + None => JwtClaims::decode(&self.access_token).and_then(|c| c.exp), + } + } +} + +impl TokenResponse { + pub fn parse(status: u16, json: &Value) -> TokenResponse { + if (200..300).contains(&status) { + return AcquiredToken::from_json(json).map_or_else( + || TokenResponse::Other { + error: "missing_access_token".to_owned(), + description: String::new(), + }, + TokenResponse::Ok, + ); + } + let error = json + .get("error") + .and_then(Value::as_str) + .unwrap_or_default(); + hashify::map!( + error.as_bytes(), + TokenResponse, + "authorization_pending" => TokenResponse::Pending, + "slow_down" => TokenResponse::SlowDown, + "expired_token" => TokenResponse::Expired, + "authorization_declined" => TokenResponse::Declined, + "bad_verification_code" => TokenResponse::BadCode, + ) + .cloned() + .unwrap_or_else(|| TokenResponse::Other { + error: error.to_owned(), + description: json + .get("error_description") + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned(), + }) + } +} + +impl DeviceCodeResponse { + pub fn parse(json: &Value) -> Result { + let text = |key: &str, default: &str| { + json.get(key) + .and_then(Value::as_str) + .unwrap_or(default) + .to_owned() + }; + let device_code = json + .get("device_code") + .and_then(Value::as_str) + .ok_or_else(|| OAuthError("device_code missing".to_owned()))? + .to_owned(); + Ok(DeviceCodeResponse { + device_code, + user_code: text("user_code", DEFAULT_USER_CODE), + verification_uri: text("verification_uri", DEFAULT_VERIFICATION_URI), + interval: json + .get("interval") + .and_then(Value::as_u64) + .unwrap_or(DEFAULT_INTERVAL_SECS), + expires_in: json + .get("expires_in") + .and_then(Value::as_u64) + .unwrap_or(DEFAULT_DEVICE_CODE_LIFETIME_SECS), + message: json + .get("message") + .and_then(Value::as_str) + .map(str::to_owned), + }) + } + + pub fn announce(&self, logger: Logger) { + match self.message.as_deref() { + Some(message) => logger.notice(format_args!("{message}")), + None => logger.notice(format_args!( + "To sign in, open {} and enter the code {}", + self.verification_uri, self.user_code + )), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use encodify::base64::{STANDARD, STANDARD_NO_PAD, URL_SAFE_NO_PAD}; + use serde_json::json; + + fn make_jwt(tid: &str, upn: &str, exp: u64) -> String { + let header = URL_SAFE_NO_PAD.encode(b"{\"alg\":\"none\"}"); + let claims = format!(r#"{{"tid":"{tid}","upn":"{upn}","exp":{exp}}}"#); + let payload = URL_SAFE_NO_PAD.encode(claims.as_bytes()); + format!("{header}.{payload}.") + } + + #[test] + fn jwt_decoder_extracts_tid_upn_and_exp() { + let claims = JwtClaims::decode(&make_jwt("tenant-1", "alice@x.com", 9999999999)).unwrap(); + assert_eq!(claims.tenant_id.as_deref(), Some("tenant-1")); + assert_eq!(claims.upn.as_deref(), Some("alice@x.com")); + assert_eq!(claims.exp, Some(9999999999)); + } + + #[test] + fn jwt_payload_decodes_in_either_alphabet_with_or_without_padding() { + let claims = r#"{"tid":"t-3","upn":"??>>@xy","exp":1}"#; + for engine in [URL_SAFE_NO_PAD, URL_SAFE, STANDARD_NO_PAD, STANDARD] { + let token = format!("h.{}.s", engine.encode(claims)); + let decoded = JwtClaims::decode(&token).unwrap(); + assert_eq!(decoded.upn.as_deref(), Some("??>>@xy"), "{token}"); + } + assert!(JwtClaims::decode("h.eyJ0aWQiOiJ0LTMifQ.s").is_some()); + assert!(JwtClaims::decode("h.eyJ0aWQiOiJ0LTMifQ=.s").is_none()); + assert!(JwtClaims::decode("h.eyJ0 aWQiOiJ0LTMifQ.s").is_none()); + } + + #[test] + fn malformed_token_returns_none() { + assert!(JwtClaims::decode("garbage").is_none()); + assert!(JwtClaims::decode("only.two").is_none()); + } + + #[test] + fn bearer_token_carries_its_claims() { + let token = make_jwt("t-2", "bob@x", 9999999999); + let acq = AcquiredToken::from_bearer(token.clone()); + assert_eq!(acq.access_token, token); + assert_eq!(acq.tenant_id.as_deref(), Some("t-2")); + assert_eq!(acq.upn.as_deref(), Some("bob@x")); + assert_eq!(acq.deadline_unix(0), Some(9999999999)); + } + + #[test] + fn expires_in_wins_over_the_jwt_expiry() { + let token = make_jwt("t", "u", 9999999999); + let acq = + AcquiredToken::from_json(&json!({"access_token": token, "expires_in": 60})).unwrap(); + assert_eq!(acq.deadline_unix(1000), Some(1060)); + } + + #[test] + fn token_response_classifies_errors() { + let classify = |error: &str| TokenResponse::parse(400, &json!({"error": error})); + assert!(matches!( + classify("authorization_pending"), + TokenResponse::Pending + )); + assert!(matches!(classify("slow_down"), TokenResponse::SlowDown)); + assert!(matches!(classify("expired_token"), TokenResponse::Expired)); + assert!(matches!( + classify("authorization_declined"), + TokenResponse::Declined + )); + assert!(matches!( + classify("bad_verification_code"), + TokenResponse::BadCode + )); + assert!(matches!( + classify("invalid_grant"), + TokenResponse::Other { ref error, .. } if error == "invalid_grant" + )); + } + + #[test] + fn token_response_success_includes_refresh() { + let json = json!({ + "access_token": make_jwt("t-2", "bob@x.com", 9999999999), + "refresh_token": "REFRESH", + "expires_in": 3599 + }); + match TokenResponse::parse(200, &json) { + TokenResponse::Ok(t) => { + assert_eq!(t.refresh_token.as_deref(), Some("REFRESH")); + assert_eq!(t.expires_in, Some(3599)); + assert_eq!(t.upn.as_deref(), Some("bob@x.com")); + } + other => panic!("expected Ok, got {other:?}"), + } + assert!(matches!( + TokenResponse::parse(200, &json!({})), + TokenResponse::Other { ref error, .. } if error == "missing_access_token" + )); + } + + #[test] + fn device_code_response_defaults() { + let parsed = DeviceCodeResponse::parse(&json!({"device_code": "D"})).unwrap(); + assert_eq!(parsed.user_code, "?"); + assert_eq!(parsed.verification_uri, "https://microsoft.com/devicelogin"); + assert_eq!(parsed.interval, 5); + assert_eq!(parsed.expires_in, 900); + assert!(DeviceCodeResponse::parse(&json!({})).is_err()); + } +} diff --git a/src/exchange/tz.rs b/src/exchange/tz.rs index d8df894..f8dc3db 100644 --- a/src/exchange/tz.rs +++ b/src/exchange/tz.rs @@ -4,181 +4,201 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -pub fn windows_to_iana(windows_name: &str) -> Option<&'static str> { - let key = windows_name.trim(); - for (w, i) in TABLE { - if w.eq_ignore_ascii_case(key) { - return Some(i); +use std::str::FromStr; + +use calcard::common::timezone::Tz; + +const MICROSOFT_TZONE: &str = "tzone://Microsoft/"; +const MICROSOFT_UTC: &str = "Utc"; +const UTC: &str = "Etc/UTC"; + +type Iana = &'static str; + +macro_rules! windows_zones { + ($($windows:literal => $iana:literal,)+) => { + #[cfg(test)] + const WINDOWS_NAMES: &[&str] = &[$($windows,)+]; + + impl ZoneId<'_> { + pub fn windows_iana(self) -> Option<&'static str> { + hashify::map_ignore_case!( + self.0.trim().as_bytes(), + Iana, + $($windows => $iana,)+ + ) + .copied() + } } - } - None + }; } -pub fn resolve_to_iana(value: &str) -> Option { - use calcard::common::timezone::Tz; - use std::str::FromStr; +#[derive(Debug, Clone, Copy)] +pub struct ZoneId<'a>(pub &'a str); - let key = value.trim(); - if key.is_empty() { - return None; +impl ZoneId<'_> { + pub fn iana(self) -> Option { + let key = self.0.trim(); + if key.is_empty() { + return None; + } + if let Some(rest) = key.strip_prefix(MICROSOFT_TZONE) { + return rest + .eq_ignore_ascii_case(MICROSOFT_UTC) + .then(|| UTC.to_owned()); + } + if let Some(iana) = self.windows_iana() { + return Some(iana.to_owned()); + } + Tz::from_str(key) + .ok() + .and_then(|tz| tz.name().map(|name| name.into_owned())) } - if let Some(rest) = key.strip_prefix("tzone://Microsoft/") { - return rest - .eq_ignore_ascii_case("Utc") - .then(|| "Etc/UTC".to_owned()); - } - if let Some(iana) = windows_to_iana(key) { - return Some(iana.to_owned()); - } - Tz::from_str(key) - .ok() - .and_then(|tz| tz.name().map(|name| name.into_owned())) } -const TABLE: &[(&str, &str)] = &[ - ("UTC", "Etc/UTC"), - ("Coordinated Universal Time", "Etc/UTC"), - ("UTC-11", "Etc/GMT+11"), - ("UTC-09", "Etc/GMT+9"), - ("UTC-08", "Etc/GMT+8"), - ("UTC-02", "Etc/GMT+2"), - ("UTC+12", "Etc/GMT-12"), - ("UTC+13", "Etc/GMT-13"), - ("Dateline Standard Time", "Etc/GMT+12"), - ("Aleutian Standard Time", "America/Adak"), - ("Hawaiian Standard Time", "Pacific/Honolulu"), - ("Marquesas Standard Time", "Pacific/Marquesas"), - ("Alaskan Standard Time", "America/Anchorage"), - ("Pacific Standard Time (Mexico)", "America/Tijuana"), - ("Pacific Standard Time", "America/Los_Angeles"), - ("US Mountain Standard Time", "America/Phoenix"), - ("Mountain Standard Time (Mexico)", "America/Mazatlan"), - ("Mountain Standard Time", "America/Denver"), - ("Yukon Standard Time", "America/Whitehorse"), - ("Central America Standard Time", "America/Guatemala"), - ("Central Standard Time", "America/Chicago"), - ("Easter Island Standard Time", "Pacific/Easter"), - ("Central Standard Time (Mexico)", "America/Mexico_City"), - ("Canada Central Standard Time", "America/Regina"), - ("SA Pacific Standard Time", "America/Bogota"), - ("Eastern Standard Time (Mexico)", "America/Cancun"), - ("Eastern Standard Time", "America/New_York"), - ("Haiti Standard Time", "America/Port-au-Prince"), - ("Cuba Standard Time", "America/Havana"), - ("US Eastern Standard Time", "America/Indianapolis"), - ("Turks And Caicos Standard Time", "America/Grand_Turk"), - ("Paraguay Standard Time", "America/Asuncion"), - ("Atlantic Standard Time", "America/Halifax"), - ("Venezuela Standard Time", "America/Caracas"), - ("Central Brazilian Standard Time", "America/Cuiaba"), - ("SA Western Standard Time", "America/La_Paz"), - ("Pacific SA Standard Time", "America/Santiago"), - ("Newfoundland Standard Time", "America/St_Johns"), - ("Tocantins Standard Time", "America/Araguaina"), - ("E. South America Standard Time", "America/Sao_Paulo"), - ("SA Eastern Standard Time", "America/Cayenne"), - ("Argentina Standard Time", "America/Argentina/Buenos_Aires"), - ("Greenland Standard Time", "America/Godthab"), - ("Montevideo Standard Time", "America/Montevideo"), - ("Magallanes Standard Time", "America/Punta_Arenas"), - ("Saint Pierre Standard Time", "America/Miquelon"), - ("Bahia Standard Time", "America/Bahia"), - ("Mid-Atlantic Standard Time", "Atlantic/South_Georgia"), - ("Azores Standard Time", "Atlantic/Azores"), - ("Cape Verde Standard Time", "Atlantic/Cape_Verde"), - ("GMT Standard Time", "Europe/London"), - ("Greenwich Standard Time", "Atlantic/Reykjavik"), - ("Sao Tome Standard Time", "Africa/Sao_Tome"), - ("Morocco Standard Time", "Africa/Casablanca"), - ("W. Europe Standard Time", "Europe/Berlin"), - ("Central Europe Standard Time", "Europe/Budapest"), - ("Romance Standard Time", "Europe/Paris"), - ("Central European Standard Time", "Europe/Warsaw"), - ("W. Central Africa Standard Time", "Africa/Lagos"), - ("Jordan Standard Time", "Asia/Amman"), - ("GTB Standard Time", "Europe/Bucharest"), - ("Middle East Standard Time", "Asia/Beirut"), - ("Egypt Standard Time", "Africa/Cairo"), - ("E. Europe Standard Time", "Europe/Chisinau"), - ("Syria Standard Time", "Asia/Damascus"), - ("West Bank Standard Time", "Asia/Hebron"), - ("South Africa Standard Time", "Africa/Johannesburg"), - ("FLE Standard Time", "Europe/Kiev"), - ("Israel Standard Time", "Asia/Jerusalem"), - ("South Sudan Standard Time", "Africa/Juba"), - ("Kaliningrad Standard Time", "Europe/Kaliningrad"), - ("Sudan Standard Time", "Africa/Khartoum"), - ("Libya Standard Time", "Africa/Tripoli"), - ("Namibia Standard Time", "Africa/Windhoek"), - ("Arabic Standard Time", "Asia/Baghdad"), - ("Turkey Standard Time", "Europe/Istanbul"), - ("Arab Standard Time", "Asia/Riyadh"), - ("Belarus Standard Time", "Europe/Minsk"), - ("Russian Standard Time", "Europe/Moscow"), - ("E. Africa Standard Time", "Africa/Nairobi"), - ("Iran Standard Time", "Asia/Tehran"), - ("Arabian Standard Time", "Asia/Dubai"), - ("Astrakhan Standard Time", "Europe/Astrakhan"), - ("Azerbaijan Standard Time", "Asia/Baku"), - ("Russia Time Zone 3", "Europe/Samara"), - ("Mauritius Standard Time", "Indian/Mauritius"), - ("Saratov Standard Time", "Europe/Saratov"), - ("Georgian Standard Time", "Asia/Tbilisi"), - ("Volgograd Standard Time", "Europe/Volgograd"), - ("Caucasus Standard Time", "Asia/Yerevan"), - ("Afghanistan Standard Time", "Asia/Kabul"), - ("West Asia Standard Time", "Asia/Tashkent"), - ("Ekaterinburg Standard Time", "Asia/Yekaterinburg"), - ("Pakistan Standard Time", "Asia/Karachi"), - ("Qyzylorda Standard Time", "Asia/Qyzylorda"), - ("India Standard Time", "Asia/Kolkata"), - ("Sri Lanka Standard Time", "Asia/Colombo"), - ("Nepal Standard Time", "Asia/Katmandu"), - ("Central Asia Standard Time", "Asia/Almaty"), - ("Bangladesh Standard Time", "Asia/Dhaka"), - ("Omsk Standard Time", "Asia/Omsk"), - ("Myanmar Standard Time", "Asia/Rangoon"), - ("SE Asia Standard Time", "Asia/Bangkok"), - ("Altai Standard Time", "Asia/Barnaul"), - ("W. Mongolia Standard Time", "Asia/Hovd"), - ("North Asia Standard Time", "Asia/Krasnoyarsk"), - ("N. Central Asia Standard Time", "Asia/Novosibirsk"), - ("Tomsk Standard Time", "Asia/Tomsk"), - ("China Standard Time", "Asia/Shanghai"), - ("North Asia East Standard Time", "Asia/Irkutsk"), - ("Singapore Standard Time", "Asia/Singapore"), - ("W. Australia Standard Time", "Australia/Perth"), - ("Taipei Standard Time", "Asia/Taipei"), - ("Ulaanbaatar Standard Time", "Asia/Ulaanbaatar"), - ("Aus Central W. Standard Time", "Australia/Eucla"), - ("Transbaikal Standard Time", "Asia/Chita"), - ("Tokyo Standard Time", "Asia/Tokyo"), - ("North Korea Standard Time", "Asia/Pyongyang"), - ("Korea Standard Time", "Asia/Seoul"), - ("Yakutsk Standard Time", "Asia/Yakutsk"), - ("Cen. Australia Standard Time", "Australia/Adelaide"), - ("AUS Central Standard Time", "Australia/Darwin"), - ("E. Australia Standard Time", "Australia/Brisbane"), - ("AUS Eastern Standard Time", "Australia/Sydney"), - ("West Pacific Standard Time", "Pacific/Port_Moresby"), - ("Tasmania Standard Time", "Australia/Hobart"), - ("Vladivostok Standard Time", "Asia/Vladivostok"), - ("Lord Howe Standard Time", "Australia/Lord_Howe"), - ("Bougainville Standard Time", "Pacific/Bougainville"), - ("Russia Time Zone 10", "Asia/Srednekolymsk"), - ("Magadan Standard Time", "Asia/Magadan"), - ("Norfolk Standard Time", "Pacific/Norfolk"), - ("Sakhalin Standard Time", "Asia/Sakhalin"), - ("Central Pacific Standard Time", "Pacific/Guadalcanal"), - ("Russia Time Zone 11", "Asia/Kamchatka"), - ("New Zealand Standard Time", "Pacific/Auckland"), - ("Fiji Standard Time", "Pacific/Fiji"), - ("Kamchatka Standard Time", "Asia/Kamchatka"), - ("Chatham Islands Standard Time", "Pacific/Chatham"), - ("Tonga Standard Time", "Pacific/Tongatapu"), - ("Samoa Standard Time", "Pacific/Apia"), - ("Line Islands Standard Time", "Pacific/Kiritimati"), -]; +windows_zones! { + "UTC" => "Etc/UTC", + "Coordinated Universal Time" => "Etc/UTC", + "UTC-11" => "Etc/GMT+11", + "UTC-09" => "Etc/GMT+9", + "UTC-08" => "Etc/GMT+8", + "UTC-02" => "Etc/GMT+2", + "UTC+12" => "Etc/GMT-12", + "UTC+13" => "Etc/GMT-13", + "Dateline Standard Time" => "Etc/GMT+12", + "Aleutian Standard Time" => "America/Adak", + "Hawaiian Standard Time" => "Pacific/Honolulu", + "Marquesas Standard Time" => "Pacific/Marquesas", + "Alaskan Standard Time" => "America/Anchorage", + "Pacific Standard Time (Mexico)" => "America/Tijuana", + "Pacific Standard Time" => "America/Los_Angeles", + "US Mountain Standard Time" => "America/Phoenix", + "Mountain Standard Time (Mexico)" => "America/Mazatlan", + "Mountain Standard Time" => "America/Denver", + "Yukon Standard Time" => "America/Whitehorse", + "Central America Standard Time" => "America/Guatemala", + "Central Standard Time" => "America/Chicago", + "Easter Island Standard Time" => "Pacific/Easter", + "Central Standard Time (Mexico)" => "America/Mexico_City", + "Canada Central Standard Time" => "America/Regina", + "SA Pacific Standard Time" => "America/Bogota", + "Eastern Standard Time (Mexico)" => "America/Cancun", + "Eastern Standard Time" => "America/New_York", + "Haiti Standard Time" => "America/Port-au-Prince", + "Cuba Standard Time" => "America/Havana", + "US Eastern Standard Time" => "America/Indianapolis", + "Turks And Caicos Standard Time" => "America/Grand_Turk", + "Paraguay Standard Time" => "America/Asuncion", + "Atlantic Standard Time" => "America/Halifax", + "Venezuela Standard Time" => "America/Caracas", + "Central Brazilian Standard Time" => "America/Cuiaba", + "SA Western Standard Time" => "America/La_Paz", + "Pacific SA Standard Time" => "America/Santiago", + "Newfoundland Standard Time" => "America/St_Johns", + "Tocantins Standard Time" => "America/Araguaina", + "E. South America Standard Time" => "America/Sao_Paulo", + "SA Eastern Standard Time" => "America/Cayenne", + "Argentina Standard Time" => "America/Argentina/Buenos_Aires", + "Greenland Standard Time" => "America/Godthab", + "Montevideo Standard Time" => "America/Montevideo", + "Magallanes Standard Time" => "America/Punta_Arenas", + "Saint Pierre Standard Time" => "America/Miquelon", + "Bahia Standard Time" => "America/Bahia", + "Mid-Atlantic Standard Time" => "Atlantic/South_Georgia", + "Azores Standard Time" => "Atlantic/Azores", + "Cape Verde Standard Time" => "Atlantic/Cape_Verde", + "GMT Standard Time" => "Europe/London", + "Greenwich Standard Time" => "Atlantic/Reykjavik", + "Sao Tome Standard Time" => "Africa/Sao_Tome", + "Morocco Standard Time" => "Africa/Casablanca", + "W. Europe Standard Time" => "Europe/Berlin", + "Central Europe Standard Time" => "Europe/Budapest", + "Romance Standard Time" => "Europe/Paris", + "Central European Standard Time" => "Europe/Warsaw", + "W. Central Africa Standard Time" => "Africa/Lagos", + "Jordan Standard Time" => "Asia/Amman", + "GTB Standard Time" => "Europe/Bucharest", + "Middle East Standard Time" => "Asia/Beirut", + "Egypt Standard Time" => "Africa/Cairo", + "E. Europe Standard Time" => "Europe/Chisinau", + "Syria Standard Time" => "Asia/Damascus", + "West Bank Standard Time" => "Asia/Hebron", + "South Africa Standard Time" => "Africa/Johannesburg", + "FLE Standard Time" => "Europe/Kiev", + "Israel Standard Time" => "Asia/Jerusalem", + "South Sudan Standard Time" => "Africa/Juba", + "Kaliningrad Standard Time" => "Europe/Kaliningrad", + "Sudan Standard Time" => "Africa/Khartoum", + "Libya Standard Time" => "Africa/Tripoli", + "Namibia Standard Time" => "Africa/Windhoek", + "Arabic Standard Time" => "Asia/Baghdad", + "Turkey Standard Time" => "Europe/Istanbul", + "Arab Standard Time" => "Asia/Riyadh", + "Belarus Standard Time" => "Europe/Minsk", + "Russian Standard Time" => "Europe/Moscow", + "E. Africa Standard Time" => "Africa/Nairobi", + "Iran Standard Time" => "Asia/Tehran", + "Arabian Standard Time" => "Asia/Dubai", + "Astrakhan Standard Time" => "Europe/Astrakhan", + "Azerbaijan Standard Time" => "Asia/Baku", + "Russia Time Zone 3" => "Europe/Samara", + "Mauritius Standard Time" => "Indian/Mauritius", + "Saratov Standard Time" => "Europe/Saratov", + "Georgian Standard Time" => "Asia/Tbilisi", + "Volgograd Standard Time" => "Europe/Volgograd", + "Caucasus Standard Time" => "Asia/Yerevan", + "Afghanistan Standard Time" => "Asia/Kabul", + "West Asia Standard Time" => "Asia/Tashkent", + "Ekaterinburg Standard Time" => "Asia/Yekaterinburg", + "Pakistan Standard Time" => "Asia/Karachi", + "Qyzylorda Standard Time" => "Asia/Qyzylorda", + "India Standard Time" => "Asia/Kolkata", + "Sri Lanka Standard Time" => "Asia/Colombo", + "Nepal Standard Time" => "Asia/Katmandu", + "Central Asia Standard Time" => "Asia/Almaty", + "Bangladesh Standard Time" => "Asia/Dhaka", + "Omsk Standard Time" => "Asia/Omsk", + "Myanmar Standard Time" => "Asia/Rangoon", + "SE Asia Standard Time" => "Asia/Bangkok", + "Altai Standard Time" => "Asia/Barnaul", + "W. Mongolia Standard Time" => "Asia/Hovd", + "North Asia Standard Time" => "Asia/Krasnoyarsk", + "N. Central Asia Standard Time" => "Asia/Novosibirsk", + "Tomsk Standard Time" => "Asia/Tomsk", + "China Standard Time" => "Asia/Shanghai", + "North Asia East Standard Time" => "Asia/Irkutsk", + "Singapore Standard Time" => "Asia/Singapore", + "W. Australia Standard Time" => "Australia/Perth", + "Taipei Standard Time" => "Asia/Taipei", + "Ulaanbaatar Standard Time" => "Asia/Ulaanbaatar", + "Aus Central W. Standard Time" => "Australia/Eucla", + "Transbaikal Standard Time" => "Asia/Chita", + "Tokyo Standard Time" => "Asia/Tokyo", + "North Korea Standard Time" => "Asia/Pyongyang", + "Korea Standard Time" => "Asia/Seoul", + "Yakutsk Standard Time" => "Asia/Yakutsk", + "Cen. Australia Standard Time" => "Australia/Adelaide", + "AUS Central Standard Time" => "Australia/Darwin", + "E. Australia Standard Time" => "Australia/Brisbane", + "AUS Eastern Standard Time" => "Australia/Sydney", + "West Pacific Standard Time" => "Pacific/Port_Moresby", + "Tasmania Standard Time" => "Australia/Hobart", + "Vladivostok Standard Time" => "Asia/Vladivostok", + "Lord Howe Standard Time" => "Australia/Lord_Howe", + "Bougainville Standard Time" => "Pacific/Bougainville", + "Russia Time Zone 10" => "Asia/Srednekolymsk", + "Magadan Standard Time" => "Asia/Magadan", + "Norfolk Standard Time" => "Pacific/Norfolk", + "Sakhalin Standard Time" => "Asia/Sakhalin", + "Central Pacific Standard Time" => "Pacific/Guadalcanal", + "Russia Time Zone 11" => "Asia/Kamchatka", + "New Zealand Standard Time" => "Pacific/Auckland", + "Fiji Standard Time" => "Pacific/Fiji", + "Kamchatka Standard Time" => "Asia/Kamchatka", + "Chatham Islands Standard Time" => "Pacific/Chatham", + "Tonga Standard Time" => "Pacific/Tongatapu", + "Samoa Standard Time" => "Pacific/Apia", + "Line Islands Standard Time" => "Pacific/Kiritimati", +} #[cfg(test)] mod tests { @@ -187,7 +207,7 @@ mod tests { #[test] fn pacific_standard_time_maps_to_los_angeles() { assert_eq!( - windows_to_iana("Pacific Standard Time"), + ZoneId("Pacific Standard Time").windows_iana(), Some("America/Los_Angeles") ); } @@ -195,20 +215,20 @@ mod tests { #[test] fn romance_standard_time_maps_to_paris() { assert_eq!( - windows_to_iana("Romance Standard Time"), + ZoneId("Romance Standard Time").windows_iana(), Some("Europe/Paris") ); } #[test] fn unknown_zone_returns_none() { - assert_eq!(windows_to_iana("Made Up Zone"), None); + assert_eq!(ZoneId("Made Up Zone").windows_iana(), None); } #[test] fn case_insensitive_lookup() { assert_eq!( - windows_to_iana("eastern standard time"), + ZoneId("eastern standard time").windows_iana(), Some("America/New_York") ); } @@ -216,55 +236,70 @@ mod tests { #[test] fn whitespace_is_trimmed() { assert_eq!( - windows_to_iana(" GMT Standard Time "), + ZoneId(" GMT Standard Time ").windows_iana(), Some("Europe/London") ); } #[test] - fn full_cldr_table_has_at_least_140_rows() { - assert!(TABLE.len() >= 140, "got {} rows", TABLE.len()); + fn full_cldr_table_has_at_least_140_rows_and_each_resolves() { + // Every Windows name in the table resolves, in any case. + assert!( + WINDOWS_NAMES.len() >= 140, + "got {} rows", + WINDOWS_NAMES.len() + ); + for name in WINDOWS_NAMES { + assert!(ZoneId(name).windows_iana().is_some(), "{name}"); + assert!( + ZoneId(&name.to_ascii_uppercase()).windows_iana().is_some(), + "{name}" + ); + } } #[test] fn spot_check_coverage_across_continents() { - assert_eq!(windows_to_iana("Tokyo Standard Time"), Some("Asia/Tokyo")); assert_eq!( - windows_to_iana("AUS Eastern Standard Time"), + ZoneId("Tokyo Standard Time").windows_iana(), + Some("Asia/Tokyo") + ); + assert_eq!( + ZoneId("AUS Eastern Standard Time").windows_iana(), Some("Australia/Sydney") ); assert_eq!( - windows_to_iana("Cen. Australia Standard Time"), + ZoneId("Cen. Australia Standard Time").windows_iana(), Some("Australia/Adelaide") ); assert_eq!( - windows_to_iana("Greenland Standard Time"), + ZoneId("Greenland Standard Time").windows_iana(), Some("America/Godthab") ); assert_eq!( - windows_to_iana("China Standard Time"), + ZoneId("China Standard Time").windows_iana(), Some("Asia/Shanghai") ); assert_eq!( - windows_to_iana("Israel Standard Time"), + ZoneId("Israel Standard Time").windows_iana(), Some("Asia/Jerusalem") ); assert_eq!( - windows_to_iana("Chatham Islands Standard Time"), + ZoneId("Chatham Islands Standard Time").windows_iana(), Some("Pacific/Chatham") ); assert_eq!( - windows_to_iana("Newfoundland Standard Time"), + ZoneId("Newfoundland Standard Time").windows_iana(), Some("America/St_Johns") ); - assert_eq!(windows_to_iana("UTC-08"), Some("Etc/GMT+8")); - assert_eq!(windows_to_iana("UTC+13"), Some("Etc/GMT-13")); + assert_eq!(ZoneId("UTC-08").windows_iana(), Some("Etc/GMT+8")); + assert_eq!(ZoneId("UTC+13").windows_iana(), Some("Etc/GMT-13")); assert_eq!( - windows_to_iana("Yukon Standard Time"), + ZoneId("Yukon Standard Time").windows_iana(), Some("America/Whitehorse") ); assert_eq!( - windows_to_iana("Aus Central W. Standard Time"), + ZoneId("Aus Central W. Standard Time").windows_iana(), Some("Australia/Eucla") ); } @@ -272,27 +307,27 @@ mod tests { #[test] fn resolve_handles_windows_iana_offset_and_microsoft_tzone() { assert_eq!( - resolve_to_iana("Romance Standard Time").as_deref(), + ZoneId("Romance Standard Time").iana().as_deref(), Some("Europe/Paris") ); assert_eq!( - resolve_to_iana("Europe/Paris").as_deref(), + ZoneId("Europe/Paris").iana().as_deref(), Some("Europe/Paris") ); - assert_eq!(resolve_to_iana("UTC").as_deref(), Some("Etc/UTC")); + assert_eq!(ZoneId("UTC").iana().as_deref(), Some("Etc/UTC")); assert_eq!( - resolve_to_iana("tzone://Microsoft/Utc").as_deref(), + ZoneId("tzone://Microsoft/Utc").iana().as_deref(), Some("Etc/UTC") ); - assert_eq!(resolve_to_iana("tzone://Microsoft/Custom"), None); - assert_eq!(resolve_to_iana("Made Up Zone"), None); - assert_eq!(resolve_to_iana(""), None); + assert_eq!(ZoneId("tzone://Microsoft/Custom").iana(), None); + assert_eq!(ZoneId("Made Up Zone").iana(), None); + assert_eq!(ZoneId("").iana(), None); } #[test] fn coordinated_universal_time_alias_resolves_to_utc() { assert_eq!( - windows_to_iana("Coordinated Universal Time"), + ZoneId("Coordinated Universal Time").windows_iana(), Some("Etc/UTC") ); } diff --git a/src/exchange/vocab.rs b/src/exchange/vocab.rs new file mode 100644 index 0000000..b9c967e --- /dev/null +++ b/src/exchange/vocab.rs @@ -0,0 +1,387 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fmt::Write; + +use chrono::Weekday; + +const SECONDS_PER_HOUR: i64 = 3600; +const SECONDS_PER_MINUTE: i64 = 60; + +#[derive(Debug, Clone, Copy)] +pub struct WeekdayName<'a>(pub &'a str); + +impl WeekdayName<'_> { + pub fn parse(self) -> Option { + hashify::map_ignore_case!( + self.0.as_bytes(), + Weekday, + "monday" => Weekday::Mon, + "tuesday" => Weekday::Tue, + "wednesday" => Weekday::Wed, + "thursday" => Weekday::Thu, + "friday" => Weekday::Fri, + "saturday" => Weekday::Sat, + "sunday" => Weekday::Sun, + ) + .copied() + } + + pub fn long(day: Weekday) -> &'static str { + match day { + Weekday::Mon => "monday", + Weekday::Tue => "tuesday", + Weekday::Wed => "wednesday", + Weekday::Thu => "thursday", + Weekday::Fri => "friday", + Weekday::Sat => "saturday", + Weekday::Sun => "sunday", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DayToken(pub Weekday); + +impl DayToken { + pub fn parse(token: &str) -> Option { + hashify::map_ignore_case!( + token.as_bytes(), + Weekday, + "mo" => Weekday::Mon, + "tu" => Weekday::Tue, + "we" => Weekday::Wed, + "th" => Weekday::Thu, + "fr" => Weekday::Fri, + "sa" => Weekday::Sat, + "su" => Weekday::Sun, + ) + .copied() + .map(DayToken) + } + + pub fn as_str(self) -> &'static str { + match self.0 { + Weekday::Mon => "mo", + Weekday::Tue => "tu", + Weekday::Wed => "we", + Weekday::Thu => "th", + Weekday::Fri => "fr", + Weekday::Sat => "sa", + Weekday::Sun => "su", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Ordinal { + First, + Second, + Third, + Fourth, + Last, +} + +impl Ordinal { + pub fn parse(name: &str) -> Option { + hashify::map_ignore_case!( + name.as_bytes(), + Ordinal, + "first" => Ordinal::First, + "second" => Ordinal::Second, + "third" => Ordinal::Third, + "fourth" => Ordinal::Fourth, + "last" => Ordinal::Last, + ) + .copied() + } + + pub fn parse_exact(name: &str) -> Option { + hashify::map!( + name.as_bytes(), + Ordinal, + "First" => Ordinal::First, + "Second" => Ordinal::Second, + "Third" => Ordinal::Third, + "Fourth" => Ordinal::Fourth, + "Last" => Ordinal::Last, + ) + .copied() + } + + pub fn position(self) -> i32 { + match self { + Ordinal::First => 1, + Ordinal::Second => 2, + Ordinal::Third => 3, + Ordinal::Fourth => 4, + Ordinal::Last => -1, + } + } + + pub fn from_position(position: i64) -> Ordinal { + match position { + 2 => Ordinal::Second, + 3 => Ordinal::Third, + 4 => Ordinal::Fourth, + -1 => Ordinal::Last, + _ => Ordinal::First, + } + } + + pub fn name(self) -> &'static str { + match self { + Ordinal::First => "first", + Ordinal::Second => "second", + Ordinal::Third => "third", + Ordinal::Fourth => "fourth", + Ordinal::Last => "last", + } + } + + pub fn pick(self, matches: &[T]) -> Option { + match self { + Ordinal::First => matches.first(), + Ordinal::Second => matches.get(1), + Ordinal::Third => matches.get(2), + Ordinal::Fourth => matches.get(3), + Ordinal::Last => matches.last(), + } + .copied() + } +} + +#[derive(Debug, Clone, Copy)] +pub struct MonthName<'a>(pub &'a str); + +impl MonthName<'_> { + pub fn number(self) -> Option { + hashify::map!( + self.0.as_bytes(), + u32, + "January" => 1, + "February" => 2, + "March" => 3, + "April" => 4, + "May" => 5, + "June" => 6, + "July" => 7, + "August" => 8, + "September" => 9, + "October" => 10, + "November" => 11, + "December" => 12, + ) + .copied() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Privacy { + Public, + Private, + Secret, +} + +impl Privacy { + pub fn from_sensitivity(value: &str) -> Privacy { + hashify::map_ignore_case!( + value.as_bytes(), + Privacy, + "private" => Privacy::Private, + "confidential" => Privacy::Secret, + ) + .copied() + .unwrap_or(Privacy::Public) + } + + pub fn as_str(self) -> &'static str { + match self { + Privacy::Public => "public", + Privacy::Private => "private", + Privacy::Secret => "secret", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Participation { + Accepted, + Tentative, + Declined, + NeedsAction, +} + +impl Participation { + pub fn from_ews(response_type: &str) -> Option { + hashify::map!( + response_type.as_bytes(), + Participation, + "Accept" => Participation::Accepted, + "Tentative" => Participation::Tentative, + "Decline" => Participation::Declined, + "Organizer" => Participation::Accepted, + ) + .copied() + } + + pub fn from_graph(response: &str) -> Option { + hashify::map!( + response.as_bytes(), + Participation, + "accepted" => Participation::Accepted, + "organizer" => Participation::Accepted, + "declined" => Participation::Declined, + "tentativelyAccepted" => Participation::Tentative, + "notResponded" => Participation::NeedsAction, + "none" => Participation::NeedsAction, + ) + .copied() + } + + pub fn as_str(self) -> &'static str { + match self { + Participation::Accepted => "accepted", + Participation::Tentative => "tentative", + Participation::Declined => "declined", + Participation::NeedsAction => "needs-action", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DurationStyle { + Ews, + Graph, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct IsoDuration(pub i64); + +impl IsoDuration { + pub fn format(self, style: DurationStyle) -> Option { + let total = self.0; + match style { + DurationStyle::Ews if total == 0 => return Some("PT0S".to_owned()), + DurationStyle::Ews if total < 0 => return None, + DurationStyle::Graph if total <= 0 => return None, + _ => {} + } + let hours = total / SECONDS_PER_HOUR; + let minutes = (total % SECONDS_PER_HOUR) / SECONDS_PER_MINUTE; + let seconds = total % SECONDS_PER_MINUTE; + let mut out = String::from("PT"); + if hours > 0 { + let _ = write!(out, "{hours}H"); + } + let bridge_minutes = style == DurationStyle::Graph && hours > 0 && seconds > 0; + if minutes > 0 || bridge_minutes { + let _ = write!(out, "{minutes}M"); + } + if seconds > 0 || (hours == 0 && minutes == 0) { + let _ = write!(out, "{seconds}S"); + } + Some(out) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn weekday_names_parse_in_any_case_and_round_trip_to_tokens() { + for (name, token) in [ + ("Monday", "mo"), + ("TUESDAY", "tu"), + ("wednesday", "we"), + ("Thursday", "th"), + ("friday", "fr"), + ("Saturday", "sa"), + ("sunday", "su"), + ] { + let day = WeekdayName(name).parse().expect(name); + assert_eq!(DayToken(day).as_str(), token); + } + assert_eq!(WeekdayName("mo").parse(), None); + assert_eq!( + DayToken::parse("MO").map(|d| WeekdayName::long(d.0)), + Some("monday") + ); + } + + #[test] + fn ordinals_keep_their_case_rules() { + assert_eq!(Ordinal::parse("LAST"), Some(Ordinal::Last)); + assert_eq!(Ordinal::parse_exact("Last"), Some(Ordinal::Last)); + assert_eq!(Ordinal::parse_exact("last"), None); + assert_eq!(Ordinal::Fourth.position(), 4); + assert_eq!(Ordinal::Last.pick(&[1, 2, 3]), Some(3)); + assert_eq!(Ordinal::Fourth.pick(&[1, 2, 3]), None); + assert_eq!(Ordinal::from_position(-1).name(), "last"); + assert_eq!(Ordinal::from_position(9), Ordinal::First); + } + + #[test] + fn month_names_are_case_sensitive() { + assert_eq!(MonthName("June").number(), Some(6)); + assert_eq!(MonthName("june").number(), None); + } + + #[test] + fn privacy_is_case_insensitive_for_both_servers() { + for (value, expected) in [ + ("Private", "private"), + ("private", "private"), + ("Confidential", "secret"), + ("confidential", "secret"), + ("Normal", "public"), + ("personal", "public"), + ] { + assert_eq!(Privacy::from_sensitivity(value).as_str(), expected); + } + } + + #[test] + fn duration_styles_keep_their_zero_and_minute_rules() { + assert_eq!( + IsoDuration(0).format(DurationStyle::Ews).as_deref(), + Some("PT0S") + ); + assert_eq!(IsoDuration(0).format(DurationStyle::Graph), None); + assert_eq!(IsoDuration(-5).format(DurationStyle::Ews), None); + assert_eq!( + IsoDuration(3605).format(DurationStyle::Ews).as_deref(), + Some("PT1H5S") + ); + assert_eq!( + IsoDuration(3605).format(DurationStyle::Graph).as_deref(), + Some("PT1H0M5S") + ); + assert_eq!( + IsoDuration(5400).format(DurationStyle::Graph).as_deref(), + Some("PT1H30M") + ); + assert_eq!( + IsoDuration(45).format(DurationStyle::Ews).as_deref(), + Some("PT45S") + ); + } + + #[test] + fn participation_vocabularies_differ_per_server() { + assert_eq!( + Participation::from_ews("Organizer"), + Some(Participation::Accepted) + ); + assert_eq!(Participation::from_ews("accepted"), None); + assert_eq!( + Participation::from_graph("tentativelyAccepted"), + Some(Participation::Tentative) + ); + assert_eq!(Participation::NeedsAction.as_str(), "needs-action"); + } +} diff --git a/src/exchange_ews/autodiscover.rs b/src/exchange_ews/autodiscover.rs deleted file mode 100644 index cb28b4f..0000000 --- a/src/exchange_ews/autodiscover.rs +++ /dev/null @@ -1,578 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use quick_xml::NsReader; -use quick_xml::events::Event; -use serde_json::Value; -use ureq::Agent; -use ureq::config::Config; -use ureq::tls::{RootCerts, TlsConfig}; - -use crate::exchange_ews::error::EwsError; -use crate::exchange_ews::parse::entity_to_char; - -const V2_HOST: &str = "https://outlook.office365.com"; -const POX_REQ_NS: &str = - "http://schemas.microsoft.com/exchange/autodiscover/outlook/requestschema/2006"; -const POX_RESP_NS: &str = - "http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a"; - -pub fn is_fully_qualified_ews_url(url: &str) -> bool { - let parsed = match url::Url::parse(url) { - Ok(u) => u, - Err(_) => return false, - }; - if parsed.scheme() != "https" { - return false; - } - parsed.path().eq_ignore_ascii_case("/EWS/Exchange.asmx") -} - -#[derive(Debug, Clone)] -pub struct DiscoveryResult { - pub ews_url: String, - pub source: DiscoverySource, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum DiscoverySource { - V2, - V1, - SuppliedUrl, -} - -pub fn discover( - supplied_url: Option<&str>, - email: Option<&str>, - auth_header: Option<&str>, - allow_invalid_certs: bool, -) -> Result { - if let Some(url) = supplied_url - && is_fully_qualified_ews_url(url) - { - return Ok(DiscoveryResult { - ews_url: url.to_owned(), - source: DiscoverySource::SuppliedUrl, - }); - } - let Some(email) = email else { - return Err(EwsError::AutodiscoverFailed( - "either a fully-qualified --url or --mailbox is required".to_owned(), - )); - }; - let agent = build_agent(allow_invalid_certs); - if let Ok(url) = autodiscover_v2(&agent, email) { - return Ok(DiscoveryResult { - ews_url: url, - source: DiscoverySource::V2, - }); - } - let mut tried: Vec = Vec::new(); - let mut redirects: u8 = 0; - let mut url_redirects: u8 = 0; - let mut current_email = email.to_owned(); - 'outer: loop { - let domain = current_email - .split('@') - .nth(1) - .ok_or_else(|| EwsError::AutodiscoverFailed("--mailbox is not an email".to_owned()))?; - let candidates = pox_candidates(domain); - for candidate in &candidates { - tried.push(candidate.clone()); - match autodiscover_v1(&agent, candidate, ¤t_email, auth_header) { - Ok(PoxOutcome::EwsUrl(url)) => { - return Ok(DiscoveryResult { - ews_url: url, - source: DiscoverySource::V1, - }); - } - Ok(PoxOutcome::RedirectAddr(addr)) => { - if redirects >= 4 { - return Err(EwsError::AutodiscoverLoop); - } - redirects += 1; - current_email = addr; - continue 'outer; - } - Ok(PoxOutcome::RedirectUrl(url)) => { - if url_redirects >= 1 { - return Err(EwsError::AutodiscoverLoop); - } - url_redirects += 1; - tried.push(url.clone()); - if let Ok(PoxOutcome::EwsUrl(u)) = - autodiscover_v1(&agent, &url, ¤t_email, auth_header) - { - return Ok(DiscoveryResult { - ews_url: u, - source: DiscoverySource::V1, - }); - } - continue; - } - Err(_) | Ok(PoxOutcome::Failed) => continue, - } - } - break; - } - if let Some(url) = supplied_url { - return Ok(DiscoveryResult { - ews_url: url.to_owned(), - source: DiscoverySource::SuppliedUrl, - }); - } - Err(EwsError::AutodiscoverFailed(format!( - "tried {} candidate(s), none returned a v1 EwsUrl", - tried.len() - ))) -} - -fn build_agent(allow_invalid_certs: bool) -> Agent { - let config: Config = Config::builder() - .http_status_as_error(false) - .tls_config( - TlsConfig::builder() - .unversioned_rustls_crypto_provider(std::sync::Arc::new( - rustls::crypto::aws_lc_rs::default_provider(), - )) - .root_certs(RootCerts::PlatformVerifier) - .disable_verification(allow_invalid_certs) - .build(), - ) - .build(); - config.new_agent() -} - -fn autodiscover_v2(agent: &Agent, email: &str) -> Result { - let url = format!( - "{V2_HOST}/autodiscover/autodiscover.json?Email={}&Protocol=Ews", - urlencode(email) - ); - let mut resp = agent - .get(&url) - .header("Accept", "application/json") - .call() - .map_err(|e| EwsError::Transport(format!("autodiscover v2: {e}")))?; - let status = resp.status().as_u16(); - if !(200..300).contains(&status) { - return Err(EwsError::AutodiscoverFailed(format!( - "v2 returned http {status}" - ))); - } - let body: Value = resp - .body_mut() - .read_json() - .map_err(|e| EwsError::Malformed(format!("v2 body: {e}")))?; - let protocol = body.get("Protocol").and_then(Value::as_str).unwrap_or(""); - if !protocol.eq_ignore_ascii_case("Ews") { - return Err(EwsError::AutodiscoverFailed(format!( - "v2 returned protocol {protocol:?}" - ))); - } - let url = body - .get("Url") - .and_then(Value::as_str) - .ok_or_else(|| EwsError::AutodiscoverFailed("v2 missing Url".to_owned()))?; - Ok(url.to_owned()) -} - -#[derive(Debug, Clone)] -enum PoxOutcome { - EwsUrl(String), - RedirectAddr(String), - RedirectUrl(String), - Failed, -} - -fn pox_candidates(domain: &str) -> Vec { - let mut out = vec![ - format!("https://autodiscover.{domain}/autodiscover/autodiscover.xml"), - format!("https://{domain}/autodiscover/autodiscover.xml"), - ]; - for SrvRecord { target, port, .. } in srv_lookup(&format!("_autodiscover._tcp.{domain}")) { - let host = target.trim_end_matches('.'); - let suffix = if port == 443 { - String::new() - } else { - format!(":{port}") - }; - out.push(format!( - "https://{host}{suffix}/autodiscover/autodiscover.xml" - )); - } - out -} - -#[derive(Debug, Clone)] -struct SrvRecord { - priority: u16, - weight: u16, - port: u16, - target: String, -} - -fn srv_lookup(qname: &str) -> Vec { - use std::net::UdpSocket; - use std::time::Duration; - - let nameservers = resolv_conf_nameservers(); - if nameservers.is_empty() { - return Vec::new(); - } - let mut query = Vec::with_capacity(64); - let txn_id: u16 = 0x1234; - query.extend_from_slice(&txn_id.to_be_bytes()); - query.extend_from_slice(&0x0100u16.to_be_bytes()); - query.extend_from_slice(&1u16.to_be_bytes()); - query.extend_from_slice(&[0u8; 6]); - encode_qname(&mut query, qname); - query.extend_from_slice(&33u16.to_be_bytes()); - query.extend_from_slice(&1u16.to_be_bytes()); - - let mut out: Vec = Vec::new(); - for ns in nameservers { - let Ok(socket) = UdpSocket::bind("0.0.0.0:0") else { - continue; - }; - let _ = socket.set_read_timeout(Some(Duration::from_secs(3))); - if socket.send_to(&query, format!("{ns}:53")).is_err() { - continue; - } - let mut buf = [0u8; 1500]; - let Ok((n, _)) = socket.recv_from(&mut buf) else { - continue; - }; - if let Some(records) = parse_srv_response(&buf[..n]) { - out = records; - break; - } - } - out.sort_by(|a, b| a.priority.cmp(&b.priority).then(b.weight.cmp(&a.weight))); - out -} - -fn resolv_conf_nameservers() -> Vec { - let path = std::path::Path::new("/etc/resolv.conf"); - let Ok(text) = std::fs::read_to_string(path) else { - return Vec::new(); - }; - let mut out: Vec = Vec::new(); - for line in text.lines() { - let line = line.split('#').next().unwrap_or("").trim(); - if let Some(rest) = line.strip_prefix("nameserver") { - let ns = rest.trim(); - if !ns.is_empty() { - out.push(ns.to_owned()); - } - } - } - out -} - -fn encode_qname(out: &mut Vec, qname: &str) { - for label in qname.trim_end_matches('.').split('.') { - if label.is_empty() { - continue; - } - out.push(label.len() as u8); - out.extend_from_slice(label.as_bytes()); - } - out.push(0); -} - -fn parse_srv_response(buf: &[u8]) -> Option> { - if buf.len() < 12 { - return None; - } - let qd = u16::from_be_bytes([buf[4], buf[5]]) as usize; - let an = u16::from_be_bytes([buf[6], buf[7]]) as usize; - let mut cursor = 12; - for _ in 0..qd { - cursor = skip_name(buf, cursor)?; - cursor = cursor.checked_add(4)?; - if cursor > buf.len() { - return None; - } - } - let mut out: Vec = Vec::new(); - for _ in 0..an { - cursor = skip_name(buf, cursor)?; - if cursor + 10 > buf.len() { - return None; - } - let rtype = u16::from_be_bytes([buf[cursor], buf[cursor + 1]]); - let rdlength = u16::from_be_bytes([buf[cursor + 8], buf[cursor + 9]]) as usize; - cursor += 10; - let rdata_end = cursor.checked_add(rdlength)?; - if rdata_end > buf.len() { - return None; - } - if rtype == 33 && rdlength >= 7 { - let priority = u16::from_be_bytes([buf[cursor], buf[cursor + 1]]); - let weight = u16::from_be_bytes([buf[cursor + 2], buf[cursor + 3]]); - let port = u16::from_be_bytes([buf[cursor + 4], buf[cursor + 5]]); - let target = read_name(buf, cursor + 6)?; - out.push(SrvRecord { - priority, - weight, - port, - target, - }); - } - cursor = rdata_end; - } - Some(out) -} - -fn skip_name(buf: &[u8], mut cursor: usize) -> Option { - loop { - if cursor >= buf.len() { - return None; - } - let len = buf[cursor]; - if len == 0 { - return Some(cursor + 1); - } - if (len & 0xC0) == 0xC0 { - return Some(cursor + 2); - } - cursor = cursor.checked_add(1 + len as usize)?; - } -} - -fn read_name(buf: &[u8], start: usize) -> Option { - let mut out = String::new(); - let mut cursor = start; - let mut hops = 0; - loop { - if cursor >= buf.len() { - return None; - } - let len = buf[cursor]; - if len == 0 { - break; - } - if (len & 0xC0) == 0xC0 { - if cursor + 1 >= buf.len() { - return None; - } - let pointer = ((len & 0x3F) as usize) << 8 | (buf[cursor + 1] as usize); - hops += 1; - if hops > 8 { - return None; - } - cursor = pointer; - continue; - } - cursor += 1; - let end = cursor.checked_add(len as usize)?; - if end > buf.len() { - return None; - } - if !out.is_empty() { - out.push('.'); - } - out.push_str(std::str::from_utf8(&buf[cursor..end]).ok()?); - cursor = end; - } - Some(out) -} - -fn autodiscover_v1( - agent: &Agent, - url: &str, - email: &str, - auth_header: Option<&str>, -) -> Result { - let body = format!( - "\ - \ - {}\ - {POX_RESP_NS}\ - ", - xml_escape(email) - ); - let mut req = agent - .post(url) - .header("Content-Type", "text/xml; charset=utf-8"); - if let Some(h) = auth_header { - req = req.header("Authorization", h); - } - let mut resp = req - .send(body.as_bytes()) - .map_err(|e| EwsError::Transport(format!("v1 POST: {e}")))?; - let status = resp.status().as_u16(); - if !(200..300).contains(&status) { - return Err(EwsError::AutodiscoverFailed(format!( - "{url} returned http {status}" - ))); - } - let bytes = resp - .body_mut() - .with_config() - .limit(2 * 1024 * 1024) - .read_to_vec() - .map_err(|e| EwsError::Malformed(format!("v1 body: {e}")))?; - parse_pox_response(&bytes) -} - -fn parse_pox_response(body: &[u8]) -> Result { - let mut xml = NsReader::from_reader(body); - xml.config_mut().trim_text(true); - let mut buf = Vec::new(); - let mut current: Option<&'static str> = None; - let mut current_type: Option = None; - let mut ews_url: Option = None; - let mut redirect_addr: Option = None; - let mut redirect_url: Option = None; - let mut action: Option = None; - let mut cur = String::new(); - loop { - buf.clear(); - let (_, ev) = xml.read_resolved_event_into(&mut buf)?; - match ev { - Event::Start(e) => { - let local = e.local_name().as_ref().to_owned(); - cur.clear(); - if local.eq_ignore_ascii_case("Protocol") { - current_type = None; - } - if local.eq_ignore_ascii_case("Type") { - current = Some("type"); - } else if local.eq_ignore_ascii_case("EwsUrl") { - if matches!(current_type.as_deref(), Some("EXPR") | Some("EXCH")) { - current = Some("ewsUrl"); - } else { - current = None; - } - } else if local.eq_ignore_ascii_case("Action") { - current = Some("action"); - } else if local.eq_ignore_ascii_case("RedirectAddr") { - current = Some("redirectAddr"); - } else if local.eq_ignore_ascii_case("RedirectUrl") { - current = Some("redirectUrl"); - } else { - current = None; - } - } - Event::End(e) => { - let local = e.local_name().as_ref().to_owned(); - if let Some(field) = current.take() { - let text = std::mem::take(&mut cur); - match field { - "type" => current_type = Some(text), - "ewsUrl" => ews_url = Some(text), - "action" => action = Some(text), - "redirectAddr" => redirect_addr = Some(text), - "redirectUrl" => redirect_url = Some(text), - _ => {} - } - } - cur.clear(); - if local.eq_ignore_ascii_case("Protocol") { - current_type = None; - } - } - Event::Text(t) => { - cur.push_str(&t); - } - Event::CData(c) => { - cur.push_str(&c); - } - Event::GeneralRef(g) => { - if let Some(c) = entity_to_char(&g) { - cur.push(c); - } - } - Event::Eof => break, - _ => {} - } - } - if let Some(addr) = redirect_addr.filter(|_| action.as_deref() == Some("redirectAddr")) { - return Ok(PoxOutcome::RedirectAddr(addr)); - } - if let Some(url) = redirect_url.filter(|_| action.as_deref() == Some("redirectUrl")) { - return Ok(PoxOutcome::RedirectUrl(url)); - } - if let Some(url) = ews_url { - return Ok(PoxOutcome::EwsUrl(url)); - } - Ok(PoxOutcome::Failed) -} - -fn xml_escape(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - for ch in s.chars() { - match ch { - '&' => out.push_str("&"), - '<' => out.push_str("<"), - '>' => out.push_str(">"), - '"' => out.push_str("""), - _ => out.push(ch), - } - } - out -} - -fn urlencode(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - for b in s.bytes() { - match b { - b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { - out.push(b as char); - } - _ => out.push_str(&format!("%{b:02X}")), - } - } - out -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn detects_fully_qualified_ews_url() { - assert!(is_fully_qualified_ews_url( - "https://outlook.office365.com/EWS/Exchange.asmx" - )); - assert!(is_fully_qualified_ews_url( - "https://exchange.example.com/EWS/Exchange.asmx" - )); - assert!(!is_fully_qualified_ews_url( - "http://outlook.office365.com/EWS/Exchange.asmx" - )); - assert!(!is_fully_qualified_ews_url( - "https://outlook.office365.com/" - )); - } - - #[test] - fn pox_response_extracts_ews_url() { - let body = format!( - "\ - \ - EXPRhttps://srv/EWS/Exchange.asmx" - ); - match parse_pox_response(body.as_bytes()).unwrap() { - PoxOutcome::EwsUrl(u) => assert_eq!(u, "https://srv/EWS/Exchange.asmx"), - _ => panic!("expected EwsUrl"), - } - } - - #[test] - fn pox_response_handles_redirect_addr() { - let body = format!( - "\ - \ - redirectAddrbob@other.com" - ); - match parse_pox_response(body.as_bytes()).unwrap() { - PoxOutcome::RedirectAddr(a) => assert_eq!(a, "bob@other.com"), - _ => panic!("expected RedirectAddr"), - } - } -} diff --git a/src/exchange_ews/client.rs b/src/exchange_ews/client.rs deleted file mode 100644 index 091a313..0000000 --- a/src/exchange_ews/client.rs +++ /dev/null @@ -1,608 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::sync::Arc; -use std::sync::Mutex; -use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; -use std::time::{Duration, Instant}; - -use ureq::Agent; -use ureq::config::{Config, RedirectAuthHeaders}; -use ureq::tls::{RootCerts, TlsConfig}; - -use crate::exchange_ews::error::EwsError; -use crate::exchange_ews::parse::{EnvelopeKind, SoapFault, read_envelope_summary}; -use crate::exchange_ews::retry::{FaultDisposition, classify_fault, classify_http_status}; -use crate::exchange_ews::soap::{EnvelopeOptions, soap_action, wrap_envelope}; -use crate::exchange_ews::types::ServerVersion; -use crate::jmap::http::{Auth, RetryPolicy, retry_after_header}; -use crate::jmap::retry::{self, Disposition, RateLimitState}; -use crate::logging::{HttpCall, LEVEL_BODIES, LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; - -const MAX_BODY: u64 = 2 * 1024 * 1024 * 1024; -const LONG_RETRY_THRESHOLD: Duration = Duration::from_secs(10); - -struct Inner { - agent: Agent, - auth: Mutex, - impersonated_smtp: Mutex>, - anchor_mailbox: Mutex>, - affinity_cookie: Mutex>, - retry: RetryPolicy, - rate_limit: RateLimitState, - version: Mutex, - log_level: AtomicU8, - retries_total: AtomicU64, - retry_after_sleeps: AtomicU64, - soap_calls: AtomicU64, - user_agent: String, -} - -#[derive(Debug, Clone, Copy)] -struct VersionState { - current: ServerVersion, - accepted_ceiling: Option, -} - -#[derive(Clone)] -pub struct EwsClient { - inner: Arc, -} - -#[derive(Debug, Clone)] -pub struct SoapResponse { - pub body: Vec, - pub server_version: Option, -} - -impl EwsClient { - pub fn new(auth: Auth, retry: RetryPolicy, allow_invalid_certs: bool) -> EwsClient { - let config: Config = retry - .apply_timeouts(Config::builder()) - .http_status_as_error(false) - .redirect_auth_headers(RedirectAuthHeaders::SameHost) - .tls_config( - TlsConfig::builder() - .unversioned_rustls_crypto_provider(std::sync::Arc::new( - rustls::crypto::aws_lc_rs::default_provider(), - )) - .root_certs(RootCerts::PlatformVerifier) - .disable_verification(allow_invalid_certs) - .build(), - ) - .build(); - EwsClient { - inner: Arc::new(Inner { - agent: config.new_agent(), - auth: Mutex::new(auth), - impersonated_smtp: Mutex::new(None), - anchor_mailbox: Mutex::new(None), - affinity_cookie: Mutex::new(None), - retry, - rate_limit: RateLimitState::new(), - version: Mutex::new(VersionState { - current: ServerVersion::Exchange2013Sp1, - accepted_ceiling: None, - }), - log_level: AtomicU8::new(LEVEL_DEFAULT), - retries_total: AtomicU64::new(0), - retry_after_sleeps: AtomicU64::new(0), - soap_calls: AtomicU64::new(0), - user_agent: format!("inbuxa-migrate/{}", env!("CARGO_PKG_VERSION")), - }), - } - } - - pub fn set_auth(&self, auth: Auth) { - if let Ok(mut g) = self.inner.auth.lock() { - *g = auth; - } - } - - pub fn set_anchor_mailbox(&self, smtp: Option) { - if let Ok(mut g) = self.inner.anchor_mailbox.lock() { - *g = smtp; - } - } - - pub fn set_impersonation(&self, smtp: Option) { - if let Ok(mut g) = self.inner.impersonated_smtp.lock() { - *g = smtp; - } - } - - pub fn set_logger(&self, logger: Logger) { - self.inner - .log_level - .store(logger.level(), Ordering::Relaxed); - } - - pub fn logger(&self) -> Logger { - Logger::new(self.inner.log_level.load(Ordering::Relaxed)) - } - - pub fn set_server_version(&self, version: ServerVersion) { - if let Ok(mut g) = self.inner.version.lock() { - g.current = g.accepted_ceiling.map_or(version, |c| version.min(c)); - } - } - - fn pin_accepted_version(&self, version: ServerVersion) { - if let Ok(mut g) = self.inner.version.lock() { - g.current = version; - g.accepted_ceiling = Some(version); - } - } - - pub fn server_version(&self) -> ServerVersion { - self.inner - .version - .lock() - .map(|g| g.current) - .unwrap_or(ServerVersion::Exchange2013Sp1) - } - - pub fn retries_observed(&self) -> u64 { - self.inner.retries_total.load(Ordering::Relaxed) - } - - pub fn retry_after_sleeps(&self) -> u64 { - self.inner.retry_after_sleeps.load(Ordering::Relaxed) - } - - pub fn soap_calls(&self) -> u64 { - self.inner.soap_calls.load(Ordering::Relaxed) - } - - pub fn rate_limit(&self) -> &RateLimitState { - &self.inner.rate_limit - } - - pub fn call(&self, url: &str, operation: &str, body: &str) -> Result { - let action = soap_action(operation); - self.inner.soap_calls.fetch_add(1, Ordering::Relaxed); - self.execute(url, body, &action) - } - - fn wrap(&self, body: &str) -> String { - let version = self.server_version(); - let smtp = self - .inner - .impersonated_smtp - .lock() - .ok() - .and_then(|g| g.clone()); - let opts = EnvelopeOptions { - version, - impersonated_smtp: smtp.as_deref(), - }; - wrap_envelope(opts, body) - } - - fn auth_header(&self) -> String { - self.inner - .auth - .lock() - .map(|auth| auth.header_value()) - .unwrap_or_default() - } - - fn anchor_header(&self) -> Option { - self.inner - .anchor_mailbox - .lock() - .ok() - .and_then(|g| g.clone()) - } - - fn affinity_cookie(&self) -> Option { - self.inner - .affinity_cookie - .lock() - .ok() - .and_then(|g| g.clone()) - } - - fn execute(&self, url: &str, body: &str, action: &str) -> Result { - let logger = self.logger(); - let policy = self.inner.retry; - let mut attempt: u32 = 0; - let mut downgraded = false; - loop { - self.inner.rate_limit.cooldown().wait(); - let sent_version = self.server_version(); - let envelope = self.wrap(body); - let outcome = self.one_attempt(url, &envelope, action); - match outcome { - AttemptOutcome::Ok { - status, - body: bytes, - retry_after, - } => { - let summary = read_envelope_summary(&bytes); - if (200..300).contains(&status) { - self.inner.rate_limit.on_success(); - match summary { - Ok(EnvelopeKind::Body { version }) => { - if downgraded { - self.pin_accepted_version(sent_version); - } - if let Some(v) = version.to_server_version() { - self.set_server_version(v); - } - return Ok(SoapResponse { - body: bytes, - server_version: version.to_server_version(), - }); - } - Ok(EnvelopeKind::Fault { fault, .. }) => { - match self.on_fault( - &fault, - sent_version, - action, - &policy, - &logger, - &mut attempt, - ) { - FaultStep::Fail(e) => return Err(e), - FaultStep::Sleep(d) => { - std::thread::sleep(d); - continue; - } - FaultStep::Downgrade(v) => { - self.set_server_version(v); - downgraded = true; - continue; - } - } - } - Err(e) => return Err(e), - } - } - if status != 401 - && status != 403 - && let Ok(EnvelopeKind::Fault { fault, .. }) = summary - { - match self.on_fault( - &fault, - sent_version, - action, - &policy, - &logger, - &mut attempt, - ) { - FaultStep::Fail(e) => return Err(e), - FaultStep::Sleep(d) => { - std::thread::sleep(d); - continue; - } - FaultStep::Downgrade(v) => { - self.set_server_version(v); - downgraded = true; - continue; - } - } - } - if status == 401 { - return Err(EwsError::Auth(format!( - "server returned 401: {}", - truncate(&bytes) - ))); - } - if status == 456 { - return Err(EwsError::Auth(format!( - "account is locked (http 456); an administrator must unlock it: {}", - truncate(&bytes) - ))); - } - if status == 403 { - return Err(EwsError::HttpStatus { - status: 403, - body: truncate(&bytes), - }); - } - match classify_http_status(status) { - Disposition::Fatal => { - return Err(EwsError::HttpStatus { - status, - body: truncate(&bytes), - }); - } - Disposition::Retryable => { - attempt += 1; - self.inner.retries_total.fetch_add(1, Ordering::Relaxed); - if attempt > policy.max_retries { - return Err(EwsError::RetriesExhausted(format!( - "{action} kept returning http {status}" - ))); - } - if retry_after.is_some() { - self.inner - .retry_after_sleeps - .fetch_add(1, Ordering::Relaxed); - } - let chosen = self.inner.rate_limit.on_throttle(&policy, retry_after); - if chosen >= LONG_RETRY_THRESHOLD { - logger.warn(&format!( - "EWS {action} rate-limited (http {status}); waiting {}s", - chosen.as_secs() - )); - } - if logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "retry {}/{} (http {status})", - attempt, policy.max_retries - ); - } - std::thread::sleep(chosen); - } - } - } - AttemptOutcome::Transport(err) => { - attempt += 1; - self.inner.retries_total.fetch_add(1, Ordering::Relaxed); - if attempt > policy.max_retries { - return Err(err); - } - let delay = retry::backoff_delay(&policy, attempt); - if delay >= LONG_RETRY_THRESHOLD { - logger.warn(&format!( - "transient transport failure ({err}); waiting {}s before retry {}/{}", - delay.as_secs(), - attempt, - policy.max_retries - )); - } - std::thread::sleep(delay); - } - } - } - } - - fn on_fault( - &self, - fault: &SoapFault, - sent_version: ServerVersion, - action: &str, - policy: &RetryPolicy, - logger: &Logger, - attempt: &mut u32, - ) -> FaultStep { - match classify_fault(&fault.response_code) { - FaultDisposition::Fatal => FaultStep::Fail(EwsError::SoapFault { - code: fault.response_code.clone(), - reason: fault.fault_string.clone(), - }), - FaultDisposition::Auth => FaultStep::Fail(EwsError::Auth(fault.fault_string.clone())), - FaultDisposition::VersionError => match sent_version.lower() { - Some(next) => { - logger.warn(&format!( - "EWS rejected schema version {} ({}); retrying as {}", - sent_version.as_str(), - fault.response_code, - next.as_str() - )); - FaultStep::Downgrade(next) - } - None => FaultStep::Fail(EwsError::SoapFault { - code: fault.response_code.clone(), - reason: format!( - "server rejected every supported EWS schema version (last tried {}): {}", - sent_version.as_str(), - fault.fault_string - ), - }), - }, - FaultDisposition::Retryable { delay } => { - *attempt += 1; - self.inner.retries_total.fetch_add(1, Ordering::Relaxed); - if *attempt > policy.max_retries { - return FaultStep::Fail(EwsError::RetriesExhausted(format!( - "{action} kept returning {code}", - code = fault.response_code - ))); - } - let chosen = self.inner.rate_limit.on_throttle(policy, delay); - if chosen >= LONG_RETRY_THRESHOLD { - logger.warn(&format!( - "EWS soap fault {} ({}); waiting {}s before retry {}/{}", - fault.response_code, - fault.fault_string, - chosen.as_secs(), - *attempt, - policy.max_retries - )); - } - if logger.enabled(LEVEL_BODIES) { - eprintln!( - "retry {}/{} {} after {:?} ({})", - *attempt, policy.max_retries, action, chosen, fault.response_code, - ); - } - FaultStep::Sleep(chosen) - } - } - } - - fn one_attempt(&self, url: &str, body: &str, action: &str) -> AttemptOutcome { - let mut req = self - .inner - .agent - .post(url) - .header("Authorization", self.auth_header()) - .header("Content-Type", "text/xml; charset=utf-8") - .header("Accept", "text/xml, application/soap+xml, application/xml") - .header("Accept-Encoding", "gzip") - .header("SOAPAction", action.to_owned()) - .header("User-Agent", self.inner.user_agent.as_str()) - .header("X-PreferServerAffinity", "True"); - if let Some(anchor) = self.anchor_header() { - req = req.header("X-AnchorMailbox", anchor); - } - if let Some(cookie) = self.affinity_cookie() { - req = req.header("X-BackEndOverrideCookie", cookie); - } - let logger = self.logger(); - let started = Instant::now(); - let result = req.send(body.as_bytes()); - match result { - Ok(mut resp) => { - let status = resp.status().as_u16(); - let response_type = resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .map(str::to_owned); - let retry_after = resp - .headers() - .get("retry-after") - .and_then(|v| v.to_str().ok()) - .and_then(retry_after_header); - if let Some(cookie) = extract_affinity_cookie(resp.headers()) - && let Ok(mut g) = self.inner.affinity_cookie.lock() - { - *g = Some(cookie); - } - match resp.body_mut().with_config().limit(MAX_BODY).read_to_vec() { - Ok(bytes) => { - logger.trace_http(&HttpCall { - proto: "EWS", - method: action, - url, - status, - elapsed: started.elapsed(), - note: None, - request: Some(body.as_bytes()), - request_type: Some("text/xml"), - response: &bytes, - response_type: response_type.as_deref(), - }); - AttemptOutcome::Ok { - status, - body: bytes, - retry_after, - } - } - Err(e) => AttemptOutcome::Transport(EwsError::Transport(format!( - "reading response body: {e}" - ))), - } - } - Err(e) => { - let err = map_ureq_error(e); - logger.trace_http_error("EWS", action, url, &err.to_string(), started.elapsed()); - AttemptOutcome::Transport(err) - } - } - } -} - -enum AttemptOutcome { - Ok { - status: u16, - body: Vec, - retry_after: Option, - }, - Transport(EwsError), -} - -enum FaultStep { - Fail(EwsError), - Sleep(Duration), - Downgrade(ServerVersion), -} - -fn extract_affinity_cookie(headers: &ureq::http::HeaderMap) -> Option { - for value in headers.get_all("set-cookie") { - let Ok(s) = value.to_str() else { continue }; - if let Some(rest) = s.trim_start().strip_prefix("X-BackEndOverrideCookie=") { - let val = rest.split(';').next().unwrap_or(rest).trim(); - if !val.is_empty() { - return Some(val.to_owned()); - } - } - } - None -} - -fn map_ureq_error(err: ureq::Error) -> EwsError { - match err { - ureq::Error::Io(e) => EwsError::Transport(format!("io: {e}")), - ureq::Error::Timeout(t) => EwsError::Transport(format!("timeout: {t}")), - ureq::Error::HostNotFound => EwsError::Transport("host not found".to_owned()), - ureq::Error::ConnectionFailed => EwsError::Transport("connection failed".to_owned()), - ureq::Error::BodyStalled => EwsError::Transport("body stalled".to_owned()), - ureq::Error::Tls(m) => EwsError::Transport(format!("tls: {m}")), - ureq::Error::TooManyRedirects => EwsError::Connect("too many redirects".to_owned()), - ureq::Error::RedirectFailed => EwsError::Connect("redirect failed".to_owned()), - ureq::Error::TlsRequired => { - EwsError::Connect("server requires TLS but transport is unsecured".to_owned()) - } - other => EwsError::Connect(other.to_string()), - } -} - -fn truncate(body: &[u8]) -> String { - let text = String::from_utf8_lossy(body); - if text.len() <= 512 { - return text.into_owned(); - } - let end = text.floor_char_boundary(512); - format!("{}...", &text[..end]) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn client_constructs_with_defaults() { - let c = EwsClient::new( - Auth::Bearer { token: "t".into() }, - RetryPolicy::new(3), - false, - ); - assert_eq!(c.server_version(), ServerVersion::Exchange2013Sp1); - assert_eq!(c.retries_observed(), 0); - assert_eq!(c.retry_after_sleeps(), 0); - } - - #[test] - fn server_version_can_be_pinned() { - let c = EwsClient::new( - Auth::Bearer { token: "t".into() }, - RetryPolicy::new(0), - false, - ); - c.set_server_version(ServerVersion::Exchange2016); - assert_eq!(c.server_version(), ServerVersion::Exchange2016); - } - - #[test] - fn accepted_downgrade_caps_reported_version() { - let c = EwsClient::new( - Auth::Bearer { token: "t".into() }, - RetryPolicy::new(0), - false, - ); - c.pin_accepted_version(ServerVersion::Exchange2013Sp1); - c.set_server_version(ServerVersion::Exchange2016); - assert_eq!(c.server_version(), ServerVersion::Exchange2013Sp1); - c.set_server_version(ServerVersion::Exchange2010Sp2); - assert_eq!(c.server_version(), ServerVersion::Exchange2010Sp2); - } - - #[test] - fn anchor_mailbox_round_trips() { - let c = EwsClient::new( - Auth::Bearer { token: "t".into() }, - RetryPolicy::new(0), - false, - ); - c.set_anchor_mailbox(Some("alice@x".to_owned())); - assert_eq!(c.anchor_header().as_deref(), Some("alice@x")); - } -} diff --git a/src/exchange_ews/contact_map.rs b/src/exchange_ews/contact_map.rs deleted file mode 100644 index 2784362..0000000 --- a/src/exchange_ews/contact_map.rs +++ /dev/null @@ -1,563 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use chrono::Datelike; -use serde_json::{Map, Value, json}; - -use crate::exchange::date::{is_outlook_no_date, nearest_midnight}; -use crate::exchange_ews::parse::{ContactItemRaw, RawContactAddress}; - -pub fn synthetic_uid(item_id: &str) -> String { - let hash = blake3::hash(item_id.as_bytes()).to_hex(); - format!("inbuxa-migrate-ews-syn-{hash}") -} - -pub fn to_jscontact(raw: &ContactItemRaw) -> Value { - let mut card = Map::new(); - card.insert("@type".to_owned(), Value::String("Card".to_owned())); - card.insert("version".to_owned(), Value::String("1.0".to_owned())); - if raw.is_group { - card.insert("kind".to_owned(), Value::String("group".to_owned())); - let mut members = Map::new(); - for m in &raw.members { - let key = match m.email.as_ref() { - Some(email) => format!("mailto:{email}"), - None => match m.name.as_ref() { - Some(name) => name.clone(), - None => continue, - }, - }; - members.insert(key, Value::Bool(true)); - } - if !members.is_empty() { - card.insert("members".to_owned(), Value::Object(members)); - } - } - if let Some(full) = raw.display_name.as_ref() { - let mut name = Map::new(); - name.insert("@type".to_owned(), Value::String("Name".to_owned())); - name.insert("full".to_owned(), Value::String(full.clone())); - let components = name_components(raw); - if !components.is_empty() { - name.insert("components".to_owned(), Value::Array(components)); - } - card.insert("name".to_owned(), Value::Object(name)); - } - if let Some(n) = raw.nickname.as_ref() { - card.insert( - "nicknames".to_owned(), - Value::Object(map_singleton("1", json!({"@type": "Nickname", "name": n}))), - ); - } - let mut organizations: Map = Map::new(); - let mut have_org = false; - let mut org_inner: Map = Map::new(); - org_inner.insert("@type".to_owned(), Value::String("Organization".to_owned())); - if let Some(co) = raw.company_name.as_ref() { - org_inner.insert("name".to_owned(), Value::String(co.clone())); - have_org = true; - } - let mut org_units: Vec = Vec::new(); - if let Some(dep) = raw.department.as_ref() { - org_units.push(json!({"@type": "OrgUnit", "name": dep})); - } - if let Some(office) = raw.office_location.as_ref() { - org_units.push(json!({"@type": "OrgUnit", "name": office})); - } - if !org_units.is_empty() { - org_inner.insert("units".to_owned(), Value::Array(org_units)); - have_org = true; - } - if have_org { - organizations.insert("1".to_owned(), Value::Object(org_inner)); - } - let mut next_org_idx = 2u32; - for extra in &raw.companies { - if matches!(raw.company_name.as_deref(), Some(c) if c == extra) { - continue; - } - organizations.insert( - next_org_idx.to_string(), - json!({"@type": "Organization", "name": extra}), - ); - next_org_idx += 1; - } - if !organizations.is_empty() { - card.insert("organizations".to_owned(), Value::Object(organizations)); - } - if let Some(t) = raw.job_title.as_ref() { - card.insert( - "titles".to_owned(), - Value::Object(map_singleton("1", json!({"@type": "Title", "name": t}))), - ); - } - { - let mut map = Map::new(); - for (i, (key, addr)) in raw - .emails - .iter() - .filter(|(_, v)| !v.trim().is_empty()) - .enumerate() - { - let mut entry = Map::new(); - entry.insert("@type".to_owned(), Value::String("EmailAddress".to_owned())); - entry.insert("address".to_owned(), Value::String(addr.clone())); - entry.insert("contexts".to_owned(), email_contexts(key)); - map.insert((i + 1).to_string(), Value::Object(entry)); - } - if !map.is_empty() { - card.insert("emails".to_owned(), Value::Object(map)); - } - } - { - let mut map = Map::new(); - for (i, (key, num)) in raw - .phones - .iter() - .filter(|(_, v)| !v.trim().is_empty()) - .enumerate() - { - let mut entry = Map::new(); - entry.insert("@type".to_owned(), Value::String("Phone".to_owned())); - entry.insert("number".to_owned(), Value::String(num.clone())); - entry.insert("contexts".to_owned(), phone_contexts(key)); - entry.insert("features".to_owned(), phone_features(key)); - map.insert((i + 1).to_string(), Value::Object(entry)); - } - if !map.is_empty() { - card.insert("phones".to_owned(), Value::Object(map)); - } - } - { - let mut map = Map::new(); - for (i, (key, addr)) in raw - .ims - .iter() - .filter(|(_, v)| !v.trim().is_empty()) - .enumerate() - { - let entry_id = (i + 1).to_string(); - let mut entry = Map::new(); - entry.insert( - "@type".to_owned(), - Value::String("OnlineService".to_owned()), - ); - entry.insert("user".to_owned(), Value::String(addr.clone())); - if let Some(service) = im_service_from_key(key) { - entry.insert("service".to_owned(), Value::String(service)); - } - map.insert(entry_id, Value::Object(entry)); - } - if !map.is_empty() { - card.insert("onlineServices".to_owned(), Value::Object(map)); - } - } - if !raw.addresses.is_empty() { - let pref_key = raw - .postal_address_index - .as_deref() - .map(str::to_ascii_lowercase); - let mut map = Map::new(); - for (i, addr) in raw.addresses.iter().enumerate() { - let entry_id = format!("{}", i + 1); - let is_pref = matches!(&pref_key, Some(p) if !p.is_empty() && p == &addr.key.to_ascii_lowercase()); - map.insert(entry_id, address_to_jscontact(addr, is_pref)); - } - card.insert("addresses".to_owned(), Value::Object(map)); - } - if let Some(url) = raw.url.as_ref() { - card.insert( - "links".to_owned(), - Value::Object(map_singleton("1", json!({"@type": "Link", "uri": url}))), - ); - } - if let Some(bday) = raw.birthday.as_ref() - && let Some(partial) = partial_date(bday) - { - card.insert( - "anniversaries".to_owned(), - Value::Object(map_singleton( - "birth", - json!({"@type": "Anniversary", "kind": "birth", "date": partial}), - )), - ); - } - if let Some(anniv) = raw.wedding_anniversary.as_ref() - && let Some(partial) = partial_date(anniv) - { - let key = "wedding"; - let entry = json!({"@type": "Anniversary", "kind": "wedding", "date": partial}); - if let Some(map) = card.get_mut("anniversaries").and_then(Value::as_object_mut) { - map.insert(key.to_owned(), entry); - } else { - card.insert( - "anniversaries".to_owned(), - Value::Object(map_singleton(key, entry)), - ); - } - } - if !raw.categories.is_empty() { - let mut keywords = Map::new(); - for cat in &raw.categories { - keywords.insert(cat.to_ascii_lowercase(), Value::Bool(true)); - } - card.insert("keywords".to_owned(), Value::Object(keywords)); - } - if let Some(notes) = raw.notes.as_ref().filter(|n| !n.trim().is_empty()) { - card.insert( - "notes".to_owned(), - Value::Object(map_singleton("1", json!({"@type": "Note", "note": notes}))), - ); - } - Value::Object(card) -} - -fn name_components(raw: &ContactItemRaw) -> Vec { - let mut out: Vec = Vec::new(); - if let Some(v) = raw.given_name.as_ref() { - out.push(json!({"@type": "NameComponent", "kind": "given", "value": v})); - } - if let Some(v) = raw.middle_name.as_ref() { - out.push(json!({"@type": "NameComponent", "kind": "given2", "value": v})); - } - if let Some(v) = raw.surname.as_ref() { - out.push(json!({"@type": "NameComponent", "kind": "surname", "value": v})); - } - if let Some(v) = raw.generation.as_ref() { - out.push(json!({"@type": "NameComponent", "kind": "generation", "value": v})); - } - out -} - -fn email_contexts(key: &str) -> Value { - let mut o = Map::new(); - let lower = key.to_ascii_lowercase(); - if lower.contains("business") || lower.contains("work") { - o.insert("work".to_owned(), Value::Bool(true)); - } else if lower.contains("home") { - o.insert("private".to_owned(), Value::Bool(true)); - } - Value::Object(o) -} - -fn phone_contexts(key: &str) -> Value { - let mut o = Map::new(); - let lower = key.to_ascii_lowercase(); - if lower.starts_with("business") || lower.starts_with("company") { - o.insert("work".to_owned(), Value::Bool(true)); - } else if lower.starts_with("home") || lower.starts_with("mobile") { - o.insert("private".to_owned(), Value::Bool(true)); - } - Value::Object(o) -} - -fn phone_features(key: &str) -> Value { - let mut o = Map::new(); - let lower = key.to_ascii_lowercase(); - if lower.contains("mobile") { - o.insert("mobile".to_owned(), Value::Bool(true)); - } else if lower.contains("fax") { - o.insert("fax".to_owned(), Value::Bool(true)); - } else if lower.contains("pager") { - o.insert("pager".to_owned(), Value::Bool(true)); - } else { - o.insert("voice".to_owned(), Value::Bool(true)); - } - Value::Object(o) -} - -fn address_to_jscontact(addr: &RawContactAddress, is_pref: bool) -> Value { - let mut o = Map::new(); - o.insert("@type".to_owned(), Value::String("Address".to_owned())); - let mut components: Vec = Vec::new(); - if let Some(s) = addr.street.as_ref() { - components.push(json!({"@type": "AddressComponent", "kind": "name", "value": s})); - } - if let Some(s) = addr.city.as_ref() { - components.push(json!({"@type": "AddressComponent", "kind": "locality", "value": s})); - } - if let Some(s) = addr.state.as_ref() { - components.push(json!({"@type": "AddressComponent", "kind": "region", "value": s})); - } - if let Some(s) = addr.postal_code.as_ref() { - components.push(json!({"@type": "AddressComponent", "kind": "postcode", "value": s})); - } - if let Some(s) = addr.country.as_ref() { - components.push(json!({"@type": "AddressComponent", "kind": "country", "value": s})); - } - if !components.is_empty() { - o.insert("components".to_owned(), Value::Array(components)); - } - let mut contexts = Map::new(); - let lower = addr.key.to_ascii_lowercase(); - if lower == "business" || lower == "work" { - contexts.insert("work".to_owned(), Value::Bool(true)); - } else if lower == "home" { - contexts.insert("private".to_owned(), Value::Bool(true)); - } - if !contexts.is_empty() { - o.insert("contexts".to_owned(), Value::Object(contexts)); - } - if is_pref { - o.insert("pref".to_owned(), Value::from(1u32)); - } - Value::Object(o) -} - -fn im_service_from_key(key: &str) -> Option { - let trimmed: String = key.chars().filter(|c| !c.is_ascii_digit()).collect(); - let canonical = trimmed.trim_start_matches("ImAddress"); - if canonical.is_empty() { - return None; - } - Some(canonical.to_owned()) -} - -fn partial_date(s: &str) -> Option { - if is_outlook_no_date(s) { - return None; - } - let date = nearest_midnight(s, None)?; - let mut o = Map::new(); - o.insert("@type".to_owned(), Value::String("PartialDate".to_owned())); - if date.year() > 0 { - o.insert("year".to_owned(), Value::from(date.year())); - } - o.insert("month".to_owned(), Value::from(date.month())); - o.insert("day".to_owned(), Value::from(date.day())); - Some(Value::Object(o)) -} - -fn map_singleton(key: &str, value: Value) -> Map { - let mut m = Map::new(); - m.insert(key.to_owned(), value); - m -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn basic_contact_round_trip() { - let raw = ContactItemRaw { - display_name: Some("Alice Doe".to_owned()), - given_name: Some("Alice".to_owned()), - surname: Some("Doe".to_owned()), - emails: vec![("EmailAddress1".to_owned(), "alice@x".to_owned())], - phones: vec![("BusinessPhone1".to_owned(), "+1".to_owned())], - ..ContactItemRaw::default() - }; - let v = to_jscontact(&raw); - assert_eq!(v["@type"], "Card"); - assert_eq!(v["name"]["full"], "Alice Doe"); - let comps = v["name"]["components"].as_array().unwrap(); - assert_eq!(comps[0]["kind"], "given"); - assert_eq!(comps[0]["value"], "Alice"); - assert_eq!(v["emails"]["1"]["address"], "alice@x"); - assert_eq!(v["phones"]["1"]["contexts"]["work"], true); - } - - #[test] - fn empty_body_and_empty_im_entry_are_omitted() { - let raw = ContactItemRaw { - display_name: Some("Random Person".to_owned()), - notes: Some(String::new()), - ims: vec![("ImAddress1".to_owned(), String::new())], - emails: vec![("EmailAddress1".to_owned(), " ".to_owned())], - phones: vec![("MobilePhone".to_owned(), "+17801234567".to_owned())], - ..ContactItemRaw::default() - }; - let v = to_jscontact(&raw); - assert!(v.get("notes").is_none(), "empty body must not emit notes"); - assert!( - v.get("onlineServices").is_none(), - "empty IM entry must not emit onlineServices" - ); - assert!( - v.get("emails").is_none(), - "whitespace-only email must not emit emails" - ); - assert_eq!(v["phones"]["1"]["number"], "+17801234567"); - } - - #[test] - fn synthetic_uid_is_stable() { - let u1 = synthetic_uid("ITEM-1"); - let u2 = synthetic_uid("ITEM-1"); - assert_eq!(u1, u2); - assert_ne!(u1, synthetic_uid("ITEM-2")); - assert!(u1.starts_with("inbuxa-migrate-ews-syn-")); - } - - #[test] - fn anniversaries_combine_birth_and_wedding() { - let raw = ContactItemRaw { - display_name: Some("X".to_owned()), - birthday: Some("1990-04-15T00:00:00Z".to_owned()), - wedding_anniversary: Some("2015-08-22T00:00:00".to_owned()), - ..ContactItemRaw::default() - }; - let v = to_jscontact(&raw); - assert_eq!(v["anniversaries"]["birth"]["kind"], "birth"); - assert_eq!(v["anniversaries"]["birth"]["date"]["month"], 4); - assert_eq!(v["anniversaries"]["wedding"]["kind"], "wedding"); - } - - #[test] - fn birthday_stored_as_local_midnight_in_utc_keeps_its_day() { - let raw = ContactItemRaw { - display_name: Some("X".to_owned()), - birthday: Some("1990-04-14T22:00:00Z".to_owned()), - ..ContactItemRaw::default() - }; - let v = to_jscontact(&raw); - assert_eq!( - v["anniversaries"]["birth"]["date"], - json!({"@type": "PartialDate", "year": 1990, "month": 4, "day": 15}) - ); - } - - #[test] - fn year_zero_birthday_has_no_year() { - let raw = ContactItemRaw { - display_name: Some("X".to_owned()), - birthday: Some("0000-04-15T00:00:00Z".to_owned()), - ..ContactItemRaw::default() - }; - let v = to_jscontact(&raw); - assert_eq!( - v["anniversaries"]["birth"]["date"], - json!({"@type": "PartialDate", "month": 4, "day": 15}) - ); - } - - #[test] - fn outlook_no_date_anniversary_is_dropped() { - let raw = ContactItemRaw { - display_name: Some("X".to_owned()), - birthday: Some("1990-04-15T00:00:00Z".to_owned()), - wedding_anniversary: Some("4501-01-01T00:00:00Z".to_owned()), - ..ContactItemRaw::default() - }; - let v = to_jscontact(&raw); - assert!(v["anniversaries"].get("wedding").is_none()); - assert_eq!(v["anniversaries"]["birth"]["date"]["day"], 15); - } - - #[test] - fn office_location_is_added_as_org_unit() { - let raw = ContactItemRaw { - display_name: Some("X".to_owned()), - company_name: Some("Initech".to_owned()), - department: Some("R&D".to_owned()), - office_location: Some("Bldg 4".to_owned()), - ..ContactItemRaw::default() - }; - let v = to_jscontact(&raw); - let units = v["organizations"]["1"]["units"].as_array().unwrap(); - assert_eq!(units[0]["name"], "R&D"); - assert_eq!(units[1]["name"], "Bldg 4"); - } - - #[test] - fn extra_companies_become_additional_organizations() { - let raw = ContactItemRaw { - display_name: Some("X".to_owned()), - company_name: Some("Initech".to_owned()), - companies: vec!["Initech".to_owned(), "Initrode".to_owned()], - ..ContactItemRaw::default() - }; - let v = to_jscontact(&raw); - assert_eq!(v["organizations"]["1"]["name"], "Initech"); - assert_eq!(v["organizations"]["2"]["name"], "Initrode"); - } - - #[test] - fn postal_address_index_sets_pref_on_matching_entry() { - let raw = ContactItemRaw { - display_name: Some("X".to_owned()), - addresses: vec![ - RawContactAddress { - key: "Home".to_owned(), - street: Some("1 St".to_owned()), - ..Default::default() - }, - RawContactAddress { - key: "Business".to_owned(), - street: Some("2 Av".to_owned()), - ..Default::default() - }, - ], - postal_address_index: Some("Business".to_owned()), - ..ContactItemRaw::default() - }; - let v = to_jscontact(&raw); - assert!(v["addresses"]["1"].get("pref").is_none()); - assert_eq!(v["addresses"]["2"]["pref"], 1); - } - - #[test] - fn im_service_extracted_from_key() { - let raw = ContactItemRaw { - display_name: Some("X".to_owned()), - ims: vec![("ImAddress1".to_owned(), "alice@msn".to_owned())], - ..ContactItemRaw::default() - }; - let v = to_jscontact(&raw); - assert_eq!(v["onlineServices"]["1"]["user"], "alice@msn"); - assert!(v["onlineServices"]["1"].get("service").is_none()); - } - - #[test] - fn distribution_list_becomes_group_card_with_members() { - let raw = ContactItemRaw { - display_name: Some("The Team".to_owned()), - is_group: true, - members: vec![ - crate::exchange_ews::parse::RawGroupMember { - name: Some("Bob".to_owned()), - email: Some("bob@x".to_owned()), - }, - crate::exchange_ews::parse::RawGroupMember { - name: Some("No Email".to_owned()), - email: None, - }, - ], - ..ContactItemRaw::default() - }; - let v = to_jscontact(&raw); - assert_eq!(v["kind"], "group"); - assert_eq!(v["name"]["full"], "The Team"); - assert_eq!(v["members"]["mailto:bob@x"], true); - assert_eq!(v["members"]["No Email"], true); - } - - #[test] - fn individual_contact_has_no_kind_or_members() { - let raw = ContactItemRaw { - display_name: Some("Alice".to_owned()), - ..ContactItemRaw::default() - }; - let v = to_jscontact(&raw); - assert!(v.get("kind").is_none()); - assert!(v.get("members").is_none()); - } - - #[test] - fn unresolvable_relations_are_dropped() { - let raw = ContactItemRaw { - display_name: Some("X".to_owned()), - manager: Some("Boss".to_owned()), - assistant: Some("Asst".to_owned()), - spouse: Some("Partner".to_owned()), - children: vec!["Kid".to_owned()], - ..ContactItemRaw::default() - }; - let v = to_jscontact(&raw); - assert!(v.get("relatedTo").is_none()); - } -} diff --git a/src/exchange_ews/oauth.rs b/src/exchange_ews/oauth.rs deleted file mode 100644 index e00e935..0000000 --- a/src/exchange_ews/oauth.rs +++ /dev/null @@ -1,386 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::io::{self, Write}; -use std::time::Duration; - -use encodify::base64::{Base64, Padding, URL_SAFE}; -use serde_json::Value; -use ureq::config::Config; -use ureq::tls::{RootCerts, TlsConfig}; - -use crate::exchange_ews::error::EwsError; - -pub const SCOPE_APP_ONLY: &str = "https://outlook.office365.com/.default"; -pub const SCOPE_DELEGATED: &str = - "https://outlook.office365.com/EWS.AccessAsUser.All offline_access"; - -const JWT_SEGMENT: Base64 = URL_SAFE.with_padding(Padding::Optional).any_alphabet(); - -#[derive(Debug, Clone)] -pub struct AcquiredToken { - pub access_token: String, - pub refresh_token: Option, - pub expires_in: Option, - pub tenant_id: Option, - pub upn: Option, - pub name: Option, -} - -#[derive(Debug, Clone)] -pub enum OAuthFlow { - PreAcquired { - token: String, - }, - ClientCredentials { - tenant: String, - client_id: String, - client_secret: String, - }, - DeviceCode { - tenant: String, - client_id: String, - }, -} - -pub fn acquire(flow: &OAuthFlow, allow_invalid_certs: bool) -> Result { - match flow { - OAuthFlow::PreAcquired { token } => { - let claims = decode_jwt_claims(token).unwrap_or_default(); - Ok(AcquiredToken { - access_token: token.clone(), - refresh_token: None, - expires_in: None, - tenant_id: claims.tenant_id, - upn: claims.upn, - name: claims.name, - }) - } - OAuthFlow::ClientCredentials { - tenant, - client_id, - client_secret, - } => client_credentials(tenant, client_id, client_secret, allow_invalid_certs), - OAuthFlow::DeviceCode { tenant, client_id } => { - device_code_flow(tenant, client_id, allow_invalid_certs) - } - } -} - -#[derive(Debug, Clone, Default)] -pub struct JwtClaims { - pub tenant_id: Option, - pub upn: Option, - pub name: Option, - pub exp: Option, -} - -pub fn decode_jwt_claims(token: &str) -> Option { - let mut parts = token.split('.'); - let _header = parts.next()?; - let payload = parts.next()?; - let bytes = JWT_SEGMENT.decode(payload).ok()?; - let value: Value = serde_json::from_slice(&bytes).ok()?; - Some(JwtClaims { - tenant_id: value.get("tid").and_then(Value::as_str).map(str::to_owned), - upn: value - .get("upn") - .and_then(Value::as_str) - .or_else(|| value.get("preferred_username").and_then(Value::as_str)) - .map(str::to_owned), - name: value.get("name").and_then(Value::as_str).map(str::to_owned), - exp: value.get("exp").and_then(Value::as_u64), - }) -} - -fn token_endpoint(tenant: &str) -> String { - format!("https://login.microsoftonline.com/{tenant}/oauth2/v2.0/token") -} - -fn device_code_endpoint(tenant: &str) -> String { - format!("https://login.microsoftonline.com/{tenant}/oauth2/v2.0/devicecode") -} - -fn build_agent(allow_invalid_certs: bool) -> ureq::Agent { - let config: Config = Config::builder() - .http_status_as_error(false) - .tls_config( - TlsConfig::builder() - .unversioned_rustls_crypto_provider(std::sync::Arc::new( - rustls::crypto::aws_lc_rs::default_provider(), - )) - .root_certs(RootCerts::PlatformVerifier) - .disable_verification(allow_invalid_certs) - .build(), - ) - .build(); - config.new_agent() -} - -fn client_credentials( - tenant: &str, - client_id: &str, - client_secret: &str, - allow_invalid_certs: bool, -) -> Result { - let agent = build_agent(allow_invalid_certs); - let body = form_encode(&[ - ("client_id", client_id), - ("client_secret", client_secret), - ("scope", SCOPE_APP_ONLY), - ("grant_type", "client_credentials"), - ]); - let endpoint = token_endpoint(tenant); - let resp = agent - .post(&endpoint) - .header("Content-Type", "application/x-www-form-urlencoded") - .send(body.as_bytes()) - .map_err(|e| EwsError::OAuth(format!("token endpoint: {e}")))?; - parse_token_response(resp) -} - -fn device_code_flow( - tenant: &str, - client_id: &str, - allow_invalid_certs: bool, -) -> Result { - let agent = build_agent(allow_invalid_certs); - let body = form_encode(&[("client_id", client_id), ("scope", SCOPE_DELEGATED)]); - let endpoint = device_code_endpoint(tenant); - let mut resp = agent - .post(&endpoint) - .header("Content-Type", "application/x-www-form-urlencoded") - .send(body.as_bytes()) - .map_err(|e| EwsError::OAuth(format!("device code endpoint: {e}")))?; - let json: Value = resp - .body_mut() - .read_json() - .map_err(|e| EwsError::OAuth(format!("device code body: {e}")))?; - let device_code = json - .get("device_code") - .and_then(Value::as_str) - .ok_or_else(|| EwsError::OAuth("device_code missing".to_owned()))? - .to_owned(); - let user_code = json.get("user_code").and_then(Value::as_str).unwrap_or("?"); - let verify_uri = json - .get("verification_uri") - .and_then(Value::as_str) - .unwrap_or("https://microsoft.com/devicelogin"); - let interval = json.get("interval").and_then(Value::as_u64).unwrap_or(5); - let message = json.get("message").and_then(Value::as_str); - if let Some(m) = message { - eprintln!("{m}"); - } else { - eprintln!("To sign in, open {verify_uri} and enter the code {user_code}"); - } - io::stdout() - .flush() - .map_err(|e| EwsError::OAuth(format!("flushing stdout: {e}")))?; - let endpoint = token_endpoint(tenant); - let mut delay = Duration::from_secs(interval); - let deadline = std::time::Instant::now() - + Duration::from_secs( - json.get("expires_in") - .and_then(Value::as_u64) - .unwrap_or(900), - ); - loop { - std::thread::sleep(delay); - if std::time::Instant::now() >= deadline { - return Err(EwsError::OAuth( - "device code expired before the user completed sign-in".to_owned(), - )); - } - let body = form_encode(&[ - ("client_id", client_id), - ("grant_type", "urn:ietf:params:oauth:grant-type:device_code"), - ("device_code", &device_code), - ]); - let mut resp = agent - .post(&endpoint) - .header("Content-Type", "application/x-www-form-urlencoded") - .send(body.as_bytes()) - .map_err(|e| EwsError::OAuth(format!("device code polling: {e}")))?; - let status = resp.status().as_u16(); - if (200..300).contains(&status) { - return parse_token_response(resp); - } - let payload: Value = resp.body_mut().read_json().unwrap_or(Value::Null); - let error = payload.get("error").and_then(Value::as_str).unwrap_or(""); - match error { - "authorization_pending" => continue, - "slow_down" => { - delay = delay.saturating_add(Duration::from_secs(5)); - continue; - } - "expired_token" => { - return Err(EwsError::OAuth( - "device code expired; restart the import".to_owned(), - )); - } - "" => { - return Err(EwsError::OAuth(format!( - "device code polling: http {status}" - ))); - } - other => { - let desc = payload - .get("error_description") - .and_then(Value::as_str) - .unwrap_or(""); - return Err(EwsError::OAuth(format!( - "device code polling: {other}: {desc}" - ))); - } - } - } -} - -fn parse_token_response( - mut resp: ureq::http::Response, -) -> Result { - let status = resp.status().as_u16(); - let json: Value = resp - .body_mut() - .read_json() - .map_err(|e| EwsError::OAuth(format!("token body: {e}")))?; - if !(200..300).contains(&status) { - let desc = json - .get("error_description") - .and_then(Value::as_str) - .unwrap_or("(no error_description)"); - return Err(EwsError::OAuth(format!("http {status}: {desc}"))); - } - let access = json - .get("access_token") - .and_then(Value::as_str) - .ok_or_else(|| EwsError::OAuth("access_token missing".to_owned()))? - .to_owned(); - let refresh_token = json - .get("refresh_token") - .and_then(Value::as_str) - .map(str::to_owned); - let expires_in = json.get("expires_in").and_then(Value::as_u64); - let claims = decode_jwt_claims(&access).unwrap_or_default(); - Ok(AcquiredToken { - access_token: access, - refresh_token, - expires_in, - tenant_id: claims.tenant_id, - upn: claims.upn, - name: claims.name, - }) -} - -pub fn refresh_with_token( - tenant: &str, - client_id: &str, - refresh_token: &str, - allow_invalid_certs: bool, -) -> Result { - let agent = build_agent(allow_invalid_certs); - let body = form_encode(&[ - ("client_id", client_id), - ("grant_type", "refresh_token"), - ("refresh_token", refresh_token), - ("scope", SCOPE_DELEGATED), - ]); - let endpoint = token_endpoint(tenant); - let resp = agent - .post(&endpoint) - .header("Content-Type", "application/x-www-form-urlencoded") - .send(body.as_bytes()) - .map_err(|e| EwsError::OAuth(format!("refresh token endpoint: {e}")))?; - parse_token_response(resp) -} - -fn form_encode(pairs: &[(&str, &str)]) -> String { - let mut out = String::new(); - for (i, (k, v)) in pairs.iter().enumerate() { - if i > 0 { - out.push('&'); - } - out.push_str(&urlencode(k)); - out.push('='); - out.push_str(&urlencode(v)); - } - out -} - -fn urlencode(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - for b in s.bytes() { - match b { - b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { - out.push(b as char); - } - _ => out.push_str(&format!("%{b:02X}")), - } - } - out -} - -#[cfg(test)] -mod tests { - use super::*; - use encodify::base64::{STANDARD, STANDARD_NO_PAD, URL_SAFE_NO_PAD}; - - fn make_jwt(tid: &str, upn: &str, exp: u64) -> String { - let header = URL_SAFE_NO_PAD.encode(b"{\"alg\":\"none\"}"); - let claims = format!(r#"{{"tid":"{tid}","upn":"{upn}","exp":{exp}}}"#); - let payload = URL_SAFE_NO_PAD.encode(claims.as_bytes()); - format!("{header}.{payload}.") - } - - #[test] - fn jwt_decoder_extracts_tid_and_upn() { - let token = make_jwt("tenant-1", "alice@x.com", 9999999999); - let claims = decode_jwt_claims(&token).unwrap(); - assert_eq!(claims.tenant_id.as_deref(), Some("tenant-1")); - assert_eq!(claims.upn.as_deref(), Some("alice@x.com")); - assert_eq!(claims.exp, Some(9999999999)); - } - - #[test] - fn jwt_payload_decodes_in_either_alphabet_with_or_without_padding() { - let claims = r#"{"tid":"t-3","upn":"??>>@xy","exp":1}"#; - for engine in [URL_SAFE_NO_PAD, URL_SAFE, STANDARD_NO_PAD, STANDARD] { - let token = format!("h.{}.s", engine.encode(claims)); - let decoded = decode_jwt_claims(&token).unwrap(); - assert_eq!(decoded.upn.as_deref(), Some("??>>@xy"), "{token}"); - } - assert!(decode_jwt_claims("h.eyJ0aWQiOiJ0LTMifQ.s").is_some()); - assert!(decode_jwt_claims("h.eyJ0aWQiOiJ0LTMifQ=.s").is_none()); - assert!(decode_jwt_claims("h.eyJ0 aWQiOiJ0LTMifQ.s").is_none()); - } - - #[test] - fn malformed_token_returns_none() { - assert!(decode_jwt_claims("garbage").is_none()); - assert!(decode_jwt_claims("only.two").is_none()); - } - - #[test] - fn pre_acquired_flow_decodes_claims() { - let token = make_jwt("t-2", "bob@x", 9999999999); - let acq = acquire( - &OAuthFlow::PreAcquired { - token: token.clone(), - }, - false, - ) - .unwrap(); - assert_eq!(acq.access_token, token); - assert_eq!(acq.tenant_id.as_deref(), Some("t-2")); - assert_eq!(acq.upn.as_deref(), Some("bob@x")); - } - - #[test] - fn urlencode_handles_special_chars() { - assert_eq!(urlencode("a b/c"), "a%20b%2Fc"); - assert_eq!(urlencode("a.b-c_d"), "a.b-c_d"); - } -} diff --git a/src/exchange_ews/parse.rs b/src/exchange_ews/parse.rs deleted file mode 100644 index 2f1ad74..0000000 --- a/src/exchange_ews/parse.rs +++ /dev/null @@ -1,3093 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::io::BufRead; - -use quick_xml::NsReader; -use quick_xml::XmlVersion; -use quick_xml::events::{BytesRef, BytesStart, Event}; -use quick_xml::name::ResolveResult; - -use crate::exchange_ews::error::EwsError; -use crate::exchange_ews::soap::{NS_MESSAGES, NS_TYPES}; -use crate::exchange_ews::types::{CalendarItemType, FolderId, ItemId, ResponseCode, ServerVersion}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Ns { - Messages, - Types, - Other, -} - -fn classify(ns: &ResolveResult<'_>) -> Ns { - match ns { - ResolveResult::Bound(prefix) => { - let p = prefix.as_ref(); - if p == NS_MESSAGES { - Ns::Messages - } else if p == NS_TYPES { - Ns::Types - } else { - Ns::Other - } - } - _ => Ns::Other, - } -} - -fn is(ns: Ns, local: &str, expected_ns: Ns, target: &str) -> bool { - ns == expected_ns && local.eq_ignore_ascii_case(target) -} - -fn attr_value(e: &BytesStart<'_>, name: &str) -> Option { - for a in e.attributes().flatten() { - let key = a.key.local_name(); - if key.as_ref().eq_ignore_ascii_case(name) { - return Some( - a.normalized_value(XmlVersion::Implicit1_0) - .map(|c| c.into_owned()) - .unwrap_or_else(|_| a.value.into_owned()), - ); - } - } - None -} - -fn capture_id_attrs(e: &BytesStart<'_>, id_out: &mut String, ck_out: &mut String) { - for a in e.attributes().flatten() { - let key = a.key.local_name(); - let kb = key.as_ref(); - let v = a - .normalized_value(XmlVersion::Implicit1_0) - .map(|c| c.into_owned()) - .unwrap_or_else(|_| a.value.into_owned()); - if kb.eq_ignore_ascii_case("Id") { - *id_out = v; - } else if kb.eq_ignore_ascii_case("ChangeKey") { - *ck_out = v; - } - } -} - -#[derive(Debug, Clone, Default)] -pub struct ServerVersionInfo { - pub major_version: Option, - pub minor_version: Option, - pub major_build: Option, - pub minor_build: Option, - pub version: Option, -} - -impl ServerVersionInfo { - pub fn to_server_version(&self) -> Option { - match (self.major_version, self.minor_version) { - (Some(maj), Some(min)) => Some(ServerVersion::from_build(maj, min)), - _ => None, - } - } -} - -fn read_version_attrs(e: &BytesStart<'_>, out: &mut ServerVersionInfo) { - for a in e.attributes().flatten() { - let key = a.key.local_name(); - let kb = key.as_ref(); - let v = a.value.into_owned(); - if kb.eq_ignore_ascii_case("MajorVersion") { - out.major_version = v.parse().ok(); - } else if kb.eq_ignore_ascii_case("MinorVersion") { - out.minor_version = v.parse().ok(); - } else if kb.eq_ignore_ascii_case("MajorBuildNumber") { - out.major_build = v.parse().ok(); - } else if kb.eq_ignore_ascii_case("MinorBuildNumber") { - out.minor_build = v.parse().ok(); - } else if kb.eq_ignore_ascii_case("Version") { - out.version = Some(v); - } - } -} - -#[derive(Debug, Clone)] -pub struct SoapFault { - pub fault_code: String, - pub fault_string: String, - pub response_code: ResponseCode, - pub back_off_ms: Option, -} - -#[derive(Debug, Clone)] -pub enum EnvelopeKind { - Fault { - version: ServerVersionInfo, - fault: SoapFault, - }, - Body { - version: ServerVersionInfo, - }, -} - -pub fn read_envelope_summary(bytes: &[u8]) -> Result { - let mut xml = NsReader::from_reader(bytes); - xml.config_mut().trim_text(true); - let mut buf = Vec::new(); - let mut version = ServerVersionInfo::default(); - let mut seen_body = false; - loop { - buf.clear(); - let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; - let ns_kind = classify(&ns); - match ev { - Event::Start(ref e) | Event::Empty(ref e) => { - let local = e.local_name().as_ref().to_owned(); - if is(ns_kind, &local, Ns::Types, "ServerVersionInfo") { - read_version_attrs(e, &mut version); - } else if local.eq_ignore_ascii_case("Fault") { - let fault = parse_fault(&mut xml)?; - return Ok(EnvelopeKind::Fault { version, fault }); - } else if local.eq_ignore_ascii_case("Body") { - seen_body = true; - } - } - Event::Eof => break, - _ => {} - } - } - if seen_body { - Ok(EnvelopeKind::Body { version }) - } else { - Err(EwsError::Malformed("envelope had no soap:Body".to_owned())) - } -} - -fn parse_fault(xml: &mut NsReader) -> Result { - let mut buf = Vec::new(); - let mut fault_code = String::new(); - let mut fault_string = String::new(); - let mut code = ResponseCode::Other(String::new()); - let mut back_off: Option = None; - let mut text_target: Option<&'static str> = None; - let mut last_value_name: Option = None; - let mut depth: u32 = 1; - loop { - buf.clear(); - let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; - let ns_kind = classify(&ns); - match ev { - Event::Start(e) => { - depth += 1; - let local = e.local_name().as_ref().to_owned(); - if local.eq_ignore_ascii_case("faultcode") { - text_target = Some("faultcode"); - } else if local.eq_ignore_ascii_case("faultstring") { - text_target = Some("faultstring"); - } else if local.eq_ignore_ascii_case("ResponseCode") { - text_target = Some("responseCode"); - } else if local.eq_ignore_ascii_case("Value") && ns_kind == Ns::Types { - last_value_name = attr_value(&e, "Name"); - text_target = Some("messageXmlValue"); - } else { - text_target = None; - } - } - Event::Empty(e) => { - let local = e.local_name().as_ref().to_owned(); - if is(ns_kind, &local, Ns::Types, "Value") { - last_value_name = attr_value(&e, "Name"); - } - } - Event::End(_) => { - if depth == 0 { - break; - } - depth -= 1; - text_target = None; - if depth == 0 { - break; - } - } - Event::Text(t) => { - let text = t.into_inner().into_owned(); - match text_target { - Some("faultcode") => fault_code = text, - Some("faultstring") => fault_string.push_str(&text), - Some("responseCode") => code = ResponseCode::parse(&text), - Some("messageXmlValue") - if last_value_name.as_deref() == Some("BackOffMilliseconds") => - { - back_off = text.trim().parse().ok(); - } - _ => {} - } - } - Event::CData(c) if text_target == Some("faultstring") => { - fault_string.push_str(&c); - } - Event::GeneralRef(ref g) if text_target == Some("faultstring") => { - if let Some(c) = entity_to_char(g) { - fault_string.push(c); - } - } - Event::Eof => break, - _ => {} - } - } - if matches!(code, ResponseCode::ServerBusy { .. }) { - code = ResponseCode::ServerBusy { - back_off_ms: back_off, - }; - } - Ok(SoapFault { - fault_code, - fault_string, - response_code: code, - back_off_ms: back_off, - }) -} - -#[derive(Debug, Clone, Default)] -pub struct FindFolderResponse { - pub folders: Vec, - pub more: bool, - pub total_in_view: Option, -} - -#[derive(Debug, Clone)] -pub struct FolderEntry { - pub element: FolderElement, - pub folder_id: FolderId, - pub parent_id: Option, - pub display_name: String, - pub folder_class: String, - pub total_count: Option, - pub child_count: Option, - pub is_hidden: bool, -} - -impl Default for FolderEntry { - fn default() -> Self { - FolderEntry { - element: FolderElement::Folder, - folder_id: FolderId { - id: String::new(), - change_key: String::new(), - }, - parent_id: None, - display_name: String::new(), - folder_class: String::new(), - total_count: None, - child_count: None, - is_hidden: false, - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum FolderElement { - Folder, - CalendarFolder, - ContactsFolder, - TasksFolder, - SearchFolder, -} - -impl FolderElement { - fn from_local(local: &str) -> Option { - if local.eq_ignore_ascii_case("Folder") { - Some(FolderElement::Folder) - } else if local.eq_ignore_ascii_case("CalendarFolder") { - Some(FolderElement::CalendarFolder) - } else if local.eq_ignore_ascii_case("ContactsFolder") { - Some(FolderElement::ContactsFolder) - } else if local.eq_ignore_ascii_case("TasksFolder") { - Some(FolderElement::TasksFolder) - } else if local.eq_ignore_ascii_case("SearchFolder") { - Some(FolderElement::SearchFolder) - } else { - None - } - } -} - -pub fn parse_find_folder_response(body: &[u8]) -> Result { - let mut xml = NsReader::from_reader(body); - xml.config_mut().trim_text(true); - let mut buf = Vec::new(); - let mut out = FindFolderResponse::default(); - loop { - buf.clear(); - let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; - let ns_kind = classify(&ns); - match ev { - Event::Start(ref e) | Event::Empty(ref e) => { - let local = e.local_name().as_ref().to_owned(); - if is(ns_kind, &local, Ns::Messages, "RootFolder") { - if let Some(v) = attr_value(e, "TotalItemsInView") { - out.total_in_view = v.trim().parse().ok(); - } - if let Some(v) = attr_value(e, "IncludesLastItemInRange") { - out.more = !matches!(v.trim(), "true" | "1"); - } - } else if ns_kind == Ns::Types - && let Some(elem) = FolderElement::from_local(&local) - { - let entry = parse_folder_element(&mut xml, elem)?; - out.folders.push(entry); - } - } - Event::Eof => break, - _ => {} - } - } - Ok(out) -} - -pub fn parse_folder_inner(inner_xml: &str) -> Result, EwsError> { - let mut xml = NsReader::from_reader(inner_xml.as_bytes()); - xml.config_mut().trim_text(true); - let mut buf = Vec::new(); - loop { - buf.clear(); - let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; - let ns_kind = classify(&ns); - match ev { - Event::Start(_) | Event::Empty(_) => { - let local = match &ev { - Event::Start(e) | Event::Empty(e) => e.local_name().as_ref().to_owned(), - _ => continue, - }; - if ns_kind == Ns::Types - && let Some(elem) = FolderElement::from_local(&local) - { - return Ok(Some(parse_folder_element(&mut xml, elem)?)); - } - } - Event::Eof => return Ok(None), - _ => {} - } - } -} - -pub fn parse_get_folder_response(body: &[u8]) -> Result, EwsError> { - let mut xml = NsReader::from_reader(body); - xml.config_mut().trim_text(true); - let mut buf = Vec::new(); - let mut out: Vec = Vec::new(); - loop { - buf.clear(); - let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; - let ns_kind = classify(&ns); - match ev { - Event::Start(_) | Event::Empty(_) => { - let local = match &ev { - Event::Start(e) | Event::Empty(e) => e.local_name().as_ref().to_owned(), - _ => continue, - }; - if ns_kind == Ns::Types - && let Some(elem) = FolderElement::from_local(&local) - { - let entry = parse_folder_element(&mut xml, elem)?; - out.push(entry); - } - } - Event::Eof => break, - _ => {} - } - } - Ok(out) -} - -fn parse_folder_element( - xml: &mut NsReader, - element: FolderElement, -) -> Result { - xml.config_mut().trim_text(false); - let mut entry = FolderEntry { - element, - ..FolderEntry::default() - }; - let mut buf = Vec::new(); - let mut depth: u32 = 1; - let mut current: Option<&'static str> = None; - let mut in_hidden_ext = false; - let mut cur = String::new(); - loop { - buf.clear(); - let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; - let ns_kind = classify(&ns); - match ev { - Event::Start(ref e) | Event::Empty(ref e) => { - let is_empty = matches!(ev, Event::Empty(_)); - if !is_empty { - depth += 1; - } - let local = e.local_name().as_ref().to_owned(); - if ns_kind == Ns::Types { - if local.eq_ignore_ascii_case("FolderId") { - capture_id_attrs( - e, - &mut entry.folder_id.id, - &mut entry.folder_id.change_key, - ); - } else if local.eq_ignore_ascii_case("ParentFolderId") { - let mut pid = String::new(); - let mut ck = String::new(); - capture_id_attrs(e, &mut pid, &mut ck); - if !pid.is_empty() { - entry.parent_id = Some(pid); - } - } else if local.eq_ignore_ascii_case("DisplayName") { - current = Some("displayName"); - cur.clear(); - } else if local.eq_ignore_ascii_case("FolderClass") { - current = Some("folderClass"); - cur.clear(); - } else if local.eq_ignore_ascii_case("TotalCount") { - current = Some("totalCount"); - cur.clear(); - } else if local.eq_ignore_ascii_case("ChildFolderCount") { - current = Some("childCount"); - cur.clear(); - } else if local.eq_ignore_ascii_case("ExtendedFieldURI") { - in_hidden_ext = attr_value(e, "PropertyTag").is_some_and(|tag| { - let tag = tag.trim(); - tag.eq_ignore_ascii_case("0x10f4") || tag == "4340" - }); - } else if in_hidden_ext && local.eq_ignore_ascii_case("Value") { - current = Some("hidden"); - cur.clear(); - } else { - current = None; - } - } - if is_empty { - current = None; - } - } - Event::End(_) => { - if let Some(field) = current.take() { - let text = std::mem::take(&mut cur); - match field { - "displayName" => entry.display_name = text.trim().to_owned(), - "folderClass" => entry.folder_class = text.trim().to_owned(), - "totalCount" => entry.total_count = text.trim().parse().ok(), - "childCount" => entry.child_count = text.trim().parse().ok(), - "hidden" => entry.is_hidden = matches!(text.trim(), "true" | "1"), - _ => {} - } - } - if depth == 0 { - break; - } - depth -= 1; - if depth == 0 { - break; - } - } - Event::Text(t) => { - cur.push_str(&t); - } - Event::CData(c) => { - cur.push_str(&c); - } - Event::GeneralRef(ref g) => { - if let Some(c) = entity_to_char(g) { - cur.push(c); - } - } - Event::Eof => break, - _ => {} - } - } - Ok(entry) -} - -#[derive(Debug, Clone, Default)] -pub struct FindItemResponse { - pub items: Vec, - pub total_in_view: Option, - pub more: bool, -} - -#[derive(Debug, Clone)] -pub struct ItemEntry { - pub element: String, - pub id: ItemId, - pub item_class: Option, -} - -pub fn parse_find_item_response(body: &[u8]) -> Result { - let mut xml = NsReader::from_reader(body); - xml.config_mut().trim_text(true); - let mut buf = Vec::new(); - let mut out = FindItemResponse::default(); - let mut in_root = false; - let mut depth = ItemDepth::default(); - let mut items_level: Option = None; - let mut reading_class = false; - loop { - buf.clear(); - let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; - let ns_kind = classify(&ns); - match ev { - Event::Start(ref e) | Event::Empty(ref e) => { - let level = depth.open(matches!(ev, Event::Empty(_))); - let local = e.local_name().as_ref().to_owned(); - if is(ns_kind, &local, Ns::Messages, "RootFolder") { - in_root = true; - if let Some(v) = attr_value(e, "TotalItemsInView") { - out.total_in_view = v.trim().parse().ok(); - } - if let Some(v) = attr_value(e, "IncludesLastItemInRange") { - out.more = !matches!(v.trim(), "true" | "1"); - } - } else if in_root && ns_kind == Ns::Types { - if items_level.is_none() && local.eq_ignore_ascii_case("Items") { - items_level = Some(level); - } else if items_level.is_some_and(|items| level == items + 1) { - depth.enter_item(level); - out.items.push(ItemEntry { - element: local.clone(), - id: ItemId::default(), - item_class: None, - }); - } else if depth.is_item_child(level) { - if local.eq_ignore_ascii_case("ItemId") - && let Some(last) = out.items.last_mut() - { - capture_id_attrs(e, &mut last.id.id, &mut last.id.change_key); - } else if local.eq_ignore_ascii_case("ItemClass") { - reading_class = !matches!(ev, Event::Empty(_)); - } - } - } - } - Event::Text(ref t) if reading_class => { - if let Some(last) = out.items.last_mut() { - let text: &str = t; - last.item_class = Some(text.trim().to_owned()); - } - } - Event::End(e) => { - depth.close(); - reading_class = false; - let local = e.local_name().as_ref().to_owned(); - if local.eq_ignore_ascii_case("RootFolder") { - in_root = false; - items_level = None; - } - } - Event::Eof => break, - _ => {} - } - } - Ok(out) -} - -pub(crate) fn entity_to_char(g: &BytesRef) -> Option { - if let Ok(Some(c)) = g.resolve_char_ref() { - return Some(c); - } - match g.as_ref() { - "amp" => Some('&'), - "lt" => Some('<'), - "gt" => Some('>'), - "quot" => Some('"'), - "apos" => Some('\''), - _ => None, - } -} - -fn is_item_element(local: &str) -> bool { - hashify::set_ignore_case!( - local.as_bytes(), - "Item", - "Message", - "SharingMessage", - "CalendarItem", - "Contact", - "DistributionList", - "MeetingMessage", - "MeetingRequest", - "MeetingResponse", - "MeetingCancellation", - "Task", - "PostItem", - ) -} - -#[derive(Default)] -struct ItemDepth { - depth: usize, - item_level: Option, -} - -impl ItemDepth { - fn open(&mut self, is_empty: bool) -> usize { - let level = self.depth + 1; - if !is_empty { - self.depth = level; - } - level - } - - fn close(&mut self) { - self.depth = self.depth.saturating_sub(1); - } - - fn enter_item(&mut self, level: usize) { - self.item_level = Some(level); - } - - fn is_item_child(&self, level: usize) -> bool { - self.item_level.is_some_and(|item| level == item + 1) - } -} - -#[derive(Debug, Clone)] -pub struct ResponseMessage { - pub success: bool, - pub response_code: ResponseCode, - pub message_text: String, - pub inner_xml: String, -} - -pub fn parse_response_messages( - body: &[u8], - response_message_local: &str, -) -> Result, EwsError> { - let mut xml = NsReader::from_reader(body); - xml.config_mut().trim_text(false); - let mut buf = Vec::new(); - let mut out: Vec = Vec::new(); - loop { - buf.clear(); - let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; - let ns_kind = classify(&ns); - match ev { - Event::Start(e) => { - let local = e.local_name().as_ref().to_owned(); - if is(ns_kind, &local, Ns::Messages, response_message_local) { - let response_class = - attr_value(&e, "ResponseClass").unwrap_or_else(|| "Success".to_owned()); - let msg = parse_one_response_message(&mut xml, response_class)?; - out.push(msg); - } - } - Event::Eof => break, - _ => {} - } - } - Ok(out) -} - -fn parse_one_response_message( - xml: &mut NsReader, - response_class: String, -) -> Result { - let mut buf = Vec::new(); - let mut response_code = ResponseCode::NoError; - let mut message_text = String::new(); - let mut inner = String::new(); - let mut depth: u32 = 1; - let mut current: Option<&'static str> = None; - let mut capture_depth: u32 = 0; - loop { - buf.clear(); - let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; - let ns_kind = classify(&ns); - let in_capture = capture_depth > 0; - match ev { - Event::Start(e) => { - depth += 1; - let local = e.local_name().as_ref().to_owned(); - if !in_capture && is(ns_kind, &local, Ns::Messages, "ResponseCode") { - current = Some("responseCode"); - } else if !in_capture && is(ns_kind, &local, Ns::Messages, "MessageText") { - current = Some("messageText"); - } else if !in_capture - && (is(ns_kind, &local, Ns::Messages, "Items") - || is(ns_kind, &local, Ns::Messages, "Attachments") - || is(ns_kind, &local, Ns::Messages, "Folders")) - { - capture_depth = 1; - current = None; - } else if in_capture { - write_start_xml(&mut inner, &e); - capture_depth += 1; - } else { - current = None; - } - } - Event::Empty(e) => { - let local = e.local_name().as_ref().to_owned(); - if in_capture { - write_empty_xml(&mut inner, &e); - } else if is(ns_kind, &local, Ns::Messages, "ResponseCode") - || is(ns_kind, &local, Ns::Messages, "MessageText") - { - } - } - Event::End(e) => { - if in_capture { - if capture_depth == 1 { - capture_depth = 0; - } else { - write_end_xml_event(&mut inner, &e); - capture_depth -= 1; - } - } - if depth == 0 { - break; - } - depth -= 1; - current = None; - if depth == 0 { - break; - } - } - Event::Text(t) => { - let text = t.into_inner().into_owned(); - if in_capture { - write_text_xml(&mut inner, &text); - } else { - match current { - Some("responseCode") => response_code = ResponseCode::parse(text.trim()), - Some("messageText") => message_text = text, - _ => {} - } - } - } - Event::CData(c) => { - let text = c.into_inner().into_owned(); - if in_capture { - write_text_xml(&mut inner, &text); - } - } - Event::GeneralRef(ref g) => { - if in_capture { - inner.push('&'); - inner.push_str(g.as_ref()); - inner.push(';'); - } - } - Event::Eof => break, - _ => {} - } - } - Ok(ResponseMessage { - success: matches!(response_class.as_str(), "Success" | "Warning"), - response_code, - message_text, - inner_xml: wrap_inner_with_namespaces(&inner), - }) -} - -fn wrap_inner_with_namespaces(inner: &str) -> String { - if inner.is_empty() { - return String::new(); - } - format!( - "{inner}", - ) -} - -fn write_start_xml(out: &mut String, e: &BytesStart<'_>) { - out.push('<'); - out.push_str(e.name().as_ref()); - for a in e.attributes().flatten() { - out.push(' '); - out.push_str(a.key.as_ref()); - out.push_str("=\""); - let val = a.value; - for ch in val.chars() { - match ch { - '&' => out.push_str("&"), - '<' => out.push_str("<"), - '>' => out.push_str(">"), - '"' => out.push_str("""), - _ => out.push(ch), - } - } - out.push('"'); - } - out.push('>'); -} - -fn write_empty_xml(out: &mut String, e: &BytesStart<'_>) { - out.push('<'); - out.push_str(e.name().as_ref()); - for a in e.attributes().flatten() { - out.push(' '); - out.push_str(a.key.as_ref()); - out.push_str("=\""); - let val = a.value; - for ch in val.chars() { - match ch { - '&' => out.push_str("&"), - '<' => out.push_str("<"), - '>' => out.push_str(">"), - '"' => out.push_str("""), - _ => out.push(ch), - } - } - out.push('"'); - } - out.push_str("/>"); -} - -fn write_end_xml_event(out: &mut String, e: &quick_xml::events::BytesEnd<'_>) { - out.push_str("'); -} - -fn write_text_xml(out: &mut String, text: &str) { - for ch in text.chars() { - match ch { - '&' => out.push_str("&"), - '<' => out.push_str("<"), - '>' => out.push_str(">"), - _ => out.push(ch), - } - } -} - -#[derive(Debug, Clone)] -pub struct SyncFolderItemsResponse { - pub sync_state: String, - pub more: bool, - pub changes: Vec, -} - -#[derive(Debug, Clone)] -pub enum SyncChange { - Create { - id: ItemId, - element: String, - item_class: Option, - }, - Update { - id: ItemId, - element: String, - item_class: Option, - }, - Delete { - id: ItemId, - }, - ReadFlagChange { - id: ItemId, - is_read: bool, - }, -} - -pub fn parse_sync_folder_items_response(body: &[u8]) -> Result { - let mut xml = NsReader::from_reader(body); - xml.config_mut().trim_text(true); - let mut buf = Vec::new(); - let mut sync_state = String::new(); - let mut more = false; - let mut changes: Vec = Vec::new(); - let mut current_text: Option<&'static str> = None; - let mut in_changes = false; - let mut current_change: Option = None; - let mut reading_is_read = false; - let mut pending_is_read: Option = None; - let mut reading_item_class = false; - loop { - buf.clear(); - let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; - let ns_kind = classify(&ns); - match ev { - Event::Start(ref e) | Event::Empty(ref e) => { - let local = e.local_name().as_ref().to_owned(); - let is_empty = matches!(ev, Event::Empty(_)); - if is(ns_kind, &local, Ns::Messages, "SyncState") { - current_text = Some("syncState"); - } else if is(ns_kind, &local, Ns::Messages, "IncludesLastItemInRange") { - current_text = Some("includesLast"); - } else if is(ns_kind, &local, Ns::Messages, "Changes") { - in_changes = true; - } else if in_changes && ns_kind == Ns::Types { - if local.eq_ignore_ascii_case("Create") { - current_change = Some(SyncChange::Create { - id: ItemId::default(), - element: String::new(), - item_class: None, - }); - } else if local.eq_ignore_ascii_case("Update") { - current_change = Some(SyncChange::Update { - id: ItemId::default(), - element: String::new(), - item_class: None, - }); - } else if local.eq_ignore_ascii_case("Delete") { - current_change = Some(SyncChange::Delete { - id: ItemId::default(), - }); - } else if local.eq_ignore_ascii_case("ReadFlagChange") { - current_change = Some(SyncChange::ReadFlagChange { - id: ItemId::default(), - is_read: false, - }); - pending_is_read = None; - } else if local.eq_ignore_ascii_case("ItemId") { - if let Some(change) = current_change.as_mut() { - let (id, ck) = item_id_mut(change); - capture_id_attrs(e, id, ck); - } - } else if local.eq_ignore_ascii_case("IsRead") { - reading_is_read = true; - } else if local.eq_ignore_ascii_case("ItemClass") { - reading_item_class = !is_empty; - } else if is_item_element(&local) { - match current_change.as_mut() { - Some(SyncChange::Create { element, .. }) - | Some(SyncChange::Update { element, .. }) => { - *element = local.clone(); - } - _ => {} - } - } - } - if is_empty { - current_text = None; - } - } - Event::End(e) => { - let local = e.local_name().as_ref().to_owned(); - let lower = local.to_ascii_lowercase(); - match lower.as_str() { - "create" | "update" | "delete" | "readflagchange" => { - if let Some(mut change) = current_change.take() { - if let SyncChange::ReadFlagChange { is_read, .. } = &mut change - && let Some(v) = pending_is_read.take() - { - *is_read = v; - } - changes.push(change); - } - } - "isread" => reading_is_read = false, - "itemclass" => reading_item_class = false, - "changes" => in_changes = false, - _ => {} - } - current_text = None; - } - Event::Text(t) => { - let text = t.into_inner().into_owned(); - if reading_is_read { - pending_is_read = Some(matches!(text.trim(), "true" | "1")); - } - if reading_item_class - && let Some( - SyncChange::Create { item_class, .. } - | SyncChange::Update { item_class, .. }, - ) = current_change.as_mut() - { - *item_class = Some(text.trim().to_owned()); - } - match current_text { - Some("syncState") => sync_state = text, - Some("includesLast") => more = !matches!(text.trim(), "true" | "1"), - _ => {} - } - } - Event::CData(c) => { - let text = c.into_inner().into_owned(); - if current_text == Some("syncState") { - sync_state = text; - } - } - Event::Eof => break, - _ => {} - } - } - Ok(SyncFolderItemsResponse { - sync_state, - more, - changes, - }) -} - -fn item_id_mut(change: &mut SyncChange) -> (&mut String, &mut String) { - match change { - SyncChange::Create { id, .. } - | SyncChange::Update { id, .. } - | SyncChange::Delete { id } - | SyncChange::ReadFlagChange { id, .. } => (&mut id.id, &mut id.change_key), - } -} - -#[derive(Debug, Clone, Default)] -pub struct MessageItem { - pub element: String, - pub id: ItemId, - pub parent_folder_id: Option, - pub mime_content: Option, - pub mime_charset: Option, - pub subject: Option, - pub date_time_received: Option, - pub is_read: Option, - pub is_draft: Option, - pub categories: Vec, - pub flag_status: Option, -} - -pub fn parse_message_item(inner_xml: &str) -> Result { - let mut xml = NsReader::from_reader(inner_xml.as_bytes()); - xml.config_mut().trim_text(false); - let mut buf = Vec::new(); - let mut item = MessageItem::default(); - let mut current: Option<&'static str> = None; - let mut mime_charset: Option = None; - let mut category_collecting = false; - let mut in_flag = false; - let mut in_flagstatus_ext = false; - let mut depth = ItemDepth::default(); - let mut cur = String::new(); - loop { - buf.clear(); - let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; - let ns_kind = classify(&ns); - match ev { - Event::Start(ref e) | Event::Empty(ref e) => { - let local = e.local_name().as_ref().to_owned(); - let is_empty = matches!(ev, Event::Empty(_)); - let level = depth.open(is_empty); - if item.element.is_empty() && is_item_element(&local) { - item.element = local.clone(); - depth.enter_item(level); - } else if ns_kind == Ns::Types { - cur.clear(); - if local.eq_ignore_ascii_case("ItemId") { - if depth.is_item_child(level) { - capture_id_attrs(e, &mut item.id.id, &mut item.id.change_key); - } - } else if local.eq_ignore_ascii_case("ParentFolderId") { - let mut pid = String::new(); - let mut ck = String::new(); - capture_id_attrs(e, &mut pid, &mut ck); - if !pid.is_empty() { - item.parent_folder_id = Some(pid); - } - } else if local.eq_ignore_ascii_case("MimeContent") { - mime_charset = attr_value(e, "CharacterSet"); - current = Some("mimeContent"); - } else if local.eq_ignore_ascii_case("Subject") { - current = Some("subject"); - } else if local.eq_ignore_ascii_case("DateTimeReceived") { - current = Some("received"); - } else if local.eq_ignore_ascii_case("IsRead") { - current = Some("isRead"); - } else if local.eq_ignore_ascii_case("IsDraft") { - current = Some("isDraft"); - } else if local.eq_ignore_ascii_case("Categories") { - category_collecting = true; - } else if category_collecting && local.eq_ignore_ascii_case("String") { - current = Some("category"); - } else if local.eq_ignore_ascii_case("Flag") { - in_flag = true; - } else if in_flag && local.eq_ignore_ascii_case("FlagStatus") { - current = Some("flagStatus"); - } else if local.eq_ignore_ascii_case("ExtendedFieldURI") { - if let Some(tag) = attr_value(e, "PropertyTag") { - let t = tag.trim().to_ascii_lowercase(); - in_flagstatus_ext = t == "0x1090" || t == "4240"; - } - } else if in_flagstatus_ext && local.eq_ignore_ascii_case("Value") { - current = Some("flagStatusExt"); - } else { - current = None; - } - } - if is_empty { - current = None; - } - } - Event::End(e) => { - depth.close(); - let local = e.local_name().as_ref().to_owned(); - let lower = local.to_ascii_lowercase(); - if let Some(field) = current.take() { - let text = std::mem::take(&mut cur); - match field { - "mimeContent" => { - item.mime_content = Some(text); - item.mime_charset = mime_charset.clone(); - } - "subject" => item.subject = Some(text), - "received" => item.date_time_received = Some(text), - "isRead" => item.is_read = Some(matches!(text.trim(), "true" | "1")), - "isDraft" => item.is_draft = Some(matches!(text.trim(), "true" | "1")), - "category" => item.categories.push(text), - "flagStatus" => item.flag_status = Some(text), - "flagStatusExt" if text.trim() == "2" => { - item.flag_status = Some("Flagged".to_owned()); - } - _ => {} - } - } - cur.clear(); - if lower == "categories" { - category_collecting = false; - } else if lower == "flag" { - in_flag = false; - } else if lower == "extendedproperty" { - in_flagstatus_ext = false; - } - } - Event::Text(ref t) => { - cur.push_str(t); - } - Event::CData(ref c) => { - cur.push_str(c); - } - Event::GeneralRef(ref g) => { - if let Some(c) = entity_to_char(g) { - cur.push(c); - } - } - Event::Eof => break, - _ => {} - } - } - Ok(item) -} - -#[derive(Debug, Clone, Default)] -pub struct CalendarItemRaw { - pub element: String, - pub id: ItemId, - pub parent_folder_id: Option, - pub uid: Option, - pub subject: Option, - pub start: Option, - pub end: Option, - pub original_start: Option, - pub is_all_day_event: Option, - pub legacy_free_busy_status: Option, - pub my_response_type: Option, - pub sensitivity: Option, - pub location: Option, - pub calendar_item_type: Option, - pub recurrence_id: Option, - pub start_tz: Option, - pub end_tz: Option, - pub recurrence: Option, - pub modified_occurrences: Vec, - pub deleted_occurrences: Vec, - pub organizer_smtp: Option, - pub organizer_routing_type: Option, - pub organizer_name: Option, - pub required_attendees: Vec, - pub optional_attendees: Vec, - pub resources: Vec, - pub reminder_is_set: Option, - pub reminder_minutes_before_start: Option, - pub is_online_meeting: Option, - pub join_online_meeting_url: Option, - pub net_show_url: Option, - pub meeting_workspace_url: Option, - pub categories: Vec, - pub created: Option, - pub last_modified: Option, - pub body_text: Option, - pub body_html: Option, - pub attachments: Vec, -} - -#[derive(Debug, Clone, Default)] -pub struct RawOccurrence { - pub item_id: ItemId, - pub start: Option, - pub end: Option, - pub original_start: Option, -} - -#[derive(Debug, Clone, Default)] -pub struct RawAttendee { - pub email: Option, - pub routing_type: Option, - pub name: Option, - pub response_type: Option, -} - -#[derive(Debug, Clone, Default)] -pub struct RawAttachmentRef { - pub attachment_id: String, - pub name: Option, - pub content_type: Option, - pub is_contact_photo: bool, - pub is_item_attachment: bool, -} - -#[derive(Debug, Clone, Default)] -pub struct RawRecurrence { - pub pattern: Option, - pub range: Option, -} - -#[derive(Debug, Clone)] -pub enum RecurrencePattern { - Daily { - interval: u32, - }, - Weekly { - interval: u32, - days_of_week: Vec, - }, - AbsoluteMonthly { - interval: u32, - day_of_month: u32, - }, - RelativeMonthly { - interval: u32, - day_of_week_index: String, - days_of_week: Vec, - }, - AbsoluteYearly { - month: String, - day_of_month: u32, - }, - RelativeYearly { - month: String, - day_of_week_index: String, - days_of_week: Vec, - }, -} - -#[derive(Debug, Clone)] -pub enum RecurrenceRange { - NoEnd { - start_date: String, - }, - EndDate { - start_date: String, - end_date: String, - }, - Numbered { - start_date: String, - number_of_occurrences: u32, - }, -} - -pub fn parse_calendar_item(inner_xml: &str) -> Result { - let mut xml = NsReader::from_reader(inner_xml.as_bytes()); - xml.config_mut().trim_text(false); - let mut buf = Vec::new(); - let mut item = CalendarItemRaw::default(); - let mut text_target: Option<&'static str> = None; - let mut category_collecting = false; - let mut attendee_kind: Option<&'static str> = None; - let mut current_attendee: Option = None; - let mut in_mailbox = false; - let mut in_organizer = false; - let mut occurrence_stack: Vec = Vec::new(); - let mut deleted_stack: Vec = Vec::new(); - let mut in_modified = false; - let mut in_deleted = false; - let mut recurrence_path: Vec = Vec::new(); - let mut recurrence_text: Option<&'static str> = None; - let mut recurrence = RawRecurrence::default(); - let mut pending = PendingRecurrence::default(); - let mut depth = ItemDepth::default(); - let mut cur = String::new(); - - loop { - buf.clear(); - let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; - let ns_kind = classify(&ns); - match ev { - Event::Start(ref e) | Event::Empty(ref e) => { - let local = e.local_name().as_ref().to_owned(); - let is_empty = matches!(ev, Event::Empty(_)); - let level = depth.open(is_empty); - if item.element.is_empty() && is_item_element(&local) { - item.element = local.clone(); - depth.enter_item(level); - continue; - } - if ns_kind != Ns::Types { - continue; - } - if !recurrence_path.is_empty() { - recurrence_path.push(local.clone()); - if local.eq_ignore_ascii_case("Interval") { - recurrence_text = Some("interval"); - } else if local.eq_ignore_ascii_case("DaysOfWeek") { - recurrence_text = Some("daysOfWeek"); - } else if local.eq_ignore_ascii_case("DayOfMonth") { - recurrence_text = Some("dayOfMonth"); - } else if local.eq_ignore_ascii_case("DayOfWeekIndex") { - recurrence_text = Some("dayOfWeekIndex"); - } else if local.eq_ignore_ascii_case("Month") { - recurrence_text = Some("month"); - } else if local.eq_ignore_ascii_case("StartDate") { - recurrence_text = Some("startDate"); - } else if local.eq_ignore_ascii_case("EndDate") { - recurrence_text = Some("endDate"); - } else if local.eq_ignore_ascii_case("NumberOfOccurrences") { - recurrence_text = Some("numberOfOccurrences"); - } else if local.eq_ignore_ascii_case("DailyRecurrence") { - pending.pattern_choice = Some("Daily"); - } else if local.eq_ignore_ascii_case("WeeklyRecurrence") { - pending.pattern_choice = Some("Weekly"); - } else if local.eq_ignore_ascii_case("AbsoluteMonthlyRecurrence") { - pending.pattern_choice = Some("AbsoluteMonthly"); - } else if local.eq_ignore_ascii_case("RelativeMonthlyRecurrence") { - pending.pattern_choice = Some("RelativeMonthly"); - } else if local.eq_ignore_ascii_case("AbsoluteYearlyRecurrence") { - pending.pattern_choice = Some("AbsoluteYearly"); - } else if local.eq_ignore_ascii_case("RelativeYearlyRecurrence") { - pending.pattern_choice = Some("RelativeYearly"); - } else if local.eq_ignore_ascii_case("NoEndRecurrence") { - pending.range_choice = Some("NoEnd"); - } else if local.eq_ignore_ascii_case("EndDateRecurrence") { - pending.range_choice = Some("EndDate"); - } else if local.eq_ignore_ascii_case("NumberedRecurrence") { - pending.range_choice = Some("Numbered"); - } else { - recurrence_text = None; - } - if is_empty { - recurrence_path.pop(); - recurrence_text = None; - } - continue; - } - cur.clear(); - if local.eq_ignore_ascii_case("ItemId") { - if in_modified { - if let Some(occ) = occurrence_stack.last_mut() { - capture_id_attrs(e, &mut occ.item_id.id, &mut occ.item_id.change_key); - } - } else if in_deleted { - if let Some(occ) = deleted_stack.last_mut() { - capture_id_attrs(e, &mut occ.item_id.id, &mut occ.item_id.change_key); - } - } else if depth.is_item_child(level) { - capture_id_attrs(e, &mut item.id.id, &mut item.id.change_key); - } - } else if local.eq_ignore_ascii_case("ParentFolderId") { - let mut pid = String::new(); - let mut ck = String::new(); - capture_id_attrs(e, &mut pid, &mut ck); - if !pid.is_empty() { - item.parent_folder_id = Some(pid); - } - } else if local.eq_ignore_ascii_case("UID") { - text_target = Some("uid"); - } else if local.eq_ignore_ascii_case("Subject") { - text_target = Some("subject"); - } else if local.eq_ignore_ascii_case("Start") { - if in_modified || in_deleted { - text_target = Some("occStart"); - } else { - text_target = Some("start"); - } - } else if local.eq_ignore_ascii_case("End") { - if in_modified || in_deleted { - text_target = Some("occEnd"); - } else { - text_target = Some("end"); - } - } else if local.eq_ignore_ascii_case("OriginalStart") { - if in_modified || in_deleted { - text_target = Some("occOrig"); - } else { - text_target = Some("originalStart"); - } - } else if local.eq_ignore_ascii_case("IsAllDayEvent") { - text_target = Some("isAllDay"); - } else if local.eq_ignore_ascii_case("LegacyFreeBusyStatus") { - text_target = Some("freeBusy"); - } else if local.eq_ignore_ascii_case("MyResponseType") { - text_target = Some("myResponse"); - } else if local.eq_ignore_ascii_case("Sensitivity") { - text_target = Some("sensitivity"); - } else if local.eq_ignore_ascii_case("Location") { - text_target = Some("location"); - } else if local.eq_ignore_ascii_case("CalendarItemType") { - text_target = Some("calendarItemType"); - } else if local.eq_ignore_ascii_case("RecurrenceId") { - text_target = Some("recurrenceId"); - } else if local.eq_ignore_ascii_case("StartTimeZone") { - if let Some(v) = attr_value(e, "Id") { - item.start_tz = Some(v); - } - } else if local.eq_ignore_ascii_case("EndTimeZone") { - if let Some(v) = attr_value(e, "Id") { - item.end_tz = Some(v); - } - } else if local.eq_ignore_ascii_case("Recurrence") { - recurrence_path.push(local.clone()); - } else if local.eq_ignore_ascii_case("ModifiedOccurrences") { - in_modified = true; - } else if local.eq_ignore_ascii_case("DeletedOccurrences") { - in_deleted = true; - } else if local.eq_ignore_ascii_case("Occurrence") { - let mut occ = RawOccurrence::default(); - capture_id_attrs(e, &mut occ.item_id.id, &mut occ.item_id.change_key); - occurrence_stack.push(occ); - } else if local.eq_ignore_ascii_case("DeletedOccurrence") { - deleted_stack.push(RawOccurrence::default()); - } else if local.eq_ignore_ascii_case("Organizer") { - in_organizer = true; - } else if local.eq_ignore_ascii_case("RequiredAttendees") { - attendee_kind = Some("required"); - } else if local.eq_ignore_ascii_case("OptionalAttendees") { - attendee_kind = Some("optional"); - } else if local.eq_ignore_ascii_case("Resources") { - attendee_kind = Some("resource"); - } else if local.eq_ignore_ascii_case("ReminderIsSet") { - text_target = Some("reminderIsSet"); - } else if local.eq_ignore_ascii_case("ReminderMinutesBeforeStart") { - text_target = Some("reminderMinutes"); - } else if local.eq_ignore_ascii_case("IsOnlineMeeting") { - text_target = Some("isOnlineMeeting"); - } else if local.eq_ignore_ascii_case("JoinOnlineMeetingUrl") { - text_target = Some("joinUrl"); - } else if local.eq_ignore_ascii_case("NetShowUrl") { - text_target = Some("netShowUrl"); - } else if local.eq_ignore_ascii_case("MeetingWorkspaceUrl") { - text_target = Some("workspaceUrl"); - } else if local.eq_ignore_ascii_case("Attendee") && attendee_kind.is_some() { - current_attendee = Some(RawAttendee::default()); - } else if local.eq_ignore_ascii_case("Mailbox") { - in_mailbox = true; - } else if local.eq_ignore_ascii_case("Name") && in_mailbox { - text_target = Some(if in_organizer { - "organizerName" - } else { - "attendeeName" - }); - } else if local.eq_ignore_ascii_case("EmailAddress") && in_mailbox { - text_target = Some(if in_organizer { - "organizerEmail" - } else { - "attendeeEmail" - }); - } else if local.eq_ignore_ascii_case("RoutingType") && in_mailbox { - text_target = Some(if in_organizer { - "organizerRouting" - } else { - "attendeeRouting" - }); - } else if local.eq_ignore_ascii_case("ResponseType") && current_attendee.is_some() { - text_target = Some("attendeeResponse"); - } else if local.eq_ignore_ascii_case("Categories") { - category_collecting = true; - } else if category_collecting && local.eq_ignore_ascii_case("String") { - text_target = Some("category"); - } else if local.eq_ignore_ascii_case("DateTimeCreated") { - text_target = Some("created"); - } else if local.eq_ignore_ascii_case("LastModifiedTime") { - text_target = Some("lastModified"); - } else if local.eq_ignore_ascii_case("Body") { - let body_type = attr_value(e, "BodyType").unwrap_or_default(); - text_target = Some(if body_type.eq_ignore_ascii_case("HTML") { - "bodyHtml" - } else { - "bodyText" - }); - } else if local.eq_ignore_ascii_case("FileAttachment") - || local.eq_ignore_ascii_case("ItemAttachment") - { - let att = parse_attachment_ref(&mut xml, &local)?; - item.attachments.push(att); - } else { - text_target = None; - } - if is_empty { - text_target = None; - } - } - Event::End(e) => { - depth.close(); - let local = e.local_name().as_ref().to_owned(); - let lower = local.to_ascii_lowercase(); - if !recurrence_path.is_empty() { - recurrence_path.pop(); - if recurrence_path.is_empty() { - pending.finalize(&mut recurrence); - item.recurrence = Some(recurrence.clone()); - pending = PendingRecurrence::default(); - } - recurrence_text = None; - continue; - } - if let Some(tt) = text_target { - let text = std::mem::take(&mut cur); - match tt { - "uid" => item.uid = Some(text), - "subject" => item.subject = Some(text), - "start" => item.start = Some(text), - "end" => item.end = Some(text), - "originalStart" => item.original_start = Some(text), - "isAllDay" => { - item.is_all_day_event = Some(matches!(text.trim(), "true" | "1")); - } - "freeBusy" => item.legacy_free_busy_status = Some(text), - "myResponse" => item.my_response_type = Some(text.trim().to_owned()), - "sensitivity" => item.sensitivity = Some(text.trim().to_owned()), - "location" => item.location = Some(text), - "calendarItemType" => { - item.calendar_item_type = CalendarItemType::parse(text.trim()); - } - "recurrenceId" => item.recurrence_id = Some(text), - "organizerName" => item.organizer_name = Some(text), - "organizerEmail" => item.organizer_smtp = Some(text), - "organizerRouting" => item.organizer_routing_type = Some(text), - "attendeeName" => { - if let Some(att) = current_attendee.as_mut() { - att.name = Some(text); - } - } - "attendeeEmail" => { - if let Some(att) = current_attendee.as_mut() { - att.email = Some(text); - } - } - "attendeeRouting" => { - if let Some(att) = current_attendee.as_mut() { - att.routing_type = Some(text); - } - } - "attendeeResponse" => { - if let Some(att) = current_attendee.as_mut() { - att.response_type = Some(text); - } - } - "reminderIsSet" => { - item.reminder_is_set = Some(matches!(text.trim(), "true" | "1")); - } - "reminderMinutes" => { - item.reminder_minutes_before_start = text.trim().parse().ok(); - } - "isOnlineMeeting" => { - item.is_online_meeting = Some(matches!(text.trim(), "true" | "1")); - } - "joinUrl" => item.join_online_meeting_url = Some(text), - "netShowUrl" => item.net_show_url = Some(text), - "workspaceUrl" => item.meeting_workspace_url = Some(text), - "category" => item.categories.push(text), - "created" => item.created = Some(text), - "lastModified" => item.last_modified = Some(text), - "bodyText" => item.body_text = Some(text), - "bodyHtml" => item.body_html = Some(text), - "occStart" => { - if let Some(occ) = occurrence_stack.last_mut() { - occ.start = Some(text.clone()); - } - if let Some(occ) = deleted_stack.last_mut() { - occ.start = Some(text); - } - } - "occEnd" => { - if let Some(occ) = occurrence_stack.last_mut() { - occ.end = Some(text); - } - } - "occOrig" => { - if let Some(occ) = occurrence_stack.last_mut() { - occ.original_start = Some(text); - } - } - _ => {} - } - } - cur.clear(); - text_target = None; - match lower.as_str() { - "modifiedoccurrences" => in_modified = false, - "deletedoccurrences" => in_deleted = false, - "occurrence" => { - if let Some(occ) = occurrence_stack.pop() { - item.modified_occurrences.push(occ); - } - } - "deletedoccurrence" => { - if let Some(occ) = deleted_stack.pop() { - item.deleted_occurrences.push(occ); - } - } - "attendee" => { - if let Some(att) = current_attendee.take() { - match attendee_kind { - Some("required") => item.required_attendees.push(att), - Some("optional") => item.optional_attendees.push(att), - Some("resource") => item.resources.push(att), - _ => {} - } - } - } - "requiredattendees" | "optionalattendees" | "resources" => attendee_kind = None, - "organizer" => in_organizer = false, - "mailbox" => in_mailbox = false, - "categories" => category_collecting = false, - _ => {} - } - } - Event::Text(ref t) => { - let text = t.as_ref().to_owned(); - if !recurrence_path.is_empty() { - match recurrence_text { - Some("interval") => pending.interval = text.trim().parse().unwrap_or(1), - Some("daysOfWeek") => { - for word in text.split_whitespace() { - pending.days_of_week.push(word.to_owned()); - } - } - Some("dayOfMonth") => { - pending.day_of_month = text.trim().parse().unwrap_or(0); - } - Some("dayOfWeekIndex") => { - pending.day_of_week_index = text.trim().to_owned() - } - Some("month") => pending.month = text.trim().to_owned(), - Some("startDate") => pending.start_date = text.trim().to_owned(), - Some("endDate") => pending.end_date = text.trim().to_owned(), - Some("numberOfOccurrences") => { - pending.number_of_occurrences = text.trim().parse().unwrap_or(0); - } - _ => {} - } - } else { - cur.push_str(&text); - } - } - Event::CData(ref c) => { - if recurrence_path.is_empty() { - cur.push_str(c); - } - } - Event::GeneralRef(ref g) => { - if recurrence_path.is_empty() - && let Some(c) = entity_to_char(g) - { - cur.push(c); - } - } - Event::Eof => break, - _ => {} - } - } - Ok(item) -} - -#[derive(Default)] -struct PendingRecurrence { - pattern_choice: Option<&'static str>, - range_choice: Option<&'static str>, - interval: u32, - days_of_week: Vec, - day_of_month: u32, - day_of_week_index: String, - month: String, - start_date: String, - end_date: String, - number_of_occurrences: u32, -} - -impl PendingRecurrence { - fn finalize(self, out: &mut RawRecurrence) { - let interval = if self.interval == 0 { 1 } else { self.interval }; - out.pattern = match self.pattern_choice { - Some("Daily") => Some(RecurrencePattern::Daily { interval }), - Some("Weekly") => Some(RecurrencePattern::Weekly { - interval, - days_of_week: self.days_of_week.clone(), - }), - Some("AbsoluteMonthly") => Some(RecurrencePattern::AbsoluteMonthly { - interval, - day_of_month: self.day_of_month, - }), - Some("RelativeMonthly") => Some(RecurrencePattern::RelativeMonthly { - interval, - day_of_week_index: self.day_of_week_index.clone(), - days_of_week: self.days_of_week.clone(), - }), - Some("AbsoluteYearly") => Some(RecurrencePattern::AbsoluteYearly { - month: self.month.clone(), - day_of_month: self.day_of_month, - }), - Some("RelativeYearly") => Some(RecurrencePattern::RelativeYearly { - month: self.month.clone(), - day_of_week_index: self.day_of_week_index.clone(), - days_of_week: self.days_of_week.clone(), - }), - _ => None, - }; - out.range = match self.range_choice { - Some("NoEnd") => Some(RecurrenceRange::NoEnd { - start_date: self.start_date, - }), - Some("EndDate") => Some(RecurrenceRange::EndDate { - start_date: self.start_date, - end_date: self.end_date, - }), - Some("Numbered") => Some(RecurrenceRange::Numbered { - start_date: self.start_date, - number_of_occurrences: self.number_of_occurrences, - }), - _ => None, - }; - } -} - -#[derive(Debug, Clone, Default)] -pub struct ContactItemRaw { - pub id: ItemId, - pub parent_folder_id: Option, - pub display_name: Option, - pub given_name: Option, - pub middle_name: Option, - pub surname: Option, - pub initials: Option, - pub nickname: Option, - pub company_name: Option, - pub department: Option, - pub job_title: Option, - pub generation: Option, - pub office_location: Option, - pub url: Option, - pub birthday: Option, - pub wedding_anniversary: Option, - pub notes: Option, - pub categories: Vec, - pub emails: Vec<(String, String)>, - pub phones: Vec<(String, String)>, - pub addresses: Vec, - pub ims: Vec<(String, String)>, - pub manager: Option, - pub spouse: Option, - pub assistant: Option, - pub children: Vec, - pub companies: Vec, - pub profession: Option, - pub postal_address_index: Option, - pub created: Option, - pub last_modified: Option, - pub attachments: Vec, - pub is_group: bool, - pub members: Vec, -} - -#[derive(Debug, Clone, Default)] -pub struct RawGroupMember { - pub name: Option, - pub email: Option, -} - -#[derive(Debug, Clone, Default)] -pub struct RawContactAddress { - pub key: String, - pub street: Option, - pub city: Option, - pub state: Option, - pub country: Option, - pub postal_code: Option, -} - -pub fn parse_contact_item(inner_xml: &str) -> Result { - let mut xml = NsReader::from_reader(inner_xml.as_bytes()); - xml.config_mut().trim_text(false); - let mut buf = Vec::new(); - let mut item = ContactItemRaw::default(); - let mut text_target: Option<&'static str> = None; - let mut category_collecting = false; - let mut children_collecting = false; - let mut companies_collecting = false; - let mut entry_container: Option<&'static str> = None; - let mut entry_key: Option = None; - let mut current_address: Option = None; - let mut address_text: Option<&'static str> = None; - let mut seen_root_contact = false; - let mut in_members = false; - let mut member_mailbox = false; - let mut current_member: Option = None; - let mut depth = ItemDepth::default(); - let mut cur = String::new(); - loop { - buf.clear(); - let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; - let ns_kind = classify(&ns); - match ev { - Event::Start(ref e) | Event::Empty(ref e) => { - let local = e.local_name().as_ref().to_owned(); - let is_empty = matches!(ev, Event::Empty(_)); - let level = depth.open(is_empty); - if !seen_root_contact - && (local.eq_ignore_ascii_case("Contact") - || local.eq_ignore_ascii_case("DistributionList")) - { - seen_root_contact = true; - depth.enter_item(level); - if local.eq_ignore_ascii_case("DistributionList") { - item.is_group = true; - } - continue; - } - if ns_kind != Ns::Types { - continue; - } - cur.clear(); - if local.eq_ignore_ascii_case("ItemId") { - if depth.is_item_child(level) { - capture_id_attrs(e, &mut item.id.id, &mut item.id.change_key); - } - } else if local.eq_ignore_ascii_case("ParentFolderId") { - let mut pid = String::new(); - let mut ck = String::new(); - capture_id_attrs(e, &mut pid, &mut ck); - if !pid.is_empty() { - item.parent_folder_id = Some(pid); - } - } else if local.eq_ignore_ascii_case("DisplayName") { - text_target = Some("displayName"); - } else if local.eq_ignore_ascii_case("GivenName") { - text_target = Some("givenName"); - } else if local.eq_ignore_ascii_case("MiddleName") { - text_target = Some("middleName"); - } else if local.eq_ignore_ascii_case("Surname") { - text_target = Some("surname"); - } else if local.eq_ignore_ascii_case("Initials") { - text_target = Some("initials"); - } else if local.eq_ignore_ascii_case("Nickname") { - text_target = Some("nickname"); - } else if local.eq_ignore_ascii_case("CompanyName") { - text_target = Some("companyName"); - } else if local.eq_ignore_ascii_case("Department") { - text_target = Some("department"); - } else if local.eq_ignore_ascii_case("JobTitle") { - text_target = Some("jobTitle"); - } else if local.eq_ignore_ascii_case("Generation") { - text_target = Some("generation"); - } else if local.eq_ignore_ascii_case("OfficeLocation") { - text_target = Some("officeLocation"); - } else if local.eq_ignore_ascii_case("BusinessHomePage") { - text_target = Some("url"); - } else if local.eq_ignore_ascii_case("Birthday") { - text_target = Some("birthday"); - } else if local.eq_ignore_ascii_case("WeddingAnniversary") { - text_target = Some("weddingAnniversary"); - } else if local.eq_ignore_ascii_case("Manager") { - text_target = Some("manager"); - } else if local.eq_ignore_ascii_case("SpouseName") { - text_target = Some("spouse"); - } else if local.eq_ignore_ascii_case("AssistantName") { - text_target = Some("assistant"); - } else if local.eq_ignore_ascii_case("Profession") { - text_target = Some("profession"); - } else if local.eq_ignore_ascii_case("PostalAddressIndex") { - text_target = Some("postalAddressIndex"); - } else if local.eq_ignore_ascii_case("Members") { - in_members = true; - } else if in_members && local.eq_ignore_ascii_case("Member") { - current_member = Some(RawGroupMember::default()); - } else if in_members && local.eq_ignore_ascii_case("Mailbox") { - member_mailbox = true; - } else if member_mailbox && local.eq_ignore_ascii_case("Name") { - text_target = Some("memberName"); - } else if member_mailbox && local.eq_ignore_ascii_case("EmailAddress") { - text_target = Some("memberEmail"); - } else if local.eq_ignore_ascii_case("Body") { - text_target = Some("notes"); - } else if local.eq_ignore_ascii_case("DateTimeCreated") { - text_target = Some("created"); - } else if local.eq_ignore_ascii_case("LastModifiedTime") { - text_target = Some("lastModified"); - } else if local.eq_ignore_ascii_case("Categories") { - category_collecting = true; - } else if category_collecting && local.eq_ignore_ascii_case("String") { - text_target = Some("category"); - } else if local.eq_ignore_ascii_case("Children") { - children_collecting = true; - } else if children_collecting && local.eq_ignore_ascii_case("String") { - text_target = Some("child"); - } else if local.eq_ignore_ascii_case("Companies") { - companies_collecting = true; - } else if companies_collecting && local.eq_ignore_ascii_case("String") { - text_target = Some("company"); - } else if local.eq_ignore_ascii_case("EmailAddresses") { - entry_container = Some("email"); - } else if local.eq_ignore_ascii_case("PhoneNumbers") { - entry_container = Some("phone"); - } else if local.eq_ignore_ascii_case("ImAddresses") { - entry_container = Some("im"); - } else if local.eq_ignore_ascii_case("PhysicalAddresses") { - entry_container = Some("address"); - } else if local.eq_ignore_ascii_case("Entry") { - entry_key = attr_value(e, "Key"); - if entry_container == Some("address") { - current_address = Some(RawContactAddress { - key: entry_key.clone().unwrap_or_default(), - ..RawContactAddress::default() - }); - } else { - text_target = Some("entryValue"); - } - } else if current_address.is_some() { - if local.eq_ignore_ascii_case("Street") { - address_text = Some("street"); - } else if local.eq_ignore_ascii_case("City") { - address_text = Some("city"); - } else if local.eq_ignore_ascii_case("State") { - address_text = Some("state"); - } else if local.eq_ignore_ascii_case("CountryOrRegion") { - address_text = Some("country"); - } else if local.eq_ignore_ascii_case("PostalCode") { - address_text = Some("postal"); - } else { - address_text = None; - } - } else if local.eq_ignore_ascii_case("FileAttachment") - || local.eq_ignore_ascii_case("ItemAttachment") - { - let att = parse_attachment_ref(&mut xml, &local)?; - item.attachments.push(att); - } else { - text_target = None; - } - if is_empty { - text_target = None; - address_text = None; - } - } - Event::End(e) => { - depth.close(); - let local = e.local_name().as_ref().to_owned(); - let lower = local.to_ascii_lowercase(); - if let Some(at) = address_text { - if let Some(addr) = current_address.as_mut() { - let text = std::mem::take(&mut cur); - match at { - "street" => addr.street = Some(text), - "city" => addr.city = Some(text), - "state" => addr.state = Some(text), - "country" => addr.country = Some(text), - "postal" => addr.postal_code = Some(text), - _ => {} - } - } - } else if let Some(tt) = text_target { - let text = std::mem::take(&mut cur); - match tt { - "displayName" => item.display_name = Some(text), - "givenName" => item.given_name = Some(text), - "middleName" => item.middle_name = Some(text), - "surname" => item.surname = Some(text), - "initials" => item.initials = Some(text), - "nickname" => item.nickname = Some(text), - "companyName" => item.company_name = Some(text), - "department" => item.department = Some(text), - "jobTitle" => item.job_title = Some(text), - "generation" => item.generation = Some(text), - "officeLocation" => item.office_location = Some(text), - "url" => item.url = Some(text), - "birthday" => item.birthday = Some(text), - "weddingAnniversary" => item.wedding_anniversary = Some(text), - "manager" => item.manager = Some(text), - "spouse" => item.spouse = Some(text), - "assistant" => item.assistant = Some(text), - "profession" => item.profession = Some(text), - "postalAddressIndex" => item.postal_address_index = Some(text), - "notes" => item.notes = Some(text), - "created" => item.created = Some(text), - "lastModified" => item.last_modified = Some(text), - "memberName" => { - if let Some(m) = current_member.as_mut() { - m.name = Some(text); - } - } - "memberEmail" => { - if let Some(m) = current_member.as_mut() { - m.email = Some(text); - } - } - "category" => item.categories.push(text), - "child" => item.children.push(text), - "company" => item.companies.push(text), - "entryValue" => { - let key = entry_key.clone().unwrap_or_default(); - match entry_container { - Some("email") => item.emails.push((key, text)), - Some("phone") => item.phones.push((key, text)), - Some("im") => item.ims.push((key, text)), - _ => {} - } - } - _ => {} - } - } - cur.clear(); - text_target = None; - address_text = None; - match lower.as_str() { - "categories" => category_collecting = false, - "children" => children_collecting = false, - "companies" => companies_collecting = false, - "emailaddresses" | "phonenumbers" | "imaddresses" | "physicaladdresses" => { - entry_container = None - } - "entry" => { - if let Some(addr) = current_address.take() { - item.addresses.push(addr); - } - entry_key = None; - } - "members" => in_members = false, - "mailbox" => member_mailbox = false, - "member" => { - if let Some(m) = current_member.take() - && (m.email.is_some() || m.name.is_some()) - { - item.members.push(m); - } - } - _ => {} - } - } - Event::Text(ref t) => { - cur.push_str(t); - } - Event::CData(ref c) => { - cur.push_str(c); - } - Event::GeneralRef(ref g) => { - if let Some(c) = entity_to_char(g) { - cur.push(c); - } - } - Event::Eof => break, - _ => {} - } - } - Ok(item) -} - -fn parse_attachment_ref( - xml: &mut NsReader, - element_local: &str, -) -> Result { - let mut att = RawAttachmentRef { - is_item_attachment: element_local.eq_ignore_ascii_case("itemattachment"), - ..RawAttachmentRef::default() - }; - let mut buf = Vec::new(); - let mut depth: u32 = 1; - let mut current: Option<&'static str> = None; - let mut cur = String::new(); - loop { - buf.clear(); - let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; - let ns_kind = classify(&ns); - match ev { - Event::Start(ref e) | Event::Empty(ref e) => { - let is_empty = matches!(ev, Event::Empty(_)); - if !is_empty { - depth += 1; - } - let local = e.local_name().as_ref().to_owned(); - if ns_kind == Ns::Types { - cur.clear(); - if local.eq_ignore_ascii_case("AttachmentId") { - if let Some(v) = attr_value(e, "Id") { - att.attachment_id = v; - } - } else if local.eq_ignore_ascii_case("Name") { - current = Some("name"); - } else if local.eq_ignore_ascii_case("ContentType") { - current = Some("contentType"); - } else if local.eq_ignore_ascii_case("IsContactPhoto") { - current = Some("isContactPhoto"); - } else { - current = None; - } - } - if is_empty { - current = None; - } - } - Event::End(_) => { - if let Some(field) = current.take() { - let text = std::mem::take(&mut cur); - match field { - "name" => att.name = Some(text), - "contentType" => att.content_type = Some(text), - "isContactPhoto" => { - att.is_contact_photo = matches!(text.trim(), "true" | "1"); - } - _ => {} - } - } - cur.clear(); - if depth == 0 { - break; - } - depth -= 1; - if depth == 0 { - break; - } - } - Event::Text(ref t) => { - cur.push_str(t); - } - Event::CData(ref c) => { - cur.push_str(c); - } - Event::GeneralRef(ref g) => { - if let Some(c) = entity_to_char(g) { - cur.push(c); - } - } - Event::Eof => break, - _ => {} - } - } - Ok(att) -} - -#[derive(Debug, Clone, Default)] -pub struct GetAttachmentInline { - pub attachment_id: String, - pub name: Option, - pub content_type: Option, - pub is_contact_photo: bool, - pub content_base64: String, -} - -pub fn parse_get_attachment_inline(body: &[u8]) -> Result, EwsError> { - let mut xml = NsReader::from_reader(body); - xml.config_mut().trim_text(true); - let mut buf = Vec::new(); - let mut out: Vec = Vec::new(); - let mut current: Option = None; - let mut text_target: Option<&'static str> = None; - loop { - buf.clear(); - let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; - let ns_kind = classify(&ns); - match ev { - Event::Start(ref e) | Event::Empty(ref e) => { - let local = e.local_name().as_ref().to_owned(); - let is_empty = matches!(ev, Event::Empty(_)); - if ns_kind == Ns::Types - && (local.eq_ignore_ascii_case("FileAttachment") - || local.eq_ignore_ascii_case("ItemAttachment")) - { - current = Some(GetAttachmentInline::default()); - } else if let Some(cur) = current.as_mut() - && ns_kind == Ns::Types - { - if local.eq_ignore_ascii_case("AttachmentId") { - if let Some(v) = attr_value(e, "Id") { - cur.attachment_id = v; - } - } else if local.eq_ignore_ascii_case("Name") { - text_target = Some("name"); - } else if local.eq_ignore_ascii_case("ContentType") { - text_target = Some("contentType"); - } else if local.eq_ignore_ascii_case("IsContactPhoto") { - text_target = Some("isContactPhoto"); - } else if local.eq_ignore_ascii_case("Content") { - text_target = Some("content"); - } else { - text_target = None; - } - } - if is_empty { - text_target = None; - } - } - Event::End(e) => { - let local = e.local_name().as_ref().to_owned(); - if (local.eq_ignore_ascii_case("FileAttachment") - || local.eq_ignore_ascii_case("ItemAttachment")) - && let Some(att) = current.take() - { - out.push(att); - } - text_target = None; - } - Event::Text(_) | Event::CData(_) => { - let text = match ev { - Event::Text(ref t) => t.as_ref().to_owned(), - Event::CData(ref c) => c.as_ref().to_owned(), - _ => unreachable!(), - }; - - if let Some(cur) = current.as_mut() { - match text_target { - Some("name") => { - cur.name.get_or_insert_with(String::new).push_str(&text); - } - Some("contentType") => { - cur.content_type - .get_or_insert_with(String::new) - .push_str(&text); - } - Some("isContactPhoto") => { - cur.is_contact_photo = matches!(text.trim(), "true" | "1"); - } - Some("content") => cur.content_base64.push_str(&text), - _ => {} - } - } - } - Event::GeneralRef(ref g) => { - if let (Some(cur), Some(c)) = (current.as_mut(), entity_to_char(g)) { - match text_target { - Some("name") => cur.name.get_or_insert_with(String::new).push(c), - Some("contentType") => { - cur.content_type.get_or_insert_with(String::new).push(c) - } - Some("content") => cur.content_base64.push(c), - _ => {} - } - } - } - Event::Eof => break, - _ => {} - } - } - Ok(out) -} - -#[cfg(test)] -mod tests { - use super::*; - - const NS: &str = " xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\" \ - xmlns:t=\"http://schemas.microsoft.com/exchange/services/2006/types\" \ - xmlns:m=\"http://schemas.microsoft.com/exchange/services/2006/messages\""; - - #[test] - fn server_busy_fault_extracts_backoff_milliseconds() { - let body = format!( - "\ - soap:Serverbusy\ - ErrorServerBusy\ - 297749\ - " - ); - let env = read_envelope_summary(body.as_bytes()).unwrap(); - match env { - EnvelopeKind::Fault { fault, version } => { - assert!(matches!( - fault.response_code, - ResponseCode::ServerBusy { - back_off_ms: Some(297749) - } - )); - assert_eq!(fault.back_off_ms, Some(297749)); - assert_eq!( - version.to_server_version(), - Some(ServerVersion::Exchange2016) - ); - } - _ => panic!("expected fault"), - } - } - - #[test] - fn body_envelope_returns_version_only() { - let body = format!( - "\ - " - ); - let env = read_envelope_summary(body.as_bytes()).unwrap(); - assert!(matches!(env, EnvelopeKind::Body { .. })); - } - - #[test] - fn find_folder_parses_two_folders() { - let body = format!( - "\ - NoError\ - \ - \ - \ - IPF.NoteInbox\ - 70\ - \ - IPF.AppointmentCalendar\ - \ - " - ); - let r = parse_find_folder_response(body.as_bytes()).unwrap(); - assert_eq!(r.folders.len(), 2); - assert_eq!(r.folders[0].folder_id.id, "F1"); - assert_eq!(r.folders[0].folder_id.change_key, "C1"); - assert_eq!(r.folders[0].display_name, "Inbox"); - assert_eq!(r.folders[0].folder_class, "IPF.Note"); - assert_eq!(r.folders[0].parent_id.as_deref(), Some("ROOT")); - assert_eq!(r.folders[1].element, FolderElement::CalendarFolder); - assert!(!r.more); - assert!(!r.folders[0].is_hidden); - } - - #[test] - fn find_folder_reads_pr_attr_hidden() { - let body = format!( - "\ - NoError\ - \ - \ - \ - IPF.ContactRecipient Cache\ - \ - true\ - \ - IPF.NoteInbox\ - \ - false\ - " - ); - let r = parse_find_folder_response(body.as_bytes()).unwrap(); - assert_eq!(r.folders.len(), 2); - assert!(r.folders[0].is_hidden); - assert_eq!(r.folders[0].display_name, "Recipient Cache"); - assert!(!r.folders[1].is_hidden); - assert_eq!(r.folders[1].display_name, "Inbox"); - } - - #[test] - fn find_item_reads_item_class() { - let body = format!( - "\ - NoError\ - \ - IPM.Contact\ - \ - " - ); - let r = parse_find_item_response(body.as_bytes()).unwrap(); - assert_eq!(r.items[0].item_class.as_deref(), Some("IPM.Contact")); - assert_eq!(r.items[1].item_class, None); - } - - #[test] - fn sync_folder_items_reads_item_class() { - let body = format!( - "\ - NoError\ - AAAA\ - true\ - \ - IPM.Task\ - IPM.Note\ - " - ); - let r = parse_sync_folder_items_response(body.as_bytes()).unwrap(); - let classes: Vec> = - r.changes - .iter() - .map(|c| match c { - SyncChange::Create { item_class, .. } - | SyncChange::Update { item_class, .. } => item_class.as_deref(), - _ => None, - }) - .collect(); - assert_eq!(classes, [Some("IPM.Task"), Some("IPM.Note")]); - } - - #[test] - fn find_item_opens_an_entry_for_every_item_element() { - let body = format!( - "\ - NoError\ - \ - \ - \ - \ - \ - " - ); - let r = parse_find_item_response(body.as_bytes()).unwrap(); - let ids: Vec<(&str, &str)> = r - .items - .iter() - .map(|i| (i.element.as_str(), i.id.id.as_str())) - .collect(); - assert_eq!( - ids, - [ - ("Message", "M1"), - ("Task", "T1"), - ("PostItem", "P1"), - ("FutureItem", "F1") - ] - ); - } - - #[test] - fn find_item_ignores_item_ids_nested_in_mailboxes() { - let body = format!( - "\ - NoError\ - \ - \ - bob@example.com\ - \ - " - ); - let r = parse_find_item_response(body.as_bytes()).unwrap(); - assert_eq!(r.items.len(), 1); - assert_eq!(r.items[0].id.id, "E1"); - } - - #[test] - fn calendar_item_keeps_its_own_id_when_attendees_carry_contact_ids() { - let xml = "\ - \ - alice@example.com\ - Contact\ - \ - bob@example.com\ - Contact\ - "; - let item = parse_calendar_item(xml).unwrap(); - assert_eq!(item.id.id, "EVENT-A"); - assert_eq!(item.id.change_key, "CK"); - assert_eq!(item.required_attendees.len(), 1); - } - - #[test] - fn message_item_keeps_its_own_id_when_recipients_carry_contact_ids() { - let xml = "\ - \ - bob@example.com\ - "; - let item = parse_message_item(xml).unwrap(); - assert_eq!(item.id.id, "MSG-A"); - assert_eq!(item.element, "MeetingRequest"); - } - - #[test] - fn distribution_list_keeps_its_own_id_when_members_carry_contact_ids() { - let xml = "\ - \ - Team\ - Bobbob@example.com\ - Contact\ - "; - let item = parse_contact_item(xml).unwrap(); - assert_eq!(item.id.id, "DL-A"); - assert!(item.is_group); - assert_eq!(item.members.len(), 1); - assert_eq!(item.members[0].email.as_deref(), Some("bob@example.com")); - } - - #[test] - fn find_item_parses_two_messages_with_pagination_flag() { - let body = format!( - "\ - NoError\ - \ - \ - \ - \ - " - ); - let r = parse_find_item_response(body.as_bytes()).unwrap(); - assert_eq!(r.items.len(), 2); - assert_eq!(r.items[0].id.id, "I1"); - assert_eq!(r.items[0].id.change_key, "CK1"); - assert_eq!(r.items[1].id.id, "I2"); - assert!(r.more); - assert_eq!(r.total_in_view, Some(3)); - } - - #[test] - fn parse_response_messages_carries_inner_xml_and_codes() { - let body = format!( - "\ - \ - NoError\ - \ - Hitrue\ - SGVsbG8=\ - \ - \ - ErrorItemNotFound\ - not found\ - \ - " - ); - let r = parse_response_messages(body.as_bytes(), "GetItemResponseMessage").unwrap(); - assert_eq!(r.len(), 2); - assert!(r[0].success); - assert!(r[0].inner_xml.contains("\ - \ - NoError\ - {message_xml}\ - " - ); - let r = parse_response_messages(body.as_bytes(), "GetItemResponseMessage").unwrap(); - parse_message_item(&r[0].inner_xml).unwrap() - } - - let flagged = parse_one( - "\ - Flagged\ - \ - \ - 2\ - \ - true", - ); - assert_eq!(flagged.flag_status.as_deref(), Some("Flagged")); - assert_eq!(flagged.is_read, Some(true)); - - let cleared = parse_one( - "\ - \ - \ - 0\ - ", - ); - assert_eq!(cleared.flag_status, None); - } - - #[test] - fn native_item_flag_still_parses_on_exchange_2013_plus() { - let body = format!( - "\ - \ - NoError\ - \ - Native\ - Flagged\ - false\ - " - ); - let r = parse_response_messages(body.as_bytes(), "GetItemResponseMessage").unwrap(); - let p = parse_message_item(&r[0].inner_xml).unwrap(); - assert_eq!( - p.flag_status.as_deref(), - Some("Flagged"), - "the 2013+ native item:Flag path must be unaffected by the pre-2013 extended-property fallback" - ); - assert_eq!(p.is_read, Some(false)); - } - - #[test] - fn sync_folder_items_emits_create_delete_readflag() { - let body = format!( - "\ - NoError\ - AAAA\ - true\ - \ - \ - \ - true\ - " - ); - let r = parse_sync_folder_items_response(body.as_bytes()).unwrap(); - assert_eq!(r.sync_state, "AAAA"); - assert!(!r.more); - assert_eq!(r.changes.len(), 3); - match &r.changes[0] { - SyncChange::Create { id, element, .. } => { - assert_eq!(id.id, "I1"); - assert_eq!(element, "Message"); - } - _ => panic!("expected Create"), - } - match &r.changes[2] { - SyncChange::ReadFlagChange { id, is_read } => { - assert_eq!(id.id, "I3"); - assert!(*is_read); - } - _ => panic!("expected ReadFlagChange"), - } - } - - #[test] - fn calendar_master_recurrence_and_occurrences_round_trip() { - let body = format!( - "\ - \ - Daily\ - uid-1\ - 2025-06-15T14:00:00Z\ - 2025-06-15T15:00:00Z\ - RecurringMaster\ - \ - 1\ - 2025-06-153\ - \ - \ - 2025-06-16T15:00:00Z2025-06-16T16:30:00Z2025-06-16T14:00:00Z\ - \ - \ - 2025-06-17T14:00:00Z\ - \ - " - ); - let parsed = parse_calendar_item(&body).unwrap(); - assert_eq!(parsed.id.id, "M1"); - assert_eq!(parsed.uid.as_deref(), Some("uid-1")); - assert_eq!( - parsed.calendar_item_type, - Some(CalendarItemType::RecurringMaster) - ); - assert_eq!(parsed.modified_occurrences.len(), 1); - assert_eq!( - parsed.modified_occurrences[0].original_start.as_deref(), - Some("2025-06-16T14:00:00Z") - ); - assert_eq!(parsed.deleted_occurrences.len(), 1); - assert_eq!( - parsed.deleted_occurrences[0].start.as_deref(), - Some("2025-06-17T14:00:00Z") - ); - let rec = parsed.recurrence.unwrap(); - assert!(matches!( - rec.pattern, - Some(RecurrencePattern::Daily { interval: 1 }) - )); - assert!(matches!( - rec.range, - Some(RecurrenceRange::Numbered { - number_of_occurrences: 3, - .. - }) - )); - } - - #[test] - fn end_date_recurrence_wire_shape_with_trailing_z_yields_clean_until() { - let body = format!( - "\ - RecDailyEndu\ - 2026-07-01T15:00:00Z2026-07-01T15:30:00Z\ - RecurringMaster\ - \ - 2\ - 2026-07-01Z2026-08-01Z\ - " - ); - let parsed = parse_calendar_item(&body).unwrap(); - let rule = crate::exchange_ews::recurrence::to_jscalendar_rule( - parsed.recurrence.as_ref().unwrap(), - ) - .unwrap(); - assert_eq!(rule["frequency"], "daily"); - assert_eq!(rule["interval"], 2); - assert_eq!( - rule["until"], "2026-08-01T23:59:59", - "trailing Z on EndDate must be stripped, not embedded mid-string" - ); - } - - #[test] - fn absolute_and_relative_recurrence_wire_shapes_round_trip() { - let cases = [ - ( - "1152026-07-01Z", - "monthly", - ), - ( - "1FridayLast2026-07-01Z", - "monthly", - ), - ( - "1January2026-07-01Z", - "yearly", - ), - ( - "MondayFirstSeptember2026-07-01Z", - "yearly", - ), - ]; - for (rec_xml, freq) in cases { - let body = format!( - "u\ - 2026-07-01T15:00:00Z2026-07-01T15:30:00Z\ - {rec_xml}" - ); - let parsed = parse_calendar_item(&body).unwrap(); - let rule = crate::exchange_ews::recurrence::to_jscalendar_rule( - parsed.recurrence.as_ref().unwrap(), - ) - .unwrap(); - assert_eq!(rule["frequency"], freq, "for {rec_xml}"); - } - let body = format!( - "u\ - 2026-07-01T15:00:00Z2026-07-01T15:30:00Z\ - 1\ - FridayLast\ - 2026-07-01Z" - ); - let parsed = parse_calendar_item(&body).unwrap(); - let rule = crate::exchange_ews::recurrence::to_jscalendar_rule( - parsed.recurrence.as_ref().unwrap(), - ) - .unwrap(); - assert_eq!(rule["byDay"][0]["day"], "fr"); - assert_eq!(rule["byDay"][0]["nthOfPeriod"], -1); - } - - #[test] - fn attendee_and_organizer_routing_type_is_parsed() { - let body = format!( - "u\ - Org\ - /o=ExLabs/cn=orgEX\ - Att\ - /o=ExLabs/cn=attEX\ - Accept\ - " - ); - let parsed = parse_calendar_item(&body).unwrap(); - assert_eq!(parsed.organizer_routing_type.as_deref(), Some("EX")); - assert_eq!(parsed.required_attendees.len(), 1); - assert_eq!( - parsed.required_attendees[0].routing_type.as_deref(), - Some("EX") - ); - assert_eq!( - parsed.required_attendees[0].email.as_deref(), - Some("/o=ExLabs/cn=att") - ); - } - - #[test] - fn calendar_parses_own_response_sensitivity_and_attachments() { - let body = format!( - "\ - \ - Private\ - \ - agenda.pdfapplication/pdf\ - 2020-01-01T00:00:00\ - \ - 2025-06-01T00:00:00Z\ - Accept\ - " - ); - let parsed = parse_calendar_item(&body).unwrap(); - assert_eq!(parsed.sensitivity.as_deref(), Some("Private")); - assert_eq!(parsed.my_response_type.as_deref(), Some("Accept")); - assert_eq!(parsed.attachments.len(), 1); - assert_eq!(parsed.attachments[0].attachment_id, "ATT1"); - assert_eq!( - parsed.last_modified.as_deref(), - Some("2025-06-01T00:00:00Z") - ); - } - - #[test] - fn calendar_parses_reminder_resources_and_online_meeting() { - let body = format!( - "\ - \ - Standup\ - uid-r\ - true\ - 10\ - 2025-06-15T14:00:00Z\ - 2025-06-15T15:00:00Z\ - true\ - https://meet/abc\ - \ - Bobbob@xAccept\ - \ - \ - Room 7room7@x\ - \ - " - ); - let parsed = parse_calendar_item(&body).unwrap(); - assert_eq!(parsed.reminder_is_set, Some(true)); - assert_eq!(parsed.reminder_minutes_before_start, Some(10)); - assert_eq!(parsed.is_online_meeting, Some(true)); - assert_eq!(parsed.net_show_url.as_deref(), Some("https://meet/abc")); - assert_eq!(parsed.required_attendees.len(), 1); - assert_eq!(parsed.required_attendees[0].email.as_deref(), Some("bob@x")); - assert_eq!(parsed.resources.len(), 1); - assert_eq!(parsed.resources[0].email.as_deref(), Some("room7@x")); - } - - #[test] - fn distribution_list_parses_members() { - let body = format!( - "\ - \ - The Team\ - \ - Bobbob@xNormal\ - Carolcarol@x\ - \ - " - ); - let parsed = parse_contact_item(&body).unwrap(); - assert!(parsed.is_group); - assert_eq!(parsed.id.id, "DL1"); - assert_eq!(parsed.display_name.as_deref(), Some("The Team")); - assert_eq!(parsed.members.len(), 2); - assert_eq!(parsed.members[0].email.as_deref(), Some("bob@x")); - assert_eq!(parsed.members[1].name.as_deref(), Some("Carol")); - } - - #[test] - fn contact_decodes_xml_entities() { - let body = format!( - "\ - \ - R&D\ - Sales & Marketing <Team>\ - AT&T\ - R&D\ - a&b@x.com\ - " - ); - let parsed = parse_contact_item(&body).unwrap(); - assert_eq!( - parsed.display_name.as_deref(), - Some("Sales & Marketing ") - ); - assert_eq!(parsed.company_name.as_deref(), Some("AT&T")); - assert_eq!(parsed.department.as_deref(), Some("R&D")); - assert_eq!(parsed.categories, vec!["R&D".to_owned()]); - assert_eq!(parsed.emails[0].1, "a&b@x.com"); - } - - #[test] - fn calendar_decodes_xml_entities_in_subject_and_location() { - let body = format!( - "\ - \ - Q&A <all hands>\ - Room A & B\ - 2025-06-15T14:00:00Z\ - 2025-06-15T15:00:00Z\ - " - ); - let parsed = parse_calendar_item(&body).unwrap(); - assert_eq!(parsed.subject.as_deref(), Some("Q&A ")); - assert_eq!(parsed.location.as_deref(), Some("Room A & B")); - } - - #[test] - fn folder_decodes_xml_entities_in_display_name() { - let body = format!( - "\ - Sales & Marketing\ - IPF.Note" - ); - let entry = parse_folder_inner(&body).unwrap().unwrap(); - assert_eq!(entry.display_name, "Sales & Marketing"); - } - - #[test] - fn response_capture_preserves_entities_for_reparse() { - let body = format!( - "\ - NoError\ - AT&T\ - " - ); - let msgs = parse_response_messages(body.as_bytes(), "GetItemResponseMessage").unwrap(); - assert!( - msgs[0].inner_xml.contains("AT&T") || msgs[0].inner_xml.contains("AT&T"), - "capture must preserve the entity for the per-item parser: {}", - msgs[0].inner_xml - ); - let parsed = parse_contact_item(&msgs[0].inner_xml).unwrap(); - assert_eq!(parsed.company_name.as_deref(), Some("AT&T")); - } - - #[test] - fn contact_parses_emails_phones_addresses() { - let body = format!( - "\ - \ - Alice Doe\ - Alice\ - Doe\ - \ - alice@example.com\ - \ - \ - +15551234\ - \ - \ - 1 MainTownUS\ - \ - " - ); - let parsed = parse_contact_item(&body).unwrap(); - assert_eq!(parsed.given_name.as_deref(), Some("Alice")); - assert_eq!(parsed.surname.as_deref(), Some("Doe")); - assert_eq!(parsed.emails.len(), 1); - assert_eq!(parsed.emails[0].0, "EmailAddress1"); - assert_eq!(parsed.emails[0].1, "alice@example.com"); - assert_eq!(parsed.phones.len(), 1); - assert_eq!(parsed.phones[0].0, "BusinessPhone1"); - assert_eq!(parsed.addresses.len(), 1); - assert_eq!(parsed.addresses[0].key, "Home"); - assert_eq!(parsed.addresses[0].street.as_deref(), Some("1 Main")); - assert_eq!(parsed.addresses[0].country.as_deref(), Some("US")); - } - - #[test] - fn get_item_with_unprefixed_types_elements_parses() { - let body = format!( - "\ - \ - NoError\ - \ - \ - Hello\ - SGk=\ - \ - " - ); - let msgs = parse_response_messages(body.as_bytes(), "GetItemResponseMessage").unwrap(); - assert_eq!(msgs.len(), 1); - let item = parse_message_item(&msgs[0].inner_xml).unwrap(); - assert_eq!(item.id.id, "NP1"); - assert_eq!(item.subject.as_deref(), Some("Hello")); - assert_eq!(item.mime_content.as_deref(), Some("SGk=")); - } - - #[test] - fn default_namespace_response_parses_via_resolved_names() { - let body = "\ - \ - \ - \ - NoError\ - \ - \ - IPF.NoteInbox\ - \ - \ - "; - let parsed = parse_find_folder_response(body.as_bytes()).unwrap(); - assert_eq!(parsed.folders.len(), 1); - assert_eq!(parsed.folders[0].folder_id.id, "D1"); - assert_eq!(parsed.folders[0].display_name, "Inbox"); - } - - #[test] - fn fault_response_code_in_messages_namespace_is_recognised() { - let body = format!( - "\ - soap:Serverbusy\ - ErrorServerBusy\ - 42\ - " - ); - let env = read_envelope_summary(body.as_bytes()).unwrap(); - match env { - EnvelopeKind::Fault { fault, .. } => { - assert!(matches!( - fault.response_code, - ResponseCode::ServerBusy { .. } - )); - assert_eq!(fault.back_off_ms, Some(42)); - } - _ => panic!("expected fault"), - } - } - - #[test] - fn change_key_attribute_with_entities_is_xml_decoded() { - let body = format!( - "\ - NoError\ - \ - \ - X\ - " - ); - let parsed = parse_find_folder_response(body.as_bytes()).unwrap(); - assert_eq!(parsed.folders[0].folder_id.change_key, "A&B"); - } - - #[test] - fn warning_response_class_is_treated_as_success() { - let body = format!( - "\ - \ - NoError\ - \ - " - ); - let r = parse_response_messages(body.as_bytes(), "GetItemResponseMessage").unwrap(); - assert_eq!(r.len(), 1); - assert!(r[0].success, "Warning should be success-equivalent"); - } - - #[test] - fn get_folder_response_message_captures_folder_inner() { - let body = format!( - "\ - \ - NoError\ - \ - Inbox\ - \ - \ - ErrorAccessDenied\ - " - ); - let msgs = parse_response_messages(body.as_bytes(), "GetFolderResponseMessage").unwrap(); - assert_eq!(msgs.len(), 2); - assert!(msgs[0].success); - let f = parse_folder_inner(&msgs[0].inner_xml).unwrap().unwrap(); - assert_eq!(f.folder_id.id, "F1"); - assert!(!msgs[1].success); - } - - #[test] - fn get_attachment_inline_decodes_content_field() { - let body = format!( - "\ - \ - NoError\ - \ - p.png\ - image/png\ - true\ - iVBORw0KGgo=\ - " - ); - let items = parse_get_attachment_inline(body.as_bytes()).unwrap(); - assert_eq!(items.len(), 1); - assert_eq!(items[0].attachment_id, "A1"); - assert_eq!(items[0].name.as_deref(), Some("p.png")); - assert_eq!(items[0].content_type.as_deref(), Some("image/png")); - assert!(items[0].is_contact_photo); - assert_eq!(items[0].content_base64, "iVBORw0KGgo="); - } -} diff --git a/src/exchange_ews/retry.rs b/src/exchange_ews/retry.rs deleted file mode 100644 index 4e08dca..0000000 --- a/src/exchange_ews/retry.rs +++ /dev/null @@ -1,181 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::time::Duration; - -use crate::exchange_ews::types::ResponseCode; -use crate::jmap::retry::Disposition; - -pub fn classify_http_status(status: u16) -> Disposition { - match status { - 429 | 502 | 503 | 504 => Disposition::Retryable, - _ => Disposition::Fatal, - } -} - -#[derive(Debug, Clone)] -pub enum FaultDisposition { - Fatal, - Auth, - VersionError, - Retryable { delay: Option }, -} - -pub fn classify_fault(code: &ResponseCode) -> FaultDisposition { - match code { - ResponseCode::ServerBusy { back_off_ms } => FaultDisposition::Retryable { - delay: back_off_ms.map(Duration::from_millis), - }, - ResponseCode::TimeoutExpired - | ResponseCode::InternalServerTransientError - | ResponseCode::MailboxStoreUnavailable - | ResponseCode::ConnectionFailed - | ResponseCode::AdUnavailable - | ResponseCode::BatchProcessingStopped => FaultDisposition::Retryable { delay: None }, - ResponseCode::AuthenticationRequired => FaultDisposition::Auth, - ResponseCode::InvalidServerVersion - | ResponseCode::IncorrectSchemaVersion - | ResponseCode::InvalidRequest - | ResponseCode::InvalidSchemaVersionForMailboxVersion => FaultDisposition::VersionError, - _ => FaultDisposition::Fatal, - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ItemAction { - Success, - Vanished, - Skip, - Retryable, - RetryDiscardSyncState, -} - -pub fn classify_item_response(code: &ResponseCode) -> ItemAction { - match code { - ResponseCode::NoError => ItemAction::Success, - ResponseCode::ItemNotFound | ResponseCode::AttachmentNotFound => ItemAction::Vanished, - ResponseCode::ServerBusy { .. } - | ResponseCode::TimeoutExpired - | ResponseCode::InternalServerTransientError - | ResponseCode::MailboxStoreUnavailable - | ResponseCode::ConnectionFailed - | ResponseCode::AdUnavailable - | ResponseCode::BatchProcessingStopped => ItemAction::Retryable, - ResponseCode::InvalidSyncStateData => ItemAction::RetryDiscardSyncState, - ResponseCode::AccessDenied - | ResponseCode::InvalidIdMalformed - | ResponseCode::MimeContentConversionFailed - | ResponseCode::ImpersonationFailed => ItemAction::Skip, - ResponseCode::AuthenticationRequired => ItemAction::Skip, - ResponseCode::InvalidServerVersion - | ResponseCode::IncorrectSchemaVersion - | ResponseCode::InvalidRequest - | ResponseCode::InvalidSchemaVersionForMailboxVersion => ItemAction::Skip, - ResponseCode::Other(_) => ItemAction::Skip, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn server_busy_with_backoff_is_retryable_with_delay() { - let code = ResponseCode::ServerBusy { - back_off_ms: Some(297749), - }; - match classify_fault(&code) { - FaultDisposition::Retryable { delay: Some(d) } => { - assert_eq!(d, Duration::from_millis(297749)); - } - _ => panic!("expected retryable with delay"), - } - } - - #[test] - fn item_responses_are_classified() { - assert_eq!( - classify_item_response(&ResponseCode::NoError), - ItemAction::Success - ); - assert_eq!( - classify_item_response(&ResponseCode::ItemNotFound), - ItemAction::Vanished - ); - assert_eq!( - classify_item_response(&ResponseCode::AttachmentNotFound), - ItemAction::Vanished - ); - assert_eq!( - classify_item_response(&ResponseCode::AccessDenied), - ItemAction::Skip - ); - assert_eq!( - classify_item_response(&ResponseCode::InvalidSyncStateData), - ItemAction::RetryDiscardSyncState - ); - assert_eq!( - classify_item_response(&ResponseCode::ServerBusy { back_off_ms: None }), - ItemAction::Retryable - ); - } - - #[test] - fn http_status_table() { - assert_eq!(classify_http_status(429), Disposition::Retryable); - assert_eq!(classify_http_status(503), Disposition::Retryable); - assert_eq!(classify_http_status(500), Disposition::Fatal); - assert_eq!(classify_http_status(401), Disposition::Fatal); - } - - #[test] - fn schema_version_codes_request_a_downgrade() { - for code in [ - ResponseCode::InvalidServerVersion, - ResponseCode::IncorrectSchemaVersion, - ResponseCode::InvalidRequest, - ResponseCode::InvalidSchemaVersionForMailboxVersion, - ] { - assert!(matches!( - classify_fault(&code), - FaultDisposition::VersionError - )); - } - } - - #[test] - fn transient_codes_are_retryable_at_item_and_fault_level() { - for code in [ - ResponseCode::InternalServerTransientError, - ResponseCode::MailboxStoreUnavailable, - ResponseCode::ConnectionFailed, - ResponseCode::AdUnavailable, - ResponseCode::BatchProcessingStopped, - ] { - assert_eq!(classify_item_response(&code), ItemAction::Retryable); - assert!(matches!( - classify_fault(&code), - FaultDisposition::Retryable { .. } - )); - } - } - - #[test] - fn mime_content_conversion_failure_skips_item() { - assert_eq!( - classify_item_response(&ResponseCode::MimeContentConversionFailed), - ItemAction::Skip - ); - } - - #[test] - fn other_unknown_codes_skip_at_item_level() { - assert_eq!( - classify_item_response(&ResponseCode::Other("ErrorWhoKnows".to_owned())), - ItemAction::Skip - ); - } -} diff --git a/src/exchange_ews/xml.rs b/src/exchange_ews/xml.rs deleted file mode 100644 index 4cc98c8..0000000 --- a/src/exchange_ews/xml.rs +++ /dev/null @@ -1,519 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use crate::exchange_ews::soap::write_escaped; -use crate::exchange_ews::types::{DistinguishedFolderId, FolderId, ItemId, ServerVersion}; - -#[derive(Debug, Clone, Copy)] -pub enum FolderRef<'a> { - Distinguished(DistinguishedFolderId), - DistinguishedIn(DistinguishedFolderId, &'a str), - Concrete(&'a FolderId), -} - -impl<'a> FolderRef<'a> { - pub fn distinguished(id: DistinguishedFolderId, mailbox: Option<&'a str>) -> FolderRef<'a> { - match mailbox { - Some(mailbox) => FolderRef::DistinguishedIn(id, mailbox), - None => FolderRef::Distinguished(id), - } - } -} - -#[derive(Debug, Clone, Copy)] -pub enum FolderShape { - IdOnly, -} - -impl FolderShape { - fn as_str(self) -> &'static str { - match self { - FolderShape::IdOnly => "IdOnly", - } - } -} - -#[derive(Debug, Clone, Copy)] -pub enum Traversal { - Shallow, - Deep, -} - -impl Traversal { - fn as_str(self) -> &'static str { - match self { - Traversal::Shallow => "Shallow", - Traversal::Deep => "Deep", - } - } -} - -pub fn find_folder_body(parent: FolderRef<'_>, traversal: Traversal) -> String { - let mut out = String::with_capacity(512); - out.push_str(""); - out.push_str("Default"); - out.push_str(""); - out.push_str(""); - out.push_str(""); - out.push_str(""); - out.push_str(""); - out.push_str(""); - out.push_str(""); - out.push_str(""); - out.push_str(""); - write_folder_ref(&mut out, parent); - out.push_str(""); - out -} - -pub fn get_folder_body(folders: &[FolderRef<'_>], shape: FolderShape) -> String { - let mut out = String::with_capacity(256 + folders.len() * 64); - out.push_str(""); - out.push_str(""); - out.push_str(shape.as_str()); - out.push_str(""); - out.push_str(""); - for f in folders { - write_folder_ref(&mut out, *f); - } - out.push_str(""); - out -} - -const ITEM_CLASS_PROPERTY: &str = - ""; - -pub fn find_item_body( - parent: FolderRef<'_>, - traversal: Traversal, - offset: u32, - page_size: u32, -) -> String { - let mut out = String::with_capacity(512); - out.push_str("IdOnly"); - out.push_str(ITEM_CLASS_PROPERTY); - out.push_str(""); - out.push_str(""); - out.push_str( - "\ - ", - ); - out.push_str(""); - write_folder_ref(&mut out, parent); - out.push_str(""); - out -} - -pub fn sync_folder_items_body( - folder: &FolderId, - sync_state: &str, - max_changes: u32, - version: ServerVersion, -) -> String { - let mut out = String::with_capacity(512); - out.push_str(""); - out.push_str("IdOnly"); - out.push_str(ITEM_CLASS_PROPERTY); - out.push_str(""); - write_folder_id_element(&mut out, folder); - out.push_str(""); - out.push_str(""); - write_escaped(&mut out, sync_state); - out.push_str(""); - out.push_str(""); - out.push_str(&max_changes.to_string()); - out.push_str(""); - if version >= ServerVersion::Exchange2010 { - out.push_str("NormalItems"); - } - out.push_str(""); - out -} - -#[derive(Debug, Clone, Copy)] -pub enum ItemShape { - Message, - CalendarItem, - Contact, -} - -pub fn get_item_body(shape: ItemShape, ids: &[ItemId], version: ServerVersion) -> String { - let mut out = String::with_capacity(256 + ids.len() * 128); - out.push_str(""); - match shape { - ItemShape::Message => { - out.push_str("IdOnly"); - out.push_str("true"); - out.push_str("Best"); - out.push_str(""); - out.push_str(""); - out.push_str(""); - out.push_str(""); - out.push_str(""); - out.push_str(""); - if version >= ServerVersion::Exchange2013 { - out.push_str(""); - } else { - out.push_str( - "", - ); - } - out.push_str(""); - } - ItemShape::CalendarItem => { - out.push_str("IdOnly"); - out.push_str("Best"); - out.push_str(""); - for f in CALENDAR_FIELDS { - out.push_str(""); - } - if version >= ServerVersion::Exchange2010 { - out.push_str(""); - out.push_str(""); - } - out.push_str(""); - } - ItemShape::Contact => { - out.push_str("AllProperties"); - } - } - out.push_str(""); - for id in ids { - write_item_id_element(&mut out, id); - } - out.push_str(""); - out -} - -const CALENDAR_FIELDS: &[&str] = &[ - "calendar:UID", - "calendar:RecurrenceId", - "calendar:Start", - "calendar:End", - "calendar:OriginalStart", - "calendar:IsAllDayEvent", - "calendar:LegacyFreeBusyStatus", - "calendar:Location", - "calendar:IsRecurring", - "calendar:CalendarItemType", - "calendar:Organizer", - "calendar:RequiredAttendees", - "calendar:OptionalAttendees", - "calendar:Resources", - "calendar:IsOnlineMeeting", - "calendar:MeetingWorkspaceUrl", - "calendar:NetShowUrl", - "calendar:MyResponseType", - "item:ReminderIsSet", - "item:ReminderMinutesBeforeStart", - "calendar:Recurrence", - "calendar:ModifiedOccurrences", - "calendar:DeletedOccurrences", - "calendar:Duration", - "item:Subject", - "item:Body", - "item:Categories", - "item:Sensitivity", - "item:Attachments", - "item:DateTimeCreated", - "item:LastModifiedTime", - "item:ParentFolderId", -]; - -pub fn get_attachment_body(ids: &[&str]) -> String { - let mut out = String::with_capacity(256 + ids.len() * 64); - out.push_str(""); - out.push_str("true"); - out.push_str("Best"); - out.push_str(""); - for id in ids { - out.push_str(""); - } - out.push_str(""); - out -} - -pub fn write_folder_id_element(out: &mut String, folder: &FolderId) { - out.push_str(""); -} - -pub fn write_item_id_element(out: &mut String, id: &ItemId) { - out.push_str(""); -} - -fn write_folder_ref(out: &mut String, parent: FolderRef<'_>) { - match parent { - FolderRef::Distinguished(d) => { - out.push_str(""); - } - FolderRef::DistinguishedIn(d, mailbox) => { - out.push_str(""); - write_escaped(out, mailbox); - out.push_str(""); - } - FolderRef::Concrete(f) => write_folder_id_element(out, f), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn find_folder_uses_deep_traversal_for_msgfolderroot() { - let body = find_folder_body( - FolderRef::Distinguished(DistinguishedFolderId::MsgFolderRoot), - Traversal::Deep, - ); - assert!(body.contains("Traversal=\"Deep\"")); - assert!(body.contains("")); - assert!(body.contains("folder:FolderClass")); - assert!(body.contains("folder:DisplayName")); - assert!(body.contains("folder:ParentFolderId")); - assert!(body.contains("PropertyTag=\"0x10F4\" PropertyType=\"Boolean\"")); - } - - #[test] - fn find_folder_archive_uses_archiveroot() { - let body = find_folder_body( - FolderRef::Distinguished(DistinguishedFolderId::ArchiveRoot), - Traversal::Deep, - ); - assert!(body.contains("")); - } - - #[test] - fn distinguished_folder_in_named_mailbox_carries_escaped_address() { - let body = find_folder_body( - FolderRef::distinguished( - DistinguishedFolderId::MsgFolderRoot, - Some("b&o@corp.example"), - ), - Traversal::Deep, - ); - assert!(body.contains( - "\ - b&o@corp.example\ - " - )); - let body = find_folder_body( - FolderRef::distinguished(DistinguishedFolderId::MsgFolderRoot, None), - Traversal::Deep, - ); - assert!(body.contains("")); - } - - #[test] - fn find_item_paginates_with_offset_and_page_size() { - let folder = FolderId::new("FID", "FCK"); - let body = find_item_body(FolderRef::Concrete(&folder), Traversal::Shallow, 200, 50); - assert!(body.contains("Offset=\"200\"")); - assert!(body.contains("MaxEntriesReturned=\"50\"")); - assert!(body.contains("")); - } - - #[test] - fn find_item_sorts_by_creation_time_for_stable_paging() { - let folder = FolderId::new("FID", "FCK"); - let body = find_item_body(FolderRef::Concrete(&folder), Traversal::Shallow, 0, 50); - assert!(body.contains("")); - assert!(body.contains("Order=\"Ascending\"")); - assert!(body.contains("item:DateTimeCreated")); - let sort_at = body.find("").unwrap(); - let parents_at = body.find("").unwrap(); - assert!( - sort_at < parents_at, - "SortOrder must precede ParentFolderIds" - ); - } - - #[test] - fn get_item_message_shape_requests_mime() { - let ids = vec![ItemId::new("I1", "CK1"), ItemId::new("I2", "")]; - let body = get_item_body(ItemShape::Message, &ids, ServerVersion::Exchange2013Sp1); - assert!(body.contains("true")); - assert!(body.contains("")); - assert!(body.contains("")); - } - - #[test] - fn item_flag_requested_only_on_exchange_2013_and_later() { - let ids = [ItemId::new("I1", "")]; - let modern = get_item_body(ItemShape::Message, &ids, ServerVersion::Exchange2013Sp1); - assert!(modern.contains("item:Flag")); - let legacy = get_item_body(ItemShape::Message, &ids, ServerVersion::Exchange2010Sp2); - assert!( - !legacy.contains("item:Flag"), - "item:Flag is an Exchange 2013 schema addition; must be omitted on 2010" - ); - assert!( - legacy.contains("PropertyTag=\"0x1090\""), - "on pre-2013 the follow-up flag must be fetched via the PidTagFlagStatus extended property" - ); - assert!( - !modern.contains("0x1090"), - "the extended-property fallback is only used when item:Flag is unavailable" - ); - assert!(legacy.contains("item:DateTimeReceived")); - assert!( - modern.contains("\"message:IsRead\"") && !modern.contains("\"item:IsRead\""), - "IsRead is a MessageType property; its FieldURI is message:IsRead" - ); - } - - #[test] - fn get_item_calendar_shape_lists_calendar_fields() { - let body = get_item_body( - ItemShape::CalendarItem, - &[ItemId::new("X", "")], - ServerVersion::Exchange2010Sp2, - ); - assert!(body.contains("calendar:Recurrence")); - assert!(body.contains("calendar:ModifiedOccurrences")); - assert!(body.contains("calendar:DeletedOccurrences")); - assert!(body.contains("calendar:UID")); - assert!(!body.contains("")); - } - - #[test] - fn get_item_contact_shape_requests_all_properties() { - let body = get_item_body( - ItemShape::Contact, - &[ItemId::new("X", "")], - ServerVersion::Exchange2013Sp1, - ); - assert!(body.contains("AllProperties")); - } - - #[test] - fn sync_folder_items_body_carries_state_and_max() { - let folder = FolderId::new("FID", "FCK"); - let body = sync_folder_items_body(&folder, "STATE", 512, ServerVersion::Exchange2013Sp1); - assert!(body.contains("")); - assert!(body.contains("STATE")); - assert!(body.contains("512")); - assert!(body.contains("NormalItems")); - assert!(body.contains("FieldURI=\"item:ItemClass\"")); - } - - #[test] - fn find_item_requests_item_class() { - let folder = FolderId::new("FID", "FCK"); - let body = find_item_body(FolderRef::Concrete(&folder), Traversal::Shallow, 0, 10); - assert!(body.contains( - "IdOnly" - )); - } - - #[test] - fn sync_scope_omitted_below_exchange_2010() { - let folder = FolderId::new("FID", ""); - let modern = sync_folder_items_body(&folder, "", 100, ServerVersion::Exchange2010); - assert!(modern.contains("NormalItems")); - let legacy = sync_folder_items_body(&folder, "", 100, ServerVersion::Exchange2007); - assert!( - !legacy.contains("SyncScope"), - "SyncScope is an Exchange 2010 addition" - ); - } - - #[test] - fn calendar_shape_requests_attachments_own_response_and_sensitivity() { - let body = get_item_body( - ItemShape::CalendarItem, - &[ItemId::new("X", "")], - ServerVersion::Exchange2007, - ); - for field in [ - "item:Attachments", - "calendar:MyResponseType", - "item:Sensitivity", - ] { - assert!(body.contains(&format!("\"{field}\"")), "{field}"); - } - } - - #[test] - fn calendar_timezone_fields_gated_on_exchange_2010() { - let modern = get_item_body( - ItemShape::CalendarItem, - &[ItemId::new("X", "")], - ServerVersion::Exchange2010, - ); - assert!(modern.contains("calendar:StartTimeZone")); - assert!(modern.contains("calendar:EndTimeZone")); - let legacy = get_item_body( - ItemShape::CalendarItem, - &[ItemId::new("X", "")], - ServerVersion::Exchange2007, - ); - assert!( - !legacy.contains("TimeZone"), - "StartTimeZone/EndTimeZone are Exchange 2010 additions" - ); - } - - #[test] - fn sync_folder_items_empty_state_round_trips() { - let folder = FolderId::new("FID", ""); - let body = sync_folder_items_body(&folder, "", 100, ServerVersion::Exchange2013Sp1); - assert!(body.contains("")); - } - - #[test] - fn get_attachment_lists_ids() { - let body = get_attachment_body(&["AID1", "AID2"]); - assert!(body.contains("")); - assert!(body.contains("")); - assert!(body.contains("true")); - } - - #[test] - fn get_folder_emits_distinguished_ids() { - let body = get_folder_body( - &[ - FolderRef::Distinguished(DistinguishedFolderId::Inbox), - FolderRef::Distinguished(DistinguishedFolderId::SentItems), - ], - FolderShape::IdOnly, - ); - assert!(body.contains("")); - assert!(body.contains("")); - assert!(body.contains("IdOnly")); - } -} diff --git a/src/exchange_graph/calendar_map.rs b/src/exchange_graph/calendar_map.rs deleted file mode 100644 index 76f63f2..0000000 --- a/src/exchange_graph/calendar_map.rs +++ /dev/null @@ -1,1017 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use chrono::NaiveDate; -use serde_json::{Map, Value, json}; - -use crate::exchange::address::{as_smtp_address, extract_smtp_address}; -use crate::exchange::date::all_day_date; -use crate::exchange::jscalendar::{drop_calendar_address_dependents, synthetic_attendee_address}; -use crate::exchange::tz::resolve_to_iana; -use crate::exchange_graph::error::GraphError; -use crate::exchange_graph::recurrence::convert_patterned_recurrence_rule; - -pub fn graph_calendar_color_to_hex(value: &str) -> Option<&'static str> { - match value { - "lightBlue" => Some("#A6CEE3"), - "lightGreen" => Some("#B2DF8A"), - "lightOrange" => Some("#FDBF6F"), - "lightGray" => Some("#CCCCCC"), - "lightYellow" => Some("#FFFF99"), - "lightTeal" => Some("#A0E7E5"), - "lightPink" => Some("#FB9A99"), - "lightBrown" => Some("#B15928"), - "lightRed" => Some("#FB9A99"), - _ => None, - } -} - -pub fn windows_or_iana_to_iana(value: &str) -> Option { - resolve_to_iana(value) -} - -fn utc_naive_to_local(utc_naive: &str, iana: &str) -> Option { - use chrono::{NaiveDateTime, TimeZone}; - use chrono_tz::Tz; - - let tz: Tz = iana.parse().ok()?; - let naive = NaiveDateTime::parse_from_str(utc_naive, "%Y-%m-%dT%H:%M:%S").ok()?; - Some( - tz.from_utc_datetime(&naive) - .naive_local() - .format("%Y-%m-%dT%H:%M:%S") - .to_string(), - ) -} - -#[derive(Debug, Clone)] -pub struct ConvertedEvent { - pub uid: String, - pub data: Value, - pub is_draft: bool, - pub use_default_alerts: bool, - pub series_master_id: Option, - pub event_type: EventType, - pub original_start: Option, - pub zone: Option, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum EventType { - SingleInstance, - SeriesMaster, - Occurrence, - Exception, -} - -pub fn classify_event_type(value: &Value) -> EventType { - match value.get("type").and_then(Value::as_str).unwrap_or("") { - "seriesMaster" => EventType::SeriesMaster, - "occurrence" => EventType::Occurrence, - "exception" => EventType::Exception, - _ => EventType::SingleInstance, - } -} - -pub fn convert_event( - graph_event: &Value, - fallback_calendar_tz: Option<&str>, -) -> Result { - let uid = graph_event - .get("iCalUId") - .and_then(Value::as_str) - .or_else(|| graph_event.get("id").and_then(Value::as_str)) - .ok_or_else(|| GraphError::Malformed("event has neither iCalUId nor id".to_owned()))? - .to_owned(); - - let event_type = classify_event_type(graph_event); - let series_master_id = graph_event - .get("seriesMasterId") - .and_then(Value::as_str) - .map(str::to_owned); - let original_start = graph_event - .get("originalStart") - .and_then(Value::as_str) - .map(str::to_owned); - let is_draft = graph_event - .get("isDraft") - .and_then(Value::as_bool) - .unwrap_or(false); - - let mut card = Map::new(); - card.insert("@type".to_owned(), Value::from("Event")); - card.insert("uid".to_owned(), Value::from(uid.clone())); - - if let Some(subject) = graph_event.get("subject").and_then(Value::as_str) - && !subject.is_empty() - { - card.insert("title".to_owned(), Value::from(subject.to_owned())); - } - - if let Some(body) = graph_event.get("body").and_then(Value::as_object) - && let Some(content) = body.get("content").and_then(Value::as_str) - && !content.is_empty() - { - card.insert("description".to_owned(), Value::from(content.to_owned())); - } - - let start_dt = extract_local_datetime(graph_event.get("start")).map(strip_fractional); - let end_dt = extract_local_datetime(graph_event.get("end")).map(strip_fractional); - - let is_all_day = graph_event - .get("isAllDay") - .and_then(Value::as_bool) - .unwrap_or(false); - - let display_tz = graph_event - .get("originalStartTimeZone") - .and_then(Value::as_str) - .and_then(windows_or_iana_to_iana) - .or_else(|| { - graph_event - .get("start") - .and_then(|s| s.get("timeZone")) - .and_then(Value::as_str) - .and_then(windows_or_iana_to_iana) - }) - .or_else(|| fallback_calendar_tz.and_then(windows_or_iana_to_iana)) - .filter(|z| z != "Etc/UTC"); - - if is_all_day { - let zone = display_tz.as_deref(); - let start = start_dt.as_deref().and_then(|s| all_day_date(s, zone)); - if let Some(start) = start { - card.insert("start".to_owned(), Value::from(all_day_key(start))); - let days = end_dt - .as_deref() - .and_then(|e| all_day_date(e, zone)) - .map_or(1, |end| (end - start).num_days().max(1)); - card.insert("duration".to_owned(), Value::from(format!("P{days}D"))); - } - card.insert("showWithoutTime".to_owned(), Value::Bool(true)); - } else { - if let Some(start) = start_dt.as_deref() { - let value = display_tz - .as_deref() - .and_then(|tz| utc_naive_to_local(start, tz)) - .unwrap_or_else(|| start.to_owned()); - card.insert("start".to_owned(), Value::from(value)); - } - if let (Some(start), Some(end)) = (start_dt.as_deref(), end_dt.as_deref()) - && let Some(dur) = duration_iso8601(start, end) - { - card.insert("duration".to_owned(), Value::from(dur)); - } - let label = display_tz.as_deref().unwrap_or("Etc/UTC"); - card.insert("timeZone".to_owned(), Value::from(label)); - } - - if let Some(true) = graph_event.get("isCancelled").and_then(Value::as_bool) { - card.insert("status".to_owned(), Value::from("cancelled")); - } else { - card.insert("status".to_owned(), Value::from("confirmed")); - } - - if let Some(sens) = graph_event.get("sensitivity").and_then(Value::as_str) { - card.insert("privacy".to_owned(), Value::from(privacy_for(sens))); - } - - if let Some(imp) = graph_event.get("importance").and_then(Value::as_str) { - card.insert("priority".to_owned(), Value::from(priority_for(imp))); - } - - if let Some(show) = graph_event.get("showAs").and_then(Value::as_str) { - card.insert( - "freeBusyStatus".to_owned(), - Value::from(free_busy_for(show)), - ); - } - - if let Some(cats) = graph_event.get("categories").and_then(Value::as_array) - && !cats.is_empty() - { - let mut map = Map::new(); - for cat in cats.iter().filter_map(Value::as_str) { - map.insert(cat.to_owned(), Value::Bool(true)); - } - if !map.is_empty() { - card.insert("keywords".to_owned(), Value::Object(map)); - } - } - - if let Some(locs) = graph_event.get("locations").and_then(Value::as_array) - && !locs.is_empty() - { - let mut locations = Map::new(); - let width = pad_width(locs.len()); - for (i, loc) in locs.iter().enumerate() { - let key = format!("loc-{:0width$}", i + 1, width = width); - let mut object = Map::new(); - object.insert("@type".to_owned(), Value::from("Location")); - let display_name = loc - .get("displayName") - .and_then(Value::as_str) - .filter(|s| !s.is_empty()); - let address_str = loc - .get("address") - .and_then(Value::as_object) - .map(|addr| { - ["street", "city", "state", "postalCode", "countryOrRegion"] - .iter() - .filter_map(|f| addr.get(*f).and_then(Value::as_str)) - .filter(|s| !s.is_empty()) - .collect::>() - .join(", ") - }) - .filter(|s| !s.is_empty()); - let name = match (display_name, address_str.as_deref()) { - (Some(dn), Some(addr)) if dn != addr => Some(format!("{dn}, {addr}")), - (Some(dn), _) => Some(dn.to_owned()), - (None, Some(addr)) => Some(addr.to_owned()), - (None, None) => None, - }; - if let Some(n) = name { - object.insert("name".to_owned(), Value::from(n)); - } - locations.insert(key, Value::Object(object)); - } - if !locations.is_empty() { - card.insert("locations".to_owned(), Value::Object(locations)); - } - } - - if let Some(recurrence) = graph_event.get("recurrence") - && !recurrence.is_null() - { - let rule = convert_patterned_recurrence_rule(recurrence)?; - card.insert("recurrenceRule".to_owned(), Value::Object(rule)); - } - - let mut participants: Vec<(String, Value)> = Vec::new(); - if let Some(org) = graph_event - .get("organizer") - .and_then(|o| o.get("emailAddress")) - { - if let Some((cal_addr, _)) = resolve_calendar_address(org) { - card.insert("organizerCalendarAddress".to_owned(), Value::from(cal_addr)); - } - let p = build_participant(org, &["owner", "chair"], None, None); - participants.push(("organizer".to_owned(), p)); - } - if let Some(attendees) = graph_event.get("attendees").and_then(Value::as_array) { - let width = pad_width(attendees.len()); - for (i, att) in attendees.iter().enumerate() { - let key = format!("att-{:0width$}", i + 1, width = width); - let Some(email) = att.get("emailAddress") else { - continue; - }; - let role = att - .get("type") - .and_then(Value::as_str) - .map(attendee_type_to_role) - .unwrap_or("required"); - let status = att - .get("status") - .and_then(|s| s.get("response")) - .and_then(Value::as_str) - .and_then(attendee_response_to_status); - let p = build_participant(email, &[role], status, Some(true)); - participants.push((key, p)); - } - } - if !participants.is_empty() { - let mut map: Map = participants.into_iter().collect(); - if !card.contains_key("organizerCalendarAddress") { - drop_calendar_address_dependents(&mut map); - } - card.insert("participants".to_owned(), Value::Object(map)); - } - - if let Some(created) = graph_event.get("createdDateTime").and_then(Value::as_str) { - card.insert( - "created".to_owned(), - Value::from(strip_fractional_utc(created)), - ); - } - if let Some(updated) = graph_event - .get("lastModifiedDateTime") - .and_then(Value::as_str) - { - card.insert( - "updated".to_owned(), - Value::from(strip_fractional_utc(updated)), - ); - } - - let use_default_alerts = false; - if let Some(true) = graph_event.get("isReminderOn").and_then(Value::as_bool) - && let Some(mins) = graph_event - .get("reminderMinutesBeforeStart") - .and_then(Value::as_i64) - { - let trigger = format!("-PT{}M", mins.unsigned_abs()); - let alert = json!({ - "@type": "Alert", - "trigger": { - "@type": "OffsetTrigger", - "relativeTo": "start", - "offset": trigger - } - }); - let mut alerts = Map::new(); - alerts.insert("alert-1".to_owned(), alert); - card.insert("alerts".to_owned(), Value::Object(alerts)); - } - - if let Some(cancels) = graph_event - .get("cancelledOccurrences") - .and_then(Value::as_array) - && !cancels.is_empty() - { - let start_time = card - .get("start") - .and_then(Value::as_str) - .and_then(|s| s.split_once('T').map(|(_, time)| time.to_owned())); - let mut overrides = Map::new(); - for entry in cancels.iter().filter_map(Value::as_str) { - if let Some(key) = cancelled_occurrence_key(entry, start_time.as_deref()) { - overrides.insert(key, json!({"excluded": true})); - } - } - if !overrides.is_empty() { - card.insert("recurrenceOverrides".to_owned(), Value::Object(overrides)); - } - } - - Ok(ConvertedEvent { - uid, - data: Value::Object(card), - is_draft, - use_default_alerts, - series_master_id, - event_type, - original_start, - zone: display_tz, - }) -} - -pub fn all_day_key(date: NaiveDate) -> String { - format!("{date}T00:00:00") -} - -fn cancelled_occurrence_key(entry: &str, start_time: Option<&str>) -> Option { - let date = entry.rsplit('.').next().filter(|d| d.len() == 10)?; - if !date.as_bytes().iter().enumerate().all(|(i, b)| match i { - 4 | 7 => *b == b'-', - _ => b.is_ascii_digit(), - }) { - return None; - } - Some(format!("{date}T{}", start_time.unwrap_or("00:00:00"))) -} - -fn extract_local_datetime(slot: Option<&Value>) -> Option { - let slot = slot?; - let dt = slot.get("dateTime").and_then(Value::as_str)?; - Some(dt.to_owned()) -} - -fn non_empty(email: &Value, key: &str) -> Option { - email - .get(key) - .and_then(Value::as_str) - .map(str::trim) - .filter(|s| !s.is_empty()) - .map(str::to_owned) -} - -fn resolve_calendar_address(email: &Value) -> Option<(String, Option)> { - if let Some(addr) = non_empty(email, "address") { - if let Some(smtp) = as_smtp_address(&addr).or_else(|| extract_smtp_address(&addr)) { - return Some((format!("mailto:{smtp}"), Some(smtp))); - } - return Some((synthetic_attendee_address(&addr), None)); - } - non_empty(email, "name").map(|name| (synthetic_attendee_address(&name), None)) -} - -fn build_participant( - email: &Value, - roles: &[&str], - participation_status: Option<&'static str>, - expect_reply: Option, -) -> Value { - let mut map = Map::new(); - map.insert("@type".to_owned(), Value::from("Participant")); - if let Some(name) = non_empty(email, "name") { - map.insert("name".to_owned(), Value::from(name)); - } - if let Some((cal_addr, smtp)) = resolve_calendar_address(email) { - map.insert("calendarAddress".to_owned(), Value::from(cal_addr)); - if let Some(addr) = smtp { - map.insert("email".to_owned(), Value::from(addr)); - } - let role_map: Map = roles - .iter() - .map(|r| ((*r).to_owned(), Value::Bool(true))) - .collect(); - map.insert("roles".to_owned(), Value::Object(role_map)); - if let Some(status) = participation_status { - map.insert( - "participationStatus".to_owned(), - Value::from(status.to_owned()), - ); - } - if let Some(b) = expect_reply { - map.insert("expectReply".to_owned(), Value::Bool(b)); - } - } - Value::Object(map) -} - -fn attendee_type_to_role(t: &str) -> &'static str { - match t { - "optional" => "optional", - "resource" => "informational", - _ => "required", - } -} - -fn attendee_response_to_status(r: &str) -> Option<&'static str> { - match r { - "accepted" | "organizer" => Some("accepted"), - "declined" => Some("declined"), - "tentativelyAccepted" => Some("tentative"), - "notResponded" | "none" => Some("needs-action"), - _ => None, - } -} - -fn pad_width(n: usize) -> usize { - let mut digits = 1; - let mut k = n; - while k >= 10 { - digits += 1; - k /= 10; - } - digits -} - -fn privacy_for(value: &str) -> &'static str { - match value { - "private" => "private", - "confidential" => "secret", - _ => "public", - } -} - -fn priority_for(value: &str) -> i64 { - match value { - "high" => 1, - "low" => 9, - _ => 5, - } -} - -fn free_busy_for(value: &str) -> &'static str { - match value { - "free" => "free", - _ => "busy", - } -} - -fn duration_iso8601(start: &str, end: &str) -> Option { - let s = parse_naive_seconds(start)?; - let e = parse_naive_seconds(end)?; - if e <= s { - return None; - } - let total = e - s; - let hours = total / 3600; - let minutes = (total % 3600) / 60; - let seconds = total % 60; - let mut out = "PT".to_owned(); - use std::fmt::Write; - if hours > 0 { - let _ = write!(out, "{hours}H"); - } - if minutes > 0 || (hours > 0 && seconds > 0) { - let _ = write!(out, "{minutes}M"); - } - if seconds > 0 || (hours == 0 && minutes == 0) { - let _ = write!(out, "{seconds}S"); - } - Some(out) -} - -fn strip_fractional(s: String) -> String { - if let Some(dot) = s.find('.') { - s[..dot].to_owned() - } else { - s - } -} - -fn strip_fractional_utc(raw: &str) -> String { - let trailing_z = raw.ends_with('Z'); - let trimmed = raw.trim_end_matches('Z'); - let base = match trimmed.find('.') { - Some(dot) => &trimmed[..dot], - None => trimmed, - }; - if trailing_z { - format!("{base}Z") - } else { - base.to_owned() - } -} - -fn parse_naive_seconds(s: &str) -> Option { - let (date, time) = s.split_once('T')?; - let mut date_parts = date.split('-'); - let year: i64 = date_parts.next()?.parse().ok()?; - let month: i64 = date_parts.next()?.parse().ok()?; - let day: i64 = date_parts.next()?.parse().ok()?; - let time = time.split('.').next()?; - let time = time.trim_end_matches('Z'); - let mut time_parts = time.split(':'); - let hh: i64 = time_parts.next()?.parse().ok()?; - let mm: i64 = time_parts.next()?.parse().ok()?; - let ss: i64 = time_parts.next().unwrap_or("0").parse().ok()?; - Some(days_from_civil(year, month, day) * 86400 + hh * 3600 + mm * 60 + ss) -} - -fn days_from_civil(y: i64, m: i64, d: i64) -> i64 { - let y = if m <= 2 { y - 1 } else { y }; - let era = (if y >= 0 { y } else { y - 399 }) / 400; - let yoe = (y - era * 400) as u64; - let m_adj: i64 = if m > 2 { m - 3 } else { m + 9 }; - let doy = ((153 * m_adj + 2) / 5 + d - 1) as u64; - let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; - era * 146097 + (doe as i64) - 719468 -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - fn sample() -> Value { - json!({ - "id": "AAA", - "iCalUId": "uid-1", - "type": "singleInstance", - "subject": "Sync up", - "body": {"contentType": "text", "content": "Agenda"}, - "start": {"dateTime": "2026-05-27T10:00:00.0000000", "timeZone": "UTC"}, - "end": {"dateTime": "2026-05-27T11:00:00.0000000", "timeZone": "UTC"}, - "isAllDay": false, - "isCancelled": false, - "sensitivity": "private", - "importance": "high", - "showAs": "busy", - "categories": ["Red", "Internal"], - "createdDateTime": "2026-05-01T08:00:00Z", - "lastModifiedDateTime": "2026-05-02T08:00:00Z", - "isReminderOn": true, - "reminderMinutesBeforeStart": 15, - "isDraft": false, - "organizer": {"emailAddress": {"name": "Alice", "address": "alice@x.com"}}, - "attendees": [ - {"emailAddress": {"name": "Bob", "address": "bob@x.com"}, "type": "required"} - ] - }) - } - - #[test] - fn classify_event_type_dispatch() { - assert_eq!( - classify_event_type(&json!({"type": "seriesMaster"})), - EventType::SeriesMaster - ); - assert_eq!( - classify_event_type(&json!({"type": "occurrence"})), - EventType::Occurrence - ); - assert_eq!( - classify_event_type(&json!({"type": "exception"})), - EventType::Exception - ); - assert_eq!(classify_event_type(&json!({})), EventType::SingleInstance); - } - - #[test] - fn single_instance_converts_to_event_value() { - let conv = convert_event(&sample(), Some("UTC")).unwrap(); - assert_eq!(conv.uid, "uid-1"); - assert_eq!(conv.data["@type"], "Event"); - assert_eq!(conv.data["title"], "Sync up"); - assert_eq!(conv.data["description"], "Agenda"); - assert_eq!(conv.data["start"], "2026-05-27T10:00:00"); - assert_eq!(conv.data["duration"], "PT1H"); - assert_eq!(conv.data["status"], "confirmed"); - assert_eq!(conv.data["privacy"], "private"); - assert_eq!(conv.data["priority"], 1); - assert_eq!(conv.data["freeBusyStatus"], "busy"); - let kws = conv.data["keywords"].as_object().unwrap(); - assert!(kws.contains_key("Red")); - assert!(kws.contains_key("Internal")); - assert!(conv.data.get("categories").is_none()); - assert!(conv.data.get("version").is_none()); - assert_eq!(conv.data["timeZone"], "Etc/UTC"); - let alerts = conv.data["alerts"].as_object().unwrap(); - assert_eq!(alerts["alert-1"]["trigger"]["offset"], "-PT15M"); - } - - #[test] - fn cancelled_event_status_is_cancelled() { - let mut v = sample(); - v["isCancelled"] = Value::Bool(true); - let conv = convert_event(&v, None).unwrap(); - assert_eq!(conv.data["status"], "cancelled"); - } - - #[test] - fn all_day_sets_show_without_time() { - let mut v = sample(); - v["isAllDay"] = Value::Bool(true); - let conv = convert_event(&v, None).unwrap(); - assert_eq!(conv.data["showWithoutTime"], true); - } - - #[test] - fn priority_mapping_covers_three_levels() { - let conv = convert_event(&sample(), None).unwrap(); - assert_eq!(conv.data["priority"], 1); - let mut v = sample(); - v["importance"] = Value::from("normal"); - assert_eq!(convert_event(&v, None).unwrap().data["priority"], 5); - v["importance"] = Value::from("low"); - assert_eq!(convert_event(&v, None).unwrap().data["priority"], 9); - } - - #[test] - fn duration_handles_subhour() { - assert_eq!( - duration_iso8601("2026-05-27T10:00:00", "2026-05-27T10:30:00").as_deref(), - Some("PT30M") - ); - } - - #[test] - fn duration_handles_multi_day() { - assert_eq!( - duration_iso8601("2026-05-27T10:00:00", "2026-05-28T11:30:45").as_deref(), - Some("PT25H30M45S") - ); - } - - #[test] - fn windows_tz_fallback_returns_iana() { - assert_eq!( - windows_or_iana_to_iana("Pacific Standard Time").as_deref(), - Some("America/Los_Angeles") - ); - assert_eq!( - windows_or_iana_to_iana("America/New_York").as_deref(), - Some("America/New_York") - ); - assert!(windows_or_iana_to_iana("Made Up Zone").is_none()); - } - - #[test] - fn organizer_and_attendees_become_participants() { - let conv = convert_event(&sample(), None).unwrap(); - let participants = conv.data["participants"].as_object().unwrap(); - assert!(participants.contains_key("organizer")); - assert_eq!(participants["organizer"]["email"], "alice@x.com"); - assert_eq!( - participants["organizer"]["calendarAddress"], - "mailto:alice@x.com" - ); - assert_eq!(participants["organizer"]["roles"]["owner"], true); - assert_eq!(participants["organizer"]["roles"]["chair"], true); - assert!(participants.contains_key("att-1")); - assert_eq!(participants["att-1"]["email"], "bob@x.com"); - assert_eq!(participants["att-1"]["roles"]["required"], true); - } - - #[test] - fn a_hybrid_x500_address_is_not_emitted_as_a_mailto() { - let mut v = sample(); - v["attendees"] = json!([ - { - "emailAddress": { - "name": "Kristina Morgental", - "address": "/o=ExchangeLabs/ou=Exchange Administrative Group \ - (FYDIBOHF23SPDLT)/cn=Recipients/cn=bdc77b18152647a29d28ce1188376dc9-kristina" - }, - "type": "required", - "status": {"response": "none"} - } - ]); - let conv = convert_event(&v, None).unwrap(); - let p = &conv.data["participants"]["att-1"]; - let addr = p["calendarAddress"].as_str().unwrap(); - assert!( - addr.starts_with("urn:x-inbuxa-migrate:attendee:"), - "a hybrid X500 reference must not become a malformed mailto, got {addr}" - ); - assert!( - p.get("email").is_none(), - "an X500 reference is not an email address" - ); - assert_eq!(p["name"], "Kristina Morgental", "the display name is kept"); - } - - #[test] - fn a_prefixed_or_bracketed_address_is_normalised() { - let mut v = sample(); - v["attendees"] = json!([ - { - "emailAddress": {"name": "Bob", "address": "SMTP:bob@example.com"}, - "type": "required", - "status": {"response": "none"} - }, - { - "emailAddress": {"name": "Eve", "address": "Eve "}, - "type": "required", - "status": {"response": "none"} - } - ]); - let conv = convert_event(&v, None).unwrap(); - assert_eq!( - conv.data["participants"]["att-1"]["calendarAddress"], - "mailto:bob@example.com" - ); - assert_eq!( - conv.data["participants"]["att-1"]["email"], - "bob@example.com" - ); - assert_eq!( - conv.data["participants"]["att-2"]["calendarAddress"], - "mailto:eve@example.com" - ); - } - - #[test] - fn a_name_only_attendee_gets_a_stable_synthetic_calendar_address() { - let mut v = sample(); - v["attendees"] = json!([ - { - "emailAddress": {"name": "Room 4"}, - "type": "resource", - "status": {"response": "accepted"} - } - ]); - let conv = convert_event(&v, None).unwrap(); - let p = &conv.data["participants"]["att-1"]; - let addr = p["calendarAddress"].as_str().unwrap(); - assert!( - addr.starts_with("urn:x-inbuxa-migrate:attendee:"), - "a name-only attendee needs a synthetic calendarAddress, got {addr}" - ); - assert!( - p.get("email").is_none(), - "there is no SMTP address to report as email" - ); - assert_eq!(p["participationStatus"], "accepted"); - assert_eq!(p["roles"]["informational"], true); - assert_eq!( - addr, - convert_event(&v, None).unwrap().data["participants"]["att-1"]["calendarAddress"] - ); - } - - #[test] - fn an_unidentifiable_attendee_keeps_no_calendar_address_dependent_property() { - let mut v = sample(); - v["attendees"] = json!([ - {"emailAddress": {}, "type": "required", "status": {"response": "accepted"}} - ]); - let conv = convert_event(&v, None).unwrap(); - let p = &conv.data["participants"]["att-1"]; - for key in [ - "calendarAddress", - "email", - "roles", - "participationStatus", - "expectReply", - ] { - assert!( - p.get(key).is_none(), - "{key} requires calendarAddress (jscalendarbis 3.4.6)" - ); - } - assert_eq!(p["@type"], "Participant"); - } - - #[test] - fn an_organizer_without_an_address_still_yields_a_calendar_address() { - let mut v = sample(); - v["organizer"] = json!({"emailAddress": {"name": "Alice Example"}}); - let conv = convert_event(&v, None).unwrap(); - let organizer_address = conv.data["organizerCalendarAddress"].as_str().unwrap(); - assert!(organizer_address.starts_with("urn:x-inbuxa-migrate:attendee:")); - assert_eq!( - conv.data["participants"]["organizer"]["calendarAddress"], - organizer_address - ); - } - - #[test] - fn participants_drop_calendar_addresses_when_no_organizer_is_known() { - let mut v = sample(); - v.as_object_mut().unwrap().remove("organizer"); - let conv = convert_event(&v, None).unwrap(); - assert!( - conv.data.get("organizerCalendarAddress").is_none(), - "no organizer is derivable from the source event" - ); - let participants = conv.data["participants"].as_object().unwrap(); - assert!(!participants.is_empty()); - for (key, participant) in participants { - assert!( - participant.get("calendarAddress").is_none(), - "{key} may not carry a calendarAddress without organizerCalendarAddress" - ); - assert!(participant.get("roles").is_none(), "{key} kept roles"); - } - } - - #[test] - fn optional_attendee_keeps_role_and_uses_status_when_present() { - let mut v = sample(); - v["attendees"] = json!([ - { - "emailAddress": {"name": "Carol", "address": "carol@x.com"}, - "type": "optional", - "status": {"response": "tentativelyAccepted"} - } - ]); - let conv = convert_event(&v, None).unwrap(); - let p = &conv.data["participants"]["att-1"]; - assert_eq!(p["roles"]["optional"], true); - assert!(p["roles"].get("required").is_none()); - assert_eq!(p["participationStatus"], "tentative"); - } - - #[test] - fn many_attendees_zero_pad_keys_for_stable_sort() { - let mut v = sample(); - let mut atts = Vec::new(); - for i in 1..=11 { - atts.push(json!({ - "emailAddress": {"address": format!("u{i}@x.com")}, - "type": "required" - })); - } - v["attendees"] = Value::Array(atts); - let conv = convert_event(&v, None).unwrap(); - let map = conv.data["participants"].as_object().unwrap(); - assert!(map.contains_key("att-01")); - assert!(map.contains_key("att-11")); - } - - #[test] - fn original_start_time_zone_is_preserved() { - let mut v = sample(); - v["originalStartTimeZone"] = Value::from("Pacific Standard Time"); - v["start"]["timeZone"] = Value::from("UTC"); - let conv = convert_event(&v, None).unwrap(); - assert_eq!(conv.data["timeZone"], "America/Los_Angeles"); - assert_eq!(conv.data["start"], "2026-05-27T03:00:00"); - } - - #[test] - fn utc_times_are_converted_into_the_original_zone() { - let mut v = sample(); - v["originalStartTimeZone"] = Value::from("Europe/Paris"); - v["start"]["dateTime"] = Value::from("2025-03-12T07:30:00.0000000"); - v["start"]["timeZone"] = Value::from("UTC"); - v["end"]["dateTime"] = Value::from("2025-03-12T08:00:00.0000000"); - v["end"]["timeZone"] = Value::from("UTC"); - let conv = convert_event(&v, None).unwrap(); - assert_eq!(conv.data["start"], "2025-03-12T08:30:00"); - assert_eq!(conv.data["timeZone"], "Europe/Paris"); - assert_eq!(conv.data["duration"], "PT30M"); - } - - #[test] - fn summer_event_shifts_by_two_hours_in_paris() { - let mut v = sample(); - v["originalStartTimeZone"] = Value::from("Romance Standard Time"); - v["start"]["dateTime"] = Value::from("2025-07-15T06:30:00.0000000"); - v["start"]["timeZone"] = Value::from("UTC"); - v["end"]["dateTime"] = Value::from("2025-07-15T07:30:00.0000000"); - v["end"]["timeZone"] = Value::from("UTC"); - let conv = convert_event(&v, None).unwrap(); - assert_eq!(conv.data["start"], "2025-07-15T08:30:00"); - assert_eq!(conv.data["timeZone"], "Europe/Paris"); - } - - #[test] - fn unresolvable_microsoft_zone_falls_back_to_utc() { - let mut v = sample(); - v["originalStartTimeZone"] = Value::from("tzone://Microsoft/Custom"); - v["start"]["dateTime"] = Value::from("2025-03-12T07:30:00.0000000"); - v["start"]["timeZone"] = Value::from("UTC"); - let conv = convert_event(&v, None).unwrap(); - assert_eq!(conv.data["start"], "2025-03-12T07:30:00"); - assert_eq!(conv.data["timeZone"], "Etc/UTC"); - } - - #[test] - fn all_day_keeps_date_and_omits_timezone() { - let mut v = sample(); - v["isAllDay"] = Value::Bool(true); - v["originalStartTimeZone"] = Value::from("Europe/Paris"); - v["start"]["dateTime"] = Value::from("2025-03-12T00:00:00.0000000"); - v["start"]["timeZone"] = Value::from("UTC"); - let conv = convert_event(&v, None).unwrap(); - assert_eq!(conv.data["start"], "2025-03-12T00:00:00"); - assert_eq!(conv.data["showWithoutTime"], true); - assert!(conv.data.get("timeZone").is_none()); - } - - #[test] - fn all_day_utc_instant_east_of_utc_keeps_its_local_date() { - let mut v = sample(); - v["isAllDay"] = Value::Bool(true); - v["originalStartTimeZone"] = Value::from("Romance Standard Time"); - v["start"]["dateTime"] = Value::from("2026-10-23T22:00:00.0000000"); - v["end"]["dateTime"] = Value::from("2026-10-25T23:00:00.0000000"); - let conv = convert_event(&v, None).unwrap(); - assert_eq!(conv.data["start"], "2026-10-24T00:00:00"); - assert_eq!(conv.data["duration"], "P2D"); - assert_eq!(conv.zone.as_deref(), Some("Europe/Paris")); - } - - #[test] - fn all_day_midnight_is_kept_whatever_the_zone() { - let mut v = sample(); - v["isAllDay"] = Value::Bool(true); - v["originalStartTimeZone"] = Value::from("New Zealand Standard Time"); - v["start"]["dateTime"] = Value::from("2026-11-12T00:00:00.0000000"); - v["end"]["dateTime"] = Value::from("2026-11-13T00:00:00.0000000"); - let conv = convert_event(&v, None).unwrap(); - assert_eq!(conv.data["start"], "2026-11-12T00:00:00"); - assert_eq!(conv.data["duration"], "P1D"); - } - - #[test] - fn duration_h_and_s_no_minutes_includes_zero_m() { - assert_eq!( - duration_iso8601("2026-05-27T10:00:00", "2026-05-27T11:00:05").as_deref(), - Some("PT1H0M5S") - ); - } - - #[test] - fn fractional_seconds_are_stripped_from_timestamps() { - let mut v = sample(); - v["createdDateTime"] = Value::from("2026-05-01T08:00:00.0000000Z"); - v["lastModifiedDateTime"] = Value::from("2026-05-02T08:00:00.123Z"); - let conv = convert_event(&v, None).unwrap(); - assert_eq!(conv.data["created"], "2026-05-01T08:00:00Z"); - assert_eq!(conv.data["updated"], "2026-05-02T08:00:00Z"); - } - - #[test] - fn organizer_calendar_address_is_set_alongside_participants() { - let conv = convert_event(&sample(), None).unwrap(); - assert_eq!(conv.data["organizerCalendarAddress"], "mailto:alice@x.com"); - } - - #[test] - fn recurrence_emits_singular_recurrence_rule_object() { - let mut v = sample(); - v["recurrence"] = json!({ - "pattern": {"type": "daily", "interval": 1}, - "range": {"type": "noEnd"} - }); - let conv = convert_event(&v, None).unwrap(); - assert!(conv.data.get("recurrenceRules").is_none()); - let rule = &conv.data["recurrenceRule"]; - assert!(rule.is_object()); - assert_eq!(rule["@type"], "RecurrenceRule"); - assert_eq!(rule["frequency"], "daily"); - } - - #[test] - fn location_with_address_only_uses_name_property() { - let mut v = sample(); - v["locations"] = json!([{ - "address": {"street": "1 Infinite Loop", "city": "Cupertino"} - }]); - let conv = convert_event(&v, None).unwrap(); - let locs = conv.data["locations"].as_object().unwrap(); - let only = locs.values().next().unwrap(); - assert_eq!(only["@type"], "Location"); - assert_eq!(only["name"], "1 Infinite Loop, Cupertino"); - assert!(only.get("description").is_none()); - } -} diff --git a/src/exchange_graph/client.rs b/src/exchange_graph/client.rs deleted file mode 100644 index c2c9e86..0000000 --- a/src/exchange_graph/client.rs +++ /dev/null @@ -1,499 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::sync::Arc; -use std::sync::Mutex; -use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; -use std::time::{Duration, Instant}; - -use serde_json::Value; -use ureq::Agent; -use ureq::config::{Config, RedirectAuthHeaders}; -use ureq::tls::{RootCerts, TlsConfig}; -use ureq::{ResponseExt, http::Uri}; - -use crate::exchange_graph::error::GraphError; -use crate::exchange_graph::retry::{HttpClass, classify_http_status, is_throttled}; -use crate::jmap::http::{RetryPolicy, cross_host, retry_after_header}; -use crate::jmap::retry::{self, RateLimitState}; -use crate::logging::{HttpCall, LEVEL_BODIES, LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; - -const MAX_BODY: u64 = 256 * 1024 * 1024; -const LONG_RETRY_THRESHOLD: Duration = Duration::from_secs(10); - -#[derive(Debug, Clone, Copy)] -pub enum Accept { - Json, - Text, - Binary, -} - -impl Accept { - fn header_value(self) -> &'static str { - match self { - Accept::Json => "application/json", - Accept::Text => "text/plain", - Accept::Binary => "application/octet-stream", - } - } -} - -#[derive(Debug, Clone)] -pub struct GraphResponse { - pub status: u16, - pub body: Vec, - pub content_type: Option, -} - -impl GraphResponse { - pub fn as_str(&self) -> Result<&str, GraphError> { - std::str::from_utf8(&self.body) - .map_err(|e| GraphError::Malformed(format!("response is not utf-8: {e}"))) - } - - pub fn json(&self) -> Result { - serde_json::from_slice(&self.body) - .map_err(|e| GraphError::Malformed(format!("response is not valid json: {e}"))) - } -} - -struct Inner { - agent: Agent, - bearer: Mutex, - retry: RetryPolicy, - rate_limit: RateLimitState, - log_level: AtomicU8, - retries_total: AtomicU64, - retry_after_sleeps: AtomicU64, - requests_total: AtomicU64, - user_agent: String, -} - -#[derive(Clone)] -pub struct GraphClient { - inner: Arc, -} - -enum Attempt { - Ok { - status: u16, - body: Vec, - retry_after: Option, - content_type: Option, - }, - Transport(GraphError), -} - -impl GraphClient { - pub fn new(bearer: String, retry: RetryPolicy, allow_invalid_certs: bool) -> GraphClient { - let config: Config = retry - .apply_timeouts(Config::builder()) - .http_status_as_error(false) - .redirect_auth_headers(RedirectAuthHeaders::SameHost) - .tls_config( - TlsConfig::builder() - .unversioned_rustls_crypto_provider(std::sync::Arc::new( - rustls::crypto::aws_lc_rs::default_provider(), - )) - .root_certs(RootCerts::PlatformVerifier) - .disable_verification(allow_invalid_certs) - .build(), - ) - .build(); - GraphClient { - inner: Arc::new(Inner { - agent: config.new_agent(), - bearer: Mutex::new(bearer), - retry, - rate_limit: RateLimitState::new(), - log_level: AtomicU8::new(LEVEL_DEFAULT), - retries_total: AtomicU64::new(0), - retry_after_sleeps: AtomicU64::new(0), - requests_total: AtomicU64::new(0), - user_agent: format!("inbuxa-migrate/{}", env!("CARGO_PKG_VERSION")), - }), - } - } - - pub fn set_bearer(&self, bearer: String) { - let mut g = match self.inner.bearer.lock() { - Ok(g) => g, - Err(p) => p.into_inner(), - }; - *g = bearer; - } - - pub fn set_logger(&self, logger: Logger) { - self.inner - .log_level - .store(logger.level(), Ordering::Relaxed); - } - - fn logger(&self) -> Logger { - Logger::new(self.inner.log_level.load(Ordering::Relaxed)) - } - - fn bearer(&self) -> String { - match self.inner.bearer.lock() { - Ok(g) => g.clone(), - Err(p) => p.into_inner().clone(), - } - } - - fn auth_header(&self) -> String { - format!("Bearer {}", self.bearer()) - } - - pub fn retries_observed(&self) -> u64 { - self.inner.retries_total.load(Ordering::Relaxed) - } - - pub fn retry_after_sleeps(&self) -> u64 { - self.inner.retry_after_sleeps.load(Ordering::Relaxed) - } - - pub fn requests_observed(&self) -> u64 { - self.inner.requests_total.load(Ordering::Relaxed) - } - - pub fn rate_limit(&self) -> &RateLimitState { - &self.inner.rate_limit - } - - pub fn get(&self, url: &str, accept: Accept) -> Result { - self.execute("GET", url, accept, &[]) - } - - pub fn get_with_prefer( - &self, - url: &str, - accept: Accept, - prefer: &[&str], - ) -> Result { - self.execute("GET", url, accept, prefer) - } - - pub fn get_json(&self, url: &str) -> Result { - self.get(url, Accept::Json).and_then(|r| r.json()) - } - - pub fn get_json_with_prefer(&self, url: &str, prefer: &[&str]) -> Result { - self.get_with_prefer(url, Accept::Json, prefer) - .and_then(|r| r.json()) - } - - fn execute( - &self, - method: &str, - url: &str, - accept: Accept, - extra_prefer: &[&str], - ) -> Result { - let policy = self.inner.retry; - let logger = self.logger(); - let mut attempt: u32 = 0; - loop { - self.inner.rate_limit.cooldown().wait(); - self.inner.requests_total.fetch_add(1, Ordering::Relaxed); - let outcome = self.one_attempt(method, url, accept, extra_prefer); - match outcome { - Attempt::Ok { - status, - body, - retry_after, - content_type, - } => { - let class = classify_http_status(status); - match class { - HttpClass::Success => { - self.inner.rate_limit.on_success(); - return Ok(GraphResponse { - status, - body, - content_type, - }); - } - HttpClass::Vanished => { - return Err(GraphError::Vanished); - } - HttpClass::Auth => { - return Err(GraphError::Auth(format!( - "server returned {status}: {}", - truncate(&body) - ))); - } - HttpClass::Fatal => { - return Err(GraphError::HttpStatus { - status, - body: truncate(&body), - }); - } - HttpClass::Retryable => { - attempt += 1; - self.inner.retries_total.fetch_add(1, Ordering::Relaxed); - if attempt > policy.max_retries { - return Err(GraphError::RetriesExhausted(format!( - "{method} {url} kept returning http {status}" - ))); - } - if retry_after.is_some() { - self.inner - .retry_after_sleeps - .fetch_add(1, Ordering::Relaxed); - } - let delay = if is_throttled(status) { - self.inner.rate_limit.on_throttle(&policy, retry_after) - } else { - match retry_after { - Some(d) => d, - None => retry::backoff_delay(&policy, attempt), - } - }; - if delay >= LONG_RETRY_THRESHOLD { - logger.warn(&format!( - "graph rate-limited (http {status}); waiting {} before retry {}/{} (shared level {})", - format_retry_wait(delay), - attempt, - policy.max_retries, - self.inner.rate_limit.level(), - )); - } - self.log_retry( - &logger, - RetryLog { - method, - url, - attempt, - delay, - status, - body: &body, - }, - ); - std::thread::sleep(delay); - } - } - } - Attempt::Transport(err) => { - if matches!(err, GraphError::Connect(_)) { - return Err(err); - } - attempt += 1; - self.inner.retries_total.fetch_add(1, Ordering::Relaxed); - if attempt > policy.max_retries { - return Err(GraphError::RetriesExhausted(format!( - "{method} {url}: {err}" - ))); - } - let delay = retry::backoff_delay(&policy, attempt); - if delay >= LONG_RETRY_THRESHOLD { - logger.warn(&format!( - "transient transport failure ({err}); waiting {} before retry {}/{}", - format_retry_wait(delay), - attempt, - policy.max_retries - )); - } - std::thread::sleep(delay); - } - } - } - } - - fn one_attempt( - &self, - method: &str, - url: &str, - accept: Accept, - extra_prefer: &[&str], - ) -> Attempt { - let mut req = match method { - "GET" => self.inner.agent.get(url), - other => { - return Attempt::Transport(GraphError::Connect(format!( - "unsupported method {other} (graph importer is read-only)" - ))); - } - }; - req = req - .header("Authorization", self.auth_header()) - .header("Accept", accept.header_value()) - .header("User-Agent", self.inner.user_agent.as_str()); - req = req.header("Prefer", "IdType=\"ImmutableId\""); - for value in extra_prefer { - req = req.header("Prefer", *value); - } - let logger = self.logger(); - let started = Instant::now(); - match req.call() { - Ok(mut resp) => { - let status = resp.status().as_u16(); - let final_uri = resp.get_uri().clone(); - let retry_after = resp - .headers() - .get("retry-after") - .and_then(|v| v.to_str().ok()) - .and_then(retry_after_header); - let content_type = resp - .headers() - .get("content-type") - .and_then(|v| v.to_str().ok()) - .map(str::to_owned); - match resp.body_mut().with_config().limit(MAX_BODY).read_to_vec() { - Ok(bytes) => { - warn_on_redirect(&logger, method, url, &final_uri); - logger.trace_http(&HttpCall { - proto: "Graph", - method, - url, - status, - elapsed: started.elapsed(), - note: None, - request: None, - request_type: None, - response: &bytes, - response_type: content_type.as_deref(), - }); - Attempt::Ok { - status, - body: bytes, - retry_after, - content_type, - } - } - Err(e) => Attempt::Transport(GraphError::Transport(format!( - "reading response body: {e}" - ))), - } - } - Err(e) => { - let err = map_ureq_error(e); - logger.trace_http_error("Graph", method, url, &err.to_string(), started.elapsed()); - Attempt::Transport(err) - } - } - } - - fn log_retry(&self, logger: &Logger, info: RetryLog<'_>) { - let RetryLog { - method, - url, - attempt, - delay, - status, - body, - } = info; - if logger.enabled(LEVEL_BODIES) { - eprintln!( - "retry {}/{} {} {} after {:?} (http {status}) body={}", - attempt, - self.inner.retry.max_retries, - method, - url, - delay, - truncate(body) - ); - } else if logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "retry {}/{} (http {status})", - attempt, self.inner.retry.max_retries - ); - } - } -} - -struct RetryLog<'a> { - method: &'a str, - url: &'a str, - attempt: u32, - delay: Duration, - status: u16, - body: &'a [u8], -} - -fn warn_on_redirect(logger: &Logger, method: &str, requested: &str, final_uri: &Uri) { - if requested.ends_with("/content") { - return; - } - let landed = final_uri.to_string(); - if cross_host(requested, &landed) { - let host = url::Url::parse(&landed) - .ok() - .and_then(|u| u.host_str().map(str::to_owned)) - .unwrap_or_else(|| "another host".to_owned()); - logger.warn(&format!( - "{method} {requested} was redirected across hosts to {host}; verify the Graph endpoint host is reachable directly without redirection" - )); - } -} - -fn map_ureq_error(err: ureq::Error) -> GraphError { - match err { - ureq::Error::Io(e) => GraphError::Transport(format!("io: {e}")), - ureq::Error::Timeout(t) => GraphError::Transport(format!("timeout: {t}")), - ureq::Error::HostNotFound => GraphError::Transport("host not found".to_owned()), - ureq::Error::ConnectionFailed => GraphError::Transport("connection failed".to_owned()), - ureq::Error::BodyStalled => GraphError::Transport("body stalled".to_owned()), - ureq::Error::Tls(m) => GraphError::Transport(format!("tls: {m}")), - ureq::Error::TooManyRedirects => GraphError::Connect("too many redirects".to_owned()), - ureq::Error::RedirectFailed => GraphError::Connect("redirect failed".to_owned()), - ureq::Error::TlsRequired => { - GraphError::Connect("server requires TLS but transport is unsecured".to_owned()) - } - other => GraphError::Connect(other.to_string()), - } -} - -fn truncate(body: &[u8]) -> String { - let text = String::from_utf8_lossy(body); - if text.len() <= 512 { - return text.into_owned(); - } - let end = text.floor_char_boundary(512); - format!("{}...", &text[..end]) -} - -fn format_retry_wait(d: Duration) -> String { - let total = d.as_secs(); - if total < 60 { - format!("{total}s") - } else if total < 3600 { - format!("{}m{:02}s", total / 60, total % 60) - } else { - format!( - "{}h{:02}m{:02}s", - total / 3600, - (total % 3600) / 60, - total % 60 - ) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn defaults_construct() { - let c = GraphClient::new("token".to_owned(), RetryPolicy::new(3), false); - assert_eq!(c.retries_observed(), 0); - assert_eq!(c.retry_after_sleeps(), 0); - assert_eq!(c.requests_observed(), 0); - assert_eq!(c.auth_header(), "Bearer token"); - } - - #[test] - fn bearer_can_be_swapped_at_runtime() { - let c = GraphClient::new("old".to_owned(), RetryPolicy::new(0), false); - c.set_bearer("new".to_owned()); - assert_eq!(c.auth_header(), "Bearer new"); - } - - #[test] - fn accept_header_values_are_what_the_spec_requires() { - assert_eq!(Accept::Json.header_value(), "application/json"); - assert_eq!(Accept::Text.header_value(), "text/plain"); - } -} diff --git a/src/exchange_graph/contact_map.rs b/src/exchange_graph/contact_map.rs deleted file mode 100644 index e9b8db7..0000000 --- a/src/exchange_graph/contact_map.rs +++ /dev/null @@ -1,669 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use chrono::Datelike; -use serde_json::{Map, Value, json}; - -use crate::exchange::date::{is_outlook_no_date, nearest_midnight}; -use crate::exchange_graph::error::GraphError; - -#[derive(Debug, Clone)] -pub struct ConvertedContact { - pub uid: String, - pub data: Value, -} - -pub fn convert_contact(graph_contact: &Value) -> Result { - let graph_id = graph_contact - .get("id") - .and_then(Value::as_str) - .ok_or_else(|| GraphError::Malformed("contact has no id".to_owned()))?; - let uid = synthetic_uid(graph_id); - - let mut card = Map::new(); - card.insert("@type".to_owned(), Value::from("Card")); - card.insert("version".to_owned(), Value::from("1.0")); - card.insert("uid".to_owned(), Value::from(uid.clone())); - card.insert("kind".to_owned(), Value::from("individual")); - - let mut name_object = Map::new(); - name_object.insert("@type".to_owned(), Value::from("Name")); - if let Some(display) = graph_contact.get("displayName").and_then(Value::as_str) - && !display.is_empty() - { - name_object.insert("full".to_owned(), Value::from(display.to_owned())); - } - let name_slots = [ - ("title", graph_contact.get("title").and_then(Value::as_str)), - ( - "given", - graph_contact.get("givenName").and_then(Value::as_str), - ), - ( - "given2", - graph_contact.get("middleName").and_then(Value::as_str), - ), - ( - "surname", - graph_contact.get("surname").and_then(Value::as_str), - ), - ( - "generation", - graph_contact.get("generation").and_then(Value::as_str), - ), - ]; - let mut components: Vec = Vec::new(); - let mut slot_index: std::collections::HashMap<&'static str, usize> = - std::collections::HashMap::new(); - for (kind, value) in name_slots { - let Some(v) = value else { continue }; - if v.is_empty() { - continue; - } - slot_index.insert(kind, components.len()); - components.push(json!({"@type": "NameComponent", "kind": kind, "value": v})); - } - if !components.is_empty() { - name_object.insert("components".to_owned(), Value::Array(components)); - } - if name_object.contains_key("full") || name_object.contains_key("components") { - card.insert("name".to_owned(), Value::Object(name_object)); - } - - if let Some(nick) = graph_contact.get("nickName").and_then(Value::as_str) - && !nick.is_empty() - { - let mut nicks = Map::new(); - nicks.insert( - "nick-1".to_owned(), - json!({"@type": "Nickname", "name": nick}), - ); - card.insert("nicknames".to_owned(), Value::Object(nicks)); - } - - let mut emails = Map::new(); - if let Some(addrs) = graph_contact - .get("emailAddresses") - .and_then(Value::as_array) - { - for (i, e) in addrs.iter().enumerate() { - let Some(addr) = e.get("address").and_then(Value::as_str) else { - continue; - }; - if addr.is_empty() { - continue; - } - let key = format!("email-{}", i + 1); - let mut entry = Map::new(); - entry.insert("@type".to_owned(), Value::from("EmailAddress")); - entry.insert("address".to_owned(), Value::from(addr.to_owned())); - if i == 0 { - entry.insert("pref".to_owned(), Value::from(1)); - } - emails.insert(key, Value::Object(entry)); - } - } - if !emails.is_empty() { - card.insert("emails".to_owned(), Value::Object(emails)); - } - - let mut phones = Map::new(); - push_phones( - &mut phones, - graph_contact - .get("businessPhones") - .and_then(Value::as_array), - "work", - ); - push_phones( - &mut phones, - graph_contact.get("homePhones").and_then(Value::as_array), - "private", - ); - if let Some(mobile) = graph_contact.get("mobilePhone").and_then(Value::as_str) - && !mobile.is_empty() - { - let key = format!("phone-{}", phones.len() + 1); - phones.insert( - key, - json!({ - "@type": "Phone", - "number": mobile, - "features": {"mobile": true, "voice": true} - }), - ); - } - if !phones.is_empty() { - card.insert("phones".to_owned(), Value::Object(phones)); - } - - let mut addresses = Map::new(); - push_address( - &mut addresses, - graph_contact.get("homeAddress"), - Some("private"), - ); - push_address( - &mut addresses, - graph_contact.get("businessAddress"), - Some("work"), - ); - push_address(&mut addresses, graph_contact.get("otherAddress"), None); - if !addresses.is_empty() { - card.insert("addresses".to_owned(), Value::Object(addresses)); - } - - let mut organizations = Map::new(); - let has_company = graph_contact - .get("companyName") - .and_then(Value::as_str) - .is_some_and(|s| !s.is_empty()); - let has_dept = graph_contact - .get("department") - .and_then(Value::as_str) - .is_some_and(|s| !s.is_empty()); - let has_office = graph_contact - .get("officeLocation") - .and_then(Value::as_str) - .is_some_and(|s| !s.is_empty()); - let has_title = graph_contact - .get("jobTitle") - .and_then(Value::as_str) - .is_some_and(|s| !s.is_empty()); - if has_company || has_dept || has_office { - let mut org = Map::new(); - org.insert("@type".to_owned(), Value::from("Organization")); - if let Some(name) = graph_contact.get("companyName").and_then(Value::as_str) - && !name.is_empty() - { - org.insert("name".to_owned(), Value::from(name.to_owned())); - } - let mut units: Vec = Vec::new(); - if let Some(dept) = graph_contact.get("department").and_then(Value::as_str) - && !dept.is_empty() - { - units.push(json!({"@type": "OrgUnit", "name": dept})); - } - if let Some(office) = graph_contact.get("officeLocation").and_then(Value::as_str) - && !office.is_empty() - { - units.push(json!({"@type": "OrgUnit", "name": office})); - } - if !units.is_empty() { - org.insert("units".to_owned(), Value::Array(units)); - } - organizations.insert("org-1".to_owned(), Value::Object(org)); - } - if has_title - && let Some(title) = graph_contact.get("jobTitle").and_then(Value::as_str) - && !title.is_empty() - { - let mut titles = Map::new(); - let mut title_obj = Map::new(); - title_obj.insert("@type".to_owned(), Value::from("Title")); - title_obj.insert("name".to_owned(), Value::from(title.to_owned())); - if organizations.contains_key("org-1") { - title_obj.insert("organizationId".to_owned(), Value::from("org-1")); - } - titles.insert("title-1".to_owned(), Value::Object(title_obj)); - card.insert("titles".to_owned(), Value::Object(titles)); - } - let has_org_one = organizations.contains_key("org-1"); - if !organizations.is_empty() { - card.insert("organizations".to_owned(), Value::Object(organizations)); - } - - if let Some(notes) = graph_contact.get("personalNotes").and_then(Value::as_str) - && !notes.is_empty() - { - let mut map = Map::new(); - map.insert("note-1".to_owned(), json!({"@type": "Note", "note": notes})); - card.insert("notes".to_owned(), Value::Object(map)); - } - - let mut localizations = Map::new(); - let mut ja = Map::new(); - if let Some(name) = graph_contact.get("yomiCompanyName").and_then(Value::as_str) - && !name.is_empty() - && has_org_one - { - ja.insert( - "organizations/org-1/name".to_owned(), - Value::from(name.to_owned()), - ); - } - if let Some(name) = graph_contact.get("yomiGivenName").and_then(Value::as_str) - && !name.is_empty() - && let Some(idx) = slot_index.get("given") - { - ja.insert( - format!("name/components/{idx}/value"), - Value::from(name.to_owned()), - ); - } - if let Some(name) = graph_contact.get("yomiSurname").and_then(Value::as_str) - && !name.is_empty() - && let Some(idx) = slot_index.get("surname") - { - ja.insert( - format!("name/components/{idx}/value"), - Value::from(name.to_owned()), - ); - } - if !ja.is_empty() { - localizations.insert("ja".to_owned(), Value::Object(ja)); - } - if !localizations.is_empty() { - card.insert("localizations".to_owned(), Value::Object(localizations)); - } - - if let Some(birthday) = graph_contact.get("birthday").and_then(Value::as_str) - && let Some(date) = parse_partial_date(birthday) - { - let mut anniversaries = Map::new(); - anniversaries.insert( - "anniversary-1".to_owned(), - json!({ - "@type": "Anniversary", - "kind": "birth", - "date": date, - }), - ); - card.insert("anniversaries".to_owned(), Value::Object(anniversaries)); - } - - if let Some(link) = graph_contact - .get("businessHomePage") - .and_then(Value::as_str) - && !link.is_empty() - { - let mut links = Map::new(); - links.insert( - "link-1".to_owned(), - json!({"@type": "Link", "uri": link, "contexts": {"work": true}}), - ); - card.insert("links".to_owned(), Value::Object(links)); - } - - if let Some(cats) = graph_contact.get("categories").and_then(Value::as_array) { - let mut keywords = Map::new(); - for cat in cats.iter().filter_map(Value::as_str) { - if !cat.trim().is_empty() { - keywords.insert(cat.to_ascii_lowercase(), Value::Bool(true)); - } - } - if !keywords.is_empty() { - card.insert("keywords".to_owned(), Value::Object(keywords)); - } - } - - if let Some(ims) = graph_contact.get("imAddresses").and_then(Value::as_array) { - let mut services = Map::new(); - for (i, addr) in ims - .iter() - .filter_map(Value::as_str) - .map(str::trim) - .filter(|a| !a.is_empty()) - .enumerate() - { - let slot = if has_uri_scheme(addr) { "uri" } else { "user" }; - services.insert( - (i + 1).to_string(), - json!({"@type": "OnlineService", slot: addr}), - ); - } - if !services.is_empty() { - card.insert("onlineServices".to_owned(), Value::Object(services)); - } - } - - if let Some(created) = graph_contact.get("createdDateTime").and_then(Value::as_str) { - card.insert( - "created".to_owned(), - Value::from(normalise_utc_datetime(created)), - ); - } - if let Some(updated) = graph_contact - .get("lastModifiedDateTime") - .and_then(Value::as_str) - { - card.insert( - "updated".to_owned(), - Value::from(normalise_utc_datetime(updated)), - ); - } - - Ok(ConvertedContact { - uid, - data: Value::Object(card), - }) -} - -fn has_uri_scheme(value: &str) -> bool { - let Some((scheme, rest)) = value.split_once(':') else { - return false; - }; - !rest.is_empty() - && !scheme.is_empty() - && scheme.starts_with(|c: char| c.is_ascii_alphabetic()) - && scheme - .chars() - .all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.')) -} - -pub fn synthetic_uid(graph_id: &str) -> String { - let hash = blake3::hash(graph_id.as_bytes()); - format!("inbuxa-migrate-graph-{}", hash.to_hex()) -} - -fn normalise_utc_datetime(raw: &str) -> String { - let trailing_z = raw.ends_with('Z'); - let trimmed = raw.trim_end_matches('Z'); - let base = match trimmed.find('.') { - Some(dot) => &trimmed[..dot], - None => trimmed, - }; - if trailing_z { - format!("{base}Z") - } else { - base.to_owned() - } -} - -fn parse_partial_date(raw: &str) -> Option { - if is_outlook_no_date(raw) { - return None; - } - let date = nearest_midnight(raw, None)?; - let mut obj = Map::new(); - obj.insert("@type".to_owned(), Value::from("PartialDate")); - if date.year() > 0 { - obj.insert("year".to_owned(), Value::from(date.year())); - } - obj.insert("month".to_owned(), Value::from(date.month())); - obj.insert("day".to_owned(), Value::from(date.day())); - Some(Value::Object(obj)) -} - -fn push_phones(target: &mut Map, list: Option<&Vec>, context: &str) { - let Some(list) = list else { return }; - for n in list.iter().filter_map(Value::as_str) { - if n.is_empty() { - continue; - } - let key = format!("phone-{}", target.len() + 1); - target.insert( - key, - json!({ - "@type": "Phone", - "number": n, - "contexts": {context: true} - }), - ); - } -} - -fn push_address(target: &mut Map, source: Option<&Value>, context: Option<&str>) { - let Some(source) = source else { return }; - let Some(map) = source.as_object() else { - return; - }; - let street = map - .get("street") - .and_then(Value::as_str) - .filter(|s| !s.is_empty()); - let component_fields: Vec<(&str, &str)> = [ - ("city", "locality"), - ("state", "region"), - ("postalCode", "postcode"), - ("countryOrRegion", "country"), - ] - .into_iter() - .filter_map(|(graph_key, jsc_kind)| { - map.get(graph_key) - .and_then(Value::as_str) - .filter(|s| !s.is_empty()) - .map(|s| (jsc_kind, s)) - }) - .collect(); - if street.is_none() && component_fields.is_empty() { - return; - } - let mut address = Map::new(); - address.insert("@type".to_owned(), Value::from("Address")); - if let Some(context) = context { - address.insert("contexts".to_owned(), json!({context: true})); - } - if let Some(s) = street { - address.insert("full".to_owned(), Value::from(s.to_owned())); - } - if !component_fields.is_empty() { - let components: Vec = component_fields - .iter() - .map(|(kind, value)| json!({"@type": "AddressComponent", "kind": kind, "value": value})) - .collect(); - address.insert("components".to_owned(), Value::Array(components)); - } - let key = format!("addr-{}", target.len() + 1); - target.insert(key, Value::Object(address)); -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn minimal_contact_yields_card_with_synthetic_uid() { - let c = json!({"id": "GRAPH-1", "displayName": "Alice"}); - let conv = convert_contact(&c).unwrap(); - assert!(conv.uid.starts_with("inbuxa-migrate-graph-")); - assert_eq!(conv.data["@type"], "Card"); - assert_eq!(conv.data["kind"], "individual"); - assert_eq!(conv.data["name"]["full"], "Alice"); - } - - #[test] - fn synthetic_uid_is_stable_per_graph_id() { - let a = synthetic_uid("AAA"); - let b = synthetic_uid("AAA"); - let c = synthetic_uid("BBB"); - assert_eq!(a, b); - assert_ne!(a, c); - } - - #[test] - fn emails_map_with_first_as_pref() { - let c = json!({ - "id": "X", - "emailAddresses": [ - {"address": "a@x.com", "name": "Alice"}, - {"address": "b@x.com", "name": "Alice B"} - ] - }); - let conv = convert_contact(&c).unwrap(); - let emails = conv.data["emails"].as_object().unwrap(); - assert_eq!(emails["email-1"]["address"], "a@x.com"); - assert_eq!(emails["email-1"]["pref"], 1); - assert_eq!(emails["email-2"]["address"], "b@x.com"); - assert!(emails["email-2"].get("pref").is_none()); - } - - #[test] - fn phones_carry_context_per_source_field() { - let c = json!({ - "id": "X", - "businessPhones": ["+1-555-2000"], - "homePhones": ["+1-555-1000"], - "mobilePhone": "+1-555-3000" - }); - let conv = convert_contact(&c).unwrap(); - let phones = conv.data["phones"].as_object().unwrap(); - assert_eq!(phones.len(), 3); - let has_work = phones - .values() - .any(|v| v["contexts"]["work"] == Value::Bool(true)); - let has_private = phones - .values() - .any(|v| v["contexts"]["private"] == Value::Bool(true)); - let has_mobile = phones - .values() - .any(|v| v["features"]["mobile"] == Value::Bool(true)); - assert!(has_work && has_private && has_mobile); - } - - #[test] - fn addresses_map_three_contexts() { - let c = json!({ - "id": "X", - "homeAddress": {"city": "Home City"}, - "businessAddress": {"city": "Work City"}, - "otherAddress": {"city": "Other City"} - }); - let conv = convert_contact(&c).unwrap(); - let addresses = conv.data["addresses"].as_object().unwrap(); - assert_eq!(addresses.len(), 3); - let has_other_context = addresses - .values() - .any(|a| a.get("contexts").and_then(|c| c.get("other")).is_some()); - assert!(!has_other_context); - let without_context = addresses - .values() - .filter(|a| a.get("contexts").is_none()) - .count(); - assert_eq!(without_context, 1); - } - - #[test] - fn org_and_title_pair_into_separate_jscontact_objects() { - let c = json!({ - "id": "X", - "companyName": "Acme", - "department": "Engineering", - "officeLocation": "Building 9", - "jobTitle": "Architect" - }); - let conv = convert_contact(&c).unwrap(); - let orgs = conv.data["organizations"].as_object().unwrap(); - assert!(orgs.contains_key("org-1")); - assert_eq!(orgs["org-1"]["name"], "Acme"); - let titles = conv.data["titles"].as_object().unwrap(); - assert_eq!(titles["title-1"]["name"], "Architect"); - assert_eq!(titles["title-1"]["organizationId"], "org-1"); - } - - #[test] - fn yomi_fields_populate_japanese_localization() { - let c = json!({ - "id": "X", - "displayName": "Yamada Taro", - "givenName": "Taro", - "surname": "Yamada", - "companyName": "Acme", - "yomiCompanyName": "アクメ", - "yomiGivenName": "タロウ", - "yomiSurname": "ヤマダ" - }); - let conv = convert_contact(&c).unwrap(); - let ja = conv.data["localizations"]["ja"].as_object().unwrap(); - assert_eq!(ja["organizations/org-1/name"], "アクメ"); - } - - #[test] - fn birthday_stored_as_local_midnight_in_utc_keeps_its_day() { - for raw in ["1990-04-14T22:00:00Z", "1990-04-15T11:59:00Z", "1990-04-15"] { - let conv = convert_contact(&json!({"id": "X", "birthday": raw})).unwrap(); - assert_eq!( - conv.data["anniversaries"]["anniversary-1"]["date"], - json!({"@type": "PartialDate", "year": 1990, "month": 4, "day": 15}) - ); - } - } - - #[test] - fn outlook_no_date_birthday_is_dropped() { - let c = json!({"id": "X", "birthday": "4501-01-01T00:00:00Z"}); - let conv = convert_contact(&c).unwrap(); - assert!(conv.data.get("anniversaries").is_none()); - } - - #[test] - fn birthday_emits_partialdate_object() { - let c = json!({"id": "X", "birthday": "1980-04-12T00:00:00Z"}); - let conv = convert_contact(&c).unwrap(); - let ann = conv.data["anniversaries"].as_object().unwrap(); - let date = &ann["anniversary-1"]["date"]; - assert_eq!(date["@type"], "PartialDate"); - assert_eq!(date["year"], 1980); - assert_eq!(date["month"], 4); - assert_eq!(date["day"], 12); - assert_eq!(ann["anniversary-1"]["kind"], "birth"); - } - - #[test] - fn mobile_phone_uses_features_not_contexts() { - let c = json!({"id": "X", "mobilePhone": "+1-555-3000"}); - let conv = convert_contact(&c).unwrap(); - let phones = conv.data["phones"].as_object().unwrap(); - let phone = phones.values().next().unwrap(); - assert_eq!(phone["features"]["mobile"], true); - assert!(phone.get("contexts").is_none()); - } - - #[test] - fn generation_maps_to_generation_kind() { - let c = json!({"id": "X", "generation": "Jr."}); - let conv = convert_contact(&c).unwrap(); - let comps = conv.data["name"]["components"].as_array().unwrap(); - assert!( - comps - .iter() - .any(|c| c["kind"] == "generation" && c["value"] == "Jr.") - ); - } - - #[test] - fn only_jobtitle_does_not_emit_empty_organization() { - let c = json!({"id": "X", "jobTitle": "Architect"}); - let conv = convert_contact(&c).unwrap(); - assert!(conv.data.get("organizations").is_none()); - let titles = conv.data["titles"].as_object().unwrap(); - assert_eq!(titles["title-1"]["name"], "Architect"); - assert!(titles["title-1"].get("organizationId").is_none()); - } - - #[test] - fn related_contacts_are_dropped_without_resolvable_uids() { - let c = json!({ - "id": "X", - "manager": "alice@x.com", - "spouseName": "Bob", - "children": ["Kid A", "Kid B"] - }); - let conv = convert_contact(&c).unwrap(); - assert!(conv.data.get("relatedTo").is_none()); - } - - #[test] - fn nameless_contact_omits_name_object() { - let c = json!({"id": "X", "businessPhones": ["+1"]}); - let conv = convert_contact(&c).unwrap(); - assert!(conv.data.get("name").is_none()); - } - - #[test] - fn timestamps_strip_fractional_seconds() { - let c = json!({ - "id": "X", - "displayName": "Alice", - "createdDateTime": "2026-05-01T08:00:00.0000000Z", - "lastModifiedDateTime": "2026-05-02T08:00:00.123Z" - }); - let conv = convert_contact(&c).unwrap(); - assert_eq!(conv.data["created"], "2026-05-01T08:00:00Z"); - assert_eq!(conv.data["updated"], "2026-05-02T08:00:00Z"); - } -} diff --git a/src/exchange_graph/expand.rs b/src/exchange_graph/expand.rs deleted file mode 100644 index ef3cf3f..0000000 --- a/src/exchange_graph/expand.rs +++ /dev/null @@ -1,637 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use chrono::{Datelike, Duration, NaiveDate, Weekday}; -use serde_json::Value; - -const MAX_STEPS: usize = 200_000; - -pub fn expected_dates( - recurrence: &Value, - window_start: NaiveDate, - window_end: NaiveDate, -) -> Option> { - let pattern = recurrence.get("pattern")?; - let range = recurrence.get("range")?; - let start = date_of(range.get("startDate"))?; - let limit = date_of(range.get("endDate")); - let count = range - .get("numberOfOccurrences") - .and_then(Value::as_u64) - .filter(|n| *n > 0) - .map(|n| n as usize); - let interval = pattern - .get("interval") - .and_then(Value::as_u64) - .filter(|i| *i > 0) - .unwrap_or(1) as i64; - let kind = pattern.get("type").and_then(Value::as_str)?; - - let mut out: Vec = Vec::new(); - let mut emitted = 0usize; - let mut steps = 0usize; - let mut cursor = start; - - let push = |date: NaiveDate, out: &mut Vec, emitted: &mut usize| { - if date < start { - return; - } - if let Some(end) = limit - && date > end - { - return; - } - *emitted += 1; - if date >= window_start && date <= window_end { - out.push(date); - } - }; - - loop { - steps += 1; - if steps > MAX_STEPS { - break; - } - let done = match kind { - "daily" => { - push(cursor, &mut out, &mut emitted); - cursor = cursor.checked_add_signed(Duration::days(interval))?; - false - } - "weekly" => { - let days = weekdays(pattern); - if days.is_empty() { - return None; - } - let first = first_day_of_week(pattern); - let week_start = start_of_week(cursor, first); - for day in &days { - let offset = weekday_offset(first, *day); - let date = week_start + Duration::days(offset); - push(date, &mut out, &mut emitted); - } - cursor = week_start.checked_add_signed(Duration::weeks(interval))?; - false - } - "absoluteMonthly" => { - let dom = pattern.get("dayOfMonth").and_then(Value::as_u64)? as u32; - if let Some(date) = clamped_day(cursor.year(), cursor.month(), dom) { - push(date, &mut out, &mut emitted); - } - cursor = add_months(cursor.with_day(1)?, interval)?; - false - } - "relativeMonthly" => { - if let Some(date) = nth_weekday_of_month( - cursor.year(), - cursor.month(), - &weekdays(pattern), - pattern - .get("index") - .and_then(Value::as_str) - .unwrap_or("first"), - ) { - push(date, &mut out, &mut emitted); - } - cursor = add_months(cursor.with_day(1)?, interval)?; - false - } - "absoluteYearly" => { - let month = pattern.get("month").and_then(Value::as_u64)? as u32; - let dom = pattern.get("dayOfMonth").and_then(Value::as_u64)? as u32; - if let Some(date) = clamped_day(cursor.year(), month, dom) { - push(date, &mut out, &mut emitted); - } - cursor = NaiveDate::from_ymd_opt(cursor.year() + interval as i32, 1, 1)?; - false - } - "relativeYearly" => { - let month = pattern.get("month").and_then(Value::as_u64)? as u32; - if let Some(date) = nth_weekday_of_month( - cursor.year(), - month, - &weekdays(pattern), - pattern - .get("index") - .and_then(Value::as_str) - .unwrap_or("first"), - ) { - push(date, &mut out, &mut emitted); - } - cursor = NaiveDate::from_ymd_opt(cursor.year() + interval as i32, 1, 1)?; - false - } - _ => true, - }; - if done { - return None; - } - if let Some(n) = count - && emitted >= n - { - break; - } - if let Some(end) = limit - && cursor > end - { - break; - } - if cursor > window_end { - break; - } - } - - out.sort_unstable(); - out.dedup(); - Some(out) -} - -fn date_of(value: Option<&Value>) -> Option { - let raw = value?.as_str()?; - NaiveDate::parse_from_str(raw.split('T').next().unwrap_or(raw), "%Y-%m-%d").ok() -} - -fn weekdays(pattern: &Value) -> Vec { - pattern - .get("daysOfWeek") - .and_then(Value::as_array) - .map(|days| { - days.iter() - .filter_map(Value::as_str) - .filter_map(parse_weekday) - .collect() - }) - .unwrap_or_default() -} - -fn parse_weekday(name: &str) -> Option { - match name.to_ascii_lowercase().as_str() { - "monday" => Some(Weekday::Mon), - "tuesday" => Some(Weekday::Tue), - "wednesday" => Some(Weekday::Wed), - "thursday" => Some(Weekday::Thu), - "friday" => Some(Weekday::Fri), - "saturday" => Some(Weekday::Sat), - "sunday" => Some(Weekday::Sun), - _ => None, - } -} - -fn first_day_of_week(pattern: &Value) -> Weekday { - pattern - .get("firstDayOfWeek") - .and_then(Value::as_str) - .and_then(parse_weekday) - .unwrap_or(Weekday::Sun) -} - -fn weekday_offset(first: Weekday, day: Weekday) -> i64 { - let from = first.num_days_from_sunday() as i64; - let to = day.num_days_from_sunday() as i64; - (to - from).rem_euclid(7) -} - -fn start_of_week(date: NaiveDate, first: Weekday) -> NaiveDate { - let offset = weekday_offset(first, date.weekday()); - date - Duration::days(offset) -} - -fn clamped_day(year: i32, month: u32, day: u32) -> Option { - if !(1..=12).contains(&month) { - return None; - } - let last = last_day_of_month(year, month)?; - NaiveDate::from_ymd_opt(year, month, day.min(last)) -} - -fn last_day_of_month(year: i32, month: u32) -> Option { - let first_next = if month == 12 { - NaiveDate::from_ymd_opt(year + 1, 1, 1)? - } else { - NaiveDate::from_ymd_opt(year, month + 1, 1)? - }; - Some((first_next - Duration::days(1)).day()) -} - -fn add_months(date: NaiveDate, months: i64) -> Option { - let total = date.year() as i64 * 12 + (date.month() as i64 - 1) + months; - let year = (total.div_euclid(12)) as i32; - let month = (total.rem_euclid(12) + 1) as u32; - NaiveDate::from_ymd_opt(year, month, 1) -} - -fn nth_weekday_of_month(year: i32, month: u32, days: &[Weekday], index: &str) -> Option { - if days.is_empty() || !(1..=12).contains(&month) { - return None; - } - let last = last_day_of_month(year, month)?; - let mut matches: Vec = Vec::new(); - for day in 1..=last { - let date = NaiveDate::from_ymd_opt(year, month, day)?; - if days.contains(&date.weekday()) { - matches.push(date); - } - } - if matches.is_empty() { - return None; - } - match index.to_ascii_lowercase().as_str() { - "first" => matches.first().copied(), - "second" => matches.get(1).copied(), - "third" => matches.get(2).copied(), - "fourth" => matches.get(3).copied(), - "last" => matches.last().copied(), - _ => matches.first().copied(), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - fn day(s: &str) -> NaiveDate { - NaiveDate::parse_from_str(s, "%Y-%m-%d").unwrap() - } - - fn expand(recurrence: Value, from: &str, to: &str) -> Vec { - expected_dates(&recurrence, day(from), day(to)) - .expect("pattern is supported") - .into_iter() - .map(|d| d.to_string()) - .collect() - } - - #[test] - fn daily_numbered_stops_at_the_count() { - let dates = expand( - json!({ - "pattern": {"type": "daily", "interval": 1}, - "range": {"type": "numbered", "startDate": "2026-09-21", - "numberOfOccurrences": 6} - }), - "2026-01-01", - "2027-01-01", - ); - assert_eq!( - dates, - [ - "2026-09-21", - "2026-09-22", - "2026-09-23", - "2026-09-24", - "2026-09-25", - "2026-09-26" - ] - ); - } - - #[test] - fn weekly_interval_two_keeps_all_three_weekdays() { - let dates = expand( - json!({ - "pattern": {"type": "weekly", "interval": 2, - "daysOfWeek": ["monday", "wednesday", "friday"], - "firstDayOfWeek": "monday"}, - "range": {"type": "endDate", "startDate": "2026-09-16", - "endDate": "2026-10-03"} - }), - "2026-01-01", - "2027-01-01", - ); - assert_eq!( - dates, - [ - "2026-09-16", - "2026-09-18", - "2026-09-28", - "2026-09-30", - "2026-10-02" - ], - "interval 2 skips the week of the 21st entirely" - ); - } - - #[test] - fn relative_monthly_third_thursday() { - let dates = expand( - json!({ - "pattern": {"type": "relativeMonthly", "interval": 1, - "daysOfWeek": ["thursday"], "index": "third"}, - "range": {"type": "numbered", "startDate": "2026-09-17", - "numberOfOccurrences": 3} - }), - "2026-01-01", - "2027-06-01", - ); - assert_eq!(dates, ["2026-09-17", "2026-10-15", "2026-11-19"]); - } - - #[test] - fn relative_monthly_last_weekday() { - let dates = expand( - json!({ - "pattern": {"type": "relativeMonthly", "interval": 1, - "daysOfWeek": ["friday"], "index": "last"}, - "range": {"type": "numbered", "startDate": "2026-01-30", - "numberOfOccurrences": 3} - }), - "2026-01-01", - "2027-01-01", - ); - assert_eq!(dates, ["2026-01-30", "2026-02-27", "2026-03-27"]); - } - - #[test] - fn absolute_monthly_clamps_to_a_short_month() { - let dates = expand( - json!({ - "pattern": {"type": "absoluteMonthly", "interval": 1, "dayOfMonth": 31}, - "range": {"type": "numbered", "startDate": "2026-01-31", - "numberOfOccurrences": 3} - }), - "2026-01-01", - "2027-01-01", - ); - assert_eq!( - dates, - ["2026-01-31", "2026-02-28", "2026-03-31"], - "day 31 clamps to the last day of February" - ); - } - - #[test] - fn absolute_yearly_keeps_a_leap_day_series_on_the_28th_in_common_years() { - let dates = expand( - json!({ - "pattern": {"type": "absoluteYearly", "interval": 1, - "month": 2, "dayOfMonth": 29}, - "range": {"type": "numbered", "startDate": "2024-02-29", - "numberOfOccurrences": 3} - }), - "2020-01-01", - "2030-01-01", - ); - assert_eq!(dates, ["2024-02-29", "2025-02-28", "2026-02-28"]); - } - - #[test] - fn window_clips_output_but_the_count_still_tracks_the_whole_series() { - let dates = expand( - json!({ - "pattern": {"type": "daily", "interval": 1}, - "range": {"type": "numbered", "startDate": "2026-09-21", - "numberOfOccurrences": 6} - }), - "2026-09-24", - "2026-09-25", - ); - assert_eq!( - dates, - ["2026-09-24", "2026-09-25"], - "only in-window dates are returned, but occurrences 1-3 still counted \ - toward the numbered range" - ); - } - - #[test] - fn no_end_series_stops_at_the_window() { - let dates = expand( - json!({ - "pattern": {"type": "daily", "interval": 1}, - "range": {"type": "noEnd", "startDate": "2026-09-21"} - }), - "2026-09-21", - "2026-09-24", - ); - assert_eq!( - dates, - ["2026-09-21", "2026-09-22", "2026-09-23", "2026-09-24"] - ); - } - - #[test] - fn unknown_pattern_is_reported_rather_than_guessed() { - assert!( - expected_dates( - &json!({ - "pattern": {"type": "somethingNew", "interval": 1}, - "range": {"type": "noEnd", "startDate": "2026-01-01"} - }), - day("2026-01-01"), - day("2027-01-01"), - ) - .is_none() - ); - } -} - -pub fn graph_pattern_from_jscalendar(rule: &Value, series_start: NaiveDate) -> Option { - use serde_json::json; - let frequency = rule.get("frequency").and_then(Value::as_str)?; - let interval = rule.get("interval").and_then(Value::as_u64).unwrap_or(1); - let by_day: Vec = rule - .get("byDay") - .and_then(Value::as_array) - .map(|days| { - days.iter() - .filter_map(|d| d.get("day").and_then(Value::as_str)) - .filter_map(long_weekday) - .map(str::to_owned) - .collect() - }) - .unwrap_or_default(); - let by_month_day = rule - .get("byMonthDay") - .and_then(Value::as_array) - .and_then(|a| a.first()) - .and_then(Value::as_i64); - let by_month = rule - .get("byMonth") - .and_then(Value::as_array) - .and_then(|a| a.first()) - .and_then(|m| { - m.as_str() - .and_then(|s| s.parse::().ok()) - .or_else(|| m.as_i64()) - }); - let set_position = rule - .get("bySetPosition") - .and_then(Value::as_array) - .and_then(|a| a.first()) - .and_then(Value::as_i64); - - let mut pattern = serde_json::Map::new(); - pattern.insert("interval".to_owned(), Value::from(interval)); - let kind = match frequency { - "daily" => "daily", - "weekly" => { - pattern.insert("daysOfWeek".to_owned(), json!(by_day)); - "weekly" - } - "monthly" if set_position.is_some() && !by_day.is_empty() => { - pattern.insert("daysOfWeek".to_owned(), json!(by_day)); - pattern.insert("index".to_owned(), Value::from(index_name(set_position?))); - "relativeMonthly" - } - "monthly" => { - pattern.insert( - "dayOfMonth".to_owned(), - Value::from(by_month_day.unwrap_or(series_start.day() as i64)), - ); - "absoluteMonthly" - } - "yearly" if set_position.is_some() && !by_day.is_empty() => { - pattern.insert("daysOfWeek".to_owned(), json!(by_day)); - pattern.insert("index".to_owned(), Value::from(index_name(set_position?))); - pattern.insert( - "month".to_owned(), - Value::from(by_month.unwrap_or(series_start.month() as i64)), - ); - "relativeYearly" - } - "yearly" => { - pattern.insert( - "month".to_owned(), - Value::from(by_month.unwrap_or(series_start.month() as i64)), - ); - pattern.insert( - "dayOfMonth".to_owned(), - Value::from(by_month_day.unwrap_or(series_start.day() as i64)), - ); - "absoluteYearly" - } - _ => return None, - }; - pattern.insert("type".to_owned(), Value::from(kind)); - if let Some(first) = rule.get("firstDayOfWeek").and_then(Value::as_str) - && let Some(long) = long_weekday(first) - { - pattern.insert("firstDayOfWeek".to_owned(), Value::from(long)); - } - - let mut range = serde_json::Map::new(); - range.insert( - "startDate".to_owned(), - Value::from(series_start.format("%Y-%m-%d").to_string()), - ); - if let Some(count) = rule.get("count").and_then(Value::as_u64) { - range.insert("type".to_owned(), Value::from("numbered")); - range.insert("numberOfOccurrences".to_owned(), Value::from(count)); - } else if let Some(until) = rule.get("until").and_then(Value::as_str) { - range.insert("type".to_owned(), Value::from("endDate")); - range.insert( - "endDate".to_owned(), - Value::from(until.split('T').next().unwrap_or(until).to_owned()), - ); - } else { - range.insert("type".to_owned(), Value::from("noEnd")); - } - - Some(json!({"pattern": Value::Object(pattern), "range": Value::Object(range)})) -} - -fn long_weekday(short: &str) -> Option<&'static str> { - match short.to_ascii_lowercase().as_str() { - "mo" => Some("monday"), - "tu" => Some("tuesday"), - "we" => Some("wednesday"), - "th" => Some("thursday"), - "fr" => Some("friday"), - "sa" => Some("saturday"), - "su" => Some("sunday"), - _ => None, - } -} - -fn index_name(set_position: i64) -> &'static str { - match set_position { - 1 => "first", - 2 => "second", - 3 => "third", - 4 => "fourth", - -1 => "last", - _ => "first", - } -} - -#[cfg(test)] -mod roundtrip_tests { - use super::*; - use crate::exchange_graph::recurrence::convert_patterned_recurrence; - use serde_json::json; - - fn day(s: &str) -> NaiveDate { - NaiveDate::parse_from_str(s, "%Y-%m-%d").unwrap() - } - - fn roundtrip(graph: Value, start: &str, from: &str, to: &str) { - let direct = expected_dates(&graph, day(from), day(to)).expect("graph pattern expands"); - let rule = convert_patterned_recurrence(&graph).expect("converts to jscalendar"); - let back = graph_pattern_from_jscalendar(&rule, day(start)) - .expect("jscalendar rule maps back to a graph pattern"); - let viarule = expected_dates(&back, day(from), day(to)).expect("mapped pattern expands"); - assert_eq!( - direct, viarule, - "a stored rule must expand to the same dates as the Graph pattern it came from" - ); - } - - #[test] - fn stored_rules_expand_identically_to_their_graph_patterns() { - roundtrip( - json!({"pattern": {"type": "daily", "interval": 2}, - "range": {"type": "numbered", "startDate": "2026-09-21", - "numberOfOccurrences": 5}}), - "2026-09-21", - "2026-01-01", - "2027-01-01", - ); - roundtrip( - json!({"pattern": {"type": "weekly", "interval": 2, - "daysOfWeek": ["monday", "wednesday", "friday"], - "firstDayOfWeek": "monday"}, - "range": {"type": "endDate", "startDate": "2026-09-16", - "endDate": "2026-12-31"}}), - "2026-09-16", - "2026-01-01", - "2027-01-01", - ); - roundtrip( - json!({"pattern": {"type": "relativeMonthly", "interval": 1, - "daysOfWeek": ["thursday"], "index": "third"}, - "range": {"type": "numbered", "startDate": "2026-09-17", - "numberOfOccurrences": 6}}), - "2026-09-17", - "2026-01-01", - "2028-01-01", - ); - roundtrip( - json!({"pattern": {"type": "absoluteMonthly", "interval": 1, "dayOfMonth": 31}, - "range": {"type": "numbered", "startDate": "2026-01-31", - "numberOfOccurrences": 4}}), - "2026-01-31", - "2026-01-01", - "2027-01-01", - ); - roundtrip( - json!({"pattern": {"type": "absoluteYearly", "interval": 1, - "month": 11, "dayOfMonth": 29}, - "range": {"type": "noEnd", "startDate": "2026-11-29"}}), - "2026-11-29", - "2026-01-01", - "2031-01-01", - ); - roundtrip( - json!({"pattern": {"type": "relativeYearly", "interval": 1, - "daysOfWeek": ["monday"], "index": "last", "month": 5}, - "range": {"type": "noEnd", "startDate": "2026-05-25"}}), - "2026-05-25", - "2026-01-01", - "2031-01-01", - ); - } -} diff --git a/src/exchange_graph/oauth.rs b/src/exchange_graph/oauth.rs deleted file mode 100644 index bf66605..0000000 --- a/src/exchange_graph/oauth.rs +++ /dev/null @@ -1,565 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::io::{self, Write}; -use std::time::{Duration, Instant}; - -use encodify::base64::{Base64, Padding, URL_SAFE}; -use serde_json::Value; -use ureq::config::Config; -use ureq::tls::{RootCerts, TlsConfig}; - -use crate::exchange_graph::error::GraphError; - -pub const SCOPES: &str = "offline_access User.Read Mail.Read MailboxSettings.Read Calendars.Read Contacts.Read Files.Read"; - -const JWT_SEGMENT: Base64 = URL_SAFE.with_padding(Padding::Optional).any_alphabet(); - -pub fn default_authority(tenant: &str) -> String { - format!("https://login.microsoftonline.com/{tenant}") -} - -pub fn device_code_endpoint(authority: &str) -> String { - format!("{authority}/oauth2/v2.0/devicecode") -} - -pub fn token_endpoint(authority: &str) -> String { - format!("{authority}/oauth2/v2.0/token") -} - -#[derive(Debug, Clone)] -pub enum OAuthFlow { - PreAcquired { - token: String, - }, - DeviceCode { - authority: String, - client_id: String, - }, -} - -#[derive(Debug, Clone, Default)] -pub struct JwtClaims { - pub tenant_id: Option, - pub upn: Option, - pub name: Option, - pub exp: Option, -} - -pub fn decode_jwt_claims(token: &str) -> Option { - let mut parts = token.split('.'); - let _header = parts.next()?; - let payload = parts.next()?; - let bytes = JWT_SEGMENT.decode(payload).ok()?; - let value: Value = serde_json::from_slice(&bytes).ok()?; - Some(JwtClaims { - tenant_id: value.get("tid").and_then(Value::as_str).map(str::to_owned), - upn: value - .get("upn") - .and_then(Value::as_str) - .or_else(|| value.get("preferred_username").and_then(Value::as_str)) - .map(str::to_owned), - name: value.get("name").and_then(Value::as_str).map(str::to_owned), - exp: value.get("exp").and_then(Value::as_u64), - }) -} - -#[derive(Debug, Clone)] -pub struct AcquiredToken { - pub access_token: String, - pub refresh_token: Option, - pub expires_in: Option, - pub tenant_id: Option, - pub upn: Option, - pub name: Option, -} - -fn build_agent(allow_invalid_certs: bool) -> ureq::Agent { - let config: Config = Config::builder() - .http_status_as_error(false) - .tls_config( - TlsConfig::builder() - .unversioned_rustls_crypto_provider(std::sync::Arc::new( - rustls::crypto::aws_lc_rs::default_provider(), - )) - .root_certs(RootCerts::PlatformVerifier) - .disable_verification(allow_invalid_certs) - .build(), - ) - .build(); - config.new_agent() -} - -pub fn acquire(flow: &OAuthFlow, allow_invalid_certs: bool) -> Result { - match flow { - OAuthFlow::PreAcquired { token } => Ok(token_from_string(token.clone())), - OAuthFlow::DeviceCode { - authority, - client_id, - } => device_code_flow(authority, client_id, allow_invalid_certs), - } -} - -fn token_from_string(token: String) -> AcquiredToken { - let claims = decode_jwt_claims(&token).unwrap_or_default(); - AcquiredToken { - access_token: token, - refresh_token: None, - expires_in: None, - tenant_id: claims.tenant_id, - upn: claims.upn, - name: claims.name, - } -} - -#[derive(Debug, Clone)] -pub struct DeviceCodeResponse { - pub device_code: String, - pub user_code: String, - pub verification_uri: String, - pub interval: u64, - pub expires_in: u64, - pub message: Option, -} - -pub fn parse_device_code_response(json: &Value) -> Result { - let device_code = json - .get("device_code") - .and_then(Value::as_str) - .ok_or_else(|| GraphError::OAuth("device_code missing".to_owned()))? - .to_owned(); - let user_code = json - .get("user_code") - .and_then(Value::as_str) - .unwrap_or("?") - .to_owned(); - let verification_uri = json - .get("verification_uri") - .and_then(Value::as_str) - .unwrap_or("https://microsoft.com/devicelogin") - .to_owned(); - let interval = json.get("interval").and_then(Value::as_u64).unwrap_or(5); - let expires_in = json - .get("expires_in") - .and_then(Value::as_u64) - .unwrap_or(900); - let message = json - .get("message") - .and_then(Value::as_str) - .map(str::to_owned); - Ok(DeviceCodeResponse { - device_code, - user_code, - verification_uri, - interval, - expires_in, - message, - }) -} - -#[derive(Debug, Clone)] -pub enum TokenResponse { - Ok(AcquiredToken), - Pending, - SlowDown, - Expired, - Declined, - BadCode, - Other { error: String, description: String }, -} - -pub fn parse_token_response(status: u16, json: &Value) -> TokenResponse { - if (200..300).contains(&status) { - let Some(access) = json.get("access_token").and_then(Value::as_str) else { - return TokenResponse::Other { - error: "missing_access_token".to_owned(), - description: String::new(), - }; - }; - let claims = decode_jwt_claims(access).unwrap_or_default(); - return TokenResponse::Ok(AcquiredToken { - access_token: access.to_owned(), - refresh_token: json - .get("refresh_token") - .and_then(Value::as_str) - .map(str::to_owned), - expires_in: json.get("expires_in").and_then(Value::as_u64), - tenant_id: claims.tenant_id, - upn: claims.upn, - name: claims.name, - }); - } - let error = json - .get("error") - .and_then(Value::as_str) - .unwrap_or("") - .to_owned(); - let description = json - .get("error_description") - .and_then(Value::as_str) - .unwrap_or("") - .to_owned(); - match error.as_str() { - "authorization_pending" => TokenResponse::Pending, - "slow_down" => TokenResponse::SlowDown, - "expired_token" => TokenResponse::Expired, - "authorization_declined" => TokenResponse::Declined, - "bad_verification_code" => TokenResponse::BadCode, - _ => TokenResponse::Other { error, description }, - } -} - -fn device_code_flow( - authority: &str, - client_id: &str, - allow_invalid_certs: bool, -) -> Result { - let agent = build_agent(allow_invalid_certs); - let body = form_encode(&[("client_id", client_id), ("scope", SCOPES)]); - let endpoint = device_code_endpoint(authority); - let mut resp = agent - .post(&endpoint) - .header("Content-Type", "application/x-www-form-urlencoded") - .send(body.as_bytes()) - .map_err(|e| GraphError::OAuth(format!("device code endpoint: {e}")))?; - let json: Value = resp - .body_mut() - .read_json() - .map_err(|e| GraphError::OAuth(format!("device code body: {e}")))?; - let response = parse_device_code_response(&json)?; - if let Some(m) = response.message.as_deref() { - eprintln!("{m}"); - } else { - eprintln!( - "To sign in, open {} and enter the code {}", - response.verification_uri, response.user_code - ); - } - let _ = io::stdout().flush(); - let token_endpoint = token_endpoint(authority); - let deadline = Instant::now() + Duration::from_secs(response.expires_in); - let mut delay = Duration::from_secs(response.interval); - loop { - std::thread::sleep(delay); - if Instant::now() >= deadline { - return Err(GraphError::OAuth( - "device code expired before user completed sign-in".to_owned(), - )); - } - let body = form_encode(&[ - ("client_id", client_id), - ("grant_type", "urn:ietf:params:oauth:grant-type:device_code"), - ("device_code", response.device_code.as_str()), - ]); - let mut resp = agent - .post(&token_endpoint) - .header("Content-Type", "application/x-www-form-urlencoded") - .send(body.as_bytes()) - .map_err(|e| GraphError::OAuth(format!("device code polling: {e}")))?; - let status = resp.status().as_u16(); - let payload: Value = resp.body_mut().read_json().unwrap_or(Value::Null); - match parse_token_response(status, &payload) { - TokenResponse::Ok(t) => return Ok(t), - TokenResponse::Pending => continue, - TokenResponse::SlowDown => { - delay = delay.saturating_add(Duration::from_secs(5)); - } - TokenResponse::Expired => { - return Err(GraphError::OAuth( - "device code expired; restart the import".to_owned(), - )); - } - TokenResponse::Declined => { - return Err(GraphError::OAuth( - "the user declined the device-code sign-in request".to_owned(), - )); - } - TokenResponse::BadCode => { - return Err(GraphError::OAuth("device code rejected".to_owned())); - } - TokenResponse::Other { error, description } => { - return Err(GraphError::OAuth(format!( - "device code polling: {error}: {description}" - ))); - } - } - } -} - -pub fn refresh_access_token( - authority: &str, - client_id: &str, - refresh_token: &str, - allow_invalid_certs: bool, -) -> Result { - let agent = build_agent(allow_invalid_certs); - let body = form_encode(&[ - ("client_id", client_id), - ("grant_type", "refresh_token"), - ("refresh_token", refresh_token), - ("scope", SCOPES), - ]); - let endpoint = token_endpoint(authority); - let mut delay = Duration::from_millis(500); - let mut attempts: u32 = 0; - loop { - attempts += 1; - let send = agent - .post(&endpoint) - .header("Content-Type", "application/x-www-form-urlencoded") - .send(body.as_bytes()); - let mut resp = match send { - Ok(r) => r, - Err(e) if attempts < 4 => { - std::thread::sleep(delay); - delay = (delay * 2).min(Duration::from_secs(8)); - eprintln!("graph token refresh transport error: {e}; retrying"); - continue; - } - Err(e) => return Err(GraphError::OAuth(format!("refresh endpoint: {e}"))), - }; - let status = resp.status().as_u16(); - let payload: Value = resp.body_mut().read_json().unwrap_or(Value::Null); - if matches!(status, 500..=599) && attempts < 4 { - std::thread::sleep(delay); - delay = (delay * 2).min(Duration::from_secs(8)); - continue; - } - return match parse_token_response(status, &payload) { - TokenResponse::Ok(t) => Ok(t), - other => Err(GraphError::OAuth(format!( - "refresh failed (http {status}): {other:?}" - ))), - }; - } -} - -fn form_encode(pairs: &[(&str, &str)]) -> String { - let mut out = String::new(); - for (i, (k, v)) in pairs.iter().enumerate() { - if i > 0 { - out.push('&'); - } - out.push_str(&urlencode(k)); - out.push('='); - out.push_str(&urlencode(v)); - } - out -} - -fn urlencode(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - for b in s.bytes() { - match b { - b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { - out.push(b as char); - } - _ => { - use std::fmt::Write; - let _ = write!(out, "%{b:02X}"); - } - } - } - out -} - -pub fn run_device_code_polling_against( - authority: &str, - client_id: &str, - response: &DeviceCodeResponse, - mut step: F, -) -> Result -where - F: FnMut(&str, &str) -> Result<(u16, Value), GraphError>, -{ - let token_endpoint = token_endpoint(authority); - let deadline = Instant::now() + Duration::from_secs(response.expires_in.max(1)); - let mut delay = Duration::from_millis(20); - loop { - if Instant::now() >= deadline { - return Err(GraphError::OAuth( - "device code expired before user completed sign-in".to_owned(), - )); - } - std::thread::sleep(delay); - let body = form_encode(&[ - ("client_id", client_id), - ("grant_type", "urn:ietf:params:oauth:grant-type:device_code"), - ("device_code", response.device_code.as_str()), - ]); - let (status, payload) = step(&token_endpoint, &body)?; - match parse_token_response(status, &payload) { - TokenResponse::Ok(t) => return Ok(t), - TokenResponse::Pending => continue, - TokenResponse::SlowDown => { - delay = delay.saturating_add(Duration::from_millis(20)); - } - TokenResponse::Expired => { - return Err(GraphError::OAuth("device code expired".to_owned())); - } - TokenResponse::Declined => { - return Err(GraphError::OAuth("device code declined by user".to_owned())); - } - TokenResponse::BadCode => { - return Err(GraphError::OAuth("device code rejected".to_owned())); - } - TokenResponse::Other { error, description } => { - return Err(GraphError::OAuth(format!("{error}: {description}"))); - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use encodify::base64::{STANDARD, STANDARD_NO_PAD, URL_SAFE_NO_PAD}; - - fn make_jwt(tid: &str, upn: &str, exp: u64) -> String { - let header = URL_SAFE_NO_PAD.encode(b"{\"alg\":\"none\"}"); - let claims = format!(r#"{{"tid":"{tid}","upn":"{upn}","exp":{exp}}}"#); - let payload = URL_SAFE_NO_PAD.encode(claims.as_bytes()); - format!("{header}.{payload}.") - } - - // Device code consent covers every --objects surface, including the drive. - #[test] - fn scopes_cover_every_surface() { - for required in [ - "offline_access", - "Mail.Read", - "MailboxSettings.Read", - "Calendars.Read", - "Contacts.Read", - "Files.Read", - ] { - assert!( - SCOPES.split_whitespace().any(|s| s == required), - "missing {required}" - ); - } - } - - #[test] - fn jwt_claim_extraction() { - let token = make_jwt("t-1", "alice@x.com", 9999999999); - let claims = decode_jwt_claims(&token).unwrap(); - assert_eq!(claims.tenant_id.as_deref(), Some("t-1")); - assert_eq!(claims.upn.as_deref(), Some("alice@x.com")); - assert_eq!(claims.exp, Some(9999999999)); - } - - #[test] - fn jwt_payload_decodes_in_either_alphabet_with_or_without_padding() { - let claims = r#"{"tid":"t-3","upn":"??>>@xy","exp":1}"#; - for engine in [URL_SAFE_NO_PAD, URL_SAFE, STANDARD_NO_PAD, STANDARD] { - let token = format!("h.{}.s", engine.encode(claims)); - let decoded = decode_jwt_claims(&token).unwrap(); - assert_eq!(decoded.upn.as_deref(), Some("??>>@xy"), "{token}"); - } - assert!(decode_jwt_claims("h.eyJ0aWQiOiJ0LTMifQ.s").is_some()); - assert!(decode_jwt_claims("h.eyJ0aWQiOiJ0LTMifQ=.s").is_none()); - assert!(decode_jwt_claims("h.eyJ0 aWQiOiJ0LTMifQ.s").is_none()); - } - - #[test] - fn malformed_token_returns_none() { - assert!(decode_jwt_claims("garbage").is_none()); - } - - #[test] - fn device_code_response_parses_required_fields() { - let json = serde_json::json!({ - "device_code": "ABCDEF", - "user_code": "XXX-YYY", - "verification_uri": "https://microsoft.com/devicelogin", - "interval": 5, - "expires_in": 900, - "message": "go sign in" - }); - let r = parse_device_code_response(&json).unwrap(); - assert_eq!(r.device_code, "ABCDEF"); - assert_eq!(r.user_code, "XXX-YYY"); - assert_eq!(r.interval, 5); - assert_eq!(r.expires_in, 900); - } - - #[test] - fn token_response_pending() { - let json = serde_json::json!({ - "error": "authorization_pending", - "error_description": "user has not yet completed" - }); - assert!(matches!( - parse_token_response(400, &json), - TokenResponse::Pending - )); - } - - #[test] - fn token_response_declined() { - let json = serde_json::json!({"error": "authorization_declined"}); - assert!(matches!( - parse_token_response(400, &json), - TokenResponse::Declined - )); - } - - #[test] - fn token_response_bad_code() { - let json = serde_json::json!({"error": "bad_verification_code"}); - assert!(matches!( - parse_token_response(400, &json), - TokenResponse::BadCode - )); - } - - #[test] - fn token_response_expired() { - let json = serde_json::json!({"error": "expired_token"}); - assert!(matches!( - parse_token_response(400, &json), - TokenResponse::Expired - )); - } - - #[test] - fn token_response_success_includes_refresh() { - let token = make_jwt("t-2", "bob@x.com", 9999999999); - let json = serde_json::json!({ - "access_token": token, - "refresh_token": "REFRESH", - "expires_in": 3599 - }); - match parse_token_response(200, &json) { - TokenResponse::Ok(t) => { - assert!(t.access_token.starts_with("eyJhbGciOiJub25lIn0")); - assert_eq!(t.refresh_token.as_deref(), Some("REFRESH")); - assert_eq!(t.expires_in, Some(3599)); - assert_eq!(t.upn.as_deref(), Some("bob@x.com")); - } - other => panic!("expected Ok, got {other:?}"), - } - } - - #[test] - fn urlencode_escapes_special_chars() { - assert_eq!(urlencode("a b/c"), "a%20b%2Fc"); - assert_eq!(urlencode("plain.123_-"), "plain.123_-"); - } - - #[test] - fn default_authority_uses_login_microsoftonline_com() { - assert_eq!( - default_authority("common"), - "https://login.microsoftonline.com/common" - ); - assert_eq!( - default_authority("contoso.onmicrosoft.com"), - "https://login.microsoftonline.com/contoso.onmicrosoft.com" - ); - } -} diff --git a/src/exchange_graph/recurrence.rs b/src/exchange_graph/recurrence.rs deleted file mode 100644 index 9044e98..0000000 --- a/src/exchange_graph/recurrence.rs +++ /dev/null @@ -1,319 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use serde_json::{Value, json}; - -use crate::exchange::date::{is_outlook_no_date, recurrence_until}; -use crate::exchange_graph::error::GraphError; - -pub fn convert_patterned_recurrence(pr: &Value) -> Result { - Ok(Value::Object(convert_patterned_recurrence_rule(pr)?)) -} - -pub fn convert_patterned_recurrence_rule( - pr: &Value, -) -> Result, GraphError> { - let pattern = pr - .get("pattern") - .ok_or_else(|| GraphError::Malformed("recurrence.pattern missing".to_owned()))?; - let range = pr - .get("range") - .ok_or_else(|| GraphError::Malformed("recurrence.range missing".to_owned()))?; - - let mut rule = serde_json::Map::new(); - rule.insert("@type".to_owned(), Value::from("RecurrenceRule")); - rule.insert("frequency".to_owned(), Value::from(frequency_for(pattern))); - - if let Some(interval) = pattern.get("interval").and_then(Value::as_u64) - && interval != 1 - { - rule.insert("interval".to_owned(), Value::from(interval)); - } - - if let Some(first_day) = pattern.get("firstDayOfWeek").and_then(Value::as_str) - && let Some(short) = day_short(first_day) - { - rule.insert("firstDayOfWeek".to_owned(), Value::from(short)); - } - - if let Some(days) = pattern.get("daysOfWeek").and_then(Value::as_array) { - let by_day: Vec = days - .iter() - .filter_map(Value::as_str) - .filter_map(day_short) - .map(|d| json!({"@type": "NDay", "day": d})) - .collect(); - if !by_day.is_empty() { - rule.insert("byDay".to_owned(), Value::Array(by_day)); - } - } - - if let Some(dom) = pattern.get("dayOfMonth").and_then(Value::as_i64) - && dom != 0 - { - rule.insert( - "byMonthDay".to_owned(), - Value::Array(vec![Value::from(dom)]), - ); - } - - if let Some(month) = pattern.get("month").and_then(Value::as_i64) - && month != 0 - { - rule.insert( - "byMonth".to_owned(), - Value::Array(vec![Value::String(month.to_string())]), - ); - } - - let pattern_type = pattern.get("type").and_then(Value::as_str).unwrap_or(""); - if matches!(pattern_type, "relativeMonthly" | "relativeYearly") - && let Some(index) = pattern.get("index").and_then(Value::as_str) - && let Some(setpos) = set_position_for(index) - { - rule.insert( - "bySetPosition".to_owned(), - Value::Array(vec![Value::from(setpos)]), - ); - } - - let range_type = range.get("type").and_then(Value::as_str).unwrap_or("noEnd"); - match range_type { - "endDate" => { - if let Some(local) = range - .get("endDate") - .and_then(Value::as_str) - .filter(|end| !is_outlook_no_date(end)) - .and_then(recurrence_until) - { - rule.insert("until".to_owned(), Value::from(local)); - } - } - "numbered" => { - if let Some(count) = range.get("numberOfOccurrences").and_then(Value::as_u64) { - rule.insert("count".to_owned(), Value::from(count)); - } - } - _ => {} - } - - Ok(rule) -} - -fn frequency_for(pattern: &Value) -> &'static str { - match pattern.get("type").and_then(Value::as_str).unwrap_or("") { - "daily" => "daily", - "weekly" => "weekly", - "absoluteMonthly" | "relativeMonthly" => "monthly", - "absoluteYearly" | "relativeYearly" => "yearly", - _ => "daily", - } -} - -fn day_short(name: &str) -> Option<&'static str> { - match name.to_ascii_lowercase().as_str() { - "monday" => Some("mo"), - "tuesday" => Some("tu"), - "wednesday" => Some("we"), - "thursday" => Some("th"), - "friday" => Some("fr"), - "saturday" => Some("sa"), - "sunday" => Some("su"), - _ => None, - } -} - -fn set_position_for(index: &str) -> Option { - match index.to_ascii_lowercase().as_str() { - "first" => Some(1), - "second" => Some(2), - "third" => Some(3), - "fourth" => Some(4), - "last" => Some(-1), - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn daily_with_interval() { - let pr = json!({ - "pattern": {"type": "daily", "interval": 3}, - "range": {"type": "noEnd"} - }); - let out = convert_patterned_recurrence(&pr).unwrap(); - let rule = &out; - assert_eq!(rule["frequency"], "daily"); - assert_eq!(rule["interval"], 3); - assert!(rule.get("until").is_none()); - assert!(rule.get("count").is_none()); - } - - #[test] - fn weekly_with_days_of_week() { - let pr = json!({ - "pattern": { - "type": "weekly", - "interval": 1, - "daysOfWeek": ["monday", "wednesday", "friday"], - "firstDayOfWeek": "monday" - }, - "range": {"type": "noEnd"} - }); - let out = convert_patterned_recurrence(&pr).unwrap(); - let rule = &out; - assert_eq!(rule["frequency"], "weekly"); - assert_eq!(rule["firstDayOfWeek"], "mo"); - let by_day = rule["byDay"].as_array().unwrap(); - assert_eq!(by_day.len(), 3); - assert_eq!(by_day[0]["day"], "mo"); - assert_eq!(by_day[1]["day"], "we"); - assert_eq!(by_day[2]["day"], "fr"); - } - - #[test] - fn absolute_monthly_uses_by_month_day() { - let pr = json!({ - "pattern": {"type": "absoluteMonthly", "interval": 1, "dayOfMonth": 15}, - "range": {"type": "noEnd"} - }); - let out = convert_patterned_recurrence(&pr).unwrap(); - let rule = &out; - assert_eq!(rule["frequency"], "monthly"); - assert_eq!(rule["byMonthDay"][0], 15); - } - - #[test] - fn relative_monthly_emits_by_set_position() { - let pr = json!({ - "pattern": { - "type": "relativeMonthly", - "interval": 1, - "daysOfWeek": ["thursday"], - "index": "third" - }, - "range": {"type": "noEnd"} - }); - let out = convert_patterned_recurrence(&pr).unwrap(); - let rule = &out; - assert_eq!(rule["frequency"], "monthly"); - assert_eq!(rule["bySetPosition"][0], 3); - assert_eq!(rule["byDay"][0]["day"], "th"); - } - - #[test] - fn index_is_ignored_unless_the_pattern_is_relative() { - for kind in ["daily", "weekly", "absoluteMonthly", "absoluteYearly"] { - let pr = json!({ - "pattern": {"type": kind, "interval": 1, "index": "first", - "daysOfWeek": ["monday"], "dayOfMonth": 1, "month": 1}, - "range": {"type": "noEnd"} - }); - let out = convert_patterned_recurrence(&pr).unwrap(); - assert!( - out.get("bySetPosition").is_none(), - "{kind} carries Graph's default index=first, which must not become bySetPosition" - ); - } - } - - #[test] - fn index_survives_for_relative_patterns() { - for kind in ["relativeMonthly", "relativeYearly"] { - let pr = json!({ - "pattern": {"type": kind, "interval": 1, "index": "third", - "daysOfWeek": ["thursday"]}, - "range": {"type": "noEnd"} - }); - let out = convert_patterned_recurrence(&pr).unwrap(); - assert_eq!(out["bySetPosition"][0], 3, "{kind} keeps its index"); - } - } - - #[test] - fn absolute_yearly_with_month_and_day() { - let pr = json!({ - "pattern": {"type": "absoluteYearly", "interval": 1, "month": 7, "dayOfMonth": 4}, - "range": {"type": "noEnd"} - }); - let out = convert_patterned_recurrence(&pr).unwrap(); - let rule = &out; - assert_eq!(rule["frequency"], "yearly"); - assert_eq!(rule["byMonth"][0], "7"); - assert_eq!(rule["byMonthDay"][0], 4); - } - - #[test] - fn relative_yearly_uses_setpos_and_byday() { - let pr = json!({ - "pattern": { - "type": "relativeYearly", "interval": 1, "daysOfWeek": ["monday"], - "index": "last", "month": 5 - }, - "range": {"type": "noEnd"} - }); - let out = convert_patterned_recurrence(&pr).unwrap(); - let rule = &out; - assert_eq!(rule["frequency"], "yearly"); - assert_eq!(rule["bySetPosition"][0], -1); - assert_eq!(rule["byMonth"][0], "5"); - } - - #[test] - fn range_end_date_maps_to_until_local_datetime() { - let pr = json!({ - "pattern": {"type": "daily", "interval": 1}, - "range": {"type": "endDate", "endDate": "2026-12-31"} - }); - let out = convert_patterned_recurrence(&pr).unwrap(); - assert_eq!(out["until"], "2026-12-31T23:59:59"); - } - - #[test] - fn range_end_on_outlook_no_date_has_no_until() { - let rule = convert_patterned_recurrence(&json!({ - "pattern": {"type": "daily", "interval": 1}, - "range": {"type": "endDate", "startDate": "2026-01-01", "endDate": "4500-12-31"} - })) - .unwrap(); - assert!(rule.get("until").is_none()); - } - - #[test] - fn range_end_datetime_preserved() { - let pr = json!({ - "pattern": {"type": "daily", "interval": 1}, - "range": {"type": "endDate", "endDate": "2026-12-31T23:59:59"} - }); - let out = convert_patterned_recurrence(&pr).unwrap(); - assert_eq!(out["until"], "2026-12-31T23:59:59"); - } - - #[test] - fn range_numbered_maps_to_count() { - let pr = json!({ - "pattern": {"type": "weekly", "interval": 1, "daysOfWeek": ["monday"]}, - "range": {"type": "numbered", "numberOfOccurrences": 10} - }); - let out = convert_patterned_recurrence(&pr).unwrap(); - assert_eq!(out["count"], 10); - } - - #[test] - fn no_end_range_omits_until_and_count() { - let pr = json!({ - "pattern": {"type": "daily", "interval": 1}, - "range": {"type": "noEnd"} - }); - let out = convert_patterned_recurrence(&pr).unwrap(); - assert!(out.get("until").is_none()); - assert!(out.get("count").is_none()); - } -} diff --git a/src/exchange_graph/retry.rs b/src/exchange_graph/retry.rs deleted file mode 100644 index c9b69f9..0000000 --- a/src/exchange_graph/retry.rs +++ /dev/null @@ -1,78 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HttpClass { - Success, - Retryable, - Auth, - Vanished, - Fatal, -} - -pub fn classify_http_status(status: u16) -> HttpClass { - match status { - 200..=299 => HttpClass::Success, - 401 => HttpClass::Auth, - 403 => HttpClass::Auth, - 404 | 410 => HttpClass::Vanished, - 429 | 500 | 502 | 503 | 504 => HttpClass::Retryable, - _ => HttpClass::Fatal, - } -} - -pub fn is_throttled(status: u16) -> bool { - matches!(status, 429 | 503) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn rate_limit_status_is_retryable() { - assert_eq!(classify_http_status(429), HttpClass::Retryable); - assert!(is_throttled(429)); - } - - #[test] - fn service_unavailable_is_retryable_and_throttle_marked() { - assert_eq!(classify_http_status(503), HttpClass::Retryable); - assert!(is_throttled(503)); - } - - #[test] - fn bad_gateway_and_timeout_retry() { - assert_eq!(classify_http_status(502), HttpClass::Retryable); - assert_eq!(classify_http_status(504), HttpClass::Retryable); - } - - #[test] - fn insufficient_storage_is_per_item_fatal() { - assert_eq!(classify_http_status(507), HttpClass::Fatal); - } - - #[test] - fn auth_failure_is_marked_distinctly() { - assert_eq!(classify_http_status(401), HttpClass::Auth); - assert_eq!(classify_http_status(403), HttpClass::Auth); - } - - #[test] - fn not_found_and_gone_are_vanished() { - assert_eq!(classify_http_status(404), HttpClass::Vanished); - assert_eq!(classify_http_status(410), HttpClass::Vanished); - } - - #[test] - fn unknown_4xx_is_fatal() { - assert_eq!(classify_http_status(400), HttpClass::Fatal); - assert_eq!(classify_http_status(405), HttpClass::Fatal); - assert_eq!(classify_http_status(408), HttpClass::Fatal); - assert_eq!(classify_http_status(422), HttpClass::Fatal); - assert_eq!(classify_http_status(423), HttpClass::Fatal); - } -} diff --git a/src/sync/export/blobpool.rs b/src/export/blobpool.rs similarity index 63% rename from src/sync/export/blobpool.rs rename to src/export/blobpool.rs index cb11704..c3cfaa7 100644 --- a/src/sync/export/blobpool.rs +++ b/src/export/blobpool.rs @@ -6,21 +6,22 @@ use std::collections::HashMap; -use super::{Net, Uploader}; -use crate::jmap::blobxfer; +use super::Exporter; +use super::uploader::Uploader; +use crate::archive::BlobId; use crate::jmap::error::JmapError; +use crate::jmap::session::Lane; use crate::jmap::wire::JmapId; -use crate::sync::import_jmap::mapping::BlobBytes; -use crate::sync::import_jmap::pool::{Pool, effective_workers}; +use crate::sync::pool::Pool; -pub const MAX_INFLIGHT_BLOBS: usize = 64; -pub const MAX_INFLIGHT_BYTES: u64 = 256 * 1024 * 1024; +const MAX_INFLIGHT_BLOBS: usize = 64; +const MAX_INFLIGHT_BYTES: u64 = 256 * 1024 * 1024; -type UploadJob = (i64, Vec); -type UploadDone = (i64, u64, Result); +type UploadJob = (BlobId, Vec); +type UploadDone = (BlobId, u64, Result); #[derive(Debug)] -pub struct Budget { +struct Budget { count: usize, bytes: u64, max_count: usize, @@ -28,7 +29,7 @@ pub struct Budget { } impl Budget { - pub fn new(max_count: usize, max_bytes: u64) -> Budget { + fn new(max_count: usize, max_bytes: u64) -> Budget { Budget { count: 0, bytes: 0, @@ -37,59 +38,60 @@ impl Budget { } } - pub fn admits(&self, len: u64) -> bool { + fn admits(&self, len: u64) -> bool { self.count == 0 || (self.count < self.max_count && self.bytes.saturating_add(len) <= self.max_bytes) } - pub fn take(&mut self, len: u64) { + fn take(&mut self, len: u64) { self.count += 1; self.bytes = self.bytes.saturating_add(len); } - pub fn release(&mut self, len: u64) { + fn release(&mut self, len: u64) { self.count = self.count.saturating_sub(1); self.bytes = self.bytes.saturating_sub(len); } - pub fn is_idle(&self) -> bool { + fn is_idle(&self) -> bool { self.count == 0 } } -pub struct UploadPool { +pub(super) struct UploadPool { pool: Pool, } impl UploadPool { - pub fn new(net: &Net, threads: usize, content_type: &'static str) -> UploadPool { - let workers = effective_workers(threads, &net.limits, true); - let net = net.clone(); - let pool = Pool::new(workers, move |(local_id, bytes): UploadJob| { + pub(super) fn new(exporter: &Exporter<'_>, content_type: &'static str) -> UploadPool { + let workers = exporter + .account + .limits() + .workers(exporter.threads, Lane::Upload); + let account = exporter.account.clone(); + let pool = Pool::new(workers, move |(blob, bytes): UploadJob| { let len = bytes.len() as u64; - let res = blobxfer::upload_bytes( - &net.client, - &net.session, - &net.account, - content_type, - &bytes, - ); - (local_id, len, res) + let res = account.upload(&bytes, content_type); + (blob, len, res) }); UploadPool { pool } } - pub fn upload(&self, uploader: &mut Uploader<'_>, blobs: &[i64]) -> HashMap { + pub(super) fn upload( + &self, + uploader: &mut Uploader<'_>, + blobs: &[BlobId], + ) -> HashMap { let mut failed = HashMap::new(); let mut budget = Budget::new(MAX_INFLIGHT_BLOBS, MAX_INFLIGHT_BYTES); - for &local_id in blobs { - if uploader.cached(local_id).is_some() { + for &blob in blobs { + if uploader.cached(blob).is_some() { continue; } - let bytes = match uploader.bytes(local_id) { + let bytes = match uploader.bytes(blob) { Ok(b) => b, Err(e) => { - failed.insert(local_id, e); + failed.insert(blob, e); continue; } }; @@ -100,7 +102,7 @@ impl UploadPool { } } budget.take(len); - self.pool.submit((local_id, bytes)); + self.pool.submit((blob, bytes)); } while !budget.is_idle() { if !self.collect_one(uploader, &mut budget, &mut failed) { @@ -114,15 +116,15 @@ impl UploadPool { &self, uploader: &mut Uploader<'_>, budget: &mut Budget, - failed: &mut HashMap, + failed: &mut HashMap, ) -> bool { match self.pool.results().recv() { - Ok((local_id, len, res)) => { + Ok((blob, len, res)) => { budget.release(len); match res { - Ok(id) => uploader.store(local_id, id), + Ok(id) => uploader.remember(blob, id), Err(e) => { - failed.insert(local_id, e); + failed.insert(blob, e); } } true diff --git a/src/export/email.rs b/src/export/email.rs new file mode 100644 index 0000000..49fcf21 --- /dev/null +++ b/src/export/email.rs @@ -0,0 +1,562 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +mod import; + +use std::collections::{HashMap, HashSet}; +use std::fmt; +use std::mem; + +use serde_json::{Map, Value, json}; + +use self::import::SingleImport; +use super::blobpool::UploadPool; +use super::target::{ClientId, Maps, TargetJson}; +use super::uploader::Uploader; +use super::{Exporter, Plan}; +use crate::archive::{BlobId, LocalId}; +use crate::console::ByteSize; +use crate::error::Error; +use crate::jmap::error::JmapError; +use crate::jmap::request::Request; +use crate::jmap::session::Limits; +use crate::jmap::wire::JmapId; +use crate::sync::TypeCounts; +use crate::sync::jmap_map::EmailRow; +use crate::sync::keys::{EmailIndex, EmailKey}; +use crate::types::ObjectType; + +const EMAIL_MIME: &str = "message/rfc822"; +const EMAIL_PARSE_MAX_ITEMS: usize = 10; +const EMAIL_KEY_PROPERTIES: [&str; 5] = ["messageId", "from", "subject", "sentAt", "to"]; + +impl EmailIndex { + fn from_target(v: &Value) -> EmailIndex { + let addresses = |k: &str| -> Vec { + v.get(k) + .and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(|x| x.get("email").and_then(Value::as_str).map(str::to_owned)) + .collect() + }) + .unwrap_or_default() + }; + let mids: Vec = v + .get("messageId") + .and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(|x| x.as_str().map(str::to_owned)) + .collect() + }) + .unwrap_or_default(); + EmailIndex::new( + &mids, + &addresses("from"), + v.get("subject").and_then(Value::as_str).unwrap_or(""), + v.get("sentAt").and_then(Value::as_str).unwrap_or(""), + &addresses("to"), + ) + } +} + +impl Exporter<'_> { + pub(super) fn reconcile_emails( + &self, + maps: &Maps, + counts: &mut TypeCounts, + ) -> Result { + let target_keys = self.email_target_keys()?; + let locals = self.email_locals()?; + let unmatched: Vec<&(LocalId, EmailRow)> = locals + .iter() + .filter(|(_, row)| { + let matched = + target_keys.contains(&EmailIndex::from_json(&row.message_match).key()); + if matched { + counts.skipped += 1; + } + !matched + }) + .collect(); + + let mut batch = EmailBatch { + exporter: self, + uploader: Uploader::new(self), + pool: (!self.is_dry_run()).then(|| UploadPool::new(self, EMAIL_MIME)), + maps, + counts, + target_keys: &target_keys, + size: EmailBatch::initial_size(self.account.limits()), + }; + let mut rest = unmatched.as_slice(); + while !rest.is_empty() { + let (chunk, tail) = rest.split_at(batch.size.min(rest.len())); + batch.export_chunk(chunk)?; + rest = tail; + } + Ok(Plan::default()) + } + + fn email_target_keys(&self) -> Result, Error> { + let ty = ObjectType::Email; + let targets = self.query_get(ty, Some(&["messageId"]))?; + let mut indices: Vec = targets.iter().map(EmailIndex::from_target).collect(); + let fallback_ids: Vec = targets + .iter() + .zip(&indices) + .filter(|(_, i)| i.mids.is_empty()) + .filter_map(|(v, _)| TargetJson(v).id().map(|id| JmapId(id.to_owned()))) + .collect(); + if !fallback_ids.is_empty() { + let got = self.account.get::( + ty.jmap_name(), + &fallback_ids, + Some(&EMAIL_KEY_PROPERTIES), + )?; + let by_id: HashMap<&str, &Value> = got + .list + .iter() + .filter_map(|v| TargetJson(v).id().map(|i| (i, v))) + .collect(); + for (v, slot) in targets.iter().zip(indices.iter_mut()) { + if let Some(full) = TargetJson(v).id().and_then(|i| by_id.get(i)) { + *slot = EmailIndex::from_target(full); + } + } + } + Ok(indices.iter().map(EmailIndex::key).collect()) + } + + fn email_locals(&self) -> Result, Error> { + let mut stmt = self + .store + .connection() + .prepare(EmailRow::SELECT) + .map_err(Error::partial)?; + let rows = stmt + .query_map([], |row| Ok((row.get(0)?, EmailRow::from_row(row)))) + .map_err(Error::partial)?; + let rows: Vec<_> = rows.collect::>().map_err(Error::partial)?; + let mut out = Vec::with_capacity(rows.len()); + for (id, email) in rows { + out.push((id, email?)); + } + Ok(out) + } +} + +struct Pending<'r> { + cid: ClientId, + row: &'r EmailRow, + mids: Map, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Attempt { + First, + Reupload, + LastReupload, +} + +impl Attempt { + fn suffix(self) -> &'static str { + match self { + Attempt::First => "", + Attempt::Reupload | Attempt::LastReupload => " after blob re-upload", + } + } + + fn upload_verb(self) -> &'static str { + match self { + Attempt::First => "blob upload", + Attempt::Reupload | Attempt::LastReupload => "blob re-upload", + } + } +} + +struct BlobHint { + mid: Option, + len: Option, +} + +impl fmt::Display for BlobHint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.mid { + Some(mid) => write!(f, "message-id <{mid}>")?, + None => f.write_str("no message-id")?, + } + if let Some(len) = self.len { + write!(f, ", {}", ByteSize(len))?; + } + Ok(()) + } +} + +struct SizeNote<'e>(&'e JmapError); + +impl fmt::Display for SizeNote<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if matches!( + self.0, + JmapError::RequestTooLarge | JmapError::SingleObjectTooLarge(_) + ) { + f.write_str("; exceeds the target server size limit, so this message is skipped and re-running will not migrate it")?; + } + Ok(()) + } +} + +struct EmailBatch<'e, 'a> { + exporter: &'e Exporter<'a>, + uploader: Uploader<'a>, + pool: Option, + maps: &'e Maps, + counts: &'e mut TypeCounts, + target_keys: &'e HashSet, + size: usize, +} + +impl<'r> EmailBatch<'_, '_> { + fn initial_size(limits: &Limits) -> usize { + usize::try_from(limits.max_objects_in_set.max(1)).unwrap_or(usize::MAX) + } + + fn halve(previous: usize) -> usize { + (previous / 2).max(1) + } + + fn hint(&self, row: &EmailRow) -> BlobHint { + BlobHint { + mid: EmailIndex::from_json(&row.message_match) + .mids + .into_iter() + .next(), + len: self.uploader.len(row.blob), + } + } + + fn mailbox_ids(&self, row: &EmailRow) -> Option> { + let mut mids = Map::new(); + for mailbox in row.mailboxes.as_slice() { + let target = self.maps.target(ObjectType::Mailbox, *mailbox)?; + mids.insert(target.0, Value::Bool(true)); + } + Some(mids) + } + + fn export_chunk(&mut self, chunk: &[&'r (LocalId, EmailRow)]) -> Result<(), Error> { + let mut pending: Vec> = Vec::with_capacity(chunk.len()); + for (local, row) in chunk.iter().copied() { + let cid = ClientId::email(*local); + match self.mailbox_ids(row) { + Some(mids) => pending.push(Pending { cid, row, mids }), + None => { + self.exporter.logger.warn(format_args!( + "Email/import {cid} ({}) skipped: mailbox not on target", + self.hint(row) + )); + self.counts.failed += 1; + } + } + } + let mut size = pending.len(); + let mut retry = self.import(pending, Attempt::First)?; + if !retry.is_empty() { + self.size = EmailBatch::halve(self.size); + } + while !retry.is_empty() { + size = EmailBatch::halve(size); + let attempt = if size == 1 { + Attempt::LastReupload + } else { + Attempt::Reupload + }; + for p in &retry { + self.uploader.invalidate(p.row.blob); + } + let mut again = Vec::new(); + while !retry.is_empty() { + let tail = retry.split_off(size.min(retry.len())); + let head = mem::replace(&mut retry, tail); + again.extend(self.import(head, attempt)?); + } + retry = again; + } + Ok(()) + } + + fn import( + &mut self, + pending: Vec>, + attempt: Attempt, + ) -> Result>, Error> { + let logger = self.exporter.logger; + let mut failed = self.upload(&pending); + let on_target = match attempt { + Attempt::First => self.parsed_on_target(&pending, &failed)?, + Attempt::Reupload | Attempt::LastReupload => HashSet::new(), + }; + let mut ready: Vec> = Vec::with_capacity(pending.len()); + let mut items: Vec<(ClientId, Value)> = Vec::with_capacity(pending.len()); + for p in pending { + let blob = match failed.remove(&p.row.blob) { + Some(e) => Err(e), + None => self + .uploader + .cached(p.row.blob) + .map(|b| b.0.clone()) + .ok_or_else(|| JmapError::malformed("blob upload did not complete")), + }; + let blob = match blob { + Ok(b) => b, + Err(e) => { + logger.warn(format_args!( + "Email/import {} ({}) {} failed: {e}{}", + p.cid, + self.hint(p.row), + attempt.upload_verb(), + SizeNote(&e) + )); + self.counts.failed += 1; + continue; + } + }; + if on_target.contains(&blob) { + self.counts.skipped += 1; + continue; + } + if self.exporter.is_dry_run() { + self.counts.created += 1; + continue; + } + let keywords: Map = p + .row + .keywords + .iter() + .map(|k| (k.clone(), Value::Bool(true))) + .collect(); + let item = json!({ + "blobId": blob, + "mailboxIds": Value::Object(p.mids.clone()), + "keywords": Value::Object(keywords), + "receivedAt": p.row.received_at, + }); + items.push((p.cid.clone(), item)); + ready.push(p); + } + if items.is_empty() { + return Ok(Vec::new()); + } + + let mut results = Vec::with_capacity(items.len()); + if let Err(e) = self.exporter.send_import(&items, &mut results) { + let mapped = Error::from(e); + if mapped.aborts_run() { + return Err(mapped); + } + for p in ready.iter().skip(results.len()) { + logger.warn(format_args!( + "Email/import {} ({}) send failed{}: {mapped}", + p.cid, + self.hint(p.row), + attempt.suffix() + )); + self.counts.failed += 1; + } + ready.truncate(results.len()); + } + + let mut retry = Vec::new(); + for (p, res) in ready.into_iter().zip(results) { + match res { + Ok(SingleImport::Created) => self.counts.created += 1, + Ok(SingleImport::Skipped) => self.counts.skipped += 1, + Ok(SingleImport::NotCreated { error_type, .. }) + if attempt != Attempt::LastReupload && error_type == "blobNotFound" => + { + retry.push(p); + } + Ok(SingleImport::NotCreated { detail, .. }) => { + logger.warn(format_args!( + "Email/import {} ({}) failed{}: {detail}", + p.cid, + self.hint(p.row), + attempt.suffix() + )); + self.counts.failed += 1; + } + Err(e) => { + logger.warn(format_args!( + "Email/import {} ({}) send failed{}: {e}{}", + p.cid, + self.hint(p.row), + attempt.suffix(), + SizeNote(&e) + )); + self.counts.failed += 1; + } + } + } + Ok(retry) + } + + fn parsed_on_target( + &self, + pending: &[Pending<'_>], + failed: &HashMap, + ) -> Result, Error> { + let mut found = HashSet::new(); + if self.exporter.is_dry_run() || self.target_keys.is_empty() { + return Ok(found); + } + let account = self.exporter.account; + let blobs: Vec<&str> = pending + .iter() + .filter(|p| !failed.contains_key(&p.row.blob)) + .filter_map(|p| self.uploader.cached(p.row.blob)) + .map(|b| b.0.as_str()) + .collect(); + let calls = usize::try_from(account.limits().max_calls_in_request.max(1)).unwrap_or(1); + for group in blobs.chunks(EMAIL_PARSE_MAX_ITEMS.saturating_mul(calls)) { + let mut req = Request::new(); + for (n, chunk) in group.chunks(EMAIL_PARSE_MAX_ITEMS).enumerate() { + req.call( + "Email/parse", + json!({ + "accountId": account.account_id(), + "blobIds": chunk, + "properties": EMAIL_KEY_PROPERTIES, + }), + format!("p{n}"), + ); + } + let resp = match account.send(&req) { + Ok(resp) => resp, + Err(e) => { + let mapped = Error::from(e); + if mapped.aborts_run() { + return Err(mapped); + } + self.exporter.logger.warn(format_args!( + "Email/parse failed, importing without a target duplicate check: {mapped}" + )); + return Ok(found); + } + }; + for mr in &resp.method_responses { + if let Err(e) = mr.check_error() { + self.exporter.logger.warn(format_args!( + "Email/parse failed, importing without a target duplicate check: {e}" + )); + continue; + } + let Some(parsed) = mr.args.get("parsed").and_then(Value::as_object) else { + continue; + }; + found.extend( + parsed + .iter() + .filter(|(_, email)| { + self.target_keys + .contains(&EmailIndex::from_target(email).key()) + }) + .map(|(blob, _)| blob.clone()), + ); + } + } + Ok(found) + } + + fn upload(&mut self, pending: &[Pending<'_>]) -> HashMap { + let Some(pool) = &self.pool else { + let mut failed = HashMap::new(); + for p in pending { + if let Err(e) = self.uploader.upload(p.row.blob, EMAIL_MIME) { + failed.insert(p.row.blob, e); + } + } + return failed; + }; + let mut seen: HashSet = HashSet::with_capacity(pending.len()); + let mut queue: Vec = Vec::with_capacity(pending.len()); + for p in pending { + let blob = p.row.blob; + if self.uploader.cached(blob).is_none() && seen.insert(blob) { + queue.push(blob); + } + } + pool.upload(&mut self.uploader, &queue) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn limits(set: u64) -> Limits { + Limits { + max_objects_in_get: 500, + max_objects_in_set: set, + max_calls_in_request: 16, + max_concurrent_requests: 4, + max_concurrent_upload: 4, + max_size_request: 10_000_000, + max_size_upload: 50_000_000, + } + } + + #[test] + fn batch_size_follows_max_objects_in_set() { + assert_eq!(EmailBatch::initial_size(&limits(500)), 500); + assert_eq!(EmailBatch::initial_size(&limits(7)), 7); + assert_eq!(EmailBatch::initial_size(&limits(0)), 1); + } + + #[test] + fn reupload_batches_halve_down_to_one() { + let mut size = 500; + let mut steps = Vec::new(); + while size > 1 { + size = EmailBatch::halve(size); + steps.push(size); + } + assert_eq!(steps, vec![250, 125, 62, 31, 15, 7, 3, 1]); + assert_eq!(EmailBatch::halve(1), 1); + assert_eq!(EmailBatch::halve(0), 1); + } + + #[test] + fn chunking_yields_ceil_n_over_limit_batches() { + let ids: Vec = (0..1001).collect(); + let sizes: Vec = ids + .chunks(EmailBatch::initial_size(&limits(500))) + .map(<[u32]>::len) + .collect(); + assert_eq!(sizes, vec![500, 500, 1]); + } + + #[test] + fn blob_hint_and_size_note_texts() { + let hint = BlobHint { + mid: Some("a@b".to_owned()), + len: Some(2048), + }; + assert_eq!(hint.to_string(), "message-id , 2.0 KB"); + let bare = BlobHint { + mid: None, + len: None, + }; + assert_eq!(bare.to_string(), "no message-id"); + assert!( + SizeNote(&JmapError::RequestTooLarge) + .to_string() + .starts_with("; exceeds") + ); + assert_eq!(SizeNote(&JmapError::malformed("x")).to_string(), ""); + } +} diff --git a/src/export/email/import.rs b/src/export/email/import.rs new file mode 100644 index 0000000..de0bb4b --- /dev/null +++ b/src/export/email/import.rs @@ -0,0 +1,138 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::{Map, Value, json}; + +use crate::export::Exporter; +use crate::export::target::{ClientId, TargetJson}; +use crate::jmap::error::JmapError; +use crate::jmap::request::{MethodCallKind, Request}; + +pub(super) enum SingleImport { + Created, + Skipped, + NotCreated { error_type: String, detail: String }, +} + +impl SingleImport { + fn from_response(args: &Value, cid: &str) -> SingleImport { + if let Some(err) = args + .get("notCreated") + .and_then(Value::as_object) + .and_then(|nc| nc.get(cid)) + { + let error_type = TargetJson(err).error_type().unwrap_or_default(); + if error_type == "alreadyExists" { + return SingleImport::Skipped; + } + return SingleImport::NotCreated { + error_type: error_type.to_owned(), + detail: err.to_string(), + }; + } + if args + .get("created") + .and_then(Value::as_object) + .is_some_and(|c| c.contains_key(cid)) + { + return SingleImport::Created; + } + SingleImport::NotCreated { + error_type: String::new(), + detail: format!("Email/import returned neither created nor notCreated for {cid}"), + } + } +} + +impl Exporter<'_> { + fn import_request(&self, items: &[(ClientId, Value)]) -> Request { + let emails: Map = items + .iter() + .map(|(cid, item)| (cid.as_str().to_owned(), item.clone())) + .collect(); + let mut req = Request::new(); + req.call( + "Email/import", + json!({ "accountId": self.account.account_id(), "emails": Value::Object(emails) }), + "i", + ); + req + } + + pub(super) fn send_import( + &self, + items: &[(ClientId, Value)], + out: &mut Vec>, + ) -> Result<(), JmapError> { + let req = self.import_request(items); + let sent = req.fits(self.account.limits()).and_then(|()| { + self.account + .client() + .retry_method(MethodCallKind::SingleObjectWrite, || { + let resp = self.account.send(&req)?; + let mr = resp.first()?; + mr.check_error()?; + Ok(items + .iter() + .map(|(cid, _)| SingleImport::from_response(&mr.args, cid.as_str())) + .collect::>()) + }) + }); + match sent { + Ok(outcomes) => { + out.extend(outcomes.into_iter().map(Ok)); + Ok(()) + } + Err(JmapError::RequestTooLarge) if items.len() > 1 => { + let (left, right) = items.split_at(items.len() / 2); + self.send_import(left, out)?; + self.send_import(right, out) + } + Err(e @ JmapError::RequestTooLarge) => { + out.push(Err(e)); + Ok(()) + } + Err(e) => Err(e), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn item_outcome_reads_each_creation_id() { + let args = json!({ + "created": { "e1": { "id": "M1" } }, + "notCreated": { + "e2": { "type": "alreadyExists", "existingId": "M9" }, + "e3": { "type": "blobNotFound" }, + "e4": { "type": "overQuota" } + } + }); + assert!(matches!( + SingleImport::from_response(&args, "e1"), + SingleImport::Created + )); + assert!(matches!( + SingleImport::from_response(&args, "e2"), + SingleImport::Skipped + )); + assert!(matches!( + SingleImport::from_response(&args, "e3"), + SingleImport::NotCreated { ref error_type, .. } if error_type == "blobNotFound" + )); + assert!(matches!( + SingleImport::from_response(&args, "e4"), + SingleImport::NotCreated { ref error_type, .. } if error_type == "overQuota" + )); + assert!(matches!( + SingleImport::from_response(&args, "e5"), + SingleImport::NotCreated { ref error_type, .. } if error_type.is_empty() + )); + } +} diff --git a/src/export/flat.rs b/src/export/flat.rs new file mode 100644 index 0000000..804a0a4 --- /dev/null +++ b/src/export/flat.rs @@ -0,0 +1,190 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashSet; + +use rusqlite::params; +use serde_json::{Value, json}; + +use super::prune::{DestroyOrder, TargetObj}; +use super::target::{ClientId, Maps, TargetJson}; +use super::{Exporter, Plan}; +use crate::archive::LocalId; +use crate::error::Error; +use crate::jmap::request::Request; +use crate::jmap::wire::JmapId; +use crate::jmap::wire::address_book::AddressBook; +use crate::jmap::wire::calendar::Calendar; +use crate::sync::TypeCounts; +use crate::sync::keys::FoldedName; +use crate::types::ObjectType; + +struct FlatLocal { + local: LocalId, + name: FoldedName, + is_default: bool, +} + +impl Exporter<'_> { + pub(super) fn reconcile_flat( + &self, + ty: ObjectType, + maps: &mut Maps, + counts: &mut TypeCounts, + ) -> Result { + let locals = self.flat_locals(ty)?; + let targets = self.get_all(ty)?; + let mut default_claimed = targets + .iter() + .any(|t| TargetJson(t).flag("isDefault").unwrap_or(false)); + let named: Vec<(&str, FoldedName)> = targets + .iter() + .filter_map(|t| { + let t = TargetJson(t); + Some((t.id()?, FoldedName::new(t.text("name")?))) + }) + .collect(); + let mut tmatched: HashSet<&str> = HashSet::new(); + + let mut to_create: Vec<&FlatLocal> = Vec::new(); + for local in &locals { + match named + .iter() + .find(|(id, name)| *name == local.name && !tmatched.contains(id)) + { + Some((tid, _)) => { + tmatched.insert(tid); + maps.insert(ty, local.local, JmapId((*tid).to_owned())); + counts.skipped += 1; + } + None => to_create.push(local), + } + } + + if !to_create.is_empty() { + let mut batch = Vec::with_capacity(to_create.len()); + for local in &to_create { + batch.push(( + ClientId::object(local.local), + self.flat_create(ty, local.local)?, + )); + } + let outcome = self.create(ty, batch)?; + maps.record_created(ty, &outcome, counts); + let defaults: Vec = to_create + .iter() + .filter(|l| l.is_default) + .map(|l| l.local) + .collect(); + for (cid, v) in &outcome.created { + if let Some(local) = ClientId::parse(cid) + && let Some(id) = TargetJson(v).id() + && defaults.contains(&local) + { + if default_claimed { + self.logger.warn(format_args!( + "{} {id}: the target account already has a default, leaving it alone", + ty.noun() + )); + } else if self.claim_default(ty, id) { + default_claimed = true; + } + } + } + for (cid, err) in &outcome.not_created { + self.warn_not_created(ty, cid, err, counts); + } + } + + let objs: Vec> = targets + .iter() + .filter_map(|t| { + let t = TargetJson(t); + let id = t.id()?; + Some(TargetObj { + id, + matched: tmatched.contains(id), + protected: t.flag("isDefault").unwrap_or(false), + may_delete: t.may_delete().unwrap_or(true), + parent: None, + }) + }) + .collect(); + Ok(Plan::prune(TargetObj::candidates( + &objs, + DestroyOrder::ById, + ))) + } + + fn flat_locals(&self, ty: ObjectType) -> Result, Error> { + let table = ty.table(); + let mut stmt = self + .store + .connection() + .prepare(&format!("SELECT id, name, is_default FROM {table}")) + .map_err(Error::partial)?; + stmt.query_map([], |r| { + Ok(FlatLocal { + local: r.get(0)?, + name: FoldedName::new(&r.get::<_, String>(1)?), + is_default: r.get::<_, i64>(2)? != 0, + }) + }) + .and_then(Iterator::collect) + .map_err(Error::partial) + } + + fn flat_create(&self, ty: ObjectType, local: LocalId) -> Result { + let select = if ty == ObjectType::AddressBook { + AddressBook::SELECT + } else { + Calendar::SELECT + }; + let mut wire = self + .store + .connection() + .query_row(&format!("{select} WHERE id = ?1"), params![local], |row| { + Ok(if ty == ObjectType::AddressBook { + AddressBook::from_row(row).map(|w| serde_json::to_value(&w)) + } else { + Calendar::from_row(row).map(|mut w| { + if let Some(dropped) = w.normalize_color() { + self.logger.progress(format_args!( + "calendar {local}: dropping color {dropped:?}, not a CSS color name or #RGB/#RRGGBB value" + )); + } + serde_json::to_value(&w) + }) + }) + }) + .map_err(Error::partial)?? + .map_err(Error::partial)?; + if let Value::Object(m) = &mut wire { + m.remove("isDefault"); + } + Ok(wire) + } + + fn claim_default(&self, ty: ObjectType, id: &str) -> bool { + if self.is_dry_run() { + return true; + } + let mut req = Request::new(); + req.call( + format!("{}/set", ty.jmap_name()), + json!({ "accountId": self.account.account_id(), "onSuccessSetIsDefault": id }), + "d", + ); + match self.account.send(&req) { + Ok(_) => true, + Err(e) => { + self.logger + .warn(format_args!("{} isDefault not set: {e}", ty.noun())); + false + } + } + } +} diff --git a/src/export/keyed.rs b/src/export/keyed.rs new file mode 100644 index 0000000..69cdd67 --- /dev/null +++ b/src/export/keyed.rs @@ -0,0 +1,83 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashSet; + +use serde_json::Value; + +use super::target::{ClientId, TargetJson}; +use super::{Exporter, Plan}; +use crate::archive::LocalId; +use crate::error::Error; +use crate::jmap::wire::identity::Identity; +use crate::sync::TypeCounts; +use crate::sync::keys::Key32; +use crate::types::ObjectType; + +impl Exporter<'_> { + pub(super) fn reconcile_identities(&self, counts: &mut TypeCounts) -> Result { + let ty = ObjectType::Identity; + let locals = self.identity_locals()?; + let targets = self.get_all(ty)?; + let target_keys: HashSet = targets + .iter() + .map(|t| { + let t = TargetJson(t); + Key32::identity( + t.text("name").unwrap_or_default(), + t.text("email").unwrap_or_default(), + ) + }) + .collect(); + + let mut seen: HashSet = HashSet::with_capacity(locals.len()); + let mut batch = Vec::new(); + for (local, key, wire) in locals { + if target_keys.contains(&key) || !seen.insert(key) { + counts.skipped += 1; + continue; + } + batch.push((ClientId::object(local), wire)); + } + if !batch.is_empty() { + let outcome = self.create(ty, batch)?; + counts.created += outcome.created.len() as u64; + for (cid, err) in &outcome.not_created { + self.logger.warn(format_args!( + "{} {cid} not created (expected for addresses the account does not own): {err}", + ty.noun() + )); + counts.skipped += 1; + } + } + Ok(Plan::default()) + } + + fn identity_locals(&self) -> Result, Error> { + let mut stmt = self + .store + .connection() + .prepare(Identity::SELECT) + .map_err(Error::partial)?; + let rows = stmt + .query_map([], |row| { + Ok((row.get::<_, LocalId>(0)?, Identity::from_row(row))) + }) + .map_err(Error::partial)?; + let rows: Vec<_> = rows.collect::>().map_err(Error::partial)?; + let mut out = Vec::with_capacity(rows.len()); + for (id, wire) in rows { + let wire = wire?; + let key = Key32::identity(&wire.name, &wire.email); + out.push(( + id, + key, + serde_json::to_value(&wire).map_err(Error::partial)?, + )); + } + Ok(out) + } +} diff --git a/src/export/mod.rs b/src/export/mod.rs new file mode 100644 index 0000000..01878fc --- /dev/null +++ b/src/export/mod.rs @@ -0,0 +1,172 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +mod blobpool; +mod email; +mod flat; +mod keyed; +mod prune; +mod run; +mod sieve; +mod target; +mod tree; +mod uidtype; +mod uploader; +mod vacation; + +use crate::archive::{Archive, Store}; +use crate::console::Initial; +use crate::error::Error; +use crate::jmap::connect::{ConnectConfig, JmapAccount}; +use crate::jmap::http::JmapClient; +use crate::logging::Logger; +use crate::sync::{RunMode, RunOptions, RunOutcome, TypeCounts}; +use crate::types::{ObjectType, ObjectTypes}; + +use self::run::ExportRun; +use self::target::Maps; + +const EXPORT_ORDER: [ObjectType; 9] = [ + ObjectType::Mailbox, + ObjectType::AddressBook, + ObjectType::Calendar, + ObjectType::FileNode, + ObjectType::Identity, + ObjectType::SieveScript, + ObjectType::Email, + ObjectType::ContactCard, + ObjectType::CalendarEvent, +]; + +pub struct Export { + pub connect: ConnectConfig, + pub objects: Option, + pub prune: Prune, + pub confirm: Confirm, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Prune { + Keep, + Destroy, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Confirm { + Prompt, + Assume, +} + +impl Export { + pub fn run(self, options: RunOptions) -> RunOutcome { + let client = JmapClient::new(self.connect.auth.clone(), options.client_options()); + let archive = match Archive::open(&options.archive) { + Ok(archive) => archive, + Err(e) => return RunOutcome::failed(Error::from(e)), + }; + let account = match JmapAccount::connect(client, &self.connect) { + Ok(account) => account, + Err(e) => return RunOutcome::failed(e), + }; + let exporter = Exporter { + store: archive.store(), + account: &account, + mode: options.mode, + threads: options.threads, + logger: options.logger, + }; + ExportRun::execute(&exporter, &self) + } + + fn selects(&self, ty: ObjectType) -> bool { + self.objects.as_ref().is_none_or(|s| s.contains(&ty)) + } +} + +#[derive(Default)] +struct Plan { + prune_candidates: Vec, + active_sieve_target: Option, +} + +impl Plan { + fn prune(prune_candidates: Vec) -> Plan { + Plan { + prune_candidates, + active_sieve_target: None, + } + } +} + +#[derive(Clone, Copy)] +struct Exporter<'a> { + store: Store<'a>, + account: &'a JmapAccount, + mode: RunMode, + threads: usize, + logger: Logger, +} + +impl Exporter<'_> { + fn is_dry_run(&self) -> bool { + self.mode == RunMode::DryRun + } + + fn has_rows(&self, ty: ObjectType) -> bool { + let table = ty.table(); + self.store + .connection() + .query_row(&format!("SELECT EXISTS(SELECT 1 FROM {table})"), [], |r| { + r.get::<_, i64>(0) + }) + .map(|n| n != 0) + .unwrap_or(false) + } + + fn work_list(&self, config: &Export) -> Vec { + let participant = ObjectType::ParticipantIdentity; + if config.selects(participant) && self.has_rows(participant) { + self.logger.info(format_args!( + "{} are not exported; the target creates them from the account's addresses", + participant.label() + )); + } + EXPORT_ORDER + .into_iter() + .filter(|ty| config.selects(*ty)) + .filter(|ty| self.has_rows(*ty)) + .filter(|ty| { + let supported = self.account.supports(*ty); + if !supported { + self.logger.warn(format_args!( + "the target does not support {}; skipping them", + Initial::Lower(ty.label()) + )); + } + supported + }) + .collect() + } + + fn reconcile( + &self, + ty: ObjectType, + maps: &mut Maps, + counts: &mut TypeCounts, + ) -> Result { + match ty { + ObjectType::Mailbox | ObjectType::FileNode => self.reconcile_tree(ty, maps, counts), + ObjectType::AddressBook | ObjectType::Calendar => self.reconcile_flat(ty, maps, counts), + ObjectType::Identity => self.reconcile_identities(counts), + ObjectType::ParticipantIdentity => Ok(Plan::default()), + ObjectType::SieveScript => self.reconcile_sieve(counts), + ObjectType::ContactCard | ObjectType::CalendarEvent => { + self.reconcile_uid_type(ty, maps, counts) + } + ObjectType::Email => self.reconcile_emails(maps, counts), + } + } +} diff --git a/src/export/prune.rs b/src/export/prune.rs new file mode 100644 index 0000000..f7e6fa6 --- /dev/null +++ b/src/export/prune.rs @@ -0,0 +1,153 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::cmp::Reverse; +use std::collections::{HashMap, HashSet}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum DestroyOrder { + LeafFirst, + ById, +} + +#[derive(Debug, Clone)] +pub(super) struct TargetObj<'t> { + pub(super) id: &'t str, + pub(super) matched: bool, + pub(super) protected: bool, + pub(super) may_delete: bool, + pub(super) parent: Option<&'t str>, +} + +impl<'t> TargetObj<'t> { + pub(super) fn candidates(objs: &[TargetObj<'t>], order: DestroyOrder) -> Vec { + let by_id: HashMap<&str, &TargetObj<'_>> = objs.iter().map(|o| (o.id, o)).collect(); + let protected = TargetObj::protected_with_ancestors(objs, &by_id); + let mut out: Vec<&TargetObj<'_>> = objs + .iter() + .filter(|o| !o.matched && o.may_delete && !protected.contains(o.id)) + .collect(); + match order { + DestroyOrder::LeafFirst => { + out.sort_by_cached_key(|o| (Reverse(o.depth(&by_id)), o.id)); + } + DestroyOrder::ById => out.sort_unstable_by_key(|o| o.id), + } + out.into_iter().map(|o| o.id.to_owned()).collect() + } + + fn protected_with_ancestors<'m>( + objs: &'m [TargetObj<'t>], + by_id: &HashMap<&str, &'m TargetObj<'t>>, + ) -> HashSet<&'t str> { + let mut protected = HashSet::new(); + for o in objs.iter().filter(|o| o.protected) { + let mut cur = Some(o.id); + while let Some(id) = cur { + if !protected.insert(id) { + break; + } + cur = by_id + .get(id) + .and_then(|x| x.parent) + .filter(|p| by_id.contains_key(p)); + } + } + protected + } + + fn depth(&self, by_id: &HashMap<&str, &TargetObj<'t>>) -> usize { + let mut depth = 0; + let mut cur = self.parent; + let mut seen: HashSet<&str> = HashSet::new(); + while let Some(p) = cur { + if !seen.insert(p) || !by_id.contains_key(p) { + break; + } + depth += 1; + cur = by_id.get(p).and_then(|o| o.parent); + } + depth + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn o<'t>( + id: &'t str, + matched: bool, + protected: bool, + parent: Option<&'t str>, + ) -> TargetObj<'t> { + TargetObj { + id, + matched, + protected, + may_delete: true, + parent, + } + } + + #[test] + fn unmatched_unprotected_are_candidates() { + let objs = vec![ + o("a", true, false, None), + o("b", false, false, None), + o("c", false, true, None), + ]; + assert_eq!( + TargetObj::candidates(&objs, DestroyOrder::ById), + vec!["b".to_owned()] + ); + } + + #[test] + fn protected_excludes_ancestors() { + let objs = vec![ + o("root", false, false, None), + o("mid", false, false, Some("root")), + o("leaf", false, true, Some("mid")), + ]; + assert!(TargetObj::candidates(&objs, DestroyOrder::LeafFirst).is_empty()); + } + + #[test] + fn may_delete_false_is_skipped() { + let mut x = o("x", false, false, None); + x.may_delete = false; + assert!(TargetObj::candidates(&[x], DestroyOrder::ById).is_empty()); + } + + #[test] + fn tree_destroy_is_leaf_first() { + let objs = vec![ + o("root", false, false, None), + o("child", false, false, Some("root")), + o("grand", false, false, Some("child")), + ]; + assert_eq!( + TargetObj::candidates(&objs, DestroyOrder::LeafFirst), + vec!["grand".to_owned(), "child".to_owned(), "root".to_owned()] + ); + } + + #[test] + fn leaf_first_breaks_depth_ties_by_id() { + let objs = vec![ + o("r", false, false, None), + o("b", false, false, Some("r")), + o("a", false, false, Some("r")), + o("loop1", false, false, Some("loop2")), + o("loop2", false, false, Some("loop1")), + ]; + assert_eq!( + TargetObj::candidates(&objs, DestroyOrder::LeafFirst), + ["loop1", "loop2", "a", "b", "r"].map(str::to_owned) + ); + } +} diff --git a/src/export/run.rs b/src/export/run.rs new file mode 100644 index 0000000..70c5171 --- /dev/null +++ b/src/export/run.rs @@ -0,0 +1,319 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; +use std::fmt; +use std::io::{self, IsTerminal, Write as _}; +use std::mem; + +use serde_json::{Value, json}; + +use super::target::Maps; +use super::{Confirm, Export, Exporter, Plan, Prune}; +use crate::console::{Column, Counted, Initial, Table, Tone}; +use crate::error::Error; +use crate::jmap::request::{Request, SetRequest, URN_VACATION_RESPONSE}; +use crate::net::http::RetryStats; +use crate::sync::{PlanCounts, RunOutcome, Summary, SummaryType, TypeCounts}; +use crate::types::ObjectType; + +const PRUNE_SAMPLE: usize = 5; +const PRUNE_COLUMNS: [Column; 3] = [ + Column::left("Type"), + Column::right("Candidates"), + Column::left("Sample"), +]; + +struct PruneSample<'p>(&'p [String]); + +impl fmt::Display for PruneSample<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for (n, id) in self.0.iter().take(PRUNE_SAMPLE).enumerate() { + if n > 0 { + f.write_str(", ")?; + } + f.write_str(id)?; + } + if self.0.len() > PRUNE_SAMPLE { + f.write_str(", ...")?; + } + Ok(()) + } +} + +pub(super) struct ExportRun<'e, 'a> { + exporter: &'e Exporter<'a>, + config: &'e Export, + work: Vec, + counts: HashMap, + vacation: Option, + plan: Vec<(SummaryType, PlanCounts)>, +} + +impl<'e, 'a> ExportRun<'e, 'a> { + pub(super) fn execute(exporter: &'e Exporter<'a>, config: &'e Export) -> RunOutcome { + let mut run = ExportRun { + exporter, + config, + work: exporter.work_list(config), + counts: HashMap::new(), + vacation: None, + plan: Vec::new(), + }; + let result = run.phases(); + let stats = exporter.account.client().stats(); + match result { + Ok(()) if exporter.is_dry_run() => { + let plan = mem::take(&mut run.plan); + RunOutcome { + summary: Summary { + plan, + ..run.summary(RetryStats::default()) + }, + error: None, + } + } + Ok(()) => RunOutcome { + summary: run.summary(stats), + error: None, + }, + Err(e) => RunOutcome { + summary: run.summary(stats), + error: Some(e), + }, + } + } + + fn phases(&mut self) -> Result<(), Error> { + let exporter = self.exporter; + let logger = exporter.logger; + let mut maps = Maps::default(); + let mut plans: HashMap = HashMap::with_capacity(self.work.len()); + + for &ty in &self.work { + logger.info(format_args!("Exporting {}", Initial::Lower(ty.label()))); + let counts = self.counts.entry(ty).or_default(); + let plan = match exporter.reconcile(ty, &mut maps, counts) { + Ok(plan) => { + if exporter.is_dry_run() { + self.plan.push(( + ty.into(), + PlanCounts { + new: counts.created, + vanished: plan.prune_candidates.len() as u64, + present: counts.skipped, + }, + )); + } + plan + } + Err(e) if e.aborts_run() => return Err(e), + Err(e) => { + logger.warn(format_args!("{} aborted: {e}", Initial::Lower(ty.label()))); + counts.failed += 1; + Plan::default() + } + }; + plans.insert(ty, plan); + } + + if self.config.prune == Prune::Destroy { + self.prune_phase(&plans)?; + } + + if self.config.selects(ObjectType::SieveScript) + && exporter + .account + .session() + .may_support(exporter.account.account_id(), URN_VACATION_RESPONSE) + { + let mut counts = TypeCounts::default(); + let present = match exporter.reconcile_vacation(&mut counts) { + Ok(present) => present, + Err(e) if e.aborts_run() => { + self.vacation = Some(counts); + return Err(e); + } + Err(e) => { + logger.warn(format_args!( + "{} aborted: {e}", + SummaryType::VacationResponse.noun() + )); + counts.failed += 1; + true + } + }; + if present { + if exporter.is_dry_run() { + self.plan.push(( + SummaryType::VacationResponse, + PlanCounts { + new: counts.updated, + ..PlanCounts::default() + }, + )); + } + self.vacation = Some(counts); + } + } + Ok(()) + } + + fn summary(mut self, stats: RetryStats) -> Summary { + let mut summary = Summary::default(); + for ty in &self.work { + if let Some(counts) = self.counts.remove(ty) { + summary.per_type.push(((*ty).into(), counts)); + } + } + if let Some(counts) = self.vacation { + summary + .per_type + .push((SummaryType::VacationResponse, counts)); + } + summary.set_retry_stats(stats); + summary + } + + fn prune_phase(&mut self, plans: &HashMap) -> Result<(), Error> { + let pending = || { + self.work.iter().filter_map(|ty| { + plans + .get(ty) + .filter(|p| !p.prune_candidates.is_empty()) + .map(|p| (*ty, p)) + }) + }; + if pending().next().is_none() { + return Ok(()); + } + let styled = self.exporter.logger.console().err; + let mut table = Table::new(&PRUNE_COLUMNS); + let mut total = 0; + for (ty, p) in pending() { + total += p.prune_candidates.len() as u64; + table + .cell(ty.label(), None) + .count(p.prune_candidates.len() as u64, Some(Tone::Bad)) + .cell(PruneSample(&p.prune_candidates), Some(Tone::Dim)); + } + let total = Counted::new(total, "object", "objects"); + let mut err = io::stderr().lock(); + let _ = writeln!( + err, + "\n{}\n", + styled.paint( + Tone::Title, + format_args!("Prune plan: {total} on the target with no match in the archive") + ) + ); + let _ = table.write(styled, &mut err); + drop(err); + if self.exporter.is_dry_run() { + return Ok(()); + } + if self.config.confirm == Confirm::Prompt && io::stdin().is_terminal() { + eprint!("Destroy {total} on the target? [y/N] "); + let _ = io::stderr().flush(); + let mut line = String::new(); + io::stdin().read_line(&mut line).map_err(Error::partial)?; + if !hashify::set!(line.trim().as_bytes(), "y", "Y", "yes") { + return Err(Error::PruneAborted); + } + } + for ty in self.work.iter().rev() { + if let Some(plan) = plans.get(ty) + && !plan.prune_candidates.is_empty() + && let Some(counts) = self.counts.get_mut(ty) + { + self.exporter.destroy(*ty, plan, counts); + } + } + Ok(()) + } +} + +impl Exporter<'_> { + fn destroy(&self, ty: ObjectType, plan: &Plan, counts: &mut TypeCounts) { + if ty == ObjectType::SieveScript && plan.active_sieve_target.is_none() { + self.deactivate_sieve_scripts(); + } + let destroy = Value::Array( + plan.prune_candidates + .iter() + .map(|s| Value::String(s.clone())) + .collect(), + ); + let extra = ty.destroy_contents_arg(); + let request = SetRequest { + destroy: Some(destroy), + extra_args: extra.as_slice(), + ..Default::default() + }; + match self.account.set(ty.jmap_name(), request) { + Ok(outcome) => { + counts.deleted += outcome.destroyed.len() as u64; + for (id, err) in &outcome.not_destroyed { + self.logger.warn(format_args!( + "prune: {} {id} not destroyed: {err}", + ty.noun() + )); + counts.skipped += 1; + } + } + Err(e) => { + self.logger.warn(format_args!( + "prune: destroying {} failed: {e}", + Initial::Lower(ty.label()) + )); + counts.failed += plan.prune_candidates.len() as u64; + } + } + } + + fn deactivate_sieve_scripts(&self) { + let mut req = Request::new(); + req.call( + "SieveScript/set", + json!({ "accountId": self.account.account_id(), "onSuccessDeactivateScript": true }), + "d", + ); + let outcome = self + .account + .send(&req) + .and_then(|resp| resp.first()?.check_error()); + if let Err(e) = outcome { + self.logger.warn(format_args!( + "prune: deactivating sieve scripts before destroy failed: {e}" + )); + } + } +} + +impl ObjectType { + fn destroy_contents_arg(self) -> Option<(&'static str, Value)> { + let name = match self { + ObjectType::AddressBook => "onDestroyRemoveContents", + ObjectType::Calendar => "onDestroyRemoveEvents", + ObjectType::FileNode => "onDestroyRemoveChildren", + _ => return None, + }; + Some((name, Value::Bool(false))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // The sample shows the first five ids and marks the rest with an ellipsis. + #[test] + fn prune_sample_truncates_after_five() { + let ids: Vec = (1..=7).map(|n| format!("m{n}")).collect(); + assert_eq!(PruneSample(&ids).to_string(), "m1, m2, m3, m4, m5, ..."); + assert_eq!(PruneSample(&ids[..2]).to_string(), "m1, m2"); + } +} diff --git a/src/export/sieve.rs b/src/export/sieve.rs new file mode 100644 index 0000000..dd0e681 --- /dev/null +++ b/src/export/sieve.rs @@ -0,0 +1,162 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; + +use serde_json::{Map, Value, json}; + +use super::target::{ClientId, TargetJson}; +use super::uploader::Uploader; +use super::{Exporter, Plan}; +use crate::archive::LocalId; +use crate::error::Error; +use crate::jmap::request::{Request, URN_VACATION_RESPONSE}; +use crate::jmap::wire::vacation_response::VacationResponse; +use crate::sync::TypeCounts; +use crate::sync::jmap_map::SieveRow; +use crate::types::ObjectType; + +const SIEVE_MEDIA_TYPE: &str = "application/sieve"; + +impl Exporter<'_> { + pub(super) fn reconcile_sieve(&self, counts: &mut TypeCounts) -> Result { + let ty = ObjectType::SieveScript; + let targets = self.get_all(ty)?; + let mut target_by_name: HashMap = HashMap::with_capacity(targets.len()); + for t in &targets { + let t = TargetJson(t); + if let (Some(id), Some(name)) = (t.id(), t.text("name")) { + target_by_name.insert(name.to_owned(), id.to_owned()); + } + } + + let target_has_vacation = self + .account + .session() + .may_support(self.account.account_id(), URN_VACATION_RESPONSE); + let vacation_exported = + target_has_vacation && self.store.vacation().exists().map_err(Error::partial)?; + + let mut locals = self.sieve_locals()?; + if vacation_exported { + let before = locals.len(); + locals.retain(|(_, row)| { + !row.name + .as_deref() + .is_some_and(VacationResponse::is_reserved_script_name) + }); + counts.skipped += (before - locals.len()) as u64; + } + + let mut active_target: Option = None; + let mut uploader = Uploader::new(self); + for (local, row) in &locals { + let matched = row + .name + .as_ref() + .and_then(|n| target_by_name.get(n)) + .cloned(); + let target_id = if let Some(id) = matched { + counts.skipped += 1; + id + } else { + let cid = ClientId::object(*local); + let build = |up: &mut Uploader<'_>| -> Result { + let blob_id = up.upload(row.blob, SIEVE_MEDIA_TYPE)?; + let mut obj = Map::new(); + if let Some(n) = &row.name { + obj.insert("name".to_owned(), Value::String(n.clone())); + } + obj.insert("blobId".to_owned(), Value::String(blob_id.0)); + Ok(Value::Object(obj)) + }; + uploader.clear_touched(); + let wire = build(&mut uploader)?; + let outcome = self.create(ty, vec![(cid.clone(), wire)])?; + let outcome = + self.retry_if_blob_missing(ty, &cid, &mut uploader, outcome, build)?; + match outcome + .created + .first() + .and_then(|(_, v)| TargetJson(v).id()) + { + Some(id) => { + counts.created += 1; + if let Some(n) = &row.name { + target_by_name.insert(n.clone(), id.to_owned()); + } + id.to_owned() + } + None => { + for (cid, err) in &outcome.not_created { + self.logger + .warn(format_args!("sieve script {cid} not created: {err}")); + } + counts.failed += 1; + continue; + } + } + }; + if row.is_active { + active_target = Some(target_id); + } + } + + let deactivate = active_target.is_none() && locals.iter().all(|(_, row)| !row.is_active); + if !self.is_dry_run() { + let account_id = self.account.account_id(); + let args = if let Some(id) = &active_target { + json!({ "accountId": account_id, "onSuccessActivateScript": id }) + } else if deactivate { + json!({ "accountId": account_id, "onSuccessDeactivateScript": true }) + } else { + json!({ "accountId": account_id }) + }; + let mut req = Request::new(); + req.call("SieveScript/set", args, "a"); + if let Err(e) = self.account.send(&req) { + self.logger + .warn(format_args!("sieve script activation failed: {e}")); + } + } + + let local_names: HashSet<&str> = locals + .iter() + .filter_map(|(_, row)| row.name.as_deref()) + .collect(); + let mut prune_candidates: Vec = target_by_name + .iter() + .filter(|(name, _)| !local_names.contains(name.as_str())) + .filter(|(name, _)| { + !(target_has_vacation && VacationResponse::is_reserved_script_name(name)) + }) + .map(|(_, id)| id.clone()) + .collect(); + prune_candidates.sort_unstable(); + + Ok(Plan { + prune_candidates, + active_sieve_target: active_target, + }) + } + + fn sieve_locals(&self) -> Result, Error> { + let mut stmt = self + .store + .connection() + .prepare(SieveRow::SELECT) + .map_err(Error::partial)?; + let rows = stmt + .query_map([], |row| Ok((row.get(0)?, SieveRow::from_row(row)))) + .map_err(Error::partial)?; + let rows: Vec<_> = rows.collect::>().map_err(Error::partial)?; + let mut out = Vec::with_capacity(rows.len()); + for (id, sieve) in rows { + out.push((id, sieve?)); + } + Ok(out) + } +} diff --git a/src/export/target.rs b/src/export/target.rs new file mode 100644 index 0000000..33e32b7 --- /dev/null +++ b/src/export/target.rs @@ -0,0 +1,192 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; +use std::fmt; + +use serde_json::{Map, Value, json}; + +use super::Exporter; +use super::uploader::Uploader; +use crate::archive::LocalId; +use crate::error::Error; +use crate::jmap::request::{SetOutcome, SetRequest}; +use crate::jmap::wire::JmapId; +use crate::sync::TypeCounts; +use crate::types::ObjectType; + +#[derive(Default)] +pub(super) struct Maps(HashMap>); + +impl Maps { + pub(super) fn insert(&mut self, ty: ObjectType, local: LocalId, target: JmapId) { + self.0.entry(ty).or_default().insert(local, target); + } + + pub(super) fn target(&self, ty: ObjectType, local: LocalId) -> Option { + self.0.get(&ty)?.get(&local).cloned() + } + + pub(super) fn record_created( + &mut self, + ty: ObjectType, + outcome: &SetOutcome, + counts: &mut TypeCounts, + ) { + for (cid, v) in &outcome.created { + if let Some(local) = ClientId::parse(cid) + && let Some(id) = TargetJson(v).id() + { + self.insert(ty, local, JmapId(id.to_owned())); + counts.created += 1; + } + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct ClientId(String); + +impl ClientId { + pub(super) fn object(local: LocalId) -> ClientId { + ClientId(format!("c{local}")) + } + + pub(super) fn email(local: LocalId) -> ClientId { + ClientId(format!("e{local}")) + } + + pub(super) fn parse(cid: &str) -> Option { + cid.strip_prefix('c')? + .parse::() + .ok() + .map(LocalId::from_raw) + } + + pub(super) fn as_str(&self) -> &str { + &self.0 + } + + pub(super) fn into_string(self) -> String { + self.0 + } +} + +impl fmt::Display for ClientId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +#[derive(Clone, Copy)] +pub(super) struct TargetJson<'v>(pub(super) &'v Value); + +impl<'v> TargetJson<'v> { + pub(super) fn id(self) -> Option<&'v str> { + self.text("id") + } + + pub(super) fn text(self, key: &str) -> Option<&'v str> { + self.0.get(key)?.as_str() + } + + pub(super) fn flag(self, key: &str) -> Option { + self.0.get(key)?.as_bool() + } + + pub(super) fn may_delete(self) -> Option { + self.0.get("myRights")?.get("mayDelete")?.as_bool() + } + + pub(super) fn error_type(self) -> Option<&'v str> { + self.text("type") + } + + pub(super) fn existing_id(self) -> Option<&'v str> { + if self.error_type() == Some("alreadyExists") { + self.text("existingId") + } else { + None + } + } +} + +impl Exporter<'_> { + pub(super) fn query_get( + &self, + ty: ObjectType, + props: Option<&[&str]>, + ) -> Result, Error> { + let ids = self.account.query_all_ids(ty.jmap_name())?; + if ids.is_empty() { + return Ok(Vec::new()); + } + Ok(self.account.get::(ty.jmap_name(), &ids, props)?.list) + } + + pub(super) fn get_all(&self, ty: ObjectType) -> Result, Error> { + Ok(self.account.get_all::(ty.jmap_name())?.list) + } + + pub(super) fn create( + &self, + ty: ObjectType, + creates: Vec<(ClientId, Value)>, + ) -> Result { + if self.is_dry_run() { + let mut outcome = SetOutcome::default(); + outcome.created.reserve(creates.len()); + for (cid, _) in creates { + let synthetic = json!({ "id": format!("dryrun-{}-{cid}", ty.jmap_name()) }); + outcome.created.push((cid.into_string(), synthetic)); + } + return Ok(outcome); + } + let map: Map = creates + .into_iter() + .map(|(cid, obj)| (cid.into_string(), obj)) + .collect(); + Ok(self.account.set( + ty.jmap_name(), + SetRequest { + create: Some(Value::Object(map)), + ..Default::default() + }, + )?) + } + + pub(super) fn retry_if_blob_missing( + &self, + ty: ObjectType, + cid: &ClientId, + uploader: &mut Uploader<'_>, + outcome: SetOutcome, + mut rebuild: impl FnMut(&mut Uploader<'_>) -> Result, + ) -> Result { + let blob_missing = outcome.not_created.iter().any(|(c, err)| { + c == cid.as_str() && TargetJson(err).error_type() == Some("blobNotFound") + }); + if !blob_missing { + return Ok(outcome); + } + uploader.invalidate_touched(); + let wire = rebuild(uploader)?; + uploader.clear_touched(); + self.create(ty, vec![(cid.clone(), wire)]) + } + + pub(super) fn warn_not_created( + &self, + ty: ObjectType, + cid: &str, + err: &Value, + counts: &mut TypeCounts, + ) { + self.logger + .warn(format_args!("{} {cid} not created: {err}", ty.noun())); + counts.failed += 1; + } +} diff --git a/src/export/tree.rs b/src/export/tree.rs new file mode 100644 index 0000000..3cea091 --- /dev/null +++ b/src/export/tree.rs @@ -0,0 +1,419 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; + +use rusqlite::params; +use serde_json::Value; + +use super::prune::{DestroyOrder, TargetObj}; +use super::target::{ClientId, Maps, TargetJson}; +use super::uploader::Uploader; +use super::{Exporter, Plan}; +use crate::archive::LocalId; +use crate::error::Error; +use crate::jmap::request::SetOutcome; +use crate::jmap::wire::JmapId; +use crate::jmap::wire::file_node::NodeKind; +use crate::jmap::wire::mailbox::Mailbox; +use crate::sync::TypeCounts; +use crate::sync::jmap_map::FileNodeRow; +use crate::sync::keys::FoldedName; +use crate::types::ObjectType; + +const TARGET_PROPS: [&str; 4] = ["role", "name", "parentId", "myRights"]; +const DEFAULT_MEDIA_TYPE: &str = "application/octet-stream"; + +struct LocalNode { + local: LocalId, + parent: Option, + name: FoldedName, + role: Option, +} + +struct TargetNode { + id: String, + parent: Option, + name: FoldedName, + role: Option, + may_delete: bool, +} + +struct TargetTree<'t> { + roots: Vec<&'t TargetNode>, + children: HashMap<&'t str, Vec<&'t TargetNode>>, +} + +enum ParentTarget { + Root, + Created(JmapId), + Missing(LocalId), +} + +impl LocalNode { + fn depth(&self, parents: &HashMap>) -> usize { + let mut depth = 0; + let mut cur = parents.get(&self.local).copied().flatten(); + let mut seen = HashSet::new(); + while let Some(p) = cur { + if !seen.insert(p) { + break; + } + depth += 1; + cur = parents.get(&p).copied().flatten(); + } + depth + } + + fn parent_target(&self, ty: ObjectType, maps: &Maps) -> ParentTarget { + match self.parent { + None => ParentTarget::Root, + Some(p) => maps + .target(ty, p) + .map_or(ParentTarget::Missing(p), ParentTarget::Created), + } + } +} + +impl ParentTarget { + fn id(&self) -> Option<&str> { + match self { + ParentTarget::Created(id) => Some(&id.0), + ParentTarget::Root | ParentTarget::Missing(_) => None, + } + } +} + +impl<'t> TargetTree<'t> { + fn new(targets: &'t [TargetNode]) -> TargetTree<'t> { + let mut tree = TargetTree { + roots: Vec::new(), + children: HashMap::new(), + }; + for t in targets { + match t.parent.as_deref() { + None => tree.roots.push(t), + Some(p) => tree.children.entry(p).or_default().push(t), + } + } + tree + } + + fn children(&self, parent: Option<&str>) -> &[&'t TargetNode] { + match parent { + None => &self.roots, + Some(p) => self.children.get(p).map_or(&[], Vec::as_slice), + } + } + + fn named(&self, parent: Option<&str>, name: &FoldedName) -> Option<&'t TargetNode> { + self.children(parent) + .iter() + .find(|t| &t.name == name) + .copied() + } +} + +impl Exporter<'_> { + pub(super) fn reconcile_tree( + &self, + ty: ObjectType, + maps: &mut Maps, + counts: &mut TypeCounts, + ) -> Result { + let locals = self.tree_locals(ty)?; + let targets = self.tree_targets(ty)?; + let tree = TargetTree::new(&targets); + + let mut matched: HashMap = HashMap::new(); + let mut tmatched: HashSet = HashSet::new(); + + if ty == ObjectType::Mailbox { + for t in &targets { + if let Some(r) = &t.role + && let Some(l) = locals.iter().find(|l| { + l.role.as_deref() == Some(r.as_str()) && !matched.contains_key(&l.local) + }) + { + matched.insert(l.local, &t.id); + tmatched.insert(t.id.clone()); + } + } + } + + let mut local_children: HashMap, Vec<&LocalNode>> = HashMap::new(); + for n in &locals { + local_children.entry(n.parent).or_default().push(n); + } + let mut stack: Vec<(Option, Option<&str>)> = Vec::with_capacity(matched.len() + 1); + stack.push((None, None)); + stack.extend(matched.iter().map(|(l, t)| (Some(*l), Some(*t)))); + while let Some((lp, tp)) = stack.pop() { + for ln in local_children.get(&lp).into_iter().flatten() { + if matched.contains_key(&ln.local) { + continue; + } + if let Some(tn) = tree + .children(tp) + .iter() + .find(|t| !tmatched.contains(&t.id) && t.name == ln.name) + { + matched.insert(ln.local, &tn.id); + tmatched.insert(tn.id.clone()); + stack.push((Some(ln.local), Some(&tn.id))); + } + } + } + + for (l, tid) in &matched { + maps.insert(ty, *l, JmapId((*tid).to_owned())); + } + counts.skipped += matched.len() as u64; + + let parents: HashMap> = + locals.iter().map(|n| (n.local, n.parent)).collect(); + let mut to_create: Vec<(usize, &LocalNode)> = locals + .iter() + .filter(|n| !matched.contains_key(&n.local)) + .map(|n| (n.depth(&parents), n)) + .collect(); + to_create.sort_by_key(|(depth, _)| *depth); + + let mut uploader = Uploader::new(self); + let mut taken_roles: HashSet<&str> = + targets.iter().filter_map(|t| t.role.as_deref()).collect(); + for level in to_create.chunk_by(|a, b| a.0 == b.0) { + let mut batch: Vec<(ClientId, Value)> = Vec::new(); + for (_, n) in level { + let parent = match n.parent_target(ty, maps) { + ParentTarget::Missing(p) => { + self.logger.warn(format_args!( + "{} local {} skipped: parent {p} not created", + ty.noun(), + n.local + )); + counts.failed += 1; + continue; + } + parent => parent, + }; + if let Some(t) = tree.named(parent.id(), &n.name) { + self.record_merge( + ty, + n.local, + t.id.clone(), + "name already present on target", + maps, + &mut tmatched, + ); + counts.skipped += 1; + continue; + } + if ty == ObjectType::FileNode { + let cid = ClientId::object(n.local); + uploader.clear_touched(); + let obj = match self.tree_create(ty, n.local, maps, &mut uploader) { + Ok(o) => o, + Err(e) => { + self.warn_tree_skip(ty, n.local, &e, counts); + continue; + } + }; + let outcome = self.create(ty, vec![(cid.clone(), obj)])?; + match self.retry_if_blob_missing(ty, &cid, &mut uploader, outcome, |up| { + self.tree_create(ty, n.local, maps, up) + }) { + Ok(outcome) => { + self.absorb_tree_outcome(ty, &outcome, maps, &mut tmatched, counts) + } + Err(e) => self.warn_tree_skip(ty, n.local, &e, counts), + } + continue; + } + match self.tree_create(ty, n.local, maps, &mut uploader) { + Ok(mut obj) => { + if let Some(r) = n.role.as_deref() { + if taken_roles.contains(r) { + if let Value::Object(m) = &mut obj { + m.remove("role"); + } + self.logger.warn(format_args!( + "mailbox local {}: role \"{r}\" already on the target, creating a plain folder", + n.local + )); + } else { + taken_roles.insert(r); + } + } + batch.push((ClientId::object(n.local), obj)); + } + Err(e) => self.warn_tree_skip(ty, n.local, &e, counts), + } + } + if batch.is_empty() { + continue; + } + let outcome = self.create(ty, batch)?; + self.absorb_tree_outcome(ty, &outcome, maps, &mut tmatched, counts); + } + + let objs: Vec> = targets + .iter() + .map(|t| TargetObj { + id: &t.id, + matched: tmatched.contains(&t.id), + protected: t.role.is_some(), + may_delete: t.may_delete, + parent: t.parent.as_deref(), + }) + .collect(); + Ok(Plan::prune(TargetObj::candidates( + &objs, + DestroyOrder::LeafFirst, + ))) + } + + fn tree_locals(&self, ty: ObjectType) -> Result, Error> { + let table = ty.table(); + let mut stmt = self + .store + .connection() + .prepare(&format!("SELECT id, parent_id, name, role FROM {table}")) + .map_err(Error::partial)?; + stmt.query_map([], |r| { + Ok(LocalNode { + local: r.get(0)?, + parent: r.get(1)?, + name: FoldedName::new(&r.get::<_, String>(2)?), + role: r.get(3)?, + }) + }) + .and_then(Iterator::collect) + .map_err(Error::partial) + } + + fn tree_targets(&self, ty: ObjectType) -> Result, Error> { + let list = self.query_get(ty, Some(&TARGET_PROPS))?; + let mut own_rights: HashMap<&str, bool> = HashMap::with_capacity(list.len()); + for v in &list { + let v = TargetJson(v); + if let (Some(id), Some(may)) = (v.id(), v.may_delete()) { + own_rights.insert(id, may); + } + } + Ok(list + .iter() + .filter_map(|v| { + let v = TargetJson(v); + let id = v.id()?; + let parent = v.text("parentId"); + let may_delete = own_rights + .get(id) + .or_else(|| parent.and_then(|p| own_rights.get(p))) + .copied() + .unwrap_or(true); + Some(TargetNode { + id: id.to_owned(), + parent: parent.map(str::to_owned), + name: FoldedName::new(v.text("name")?), + role: v.text("role").map(str::to_owned), + may_delete, + }) + }) + .collect()) + } + + fn record_merge( + &self, + ty: ObjectType, + local: LocalId, + target_id: String, + reason: &str, + maps: &mut Maps, + tmatched: &mut HashSet, + ) { + self.logger.warn(format_args!( + "{} local {local} merged into existing target {target_id}: {reason}", + ty.noun() + )); + maps.insert(ty, local, JmapId(target_id.clone())); + tmatched.insert(target_id); + } + + fn absorb_tree_outcome( + &self, + ty: ObjectType, + outcome: &SetOutcome, + maps: &mut Maps, + tmatched: &mut HashSet, + counts: &mut TypeCounts, + ) { + maps.record_created(ty, outcome, counts); + for (cid, err) in &outcome.not_created { + if let Some(local) = ClientId::parse(cid) + && let Some(existing) = TargetJson(err).existing_id() + { + self.record_merge( + ty, + local, + existing.to_owned(), + "alreadyExists: reusing existing target", + maps, + tmatched, + ); + counts.skipped += 1; + continue; + } + self.warn_not_created(ty, cid, err, counts); + } + } + + fn warn_tree_skip(&self, ty: ObjectType, local: LocalId, e: &Error, counts: &mut TypeCounts) { + self.logger + .warn(format_args!("{} local {local} skipped: {e}", ty.noun())); + counts.failed += 1; + } + + fn tree_create( + &self, + ty: ObjectType, + local: LocalId, + maps: &Maps, + uploader: &mut Uploader<'_>, + ) -> Result { + let conn = self.store.connection(); + let resolve = |ty, id| maps.target(ty, id); + if ty == ObjectType::Mailbox { + let wire = conn + .query_row( + &format!("{} WHERE id = ?1", Mailbox::SELECT), + params![local], + |row| Ok(Mailbox::from_row(row, &resolve)), + ) + .map_err(Error::partial)??; + return serde_json::to_value(&wire).map_err(Error::partial); + } + let row = conn + .query_row( + &format!("{} WHERE id = ?1", FileNodeRow::SELECT), + params![local], + |row| Ok(FileNodeRow::from_row(row, &resolve)), + ) + .map_err(Error::partial)??; + let mut wire = row.wire; + if let (NodeKind::File(file), Some(blob)) = (&mut wire.kind, row.blob) { + let media_type = file.media_type.as_deref().unwrap_or(DEFAULT_MEDIA_TYPE); + file.blob_id = Some(uploader.upload(blob, media_type)?); + } + let mut obj = serde_json::to_value(&wire).map_err(Error::partial)?; + if let Value::Object(m) = &mut obj { + m.remove("created"); + m.remove("modified"); + m.remove("isSubscribed"); + } + Ok(obj) + } +} diff --git a/src/export/uidtype.rs b/src/export/uidtype.rs new file mode 100644 index 0000000..536d4d0 --- /dev/null +++ b/src/export/uidtype.rs @@ -0,0 +1,189 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; +use std::fmt; + +use rusqlite::params; +use serde::Deserialize; +use serde_json::Value; + +use super::prune::{DestroyOrder, TargetObj}; +use super::target::{ClientId, Maps, TargetJson}; +use super::uploader::Uploader; +use super::{Exporter, Plan}; +use crate::archive::LocalId; +use crate::error::Error; +use crate::jmap::wire::JmapId; +use crate::jmap::wire::calendar_event::CalendarEvent; +use crate::jmap::wire::contact_card::ContactCard; +use crate::sync::TypeCounts; +use crate::types::ObjectType; + +#[derive(Deserialize)] +struct EventUid<'d> { + #[serde(borrow)] + uid: Option>, +} + +struct ItemLabel<'u> { + ty: ObjectType, + local: LocalId, + uid: &'u str, +} + +impl fmt::Display for ItemLabel<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{} local {}", self.ty.jmap_name(), self.local)?; + if !self.uid.is_empty() { + write!(f, " (uid {})", self.uid)?; + } + Ok(()) + } +} + +impl Exporter<'_> { + pub(super) fn reconcile_uid_type( + &self, + ty: ObjectType, + maps: &mut Maps, + counts: &mut TypeCounts, + ) -> Result { + let targets = self.query_get(ty, None)?; + let mut by_uid: HashMap<&str, &str> = HashMap::with_capacity(targets.len()); + for t in &targets { + let t = TargetJson(t); + if let (Some(uid), Some(id)) = (t.text("uid"), t.id()) { + by_uid.entry(uid).or_insert(id); + } + } + + let rows = self.uid_locals(ty)?; + let mut matched_uids: HashSet<&str> = HashSet::new(); + let uploader = Uploader::new(self); + for (local, uid) in &rows { + let label = ItemLabel { + ty, + local: *local, + uid, + }; + if let Some((target_uid, tid)) = by_uid.get_key_value(uid.as_str()) { + maps.insert(ty, *local, JmapId((*tid).to_owned())); + matched_uids.insert(target_uid); + counts.skipped += 1; + continue; + } + let wire = match self.uid_create(ty, *local, maps, &uploader) { + Ok(w) => w, + Err(e) if e.aborts_run() => return Err(e), + Err(e) => { + self.logger.warn(format_args!("{label} skipped: {e}")); + counts.failed += 1; + continue; + } + }; + let outcome = match self.create(ty, vec![(ClientId::object(*local), wire)]) { + Ok(o) => o, + Err(e) if e.aborts_run() => return Err(e), + Err(e) => { + self.logger.warn(format_args!("{label} not created: {e}")); + counts.failed += 1; + continue; + } + }; + maps.record_created(ty, &outcome, counts); + for (cid, err) in &outcome.not_created { + self.warn_not_created(ty, cid, err, counts); + } + } + + let objs: Vec> = targets + .iter() + .filter_map(|t| { + let t = TargetJson(t); + Some(TargetObj { + id: t.id()?, + matched: t.text("uid").is_some_and(|u| matched_uids.contains(u)), + protected: false, + may_delete: true, + parent: None, + }) + }) + .collect(); + Ok(Plan::prune(TargetObj::candidates( + &objs, + DestroyOrder::ById, + ))) + } + + fn uid_locals(&self, ty: ObjectType) -> Result, Error> { + let select = if ty == ObjectType::ContactCard { + ContactCard::SELECT + } else { + CalendarEvent::SELECT + }; + let mut stmt = self + .store + .connection() + .prepare(select) + .map_err(Error::partial)?; + stmt.query_map([], |r| { + let uid = if ty == ObjectType::ContactCard { + r.get::<_, String>(1)? + } else { + let data = r.get_ref(4)?.as_str()?; + serde_json::from_str::>(data) + .ok() + .and_then(|e| e.uid) + .map(Cow::into_owned) + .unwrap_or_default() + }; + Ok((r.get(0)?, uid)) + }) + .and_then(Iterator::collect) + .map_err(Error::partial) + } + + fn uid_create( + &self, + ty: ObjectType, + local: LocalId, + maps: &Maps, + uploader: &Uploader<'_>, + ) -> Result { + let conn = self.store.connection(); + let resolve = |ty, id| maps.target(ty, id); + let bytes = |blob| uploader.bytes(blob); + if ty == ObjectType::ContactCard { + let (uid, books, data): (String, String, String) = conn + .query_row( + &format!("{} WHERE id = ?1", ContactCard::SELECT), + params![local], + |r| Ok((r.get(1)?, r.get(2)?, r.get(3)?)), + ) + .map_err(Error::partial)?; + return Ok(ContactCard::export_json( + &uid, &books, &data, &resolve, &bytes, + )?); + } + let (calendars, is_draft, use_default_alerts, data): (String, i64, i64, String) = conn + .query_row( + &format!("{} AND id = ?1", CalendarEvent::SELECT), + params![local], + |r| Ok((r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)), + ) + .map_err(Error::partial)?; + Ok(CalendarEvent::export_json( + &calendars, + is_draft != 0, + use_default_alerts != 0, + &data, + &resolve, + &bytes, + )?) + } +} diff --git a/src/export/uploader.rs b/src/export/uploader.rs new file mode 100644 index 0000000..95fa53e --- /dev/null +++ b/src/export/uploader.rs @@ -0,0 +1,88 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use super::Exporter; +use crate::archive::BlobId; +use crate::jmap::error::JmapError; +use crate::jmap::wire::JmapId; + +pub(super) struct Uploader<'a> { + exporter: Exporter<'a>, + cache: HashMap, + touched: Vec, +} + +impl<'a> Uploader<'a> { + pub(super) fn new(exporter: &Exporter<'a>) -> Uploader<'a> { + Uploader { + exporter: *exporter, + cache: HashMap::new(), + touched: Vec::new(), + } + } + + pub(super) fn upload(&mut self, blob: BlobId, content_type: &str) -> Result { + self.touched.push(blob); + if let Some(id) = self.cache.get(&blob) { + return Ok(id.clone()); + } + let id = if self.exporter.is_dry_run() { + self.exporter + .store + .blobs() + .len(blob)? + .ok_or_else(|| Uploader::missing(blob))?; + JmapId(format!("dryrun-blob-{blob}")) + } else { + self.exporter + .account + .upload(&self.bytes(blob)?, content_type)? + }; + self.cache.insert(blob, id.clone()); + Ok(id) + } + + pub(super) fn bytes(&self, blob: BlobId) -> Result, JmapError> { + self.exporter + .store + .blobs() + .bytes(blob)? + .ok_or_else(|| Uploader::missing(blob)) + } + + pub(super) fn len(&self, blob: BlobId) -> Option { + self.exporter.store.blobs().len(blob).ok().flatten() + } + + pub(super) fn cached(&self, blob: BlobId) -> Option<&JmapId> { + self.cache.get(&blob) + } + + pub(super) fn remember(&mut self, blob: BlobId, id: JmapId) { + self.cache.insert(blob, id); + } + + pub(super) fn invalidate(&mut self, blob: BlobId) { + self.cache.remove(&blob); + } + + pub(super) fn clear_touched(&mut self) { + self.touched.clear(); + } + + pub(super) fn invalidate_touched(&mut self) { + for blob in self.touched.drain(..) { + self.cache.remove(&blob); + } + } + + #[cold] + fn missing(blob: BlobId) -> JmapError { + JmapError::malformed(format!("blob local id {blob} missing")) + } +} diff --git a/src/export/vacation.rs b/src/export/vacation.rs new file mode 100644 index 0000000..92c7c4b --- /dev/null +++ b/src/export/vacation.rs @@ -0,0 +1,40 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::json; + +use super::Exporter; +use crate::error::Error; +use crate::jmap::request::SetRequest; +use crate::jmap::wire::vacation_response::{SINGLETON_ID, TYPE_NAME, VacationResponse}; +use crate::sync::TypeCounts; + +impl Exporter<'_> { + pub(super) fn reconcile_vacation(&self, counts: &mut TypeCounts) -> Result { + let Some(stored) = self.store.vacation().load().map_err(Error::partial)? else { + return Ok(false); + }; + if self.is_dry_run() { + counts.updated += 1; + return Ok(true); + } + let vacation = VacationResponse::from(stored); + let outcome = self.account.set( + TYPE_NAME, + SetRequest { + update: Some(json!({ SINGLETON_ID: vacation })), + ..Default::default() + }, + )?; + counts.updated += outcome.updated.len() as u64; + for (id, err) in &outcome.not_updated { + self.logger + .warn(format_args!("vacation response {id} not updated: {err}")); + counts.failed += 1; + } + Ok(true) + } +} diff --git a/src/imap/automap.rs b/src/imap/automap.rs index 52d1a94..b174e77 100644 --- a/src/imap/automap.rs +++ b/src/imap/automap.rs @@ -4,116 +4,111 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -pub fn role_from_special_use(attributes: &[String]) -> Option<&'static str> { - for attr in attributes { - match attr.to_ascii_lowercase().as_str() { - "\\all" => return Some("all"), - "\\archive" => return Some("archive"), - "\\drafts" => return Some("drafts"), - "\\flagged" => return Some("flagged"), - "\\important" => return Some("important"), - "\\junk" => return Some("junk"), - "\\sent" => return Some("sent"), - "\\trash" => return Some("trash"), - _ => {} +type Role = &'static str; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Automap { + Enabled, + Disabled, +} + +pub struct SpecialUse; + +impl SpecialUse { + pub fn role(attributes: &[String]) -> Option<&'static str> { + attributes.iter().find_map(|attr| { + hashify::map_ignore_case!(attr.as_bytes(), Role, + "\\all" => "all", + "\\archive" => "archive", + "\\drafts" => "drafts", + "\\flagged" => "flagged", + "\\important" => "important", + "\\junk" => "junk", + "\\sent" => "sent", + "\\trash" => "trash", + ) + .copied() + }) + } +} + +impl Automap { + pub fn role( + self, + folder_name: &str, + attributes: &[String], + namespace_prefix: &str, + ) -> Option<&'static str> { + if folder_name.eq_ignore_ascii_case("INBOX") { + return Some("inbox"); } - } - None -} - -pub fn role_for_folder( - folder_name: &str, - attributes: &[String], - namespace_prefix: &str, - automap_enabled: bool, -) -> Option<&'static str> { - if folder_name.eq_ignore_ascii_case("INBOX") { - return Some("inbox"); - } - if let Some(r) = role_from_special_use(attributes) { - return Some(r); - } - if !automap_enabled { - return None; - } - let stripped = strip_prefix_case_insensitive(folder_name, namespace_prefix); - let leaf = leaf_component(stripped); - role_from_name(stripped, leaf) -} - -fn role_from_name(full: &str, leaf: &str) -> Option<&'static str> { - for (role, patterns) in ROLE_PATTERNS { - for p in *patterns { - if leaf.eq_ignore_ascii_case(p) || full.eq_ignore_ascii_case(p) { - return Some(role); - } + if let Some(role) = SpecialUse::role(attributes) { + return Some(role); } + if self == Automap::Disabled { + return None; + } + let stripped = folder_name + .get(..namespace_prefix.len()) + .filter(|head| head.eq_ignore_ascii_case(namespace_prefix)) + .and_then(|_| folder_name.get(namespace_prefix.len()..)) + .unwrap_or(folder_name); + let leaf = stripped + .rsplit_once(['/', '.']) + .map_or(stripped, |(_, leaf)| leaf); + Automap::role_from_name(leaf).or_else(|| Automap::role_from_name(stripped)) } - None -} -fn strip_prefix_case_insensitive<'a>(name: &'a str, prefix: &str) -> &'a str { - if prefix.is_empty() { - return name; - } - if name.len() < prefix.len() { - return name; - } - if name[..prefix.len()].eq_ignore_ascii_case(prefix) { - &name[prefix.len()..] - } else { - name + fn role_from_name(name: &str) -> Option<&'static str> { + hashify::map_ignore_case!(name.as_bytes(), Role, + "Sent" => "sent", + "Sent Mail" => "sent", + "Sent Items" => "sent", + "Sent Messages" => "sent", + "Sent-Mail" => "sent", + "[Gmail]/Sent Mail" => "sent", + "[Gmail]/Sent" => "sent", + "Drafts" => "drafts", + "Draft" => "drafts", + "Brouillons" => "drafts", + "[Gmail]/Drafts" => "drafts", + "Trash" => "trash", + "Deleted" => "trash", + "Deleted Items" => "trash", + "Deleted Messages" => "trash", + "[Gmail]/Trash" => "trash", + "[Gmail]/Bin" => "trash", + "Junk" => "junk", + "Spam" => "junk", + "Junk E-mail" => "junk", + "Junk Email" => "junk", + "[Gmail]/Spam" => "junk", + "[Gmail]/All Mail" => "all", + "Archive" => "archive", + "Archives" => "archive", + ) + .copied() } } -fn leaf_component(name: &str) -> &str { - if let Some(idx) = name.rfind(['/', '.']) { - &name[idx + 1..] - } else { - name - } -} - -const ROLE_PATTERNS: &[(&str, &[&str])] = &[ - ( - "sent", - &[ - "Sent", - "Sent Mail", - "Sent Items", - "Sent Messages", - "Sent-Mail", - "[Gmail]/Sent Mail", - "[Gmail]/Sent", - ], - ), - ( - "drafts", - &["Drafts", "Draft", "Brouillons", "[Gmail]/Drafts"], - ), - ( - "trash", - &[ - "Trash", - "Deleted", - "Deleted Items", - "Deleted Messages", - "[Gmail]/Trash", - "[Gmail]/Bin", - ], - ), - ( - "junk", - &["Junk", "Spam", "Junk E-mail", "Junk Email", "[Gmail]/Spam"], - ), - ("all", &["[Gmail]/All Mail"]), - ("archive", &["Archive", "Archives"]), -]; - #[cfg(test)] mod tests { use super::*; + fn role_for_folder( + name: &str, + attributes: &[String], + prefix: &str, + enabled: bool, + ) -> Option<&'static str> { + let automap = if enabled { + Automap::Enabled + } else { + Automap::Disabled + }; + automap.role(name, attributes, prefix) + } + fn attrs(s: &[&str]) -> Vec { s.iter().map(|x| (*x).to_owned()).collect() } @@ -145,7 +140,7 @@ mod tests { ("\\Sent", "sent"), ("\\Trash", "trash"), ] { - let r = role_from_special_use(&attrs(&[attr])); + let r = SpecialUse::role(&attrs(&[attr])); assert_eq!(r, Some(expected), "for {attr}"); } } @@ -162,7 +157,7 @@ mod tests { "\\Noinferiors", "\\Remote", ] { - assert_eq!(role_from_special_use(&attrs(&[s])), None); + assert_eq!(SpecialUse::role(&attrs(&[s])), None); } } @@ -233,4 +228,10 @@ mod tests { assert_eq!(role_for_folder("Archive", &[], "", true), Some("archive")); assert_eq!(role_for_folder("Archives", &[], "", true), Some("archive")); } + + #[test] + fn namespace_prefix_inside_a_multibyte_char_does_not_panic() { + assert_eq!(role_for_folder("a日本", &[], "INBOX.", true), None); + assert_eq!(role_for_folder("日本/Sent", &[], "日", true), Some("sent")); + } } diff --git a/src/imap/client.rs b/src/imap/client.rs index 066ce26..64fbc1e 100644 --- a/src/imap/client.rs +++ b/src/imap/client.rs @@ -5,37 +5,106 @@ */ use std::collections::BTreeSet; -use std::io::{BufReader, Read, Write}; +use std::fmt::Write as _; +use std::io::{BufReader, Write}; use std::time::Instant; -use super::command::{self, CommandBuilder}; -use super::error::{ImapError, NoError}; -use super::response::{Response, Status, StatusLine, Untagged, parse_response}; -use super::transport::{Connector, ImapStream}; -use crate::logging::Logger; +use encodify::base64::STANDARD; + +use super::command::{Command, Tags}; +use super::error::ImapError; +use super::response::{Response, StatusLine, Untagged}; +use crate::logging::{Level, Logger}; +use crate::net::sasl::SaslPayload; +use crate::net::stream::{ConnectMode, Connector, Stream}; + +const CRLF: &[u8] = b"\r\n"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Capability { + StartTls, + LoginDisabled, + SaslIr, + Enable, + Utf8Accept, + CompressDeflate, + LiteralPlus, + LiteralMinus, + AuthPlain, + AuthXoauth2, + AuthOauthBearer, + ListExtended, + ListStatus, + SpecialUse, + Namespace, + Esearch, +} + +impl Capability { + pub const fn name(self) -> &'static str { + match self { + Capability::StartTls => "STARTTLS", + Capability::LoginDisabled => "LOGINDISABLED", + Capability::SaslIr => "SASL-IR", + Capability::Enable => "ENABLE", + Capability::Utf8Accept => "UTF8=ACCEPT", + Capability::CompressDeflate => "COMPRESS=DEFLATE", + Capability::LiteralPlus => "LITERAL+", + Capability::LiteralMinus => "LITERAL-", + Capability::AuthPlain => "AUTH=PLAIN", + Capability::AuthXoauth2 => "AUTH=XOAUTH2", + Capability::AuthOauthBearer => "AUTH=OAUTHBEARER", + Capability::ListExtended => "LIST-EXTENDED", + Capability::ListStatus => "LIST-STATUS", + Capability::SpecialUse => "SPECIAL-USE", + Capability::Namespace => "NAMESPACE", + Capability::Esearch => "ESEARCH", + } + } +} + +#[derive(Debug, Clone, Default)] +pub struct Capabilities(BTreeSet>); + +impl Capabilities { + pub fn has(&self, capability: Capability) -> bool { + self.0.contains(capability.name()) + } + + pub fn contains(&self, token: &str) -> bool { + self.0.contains(token.to_ascii_uppercase().as_str()) + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + fn replace<'t>(&mut self, tokens: impl IntoIterator) { + self.0.clear(); + self.insert(tokens); + } + + pub fn insert<'t>(&mut self, tokens: impl IntoIterator) { + self.0 + .extend(tokens.into_iter().map(|t| t.to_ascii_uppercase().into())); + } +} + +#[derive(Debug)] +pub struct CollectedResponse { + pub line: StatusLine, + pub untagged: Vec, +} pub struct ImapClient { - reader: BufReader>, - tags: CommandBuilder, - pub capabilities: BTreeSet, - pub host: String, + reader: BufReader, + tags: Tags, + capabilities: Capabilities, + host: String, closed: bool, utf8_accept: bool, logger: Logger, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ConnectMode { - ImplicitTls, - Plain, - StartTls, -} - -#[derive(Debug, Clone)] -pub struct Greeting { - pub status: Status, - pub text: String, - pub code: Option, + line: String, } impl ImapClient { @@ -46,55 +115,56 @@ impl ImapClient { mode: ConnectMode, logger: Logger, ) -> Result { - let stream: Box = match mode { + let stream = match mode { ConnectMode::ImplicitTls => connector.connect_tls(host, port)?, ConnectMode::Plain | ConnectMode::StartTls => connector.connect_plain(host, port)?, }; - let mut client = ImapClient { - reader: BufReader::new(stream), - tags: CommandBuilder::new(), - capabilities: BTreeSet::new(), - host: host.to_owned(), - closed: false, - utf8_accept: false, - logger, - }; + let mut client = ImapClient::over(stream, host, logger); client.read_greeting()?; - client.send_capability()?; - if matches!(mode, ConnectMode::StartTls) { - if !client.has_capability("STARTTLS") { + client.refresh_capabilities()?; + if mode == ConnectMode::StartTls { + if !client.has(Capability::StartTls) { return Err(ImapError::Unsupported( "server does not advertise STARTTLS but cleartext credentials are forbidden" .into(), )); } client.starttls(connector)?; - client.send_capability()?; + client.refresh_capabilities()?; } Ok(client) } - pub fn has_capability(&self, name: &str) -> bool { - self.capabilities - .iter() - .any(|c| c.eq_ignore_ascii_case(name)) + fn over(stream: Stream, host: &str, logger: Logger) -> ImapClient { + ImapClient { + reader: BufReader::new(stream), + tags: Tags::default(), + capabilities: Capabilities::default(), + host: host.to_owned(), + closed: false, + utf8_accept: false, + logger, + line: String::new(), + } } - fn read_greeting(&mut self) -> Result { - let resp = parse_response(&mut self.reader)?; - match resp { + pub fn has(&self, capability: Capability) -> bool { + self.capabilities.has(capability) + } + + pub fn capabilities(&self) -> &Capabilities { + &self.capabilities + } + + pub fn utf8_accept(&self) -> bool { + self.utf8_accept + } + + fn read_greeting(&mut self) -> Result<(), ImapError> { + match Response::read(&mut self.reader)? { Response::Untagged(Untagged::StatusLine(line)) => { - if let Some(code) = &line.code - && code == "CAPABILITY" - && let Some(args) = &line.code_args - { - self.capabilities = parse_caps(args); - } - Ok(Greeting { - status: line.status, - text: line.text, - code: line.code, - }) + self.absorb_code(&line); + Ok(()) } Response::Untagged(Untagged::Bye(text)) => Err(ImapError::Bye(text)), other => Err(ImapError::Protocol(format!( @@ -104,16 +174,7 @@ impl ImapClient { } pub fn refresh_capabilities(&mut self) -> Result<(), ImapError> { - self.send_capability() - } - - fn send_capability(&mut self) -> Result<(), ImapError> { - let resp = self.run_collect(command::capability())?; - for u in resp.untagged { - if let Untagged::Capability(caps) = u { - self.capabilities = caps.into_iter().collect(); - } - } + self.run_collect(&Command::Capability)?; if self.capabilities.is_empty() { return Err(ImapError::Protocol( "server returned no CAPABILITY response".into(), @@ -123,40 +184,32 @@ impl ImapClient { } fn starttls(&mut self, connector: &Connector) -> Result<(), ImapError> { - self.run_collect(command::starttls())?; - let stream = self.reader.get_ref(); - if stream.is_tls() { + self.run_collect(&Command::StartTls)?; + if self.reader.get_ref().is_tls() { return Err(ImapError::Protocol( "STARTTLS issued but stream already TLS".into(), )); } - let buffered = std::mem::replace(&mut self.reader, BufReader::new(noop_stream())); - let inner = buffered.into_inner(); - let upgraded = inner.upgrade_tls(connector, &self.host)?; - self.reader = BufReader::new(upgraded); + let stream = self.reader.get_mut().take(); + self.reader = BufReader::new(stream.upgrade_tls(connector, &self.host)?); Ok(()) } - pub fn utf8_accept(&self) -> bool { - self.utf8_accept - } - pub fn enable(&mut self, extensions: &[&str]) -> Result, ImapError> { - if !self.has_capability("ENABLE") { + if !self.has(Capability::Enable) { return Err(ImapError::Unsupported( "server does not advertise ENABLE".into(), )); } - let resp = self.run_collect(&command::enable(extensions))?; - let mut enabled: Vec = Vec::new(); - for u in resp.untagged { - if let Untagged::Enabled(exts) = u { + let mut enabled = Vec::new(); + for untagged in self.run_collect(&Command::Enable(extensions))?.untagged { + if let Untagged::Enabled(exts) = untagged { enabled.extend(exts); } } if enabled .iter() - .any(|e| e.eq_ignore_ascii_case("UTF8=ACCEPT")) + .any(|e| e.eq_ignore_ascii_case(Capability::Utf8Accept.name())) { self.utf8_accept = true; } @@ -164,197 +217,98 @@ impl ImapClient { } pub fn compress_deflate(&mut self) -> Result<(), ImapError> { - if !self.has_capability("COMPRESS=DEFLATE") { + if !self.has(Capability::CompressDeflate) { return Err(ImapError::Unsupported( "server does not advertise COMPRESS=DEFLATE".into(), )); } - self.run_collect(command::compress_deflate())?; - let buffered = std::mem::replace(&mut self.reader, BufReader::new(noop_stream())); - let primer = buffered.buffer().to_vec(); - let inner = buffered.into_inner(); - let deflated = super::transport::DeflateImapStream::wrap(inner, &primer); - self.reader = BufReader::new(deflated); + self.run_collect(&Command::CompressDeflate)?; + let primer = self.reader.buffer().to_vec(); + let stream = self.reader.get_mut().take(); + self.reader = BufReader::new(stream.compress(&primer)); Ok(()) } pub fn login(&mut self, user: &str, password: &str) -> Result<(), ImapError> { - if self.capabilities.iter().any(|c| c == "LOGINDISABLED") { + if self.has(Capability::LoginDisabled) { return Err(ImapError::Unsupported( "server advertises LOGINDISABLED".into(), )); } - let cmd = command::login(user, password); - match self.run_collect(&cmd) { - Ok(_) => Ok(()), - Err(ImapError::No(no)) if no.is_auth_failed() => Err(ImapError::AuthFailed(no.text)), - Err(e) => Err(e), - } + self.run_collect(&Command::Login { user, password }) + .map(drop) + .map_err(ImapError::into_auth_failure) } - pub fn authenticate_plain(&mut self, authcid: &str, password: &str) -> Result<(), ImapError> { - let mut payload = Vec::with_capacity(authcid.len() + password.len() + 3); - payload.push(0); - payload.extend_from_slice(authcid.as_bytes()); - payload.push(0); - payload.extend_from_slice(password.as_bytes()); - if self.has_capability("SASL-IR") { - let cmd = command::authenticate_with_ir("PLAIN", &payload); - return match self.run_collect(&cmd) { - Ok(_) => Ok(()), - Err(ImapError::No(no)) if no.is_auth_failed() => { - Err(ImapError::AuthFailed(no.text)) - } - Err(e) => Err(e), - }; + pub fn authenticate(&mut self, payload: SaslPayload<'_>) -> Result<(), ImapError> { + if self.has(Capability::SaslIr) { + return self + .run_collect(&Command::AuthenticateIr(payload)) + .map(drop) + .map_err(ImapError::into_auth_failure); } - let (tag, bytes) = self.tags.build(&command::authenticate("PLAIN")); - self.write_all(&bytes)?; - loop { - let resp = parse_response(&mut self.reader)?; - match resp { - Response::Continuation(_) => { - self.write_all(&command::sasl_response(&payload))?; - } - Response::Tagged { tag: t, line } if t == tag => { - self.update_capabilities_from_code(&line); - return self.finish_tagged(line); - } - Response::Untagged(Untagged::Capability(caps)) => { - self.capabilities = caps.into_iter().collect(); - } - _ => {} - } - } - } - - pub fn authenticate_xoauth2(&mut self, user: &str, token: &str) -> Result<(), ImapError> { - let mut payload = String::with_capacity(user.len() + token.len() + 32); - payload.push_str("user="); - payload.push_str(user); - payload.push('\x01'); - payload.push_str("auth=Bearer "); - payload.push_str(token); - payload.push('\x01'); - payload.push('\x01'); - self.drive_bearer_sasl("XOAUTH2", payload.as_bytes()) - } - - pub fn authenticate_oauthbearer(&mut self, user: &str, token: &str) -> Result<(), ImapError> { - let mut payload = String::with_capacity(user.len() + token.len() + 32); - payload.push_str("n,a="); - payload.push_str(user); - payload.push(','); - payload.push('\x01'); - payload.push_str("auth=Bearer "); - payload.push_str(token); - payload.push('\x01'); - payload.push('\x01'); - self.drive_bearer_sasl("OAUTHBEARER", payload.as_bytes()) - } - - fn drive_bearer_sasl(&mut self, mechanism: &str, payload: &[u8]) -> Result<(), ImapError> { - if self.has_capability("SASL-IR") { - let cmd = command::authenticate_with_ir(mechanism, payload); - return match self.run_collect(&cmd) { - Ok(_) => Ok(()), - Err(ImapError::No(no)) if no.is_auth_failed() => { - Err(ImapError::AuthFailed(no.text)) - } - Err(e) => Err(e), - }; - } - let (tag, bytes) = self.tags.build(&command::authenticate(mechanism)); - self.write_all(&bytes)?; + let tag = self.send(&Command::Authenticate(payload.mechanism()))?; + let response = payload.bytes(); let mut sent = false; loop { - let resp = parse_response(&mut self.reader)?; - match resp { + match Response::read(&mut self.reader)? { Response::Continuation(_) => { - if sent { - self.write_all(b"\r\n")?; + if sent && !matches!(payload, SaslPayload::Plain { .. }) { + self.write_all(CRLF)?; } else { - self.write_all(&command::sasl_response(payload))?; + let mut line = + Vec::with_capacity(STANDARD.encoded_len(response.len()) + CRLF.len()); + STANDARD.encode_append(&response, &mut line); + line.extend_from_slice(CRLF); + self.write_all(&line)?; sent = true; } } Response::Tagged { tag: t, line } if t == tag => { - self.update_capabilities_from_code(&line); - return self.finish_tagged(line); + self.absorb_code(&line); + return line + .into_result() + .map(drop) + .map_err(ImapError::into_auth_failure); } Response::Untagged(Untagged::Capability(caps)) => { - self.capabilities = caps.into_iter().collect(); + self.capabilities.replace(caps.iter().map(String::as_str)); } _ => {} } } } - fn update_capabilities_from_code(&mut self, line: &StatusLine) { - if line.code.as_deref() == Some("CAPABILITY") - && let Some(args) = &line.code_args - { - self.capabilities = parse_caps(args); - } - } - pub fn noop(&mut self) -> Result { - self.run_collect(command::noop()) + self.run_collect(&Command::Noop) } pub fn logout(&mut self) -> Result<(), ImapError> { if self.closed { return Ok(()); } - let _ = self.run_collect(command::logout()); + let _ = self.run_collect(&Command::Logout); self.closed = true; Ok(()) } - pub fn run(&mut self, command: &str) -> Result { - self.run_collect(command) - } - - pub fn run_streamed( + pub fn run_streamed( &mut self, - command: &str, - mut on_untagged: F, - ) -> Result - where - F: FnMut(Untagged), - { - let (tag, bytes) = self.tags.build(command); - if command::contains_literal(command.as_bytes()) - && !self.has_capability("LITERAL+") - && !self.has_capability("LITERAL-") - { - return Err(ImapError::Unsupported( - "command contains a non-ASCII literal but server does not advertise LITERAL+ or LITERAL-" - .into(), - )); - } + command: &Command<'_>, + mut on_untagged: impl FnMut(Untagged), + ) -> Result { + let tag = self.send(command)?; let started = Instant::now(); - self.write_all(&bytes)?; loop { - let resp = parse_response(&mut self.reader)?; - match resp { + match Response::read(&mut self.reader)? { Response::Tagged { tag: t, line } if t == tag => { - self.update_capabilities_from_code(&line); - self.logger.trace_cmd( - "IMAP", - command, - &format!("{} {}", status_word(&line.status), line.text), - started.elapsed(), - ); - return match line.status { - Status::Ok => Ok(line), - Status::No => Err(ImapError::No(line.into_no_error())), - Status::Bad => Err(ImapError::Bad(line.text)), - Status::Bye => Err(ImapError::Bye(line.text)), - Status::PreAuth => Err(ImapError::Protocol( - "unexpected PREAUTH on tagged response".into(), - )), - }; + self.absorb_code(&line); + if self.logger.enabled(Level::Method) { + let status = format!("{} {}", line.status, line.text); + self.logger + .trace_cmd("IMAP", &self.line, &status, started.elapsed()); + } + return line.into_result(); } Response::Tagged { tag: other, .. } => { return Err(ImapError::Protocol(format!( @@ -365,11 +319,11 @@ impl ImapClient { self.closed = true; return Err(ImapError::Bye(text)); } - Response::Untagged(u) => { - if let Untagged::Capability(caps) = &u { - self.capabilities = caps.iter().cloned().collect(); + Response::Untagged(untagged) => { + if let Untagged::Capability(caps) = &untagged { + self.capabilities.replace(caps.iter().map(String::as_str)); } - on_untagged(u); + on_untagged(untagged); } Response::Continuation(_) => { return Err(ImapError::Protocol( @@ -380,85 +334,39 @@ impl ImapClient { } } - pub fn run_collect(&mut self, command: &str) -> Result { - let (tag, bytes) = self.tags.build(command); - if command::contains_literal(command.as_bytes()) - && !self.has_capability("LITERAL+") - && !self.has_capability("LITERAL-") - { - return Err(ImapError::Unsupported( - "command contains a non-ASCII literal but server does not advertise LITERAL+ or LITERAL-" - .into(), - )); - } - let started = Instant::now(); - self.write_all(&bytes)?; + pub fn run_collect(&mut self, command: &Command<'_>) -> Result { let mut untagged = Vec::new(); - loop { - let resp = parse_response(&mut self.reader)?; - match resp { - Response::Tagged { tag: t, line } if t == tag => { - self.update_capabilities_from_code(&line); - self.logger.trace_cmd( - "IMAP", - command, - &format!("{} {}", status_word(&line.status), line.text), - started.elapsed(), - ); - return match line.status { - Status::Ok => Ok(CollectedResponse { - tag, - line, - untagged, - }), - Status::No => Err(ImapError::No(line.into_no_error())), - Status::Bad => Err(ImapError::Bad(line.text)), - Status::Bye => Err(ImapError::Bye(line.text)), - Status::PreAuth => Err(ImapError::Protocol( - "unexpected PREAUTH on tagged response".into(), - )), - }; - } - Response::Tagged { tag: other, .. } => { - return Err(ImapError::Protocol(format!( - "expected tag {tag}, got {other}" - ))); - } - Response::Untagged(Untagged::Bye(text)) => { - self.closed = true; - return Err(ImapError::Bye(text)); - } - Response::Untagged(u) => { - if let Untagged::Capability(caps) = &u { - self.capabilities = caps.iter().cloned().collect(); - } - untagged.push(u); - } - Response::Continuation(_) => { - return Err(ImapError::Protocol( - "unexpected continuation outside AUTHENTICATE".into(), - )); - } - } - } + let line = self.run_streamed(command, |u| untagged.push(u))?; + Ok(CollectedResponse { line, untagged }) } - fn finish_tagged(&mut self, line: StatusLine) -> Result<(), ImapError> { - match line.status { - Status::Ok => Ok(()), - Status::No => { - let no = NoError::new(line.text, line.code); - if no.is_auth_failed() { - Err(ImapError::AuthFailed(no.text)) - } else { - Err(ImapError::No(no)) - } - } - Status::Bad => Err(ImapError::Bad(line.text)), - Status::Bye => Err(ImapError::Bye(line.text)), - Status::PreAuth => Err(ImapError::Protocol( - "unexpected PREAUTH on tagged response".into(), - )), + fn send(&mut self, command: &Command<'_>) -> Result { + self.line.clear(); + let _ = write!(self.line, "{command}"); + let tag = self.tags.issue(); + if Command::has_literal(&self.line) + && !self.has(Capability::LiteralPlus) + && !self.has(Capability::LiteralMinus) + { + return Err(ImapError::Unsupported( + "command contains a non-ASCII literal but server does not advertise LITERAL+ or LITERAL-" + .into(), + )); + } + let stream = self.reader.get_mut(); + stream.write_all(tag.as_bytes())?; + stream.write_all(b" ")?; + stream.write_all(self.line.as_bytes())?; + stream.write_all(CRLF)?; + stream.flush()?; + Ok(tag) + } + + fn absorb_code(&mut self, line: &StatusLine) { + if line.code.as_deref() == Some("CAPABILITY") + && let Some(args) = &line.code_args + { + self.capabilities.replace(args.split_ascii_whitespace()); } } @@ -472,124 +380,32 @@ impl ImapClient { impl Drop for ImapClient { fn drop(&mut self) { - if !self.closed { - let _ = self.logout(); - } - } -} - -#[derive(Debug)] -pub struct CollectedResponse { - pub tag: String, - pub line: StatusLine, - pub untagged: Vec, -} - -fn status_word(status: &Status) -> &'static str { - match status { - Status::Ok => "OK", - Status::No => "NO", - Status::Bad => "BAD", - Status::Bye => "BYE", - Status::PreAuth => "PREAUTH", - } -} - -fn parse_caps(s: &str) -> BTreeSet { - s.split_ascii_whitespace().map(|w| w.to_owned()).collect() -} - -fn noop_stream() -> Box { - Box::new(NoopStream) -} - -struct NoopStream; - -impl Read for NoopStream { - fn read(&mut self, _buf: &mut [u8]) -> std::io::Result { - Ok(0) - } -} - -impl Write for NoopStream { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - Ok(buf.len()) - } - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } -} - -impl ImapStream for NoopStream { - fn upgrade_tls( - self: Box, - _connector: &Connector, - _host: &str, - ) -> Result, ImapError> { - Ok(self) - } - fn is_tls(&self) -> bool { - false + let _ = self.logout(); } } #[cfg(test)] mod tests { use super::*; - use std::io::Cursor; - - struct MockStream { - reader: Cursor>, - written: Vec, - } - - impl MockStream { - fn boxed(server_says: &[u8]) -> Box { - Box::new(MockStream { - reader: Cursor::new(server_says.to_vec()), - written: Vec::new(), - }) - } - } - - impl Read for MockStream { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - self.reader.read(buf) - } - } - - impl Write for MockStream { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - self.written.extend_from_slice(buf); - Ok(buf.len()) - } - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } - } - - impl ImapStream for MockStream { - fn upgrade_tls( - self: Box, - _c: &Connector, - _h: &str, - ) -> Result, ImapError> { - Ok(self) - } - fn is_tls(&self) -> bool { - false - } - } + use crate::imap::response::Status; + use crate::net::stream::Stream; fn client_with(server: &[u8]) -> ImapClient { - ImapClient { - reader: BufReader::new(MockStream::boxed(server)), - tags: CommandBuilder::new(), - capabilities: BTreeSet::new(), - host: "test.example".to_owned(), - closed: false, - utf8_accept: false, - logger: Logger::from_flags(true, 0), + ImapClient::over( + Stream::mock(server), + "test.example", + Logger::new(Level::Quiet), + ) + } + + fn written(client: &ImapClient) -> String { + String::from_utf8(client.reader.get_ref().written().to_vec()).unwrap() + } + + fn plain() -> SaslPayload<'static> { + SaslPayload::Plain { + authcid: "alice", + password: "p@ss", } } @@ -597,17 +413,24 @@ mod tests { fn run_collect_drains_untagged_and_returns_ok() { let server = b"* CAPABILITY IMAP4rev1 STARTTLS\r\nA0001 OK done\r\n"; let mut c = client_with(server); - let r = c.run_collect("CAPABILITY").unwrap(); + let r = c.run_collect(&Command::Capability).unwrap(); assert_eq!(r.untagged.len(), 1); assert!(matches!(r.line.status, Status::Ok)); + assert!(c.has(Capability::StartTls)); + assert_eq!(written(&c), "A0001 CAPABILITY\r\n"); + } + + #[test] + fn commands_go_out_tagged_and_crlf_terminated() { + let mut c = client_with(b"A0001 OK done\r\n"); + c.noop().unwrap(); + assert_eq!(written(&c), "A0001 NOOP\r\n"); } #[test] fn run_collect_propagates_no_as_imaperror_no() { - let server = b"A0001 NO mailbox not found\r\n"; - let mut c = client_with(server); - let err = c.run_collect("SELECT bogus").unwrap_err(); - match err { + let mut c = client_with(b"A0001 NO mailbox not found\r\n"); + match c.run_collect(&Command::Select("bogus")).unwrap_err() { ImapError::No(no) => assert_eq!(no.text, "mailbox not found"), other => panic!("expected No, got {other:?}"), } @@ -615,69 +438,93 @@ mod tests { #[test] fn run_collect_propagates_bad_as_imaperror_bad() { - let server = b"A0001 BAD syntax error\r\n"; - let mut c = client_with(server); - let err = c.run_collect("BOGUS").unwrap_err(); + let mut c = client_with(b"A0001 BAD syntax error\r\n"); + let err = c.run_collect(&Command::Raw("BOGUS")).unwrap_err(); assert!(matches!(err, ImapError::Bad(_))); } #[test] - fn run_collect_translates_authenticationfailed_via_login_path() { - let server = b"A0001 NO [AUTHENTICATIONFAILED] bad creds\r\n"; - let mut c = client_with(server); + fn login_translates_authenticationfailed() { + let mut c = client_with(b"A0001 NO [AUTHENTICATIONFAILED] bad creds\r\n"); let err = c.login("alice", "wrong").unwrap_err(); assert!(matches!(err, ImapError::AuthFailed(_))); } #[test] fn login_refused_when_logindisabled() { - let server = b"A0001 OK any\r\n"; - let mut c = client_with(server); - c.capabilities.insert("LOGINDISABLED".to_owned()); + let mut c = client_with(b"A0001 OK any\r\n"); + c.capabilities.insert(["LOGINDISABLED"]); let err = c.login("a", "b").unwrap_err(); assert!(matches!(err, ImapError::Unsupported(_))); } #[test] fn bye_mid_response_closes_client() { - let server = b"* BYE going away\r\n"; - let mut c = client_with(server); - let err = c.run_collect("NOOP").unwrap_err(); + let mut c = client_with(b"* BYE going away\r\n"); + let err = c.run_collect(&Command::Noop).unwrap_err(); assert!(matches!(err, ImapError::Bye(_))); assert!(c.closed); } #[test] fn authenticate_plain_sasl_ir_path() { - let server = b"A0001 OK auth done\r\n"; - let mut c = client_with(server); - c.capabilities.insert("SASL-IR".to_owned()); - c.authenticate_plain("alice", "p@ss").unwrap(); + let mut c = client_with(b"A0001 OK auth done\r\n"); + c.capabilities.insert(["SASL-IR"]); + c.authenticate(plain()).unwrap(); + assert_eq!(written(&c), "A0001 AUTHENTICATE PLAIN AGFsaWNlAHBAc3M=\r\n"); } #[test] fn authenticate_plain_accepts_dovecot_post_login_untagged_capability() { let server = b"* CAPABILITY IMAP4rev1 SASL-IR MOVE IMAPSIEVE=sieve://127.0.0.1:4190 QUOTA\r\nA0001 OK [CAPABILITY IMAP4rev1 SASL-IR MOVE IMAPSIEVE=sieve://127.0.0.1:4190 QUOTA] Logged in\r\n"; let mut c = client_with(server); - c.capabilities.insert("SASL-IR".to_owned()); - c.authenticate_plain("alice", "p@ss").unwrap(); - assert!(c.has_capability("MOVE")); - assert!(c.has_capability("IMAPSIEVE=sieve://127.0.0.1:4190")); + c.capabilities.insert(["SASL-IR"]); + c.authenticate(plain()).unwrap(); + assert!(c.capabilities().contains("MOVE")); + assert!( + c.capabilities() + .contains("IMAPSIEVE=sieve://127.0.0.1:4190") + ); } #[test] fn authenticate_plain_continuation_path() { - let server = b"+ \r\nA0001 OK auth done\r\n"; - let mut c = client_with(server); - c.authenticate_plain("alice", "p@ss").unwrap(); + let mut c = client_with(b"+ \r\nA0001 OK auth done\r\n"); + c.authenticate(plain()).unwrap(); + assert_eq!( + written(&c), + "A0001 AUTHENTICATE PLAIN\r\nAGFsaWNlAHBAc3M=\r\n" + ); } #[test] - fn has_capability_is_case_insensitive() { + fn authenticate_bearer_answers_an_error_challenge_with_an_empty_line() { + let mut c = client_with( + b"+ \r\n+ eyJzdGF0dXMiOiI0MDEifQ==\r\nA0001 NO [AUTHENTICATIONFAILED] no\r\n", + ); + let err = c + .authenticate(SaslPayload::XOauth2 { + user: "u", + token: "t", + }) + .unwrap_err(); + assert!(matches!(err, ImapError::AuthFailed(_))); + assert!(written(&c).ends_with("\r\n\r\n")); + } + + #[test] + fn capabilities_are_case_insensitive() { let mut c = client_with(b""); - c.capabilities.insert("STARTTLS".to_owned()); - assert!(c.has_capability("starttls")); - assert!(c.has_capability("STARTTLS")); - assert!(!c.has_capability("MOVE")); + c.capabilities.insert(["starttls"]); + assert!(c.has(Capability::StartTls)); + assert!(c.capabilities().contains("StartTLS")); + assert!(!c.capabilities().contains("MOVE")); + } + + #[test] + fn literal_without_literal_plus_is_refused() { + let mut c = client_with(b""); + let err = c.run_collect(&Command::Select("éclair")).unwrap_err(); + assert!(matches!(err, ImapError::Unsupported(_))); } } diff --git a/src/imap/command.rs b/src/imap/command.rs index 611bdd8..54a38a2 100644 --- a/src/imap/command.rs +++ b/src/imap/command.rs @@ -4,418 +4,446 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use std::fmt::Write as _; +use std::fmt::{self, Write as _}; use encodify::base64::STANDARD; -const AUTHENTICATE: &str = "AUTHENTICATE "; -const CRLF: &[u8] = b"\r\n"; +use crate::net::sasl::SaslPayload; -pub struct CommandBuilder { - next_tag: u32, +#[derive(Clone, Copy)] +pub enum Command<'a> { + Capability, + StartTls, + CompressDeflate, + Noop, + Logout, + Namespace, + UidSearchAll, + UidSearchEsearchAll, + UidFetchAllUids, + Enable(&'a [&'a str]), + Login { + user: &'a str, + password: &'a str, + }, + Authenticate(&'static str), + AuthenticateIr(SaslPayload<'a>), + List { + reference: &'a str, + patterns: &'a [&'a str], + selection: &'a [&'a str], + returns: &'a [&'a str], + }, + Lsub { + reference: &'a str, + pattern: &'a str, + }, + Select(&'a str), + Status { + mailbox: &'a str, + items: &'a [&'a str], + }, + UidFetch { + set: &'a UidSet, + items: &'a [&'a str], + }, + Raw(&'a str), } -impl Default for CommandBuilder { - fn default() -> Self { - Self::new() +impl Command<'_> { + pub fn has_literal(text: &str) -> bool { + let bytes = text.as_bytes(); + text.match_indices('{').any(|(pos, _)| { + let Some(after) = bytes.get(pos + 1..).filter(|after| after.len() >= 2) else { + return false; + }; + let digits = after.iter().take_while(|b| b.is_ascii_digit()).count(); + let rest = after.get(digits..).unwrap_or_default(); + let rest = rest.strip_prefix(b"+").unwrap_or(rest); + rest.first() == Some(&b'}') + }) + } + + fn write_list(f: &mut fmt::Formatter<'_>, items: &[&str], quoted: Quote) -> fmt::Result { + f.write_char('(')?; + for (i, item) in items.iter().enumerate() { + if i > 0 { + f.write_char(' ')?; + } + match quoted { + Quote::Astring => write!(f, "{}", Astring(item))?, + Quote::Bare => f.write_str(item)?, + } + } + f.write_char(')') } } -impl CommandBuilder { - pub fn new() -> Self { - Self { next_tag: 1 } +#[derive(Clone, Copy)] +enum Quote { + Astring, + Bare, +} + +impl fmt::Display for Command<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + Command::Capability => f.write_str("CAPABILITY"), + Command::StartTls => f.write_str("STARTTLS"), + Command::CompressDeflate => f.write_str("COMPRESS DEFLATE"), + Command::Noop => f.write_str("NOOP"), + Command::Logout => f.write_str("LOGOUT"), + Command::Namespace => f.write_str("NAMESPACE"), + Command::UidSearchAll => f.write_str("UID SEARCH ALL"), + Command::UidSearchEsearchAll => f.write_str("UID SEARCH RETURN (ALL) ALL"), + Command::UidFetchAllUids => f.write_str("UID FETCH 1:* (UID)"), + Command::Enable(extensions) => { + f.write_str("ENABLE")?; + extensions.iter().try_for_each(|ext| write!(f, " {ext}")) + } + Command::Login { user, password } => { + write!(f, "LOGIN {} {}", Astring(user), Astring(password)) + } + Command::Authenticate(mechanism) => write!(f, "AUTHENTICATE {mechanism}"), + Command::AuthenticateIr(payload) => { + let mut encoded = String::new(); + STANDARD.encode_append(payload.bytes(), &mut encoded); + write!(f, "AUTHENTICATE {} {encoded}", payload.mechanism()) + } + Command::List { + reference, + patterns, + selection, + returns, + } => { + f.write_str("LIST")?; + if !selection.is_empty() { + f.write_char(' ')?; + Command::write_list(f, selection, Quote::Bare)?; + } + write!(f, " {}", Astring(reference))?; + match patterns { + [single] => write!(f, " {}", Astring(single))?, + many => { + f.write_char(' ')?; + Command::write_list(f, many, Quote::Astring)?; + } + } + if !returns.is_empty() { + f.write_str(" RETURN ")?; + Command::write_list(f, returns, Quote::Bare)?; + } + Ok(()) + } + Command::Lsub { reference, pattern } => { + write!(f, "LSUB {} {}", Astring(reference), Astring(pattern)) + } + Command::Select(mailbox) => write!(f, "SELECT {}", Astring(mailbox)), + Command::Status { mailbox, items } => { + write!(f, "STATUS {} ", Astring(mailbox))?; + Command::write_list(f, items, Quote::Bare) + } + Command::UidFetch { set, items } => { + write!(f, "UID FETCH {} ", set.0)?; + Command::write_list(f, items, Quote::Bare) + } + Command::Raw(text) => f.write_str(text), + } + } +} + +pub struct Astring<'a>(pub &'a str); + +impl fmt::Display for Astring<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s = self.0; + if s.bytes().any(|b| !(0x20..=0x7e).contains(&b)) { + return write!(f, "{{{}+}}\r\n{s}", s.len()); + } + f.write_char('"')?; + for chunk in s.split_inclusive(['"', '\\']) { + match chunk.split_at_checked(chunk.len() - 1) { + Some((head, last @ ("\"" | "\\"))) => { + f.write_str(head)?; + f.write_char('\\')?; + f.write_str(last)?; + } + _ => f.write_str(chunk)?, + } + } + f.write_char('"') + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UidSet(String); + +impl UidSet { + pub fn from_sorted(uids: &[u32]) -> UidSet { + let mut out = String::with_capacity(uids.len() * 8); + let mut iter = uids.iter().copied().peekable(); + while let Some(start) = iter.next() { + let mut end = start; + while let Some(&next) = iter.peek() { + if Some(next) != end.checked_add(1) { + break; + } + end = next; + iter.next(); + } + if !out.is_empty() { + out.push(','); + } + let _ = if start == end { + write!(out, "{start}") + } else { + write!(out, "{start}:{end}") + }; + } + UidSet(out) } - pub fn next_tag(&mut self) -> String { - let tag = format!("A{:04}", self.next_tag); - self.next_tag = self.next_tag.wrapping_add(1).max(1); + pub fn from_unsorted(uids: &[u32]) -> UidSet { + let mut sorted = uids.to_vec(); + sorted.sort_unstable(); + sorted.dedup(); + UidSet::from_sorted(&sorted) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +pub struct Tags { + next: u32, +} + +impl Default for Tags { + fn default() -> Tags { + Tags { next: 1 } + } +} + +impl Tags { + pub fn issue(&mut self) -> String { + let tag = format!("A{:04}", self.next); + self.next = self.next.wrapping_add(1).max(1); tag } - - pub fn build(&mut self, command: &str) -> (String, Vec) { - let tag = self.next_tag(); - let mut out = String::with_capacity(tag.len() + command.len() + 4); - out.push_str(&tag); - out.push(' '); - out.push_str(command); - out.push_str("\r\n"); - (tag, out.into_bytes()) - } -} - -pub fn quote_astring(s: &str) -> String { - let needs_literal = s.bytes().any(|b| !(0x20..=0x7e).contains(&b)); - if needs_literal { - format!("{{{}+}}\r\n{}", s.len(), s) - } else { - let mut out = String::with_capacity(s.len() + 2); - out.push('"'); - for c in s.chars() { - if c == '"' || c == '\\' { - out.push('\\'); - } - out.push(c); - } - out.push('"'); - out - } -} - -pub fn contains_literal(bytes: &[u8]) -> bool { - let mut i = 0; - while i + 2 < bytes.len() { - if bytes[i] == b'{' { - let mut j = i + 1; - while j < bytes.len() && bytes[j].is_ascii_digit() { - j += 1; - } - if j < bytes.len() && bytes[j] == b'+' { - j += 1; - } - if j < bytes.len() && bytes[j] == b'}' { - return true; - } - } - i += 1; - } - false -} - -pub fn login(user: &str, password: &str) -> String { - format!("LOGIN {} {}", quote_astring(user), quote_astring(password)) -} - -pub fn authenticate(mechanism: &str) -> String { - format!("AUTHENTICATE {mechanism}") -} - -pub fn authenticate_with_ir(mechanism: &str, initial_response: &[u8]) -> String { - let mut out = String::with_capacity( - AUTHENTICATE.len() + mechanism.len() + 1 + STANDARD.encoded_len(initial_response.len()), - ); - out.push_str(AUTHENTICATE); - out.push_str(mechanism); - out.push(' '); - STANDARD.encode_append(initial_response, &mut out); - out -} - -pub fn sasl_response(response: &[u8]) -> Vec { - let mut line = Vec::with_capacity(STANDARD.encoded_len(response.len()) + CRLF.len()); - STANDARD.encode_append(response, &mut line); - line.extend_from_slice(CRLF); - line -} - -pub fn capability() -> &'static str { - "CAPABILITY" -} - -pub fn starttls() -> &'static str { - "STARTTLS" -} - -pub fn compress_deflate() -> &'static str { - "COMPRESS DEFLATE" -} - -pub fn noop() -> &'static str { - "NOOP" -} - -pub fn logout() -> &'static str { - "LOGOUT" -} - -pub fn namespace() -> &'static str { - "NAMESPACE" -} - -pub fn enable(extensions: &[&str]) -> String { - let mut out = String::from("ENABLE"); - for ext in extensions { - out.push(' '); - out.push_str(ext); - } - out -} - -pub fn list(reference: &str, pattern: &str) -> String { - format!( - "LIST {} {}", - quote_astring(reference), - quote_astring(pattern) - ) -} - -pub fn list_extended( - reference: &str, - patterns: &[&str], - selection: &[&str], - return_options: &[&str], -) -> String { - let mut out = String::from("LIST"); - if !selection.is_empty() { - out.push_str(" ("); - for (i, s) in selection.iter().enumerate() { - if i > 0 { - out.push(' '); - } - out.push_str(s); - } - out.push(')'); - } - out.push(' '); - out.push_str("e_astring(reference)); - if patterns.len() == 1 { - out.push(' '); - out.push_str("e_astring(patterns[0])); - } else { - out.push_str(" ("); - for (i, p) in patterns.iter().enumerate() { - if i > 0 { - out.push(' '); - } - out.push_str("e_astring(p)); - } - out.push(')'); - } - if !return_options.is_empty() { - out.push_str(" RETURN ("); - for (i, r) in return_options.iter().enumerate() { - if i > 0 { - out.push(' '); - } - out.push_str(r); - } - out.push(')'); - } - out -} - -pub fn lsub(reference: &str, pattern: &str) -> String { - format!( - "LSUB {} {}", - quote_astring(reference), - quote_astring(pattern) - ) -} - -pub fn select(mailbox: &str) -> String { - format!("SELECT {}", quote_astring(mailbox)) -} - -pub fn examine(mailbox: &str) -> String { - format!("EXAMINE {}", quote_astring(mailbox)) -} - -pub fn status(mailbox: &str, items: &[&str]) -> String { - let mut out = String::from("STATUS "); - out.push_str("e_astring(mailbox)); - out.push_str(" ("); - for (i, s) in items.iter().enumerate() { - if i > 0 { - out.push(' '); - } - out.push_str(s); - } - out.push(')'); - out -} - -pub fn uid_search_esearch_all() -> &'static str { - "UID SEARCH RETURN (ALL) ALL" -} - -pub fn uid_search_all() -> &'static str { - "UID SEARCH ALL" -} - -pub fn uid_fetch_all_uids() -> &'static str { - "UID FETCH 1:* (UID)" -} - -pub fn uid_fetch(set: &str, items: &[&str]) -> String { - let mut out = String::from("UID FETCH "); - out.push_str(set); - out.push_str(" ("); - for (i, s) in items.iter().enumerate() { - if i > 0 { - out.push(' '); - } - out.push_str(s); - } - out.push(')'); - out -} - -pub fn format_uid_set(uids: &[u32], is_sorted: bool) -> String { - if uids.is_empty() { - return String::new(); - } - let owned_sorted: Vec; - let slice: &[u32] = if is_sorted { - uids - } else { - let mut s = uids.to_vec(); - s.sort_unstable(); - s.dedup(); - owned_sorted = s; - &owned_sorted[..] - }; - - let mut out = String::with_capacity(slice.len() * 8); - let mut i = 0; - let mut first = true; - while i < slice.len() { - let start = slice[i]; - let mut end = start; - let mut j = i + 1; - while j < slice.len() && slice[j] == end + 1 { - end = slice[j]; - j += 1; - } - if !first { - out.push(','); - } - if start == end { - let _ = write!(&mut out, "{start}"); - } else { - let _ = write!(&mut out, "{start}:{end}"); - } - first = false; - i = j; - } - out } #[cfg(test)] mod tests { use super::*; + fn text(command: Command<'_>) -> String { + command.to_string() + } + + fn reference_uid_set(uids: &[u32]) -> String { + let mut out = String::new(); + let mut i = 0; + while i < uids.len() { + let start = uids[i]; + let mut end = start; + let mut j = i + 1; + while j < uids.len() && uids[j] == end + 1 { + end = uids[j]; + j += 1; + } + if !out.is_empty() { + out.push(','); + } + if start == end { + out.push_str(&start.to_string()); + } else { + out.push_str(&format!("{start}:{end}")); + } + i = j; + } + out + } + + fn reference_has_literal(bytes: &[u8]) -> bool { + let mut i = 0; + while i + 2 < bytes.len() { + if bytes[i] == b'{' { + let mut j = i + 1; + while j < bytes.len() && bytes[j].is_ascii_digit() { + j += 1; + } + if j < bytes.len() && bytes[j] == b'+' { + j += 1; + } + if j < bytes.len() && bytes[j] == b'}' { + return true; + } + } + i += 1; + } + false + } + #[test] - fn next_tag_increments() { - let mut b = CommandBuilder::new(); - assert_eq!(b.next_tag(), "A0001"); - assert_eq!(b.next_tag(), "A0002"); - assert_eq!(b.next_tag(), "A0003"); + fn tags_increment() { + let mut tags = Tags::default(); + assert_eq!(tags.issue(), "A0001"); + assert_eq!(tags.issue(), "A0002"); + assert_eq!(tags.issue(), "A0003"); } #[test] fn authenticate_with_ir_appends_the_base64_initial_response() { assert_eq!( - authenticate_with_ir("PLAIN", b"\0foo\0bar"), + text(Command::AuthenticateIr(SaslPayload::Plain { + authcid: "foo", + password: "bar" + })), "AUTHENTICATE PLAIN AGZvbwBiYXI=" ); - assert_eq!( - authenticate_with_ir("XOAUTH2", b""), - "AUTHENTICATE XOAUTH2 " - ); } #[test] - fn sasl_response_is_base64_with_crlf() { - assert_eq!(sasl_response(b"\0foo\0bar"), b"AGZvbwBiYXI=\r\n"); - assert_eq!(sasl_response(b""), b"\r\n"); - } - - #[test] - fn build_appends_crlf() { - let mut b = CommandBuilder::new(); - let (tag, bytes) = b.build("NOOP"); - assert_eq!(tag, "A0001"); - assert_eq!(bytes, b"A0001 NOOP\r\n"); - } - - #[test] - fn quote_astring_quotes_safe_text() { - assert_eq!(quote_astring("hello"), "\"hello\""); - assert_eq!(quote_astring("INBOX"), "\"INBOX\""); - } - - #[test] - fn quote_astring_escapes_quote_and_backslash() { - assert_eq!(quote_astring("a\"b"), "\"a\\\"b\""); - assert_eq!(quote_astring("a\\b"), "\"a\\\\b\""); - } - - #[test] - fn quote_astring_switches_to_literal_for_non_ascii() { - let q = quote_astring("éclair"); - assert!(q.starts_with("{")); - assert!(q.contains("\r\n")); - assert!(q.ends_with("éclair")); + fn astring_quotes_escapes_or_switches_to_literal() { + assert_eq!(Astring("hello").to_string(), "\"hello\""); + assert_eq!(Astring("INBOX").to_string(), "\"INBOX\""); + assert_eq!(Astring("a\"b").to_string(), "\"a\\\"b\""); + assert_eq!(Astring("a\\b").to_string(), "\"a\\\\b\""); + assert_eq!(Astring("\"\\").to_string(), "\"\\\"\\\\\""); + assert_eq!(Astring("éclair").to_string(), "{7+}\r\néclair"); } #[test] fn login_command_format() { - let s = login("alice", "p@ss"); - assert_eq!(s, "LOGIN \"alice\" \"p@ss\""); - } - - #[test] - fn list_extended_with_return_clause() { - let s = list_extended("", &["*"], &[], &["SPECIAL-USE", "SUBSCRIBED"]); - assert_eq!(s, "LIST \"\" \"*\" RETURN (SPECIAL-USE SUBSCRIBED)"); - } - - #[test] - fn list_extended_with_selection_and_multi_pattern() { - let s = list_extended("", &["INBOX", "Sent"], &["SUBSCRIBED"], &[]); - assert_eq!(s, "LIST (SUBSCRIBED) \"\" (\"INBOX\" \"Sent\")"); - } - - #[test] - fn status_items_serialise_in_order() { - let s = status("INBOX", &["UIDVALIDITY", "UIDNEXT", "MESSAGES"]); - assert_eq!(s, "STATUS \"INBOX\" (UIDVALIDITY UIDNEXT MESSAGES)"); - } - - #[test] - fn uid_fetch_with_items() { - let s = uid_fetch("1:100", &["UID", "FLAGS", "INTERNALDATE", "RFC822.SIZE"]); - assert_eq!(s, "UID FETCH 1:100 (UID FLAGS INTERNALDATE RFC822.SIZE)"); - } - - #[test] - fn format_uid_set_collapses_consecutive_sorted_input() { - assert_eq!(format_uid_set(&[], true), ""); - assert_eq!(format_uid_set(&[5], true), "5"); - assert_eq!(format_uid_set(&[1, 2, 3, 5, 7, 8], true), "1:3,5,7:8"); - } - - #[test] - fn format_uid_set_sorts_when_caller_unsure() { - assert_eq!(format_uid_set(&[3, 1, 2], false), "1:3"); - assert_eq!(format_uid_set(&[1, 1, 1, 2, 2, 3], false), "1:3"); - } - - #[test] - fn enable_with_multiple_extensions() { assert_eq!( - enable(&["UTF8=ACCEPT", "CONDSTORE"]), - "ENABLE UTF8=ACCEPT CONDSTORE" + text(Command::Login { + user: "alice", + password: "p@ss" + }), + "LOGIN \"alice\" \"p@ss\"" ); } #[test] - fn quote_astring_uses_literal_plus_for_non_ascii() { - let q = quote_astring("éclair"); - assert!(q.starts_with("{")); - assert!(q.contains("+}\r\n")); - assert!(q.ends_with("éclair")); + fn list_forms() { + let list = |patterns, selection, returns| { + text(Command::List { + reference: "", + patterns, + selection, + returns, + }) + }; + assert_eq!(list(&["*"], &[], &[]), "LIST \"\" \"*\""); + assert_eq!( + list(&["*"], &[], &["SPECIAL-USE", "SUBSCRIBED"]), + "LIST \"\" \"*\" RETURN (SPECIAL-USE SUBSCRIBED)" + ); + assert_eq!( + list(&["INBOX", "Sent"], &["SUBSCRIBED"], &[]), + "LIST (SUBSCRIBED) \"\" (\"INBOX\" \"Sent\")" + ); + assert_eq!( + text(Command::Lsub { + reference: "", + pattern: "*" + }), + "LSUB \"\" \"*\"" + ); } #[test] - fn contains_literal_detects_literal_plus_and_sync() { - assert!(contains_literal(b"LOGIN \"u\" {6+}\r\nsecret")); - assert!(contains_literal(b"LOGIN \"u\" {6}\r\nsecret")); - assert!(!contains_literal(b"LOGIN \"u\" \"p\"")); + fn status_fetch_enable_select() { + assert_eq!( + text(Command::Status { + mailbox: "INBOX", + items: &["UIDVALIDITY", "UIDNEXT", "MESSAGES"] + }), + "STATUS \"INBOX\" (UIDVALIDITY UIDNEXT MESSAGES)" + ); + let set = UidSet::from_sorted(&[1, 2, 3, 100]); + assert_eq!( + text(Command::UidFetch { + set: &set, + items: &["UID", "FLAGS"] + }), + "UID FETCH 1:3,100 (UID FLAGS)" + ); + let uids: Vec = (1..=100).collect(); + let set = UidSet::from_sorted(&uids); + assert_eq!( + text(Command::UidFetch { + set: &set, + items: &["UID", "FLAGS", "INTERNALDATE", "RFC822.SIZE"] + }), + "UID FETCH 1:100 (UID FLAGS INTERNALDATE RFC822.SIZE)" + ); + assert_eq!( + text(Command::Enable(&["UTF8=ACCEPT", "CONDSTORE"])), + "ENABLE UTF8=ACCEPT CONDSTORE" + ); + assert_eq!(text(Command::Select("INBOX")), "SELECT \"INBOX\""); } #[test] - fn format_uid_set_trusts_sorted_flag() { - let s = format_uid_set(&[1, 2, 3, 7, 9], true); - assert_eq!(s, "1:3,7,9"); + fn uid_set_collapses_and_sorts() { + assert_eq!(UidSet::from_sorted(&[]).as_str(), ""); + assert_eq!(UidSet::from_sorted(&[5]).as_str(), "5"); + assert_eq!( + UidSet::from_sorted(&[1, 2, 3, 5, 7, 8]).as_str(), + "1:3,5,7:8" + ); + assert_eq!(UidSet::from_sorted(&[1, 2, 3, 7, 9]).as_str(), "1:3,7,9"); + assert_eq!(UidSet::from_unsorted(&[3, 1, 2]).as_str(), "1:3"); + assert_eq!(UidSet::from_unsorted(&[1, 1, 1, 2, 2, 3]).as_str(), "1:3"); + assert_eq!( + UidSet::from_sorted(&[u32::MAX - 1, u32::MAX]).as_str(), + "4294967294:4294967295" + ); + } + + #[test] + fn uid_set_matches_reference() { + let mut uids = Vec::new(); + let mut x: u32 = 7; + for _ in 0..2000 { + x = x.wrapping_mul(1_103_515_245).wrapping_add(12345); + uids.push(x % 600); + } + uids.sort_unstable(); + uids.dedup(); + for window in [1, 2, 3, 17, uids.len()] { + for chunk in uids.chunks(window) { + assert_eq!( + UidSet::from_sorted(chunk).as_str(), + reference_uid_set(chunk) + ); + } + } + } + + #[test] + fn has_literal_matches_reference() { + for input in [ + &b"LOGIN \"u\" {6+}\r\nsecret"[..], + b"LOGIN \"u\" {6}\r\nsecret", + b"LOGIN \"u\" \"p\"", + b"{}", + b"{+}", + b"x{", + b"{1", + b"{1}", + b"ab{12+", + b"{{}", + b"", + ] { + let text = std::str::from_utf8(input).unwrap(); + assert_eq!( + Command::has_literal(text), + reference_has_literal(input), + "{text:?}" + ); + } } } diff --git a/src/imap/error.rs b/src/imap/error.rs index 8dec7a1..2160e88 100644 --- a/src/imap/error.rs +++ b/src/imap/error.rs @@ -7,6 +7,8 @@ use std::fmt; use std::io; +use crate::net::stream::StreamError; + #[derive(Debug, thiserror::Error)] pub enum ImapError { #[error("transport error: {0}")] @@ -40,6 +42,35 @@ pub enum ImapError { Unsupported(String), } +impl ImapError { + pub fn into_auth_failure(self) -> ImapError { + match self { + ImapError::No(no) if no.is_auth_failed() => ImapError::AuthFailed(no.text), + other => other, + } + } + + pub fn is_auth_failed(&self) -> bool { + match self { + ImapError::AuthFailed(_) => true, + ImapError::No(no) => no.is_auth_failed(), + _ => false, + } + } +} + +impl From for ImapError { + fn from(e: StreamError) -> ImapError { + match e { + StreamError::Io(e) => ImapError::Io(e), + StreamError::Tls(m) => ImapError::Tls(m), + StreamError::CompressedStartTls => { + ImapError::Protocol("cannot STARTTLS after COMPRESS DEFLATE".into()) + } + } + } +} + #[derive(Debug, Clone)] pub struct NoError { pub text: String, @@ -71,10 +102,14 @@ impl NoError { } pub fn is_auth_failed(&self) -> bool { - matches!( - self.code.as_deref(), - Some("AUTHENTICATIONFAILED") | Some("PRIVACYREQUIRED") | Some("AUTHORIZATIONFAILED") - ) + self.code.as_deref().is_some_and(|code| { + hashify::set!( + code.as_bytes(), + "AUTHENTICATIONFAILED", + "PRIVACYREQUIRED", + "AUTHORIZATIONFAILED", + ) + }) } } diff --git a/src/imap/mod.rs b/src/imap/mod.rs index af211de..b4cc34e 100644 --- a/src/imap/mod.rs +++ b/src/imap/mod.rs @@ -11,7 +11,7 @@ pub mod error; pub mod name; pub mod response; pub mod retry; -pub mod transport; +pub mod session; -pub use client::{CollectedResponse, ConnectMode, Greeting, ImapClient}; +pub use client::{Capabilities, Capability, CollectedResponse, ImapClient}; pub use error::ImapError; diff --git a/src/imap/name.rs b/src/imap/name.rs index 66b8e38..6ecf3a8 100644 --- a/src/imap/name.rs +++ b/src/imap/name.rs @@ -9,52 +9,73 @@ use encodify::utf7::{self, Utf7}; use super::error::ImapError; const RECEIVED_NAME: Utf7 = utf7::IMAP.lenient(); +const INBOX: &str = "INBOX"; -pub fn canonicalise_inbox(name: &str) -> String { - if name.eq_ignore_ascii_case("INBOX") { - "INBOX".to_owned() - } else { - name.to_owned() +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NameEncoding { + ModifiedUtf7, + Utf8, +} + +impl NameEncoding { + pub fn mailbox(self, wire: &str) -> Result { + let mut name = self.decode(wire)?; + if name.eq_ignore_ascii_case(INBOX) { + name.make_ascii_uppercase(); + } + Ok(name) } -} -pub fn decode_mailbox_name(input: &str) -> Result { - decode_mailbox_name_with(input, false) -} - -pub fn decode_mailbox_name_with(input: &str, utf8_accept: bool) -> Result { - if utf8_accept || ends_in_open_shift(input) { - return Ok(input.to_owned()); + pub fn decode(self, wire: &str) -> Result { + if self == NameEncoding::Utf8 || NameEncoding::ends_in_open_shift(wire) { + return Ok(wire.to_owned()); + } + RECEIVED_NAME + .decode(wire) + .map_err(|e| ImapError::Parse(format!("modified UTF-7 mailbox name: {e}"))) } - RECEIVED_NAME - .decode(input) - .map_err(|e| ImapError::Parse(format!("modified UTF-7 mailbox name: {e}"))) -} -fn ends_in_open_shift(name: &str) -> bool { - name.rsplit_once('&') - .is_some_and(|(_, tail)| !tail.is_empty() && !tail.contains('-')) + fn ends_in_open_shift(name: &str) -> bool { + name.rsplit_once('&') + .is_some_and(|(_, tail)| !tail.is_empty() && !tail.contains('-')) + } } #[cfg(test)] mod tests { use super::*; + fn utf8_mailbox(wire: &str, _: bool) -> Result { + NameEncoding::Utf8.mailbox(wire) + } + + fn canonicalise_inbox(name: &str) -> String { + NameEncoding::Utf8.mailbox(name).unwrap() + } + #[test] fn ascii_passes_through_decode() { - assert_eq!(decode_mailbox_name("INBOX").unwrap(), "INBOX"); - assert_eq!(decode_mailbox_name("Sent/Items").unwrap(), "Sent/Items"); + assert_eq!( + NameEncoding::ModifiedUtf7.mailbox("INBOX").unwrap(), + "INBOX" + ); + assert_eq!( + NameEncoding::ModifiedUtf7.mailbox("Sent/Items").unwrap(), + "Sent/Items" + ); } #[test] fn ampersand_dash_decodes_to_literal_amp() { - assert_eq!(decode_mailbox_name("R&-D").unwrap(), "R&D"); - assert_eq!(decode_mailbox_name("&-").unwrap(), "&"); + assert_eq!(NameEncoding::ModifiedUtf7.mailbox("R&-D").unwrap(), "R&D"); + assert_eq!(NameEncoding::ModifiedUtf7.mailbox("&-").unwrap(), "&"); } #[test] fn non_ascii_decode_japanese() { - let decoded = decode_mailbox_name("~peter/mail/&ZeVnLIqe-/&U,BTFw-").unwrap(); + let decoded = NameEncoding::ModifiedUtf7 + .mailbox("~peter/mail/&ZeVnLIqe-/&U,BTFw-") + .unwrap(); assert_eq!(decoded, "~peter/mail/日本語/台北"); } @@ -69,42 +90,57 @@ mod tests { "&ZeVnLIqe", "Envoy&AOk", ] { - assert_eq!(decode_mailbox_name(name).unwrap(), name); + assert_eq!(NameEncoding::ModifiedUtf7.mailbox(name).unwrap(), name); } } #[test] fn trailing_lone_ampersand_is_literal() { - assert_eq!(decode_mailbox_name("Sales&").unwrap(), "Sales&"); - assert_eq!(decode_mailbox_name("&AOk-s&").unwrap(), "és&"); - assert_eq!(decode_mailbox_name("&").unwrap(), "&"); + assert_eq!( + NameEncoding::ModifiedUtf7.mailbox("Sales&").unwrap(), + "Sales&" + ); + assert_eq!( + NameEncoding::ModifiedUtf7.mailbox("&AOk-s&").unwrap(), + "és&" + ); + assert_eq!(NameEncoding::ModifiedUtf7.mailbox("&").unwrap(), "&"); } #[test] fn encoded_name_with_a_later_escaped_ampersand_decodes() { - assert_eq!(decode_mailbox_name("Envoy&AOk-s").unwrap(), "Envoyés"); - assert_eq!(decode_mailbox_name("&AOk-/R&-D").unwrap(), "é/R&D"); + assert_eq!( + NameEncoding::ModifiedUtf7.mailbox("Envoy&AOk-s").unwrap(), + "Envoyés" + ); + assert_eq!( + NameEncoding::ModifiedUtf7.mailbox("&AOk-/R&-D").unwrap(), + "é/R&D" + ); } #[test] fn byte_outside_the_alphabet_inside_a_shift_errors() { for name in ["&AOk.s-", "&AOk&-", "&AOké-"] { - let err = decode_mailbox_name(name).unwrap_err(); + let err = NameEncoding::ModifiedUtf7.mailbox(name).unwrap_err(); assert!(matches!(err, ImapError::Parse(_)), "{name}: {err:?}"); } } #[test] fn unpaired_surrogate_errors() { - let err = decode_mailbox_name("&2D0-").unwrap_err(); + let err = NameEncoding::ModifiedUtf7.mailbox("&2D0-").unwrap_err(); assert!(matches!(err, ImapError::Parse(_))); } #[test] fn encoded_printable_ascii_and_leftover_bits_are_accepted() { - assert_eq!(decode_mailbox_name("&AGE-").unwrap(), "a"); - assert_eq!(decode_mailbox_name("&AOl-").unwrap(), "é"); - assert_eq!(decode_mailbox_name("&AOk-&AOk-").unwrap(), "éé"); + assert_eq!(NameEncoding::ModifiedUtf7.mailbox("&AGE-").unwrap(), "a"); + assert_eq!(NameEncoding::ModifiedUtf7.mailbox("&AOl-").unwrap(), "é"); + assert_eq!( + NameEncoding::ModifiedUtf7.mailbox("&AOk-&AOk-").unwrap(), + "éé" + ); } #[test] @@ -119,7 +155,7 @@ mod tests { "Junk E-mail", ] { let enc = utf7::IMAP.encode(s); - let dec = decode_mailbox_name(&enc).unwrap(); + let dec = NameEncoding::ModifiedUtf7.mailbox(&enc).unwrap(); assert_eq!(dec, s, "roundtrip failed for {s:?}, enc={enc:?}"); } } @@ -134,25 +170,38 @@ mod tests { #[test] fn raw_utf8_name_survives_mutf7_decode_untouched() { - assert_eq!(decode_mailbox_name("Envoyés").unwrap(), "Envoyés"); assert_eq!( - decode_mailbox_name("L/Le Vent Se Lève").unwrap(), + NameEncoding::ModifiedUtf7.mailbox("Envoyés").unwrap(), + "Envoyés" + ); + assert_eq!( + NameEncoding::ModifiedUtf7 + .mailbox("L/Le Vent Se Lève") + .unwrap(), "L/Le Vent Se Lève" ); - assert_eq!(decode_mailbox_name("R&-D é").unwrap(), "R&D é"); + assert_eq!( + NameEncoding::ModifiedUtf7.mailbox("R&-D é").unwrap(), + "R&D é" + ); } #[test] fn raw_utf8_name_round_trips_through_encode() { for s in ["Envoyés", "Gönderilmiş Postalar", "Çöp kutusu"] { - assert_eq!(decode_mailbox_name(&utf7::IMAP.encode(s)).unwrap(), s); - assert_eq!(decode_mailbox_name(s).unwrap(), s); + assert_eq!( + NameEncoding::ModifiedUtf7 + .mailbox(&utf7::IMAP.encode(s)) + .unwrap(), + s + ); + assert_eq!(NameEncoding::ModifiedUtf7.mailbox(s).unwrap(), s); } } #[test] fn utf8_accept_skips_mutf7_decode_so_ampersand_passes_through() { - assert_eq!(decode_mailbox_name_with("R&D", true).unwrap(), "R&D"); - assert_eq!(decode_mailbox_name_with("日本語", true).unwrap(), "日本語"); + assert_eq!(utf8_mailbox("R&D", true).unwrap(), "R&D"); + assert_eq!(utf8_mailbox("日本語", true).unwrap(), "日本語"); } } diff --git a/src/imap/response.rs b/src/imap/response.rs deleted file mode 100644 index 1bbc097..0000000 --- a/src/imap/response.rs +++ /dev/null @@ -1,1207 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::BTreeMap; -use std::io::BufRead; - -use super::error::{ImapError, NoError}; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Status { - Ok, - No, - Bad, - Bye, - PreAuth, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct StatusLine { - pub status: Status, - pub code: Option, - pub code_args: Option, - pub text: String, -} - -impl StatusLine { - pub fn into_no_error(self) -> NoError { - NoError::new(self.text, self.code) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Value { - Atom(String), - Str(String), - Bytes(Vec), - Number(u64), - Nil, - List(Vec), -} - -impl Value { - pub fn as_str(&self) -> Option<&str> { - match self { - Value::Atom(s) | Value::Str(s) => Some(s), - _ => None, - } - } - - pub fn as_number(&self) -> Option { - match self { - Value::Number(n) => Some(*n), - _ => None, - } - } - - pub fn as_list(&self) -> Option<&[Value]> { - match self { - Value::List(items) => Some(items.as_slice()), - _ => None, - } - } - - pub fn into_bytes(self) -> Option> { - match self { - Value::Bytes(b) => Some(b), - Value::Str(s) => Some(s.into_bytes()), - _ => None, - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Untagged { - Capability(Vec), - List { - attributes: Vec, - delimiter: Option, - name: String, - }, - Lsub { - attributes: Vec, - delimiter: Option, - name: String, - }, - Status { - mailbox: String, - items: BTreeMap, - }, - Search(Vec), - Esearch { - tag: Option, - all: Vec, - count: Option, - }, - Exists(u32), - Recent(u32), - Expunge(u32), - Fetch { - seq: u32, - items: Vec<(String, Value)>, - }, - Flags(Vec), - Namespace { - personal: Vec, - others: Vec, - shared: Vec, - }, - Enabled(Vec), - StatusLine(StatusLine), - Bye(String), - Raw(String), -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct NamespaceEntry { - pub prefix: String, - pub delimiter: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Response { - Tagged { tag: String, line: StatusLine }, - Untagged(Untagged), - Continuation(String), -} - -pub fn parse_response(reader: &mut R) -> Result { - let mut parser = Parser::new(reader); - parser.parse_response() -} - -struct Parser<'r, R: BufRead> { - reader: &'r mut R, - buf: Vec, - pos: usize, -} - -impl<'r, R: BufRead> Parser<'r, R> { - fn new(reader: &'r mut R) -> Self { - Self { - reader, - buf: Vec::new(), - pos: 0, - } - } - - fn parse_response(&mut self) -> Result { - self.read_line()?; - if self.buf.is_empty() { - return Err(ImapError::Disconnected); - } - if self.peek() == Some(b'+') { - self.bump(); - self.skip_ws(); - let text = self.read_until_crlf(); - return Ok(Response::Continuation(text)); - } - if self.peek() == Some(b'*') { - self.bump(); - self.skip_ws(); - return self.parse_untagged().map(Response::Untagged); - } - let tag = self.read_atom_string()?; - self.skip_ws(); - let line = self.parse_status_line()?; - Ok(Response::Tagged { tag, line }) - } - - fn parse_untagged(&mut self) -> Result { - let first = self.read_atom_string()?; - let upper = first.to_ascii_uppercase(); - match upper.as_str() { - "OK" | "NO" | "BAD" | "BYE" | "PREAUTH" => { - self.skip_ws(); - let mut line = self.parse_status_line_body()?; - line.status = match upper.as_str() { - "OK" => Status::Ok, - "NO" => Status::No, - "BAD" => Status::Bad, - "BYE" => Status::Bye, - "PREAUTH" => Status::PreAuth, - _ => unreachable!(), - }; - if matches!(line.status, Status::Bye) { - return Ok(Untagged::Bye(line.text)); - } - Ok(Untagged::StatusLine(line)) - } - "CAPABILITY" => Ok(Untagged::Capability(self.read_space_separated()?)), - "LIST" | "LSUB" => { - self.skip_ws(); - self.expect(b'(')?; - let mut attrs = Vec::new(); - while self.peek() != Some(b')') { - self.skip_ws(); - if self.peek() == Some(b')') { - break; - } - attrs.push(self.read_atom_string()?); - } - self.expect(b')')?; - self.skip_ws(); - let delim_value = self.parse_value()?; - let delimiter = match delim_value { - Value::Nil => None, - Value::Str(s) | Value::Atom(s) => s.chars().next(), - Value::Bytes(b) if !b.is_empty() => Some(b[0] as char), - _ => None, - }; - self.skip_ws(); - let name_val = self.parse_value()?; - let name = match name_val { - Value::Str(s) | Value::Atom(s) => s, - Value::Number(n) => n.to_string(), - Value::Bytes(b) => String::from_utf8_lossy(&b).into_owned(), - _ => return Err(ImapError::Parse("LIST mailbox name missing".into())), - }; - if upper == "LIST" { - Ok(Untagged::List { - attributes: attrs, - delimiter, - name, - }) - } else { - Ok(Untagged::Lsub { - attributes: attrs, - delimiter, - name, - }) - } - } - "STATUS" => { - self.skip_ws(); - let mailbox = match self.parse_value()? { - Value::Str(s) | Value::Atom(s) => s, - Value::Number(n) => n.to_string(), - Value::Bytes(b) => String::from_utf8_lossy(&b).into_owned(), - _ => return Err(ImapError::Parse("STATUS mailbox missing".into())), - }; - self.skip_ws(); - self.expect(b'(')?; - let mut items = BTreeMap::new(); - while self.peek() != Some(b')') { - self.skip_ws(); - if self.peek() == Some(b')') { - break; - } - let key = self.read_atom_string()?.to_ascii_uppercase(); - self.skip_ws(); - let val = self.parse_value()?; - if let Some(n) = val.as_number() { - items.insert(key, n); - } - } - self.expect(b')')?; - Ok(Untagged::Status { mailbox, items }) - } - "SEARCH" => { - let mut ids = Vec::new(); - while !self.at_end() { - self.skip_ws(); - if self.at_end() { - break; - } - let v = self.parse_value()?; - if let Some(n) = v.as_number() { - ids.push(n as u32); - } - } - Ok(Untagged::Search(ids)) - } - "ESEARCH" => self.parse_esearch(), - "FLAGS" => { - self.skip_ws(); - let v = self.parse_value()?; - let flags = match v { - Value::List(items) => items - .into_iter() - .filter_map(|x| match x { - Value::Atom(s) | Value::Str(s) => Some(s), - _ => None, - }) - .collect(), - _ => Vec::new(), - }; - Ok(Untagged::Flags(flags)) - } - "NAMESPACE" => self.parse_namespace(), - "ENABLED" => Ok(Untagged::Enabled(self.read_space_separated()?)), - _ => { - if let Ok(num) = upper.parse::() { - self.skip_ws(); - let kw = self.read_atom_string()?.to_ascii_uppercase(); - match kw.as_str() { - "EXISTS" => return Ok(Untagged::Exists(num)), - "RECENT" => return Ok(Untagged::Recent(num)), - "EXPUNGE" => return Ok(Untagged::Expunge(num)), - "FETCH" => return self.parse_fetch_body(num), - _ => { - let rest = self.read_until_crlf(); - return Ok(Untagged::Raw(format!("{num} {kw} {rest}"))); - } - } - } - let rest = self.read_until_crlf(); - Ok(Untagged::Raw(format!("{first} {rest}"))) - } - } - } - - fn parse_esearch(&mut self) -> Result { - let mut tag = None; - let mut all = Vec::new(); - let mut count = None; - if self.peek() == Some(b' ') { - self.skip_ws(); - } - if self.peek() == Some(b'(') { - self.bump(); - self.skip_ws(); - let key = self.read_atom_string()?.to_ascii_uppercase(); - if key == "TAG" { - self.skip_ws(); - if let Value::Str(s) = self.parse_value()? { - tag = Some(s); - } - } - self.skip_ws(); - self.expect(b')')?; - } - while !self.at_end() { - self.skip_ws(); - if self.at_end() { - break; - } - let key = self.read_atom_string()?.to_ascii_uppercase(); - match key.as_str() { - "UID" => {} - "ALL" => { - self.skip_ws(); - let s = self.read_seq_set_token(); - all = parse_seq_set(&s); - } - "COUNT" => { - self.skip_ws(); - if let Value::Number(n) = self.parse_value()? { - count = Some(n as u32); - } - } - _ => { - self.skip_ws(); - let _ = self.parse_value(); - } - } - } - Ok(Untagged::Esearch { tag, all, count }) - } - - fn parse_namespace(&mut self) -> Result { - self.skip_ws(); - let personal = self.parse_namespace_list()?; - self.skip_ws(); - let others = self.parse_namespace_list()?; - self.skip_ws(); - let shared = self.parse_namespace_list()?; - Ok(Untagged::Namespace { - personal, - others, - shared, - }) - } - - fn parse_namespace_list(&mut self) -> Result, ImapError> { - match self.parse_value()? { - Value::Nil => Ok(Vec::new()), - Value::List(entries) => { - let mut out = Vec::new(); - for e in entries { - if let Value::List(inner) = e - && let Some(prefix) = inner.first().and_then(|v| v.as_str()) - { - let delim = inner.get(1).and_then(|v| match v { - Value::Nil => None, - Value::Str(s) | Value::Atom(s) => s.chars().next(), - _ => None, - }); - out.push(NamespaceEntry { - prefix: prefix.to_owned(), - delimiter: delim, - }); - } - } - Ok(out) - } - _ => Ok(Vec::new()), - } - } - - fn parse_fetch_body(&mut self, seq: u32) -> Result { - self.skip_ws(); - self.expect(b'(')?; - let mut items = Vec::new(); - while self.peek() != Some(b')') { - self.skip_ws(); - if self.peek() == Some(b')') { - break; - } - let name = self.read_fetch_item_name()?; - self.skip_ws(); - let val = self.parse_value()?; - items.push((name, val)); - } - self.expect(b')')?; - Ok(Untagged::Fetch { seq, items }) - } - - fn read_fetch_item_name(&mut self) -> Result { - let start = self.pos; - while let Some(b) = self.peek() { - match b { - b' ' | b'\r' | b'\n' | b')' | b'(' => break, - _ => self.bump(), - } - } - let bytes = &self.buf[start..self.pos]; - std::str::from_utf8(bytes) - .map(|s| s.to_ascii_uppercase()) - .map_err(|e| ImapError::Parse(format!("non-utf8 fetch item name: {e}"))) - } - - fn parse_status_line(&mut self) -> Result { - let kw = self.read_atom_string()?; - self.skip_ws(); - let mut line = self.parse_status_line_body()?; - line.status = match kw.to_ascii_uppercase().as_str() { - "OK" => Status::Ok, - "NO" => Status::No, - "BAD" => Status::Bad, - "BYE" => Status::Bye, - "PREAUTH" => Status::PreAuth, - other => return Err(ImapError::Parse(format!("unknown status: {other}"))), - }; - Ok(line) - } - - fn parse_status_line_body(&mut self) -> Result { - let mut code = None; - let mut code_args = None; - if self.peek() == Some(b'[') { - self.bump(); - let start = self.pos; - let mut depth = 1usize; - while let Some(b) = self.peek() { - if b == b'[' { - depth += 1; - self.bump(); - } else if b == b']' { - depth -= 1; - if depth == 0 { - break; - } - self.bump(); - } else { - self.bump(); - } - } - let inside = &self.buf[start..self.pos]; - let inside_str = std::str::from_utf8(inside) - .map_err(|e| ImapError::Parse(format!("non-utf8 status code: {e}")))?; - let (head, tail) = match inside_str.split_once(' ') { - Some((h, t)) => (h.to_owned(), Some(t.to_owned())), - None => (inside_str.to_owned(), None), - }; - code = Some(head.to_ascii_uppercase()); - code_args = tail; - self.expect(b']')?; - self.skip_ws(); - } - let text = self.read_until_crlf(); - Ok(StatusLine { - status: Status::Ok, - code, - code_args, - text, - }) - } - - fn parse_value(&mut self) -> Result { - self.skip_ws(); - match self.peek() { - Some(b'"') => self.parse_quoted(), - Some(b'{') => self.parse_literal(), - Some(b'(') => self.parse_list(), - Some(b) - if b.is_ascii_digit() - || b == b'-' - || b == b'+' - || is_atom_byte(b) - || b == b'\\' => - { - self.parse_atom_or_number() - } - None => Err(ImapError::Parse("expected value, got EOL".into())), - Some(other) => Err(ImapError::Parse(format!( - "unexpected byte {other:#04x} in value" - ))), - } - } - - fn parse_quoted(&mut self) -> Result { - self.expect(b'"')?; - let mut raw: Vec = Vec::new(); - while let Some(b) = self.peek() { - match b { - b'"' => { - self.bump(); - return Ok(match String::from_utf8(raw) { - Ok(s) => Value::Str(s), - Err(e) => Value::Bytes(e.into_bytes()), - }); - } - b'\\' => { - self.bump(); - let next = self - .peek() - .ok_or_else(|| ImapError::Parse("trailing \\ in quoted".into()))?; - raw.push(next); - self.bump(); - } - _ => { - raw.push(b); - self.bump(); - } - } - } - Err(ImapError::Parse("unterminated quoted string".into())) - } - - fn parse_literal(&mut self) -> Result { - self.expect(b'{')?; - let mut len_str = String::new(); - while let Some(b) = self.peek() { - if b.is_ascii_digit() { - len_str.push(b as char); - self.bump(); - } else { - break; - } - } - if self.peek() == Some(b'+') { - self.bump(); - } - self.expect(b'}')?; - let len: usize = len_str - .parse() - .map_err(|e| ImapError::Parse(format!("literal length: {e}")))?; - if !self.at_end_of_line() { - return Err(ImapError::Parse("garbage after literal header".into())); - } - let mut data = vec![0u8; len]; - self.reader.read_exact(&mut data).map_err(ImapError::Io)?; - self.buf.clear(); - self.read_line()?; - if let Ok(s) = std::str::from_utf8(&data) { - Ok(Value::Str(s.to_owned())) - } else { - Ok(Value::Bytes(data)) - } - } - - fn parse_list(&mut self) -> Result { - self.expect(b'(')?; - let mut items = Vec::new(); - loop { - self.skip_ws(); - if self.peek() == Some(b')') { - self.bump(); - return Ok(Value::List(items)); - } - if self.peek().is_none() { - return Err(ImapError::Parse("unterminated list".into())); - } - items.push(self.parse_value()?); - } - } - - fn parse_atom_or_number(&mut self) -> Result { - let start = self.pos; - while let Some(b) = self.peek() { - if is_atom_byte(b) || b == b'\\' || b == b'*' || b == b']' || b == b'+' { - self.bump(); - } else { - break; - } - } - let raw = &self.buf[start..self.pos]; - let s = std::str::from_utf8(raw) - .map_err(|e| ImapError::Parse(format!("non-utf8 atom: {e}")))? - .to_owned(); - if s.is_empty() { - return Err(ImapError::Parse("empty atom".into())); - } - if s.eq_ignore_ascii_case("NIL") { - return Ok(Value::Nil); - } - if let Ok(n) = s.parse::() { - return Ok(Value::Number(n)); - } - Ok(Value::Atom(s)) - } - - fn read_atom_string(&mut self) -> Result { - let start = self.pos; - while let Some(b) = self.peek() { - if is_atom_byte(b) { - self.bump(); - } else { - break; - } - } - if start == self.pos { - return Err(ImapError::Parse("expected atom".into())); - } - let bytes = &self.buf[start..self.pos]; - std::str::from_utf8(bytes) - .map(|s| s.to_owned()) - .map_err(|e| ImapError::Parse(format!("non-utf8 atom: {e}"))) - } - - fn read_space_separated(&mut self) -> Result, ImapError> { - let end = self.buf.len().saturating_sub(self.tail_crlf_len()); - let rest = self.buf.get(self.pos..end).unwrap_or_default(); - self.pos = self.pos.max(end); - std::str::from_utf8(rest) - .map(|s| s.split_ascii_whitespace().map(str::to_owned).collect()) - .map_err(|e| ImapError::Parse(format!("non-utf8 token: {e}"))) - } - - fn read_seq_set_token(&mut self) -> String { - let start = self.pos; - while let Some(b) = self.peek() { - if b == b' ' || b == b'\r' || b == b'\n' { - break; - } - self.bump(); - } - String::from_utf8_lossy(&self.buf[start..self.pos]).into_owned() - } - - fn read_until_crlf(&mut self) -> String { - let start = self.pos; - let end = self.buf.len().saturating_sub(2); - let end = end.max(start); - self.pos = self.buf.len(); - String::from_utf8_lossy(&self.buf[start..end]) - .trim_end() - .to_owned() - } - - fn read_line(&mut self) -> Result<(), ImapError> { - self.buf.clear(); - let n = self.reader.read_until(b'\n', &mut self.buf)?; - self.pos = 0; - if n == 0 { - return Ok(()); - } - Ok(()) - } - - fn peek(&self) -> Option { - let end = self.buf.len().saturating_sub(self.tail_crlf_len()); - if self.pos >= end { - None - } else { - Some(self.buf[self.pos]) - } - } - - fn bump(&mut self) { - self.pos += 1; - } - - fn skip_ws(&mut self) { - while self.peek() == Some(b' ') { - self.bump(); - } - } - - fn at_end(&self) -> bool { - self.peek().is_none() - } - - fn at_end_of_line(&self) -> bool { - let end = self.buf.len().saturating_sub(self.tail_crlf_len()); - self.pos >= end - } - - fn tail_crlf_len(&self) -> usize { - if self.buf.ends_with(b"\r\n") { - 2 - } else if self.buf.ends_with(b"\n") { - 1 - } else { - 0 - } - } - - fn expect(&mut self, b: u8) -> Result<(), ImapError> { - if self.peek() == Some(b) { - self.bump(); - Ok(()) - } else { - Err(ImapError::Parse(format!( - "expected {:?}, got {:?}", - b as char, - self.peek().map(|x| x as char) - ))) - } - } -} - -fn is_atom_byte(b: u8) -> bool { - b.is_ascii_graphic() && !matches!(b, b'(' | b')' | b'{' | b'%' | b'"' | b']') -} - -fn parse_seq_set(s: &str) -> Vec { - let mut out = Vec::new(); - for part in s.split(',') { - let part = part.trim(); - if part.is_empty() { - continue; - } - if let Some((lo, hi)) = part.split_once(':') { - let lo: u32 = match lo.parse() { - Ok(n) => n, - Err(_) => continue, - }; - let hi: u32 = match hi.parse() { - Ok(n) => n, - Err(_) => continue, - }; - let (a, b) = if lo <= hi { (lo, hi) } else { (hi, lo) }; - for n in a..=b { - out.push(n); - } - } else if let Ok(n) = part.parse::() { - out.push(n); - } - } - out -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Cursor; - - fn parse(input: &[u8]) -> Response { - parse_response(&mut Cursor::new(input)).unwrap() - } - - #[test] - fn tagged_ok() { - let r = parse(b"A001 OK LOGIN completed\r\n"); - match r { - Response::Tagged { tag, line } => { - assert_eq!(tag, "A001"); - assert_eq!(line.status, Status::Ok); - assert_eq!(line.text, "LOGIN completed"); - } - _ => panic!("expected Tagged"), - } - } - - #[test] - fn tagged_no_with_code() { - let r = parse(b"A002 NO [AUTHENTICATIONFAILED] bad credentials\r\n"); - match r { - Response::Tagged { tag, line } => { - assert_eq!(tag, "A002"); - assert_eq!(line.status, Status::No); - assert_eq!(line.code.as_deref(), Some("AUTHENTICATIONFAILED")); - assert_eq!(line.text, "bad credentials"); - } - _ => panic!("expected Tagged"), - } - } - - #[test] - fn untagged_capability() { - let r = parse(b"* CAPABILITY IMAP4rev1 STARTTLS AUTH=PLAIN AUTH=LOGIN\r\n"); - match r { - Response::Untagged(Untagged::Capability(caps)) => { - assert_eq!( - caps, - vec!["IMAP4rev1", "STARTTLS", "AUTH=PLAIN", "AUTH=LOGIN"] - ); - } - _ => panic!("expected Capability"), - } - } - - #[test] - fn untagged_capability_with_literal_plus() { - let r = parse( - b"* CAPABILITY IMAP4rev2 IMAP4rev1 ENABLE SASL-IR LITERAL+ ID UTF8=ACCEPT JMAPACCESS AUTH=PLAIN AUTH=OAUTHBEARER AUTH=XOAUTH2\r\n", - ); - match r { - Response::Untagged(Untagged::Capability(caps)) => { - assert!(caps.iter().any(|c| c == "LITERAL+")); - assert!(caps.iter().any(|c| c == "AUTH=OAUTHBEARER")); - assert_eq!(caps.len(), 11); - } - _ => panic!("expected Capability"), - } - } - - #[test] - fn untagged_capability_accepts_uri_valued_tokens() { - let r = parse( - b"* CAPABILITY IMAP4rev1 SASL-IR SPECIAL-USE IMAPSIEVE=sieve://127.0.0.1:4190 QUOTA ACL RIGHTS=texk\r\n", - ); - match r { - Response::Untagged(Untagged::Capability(caps)) => assert_eq!( - caps, - vec![ - "IMAP4rev1", - "SASL-IR", - "SPECIAL-USE", - "IMAPSIEVE=sieve://127.0.0.1:4190", - "QUOTA", - "ACL", - "RIGHTS=texk", - ] - ), - other => panic!("expected Capability, got {other:?}"), - } - } - - #[test] - fn untagged_enabled_accepts_non_atom_tokens() { - let r = parse(b"* ENABLED UTF8=ACCEPT X-VENDOR=a:b,c\r\n"); - match r { - Response::Untagged(Untagged::Enabled(exts)) => { - assert_eq!(exts, vec!["UTF8=ACCEPT", "X-VENDOR=a:b,c"]); - } - other => panic!("expected Enabled, got {other:?}"), - } - } - - #[test] - fn atom_accepts_colon_comma_and_brackets() { - let r = parse(b"* LIST (\\HasNoChildren) \"/\" Work:2026,Q1[a}\r\n"); - match r { - Response::Untagged(Untagged::List { name, .. }) => { - assert_eq!(name, "Work:2026,Q1[a}"); - } - other => panic!("expected List, got {other:?}"), - } - } - - #[test] - fn atom_stops_at_atom_specials() { - for b in b"(){%\"] \x7f\x00" { - assert!(!is_atom_byte(*b), "{b:#04x} must not be an atom byte"); - } - for b in b":,[}=/.-_+" { - assert!(is_atom_byte(*b), "{:?} must be an atom byte", *b as char); - } - } - - #[test] - fn quoted_mailbox_name_with_utf8_bytes_is_decoded_as_utf8() { - let r = parse("* LIST (\\HasNoChildren) \"/\" \"Envoy\u{e9}s\"\r\n".as_bytes()); - match r { - Response::Untagged(Untagged::List { name, .. }) => assert_eq!(name, "Envoyés"), - other => panic!("expected List, got {other:?}"), - } - } - - #[test] - fn quoted_mailbox_name_with_modified_utf7_is_left_untouched() { - let r = - parse(b"* LIST (\\HasNoChildren) \"/\" \"[Gmail]/G&APY-nderilmi&AV8- Postalar\"\r\n"); - match r { - Response::Untagged(Untagged::List { name, .. }) => { - assert_eq!(name, "[Gmail]/G&APY-nderilmi&AV8- Postalar"); - } - other => panic!("expected List, got {other:?}"), - } - } - - #[test] - fn quoted_mailbox_name_with_invalid_utf8_falls_back_to_lossy() { - let mut line: Vec = b"* LIST (\\HasNoChildren) \"/\" \"bad".to_vec(); - line.push(0xff); - line.extend_from_slice(b"name\"\r\n"); - let r = parse(&line); - match r { - Response::Untagged(Untagged::List { name, .. }) => { - assert!(name.starts_with("bad") && name.ends_with("name")); - } - other => panic!("expected List, got {other:?}"), - } - } - - #[test] - fn untagged_list_with_special_use() { - let r = parse(b"* LIST (\\HasNoChildren \\Sent) \"/\" \"Sent Mail\"\r\n"); - match r { - Response::Untagged(Untagged::List { - attributes, - delimiter, - name, - }) => { - assert_eq!(attributes, vec!["\\HasNoChildren", "\\Sent"]); - assert_eq!(delimiter, Some('/')); - assert_eq!(name, "Sent Mail"); - } - _ => panic!("expected List"), - } - } - - #[test] - fn untagged_list_with_nil_delimiter() { - let r = parse(b"* LIST () NIL INBOX\r\n"); - match r { - Response::Untagged(Untagged::List { - attributes, - delimiter, - name, - }) => { - assert!(attributes.is_empty()); - assert_eq!(delimiter, None); - assert_eq!(name, "INBOX"); - } - _ => panic!("expected List"), - } - } - - #[test] - fn untagged_list_with_numeric_atom_name() { - let r = parse(b"* LIST (\\Subscribed \\HasNoChildren \\UnMarked) \"/\" 200\r\n"); - match r { - Response::Untagged(Untagged::List { - attributes, - delimiter, - name, - }) => { - assert_eq!( - attributes, - vec!["\\Subscribed", "\\HasNoChildren", "\\UnMarked"] - ); - assert_eq!(delimiter, Some('/')); - assert_eq!(name, "200"); - } - _ => panic!("expected List"), - } - } - - #[test] - fn untagged_status() { - let r = parse(b"* STATUS INBOX (UIDVALIDITY 12345 UIDNEXT 42 MESSAGES 7)\r\n"); - match r { - Response::Untagged(Untagged::Status { mailbox, items }) => { - assert_eq!(mailbox, "INBOX"); - assert_eq!(items.get("UIDVALIDITY"), Some(&12345)); - assert_eq!(items.get("UIDNEXT"), Some(&42)); - assert_eq!(items.get("MESSAGES"), Some(&7)); - } - _ => panic!("expected Status"), - } - } - - #[test] - fn untagged_status_with_numeric_atom_mailbox() { - let r = parse(b"* STATUS 200 (MESSAGES 5 UIDNEXT 6 UIDVALIDITY 1778253936)\r\n"); - match r { - Response::Untagged(Untagged::Status { mailbox, items }) => { - assert_eq!(mailbox, "200"); - assert_eq!(items.get("MESSAGES"), Some(&5)); - assert_eq!(items.get("UIDNEXT"), Some(&6)); - assert_eq!(items.get("UIDVALIDITY"), Some(&1778253936)); - } - _ => panic!("expected Status"), - } - } - - #[test] - fn untagged_search_with_uids() { - let r = parse(b"* SEARCH 1 3 5 7\r\n"); - match r { - Response::Untagged(Untagged::Search(ids)) => { - assert_eq!(ids, vec![1, 3, 5, 7]); - } - _ => panic!("expected Search"), - } - } - - #[test] - fn untagged_search_empty() { - let r = parse(b"* SEARCH\r\n"); - match r { - Response::Untagged(Untagged::Search(ids)) => { - assert!(ids.is_empty()); - } - _ => panic!("expected Search"), - } - } - - #[test] - fn untagged_esearch_all_set() { - let r = parse(b"* ESEARCH (TAG \"q1\") UID ALL 1:3,7\r\n"); - match r { - Response::Untagged(Untagged::Esearch { tag, all, .. }) => { - assert_eq!(tag.as_deref(), Some("q1")); - assert_eq!(all, vec![1, 2, 3, 7]); - } - _ => panic!("expected Esearch"), - } - } - - #[test] - fn untagged_exists_recent() { - let r = parse(b"* 42 EXISTS\r\n"); - assert!(matches!(r, Response::Untagged(Untagged::Exists(42)))); - let r = parse(b"* 3 RECENT\r\n"); - assert!(matches!(r, Response::Untagged(Untagged::Recent(3)))); - } - - #[test] - fn untagged_fetch_flags_and_uid() { - let r = parse( - b"* 17 FETCH (UID 100 FLAGS (\\Seen \\Flagged) RFC822.SIZE 4242 INTERNALDATE \"17-Jul-1996 02:44:25 -0700\")\r\n", - ); - match r { - Response::Untagged(Untagged::Fetch { seq, items }) => { - assert_eq!(seq, 17); - let uid = items.iter().find(|(n, _)| n == "UID").unwrap(); - assert_eq!(uid.1.as_number(), Some(100)); - let flags = items.iter().find(|(n, _)| n == "FLAGS").unwrap(); - let list = flags.1.as_list().unwrap(); - assert_eq!(list.len(), 2); - let size = items.iter().find(|(n, _)| n == "RFC822.SIZE").unwrap(); - assert_eq!(size.1.as_number(), Some(4242)); - let internaldate = items.iter().find(|(n, _)| n == "INTERNALDATE").unwrap(); - assert_eq!(internaldate.1.as_str(), Some("17-Jul-1996 02:44:25 -0700")); - } - _ => panic!("expected Fetch"), - } - } - - #[test] - fn untagged_fetch_with_body_literal() { - let input = b"* 1 FETCH (UID 5 BODY[] {11}\r\nHello world)\r\n"; - let r = parse(input); - match r { - Response::Untagged(Untagged::Fetch { seq, items }) => { - assert_eq!(seq, 1); - let body = items.iter().find(|(n, _)| n == "BODY[]").unwrap(); - match &body.1 { - Value::Str(s) => assert_eq!(s, "Hello world"), - Value::Bytes(b) => assert_eq!(b, b"Hello world"), - other => panic!("expected literal, got {other:?}"), - } - } - _ => panic!("expected Fetch"), - } - } - - #[test] - fn untagged_fetch_with_binary_literal() { - let mut input = b"* 1 FETCH (UID 5 BODY[] {4}\r\n".to_vec(); - input.extend_from_slice(&[0xFF, 0xFE, 0x00, 0x01]); - input.extend_from_slice(b")\r\n"); - let r = parse_response(&mut Cursor::new(input)).unwrap(); - match r { - Response::Untagged(Untagged::Fetch { items, .. }) => { - let body = items.iter().find(|(n, _)| n == "BODY[]").unwrap(); - match &body.1 { - Value::Bytes(b) => assert_eq!(b, &[0xFF, 0xFE, 0x00, 0x01]), - other => panic!("expected Bytes, got {other:?}"), - } - } - _ => panic!("expected Fetch"), - } - } - - #[test] - fn untagged_flags() { - let r = parse(b"* FLAGS (\\Answered \\Flagged \\Draft \\Deleted \\Seen)\r\n"); - match r { - Response::Untagged(Untagged::Flags(flags)) => { - assert_eq!(flags.len(), 5); - assert!(flags.contains(&"\\Seen".to_owned())); - } - _ => panic!("expected Flags"), - } - } - - #[test] - fn untagged_bye() { - let r = parse(b"* BYE server logging out\r\n"); - match r { - Response::Untagged(Untagged::Bye(text)) => { - assert_eq!(text, "server logging out"); - } - _ => panic!("expected Bye"), - } - } - - #[test] - fn untagged_namespace() { - let r = parse(b"* NAMESPACE ((\"INBOX.\" \".\")) NIL NIL\r\n"); - match r { - Response::Untagged(Untagged::Namespace { - personal, - others, - shared, - }) => { - assert_eq!(personal.len(), 1); - assert_eq!(personal[0].prefix, "INBOX."); - assert_eq!(personal[0].delimiter, Some('.')); - assert!(others.is_empty()); - assert!(shared.is_empty()); - } - _ => panic!("expected Namespace"), - } - } - - #[test] - fn continuation_request() { - let r = parse(b"+ Ready for additional command text\r\n"); - match r { - Response::Continuation(text) => { - assert_eq!(text, "Ready for additional command text"); - } - _ => panic!("expected Continuation"), - } - } - - #[test] - fn untagged_status_with_response_code() { - let r = parse(b"* OK [PERMANENTFLAGS (\\Seen \\Flagged \\*)] Limited\r\n"); - match r { - Response::Untagged(Untagged::StatusLine(line)) => { - assert_eq!(line.status, Status::Ok); - assert_eq!(line.code.as_deref(), Some("PERMANENTFLAGS")); - assert!(line.code_args.as_deref().is_some()); - assert_eq!(line.text, "Limited"); - } - _ => panic!("expected StatusLine"), - } - } - - #[test] - fn untagged_ok_with_uidvalidity() { - let r = parse(b"* OK [UIDVALIDITY 3857529045] UIDs valid\r\n"); - match r { - Response::Untagged(Untagged::StatusLine(line)) => { - assert_eq!(line.code.as_deref(), Some("UIDVALIDITY")); - assert_eq!(line.code_args.as_deref(), Some("3857529045")); - } - _ => panic!("expected StatusLine"), - } - } - - #[test] - fn quoted_string_with_escape() { - let r = parse(b"* LIST () \"/\" \"He said \\\"hi\\\"\"\r\n"); - match r { - Response::Untagged(Untagged::List { name, .. }) => { - assert_eq!(name, "He said \"hi\""); - } - _ => panic!("expected List"), - } - } - - #[test] - fn seq_set_parses_ranges_and_singletons() { - assert_eq!(parse_seq_set("1:3,5,7:8"), vec![1, 2, 3, 5, 7, 8]); - assert_eq!(parse_seq_set("42"), vec![42]); - assert!(parse_seq_set("").is_empty()); - } - - #[test] - fn empty_read_is_disconnected() { - let mut empty: &[u8] = b""; - let err = parse_response(&mut empty).unwrap_err(); - assert!(matches!(err, ImapError::Disconnected)); - } - - #[test] - fn untagged_enabled_parses() { - let r = parse(b"* ENABLED UTF8=ACCEPT CONDSTORE\r\n"); - match r { - Response::Untagged(Untagged::Enabled(exts)) => { - assert_eq!(exts, vec!["UTF8=ACCEPT", "CONDSTORE"]); - } - other => panic!("expected Enabled, got {other:?}"), - } - } -} diff --git a/src/imap/response/lexer.rs b/src/imap/response/lexer.rs new file mode 100644 index 0000000..297c5e1 --- /dev/null +++ b/src/imap/response/lexer.rs @@ -0,0 +1,278 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::{self, BufRead, Read}; +use std::str; + +use super::Value; +use super::parser::{LITERAL_RESERVE, MAX_SEQ_SET, Parser}; +use crate::imap::error::ImapError; + +impl Parser<'_, R> { + pub(super) fn parse_value(&mut self) -> Result { + self.skip_ws(); + match self.peek() { + Some(b'"') => self.parse_quoted(), + Some(b'{') => self.parse_literal(), + Some(b'(') => self.parse_list(), + Some(b) + if b.is_ascii_digit() + || b == b'-' + || b == b'+' + || AtomByte::is(b) + || b == b'\\' => + { + self.parse_atom_or_number() + } + None => Err(ImapError::Parse("expected value, got EOL".into())), + Some(other) => Err(ImapError::Parse(format!( + "unexpected byte {other:#04x} in value" + ))), + } + } + + pub(super) fn parse_quoted(&mut self) -> Result { + self.expect(b'"')?; + let mut raw: Vec = Vec::new(); + while let Some(b) = self.peek() { + match b { + b'"' => { + self.bump(); + return Ok(match String::from_utf8(raw) { + Ok(s) => Value::Str(s), + Err(e) => Value::Bytes(e.into_bytes()), + }); + } + b'\\' => { + self.bump(); + let next = self + .peek() + .ok_or_else(|| ImapError::Parse("trailing \\ in quoted".into()))?; + raw.push(next); + self.bump(); + } + _ => { + raw.push(b); + self.bump(); + } + } + } + Err(ImapError::Parse("unterminated quoted string".into())) + } + + pub(super) fn parse_literal(&mut self) -> Result { + self.expect(b'{')?; + let start = self.pos; + while self.peek().is_some_and(|b| b.is_ascii_digit()) { + self.bump(); + } + let len_str = String::from_utf8_lossy(self.span(start)).into_owned(); + if self.peek() == Some(b'+') { + self.bump(); + } + self.expect(b'}')?; + let len: usize = len_str + .parse() + .map_err(|e| ImapError::Parse(format!("literal length: {e}")))?; + if !self.at_end_of_line() { + return Err(ImapError::Parse("garbage after literal header".into())); + } + let mut data = Vec::with_capacity(len.min(LITERAL_RESERVE)); + self.reader + .by_ref() + .take(u64::try_from(len).unwrap_or(u64::MAX)) + .read_to_end(&mut data)?; + if data.len() != len { + return Err(ImapError::Io(io::Error::new( + io::ErrorKind::UnexpectedEof, + "failed to fill whole buffer", + ))); + } + self.buf.clear(); + self.read_line()?; + Ok(match String::from_utf8(data) { + Ok(s) => Value::Str(s), + Err(e) => Value::Bytes(e.into_bytes()), + }) + } + + pub(super) fn parse_list(&mut self) -> Result { + self.expect(b'(')?; + let mut items = Vec::new(); + loop { + self.skip_ws(); + if self.peek() == Some(b')') { + self.bump(); + return Ok(Value::List(items)); + } + if self.peek().is_none() { + return Err(ImapError::Parse("unterminated list".into())); + } + items.push(self.parse_value()?); + } + } + + pub(super) fn parse_atom_or_number(&mut self) -> Result { + let start = self.pos; + while let Some(b) = self.peek() { + if AtomByte::is(b) || b == b'\\' || b == b'*' || b == b']' || b == b'+' { + self.bump(); + } else { + break; + } + } + let s = str::from_utf8(self.span(start)) + .map_err(|e| ImapError::Parse(format!("non-utf8 atom: {e}")))? + .to_owned(); + if s.is_empty() { + return Err(ImapError::Parse("empty atom".into())); + } + if s.eq_ignore_ascii_case("NIL") { + return Ok(Value::Nil); + } + if let Ok(n) = s.parse::() { + return Ok(Value::Number(n)); + } + Ok(Value::Atom(s)) + } + + pub(super) fn read_atom_string(&mut self) -> Result { + let start = self.pos; + while let Some(b) = self.peek() { + if AtomByte::is(b) { + self.bump(); + } else { + break; + } + } + if start == self.pos { + return Err(ImapError::Parse("expected atom".into())); + } + str::from_utf8(self.span(start)) + .map(str::to_owned) + .map_err(|e| ImapError::Parse(format!("non-utf8 atom: {e}"))) + } + + pub(super) fn read_space_separated(&mut self) -> Result, ImapError> { + let end = self.buf.len().saturating_sub(self.tail_crlf_len()); + let rest = self.buf.get(self.pos..end).unwrap_or_default(); + self.pos = self.pos.max(end); + str::from_utf8(rest) + .map(|s| s.split_ascii_whitespace().map(str::to_owned).collect()) + .map_err(|e| ImapError::Parse(format!("non-utf8 token: {e}"))) + } + + pub(super) fn read_seq_set(&mut self) -> Result, ImapError> { + let start = self.pos; + while let Some(b) = self.peek() { + if b == b' ' || b == b'\r' || b == b'\n' { + break; + } + self.bump(); + } + SeqSet::expand(&String::from_utf8_lossy(self.span(start))) + } + + pub(super) fn span(&self, start: usize) -> &[u8] { + self.buf.get(start..self.pos).unwrap_or_default() + } + + pub(super) fn read_until_crlf(&mut self) -> String { + let start = self.pos; + let end = self.buf.len().saturating_sub(2).max(start); + self.pos = self.buf.len(); + String::from_utf8_lossy(self.buf.get(start..end).unwrap_or_default()) + .trim_end() + .to_owned() + } + + pub(super) fn read_line(&mut self) -> Result<(), ImapError> { + self.buf.clear(); + self.reader.read_until(b'\n', &mut self.buf)?; + self.pos = 0; + Ok(()) + } + + pub(super) fn peek(&self) -> Option { + let end = self.buf.len().saturating_sub(self.tail_crlf_len()); + self.buf.get(self.pos..end)?.first().copied() + } + + pub(super) fn bump(&mut self) { + self.pos += 1; + } + + pub(super) fn skip_ws(&mut self) { + while self.peek() == Some(b' ') { + self.bump(); + } + } + + pub(super) fn at_end(&self) -> bool { + self.peek().is_none() + } + + pub(super) fn at_end_of_line(&self) -> bool { + let end = self.buf.len().saturating_sub(self.tail_crlf_len()); + self.pos >= end + } + + pub(super) fn tail_crlf_len(&self) -> usize { + if self.buf.ends_with(b"\r\n") { + 2 + } else if self.buf.ends_with(b"\n") { + 1 + } else { + 0 + } + } + + pub(super) fn expect(&mut self, b: u8) -> Result<(), ImapError> { + if self.peek() == Some(b) { + self.bump(); + Ok(()) + } else { + Err(ImapError::Parse(format!( + "expected {:?}, got {:?}", + b as char, + self.peek().map(|x| x as char) + ))) + } + } +} + +pub(super) struct SeqSet; + +impl SeqSet { + pub(super) fn expand(s: &str) -> Result, ImapError> { + let mut out = Vec::new(); + for part in s.split(',').map(str::trim).filter(|p| !p.is_empty()) { + if let Some((lo, hi)) = part.split_once(':') { + let (Ok(lo), Ok(hi)) = (lo.parse::(), hi.parse::()) else { + continue; + }; + let (a, b) = if lo <= hi { (lo, hi) } else { (hi, lo) }; + if out.len() + (b - a) as usize >= MAX_SEQ_SET { + return Err(ImapError::Parse(format!( + "sequence set expands beyond {MAX_SEQ_SET} ids" + ))); + } + out.extend(a..=b); + } else if let Ok(n) = part.parse::() { + out.push(n); + } + } + Ok(out) + } +} + +pub(super) struct AtomByte; + +impl AtomByte { + pub(super) fn is(b: u8) -> bool { + b.is_ascii_graphic() && !matches!(b, b'(' | b')' | b'{' | b'%' | b'"' | b']') + } +} diff --git a/src/imap/response/mod.rs b/src/imap/response/mod.rs new file mode 100644 index 0000000..e8d0f19 --- /dev/null +++ b/src/imap/response/mod.rs @@ -0,0 +1,642 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::BTreeMap; +use std::fmt; +use std::io::BufRead; + +use super::error::{ImapError, NoError}; + +mod lexer; +mod parser; + +use parser::Parser; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Status { + Ok, + No, + Bad, + Bye, + PreAuth, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StatusLine { + pub status: Status, + pub code: Option, + pub code_args: Option, + pub text: String, +} + +impl StatusLine { + pub fn into_no_error(self) -> NoError { + NoError::new(self.text, self.code) + } + + pub fn into_result(self) -> Result { + match self.status { + Status::Ok => Ok(self), + Status::No => Err(ImapError::No(self.into_no_error())), + Status::Bad => Err(ImapError::Bad(self.text)), + Status::Bye => Err(ImapError::Bye(self.text)), + Status::PreAuth => Err(ImapError::Protocol( + "unexpected PREAUTH on tagged response".into(), + )), + } + } +} + +impl fmt::Display for Status { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Status::Ok => "OK", + Status::No => "NO", + Status::Bad => "BAD", + Status::Bye => "BYE", + Status::PreAuth => "PREAUTH", + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Value { + Atom(String), + Str(String), + Bytes(Vec), + Number(u64), + Nil, + List(Vec), +} + +impl Value { + pub fn as_str(&self) -> Option<&str> { + match self { + Value::Atom(s) | Value::Str(s) => Some(s), + _ => None, + } + } + + pub fn as_number(&self) -> Option { + match self { + Value::Number(n) => Some(*n), + _ => None, + } + } + + pub fn as_list(&self) -> Option<&[Value]> { + match self { + Value::List(items) => Some(items.as_slice()), + _ => None, + } + } + + fn into_name(self) -> Option { + match self { + Value::Str(s) | Value::Atom(s) => Some(s), + Value::Number(n) => Some(n.to_string()), + Value::Bytes(b) => Some(String::from_utf8_lossy(&b).into_owned()), + _ => None, + } + } + + pub fn into_bytes(self) -> Option> { + match self { + Value::Bytes(b) => Some(b), + Value::Str(s) => Some(s.into_bytes()), + _ => None, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Untagged { + Capability(Vec), + List { + attributes: Vec, + delimiter: Option, + name: String, + }, + Lsub { + attributes: Vec, + delimiter: Option, + name: String, + }, + Status { + mailbox: String, + items: BTreeMap, + }, + Search(Vec), + Esearch { + tag: Option, + all: Vec, + count: Option, + }, + Exists(u32), + Recent(u32), + Expunge(u32), + Fetch { + seq: u32, + items: Vec<(String, Value)>, + }, + Flags(Vec), + Namespace { + personal: Vec, + others: Vec, + shared: Vec, + }, + Enabled(Vec), + StatusLine(StatusLine), + Bye(String), + Raw(String), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NamespaceEntry { + pub prefix: String, + pub delimiter: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Response { + Tagged { tag: String, line: StatusLine }, + Untagged(Untagged), + Continuation(String), +} + +impl Response { + pub fn read(reader: &mut R) -> Result { + Parser::new(reader).parse_response() + } +} + +#[cfg(test)] +mod tests { + use super::lexer::{AtomByte, SeqSet}; + use super::*; + use std::io::Cursor; + + fn parse(input: &[u8]) -> Response { + Response::read(&mut Cursor::new(input)).unwrap() + } + + #[test] + fn tagged_ok() { + let r = parse(b"A001 OK LOGIN completed\r\n"); + match r { + Response::Tagged { tag, line } => { + assert_eq!(tag, "A001"); + assert_eq!(line.status, Status::Ok); + assert_eq!(line.text, "LOGIN completed"); + } + _ => panic!("expected Tagged"), + } + } + + #[test] + fn tagged_no_with_code() { + let r = parse(b"A002 NO [AUTHENTICATIONFAILED] bad credentials\r\n"); + match r { + Response::Tagged { tag, line } => { + assert_eq!(tag, "A002"); + assert_eq!(line.status, Status::No); + assert_eq!(line.code.as_deref(), Some("AUTHENTICATIONFAILED")); + assert_eq!(line.text, "bad credentials"); + } + _ => panic!("expected Tagged"), + } + } + + #[test] + fn untagged_capability() { + let r = parse(b"* CAPABILITY IMAP4rev1 STARTTLS AUTH=PLAIN AUTH=LOGIN\r\n"); + match r { + Response::Untagged(Untagged::Capability(caps)) => { + assert_eq!( + caps, + vec!["IMAP4rev1", "STARTTLS", "AUTH=PLAIN", "AUTH=LOGIN"] + ); + } + _ => panic!("expected Capability"), + } + } + + #[test] + fn untagged_capability_with_literal_plus() { + let r = parse( + b"* CAPABILITY IMAP4rev2 IMAP4rev1 ENABLE SASL-IR LITERAL+ ID UTF8=ACCEPT JMAPACCESS AUTH=PLAIN AUTH=OAUTHBEARER AUTH=XOAUTH2\r\n", + ); + match r { + Response::Untagged(Untagged::Capability(caps)) => { + assert!(caps.iter().any(|c| c == "LITERAL+")); + assert!(caps.iter().any(|c| c == "AUTH=OAUTHBEARER")); + assert_eq!(caps.len(), 11); + } + _ => panic!("expected Capability"), + } + } + + #[test] + fn untagged_capability_accepts_uri_valued_tokens() { + let r = parse( + b"* CAPABILITY IMAP4rev1 SASL-IR SPECIAL-USE IMAPSIEVE=sieve://127.0.0.1:4190 QUOTA ACL RIGHTS=texk\r\n", + ); + match r { + Response::Untagged(Untagged::Capability(caps)) => assert_eq!( + caps, + vec![ + "IMAP4rev1", + "SASL-IR", + "SPECIAL-USE", + "IMAPSIEVE=sieve://127.0.0.1:4190", + "QUOTA", + "ACL", + "RIGHTS=texk", + ] + ), + other => panic!("expected Capability, got {other:?}"), + } + } + + #[test] + fn untagged_enabled_accepts_non_atom_tokens() { + let r = parse(b"* ENABLED UTF8=ACCEPT X-VENDOR=a:b,c\r\n"); + match r { + Response::Untagged(Untagged::Enabled(exts)) => { + assert_eq!(exts, vec!["UTF8=ACCEPT", "X-VENDOR=a:b,c"]); + } + other => panic!("expected Enabled, got {other:?}"), + } + } + + #[test] + fn atom_accepts_colon_comma_and_brackets() { + let r = parse(b"* LIST (\\HasNoChildren) \"/\" Work:2026,Q1[a}\r\n"); + match r { + Response::Untagged(Untagged::List { name, .. }) => { + assert_eq!(name, "Work:2026,Q1[a}"); + } + other => panic!("expected List, got {other:?}"), + } + } + + #[test] + fn atom_stops_at_atom_specials() { + for b in b"(){%\"] \x7f\x00" { + assert!(!AtomByte::is(*b), "{b:#04x} must not be an atom byte"); + } + for b in b":,[}=/.-_+" { + assert!(AtomByte::is(*b), "{:?} must be an atom byte", *b as char); + } + } + + #[test] + fn quoted_mailbox_name_with_utf8_bytes_is_decoded_as_utf8() { + let r = parse("* LIST (\\HasNoChildren) \"/\" \"Envoy\u{e9}s\"\r\n".as_bytes()); + match r { + Response::Untagged(Untagged::List { name, .. }) => assert_eq!(name, "Envoyés"), + other => panic!("expected List, got {other:?}"), + } + } + + #[test] + fn quoted_mailbox_name_with_modified_utf7_is_left_untouched() { + let r = + parse(b"* LIST (\\HasNoChildren) \"/\" \"[Gmail]/G&APY-nderilmi&AV8- Postalar\"\r\n"); + match r { + Response::Untagged(Untagged::List { name, .. }) => { + assert_eq!(name, "[Gmail]/G&APY-nderilmi&AV8- Postalar"); + } + other => panic!("expected List, got {other:?}"), + } + } + + #[test] + fn quoted_mailbox_name_with_invalid_utf8_falls_back_to_lossy() { + let mut line: Vec = b"* LIST (\\HasNoChildren) \"/\" \"bad".to_vec(); + line.push(0xff); + line.extend_from_slice(b"name\"\r\n"); + let r = parse(&line); + match r { + Response::Untagged(Untagged::List { name, .. }) => { + assert!(name.starts_with("bad") && name.ends_with("name")); + } + other => panic!("expected List, got {other:?}"), + } + } + + #[test] + fn untagged_list_with_special_use() { + let r = parse(b"* LIST (\\HasNoChildren \\Sent) \"/\" \"Sent Mail\"\r\n"); + match r { + Response::Untagged(Untagged::List { + attributes, + delimiter, + name, + }) => { + assert_eq!(attributes, vec!["\\HasNoChildren", "\\Sent"]); + assert_eq!(delimiter, Some('/')); + assert_eq!(name, "Sent Mail"); + } + _ => panic!("expected List"), + } + } + + #[test] + fn untagged_list_with_nil_delimiter() { + let r = parse(b"* LIST () NIL INBOX\r\n"); + match r { + Response::Untagged(Untagged::List { + attributes, + delimiter, + name, + }) => { + assert!(attributes.is_empty()); + assert_eq!(delimiter, None); + assert_eq!(name, "INBOX"); + } + _ => panic!("expected List"), + } + } + + #[test] + fn untagged_list_with_numeric_atom_name() { + let r = parse(b"* LIST (\\Subscribed \\HasNoChildren \\UnMarked) \"/\" 200\r\n"); + match r { + Response::Untagged(Untagged::List { + attributes, + delimiter, + name, + }) => { + assert_eq!( + attributes, + vec!["\\Subscribed", "\\HasNoChildren", "\\UnMarked"] + ); + assert_eq!(delimiter, Some('/')); + assert_eq!(name, "200"); + } + _ => panic!("expected List"), + } + } + + #[test] + fn untagged_status() { + let r = parse(b"* STATUS INBOX (UIDVALIDITY 12345 UIDNEXT 42 MESSAGES 7)\r\n"); + match r { + Response::Untagged(Untagged::Status { mailbox, items }) => { + assert_eq!(mailbox, "INBOX"); + assert_eq!(items.get("UIDVALIDITY"), Some(&12345)); + assert_eq!(items.get("UIDNEXT"), Some(&42)); + assert_eq!(items.get("MESSAGES"), Some(&7)); + } + _ => panic!("expected Status"), + } + } + + #[test] + fn untagged_status_with_numeric_atom_mailbox() { + let r = parse(b"* STATUS 200 (MESSAGES 5 UIDNEXT 6 UIDVALIDITY 1778253936)\r\n"); + match r { + Response::Untagged(Untagged::Status { mailbox, items }) => { + assert_eq!(mailbox, "200"); + assert_eq!(items.get("MESSAGES"), Some(&5)); + assert_eq!(items.get("UIDNEXT"), Some(&6)); + assert_eq!(items.get("UIDVALIDITY"), Some(&1778253936)); + } + _ => panic!("expected Status"), + } + } + + #[test] + fn untagged_search_with_uids() { + let r = parse(b"* SEARCH 1 3 5 7\r\n"); + match r { + Response::Untagged(Untagged::Search(ids)) => { + assert_eq!(ids, vec![1, 3, 5, 7]); + } + _ => panic!("expected Search"), + } + } + + #[test] + fn untagged_search_empty() { + let r = parse(b"* SEARCH\r\n"); + match r { + Response::Untagged(Untagged::Search(ids)) => { + assert!(ids.is_empty()); + } + _ => panic!("expected Search"), + } + } + + #[test] + fn untagged_esearch_all_set() { + let r = parse(b"* ESEARCH (TAG \"q1\") UID ALL 1:3,7\r\n"); + match r { + Response::Untagged(Untagged::Esearch { tag, all, .. }) => { + assert_eq!(tag.as_deref(), Some("q1")); + assert_eq!(all, vec![1, 2, 3, 7]); + } + _ => panic!("expected Esearch"), + } + } + + #[test] + fn untagged_exists_recent() { + let r = parse(b"* 42 EXISTS\r\n"); + assert!(matches!(r, Response::Untagged(Untagged::Exists(42)))); + let r = parse(b"* 3 RECENT\r\n"); + assert!(matches!(r, Response::Untagged(Untagged::Recent(3)))); + } + + #[test] + fn untagged_fetch_flags_and_uid() { + let r = parse( + b"* 17 FETCH (UID 100 FLAGS (\\Seen \\Flagged) RFC822.SIZE 4242 INTERNALDATE \"17-Jul-1996 02:44:25 -0700\")\r\n", + ); + match r { + Response::Untagged(Untagged::Fetch { seq, items }) => { + assert_eq!(seq, 17); + let uid = items.iter().find(|(n, _)| n == "UID").unwrap(); + assert_eq!(uid.1.as_number(), Some(100)); + let flags = items.iter().find(|(n, _)| n == "FLAGS").unwrap(); + let list = flags.1.as_list().unwrap(); + assert_eq!(list.len(), 2); + let size = items.iter().find(|(n, _)| n == "RFC822.SIZE").unwrap(); + assert_eq!(size.1.as_number(), Some(4242)); + let internaldate = items.iter().find(|(n, _)| n == "INTERNALDATE").unwrap(); + assert_eq!(internaldate.1.as_str(), Some("17-Jul-1996 02:44:25 -0700")); + } + _ => panic!("expected Fetch"), + } + } + + #[test] + fn untagged_fetch_with_body_literal() { + let input = b"* 1 FETCH (UID 5 BODY[] {11}\r\nHello world)\r\n"; + let r = parse(input); + match r { + Response::Untagged(Untagged::Fetch { seq, items }) => { + assert_eq!(seq, 1); + let body = items.iter().find(|(n, _)| n == "BODY[]").unwrap(); + match &body.1 { + Value::Str(s) => assert_eq!(s, "Hello world"), + Value::Bytes(b) => assert_eq!(b, b"Hello world"), + other => panic!("expected literal, got {other:?}"), + } + } + _ => panic!("expected Fetch"), + } + } + + #[test] + fn untagged_fetch_with_binary_literal() { + let mut input = b"* 1 FETCH (UID 5 BODY[] {4}\r\n".to_vec(); + input.extend_from_slice(&[0xFF, 0xFE, 0x00, 0x01]); + input.extend_from_slice(b")\r\n"); + let r = Response::read(&mut Cursor::new(input)).unwrap(); + match r { + Response::Untagged(Untagged::Fetch { items, .. }) => { + let body = items.iter().find(|(n, _)| n == "BODY[]").unwrap(); + match &body.1 { + Value::Bytes(b) => assert_eq!(b, &[0xFF, 0xFE, 0x00, 0x01]), + other => panic!("expected Bytes, got {other:?}"), + } + } + _ => panic!("expected Fetch"), + } + } + + #[test] + fn untagged_flags() { + let r = parse(b"* FLAGS (\\Answered \\Flagged \\Draft \\Deleted \\Seen)\r\n"); + match r { + Response::Untagged(Untagged::Flags(flags)) => { + assert_eq!(flags.len(), 5); + assert!(flags.contains(&"\\Seen".to_owned())); + } + _ => panic!("expected Flags"), + } + } + + #[test] + fn untagged_bye() { + let r = parse(b"* BYE server logging out\r\n"); + match r { + Response::Untagged(Untagged::Bye(text)) => { + assert_eq!(text, "server logging out"); + } + _ => panic!("expected Bye"), + } + } + + #[test] + fn untagged_namespace() { + let r = parse(b"* NAMESPACE ((\"INBOX.\" \".\")) NIL NIL\r\n"); + match r { + Response::Untagged(Untagged::Namespace { + personal, + others, + shared, + }) => { + assert_eq!(personal.len(), 1); + assert_eq!(personal[0].prefix, "INBOX."); + assert_eq!(personal[0].delimiter, Some('.')); + assert!(others.is_empty()); + assert!(shared.is_empty()); + } + _ => panic!("expected Namespace"), + } + } + + #[test] + fn continuation_request() { + let r = parse(b"+ Ready for additional command text\r\n"); + match r { + Response::Continuation(text) => { + assert_eq!(text, "Ready for additional command text"); + } + _ => panic!("expected Continuation"), + } + } + + #[test] + fn untagged_status_with_response_code() { + let r = parse(b"* OK [PERMANENTFLAGS (\\Seen \\Flagged \\*)] Limited\r\n"); + match r { + Response::Untagged(Untagged::StatusLine(line)) => { + assert_eq!(line.status, Status::Ok); + assert_eq!(line.code.as_deref(), Some("PERMANENTFLAGS")); + assert!(line.code_args.as_deref().is_some()); + assert_eq!(line.text, "Limited"); + } + _ => panic!("expected StatusLine"), + } + } + + #[test] + fn untagged_ok_with_uidvalidity() { + let r = parse(b"* OK [UIDVALIDITY 3857529045] UIDs valid\r\n"); + match r { + Response::Untagged(Untagged::StatusLine(line)) => { + assert_eq!(line.code.as_deref(), Some("UIDVALIDITY")); + assert_eq!(line.code_args.as_deref(), Some("3857529045")); + } + _ => panic!("expected StatusLine"), + } + } + + #[test] + fn quoted_string_with_escape() { + let r = parse(b"* LIST () \"/\" \"He said \\\"hi\\\"\"\r\n"); + match r { + Response::Untagged(Untagged::List { name, .. }) => { + assert_eq!(name, "He said \"hi\""); + } + _ => panic!("expected List"), + } + } + + #[test] + fn seq_set_parses_ranges_and_singletons() { + assert_eq!(SeqSet::expand("1:3,5,7:8").unwrap(), vec![1, 2, 3, 5, 7, 8]); + assert_eq!(SeqSet::expand("42").unwrap(), vec![42]); + assert!(SeqSet::expand("").unwrap().is_empty()); + } + + #[test] + fn empty_read_is_disconnected() { + let mut empty: &[u8] = b""; + let err = Response::read(&mut empty).unwrap_err(); + assert!(matches!(err, ImapError::Disconnected)); + } + + #[test] + fn untagged_enabled_parses() { + let r = parse(b"* ENABLED UTF8=ACCEPT CONDSTORE\r\n"); + match r { + Response::Untagged(Untagged::Enabled(exts)) => { + assert_eq!(exts, vec!["UTF8=ACCEPT", "CONDSTORE"]); + } + other => panic!("expected Enabled, got {other:?}"), + } + } + + #[test] + fn oversized_literal_header_fails_without_preallocating() { + let err = Response::read(&mut Cursor::new( + &b"* 1 FETCH (BODY[] {4294967295}\r\nshort"[..], + )) + .unwrap_err(); + assert!(matches!(err, ImapError::Io(_)), "{err:?}"); + } + + #[test] + fn seq_set_refuses_to_expand_past_the_cap() { + assert!(SeqSet::expand("1:4294967295").is_err()); + assert_eq!(SeqSet::expand("4294967290:4294967295").unwrap().len(), 6); + } +} diff --git a/src/imap/response/parser.rs b/src/imap/response/parser.rs new file mode 100644 index 0000000..71e2dad --- /dev/null +++ b/src/imap/response/parser.rs @@ -0,0 +1,435 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::BTreeMap; +use std::io::BufRead; + +use super::{NamespaceEntry, Response, Status, StatusLine, Untagged, Value}; +use crate::imap::error::ImapError; + +pub(super) const LITERAL_RESERVE: usize = 1 << 20; +pub(super) const MAX_SEQ_SET: usize = 1 << 24; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Keyword { + Status(Status), + Capability, + List, + Lsub, + StatusItems, + Search, + Esearch, + Flags, + Namespace, + Enabled, +} + +impl Keyword { + fn parse(word: &str) -> Option { + hashify::map_ignore_case!(word.as_bytes(), Keyword, + "OK" => Keyword::Status(Status::Ok), + "NO" => Keyword::Status(Status::No), + "BAD" => Keyword::Status(Status::Bad), + "BYE" => Keyword::Status(Status::Bye), + "PREAUTH" => Keyword::Status(Status::PreAuth), + "CAPABILITY" => Keyword::Capability, + "LIST" => Keyword::List, + "LSUB" => Keyword::Lsub, + "STATUS" => Keyword::StatusItems, + "SEARCH" => Keyword::Search, + "ESEARCH" => Keyword::Esearch, + "FLAGS" => Keyword::Flags, + "NAMESPACE" => Keyword::Namespace, + "ENABLED" => Keyword::Enabled, + ) + .copied() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum MessageKeyword { + Exists, + Recent, + Expunge, + Fetch, +} + +impl MessageKeyword { + fn parse(word: &str) -> Option { + hashify::map_ignore_case!(word.as_bytes(), MessageKeyword, + "EXISTS" => MessageKeyword::Exists, + "RECENT" => MessageKeyword::Recent, + "EXPUNGE" => MessageKeyword::Expunge, + "FETCH" => MessageKeyword::Fetch, + ) + .copied() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum EsearchItem { + Uid, + All, + Count, +} + +impl EsearchItem { + fn parse(word: &str) -> Option { + hashify::map_ignore_case!(word.as_bytes(), EsearchItem, + "UID" => EsearchItem::Uid, + "ALL" => EsearchItem::All, + "COUNT" => EsearchItem::Count, + ) + .copied() + } +} + +pub(super) struct Parser<'r, R: BufRead> { + pub(super) reader: &'r mut R, + pub(super) buf: Vec, + pub(super) pos: usize, +} + +impl<'r, R: BufRead> Parser<'r, R> { + pub(super) fn new(reader: &'r mut R) -> Self { + Self { + reader, + buf: Vec::new(), + pos: 0, + } + } + + pub(super) fn parse_response(&mut self) -> Result { + self.read_line()?; + if self.buf.is_empty() { + return Err(ImapError::Disconnected); + } + if self.peek() == Some(b'+') { + self.bump(); + self.skip_ws(); + let text = self.read_until_crlf(); + return Ok(Response::Continuation(text)); + } + if self.peek() == Some(b'*') { + self.bump(); + self.skip_ws(); + return self.parse_untagged().map(Response::Untagged); + } + let tag = self.read_atom_string()?; + self.skip_ws(); + let line = self.parse_status_line()?; + Ok(Response::Tagged { tag, line }) + } + + fn parse_untagged(&mut self) -> Result { + let first = self.read_atom_string()?; + let Some(keyword) = Keyword::parse(&first) else { + return self.parse_numbered(first); + }; + match keyword { + Keyword::Status(status) => { + self.skip_ws(); + let mut line = self.parse_status_line_body()?; + line.status = status; + if status == Status::Bye { + return Ok(Untagged::Bye(line.text)); + } + Ok(Untagged::StatusLine(line)) + } + Keyword::Capability => Ok(Untagged::Capability(self.read_space_separated()?)), + Keyword::List | Keyword::Lsub => { + self.skip_ws(); + self.expect(b'(')?; + let mut attributes = Vec::new(); + while self.peek() != Some(b')') { + self.skip_ws(); + if self.peek() == Some(b')') { + break; + } + attributes.push(self.read_atom_string()?); + } + self.expect(b')')?; + self.skip_ws(); + let delimiter = match self.parse_value()? { + Value::Str(s) | Value::Atom(s) => s.chars().next(), + Value::Bytes(b) => b.first().map(|&b| char::from(b)), + _ => None, + }; + self.skip_ws(); + let name = self + .parse_value()? + .into_name() + .ok_or_else(|| ImapError::Parse("LIST mailbox name missing".into()))?; + Ok(if keyword == Keyword::List { + Untagged::List { + attributes, + delimiter, + name, + } + } else { + Untagged::Lsub { + attributes, + delimiter, + name, + } + }) + } + Keyword::StatusItems => { + self.skip_ws(); + let mailbox = self + .parse_value()? + .into_name() + .ok_or_else(|| ImapError::Parse("STATUS mailbox missing".into()))?; + self.skip_ws(); + self.expect(b'(')?; + let mut items = BTreeMap::new(); + while self.peek() != Some(b')') { + self.skip_ws(); + if self.peek() == Some(b')') { + break; + } + let mut key = self.read_atom_string()?; + key.make_ascii_uppercase(); + self.skip_ws(); + if let Some(n) = self.parse_value()?.as_number() { + items.insert(key, n); + } + } + self.expect(b')')?; + Ok(Untagged::Status { mailbox, items }) + } + Keyword::Search => { + let mut ids = Vec::new(); + while !self.at_end() { + self.skip_ws(); + if self.at_end() { + break; + } + if let Some(n) = self + .parse_value()? + .as_number() + .and_then(|n| u32::try_from(n).ok()) + { + ids.push(n); + } + } + Ok(Untagged::Search(ids)) + } + Keyword::Esearch => self.parse_esearch(), + Keyword::Flags => { + self.skip_ws(); + let flags = match self.parse_value()? { + Value::List(items) => items + .into_iter() + .filter_map(|x| match x { + Value::Atom(s) | Value::Str(s) => Some(s), + _ => None, + }) + .collect(), + _ => Vec::new(), + }; + Ok(Untagged::Flags(flags)) + } + Keyword::Namespace => self.parse_namespace(), + Keyword::Enabled => Ok(Untagged::Enabled(self.read_space_separated()?)), + } + } + + fn parse_numbered(&mut self, first: String) -> Result { + let Ok(num) = first.parse::() else { + let rest = self.read_until_crlf(); + return Ok(Untagged::Raw(format!("{first} {rest}"))); + }; + self.skip_ws(); + let mut kw = self.read_atom_string()?; + match MessageKeyword::parse(&kw) { + Some(MessageKeyword::Exists) => Ok(Untagged::Exists(num)), + Some(MessageKeyword::Recent) => Ok(Untagged::Recent(num)), + Some(MessageKeyword::Expunge) => Ok(Untagged::Expunge(num)), + Some(MessageKeyword::Fetch) => self.parse_fetch_body(num), + None => { + kw.make_ascii_uppercase(); + let rest = self.read_until_crlf(); + Ok(Untagged::Raw(format!("{num} {kw} {rest}"))) + } + } + } + + fn parse_esearch(&mut self) -> Result { + let mut tag = None; + let mut all = Vec::new(); + let mut count = None; + if self.peek() == Some(b' ') { + self.skip_ws(); + } + if self.peek() == Some(b'(') { + self.bump(); + self.skip_ws(); + let key = self.read_atom_string()?; + if key.eq_ignore_ascii_case("TAG") { + self.skip_ws(); + if let Value::Str(s) = self.parse_value()? { + tag = Some(s); + } + } + self.skip_ws(); + self.expect(b')')?; + } + while !self.at_end() { + self.skip_ws(); + if self.at_end() { + break; + } + let key = self.read_atom_string()?; + match EsearchItem::parse(&key) { + Some(EsearchItem::Uid) => {} + Some(EsearchItem::All) => { + self.skip_ws(); + all = self.read_seq_set()?; + } + Some(EsearchItem::Count) => { + self.skip_ws(); + if let Value::Number(n) = self.parse_value()? { + count = u32::try_from(n).ok(); + } + } + None => { + self.skip_ws(); + let _ = self.parse_value(); + } + } + } + Ok(Untagged::Esearch { tag, all, count }) + } + + fn parse_namespace(&mut self) -> Result { + self.skip_ws(); + let personal = self.parse_namespace_list()?; + self.skip_ws(); + let others = self.parse_namespace_list()?; + self.skip_ws(); + let shared = self.parse_namespace_list()?; + Ok(Untagged::Namespace { + personal, + others, + shared, + }) + } + + fn parse_namespace_list(&mut self) -> Result, ImapError> { + match self.parse_value()? { + Value::Nil => Ok(Vec::new()), + Value::List(entries) => { + let mut out = Vec::new(); + for e in entries { + if let Value::List(inner) = e + && let Some(prefix) = inner.first().and_then(|v| v.as_str()) + { + let delim = inner.get(1).and_then(|v| match v { + Value::Nil => None, + Value::Str(s) | Value::Atom(s) => s.chars().next(), + _ => None, + }); + out.push(NamespaceEntry { + prefix: prefix.to_owned(), + delimiter: delim, + }); + } + } + Ok(out) + } + _ => Ok(Vec::new()), + } + } + + fn parse_fetch_body(&mut self, seq: u32) -> Result { + self.skip_ws(); + self.expect(b'(')?; + let mut items = Vec::new(); + while self.peek() != Some(b')') { + self.skip_ws(); + if self.peek() == Some(b')') { + break; + } + let name = self.read_fetch_item_name()?; + self.skip_ws(); + let val = self.parse_value()?; + items.push((name, val)); + } + self.expect(b')')?; + Ok(Untagged::Fetch { seq, items }) + } + + fn read_fetch_item_name(&mut self) -> Result { + let start = self.pos; + while let Some(b) = self.peek() { + match b { + b' ' | b'\r' | b'\n' | b')' | b'(' => break, + _ => self.bump(), + } + } + str::from_utf8(self.span(start)) + .map(|s| s.to_ascii_uppercase()) + .map_err(|e| ImapError::Parse(format!("non-utf8 fetch item name: {e}"))) + } + + fn parse_status_line(&mut self) -> Result { + let kw = self.read_atom_string()?; + self.skip_ws(); + let mut line = self.parse_status_line_body()?; + line.status = match Keyword::parse(&kw) { + Some(Keyword::Status(status)) => status, + _ => { + return Err(ImapError::Parse(format!( + "unknown status: {}", + kw.to_ascii_uppercase() + ))); + } + }; + Ok(line) + } + + fn parse_status_line_body(&mut self) -> Result { + let mut code = None; + let mut code_args = None; + if self.peek() == Some(b'[') { + self.bump(); + let start = self.pos; + let mut depth = 1usize; + while let Some(b) = self.peek() { + if b == b'[' { + depth += 1; + self.bump(); + } else if b == b']' { + depth -= 1; + if depth == 0 { + break; + } + self.bump(); + } else { + self.bump(); + } + } + let inside_str = str::from_utf8(self.span(start)) + .map_err(|e| ImapError::Parse(format!("non-utf8 status code: {e}")))?; + let (head, tail) = match inside_str.split_once(' ') { + Some((h, t)) => (h.to_owned(), Some(t.to_owned())), + None => (inside_str.to_owned(), None), + }; + code = Some(head.to_ascii_uppercase()); + code_args = tail; + self.expect(b']')?; + self.skip_ws(); + } + let text = self.read_until_crlf(); + Ok(StatusLine { + status: Status::Ok, + code, + code_args, + text, + }) + } +} diff --git a/src/imap/retry.rs b/src/imap/retry.rs index eb2e615..def9154 100644 --- a/src/imap/retry.rs +++ b/src/imap/retry.rs @@ -4,163 +4,54 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; -use std::time::Duration; - use super::error::ImapError; +use crate::net::backoff::{Fault, Retryable}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Disposition { TransportDrop, Transient, Permanent, - PerMessageRecoverable, - PerFolderPermanent, FeatureNegotiation, AuthFailed, } -#[derive(Debug, Clone, Copy)] -pub struct RetryPolicy { - pub max_retries: u32, -} +impl ImapError { + pub fn disposition(&self) -> Disposition { + match self { + ImapError::Io(_) | ImapError::Disconnected | ImapError::Bye(_) => { + Disposition::TransportDrop + } + ImapError::AuthFailed(_) => Disposition::AuthFailed, + ImapError::Bad(_) => Disposition::FeatureNegotiation, + ImapError::No(no) if no.is_auth_failed() => Disposition::AuthFailed, + ImapError::No(no) if no.is_transient() => Disposition::Transient, + ImapError::No(_) + | ImapError::Tls(_) + | ImapError::Protocol(_) + | ImapError::Parse(_) + | ImapError::Unsupported(_) => Disposition::Permanent, + } + } -impl RetryPolicy { - pub fn new(max_retries: u32) -> Self { - Self { max_retries } + pub fn is_negotiation_failure(&self) -> bool { + match self { + ImapError::Bad(_) | ImapError::Unsupported(_) => true, + ImapError::No(no) => !no.is_transient(), + _ => false, + } } } -const SHARED_BASE_MS: u64 = 1_000; -const SHARED_CAP_MS: u64 = 30_000; -const TRANSPORT_BASE_MS: u64 = 500; -const TRANSPORT_CAP_MS: u64 = 30_000; -const LEVEL_CAP: u32 = 16; - -#[derive(Default, Clone)] -pub struct BackoffState { - inner: Arc, -} - -#[derive(Default)] -struct BackoffInner { - level: AtomicU32, - transient_retries: AtomicU32, - transport_retries: AtomicU32, -} - -impl BackoffState { - pub fn new() -> Self { - Self::default() - } - - pub fn reset(&self) { - self.inner.level.store(0, Ordering::Relaxed); - } - - pub fn current_level(&self) -> u32 { - self.inner.level.load(Ordering::Relaxed) - } - - pub fn transient_retries(&self) -> u32 { - self.inner.transient_retries.load(Ordering::Relaxed) - } - - pub fn transport_retries(&self) -> u32 { - self.inner.transport_retries.load(Ordering::Relaxed) - } - - pub fn total_retries(&self) -> u64 { - self.transient_retries() as u64 + self.transport_retries() as u64 - } - - pub fn next_shared_delay(&self) -> Duration { - let prev = self.inner.level.fetch_add(1, Ordering::Relaxed); - self.inner.transient_retries.fetch_add(1, Ordering::Relaxed); - let n = prev.saturating_add(1).min(LEVEL_CAP); - let shift = n.saturating_sub(1).min(15); - let scaled = SHARED_BASE_MS.saturating_mul(1u64 << shift); - let scaled = scaled.min(SHARED_CAP_MS); - let half = scaled / 2; - Duration::from_millis(jitter_in_range(half, scaled)) - } - - pub fn transport_delay(&self, attempt: u32) -> Duration { - self.inner.transport_retries.fetch_add(1, Ordering::Relaxed); - let n = attempt.min(LEVEL_CAP); - let shift = n.saturating_sub(1).min(15); - let scaled = TRANSPORT_BASE_MS.saturating_mul(1u64 << shift); - let scaled = scaled.min(TRANSPORT_CAP_MS); - Duration::from_millis(jitter_in_range(0, scaled)) - } -} - -fn jitter_in_range(lo: u64, hi: u64) -> u64 { - if hi <= lo { - return lo; - } - let span = hi - lo; - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.subsec_nanos() as u64) - .unwrap_or(0); - lo + (nanos % span) -} - -pub fn is_negotiation_failure(err: &ImapError) -> bool { - match err { - ImapError::Bad(_) => true, - ImapError::No(no) => !no.is_transient(), - ImapError::Unsupported(_) => true, - _ => false, - } -} - -pub fn classify(err: &ImapError) -> Disposition { - match err { - ImapError::Io(_) => Disposition::TransportDrop, - ImapError::Tls(_) => Disposition::Permanent, - ImapError::Disconnected => Disposition::TransportDrop, - ImapError::Bye(_) => Disposition::TransportDrop, - ImapError::AuthFailed(_) => Disposition::AuthFailed, - ImapError::Bad(_) => Disposition::FeatureNegotiation, - ImapError::No(no) => { - if no.is_auth_failed() { - Disposition::AuthFailed - } else if no.is_transient() { - Disposition::Transient - } else { - Disposition::Permanent +impl Retryable for ImapError { + fn fault(&self) -> Fault { + match self.disposition() { + Disposition::TransportDrop => Fault::TransportDrop, + Disposition::Transient => Fault::Transient, + Disposition::Permanent | Disposition::FeatureNegotiation | Disposition::AuthFailed => { + Fault::Fatal } } - ImapError::Protocol(_) => Disposition::Permanent, - ImapError::Parse(_) => Disposition::Permanent, - ImapError::Unsupported(_) => Disposition::Permanent, - } -} - -pub fn folder_disposition(err: &ImapError) -> Disposition { - let base = classify(err); - if matches!( - base, - Disposition::Permanent | Disposition::FeatureNegotiation - ) { - Disposition::PerFolderPermanent - } else { - base - } -} - -pub fn message_disposition(err: &ImapError) -> Disposition { - let base = classify(err); - if matches!( - base, - Disposition::Permanent | Disposition::FeatureNegotiation - ) { - Disposition::PerMessageRecoverable - } else { - base } } @@ -170,6 +61,10 @@ mod tests { use super::*; use std::io::{self, ErrorKind}; + fn no(text: &str, code: Option<&str>) -> ImapError { + ImapError::No(NoError::new(text, code.map(str::to_owned))) + } + #[test] fn transport_errors_are_transport_drop() { for k in [ @@ -180,173 +75,62 @@ mod tests { ErrorKind::TimedOut, ] { let e = ImapError::Io(io::Error::new(k, "x")); - assert_eq!(classify(&e), Disposition::TransportDrop, "{k:?}"); + assert_eq!(e.disposition(), Disposition::TransportDrop, "{k:?}"); } - } - - #[test] - fn bye_is_transport_drop() { assert_eq!( - classify(&ImapError::Bye("going away".into())), + ImapError::Bye("going away".into()).disposition(), Disposition::TransportDrop ); + assert_eq!(ImapError::Disconnected.fault(), Fault::TransportDrop); } #[test] fn auth_failed_classified_separately() { - let e = ImapError::AuthFailed("bad creds".into()); - assert_eq!(classify(&e), Disposition::AuthFailed); - let e = ImapError::No(NoError::new("x", Some("AUTHENTICATIONFAILED".into()))); - assert_eq!(classify(&e), Disposition::AuthFailed); - } - - #[test] - fn transient_no_text_marked_transient() { - let e = ImapError::No(NoError::new("Try again later", None)); - assert_eq!(classify(&e), Disposition::Transient); - let e = ImapError::No(NoError::new("user over quota", None)); - assert_eq!(classify(&e), Disposition::Transient); - } - - #[test] - fn non_transient_no_is_permanent() { - let e = ImapError::No(NoError::new("mailbox does not exist", None)); - assert_eq!(classify(&e), Disposition::Permanent); - } - - #[test] - fn bad_is_feature_negotiation() { - let e = ImapError::Bad("unrecognised command".into()); - assert_eq!(classify(&e), Disposition::FeatureNegotiation); - } - - #[test] - fn protocol_and_parse_are_permanent() { assert_eq!( - classify(&ImapError::Protocol("x".into())), - Disposition::Permanent + ImapError::AuthFailed("bad creds".into()).disposition(), + Disposition::AuthFailed ); assert_eq!( - classify(&ImapError::Parse("x".into())), + no("x", Some("AUTHENTICATIONFAILED")).disposition(), + Disposition::AuthFailed + ); + } + + #[test] + fn no_responses_split_by_text() { + assert_eq!( + no("Try again later", None).disposition(), + Disposition::Transient + ); + assert_eq!(no("user over quota", None).fault(), Fault::Transient); + assert_eq!( + no("mailbox does not exist", None).disposition(), Disposition::Permanent ); } #[test] - fn folder_disposition_lifts_permanent_to_per_folder() { - let e = ImapError::No(NoError::new("no such mailbox", None)); - assert_eq!(folder_disposition(&e), Disposition::PerFolderPermanent); - } - - #[test] - fn folder_disposition_keeps_transient_class() { - let e = ImapError::No(NoError::new("try again", None)); - assert_eq!(folder_disposition(&e), Disposition::Transient); - } - - #[test] - fn message_disposition_demotes_permanent_to_per_message() { - let e = ImapError::Parse("bad fetch".into()); - assert_eq!(message_disposition(&e), Disposition::PerMessageRecoverable); - } - - #[test] - fn shared_delay_grows_with_consecutive_throttles() { - let s = BackoffState::new(); - let d1 = s.next_shared_delay(); - let d3 = { - let _ = s.next_shared_delay(); - s.next_shared_delay() - }; - assert!(d1 <= Duration::from_millis(SHARED_BASE_MS)); - assert!( - d3 >= d1, - "expected non-decreasing delay, got {d1:?} -> {d3:?}" + fn bad_protocol_and_parse_are_fatal() { + assert_eq!( + ImapError::Bad("unrecognised command".into()).disposition(), + Disposition::FeatureNegotiation ); - assert!(d3 <= Duration::from_millis(SHARED_CAP_MS)); - } - - #[test] - fn shared_delay_caps_at_cap_ms() { - let s = BackoffState::new(); - for _ in 0..40 { - let _ = s.next_shared_delay(); + for e in [ + ImapError::Protocol("x".into()), + ImapError::Parse("x".into()), + ] { + assert_eq!(e.disposition(), Disposition::Permanent, "{e:?}"); + assert_eq!(e.fault(), Fault::Fatal, "{e:?}"); } - let d = s.next_shared_delay(); - assert!(d <= Duration::from_millis(SHARED_CAP_MS)); } #[test] - fn reset_clears_level() { - let s = BackoffState::new(); - let _ = s.next_shared_delay(); - let _ = s.next_shared_delay(); - assert!(s.current_level() > 0); - s.reset(); - assert_eq!(s.current_level(), 0); - } - - #[test] - fn transport_delay_does_not_touch_shared_level() { - let s = BackoffState::new(); - let _ = s.transport_delay(1); - let _ = s.transport_delay(2); - let _ = s.transport_delay(3); - assert_eq!(s.current_level(), 0, "transport delays are local"); - } - - #[test] - fn transport_delay_bounded() { - let s = BackoffState::new(); - let d1 = s.transport_delay(1); - let d6 = s.transport_delay(6); - assert!(d1 <= Duration::from_millis(TRANSPORT_BASE_MS)); - assert!(d6 <= Duration::from_millis(TRANSPORT_CAP_MS)); - } - - #[test] - fn shared_state_is_cloneable_and_shared() { - let a = BackoffState::new(); - let b = a.clone(); - let _ = a.next_shared_delay(); - assert_eq!(a.current_level(), b.current_level()); - } - - #[test] - fn is_negotiation_failure_matches_bad_unsupported_and_nontransient_no() { - assert!(is_negotiation_failure(&ImapError::Bad("x".into()))); - assert!(is_negotiation_failure(&ImapError::Unsupported("x".into()))); - assert!(is_negotiation_failure(&ImapError::No(NoError::new( - "no such mailbox", - None - )))); - } - - #[test] - fn is_negotiation_failure_excludes_transient_no_and_others() { - assert!(!is_negotiation_failure(&ImapError::No(NoError::new( - "try again later", - None - )))); - assert!(!is_negotiation_failure(&ImapError::Bye("x".into()))); - assert!(!is_negotiation_failure(&ImapError::Disconnected)); - } - - #[test] - fn transient_retries_counter_increments_on_shared_delay() { - let s = BackoffState::new(); - assert_eq!(s.transient_retries(), 0); - let _ = s.next_shared_delay(); - let _ = s.next_shared_delay(); - assert_eq!(s.transient_retries(), 2); - } - - #[test] - fn transport_retries_counter_increments_on_transport_delay() { - let s = BackoffState::new(); - assert_eq!(s.transport_retries(), 0); - let _ = s.transport_delay(1); - let _ = s.transport_delay(2); - assert_eq!(s.transport_retries(), 2); + fn negotiation_failure_matches_bad_unsupported_and_permanent_no() { + assert!(ImapError::Bad("x".into()).is_negotiation_failure()); + assert!(ImapError::Unsupported("x".into()).is_negotiation_failure()); + assert!(no("no such mailbox", None).is_negotiation_failure()); + assert!(!no("try again later", None).is_negotiation_failure()); + assert!(!ImapError::Bye("x".into()).is_negotiation_failure()); + assert!(!ImapError::Disconnected.is_negotiation_failure()); } } diff --git a/src/imap/session.rs b/src/imap/session.rs new file mode 100644 index 0000000..98defe3 --- /dev/null +++ b/src/imap/session.rs @@ -0,0 +1,188 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use super::client::{Capability, ImapClient}; +use super::error::ImapError; +use super::name::NameEncoding; +use crate::error::Error; +use crate::logging::Logger; +use crate::net::sasl::{SaslAuth, SaslPayload}; +use crate::net::stream::{Cleartext, ConnectMode, Connector, Endpoint, Scheme, UrlTls}; + +pub const IMAP: Scheme = Scheme { + plain: "imap", + tls: "imaps", + plain_port: 143, + tls_port: 993, +}; + +const UTF8_ACCEPT: &[&str] = &["UTF8=ACCEPT"]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Compress { + Requested, + Off, +} + +pub struct ImapSession { + connector: Connector, + endpoint: Endpoint, + mode: ConnectMode, + auth: SaslAuth, + compress: Compress, + logger: Logger, +} + +impl ImapSession { + pub fn new( + connector: Connector, + endpoint: Endpoint, + cleartext: Cleartext, + auth: SaslAuth, + compress: Compress, + logger: Logger, + ) -> ImapSession { + let mode = match (endpoint.tls, cleartext) { + (UrlTls::Implicit, _) => ConnectMode::ImplicitTls, + (UrlTls::Negotiated, Cleartext::Allow) => ConnectMode::Plain, + (UrlTls::Negotiated, Cleartext::Forbid) => ConnectMode::StartTls, + }; + ImapSession { + connector, + endpoint, + mode, + auth, + compress, + logger, + } + } + + pub fn session_url(&self) -> String { + self.endpoint.session_url(&IMAP) + } + + pub fn logger(&self) -> Logger { + self.logger + } + + fn connect(&self) -> Result { + ImapClient::connect( + &self.connector, + &self.endpoint.host, + self.endpoint.port, + self.mode, + self.logger, + ) + } + + pub fn open(&self) -> Result<(ImapClient, String), Error> { + let logger = self.logger; + let mut client = self.connect().map_err(Error::connection)?; + let account_id = self.authenticate(&mut client)?; + logger.progress(format_args!( + "Authenticated as {account_id} on {}", + self.session_url() + )); + client + .refresh_capabilities() + .map_err(|e| Error::Connection(format!("post-auth CAPABILITY: {e}")))?; + if self.compress == Compress::Requested { + if client.has(Capability::CompressDeflate) { + client + .compress_deflate() + .map_err(|e| Error::Connection(format!("COMPRESS DEFLATE: {e}")))?; + logger.progress(format_args!("COMPRESS DEFLATE enabled")); + } else { + logger.progress(format_args!( + "The server does not advertise COMPRESS=DEFLATE; --compress is ignored" + )); + } + } + if client.has(Capability::Enable) && client.has(Capability::Utf8Accept) { + match client.enable(UTF8_ACCEPT) { + Ok(_) => logger.progress(format_args!("UTF8=ACCEPT enabled")), + Err(e) => logger.progress(format_args!( + "ENABLE UTF8=ACCEPT failed ({e}); using modified UTF-7" + )), + } + } + Ok((client, account_id)) + } + + pub fn reopen(&self) -> Result { + let mut client = self.connect()?; + self.authenticate(&mut client) + .map_err(|e| ImapError::AuthFailed(e.to_string()))?; + let _ = client.refresh_capabilities(); + if self.compress == Compress::Requested && client.has(Capability::CompressDeflate) { + client.compress_deflate()?; + } + if client.has(Capability::Enable) && client.has(Capability::Utf8Accept) { + let _ = client.enable(UTF8_ACCEPT); + } + Ok(client) + } + + fn authenticate(&self, client: &mut ImapClient) -> Result { + match &self.auth { + SaslAuth::Basic { user, password } => { + if client.has(Capability::AuthPlain) { + match client.authenticate(SaslPayload::Plain { + authcid: user, + password, + }) { + Ok(()) => return Ok(user.clone()), + Err(e) if e.is_auth_failed() => { + return Err(Error::Connection(format!("auth failed: {e}"))); + } + Err(e) if e.is_negotiation_failure() => {} + Err(e) => return Err(Error::Connection(e.to_string())), + } + } + match client.login(user, password) { + Ok(()) => Ok(user.clone()), + Err(e) if e.is_auth_failed() => { + Err(Error::Connection(format!("auth failed: {e}"))) + } + Err(e) => Err(Error::Connection(format!("LOGIN failed: {e}"))), + } + } + SaslAuth::Bearer { user, token } => { + let has_xoauth2 = client.has(Capability::AuthXoauth2); + let has_oauthbearer = client.has(Capability::AuthOauthBearer); + if !has_xoauth2 && !has_oauthbearer { + return Err(Error::Connection( + "server does not advertise AUTH=XOAUTH2 or AUTH=OAUTHBEARER".into(), + )); + } + if has_xoauth2 { + match client.authenticate(SaslPayload::XOauth2 { user, token }) { + Ok(()) => return Ok(user.clone()), + Err(e) if e.is_auth_failed() => { + return Err(Error::Connection(format!("auth failed: {e}"))); + } + Err(e) if e.is_negotiation_failure() && has_oauthbearer => {} + Err(e) => return Err(Error::Connection(format!("XOAUTH2 failed: {e}"))), + } + } + client + .authenticate(SaslPayload::OAuthBearer { user, token }) + .map_err(|e| Error::Connection(format!("OAUTHBEARER failed: {e}")))?; + Ok(user.clone()) + } + } + } +} + +impl ImapClient { + pub fn name_encoding(&self) -> NameEncoding { + if self.utf8_accept() { + NameEncoding::Utf8 + } else { + NameEncoding::ModifiedUtf7 + } + } +} diff --git a/src/imap/transport.rs b/src/imap/transport.rs deleted file mode 100644 index 8ede26a..0000000 --- a/src/imap/transport.rs +++ /dev/null @@ -1,434 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::io::{self, Read, Write}; -use std::net::TcpStream; -use std::sync::Arc; -use std::time::Duration; - -use rustls::pki_types::ServerName; -use rustls::{ClientConfig, ClientConnection, StreamOwned}; -use rustls_platform_verifier::BuilderVerifierExt; - -use super::error::ImapError; - -const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); - -pub trait ImapStream: Read + Write + Send { - fn upgrade_tls( - self: Box, - connector: &Connector, - host: &str, - ) -> Result, ImapError>; - - fn is_tls(&self) -> bool; -} - -pub struct Connector { - config: Arc, -} - -impl Connector { - pub fn new(allow_invalid_certs: bool) -> Result { - let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider()); - let builder = ClientConfig::builder_with_provider(provider) - .with_safe_default_protocol_versions() - .map_err(|e| ImapError::Tls(format!("rustls protocol versions: {e}")))?; - let config = if allow_invalid_certs { - builder - .dangerous() - .with_custom_certificate_verifier(Arc::new(NoVerify)) - .with_no_client_auth() - } else { - builder - .with_platform_verifier() - .map_err(|e| ImapError::Tls(format!("rustls platform verifier: {e}")))? - .with_no_client_auth() - }; - Ok(Connector { - config: Arc::new(config), - }) - } - - pub fn connect_plain(&self, host: &str, port: u16) -> Result, ImapError> { - let tcp = tcp_connect(host, port)?; - Ok(Box::new(TcpImapStream { tcp })) - } - - pub fn connect_tls(&self, host: &str, port: u16) -> Result, ImapError> { - let tcp = tcp_connect(host, port)?; - self.wrap_tls(tcp, host) - } - - fn wrap_tls(&self, tcp: TcpStream, host: &str) -> Result, ImapError> { - let server_name = ServerName::try_from(host.to_owned()) - .map_err(|e| ImapError::Tls(format!("invalid server name {host}: {e}")))?; - let conn = ClientConnection::new(self.config.clone(), server_name) - .map_err(|e| ImapError::Tls(format!("rustls client: {e}")))?; - Ok(Box::new(TlsImapStream { - inner: StreamOwned::new(conn, tcp), - })) - } -} - -fn tcp_connect(host: &str, port: u16) -> Result { - let tcp = TcpStream::connect((host, port))?; - tcp.set_read_timeout(Some(DEFAULT_TIMEOUT))?; - tcp.set_write_timeout(Some(DEFAULT_TIMEOUT))?; - tcp.set_nodelay(true)?; - Ok(tcp) -} - -pub struct TcpImapStream { - tcp: TcpStream, -} - -impl Read for TcpImapStream { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - self.tcp.read(buf) - } -} - -impl Write for TcpImapStream { - fn write(&mut self, buf: &[u8]) -> io::Result { - self.tcp.write(buf) - } - fn flush(&mut self) -> io::Result<()> { - self.tcp.flush() - } -} - -impl ImapStream for TcpImapStream { - fn upgrade_tls( - self: Box, - connector: &Connector, - host: &str, - ) -> Result, ImapError> { - connector.wrap_tls(self.tcp, host) - } - - fn is_tls(&self) -> bool { - false - } -} - -pub struct TlsImapStream { - inner: StreamOwned, -} - -impl Read for TlsImapStream { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - self.inner.read(buf) - } -} - -impl Write for TlsImapStream { - fn write(&mut self, buf: &[u8]) -> io::Result { - self.inner.write(buf) - } - fn flush(&mut self) -> io::Result<()> { - self.inner.flush() - } -} - -impl ImapStream for TlsImapStream { - fn upgrade_tls( - self: Box, - _connector: &Connector, - _host: &str, - ) -> Result, ImapError> { - Ok(self) - } - - fn is_tls(&self) -> bool { - true - } -} - -pub struct DeflateImapStream { - inner: Box, - decompress: flate2::Decompress, - compress: flate2::Compress, - rx_raw: Vec, - rx_raw_pos: usize, - rx_decoded: Vec, - rx_decoded_pos: usize, -} - -impl DeflateImapStream { - pub fn wrap(inner: Box, primer: &[u8]) -> Box { - Box::new(DeflateImapStream { - inner, - decompress: flate2::Decompress::new(false), - compress: flate2::Compress::new(flate2::Compression::default(), false), - rx_raw: primer.to_vec(), - rx_raw_pos: 0, - rx_decoded: Vec::new(), - rx_decoded_pos: 0, - }) - } -} - -impl Read for DeflateImapStream { - fn read(&mut self, out: &mut [u8]) -> io::Result { - loop { - if self.rx_decoded_pos < self.rx_decoded.len() { - let avail = &self.rx_decoded[self.rx_decoded_pos..]; - let n = avail.len().min(out.len()); - out[..n].copy_from_slice(&avail[..n]); - self.rx_decoded_pos += n; - return Ok(n); - } - if self.rx_raw_pos >= self.rx_raw.len() { - self.rx_raw.clear(); - self.rx_raw_pos = 0; - self.rx_raw.resize(8192, 0); - let n = self.inner.read(&mut self.rx_raw)?; - if n == 0 { - return Ok(0); - } - self.rx_raw.truncate(n); - } - self.rx_decoded.clear(); - self.rx_decoded_pos = 0; - self.rx_decoded.resize(out.len().max(8192) * 8, 0); - let in_before = self.decompress.total_in(); - let out_before = self.decompress.total_out(); - let status = self.decompress.decompress( - &self.rx_raw[self.rx_raw_pos..], - &mut self.rx_decoded, - flate2::FlushDecompress::None, - ); - match status { - Ok(_) => {} - Err(e) => { - return Err(io::Error::new(io::ErrorKind::InvalidData, e)); - } - } - let consumed = (self.decompress.total_in() - in_before) as usize; - let produced = (self.decompress.total_out() - out_before) as usize; - self.rx_raw_pos += consumed; - self.rx_decoded.truncate(produced); - if produced == 0 && consumed == 0 { - self.rx_raw.clear(); - self.rx_raw_pos = 0; - continue; - } - } - } -} - -impl Write for DeflateImapStream { - fn write(&mut self, buf: &[u8]) -> io::Result { - let mut out = vec![0u8; buf.len() + 256]; - let in_before = self.compress.total_in(); - let out_before = self.compress.total_out(); - self.compress - .compress(buf, &mut out, flate2::FlushCompress::Sync) - .map_err(io::Error::other)?; - let consumed = (self.compress.total_in() - in_before) as usize; - let produced = (self.compress.total_out() - out_before) as usize; - self.inner.write_all(&out[..produced])?; - Ok(consumed) - } - - fn flush(&mut self) -> io::Result<()> { - self.inner.flush() - } -} - -impl ImapStream for DeflateImapStream { - fn upgrade_tls( - self: Box, - _connector: &Connector, - _host: &str, - ) -> Result, ImapError> { - Err(ImapError::Protocol( - "cannot STARTTLS after COMPRESS DEFLATE".into(), - )) - } - - fn is_tls(&self) -> bool { - self.inner.is_tls() - } -} - -#[derive(Debug)] -struct NoVerify; - -impl rustls::client::danger::ServerCertVerifier for NoVerify { - fn verify_server_cert( - &self, - _end_entity: &rustls::pki_types::CertificateDer<'_>, - _intermediates: &[rustls::pki_types::CertificateDer<'_>], - _server_name: &rustls::pki_types::ServerName<'_>, - _ocsp_response: &[u8], - _now: rustls::pki_types::UnixTime, - ) -> Result { - Ok(rustls::client::danger::ServerCertVerified::assertion()) - } - - fn verify_tls12_signature( - &self, - _message: &[u8], - _cert: &rustls::pki_types::CertificateDer<'_>, - _dss: &rustls::DigitallySignedStruct, - ) -> Result { - Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) - } - - fn verify_tls13_signature( - &self, - _message: &[u8], - _cert: &rustls::pki_types::CertificateDer<'_>, - _dss: &rustls::DigitallySignedStruct, - ) -> Result { - Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) - } - - fn supported_verify_schemes(&self) -> Vec { - use rustls::SignatureScheme::*; - vec![ - RSA_PKCS1_SHA256, - RSA_PKCS1_SHA384, - RSA_PKCS1_SHA512, - ECDSA_NISTP256_SHA256, - ECDSA_NISTP384_SHA384, - ECDSA_NISTP521_SHA512, - RSA_PSS_SHA256, - RSA_PSS_SHA384, - RSA_PSS_SHA512, - ED25519, - ] - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn connector_constructs_with_invalid_certs_disabled() { - let _c = Connector::new(false).unwrap(); - } - - #[test] - fn connector_constructs_with_invalid_certs_allowed() { - let _c = Connector::new(true).unwrap(); - } - - struct PeerStream { - outbound: Vec, - inbound: Vec, - inbound_pos: usize, - } - - impl PeerStream { - fn new() -> Self { - Self { - outbound: Vec::new(), - inbound: Vec::new(), - inbound_pos: 0, - } - } - } - - impl Read for PeerStream { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - if self.inbound_pos >= self.inbound.len() { - return Ok(0); - } - let avail = &self.inbound[self.inbound_pos..]; - let n = avail.len().min(buf.len()); - buf[..n].copy_from_slice(&avail[..n]); - self.inbound_pos += n; - Ok(n) - } - } - - impl Write for PeerStream { - fn write(&mut self, buf: &[u8]) -> io::Result { - self.outbound.extend_from_slice(buf); - Ok(buf.len()) - } - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } - } - - impl ImapStream for PeerStream { - fn upgrade_tls( - self: Box, - _c: &Connector, - _h: &str, - ) -> Result, ImapError> { - Ok(self) - } - fn is_tls(&self) -> bool { - false - } - } - - #[test] - fn deflate_writes_round_trip_through_peer_decompressor() { - let peer = Box::new(PeerStream::new()); - let mut stream = DeflateImapStream::wrap(peer, b""); - stream.write_all(b"hello world").unwrap(); - - stream.write_all(b" again").unwrap(); - stream.flush().unwrap(); - } - - #[test] - fn deflate_reads_decompressed_payload_from_inner() { - let mut peer = PeerStream::new(); - let mut compressor = flate2::Compress::new(flate2::Compression::default(), false); - let plaintext = b"* OK [CAPABILITY IMAP4rev2 COMPRESS=DEFLATE] hi\r\n"; - let mut out = vec![0u8; plaintext.len() + 256]; - compressor - .compress(plaintext, &mut out, flate2::FlushCompress::Sync) - .unwrap(); - let n = compressor.total_out() as usize; - peer.inbound.extend_from_slice(&out[..n]); - - let mut stream = DeflateImapStream::wrap(Box::new(peer), b""); - let mut got = vec![0u8; 256]; - let mut total = 0usize; - while total < plaintext.len() { - let n = stream.read(&mut got[total..]).unwrap(); - if n == 0 { - break; - } - total += n; - } - assert_eq!(&got[..total], plaintext); - } - - #[test] - fn deflate_consumes_primer_before_reading_inner() { - let mut compressor = flate2::Compress::new(flate2::Compression::default(), false); - let plaintext = b"primed payload\r\n"; - let mut out = vec![0u8; plaintext.len() + 256]; - compressor - .compress(plaintext, &mut out, flate2::FlushCompress::Sync) - .unwrap(); - let n = compressor.total_out() as usize; - let primer = out[..n].to_vec(); - - let peer = Box::new(PeerStream::new()); - let mut stream = DeflateImapStream::wrap(peer, &primer); - let mut buf = vec![0u8; 256]; - let mut total = 0usize; - while total < plaintext.len() { - let n = stream.read(&mut buf[total..]).unwrap(); - if n == 0 { - break; - } - total += n; - } - assert_eq!(&buf[..total], plaintext); - } -} diff --git a/src/import/dav/collections.rs b/src/import/dav/collections.rs new file mode 100644 index 0000000..c6dfd66 --- /dev/null +++ b/src/import/dav/collections.rs @@ -0,0 +1,495 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::borrow::Cow; +use std::collections::HashSet; + +use rusqlite::{OptionalExtension, params}; + +use crate::archive::rows::{AddressBookRecord, CalendarRecord, Subscribed}; +use crate::archive::{LocalId, SourceId, Store}; +use crate::dav::discover::DiscoveredCollection; +use crate::dav::href::Href; +use crate::error::Error; +use crate::logging::Logger; +use crate::sync::TypeCounts; +use crate::sync::vanish::{Children, OnRowError, RowFailure, VanishedSweep}; +use crate::types::ObjectType; + +const DELETE_CALENDAR_MEMBERS: &str = "DELETE FROM calendar_events + WHERE EXISTS (SELECT 1 FROM json_each(calendar_ids) WHERE value = ?1)"; +const DELETE_ADDRESS_BOOK_MEMBERS: &str = "DELETE FROM contact_cards + WHERE EXISTS (SELECT 1 FROM json_each(address_book_ids) WHERE value = ?1)"; +const UNTITLED: &str = "Untitled"; +const ALL_AVAILABILITY: &str = "all"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum CollectionKind { + Calendar, + AddressBook, +} + +impl CollectionKind { + pub(super) const fn container(self) -> ObjectType { + match self { + CollectionKind::Calendar => ObjectType::Calendar, + CollectionKind::AddressBook => ObjectType::AddressBook, + } + } + + pub(super) const fn item(self) -> ObjectType { + match self { + CollectionKind::Calendar => ObjectType::CalendarEvent, + CollectionKind::AddressBook => ObjectType::ContactCard, + } + } + + fn upsert( + self, + store: Store<'_>, + collection: CollectionMeta<'_>, + existing: Option, + ) -> rusqlite::Result { + let name = collection.name(); + let props = &collection.0.props; + match self { + CollectionKind::Calendar => { + let kept = match existing { + Some(id) => StoredCollection(store).calendar(id)?, + None => None, + } + .unwrap_or_default(); + let record = CalendarRecord { + name: &name, + description: props.calendar_description.as_deref(), + color: props.calendar_color.as_deref(), + sort_order: props.calendar_order.unwrap_or(0).max(0), + subscribed: kept.subscribed, + is_visible: kept.is_visible, + is_default: kept.is_default, + include_in_availability: &kept.include_in_availability, + default_alerts_with_time: kept.default_alerts_with_time.as_deref(), + default_alerts_without_time: kept.default_alerts_without_time.as_deref(), + time_zone: collection.time_zone(), + }; + let calendars = store.calendars(); + match existing { + Some(id) => calendars.update(id, &record).map(|_| id), + None => calendars.insert(&record), + } + } + CollectionKind::AddressBook => { + let kept = match existing { + Some(id) => StoredCollection(store).address_book(id)?, + None => None, + } + .unwrap_or_default(); + let record = AddressBookRecord { + name: &name, + description: props.addressbook_description.as_deref(), + sort_order: kept.sort_order, + is_default: kept.is_default, + subscribed: kept.subscribed, + }; + let books = store.address_books(); + match existing { + Some(id) => books.update(id, &record).map(|_| id), + None => books.insert(&record), + } + } + } + } + + fn delete_members(self, store: Store<'_>, container: LocalId) -> rusqlite::Result<()> { + let sql = match self { + CollectionKind::Calendar => DELETE_CALENDAR_MEMBERS, + CollectionKind::AddressBook => DELETE_ADDRESS_BOOK_MEMBERS, + }; + store.connection().execute(sql, params![container])?; + Ok(()) + } +} + +#[derive(Clone, Copy)] +struct CollectionMeta<'c>(&'c DiscoveredCollection); + +impl<'c> CollectionMeta<'c> { + fn name(self) -> Cow<'c, str> { + if let Some(name) = self.0.props.displayname.as_deref() + && !name.trim().is_empty() + { + return Cow::Borrowed(name); + } + let component = self.0.href.last_component(); + if component.is_empty() { + Cow::Borrowed(UNTITLED) + } else { + Cow::Owned(component) + } + } + + fn time_zone(self) -> Option<&'c str> { + self.0 + .props + .calendar_timezone + .as_deref()? + .lines() + .filter_map(|line| line.trim().strip_prefix("TZID:")) + .map(str::trim) + .find(|tzid| !tzid.is_empty()) + } +} + +#[derive(Clone, Copy)] +pub(super) struct CollectionSync<'a> { + pub(super) store: Store<'a>, + pub(super) source: SourceId, + pub(super) logger: Logger, +} + +impl CollectionSync<'_> { + pub(super) fn reconcile<'d>( + &self, + kind: CollectionKind, + discovered: &'d [DiscoveredCollection], + counts: &mut TypeCounts, + ) -> Result, Error> { + let container = kind.container(); + let mut existing = self + .store + .dav_ids(self.source) + .all(container) + .map_err(Error::partial)?; + let mut upserted = Vec::with_capacity(discovered.len()); + for collection in discovered { + let href = collection.href.as_str(); + let known = existing.get(href).copied(); + let tx = self.store.begin().map_err(Error::partial)?; + match kind.upsert(tx.store(), CollectionMeta(collection), known) { + Ok(local) => { + if known.is_none() { + tx.store() + .dav_ids(self.source) + .insert(container, href, href, "", local) + .map_err(Error::partial)?; + counts.created += 1; + } else { + counts.fetched += 1; + } + tx.commit().map_err(Error::partial)?; + upserted.push((&collection.href, local)); + self.logger + .progress(format_args!("Collection {href} stored as {local}")); + } + Err(e) => { + drop(tx); + self.logger + .warn(format_args!("cannot store collection {href}: {e}")); + counts.failed += 1; + } + } + } + let server: HashSet<&str> = discovered.iter().map(|c| c.href.as_str()).collect(); + existing.retain(|href, _| !server.contains(href.as_str())); + let mut vanished: Vec<(String, LocalId)> = existing.into_iter().collect(); + vanished.sort_unstable(); + self.sweep(kind, &vanished, counts)?; + Ok(upserted) + } + + fn sweep( + &self, + kind: CollectionKind, + vanished: &[(String, LocalId)], + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let source = self.source; + let logger = self.logger; + VanishedSweep::new(self.store, OnRowError::Count, Children::Restrict) + .run( + kind.container(), + vanished, + counts, + |store, href, local| { + let ids = store.dav_ids(source); + ids.delete_collection(kind.item(), href)?; + ids.delete_collection(kind.container(), href)?; + kind.delete_members(store, local) + }, + |href, _, failure| { + if let RowFailure::Sql(e) = failure { + logger.warn(format_args!( + "cannot delete vanished collection {href}: {e}" + )); + } + }, + ) + .map_err(Error::partial) + } +} + +struct KeptCalendar { + subscribed: Subscribed, + is_visible: bool, + is_default: bool, + include_in_availability: String, + default_alerts_with_time: Option, + default_alerts_without_time: Option, +} + +impl Default for KeptCalendar { + fn default() -> KeptCalendar { + KeptCalendar { + subscribed: Subscribed::Yes, + is_visible: true, + is_default: false, + include_in_availability: ALL_AVAILABILITY.to_owned(), + default_alerts_with_time: None, + default_alerts_without_time: None, + } + } +} + +struct KeptAddressBook { + sort_order: i64, + is_default: bool, + subscribed: Subscribed, +} + +impl Default for KeptAddressBook { + fn default() -> KeptAddressBook { + KeptAddressBook { + sort_order: 0, + is_default: false, + subscribed: Subscribed::Yes, + } + } +} + +#[derive(Clone, Copy)] +struct StoredCollection<'a>(Store<'a>); + +impl StoredCollection<'_> { + fn calendar(self, id: LocalId) -> rusqlite::Result> { + self.0 + .connection() + .prepare_cached( + "SELECT is_subscribed, is_visible, is_default, include_in_availability, + default_alerts_with_time, default_alerts_without_time + FROM calendars WHERE id = ?1", + )? + .query_row([id], |row| { + Ok(KeptCalendar { + subscribed: row.get(0)?, + is_visible: row.get(1)?, + is_default: row.get(2)?, + include_in_availability: row.get(3)?, + default_alerts_with_time: row.get(4)?, + default_alerts_without_time: row.get(5)?, + }) + }) + .optional() + } + + fn address_book(self, id: LocalId) -> rusqlite::Result> { + self.0 + .connection() + .prepare_cached( + "SELECT sort_order, is_default, is_subscribed FROM address_books WHERE id = ?1", + )? + .query_row([id], |row| { + Ok(KeptAddressBook { + sort_order: row.get(0)?, + is_default: row.get(1)?, + subscribed: row.get(2)?, + }) + }) + .optional() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::Archive; + use crate::archive::rows::{CalendarEventRecord, EventDataType}; + use crate::archive::sources::{SourceKey, SourceKind}; + use crate::dav::parse::ResourceProps; + use crate::logging::Level; + + fn collection(href: &str, props: ResourceProps) -> DiscoveredCollection { + DiscoveredCollection { + url: format!("https://x{href}"), + href: Href::from_normalised(href.to_owned()), + props, + } + } + + fn count(archive: &Archive, table: &str) -> i64 { + archive + .connection() + .query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0)) + .unwrap() + } + + #[test] + fn time_zone_is_the_first_tzid() { + let c = collection( + "/cal/", + ResourceProps { + calendar_timezone: Some( + "BEGIN:VTIMEZONE\r\nTZID:Europe/Berlin\r\nEND:VTIMEZONE\r\n".to_owned(), + ), + ..Default::default() + }, + ); + assert_eq!(CollectionMeta(&c).time_zone(), Some("Europe/Berlin")); + } + + #[test] + fn time_zone_is_none_when_absent() { + let mut c = collection("/cal/", ResourceProps::default()); + assert!(CollectionMeta(&c).time_zone().is_none()); + c.props.calendar_timezone = Some("nothing here".to_owned()); + assert!(CollectionMeta(&c).time_zone().is_none()); + } + + #[test] + fn name_falls_back_to_the_href_when_displayname_is_empty() { + let c = collection("/dav/cal/u/work/", ResourceProps::default()); + assert_eq!(CollectionMeta(&c).name(), "work"); + } + + #[test] + fn name_uses_the_displayname_when_present() { + let c = collection( + "/dav/cal/u/default/", + ResourceProps { + displayname: Some("Default".to_owned()), + ..Default::default() + }, + ); + assert_eq!(CollectionMeta(&c).name(), "Default"); + } + + #[test] + fn vanished_calendar_cascades_events_and_sync_id_rows() { + let archive = Archive::open_in_memory().unwrap(); + let store = archive.store(); + let source = store + .sources() + .upsert( + &SourceKey { + kind: SourceKind::CalDav, + url: "https://x", + account_id: "u", + }, + Some("u"), + "u", + ) + .unwrap(); + let sync = CollectionSync { + store, + source, + logger: Logger::new(Level::Quiet), + }; + let discovered = [collection("/cal/work/", ResourceProps::default())]; + let mut counts = TypeCounts::default(); + let upserted = sync + .reconcile(CollectionKind::Calendar, &discovered, &mut counts) + .unwrap(); + let [(_, calendar)] = upserted.as_slice() else { + panic!("one calendar expected"); + }; + let event = store + .calendar_events() + .insert(&CalendarEventRecord { + calendars: &[*calendar], + is_draft: false, + use_default_alerts: false, + data_type: EventDataType::Event, + data: "{}", + }) + .unwrap(); + store + .dav_ids(source) + .insert( + ObjectType::CalendarEvent, + "/cal/work/", + "/cal/work/e.ics", + "\"v\"", + event, + ) + .unwrap(); + + let mut counts = TypeCounts::default(); + sync.reconcile(CollectionKind::Calendar, &[], &mut counts) + .unwrap(); + + assert_eq!((counts.deleted, counts.failed), (1, 0)); + assert_eq!(count(&archive, "calendars"), 0, "calendar row gone"); + assert_eq!(count(&archive, "calendar_events"), 0, "event rows cascaded"); + assert_eq!(count(&archive, "sync_id_dav"), 0, "both id types cleared"); + } + + #[test] + fn calendar_update_keeps_the_columns_dav_never_rewrites() { + let archive = Archive::open_in_memory().unwrap(); + let store = archive.store(); + let source = store + .sources() + .upsert( + &SourceKey { + kind: SourceKind::CalDav, + url: "https://x", + account_id: "u", + }, + Some("u"), + "u", + ) + .unwrap(); + let sync = CollectionSync { + store, + source, + logger: Logger::new(Level::Quiet), + }; + let discovered = [collection("/cal/work/", ResourceProps::default())]; + sync.reconcile( + CollectionKind::Calendar, + &discovered, + &mut TypeCounts::default(), + ) + .unwrap(); + archive + .connection() + .execute_batch( + "UPDATE calendars SET is_visible = 0, is_subscribed = 0, + include_in_availability = 'none', default_alerts_with_time = '{}'", + ) + .unwrap(); + + sync.reconcile( + CollectionKind::Calendar, + &discovered, + &mut TypeCounts::default(), + ) + .unwrap(); + + let kept: (bool, bool, String, Option) = archive + .connection() + .query_row( + "SELECT is_visible, is_subscribed, include_in_availability, + default_alerts_with_time FROM calendars", + [], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)), + ) + .unwrap(); + assert_eq!( + kept, + (false, false, "none".to_owned(), Some("{}".to_owned())) + ); + } +} diff --git a/src/import/dav/coordinator.rs b/src/import/dav/coordinator.rs new file mode 100644 index 0000000..d8f2c89 --- /dev/null +++ b/src/import/dav/coordinator.rs @@ -0,0 +1,486 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fmt::Write; + +use url::Url; + +use crate::archive::sources::{SourceChange, SourceKey, SourceKind}; +use crate::archive::{Archive, SourceId}; +use crate::console::{Counted, Grouped}; +use crate::dav::client::DavClient; +use crate::dav::discover::{DavKind, Discovery, DiscoveryError}; +use crate::dav::href::Href; +use crate::dav::xml::Propfind; +use crate::error::Error; +use crate::logging::Logger; +use crate::net::auth::Auth; +use crate::sync::run::{GcFailure, ImportRun, SourceLabel}; +use crate::sync::{RunMode, RunOptions, RunOutcome, Summary, TypeCounts}; +use crate::types::ObjectType; + +use super::collections::{CollectionKind, CollectionSync}; +use super::items::ItemSync; +use super::replace::Removed; +use super::tree::FileTree; + +const BEARER_USERNAME: &str = "(bearer)"; + +#[derive(Debug, Clone, Copy)] +pub enum DavKindArg { + Caldav, + Carddav, + Webdav, +} + +impl DavKindArg { + pub fn kind(self) -> DavKind { + match self { + DavKindArg::Caldav => DavKind::Caldav, + DavKindArg::Carddav => DavKind::Carddav, + DavKindArg::Webdav => DavKind::Webdav, + } + } + + pub fn source_kind(self) -> SourceKind { + match self { + DavKindArg::Caldav => SourceKind::CalDav, + DavKindArg::Carddav => SourceKind::CardDav, + DavKindArg::Webdav => SourceKind::WebDav, + } + } +} + +#[derive(Debug)] +pub struct DavImport { + pub kind: DavKindArg, + pub url: String, + pub auth: Auth, + pub allow_cleartext: bool, + pub dav_connections: usize, + pub multiget_batch: usize, + pub source_change: SourceChange, +} + +impl DavImport { + pub fn run(self, options: RunOptions) -> RunOutcome { + let failed = |error: Error| RunOutcome { + summary: Summary::default(), + error: Some(error), + }; + if let Err(e) = self.check_scheme() { + return failed(e); + } + let archive = match Archive::open(&options.archive) { + Ok(archive) => archive, + Err(e) => return failed(e.into()), + }; + let client = DavClient::new(self.auth.clone(), options.client_options()); + let discovery = match Discovery::run(&client, self.kind.kind(), &self.url) { + Ok(discovery) => discovery, + Err(e) => return failed(e.into()), + }; + let session_url = SourceIdentity::base_url(&self.url); + let account_id = SourceIdentity::account_url( + discovery + .principal_url + .as_deref() + .unwrap_or(&discovery.home_set_url), + ); + let key = SourceKey { + kind: self.kind.source_kind(), + url: &session_url, + account_id: &account_id, + }; + let username = self.username(); + let label = SourceLabel { + name: Some(&account_id), + username: &username, + }; + let logger = options.logger; + let report = || { + logger.progress(format_args!( + "Found {} under {} (principal {})", + Counted::new( + discovery.collections.len() as u64, + "collection", + "collections" + ), + discovery.home_set_url, + discovery.principal_url.as_deref().unwrap_or("unknown") + )) + }; + let replace = self.source_change == SourceChange::Replace; + let mut removed = Removed::default(); + if replace { + report(); + if options.mode == RunMode::Apply { + removed = match Removed::other_sources(archive.store(), &key, logger) { + Ok(removed) => removed, + Err(e) => return failed(e), + }; + } + } + let run = match ImportRun::start_on(archive, options, &key, label, self.source_change) { + Ok(run) => run, + Err(e) => return failed(e), + }; + if !replace { + report(); + } + let mut dav = DavRun { + run, + client, + discovery, + config: self, + }; + let result = dav.reconcile(&key, removed); + let stats = dav.client.stats(); + match result { + Ok(()) => dav.run.finish(stats, GcFailure::WarnAsPartial), + Err(e) => dav.run.abort(e, stats), + } + } + + fn username(&self) -> String { + match &self.auth { + Auth::Basic { user, .. } => user.clone(), + Auth::Bearer { .. } => BEARER_USERNAME.to_owned(), + } + } + + fn check_scheme(&self) -> Result<(), Error> { + let url = &self.url; + let parsed = + Url::parse(url).map_err(|e| Error::Usage(format!("invalid --url {url:?}: {e}")))?; + match parsed.scheme() { + "https" => Ok(()), + "http" if self.allow_cleartext => Ok(()), + "http" => Err(Error::Connection( + "http:// URL requires --allow-cleartext".to_owned(), + )), + other => Err(Error::Usage(format!( + "--url scheme must be http or https, got {other}" + ))), + } + } +} + +impl From for Error { + fn from(err: DiscoveryError) -> Error { + match err { + DiscoveryError::NotFound { url } => Error::Usage(format!( + "no DAV collections found under --url {url:?}; \ + check the URL and that the account has access" + )), + DiscoveryError::Transport(e) => Error::from(e), + DiscoveryError::Href(e) => Error::Connection(format!("DAV discovery: bad href: {e}")), + } + } +} + +struct SourceIdentity; + +impl SourceIdentity { + fn base_url(url: &str) -> String { + let Ok(parsed) = Url::parse(url) else { + return url.to_owned(); + }; + let mut base = format!("{}://{}", parsed.scheme(), parsed.host_str().unwrap_or("")); + if let Some(port) = parsed.port() { + let _ = write!(base, ":{port}"); + } + base + } + + fn account_url(url: &str) -> String { + let Ok(mut parsed) = Url::parse(url) else { + return url.to_owned(); + }; + parsed.set_query(None); + parsed.set_fragment(None); + if !parsed.path().ends_with('/') { + let path = format!("{}/", parsed.path()); + parsed.set_path(&path); + } + parsed.into() + } +} + +#[derive(Default)] +struct PhaseCounts { + container: TypeCounts, + items: TypeCounts, +} + +struct DavRun { + run: ImportRun, + client: DavClient, + discovery: Discovery, + config: DavImport, +} + +impl DavRun { + fn logger(&self) -> Logger { + self.run.options().logger + } + + fn phase(&self) -> Option { + match self.config.kind { + DavKindArg::Caldav => Some(CollectionKind::Calendar), + DavKindArg::Carddav => Some(CollectionKind::AddressBook), + DavKindArg::Webdav => None, + } + } + + fn reconcile(&mut self, key: &SourceKey<'_>, removed: Removed) -> Result<(), Error> { + if self.run.options().mode == RunMode::DryRun { + if self.config.source_change == SourceChange::Replace { + let logger = self.logger(); + for other in self.run.store().sources().others(key)? { + logger.info(format_args!( + "Would remove the {} source {} (account {})", + key.kind.label(), + other.url, + other.account_id + )); + } + } + return self.dry_run(); + } + let result = match self.phase() { + Some(phase) => self.collections(phase), + None => self.files(), + }; + removed.add_to(self.run.summary()); + result + } + + fn source(&self) -> Result { + self.run + .claim() + .id() + .ok_or_else(|| Error::partial("DAV source is not registered")) + } + + fn collections(&mut self, kind: CollectionKind) -> Result<(), Error> { + let mut counts = PhaseCounts::default(); + let result = self.collections_into(kind, &mut counts); + *self.run.counts(kind.container()) = counts.container; + *self.run.counts(kind.item()) = counts.items; + result + } + + fn collections_into( + &self, + kind: CollectionKind, + counts: &mut PhaseCounts, + ) -> Result<(), Error> { + let logger = self.logger(); + let source = self.source()?; + let store = self.run.store(); + let container = kind.container(); + let upserted = CollectionSync { + store, + source, + logger, + } + .reconcile(kind, &self.discovery.collections, &mut counts.container)?; + logger.info(format_args!( + "{}: {} upserted, {} deleted, {} failed", + container.label(), + Grouped(counts.container.created + counts.container.fetched), + Grouped(counts.container.deleted), + Grouped(counts.container.failed) + )); + let items = ItemSync { + client: &self.client, + store, + source, + base_url: &self.discovery.home_set_url, + multiget_batch: self.config.multiget_batch, + dav_connections: self.config.dav_connections, + logger, + }; + for (href, local) in upserted { + match items.reconcile(kind, href.as_str(), local, &mut counts.items) { + Ok(()) => {} + Err(e) if e.aborts_run() => return Err(e), + Err(e) => { + logger.warn(format_args!( + "items of {} {} failed: {e}", + container.noun(), + href.as_str() + )); + counts.items.failed += 1; + } + } + } + Ok(()) + } + + fn files(&mut self) -> Result<(), Error> { + let mut file_counts = TypeCounts::default(); + let result = match self.discovery.collections.first() { + Some(root) => FileTree { + client: &self.client, + store: self.run.store(), + source: self.source()?, + base_url: &self.discovery.home_set_url, + dav_connections: self.config.dav_connections, + logger: self.logger(), + } + .reconcile(root, &mut file_counts), + None => Ok(()), + }; + *self.run.counts(ObjectType::FileNode) = file_counts; + result + } + + fn dry_run(&mut self) -> Result<(), Error> { + let mut counts = PhaseCounts::default(); + let mut plan = PhasePlan::default(); + let result = self.dry_run_into(&mut counts, &mut plan); + let (container, item) = match self.phase() { + Some(kind) => (kind.container(), Some(kind.item())), + None => (ObjectType::FileNode, None), + }; + let summary = self.run.summary(); + *summary.counts(container) = counts.container; + if let Some(item) = item { + *summary.counts(item) = counts.items; + } + if result.is_ok() { + match item { + Some(item) => { + summary.plan(container).new = plan.containers; + summary.plan(item).new = plan.items; + } + None => summary.plan(container).new = plan.items, + } + } + result + } + + fn dry_run_into(&self, counts: &mut PhaseCounts, plan: &mut PhasePlan) -> Result<(), Error> { + let logger = self.logger(); + let discovery = &self.discovery; + let container = self + .phase() + .map_or(ObjectType::FileNode, |kind| kind.container()); + plan.containers = discovery.collections.len() as u64; + logger.note(format_args!( + "a DAV dry run does not compare with the archive; every {} counts as new", + container.noun() + )); + let (targets, listing, failed) = match self.config.kind { + DavKindArg::Webdav => ( + discovery.collections.get(..1).unwrap_or_default(), + Propfind::WebdavListing, + &mut counts.container.failed, + ), + DavKindArg::Caldav | DavKindArg::Carddav => ( + discovery.collections.as_slice(), + Propfind::DavItems, + &mut counts.items.failed, + ), + }; + for coll in targets { + let url = Href::join_absolute(&discovery.home_set_url, coll.href.as_str()) + .map_err(Error::partial)?; + match self + .client + .propfind_responses(&url, 1, listing.body(), &url) + .map_err(Error::partial) + { + Ok(ms) => { + plan.items += ms + .responses + .iter() + .filter(|r| !r.props.is_collection) + .count() as u64; + } + Err(e) if e.aborts_run() => return Err(e), + Err(e) => { + logger.warn(format_args!( + "cannot list {} {}: {e}", + container.noun(), + coll.href.as_str() + )); + *failed += 1; + } + } + } + Ok(()) + } +} + +#[derive(Default)] +struct PhasePlan { + containers: u64, + items: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn import(url: &str, allow_cleartext: bool) -> DavImport { + DavImport { + kind: DavKindArg::Caldav, + url: url.to_owned(), + auth: Auth::Bearer { + token: "t".to_owned(), + }, + allow_cleartext, + dav_connections: 1, + multiget_batch: 1, + source_change: SourceChange::Reject, + } + } + + #[test] + fn https_url_is_accepted_without_allow_cleartext() { + assert!(import("https://x/", false).check_scheme().is_ok()); + } + + #[test] + fn http_url_requires_allow_cleartext() { + let err = import("http://x/", false).check_scheme().unwrap_err(); + assert!(matches!(err, Error::Connection(_))); + assert!(import("http://x/", true).check_scheme().is_ok()); + } + + #[test] + fn ftp_url_rejected_as_usage() { + let err = import("ftp://x/", false).check_scheme().unwrap_err(); + assert!(matches!(err, Error::Usage(_))); + } + + #[test] + fn base_url_strips_path() { + assert_eq!( + SourceIdentity::base_url("https://dav.example.com/cal/"), + "https://dav.example.com" + ); + } + + #[test] + fn base_url_keeps_explicit_port() { + assert_eq!( + SourceIdentity::base_url("https://dav.example.com:8443/cal/"), + "https://dav.example.com:8443" + ); + } + + #[test] + fn account_url_drops_query_and_adds_trailing_slash() { + assert_eq!( + SourceIdentity::account_url("https://dav.example.com/p/alice?x=1#f"), + "https://dav.example.com/p/alice/" + ); + } +} diff --git a/src/import/dav/items.rs b/src/import/dav/items.rs new file mode 100644 index 0000000..aa8a928 --- /dev/null +++ b/src/import/dav/items.rs @@ -0,0 +1,581 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; +use std::{slice, str}; + +use crate::archive::ids::dav::DavItem; +use crate::archive::rows::{CalendarEventRecord, ContactCardRecord}; +use crate::archive::{LocalId, SourceId, Store}; +use crate::console::Grouped; +use crate::dav::client::{DavClient, DavHttpResponse, MultiStatus}; +use crate::dav::error::DavError; +use crate::dav::href::Href; +use crate::dav::parse::DavResponse; +use crate::dav::retry::DavOutcome; +use crate::dav::xml::{Multiget, Propfind}; +use crate::error::Error; +use crate::logging::{Level, Logger}; +use crate::sync::TypeCounts; +use crate::sync::calcard::{ExtractedFields, JsCalendarEntry, JsContact}; +use crate::sync::pool::Pool; +use crate::sync::vanish::{Children, OnRowError, VanishedSweep}; + +use super::MAX_DAV_CONNECTIONS; +use super::collections::CollectionKind; + +const COMMIT_BATCH: usize = 64; +const METHOD_NOT_ALLOWED: u16 = 405; +const NOT_IMPLEMENTED: u16 = 501; + +pub(super) struct ItemSync<'a> { + pub(super) client: &'a DavClient, + pub(super) store: Store<'a>, + pub(super) source: SourceId, + pub(super) base_url: &'a str, + pub(super) multiget_batch: usize, + pub(super) dav_connections: usize, + pub(super) logger: Logger, +} + +impl ItemSync<'_> { + pub(super) fn reconcile( + &self, + kind: CollectionKind, + collection_href: &str, + container: LocalId, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let url = Href::join_absolute(self.base_url, collection_href).map_err(Error::partial)?; + let server = self.enumerate(&url)?; + let local = self + .store + .dav_ids(self.source) + .items_in(kind.item(), collection_href) + .map_err(Error::partial)?; + let collection = CollectionItems { + sync: self, + kind, + href: collection_href, + container, + url, + local, + }; + collection.reconcile(server, counts) + } + + fn enumerate(&self, url: &str) -> Result, Error> { + let ms = self + .client + .propfind_responses(url, 1, Propfind::DavItems.body(), url) + .map_err(Error::partial)?; + if DavOutcome::is_error(ms.status) { + return Err(Error::Partial(format!( + "enumerate {url}: http {}", + ms.status + ))); + } + let self_href = Href::parse(url, "") + .map(Href::into_string) + .unwrap_or_default(); + Ok(ms + .responses + .into_iter() + .filter(|r| r.href.as_str() != self_href && !r.props.is_collection) + .map(|r| ServerItem { + href: r.href.into_string(), + etag: r.props.etag.unwrap_or_default(), + }) + .collect()) + } +} + +#[derive(Debug, Clone)] +struct ServerItem { + href: String, + etag: String, +} + +struct MultigetJob { + body: String, + items: Vec, +} + +struct MultigetReply { + items: Vec, + result: Result, +} + +struct GetJob { + url: String, + item: ServerItem, +} + +struct GetReply { + item: ServerItem, + result: Result, +} + +struct CollectionItems<'a> { + sync: &'a ItemSync<'a>, + kind: CollectionKind, + href: &'a str, + container: LocalId, + url: String, + local: HashMap, +} + +impl CollectionItems<'_> { + fn reconcile(&self, server: Vec, counts: &mut TypeCounts) -> Result<(), Error> { + let server_hrefs: HashSet<&str> = server.iter().map(|s| s.href.as_str()).collect(); + let mut vanished: Vec<(String, LocalId)> = self + .local + .iter() + .filter(|(href, _)| !server_hrefs.contains(href.as_str())) + .map(|(href, item)| (href.clone(), item.local)) + .collect(); + vanished.sort_unstable(); + drop(server_hrefs); + let to_fetch: Vec = server + .into_iter() + .filter(|s| { + self.local + .get(&s.href) + .is_none_or(|known| s.etag.is_empty() || s.etag != known.etag) + }) + .collect(); + + if !to_fetch.is_empty() { + let fallback = self.multiget(to_fetch, counts)?; + if !fallback.is_empty() { + self.get_each(fallback, counts)?; + } + } + self.sweep(&vanished, counts)?; + + self.sync.logger.progress(format_args!( + "Collection {}: {} fetched, {} deleted, {} failed", + self.href, + Grouped(counts.fetched), + Grouped(counts.deleted), + Grouped(counts.failed) + )); + Ok(()) + } + + fn workers(&self) -> usize { + self.sync.dav_connections.clamp(1, MAX_DAV_CONNECTIONS) + } + + fn multiget( + &self, + to_fetch: Vec, + counts: &mut TypeCounts, + ) -> Result, Error> { + let logger = self.sync.logger; + let url = self.url.as_str(); + let batch_size = self.sync.multiget_batch.max(1); + let client = self.sync.client.clone(); + let pool_url = self.url.clone(); + let pool: Pool = + Pool::new(self.workers(), move |job: MultigetJob| { + let result = client.report_responses(&pool_url, 1, &job.body, &pool_url); + MultigetReply { + items: job.items, + result, + } + }); + let query = self.kind.multiget(); + let mut submitted = 0usize; + let mut to_fetch = to_fetch.into_iter(); + loop { + let items: Vec = to_fetch.by_ref().take(batch_size).collect(); + if items.is_empty() { + break; + } + pool.submit(MultigetJob { + body: query.body(items.iter().map(|s| s.href.as_str())), + items, + }); + submitted += 1; + } + + let mut fallback = Vec::new(); + for _ in 0..submitted { + let Ok(reply) = pool.results().recv() else { + break; + }; + match reply.result { + Err(DavError::HttpStatus { status, .. }) + if status == METHOD_NOT_ALLOWED || status == NOT_IMPLEMENTED => + { + logger.warn(format_args!( + "multiget {url} returned {status}; the server may not implement multiget, falling back to per-item GET" + )); + fallback.extend(reply.items); + } + Err(e) => { + logger.warn(format_args!( + "multiget {url}: {e}; falling back to per-item GET" + )); + fallback.extend(reply.items); + } + Ok(ms) if DavOutcome::is_error(ms.status) => { + logger.warn(format_args!( + "multiget {url} returned {}; chunk failed", + ms.status + )); + counts.failed += reply.items.len() as u64; + } + Ok(ms) => self.store_multiget(&reply.items, ms, counts)?, + } + } + Ok(fallback) + } + + fn store_multiget( + &self, + requested: &[ServerItem], + ms: MultiStatus, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let requested: HashSet<&str> = requested.iter().map(|s| s.href.as_str()).collect(); + let mut tx = self.sync.store.begin().map_err(Error::partial)?; + let mut pending = 0usize; + for response in ms.responses { + if !requested.contains(response.href.as_str()) { + self.sync.logger.warn(format_args!( + "multiget {}: server returned unsolicited href {}; skipping", + self.url, + response.href.as_str() + )); + continue; + } + self.store_response(tx.store(), &response, counts); + pending += 1; + if pending >= COMMIT_BATCH { + tx.commit().map_err(Error::partial)?; + tx = self.sync.store.begin().map_err(Error::partial)?; + pending = 0; + } + } + tx.commit().map_err(Error::partial) + } + + fn store_response(&self, store: Store<'_>, response: &DavResponse, counts: &mut TypeCounts) { + if response.is_gone() { + return; + } + let href = response.href.as_str(); + let etag = response.props.etag.as_deref().unwrap_or_default(); + let data = match self.kind { + CollectionKind::Calendar => response.props.calendar_data.as_deref(), + CollectionKind::AddressBook => response.props.address_data.as_deref(), + }; + let Some(raw) = data else { + self.sync.logger.warn(format_args!( + "multiget {href}: missing {} payload", + self.kind.data_field() + )); + counts.failed += 1; + return; + }; + match self.write(store, href, etag, raw) { + Ok(()) => counts.fetched += 1, + Err(e) => { + self.sync.logger.warn(format_args!("item {href}: {e}")); + counts.failed += 1; + } + } + } + + fn get_each(&self, items: Vec, counts: &mut TypeCounts) -> Result<(), Error> { + let logger = self.sync.logger; + let client = self.sync.client.clone(); + let pool: Pool = Pool::new(self.workers(), move |job: GetJob| GetReply { + result: client.get(&job.url), + item: job.item, + }); + + let mut submitted = 0usize; + for item in items { + match Href::join_absolute(&self.url, &item.href) { + Ok(url) => { + pool.submit(GetJob { url, item }); + submitted += 1; + } + Err(e) => { + logger.warn(format_args!("bad item URL {}: {e}", item.href)); + counts.failed += 1; + } + } + } + + let mut tx = self.sync.store.begin().map_err(Error::partial)?; + let mut pending = 0usize; + for _ in 0..submitted { + let Ok(reply) = pool.results().recv() else { + break; + }; + let href = reply.item.href.as_str(); + let body = match reply.result { + Err(e) => { + logger.warn(format_args!("GET {href}: {e}")); + counts.failed += 1; + continue; + } + Ok(body) if DavOutcome::is_gone(body.status) => continue, + Ok(body) if DavOutcome::is_error(body.status) => { + logger.warn(format_args!("GET {href} returned HTTP {}", body.status)); + counts.failed += 1; + continue; + } + Ok(body) => body, + }; + let etag = body.etag.as_deref().unwrap_or(&reply.item.etag); + let raw = match str::from_utf8(&body.body) { + Ok(raw) => raw, + Err(e) => { + logger.warn(format_args!( + "GET {href} returned a body that is not UTF-8: {e}" + )); + counts.failed += 1; + continue; + } + }; + if let Err(e) = self.write(tx.store(), href, etag, raw) { + logger.warn(format_args!("item {href}: {e}")); + counts.failed += 1; + continue; + } + counts.fetched += 1; + pending += 1; + if pending >= COMMIT_BATCH { + tx.commit().map_err(Error::partial)?; + tx = self.sync.store.begin().map_err(Error::partial)?; + pending = 0; + } + } + tx.commit().map_err(Error::partial) + } + + fn write(&self, store: Store<'_>, href: &str, etag: &str, raw: &str) -> Result<(), Error> { + let existing = self.local.get(href).map(|item| item.local); + let local = match self.kind { + CollectionKind::Calendar => self.write_event(store, existing, href, raw), + CollectionKind::AddressBook => self.write_card(store, existing, href, raw), + }?; + let ids = store.dav_ids(self.sync.source); + match existing { + None => ids.insert(self.kind.item(), self.href, href, etag, local), + Some(_) => ids.update_etag(self.kind.item(), href, etag), + } + .map_err(Error::partial) + } + + fn write_event( + &self, + store: Store<'_>, + existing: Option, + href: &str, + raw: &str, + ) -> Result { + let entries = JsCalendarEntry::parse_all(raw) + .map_err(|e| Error::Partial(format!("iCalendar parse: {e}")))?; + let total = entries.len(); + let Some(mut first) = entries.into_iter().next() else { + return Err(Error::Partial( + "iCalendar contained no parseable entries".to_owned(), + )); + }; + let logger = self.sync.logger; + if total > 1 && logger.enabled(Level::Default) { + logger.warn(format_args!( + "{href}: iCalendar resource has {total} entries; storing the first ({}), \ + others dropped (recurrence overrides on the master are preserved)", + first.data_type.as_str(), + )); + } + let ExtractedFields { + is_draft, + use_default_alerts, + .. + } = first.take_extracted(); + let data = first.data.to_string(); + let record = CalendarEventRecord { + calendars: &[self.container], + is_draft, + use_default_alerts, + data_type: first.data_type, + data: &data, + }; + let events = store.calendar_events(); + match existing { + Some(id) => events.update(id, &record).map(|_| id), + None => events.insert(&record), + } + .map_err(Error::partial) + } + + fn write_card( + &self, + store: Store<'_>, + existing: Option, + href: &str, + raw: &str, + ) -> Result { + let card = JsContact::from_vcard(raw, href) + .map_err(|e| Error::Partial(format!("vCard parse: {e}")))?; + let data = card.data.to_string(); + let record = ContactCardRecord { + uid: &card.uid, + address_books: &[self.container], + data: &data, + }; + let cards = store.contact_cards(); + match existing { + Some(id) => cards.update(id, &record).map(|_| id), + None => cards.insert(&record), + } + .map_err(Error::partial) + } + + fn sweep(&self, vanished: &[(String, LocalId)], counts: &mut TypeCounts) -> Result<(), Error> { + let source = self.sync.source; + let item = self.kind.item(); + let sweep = VanishedSweep::new(self.sync.store, OnRowError::Abort, Children::Restrict); + for row in vanished { + sweep + .run( + item, + slice::from_ref(row), + counts, + |store, href, _| store.dav_ids(source).delete(item, href), + |_, _, _| {}, + ) + .map_err(Error::partial)?; + } + Ok(()) + } +} + +impl CollectionKind { + fn multiget(self) -> Multiget { + match self { + CollectionKind::Calendar => Multiget::Calendar, + CollectionKind::AddressBook => Multiget::Addressbook, + } + } + + fn data_field(self) -> &'static str { + match self { + CollectionKind::Calendar => "calendar_data", + CollectionKind::AddressBook => "address_data", + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::Archive; + use crate::archive::rows::{CalendarEventRecord, EventDataType}; + use crate::archive::sources::{SourceKey, SourceKind}; + use crate::net::auth::Auth; + use crate::net::http::{ClientOptions, TlsPolicy}; + use crate::net::retry::RetryPolicy; + use crate::types::ObjectType; + + #[test] + fn vanished_item_delete_failure_aborts_the_collection_after_committing_earlier_rows() { + let archive = Archive::open_in_memory().unwrap(); + let store = archive.store(); + let source = store + .sources() + .upsert( + &SourceKey { + kind: SourceKind::CalDav, + url: "https://x", + account_id: "u", + }, + Some("u"), + "u", + ) + .unwrap(); + let mut vanished = Vec::new(); + for name in ["a", "b", "c"] { + let id = store + .calendar_events() + .insert(&CalendarEventRecord { + calendars: &[], + is_draft: false, + use_default_alerts: false, + data_type: EventDataType::Event, + data: "{}", + }) + .unwrap(); + let href = format!("/cal/work/{name}.ics"); + store + .dav_ids(source) + .insert(ObjectType::CalendarEvent, "/cal/work/", &href, "\"v\"", id) + .unwrap(); + vanished.push((href, id)); + } + archive + .connection() + .execute_batch(&format!( + "CREATE TRIGGER refuse BEFORE DELETE ON calendar_events WHEN old.id = {} + BEGIN SELECT RAISE(ABORT, 'refused'); END", + vanished[1].1 + )) + .unwrap(); + let logger = Logger::new(Level::Quiet); + let client = DavClient::new( + Auth::Basic { + user: "u".into(), + password: "p".into(), + }, + ClientOptions { + retry: RetryPolicy::new(0), + tls: TlsPolicy::Verify, + logger, + }, + ); + let sync = ItemSync { + client: &client, + store, + source, + base_url: "https://x", + multiget_batch: 1, + dav_connections: 1, + logger, + }; + let items = CollectionItems { + sync: &sync, + kind: CollectionKind::Calendar, + href: "/cal/work/", + container: LocalId::from_raw(1), + url: String::new(), + local: HashMap::new(), + }; + let mut counts = TypeCounts::default(); + + let err = items.sweep(&vanished, &mut counts).unwrap_err(); + + assert!(matches!(err, Error::Partial(_)), "{err:?}"); + assert_eq!((counts.deleted, counts.failed), (1, 0)); + let ids = store.dav_ids(source); + let left: Vec = vanished + .iter() + .map(|(href, _)| { + ids.local_id(ObjectType::CalendarEvent, href) + .unwrap() + .is_some() + }) + .collect(); + assert_eq!(left, [false, true, true]); + } +} diff --git a/src/sync/import_maildir.rs b/src/import/dav/mod.rs similarity index 55% rename from src/sync/import_maildir.rs rename to src/import/dav/mod.rs index 2ebf43b..0012450 100644 --- a/src/sync/import_maildir.rs +++ b/src/import/dav/mod.rs @@ -4,9 +4,12 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +mod collections; pub mod coordinator; -pub mod keywords; -pub mod messages; +mod items; +pub mod replace; pub mod tree; -pub use coordinator::{MaildirImportConfig, run, run_reporting}; +pub use coordinator::{DavImport, DavKindArg}; + +const MAX_DAV_CONNECTIONS: usize = 8; diff --git a/src/import/dav/replace.rs b/src/import/dav/replace.rs new file mode 100644 index 0000000..7e9f7f4 --- /dev/null +++ b/src/import/dav/replace.rs @@ -0,0 +1,314 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::mem; + +use crate::archive::sources::SourceKey; +use crate::archive::{SourceId, Store}; +use crate::error::Error; +use crate::logging::Logger; +use crate::sync::{Summary, SummaryType}; +use crate::types::ObjectType; + +const PURGE_ORDER: [ObjectType; 5] = [ + ObjectType::CalendarEvent, + ObjectType::ContactCard, + ObjectType::Calendar, + ObjectType::AddressBook, + ObjectType::FileNode, +]; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct Removed { + pub calendar_events: u64, + pub contact_cards: u64, + pub calendars: u64, + pub address_books: u64, + pub file_nodes: u64, +} + +impl Removed { + pub fn other_sources( + store: Store<'_>, + key: &SourceKey<'_>, + logger: Logger, + ) -> Result { + let others = store.sources().others(key)?; + let mut removed = Removed::default(); + if others.is_empty() { + return Ok(removed); + } + let tx = store.begin()?; + for other in &others { + removed.remove_source(tx.store(), other.id)?; + logger.info(format_args!( + "Removed the {} source {} (account {})", + key.kind.label(), + other.url, + other.account_id + )); + } + tx.commit()?; + Ok(removed) + } + + pub fn add_to(mut self, summary: &mut Summary) { + for (ty, counts) in &mut summary.per_type { + if let SummaryType::Object(ty) = *ty + && let Some(n) = self.slot(ty) + { + counts.deleted += mem::take(n); + } + } + } + + fn remove_source(&mut self, store: Store<'_>, source: SourceId) -> rusqlite::Result<()> { + let ids = store.dav_ids(source); + for ty in PURGE_ORDER { + let n = ids.purge(ty)?; + if let Some(slot) = self.slot(ty) { + *slot += n; + } + } + store.sources().delete(source) + } + + fn slot(&mut self, ty: ObjectType) -> Option<&mut u64> { + match ty { + ObjectType::CalendarEvent => Some(&mut self.calendar_events), + ObjectType::ContactCard => Some(&mut self.contact_cards), + ObjectType::Calendar => Some(&mut self.calendars), + ObjectType::AddressBook => Some(&mut self.address_books), + ObjectType::FileNode => Some(&mut self.file_nodes), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::rows::{ + CalendarEventRecord, CalendarRecord, EventDataType, FileNodeRecord, NodeType, Subscribed, + }; + use crate::archive::sources::SourceKind; + use crate::archive::{Archive, LocalId}; + use crate::logging::Level; + + fn source<'k>( + archive: &Archive, + kind: SourceKind, + url: &'k str, + account_id: &'k str, + ) -> (SourceKey<'k>, SourceId) { + let key = SourceKey { + kind, + url, + account_id, + }; + let id = archive.store().sources().upsert(&key, None, "u").unwrap(); + (key, id) + } + + fn file_node( + archive: &Archive, + source: SourceId, + parent: Option, + name: &str, + href: &str, + ) -> LocalId { + let store = archive.store(); + let id = store + .file_nodes() + .insert(&FileNodeRecord { + parent, + node_type: NodeType::Directory, + blob: None, + target: None, + name, + media_type: None, + created: "2026-01-01T00:00:00Z", + modified: None, + subscribed: Subscribed::Yes, + role: None, + executable: false, + }) + .unwrap(); + store + .dav_ids(source) + .insert(ObjectType::FileNode, href, href, "", id) + .unwrap(); + id + } + + fn calendar_with_event(archive: &Archive, source: SourceId) { + let store = archive.store(); + let calendar = store + .calendars() + .insert(&CalendarRecord { + name: "Work", + description: None, + color: None, + sort_order: 0, + subscribed: Subscribed::Yes, + is_visible: true, + is_default: false, + include_in_availability: "all", + default_alerts_with_time: None, + default_alerts_without_time: None, + time_zone: None, + }) + .unwrap(); + let ids = store.dav_ids(source); + ids.insert(ObjectType::Calendar, "/c/", "/c/", "", calendar) + .unwrap(); + let event = store + .calendar_events() + .insert(&CalendarEventRecord { + calendars: &[calendar], + is_draft: false, + use_default_alerts: false, + data_type: EventDataType::Event, + data: "{}", + }) + .unwrap(); + ids.insert(ObjectType::CalendarEvent, "/c/", "/c/1.ics", "", event) + .unwrap(); + } + + fn count(archive: &Archive, table: &str) -> i64 { + archive + .connection() + .query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0)) + .unwrap() + } + + fn quiet() -> Logger { + Logger::new(Level::Quiet) + } + + #[test] + fn replacing_a_webdav_source_drops_the_old_tree_leaf_first() { + let archive = Archive::open_in_memory().unwrap(); + let (_, old) = source( + &archive, + SourceKind::WebDav, + "https://old", + "https://old/u/", + ); + let a = file_node(&archive, old, None, "a", "/a/"); + let b = file_node(&archive, old, Some(a), "b", "/a/b/"); + file_node(&archive, old, Some(b), "c", "/a/b/c/"); + let (new_key, new) = source( + &archive, + SourceKind::WebDav, + "https://new", + "https://new/u/", + ); + file_node(&archive, new, None, "a", "/a/"); + + let removed = Removed::other_sources(archive.store(), &new_key, quiet()).unwrap(); + + assert_eq!(removed.file_nodes, 3); + assert_eq!(count(&archive, "file_nodes"), 1); + assert_eq!(count(&archive, "sources"), 1); + assert_eq!(count(&archive, "sync_id_dav"), 1); + let kept: SourceId = archive + .connection() + .query_row("SELECT source_id FROM sync_id_dav", [], |r| r.get(0)) + .unwrap(); + assert_eq!(kept, new); + } + + #[test] + fn replacing_leaves_other_kinds_untouched() { + let archive = Archive::open_in_memory().unwrap(); + let (_, cal) = source( + &archive, + SourceKind::CalDav, + "https://old", + "https://old/u/", + ); + calendar_with_event(&archive, cal); + let (_, old_web) = source( + &archive, + SourceKind::WebDav, + "https://old", + "https://old/u/", + ); + file_node(&archive, old_web, None, "a", "/a/"); + let (new_web, _) = source( + &archive, + SourceKind::WebDav, + "https://new", + "https://new/u/", + ); + + let removed = Removed::other_sources(archive.store(), &new_web, quiet()).unwrap(); + + assert_eq!(removed.file_nodes, 1); + assert_eq!(removed.calendars + removed.calendar_events, 0); + assert_eq!(count(&archive, "calendars"), 1); + assert_eq!(count(&archive, "calendar_events"), 1); + assert_eq!(count(&archive, "file_nodes"), 0); + } + + #[test] + fn replacing_a_caldav_source_drops_its_calendars_and_events() { + let archive = Archive::open_in_memory().unwrap(); + let (_, old) = source( + &archive, + SourceKind::CalDav, + "https://old", + "https://old/u/", + ); + calendar_with_event(&archive, old); + let (new_key, _) = source( + &archive, + SourceKind::CalDav, + "https://new", + "https://new/u/", + ); + + let removed = Removed::other_sources(archive.store(), &new_key, quiet()).unwrap(); + + assert_eq!((removed.calendars, removed.calendar_events), (1, 1)); + assert_eq!(count(&archive, "calendars"), 0); + assert_eq!(count(&archive, "calendar_events"), 0); + assert_eq!(count(&archive, "sync_id_dav"), 0); + } + + #[test] + fn no_other_source_is_a_no_op() { + let archive = Archive::open_in_memory().unwrap(); + let (key, _) = source( + &archive, + SourceKind::WebDav, + "https://only", + "https://only/u/", + ); + let removed = Removed::other_sources(archive.store(), &key, quiet()).unwrap(); + assert_eq!(removed, Removed::default()); + assert_eq!(count(&archive, "sources"), 1); + } + + #[test] + fn removed_counts_land_on_matching_summary_rows() { + let mut summary = Summary::default(); + summary.counts(ObjectType::FileNode); + Removed { + file_nodes: 4, + calendars: 9, + ..Removed::default() + } + .add_to(&mut summary); + assert_eq!(summary.per_type.len(), 1); + assert_eq!( + summary.get(ObjectType::FileNode).map(|c| c.deleted), + Some(4) + ); + } +} diff --git a/src/import/dav/tree.rs b/src/import/dav/tree.rs new file mode 100644 index 0000000..5877395 --- /dev/null +++ b/src/import/dav/tree.rs @@ -0,0 +1,723 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashSet, VecDeque}; +use std::io::Read; + +use rusqlite::OptionalExtension; +use time::format_description::well_known::Rfc3339 as Rfc3339Format; +use time::macros::format_description; +use time::{OffsetDateTime, PrimitiveDateTime}; + +use crate::archive::rows::{FileNodeRecord, NodeType, Subscribed}; +use crate::archive::{BlobId, LocalId, SourceId, Store}; +use crate::console::{Counted, Grouped}; +use crate::dav::client::{DavClient, DavStream}; +use crate::dav::discover::DiscoveredCollection; +use crate::dav::error::DavError; +use crate::dav::href::Href; +use crate::dav::parse::DavResponse; +use crate::dav::retry::DavOutcome; +use crate::dav::xml::Propfind; +use crate::error::Error; +use crate::logging::{Level, Logger}; +use crate::sync::TypeCounts; +use crate::sync::dates::MissingDate; +use crate::sync::pool::Pool; +use crate::sync::tree::FolderPath; +use crate::sync::vanish::{Children, OnRowError, RowFailure, VanishedSweep}; +use crate::types::ObjectType; + +use super::MAX_DAV_CONNECTIONS; + +const FILE_COMMIT_BATCH: usize = 16; +const UNNAMED: &str = "_"; +const HREF_DELIMITER: char = '/'; + +pub struct FileTree<'a> { + pub client: &'a DavClient, + pub store: Store<'a>, + pub source: SourceId, + pub base_url: &'a str, + pub dav_connections: usize, + pub logger: Logger, +} + +struct FilePlan { + item_href: String, + parent_href: String, + parent_local: Option, + existing_local: Option, + etag: String, + content_type: Option, + last_modified: Option, + creation_date: Option, + displayname: Option, + url: String, +} + +struct FileFetched { + plan: FilePlan, + result: Result, +} + +struct FetchedBody { + status: u16, + bytes: Vec, + etag: Option, + content_type: Option, + last_modified: Option, +} + +impl FetchedBody { + fn read(mut stream: DavStream) -> Result { + let mut bytes = Vec::new(); + stream + .read_to_end(&mut bytes) + .map_err(|e| DavError::Transport(format!("read body: {e}")))?; + Ok(FetchedBody { + status: stream.status, + bytes, + etag: stream.etag, + content_type: stream.content_type, + last_modified: stream.last_modified, + }) + } +} + +struct Directory { + url: String, + href: String, + local: Option, +} + +struct Walk { + seen: HashSet, + plans: Vec, +} + +impl FileTree<'_> { + pub fn reconcile( + &self, + root: &DiscoveredCollection, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let logger = self.logger; + let root_href = root.href.as_str(); + let known = self + .store + .dav_ids(self.source) + .all(ObjectType::FileNode) + .map_err(Error::partial)?; + let mut walk = Walk { + seen: HashSet::from([root_href.to_owned()]), + plans: Vec::new(), + }; + let mut visited = HashSet::from([root_href.to_owned()]); + let mut queue = VecDeque::from([Directory { + url: Href::join_absolute(self.base_url, root_href).map_err(Error::partial)?, + href: root_href.to_owned(), + local: None, + }]); + + while let Some(directory) = queue.pop_front() { + match self.list(&directory, &mut walk, counts) { + Ok(children) => { + for child in children { + if visited.insert(child.href.clone()) { + queue.push_back(child); + } else { + logger.warn(format_args!( + "cycle detected at {}; not recursing", + child.href + )); + } + } + } + Err(e) if e.aborts_run() => return Err(e), + Err(e) => { + logger.warn(format_args!("PROPFIND {} failed: {e}", directory.url)); + counts.failed += 1; + } + } + } + + self.fetch(walk.plans, counts)?; + + let mut vanished: Vec<(String, LocalId)> = known + .into_iter() + .filter(|(href, _)| !walk.seen.contains(href)) + .collect(); + self.sweep(&mut vanished, counts) + } + + fn list( + &self, + directory: &Directory, + walk: &mut Walk, + counts: &mut TypeCounts, + ) -> Result, Error> { + let url = directory.url.as_str(); + let ms = self + .client + .propfind_responses(url, 1, Propfind::WebdavListing.body(), url) + .map_err(Error::partial)?; + if DavOutcome::is_error(ms.status) { + return Err(Error::Partial(format!("http {}", ms.status))); + } + let self_href = Href::parse(url, "") + .map(Href::into_string) + .unwrap_or_default(); + let mut children = Vec::new(); + for response in ms.responses { + let href = response.href.as_str(); + if href == self_href || href == directory.href { + continue; + } + walk.seen.insert(href.to_owned()); + if response.props.is_collection { + let local = self.upsert_directory(&response, directory.local, counts)?; + children.push(Directory { + url: Href::join_absolute(url, href).map_err(Error::partial)?, + href: response.href.into_string(), + local: Some(local), + }); + } else { + let href = response.href.as_str().to_owned(); + match self.plan_file(response, directory) { + Ok(Some(plan)) => walk.plans.push(plan), + Ok(None) => {} + Err(e) => { + self.logger.warn(format_args!("file {href}: {e}")); + counts.failed += 1; + } + } + } + } + self.logger.progress(format_args!( + "Directory {url}: {} queued ({} fetched, {} failed so far)", + Counted::new(walk.plans.len() as u64, "file", "files"), + Grouped(counts.fetched), + Grouped(counts.failed) + )); + Ok(children) + } + + fn plan_file( + &self, + response: DavResponse, + directory: &Directory, + ) -> Result, Error> { + let item_href = response.href.into_string(); + let props = response.props; + let etag = props.etag.unwrap_or_default(); + let known = self + .store + .dav_ids(self.source) + .item(ObjectType::FileNode, &item_href) + .map_err(Error::partial)?; + if let Some(known) = &known + && !etag.is_empty() + && known.etag == etag + { + return Ok(None); + } + let url = Href::join_absolute(&directory.url, &item_href).map_err(Error::partial)?; + Ok(Some(FilePlan { + item_href, + parent_href: directory.href.clone(), + parent_local: directory.local, + existing_local: known.map(|known| known.local), + etag, + content_type: props.content_type, + last_modified: props.last_modified, + creation_date: props.creation_date, + displayname: props.displayname, + url, + })) + } + + fn upsert_directory( + &self, + response: &DavResponse, + parent: Option, + counts: &mut TypeCounts, + ) -> Result { + let href = response.href.as_str(); + let name = NodeHref(&response.href).directory_name(response.props.displayname.as_deref()); + let modified = DavDate(response.props.last_modified.as_deref()).normalised(); + let existing = self + .store + .dav_ids(self.source) + .local_id(ObjectType::FileNode, href) + .map_err(Error::partial)?; + let tx = self.store.begin().map_err(Error::partial)?; + let store = tx.store(); + let kept = match existing { + Some(local) => StoredNode(store).kept(local).map_err(Error::partial)?, + None => None, + }; + let created; + let record = match &kept { + Some(kept) => FileNodeRecord { + parent, + node_type: kept.node_type, + blob: kept.blob, + target: kept.target.as_deref(), + name: &name, + media_type: kept.media_type.as_deref(), + created: &kept.created, + modified: modified.as_deref(), + subscribed: kept.subscribed, + role: kept.role.as_deref(), + executable: kept.executable, + }, + None => { + created = DavDate(response.props.creation_date.as_deref()).or_now(); + FileNodeRecord { + parent, + node_type: NodeType::Directory, + blob: None, + target: None, + name: &name, + media_type: None, + created: &created, + modified: modified.as_deref(), + subscribed: Subscribed::Yes, + role: None, + executable: false, + } + } + }; + let nodes = store.file_nodes(); + let local = match existing { + Some(local) => { + nodes.update(local, &record).map_err(Error::partial)?; + counts.fetched += 1; + local + } + None => { + let local = nodes.insert(&record).map_err(Error::partial)?; + store + .dav_ids(self.source) + .insert( + ObjectType::FileNode, + response.href.parent().as_str(), + href, + "", + local, + ) + .map_err(Error::partial)?; + counts.created += 1; + local + } + }; + tx.commit().map_err(Error::partial)?; + Ok(local) + } + + fn fetch(&self, plans: Vec, counts: &mut TypeCounts) -> Result<(), Error> { + if plans.is_empty() { + return Ok(()); + } + let logger = self.logger; + let workers = self.dav_connections.clamp(1, MAX_DAV_CONNECTIONS); + let client = self.client.clone(); + let pool: Pool = Pool::new(workers, move |plan: FilePlan| { + let result = client.get_stream(&plan.url).and_then(FetchedBody::read); + FileFetched { plan, result } + }); + let submitted = plans.len(); + for plan in plans { + pool.submit(plan); + } + + let mut tx = self.store.begin().map_err(Error::partial)?; + let mut pending = 0usize; + for _ in 0..submitted { + let Ok(fetched) = pool.results().recv() else { + break; + }; + let href = fetched.plan.item_href.as_str(); + match fetched.result { + Err(e) => { + logger.warn(format_args!("file {href}: {e}")); + counts.failed += 1; + } + Ok(body) if DavOutcome::is_gone(body.status) => {} + Ok(body) if DavOutcome::is_error(body.status) => { + logger.warn(format_args!("GET {href} returned HTTP {}", body.status)); + counts.failed += 1; + } + Ok(body) => match self.commit_file(tx.store(), &fetched.plan, body) { + Ok(()) => { + counts.fetched += 1; + pending += 1; + if pending >= FILE_COMMIT_BATCH { + tx.commit().map_err(Error::partial)?; + tx = self.store.begin().map_err(Error::partial)?; + pending = 0; + } + } + Err(e) => { + logger.warn(format_args!("file {href}: {e}")); + counts.failed += 1; + } + }, + } + } + tx.commit().map_err(Error::partial) + } + + fn commit_file( + &self, + store: Store<'_>, + plan: &FilePlan, + body: FetchedBody, + ) -> Result<(), Error> { + let etag = body.etag.as_deref().unwrap_or(&plan.etag); + let media_type = body + .content_type + .as_deref() + .or(plan.content_type.as_deref()); + let modified = DavDate( + body.last_modified + .as_deref() + .or(plan.last_modified.as_deref()), + ) + .normalised(); + let created = DavDate(plan.creation_date.as_deref()).or_now(); + let name = plan.name(self.logger); + let blob = store.blobs().intern(&body.bytes).map_err(Error::partial)?; + let kept = match plan.existing_local { + Some(local) => StoredNode(store).kept(local).map_err(Error::partial)?, + None => None, + }; + let record = FileNodeRecord { + parent: plan.parent_local, + node_type: NodeType::File, + blob: Some(blob), + target: None, + name: &name, + media_type, + created: &created, + modified: modified.as_deref(), + subscribed: kept.as_ref().map_or(Subscribed::Yes, |k| k.subscribed), + role: kept.as_ref().and_then(|k| k.role.as_deref()), + executable: kept.as_ref().is_some_and(|k| k.executable), + }; + let nodes = store.file_nodes(); + let ids = store.dav_ids(self.source); + match plan.existing_local { + Some(local) => { + nodes.update(local, &record).map_err(Error::partial)?; + ids.update_etag(ObjectType::FileNode, &plan.item_href, etag) + } + None => { + let local = nodes.insert(&record).map_err(Error::partial)?; + ids.insert( + ObjectType::FileNode, + &plan.parent_href, + &plan.item_href, + etag, + local, + ) + } + } + .map_err(Error::partial) + } + + fn sweep( + &self, + vanished: &mut [(String, LocalId)], + counts: &mut TypeCounts, + ) -> Result<(), Error> { + if vanished.is_empty() { + return Ok(()); + } + FolderPath::sort_children_first_by(vanished, HREF_DELIMITER, |(href, _)| href); + let source = self.source; + let logger = self.logger; + VanishedSweep::new(self.store, OnRowError::Count, Children::Restrict) + .run( + ObjectType::FileNode, + vanished, + counts, + |store, href, _| store.dav_ids(source).delete(ObjectType::FileNode, href), + |href, _, failure| { + if let RowFailure::Sql(e) = failure { + logger.warn(format_args!( + "cannot delete vanished file node {href}: {e}; skipped" + )); + } + }, + ) + .map_err(Error::partial)?; + logger.progress(format_args!( + "Deleted {}", + Counted::new( + vanished.len() as u64, + "vanished file node", + "vanished file nodes" + ) + )); + Ok(()) + } +} + +impl FilePlan { + fn name(&self, logger: Logger) -> String { + let basename = NodeHref::basename_of(&self.item_href); + if let Some(displayname) = &self.displayname + && !displayname.trim().is_empty() + && displayname.trim() != basename + && logger.enabled(Level::Bodies) + { + logger.progress(format_args!( + "File {}: display name \"{displayname}\" differs from \"{basename}\"; using the latter", + self.item_href + )); + } + basename + } +} + +#[derive(Clone, Copy)] +struct NodeHref<'h>(&'h Href); + +impl NodeHref<'_> { + fn basename(self) -> String { + let component = self.0.last_component(); + if component.is_empty() { + UNNAMED.to_owned() + } else { + component + } + } + + fn basename_of(href: &str) -> String { + NodeHref(&Href::from_normalised(href.to_owned())).basename() + } + + fn directory_name(self, displayname: Option<&str>) -> String { + match displayname { + Some(name) if !name.trim().is_empty() => name.to_owned(), + _ => self.basename(), + } + } +} + +struct KeptNode { + node_type: NodeType, + blob: Option, + target: Option, + media_type: Option, + created: String, + subscribed: Subscribed, + role: Option, + executable: bool, +} + +#[derive(Clone, Copy)] +struct StoredNode<'a>(Store<'a>); + +impl StoredNode<'_> { + fn kept(self, id: LocalId) -> rusqlite::Result> { + self.0 + .connection() + .prepare_cached( + "SELECT node_type, blob_id, target, media_type, created, is_subscribed, role, + executable + FROM file_nodes WHERE id = ?1", + )? + .query_row([id], |row| { + Ok(KeptNode { + node_type: row.get(0)?, + blob: row.get(1)?, + target: row.get(2)?, + media_type: row.get(3)?, + created: row.get(4)?, + subscribed: row.get(5)?, + role: row.get(6)?, + executable: row.get(7)?, + }) + }) + .optional() + } +} + +#[derive(Clone, Copy)] +struct DavDate<'a>(Option<&'a str>); + +impl DavDate<'_> { + fn normalised(self) -> Option { + let raw = self.0?.trim(); + if raw.is_empty() { + return None; + } + if let Ok(t) = OffsetDateTime::parse(raw, &Rfc3339Format) { + return t.format(&Rfc3339Format).ok(); + } + let imf_fixdate = format_description!( + "[weekday repr:short], [day] [month repr:short] [year] \ + [hour]:[minute]:[second] GMT" + ); + PrimitiveDateTime::parse(raw, imf_fixdate) + .ok() + .and_then(|p| p.assume_utc().format(&Rfc3339Format).ok()) + } + + fn or_now(self) -> String { + self.normalised() + .unwrap_or_else(|| MissingDate::Now.resolve()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn basename_handles_percent_encoding() { + assert_eq!( + NodeHref::basename_of("/dav/file/u/work/note%201.txt"), + "note 1.txt" + ); + } + + fn plan_with(displayname: Option<&str>, href: &str) -> FilePlan { + FilePlan { + item_href: href.to_owned(), + parent_href: "/dav/file/u/".to_owned(), + parent_local: Some(LocalId::from_raw(1)), + existing_local: None, + etag: String::new(), + content_type: None, + last_modified: None, + creation_date: None, + displayname: displayname.map(str::to_owned), + url: format!("https://x{href}"), + } + } + + #[test] + fn file_name_prefers_the_url_basename_over_the_displayname() { + let plan = plan_with(Some("Pretty Name.txt"), "/dav/file/u/photo.jpg"); + assert_eq!(plan.name(Logger::new(Level::Default)), "photo.jpg"); + } + + #[test] + fn file_name_is_the_basename_without_a_displayname() { + let plan = plan_with(None, "/dav/file/u/photo.jpg"); + assert_eq!(plan.name(Logger::new(Level::Default)), "photo.jpg"); + } + + #[test] + fn directory_name_prefers_a_non_blank_displayname() { + let href = Href::from_normalised("/dav/file/u/work/".to_owned()); + assert_eq!(NodeHref(&href).directory_name(Some("Work")), "Work"); + assert_eq!(NodeHref(&href).directory_name(Some(" ")), "work"); + assert_eq!(NodeHref(&href).directory_name(None), "work"); + } + + #[test] + fn dav_date_parses_imf_fixdate() { + let n = DavDate(Some("Wed, 27 May 2026 07:59:07 GMT")) + .normalised() + .expect("imf parse"); + assert!(n.starts_with("2026-05-27T07:59:07")); + } + + #[test] + fn dav_date_is_none_on_garbage() { + assert!(DavDate(Some("not a date")).normalised().is_none()); + assert!(DavDate(None).normalised().is_none()); + } + + #[test] + fn directory_update_keeps_the_columns_dav_never_rewrites() { + use crate::archive::Archive; + use crate::archive::sources::{SourceKey, SourceKind}; + use crate::dav::parse::ResourceProps; + use crate::net::auth::Auth; + use crate::net::http::{ClientOptions, TlsPolicy}; + use crate::net::retry::RetryPolicy; + + let archive = Archive::open_in_memory().unwrap(); + let store = archive.store(); + let source = store + .sources() + .upsert( + &SourceKey { + kind: SourceKind::WebDav, + url: "https://x", + account_id: "u", + }, + Some("u"), + "u", + ) + .unwrap(); + let logger = Logger::new(Level::Quiet); + let client = DavClient::new( + Auth::Basic { + user: "u".into(), + password: "p".into(), + }, + ClientOptions { + retry: RetryPolicy::new(0), + tls: TlsPolicy::Verify, + logger, + }, + ); + let tree = FileTree { + client: &client, + store, + source, + base_url: "https://x/dav/", + dav_connections: 1, + logger, + }; + let response = DavResponse { + href: Href::from_normalised("/dav/docs/".to_owned()), + status: None, + props: ResourceProps::default(), + propstat_errors: Vec::new(), + }; + let local = tree + .upsert_directory(&response, None, &mut TypeCounts::default()) + .unwrap(); + archive + .connection() + .execute_batch( + "UPDATE file_nodes SET is_subscribed = 0, role = 'archive', executable = 1, + created = '2001-01-01T00:00:00Z'", + ) + .unwrap(); + + let again = tree + .upsert_directory(&response, None, &mut TypeCounts::default()) + .unwrap(); + + assert_eq!(again, local); + let kept: (bool, Option, bool, String) = archive + .connection() + .query_row( + "SELECT is_subscribed, role, executable, created FROM file_nodes", + [], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)), + ) + .unwrap(); + assert_eq!( + kept, + ( + false, + Some("archive".to_owned()), + true, + "2001-01-01T00:00:00Z".to_owned() + ) + ); + } +} diff --git a/src/import/ews/attachments.rs b/src/import/ews/attachments.rs new file mode 100644 index 0000000..a31f28b --- /dev/null +++ b/src/import/ews/attachments.rs @@ -0,0 +1,61 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use encodify::base64::LENIENT; + +use crate::exchange::ews::error::EwsError; +use crate::exchange::ews::parse::GetAttachmentInline; +use crate::exchange::ews::xml::EwsRequest; + +use super::items::ItemRunCtx; + +const FALLBACK_MEDIA_TYPE: &str = "application/octet-stream"; + +#[derive(Debug, Clone)] +pub struct FetchedAttachment { + pub attachment_id: String, + pub bytes: Vec, + pub media_type: String, +} + +impl ItemRunCtx<'_> { + pub fn fetch_attachment( + &self, + attachment_id: &str, + ) -> Result, EwsError> { + Ok(self.fetch_attachments(&[attachment_id])?.pop()) + } + + pub fn fetch_attachments( + &self, + attachment_ids: &[&str], + ) -> Result, EwsError> { + if attachment_ids.is_empty() { + return Ok(Vec::new()); + } + let mut out = Vec::with_capacity(attachment_ids.len()); + for chunk in attachment_ids.chunks(self.attachment_batch.max(1)) { + let body = EwsRequest::GetAttachment { ids: chunk }; + let resp = self.client.call(self.url, &body)?; + for att in GetAttachmentInline::parse_all(&resp.body)? { + let bytes = LENIENT.decode(&att.content_base64).map_err(|e| { + EwsError::Malformed(format!("attachment {}: base64: {e}", att.attachment_id)) + })?; + if bytes.is_empty() { + continue; + } + out.push(FetchedAttachment { + attachment_id: att.attachment_id, + bytes, + media_type: att + .content_type + .unwrap_or_else(|| FALLBACK_MEDIA_TYPE.to_owned()), + }); + } + } + Ok(out) + } +} diff --git a/src/import/ews/calendar.rs b/src/import/ews/calendar.rs new file mode 100644 index 0000000..f0fd58e --- /dev/null +++ b/src/import/ews/calendar.rs @@ -0,0 +1,171 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::{Map, Value}; + +use crate::archive::rows::{CalendarEventRecord, EventDataType}; +use crate::error::Error; +use crate::exchange::date::WireDate; +use crate::exchange::ews::parse::CalendarItemRaw; +use crate::exchange::ews::types::{CalendarItemType, ItemId}; +use crate::exchange::ews::xml::ItemShape; +use crate::sync::TypeCounts; +use crate::types::ObjectType; + +use super::coordinator::EwsCoordinator; +use super::folder_sync::Placement; + +impl EwsCoordinator<'_> { + pub fn import_event( + &self, + xml: &str, + at: &Placement<'_>, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let raw = CalendarItemRaw::parse(xml) + .map_err(|e| Error::Partial(format!("unparsable item: {e}")))?; + if raw.id.id.is_empty() { + counts.failed += 1; + return Ok(()); + } + if matches!( + raw.calendar_item_type, + Some(CalendarItemType::Occurrence | CalendarItemType::Exception) + ) || raw + .start + .as_deref() + .is_some_and(|s| WireDate(s).is_outlook_no_date()) + { + counts.skipped += 1; + return Ok(()); + } + let modified = self.fetch_modified_occurrences(&raw); + let mut built = raw.to_jscalendar_with_exceptions(&modified, self.items.owner_smtp); + self.attach_links(&raw, &mut built.data)?; + let calendars = [at.folder]; + let data = built.data.to_string(); + let record = CalendarEventRecord { + calendars: &calendars, + is_draft: built.is_draft, + use_default_alerts: built.use_default_alerts, + data_type: EventDataType::Event, + data: &data, + }; + + let tx = self.store.begin().map_err(Error::partial)?; + let store = tx.store(); + let ids = self.ids_in(store); + match at.existing.get(&raw.id.id) { + Some(&id) => { + store + .calendar_events() + .update(id, &record) + .map_err(Error::partial)?; + ids.update_change_key(ObjectType::CalendarEvent, &raw.id.id, &raw.id.change_key) + .map_err(Error::partial)?; + counts.fetched += 1; + } + None => { + let id = store + .calendar_events() + .insert(&record) + .map_err(Error::partial)?; + ids.insert( + ObjectType::CalendarEvent, + at.folder_id, + &raw.id.id, + &raw.id.change_key, + id, + ) + .map_err(Error::partial)?; + counts.created += 1; + } + } + tx.commit().map_err(Error::partial) + } + + fn fetch_modified_occurrences(&self, raw: &CalendarItemRaw) -> Vec { + let ids: Vec = raw + .modified_occurrences + .iter() + .filter(|o| !o.item_id.id.is_empty()) + .map(|o| o.item_id.clone()) + .collect(); + if ids.is_empty() { + return Vec::new(); + } + match self.items.get_items(ItemShape::CalendarItem, &ids) { + Ok(outcome) => outcome + .messages + .iter() + .filter(|m| m.success) + .filter_map(|m| CalendarItemRaw::parse(&m.inner_xml).ok()) + .filter(|occ| !occ.id.id.is_empty()) + .collect(), + Err(e) => { + self.items.logger.warn(format_args!( + "GetItem (calendar exceptions) failed: {e}; overrides keep time-only changes" + )); + Vec::new() + } + } + } + + fn attach_links(&self, raw: &CalendarItemRaw, data: &mut Value) -> Result<(), Error> { + let refs: Vec<_> = raw + .attachments + .iter() + .filter(|a| !a.is_item_attachment && !a.attachment_id.is_empty()) + .collect(); + if refs.is_empty() { + return Ok(()); + } + let ids: Vec<&str> = refs.iter().map(|a| a.attachment_id.as_str()).collect(); + let fetched = match self.items.fetch_attachments(&ids) { + Ok(fetched) => fetched, + Err(e) => { + self.items.logger.warn(format_args!( + "calendar attachments fetch failed: {e}; continuing without enclosures" + )); + return Ok(()); + } + }; + let mut links = Map::new(); + for (idx, att) in (1u32..).zip(fetched) { + let blob = self + .store + .blobs() + .intern(&att.bytes) + .map_err(Error::partial)?; + let declared = refs.iter().find(|r| r.attachment_id == att.attachment_id); + let content_type = declared + .and_then(|r| r.content_type.as_deref()) + .filter(|s| !s.is_empty()) + .map_or(att.media_type, str::to_owned); + let mut entry = Map::new(); + entry.insert("@type".to_owned(), Value::String("Link".to_owned())); + entry.insert("@blob".to_owned(), Value::from(blob.get())); + entry.insert("contentType".to_owned(), Value::String(content_type)); + if let Some(name) = declared.and_then(|r| r.name.clone()) { + entry.insert("title".to_owned(), Value::String(name)); + } + entry.insert("rel".to_owned(), Value::String("enclosure".to_owned())); + links.insert(idx.to_string(), Value::Object(entry)); + } + if links.is_empty() { + return Ok(()); + } + if let Some(obj) = data.as_object_mut() { + match obj.get_mut("links").and_then(Value::as_object_mut) { + Some(existing) => existing.extend(links), + None => { + obj.insert("links".to_owned(), Value::Object(links)); + } + } + } + Ok(()) + } +} diff --git a/src/import/ews/contacts.rs b/src/import/ews/contacts.rs new file mode 100644 index 0000000..6979b65 --- /dev/null +++ b/src/import/ews/contacts.rs @@ -0,0 +1,104 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::{Map, Value, json}; + +use crate::archive::rows::ContactCardRecord; +use crate::error::Error; +use crate::exchange::ews::parse::ContactItemRaw; +use crate::sync::TypeCounts; +use crate::types::ObjectType; + +use super::coordinator::EwsCoordinator; +use super::folder_sync::Placement; + +impl EwsCoordinator<'_> { + pub fn import_contact( + &self, + xml: &str, + at: &Placement<'_>, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let item = ContactItemRaw::parse(xml) + .map_err(|e| Error::Partial(format!("unparsable item: {e}")))?; + if item.id.id.is_empty() { + counts.failed += 1; + return Ok(()); + } + let mut card = item.to_jscontact(); + if let Some(photo) = item.attachments.iter().find(|a| a.is_contact_photo) + && let Err(e) = self.attach_photo(&photo.attachment_id, &mut card) + { + self.items + .logger + .warn(format_args!("contact photo fetch failed: {e}; continuing")); + } + let uid = ContactItemRaw::synthetic_uid(&item.id.id); + let books = [at.folder]; + let data = card.to_string(); + let record = ContactCardRecord { + uid: &uid, + address_books: &books, + data: &data, + }; + + let tx = self.store.begin().map_err(Error::partial)?; + let store = tx.store(); + let ids = self.ids_in(store); + match at.existing.get(&item.id.id) { + Some(&id) => { + store + .contact_cards() + .update(id, &record) + .map_err(Error::partial)?; + ids.update_change_key(ObjectType::ContactCard, &item.id.id, &item.id.change_key) + .map_err(Error::partial)?; + counts.fetched += 1; + } + None => { + let id = store + .contact_cards() + .insert(&record) + .map_err(Error::partial)?; + ids.insert( + ObjectType::ContactCard, + at.folder_id, + &item.id.id, + &item.id.change_key, + id, + ) + .map_err(Error::partial)?; + counts.created += 1; + } + } + tx.commit().map_err(Error::partial) + } + + fn attach_photo(&self, attachment_id: &str, card: &mut Value) -> Result<(), Error> { + let Some(photo) = self.items.fetch_attachment(attachment_id)? else { + return Ok(()); + }; + let blob = self + .store + .blobs() + .intern(&photo.bytes) + .map_err(Error::partial)?; + let mut media = Map::new(); + media.insert( + "photo".to_owned(), + json!({ + "@type": "Media", + "kind": "photo", + "@blob": blob.get(), + "mediaType": photo.media_type, + }), + ); + if let Some(m) = card.as_object_mut() { + m.insert("media".to_owned(), Value::Object(media)); + } + Ok(()) + } +} diff --git a/src/import/ews/coordinator.rs b/src/import/ews/coordinator.rs new file mode 100644 index 0000000..384525c --- /dev/null +++ b/src/import/ews/coordinator.rs @@ -0,0 +1,602 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fmt::Write as _; + +use crate::archive::ids::ews::EwsIds; +use crate::archive::sources::{SourceChange, SourceKey, SourceKind}; +use crate::archive::{Archive, LocalId, SourceId, Store}; +use crate::error::Error; +use crate::exchange::ews::EwsClient; +use crate::exchange::ews::autodiscover::{Autodiscover, DiscoveryResult}; +use crate::exchange::ews::oauth::{AcquiredToken, OAuthFlow, REFRESH_SCHEDULE}; +use crate::exchange::ews::types::MailboxKind; +use crate::exchange::oauth::{RefreshPlan, TokenRefresher}; +use crate::logging::Logger; +use crate::net::auth::Auth; +use crate::net::http::{ClientOptions, RetryStats}; +use crate::sync::run::{GcFailure, ImportRun, SourceClaim, SourceLabel}; +use crate::sync::vanish::{Children, OnRowError, VanishedSweep}; +use crate::sync::{RunOptions, RunOutcome, Summary, TypeCounts}; +use crate::types::ObjectType; + +use super::folder_sync::ItemKind; +use super::folders::FolderPlan; +use super::items::{ItemEnumeration, ItemRunCtx, SyncBatch}; + +const LABELS: [ObjectType; 6] = [ + ObjectType::Mailbox, + ObjectType::Calendar, + ObjectType::AddressBook, + ObjectType::Email, + ObjectType::CalendarEvent, + ObjectType::ContactCard, +]; +const DRY_RUN_LABELS: [ObjectType; 3] = [ + ObjectType::Mailbox, + ObjectType::Calendar, + ObjectType::AddressBook, +]; +const DEFAULT_PATH: &str = "/EWS/Exchange.asmx"; +const ENDPOINT: &str = "Exchange.asmx"; +const EWS_SEGMENT: &str = "EWS"; +const BASIC_AUTH_BLOCKED_HOSTS: [&str; 4] = [ + "outlook.office365.com", + "outlook.office.com", + "outlook.office365.us", + "office365.us", +]; + +#[derive(Debug, Clone)] +pub enum EwsAuth { + Basic { user: String, password: String }, + Bearer { token: String }, + OAuth(OAuthFlow), +} + +#[derive(Debug, Clone)] +pub struct EwsImport { + pub url: Option, + pub mailbox: Option, + pub mailbox_kind: MailboxKind, + pub auth: EwsAuth, + pub ews_connections: usize, + pub getitem_batch: usize, + pub attachment_batch: usize, + pub enumeration: ItemEnumeration, + pub source_change: SourceChange, +} + +pub struct EwsCoordinator<'a> { + pub store: Store<'a>, + pub source: SourceId, + pub items: ItemRunCtx<'a>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SessionUrl(String); + +#[derive(Default)] +struct EwsCounts([TypeCounts; LABELS.len()]); + +struct Session { + auth: Auth, + acquired: Option, + mailbox: String, + url: SessionUrl, +} + +impl<'a> EwsCoordinator<'a> { + pub fn ids(&self) -> EwsIds<'a> { + self.store.ews_ids(self.source) + } + + pub fn ids_in<'s>(&self, store: Store<'s>) -> EwsIds<'s> { + store.ews_ids(self.source) + } + + pub fn sweep>( + &self, + ty: ObjectType, + vanished: &[(K, LocalId)], + counts: &mut TypeCounts, + ) -> Result<(), Error> { + VanishedSweep::new(self.store, OnRowError::Count, Children::Restrict) + .run( + ty, + vanished, + counts, + |store, item_id, _| self.ids_in(store).delete(ty, item_id.as_ref()), + |_, _, _| {}, + ) + .map_err(Error::partial) + } +} + +impl EwsCounts { + fn split(&mut self, kind: ItemKind) -> (&mut TypeCounts, &mut TypeCounts) { + let [mailbox, calendar, addressbook, email, event, contact] = &mut self.0; + match kind { + ItemKind::Message => (mailbox, email), + ItemKind::Contact => (addressbook, contact), + ItemKind::Event => (calendar, event), + } + } + + fn store_into(self, summary: &mut Summary) { + for (label, counts) in LABELS.into_iter().zip(self.0) { + *summary.counts(label) = counts; + } + } +} + +impl SessionUrl { + pub fn normalise(raw: &str) -> SessionUrl { + let Ok(parsed) = url::Url::parse(raw) else { + return SessionUrl(raw.to_owned()); + }; + let host = parsed.host_str().unwrap_or("").to_ascii_lowercase(); + let mut out = String::with_capacity(raw.len()); + let _ = write!(out, "{}://{host}", parsed.scheme()); + if let Some(port) = parsed.port() { + let _ = write!(out, ":{port}"); + } + let path = parsed.path(); + match path.rsplit_once('/') { + None | Some(("", "")) => out.push_str(DEFAULT_PATH), + Some((dir, last)) if last.eq_ignore_ascii_case(ENDPOINT) => { + match dir.rsplit_once('/') { + Some((head, segment)) if segment.eq_ignore_ascii_case(EWS_SEGMENT) => { + let _ = write!(out, "{head}/{EWS_SEGMENT}/{ENDPOINT}"); + } + _ => { + let _ = write!(out, "{dir}/{ENDPOINT}"); + } + } + } + Some(_) => out.push_str(path), + } + SessionUrl(out) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl EwsAuth { + fn resolve(&self, options: ClientOptions) -> Result<(Auth, Option), Error> { + let acquired = match self { + EwsAuth::Basic { user, password } => { + return Ok(( + Auth::Basic { + user: user.clone(), + password: password.clone(), + }, + None, + )); + } + EwsAuth::Bearer { token } => AcquiredToken::from_bearer(token.clone()), + EwsAuth::OAuth(flow) => flow.acquire(options)?, + }; + Ok(( + Auth::Bearer { + token: acquired.access_token.clone(), + }, + Some(acquired), + )) + } + + fn check_basic_policy(&self, url: &str) -> Result<(), Error> { + let EwsAuth::Basic { .. } = self else { + return Ok(()); + }; + let parsed = url::Url::parse(url) + .map_err(|e| Error::Usage(format!("invalid EWS URL {url:?}: {e}")))?; + if parsed.scheme() != "https" { + return Err(Error::Connection( + "EWS requires https:// (basic auth refused on cleartext)".to_owned(), + )); + } + let host = parsed.host_str().unwrap_or("").to_ascii_lowercase(); + if BASIC_AUTH_BLOCKED_HOSTS + .iter() + .any(|blocked| host.ends_with(blocked)) + { + return Err(Error::Connection( + "Basic auth is disabled on Exchange Online; use --auth-bearer".to_owned(), + )); + } + Ok(()) + } +} + +impl Session { + fn username<'s>(&'s self, auth: &'s EwsAuth) -> &'s str { + match auth { + EwsAuth::Basic { user, .. } => user, + EwsAuth::Bearer { .. } | EwsAuth::OAuth(_) => self + .acquired + .as_ref() + .and_then(|a| a.upn.as_deref()) + .unwrap_or(&self.mailbox), + } + } +} + +impl EwsImport { + pub fn run(self, options: RunOptions) -> RunOutcome { + let prepared = Archive::open(&options.archive) + .map_err(Error::from) + .and_then(|archive| self.connect(&options).map(|session| (archive, session))); + match prepared { + Ok((archive, session)) => self.run_session(archive, session, options), + Err(error) => RunOutcome::failed(error), + } + } + + fn run_session(&self, archive: Archive, session: Session, options: RunOptions) -> RunOutcome { + let logger = options.logger; + let account_id = self.mailbox_kind.account_id(&session.mailbox).to_string(); + let key = SourceKey { + kind: SourceKind::ExchangeEws, + url: session.url.as_str(), + account_id: &account_id, + }; + let label = SourceLabel { + name: session.acquired.as_ref().and_then(|a| a.name.as_deref()), + username: session.username(&self.auth), + }; + let client_options = options.client_options(); + let mut run = match ImportRun::start_on(archive, options, &key, label, self.source_change) { + Ok(run) => run, + Err(error) => return RunOutcome::failed(error), + }; + + let Session { + auth, + acquired, + mailbox, + url, + } = session; + let client = EwsClient::new(auth, client_options); + let owner = (self.mailbox_kind != MailboxKind::PublicFolders).then_some(mailbox.as_str()); + client.set_anchor_mailbox(owner.map(str::to_owned)); + if let EwsAuth::OAuth(OAuthFlow::ClientCredentials { .. }) = &self.auth { + client.set_impersonation(Some(mailbox.clone())); + } + let _refresher = self.spawn_refresher(&client, acquired.as_ref(), client_options, logger); + let items = ItemRunCtx { + client: &client, + url: url.as_str(), + owner_smtp: owner, + batch_size: self.getitem_batch.max(1), + attachment_batch: self.attachment_batch.max(1), + connections: self.ews_connections, + enumeration: self.enumeration, + sync_batch: SyncBatch::MAX, + logger, + }; + + let source = match run.claim() { + SourceClaim::Registered(source) => *source, + SourceClaim::DryRun(_) => { + return match self.dry_run(&items) { + Ok(found) => { + for (ty, found) in DRY_RUN_LABELS.into_iter().zip(found) { + run.counts(ty).created = found; + run.summary().plan(ty).new = found; + } + run.finish(RetryStats::default(), GcFailure::Warn) + } + Err(error) => run.abort(error, RetryStats::default()), + }; + } + }; + let ctx = EwsCoordinator { + store: run.store(), + source, + items, + }; + let mut counts = EwsCounts::default(); + let result = self.reconcile(&ctx, &mut counts); + counts.store_into(run.summary()); + let stats = client.stats(); + match result { + Ok(()) => run.finish(stats, GcFailure::Warn), + Err(error) => run.abort(error, stats), + } + } + + fn reconcile(&self, ctx: &EwsCoordinator<'_>, counts: &mut EwsCounts) -> Result<(), Error> { + let items = &ctx.items; + let plan = FolderPlan::discover( + items.client, + items.url, + self.mailbox_kind, + self.mailbox.as_deref(), + items.logger, + )?; + for kind in [ItemKind::Message, ItemKind::Event, ItemKind::Contact] { + let (folders, members) = counts.split(kind); + ctx.reconcile_folders(kind, &plan, folders, members)?; + } + for kind in ItemKind::ALL { + ctx.reconcile_items(kind, &plan, counts.split(kind).1)?; + } + Ok(()) + } + + fn dry_run(&self, items: &ItemRunCtx<'_>) -> Result<[u64; 3], Error> { + let plan = FolderPlan::discover( + items.client, + items.url, + self.mailbox_kind, + self.mailbox.as_deref(), + items.logger, + )?; + Ok([plan.mail.len(), plan.calendar.len(), plan.contacts.len()].map(|n| n as u64)) + } + + fn connect(&self, options: &RunOptions) -> Result { + let (auth, acquired) = self.auth.resolve(options.client_options())?; + let discovery = self.autodiscover(acquired.as_ref(), options.client_options())?; + options.logger.progress(format_args!( + "EWS endpoint {} ({})", + discovery.ews_url, discovery.source + )); + let mailbox = self.resolve_mailbox(acquired.as_ref())?; + let url = SessionUrl::normalise(&discovery.ews_url); + self.auth.check_basic_policy(url.as_str())?; + Ok(Session { + auth, + acquired, + mailbox, + url, + }) + } + + fn autodiscover( + &self, + acquired: Option<&AcquiredToken>, + options: ClientOptions, + ) -> Result { + let email = self + .mailbox + .as_deref() + .or_else(|| acquired.and_then(|a| a.upn.as_deref())); + Ok(Autodiscover::new(options).discover(self.url.as_deref(), email, None)?) + } + + fn resolve_mailbox(&self, acquired: Option<&AcquiredToken>) -> Result { + if let Some(mailbox) = &self.mailbox { + return Ok(mailbox.clone()); + } + if let Some(upn) = acquired.and_then(|a| a.upn.as_ref()) { + return Ok(upn.clone()); + } + if let EwsAuth::Basic { user, .. } = &self.auth + && user.contains('@') + { + return Ok(user.clone()); + } + Err(Error::Account( + "could not resolve mailbox SMTP; pass --mailbox".to_owned(), + )) + } + + fn spawn_refresher( + &self, + client: &EwsClient, + initial: Option<&AcquiredToken>, + options: ClientOptions, + logger: Logger, + ) -> Option { + let EwsAuth::OAuth(flow) = &self.auth else { + return None; + }; + let initial = initial?; + let deadline_unix = initial.deadline_unix(TokenRefresher::unix_now())?; + let flow = flow.clone(); + let client = client.clone(); + TokenRefresher::spawn( + RefreshPlan { + label: "EWS", + schedule: REFRESH_SCHEDULE, + deadline_unix, + refresh_token: initial.refresh_token.clone(), + logger, + }, + move |refresh_token| flow.renew(options, refresh_token), + move |token| { + client.set_auth(Auth::Bearer { + token: token.access_token.clone(), + }) + }, + ) + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::*; + use crate::logging::Level; + use crate::net::http::TlsPolicy; + use crate::sync::{RunMode, SummaryType}; + + const NS: &str = " xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\" \ + xmlns:t=\"http://schemas.microsoft.com/exchange/services/2006/types\" \ + xmlns:m=\"http://schemas.microsoft.com/exchange/services/2006/messages\""; + + fn soap(server: &mut mockito::Server, action: &str, inner: &str) -> mockito::Mock { + server + .mock("POST", "/EWS/Exchange.asmx") + .match_header( + "soapaction", + mockito::Matcher::Regex(format!("/{action}\"$")), + ) + .with_status(200) + .with_header("content-type", "text/xml; charset=utf-8") + .with_body(format!( + "{inner}" + )) + .create() + } + + fn basic(user: &str) -> EwsAuth { + EwsAuth::Basic { + user: user.to_owned(), + password: "p".to_owned(), + } + } + + #[test] + fn basic_against_office365_endpoint_is_refused() { + let res = + basic("alice@x").check_basic_policy("https://outlook.office365.com/EWS/Exchange.asmx"); + assert!(matches!(res, Err(Error::Connection(_)))); + } + + #[test] + fn basic_over_https_on_prem_is_allowed() { + assert!( + basic("alice@x") + .check_basic_policy("https://exchange.example.com/EWS/Exchange.asmx") + .is_ok() + ); + } + + #[test] + fn basic_over_http_is_refused() { + assert!( + basic("alice") + .check_basic_policy("http://exchange.example.com/EWS/Exchange.asmx") + .is_err() + ); + } + + #[test] + fn normalise_ews_url_lowercases_host_and_keeps_canonical_path() { + let normalise = |raw| SessionUrl::normalise(raw).0; + assert_eq!( + normalise("https://OUTLOOK.OFFICE365.COM/EWS/Exchange.asmx"), + "https://outlook.office365.com/EWS/Exchange.asmx" + ); + assert_eq!( + normalise("https://outlook.office365.com:443/ews/exchange.asmx"), + "https://outlook.office365.com/EWS/Exchange.asmx" + ); + assert_eq!( + normalise("https://srv.example.com:8443/EWS/Exchange.asmx"), + "https://srv.example.com:8443/EWS/Exchange.asmx" + ); + assert_eq!( + normalise("https://srv.example.com"), + "https://srv.example.com/EWS/Exchange.asmx" + ); + assert_eq!( + normalise("https://srv.example.com/"), + "https://srv.example.com/EWS/Exchange.asmx" + ); + assert_eq!( + normalise("https://srv.example.com/Exchange.ASMX"), + "https://srv.example.com/Exchange.asmx" + ); + assert_eq!( + normalise("https://srv.example.com/a/ews/EXCHANGE.asmx"), + "https://srv.example.com/a/EWS/Exchange.asmx" + ); + assert_eq!( + normalise("https://srv.example.com/owa/x"), + "https://srv.example.com/owa/x" + ); + assert_eq!(normalise("not a url"), "not a url"); + } + + // A SQL failure on calendars after the mailboxes keeps the mailbox counts (R15). + #[test] + fn abort_after_mailboxes_keeps_the_mailbox_counts() { + let mut server = mockito::Server::new(); + let _find = soap( + &mut server, + "FindFolder", + "\ + NoError\ + \ + \ + InboxIPF.Note\ + \ + CalendarIPF.Appointment\ + ", + ); + let _get = soap( + &mut server, + "GetFolder", + "\ + NoError\ + \ + ", + ); + let archive = Archive::open_in_memory().unwrap(); + archive + .connection() + .execute_batch( + "CREATE TEMP TRIGGER no_calendars BEFORE INSERT ON calendars \ + BEGIN SELECT RAISE(ABORT, 'calendars refused'); END", + ) + .unwrap(); + let import = EwsImport { + url: None, + mailbox: Some("alice@x.com".to_owned()), + mailbox_kind: MailboxKind::Primary, + auth: EwsAuth::Bearer { + token: "t".to_owned(), + }, + ews_connections: 1, + getitem_batch: 10, + attachment_batch: 10, + enumeration: ItemEnumeration::FindItem, + source_change: SourceChange::Reject, + }; + let session = Session { + auth: Auth::Bearer { + token: "t".to_owned(), + }, + acquired: None, + mailbox: "alice@x.com".to_owned(), + url: SessionUrl(format!("{}/EWS/Exchange.asmx", server.url())), + }; + let options = RunOptions { + archive: PathBuf::new(), + threads: 1, + mode: RunMode::Apply, + max_retries: 0, + tls: TlsPolicy::Verify, + logger: Logger::new(Level::Quiet), + }; + let outcome = import.run_session(archive, session, options); + assert!( + matches!(outcome.error, Some(Error::Partial(_))), + "the calendar insert must abort the run: {:?}", + outcome.error + ); + let summary = &outcome.summary; + assert_eq!(summary.get(ObjectType::Mailbox).map(|c| c.created), Some(1)); + assert_eq!( + summary.get(ObjectType::Calendar).map(|c| c.created), + Some(0) + ); + assert!( + summary + .per_type + .iter() + .map(|(t, _)| *t) + .eq(LABELS.map(SummaryType::from)) + ); + } +} diff --git a/src/import/ews/folder_sync.rs b/src/import/ews/folder_sync.rs new file mode 100644 index 0000000..08bc1e6 --- /dev/null +++ b/src/import/ews/folder_sync.rs @@ -0,0 +1,230 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; +use std::fmt::Write as _; +use std::mem; + +use crate::archive::LocalId; +use crate::console::{Counted, Grouped}; +use crate::error::Error; +use crate::exchange::ews::parse::ResponseMessage; +use crate::exchange::ews::types::{FolderId, ResponseCode}; +use crate::exchange::ews::xml::ItemShape; +use crate::logging::Level; +use crate::sync::TypeCounts; +use crate::types::ObjectType; + +use super::coordinator::EwsCoordinator; +use super::folders::{ClassifiedFolder, FolderPlan}; +use super::items::{EnumerationMode, EnumerationOutcome}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ItemKind { + Message, + Contact, + Event, +} + +pub struct Placement<'p> { + pub folder: LocalId, + pub folder_id: &'p str, + pub existing: &'p HashMap, +} + +impl ItemKind { + pub const ALL: [ItemKind; 3] = [ItemKind::Message, ItemKind::Contact, ItemKind::Event]; + + pub fn container(self) -> ObjectType { + match self { + ItemKind::Message => ObjectType::Mailbox, + ItemKind::Contact => ObjectType::AddressBook, + ItemKind::Event => ObjectType::Calendar, + } + } + + pub fn item(self) -> ObjectType { + match self { + ItemKind::Message => ObjectType::Email, + ItemKind::Contact => ObjectType::ContactCard, + ItemKind::Event => ObjectType::CalendarEvent, + } + } + + fn shape(self) -> ItemShape { + match self { + ItemKind::Message => ItemShape::Message, + ItemKind::Contact => ItemShape::Contact, + ItemKind::Event => ItemShape::CalendarItem, + } + } + + pub fn folders(self, plan: &FolderPlan) -> &[ClassifiedFolder] { + match self { + ItemKind::Message => &plan.mail, + ItemKind::Contact => &plan.contacts, + ItemKind::Event => &plan.calendar, + } + } +} + +impl EwsCoordinator<'_> { + pub fn reconcile_items( + &self, + kind: ItemKind, + plan: &FolderPlan, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + for folder in kind.folders(plan) { + let entry = &folder.folder; + let Some(local) = self + .ids() + .local_id(kind.container(), &entry.folder_id.id) + .map_err(Error::partial)? + else { + continue; + }; + if let Err(e) = + self.reconcile_folder(kind, &entry.folder_id, &entry.display_name, local, counts) + { + self.items.logger.warn(format_args!( + "{} \"{}\" failed: {e}", + kind.container().noun(), + entry.display_name + )); + counts.failed += 1; + } + } + Ok(()) + } + + fn reconcile_folder( + &self, + kind: ItemKind, + folder: &FolderId, + folder_name: &str, + local_folder: LocalId, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let ids = self.ids(); + let prior = ids + .sync_state(kind.container(), &folder.id) + .map_err(Error::partial)?; + let mut outcome = self.items.enumerate(folder, prior.as_deref())?; + match kind { + ItemKind::Message => self.skip_non_mail(&mut outcome, folder_name, counts), + ItemKind::Event => outcome.items.retain(|s| { + hashify::set_ignore_case!(s.element.as_bytes(), "calendaritem", "item") + }), + ItemKind::Contact => {} + } + let local = ids + .items_in(kind.item(), &folder.id) + .map_err(Error::partial)?; + let mut plan = outcome.plan(&local); + let logger = self.items.logger; + match kind { + ItemKind::Message => logger.progress(format_args!( + "{} in \"{folder_name}\": {} new, {} changed, {} vanished, {} unchanged", + kind.item().label(), + Grouped(plan.new.len() as u64), + Grouped(plan.present_changed.len() as u64), + Grouped(plan.vanished.len() as u64), + Grouped(plan.present_unchanged as u64) + )), + ItemKind::Contact | ItemKind::Event => logger.progress(format_args!( + "{} in \"{folder_name}\": {} new, {} changed, {} vanished", + kind.item().label(), + Grouped(plan.new.len() as u64), + Grouped(plan.present_changed.len() as u64), + Grouped(plan.vanished.len() as u64) + )), + } + let vanished = mem::take(&mut plan.vanished); + let (to_fetch, existing) = plan.into_fetch(); + if !to_fetch.is_empty() { + let at = Placement { + folder: local_folder, + folder_id: &folder.id, + existing: &existing, + }; + let failed_items = + self.items + .for_each_fetched_item(kind.shape(), &to_fetch, |msg| { + self.import_item(kind, msg, &at, counts) + })?; + counts.failed += failed_items; + } + self.sweep(kind.item(), &vanished, counts)?; + if let EnumerationMode::Delta { new_sync_state, .. } = &outcome.mode { + ids.set_sync_state(kind.container(), &folder.id, new_sync_state) + .map_err(Error::partial)?; + } + Ok(()) + } + + fn import_item( + &self, + kind: ItemKind, + msg: ResponseMessage, + at: &Placement<'_>, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + if !msg.success { + if msg.response_code == ResponseCode::ItemNotFound { + counts.skipped += 1; + return Ok(()); + } + counts.failed += 1; + let logger = self.items.logger; + let noun = kind.item().noun(); + match kind { + ItemKind::Message => logger.warn(format_args!( + "GetItem ({noun}) failed: {} {}", + msg.response_code, msg.message_text + )), + ItemKind::Contact | ItemKind::Event => logger.warn(format_args!( + "GetItem ({noun}) failed: {}", + msg.response_code + )), + } + return Ok(()); + } + match kind { + ItemKind::Message => self.import_message(&msg.inner_xml, at, counts), + ItemKind::Contact => self.import_contact(&msg.inner_xml, at, counts), + ItemKind::Event => self.import_event(&msg.inner_xml, at, counts), + } + } + + fn skip_non_mail( + &self, + outcome: &mut EnumerationOutcome, + folder_name: &str, + counts: &mut TypeCounts, + ) { + let skipped = outcome.drop_non_mail(); + if skipped.is_empty() { + return; + } + let total: u64 = skipped.iter().map(|(_, n)| n).sum(); + counts.skipped += total; + let logger = self.items.logger; + if logger.enabled(Level::Default) { + let mut summary = String::new(); + for (class, n) in &skipped { + if !summary.is_empty() { + summary.push_str(", "); + } + let _ = write!(summary, "{} {class}", Grouped(*n)); + } + logger.info(format_args!( + "Skipped {} in \"{folder_name}\" ({summary})", + Counted::new(total, "non-mail item", "non-mail items") + )); + } + } +} diff --git a/src/import/ews/folders.rs b/src/import/ews/folders.rs new file mode 100644 index 0000000..fb3d333 --- /dev/null +++ b/src/import/ews/folders.rs @@ -0,0 +1,777 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; + +use crate::archive::ids::ews::FolderRow; +use crate::archive::rows::{AddressBookRecord, CalendarRecord, MailboxRecord, Subscribed}; +use crate::archive::{LocalId, Store}; +use crate::console::Counted; +use crate::error::Error; +use crate::exchange::ews::EwsClient; +use crate::exchange::ews::error::EwsError; +use crate::exchange::ews::parse::{ + FindFolderResponse, FindItemResponse, FolderEntry, ResponseMessage, +}; +use crate::exchange::ews::types::{DistinguishedFolderId, FolderClass, MailboxKind}; +use crate::exchange::ews::xml::{EwsRequest, FolderRef, FolderShape, Traversal}; +use crate::logging::Logger; +use crate::sync::TypeCounts; +use crate::sync::tree::{DanglingParent, ParentMap}; + +use super::coordinator::EwsCoordinator; +use super::folder_sync::ItemKind; + +const CLASS_SAMPLE_SIZE: u32 = 10; +const AVAILABILITY_ALL: &str = "all"; + +const WELL_KNOWN_ROLES: &[(DistinguishedFolderId, Option<&str>)] = &[ + (DistinguishedFolderId::Inbox, Some("inbox")), + (DistinguishedFolderId::SentItems, Some("sent")), + (DistinguishedFolderId::Drafts, Some("drafts")), + (DistinguishedFolderId::DeletedItems, Some("trash")), + (DistinguishedFolderId::JunkEmail, Some("junk")), + (DistinguishedFolderId::Archive, Some("archive")), + (DistinguishedFolderId::Outbox, None), + (DistinguishedFolderId::ConversationHistory, None), +]; + +pub struct FolderPlan { + pub mail: Vec, + pub calendar: Vec, + pub contacts: Vec, + pub well_known_roles: HashMap, + pub root_folder_id: String, +} + +#[derive(Debug, Clone)] +pub struct ClassifiedFolder { + pub folder: FolderEntry, + pub kind: FolderClass, +} + +struct Discovery<'a> { + client: &'a EwsClient, + url: &'a str, + mailbox_kind: MailboxKind, + mailbox: Option<&'a str>, + logger: Logger, +} + +enum FolderRecord<'r> { + Mailbox(MailboxRecord<'r>), + Calendar(CalendarRecord<'r>), + AddressBook(AddressBookRecord<'r>), +} + +impl FolderPlan { + pub fn discover( + client: &EwsClient, + url: &str, + mailbox_kind: MailboxKind, + mailbox: Option<&str>, + logger: Logger, + ) -> Result { + Discovery { + client, + url, + mailbox_kind, + mailbox: mailbox.filter(|_| mailbox_kind != MailboxKind::PublicFolders), + logger, + } + .plan() + } + + fn drop_hidden(mut folders: Vec) -> Vec { + if !folders.iter().any(|f| f.is_hidden) { + return folders; + } + let by_id: HashMap<&str, &FolderEntry> = folders + .iter() + .map(|f| (f.folder_id.id.as_str(), f)) + .collect(); + let keep: Vec = folders + .iter() + .map(|f| { + let mut current = Some(f); + for _ in 0..=by_id.len() { + let Some(folder) = current else { + return true; + }; + if folder.is_hidden { + return false; + } + current = folder + .parent_id + .as_deref() + .and_then(|p| by_id.get(p).copied()); + } + true + }) + .collect(); + let mut keep = keep.into_iter(); + folders.retain(|_| keep.next().unwrap_or(true)); + folders + } + + fn reparent_under_mail(mail: &mut [ClassifiedFolder], parents: &HashMap) { + let nearest: Vec> = { + let mail_ids: HashSet<&str> = mail + .iter() + .map(|f| f.folder.folder_id.id.as_str()) + .collect(); + mail.iter() + .map(|folder| { + let mut parent = folder.folder.parent_id.as_deref(); + for _ in 0..=parents.len() { + match parent { + Some(p) if !mail_ids.contains(p) => match parents.get(p) { + Some(grandparent) => parent = Some(grandparent), + None => break, + }, + _ => break, + } + } + parent.map(str::to_owned) + }) + .collect() + }; + for (folder, parent) in mail.iter_mut().zip(nearest) { + folder.folder.parent_id = parent; + } + } + + fn parents_first(folders: &[ClassifiedFolder]) -> Vec<&ClassifiedFolder> { + let mut tree = ParentMap::with_capacity(DanglingParent::Ignored, folders.len()); + tree.extend(folders.iter().map(|f| { + ( + f.folder.folder_id.id.as_str(), + f.folder.parent_id.as_deref(), + ) + })); + let mut ordered: Vec<&ClassifiedFolder> = folders.iter().collect(); + ordered.sort_by_cached_key(|f| tree.depth(&f.folder.folder_id.id.as_str())); + ordered + } + + fn parent_of( + &self, + folder: &ClassifiedFolder, + placed: &HashMap<&str, LocalId>, + ) -> Option { + let parent = folder.folder.parent_id.as_deref()?; + if parent == self.root_folder_id || parent.is_empty() { + return None; + } + placed.get(parent).copied() + } +} + +impl Discovery<'_> { + fn plan(&self) -> Result { + let request = EwsRequest::FindFolder { + parent: FolderRef::distinguished(self.mailbox_kind.distinguished_root(), self.mailbox), + traversal: Traversal::Deep, + }; + let resp = self.client.call(self.url, &request)?; + let parsed = FindFolderResponse::parse(&resp.body)?; + self.logger.progress(format_args!( + "FindFolder returned {}", + Counted::new(parsed.folders.len() as u64, "folder", "folders") + )); + let folders = FolderPlan::drop_hidden(parsed.folders); + let parents: HashMap = folders + .iter() + .filter_map(|f| Some((f.folder_id.id.clone(), f.parent_id.clone()?))) + .collect(); + let mut mail = Vec::new(); + let mut calendar = Vec::new(); + let mut contacts = Vec::new(); + for folder in folders { + let mut class = FolderClass::from_ipf(&folder.folder_class); + if class == FolderClass::Mail && FolderClass::is_mail_fallback(&folder.folder_class) { + let classless = folder.folder_class.trim().is_empty(); + if classless && self.mailbox_kind != MailboxKind::Archive { + class = self.sample_kind(&folder); + } + let kind = match class { + FolderClass::Calendar => "calendar", + FolderClass::Contacts => "contacts", + FolderClass::Mail | FolderClass::Skipped => "mail", + }; + let name = &folder.display_name; + if classless { + self.logger.note(format_args!( + "folder \"{name}\" has no FolderClass; importing it as {kind}" + )); + } else { + self.logger.note(format_args!( + "folder \"{name}\" has an unrecognized FolderClass \"{}\"; importing it as {kind}", + folder.folder_class + )); + } + } + let bucket = match (self.mailbox_kind, class) { + (_, FolderClass::Mail) => &mut mail, + (MailboxKind::Archive, _) | (_, FolderClass::Skipped) => continue, + (_, FolderClass::Calendar) => &mut calendar, + (_, FolderClass::Contacts) => &mut contacts, + }; + bucket.push(ClassifiedFolder { + folder, + kind: class, + }); + } + FolderPlan::reparent_under_mail(&mut mail, &parents); + Ok(FolderPlan { + mail, + calendar, + contacts, + well_known_roles: self.well_known_roles()?, + root_folder_id: self.root_folder_id()?, + }) + } + + fn sample_kind(&self, folder: &FolderEntry) -> FolderClass { + let request = EwsRequest::FindItem { + parent: FolderRef::Concrete(&folder.folder_id), + traversal: Traversal::Shallow, + offset: 0, + page_size: CLASS_SAMPLE_SIZE, + }; + let sampled = self + .client + .call(self.url, &request) + .and_then(|resp| FindItemResponse::parse(&resp.body)); + match sampled { + Ok(found) => FolderClass::from_item_classes( + found + .items + .iter() + .filter_map(|item| item.item_class.as_deref()), + ), + Err(e) => { + self.logger.progress(format_args!( + "FindItem sample of \"{}\" failed: {e}; treating it as mail", + folder.display_name + )); + FolderClass::Mail + } + } + } + + fn well_known_roles(&self) -> Result, EwsError> { + if self.mailbox_kind != MailboxKind::Primary { + return Ok(HashMap::new()); + } + let folders: Vec> = WELL_KNOWN_ROLES + .iter() + .map(|(d, _)| FolderRef::distinguished(*d, self.mailbox)) + .collect(); + let request = EwsRequest::GetFolder { + folders: &folders, + shape: FolderShape::IdOnly, + }; + let resp = match self.client.call(self.url, &request) { + Ok(r) => r, + Err(e) => { + self.logger.progress(format_args!( + "GetFolder for distinguished ids failed: {e}; roles will be empty" + )); + return Ok(HashMap::new()); + } + }; + let messages = + ResponseMessage::parse_all(&resp.body, "GetFolderResponseMessage").unwrap_or_default(); + let mut map = HashMap::new(); + for (msg, (_, role)) in messages.into_iter().zip(WELL_KNOWN_ROLES) { + if !msg.success { + continue; + } + let Ok(Some(entry)) = FolderEntry::parse_inner(&msg.inner_xml) else { + continue; + }; + if let Some(role) = role + && !entry.folder_id.id.is_empty() + { + map.insert(entry.folder_id.id, *role); + } + } + Ok(map) + } + + fn root_folder_id(&self) -> Result { + let request = EwsRequest::GetFolder { + folders: &[FolderRef::distinguished( + self.mailbox_kind.distinguished_root(), + self.mailbox, + )], + shape: FolderShape::IdOnly, + }; + let resp = self.client.call(self.url, &request)?; + Ok(FolderEntry::parse_get_response(&resp.body)? + .into_iter() + .next() + .map(|e| e.folder_id.id) + .unwrap_or_default()) + } +} + +impl<'r> FolderRecord<'r> { + fn new( + kind: ItemKind, + name: &'r str, + parent: Option, + role: Option<&'r str>, + ) -> FolderRecord<'r> { + match kind { + ItemKind::Message => FolderRecord::Mailbox(MailboxRecord { + name, + parent, + role, + sort_order: 0, + subscribed: Subscribed::Yes, + }), + ItemKind::Event => FolderRecord::Calendar(CalendarRecord { + name, + description: None, + color: None, + sort_order: 0, + subscribed: Subscribed::Yes, + is_visible: true, + is_default: false, + include_in_availability: AVAILABILITY_ALL, + default_alerts_with_time: None, + default_alerts_without_time: None, + time_zone: None, + }), + ItemKind::Contact => FolderRecord::AddressBook(AddressBookRecord { + name, + description: None, + sort_order: 0, + is_default: false, + subscribed: Subscribed::Yes, + }), + } + } + + fn insert(&self, store: Store<'_>) -> rusqlite::Result { + match self { + FolderRecord::Mailbox(r) => store.mailboxes().insert(r), + FolderRecord::Calendar(r) => store.calendars().insert(r), + FolderRecord::AddressBook(r) => store.address_books().insert(r), + } + } + + fn update(&self, store: Store<'_>, id: LocalId) -> rusqlite::Result<()> { + match self { + FolderRecord::Mailbox(r) => store.mailboxes().update(id, r), + FolderRecord::Calendar(r) => store.calendars().update(id, r), + FolderRecord::AddressBook(r) => store.address_books().update(id, r), + } + .map(drop) + } +} + +impl EwsCoordinator<'_> { + pub fn reconcile_folders( + &self, + kind: ItemKind, + plan: &FolderPlan, + folder_counts: &mut TypeCounts, + item_counts: &mut TypeCounts, + ) -> Result<(), Error> { + let container = kind.container(); + let ids = self.ids(); + let local = ids.all(container).map_err(Error::partial)?; + let folders = FolderPlan::parents_first(kind.folders(plan)); + let mut placed: HashMap<&str, LocalId> = local + .iter() + .map(|(id, row)| (id.as_str(), row.local_id)) + .collect(); + for folder in &folders { + let id = folder.folder.folder_id.id.as_str(); + let (parent, role) = match kind { + ItemKind::Message => ( + plan.parent_of(folder, &placed), + plan.well_known_roles.get(id).copied(), + ), + ItemKind::Contact | ItemKind::Event => (None, None), + }; + let record = FolderRecord::new(kind, &folder.folder.display_name, parent, role); + let local_id = + self.upsert_folder(kind, folder, local.get(id), &record, folder_counts)?; + placed.insert(id, local_id); + } + + let on_server: HashSet<&str> = folders + .iter() + .map(|f| f.folder.folder_id.id.as_str()) + .collect(); + let mut vanished: Vec<(&str, LocalId)> = local + .iter() + .filter(|(id, _)| !on_server.contains(id.as_str())) + .map(|(id, row)| (id.as_str(), row.local_id)) + .collect(); + if vanished.is_empty() { + return Ok(()); + } + let mut tree = ParentMap::with_capacity(DanglingParent::Ignored, local.len()); + tree.extend( + local + .values() + .map(|row| (row.item_id.as_str(), Some(row.folder_id.as_str()))), + ); + tree.children_first(&mut vanished, |(id, _)| id); + let mut children = Vec::new(); + for (folder_id, _) in &vanished { + children.extend( + ids.items_in(kind.item(), folder_id) + .map_err(Error::partial)? + .into_iter() + .map(|row| (row.item_id, row.local_id)), + ); + } + self.sweep(kind.item(), &children, item_counts)?; + self.sweep(container, &vanished, folder_counts) + } + + fn upsert_folder( + &self, + kind: ItemKind, + folder: &ClassifiedFolder, + existing: Option<&FolderRow>, + record: &FolderRecord<'_>, + counts: &mut TypeCounts, + ) -> Result { + let container = kind.container(); + let entry = &folder.folder; + let tx = self.store.begin().map_err(Error::partial)?; + let store = tx.store(); + let ids = self.ids_in(store); + let local_id = match existing { + Some(row) => { + record.update(store, row.local_id).map_err(Error::partial)?; + ids.update_change_key(container, &entry.folder_id.id, &entry.folder_id.change_key) + .map_err(Error::partial)?; + counts.fetched += 1; + row.local_id + } + None => { + let id = record.insert(store).map_err(Error::partial)?; + ids.insert( + container, + entry.parent_id.as_deref().unwrap_or(""), + &entry.folder_id.id, + &entry.folder_id.change_key, + id, + ) + .map_err(Error::partial)?; + counts.created += 1; + id + } + }; + tx.commit().map_err(Error::partial)?; + Ok(local_id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::rows::EmailRecord; + use crate::archive::sources::{SourceKey, SourceKind}; + use crate::archive::{Archive, SourceId}; + use crate::exchange::ews::types::FolderId; + use crate::import::ews::items::{ItemEnumeration, ItemRunCtx, SyncBatch}; + use crate::logging::Level; + use crate::net::auth::Auth; + use crate::net::http::{ClientOptions, TlsPolicy}; + use crate::net::retry::RetryPolicy; + use crate::types::ObjectType; + + fn client() -> EwsClient { + EwsClient::new( + Auth::Bearer { + token: "t".to_owned(), + }, + ClientOptions { + retry: RetryPolicy::new(0), + tls: TlsPolicy::Verify, + logger: Logger::new(Level::Quiet), + }, + ) + } + + fn source(archive: &Archive) -> SourceId { + archive + .store() + .sources() + .upsert( + &SourceKey { + kind: SourceKind::ExchangeEws, + url: "https://x/EWS/Exchange.asmx", + account_id: "u@d", + }, + Some("u"), + "u@d", + ) + .unwrap() + } + + fn coordinator<'a>(archive: &'a Archive, client: &'a EwsClient) -> EwsCoordinator<'a> { + EwsCoordinator { + store: archive.store(), + source: source(archive), + items: ItemRunCtx { + client, + url: "https://x/EWS/Exchange.asmx", + owner_smtp: None, + batch_size: 10, + attachment_batch: 10, + connections: 1, + enumeration: ItemEnumeration::FindItem, + sync_batch: SyncBatch::MAX, + logger: Logger::new(Level::Quiet), + }, + } + } + + fn entry(id: &str, parent: &str, class: &str, hidden: bool) -> FolderEntry { + FolderEntry { + folder_id: FolderId { + id: id.to_owned(), + change_key: "CK".to_owned(), + }, + parent_id: Some(parent.to_owned()), + display_name: id.to_owned(), + folder_class: class.to_owned(), + is_hidden: hidden, + ..FolderEntry::default() + } + } + + fn classified(id: &str, parent: &str, kind: FolderClass) -> ClassifiedFolder { + ClassifiedFolder { + folder: entry(id, parent, "", false), + kind, + } + } + + fn plan(mail: Vec, calendar: Vec) -> FolderPlan { + FolderPlan { + mail, + calendar, + contacts: Vec::new(), + well_known_roles: HashMap::new(), + root_folder_id: "ROOT".to_owned(), + } + } + + fn count(archive: &Archive, table: &str) -> i64 { + archive + .connection() + .query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0)) + .unwrap() + } + + fn reconcile(ctx: &EwsCoordinator<'_>, plan: &FolderPlan) -> [TypeCounts; 2] { + let mut counts: [TypeCounts; 2] = Default::default(); + let [folders, items] = &mut counts; + for kind in [ItemKind::Message, ItemKind::Event] { + ctx.reconcile_folders(kind, plan, folders, items).unwrap(); + } + counts + } + + // A vanished folder takes its archived items with it, id rows included. + #[test] + fn vanished_folder_cascades_deletion_of_its_child_items() { + let archive = Archive::open_in_memory().unwrap(); + let client = client(); + let ctx = coordinator(&archive, &client); + reconcile( + &ctx, + &plan( + vec![classified("FOLDER", "ROOT", FolderClass::Mail)], + vec![], + ), + ); + let store = archive.store(); + let blob = store.blobs().intern(b"m").unwrap(); + let folder = ctx + .ids() + .local_id(ObjectType::Mailbox, "FOLDER") + .unwrap() + .unwrap(); + let email = store + .emails() + .insert(&EmailRecord { + blob, + received_at: "", + mailboxes: &[folder], + keywords: "[]", + message_match: None, + }) + .unwrap(); + ctx.ids() + .insert(ObjectType::Email, "FOLDER", "MAIL", "CK", email) + .unwrap(); + + let [folders, items] = reconcile(&ctx, &plan(vec![], vec![])); + assert_eq!(folders.deleted, 1); + assert_eq!(items.deleted, 1, "child email must be cascade-deleted"); + assert_eq!(count(&archive, "mailboxes"), 0); + assert_eq!(count(&archive, "emails"), 0); + assert_eq!(count(&archive, "sync_id_exchange_ews"), 0); + } + + // A vanished parent and child are both removed, the child first. + #[test] + fn vanished_subtree_is_deleted_leaf_first() { + let archive = Archive::open_in_memory().unwrap(); + let client = client(); + let ctx = coordinator(&archive, &client); + reconcile( + &ctx, + &plan( + vec![ + classified("CHILD", "PARENT", FolderClass::Mail), + classified("PARENT", "ROOT", FolderClass::Mail), + classified("GRAND", "CHILD", FolderClass::Mail), + ], + vec![], + ), + ); + let parent_of_child: Option = archive + .connection() + .query_row( + "SELECT parent_id FROM mailboxes WHERE name = 'CHILD'", + [], + |r| r.get(0), + ) + .unwrap(); + assert!(parent_of_child.is_some(), "parents are created first"); + let [folders, _] = reconcile(&ctx, &plan(vec![], vec![])); + assert_eq!((folders.deleted, folders.failed), (3, 0)); + assert_eq!(count(&archive, "mailboxes"), 0); + } + + #[test] + fn hidden_folders_are_dropped_with_their_descendants() { + let folders = vec![ + entry("CONTACTS", "ROOT", "IPF.Contact", false), + entry("RECIPIENT_CACHE", "CONTACTS", "IPF.Contact", true), + entry("YAMMER", "ROOT", "IPF.Note", true), + entry("YAMMER_FEEDS", "YAMMER", "IPF.Note", false), + entry("YAMMER_DEEP", "YAMMER_FEEDS", "IPF.Note", false), + entry("INBOX", "ROOT", "IPF.Note", false), + ]; + let kept: Vec = FolderPlan::drop_hidden(folders) + .into_iter() + .map(|f| f.folder_id.id) + .collect(); + assert_eq!(kept, ["CONTACTS", "INBOX"]); + } + + #[test] + fn mail_folder_is_filed_under_its_nearest_mail_ancestor() { + let parents: HashMap = [ + ("PROJECTS", "ROOT"), + ("PCAL", "PROJECTS"), + ("SUB", "PCAL"), + ("CAL", "ROOT"), + ("NOTES", "CAL"), + ("INBOX", "ROOT"), + ("A", "B"), + ("B", "A"), + ("LOOP", "A"), + ] + .into_iter() + .map(|(id, parent)| (id.to_owned(), parent.to_owned())) + .collect(); + let mut mail = vec![ + classified("PROJECTS", "ROOT", FolderClass::Mail), + classified("SUB", "PCAL", FolderClass::Mail), + classified("NOTES", "CAL", FolderClass::Mail), + classified("INBOX", "ROOT", FolderClass::Mail), + classified("LOOP", "A", FolderClass::Mail), + ]; + FolderPlan::reparent_under_mail(&mut mail, &parents); + let parent_of = |id: &str| { + mail.iter() + .find(|f| f.folder.folder_id.id == id) + .and_then(|f| f.folder.parent_id.clone()) + }; + assert_eq!(parent_of("SUB").as_deref(), Some("PROJECTS")); + assert_eq!(parent_of("NOTES").as_deref(), Some("ROOT")); + assert_eq!(parent_of("INBOX").as_deref(), Some("ROOT")); + assert_ne!(parent_of("LOOP").as_deref(), Some("PROJECTS")); + } + + #[test] + fn classless_folder_kind_comes_from_its_items() { + let mut server = mockito::Server::new(); + let url = format!("{}/EWS/Exchange.asmx", server.url()); + let _m = server + .mock("POST", "/EWS/Exchange.asmx") + .with_status(200) + .with_header("content-type", "text/xml; charset=utf-8") + .with_body( + "\ + \ + NoError\ + \ + IPM.Contact\ + \ + ", + ) + .create(); + let client = client(); + let discovery = Discovery { + client: &client, + url: &url, + mailbox_kind: MailboxKind::Primary, + mailbox: None, + logger: Logger::new(Level::Default), + }; + assert_eq!( + discovery.sample_kind(&entry("LOCAL", "ROOT", "", false)), + FolderClass::Contacts + ); + } + + #[test] + fn rerun_does_not_take_a_calendar_local_id_as_mailbox_parent() { + let archive = Archive::open_in_memory().unwrap(); + let client = client(); + let ctx = coordinator(&archive, &client); + let plan = plan( + vec![ + classified("INBOX", "ROOT", FolderClass::Mail), + classified("NOTES", "CAL", FolderClass::Mail), + ], + vec![ + classified("CAL", "ROOT", FolderClass::Calendar), + classified("CAL2", "ROOT", FolderClass::Calendar), + ], + ); + for _ in 0..2 { + reconcile(&ctx, &plan); + let parent: Option = archive + .connection() + .query_row( + "SELECT parent_id FROM mailboxes WHERE name = 'NOTES'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(parent, None); + } + } +} diff --git a/src/import/ews/items.rs b/src/import/ews/items.rs new file mode 100644 index 0000000..7521f69 --- /dev/null +++ b/src/import/ews/items.rs @@ -0,0 +1,588 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; + +use crate::archive::LocalId; +use crate::archive::ids::ews::ItemRow; +use crate::console::Counted; +use crate::error::Error; +use crate::exchange::ews::EwsClient; +use crate::exchange::ews::error::EwsError; +use crate::exchange::ews::parse::{ + FindItemResponse, ResponseMessage, SyncChange, SyncFolderItemsResponse, +}; +use crate::exchange::ews::types::{FolderId, ItemId}; +use crate::exchange::ews::xml::{EwsRequest, FolderRef, ItemShape, Traversal}; +use crate::logging::Logger; +use crate::sync::pool::Pool; + +pub const MAX_EWS_CONNECTIONS: usize = 8; +const FIND_ITEM_PAGE_SIZE: u32 = 500; +const MAX_FIND_ITEM_OFFSET: u32 = 1_000_000; +const MAX_SYNC_ROUNDS: u32 = 200; +const GET_ITEM_RESPONSE: &str = "GetItemResponseMessage"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ItemEnumeration { + SyncFolderItems, + FindItem, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SyncBatch(u32); + +impl SyncBatch { + pub const MAX: SyncBatch = SyncBatch(512); + const MIN: SyncBatch = SyncBatch(32); + + pub fn new(changes: u32) -> SyncBatch { + SyncBatch(changes.clamp(SyncBatch::MIN.0, SyncBatch::MAX.0)) + } + + pub fn get(self) -> u32 { + self.0 + } + + pub fn shrink(self) -> SyncBatch { + SyncBatch((self.0 / 2).max(SyncBatch::MIN.0)) + } +} + +pub struct ItemRunCtx<'a> { + pub client: &'a EwsClient, + pub url: &'a str, + pub owner_smtp: Option<&'a str>, + pub batch_size: usize, + pub attachment_batch: usize, + pub connections: usize, + pub enumeration: ItemEnumeration, + pub sync_batch: SyncBatch, + pub logger: Logger, +} + +#[derive(Debug, Clone)] +pub struct EnumeratedItem { + pub element: String, + pub id: ItemId, + pub item_class: Option, +} + +#[derive(Debug, Clone)] +pub struct EnumerationOutcome { + pub items: Vec, + pub mode: EnumerationMode, +} + +#[derive(Debug, Clone)] +pub enum EnumerationMode { + Full, + Delta { + deletions: Vec, + new_sync_state: String, + }, +} + +#[derive(Debug, Clone, Default)] +pub struct DiffPlan { + pub new: Vec, + pub vanished: Vec<(String, LocalId)>, + pub present_changed: Vec<(ItemId, LocalId)>, + pub present_unchanged: usize, +} + +struct FetchedBatch { + len: usize, + result: Result, EwsError>, +} + +pub struct GetItemBatchOutcome { + pub messages: Vec, + pub failed_items: u64, +} + +impl ItemRunCtx<'_> { + fn batch(&self) -> usize { + self.batch_size.max(1) + } + + fn workers(&self) -> usize { + self.connections.clamp(1, MAX_EWS_CONNECTIONS) + } + + pub fn enumerate( + &self, + folder: &FolderId, + prior_sync_state: Option<&str>, + ) -> Result { + if self.enumeration == ItemEnumeration::SyncFolderItems + && let Some(state) = prior_sync_state + && !state.is_empty() + && let Some(outcome) = self.sync_folder_items(folder, state)? + { + return Ok(outcome); + } + self.find_items(folder).map(|items| EnumerationOutcome { + items, + mode: EnumerationMode::Full, + }) + } + + fn sync_folder_items( + &self, + folder: &FolderId, + prior_state: &str, + ) -> Result, EwsError> { + let mut sync_state = prior_state.to_owned(); + let mut items = Vec::new(); + let mut deletions = Vec::new(); + let mut rounds = 0; + let mut batch = self.sync_batch; + let version = self.client.server_version(); + loop { + let retries_before = self.client.stats().retries_observed; + let body = EwsRequest::SyncFolderItems { + folder, + sync_state: &sync_state, + max_changes: batch.get(), + version, + }; + let resp = match self.client.call(self.url, &body) { + Ok(r) => r, + Err(EwsError::SoapFault { code, .. }) if code.restarts_full_sync() => { + return Ok(None); + } + Err(e) => return Err(e), + }; + let parsed = SyncFolderItemsResponse::parse(&resp.body)?; + for change in parsed.changes { + match change { + SyncChange::Create { + id, + element, + item_class, + } + | SyncChange::Update { + id, + element, + item_class, + } => items.push(EnumeratedItem { + element, + id, + item_class, + }), + SyncChange::Delete { id } => deletions.push(id.id), + SyncChange::ReadFlagChange { id, .. } => items.push(EnumeratedItem { + element: "Message".to_owned(), + id, + item_class: None, + }), + } + } + if self.client.stats().retries_observed > retries_before { + let shrunk = batch.shrink(); + if shrunk != batch { + self.logger.warn(format_args!( + "SyncFolderItems throttled; shrinking the change batch from {} to {}", + batch.get(), + shrunk.get() + )); + batch = shrunk; + } + } + sync_state = parsed.sync_state; + if !parsed.more { + break; + } + rounds += 1; + if rounds > MAX_SYNC_ROUNDS { + break; + } + } + Ok(Some(EnumerationOutcome { + items, + mode: EnumerationMode::Delta { + deletions, + new_sync_state: sync_state, + }, + })) + } + + fn find_items(&self, folder: &FolderId) -> Result, EwsError> { + let mut items = Vec::new(); + let mut offset: u32 = 0; + loop { + let body = EwsRequest::FindItem { + parent: FolderRef::Concrete(folder), + traversal: Traversal::Shallow, + offset, + page_size: FIND_ITEM_PAGE_SIZE, + }; + let resp = self.client.call(self.url, &body)?; + let parsed = FindItemResponse::parse(&resp.body)?; + let returned_none = parsed.items.is_empty(); + items.extend(parsed.items.into_iter().map(|entry| EnumeratedItem { + element: entry.element, + id: entry.id, + item_class: entry.item_class, + })); + if !parsed.more || returned_none { + break; + } + offset = offset.saturating_add(FIND_ITEM_PAGE_SIZE); + if offset > MAX_FIND_ITEM_OFFSET { + break; + } + } + Ok(items) + } + + pub fn get_items( + &self, + shape: ItemShape, + ids: &[ItemId], + ) -> Result { + let batch = self.batch(); + let workers = self.workers(); + let version = self.client.server_version(); + let mut failed_items: u64 = 0; + let mut messages = Vec::new(); + if workers <= 1 || ids.len() <= batch { + for chunk in ids.chunks(batch) { + let body = EwsRequest::GetItem { + shape, + ids: chunk, + version, + }; + let parsed = self + .client + .call(self.url, &body) + .map_err(|e| (e, "GetItem batch failed")) + .and_then(|resp| { + ResponseMessage::parse_all(&resp.body, GET_ITEM_RESPONSE) + .map_err(|e| (e, "GetItem batch parse failed")) + }); + match parsed { + Ok(mut msgs) => messages.append(&mut msgs), + Err((e, what)) if e.is_per_batch() => { + self.logger.warn(format_args!( + "{what} ({e}); {} left for the next run", + Counted::new(chunk.len() as u64, "item", "items") + )); + failed_items += chunk.len() as u64; + } + Err((e, _)) => return Err(e), + } + } + return Ok(GetItemBatchOutcome { + messages, + failed_items, + }); + } + let client = self.client.clone(); + let url = self.url.to_owned(); + let pool: Pool, FetchedBatch> = + Pool::new(workers, move |chunk: Vec| { + let body = EwsRequest::GetItem { + shape, + ids: &chunk, + version, + }; + FetchedBatch { + len: chunk.len(), + result: client + .call(&url, &body) + .and_then(|resp| ResponseMessage::parse_all(&resp.body, GET_ITEM_RESPONSE)), + } + }); + let mut submitted = 0usize; + for chunk in ids.chunks(batch) { + pool.submit(chunk.to_vec()); + submitted += 1; + } + let mut abort = None; + for _ in 0..submitted { + let Ok(FetchedBatch { len: n, result }) = pool.results().recv() else { + break; + }; + match result { + Ok(mut msgs) => messages.append(&mut msgs), + Err(e) if e.is_per_batch() => { + self.logger.warn(format_args!( + "GetItem batch failed ({e}); {} left for the next run", + Counted::new(n as u64, "item", "items") + )); + failed_items += n as u64; + } + Err(e) => { + abort.get_or_insert(e); + } + } + } + match abort { + Some(e) => Err(e), + None => Ok(GetItemBatchOutcome { + messages, + failed_items, + }), + } + } + + pub fn for_each_fetched_item( + &self, + shape: ItemShape, + ids: &[ItemId], + mut on_message: impl FnMut(ResponseMessage) -> Result<(), Error>, + ) -> Result { + let batch = self.batch(); + let window = batch.saturating_mul(self.workers()).max(batch); + let mut failed_items = 0u64; + for win in ids.chunks(window) { + let outcome = self.get_items(shape, win)?; + failed_items = failed_items.saturating_add(outcome.failed_items); + for msg in outcome.messages { + if let Err(e) = on_message(msg) { + if e.aborts_run() { + return Err(e); + } + self.logger.warn(format_args!("EWS item not imported: {e}")); + failed_items = failed_items.saturating_add(1); + } + } + } + Ok(failed_items) + } +} + +impl EnumerationOutcome { + pub fn plan(&self, local: &[ItemRow]) -> DiffPlan { + match &self.mode { + EnumerationMode::Full => DiffPlan::full(&self.items, local), + EnumerationMode::Delta { deletions, .. } => { + DiffPlan::delta(&self.items, deletions, local) + } + } + } +} + +impl DiffPlan { + fn delta(changes: &[EnumeratedItem], deletions: &[String], local: &[ItemRow]) -> DiffPlan { + let local: HashMap<&str, &ItemRow> = + local.iter().map(|r| (r.item_id.as_str(), r)).collect(); + let mut plan = DiffPlan::default(); + for s in changes { + match local.get(s.id.id.as_str()) { + None => plan.new.push(s.id.clone()), + Some(row) => plan.present_changed.push((s.id.clone(), row.local_id)), + } + } + plan.vanished.extend( + deletions + .iter() + .filter_map(|d| local.get(d.as_str()).map(|row| (d.clone(), row.local_id))), + ); + plan + } + + pub fn full(server: &[EnumeratedItem], local: &[ItemRow]) -> DiffPlan { + let by_id: HashMap<&str, &ItemRow> = + local.iter().map(|r| (r.item_id.as_str(), r)).collect(); + let mut on_server = HashSet::with_capacity(server.len()); + let mut plan = DiffPlan::default(); + for s in server { + on_server.insert(s.id.id.as_str()); + match by_id.get(s.id.id.as_str()) { + None => plan.new.push(s.id.clone()), + Some(row) if s.id.change_key.is_empty() || s.id.change_key == row.change_key => { + plan.present_unchanged += 1; + } + Some(row) => plan.present_changed.push((s.id.clone(), row.local_id)), + } + } + plan.vanished.extend( + local + .iter() + .filter(|row| !on_server.contains(row.item_id.as_str())) + .map(|row| (row.item_id.clone(), row.local_id)), + ); + plan + } + + pub fn into_fetch(self) -> (Vec, HashMap) { + let mut to_fetch = self.new; + let mut existing = HashMap::with_capacity(self.present_changed.len()); + to_fetch.reserve(self.present_changed.len()); + for (id, local) in self.present_changed { + existing.insert(id.id.clone(), local); + to_fetch.push(id); + } + (to_fetch, existing) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::exchange::ews::parse::MessageItem; + use crate::logging::Level; + use crate::net::auth::Auth; + use crate::net::http::{ClientOptions, TlsPolicy}; + use crate::net::retry::RetryPolicy; + + const NS: &str = " xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\" \ + xmlns:t=\"http://schemas.microsoft.com/exchange/services/2006/types\" \ + xmlns:m=\"http://schemas.microsoft.com/exchange/services/2006/messages\""; + + fn fetch_three(fail_with: fn() -> Error) -> (Result, Vec) { + let mut server = mockito::Server::new(); + let url = format!("{}/EWS/Exchange.asmx", server.url()); + let messages: String = ["A", "B", "C"] + .iter() + .map(|id| { + format!( + "\ + NoError\ + \ + " + ) + }) + .collect(); + let _m = server + .mock("POST", "/EWS/Exchange.asmx") + .with_status(200) + .with_header("content-type", "text/xml; charset=utf-8") + .with_body(format!( + "\ + {messages}" + )) + .create(); + let client = EwsClient::new( + Auth::Bearer { + token: "t".to_owned(), + }, + ClientOptions { + retry: RetryPolicy::new(0), + tls: TlsPolicy::Verify, + logger: Logger::new(Level::Quiet), + }, + ); + let ctx = ItemRunCtx { + client: &client, + url: &url, + owner_smtp: None, + batch_size: 10, + attachment_batch: 10, + connections: 1, + enumeration: ItemEnumeration::FindItem, + sync_batch: SyncBatch::MAX, + logger: Logger::new(Level::Default), + }; + let ids: Vec = ["A", "B", "C"] + .iter() + .map(|id| ItemId::new(*id, "K")) + .collect(); + let mut seen = Vec::new(); + let result = ctx.for_each_fetched_item(ItemShape::Message, &ids, |msg| { + let parsed = MessageItem::parse(&msg.inner_xml).map_err(Error::partial)?; + seen.push(parsed.id.id.clone()); + if parsed.id.id == "A" { + return Err(fail_with()); + } + Ok(()) + }); + (result, seen) + } + + fn row(item_id: &str, change_key: &str, local_id: i64) -> ItemRow { + ItemRow { + item_id: item_id.to_owned(), + change_key: change_key.to_owned(), + local_id: LocalId::from_raw(local_id), + } + } + + fn message(id: &str, change_key: &str) -> EnumeratedItem { + EnumeratedItem { + element: "Message".to_owned(), + id: ItemId::new(id, change_key), + item_class: None, + } + } + + #[test] + fn per_item_failure_does_not_abort_the_folder() { + let (result, seen) = fetch_three(|| Error::Partial("UNIQUE constraint failed".to_owned())); + assert_eq!(result.unwrap(), 1); + assert_eq!(seen, ["A", "B", "C"]); + } + + #[test] + fn run_aborting_failure_still_stops() { + let (result, seen) = fetch_three(|| Error::Connection("gone".to_owned())); + assert!(result.is_err()); + assert_eq!(seen, ["A"]); + } + + #[test] + fn sync_batch_shrinks_by_half_down_to_floor() { + let shrink = |n| SyncBatch(n).shrink().get(); + assert_eq!(shrink(512), 256); + assert_eq!(shrink(256), 128); + assert_eq!(shrink(64), 32); + assert_eq!(shrink(32), 32); + assert_eq!(shrink(40), 32); + assert_eq!(SyncBatch::new(4096), SyncBatch::MAX); + assert_eq!(SyncBatch::new(1).get(), 32); + } + + #[test] + fn diff_delta_only_yields_changes_from_server_response() { + let outcome = EnumerationOutcome { + items: vec![message("A", "ck-2")], + mode: EnumerationMode::Delta { + deletions: vec!["Z".to_owned()], + new_sync_state: "STATE2".to_owned(), + }, + }; + let local = [ + row("A", "ck-1", 1), + row("B", "ck-1", 2), + row("Z", "ck-9", 99), + ]; + let plan = outcome.plan(&local); + assert!(plan.new.is_empty()); + assert_eq!(plan.present_changed.len(), 1); + assert_eq!(plan.present_changed[0].0.id, "A"); + assert_eq!(plan.vanished, [("Z".to_owned(), LocalId::from_raw(99))]); + assert_eq!(plan.present_unchanged, 0); + } + + #[test] + fn diff_splits_into_new_unchanged_changed_vanished() { + let server = [ + message("A", "ck-1"), + message("B", "ck-1"), + message("C", "ck-2"), + ]; + let local = [ + row("A", "ck-1", 1), + row("B", "ck-0", 2), + row("Z", "ck-9", 99), + ]; + let plan = DiffPlan::full(&server, &local); + assert_eq!(plan.new.len(), 1); + assert_eq!(plan.new[0].id, "C"); + assert_eq!(plan.present_unchanged, 1); + assert_eq!(plan.present_changed.len(), 1); + assert_eq!(plan.present_changed[0].0.id, "B"); + assert_eq!(plan.vanished.len(), 1); + assert_eq!(plan.vanished[0].0, "Z"); + let (to_fetch, existing) = plan.into_fetch(); + let fetched: Vec<&str> = to_fetch.iter().map(|id| id.id.as_str()).collect(); + assert_eq!(fetched, ["C", "B"]); + assert_eq!(existing.get("B"), Some(&LocalId::from_raw(2))); + } +} diff --git a/src/import/ews/messages.rs b/src/import/ews/messages.rs new file mode 100644 index 0000000..6142d67 --- /dev/null +++ b/src/import/ews/messages.rs @@ -0,0 +1,210 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use encodify::base64::LENIENT; + +use crate::archive::rows::EmailRecord; +use crate::error::Error; +use crate::exchange::ews::parse::MessageItem; +use crate::exchange::ews::types::ItemClass; +use crate::sync::TypeCounts; +use crate::sync::dates::Rfc3339; +use crate::sync::emailmeta::EmailMeta; +use crate::sync::keywords::{Keywords, MessageFlags}; +use crate::types::ObjectType; + +use super::coordinator::EwsCoordinator; +use super::folder_sync::Placement; +use super::items::EnumerationOutcome; + +const FLAGGED: &str = "Flagged"; + +impl EwsCoordinator<'_> { + pub fn import_message( + &self, + xml: &str, + at: &Placement<'_>, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let item = + MessageItem::parse(xml).map_err(|e| Error::Partial(format!("unparsable item: {e}")))?; + if item.id.id.is_empty() { + counts.failed += 1; + return Ok(()); + } + let logger = self.items.logger; + let Some(mime_b64) = item.mime_content.as_ref() else { + counts.skipped += 1; + logger.warn(format_args!( + "message {} has no MimeContent; skipping", + item.id.id + )); + return Ok(()); + }; + let bytes = match LENIENT.decode(mime_b64) { + Ok(b) => b, + Err(e) => { + counts.failed += 1; + logger.warn(format_args!( + "message {} MimeContent base64 decode failed: {e}", + item.id.id + )); + return Ok(()); + } + }; + let message_match = EmailMeta::from_blob(&bytes).index.to_json(); + let keywords = item.keywords().to_json(); + let mailboxes = [at.folder]; + let received_at = item.date_time_received.as_deref().unwrap_or(Rfc3339::EPOCH); + + let tx = self.store.begin().map_err(Error::partial)?; + let store = tx.store(); + let ids = self.ids_in(store); + let record = EmailRecord { + blob: store.blobs().intern(&bytes).map_err(Error::partial)?, + received_at, + mailboxes: &mailboxes, + keywords: &keywords, + message_match: Some(&message_match), + }; + match at.existing.get(&item.id.id) { + Some(&id) => { + store.emails().update(id, &record).map_err(Error::partial)?; + ids.update_change_key(ObjectType::Email, &item.id.id, &item.id.change_key) + .map_err(Error::partial)?; + counts.fetched += 1; + } + None => { + let id = store.emails().insert(&record).map_err(Error::partial)?; + ids.insert( + ObjectType::Email, + at.folder_id, + &item.id.id, + &item.id.change_key, + id, + ) + .map_err(Error::partial)?; + counts.created += 1; + } + } + tx.commit().map_err(Error::partial) + } +} + +impl MessageItem { + fn keywords(&self) -> Keywords { + let mut keywords = MessageFlags { + read: self.is_read == Some(true), + draft: self.is_draft == Some(true), + flagged: self.flag_status.as_deref() == Some(FLAGGED), + } + .keywords(); + for category in &self.categories { + keywords.push_category(category); + } + keywords + } +} + +impl EnumerationOutcome { + pub fn drop_non_mail(&mut self) -> Vec<(String, u64)> { + let mut skipped: Vec<(String, u64)> = Vec::new(); + self.items.retain(|item| { + let Some(class) = item + .item_class + .as_deref() + .filter(|c| ItemClass(c).is_non_mail()) + else { + return true; + }; + match skipped + .iter_mut() + .find(|(c, _)| c.eq_ignore_ascii_case(class)) + { + Some((_, n)) => *n += 1, + None => skipped.push((class.to_owned(), 1)), + } + false + }); + skipped + } +} + +#[cfg(test)] +mod tests { + use crate::exchange::ews::parse::MessageItem; + use crate::exchange::ews::types::ItemId; + use crate::import::ews::items::{EnumeratedItem, EnumerationMode, EnumerationOutcome}; + + #[test] + fn non_mail_items_are_dropped_and_counted_per_class() { + let mut outcome = EnumerationOutcome { + items: [ + ("M1", Some("IPM.Note")), + ("C1", Some("IPM.Contact")), + ("R1", Some("IPM.Schedule.Meeting.Request")), + ("C2", Some("IPM.Contact")), + ("T1", Some("IPM.Task")), + ("P1", Some("IPM.Post")), + ("X1", None), + ] + .into_iter() + .map(|(id, class)| EnumeratedItem { + element: "Item".to_owned(), + id: ItemId::new(id, "K"), + item_class: class.map(str::to_owned), + }) + .collect(), + mode: EnumerationMode::Full, + }; + let skipped = outcome.drop_non_mail(); + let kept: Vec<&str> = outcome.items.iter().map(|i| i.id.id.as_str()).collect(); + assert_eq!(kept, ["M1", "R1", "P1", "X1"]); + assert_eq!( + skipped, + [("IPM.Contact".to_owned(), 2), ("IPM.Task".to_owned(), 1)] + ); + } + + #[test] + fn read_draft_flagged_and_categories_map_to_jmap_keywords() { + let item = MessageItem { + is_read: Some(true), + is_draft: Some(true), + flag_status: Some("Flagged".to_owned()), + categories: vec!["Red Category".to_owned(), "VIP".to_owned()], + ..MessageItem::default() + }; + assert_eq!( + item.keywords().to_json(), + r#"["$seen","$draft","$flagged","red_category","vip"]"# + ); + } + + #[test] + fn categories_become_valid_deduplicated_keywords() { + let item = MessageItem { + categories: vec![ + "Catégorie Rouge".to_owned(), + "catégorie rouge".to_owned(), + "$seen".to_owned(), + "( )".to_owned(), + ], + ..MessageItem::default() + }; + assert_eq!(item.keywords().to_json(), r#"["cat_gorie_rouge","seen"]"#); + } + + #[test] + fn unread_unflagged_message_has_no_seen_or_flagged() { + let item = MessageItem { + is_read: Some(false), + flag_status: Some("NotFlagged".to_owned()), + ..MessageItem::default() + }; + assert_eq!(item.keywords().to_json(), "[]"); + } +} diff --git a/src/sync/import_exchange_ews.rs b/src/import/ews/mod.rs similarity index 72% rename from src/sync/import_exchange_ews.rs rename to src/import/ews/mod.rs index 877a6b7..8e7b2c1 100644 --- a/src/sync/import_exchange_ews.rs +++ b/src/import/ews/mod.rs @@ -8,8 +8,10 @@ pub mod attachments; pub mod calendar; pub mod contacts; pub mod coordinator; +pub mod folder_sync; pub mod folders; pub mod items; pub mod messages; -pub use coordinator::{EwsAuth, EwsImportConfig, run}; +pub use coordinator::{EwsAuth, EwsImport}; +pub use items::ItemEnumeration; diff --git a/src/import/graph/calendar.rs b/src/import/graph/calendar.rs new file mode 100644 index 0000000..2b1e67c --- /dev/null +++ b/src/import/graph/calendar.rs @@ -0,0 +1,308 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashSet; +use std::mem; + +use encodify::base64::STANDARD; +use serde_json::{Map, Value, json}; + +use crate::archive::Store; +use crate::archive::rows::{CalendarEventRecord, EventDataType}; +use crate::console::{Counted, Grouped}; +use crate::error::Error; +use crate::exchange::graph::api::{ + Endpoints, PREFER_BODY_HTML, PREFER_BODY_TEXT, PREFER_TIMEZONE_UTC, +}; +use crate::exchange::graph::calendar_map::{ConvertedEvent, EventType, GraphEvent}; +use crate::exchange::graph::client::GraphClient; +use crate::exchange::graph::error::GraphError; +use crate::exchange::graph::types::EventBodyFormat; +use crate::sync::TypeCounts; +use crate::sync::diff::IdDiff; +use crate::sync::pool::Pool; +use crate::types::ObjectType; + +use super::coordinator::{CHUNK_SIZE, EventAttachments, GraphCoordinator}; +use super::folders::GraphFolder; +use super::series::ExceptionWindow; + +const FILE_ATTACHMENT: &str = "#microsoft.graph.fileAttachment"; +const DEFAULT_ATTACHMENT_NAME: &str = "attachment"; +const OCTET_STREAM: &str = "application/octet-stream"; + +struct EventAttachment { + name: String, + content_type: String, + bytes: Vec, +} + +impl EventAttachment { + fn fetch_all(client: &GraphClient, endpoints: &Endpoints, event_id: &str) -> Vec { + let Ok(body) = client.get_json_with_prefer(&endpoints.event_attachments(event_id), &[]) + else { + return Vec::new(); + }; + let Some(items) = body.get("value").and_then(Value::as_array) else { + return Vec::new(); + }; + items + .iter() + .filter_map(|item| { + let text = |key: &str| item.get(key).and_then(Value::as_str); + if text("@odata.type") != Some(FILE_ATTACHMENT) { + return None; + } + let bytes = STANDARD.decode(text("contentBytes")?).ok()?; + (!bytes.is_empty()).then(|| EventAttachment { + name: text("name").unwrap_or(DEFAULT_ATTACHMENT_NAME).to_owned(), + content_type: text("contentType").unwrap_or(OCTET_STREAM).to_owned(), + bytes, + }) + }) + .collect() + } + + fn link_all( + attachments: &[EventAttachment], + store: Store<'_>, + card: &mut Value, + ) -> rusqlite::Result<()> { + if attachments.is_empty() { + return Ok(()); + } + let blobs = store.blobs(); + let mut links = Map::with_capacity(attachments.len()); + for (index, attachment) in (1u32..).zip(attachments) { + links.insert( + index.to_string(), + json!({ + "@type": "Link", + "@blob": blobs.intern(&attachment.bytes)?.get(), + "contentType": attachment.content_type, + "title": attachment.name, + "rel": "enclosure", + }), + ); + } + if let Some(map) = card.as_object_mut() { + match map.get_mut("links").and_then(Value::as_object_mut) { + Some(existing) => existing.extend(links), + None => { + map.insert("links".to_owned(), Value::Object(links)); + } + } + } + Ok(()) + } +} + +struct FetchedEvent { + graph_id: String, + event: Result, + attachments: Vec, +} + +struct NewEvent { + graph_id: String, + converted: ConvertedEvent, + attachments: Vec, +} + +impl GraphCoordinator<'_> { + pub fn reconcile_events( + &self, + calendars: &[GraphFolder], + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let local = self.known(ObjectType::CalendarEvent)?; + let windows = ExceptionWindow::around_today(self.exception_window_years); + let mut server_total: HashSet = HashSet::new(); + let mut any_failure = false; + + for calendar in calendars { + let url = self + .endpoints + .calendar_events_ids(&calendar.graph_id, self.top); + let stubs = match self.client.collect_all_values(&url, &[PREFER_TIMEZONE_UTC]) { + Ok(stubs) => stubs, + Err(e) => { + self.logger.warn(format_args!( + "calendar {} stub fetch failed: {e}", + calendar.graph_id + )); + counts.failed += 1; + any_failure = true; + continue; + } + }; + let mut wanted = Vec::new(); + let mut occurrences = 0usize; + let mut series_masters = Vec::new(); + for stub in &stubs { + let event_type = GraphEvent(stub).event_type(); + if event_type == EventType::Occurrence { + occurrences += 1; + continue; + } + let Some(id) = stub.get("id").and_then(Value::as_str) else { + continue; + }; + if event_type == EventType::SeriesMaster { + series_masters.push(id.to_owned()); + } + if local.contains_key(id) { + counts.fetched += 1; + server_total.insert(id.to_owned()); + } else if server_total.insert(id.to_owned()) { + wanted.push(id.to_owned()); + } + } + self.logger.progress(format_args!( + "Calendar {}: {} listed, {} new, {} skipped", + calendar.graph_id, + Counted::new(stubs.len() as u64, "event", "events"), + Grouped(wanted.len() as u64), + Counted::new(occurrences as u64, "occurrence", "occurrences"), + )); + + let mut converted = Vec::with_capacity(wanted.len()); + for fetched in self.fetch_events(wanted) { + let graph_id = fetched.graph_id; + match fetched.event { + Ok(raw) if GraphEvent(&raw).starts_on_outlook_no_date() => counts.skipped += 1, + Ok(raw) => match GraphEvent(&raw).convert(None) { + Ok(event) => converted.push(NewEvent { + graph_id, + converted: event, + attachments: fetched.attachments, + }), + Err(e) => { + counts.failed += 1; + self.logger.warn(format_args!( + "calendar event {graph_id} could not be converted: {e}" + )); + } + }, + Err(GraphError::Vanished) => counts.skipped += 1, + Err(e) => { + counts.failed += 1; + self.logger + .warn(format_args!("calendar event {graph_id} fetch failed: {e}")); + } + } + } + self.insert_events(calendar, converted, counts)?; + + if !series_masters.is_empty() { + self.expand_series(&calendar.graph_id, &series_masters, &windows, counts)?; + } + } + + if any_failure { + self.logger.warn(format_args!( + "vanished calendar events not removed: one or more calendars failed to enumerate; \ + a clean re-run will reconcile deletions" + )); + return Ok(()); + } + let vanished = IdDiff::compute(server_total, &local).vanished; + self.sweep(ObjectType::CalendarEvent, &vanished, counts) + } + + fn fetch_events(&self, ids: Vec) -> Vec { + let client = self.client.clone(); + let endpoints = self.endpoints.clone(); + let prefer = [ + PREFER_TIMEZONE_UTC, + match self.event_body_format { + EventBodyFormat::Text => PREFER_BODY_TEXT, + EventBodyFormat::Html => PREFER_BODY_HTML, + }, + ]; + let want_attachments = self.event_attachments == EventAttachments::Fetch; + let pool: Pool = Pool::new(self.workers, move |id: String| { + let event = client.get_json_with_prefer(&endpoints.event(&id), &prefer); + let attachments = match &event { + Ok(raw) + if want_attachments + && raw.get("hasAttachments").and_then(Value::as_bool) == Some(true) => + { + EventAttachment::fetch_all(&client, &endpoints, &id) + } + _ => Vec::new(), + }; + FetchedEvent { + graph_id: id, + event, + attachments, + } + }); + let total = ids.len(); + for id in ids { + pool.submit(id); + } + pool.results().iter().take(total).collect() + } + + fn insert_events( + &self, + calendar: &GraphFolder, + mut events: Vec, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let calendars = [calendar.local_id]; + for chunk in events.chunks_mut(CHUNK_SIZE) { + let tx = self.store.begin()?; + let store = tx.store(); + for event in chunk { + let mut card = mem::take(&mut event.converted.data); + EventAttachment::link_all(&event.attachments, store, &mut card)?; + let id = store.calendar_events().insert(&CalendarEventRecord { + calendars: &calendars, + is_draft: event.converted.is_draft, + use_default_alerts: event.converted.use_default_alerts, + data_type: EventDataType::Event, + data: &card.to_string(), + })?; + self.ids_in(store) + .insert(ObjectType::CalendarEvent, &event.graph_id, id)?; + counts.created += 1; + } + tx.commit()?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cancelled_occurrence_keys_drop_the_oid_prefix() { + let raw = json!({ + "id": "M", + "start": {"dateTime": "2026-09-21T11:00:00.0000000", "timeZone": "UTC"}, + "end": {"dateTime": "2026-09-21T11:30:00.0000000", "timeZone": "UTC"}, + "recurrence": { + "pattern": {"type": "daily", "interval": 1}, + "range": {"type": "numbered", "startDate": "2026-09-21", + "numberOfOccurrences": 6} + }, + "cancelledOccurrences": ["OID.AAMkAGI2TGuLAAA=.2026-09-24"] + }); + let converted = GraphEvent(&raw).convert(None).unwrap(); + let overrides = converted.data["recurrenceOverrides"].as_object().unwrap(); + assert!( + overrides.contains_key("2026-09-24T11:00:00"), + "beta cancelledOccurrences are OID..; the key must be a LocalDateTime, \ + got {:?}", + overrides.keys().collect::>() + ); + assert_eq!(overrides["2026-09-24T11:00:00"]["excluded"], true); + } +} diff --git a/src/import/graph/contacts.rs b/src/import/graph/contacts.rs new file mode 100644 index 0000000..1ff00da --- /dev/null +++ b/src/import/graph/contacts.rs @@ -0,0 +1,264 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashSet; +use std::mem; + +use serde_json::{Value, json}; + +use crate::archive::Store; +use crate::archive::rows::ContactCardRecord; +use crate::console::Counted; +use crate::error::Error; +use crate::exchange::graph::api::Endpoints; +use crate::exchange::graph::client::{Accept, GraphClient}; +use crate::exchange::graph::contact_map::{ConvertedContact, GraphContact}; +use crate::exchange::graph::error::GraphError; +use crate::sync::TypeCounts; +use crate::sync::diff::IdDiff; +use crate::sync::pool::Pool; +use crate::types::ObjectType; + +use super::coordinator::{CHUNK_SIZE, ContactPhotos, GraphCoordinator}; +use super::folders::GraphFolder; + +const OCTET_STREAM: &str = "application/octet-stream"; + +#[derive(Clone, Copy)] +struct ImageBytes<'a>(&'a [u8]); + +impl ImageBytes<'_> { + fn media_type(self) -> Option<&'static str> { + let bytes = self.0; + if bytes.starts_with(b"\x89PNG\r\n\x1a\n") { + Some("image/png") + } else if bytes.starts_with(&[0xFF, 0xD8, 0xFF]) { + Some("image/jpeg") + } else if bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a") { + Some("image/gif") + } else if bytes.starts_with(b"RIFF") && bytes.get(8..12) == Some(b"WEBP") { + Some("image/webp") + } else if bytes.starts_with(b"BM") { + Some("image/bmp") + } else { + None + } + } +} + +struct ContactPhoto { + bytes: Vec, + media_type: String, +} + +impl ContactPhoto { + fn fetch(client: &GraphClient, endpoints: &Endpoints, id: &str) -> Option { + let response = client + .get_with_prefer(&endpoints.contact_photo(id), Accept::Binary, &[]) + .ok()?; + if response.body.is_empty() { + return None; + } + let media_type = match ImageBytes(&response.body).media_type() { + Some(sniffed) => sniffed.to_owned(), + None => response + .content_type + .unwrap_or_else(|| OCTET_STREAM.to_owned()), + }; + Some(ContactPhoto { + bytes: response.body, + media_type, + }) + } + + fn attach(&self, store: Store<'_>, card: &mut Value) -> rusqlite::Result<()> { + let blob = store.blobs().intern(&self.bytes)?; + if let Some(map) = card.as_object_mut() { + map.insert( + "media".to_owned(), + json!({ + "photo": { + "@type": "Media", + "kind": "photo", + "@blob": blob.get(), + "mediaType": self.media_type, + } + }), + ); + } + Ok(()) + } +} + +struct FetchedContact { + graph_id: String, + contact: Result, + photo: Option, +} + +struct NewContact { + graph_id: String, + converted: ConvertedContact, + photo: Option, +} + +impl GraphCoordinator<'_> { + pub fn reconcile_contacts( + &self, + books: &[GraphFolder], + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let local = self.known(ObjectType::ContactCard)?; + let mut server_total: HashSet = HashSet::new(); + let mut any_failure = false; + + for book in books { + let url = self + .endpoints + .contact_folder_contacts_ids(&book.graph_id, self.top); + let ids = match self.client.collect_all_ids(&url, &[]) { + Ok(ids) => ids, + Err(e) => { + self.logger.warn(format_args!( + "address book {} enumeration failed: {e}", + book.graph_id + )); + counts.failed += 1; + any_failure = true; + continue; + } + }; + self.logger.progress(format_args!( + "Address book {}: {} listed", + book.graph_id, + Counted::new(ids.len() as u64, "contact card", "contact cards") + )); + let mut wanted = Vec::new(); + for id in ids { + if local.contains_key(&id) { + counts.fetched += 1; + server_total.insert(id); + } else if server_total.insert(id.clone()) { + wanted.push(id); + } + } + if wanted.is_empty() { + continue; + } + let mut converted = Vec::with_capacity(wanted.len()); + for fetched in self.fetch_contacts(wanted) { + let graph_id = fetched.graph_id; + match fetched.contact { + Ok(raw) => match GraphContact(&raw).convert() { + Ok(contact) => converted.push(NewContact { + graph_id, + converted: contact, + photo: fetched.photo, + }), + Err(e) => { + counts.failed += 1; + self.logger.warn(format_args!( + "contact card {graph_id} could not be converted: {e}" + )); + } + }, + Err(GraphError::Vanished) => counts.skipped += 1, + Err(e) => { + counts.failed += 1; + self.logger + .warn(format_args!("contact card {graph_id} fetch failed: {e}")); + } + } + } + self.insert_contacts(book, converted, counts)?; + } + + if any_failure { + self.logger.warn(format_args!( + "vanished contact cards not removed: one or more address books failed to \ + enumerate; a clean re-run will reconcile deletions" + )); + return Ok(()); + } + let vanished = IdDiff::compute(server_total, &local).vanished; + self.sweep(ObjectType::ContactCard, &vanished, counts) + } + + fn fetch_contacts(&self, ids: Vec) -> Vec { + let client = self.client.clone(); + let endpoints = self.endpoints.clone(); + let want_photos = self.contact_photos == ContactPhotos::Fetch; + let pool: Pool = Pool::new(self.workers, move |id: String| { + let contact = client.get_json_with_prefer(&endpoints.contact(&id), &[]); + let photo = if want_photos && contact.is_ok() { + ContactPhoto::fetch(&client, &endpoints, &id) + } else { + None + }; + FetchedContact { + graph_id: id, + contact, + photo, + } + }); + let total = ids.len(); + for id in ids { + pool.submit(id); + } + pool.results().iter().take(total).collect() + } + + fn insert_contacts( + &self, + book: &GraphFolder, + mut contacts: Vec, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let books = [book.local_id]; + for chunk in contacts.chunks_mut(CHUNK_SIZE) { + let tx = self.store.begin()?; + let store = tx.store(); + for contact in chunk { + let mut card = mem::take(&mut contact.converted.data); + if let Some(photo) = &contact.photo { + photo.attach(store, &mut card)?; + } + let id = store.contact_cards().insert(&ContactCardRecord { + uid: &contact.converted.uid, + address_books: &books, + data: &card.to_string(), + })?; + self.ids_in(store) + .insert(ObjectType::ContactCard, &contact.graph_id, id)?; + counts.created += 1; + } + tx.commit()?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn photo_type_comes_from_the_bytes_not_the_header() { + let png = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR"; + assert_eq!(ImageBytes(png).media_type(), Some("image/png")); + assert_eq!( + ImageBytes(&[0xFF, 0xD8, 0xFF, 0xE0]).media_type(), + Some("image/jpeg") + ); + assert_eq!(ImageBytes(b"GIF89a...").media_type(), Some("image/gif")); + assert_eq!( + ImageBytes(b"RIFF____WEBPVP8 ").media_type(), + Some("image/webp") + ); + assert_eq!(ImageBytes(b"RIFF").media_type(), None); + assert_eq!(ImageBytes(b"not an image").media_type(), None); + } +} diff --git a/src/import/graph/coordinator.rs b/src/import/graph/coordinator.rs new file mode 100644 index 0000000..d2356df --- /dev/null +++ b/src/import/graph/coordinator.rs @@ -0,0 +1,515 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; +use std::mem; + +use serde_json::Value; + +use crate::archive::ids::graph::GraphIds; +use crate::archive::sources::{SourceChange, SourceKey, SourceKind}; +use crate::archive::{Archive, LocalId, SourceId, Store}; +use crate::error::Error; +use crate::exchange::graph::ResolvedPrincipal; +use crate::exchange::graph::api::Endpoints; +use crate::exchange::graph::client::GraphClient; +use crate::exchange::graph::error::GraphError; +use crate::exchange::graph::oauth::{REFRESH_ATTEMPTS, REFRESH_SCHEDULE, SCOPES}; +use crate::exchange::graph::types::{EventBodyFormat, MailboxKind, Surfaces}; +use crate::exchange::oauth::{ + AcquiredToken, Authority, Dialect, OAuthClient, OAuthError, RefreshPlan, TokenRefresher, +}; +use crate::logging::Logger; +use crate::net::http::{ClientOptions, RetryStats}; +use crate::sync::run::{GcFailure, ImportRun, SourceClaim, SourceLabel}; +use crate::sync::tree::{DanglingParent, ParentMap}; +use crate::sync::vanish::{Children, OnRowError, RowFailure, VanishedSweep}; +use crate::sync::{RunOptions, RunOutcome, Summary, TypeCounts}; +use crate::types::ObjectType; + +#[derive(Debug, Clone)] +pub enum GraphAuth { + PreAcquired { + token: String, + }, + DeviceCode { + authority: String, + client_id: String, + }, +} + +#[derive(Debug, Clone)] +pub struct GraphImport { + pub auth: GraphAuth, + pub api_base: String, + pub user_target: Option, + pub mailbox_kind: MailboxKind, + pub surfaces: Surfaces, + pub event_body_format: EventBodyFormat, + pub graph_connections: usize, + pub top: usize, + pub exception_window_years: i32, + pub contact_photos: ContactPhotos, + pub event_attachments: EventAttachments, + pub source_change: SourceChange, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ContactPhotos { + Fetch, + Skip, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EventAttachments { + Fetch, + Skip, +} + +pub const CHUNK_SIZE: usize = 100; +pub const MAX_GRAPH_CONNECTIONS: usize = 16; +pub const MAX_PAGE_SIZE: usize = 1000; +const LABELS: [ObjectType; 7] = [ + ObjectType::Mailbox, + ObjectType::Email, + ObjectType::Calendar, + ObjectType::CalendarEvent, + ObjectType::AddressBook, + ObjectType::ContactCard, + ObjectType::FileNode, +]; +const DRY_RUN_LABELS: [ObjectType; 3] = [ + ObjectType::Mailbox, + ObjectType::Calendar, + ObjectType::AddressBook, +]; + +pub struct GraphCoordinator<'a> { + pub store: Store<'a>, + pub client: &'a GraphClient, + pub endpoints: &'a Endpoints, + pub source: SourceId, + pub top: usize, + pub workers: usize, + pub logger: Logger, + pub event_body_format: EventBodyFormat, + pub exception_window_years: i32, + pub contact_photos: ContactPhotos, + pub event_attachments: EventAttachments, +} + +impl<'a> GraphCoordinator<'a> { + pub fn ids(&self) -> GraphIds<'a> { + self.store.graph_ids(self.source) + } + + pub fn ids_in<'s>(&self, store: Store<'s>) -> GraphIds<'s> { + store.graph_ids(self.source) + } + + pub fn known(&self, ty: ObjectType) -> Result, Error> { + Ok(self.ids().all(ty)?) + } + + pub fn sweep( + &self, + ty: ObjectType, + vanished: &[(String, LocalId)], + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let noun = ty.noun(); + let hint = match ty { + ObjectType::Mailbox => " (live children?)", + _ => "", + }; + let logger = self.logger; + VanishedSweep::new(self.store, OnRowError::Count, Children::Restrict).run( + ty, + vanished, + counts, + |store, graph_id, _| self.ids_in(store).delete(ty, graph_id), + |graph_id, local_id, failure| match failure { + RowFailure::Sql(e) => logger.warn(format_args!( + "{noun} {graph_id} (local id {local_id}) could not be deleted{hint}: {e}" + )), + RowFailure::LiveChildren => logger.warn(format_args!( + "{noun} {graph_id} (local id {local_id}) could not be deleted{hint}" + )), + }, + )?; + Ok(()) + } + + pub fn parents(&self, ty: ObjectType) -> Result, Error> { + let mut stmt = self + .store + .connection() + .prepare(&format!("SELECT id, parent_id FROM {}", ty.table()))?; + let rows = stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?; + let mut parents = ParentMap::new(DanglingParent::Counts); + for row in rows { + let (id, parent) = row?; + parents.insert(id, parent); + } + Ok(parents) + } +} + +#[derive(Clone, Copy)] +struct UserTarget<'a>(&'a str); + +impl UserTarget<'_> { + fn is_uuid(self) -> bool { + self.0.len() == 36 + && self.0.chars().enumerate().all(|(i, c)| match i { + 8 | 13 | 18 | 23 => c == '-', + _ => c.is_ascii_hexdigit(), + }) + } +} + +#[derive(Default)] +struct SurfaceCounts([TypeCounts; LABELS.len()]); + +impl SurfaceCounts { + fn store_into(self, summary: &mut Summary) { + for (label, counts) in LABELS.into_iter().zip(self.0) { + *summary.counts(label) = counts; + } + } +} + +struct Session { + client: GraphClient, + endpoints: Endpoints, + principal: ResolvedPrincipal, + acquired: AcquiredToken, +} + +impl GraphImport { + pub fn run(self, options: RunOptions) -> RunOutcome { + let logger = options.logger; + let prepared = Archive::open(&options.archive) + .map_err(Error::from) + .and_then(|archive| self.connect(&options).map(|session| (archive, session))); + let (archive, session) = match prepared { + Ok(prepared) => prepared, + Err(error) => return RunOutcome::failed(error), + }; + let session_url = self.session_url(); + let account_id = self + .mailbox_kind + .account_id(&session.principal.id) + .to_string(); + let key = SourceKey { + kind: SourceKind::ExchangeGraph, + url: &session_url, + account_id: &account_id, + }; + let label = SourceLabel { + name: Some(&session.principal.user_principal_name), + username: &session.principal.user_principal_name, + }; + let refresh_options = options.client_options(); + let mut run = match ImportRun::start_on(archive, options, &key, label, self.source_change) { + Ok(run) => run, + Err(error) => return RunOutcome::failed(error), + }; + + if self.mailbox_kind == MailboxKind::Archive { + logger.warn(format_args!( + "--mailbox-kind archive: Microsoft Online Archive holds only mail; \ + calendar and contact surfaces are skipped for this run" + )); + } + + let source = match run.claim() { + SourceClaim::Registered(source) => *source, + SourceClaim::DryRun(_) => { + return match self.dry_run(&session) { + Ok(found) => { + for (ty, found) in DRY_RUN_LABELS.into_iter().zip(found) { + run.counts(ty).created = found; + run.summary().plan(ty).new = found; + } + run.finish(RetryStats::default(), GcFailure::Warn) + } + Err(error) => run.abort(error, RetryStats::default()), + }; + } + }; + let _refresher = + self.spawn_refresher(&session.client, &session.acquired, refresh_options, logger); + let ctx = GraphCoordinator { + store: run.store(), + client: &session.client, + endpoints: &session.endpoints, + source, + top: self.top.clamp(1, MAX_PAGE_SIZE), + workers: self.graph_connections.clamp(1, MAX_GRAPH_CONNECTIONS), + logger, + event_body_format: self.event_body_format, + exception_window_years: self.exception_window_years, + contact_photos: self.contact_photos, + event_attachments: self.event_attachments, + }; + let mut counts = SurfaceCounts::default(); + let result = self.reconcile(&ctx, &mut counts); + counts.store_into(run.summary()); + let stats = session.client.stats(); + match result { + Ok(()) => run.finish(stats, GcFailure::Warn), + Err(error) => run.abort(error, stats), + } + } + + fn reconcile( + &self, + ctx: &GraphCoordinator<'_>, + counts: &mut SurfaceCounts, + ) -> Result<(), Error> { + let primary = self.mailbox_kind != MailboxKind::Archive; + let [mailboxes, emails, calendars, events, books, contacts, files] = &mut counts.0; + if self.surfaces.mail { + let folders = ctx.reconcile_mail_folders(self.mailbox_kind, mailboxes)?; + ctx.reconcile_messages(&folders, emails)?; + } + if self.surfaces.calendar && primary { + let found = ctx.reconcile_calendars(calendars)?; + ctx.reconcile_events(&found, events)?; + } + if self.surfaces.contacts && primary { + let found = ctx.reconcile_address_books(books)?; + ctx.reconcile_contacts(&found, contacts)?; + } + if self.surfaces.files && primary { + ctx.reconcile_files(files)?; + } + Ok(()) + } + + fn dry_run(&self, session: &Session) -> Result<[u64; 3], Error> { + let client = &session.client; + let endpoints = &session.endpoints; + let mut found = [0u64; 3]; + let [mailboxes, calendars, books] = &mut found; + *mailboxes = + GraphImport::enumerate_mail_folders(client, endpoints, self.mailbox_kind, self.top)? + .len() as u64; + if self.mailbox_kind != MailboxKind::Archive { + *calendars = client + .collect_all_values(&endpoints.calendars(self.top), &[])? + .len() as u64; + *books = client + .collect_all_values(&endpoints.contact_folders(self.top), &[])? + .len() as u64; + } + Ok(found) + } + + pub fn enumerate_mail_folders( + client: &GraphClient, + endpoints: &Endpoints, + mailbox_kind: MailboxKind, + top: usize, + ) -> Result, GraphError> { + let mut all = Vec::new(); + let mut frontier: Vec = Vec::new(); + let mut seen: HashSet = HashSet::new(); + let mut visit = |level: Vec, frontier: &mut Vec| { + for folder in level { + let Some(id) = folder.get("id").and_then(Value::as_str) else { + continue; + }; + if seen.insert(id.to_owned()) { + frontier.push(id.to_owned()); + all.push(folder); + } + } + }; + let initial = endpoints.mail_folders_root(mailbox_kind, top); + visit(client.collect_all_values(&initial, &[])?, &mut frontier); + while let Some(parent) = frontier.pop() { + let url = endpoints.mail_folder_child_folders(&parent, top); + visit(client.collect_all_values(&url, &[])?, &mut frontier); + } + Ok(all) + } + + fn connect(&self, options: &RunOptions) -> Result { + let acquired = self.acquire(options)?; + let client = GraphClient::new(acquired.access_token.clone(), options.client_options()); + let endpoints = self.endpoints(&client)?; + let principal = ResolvedPrincipal::resolve(&client, &endpoints, acquired.upn.as_deref())?; + Ok(Session { + client, + endpoints, + principal, + acquired, + }) + } + + fn acquire(&self, options: &RunOptions) -> Result { + match &self.auth { + GraphAuth::PreAcquired { token } => Ok(AcquiredToken::from_bearer(token.clone())), + GraphAuth::DeviceCode { + authority, + client_id, + } => { + let authority = Authority::new(authority.clone()); + OAuthClient::new( + options.client_options(), + &authority, + client_id, + SCOPES, + Dialect::Graph, + ) + .device_code() + .map_err(|e| Error::from(GraphError::from(e))) + } + } + } + + fn endpoints(&self, client: &GraphClient) -> Result { + let Some(target) = self.user_target.as_deref() else { + return Ok(Endpoints::for_me(&self.api_base)); + }; + let resolved = if UserTarget(target).is_uuid() { + target.to_owned() + } else { + self.resolve_user_id(client, target) + .map_err(|e| Error::Connection(format!("user resolution: {e}")))? + }; + Ok(Endpoints::for_user(&self.api_base, &resolved)) + } + + fn resolve_user_id(&self, client: &GraphClient, upn_or_id: &str) -> Result { + let url = format!( + "{}/users/{}?$select=id,userPrincipalName", + self.api_base.trim_end_matches('/'), + upn_or_id + ); + let mut body = client.get_json(&url)?; + match body.get_mut("id") { + Some(Value::String(id)) => Ok(mem::take(id)), + _ => Err(GraphError::Malformed( + "user resolution missing id".to_owned(), + )), + } + } + + fn session_url(&self) -> String { + let authority = match &self.auth { + GraphAuth::DeviceCode { authority, .. } => Cow::Borrowed(authority.as_str()), + GraphAuth::PreAcquired { .. } => { + Cow::Owned(Authority::for_tenant("common").into_string()) + } + }; + format!( + "{}|{}", + authority.trim_end_matches('/'), + self.api_base.trim_end_matches('/') + ) + } + + fn spawn_refresher( + &self, + client: &GraphClient, + initial: &AcquiredToken, + options: ClientOptions, + logger: Logger, + ) -> Option { + let GraphAuth::DeviceCode { + authority, + client_id, + } = &self.auth + else { + return None; + }; + let refresh_token = initial.refresh_token.clone()?; + let deadline_unix = initial.deadline_unix(TokenRefresher::unix_now())?; + let authority = Authority::new(authority.clone()); + let client_id = client_id.clone(); + let client = client.clone(); + TokenRefresher::spawn( + RefreshPlan { + label: "graph", + schedule: REFRESH_SCHEDULE, + deadline_unix, + refresh_token: Some(refresh_token), + logger, + }, + move |refresh_token| { + let refresh_token = refresh_token.unwrap_or_default(); + OAuthClient::new(options, &authority, &client_id, SCOPES, Dialect::Graph) + .refresh(refresh_token, REFRESH_ATTEMPTS) + .map_err(|e| OAuthError(GraphError::from(e).to_string())) + }, + move |token| client.set_bearer(token.access_token.clone()), + ) + } +} + +impl ResolvedPrincipal { + fn resolve( + client: &GraphClient, + endpoints: &Endpoints, + fallback_upn: Option<&str>, + ) -> Result { + let mut value = client.get_json(&endpoints.me_select_id_upn())?; + let id = match value.get_mut("id") { + Some(Value::String(id)) => mem::take(id), + _ => return Err(Error::Connection("graph principal has no id".to_owned())), + }; + let user_principal_name = match value.get_mut("userPrincipalName") { + Some(Value::String(upn)) => mem::take(upn), + _ => fallback_upn.map(str::to_owned).unwrap_or_default(), + }; + Ok(ResolvedPrincipal { + id, + user_principal_name, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn canonical_session_url_concatenates_authority_and_base() { + let config = GraphImport { + auth: GraphAuth::DeviceCode { + authority: "https://login.microsoftonline.com/common".to_owned(), + client_id: "uuid".to_owned(), + }, + api_base: "https://graph.microsoft.com/v1.0".to_owned(), + user_target: None, + mailbox_kind: MailboxKind::Primary, + surfaces: Surfaces::ALL, + event_body_format: EventBodyFormat::Text, + graph_connections: 4, + top: 100, + exception_window_years: 5, + contact_photos: ContactPhotos::Fetch, + event_attachments: EventAttachments::Fetch, + source_change: SourceChange::Reject, + }; + let url = config.session_url(); + assert_eq!( + url, + "https://login.microsoftonline.com/common|https://graph.microsoft.com/v1.0" + ); + } + + #[test] + fn looks_like_uuid_only_matches_well_formed() { + assert!(UserTarget("12345678-1234-1234-1234-123456789abc").is_uuid()); + assert!(!UserTarget("alice@x.com").is_uuid()); + assert!(!UserTarget("12345678123412341234123456789abc").is_uuid()); + } +} diff --git a/src/import/graph/files.rs b/src/import/graph/files.rs new file mode 100644 index 0000000..bf66308 --- /dev/null +++ b/src/import/graph/files.rs @@ -0,0 +1,302 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; + +use serde_json::Value; + +use crate::archive::rows::{FileNodeRecord, NodeType, Subscribed}; +use crate::archive::{BlobId, LocalId}; +use crate::console::Counted; +use crate::error::Error; +use crate::exchange::graph::client::Accept; +use crate::exchange::graph::error::GraphError; +use crate::sync::TypeCounts; +use crate::sync::dates::Rfc3339; +use crate::sync::diff::IdDiff; +use crate::sync::pool::Pool; +use crate::sync::tree::{DanglingParent, ParentMap}; +use crate::types::ObjectType; + +use super::coordinator::{CHUNK_SIZE, GraphCoordinator}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DriveFacet { + Folder, + File, +} + +#[derive(Debug, Clone)] +struct DriveNode { + graph_id: String, + name: String, + facet: DriveFacet, + media_type: Option, + created: String, + modified: Option, + parent_graph_id: String, +} + +impl DriveNode { + fn parse(value: &Value, parent_graph_id: &str) -> Option { + let has = |facet: &str| value.get(facet).is_some_and(|f| !f.is_null()); + let facet = if has("folder") { + DriveFacet::Folder + } else if has("file") { + DriveFacet::File + } else { + return None; + }; + let text = |key: &str| value.get(key).and_then(Value::as_str); + Some(DriveNode { + graph_id: text("id")?.to_owned(), + name: text("name")?.to_owned(), + facet, + media_type: value + .get("file") + .and_then(|f| f.get("mimeType")) + .and_then(Value::as_str) + .map(str::to_owned), + created: text("createdDateTime").unwrap_or(Rfc3339::EPOCH).to_owned(), + modified: text("lastModifiedDateTime").map(str::to_owned), + parent_graph_id: parent_graph_id.to_owned(), + }) + } + + fn record<'r>(&'r self, parent: Option, blob: Option) -> FileNodeRecord<'r> { + let (node_type, media_type) = match self.facet { + DriveFacet::Folder => (NodeType::Directory, None), + DriveFacet::File => (NodeType::File, self.media_type.as_deref()), + }; + FileNodeRecord { + parent, + node_type, + blob, + target: None, + name: &self.name, + media_type, + created: &self.created, + modified: self.modified.as_deref(), + subscribed: Subscribed::Yes, + role: None, + executable: false, + } + } +} + +impl GraphCoordinator<'_> { + pub fn reconcile_files(&self, counts: &mut TypeCounts) -> Result<(), Error> { + let Some(root_id) = self.drive_root_id() else { + self.logger.warn(format_args!( + "no OneDrive drive for this account; files skipped" + )); + return Ok(()); + }; + let nodes = match self.enumerate_drive(root_id) { + Ok(nodes) => nodes, + Err(e) => { + counts.failed += 1; + self.logger + .warn(format_args!("OneDrive enumeration failed: {e}")); + return Ok(()); + } + }; + self.logger.progress(format_args!( + "Found {} in OneDrive", + Counted::new(nodes.len() as u64, "item", "items") + )); + + let local = self.known(ObjectType::FileNode)?; + let (mut folders, files): (Vec, Vec) = nodes + .into_iter() + .partition(|node| node.facet == DriveFacet::Folder); + let mut tree = ParentMap::with_capacity(DanglingParent::Counts, folders.len()); + tree.extend( + folders + .iter() + .map(|node| (node.graph_id.clone(), Some(node.parent_graph_id.clone()))), + ); + tree.parents_first(&mut folders, |node| &node.graph_id); + + let mut by_graph_id: HashMap<&str, LocalId> = HashMap::with_capacity(folders.len()); + for node in &folders { + let parent = by_graph_id.get(node.parent_graph_id.as_str()).copied(); + let tx = self.store.begin()?; + let store = tx.store(); + let record = node.record(parent, None); + let id = match local.get(&node.graph_id) { + Some(&id) => { + store.file_nodes().update(id, &record)?; + counts.fetched += 1; + id + } + None => { + let id = store.file_nodes().insert(&record)?; + self.ids_in(store) + .insert(ObjectType::FileNode, &node.graph_id, id)?; + counts.created += 1; + id + } + }; + tx.commit()?; + by_graph_id.insert(&node.graph_id, id); + } + + let mut wanted = Vec::new(); + for node in &files { + if local.contains_key(&node.graph_id) { + counts.fetched += 1; + } else { + wanted.push(node); + } + } + self.download_files(&wanted, &by_graph_id, counts)?; + + let server = folders + .iter() + .chain(&files) + .map(|node| node.graph_id.clone()); + let mut vanished = IdDiff::compute(server, &local).vanished; + self.parents(ObjectType::FileNode)? + .children_first(&mut vanished, |(_, id)| id); + self.sweep(ObjectType::FileNode, &vanished, counts) + } + + fn drive_root_id(&self) -> Option { + let mut value = self.client.get_json(&self.endpoints.drive_root()).ok()?; + match value.get_mut("id")?.take() { + Value::String(id) => Some(id), + _ => None, + } + } + + fn enumerate_drive(&self, root_id: String) -> Result, GraphError> { + let mut out = Vec::new(); + let mut seen = HashSet::from([root_id.clone()]); + let mut frontier = vec![root_id]; + while let Some(parent) = frontier.pop() { + let url = self.endpoints.drive_children(&parent, self.top); + for child in self.client.collect_all_values(&url, &[])? { + let Some(node) = DriveNode::parse(&child, &parent) else { + continue; + }; + if !seen.insert(node.graph_id.clone()) { + continue; + } + if node.facet == DriveFacet::Folder { + frontier.push(node.graph_id.clone()); + } + out.push(node); + } + } + Ok(out) + } + + fn download_files( + &self, + nodes: &[&DriveNode], + by_graph_id: &HashMap<&str, LocalId>, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + if nodes.is_empty() { + return Ok(()); + } + let node_by_id: HashMap<&str, &DriveNode> = nodes + .iter() + .map(|node| (node.graph_id.as_str(), *node)) + .collect(); + let client = self.client.clone(); + let endpoints = self.endpoints.clone(); + let pool: Pool, GraphError>)> = + Pool::new(self.workers, move |id: String| { + let result = client + .get_with_prefer(&endpoints.drive_item_content(&id), Accept::Binary, &[]) + .map(|response| response.body); + (id, result) + }); + for node in nodes { + pool.submit(node.graph_id.clone()); + } + + let mut batch = None; + let mut in_batch = 0usize; + for _ in 0..nodes.len() { + let Ok((graph_id, result)) = pool.results().recv() else { + break; + }; + let Some(node) = node_by_id.get(graph_id.as_str()).copied() else { + continue; + }; + let bytes = match result { + Ok(bytes) => bytes, + Err(GraphError::Vanished) => { + counts.skipped += 1; + continue; + } + Err(e) => { + counts.failed += 1; + self.logger.warn(format_args!( + "OneDrive item {graph_id} download failed: {e}" + )); + continue; + } + }; + let tx = match batch.take() { + Some(tx) => tx, + None => self.store.begin()?, + }; + let store = tx.store(); + let parent = by_graph_id.get(node.parent_graph_id.as_str()).copied(); + let blob = store.blobs().intern(&bytes)?; + let id = store + .file_nodes() + .insert(&node.record(parent, Some(blob)))?; + self.ids_in(store) + .insert(ObjectType::FileNode, &node.graph_id, id)?; + counts.created += 1; + in_batch += 1; + if in_batch >= CHUNK_SIZE { + tx.commit()?; + in_batch = 0; + } else { + batch = Some(tx); + } + } + if let Some(tx) = batch { + tx.commit()?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn parse_rejects_items_without_file_or_folder_facet() { + let vault = json!({"id": "V", "name": "Personal Vault", "remoteItem": {}}); + assert!(DriveNode::parse(&vault, "root").is_none()); + } + + #[test] + fn parse_reads_folder_and_file_facets() { + let folder = json!({"id": "F", "name": "Docs", "folder": {"childCount": 2}}); + let parsed = DriveNode::parse(&folder, "root").expect("folder parses"); + assert_eq!(parsed.facet, DriveFacet::Folder); + + let file = json!({ + "id": "X", "name": "a.txt", "size": 12, + "file": {"mimeType": "text/plain"}, + "createdDateTime": "2026-01-01T00:00:00Z" + }); + let parsed = DriveNode::parse(&file, "root").expect("file parses"); + assert_eq!(parsed.facet, DriveFacet::File); + assert_eq!(parsed.media_type.as_deref(), Some("text/plain")); + } +} diff --git a/src/import/graph/folders.rs b/src/import/graph/folders.rs new file mode 100644 index 0000000..46bfc57 --- /dev/null +++ b/src/import/graph/folders.rs @@ -0,0 +1,347 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; + +use serde_json::Value; + +use crate::archive::LocalId; +use crate::archive::rows::{AddressBookRecord, CalendarRecord, MailboxRecord, Subscribed}; +use crate::console::Counted; +use crate::error::Error; +use crate::exchange::graph::calendar_map::CalendarColor; +use crate::exchange::graph::types::MailboxKind; +use crate::exchange::tz::ZoneId; +use crate::sync::TypeCounts; +use crate::sync::diff::IdDiff; +use crate::sync::tree::{DanglingParent, ParentMap}; +use crate::types::ObjectType; + +use super::coordinator::{CHUNK_SIZE, GraphCoordinator, GraphImport}; + +const WELL_KNOWN_ROLES: [(&str, &str); 6] = [ + ("inbox", "inbox"), + ("drafts", "drafts"), + ("sentitems", "sent"), + ("deleteditems", "trash"), + ("junkemail", "junk"), + ("archive", "archive"), +]; + +#[derive(Debug, Clone)] +pub struct GraphFolder { + pub graph_id: String, + pub local_id: LocalId, +} + +#[derive(Clone, Copy)] +struct FolderValue<'v>(&'v Value); + +impl<'v> FolderValue<'v> { + fn str(self, key: &str) -> Option<&'v str> { + self.0.get(key).and_then(Value::as_str) + } + + fn id(self) -> Option<&'v str> { + self.str("id") + } + + fn flag(self, key: &str) -> bool { + self.0.get(key).and_then(Value::as_bool).unwrap_or(false) + } +} + +impl GraphCoordinator<'_> { + pub fn reconcile_mail_folders( + &self, + mailbox_kind: MailboxKind, + counts: &mut TypeCounts, + ) -> Result, Error> { + let server = GraphImport::enumerate_mail_folders( + self.client, + self.endpoints, + mailbox_kind, + self.top, + )?; + self.logger.progress(format_args!( + "Found {}", + Counted::new(server.len() as u64, "mailbox", "mailboxes") + )); + let well_known = match mailbox_kind { + MailboxKind::Primary => self.well_known_roles(), + MailboxKind::Archive => HashMap::new(), + }; + let local = self.known(ObjectType::Mailbox)?; + + let mut tree = ParentMap::with_capacity(DanglingParent::Ignored, server.len()); + let mut entries: Vec<(String, Value)> = Vec::with_capacity(server.len()); + for value in server { + let folder = FolderValue(&value); + let Some(id) = folder.id().map(str::to_owned) else { + continue; + }; + tree.insert(id.clone(), folder.str("parentFolderId").map(str::to_owned)); + entries.push((id, value)); + } + tree.parents_first(&mut entries, |(id, _)| id); + let mut by_id: HashMap<&str, LocalId> = HashMap::with_capacity(entries.len()); + let mut out = Vec::with_capacity(entries.len()); + + for chunk in entries.chunks(CHUNK_SIZE) { + let tx = self.store.begin()?; + let store = tx.store(); + let mailboxes = store.mailboxes(); + for (graph_id, value) in chunk { + let folder = FolderValue(value); + let record = MailboxRecord { + name: folder.str("displayName").unwrap_or("Unnamed"), + parent: folder + .str("parentFolderId") + .and_then(|parent| by_id.get(parent)) + .copied(), + role: well_known.get(graph_id.as_str()).copied(), + sort_order: 0, + subscribed: if folder.flag("isHidden") { + Subscribed::No + } else { + Subscribed::Yes + }, + }; + let local_id = match local.get(graph_id) { + Some(&id) => { + mailboxes.update(id, &record)?; + counts.fetched += 1; + id + } + None => { + let id = mailboxes.insert(&record)?; + self.ids_in(store) + .insert(ObjectType::Mailbox, graph_id, id)?; + counts.created += 1; + id + } + }; + by_id.insert(graph_id, local_id); + out.push(GraphFolder { + graph_id: graph_id.clone(), + local_id, + }); + } + tx.commit()?; + } + + let mut vanished = IdDiff::compute(out.iter().map(|f| f.graph_id.clone()), &local).vanished; + self.parents(ObjectType::Mailbox)? + .children_first(&mut vanished, |(_, id)| id); + self.sweep(ObjectType::Mailbox, &vanished, counts)?; + Ok(out) + } + + pub fn reconcile_calendars(&self, counts: &mut TypeCounts) -> Result, Error> { + let server = self + .client + .collect_all_values(&self.endpoints.calendars(self.top), &[])?; + let time_zone = self.mailbox_time_zone(); + let local = self.known(ObjectType::Calendar)?; + let records: Vec<(&str, CalendarRecord<'_>)> = server + .iter() + .filter_map(|value| { + let folder = FolderValue(value); + let graph_id = folder.id()?; + Some(( + graph_id, + CalendarRecord { + name: folder.str("name").unwrap_or("Calendar"), + description: None, + color: folder + .str("hexColor") + .or_else(|| folder.str("color").and_then(|c| CalendarColor(c).hex())), + sort_order: 0, + subscribed: Subscribed::Yes, + is_visible: true, + is_default: folder.flag("isDefaultCalendar"), + include_in_availability: "all", + default_alerts_with_time: None, + default_alerts_without_time: None, + time_zone: time_zone.as_deref(), + }, + )) + }) + .collect(); + let vanished = + IdDiff::compute(records.iter().map(|(id, _)| (*id).to_owned()), &local).vanished; + self.sweep(ObjectType::Calendar, &vanished, counts)?; + let demoted = |(graph_id, record): &&(&str, CalendarRecord<'_>)| { + !record.is_default && local.contains_key(*graph_id) + }; + let write_order: Vec<&(&str, CalendarRecord<'_>)> = records + .iter() + .filter(demoted) + .chain(records.iter().filter(|r| !demoted(r))) + .collect(); + let mut inserted: HashMap<&str, LocalId> = HashMap::new(); + + for chunk in write_order.chunks(CHUNK_SIZE) { + let tx = self.store.begin()?; + let store = tx.store(); + let calendars = store.calendars(); + for &&(graph_id, ref record) in chunk { + match local.get(graph_id) { + Some(&id) => { + calendars.update(id, record)?; + counts.fetched += 1; + } + None => { + let id = calendars.insert(record)?; + self.ids_in(store) + .insert(ObjectType::Calendar, graph_id, id)?; + counts.created += 1; + inserted.insert(graph_id, id); + } + } + } + tx.commit()?; + } + + let out: Vec = records + .iter() + .filter_map(|&(graph_id, _)| { + let local_id = local + .get(graph_id) + .or_else(|| inserted.get(graph_id)) + .copied()?; + Some(GraphFolder { + graph_id: graph_id.to_owned(), + local_id, + }) + }) + .collect(); + Ok(out) + } + + pub fn reconcile_address_books( + &self, + counts: &mut TypeCounts, + ) -> Result, Error> { + let mut server = self + .client + .collect_all_values(&self.endpoints.contact_folders(self.top), &[])?; + let mut seen: HashSet = server + .iter() + .filter_map(|value| FolderValue(value).id()) + .map(str::to_owned) + .collect(); + let mut default_id = None; + if let Some(default) = self.default_contact_folder() + && let Some(id) = FolderValue(&default).id() + { + default_id = Some(id.to_owned()); + if seen.insert(id.to_owned()) { + server.insert(0, default); + } + } + let mut frontier: Vec = seen.iter().cloned().collect(); + while let Some(parent) = frontier.pop() { + let url = self.endpoints.contact_folder_children(&parent, self.top); + for child in self.client.collect_all_values(&url, &[])? { + let Some(id) = FolderValue(&child).id() else { + continue; + }; + if seen.insert(id.to_owned()) { + frontier.push(id.to_owned()); + server.push(child); + } + } + } + + let local = self.known(ObjectType::AddressBook)?; + let mut out = Vec::with_capacity(server.len()); + for chunk in server.chunks(CHUNK_SIZE) { + let tx = self.store.begin()?; + let store = tx.store(); + let books = store.address_books(); + for value in chunk { + let folder = FolderValue(value); + let Some(graph_id) = folder.id() else { + continue; + }; + let record = AddressBookRecord { + name: folder.str("displayName").unwrap_or("Contacts"), + description: None, + sort_order: 0, + is_default: default_id.as_deref() == Some(graph_id), + subscribed: Subscribed::Yes, + }; + let local_id = match local.get(graph_id) { + Some(&id) => { + books.update(id, &record)?; + counts.fetched += 1; + id + } + None => { + let id = books.insert(&record)?; + self.ids_in(store) + .insert(ObjectType::AddressBook, graph_id, id)?; + counts.created += 1; + id + } + }; + out.push(GraphFolder { + graph_id: graph_id.to_owned(), + local_id, + }); + } + tx.commit()?; + } + + let vanished = IdDiff::compute(out.iter().map(|f| f.graph_id.clone()), &local).vanished; + self.sweep(ObjectType::AddressBook, &vanished, counts)?; + Ok(out) + } + + fn default_contact_folder(&self) -> Option { + match self + .client + .get_json(&self.endpoints.default_contact_folder()) + { + Ok(value) if FolderValue(&value).id().is_some() => return Some(value), + Ok(_) => {} + Err(e) => self.logger.warn(format_args!( + "default contact folder lookup failed ({e}); \ + deriving it from an existing contact instead" + )), + } + let probe = self + .client + .get_json(&self.endpoints.any_contact_parent()) + .ok()?; + let parent = FolderValue(probe.get("value")?.as_array()?.first()?).str("parentFolderId")?; + self.client + .get_json(&self.endpoints.contact_folder(parent)) + .ok() + } + + fn mailbox_time_zone(&self) -> Option { + let value = self + .client + .get_json(&self.endpoints.mailbox_settings_timezone()) + .ok()?; + ZoneId(FolderValue(&value).str("timeZone")?).iana() + } + + fn well_known_roles(&self) -> HashMap { + let mut out = HashMap::with_capacity(WELL_KNOWN_ROLES.len()); + for (short_name, role) in WELL_KNOWN_ROLES { + let url = self.endpoints.well_known_folder(short_name); + if let Ok(value) = self.client.get_json(&url) + && let Some(id) = FolderValue(&value).id() + { + out.insert(id.to_owned(), role); + } + } + out + } +} diff --git a/src/import/graph/messages.rs b/src/import/graph/messages.rs new file mode 100644 index 0000000..6b8885e --- /dev/null +++ b/src/import/graph/messages.rs @@ -0,0 +1,215 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; + +use serde_json::Value; + +use crate::archive::rows::EmailRecord; +use crate::console::Counted; +use crate::error::Error; +use crate::exchange::graph::client::Accept; +use crate::exchange::graph::error::GraphError; +use crate::sync::TypeCounts; +use crate::sync::dates::MissingDate; +use crate::sync::diff::IdDiff; +use crate::sync::emailmeta::EmailMeta; +use crate::sync::keywords::{Keywords, MessageFlags}; +use crate::sync::pool::Pool; +use crate::types::ObjectType; + +use super::coordinator::{CHUNK_SIZE, GraphCoordinator}; +use super::folders::GraphFolder; + +const FLAGGED: &str = "flagged"; + +#[derive(Clone, Copy)] +struct MessageStub<'v>(&'v Value); + +impl MessageStub<'_> { + fn is(self, key: &str) -> bool { + self.0.get(key).and_then(Value::as_bool) == Some(true) + } + + fn keywords(self) -> Keywords { + let mut keywords = MessageFlags { + read: self.is("isRead"), + draft: self.is("isDraft"), + flagged: self + .0 + .get("flag") + .and_then(|f| f.get("flagStatus")) + .and_then(Value::as_str) + == Some(FLAGGED), + } + .keywords(); + if let Some(categories) = self.0.get("categories").and_then(Value::as_array) { + for category in categories.iter().filter_map(Value::as_str) { + keywords.push_category(category); + } + } + keywords + } +} + +impl GraphCoordinator<'_> { + pub fn reconcile_messages( + &self, + folders: &[GraphFolder], + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let local = self.known(ObjectType::Email)?; + let mut server_total: HashSet = HashSet::new(); + let mut any_failure = false; + + for folder in folders { + let url = self + .endpoints + .folder_messages_ids(&folder.graph_id, self.top); + let stubs = match self.client.collect_all_values(&url, &[]) { + Ok(stubs) => stubs, + Err(e) => { + self.logger.warn(format_args!( + "mailbox {} enumeration failed: {e}", + folder.graph_id + )); + counts.failed += 1; + any_failure = true; + continue; + } + }; + self.logger.progress(format_args!( + "Mailbox {}: {} listed", + folder.graph_id, + Counted::new(stubs.len() as u64, "email", "emails") + )); + let mut wanted: Vec<(String, String)> = Vec::new(); + for stub in &stubs { + let Some(id) = stub.get("id").and_then(Value::as_str) else { + continue; + }; + if local.contains_key(id) { + counts.fetched += 1; + server_total.insert(id.to_owned()); + } else if server_total.insert(id.to_owned()) { + wanted.push((id.to_owned(), MessageStub(stub).keywords().to_json())); + } + } + if !wanted.is_empty() { + self.fetch_messages(folder, &wanted, counts)?; + } + } + + if any_failure { + self.logger.warn(format_args!( + "vanished emails not removed: one or more mailboxes failed to enumerate; \ + a clean re-run will reconcile deletions" + )); + return Ok(()); + } + let vanished = IdDiff::compute(server_total, &local).vanished; + self.sweep(ObjectType::Email, &vanished, counts) + } + + fn fetch_messages( + &self, + folder: &GraphFolder, + wanted: &[(String, String)], + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let client = self.client.clone(); + let endpoints = self.endpoints.clone(); + let pool: Pool, GraphError>)> = + Pool::new(self.workers, move |id: String| { + let result = client + .get_with_prefer(&endpoints.message_mime(&id), Accept::Text, &[]) + .map(|response| response.body); + (id, result) + }); + for (id, _) in wanted { + pool.submit(id.clone()); + } + let keywords: HashMap<&str, &str> = wanted + .iter() + .map(|(id, keywords)| (id.as_str(), keywords.as_str())) + .collect(); + let mailboxes = [folder.local_id]; + let mut batch = None; + let mut in_batch = 0usize; + for _ in 0..wanted.len() { + let Ok((graph_id, result)) = pool.results().recv() else { + break; + }; + let bytes = match result { + Ok(bytes) => bytes, + Err(GraphError::Vanished) => { + counts.skipped += 1; + continue; + } + Err(e) => { + counts.failed += 1; + self.logger + .warn(format_args!("email {graph_id} fetch failed: {e}")); + continue; + } + }; + let tx = match batch.take() { + Some(tx) => tx, + None => self.store.begin()?, + }; + let store = tx.store(); + let meta = EmailMeta::from_blob(&bytes); + let received_at = meta.date.unwrap_or_else(|| MissingDate::Epoch.resolve()); + let message_match = meta.index.to_json(); + let id = store.emails().insert(&EmailRecord { + blob: store.blobs().intern(&bytes)?, + received_at: &received_at, + mailboxes: &mailboxes, + keywords: keywords.get(graph_id.as_str()).copied().unwrap_or("[]"), + message_match: Some(&message_match), + })?; + self.ids_in(store) + .insert(ObjectType::Email, &graph_id, id)?; + counts.created += 1; + in_batch += 1; + if in_batch >= CHUNK_SIZE { + tx.commit()?; + in_batch = 0; + } else { + batch = Some(tx); + } + } + if let Some(tx) = batch { + tx.commit()?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn stub_keywords_keep_the_flag_order_then_categories() { + let stub = json!({ + "isRead": true, + "isDraft": true, + "flag": {"flagStatus": "flagged"}, + "categories": ["Review / Legal", "review / legal", "$seen", " "] + }); + assert_eq!( + MessageStub(&stub).keywords().to_json(), + r#"["$seen","$draft","$flagged","review_/_legal","seen"]"# + ); + assert_eq!( + MessageStub(&json!({"isRead": false})).keywords().to_json(), + "[]" + ); + } +} diff --git a/src/sync/import_exchange_graph.rs b/src/import/graph/mod.rs similarity index 70% rename from src/sync/import_exchange_graph.rs rename to src/import/graph/mod.rs index 1d29802..e8c8f86 100644 --- a/src/sync/import_exchange_graph.rs +++ b/src/import/graph/mod.rs @@ -10,5 +10,6 @@ pub mod coordinator; pub mod files; pub mod folders; pub mod messages; +pub mod series; -pub use coordinator::{GraphAuth, GraphImportConfig, run}; +pub use coordinator::{ContactPhotos, EventAttachments, GraphAuth, GraphImport}; diff --git a/src/import/graph/series.rs b/src/import/graph/series.rs new file mode 100644 index 0000000..efec8d0 --- /dev/null +++ b/src/import/graph/series.rs @@ -0,0 +1,544 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; +use std::fmt::Write as _; + +use chrono::{DateTime, Duration, NaiveDate, NaiveDateTime, TimeZone, Utc}; +use chrono_tz::Tz; +use rusqlite::{OptionalExtension, params}; +use serde_json::map::Entry; +use serde_json::{Map, Value, json}; + +use crate::console::Counted; +use crate::error::Error; +use crate::exchange::date::WireDate; +use crate::exchange::graph::api::{EXCEPTION_WINDOW_MAX_DAYS, PREFER_TIMEZONE_UTC}; +use crate::exchange::graph::calendar_map::{ConvertedEvent, GraphEvent}; +use crate::exchange::graph::expand::JsRule; +use crate::exchange::graph::recurrence::GraphRecurrence; +use crate::exchange::jscalendar::OverridePatch; +use crate::exchange::tz::ZoneId; +use crate::sync::TypeCounts; +use crate::types::ObjectType; + +use super::coordinator::GraphCoordinator; + +const DAYS_PER_YEAR: i64 = 365; +const DATE_FORMAT: &str = "%Y-%m-%d"; +const WINDOW_BOUND_FORMAT: &str = "%Y-%m-%dT00:00:00"; +const LOCAL_FORMAT: &str = "%Y-%m-%dT%H:%M:%S"; +const LOCAL_FRACTION_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.f"; +const OVERRIDES: &str = "recurrenceOverrides"; + +#[derive(Debug, Clone, Copy)] +pub struct ExceptionWindow { + from: NaiveDate, + to: NaiveDate, +} + +impl ExceptionWindow { + pub fn around_today(years: i32) -> [ExceptionWindow; 2] { + let today = Utc::now().date_naive(); + let span = Duration::days( + i64::from(years.max(1)) + .saturating_mul(DAYS_PER_YEAR) + .min(EXCEPTION_WINDOW_MAX_DAYS), + ); + [ + ExceptionWindow { + from: today - span, + to: today, + }, + ExceptionWindow { + from: today, + to: today + span, + }, + ] + } + + fn bounds(self) -> (String, String) { + ( + self.from.format(WINDOW_BOUND_FORMAT).to_string(), + self.to.format(WINDOW_BOUND_FORMAT).to_string(), + ) + } +} + +struct Occurrence { + start: String, + zone: Option, +} + +pub struct OverrideKey; + +impl OverrideKey { + pub fn normalise(raw: &str, master_tz: Option<&str>) -> String { + if let Some(tz_name) = master_tz + && let Some(local) = OverrideKey::to_local(raw, tz_name) + { + return local; + } + OverrideKey::strip_offset_and_fraction(raw) + } + + fn to_local(raw: &str, tz_name: &str) -> Option { + let tz: Tz = tz_name.parse().ok()?; + let local = match DateTime::parse_from_rfc3339(raw) { + Ok(parsed) => tz.from_utc_datetime(&parsed.naive_utc()).naive_local(), + Err(_) => NaiveDateTime::parse_from_str(raw, LOCAL_FRACTION_FORMAT) + .or_else(|_| NaiveDateTime::parse_from_str(raw, LOCAL_FORMAT)) + .ok()?, + }; + Some(local.format(LOCAL_FORMAT).to_string()) + } + + fn strip_offset_and_fraction(raw: &str) -> String { + let no_offset = if let Some((head, _)) = raw.split_once('Z') { + head + } else if let Some((head, _)) = raw.split_once('+') { + head + } else if let Some((head, _)) = raw.rsplit_once('-') + && head.len() > 10 + { + head + } else { + raw + }; + let trimmed = no_offset + .split_once('.') + .map_or(no_offset, |(head, _)| head); + if trimmed.contains('T') { + trimmed.to_owned() + } else { + format!("{trimmed}T00:00:00") + } + } +} + +struct MasterCard { + data: Value, + changed: bool, +} + +impl MasterCard { + fn new(data: Value) -> MasterCard { + MasterCard { + data, + changed: false, + } + } + + fn key(&self, utc: &str, zone: Option<&str>) -> String { + if self.data.get("showWithoutTime").and_then(Value::as_bool) == Some(true) + && let Some(date) = WireDate(utc).all_day(zone) + { + return GraphEvent::all_day_key(date); + } + OverrideKey::normalise(utc, self.data.get("timeZone").and_then(Value::as_str)) + } + + fn local_date(&self, utc: &str, zone: Option<&str>) -> String { + let mut key = self.key(utc, zone); + if let Some((date, _)) = key.split_once('T') { + let len = date.len(); + key.truncate(len); + } + key + } + + fn overrides(&mut self) -> Option<&mut Map> { + self.data + .as_object_mut()? + .entry(OVERRIDES) + .or_insert_with(|| Value::Object(Map::new())) + .as_object_mut() + } + + fn set_override(&mut self, key: String, patch: Value) { + let Some(overrides) = self.overrides() else { + return; + }; + let changed = match overrides.entry(key) { + Entry::Vacant(slot) => { + slot.insert(patch); + true + } + Entry::Occupied(mut slot) if *slot.get() != patch => { + slot.insert(patch); + true + } + Entry::Occupied(_) => false, + }; + self.changed |= changed; + } + + fn exclude(&mut self, keys: Vec) { + if keys.is_empty() { + return; + } + let Some(overrides) = self.overrides() else { + return; + }; + let mut changed = false; + for key in keys { + if let Entry::Vacant(slot) = overrides.entry(key) { + slot.insert(json!({"excluded": true})); + changed = true; + } + } + self.changed |= changed; + } + + fn cancelled_keys( + &self, + occurrences: &[Occurrence], + exceptions: &[&ConvertedEvent], + windows: &[ExceptionWindow], + ) -> Vec { + let Some(rule) = self.data.get("recurrenceRule") else { + return Vec::new(); + }; + let Some(start) = self.data.get("start").and_then(Value::as_str) else { + return Vec::new(); + }; + let (start_date, start_time) = start.split_once('T').unwrap_or((start, "00:00:00")); + let Ok(series_start) = NaiveDate::parse_from_str(start_date, DATE_FORMAT) else { + return Vec::new(); + }; + let Some(pattern) = JsRule(rule).to_graph_pattern(series_start) else { + return Vec::new(); + }; + let mut actual: HashSet = + HashSet::with_capacity(occurrences.len() + exceptions.len()); + for occurrence in occurrences { + actual.insert(self.local_date(&occurrence.start, occurrence.zone.as_deref())); + } + for exception in exceptions { + if let Some(raw) = exception.original_start.as_deref() { + actual.insert(self.local_date(raw, exception.zone.as_deref())); + } + } + + let mut out = Vec::new(); + let mut key = String::new(); + for window in windows { + let Some(expected) = GraphRecurrence(&pattern).expected_dates(window.from, window.to) + else { + return Vec::new(); + }; + for date in expected { + key.clear(); + let _ = write!(key, "{}", date.format(DATE_FORMAT)); + if !actual.contains(&key) { + out.push(format!("{key}T{start_time}")); + } + } + } + out.sort_unstable(); + out.dedup(); + out + } +} + +impl GraphCoordinator<'_> { + pub fn expand_series( + &self, + calendar_id: &str, + series_masters: &[String], + windows: &[ExceptionWindow], + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let exceptions = self.fetch_exceptions(calendar_id, windows, counts); + let occurrences = self.fetch_occurrences(calendar_id, windows, counts); + let mut by_master: HashMap<&str, Vec<&ConvertedEvent>> = HashMap::new(); + for exception in &exceptions { + if let Some(master) = exception.series_master_id.as_deref() { + by_master.entry(master).or_default().push(exception); + } + } + + let conn = self.store.connection(); + let known = self.known(ObjectType::CalendarEvent)?; + for master_id in series_masters { + let Some(&local_id) = known.get(master_id) else { + continue; + }; + let Ok(Some(stored)) = conn + .query_row( + "SELECT data FROM calendar_events WHERE id = ?1", + params![local_id], + |row| row.get::<_, String>(0), + ) + .optional() + else { + continue; + }; + let Ok(data) = serde_json::from_str::(&stored) else { + continue; + }; + let mut card = MasterCard::new(data); + let master_exceptions = by_master + .get(master_id.as_str()) + .map_or(&[][..], Vec::as_slice); + for exception in master_exceptions { + if let Some(raw) = exception.original_start.as_deref() { + let key = card.key(raw, exception.zone.as_deref()); + card.set_override(key, OverridePatch::from_event(&exception.data)); + } + } + let master_occurrences = occurrences + .get(master_id.as_str()) + .map_or(&[][..], Vec::as_slice); + let cancelled = card.cancelled_keys(master_occurrences, master_exceptions, windows); + card.exclude(cancelled); + + if card.changed { + conn.execute( + "UPDATE calendar_events SET data = ?1 WHERE id = ?2", + params![card.data.to_string(), local_id], + )?; + counts.updated += 1; + } + } + Ok(()) + } + + fn fetch_exceptions( + &self, + calendar_id: &str, + windows: &[ExceptionWindow], + counts: &mut TypeCounts, + ) -> Vec { + let mut out = Vec::new(); + let mut seen: HashSet = HashSet::new(); + for window in windows { + let (from, to) = window.bounds(); + let url = self + .endpoints + .calendar_exceptions(calendar_id, &from, &to, self.top); + let values = match self.client.collect_all_values(&url, &[PREFER_TIMEZONE_UTC]) { + Ok(values) => values, + Err(e) => { + counts.failed += 1; + self.logger.warn(format_args!( + "calendar {calendar_id}: exception lookup {from}..{to} failed: {e}" + )); + continue; + } + }; + self.logger.progress(format_args!( + "Calendar {calendar_id}: {} in {from}..{to}", + Counted::new(values.len() as u64, "exception", "exceptions") + )); + for raw in values { + let Some(exception_id) = raw.get("id").and_then(Value::as_str) else { + continue; + }; + if !seen.insert(exception_id.to_owned()) { + continue; + } + match GraphEvent(&raw).convert(None) { + Ok(converted) => out.push(converted), + Err(e) => { + counts.failed += 1; + self.logger.warn(format_args!( + "exception {exception_id} could not be converted: {e}" + )); + } + } + } + } + out + } + + fn fetch_occurrences( + &self, + calendar_id: &str, + windows: &[ExceptionWindow], + counts: &mut TypeCounts, + ) -> HashMap> { + let mut out: HashMap> = HashMap::new(); + for window in windows { + let (from, to) = window.bounds(); + let url = self + .endpoints + .calendar_occurrences(calendar_id, &from, &to, self.top); + let values = match self.client.collect_all_values(&url, &[PREFER_TIMEZONE_UTC]) { + Ok(values) => values, + Err(e) => { + counts.failed += 1; + self.logger.warn(format_args!( + "calendar {calendar_id}: occurrence sweep {from}..{to} failed: {e}; \ + deleted occurrences cannot be detected for this calendar" + )); + continue; + } + }; + for raw in values { + let Some(master) = raw.get("seriesMasterId").and_then(Value::as_str) else { + continue; + }; + let Some(start) = raw + .get("start") + .and_then(|s| s.get("dateTime")) + .and_then(Value::as_str) + else { + continue; + }; + let zone = raw + .get("originalStartTimeZone") + .and_then(Value::as_str) + .and_then(|z| ZoneId(z).iana()); + out.entry(master.to_owned()).or_default().push(Occurrence { + start: start.to_owned(), + zone, + }); + } + } + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn all_day_daily_master() -> MasterCard { + MasterCard::new(json!({ + "@type": "Event", + "start": "2026-11-12T00:00:00", + "duration": "P1D", + "showWithoutTime": true, + "recurrenceRule": {"@type": "RecurrenceRule", "frequency": "daily", "count": 3} + })) + } + + fn date(text: &str) -> NaiveDate { + NaiveDate::parse_from_str(text, DATE_FORMAT).unwrap() + } + + #[test] + fn all_day_override_key_takes_the_date_in_the_event_zone() { + let master = all_day_daily_master(); + let paris = Some("Europe/Paris"); + assert_eq!( + master.key("2026-11-12T23:00:00Z", paris), + "2026-11-13T00:00:00" + ); + assert_eq!( + master.key("2026-11-13T00:00:00Z", paris), + "2026-11-13T00:00:00" + ); + assert_eq!( + master.local_date("2026-11-13T23:00:00.0000000", paris), + "2026-11-14" + ); + } + + #[test] + fn all_day_occurrences_east_of_utc_are_not_reported_cancelled() { + let master = all_day_daily_master(); + let occurrences: Vec = + ["2026-11-11T23:00:00.0000000", "2026-11-13T23:00:00.0000000"] + .into_iter() + .map(|start| Occurrence { + start: start.to_owned(), + zone: Some("Europe/Paris".to_owned()), + }) + .collect(); + let windows = [ExceptionWindow { + from: date("2026-11-01"), + to: date("2026-11-30"), + }]; + let cancelled = master.cancelled_keys(&occurrences, &[], &windows); + assert_eq!(cancelled, vec!["2026-11-13T00:00:00".to_owned()]); + } + + #[test] + fn exception_windows_never_exceed_the_graph_limit() { + for requested in [1, 5, 50] { + for window in ExceptionWindow::around_today(requested) { + let days = (window.to - window.from).num_days(); + assert!(days > 0, "window runs forwards: {window:?}"); + assert!( + days <= EXCEPTION_WINDOW_MAX_DAYS, + "requested {requested}y produced {days} days, over Graph's cap" + ); + let (from, to) = window.bounds(); + assert!(from.ends_with("T00:00:00") && to.ends_with("T00:00:00")); + } + } + } + + #[test] + fn overrides_mark_the_card_changed_only_on_a_real_change() { + let mut card = all_day_daily_master(); + card.exclude(Vec::new()); + assert!(!card.changed); + assert!(card.data.get(OVERRIDES).is_none()); + card.set_override("2026-11-13T00:00:00".to_owned(), json!({"title": "x"})); + assert!(card.changed); + card.changed = false; + card.set_override("2026-11-13T00:00:00".to_owned(), json!({"title": "x"})); + card.exclude(vec!["2026-11-13T00:00:00".to_owned()]); + assert!(!card.changed); + card.exclude(vec!["2026-11-14T00:00:00".to_owned()]); + assert!(card.changed); + assert_eq!( + card.data[OVERRIDES]["2026-11-14T00:00:00"]["excluded"], + true + ); + } + + #[test] + fn override_key_utc_converts_to_master_local_time() { + let local = OverrideKey::normalise("2026-05-11T15:00:00Z", Some("America/New_York")); + assert_eq!(local, "2026-05-11T11:00:00"); + } + + #[test] + fn override_key_no_tz_strips_z_only() { + let local = OverrideKey::normalise("2026-05-11T15:00:00Z", None); + assert_eq!(local, "2026-05-11T15:00:00"); + } + + #[test] + fn override_key_with_explicit_offset_converts() { + let local = OverrideKey::normalise("2026-05-11T15:00:00+00:00", Some("Europe/London")); + assert_eq!(local, "2026-05-11T16:00:00"); + } + + #[test] + fn override_key_naive_localdatetime_passes_through() { + let local = OverrideKey::normalise("2026-05-11T15:00:00", Some("America/New_York")); + assert_eq!(local, "2026-05-11T15:00:00"); + } + + #[test] + fn override_key_unknown_timezone_strips_offset_only() { + let local = OverrideKey::normalise("2026-05-11T15:00:00Z", Some("Not/A_Zone")); + assert_eq!(local, "2026-05-11T15:00:00"); + } + + #[test] + fn override_key_fractional_seconds_stripped() { + let local = OverrideKey::normalise("2026-05-11T15:00:00.1234567Z", None); + assert_eq!(local, "2026-05-11T15:00:00"); + } + + #[test] + fn override_key_strips_a_negative_offset_but_not_a_date_hyphen() { + assert_eq!( + OverrideKey::normalise("2026-05-11T15:00:00-05:00", None), + "2026-05-11T15:00:00" + ); + assert_eq!( + OverrideKey::normalise("2026-05-11", None), + "2026-05-11T00:00:00" + ); + } +} diff --git a/src/import/imap/control.rs b/src/import/imap/control.rs new file mode 100644 index 0000000..b1983cf --- /dev/null +++ b/src/import/imap/control.rs @@ -0,0 +1,183 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::sync::Arc; + +use crate::error::Error; +use crate::imap::client::{Capability, CollectedResponse, ImapClient}; +use crate::imap::command::Command; +use crate::imap::error::ImapError; +use crate::imap::name::NameEncoding; +use crate::imap::response::{NamespaceEntry, Untagged}; +use crate::imap::session::ImapSession; +use crate::logging::Logger; +use crate::net::backoff::{BackoffState, Fault, RetryError}; +use crate::net::http::RetryStats; + +use super::folders::ResolvedFolder; +use super::messages::UidSearch; + +pub(super) struct Control { + client: ImapClient, + session: Arc, + max_retries: u32, + backoff: BackoffState, + logger: Logger, +} + +#[derive(Debug, Clone, Copy, Default)] +pub(super) struct Selected { + pub uidvalidity: u32, + pub uidnext: u32, +} + +#[derive(Clone, Copy)] +enum SelectCode { + UidValidity, + UidNext, +} + +impl Selected { + fn from_response(resp: &CollectedResponse) -> Selected { + let mut out = Selected::default(); + for u in &resp.untagged { + let Untagged::StatusLine(line) = u else { + continue; + }; + let (Some(code), Some(args)) = (&line.code, &line.code_args) else { + continue; + }; + let slot = match hashify::map!(code.as_bytes(), SelectCode, + "UIDVALIDITY" => SelectCode::UidValidity, + "UIDNEXT" => SelectCode::UidNext, + ) { + Some(SelectCode::UidValidity) => &mut out.uidvalidity, + Some(SelectCode::UidNext) => &mut out.uidnext, + None => continue, + }; + if let Ok(n) = args.parse() { + *slot = n; + } + } + out + } +} + +impl Control { + pub fn new( + client: ImapClient, + session: Arc, + max_retries: u32, + backoff: BackoffState, + ) -> Control { + let logger = session.logger(); + Control { + client, + session, + max_retries, + backoff, + logger, + } + } + + pub fn client(&self) -> &ImapClient { + &self.client + } + + pub fn name_encoding(&self) -> NameEncoding { + self.client.name_encoding() + } + + pub fn stats(&self) -> RetryStats { + self.backoff.stats() + } + + pub fn run_collect(&mut self, command: &Command<'_>) -> Result { + let session = &self.session; + let logger = self.logger; + self.backoff + .run( + self.max_retries, + &mut self.client, + |client| client.run_collect(command), + |client, _| { + *client = session.reopen()?; + logger.info(format_args!("Reconnected the IMAP control connection")); + Ok(()) + }, + |_| {}, + ) + .map_err(|e| match e { + RetryError::Exhausted { + error, + fault: Fault::TransportDrop, + } => Error::Connection(format!("control connection lost: {error}")), + RetryError::Exhausted { error, .. } => Error::Partial(format!( + "{:?}: {error} (max retries exhausted)", + command.to_string() + )), + RetryError::Fatal(error) => { + Error::Partial(format!("{:?}: {error}", command.to_string())) + } + RetryError::Reconnect(error) => { + Error::Connection(format!("control reconnect failed: {error}")) + } + }) + } + + pub fn select(&mut self, folder: &ResolvedFolder) -> Result { + self.run_collect(&Command::Select(&folder.wire_name)) + .map(|resp| Selected::from_response(&resp)) + } + + pub fn noop(&mut self) { + let _ = self.run_collect(&Command::Noop); + } + + pub fn uids(&mut self) -> Result, ImapError> { + let session = &self.session; + self.backoff + .run( + self.max_retries, + &mut self.client, + UidSearch::all, + |client, _| { + *client = session.reopen()?; + Ok(()) + }, + |_| {}, + ) + .map_err(RetryError::into_inner) + } + + pub fn namespace_prefix(&mut self) -> String { + if !self.client.has(Capability::Namespace) { + return String::new(); + } + match self.client.run_collect(&Command::Namespace) { + Ok(resp) => resp + .untagged + .into_iter() + .find_map(|u| match u { + Untagged::Namespace { personal, .. } => personal + .into_iter() + .next() + .map(|NamespaceEntry { prefix, .. }| prefix), + _ => None, + }) + .unwrap_or_default(), + Err(e) => { + self.logger + .progress(format_args!("NAMESPACE failed ({e}); using no prefix")); + String::new() + } + } + } + + pub fn logout(&mut self) { + let _ = self.client.logout(); + } +} diff --git a/src/import/imap/dry_run.rs b/src/import/imap/dry_run.rs new file mode 100644 index 0000000..cdfd266 --- /dev/null +++ b/src/import/imap/dry_run.rs @@ -0,0 +1,112 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; + +use crate::archive::{LocalId, SourceId}; +use crate::error::Error; +use crate::sync::PlanCounts; +use crate::sync::diff::IdDiff; +use crate::types::ObjectType; + +use super::folders::ResolvedFolder; +use super::run::ImapRun; + +impl ImapRun { + pub(super) fn dry_run( + &mut self, + source: Option, + folders: &[ResolvedFolder], + ) -> Result<(), Error> { + let logger = self.run.options().logger; + let store = self.run.store(); + let ids = source.map(|sid| store.imap_ids(sid)); + let state = source.map(|sid| store.imap_folder_state(sid)); + let failed = &mut self.counts.email.failed; + let mut mailbox = PlanCounts::default(); + let mut email = PlanCounts::default(); + + let local_mailboxes = match ids { + Some(ids) => ids.folders()?, + None => HashMap::new(), + }; + let server: HashSet<&str> = folders.iter().map(|f| f.name.as_str()).collect(); + let vanished: Vec<&str> = local_mailboxes + .keys() + .map(String::as_str) + .filter(|n| !server.contains(n)) + .collect(); + mailbox.new = folders + .iter() + .filter(|f| !local_mailboxes.contains_key(&f.name)) + .count() as u64; + mailbox.present = folders.len() as u64 - mailbox.new; + mailbox.vanished = vanished.len() as u64; + + for folder in folders.iter().filter(|f| f.selectable) { + let selected = match self.control.select(folder) { + Ok(selected) => selected, + Err(e) => { + logger.note(format_args!( + "cannot select folder \"{}\": {e}", + folder.name + )); + *failed += 1; + continue; + } + }; + let server_uidvalidity = selected.uidvalidity; + let server_uids = match self.control.uids() { + Ok(uids) => uids, + Err(e) => { + logger.note(format_args!( + "cannot list the UIDs of folder \"{}\": {e}", + folder.name + )); + *failed += 1; + continue; + } + }; + let prev = match state { + Some(state) => state.get(&folder.name)?, + None => None, + }; + let changed = prev + .as_ref() + .filter(|p| p.uidvalidity != server_uidvalidity && server_uidvalidity != 0); + let local: HashMap = match ids { + Some(ids) if changed.is_none() => ids + .items_in(&folder.name)? + .into_iter() + .filter(|((v, _), _)| server_uidvalidity == 0 || *v == server_uidvalidity) + .map(|((_, uid), id)| (uid, id)) + .collect(), + _ => HashMap::new(), + }; + let plan = IdDiff::compute(server_uids, &local).plan(); + email.new += plan.new; + email.vanished += plan.vanished; + email.present += plan.present; + if let Some(prev) = changed { + logger.progress(format_args!( + "UIDVALIDITY of folder \"{}\" changed from {} to {server_uidvalidity}; every message counts as new", + folder.name, prev.uidvalidity + )); + } + } + if let Some(ids) = ids { + for name in &vanished { + email.vanished += ids.items_in(name)?.len() as u64; + } + } + let summary = self.run.summary(); + *summary.plan(ObjectType::Mailbox) = mailbox; + *summary.plan(ObjectType::Email) = email; + + self.control.logout(); + Ok(()) + } +} diff --git a/src/sync/import_imap/fetch.rs b/src/import/imap/fetch.rs similarity index 56% rename from src/sync/import_imap/fetch.rs rename to src/import/imap/fetch.rs index 7bc2fc4..cbc43cb 100644 --- a/src/sync/import_imap/fetch.rs +++ b/src/import/imap/fetch.rs @@ -15,50 +15,55 @@ pub struct FetchAttrs { pub body: Option>, } -pub fn extract(u: &Untagged) -> Option { - let Untagged::Fetch { items, .. } = u else { - return None; - }; - let mut out = FetchAttrs::default(); - for (name, value) in items { - match name.as_str() { - "UID" => { - if let Some(n) = value.as_number() { - out.uid = Some(n as u32); - } - } - "FLAGS" => { - if let Value::List(items) = value { +#[derive(Clone, Copy)] +enum FetchItem { + Uid, + Flags, + InternalDate, + Size, + Body, +} + +impl FetchAttrs { + pub fn from_untagged(u: Untagged) -> Option { + let Untagged::Fetch { items, .. } = u else { + return None; + }; + let mut out = FetchAttrs::default(); + for (name, value) in items { + let Some(item) = hashify::map!(name.as_bytes(), FetchItem, + "UID" => FetchItem::Uid, + "FLAGS" => FetchItem::Flags, + "INTERNALDATE" => FetchItem::InternalDate, + "RFC822.SIZE" => FetchItem::Size, + "BODY[]" => FetchItem::Body, + "RFC822" => FetchItem::Body, + ) else { + continue; + }; + match (item, value) { + (FetchItem::Uid, Value::Number(n)) => out.uid = u32::try_from(n).ok(), + (FetchItem::Flags, Value::List(items)) => { out.flags = items - .iter() + .into_iter() .filter_map(|v| match v { - Value::Atom(s) | Value::Str(s) => Some(s.clone()), + Value::Atom(s) | Value::Str(s) => Some(s), _ => None, }) .collect(); } - } - "INTERNALDATE" => { - if let Some(s) = value.as_str() { - out.internaldate = Some(s.to_owned()); + (FetchItem::InternalDate, Value::Atom(s) | Value::Str(s)) => { + out.internaldate = Some(s); } + (FetchItem::Size, Value::Number(n)) => out.size = Some(n), + (FetchItem::Body, Value::Bytes(b)) => out.body = Some(b), + (FetchItem::Body, Value::Str(s)) => out.body = Some(s.into_bytes()), + (FetchItem::Body, _) => out.body = None, + _ => {} } - "RFC822.SIZE" => { - if let Some(n) = value.as_number() { - out.size = Some(n); - } - } - n if n == "BODY[]" || n == "RFC822" => { - out.body = match value { - Value::Bytes(b) => Some(b.clone()), - Value::Str(s) => Some(s.clone().into_bytes()), - _ => None, - }; - } - _ => {} } + Some(out) } - Some(out) } #[cfg(test)] @@ -67,9 +72,11 @@ mod tests { use std::io::Cursor; fn parse_fetch(input: &[u8]) -> FetchAttrs { - let resp = crate::imap::response::parse_response(&mut Cursor::new(input)).unwrap(); + let resp = crate::imap::response::Response::read(&mut Cursor::new(input)).unwrap(); match resp { - crate::imap::response::Response::Untagged(u) => extract(&u).expect("fetch attrs"), + crate::imap::response::Response::Untagged(u) => { + FetchAttrs::from_untagged(u).expect("fetch attrs") + } _ => panic!("expected untagged"), } } diff --git a/src/sync/import_imap/folders.rs b/src/import/imap/folders.rs similarity index 52% rename from src/sync/import_imap/folders.rs rename to src/import/imap/folders.rs index 9020a41..73410b0 100644 --- a/src/sync/import_imap/folders.rs +++ b/src/import/imap/folders.rs @@ -8,9 +8,10 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use regex::Regex; -use crate::imap::automap; -use crate::imap::name::{canonicalise_inbox, decode_mailbox_name_with}; +use crate::imap::automap::Automap; +use crate::imap::name::NameEncoding; use crate::imap::response::Untagged; +use crate::sync::tree::FolderPath; #[derive(Debug, Clone, Default)] pub struct FolderStatus { @@ -43,243 +44,213 @@ pub struct ResolvedFolder { pub status: Option, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FolderScope { + All, + Subscribed, +} + pub struct FolderFilters { pub include: Vec, pub exclude: Vec, pub exclude_special: Vec, pub explicit: Vec, - pub subscribed_only: bool, - pub automap_enabled: bool, + pub scope: FolderScope, + pub automap: Automap, pub namespace_prefix: String, } -pub fn collect_from_list( - untagged: &[Untagged], - utf8_accept: bool, -) -> Result, String> { - let mut subscribed = HashSet::new(); - let mut statuses: BTreeMap = BTreeMap::new(); - let mut found = BTreeMap::new(); - for u in untagged { - match u { - Untagged::Lsub { name, .. } => { - let decoded = - decode_mailbox_name_with(name, utf8_accept).map_err(|e| e.to_string())?; - subscribed.insert(canonicalise_inbox(&decoded)); +impl DiscoveredFolder { + pub fn collect( + untagged: &[Untagged], + encoding: NameEncoding, + ) -> Result, String> { + let mut subscribed = HashSet::new(); + let mut statuses: HashMap = HashMap::new(); + let mut found = BTreeMap::new(); + for u in untagged { + match u { + Untagged::Lsub { name, .. } => { + subscribed.insert(encoding.mailbox(name).map_err(|e| e.to_string())?); + } + Untagged::List { + attributes, + delimiter, + name, + } => { + let canonical = encoding.mailbox(name).map_err(|e| e.to_string())?; + let selectable = !attributes.iter().any(|a| { + hashify::set_ignore_case!(a.as_bytes(), "\\noselect", "\\nonexistent") + }); + if attributes + .iter() + .any(|a| a.eq_ignore_ascii_case("\\subscribed")) + { + subscribed.insert(canonical.clone()); + } + found.insert( + canonical.clone(), + DiscoveredFolder { + name: canonical, + wire_name: name.clone(), + delimiter: *delimiter, + attributes: attributes.clone(), + subscribed: false, + selectable, + status: None, + }, + ); + } + Untagged::Status { mailbox, items } => { + let canonical = encoding.mailbox(mailbox).map_err(|e| e.to_string())?; + statuses.insert( + canonical, + FolderStatus { + uidvalidity: items.get("UIDVALIDITY").copied(), + uidnext: items.get("UIDNEXT").copied(), + messages: items.get("MESSAGES").copied(), + }, + ); + } + _ => {} } - Untagged::List { - attributes, - delimiter, - name, - } => { - let decoded = - decode_mailbox_name_with(name, utf8_accept).map_err(|e| e.to_string())?; - let canonical = canonicalise_inbox(&decoded); - let attrs_lower: Vec = - attributes.iter().map(|a| a.to_ascii_lowercase()).collect(); - let selectable = !attrs_lower + } + Ok(found + .into_values() + .map(|mut f| { + f.subscribed = subscribed.contains(&f.name); + f.status = statuses.remove(&f.name); + f + }) + .collect()) + } +} + +impl ResolvedFolder { + fn path_parts(name: &str, delimiter: Option) -> (String, Option) { + match delimiter { + Some(d) => { + let path = FolderPath::new(name, d); + (path.leaf().to_owned(), path.parent().map(str::to_owned)) + } + None => (name.to_owned(), None), + } + } + + fn depth(&self) -> usize { + match (self.parent_path.as_deref(), self.delimiter) { + (Some(parent), Some(d)) => FolderPath::new(parent, d).depth() + 1, + _ => 0, + } + } + + pub fn sort_parents_first(resolved: &mut [ResolvedFolder]) { + resolved.sort_by(|a, b| a.depth().cmp(&b.depth()).then_with(|| a.name.cmp(&b.name))); + } +} + +impl FolderFilters { + pub fn apply(&self, folders: Vec) -> Vec { + let mut resolved: Vec = Vec::with_capacity(folders.len()); + let mut others: HashMap = HashMap::new(); + for f in folders { + if !f.selectable + || !self.matches(&f.name) + || (self.scope == FolderScope::Subscribed && !f.subscribed) + { + others.insert(f.name.clone(), f); + continue; + } + let role = self + .automap + .role(&f.name, &f.attributes, &self.namespace_prefix); + if let Some(r) = role + && self + .exclude_special .iter() - .any(|a| a == "\\noselect" || a == "\\nonexistent"); - let extended_subscribed = attrs_lower.iter().any(|a| a == "\\subscribed"); - if extended_subscribed { - subscribed.insert(canonical.clone()); - } - found.insert( - canonical.clone(), - DiscoveredFolder { - name: canonical, - wire_name: name.clone(), - delimiter: *delimiter, - attributes: attributes.clone(), - subscribed: false, - selectable, - status: None, - }, - ); + .any(|s| s.eq_ignore_ascii_case(r)) + { + others.insert(f.name.clone(), f); + continue; } - Untagged::Status { mailbox, items } => { - let decoded = - decode_mailbox_name_with(mailbox, utf8_accept).map_err(|e| e.to_string())?; - let canonical = canonicalise_inbox(&decoded); - let mut st = FolderStatus::default(); - if let Some(v) = items.get("UIDVALIDITY") { - st.uidvalidity = Some(*v); + let (leaf, parent_path) = ResolvedFolder::path_parts(&f.name, f.delimiter); + resolved.push(ResolvedFolder { + name: f.name, + wire_name: f.wire_name, + leaf, + parent_path, + delimiter: f.delimiter, + role, + subscribed: f.subscribed, + selectable: true, + status: f.status, + }); + } + let mut present: HashSet = resolved.iter().map(|f| f.name.clone()).collect(); + let mut structural: Vec = Vec::new(); + for child in &resolved { + let mut parent = child.parent_path.clone(); + let mut child_wire = child.wire_name.clone(); + let mut delim = child.delimiter; + while let Some(name) = parent { + if present.contains(&name) { + break; } - if let Some(v) = items.get("UIDNEXT") { - st.uidnext = Some(*v); - } - if let Some(v) = items.get("MESSAGES") { - st.messages = Some(*v); - } - statuses.insert(canonical, st); - } - _ => {} - } - } - let mut out: Vec = found.into_values().collect(); - for f in &mut out { - if subscribed.contains(&f.name) { - f.subscribed = true; - } - if let Some(s) = statuses.remove(&f.name) { - f.status = Some(s); - } - } - Ok(out) -} - -pub fn apply_filters( - folders: Vec, - filters: &FolderFilters, -) -> Vec { - let mut resolved: Vec = Vec::with_capacity(folders.len()); - let mut others: HashMap = HashMap::new(); - for f in folders { - if !f.selectable - || !match_filters(&f.name, filters) - || (filters.subscribed_only && !f.subscribed) - { - others.insert(f.name.clone(), f); - continue; - } - let role = automap::role_for_folder( - &f.name, - &f.attributes, - &filters.namespace_prefix, - filters.automap_enabled, - ); - if let Some(r) = role - && filters - .exclude_special - .iter() - .any(|s| s.eq_ignore_ascii_case(r)) - { - others.insert(f.name.clone(), f); - continue; - } - let (leaf, parent_path) = split_parent(&f.name, f.delimiter); - resolved.push(ResolvedFolder { - name: f.name, - wire_name: f.wire_name, - leaf, - parent_path, - delimiter: f.delimiter, - role, - subscribed: f.subscribed, - selectable: true, - status: f.status, - }); - } - let mut present: HashSet = resolved.iter().map(|f| f.name.clone()).collect(); - let mut structural: Vec = Vec::new(); - for child in &resolved { - let mut parent = child.parent_path.clone(); - let mut child_wire = child.wire_name.clone(); - let mut delim = child.delimiter; - while let Some(name) = parent { - if present.contains(&name) { - break; - } - let ancestor = match others.remove(&name) { - Some(d) => { - let delimiter = d.delimiter.or(delim); - let (leaf, parent_path) = split_parent(&d.name, delimiter); - ResolvedFolder { - name: d.name, - wire_name: d.wire_name, - leaf, - parent_path, - delimiter, - role: None, - subscribed: d.subscribed, - selectable: false, - status: None, + let ancestor = match others.remove(&name) { + Some(d) => { + let delimiter = d.delimiter.or(delim); + let (leaf, parent_path) = ResolvedFolder::path_parts(&d.name, delimiter); + ResolvedFolder { + name: d.name, + wire_name: d.wire_name, + leaf, + parent_path, + delimiter, + role: None, + subscribed: d.subscribed, + selectable: false, + status: None, + } } - } - None => { - let wire_name = match delim { - Some(d) => child_wire - .rsplit_once(d) - .map_or_else(|| name.clone(), |(p, _)| p.to_owned()), - None => name.clone(), - }; - let (leaf, parent_path) = split_parent(&name, delim); - ResolvedFolder { - name, - wire_name, - leaf, - parent_path, - delimiter: delim, - role: None, - subscribed: false, - selectable: false, - status: None, + None => { + let wire_name = match delim.and_then(|d| child_wire.rsplit_once(d)) { + Some((p, _)) => p.to_owned(), + None => name.clone(), + }; + let (leaf, parent_path) = ResolvedFolder::path_parts(&name, delim); + ResolvedFolder { + name, + wire_name, + leaf, + parent_path, + delimiter: delim, + role: None, + subscribed: false, + selectable: false, + status: None, + } } - } - }; - present.insert(ancestor.name.clone()); - parent = ancestor.parent_path.clone(); - child_wire.clone_from(&ancestor.wire_name); - delim = ancestor.delimiter; - structural.push(ancestor); + }; + present.insert(ancestor.name.clone()); + parent = ancestor.parent_path.clone(); + child_wire.clone_from(&ancestor.wire_name); + delim = ancestor.delimiter; + structural.push(ancestor); + } } + resolved.extend(structural); + resolved } - resolved.extend(structural); - resolved -} -fn match_filters(name: &str, filters: &FolderFilters) -> bool { - if !filters.explicit.is_empty() { - return filters.explicit.iter().any(|e| e == name); + fn matches(&self, name: &str) -> bool { + if !self.explicit.is_empty() { + return self.explicit.iter().any(|e| e == name); + } + if !self.include.is_empty() && !self.include.iter().any(|r| r.is_match(name)) { + return false; + } + !self.exclude.iter().any(|r| r.is_match(name)) } - if !filters.include.is_empty() && !filters.include.iter().any(|r| r.is_match(name)) { - return false; - } - if filters.exclude.iter().any(|r| r.is_match(name)) { - return false; - } - true -} - -fn split_parent(name: &str, delim: Option) -> (String, Option) { - let Some(d) = delim else { - return (name.to_owned(), None); - }; - if let Some(idx) = name.rfind(d) { - let parent = &name[..idx]; - let leaf = &name[idx + d.len_utf8()..]; - (leaf.to_owned(), Some(parent.to_owned())) - } else { - (name.to_owned(), None) - } -} - -pub fn sort_by_depth(resolved: &mut [ResolvedFolder]) { - resolved.sort_by(|a, b| { - let da = a.parent_path.as_deref().map_or(0, |s| { - a.delimiter.map(|d| s.matches(d).count() + 1).unwrap_or(0) - }); - let db = b.parent_path.as_deref().map_or(0, |s| { - b.delimiter.map(|d| s.matches(d).count() + 1).unwrap_or(0) - }); - da.cmp(&db).then_with(|| a.name.cmp(&b.name)) - }); -} - -pub fn vanished_folders(local: &HashMap, server: &HashSet) -> Vec { - local - .keys() - .filter(|k| !server.contains(*k)) - .cloned() - .collect() -} - -pub fn vanished_depth_sort(names: &mut [String], delimiter: char) { - names.sort_by(|a, b| { - let depth_a = a.matches(delimiter).count(); - let depth_b = b.matches(delimiter).count(); - depth_b.cmp(&depth_a).then_with(|| a.cmp(b)) - }); } #[cfg(test)] @@ -301,7 +272,7 @@ mod tests { lst("Sent", "/", &["\\Sent"]), lst("Hidden", "/", &["\\Noselect"]), ]; - let folders = collect_from_list(&resps, false).unwrap(); + let folders = DiscoveredFolder::collect(&resps, NameEncoding::ModifiedUtf7).unwrap(); assert_eq!(folders.len(), 3); let inbox = folders.iter().find(|f| f.name == "INBOX").unwrap(); assert!(inbox.selectable); @@ -320,7 +291,7 @@ mod tests { name: "Inbox".to_owned(), }, ]; - let folders = collect_from_list(&resps, false).unwrap(); + let folders = DiscoveredFolder::collect(&resps, NameEncoding::ModifiedUtf7).unwrap(); let inbox = folders.iter().find(|f| f.name == "INBOX").unwrap(); assert!(inbox.subscribed); let sent = folders.iter().find(|f| f.name == "Sent").unwrap(); @@ -342,7 +313,7 @@ mod tests { items: inbox_status, }, ]; - let folders = collect_from_list(&resps, false).unwrap(); + let folders = DiscoveredFolder::collect(&resps, NameEncoding::ModifiedUtf7).unwrap(); let inbox = folders.iter().find(|f| f.name == "INBOX").unwrap(); let st = inbox.status.as_ref().expect("INBOX has status"); assert_eq!(st.uidvalidity, Some(12345)); @@ -356,7 +327,7 @@ mod tests { #[test] fn collect_decodes_modified_utf7_names() { let resps = vec![lst("&ZeVnLIqe-", "/", &[])]; - let folders = collect_from_list(&resps, false).unwrap(); + let folders = DiscoveredFolder::collect(&resps, NameEncoding::ModifiedUtf7).unwrap(); assert_eq!(folders[0].name, "日本語"); } @@ -366,24 +337,24 @@ mod tests { exclude: Vec::new(), exclude_special: Vec::new(), explicit: Vec::new(), - subscribed_only: false, - automap_enabled: true, + scope: FolderScope::All, + automap: Automap::Enabled, namespace_prefix: String::new(), } } #[test] fn filters_default_keep_everything() { - let folders = collect_from_list( + let folders = DiscoveredFolder::collect( &[ lst("INBOX", "/", &[]), lst("Sent", "/", &["\\Sent"]), lst("Drafts", "/", &[]), ], - false, + NameEncoding::ModifiedUtf7, ) .unwrap(); - let res = apply_filters(folders, &filters_default()); + let res = filters_default().apply(folders); assert_eq!(res.len(), 3); let sent = res.iter().find(|f| f.name == "Sent").unwrap(); assert_eq!(sent.role, Some("sent")); @@ -391,18 +362,18 @@ mod tests { #[test] fn include_excludes_non_matching() { - let folders = collect_from_list( + let folders = DiscoveredFolder::collect( &[ lst("INBOX", "/", &[]), lst("Trash", "/", &[]), lst("Sent", "/", &["\\Sent"]), ], - false, + NameEncoding::ModifiedUtf7, ) .unwrap(); let mut f = filters_default(); f.include = vec![Regex::new(r"^(INBOX|Sent)$").unwrap()]; - let res = apply_filters(folders, &f); + let res = f.apply(folders); assert_eq!(res.len(), 2); assert!(res.iter().any(|x| x.name == "INBOX")); assert!(res.iter().any(|x| x.name == "Sent")); @@ -410,77 +381,83 @@ mod tests { #[test] fn exclude_drops_matching() { - let folders = - collect_from_list(&[lst("INBOX", "/", &[]), lst("Trash", "/", &[])], false).unwrap(); + let folders = DiscoveredFolder::collect( + &[lst("INBOX", "/", &[]), lst("Trash", "/", &[])], + NameEncoding::ModifiedUtf7, + ) + .unwrap(); let mut f = filters_default(); f.exclude = vec![Regex::new(r"^Trash$").unwrap()]; - let res = apply_filters(folders, &f); + let res = f.apply(folders); assert_eq!(res.len(), 1); assert_eq!(res[0].name, "INBOX"); } #[test] fn exclude_special_drops_by_role() { - let folders = collect_from_list( + let folders = DiscoveredFolder::collect( &[lst("INBOX", "/", &[]), lst("Trash", "/", &["\\Trash"])], - false, + NameEncoding::ModifiedUtf7, ) .unwrap(); let mut f = filters_default(); f.exclude_special = vec!["trash".to_owned()]; - let res = apply_filters(folders, &f); + let res = f.apply(folders); assert_eq!(res.len(), 1); assert_eq!(res[0].name, "INBOX"); } #[test] fn explicit_folder_list_overrides_include_exclude() { - let folders = collect_from_list( + let folders = DiscoveredFolder::collect( &[ lst("INBOX", "/", &[]), lst("Sent", "/", &[]), lst("Trash", "/", &[]), ], - false, + NameEncoding::ModifiedUtf7, ) .unwrap(); let mut f = filters_default(); f.explicit = vec!["Sent".to_owned()]; - let res = apply_filters(folders, &f); + let res = f.apply(folders); assert_eq!(res.len(), 1); assert_eq!(res[0].name, "Sent"); } #[test] fn subscribed_only_filters_unsubscribed() { - let mut folders = collect_from_list( + let mut folders = DiscoveredFolder::collect( &[lst("INBOX", "/", &["\\Subscribed"]), lst("Other", "/", &[])], - false, + NameEncoding::ModifiedUtf7, ) .unwrap(); let mut f = filters_default(); - f.subscribed_only = true; + f.scope = FolderScope::Subscribed; for x in &mut folders { if x.name == "INBOX" { x.subscribed = true; } } - let res = apply_filters(folders, &f); + let res = f.apply(folders); assert_eq!(res.len(), 1); assert_eq!(res[0].name, "INBOX"); } #[test] fn split_parent_handles_root_and_nested() { - assert_eq!(split_parent("INBOX", Some('/')), ("INBOX".to_owned(), None)); assert_eq!( - split_parent("Projects/Alpha", Some('/')), + ResolvedFolder::path_parts("INBOX", Some('/')), + ("INBOX".to_owned(), None) + ); + assert_eq!( + ResolvedFolder::path_parts("Projects/Alpha", Some('/')), ("Alpha".to_owned(), Some("Projects".to_owned())) ); assert_eq!( - split_parent("Projects/Alpha/Beta", Some('/')), + ResolvedFolder::path_parts("Projects/Alpha/Beta", Some('/')), ("Beta".to_owned(), Some("Projects/Alpha".to_owned())) ); } @@ -488,7 +465,7 @@ mod tests { #[test] fn split_parent_with_no_delimiter_returns_root() { assert_eq!( - split_parent("Anything", None), + ResolvedFolder::path_parts("Anything", None), ("Anything".to_owned(), None) ); } @@ -530,32 +507,12 @@ mod tests { status: None, }, ]; - sort_by_depth(&mut r); + ResolvedFolder::sort_parents_first(&mut r); assert_eq!(r[0].name, "INBOX"); assert_eq!(r[1].name, "Projects"); assert_eq!(r[2].name, "Projects/Alpha"); } - #[test] - fn vanished_folders_is_set_minus() { - let mut local = HashMap::new(); - local.insert("INBOX".to_owned(), 1); - local.insert("Old".to_owned(), 2); - local.insert("Sent".to_owned(), 3); - let mut server = HashSet::new(); - server.insert("INBOX".to_owned()); - server.insert("Sent".to_owned()); - let v = vanished_folders(&local, &server); - assert_eq!(v, vec!["Old".to_owned()]); - } - - #[test] - fn vanished_depth_sort_deepest_first() { - let mut names = vec!["A".to_owned(), "A/B/C".to_owned(), "A/B".to_owned()]; - vanished_depth_sort(&mut names, '/'); - assert_eq!(names, vec!["A/B/C", "A/B", "A"]); - } - #[test] fn untagged_status_does_not_become_a_folder() { let resps = vec![ @@ -565,7 +522,7 @@ mod tests { items: BTreeMap::new(), }, ]; - let folders = collect_from_list(&resps, false).unwrap(); + let folders = DiscoveredFolder::collect(&resps, NameEncoding::ModifiedUtf7).unwrap(); assert_eq!(folders.len(), 1); } @@ -575,17 +532,17 @@ mod tests { #[test] fn noselect_parent_is_kept_as_structural() { - let folders = collect_from_list( + let folders = DiscoveredFolder::collect( &[ lst("INBOX", "/", &["\\HasChildren"]), lst("INBOX/Projects", "/", &["\\Noselect", "\\HasChildren"]), lst("INBOX/Projects/Invoices", "/", &["\\HasNoChildren"]), ], - false, + NameEncoding::ModifiedUtf7, ) .unwrap(); - let mut res = apply_filters(folders, &filters_default()); - sort_by_depth(&mut res); + let mut res = filters_default().apply(folders); + ResolvedFolder::sort_parents_first(&mut res); let names: Vec<&str> = res.iter().map(|f| f.name.as_str()).collect(); assert_eq!( names, @@ -601,24 +558,24 @@ mod tests { #[test] fn noselect_without_kept_children_is_dropped() { - let folders = collect_from_list( + let folders = DiscoveredFolder::collect( &[lst("INBOX", "/", &[]), lst("Empty", "/", &["\\Noselect"])], - false, + NameEncoding::ModifiedUtf7, ) .unwrap(); - let res = apply_filters(folders, &filters_default()); + let res = filters_default().apply(folders); assert_eq!(res.len(), 1); } #[test] fn consecutive_structural_ancestors_and_synthesised_ones() { - let folders = collect_from_list( + let folders = DiscoveredFolder::collect( &[lst("A", "/", &["\\NonExistent"]), lst("A/B/C/D", "/", &[])], - false, + NameEncoding::ModifiedUtf7, ) .unwrap(); - let mut res = apply_filters(folders, &filters_default()); - sort_by_depth(&mut res); + let mut res = filters_default().apply(folders); + ResolvedFolder::sort_parents_first(&mut res); let names: Vec<&str> = res.iter().map(|f| f.name.as_str()).collect(); assert_eq!(names, ["A", "A/B", "A/B/C", "A/B/C/D"]); assert!( @@ -632,17 +589,17 @@ mod tests { #[test] fn trailing_space_and_same_leaf_under_different_parents() { - let folders = collect_from_list( + let folders = DiscoveredFolder::collect( &[ lst("Work ", ".", &["\\Noselect"]), lst("Work", ".", &[]), lst("Work .Archive", ".", &[]), lst("Work.Archive", ".", &[]), ], - false, + NameEncoding::ModifiedUtf7, ) .unwrap(); - let res = apply_filters(folders, &filters_default()); + let res = filters_default().apply(folders); assert_eq!(res.len(), 4); let spaced = find(&res, "Work "); assert!(!spaced.selectable); @@ -660,8 +617,12 @@ mod tests { #[test] fn synthesised_ancestor_derives_wire_name_from_child() { - let folders = collect_from_list(&[lst("&ZeVnLIqe-/Sub", "/", &[])], false).unwrap(); - let res = apply_filters(folders, &filters_default()); + let folders = DiscoveredFolder::collect( + &[lst("&ZeVnLIqe-/Sub", "/", &[])], + NameEncoding::ModifiedUtf7, + ) + .unwrap(); + let res = filters_default().apply(folders); let parent = find(&res, "日本語"); assert_eq!(parent.wire_name, "&ZeVnLIqe-"); assert!(!parent.selectable); @@ -684,15 +645,15 @@ mod tests { }; let mut f = filters_default(); f.include = vec![Regex::new("Docs$").unwrap()]; - check(apply_filters(collect_from_list(&list, false).unwrap(), &f)); + check(f.apply(DiscoveredFolder::collect(&list, NameEncoding::ModifiedUtf7).unwrap())); let mut f = filters_default(); f.exclude = vec![Regex::new("^(INBOX|Other|Projects)$").unwrap()]; - check(apply_filters(collect_from_list(&list, false).unwrap(), &f)); + check(f.apply(DiscoveredFolder::collect(&list, NameEncoding::ModifiedUtf7).unwrap())); let mut f = filters_default(); f.explicit = vec!["Projects/Alpha/Docs".to_owned()]; - check(apply_filters(collect_from_list(&list, false).unwrap(), &f)); + check(f.apply(DiscoveredFolder::collect(&list, NameEncoding::ModifiedUtf7).unwrap())); let mut f = filters_default(); - f.subscribed_only = true; - check(apply_filters(collect_from_list(&list, false).unwrap(), &f)); + f.scope = FolderScope::Subscribed; + check(f.apply(DiscoveredFolder::collect(&list, NameEncoding::ModifiedUtf7).unwrap())); } } diff --git a/src/import/imap/internaldate.rs b/src/import/imap/internaldate.rs new file mode 100644 index 0000000..19d60de --- /dev/null +++ b/src/import/imap/internaldate.rs @@ -0,0 +1,187 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fmt::Write as _; + +use crate::error::Error; + +#[derive(Debug, Clone, Copy)] +pub struct InternalDate<'a>(pub &'a str); + +impl InternalDate<'_> { + pub fn to_rfc3339(self) -> Result { + let s = self.0.trim_start_matches(' ').trim_end(); + let (day, rest) = s + .split_once('-') + .ok_or_else(|| Error::Partial(format!("INTERNALDATE: missing day in {s:?}")))?; + let (mon, rest) = rest + .split_once('-') + .ok_or_else(|| Error::Partial(format!("INTERNALDATE: missing month in {s:?}")))?; + let (year, rest) = rest + .split_once(' ') + .ok_or_else(|| Error::Partial(format!("INTERNALDATE: missing year in {s:?}")))?; + let (time, zone) = rest + .split_once(' ') + .ok_or_else(|| Error::Partial(format!("INTERNALDATE: missing zone in {s:?}")))?; + let (h, ms) = time + .split_once(':') + .ok_or_else(|| Error::Partial(format!("INTERNALDATE: bad time {time:?}")))?; + let (m, sec) = ms + .split_once(':') + .ok_or_else(|| Error::Partial(format!("INTERNALDATE: bad time {time:?}")))?; + let day: u32 = day + .trim() + .parse() + .map_err(|e| Error::Partial(format!("INTERNALDATE day {day:?}: {e}")))?; + let mon_idx = InternalDate::month(mon)?; + let year: i32 = year + .parse() + .map_err(|e| Error::Partial(format!("INTERNALDATE year {year:?}: {e}")))?; + let h: u32 = h + .parse() + .map_err(|e| Error::Partial(format!("INTERNALDATE hour {h:?}: {e}")))?; + let m: u32 = m + .parse() + .map_err(|e| Error::Partial(format!("INTERNALDATE minute {m:?}: {e}")))?; + let sec: u32 = sec + .parse() + .map_err(|e| Error::Partial(format!("INTERNALDATE second {sec:?}: {e}")))?; + let (sign, hh, mm) = InternalDate::zone(zone)?; + let mut out = String::with_capacity(32); + let _ = write!( + out, + "{year:04}-{mon_idx:02}-{day:02}T{h:02}:{m:02}:{sec:02}" + ); + if hh == 0 && mm == 0 { + out.push('Z'); + } else { + let _ = write!(out, "{sign}{hh:02}:{mm:02}"); + } + Ok(out) + } + + fn month(s: &str) -> Result { + hashify::map!(s.as_bytes(), u32, + "Jan" => 1, + "Feb" => 2, + "Mar" => 3, + "Apr" => 4, + "May" => 5, + "Jun" => 6, + "Jul" => 7, + "Aug" => 8, + "Sep" => 9, + "Oct" => 10, + "Nov" => 11, + "Dec" => 12, + ) + .copied() + .ok_or_else(|| Error::Partial(format!("INTERNALDATE month {s:?}"))) + } + + fn zone(s: &str) -> Result<(char, u32, u32), Error> { + let s = s.trim(); + let mut chars = s.chars(); + let (Some(sign), Some(hh), Some(mm), 5) = (chars.next(), s.get(1..3), s.get(3..5), s.len()) + else { + return Err(Error::Partial(format!("INTERNALDATE zone {s:?}"))); + }; + if sign != '+' && sign != '-' { + return Err(Error::Partial(format!("INTERNALDATE zone sign {sign}"))); + } + let hh: u32 = hh + .parse() + .map_err(|e| Error::Partial(format!("INTERNALDATE zone hour: {e}")))?; + let mm: u32 = mm + .parse() + .map_err(|e| Error::Partial(format!("INTERNALDATE zone min: {e}")))?; + Ok((sign, hh, mm)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn utc_zone_becomes_z() { + assert_eq!( + InternalDate("17-Jul-1996 02:44:25 +0000") + .to_rfc3339() + .unwrap(), + "1996-07-17T02:44:25Z" + ); + } + + #[test] + fn positive_offset_preserved() { + assert_eq!( + InternalDate("17-Jul-1996 02:44:25 +0200") + .to_rfc3339() + .unwrap(), + "1996-07-17T02:44:25+02:00" + ); + } + + #[test] + fn negative_offset_preserved() { + assert_eq!( + InternalDate("17-Jul-1996 02:44:25 -0700") + .to_rfc3339() + .unwrap(), + "1996-07-17T02:44:25-07:00" + ); + } + + #[test] + fn single_digit_day_padded() { + assert_eq!( + InternalDate("5-Jan-2024 09:05:01 +0000") + .to_rfc3339() + .unwrap(), + "2024-01-05T09:05:01Z" + ); + } + + #[test] + fn leading_space_single_digit_day() { + assert_eq!( + InternalDate(" 5-Jan-2024 09:05:01 +0000") + .to_rfc3339() + .unwrap(), + "2024-01-05T09:05:01Z" + ); + } + + #[test] + fn invalid_month_errors() { + assert!( + InternalDate("1-XYZ-2024 00:00:00 +0000") + .to_rfc3339() + .is_err() + ); + } + + #[test] + fn zone_with_a_multibyte_char_across_the_hour_boundary_does_not_panic() { + assert!( + InternalDate("1-Jan-2024 00:00:00 +0\u{e9}0") + .to_rfc3339() + .is_err() + ); + assert!( + InternalDate("1-Jan-2024 00:00:00 \u{e9}000") + .to_rfc3339() + .is_err() + ); + } + + #[test] + fn malformed_input_errors() { + assert!(InternalDate("garbage").to_rfc3339().is_err()); + assert!(InternalDate("17-Jul-1996").to_rfc3339().is_err()); + } +} diff --git a/src/import/imap/keywords.rs b/src/import/imap/keywords.rs new file mode 100644 index 0000000..34afc99 --- /dev/null +++ b/src/import/imap/keywords.rs @@ -0,0 +1,185 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::sync::keywords::{Keywords, SystemKeyword}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeletedMessages { + Skip, + Keep, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FlagAction { + Keep(&'static str), + Drop, + Verbatim, + Deleted, +} + +impl FlagAction { + pub fn of(flag: &str) -> FlagAction { + let Some(system) = flag.strip_prefix('\\') else { + return FlagAction::Verbatim; + }; + hashify::map_ignore_case!(system.as_bytes(), FlagAction, + "seen" => FlagAction::Keep("$seen"), + "flagged" => FlagAction::Keep("$flagged"), + "answered" => FlagAction::Keep("$answered"), + "draft" => FlagAction::Keep("$draft"), + "recent" => FlagAction::Drop, + "deleted" => FlagAction::Deleted, + ) + .copied() + .unwrap_or(FlagAction::Verbatim) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Translation { + pub keywords: Keywords, + pub has_deleted_flag: bool, +} + +impl Translation { + pub fn from_flags(imap_flags: &[String], deleted: DeletedMessages) -> Translation { + let mut out = Translation { + keywords: Keywords::with_capacity(imap_flags.len()), + has_deleted_flag: false, + }; + for f in imap_flags { + if f.is_empty() { + continue; + } + match FlagAction::of(f) { + FlagAction::Keep(k) => out.keywords.push(k), + FlagAction::Drop => {} + FlagAction::Verbatim => out.keywords.push(f), + FlagAction::Deleted => { + out.has_deleted_flag = true; + if deleted == DeletedMessages::Keep { + out.keywords.push_system(SystemKeyword::Deleted); + } + } + } + } + out + } + + pub fn skips(&self, deleted: DeletedMessages) -> bool { + self.has_deleted_flag && deleted == DeletedMessages::Skip + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn flags(s: &[&str]) -> Vec { + s.iter().map(|x| (*x).to_owned()).collect() + } + + #[test] + fn system_flag_mapping() { + assert!(matches!( + FlagAction::of("\\Seen"), + FlagAction::Keep("$seen") + )); + assert!(matches!( + FlagAction::of("\\Flagged"), + FlagAction::Keep("$flagged") + )); + assert!(matches!( + FlagAction::of("\\Answered"), + FlagAction::Keep("$answered") + )); + assert!(matches!( + FlagAction::of("\\Draft"), + FlagAction::Keep("$draft") + )); + assert!(matches!(FlagAction::of("\\Recent"), FlagAction::Drop)); + assert!(matches!(FlagAction::of("\\Deleted"), FlagAction::Deleted)); + } + + #[test] + fn unknown_system_flag_passes_verbatim() { + assert!(matches!( + FlagAction::of("\\Forwarded"), + FlagAction::Verbatim + )); + assert!(matches!(FlagAction::of("\\MDNSent"), FlagAction::Verbatim)); + } + + #[test] + fn custom_keywords_pass_verbatim_case() { + let t = Translation::from_flags( + &flags(&["$Junk", "NotJunk", "MyLabel"]), + DeletedMessages::Skip, + ); + assert_eq!( + t.keywords.iter().collect::>(), + vec!["$Junk", "NotJunk", "MyLabel"] + ); + assert!(!t.has_deleted_flag); + } + + #[test] + fn translates_seen_and_flagged() { + let t = Translation::from_flags(&flags(&["\\Seen", "\\Flagged"]), DeletedMessages::Skip); + assert_eq!( + t.keywords.iter().collect::>(), + vec!["$seen", "$flagged"] + ); + } + + #[test] + fn drops_recent() { + let t = Translation::from_flags(&flags(&["\\Seen", "\\Recent"]), DeletedMessages::Skip); + assert_eq!(t.keywords.iter().collect::>(), vec!["$seen"]); + } + + #[test] + fn deleted_flag_without_include_marks_only() { + let t = Translation::from_flags(&flags(&["\\Deleted", "\\Seen"]), DeletedMessages::Skip); + assert!(t.has_deleted_flag); + assert_eq!(t.keywords.iter().collect::>(), vec!["$seen"]); + } + + #[test] + fn deleted_flag_with_include_adds_dollar_deleted() { + let t = Translation::from_flags(&flags(&["\\Deleted", "\\Seen"]), DeletedMessages::Keep); + assert!(t.has_deleted_flag); + assert_eq!( + t.keywords.iter().collect::>(), + vec!["$deleted", "$seen"] + ); + } + + #[test] + fn duplicate_keywords_deduped_case_insensitively() { + let t = + Translation::from_flags(&flags(&["mykey", "MYKEY", "MyKey"]), DeletedMessages::Skip); + assert_eq!(t.keywords.iter().collect::>(), vec!["mykey"]); + } + + #[test] + fn empty_flag_strings_ignored() { + let t = Translation::from_flags(&flags(&["", "\\Seen", ""]), DeletedMessages::Skip); + assert_eq!(t.keywords.iter().collect::>(), vec!["$seen"]); + } + + #[test] + fn order_preserved() { + let t = Translation::from_flags( + &flags(&["MyLabel", "\\Seen", "$Important", "\\Flagged"]), + DeletedMessages::Skip, + ); + assert_eq!( + t.keywords.iter().collect::>(), + vec!["MyLabel", "$seen", "$Important", "$flagged"] + ); + } +} diff --git a/src/import/imap/messages.rs b/src/import/imap/messages.rs new file mode 100644 index 0000000..077e623 --- /dev/null +++ b/src/import/imap/messages.rs @@ -0,0 +1,57 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::imap::client::{Capability, ImapClient}; +use crate::imap::command::Command; +use crate::imap::error::ImapError; +use crate::imap::response::{Untagged, Value}; + +pub struct UidSearch; + +impl UidSearch { + pub fn all(client: &mut ImapClient) -> Result, ImapError> { + if client.has(Capability::Esearch) { + match client.run_collect(&Command::UidSearchEsearchAll) { + Ok(resp) => { + if let Some(all) = resp.untagged.into_iter().find_map(|u| match u { + Untagged::Esearch { all, .. } => Some(all), + _ => None, + }) { + return Ok(all); + } + } + Err(e) if e.is_negotiation_failure() => {} + Err(e) => return Err(e), + } + } + match client.run_collect(&Command::UidSearchAll) { + Ok(resp) => Ok(resp + .untagged + .into_iter() + .find_map(|u| match u { + Untagged::Search(uids) => Some(uids), + _ => None, + }) + .unwrap_or_default()), + Err(e) if e.is_negotiation_failure() => UidSearch::via_fetch(client), + Err(e) => Err(e), + } + } + + fn via_fetch(client: &mut ImapClient) -> Result, ImapError> { + let resp = client.run_collect(&Command::UidFetchAllUids)?; + let mut out = Vec::new(); + for u in resp.untagged { + if let Untagged::Fetch { items, .. } = u { + out.extend(items.into_iter().filter_map(|(n, v)| match v { + Value::Number(num) if n == "UID" => u32::try_from(num).ok(), + _ => None, + })); + } + } + Ok(out) + } +} diff --git a/src/import/imap/mod.rs b/src/import/imap/mod.rs new file mode 100644 index 0000000..96d39f9 --- /dev/null +++ b/src/import/imap/mod.rs @@ -0,0 +1,51 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +mod control; +mod dry_run; +pub mod fetch; +pub mod folders; +pub mod internaldate; +pub mod keywords; +pub mod messages; +pub mod pool; +mod reconcile; +mod run; + +use regex::Regex; + +use crate::archive::sources::SourceChange; +use crate::imap::automap::Automap; +use crate::imap::session::Compress; +use crate::net::stream::Cleartext; + +pub use crate::net::sasl::SaslAuth; +pub use folders::FolderScope; +pub use keywords::DeletedMessages; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MimeKeywords { + Skip, + Extract, +} + +pub struct ImapImport { + pub url: String, + pub auth: SaslAuth, + pub cleartext: Cleartext, + pub compress: Compress, + pub include: Vec, + pub exclude: Vec, + pub exclude_special: Vec, + pub folder: Vec, + pub scope: FolderScope, + pub automap: Automap, + pub deleted: DeletedMessages, + pub mime_keywords: MimeKeywords, + pub fetch_batch: usize, + pub imap_connections: usize, + pub source_change: SourceChange, +} diff --git a/src/import/imap/pool.rs b/src/import/imap/pool.rs new file mode 100644 index 0000000..2eb52d2 --- /dev/null +++ b/src/import/imap/pool.rs @@ -0,0 +1,211 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::Arc; +use std::thread::{self, JoinHandle}; +use std::time::Duration; + +use crossbeam_channel::{Receiver, RecvTimeoutError, Sender, unbounded}; + +use crate::imap::client::ImapClient; +use crate::imap::command::{Command, UidSet}; +use crate::imap::error::ImapError; +use crate::imap::response::Untagged; +use crate::imap::retry::Disposition; +use crate::imap::session::ImapSession; +use crate::net::backoff::BackoffState; + +use super::fetch::FetchAttrs; + +pub const MAX_IMAP_CONNECTIONS: usize = 8; + +const FETCH_ITEMS: &[&str] = &["UID", "FLAGS", "INTERNALDATE", "RFC822.SIZE", "BODY.PEEK[]"]; + +pub struct FetchJob { + pub folder: Arc, + pub wire_name: Arc, + pub uids: Vec, +} + +pub enum FetchEvent { + Item(FetchAttrs), + ChunkDone { + folder: Arc, + requested: usize, + outcome: Result<(), ImapError>, + }, +} + +pub struct WorkerArgs { + pub session: Arc, + pub max_retries: u32, + pub backoff: BackoffState, +} + +pub struct WorkerPool { + job_tx: Sender, + event_rx: Receiver, + handles: Vec>, +} + +impl WorkerPool { + pub fn start(args: WorkerArgs, pool_size: usize) -> WorkerPool { + let size = pool_size.clamp(1, MAX_IMAP_CONNECTIONS); + let (job_tx, job_rx) = unbounded::(); + let (event_tx, event_rx) = unbounded::(); + let args = Arc::new(args); + let handles = (0..size) + .map(|_| { + let mut worker = Worker { + args: args.clone(), + client: None, + selected: None, + }; + let job_rx = job_rx.clone(); + let event_tx = event_tx.clone(); + thread::spawn(move || worker.run(&job_rx, &event_tx)) + }) + .collect(); + WorkerPool { + job_tx, + event_rx, + handles, + } + } + + pub fn submit(&self, job: FetchJob) { + let _ = self.job_tx.send(job); + } + + pub fn recv_timeout(&self, timeout: Duration) -> Result { + self.event_rx.recv_timeout(timeout) + } + + pub fn shutdown(self) { + drop(self.job_tx); + for h in self.handles { + let _ = h.join(); + } + } +} + +struct Worker { + args: Arc, + client: Option, + selected: Option>, +} + +impl Worker { + fn run(&mut self, job_rx: &Receiver, event_tx: &Sender) { + while let Ok(job) = job_rx.recv() { + let outcome = + match catch_unwind(AssertUnwindSafe(|| self.run_with_retry(&job, event_tx))) { + Ok(r) => r, + Err(_) => { + self.disconnect(); + Err(ImapError::Protocol("worker thread panicked".into())) + } + }; + let _ = event_tx.send(FetchEvent::ChunkDone { + folder: job.folder, + requested: job.uids.len(), + outcome, + }); + } + } + + fn disconnect(&mut self) { + self.client = None; + self.selected = None; + } + + fn run_with_retry( + &mut self, + job: &FetchJob, + event_tx: &Sender, + ) -> Result<(), ImapError> { + let args = self.args.clone(); + let mut transient_attempts: u32 = 0; + let mut transport_attempts: u32 = 0; + loop { + if self.client.is_none() { + match args.session.reopen() { + Ok(c) => { + self.client = Some(c); + self.selected = None; + } + Err(e) => { + if e.disposition() == Disposition::TransportDrop + && transport_attempts < args.max_retries + { + transport_attempts += 1; + thread::sleep(args.backoff.transport_delay(transport_attempts)); + continue; + } + return Err(e); + } + } + } + let Some(client) = self.client.as_mut() else { + return Err(ImapError::Protocol( + "worker pool: client slot empty after connect".into(), + )); + }; + match Worker::fetch(client, &mut self.selected, job, event_tx) { + Ok(()) => { + args.backoff.reset(); + return Ok(()); + } + Err(e) => match e.disposition() { + Disposition::TransportDrop => { + self.disconnect(); + if transport_attempts >= args.max_retries { + return Err(e); + } + transport_attempts += 1; + thread::sleep(args.backoff.transport_delay(transport_attempts)); + } + Disposition::Transient => { + if transient_attempts >= args.max_retries { + return Err(e); + } + transient_attempts += 1; + thread::sleep(args.backoff.next_shared_delay()); + } + _ => return Err(e), + }, + } + } + } + + fn fetch( + client: &mut ImapClient, + selected: &mut Option>, + job: &FetchJob, + event_tx: &Sender, + ) -> Result<(), ImapError> { + if selected.as_deref() != Some(&*job.folder) { + client.run_collect(&Command::Select(&job.wire_name))?; + *selected = Some(job.folder.clone()); + } + let set = UidSet::from_sorted(&job.uids); + client.run_streamed( + &Command::UidFetch { + set: &set, + items: FETCH_ITEMS, + }, + |u| { + if let Untagged::Fetch { .. } = &u + && let Some(attrs) = FetchAttrs::from_untagged(u) + { + let _ = event_tx.send(FetchEvent::Item(attrs)); + } + }, + )?; + Ok(()) + } +} diff --git a/src/import/imap/reconcile.rs b/src/import/imap/reconcile.rs new file mode 100644 index 0000000..72afa2f --- /dev/null +++ b/src/import/imap/reconcile.rs @@ -0,0 +1,316 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashSet; +use std::sync::Arc; +use std::time::Duration; + +use crossbeam_channel::RecvTimeoutError; + +use crate::archive::rows::{EmailRecord, Write}; +use crate::archive::state::imap::FolderState; +use crate::archive::{LocalId, SourceId, Store}; +use crate::console::{Counted, Grouped}; +use crate::error::Error; +use crate::imap::command::{Command, UidSet}; +use crate::sync::TypeCounts; +use crate::sync::dates::{MissingDate, Rfc3339}; +use crate::sync::diff::IdDiff; +use crate::sync::emailmeta::EmailMeta; + +use super::MimeKeywords; +use super::fetch::FetchAttrs; +use super::folders::ResolvedFolder; +use super::internaldate::InternalDate; +use super::keywords::Translation; +use super::pool::{FetchEvent, FetchJob, WorkerPool}; +use super::run::{ImapRun, MessageOptions}; + +const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(45); +const FLAG_ITEMS: &[&str] = &["UID", "FLAGS"]; + +struct FetchTarget<'a> { + folder: &'a str, + uidvalidity: u32, + mailbox: LocalId, +} + +impl ImapRun { + pub(super) fn reconcile_folder( + &mut self, + source: SourceId, + pool: &WorkerPool, + folder: &ResolvedFolder, + opts: MessageOptions, + ) -> Result<(), Error> { + let logger = opts.logger; + let store = self.run.store(); + let ids = store.imap_ids(source); + let state = store.imap_folder_state(source); + + if let Some(status) = folder.status.as_ref() + && status.messages == Some(0) + && let Some(Ok(uidvalidity)) = status.uidvalidity.map(u32::try_from) + && let Some(Ok(uidnext)) = status.uidnext.map(u32::try_from) + && ids.items_in(&folder.name)?.is_empty() + { + state.upsert( + &folder.name, + &FolderState { + uidvalidity, + uidnext, + last_seen: Rfc3339::now(), + }, + )?; + logger.progress(format_args!( + "Folder \"{}\" is empty (LIST-STATUS); skipped", + folder.name + )); + return Ok(()); + } + + let selected = self.control.select(folder)?; + let uidvalidity = selected.uidvalidity; + if uidvalidity == 0 { + return Err(Error::Partial(format!( + "SELECT {:?}: server gave no UIDVALIDITY", + folder.name + ))); + } + + let counts = &mut self.counts.email; + if let Some(prev) = state.get(&folder.name)? + && prev.uidvalidity != uidvalidity + { + logger.note(format_args!( + "UIDVALIDITY of folder \"{}\" changed from {} to {}; its archived messages are replaced", + folder.name, prev.uidvalidity, uidvalidity + )); + let tx = store.begin()?; + let ids = tx.store().imap_ids(source); + let rows = tx.store().emails(); + for local in ids.items_in(&folder.name)?.into_values() { + rows.delete(local)?; + counts.deleted += 1; + } + ids.delete_items_in(&folder.name)?; + tx.commit()?; + } + + let server = self + .control + .uids() + .map_err(|e| Error::Partial(format!("UID enum {:?}: {e}", folder.name)))?; + let local = ids.items_at_validity(&folder.name, uidvalidity)?; + let mut diff = IdDiff::compute(server, &local); + diff.new.sort_unstable(); + diff.present.sort_unstable_by_key(|(uid, _)| *uid); + + logger.progress(format_args!( + "Folder \"{}\": {} new, {} vanished, {} present", + folder.name, + Grouped(diff.new.len() as u64), + Grouped(diff.vanished.len() as u64), + Grouped(diff.present.len() as u64) + )); + + if !diff.vanished.is_empty() { + let tx = store.begin()?; + let ids = tx.store().imap_ids(source); + let rows = tx.store().emails(); + for &(uid, local) in &diff.vanished { + rows.delete(local)?; + ids.delete_item(&folder.name, uidvalidity, uid)?; + counts.deleted += 1; + } + tx.commit()?; + } + + if !diff.new.is_empty() { + let mailbox = ids.folder_id(&folder.name)?.ok_or_else(|| { + Error::Partial(format!( + "mailbox {:?} missing from sync_id_imap", + folder.name + )) + })?; + let folder_name: Arc = Arc::from(folder.name.as_str()); + let wire_name: Arc = Arc::from(folder.wire_name.as_str()); + let mut batches = 0usize; + for batch in diff.new.chunks(opts.fetch_batch) { + pool.submit(FetchJob { + folder: folder_name.clone(), + wire_name: wire_name.clone(), + uids: batch.to_vec(), + }); + batches += 1; + } + let target = FetchTarget { + folder: &folder.name, + uidvalidity, + mailbox, + }; + let tx = store.begin()?; + let mut done = 0usize; + while done < batches { + let event = loop { + match pool.recv_timeout(KEEPALIVE_INTERVAL) { + Ok(event) => break event, + Err(RecvTimeoutError::Timeout) => self.control.noop(), + Err(RecvTimeoutError::Disconnected) => { + return Err(Error::Partial("worker channel disconnected".to_owned())); + } + } + }; + match event { + FetchEvent::Item(attrs) => { + ImapRun::insert_message(tx.store(), source, &target, attrs, opts, counts)?; + } + FetchEvent::ChunkDone { + folder: chunk_folder, + requested, + outcome, + } => { + done += 1; + if let Err(e) = outcome { + logger.note(format_args!( + "UID FETCH chunk failed in folder \"{chunk_folder}\": {e}" + )); + counts.failed += requested as u64; + } + } + } + } + tx.commit()?; + } + + if !diff.present.is_empty() { + let tx = store.begin()?; + let mut updated = 0; + let mut uids: Vec = Vec::with_capacity(opts.fetch_batch); + for batch in diff.present.chunks(opts.fetch_batch) { + uids.clear(); + uids.extend(batch.iter().map(|(uid, _)| *uid)); + let set = UidSet::from_sorted(&uids); + let resp = self.control.run_collect(&Command::UidFetch { + set: &set, + items: FLAG_ITEMS, + })?; + for u in resp.untagged { + let Some(attrs) = FetchAttrs::from_untagged(u) else { + continue; + }; + let Some(&id) = attrs.uid.and_then(|uid| local.get(&uid)) else { + continue; + }; + if ImapRun::refresh_keywords(tx.store(), id, &attrs, opts)? == Write::Changed { + updated += 1; + } + } + } + tx.commit()?; + counts.updated += updated; + } + + state.upsert( + &folder.name, + &FolderState { + uidvalidity, + uidnext: selected.uidnext, + last_seen: Rfc3339::now(), + }, + )?; + Ok(()) + } + + fn insert_message( + store: Store<'_>, + source: SourceId, + target: &FetchTarget<'_>, + attrs: FetchAttrs, + opts: MessageOptions, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let Some(uid) = attrs.uid else { + return Ok(()); + }; + let Some(body) = attrs.body.as_deref() else { + counts.skipped += 1; + return Ok(()); + }; + if let Some(declared) = attrs.size + && u64::try_from(body.len()).ok() != Some(declared) + { + opts.logger.note(format_args!( + "message UID {uid} in folder \"{}\" has {} but RFC822.SIZE says {}; importing the fetched literal", + target.folder, + Counted::new(body.len() as u64, "byte", "bytes"), + Grouped(declared) + )); + } + let translation = Translation::from_flags(&attrs.flags, opts.deleted); + if translation.skips(opts.deleted) { + counts.skipped += 1; + return Ok(()); + } + let received_at = match attrs.internaldate.as_deref() { + Some(s) => InternalDate(s).to_rfc3339()?, + None => MissingDate::Now.resolve(), + }; + let blob = store.blobs().intern(body)?; + let mut keywords = translation.keywords; + let index = match opts.mime_keywords { + MimeKeywords::Extract => EmailMeta::index_with_mime_keywords(body, &mut keywords), + MimeKeywords::Skip => EmailMeta::from_blob(body).index, + }; + let message_match = index.to_json(); + let keywords = keywords.to_json(); + let local = store.emails().insert(&EmailRecord { + blob, + received_at: &received_at, + mailboxes: &[target.mailbox], + keywords: &keywords, + message_match: Some(&message_match), + })?; + store + .imap_ids(source) + .insert_item(target.folder, target.uidvalidity, uid, local)?; + counts.created += 1; + counts.fetched += 1; + Ok(()) + } + + fn refresh_keywords( + store: Store<'_>, + id: LocalId, + attrs: &FetchAttrs, + opts: MessageOptions, + ) -> Result { + let translation = Translation::from_flags(&attrs.flags, opts.deleted); + if translation.skips(opts.deleted) { + return Ok(Write::Unchanged); + } + let rows = store.emails(); + let Some((blob, stored)) = rows.keywords(id)? else { + return Ok(Write::Unchanged); + }; + let stored: Vec = serde_json::from_str(&stored).unwrap_or_default(); + let mut keywords = translation.keywords; + if opts.mime_keywords == MimeKeywords::Extract + && let Some(blob) = store.blobs().bytes(blob)? + { + EmailMeta::add_mime_keywords(&blob, &mut keywords); + } + let unchanged = { + let stored: HashSet<&str> = stored.iter().map(String::as_str).collect(); + let fresh: HashSet<&str> = keywords.iter().collect(); + stored == fresh + }; + if unchanged { + return Ok(Write::Unchanged); + } + Ok(rows.update_keywords(id, &keywords.to_json())?) + } +} diff --git a/src/import/imap/run.rs b/src/import/imap/run.rs new file mode 100644 index 0000000..4440424 --- /dev/null +++ b/src/import/imap/run.rs @@ -0,0 +1,358 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use crate::archive::rows::{MailboxRecord, Subscribed}; +use crate::archive::sources::{SourceKey, SourceKind}; +use crate::archive::{Archive, LocalId, SourceId}; +use crate::error::Error; +use crate::imap::client::Capability; +use crate::imap::command::Command; +use crate::imap::session::{IMAP, ImapSession}; +use crate::logging::Logger; +use crate::net::backoff::BackoffState; +use crate::net::stream::{Connector, Endpoint}; +use crate::sync::run::{GcFailure, ImportRun, SourceClaim, SourceLabel}; +use crate::sync::tree::FolderPath; +use crate::sync::vanish::{Children, OnRowError, RowFailure, VanishedSweep}; +use crate::sync::{RunMode, RunOptions, RunOutcome, Summary, TypeCounts}; +use crate::types::ObjectType; + +use super::control::Control; +use super::folders::{DiscoveredFolder, FolderFilters, ResolvedFolder}; +use super::pool::{WorkerArgs, WorkerPool}; +use super::{DeletedMessages, ImapImport, MimeKeywords}; + +const DEFAULT_DELIMITER: char = '/'; + +#[derive(Clone, Copy)] +pub(super) struct MessageOptions { + pub fetch_batch: usize, + pub deleted: DeletedMessages, + pub mime_keywords: MimeKeywords, + pub logger: Logger, +} + +#[derive(Default)] +pub(super) struct Counts { + pub mailbox: TypeCounts, + pub email: TypeCounts, +} + +pub(super) struct ImapRun { + pub run: ImportRun, + pub control: Control, + pub counts: Counts, +} + +impl ImapImport { + pub fn run(self, options: RunOptions) -> RunOutcome { + let failed = |error: Error| RunOutcome { + summary: Summary::default(), + error: Some(error), + }; + let archive = match Archive::open(&options.archive) { + Ok(archive) => archive, + Err(e) => return failed(e.into()), + }; + let ImapImport { + url, + auth, + cleartext, + compress, + include, + exclude, + exclude_special, + folder, + scope, + automap, + deleted, + mime_keywords, + fetch_batch, + imap_connections, + source_change, + } = self; + let session = match Endpoint::parse(&url, &IMAP).and_then(|endpoint| { + let connector = Connector::new(options.tls).map_err(Error::connection)?; + Ok(ImapSession::new( + connector, + endpoint, + cleartext, + auth, + compress, + options.logger, + )) + }) { + Ok(session) => Arc::new(session), + Err(e) => return failed(e), + }; + let session_url = session.session_url(); + let (client, account_id) = match session.open() { + Ok(opened) => opened, + Err(e) => return failed(e), + }; + let key = SourceKey { + kind: SourceKind::Imap, + url: &session_url, + account_id: &account_id, + }; + let label = SourceLabel { + name: Some(&account_id), + username: &account_id, + }; + let max_retries = options.max_retries; + let messages = MessageOptions { + fetch_batch: fetch_batch.max(1), + deleted, + mime_keywords, + logger: options.logger, + }; + let run = match ImportRun::start_on(archive, options, &key, label, source_change) { + Ok(run) => run, + Err(e) => return failed(e), + }; + let backoff = BackoffState::new(); + let mut imap = ImapRun { + run, + control: Control::new(client, session.clone(), max_retries, backoff.clone()), + counts: Counts::default(), + }; + let filters = FolderFilters { + include, + exclude, + exclude_special, + explicit: folder, + scope, + automap, + namespace_prefix: String::new(), + }; + let workers = WorkerArgs { + session, + max_retries, + backoff, + }; + let result = imap.execute(filters, messages, workers, imap_connections); + imap.finish(result) + } +} + +impl ImapRun { + fn logger(&self) -> Logger { + self.run.options().logger + } + + fn execute( + &mut self, + mut filters: FolderFilters, + messages: MessageOptions, + workers: WorkerArgs, + connections: usize, + ) -> Result<(), Error> { + filters.namespace_prefix = self.control.namespace_prefix(); + let resolved = self.discover(&filters)?; + let source = match *self.run.claim() { + SourceClaim::DryRun(source) => return self.dry_run(source, &resolved), + SourceClaim::Registered(source) => source, + }; + self.sync_mailboxes(source, &resolved)?; + let pool = WorkerPool::start(workers, connections.max(1)); + let result = self.sync_folders(source, &pool, &resolved, messages); + pool.shutdown(); + result + } + + fn finish(self, result: Result<(), Error>) -> RunOutcome { + let ImapRun { + mut run, + mut control, + counts, + } = self; + if run.options().mode == RunMode::Apply { + control.logout(); + } + *run.counts(ObjectType::Mailbox) = counts.mailbox; + *run.counts(ObjectType::Email) = counts.email; + let stats = control.stats(); + match result { + Ok(()) => run.finish(stats, GcFailure::Warn), + Err(error) => run.abort(error, stats), + } + } + + fn discover(&mut self, filters: &FolderFilters) -> Result, Error> { + let client = self.control.client(); + let extended = client.has(Capability::ListExtended); + let list_status = extended && client.has(Capability::ListStatus); + let mut returns: Vec<&str> = Vec::with_capacity(4); + if extended { + if client.has(Capability::SpecialUse) { + returns.push("SPECIAL-USE"); + } + returns.push("SUBSCRIBED"); + if list_status { + returns.push("CHILDREN"); + returns.push("STATUS (UIDVALIDITY UIDNEXT MESSAGES)"); + } + } + let mut untagged = self + .control + .run_collect(&Command::List { + reference: "", + patterns: &["*"], + selection: &[], + returns: &returns, + })? + .untagged; + if !list_status + && let Ok(lsub) = self.control.run_collect(&Command::Lsub { + reference: "", + pattern: "*", + }) + { + untagged.extend(lsub.untagged); + } + let discovered = DiscoveredFolder::collect(&untagged, self.control.name_encoding()) + .map_err(|e| Error::Partial(format!("LIST parse: {e}")))?; + let mut resolved = filters.apply(discovered); + ResolvedFolder::sort_parents_first(&mut resolved); + Ok(resolved) + } + + fn sync_mailboxes( + &mut self, + source: SourceId, + resolved: &[ResolvedFolder], + ) -> Result<(), Error> { + let local = self.run.store().imap_ids(source).folders()?; + self.upsert_mailboxes(source, resolved)?; + let server: HashSet<&str> = resolved.iter().map(|f| f.name.as_str()).collect(); + let delimiter = resolved + .iter() + .find_map(|f| f.delimiter) + .unwrap_or(DEFAULT_DELIMITER); + let mut vanished: Vec<(String, LocalId)> = local + .into_iter() + .filter(|(name, _)| !server.contains(name.as_str())) + .collect(); + FolderPath::sort_children_first_by(&mut vanished, delimiter, |(name, _)| name); + let logger = self.logger(); + let emails = &mut self.counts.email; + VanishedSweep::new( + self.run.store(), + OnRowError::Abort, + Children::RefuseSubfolders, + ) + .run( + ObjectType::Mailbox, + &vanished, + &mut self.counts.mailbox, + |store, name, _| { + let ids = store.imap_ids(source); + let rows = store.emails(); + for local in ids.items_in(name)?.into_values() { + rows.delete(local)?; + emails.deleted += 1; + } + ids.delete_items_in(name)?; + store.imap_folder_state(source).delete(name)?; + ids.delete_folder(name) + }, + |name, _, failure| { + if let RowFailure::LiveChildren = failure { + logger.note(format_args!( + "folder \"{name}\" vanished from the server but still has child mailboxes in the archive; not deleted" + )); + } + }, + )?; + Ok(()) + } + + fn upsert_mailboxes( + &mut self, + source: SourceId, + folders: &[ResolvedFolder], + ) -> Result<(), Error> { + let logger = self.logger(); + let tx = self.run.store().begin()?; + let store = tx.store(); + let ids = store.imap_ids(source); + let mailboxes = store.mailboxes(); + let mut local_ids: HashMap<&str, LocalId> = HashMap::with_capacity(folders.len()); + for folder in folders { + let parent = match folder.parent_path.as_deref() { + Some(path) => { + let found = match local_ids.get(path) { + Some(&id) => Some(id), + None => ids.folder_id(path)?, + }; + if found.is_none() { + logger.note(format_args!( + "parent \"{path}\" of folder \"{}\" could not be resolved; stored at the top level", + folder.name + )); + } + found + } + None => None, + }; + let record = MailboxRecord { + name: &folder.leaf, + parent, + role: folder.role, + sort_order: 0, + subscribed: if folder.subscribed { + Subscribed::Yes + } else { + Subscribed::No + }, + }; + let id = match ids.folder_id(&folder.name)? { + Some(id) => { + mailboxes.update(id, &record)?; + id + } + None => { + let id = mailboxes.insert(&record)?; + ids.insert_folder(&folder.name, id)?; + self.counts.mailbox.created += 1; + self.counts.mailbox.fetched += 1; + id + } + }; + local_ids.insert(&folder.name, id); + } + tx.commit()?; + Ok(()) + } + + fn sync_folders( + &mut self, + source: SourceId, + pool: &WorkerPool, + resolved: &[ResolvedFolder], + messages: MessageOptions, + ) -> Result<(), Error> { + for (i, folder) in resolved.iter().filter(|f| f.selectable).enumerate() { + if i > 0 { + self.control.noop(); + } + match self.reconcile_folder(source, pool, folder, messages) { + Ok(()) => {} + Err(e) if e.aborts_run() => return Err(e), + Err(e) => { + messages + .logger + .note(format_args!("folder \"{}\" failed: {e}", folder.name)); + self.counts.email.failed += 1; + } + } + } + Ok(()) + } +} diff --git a/src/import/jmap/blobs.rs b/src/import/jmap/blobs.rs new file mode 100644 index 0000000..378beaa --- /dev/null +++ b/src/import/jmap/blobs.rs @@ -0,0 +1,197 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use serde_json::Value; + +use super::JmapImporter; +use crate::archive::BlobId; +use crate::error::Error; +use crate::jmap::error::JmapError; +use crate::jmap::session::Lane; +use crate::sync::emailmeta::EmailMeta; +use crate::sync::pool::Pool; +use crate::types::ObjectType; + +const EMAIL_MEDIA_TYPE: &str = "message/rfc822"; +const SIEVE_MEDIA_TYPE: &str = "application/sieve"; +const OCTET_STREAM: &str = "application/octet-stream"; + +pub(super) struct BlobRef { + blob_id: String, + type_hint: Cow<'static, str>, + name: Cow<'static, str>, +} + +type Downloaded = (String, Result, JmapError>); + +impl BlobRef { + pub(super) fn collect<'a>( + ty: ObjectType, + objects: impl Iterator, + ) -> Vec { + let mut seen: HashSet<&'a str> = HashSet::new(); + let mut refs = Vec::new(); + for o in objects { + match ty { + ObjectType::Email => { + BlobRef::push_top(o, &mut seen, &mut refs, EMAIL_MEDIA_TYPE, "email") + } + ObjectType::SieveScript => { + BlobRef::push_top(o, &mut seen, &mut refs, SIEVE_MEDIA_TYPE, "script") + } + ObjectType::FileNode => { + if let Some(b) = o.get("blobId").and_then(Value::as_str) + && seen.insert(b) + { + let media_type = o + .get("mediaType") + .and_then(Value::as_str) + .unwrap_or(OCTET_STREAM); + let name = o.get("name").and_then(Value::as_str).unwrap_or("file"); + refs.push(BlobRef { + blob_id: b.to_owned(), + type_hint: Cow::Owned(media_type.to_owned()), + name: Cow::Owned(name.to_owned()), + }); + } + } + ObjectType::ContactCard | ObjectType::CalendarEvent => { + BlobRef::walk(o, &mut seen, &mut refs) + } + _ => return refs, + } + } + refs + } + + fn push_top<'a>( + object: &'a Value, + seen: &mut HashSet<&'a str>, + refs: &mut Vec, + type_hint: &'static str, + name: &'static str, + ) { + if let Some(b) = object.get("blobId").and_then(Value::as_str) + && seen.insert(b) + { + refs.push(BlobRef { + blob_id: b.to_owned(), + type_hint: Cow::Borrowed(type_hint), + name: Cow::Borrowed(name), + }); + } + } + + fn walk<'a>(value: &'a Value, seen: &mut HashSet<&'a str>, refs: &mut Vec) { + match value { + Value::Object(map) => { + if let Some(Value::String(b)) = map.get("blobId") + && seen.insert(b) + { + refs.push(BlobRef { + blob_id: b.clone(), + type_hint: Cow::Borrowed(OCTET_STREAM), + name: Cow::Borrowed("blob"), + }); + } + for v in map.values() { + BlobRef::walk(v, seen, refs); + } + } + Value::Array(items) => { + for v in items { + BlobRef::walk(v, seen, refs); + } + } + _ => {} + } + } +} + +pub(super) struct StoredEmail { + pub(super) match_key: String, + pub(super) date_header: Option, +} + +pub(super) struct StoredBlob { + pub(super) local_id: BlobId, + pub(super) email: Option, +} + +#[derive(Default)] +pub(super) struct StoredBlobs(HashMap); + +impl StoredBlobs { + pub(super) fn get(&self, jmap_blob_id: &str, kind: &str) -> Result<&StoredBlob, JmapError> { + self.0 + .get(jmap_blob_id) + .ok_or_else(|| JmapError::malformed(format!("{kind} blob missing"))) + } +} + +impl StoredBlobs { + pub(super) fn intern(&self, jmap_blob_id: &str) -> Result { + self.0 + .get(jmap_blob_id) + .map(|b| b.local_id) + .ok_or_else(|| JmapError::malformed(format!("blob {jmap_blob_id} was not prefetched"))) + } +} + +impl JmapImporter<'_> { + pub(super) fn download_blobs( + &self, + ty: ObjectType, + refs: Vec, + ) -> Result { + if refs.is_empty() { + return Ok(StoredBlobs::default()); + } + let workers = self.account.limits().workers(self.threads, Lane::Upload); + let account = Arc::clone(self.account); + let pool: Pool = Pool::new(workers, move |r: BlobRef| { + let result = account.download(&r.blob_id, &r.type_hint, &r.name); + (r.blob_id, result) + }); + let total = refs.len(); + for r in refs { + pool.submit(r); + } + let blobs = self.store.blobs(); + let mut out = HashMap::with_capacity(total); + for _ in 0..total { + let Ok((blob_id, result)) = pool.results().recv() else { + break; + }; + match result { + Ok(bytes) => { + let local_id = blobs.intern(&bytes).map_err(Error::partial)?; + let email = (ty == ObjectType::Email).then(|| { + let meta = EmailMeta::from_blob(&bytes); + StoredEmail { + match_key: meta.index.to_json(), + date_header: meta.date, + } + }); + out.insert(blob_id, StoredBlob { local_id, email }); + } + Err(e) => self + .logger + .warn(format_args!("blob {blob_id} download failed: {e}")), + } + } + self.logger.progress(format_args!( + " Blobs: {} / {} downloaded", + out.len(), + total + )); + Ok(StoredBlobs(out)) + } +} diff --git a/src/import/jmap/kinds.rs b/src/import/jmap/kinds.rs new file mode 100644 index 0000000..a51c15b --- /dev/null +++ b/src/import/jmap/kinds.rs @@ -0,0 +1,85 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::archive::rows::MemberList; +use crate::types::ObjectType; + +const MAILBOX_PROPS: &[&str] = &["name", "parentId", "role", "sortOrder", "isSubscribed"]; +const EMAIL_PROPS: &[&str] = &["blobId", "receivedAt", "mailboxIds", "keywords"]; +const EMAIL_UPDATE_PROPS: &[&str] = &["mailboxIds", "keywords"]; +const SIEVE_PROPS: &[&str] = &["name", "isActive", "blobId"]; +const FILE_NODE_PROPS: &[&str] = &[ + "@type", + "parentId", + "blobId", + "target", + "name", + "mediaType", + "executable", + "created", + "modified", + "isSubscribed", + "role", +]; + +pub(super) const IMPORT_ORDER: [ObjectType; 10] = [ + ObjectType::Mailbox, + ObjectType::AddressBook, + ObjectType::Calendar, + ObjectType::FileNode, + ObjectType::Identity, + ObjectType::SieveScript, + ObjectType::ParticipantIdentity, + ObjectType::Email, + ObjectType::ContactCard, + ObjectType::CalendarEvent, +]; + +impl ObjectType { + pub(super) const fn is_queryless(self) -> bool { + matches!( + self, + ObjectType::Identity + | ObjectType::AddressBook + | ObjectType::Calendar + | ObjectType::ParticipantIdentity + ) + } + + pub(super) const fn is_tree(self) -> bool { + matches!(self, ObjectType::Mailbox | ObjectType::FileNode) + } + + pub(super) const fn supports_changes(self) -> bool { + !matches!(self, ObjectType::SieveScript) + } + + pub(super) const fn get_props(self) -> Option<&'static [&'static str]> { + match self { + ObjectType::Mailbox => Some(MAILBOX_PROPS), + ObjectType::Email => Some(EMAIL_PROPS), + ObjectType::SieveScript => Some(SIEVE_PROPS), + ObjectType::FileNode => Some(FILE_NODE_PROPS), + _ => None, + } + } + + pub(super) const fn update_props(self) -> Option<&'static [&'static str]> { + match self { + ObjectType::Email => Some(EMAIL_UPDATE_PROPS), + other => other.get_props(), + } + } + + pub(super) const fn member_list(self) -> Option { + match self { + ObjectType::Mailbox => Some(MemberList::EmailMailboxes), + ObjectType::Calendar => Some(MemberList::EventCalendars), + ObjectType::AddressBook => Some(MemberList::CardAddressBooks), + _ => None, + } + } +} diff --git a/src/import/jmap/mod.rs b/src/import/jmap/mod.rs new file mode 100644 index 0000000..e1d5bfa --- /dev/null +++ b/src/import/jmap/mod.rs @@ -0,0 +1,248 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +mod blobs; +mod kinds; +mod reconcile; +mod tree; + +use std::mem; +use std::sync::Arc; + +use serde_json::Value; + +use self::kinds::IMPORT_ORDER; +use crate::archive::sources::{SourceChange, SourceKey, SourceKind}; +use crate::archive::{Archive, SourceId, Store}; +use crate::console::Initial; +use crate::error::Error; +use crate::jmap::account::AccountSelector; +use crate::jmap::connect::{ConnectConfig, JmapAccount}; +use crate::jmap::http::JmapClient; +use crate::jmap::request::URN_VACATION_RESPONSE; +use crate::jmap::wire::JmapId; +use crate::jmap::wire::vacation_response::{self, VacationResponse}; +use crate::logging::Logger; +use crate::net::auth::Auth; +use crate::sync::diff::IdDiff; +use crate::sync::run::{GcFailure, ImportRun, SourceClaim, SourceLabel}; +use crate::sync::{PlanCounts, RunOptions, RunOutcome, Summary, SummaryType, TypeCounts}; +use crate::types::{ObjectType, ObjectTypes}; + +const BEARER_USERNAME: &str = "(bearer)"; + +pub struct JmapImport { + pub connect: ConnectConfig, + pub objects: Option, + pub source_change: SourceChange, +} + +impl JmapImport { + pub fn run(self, options: RunOptions) -> RunOutcome { + let archive = match Archive::open(&options.archive) { + Ok(archive) => archive, + Err(e) => return RunOutcome::failed(Error::from(e)), + }; + let client = JmapClient::new(self.connect.auth.clone(), options.client_options()); + let account = match JmapAccount::connect(client, &self.connect) { + Ok(account) => Arc::new(account), + Err(e) => return RunOutcome::failed(e), + }; + let key = SourceKey { + kind: SourceKind::Jmap, + url: &self.connect.url, + account_id: account.account_id(), + }; + let label = SourceLabel { + name: match &self.connect.account { + AccountSelector::Name(name) => Some(name), + AccountSelector::Id(_) => None, + }, + username: match account.client().auth() { + Auth::Basic { user, .. } => user, + Auth::Bearer { .. } => BEARER_USERNAME, + }, + }; + let mut run = match ImportRun::start_on(archive, options, &key, label, self.source_change) { + Ok(run) => run, + Err(e) => return RunOutcome::failed(e), + }; + + let mut summary = Summary::default(); + let result = self.import_types(&run, &account, &mut summary); + let stats = account.client().stats(); + let plan = mem::take(&mut summary.plan); + *run.summary() = summary; + match result { + Err(e) => run.abort(e, stats), + Ok(()) if matches!(run.claim(), SourceClaim::DryRun(_)) => RunOutcome { + summary: Summary { + per_type: mem::take(&mut run.summary().per_type), + plan, + ..Summary::default() + }, + error: None, + }, + Ok(()) => run.finish(stats, GcFailure::WarnAsPartial), + } + } + + fn work_list<'a>(&'a self, account: &'a JmapAccount) -> impl Iterator + 'a { + IMPORT_ORDER.into_iter().filter(move |ty| { + account.supports(*ty) && self.objects.as_ref().is_none_or(|s| s.contains(ty)) + }) + } + + fn import_types( + &self, + run: &ImportRun, + account: &Arc, + summary: &mut Summary, + ) -> Result<(), Error> { + let logger = run.options().logger; + let store = run.store(); + match *run.claim() { + SourceClaim::DryRun(found) => { + for ty in self.work_list(account) { + logger.info(format_args!("Importing {}", Initial::Lower(ty.label()))); + let mut counts = TypeCounts::default(); + let result = Listing::plan(store, found, account, ty) + .map(|plan| *summary.plan(ty) = plan); + JmapImport::settle(logger, ty.into(), result, &mut counts)?; + *summary.counts(ty) = counts; + } + Ok(()) + } + SourceClaim::Registered(source) => { + let importer = JmapImporter { + store, + source, + account, + threads: run.options().threads, + logger, + }; + for ty in self.work_list(account) { + logger.info(format_args!("Importing {}", Initial::Lower(ty.label()))); + let mut counts = TypeCounts::default(); + let result = importer.reconcile(ty, &mut counts); + JmapImport::settle(logger, ty.into(), result, &mut counts)?; + *summary.counts(ty) = counts; + } + if self + .objects + .as_ref() + .is_none_or(|s| s.contains(&ObjectType::SieveScript)) + && account + .session() + .may_support(account.account_id(), URN_VACATION_RESPONSE) + { + logger.info(format_args!("Importing the vacation response")); + let mut counts = TypeCounts::default(); + let result = importer + .import_vacation_response() + .map(|fetched| counts.fetched = fetched); + JmapImport::settle(logger, SummaryType::VacationResponse, result, &mut counts)?; + *summary.counts(SummaryType::VacationResponse) = counts; + } + Ok(()) + } + } + } + + fn settle( + logger: Logger, + ty: SummaryType, + result: Result<(), Error>, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + match result { + Ok(()) => Ok(()), + Err(e) if e.aborts_run() => Err(e), + Err(e) => { + logger.warn(format_args!( + "{} aborted: {e}; continuing, the run will exit 5", + Initial::Lower(ty.label()) + )); + counts.failed += 1; + Ok(()) + } + } + } +} + +struct Listing { + ids: Vec, + preloaded: Option>, + state: Option, +} + +impl Listing { + fn plan( + store: Store<'_>, + source: Option, + account: &JmapAccount, + ty: ObjectType, + ) -> Result { + let local = match source { + Some(source) => store.jmap_ids(source).all(ty).map_err(Error::partial)?, + None => Default::default(), + }; + let listing = Listing::fetch(account, ty)?; + Ok(IdDiff::compute(listing.ids, &local).plan()) + } + + fn fetch(account: &JmapAccount, ty: ObjectType) -> Result { + if ty.is_queryless() { + let got = account.get_all::(ty.jmap_name())?; + let ids = got + .list + .iter() + .filter_map(|v| v.get("id").and_then(Value::as_str).map(str::to_owned)) + .collect(); + Ok(Listing { + ids, + preloaded: Some(got.list), + state: got.state, + }) + } else { + let ids = account.query_all_ids(ty.jmap_name())?; + Ok(Listing { + ids: ids.into_iter().map(|JmapId(id)| id).collect(), + preloaded: None, + state: None, + }) + } + } +} + +struct JmapImporter<'a> { + store: Store<'a>, + source: SourceId, + account: &'a Arc, + threads: usize, + logger: Logger, +} + +impl JmapImporter<'_> { + fn import_vacation_response(&self) -> Result { + let got = self + .account + .get_all::(vacation_response::TYPE_NAME)?; + let vacation = self.store.vacation(); + match got.list.iter().find(|v| !v.is_unset()) { + Some(found) => { + vacation + .store(self.source, &found.record()) + .map_err(Error::partial)?; + Ok(1) + } + None => { + vacation.clear(self.source).map_err(Error::partial)?; + Ok(0) + } + } + } +} diff --git a/src/import/jmap/reconcile.rs b/src/import/jmap/reconcile.rs new file mode 100644 index 0000000..05a2ac1 --- /dev/null +++ b/src/import/jmap/reconcile.rs @@ -0,0 +1,599 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use serde::Deserialize; +use serde_json::Value; + +use super::blobs::{BlobRef, StoredBlobs}; +use super::tree::TreeOrder; +use super::{JmapImporter, Listing}; +use crate::archive::rows::Write; +use crate::archive::{BlobId, LocalId, Store}; +use crate::console::Initial; +use crate::error::Error; +use crate::jmap::error::JmapError; +use crate::jmap::session::Lane; +use crate::jmap::wire::JmapId; +use crate::jmap::wire::address_book::AddressBook; +use crate::jmap::wire::calendar::Calendar; +use crate::jmap::wire::calendar_event::CalendarEvent; +use crate::jmap::wire::common::Lenient; +use crate::jmap::wire::contact_card::ContactCard; +use crate::jmap::wire::email::Email; +use crate::jmap::wire::file_node::{FileNode, FileProps, NodeKind}; +use crate::jmap::wire::identity::Identity; +use crate::jmap::wire::mailbox::Mailbox; +use crate::jmap::wire::participant_identity::ParticipantIdentity; +use crate::jmap::wire::sieve_script::SieveScript; +use crate::sync::TypeCounts; +use crate::sync::diff::IdDiff; +use crate::sync::pool::Pool; +use crate::sync::vanish::{Children, OnRowError, VanishedSweep}; +use crate::types::ObjectType; + +const WRITE_BATCH: usize = 200; + +type Fetched = Result, JmapError>; + +enum Cursor { + Stored(String), + Fresh(Option), +} + +impl JmapImporter<'_> { + pub(super) fn reconcile(&self, ty: ObjectType, counts: &mut TypeCounts) -> Result<(), Error> { + let ids = self.store.jmap_ids(self.source); + let local = ids.all(ty).map_err(Error::partial)?; + let listing = Listing::fetch(self.account, ty)?; + let diff = IdDiff::compute(listing.ids, &local); + + let stored = self + .store + .jmap_state(self.source) + .get(ty) + .map_err(Error::partial)?; + let cursor = match stored { + Some(since) if ty.supports_changes() => Cursor::Stored(since), + _ if !ty.supports_changes() => Cursor::Fresh(None), + _ if ty.is_queryless() => Cursor::Fresh(listing.state), + None => Cursor::Fresh(self.account.state(ty.jmap_name())?), + Some(_) => Cursor::Fresh(None), + }; + + if !diff.new.is_empty() { + let objects = match listing.preloaded { + Some(list) => { + let want: HashSet<&str> = diff.new.iter().map(String::as_str).collect(); + list.into_iter() + .filter(|v| { + v.get("id") + .and_then(Value::as_str) + .is_some_and(|s| want.contains(s)) + }) + .collect() + } + None => { + let new: Vec<&str> = diff.new.iter().map(String::as_str).collect(); + self.fetch(ty, &new, ty.get_props(), counts) + } + }; + self.insert_objects(ty, objects, &local, counts)?; + } + + if !diff.vanished.is_empty() { + self.delete_vanished(ty, &diff.vanished)?; + counts.deleted += diff.vanished.len() as u64; + } + + self.reconcile_updates(ty, &diff.present, cursor, counts)?; + + self.logger.info(format_args!( + " {}: {} fetched, {} deleted, {} skipped, {} failed", + ty.label(), + counts.fetched, + counts.deleted, + counts.skipped, + counts.failed + )); + Ok(()) + } + + fn fetch( + &self, + ty: ObjectType, + ids: &[&str], + props: Option<&'static [&'static str]>, + counts: &mut TypeCounts, + ) -> Vec { + let chunk = + usize::try_from(self.account.limits().max_objects_in_get.max(1)).unwrap_or(usize::MAX); + let workers = self.account.limits().workers(self.threads, Lane::Api); + let account = Arc::clone(self.account); + let pool: Pool, Fetched> = Pool::new(workers, move |ids: Vec| { + account + .get::(ty.jmap_name(), &ids, props) + .map(|r| r.list) + }); + + let mut batches = 0u64; + for part in ids.chunks(chunk) { + pool.submit(part.iter().map(|id| JmapId((*id).to_owned())).collect()); + batches += 1; + } + + let mut out = Vec::with_capacity(ids.len()); + let mut done = 0u64; + for result in pool.finish() { + match result { + Ok(list) => out.extend(list), + Err(e) => { + self.logger + .warn(format_args!("{}/get chunk failed: {e}", ty.jmap_name())); + counts.failed += 1; + } + } + done += 1; + } + self.logger.progress(format_args!( + " {}: {done} / {batches} chunks fetched", + ty.label() + )); + out + } + + fn insert_objects( + &self, + ty: ObjectType, + objects: Vec, + known: &HashMap, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let (ordered, warnings) = TreeOrder::of_objects(ty, &objects, |p| known.contains_key(p)); + for w in &warnings { + self.logger.warn(format_args!("{w}")); + } + + for batch in ordered.chunks(WRITE_BATCH) { + let refs = BlobRef::collect(ty, batch.iter().filter_map(|&i| objects.get(i))); + let blobs = self.download_blobs(ty, refs)?; + let tx = self.store.begin().map_err(Error::partial)?; + let store = tx.store(); + for obj in batch.iter().filter_map(|&i| objects.get(i)) { + let Some(jmap_id) = obj.get("id").and_then(Value::as_str) else { + counts.failed += 1; + continue; + }; + match self.insert_one(store, ty, obj, &blobs) { + Ok(local_id) => { + store + .jmap_ids(self.source) + .insert(ty, jmap_id, local_id) + .map_err(Error::partial)?; + counts.fetched += 1; + } + Err(e) => { + self.logger + .warn(format_args!("{} {jmap_id} skipped: {e}", ty.noun())); + counts.failed += 1; + } + } + } + tx.commit().map_err(Error::partial)?; + } + Ok(()) + } + + fn insert_one( + &self, + store: Store<'_>, + ty: ObjectType, + obj: &Value, + blobs: &StoredBlobs, + ) -> Result { + let ids = store.jmap_ids(self.source); + let resolve = |ty: ObjectType, id: &str| ids.local_id(ty, id).ok().flatten(); + let mut intern = |id: &str| blobs.intern(id); + match ty { + ObjectType::Mailbox => Mailbox::deserialize(obj)?.insert(store, &resolve), + ObjectType::Identity => Identity::deserialize(obj)?.insert(store), + ObjectType::AddressBook => AddressBook::deserialize(obj)?.insert(store), + ObjectType::Calendar => Calendar::deserialize(obj)?.insert(store), + ObjectType::ParticipantIdentity => ParticipantIdentity::deserialize(obj)?.insert(store), + ObjectType::Email => { + let mut w = Email::deserialize(obj)?; + let stored = blobs.get(&w.blob_id.0, "email")?; + let meta = stored + .email + .as_ref() + .ok_or_else(|| JmapError::malformed("email blob missing"))?; + if w.received_at.is_none() { + w.received_at = meta + .date_header + .as_deref() + .and_then(Lenient::parse_utc_date); + let id = obj.get("id").and_then(Value::as_str).unwrap_or("?"); + let raw = obj + .get("receivedAt") + .and_then(Value::as_str) + .unwrap_or(""); + self.logger.warn(format_args!( + "email {id}: unusable receivedAt {raw:?}; falling back to {}", + match &w.received_at { + Some(_) => "the Date header", + None => "1970-01-01T00:00:00Z", + } + )); + } + w.insert(store, stored.local_id, &meta.match_key, &resolve) + } + ObjectType::SieveScript => { + let w = SieveScript::deserialize(obj)?; + let blob = blobs.get(&w.blob_id.0, "sieve")?.local_id; + w.insert(store, blob) + } + ObjectType::FileNode => { + let w = FileNode::deserialize(obj)?; + let blob = JmapImporter::file_blob(&w, blobs)?; + w.insert(store, blob, &resolve) + } + ObjectType::ContactCard => { + ContactCard::deserialize(obj)?.insert(store, &resolve, &mut intern) + } + ObjectType::CalendarEvent => { + CalendarEvent::deserialize(obj)?.insert(store, &resolve, &mut intern) + } + } + } + + fn file_blob(node: &FileNode, blobs: &StoredBlobs) -> Result, JmapError> { + match &node.kind { + NodeKind::File(FileProps { + blob_id: Some(b), .. + }) => Ok(Some(blobs.get(&b.0, "file")?.local_id)), + _ => Ok(None), + } + } + + fn reconcile_updates( + &self, + ty: ObjectType, + present: &[(String, LocalId)], + cursor: Cursor, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + let in_order = || present.iter().map(|(id, _)| id.as_str()); + let by_id: HashMap<&str, LocalId> = present + .iter() + .map(|(id, local)| (id.as_str(), *local)) + .collect(); + let since = match cursor { + Cursor::Stored(since) => since, + Cursor::Fresh(run_state) => { + let clean = self.fetch_and_update(ty, in_order(), &by_id, counts)?; + if let Some(state) = run_state { + self.advance_state(ty, &state, clean)?; + } + return Ok(()); + } + }; + match self.account.changes(ty.jmap_name(), &since) { + Ok(changes) => { + let updated = changes + .updated + .iter() + .map(|j| j.0.as_str()) + .filter(|j| by_id.contains_key(j)); + let clean = self.fetch_and_update(ty, updated, &by_id, counts)?; + self.advance_state(ty, &changes.new_state, clean) + } + Err(err @ (JmapError::CannotCalculateChanges | JmapError::UnknownMethod)) => { + let reason = if matches!(err, JmapError::UnknownMethod) { + "server does not implement /changes" + } else { + "server cannot calculate changes from stored state" + }; + self.logger.warn(format_args!( + "{}: {reason}; refreshing every present one", + Initial::Lower(ty.label()) + )); + let captured = self.account.state(ty.jmap_name())?; + let clean = self.fetch_and_update(ty, in_order(), &by_id, counts)?; + match captured { + Some(state) => self.advance_state(ty, &state, clean), + None => Ok(()), + } + } + Err(e) => Err(Error::from(e)), + } + } + + fn advance_state(&self, ty: ObjectType, state: &str, clean: bool) -> Result<(), Error> { + if !clean { + self.logger.warn(format_args!( + "{}: holding the sync state; some updates failed and are retried on the next run", + Initial::Lower(ty.label()) + )); + return Ok(()); + } + self.store + .jmap_state(self.source) + .upsert(ty, state) + .map_err(Error::partial) + } + + fn fetch_and_update<'i>( + &self, + ty: ObjectType, + ids: impl Iterator, + present: &HashMap<&str, LocalId>, + counts: &mut TypeCounts, + ) -> Result { + let ids: Vec<&str> = ids.collect(); + if ids.is_empty() { + return Ok(true); + } + let failed_before = counts.failed; + let objects = self.fetch(ty, &ids, ty.update_props(), counts); + self.update_objects(ty, &objects, present, counts)?; + Ok(counts.failed == failed_before) + } + + fn update_objects( + &self, + ty: ObjectType, + objects: &[Value], + present: &HashMap<&str, LocalId>, + counts: &mut TypeCounts, + ) -> Result<(), Error> { + for batch in objects.chunks(WRITE_BATCH) { + let blobs = self.download_blobs(ty, BlobRef::collect(ty, batch.iter()))?; + let tx = self.store.begin().map_err(Error::partial)?; + let store = tx.store(); + for obj in batch { + let Some(jmap_id) = obj.get("id").and_then(Value::as_str) else { + counts.failed += 1; + continue; + }; + let Some(&local_id) = present.get(jmap_id) else { + continue; + }; + match self.update_one(store, ty, local_id, obj, &blobs) { + Ok(Write::Changed) => counts.updated += 1, + Ok(Write::Unchanged) => {} + Err(e) => { + self.logger + .warn(format_args!("{} {jmap_id} update skipped: {e}", ty.noun())); + counts.failed += 1; + } + } + } + tx.commit().map_err(Error::partial)?; + } + Ok(()) + } + + fn update_one( + &self, + store: Store<'_>, + ty: ObjectType, + local_id: LocalId, + obj: &Value, + blobs: &StoredBlobs, + ) -> Result { + let ids = store.jmap_ids(self.source); + let resolve = |ty: ObjectType, id: &str| ids.local_id(ty, id).ok().flatten(); + let mut intern = |id: &str| blobs.intern(id); + match ty { + ObjectType::Mailbox => Mailbox::deserialize(obj)?.update(store, local_id, &resolve), + ObjectType::Identity => Identity::deserialize(obj)?.update(store, local_id), + ObjectType::AddressBook => AddressBook::deserialize(obj)?.update(store, local_id), + ObjectType::Calendar => Calendar::deserialize(obj)?.update(store, local_id), + ObjectType::ParticipantIdentity => { + ParticipantIdentity::deserialize(obj)?.update(store, local_id) + } + ObjectType::Email => Email::update_placement(store, local_id, obj, &resolve), + ObjectType::SieveScript => { + let w = SieveScript::deserialize(obj)?; + let blob = blobs.get(&w.blob_id.0, "sieve")?.local_id; + w.update(store, local_id, blob) + } + ObjectType::FileNode => { + let w = FileNode::deserialize(obj)?; + let blob = JmapImporter::file_blob(&w, blobs)?; + w.update(store, local_id, blob, &resolve) + } + ObjectType::ContactCard => { + ContactCard::deserialize(obj)?.update(store, local_id, &resolve, &mut intern) + } + ObjectType::CalendarEvent => { + CalendarEvent::deserialize(obj)?.update(store, local_id, &resolve, &mut intern) + } + } + } + + fn delete_vanished(&self, ty: ObjectType, vanished: &[(String, LocalId)]) -> Result<(), Error> { + let children = if ty.is_tree() { + Children::DetachSurvivors + } else { + Children::Restrict + }; + let mut scratch = TypeCounts::default(); + VanishedSweep::new(self.store, OnRowError::Abort, children) + .run_then( + ty, + vanished, + &mut scratch, + |store, jmap_id, _| store.jmap_ids(self.source).delete(ty, jmap_id), + |_, _, _| {}, + |store| match ty.member_list() { + Some(list) => { + let removed: HashSet = + vanished.iter().map(|(_, id)| *id).collect(); + store.prune_members(list, &removed) + } + None => Ok(()), + }, + ) + .map_err(Error::partial)?; + self.logger.progress(format_args!( + " {}: {} vanished from the source, deleted", + ty.label(), + vanished.len() + )); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use serde_json::json; + + use super::*; + use crate::archive::sources::{SourceKey, SourceKind}; + use crate::archive::{Archive, SourceId}; + use crate::jmap::connect::JmapAccount; + use crate::jmap::http::JmapClient; + use crate::jmap::session::Session; + use crate::logging::{Level, Logger}; + use crate::net::auth::Auth; + use crate::net::http::{ClientOptions, TlsPolicy}; + use crate::net::retry::RetryPolicy; + + fn account() -> Arc { + let session: Session = serde_json::from_value(json!({ + "apiUrl": "http://unused.invalid/api", + "uploadUrl": "http://unused.invalid/up", + "downloadUrl": "http://unused.invalid/down", + "capabilities": { "urn:ietf:params:jmap:core": { + "maxObjectsInGet": 500, "maxObjectsInSet": 500, "maxCallsInRequest": 16, + "maxConcurrentRequests": 4, "maxConcurrentUpload": 4, + "maxSizeRequest": 10000000, "maxSizeUpload": 50000000 } }, + "accounts": { "w": { "name": "n" } } + })) + .unwrap(); + let limits = session.core_limits().unwrap(); + let client = JmapClient::new( + Auth::Basic { + user: "u".into(), + password: "p".into(), + }, + ClientOptions { + retry: RetryPolicy::new(0), + tls: TlsPolicy::Verify, + logger: Logger::new(Level::Quiet), + }, + ); + Arc::new(JmapAccount::from_parts(client, session, "w".into(), limits)) + } + + fn source(archive: &Archive) -> SourceId { + archive + .store() + .sources() + .upsert( + &SourceKey { + kind: SourceKind::Jmap, + url: "u", + account_id: "w", + }, + None, + "u", + ) + .unwrap() + } + + fn sweep(archive: &Archive, source: SourceId, ty: ObjectType, vanished: &[(&str, i64)]) { + let account = account(); + let importer = JmapImporter { + store: archive.store(), + source, + account: &account, + threads: 1, + logger: Logger::new(Level::Quiet), + }; + let vanished: Vec<(String, LocalId)> = vanished + .iter() + .map(|(k, id)| ((*k).to_owned(), LocalId::from_raw(*id))) + .collect(); + importer.delete_vanished(ty, &vanished).unwrap(); + } + + #[test] + fn vanished_tree_deletes_leaf_first_and_nulls_true_orphan() { + let archive = Archive::open_in_memory().unwrap(); + let sid = source(&archive); + let c = archive.connection(); + c.execute_batch( + "INSERT INTO mailboxes (id,name,parent_id) VALUES (1,'root',NULL); + INSERT INTO mailboxes (id,name,parent_id) VALUES (2,'mid',1); + INSERT INTO mailboxes (id,name,parent_id) VALUES (3,'leaf',2); + INSERT INTO mailboxes (id,name,parent_id) VALUES (4,'survivor',2);", + ) + .unwrap(); + let ids = archive.store().jmap_ids(sid); + for (jid, local) in [("R", 1), ("M", 2), ("L", 3), ("S", 4)] { + ids.insert(ObjectType::Mailbox, jid, LocalId::from_raw(local)) + .unwrap(); + } + sweep( + &archive, + sid, + ObjectType::Mailbox, + &[("L", 3), ("M", 2), ("R", 1)], + ); + let remaining: i64 = c + .query_row("SELECT count(*) FROM mailboxes", [], |r| r.get(0)) + .unwrap(); + assert_eq!(remaining, 1, "only the surviving child remains"); + let parent: Option = c + .query_row("SELECT parent_id FROM mailboxes WHERE id=4", [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!(parent, None, "true orphan's parent was nulled"); + assert!( + ids.local_id(ObjectType::Mailbox, "L").unwrap().is_none(), + "vanished sync_id rows removed" + ); + } + + #[test] + fn vanished_mailbox_is_pruned_from_email_mailbox_ids() { + let archive = Archive::open_in_memory().unwrap(); + let sid = source(&archive); + let c = archive.connection(); + c.execute_batch( + "INSERT INTO mailboxes (id,name,parent_id) VALUES (1,'a',NULL); + INSERT INTO mailboxes (id,name,parent_id) VALUES (2,'b',NULL);", + ) + .unwrap(); + let blob = archive.store().blobs().intern(b"msg").unwrap(); + c.execute( + "INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords) + VALUES (?1,'2020-01-01T00:00:00Z','[1,2]','[]')", + rusqlite::params![blob], + ) + .unwrap(); + let ids = archive.store().jmap_ids(sid); + ids.insert(ObjectType::Mailbox, "A", LocalId::from_raw(1)) + .unwrap(); + ids.insert(ObjectType::Mailbox, "B", LocalId::from_raw(2)) + .unwrap(); + sweep(&archive, sid, ObjectType::Mailbox, &[("A", 1)]); + let mailbox_ids: String = c + .query_row("SELECT mailbox_ids FROM emails", [], |r| r.get(0)) + .unwrap(); + assert_eq!( + mailbox_ids, "[2]", + "deleted mailbox id removed from referring email" + ); + } +} diff --git a/src/import/jmap/tree.rs b/src/import/jmap/tree.rs new file mode 100644 index 0000000..e7c72b6 --- /dev/null +++ b/src/import/jmap/tree.rs @@ -0,0 +1,224 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use serde_json::Value; + +use crate::types::ObjectType; + +pub(super) struct TreeOrder { + pub(super) order: Vec, + pub(super) orphans: Vec, + pub(super) cycle_roots: Vec, +} + +impl TreeOrder { + pub(super) fn compute( + items: &[(String, Option)], + is_known: impl Fn(&str) -> bool, + ) -> TreeOrder { + let index: HashMap<&str, usize> = items + .iter() + .enumerate() + .map(|(i, (id, _))| (id.as_str(), i)) + .collect(); + let mut order = Vec::with_capacity(items.len()); + let mut orphans = Vec::new(); + let mut cycle_roots = Vec::new(); + let mut placed = vec![false; items.len()]; + + while order.len() < items.len() { + let mut progressed = false; + for (i, (_, parent)) in items.iter().enumerate() { + if placed.get(i).copied().unwrap_or(true) { + continue; + } + let ready = match parent { + None => true, + Some(p) => match index.get(p.as_str()) { + Some(&pi) => placed.get(pi).copied().unwrap_or(false), + None => { + if !is_known(p) { + orphans.push(i); + } + true + } + }, + }; + if ready { + order.push(i); + if let Some(slot) = placed.get_mut(i) { + *slot = true; + } + progressed = true; + } + } + if !progressed { + let Some((root, _)) = items + .iter() + .zip(&placed) + .enumerate() + .filter(|(_, (_, placed))| !**placed) + .min_by(|(_, (a, _)), (_, (b, _))| a.0.cmp(&b.0)) + else { + break; + }; + cycle_roots.push(root); + order.push(root); + if let Some(slot) = placed.get_mut(root) { + *slot = true; + } + } + } + + TreeOrder { + order, + orphans, + cycle_roots, + } + } + + pub(super) fn of_objects( + ty: ObjectType, + objects: &[Value], + is_known: impl Fn(&str) -> bool, + ) -> (Vec, Vec) { + if !ty.is_tree() { + return ((0..objects.len()).collect(), Vec::new()); + } + let items: Vec<(String, Option)> = objects + .iter() + .map(|o| { + ( + o.get("id") + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned(), + o.get("parentId").and_then(Value::as_str).map(str::to_owned), + ) + }) + .collect(); + let result = TreeOrder::compute(&items, is_known); + let mut warnings = Vec::with_capacity(result.orphans.len() + result.cycle_roots.len()); + for (id, parent) in result.orphans.iter().filter_map(|&i| items.get(i)) { + warnings.push(format!( + "{} {id}: parent {} not present in batch or archive; inserting with NULL parent", + ty.jmap_name(), + parent.as_deref().unwrap_or(""), + )); + } + for (id, _) in result.cycle_roots.iter().filter_map(|&i| items.get(i)) { + warnings.push(format!( + "{} {id}: parent cycle detected; broken at this node", + ty.jmap_name(), + )); + } + (result.order, warnings) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use super::*; + + fn item(id: &str, parent: Option<&str>) -> (String, Option) { + (id.to_owned(), parent.map(str::to_owned)) + } + + fn pos(order: &[usize], items: &[(String, Option)], id: &str) -> usize { + order + .iter() + .position(|&i| items[i].0 == id) + .expect("id in order") + } + + #[test] + fn parents_precede_children() { + let items = vec![item("c", Some("b")), item("b", Some("a")), item("a", None)]; + let t = TreeOrder::compute(&items, |_| false); + assert_eq!(t.order.len(), 3); + assert!(pos(&t.order, &items, "a") < pos(&t.order, &items, "b")); + assert!(pos(&t.order, &items, "b") < pos(&t.order, &items, "c")); + assert!(t.orphans.is_empty()); + assert!(t.cycle_roots.is_empty()); + } + + #[test] + fn parent_not_fetched_and_not_existing_is_orphan() { + let items = vec![item("x", Some("missing"))]; + let t = TreeOrder::compute(&items, |_| false); + assert_eq!(t.order, vec![0]); + assert_eq!(t.orphans, vec![0]); + } + + #[test] + fn parent_already_in_db_is_not_orphan() { + let items = vec![item("x", Some("p"))]; + let existing: HashSet = ["p".to_owned()].into_iter().collect(); + let t = TreeOrder::compute(&items, |p| existing.contains(p)); + assert_eq!(t.order, vec![0]); + assert!(t.orphans.is_empty()); + } + + #[test] + fn cycle_is_broken_by_smallest_id() { + let items = vec![item("b", Some("a")), item("a", Some("b"))]; + let t = TreeOrder::compute(&items, |_| false); + assert_eq!(t.order.len(), 2); + assert_eq!(t.cycle_roots.len(), 1); + assert_eq!(items[t.cycle_roots[0]].0, "a"); + assert!(pos(&t.order, &items, "a") < pos(&t.order, &items, "b")); + } +} + +#[cfg(test)] +mod object_tests { + use std::collections::HashSet; + + use serde_json::json; + + use super::TreeOrder; + use crate::types::ObjectType; + + #[test] + fn order_objects_does_not_warn_when_parent_already_in_archive() { + let objects = vec![json!({"id":"child","parentId":"existing"})]; + let existing: HashSet = ["existing".to_owned()].into_iter().collect(); + let (ordered, warnings) = + TreeOrder::of_objects(ObjectType::Mailbox, &objects, |p| existing.contains(p)); + assert_eq!(ordered, vec![0]); + assert!( + warnings.is_empty(), + "no orphan warning expected: {warnings:?}" + ); + } + + #[test] + fn order_objects_warns_on_orphan_parent() { + let objects = vec![json!({"id":"orphan","parentId":"ghost"})]; + let (ordered, warnings) = TreeOrder::of_objects(ObjectType::Mailbox, &objects, |_| false); + assert_eq!(ordered, vec![0]); + assert_eq!(warnings.len(), 1); + assert!(warnings[0].contains("ghost"), "msg was: {}", warnings[0]); + } + + #[test] + fn order_objects_warns_on_cycle() { + let objects = vec![ + json!({"id":"a","parentId":"b"}), + json!({"id":"b","parentId":"a"}), + ]; + let (ordered, warnings) = TreeOrder::of_objects(ObjectType::Mailbox, &objects, |_| false); + assert_eq!(ordered.len(), 2); + assert!( + warnings.iter().any(|w| w.contains("cycle")), + "expected a cycle warning, got: {warnings:?}" + ); + } +} diff --git a/src/import/maildir/keywords.rs b/src/import/maildir/keywords.rs new file mode 100644 index 0000000..bda9bc3 --- /dev/null +++ b/src/import/maildir/keywords.rs @@ -0,0 +1,220 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use mail_parser::mailbox::maildir::Flag; +use time::OffsetDateTime; + +use crate::import::imap::DeletedMessages; +use crate::sync::keywords::Keywords; + +const MAX_DELIVERY_DIGITS: usize = 12; +const DELIVERY_CLOCK_SKEW_SECS: u64 = 86_400; + +#[derive(Debug, Default)] +pub(super) struct Translation { + pub keywords: Keywords, + pub has_trashed_flag: bool, +} + +impl Translation { + pub fn from_flags(flags: &[Flag], deleted: DeletedMessages) -> Translation { + let mut out = Translation { + keywords: Keywords::with_capacity(flags.len()), + has_trashed_flag: false, + }; + for flag in flags { + match flag { + Flag::Passed => out.keywords.push("$forwarded"), + Flag::Replied => out.keywords.push("$answered"), + Flag::Seen => out.keywords.push("$seen"), + Flag::Draft => out.keywords.push("$draft"), + Flag::Flagged => out.keywords.push("$flagged"), + Flag::Trashed => { + out.has_trashed_flag = true; + if deleted == DeletedMessages::Keep { + out.keywords.push("$deleted"); + } + } + } + } + out.keywords.sort(); + out + } +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct MaildirFilename<'a>(pub &'a str); + +impl<'a> MaildirFilename<'a> { + pub fn flags(self) -> Vec { + let Some((_, info)) = self.0.rsplit_once("2,") else { + return Vec::new(); + }; + let mut out = Vec::new(); + for &ch in info.as_bytes() { + match ch { + b'P' => out.push(Flag::Passed), + b'R' => out.push(Flag::Replied), + b'S' => out.push(Flag::Seen), + b'T' => out.push(Flag::Trashed), + b'D' => out.push(Flag::Draft), + b'F' => out.push(Flag::Flagged), + other if !other.is_ascii_alphanumeric() => break, + _ => {} + } + } + out + } + + pub fn unique_id(self) -> &'a str { + self.0.split_once(':').map_or(self.0, |(unique, _)| unique) + } + + pub fn delivery_time(self) -> Option { + let digits = self + .0 + .split(|c: char| !c.is_ascii_digit()) + .next() + .unwrap_or_default(); + if digits.is_empty() || digits.len() > MAX_DELIVERY_DIGITS { + return None; + } + let secs: u64 = digits.parse().ok()?; + if secs == 0 { + return None; + } + let now = u64::try_from(OffsetDateTime::now_utc().unix_timestamp()).unwrap_or(0); + if secs > now.saturating_add(DELIVERY_CLOCK_SKEW_SECS) { + return None; + } + Some(secs) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_flags_yields_empty_keywords() { + let t = Translation::from_flags(&[], DeletedMessages::Skip); + assert!(t.keywords.is_empty()); + assert!(!t.has_trashed_flag); + } + + #[test] + fn each_flag_maps_to_expected_jmap_keyword() { + let t = Translation::from_flags( + &[ + Flag::Passed, + Flag::Replied, + Flag::Seen, + Flag::Draft, + Flag::Flagged, + ], + DeletedMessages::Skip, + ); + let mut expected = vec!["$forwarded", "$answered", "$seen", "$draft", "$flagged"]; + expected.sort(); + assert_eq!(t.keywords.iter().collect::>(), expected); + assert!(!t.has_trashed_flag); + } + + #[test] + fn trashed_dropped_by_default_marks_flag() { + let t = Translation::from_flags(&[Flag::Trashed, Flag::Seen], DeletedMessages::Skip); + assert!(t.has_trashed_flag); + assert_eq!(t.keywords.iter().collect::>(), ["$seen"]); + } + + #[test] + fn trashed_with_include_deleted_adds_dollar_deleted() { + let t = Translation::from_flags(&[Flag::Trashed, Flag::Seen], DeletedMessages::Keep); + assert!(t.has_trashed_flag); + let mut expected = vec!["$seen", "$deleted"]; + expected.sort(); + assert_eq!(t.keywords.iter().collect::>(), expected); + } + + #[test] + fn order_independent_keywords_are_stable() { + let a = Translation::from_flags(&[Flag::Seen, Flag::Flagged], DeletedMessages::Skip); + let b = Translation::from_flags(&[Flag::Flagged, Flag::Seen], DeletedMessages::Skip); + assert_eq!(a.keywords.to_json(), b.keywords.to_json()); + } + + #[test] + fn duplicate_flags_dedup() { + let t = + Translation::from_flags(&[Flag::Seen, Flag::Seen, Flag::Seen], DeletedMessages::Skip); + assert_eq!(t.keywords.iter().collect::>(), ["$seen"]); + } + + #[test] + fn unique_id_strips_at_first_colon() { + assert_eq!( + MaildirFilename("1739471123.M001P01234V0.host:2,RS").unique_id(), + "1739471123.M001P01234V0.host" + ); + } + + #[test] + fn unique_id_no_colon_is_whole_basename() { + assert_eq!( + MaildirFilename("1739471123.M001P0.host").unique_id(), + "1739471123.M001P0.host" + ); + } + + #[test] + fn unique_id_strips_legacy_one_form() { + assert_eq!( + MaildirFilename("1234.M5.host:1,experimental").unique_id(), + "1234.M5.host" + ); + } + + #[test] + fn flags_from_filename_recognises_full_set() { + let mut flags = MaildirFilename("name:2,DFPRST").flags(); + flags.sort(); + let mut want = vec![ + Flag::Draft, + Flag::Flagged, + Flag::Passed, + Flag::Replied, + Flag::Seen, + Flag::Trashed, + ]; + want.sort(); + assert_eq!(flags, want); + } + + #[test] + fn flags_from_filename_no_info_section_yields_empty() { + assert!(MaildirFilename("plain").flags().is_empty()); + assert!(MaildirFilename("name:").flags().is_empty()); + assert!(MaildirFilename("name:1,X").flags().is_empty()); + } + + #[test] + fn flags_from_filename_dovecot_extension_metadata_passes_through() { + let mut flags = MaildirFilename("uid,S=1234,W=1300:2,RS").flags(); + flags.sort(); + let mut want = vec![Flag::Replied, Flag::Seen]; + want.sort(); + assert_eq!(flags, want); + } + + #[test] + fn flags_from_filename_unknown_alpha_chars_skipped() { + let mut flags = MaildirFilename("uid:2,SaT").flags(); + flags.sort(); + let mut want = vec![Flag::Seen, Flag::Trashed]; + want.sort(); + assert_eq!(flags, want); + } +} diff --git a/src/import/maildir/messages.rs b/src/import/maildir/messages.rs new file mode 100644 index 0000000..8997be8 --- /dev/null +++ b/src/import/maildir/messages.rs @@ -0,0 +1,669 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; +use std::time::UNIX_EPOCH; + +use mail_parser::mailbox::maildir::Flag; +use rusqlite::params; + +use crate::archive::rows::EmailRecord; +use crate::archive::{LocalId, SourceId, Store}; +use crate::console::Grouped; +use crate::import::imap::DeletedMessages; +use crate::logging::Logger; +use crate::sync::TypeCounts; +use crate::sync::dates::{MissingDate, Rfc3339}; +use crate::sync::emailmeta::EmailMeta; + +use super::keywords::{MaildirFilename, Translation}; + +const PROGRESS_TICK: u64 = 1000; +const EMPTY_KEYWORDS: &str = "[]"; + +#[derive(Debug, Clone)] +pub(super) struct DiskEntry { + pub unique_id: String, + pub filename: String, + pub path: PathBuf, + pub flags: Vec, + pub mtime_unix: u64, +} + +#[derive(Debug, Clone, Default)] +pub(super) struct DiskListing { + pub entries: Vec, + pub io_failures: u64, +} + +#[derive(Debug, Clone, Default)] +pub(super) struct FolderDiff { + pub new: Vec, + pub present: Vec<(DiskEntry, LocalId)>, + pub vanished: Vec<(String, LocalId)>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum PresentOutcome { + Unchanged, + KeywordsUpdated, + DeleteRequested, +} + +#[derive(Debug, thiserror::Error)] +pub(super) enum InsertError { + #[error("io: {0}")] + Io(#[from] io::Error), + #[error("db: {0}")] + Db(#[from] rusqlite::Error), +} + +#[derive(Default)] +pub(super) struct StoredKeywords(HashMap); + +#[derive(Clone, Copy)] +pub(super) struct FolderWriter<'a> { + pub store: Store<'a>, + pub source: SourceId, + pub folder: &'a str, + pub mailbox: LocalId, + pub deleted: DeletedMessages, +} + +impl DiskListing { + pub fn read(folder_path: &Path) -> io::Result { + let mut out = DiskListing::default(); + let mut seen: HashSet = HashSet::new(); + for sub in ["cur", "new"] { + let dir = folder_path.join(sub); + if !dir.is_dir() { + continue; + } + for entry in fs::read_dir(&dir)? { + let Ok(entry) = entry else { + out.io_failures += 1; + continue; + }; + let Ok(file_type) = entry.file_type() else { + out.io_failures += 1; + continue; + }; + if !file_type.is_file() { + continue; + } + let path = entry.path(); + let Some(raw_name) = path.file_name() else { + continue; + }; + let filename = raw_name.to_string_lossy().into_owned(); + if filename.starts_with('.') { + continue; + } + let name = MaildirFilename(&filename); + let unique_id = name.unique_id(); + if unique_id.is_empty() || !seen.insert(unique_id.to_owned()) { + continue; + } + let mtime_unix = entry + .metadata() + .and_then(|m| m.modified()) + .ok() + .and_then(|t| t.duration_since(UNIX_EPOCH).ok()) + .map_or(0, |d| d.as_secs()); + out.entries.push(DiskEntry { + unique_id: unique_id.to_owned(), + flags: name.flags(), + filename, + path, + mtime_unix, + }); + } + } + Ok(out) + } +} + +impl DiskEntry { + pub fn received_at(&self, date_header: Option<&str>) -> String { + if let Some(secs) = MaildirFilename(&self.filename).delivery_time() { + return Rfc3339::from_unix(secs); + } + if let Some(date) = date_header { + return date.to_owned(); + } + if self.mtime_unix > 0 { + return Rfc3339::from_unix(self.mtime_unix); + } + MissingDate::Now.resolve() + } +} + +impl FolderDiff { + pub fn compute(disk: Vec, local: &HashMap) -> FolderDiff { + let on_disk: HashSet<&str> = disk.iter().map(|e| e.unique_id.as_str()).collect(); + let mut vanished: Vec<(String, LocalId)> = local + .iter() + .filter(|(uid, _)| !on_disk.contains(uid.as_str())) + .map(|(uid, &id)| (uid.clone(), id)) + .collect(); + vanished.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + let mut diff = FolderDiff { + vanished, + ..FolderDiff::default() + }; + for entry in disk { + match local.get(&entry.unique_id) { + None => diff.new.push(entry), + Some(&id) => diff.present.push((entry, id)), + } + } + diff + } +} + +impl StoredKeywords { + pub fn load( + store: Store<'_>, + source: SourceId, + folder: &str, + ) -> rusqlite::Result { + let mut stmt = store.connection().prepare_cached( + "SELECT e.id, e.keywords FROM emails e + JOIN sync_id_maildir m ON m.local_id = e.id AND m.type_name = 'email' + WHERE m.source_id = ?1 AND m.folder = ?2", + )?; + let rows = stmt.query_map(params![source, folder], |row| { + Ok((row.get(0)?, row.get(1)?)) + })?; + rows.collect::>().map(StoredKeywords) + } + + pub fn get(&self, id: LocalId) -> &str { + self.0.get(&id).map_or(EMPTY_KEYWORDS, String::as_str) + } +} + +impl FolderWriter<'_> { + pub fn insert_new(&self, entry: &DiskEntry) -> Result, InsertError> { + let bytes = fs::read(&entry.path)?; + let translation = Translation::from_flags(&entry.flags, self.deleted); + if translation.has_trashed_flag && self.deleted == DeletedMessages::Skip { + return Ok(None); + } + let EmailMeta { + index, + date: date_header, + } = EmailMeta::from_blob(&bytes); + let received_at = entry.received_at(date_header.as_deref()); + let blob = self.store.blobs().intern(&bytes)?; + let message_match = index.to_json(); + let local = self.store.emails().insert(&EmailRecord { + blob, + received_at: &received_at, + mailboxes: &[self.mailbox], + keywords: &translation.keywords.to_json(), + message_match: Some(&message_match), + })?; + self.store + .maildir_ids(self.source) + .insert_item(self.folder, &entry.unique_id, local)?; + Ok(Some(local)) + } + + pub fn refresh_flags( + &self, + local: LocalId, + entry: &DiskEntry, + stored: &str, + ) -> rusqlite::Result { + let translation = Translation::from_flags(&entry.flags, self.deleted); + if translation.has_trashed_flag && self.deleted == DeletedMessages::Skip { + return Ok(PresentOutcome::DeleteRequested); + } + let expected = translation.keywords.to_json(); + if stored == expected { + return Ok(PresentOutcome::Unchanged); + } + self.store.emails().update_keywords(local, &expected)?; + Ok(PresentOutcome::KeywordsUpdated) + } + + pub fn delete(&self, unique_id: &str, local: LocalId) -> rusqlite::Result<()> { + self.store.emails().delete(local)?; + self.store + .maildir_ids(self.source) + .delete_item(self.folder, unique_id) + } + + pub fn apply( + &self, + diff: &FolderDiff, + stored: &StoredKeywords, + counts: &mut TypeCounts, + logger: Logger, + ) { + let folder = self.folder; + let total_new = diff.new.len(); + let mut inserted: u64 = 0; + for entry in &diff.new { + match self.insert_new(entry) { + Ok(Some(_)) => { + counts.created += 1; + counts.fetched += 1; + inserted += 1; + if inserted.is_multiple_of(PROGRESS_TICK) { + logger.progress(format_args!( + "Folder \"{folder}\": {} / {} inserted", + Grouped(inserted), + Grouped(total_new as u64) + )); + } + } + Ok(None) => counts.skipped += 1, + Err(e) => { + logger.warn(format_args!( + "cannot import \"{folder}/{}\": {e}", + entry.filename + )); + counts.failed += 1; + } + } + } + for (entry, local) in &diff.present { + match self.refresh_flags(*local, entry, stored.get(*local)) { + Ok(PresentOutcome::Unchanged) => counts.skipped += 1, + Ok(PresentOutcome::KeywordsUpdated) => counts.fetched += 1, + Ok(PresentOutcome::DeleteRequested) => { + match self.delete(&entry.unique_id, *local) { + Ok(()) => counts.deleted += 1, + Err(e) => { + logger.warn(format_args!( + "cannot delete trashed message \"{folder}/{}\": {e}", + entry.filename + )); + counts.failed += 1; + } + } + } + Err(e) => { + logger.warn(format_args!( + "cannot update the flags of \"{folder}/{}\": {e}", + entry.filename + )); + counts.failed += 1; + } + } + } + for (unique_id, local) in &diff.vanished { + match self.delete(unique_id, *local) { + Ok(()) => counts.deleted += 1, + Err(e) => { + logger.warn(format_args!( + "cannot delete vanished message \"{folder}/{unique_id}\": {e}" + )); + counts.failed += 1; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use time::OffsetDateTime; + + use super::*; + use crate::archive::Archive; + use crate::archive::rows::{MailboxRecord, Subscribed}; + use crate::archive::sources::{SourceKey, SourceKind}; + + struct Fixture { + archive: Archive, + source: SourceId, + mailbox: LocalId, + } + + impl Fixture { + fn new() -> Fixture { + let archive = Archive::open_in_memory().unwrap(); + let store = archive.store(); + let source = store + .sources() + .upsert( + &SourceKey { + kind: SourceKind::Maildir, + url: "file:///tmp/Maildir", + account_id: "/tmp/Maildir", + }, + Some("Maildir"), + "", + ) + .unwrap(); + let mailbox = store + .mailboxes() + .insert(&MailboxRecord { + name: "INBOX", + parent: None, + role: Some("inbox"), + sort_order: 0, + subscribed: Subscribed::Yes, + }) + .unwrap(); + store + .maildir_ids(source) + .insert_folder("INBOX", mailbox) + .unwrap(); + Fixture { + archive, + source, + mailbox, + } + } + + fn writer(&self, deleted: DeletedMessages) -> FolderWriter<'_> { + FolderWriter { + store: self.archive.store(), + source: self.source, + folder: "INBOX", + mailbox: self.mailbox, + deleted, + } + } + + fn count(&self, table: &str) -> i64 { + self.archive + .connection() + .query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0)) + .unwrap() + } + + fn keywords(&self, local: LocalId) -> String { + self.archive + .connection() + .query_row("SELECT keywords FROM emails WHERE id = ?1", [local], |r| { + r.get(0) + }) + .unwrap() + } + + fn stored(&self) -> StoredKeywords { + StoredKeywords::load(self.archive.store(), self.source, "INBOX").unwrap() + } + } + + fn write_maildir_message( + folder_path: &Path, + sub: &str, + filename: &str, + body: &[u8], + ) -> PathBuf { + let dir = folder_path.join(sub); + fs::create_dir_all(&dir).unwrap(); + let p = dir.join(filename); + fs::write(&p, body).unwrap(); + p + } + + fn ensure_folder_skel(folder_path: &Path) { + for sub in ["cur", "new", "tmp"] { + fs::create_dir_all(folder_path.join(sub)).unwrap(); + } + } + + fn only_entry(path: &Path) -> DiskEntry { + DiskListing::read(path).unwrap().entries.remove(0) + } + + fn received_at(filename: &str, mtime_unix: u64, date_header: Option<&str>) -> String { + DiskEntry { + unique_id: String::new(), + filename: filename.to_owned(), + path: PathBuf::new(), + flags: Vec::new(), + mtime_unix, + } + .received_at(date_header) + } + + #[test] + fn list_folder_returns_cur_and_new_skipping_tmp() { + let td = tempfile::tempdir().unwrap(); + ensure_folder_skel(td.path()); + write_maildir_message(td.path(), "cur", "1.M0.host:2,S", b"a"); + write_maildir_message(td.path(), "new", "2.M0.host", b"b"); + write_maildir_message(td.path(), "tmp", "3.M0.host", b"c"); + let listing = DiskListing::read(td.path()).unwrap(); + assert_eq!(listing.entries.len(), 2); + assert!(listing.entries.iter().any(|e| e.unique_id == "1.M0.host")); + assert!(listing.entries.iter().any(|e| e.unique_id == "2.M0.host")); + } + + #[test] + fn list_folder_skips_dotfiles() { + let td = tempfile::tempdir().unwrap(); + ensure_folder_skel(td.path()); + write_maildir_message(td.path(), "cur", ".uidvalidity", b"meta"); + write_maildir_message(td.path(), "cur", ".dovecot-uidlist", b"meta"); + write_maildir_message(td.path(), "cur", "1.M0.host:2,S", b"a"); + let listing = DiskListing::read(td.path()).unwrap(); + assert_eq!(listing.entries.len(), 1); + assert_eq!(listing.entries[0].unique_id, "1.M0.host"); + } + + #[test] + fn list_folder_parses_dovecot_extension_filename() { + let td = tempfile::tempdir().unwrap(); + ensure_folder_skel(td.path()); + write_maildir_message( + td.path(), + "cur", + "1739471123.M001P01234V0000I0000abcd.host,S=42,W=44:2,RS", + b"a", + ); + let listing = DiskListing::read(td.path()).unwrap(); + assert_eq!(listing.entries.len(), 1); + let e = &listing.entries[0]; + assert_eq!( + e.unique_id, + "1739471123.M001P01234V0000I0000abcd.host,S=42,W=44" + ); + assert!(e.flags.contains(&Flag::Replied)); + assert!(e.flags.contains(&Flag::Seen)); + } + + #[cfg(target_os = "linux")] + #[test] + fn list_folder_renders_non_utf8_filename_lossy() { + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + let td = tempfile::tempdir().unwrap(); + ensure_folder_skel(td.path()); + let mut bytes = b"1.M0.host".to_vec(); + bytes.extend_from_slice(&[0xff, 0xfe]); + bytes.extend_from_slice(b":2,S"); + let bad_name = OsStr::from_bytes(&bytes); + let p = td.path().join("cur").join(bad_name); + fs::write(&p, b"body").unwrap(); + let listing = DiskListing::read(td.path()).unwrap(); + assert_eq!(listing.entries.len(), 1); + let e = &listing.entries[0]; + assert!( + e.unique_id.contains('\u{FFFD}'), + "lossy substitution: {:?}", + e.unique_id + ); + assert!(e.unique_id.starts_with("1.M0.host")); + assert!(e.flags.contains(&Flag::Seen)); + } + + #[test] + fn diff_partitions_new_present_vanished() { + let entries = vec![ + DiskEntry { + unique_id: "a".into(), + filename: "a".into(), + path: PathBuf::new(), + flags: vec![], + mtime_unix: 0, + }, + DiskEntry { + unique_id: "b".into(), + filename: "b".into(), + path: PathBuf::new(), + flags: vec![], + mtime_unix: 0, + }, + ]; + let mut local = HashMap::new(); + local.insert("b".to_owned(), LocalId::from_raw(10)); + local.insert("c".to_owned(), LocalId::from_raw(11)); + let d = FolderDiff::compute(entries, &local); + assert_eq!(d.new.len(), 1); + assert_eq!(d.new[0].unique_id, "a"); + assert_eq!(d.present.len(), 1); + assert_eq!(d.present[0].0.unique_id, "b"); + assert_eq!(d.present[0].1, LocalId::from_raw(10)); + assert_eq!(d.vanished, vec![("c".to_owned(), LocalId::from_raw(11))]); + } + + #[test] + fn insert_new_writes_blob_email_and_sync_row() { + let fx = Fixture::new(); + let td = tempfile::tempdir().unwrap(); + ensure_folder_skel(td.path()); + let body = b"From: a@x\r\nSubject: hi\r\nMessage-ID: \r\n\r\nhi"; + write_maildir_message(td.path(), "cur", "uid1.M0.host:2,S", body); + let entry = only_entry(td.path()); + let inserted = fx + .writer(DeletedMessages::Skip) + .insert_new(&entry) + .unwrap() + .unwrap(); + assert_eq!(fx.count("emails"), 1); + let local = fx + .archive + .store() + .maildir_ids(fx.source) + .item_id("INBOX", "uid1.M0.host") + .unwrap(); + assert_eq!(local, Some(inserted)); + assert!(fx.keywords(inserted).contains("$seen")); + } + + #[test] + fn insert_new_skips_trashed_without_include_deleted() { + let fx = Fixture::new(); + let td = tempfile::tempdir().unwrap(); + ensure_folder_skel(td.path()); + write_maildir_message(td.path(), "cur", "uid.host:2,T", b"x"); + let entry = only_entry(td.path()); + let inserted = fx.writer(DeletedMessages::Skip).insert_new(&entry).unwrap(); + assert!(inserted.is_none()); + assert_eq!(fx.count("emails"), 0); + } + + #[test] + fn refresh_flags_updates_only_when_changed() { + let fx = Fixture::new(); + let writer = fx.writer(DeletedMessages::Skip); + let td = tempfile::tempdir().unwrap(); + ensure_folder_skel(td.path()); + write_maildir_message(td.path(), "cur", "u.host:2,", b"hi"); + let entry = only_entry(td.path()); + let local = writer.insert_new(&entry).unwrap().unwrap(); + let stored = fx.stored(); + assert_eq!( + writer + .refresh_flags(local, &entry, stored.get(local)) + .unwrap(), + PresentOutcome::Unchanged + ); + fs::rename(&entry.path, entry.path.with_file_name("u.host:2,S")).unwrap(); + let entry2 = only_entry(td.path()); + assert_eq!( + writer + .refresh_flags(local, &entry2, stored.get(local)) + .unwrap(), + PresentOutcome::KeywordsUpdated + ); + assert!(fx.keywords(local).contains("$seen")); + } + + #[test] + fn refresh_flags_returns_delete_when_trashed_added_and_include_deleted_off() { + let fx = Fixture::new(); + let writer = fx.writer(DeletedMessages::Skip); + let td = tempfile::tempdir().unwrap(); + ensure_folder_skel(td.path()); + write_maildir_message(td.path(), "cur", "u.host:2,", b"hi"); + let entry = only_entry(td.path()); + let local = writer.insert_new(&entry).unwrap().unwrap(); + fs::rename(&entry.path, entry.path.with_file_name("u.host:2,T")).unwrap(); + let entry2 = only_entry(td.path()); + assert_eq!( + writer.refresh_flags(local, &entry2, "[]").unwrap(), + PresentOutcome::DeleteRequested + ); + } + + #[test] + fn refresh_flags_keeps_trashed_with_include_deleted_on() { + let fx = Fixture::new(); + let writer = fx.writer(DeletedMessages::Keep); + let td = tempfile::tempdir().unwrap(); + ensure_folder_skel(td.path()); + write_maildir_message(td.path(), "cur", "u.host:2,T", b"hi"); + let entry = only_entry(td.path()); + let local = writer.insert_new(&entry).unwrap().unwrap(); + let stored = fx.stored(); + assert_eq!( + writer + .refresh_flags(local, &entry, stored.get(local)) + .unwrap(), + PresentOutcome::Unchanged + ); + } + + #[test] + fn filename_delivery_time_wins_over_mtime() { + let got = received_at( + "1763065233.M12345P678.host,S=1234:2,S", + 99, + Some("2025-05-12T10:00:00+02:00"), + ); + assert_eq!(got, "2025-11-13T20:20:33Z"); + } + + #[test] + fn date_header_used_when_the_filename_carries_no_timestamp() { + let got = received_at( + "M12345P678.host:2,S", + 1763065233, + Some("2025-05-12T10:00:00+02:00"), + ); + assert_eq!(got, "2025-05-12T10:00:00+02:00"); + } + + #[test] + fn mtime_is_the_last_resort_before_now() { + let got = received_at("M12345P678.host:2,S", 1763065233, None); + assert_eq!(got, "2025-11-13T20:20:33Z"); + } + + #[test] + fn implausible_filename_timestamps_are_rejected() { + let delivery = |name: &str| MaildirFilename(name).delivery_time(); + assert_eq!(delivery("0.M1P1.host"), None); + assert_eq!(delivery("M1P1.host"), None); + assert_eq!(delivery("99999999999999.M1P1.host"), None); + let far_future = (OffsetDateTime::now_utc().unix_timestamp() as u64) + 400_000; + assert_eq!(delivery(&format!("{far_future}.M1P1.host")), None); + assert_eq!(delivery("1763065233.M12345P678.host"), Some(1763065233)); + } +} diff --git a/src/import/maildir/mod.rs b/src/import/maildir/mod.rs new file mode 100644 index 0000000..b17bdff --- /dev/null +++ b/src/import/maildir/mod.rs @@ -0,0 +1,28 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +mod keywords; +mod messages; +mod run; +mod tree; + +use std::path::PathBuf; + +use regex::Regex; + +use crate::archive::sources::SourceChange; +use crate::imap::automap::Automap; +use crate::import::imap::DeletedMessages; + +pub struct MaildirImport { + pub maildir: PathBuf, + pub include: Vec, + pub exclude: Vec, + pub folder: Vec, + pub automap: Automap, + pub deleted: DeletedMessages, + pub source_change: SourceChange, +} diff --git a/src/import/maildir/run.rs b/src/import/maildir/run.rs new file mode 100644 index 0000000..af7df8f --- /dev/null +++ b/src/import/maildir/run.rs @@ -0,0 +1,430 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; +use std::fs; +use std::path::Path; + +use url::Url; + +use crate::archive::rows::{MailboxRecord, Subscribed}; +use crate::archive::sources::{SourceKey, SourceKind}; +use crate::archive::{Archive, LocalId, SourceId}; +use crate::console::Grouped; +use crate::error::Error; +use crate::imap::automap::Automap; +use crate::import::imap::DeletedMessages; +use crate::logging::Logger; +use crate::net::http::RetryStats; +use crate::sync::run::{GcFailure, ImportRun, SourceClaim, SourceLabel}; +use crate::sync::tree::FolderPath; +use crate::sync::vanish::{Children, OnRowError, RowFailure, VanishedSweep}; +use crate::sync::{PlanCounts, RunOptions, RunOutcome, Summary, TypeCounts}; +use crate::types::ObjectType; + +use super::MaildirImport; +use super::messages::{DiskListing, FolderDiff, FolderWriter, StoredKeywords}; +use super::tree::{DELIMITER, DiscoverError, FolderFilters, ResolvedFolder}; + +#[derive(Default)] +struct Counts { + mailbox: TypeCounts, + email: TypeCounts, +} + +struct MaildirRun { + run: ImportRun, + deleted: DeletedMessages, + counts: Counts, +} + +impl MaildirImport { + pub fn run(self, options: RunOptions) -> RunOutcome { + let failed = |error: Error| RunOutcome { + summary: Summary::default(), + error: Some(error), + }; + let logger = options.logger; + if options.threads > 1 { + logger.progress(format_args!( + "Maildir import runs on one thread; --threads is ignored" + )); + } + let MaildirImport { + maildir, + include, + exclude, + folder, + automap, + deleted, + source_change, + } = self; + let canonical = match fs::canonicalize(&maildir) { + Ok(canonical) => canonical, + Err(e) => return failed(Error::Usage(format!("--maildir {maildir:?}: {e}"))), + }; + MaildirImport::warn_if_subfolder(&canonical, logger); + let filters = FolderFilters { + include, + exclude, + explicit: folder, + }; + let resolved = match MaildirImport::resolve(&canonical, automap, &filters) { + Ok(resolved) => resolved, + Err(e) => return failed(e), + }; + let archive = match Archive::open(&options.archive) { + Ok(archive) => archive, + Err(e) => return failed(e.into()), + }; + let session_url = match MaildirImport::source_url(&canonical) { + Ok(url) => url, + Err(e) => return failed(e), + }; + let account_id = canonical.to_string_lossy(); + let key = SourceKey { + kind: SourceKind::Maildir, + url: &session_url, + account_id: &account_id, + }; + let label = SourceLabel { + name: canonical.file_name().and_then(|n| n.to_str()), + username: "", + }; + let run = match ImportRun::start_on(archive, options, &key, label, source_change) { + Ok(run) => run, + Err(e) => return failed(e), + }; + logger.progress(format_args!("Maildir source {session_url}")); + let mut maildir = MaildirRun { + run, + deleted, + counts: Counts::default(), + }; + let result = maildir.execute(&resolved); + maildir.finish(result) + } + + fn resolve( + canonical: &Path, + automap: Automap, + filters: &FolderFilters, + ) -> Result, Error> { + let discovered = ResolvedFolder::discover(canonical, automap).map_err(|e| { + Error::Usage(match e { + DiscoverError::NotFound(p) => format!("maildir path does not exist: {p:?}"), + DiscoverError::NotADirectory(p) => format!("not a directory: {p:?}"), + DiscoverError::NotAMaildir(p) => { + format!("not a Maildir: {p:?} (missing cur/ subdirectory)") + } + DiscoverError::NotMaildirPlus(name) => format!( + "only Maildir++ layout is supported; found non-prefixed subfolder {name:?} \ + (subfolders must be named with a leading '.', e.g. '.{name}')" + ), + DiscoverError::Io(p, e) => format!("walking {p:?}: {e}"), + }) + })?; + if !filters.explicit.is_empty() + && (!filters.include.is_empty() || !filters.exclude.is_empty()) + { + return Err(Error::Usage( + "--folder is mutually exclusive with --include/--exclude".to_owned(), + )); + } + let mut resolved = filters.apply(discovered); + ResolvedFolder::restore_ephemeral_parents(&mut resolved, canonical); + Ok(resolved) + } + + fn source_url(path: &Path) -> Result { + Url::from_file_path(path).map(String::from).map_err(|_| { + Error::Usage(format!( + "cannot encode maildir path as a file:// URL: {path:?}" + )) + }) + } + + fn warn_if_subfolder(canonical: &Path, logger: Logger) { + let Some(basename) = canonical.file_name().and_then(|n| n.to_str()) else { + return; + }; + if !basename.starts_with('.') { + return; + } + let Some(parent) = canonical.parent() else { + return; + }; + if !parent.join("cur").is_dir() { + return; + } + logger.note(format_args!( + "{} looks like a Maildir++ subfolder of {} (its name starts with '.' and the \ + parent has its own cur/); importing it anyway, but you may have meant the parent", + canonical.display(), + parent.display() + )); + } +} + +impl MaildirRun { + fn logger(&self) -> Logger { + self.run.options().logger + } + + fn execute(&mut self, resolved: &[ResolvedFolder]) -> Result<(), Error> { + let source = match *self.run.claim() { + SourceClaim::DryRun(source) => return self.dry_run(source, resolved), + SourceClaim::Registered(source) => source, + }; + let local = self.run.store().maildir_ids(source).folders()?; + let mailboxes = self.upsert_mailboxes(source, resolved)?; + self.sweep_vanished(source, local, resolved)?; + let logger = self.logger(); + for folder in resolved.iter().filter(|f| !f.is_ephemeral()) { + let Some(&mailbox) = mailboxes.get(folder.name.as_str()) else { + logger.note(format_args!( + "folder \"{}\" is missing from the mailbox id map", + folder.name + )); + self.counts.email.failed += 1; + continue; + }; + if let Err(e) = self.reconcile_folder(source, folder, mailbox) { + if e.aborts_run() { + return Err(e); + } + logger.note(format_args!("folder \"{}\" failed: {e}", folder.name)); + self.counts.email.failed += 1; + } + } + Ok(()) + } + + fn finish(self, result: Result<(), Error>) -> RunOutcome { + let MaildirRun { + mut run, counts, .. + } = self; + *run.counts(ObjectType::Mailbox) = counts.mailbox; + *run.counts(ObjectType::Email) = counts.email; + match result { + Ok(()) => run.finish(RetryStats::default(), GcFailure::Abort), + Err(error) => run.abort(error, RetryStats::default()), + } + } + + fn upsert_mailboxes<'r>( + &mut self, + source: SourceId, + folders: &'r [ResolvedFolder], + ) -> Result, Error> { + let tx = self.run.archive().transaction()?; + let store = tx.store(); + let ids = store.maildir_ids(source); + let mailboxes = store.mailboxes(); + let mut local_ids: HashMap<&str, LocalId> = HashMap::with_capacity(folders.len()); + for folder in folders { + let parent = match folder.parent_path.as_deref() { + Some(p) => match local_ids.get(p) { + Some(&id) => Some(id), + None => ids.folder_id(p).ok().flatten(), + }, + None => None, + }; + let record = MailboxRecord { + name: &folder.leaf, + parent, + role: folder.role, + sort_order: 0, + subscribed: if folder.is_ephemeral() { + Subscribed::No + } else { + Subscribed::Yes + }, + }; + let id = match ids.folder_id(&folder.name)? { + Some(id) => { + mailboxes.update(id, &record)?; + id + } + None => { + let id = mailboxes.insert(&record)?; + ids.insert_folder(&folder.name, id)?; + self.counts.mailbox.created += 1; + self.counts.mailbox.fetched += 1; + id + } + }; + local_ids.insert(&folder.name, id); + } + tx.commit()?; + Ok(local_ids) + } + + fn sweep_vanished( + &mut self, + source: SourceId, + local: HashMap, + resolved: &[ResolvedFolder], + ) -> Result<(), Error> { + let on_disk: HashSet<&str> = resolved.iter().map(|f| f.name.as_str()).collect(); + let mut vanished: Vec<(String, LocalId)> = local + .into_iter() + .filter(|(name, _)| !on_disk.contains(name.as_str())) + .collect(); + FolderPath::sort_children_first_by(&mut vanished, DELIMITER, |(name, _)| name); + let logger = self.logger(); + let emails = &mut self.counts.email; + VanishedSweep::new( + self.run.store(), + OnRowError::Abort, + Children::RefuseSubfolders, + ) + .run( + ObjectType::Mailbox, + &vanished, + &mut self.counts.mailbox, + |store, name, _| { + let ids = store.maildir_ids(source); + let rows = store.emails(); + for local in ids.items_in(name)?.into_values() { + rows.delete(local)?; + emails.deleted += 1; + } + ids.delete_items_in(name)?; + ids.delete_folder(name) + }, + |name, _, failure| { + if let RowFailure::LiveChildren = failure { + logger.note(format_args!( + "folder \"{name}\" vanished from disk but still has child mailboxes in \ + the archive; not deleted" + )); + } + }, + )?; + Ok(()) + } + + fn reconcile_folder( + &mut self, + source: SourceId, + folder: &ResolvedFolder, + mailbox: LocalId, + ) -> Result<(), Error> { + let listing = DiskListing::read(&folder.path) + .map_err(|e| Error::Partial(format!("list {:?}: {e}", folder.path)))?; + let counts = &mut self.counts.email; + counts.failed += listing.io_failures; + let store = self.run.store(); + let local = store.maildir_ids(source).items_in(&folder.name)?; + let diff = FolderDiff::compute(listing.entries, &local); + let logger = self.run.options().logger; + logger.progress(format_args!( + "Folder \"{}\": {} new, {} vanished, {} present", + folder.name, + Grouped(diff.new.len() as u64), + Grouped(diff.vanished.len() as u64), + Grouped(diff.present.len() as u64) + )); + let stored = StoredKeywords::load(store, source, &folder.name).unwrap_or_default(); + let deleted = self.deleted; + let tx = self.run.archive().transaction()?; + FolderWriter { + store: tx.store(), + source, + folder: &folder.name, + mailbox, + deleted, + } + .apply(&diff, &stored, &mut self.counts.email, logger); + tx.commit()?; + Ok(()) + } + + fn dry_run( + &mut self, + source: Option, + folders: &[ResolvedFolder], + ) -> Result<(), Error> { + let logger = self.logger(); + let store = self.run.store(); + let ids = source.map(|source| store.maildir_ids(source)); + let local_mailboxes = match ids { + Some(ids) => ids.folders()?, + None => HashMap::new(), + }; + let failed = &mut self.counts.email.failed; + let mut mailbox = PlanCounts::default(); + let mut email = PlanCounts::default(); + let on_disk: HashSet<&str> = folders.iter().map(|f| f.name.as_str()).collect(); + mailbox.new = folders + .iter() + .filter(|f| !local_mailboxes.contains_key(&f.name)) + .count() as u64; + mailbox.present = folders.len() as u64 - mailbox.new; + for folder in folders.iter().filter(|f| !f.is_ephemeral()) { + let listing = match DiskListing::read(&folder.path) { + Ok(listing) => listing, + Err(e) => { + logger.warn(format_args!("cannot list folder \"{}\": {e}", folder.name)); + *failed += 1; + continue; + } + }; + *failed += listing.io_failures; + let local = match ids { + Some(ids) => ids.items_in(&folder.name)?, + None => HashMap::new(), + }; + let disk: HashSet<&str> = listing + .entries + .iter() + .map(|e| e.unique_id.as_str()) + .collect(); + let present = disk.iter().filter(|u| local.contains_key(**u)).count() as u64; + email.new += disk.len() as u64 - present; + email.present += present; + email.vanished += local.keys().filter(|k| !disk.contains(k.as_str())).count() as u64; + } + for name in local_mailboxes + .keys() + .filter(|n| !on_disk.contains(n.as_str())) + { + mailbox.vanished += 1; + if let Some(ids) = ids { + email.vanished += ids.items_in(name)?.len() as u64; + } + } + let summary = self.run.summary(); + *summary.plan(ObjectType::Mailbox) = mailbox; + *summary.plan(ObjectType::Email) = email; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn source_url_for_ascii_path_is_passthrough() { + let url = MaildirImport::source_url(Path::new("/home/alice/Maildir")).unwrap(); + assert_eq!(url, "file:///home/alice/Maildir"); + } + + #[test] + fn source_url_percent_encodes_spaces_and_unicode() { + let url = MaildirImport::source_url(Path::new("/home/alice/My Maildir/Sübfolder")).unwrap(); + assert!(url.starts_with("file:///home/alice/My%20Maildir/")); + assert!(url.contains("S%C3%BCbfolder")); + } + + #[test] + fn source_url_rejects_relative_paths() { + let err = MaildirImport::source_url(Path::new("./relative")).unwrap_err(); + assert!(matches!(err, Error::Usage(_))); + } +} diff --git a/src/sync/import_maildir/tree.rs b/src/import/maildir/tree.rs similarity index 55% rename from src/sync/import_maildir/tree.rs rename to src/import/maildir/tree.rs index 4bf0b71..a92a36f 100644 --- a/src/sync/import_maildir/tree.rs +++ b/src/import/maildir/tree.rs @@ -4,34 +4,46 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use std::{ - borrow::Cow, - path::{Path, PathBuf}, -}; +use std::borrow::Cow; +use std::collections::HashSet; +use std::fs; +use std::iter; +use std::path::{Path, PathBuf}; use regex::Regex; -use crate::imap::{automap, name::decode_mailbox_name}; +use crate::imap::automap::Automap; +use crate::imap::name::NameEncoding; +use crate::sync::tree::FolderPath; + +const INBOX: &str = "INBOX"; +pub(super) const DELIMITER: char = '.'; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Presence { + OnDisk, + Ephemeral, +} #[derive(Debug, Clone, PartialEq, Eq)] -pub struct ResolvedFolder { +pub(super) struct ResolvedFolder { pub name: String, pub decoded_name: String, pub leaf: String, pub parent_path: Option, pub path: PathBuf, pub role: Option<&'static str>, - pub ephemeral: bool, + pub presence: Presence, } -pub struct FolderFilters { +pub(super) struct FolderFilters { pub include: Vec, pub exclude: Vec, pub explicit: Vec, } #[derive(Debug, thiserror::Error)] -pub enum DiscoverError { +pub(super) enum DiscoverError { #[error("path does not exist: {0}")] NotFound(PathBuf), #[error("path is not a directory: {0}")] @@ -47,220 +59,197 @@ pub enum DiscoverError { Io(PathBuf, std::io::Error), } -pub fn discover(root: &Path, automap_enabled: bool) -> Result, DiscoverError> { - if !root.exists() { - return Err(DiscoverError::NotFound(root.to_path_buf())); - } - let meta = std::fs::metadata(root).map_err(|e| DiscoverError::Io(root.to_path_buf(), e))?; - if !meta.is_dir() { - return Err(DiscoverError::NotADirectory(root.to_path_buf())); - } - if !root.join("cur").is_dir() { - return Err(DiscoverError::NotAMaildir(root.to_path_buf())); - } - let mut folders = vec![ResolvedFolder { - name: "INBOX".to_owned(), - decoded_name: "INBOX".to_owned(), - leaf: "INBOX".to_owned(), - parent_path: None, - path: root.to_path_buf(), - role: Some("inbox"), - ephemeral: false, - }]; +#[derive(Debug, Clone, Copy)] +pub(super) struct MaildirName<'a>(pub &'a str); - let mut subfolders: Vec<(String, PathBuf)> = Vec::new(); - let entries = std::fs::read_dir(root).map_err(|e| DiscoverError::Io(root.to_path_buf(), e))?; - for entry in entries { - let entry = entry.map_err(|e| DiscoverError::Io(root.to_path_buf(), e))?; - let path = entry.path(); - if !path.is_dir() { - continue; +impl<'a> MaildirName<'a> { + fn folder_path(self) -> FolderPath<'a> { + FolderPath::new(self.0, DELIMITER) + } + + pub fn parent(self) -> Option<&'a str> { + self.folder_path().parent() + } + + pub fn leaf(self) -> &'a str { + self.folder_path().leaf() + } + + pub fn depth(self) -> usize { + self.folder_path().depth() + } + + pub fn ancestors_and_self(self) -> impl Iterator { + let name = self.0; + name.char_indices() + .filter(|&(_, c)| c == DELIMITER) + .filter_map(move |(end, _)| name.get(..end)) + .chain(iter::once(name)) + } + + fn decode_segment(segment: &str) -> Cow<'_, str> { + if !segment.contains('&') { + return Cow::Borrowed(segment); } - let Some(raw_name) = path.file_name() else { - continue; - }; - let name: String = raw_name.to_string_lossy().into_owned(); - if matches!(name.as_str(), "cur" | "new" | "tmp") { - continue; - } - let has_cur = path.join("cur").is_dir(); - if let Some(stripped) = name.strip_prefix('.') { - if has_cur && !stripped.is_empty() { - subfolders.push((stripped.to_owned(), path)); + NameEncoding::ModifiedUtf7 + .decode(segment) + .map_or(Cow::Borrowed(segment), Cow::Owned) + } + + pub fn decoded(self) -> String { + let mut decoded = String::with_capacity(self.0.len()); + for (index, segment) in self.0.split(DELIMITER).enumerate() { + if index > 0 { + decoded.push(DELIMITER); } - } else if has_cur { - return Err(DiscoverError::NotMaildirPlus(name)); + decoded.push_str(&MaildirName::decode_segment(segment)); } + decoded } - subfolders.sort_by(|a, b| a.0.cmp(&b.0)); - - let mut known: Vec = vec!["INBOX".to_owned()]; - for (canonical, path) in subfolders { - let parent_path = split_parent(&canonical); - if let Some(p) = parent_path.as_deref() { - ensure_ephemeral_parents(p, &mut known, &mut folders, root); - } - let decoded_name = decode_name(&canonical); - let role = if automap_enabled { - automap::role_for_folder(&decoded_name, &[], "", true) - } else { - None - }; - folders.push(ResolvedFolder { - leaf: decode_segment(leaf_of(&canonical)).into_owned(), - name: canonical.clone(), - decoded_name, - parent_path, - path, - role, - ephemeral: false, - }); - known.push(canonical); + pub fn decoded_leaf(self) -> String { + MaildirName::decode_segment(self.leaf()).into_owned() } - Ok(folders) } -fn ensure_ephemeral_parents( - parent_path: &str, - known: &mut Vec, - folders: &mut Vec, - root: &Path, -) { - for ancestor in ancestor_chain(parent_path) { - if known.iter().any(|n| n == &ancestor) { - continue; - } - let parent_path = split_parent(&ancestor); - folders.push(ResolvedFolder { - decoded_name: decode_name(&ancestor), - leaf: decode_segment(leaf_of(&ancestor)).into_owned(), - name: ancestor.clone(), - parent_path, +impl ResolvedFolder { + fn ephemeral(name: &str, root: &Path) -> ResolvedFolder { + let maildir = MaildirName(name); + ResolvedFolder { + decoded_name: maildir.decoded(), + leaf: maildir.decoded_leaf(), + parent_path: maildir.parent().map(str::to_owned), + name: name.to_owned(), path: root.to_path_buf(), role: None, - ephemeral: true, - }); - known.push(ancestor); - } -} - -fn ancestor_chain(name: &str) -> Vec { - let mut out = Vec::new(); - let mut acc = String::new(); - for part in name.split('.') { - if !acc.is_empty() { - acc.push('.'); + presence: Presence::Ephemeral, } - acc.push_str(part); - out.push(acc.clone()); } - out -} -fn split_parent(name: &str) -> Option { - name.rfind('.').map(|i| name[..i].to_owned()) -} - -fn decode_segment(segment: &str) -> Cow<'_, str> { - if !segment.contains('&') { - return Cow::Borrowed(segment); + pub fn is_ephemeral(&self) -> bool { + self.presence == Presence::Ephemeral } - decode_mailbox_name(segment).map_or(Cow::Borrowed(segment), Cow::Owned) -} -fn decode_name(name: &str) -> String { - let mut decoded = String::with_capacity(name.len()); - for (i, segment) in name.split('.').enumerate() { - if i > 0 { - decoded.push('.'); + pub fn discover(root: &Path, automap: Automap) -> Result, DiscoverError> { + if !root.exists() { + return Err(DiscoverError::NotFound(root.to_path_buf())); } - decoded.push_str(&decode_segment(segment)); - } - decoded -} + let meta = fs::metadata(root).map_err(|e| DiscoverError::Io(root.to_path_buf(), e))?; + if !meta.is_dir() { + return Err(DiscoverError::NotADirectory(root.to_path_buf())); + } + if !root.join("cur").is_dir() { + return Err(DiscoverError::NotAMaildir(root.to_path_buf())); + } + let mut folders = vec![ResolvedFolder { + name: INBOX.to_owned(), + decoded_name: INBOX.to_owned(), + leaf: INBOX.to_owned(), + parent_path: None, + path: root.to_path_buf(), + role: Some("inbox"), + presence: Presence::OnDisk, + }]; -fn leaf_of(name: &str) -> &str { - match name.rfind('.') { - Some(i) => &name[i + 1..], - None => name, - } -} - -pub fn apply_filters(folders: Vec, filters: &FolderFilters) -> Vec { - folders - .into_iter() - .filter(|f| match_filters(&f.decoded_name, filters)) - .collect() -} - -pub fn restore_ephemeral_parents(folders: &mut Vec, root: &Path) { - let known: std::collections::HashSet = folders.iter().map(|f| f.name.clone()).collect(); - let mut to_add: Vec = Vec::new(); - for f in folders.iter() { - if let Some(parent) = f.parent_path.as_deref() { - for ancestor in ancestor_chain(parent) { - if known.contains(&ancestor) || to_add.iter().any(|x| x.name == ancestor) { - continue; + let mut subfolders: Vec<(String, PathBuf)> = Vec::new(); + let entries = fs::read_dir(root).map_err(|e| DiscoverError::Io(root.to_path_buf(), e))?; + for entry in entries { + let entry = entry.map_err(|e| DiscoverError::Io(root.to_path_buf(), e))?; + let path = entry.path(); + if !path.is_dir() { + continue; + } + let Some(raw_name) = path.file_name() else { + continue; + }; + let name = raw_name.to_string_lossy(); + if matches!(name.as_ref(), "cur" | "new" | "tmp") { + continue; + } + let has_cur = path.join("cur").is_dir(); + if let Some(stripped) = name.strip_prefix(DELIMITER) { + if has_cur && !stripped.is_empty() { + subfolders.push((stripped.to_owned(), path)); } - let parent_path = split_parent(&ancestor); - to_add.push(ResolvedFolder { - decoded_name: decode_name(&ancestor), - leaf: decode_segment(leaf_of(&ancestor)).into_owned(), - name: ancestor, - parent_path, - path: root.to_path_buf(), - role: None, - ephemeral: true, - }); + } else if has_cur { + return Err(DiscoverError::NotMaildirPlus(name.into_owned())); } } + + subfolders.sort_by(|a, b| a.0.cmp(&b.0)); + + let mut known: HashSet = HashSet::with_capacity(subfolders.len() + 1); + known.insert(INBOX.to_owned()); + folders.reserve(subfolders.len()); + for (canonical, path) in subfolders { + let maildir = MaildirName(&canonical); + if let Some(parent) = maildir.parent() { + for ancestor in MaildirName(parent).ancestors_and_self() { + if !known.contains(ancestor) { + known.insert(ancestor.to_owned()); + folders.push(ResolvedFolder::ephemeral(ancestor, root)); + } + } + } + let decoded_name = maildir.decoded(); + let role = match automap { + Automap::Enabled => automap.role(&decoded_name, &[], ""), + Automap::Disabled => None, + }; + folders.push(ResolvedFolder { + leaf: maildir.decoded_leaf(), + parent_path: maildir.parent().map(str::to_owned), + name: canonical.clone(), + decoded_name, + path, + role, + presence: Presence::OnDisk, + }); + known.insert(canonical); + } + Ok(folders) } - if to_add.is_empty() { - return; + + pub fn restore_ephemeral_parents(folders: &mut Vec, root: &Path) { + let mut known: HashSet<&str> = folders.iter().map(|f| f.name.as_str()).collect(); + let mut missing: Vec = Vec::new(); + for parent in folders.iter().filter_map(|f| f.parent_path.as_deref()) { + for ancestor in MaildirName(parent).ancestors_and_self() { + if known.insert(ancestor) { + missing.push(ResolvedFolder::ephemeral(ancestor, root)); + } + } + } + if missing.is_empty() { + return; + } + folders.extend(missing); + folders.sort_by_key(|f| MaildirName(&f.name).depth()); } - folders.extend(to_add); - folders.sort_by_key(|f| f.name.matches('.').count()); } -fn match_filters(name: &str, filters: &FolderFilters) -> bool { - if !filters.explicit.is_empty() { - return filters.explicit.iter().any(|e| e == name); +impl FolderFilters { + pub fn apply(&self, folders: Vec) -> Vec { + folders + .into_iter() + .filter(|f| self.matches(&f.decoded_name)) + .collect() } - if !filters.include.is_empty() && !filters.include.iter().any(|r| r.is_match(name)) { - return false; - } - if filters.exclude.iter().any(|r| r.is_match(name)) { - return false; - } - true -} -pub fn vanished_folders<'a, I>(local: I, server: &[ResolvedFolder]) -> Vec -where - I: IntoIterator, -{ - let server_set: std::collections::HashSet<&str> = - server.iter().map(|f| f.name.as_str()).collect(); - local - .into_iter() - .filter(|n| !server_set.contains(*n)) - .map(str::to_owned) - .collect() -} - -pub fn vanished_depth_sort(names: &mut [String]) { - names.sort_by(|a, b| { - let da = a.matches('.').count(); - let db = b.matches('.').count(); - db.cmp(&da).then_with(|| a.cmp(b)) - }); + fn matches(&self, name: &str) -> bool { + if !self.explicit.is_empty() { + return self.explicit.iter().any(|e| e == name); + } + if !self.include.is_empty() && !self.include.iter().any(|r| r.is_match(name)) { + return false; + } + !self.exclude.iter().any(|r| r.is_match(name)) + } } #[cfg(test)] mod tests { use super::*; - use std::fs; fn make_maildir(root: &Path, folders: &[&str]) { for sub in ["cur", "new", "tmp"] { @@ -278,7 +267,7 @@ mod tests { fn discover_just_inbox_when_root_alone() { let td = tempfile::tempdir().unwrap(); make_maildir(td.path(), &[]); - let folders = discover(td.path(), true).unwrap(); + let folders = ResolvedFolder::discover(td.path(), Automap::Enabled).unwrap(); assert_eq!(folders.len(), 1); assert_eq!(folders[0].name, "INBOX"); assert_eq!(folders[0].role, Some("inbox")); @@ -289,7 +278,7 @@ mod tests { fn discover_dovecot_style_subfolders() { let td = tempfile::tempdir().unwrap(); make_maildir(td.path(), &[".Sent", ".Drafts", ".Trash", ".Junk"]); - let folders = discover(td.path(), true).unwrap(); + let folders = ResolvedFolder::discover(td.path(), Automap::Enabled).unwrap(); let names: Vec<&str> = folders.iter().map(|f| f.name.as_str()).collect(); assert!(names.contains(&"INBOX")); assert!(names.contains(&"Sent")); @@ -308,7 +297,7 @@ mod tests { fn discover_courier_maildir_plus_deep_hierarchy() { let td = tempfile::tempdir().unwrap(); make_maildir(td.path(), &[".Archive", ".Archive.2024", ".Archive.2025"]); - let folders = discover(td.path(), true).unwrap(); + let folders = ResolvedFolder::discover(td.path(), Automap::Enabled).unwrap(); let names: Vec<&str> = folders.iter().map(|f| f.name.as_str()).collect(); assert!(names.contains(&"Archive")); assert!(names.contains(&"Archive.2024")); @@ -326,16 +315,16 @@ mod tests { fn discover_inserts_ephemeral_parent_for_orphan_branch() { let td = tempfile::tempdir().unwrap(); make_maildir(td.path(), &[".Lists.maildir-dev"]); - let folders = discover(td.path(), true).unwrap(); + let folders = ResolvedFolder::discover(td.path(), Automap::Enabled).unwrap(); let lists = folders.iter().find(|f| f.name == "Lists").unwrap(); - assert!(lists.ephemeral); + assert!(lists.is_ephemeral()); assert!(lists.role.is_none()); assert!(lists.parent_path.is_none()); let leaf = folders .iter() .find(|f| f.name == "Lists.maildir-dev") .unwrap(); - assert!(!leaf.ephemeral); + assert!(!leaf.is_ephemeral()); assert_eq!(leaf.parent_path.as_deref(), Some("Lists")); assert_eq!(leaf.leaf, "maildir-dev"); } @@ -347,7 +336,7 @@ mod tests { fs::create_dir(td.path().join(".dovecot.imap")).unwrap(); fs::write(td.path().join("dovecot.index.log"), "").unwrap(); - let folders = discover(td.path(), true).unwrap(); + let folders = ResolvedFolder::discover(td.path(), Automap::Enabled).unwrap(); assert_eq!(folders.len(), 2); assert!(folders.iter().any(|f| f.name == "INBOX")); assert!(folders.iter().any(|f| f.name == "Sent")); @@ -357,7 +346,7 @@ mod tests { fn discover_rejects_dovecot_layout_fs_tree() { let td = tempfile::tempdir().unwrap(); make_maildir(td.path(), &["Sent"]); - let err = discover(td.path(), true).unwrap_err(); + let err = ResolvedFolder::discover(td.path(), Automap::Enabled).unwrap_err(); assert!(matches!(err, DiscoverError::NotMaildirPlus(_))); } @@ -365,13 +354,17 @@ mod tests { fn discover_rejects_path_without_cur_subdir() { let td = tempfile::tempdir().unwrap(); fs::create_dir_all(td.path().join("new")).unwrap(); - let err = discover(td.path(), true).unwrap_err(); + let err = ResolvedFolder::discover(td.path(), Automap::Enabled).unwrap_err(); assert!(matches!(err, DiscoverError::NotAMaildir(_))); } #[test] fn discover_rejects_nonexistent_path() { - let err = discover(Path::new("/definitely/not/a/maildir/at/all"), true).unwrap_err(); + let err = ResolvedFolder::discover( + Path::new("/definitely/not/a/maildir/at/all"), + Automap::Enabled, + ) + .unwrap_err(); assert!(matches!(err, DiscoverError::NotFound(_))); } @@ -382,7 +375,7 @@ mod tests { td.path(), &[".Archive.2025", ".Archive", ".Lists.maildir-dev"], ); - let folders = discover(td.path(), true).unwrap(); + let folders = ResolvedFolder::discover(td.path(), Automap::Enabled).unwrap(); let index = |name: &str| folders.iter().position(|f| f.name == name).unwrap(); assert!(index("Archive") < index("Archive.2025")); assert!(index("Lists") < index("Lists.maildir-dev")); @@ -391,19 +384,17 @@ mod tests { #[test] fn ancestor_chain_walks_dot_segments() { - assert_eq!(ancestor_chain("A"), vec!["A".to_owned()]); - assert_eq!( - ancestor_chain("A.B.C"), - vec!["A".to_owned(), "A.B".to_owned(), "A.B.C".to_owned()] - ); + let chain = |name| MaildirName(name).ancestors_and_self().collect::>(); + assert_eq!(chain("A"), ["A"]); + assert_eq!(chain("A.B.C"), ["A", "A.B", "A.B.C"]); } #[test] fn leaf_and_split_handle_root_and_nested() { - assert_eq!(leaf_of("INBOX"), "INBOX"); - assert_eq!(leaf_of("Archive.2025"), "2025"); - assert_eq!(split_parent("INBOX"), None); - assert_eq!(split_parent("Archive.2025"), Some("Archive".to_owned())); + assert_eq!(MaildirName("INBOX").leaf(), "INBOX"); + assert_eq!(MaildirName("Archive.2025").leaf(), "2025"); + assert_eq!(MaildirName("INBOX").parent(), None); + assert_eq!(MaildirName("Archive.2025").parent(), Some("Archive")); } fn folders_for(names: &[&str]) -> Vec { @@ -416,7 +407,7 @@ mod tests { parent_path: None, path: PathBuf::new(), role: None, - ephemeral: false, + presence: Presence::OnDisk, }) .collect() } @@ -429,7 +420,7 @@ mod tests { exclude: Vec::new(), explicit: Vec::new(), }; - let kept = apply_filters(folders, &filters); + let kept = filters.apply(folders); assert_eq!(kept.len(), 2); assert!(kept.iter().any(|f| f.name == "INBOX")); assert!(kept.iter().any(|f| f.name == "Sent")); @@ -443,7 +434,7 @@ mod tests { exclude: vec![Regex::new("^Trash$").unwrap()], explicit: Vec::new(), }; - let kept = apply_filters(folders, &filters); + let kept = filters.apply(folders); assert_eq!(kept.len(), 1); assert_eq!(kept[0].name, "INBOX"); } @@ -456,7 +447,7 @@ mod tests { exclude: Vec::new(), explicit: vec!["Sent".to_owned()], }; - let kept = apply_filters(folders, &filters); + let kept = filters.apply(folders); assert_eq!(kept.len(), 1); assert_eq!(kept[0].name, "Sent"); } @@ -465,7 +456,7 @@ mod tests { fn discover_with_automap_off_clears_roles_except_inbox() { let td = tempfile::tempdir().unwrap(); make_maildir(td.path(), &[".Sent", ".Drafts"]); - let folders = discover(td.path(), false).unwrap(); + let folders = ResolvedFolder::discover(td.path(), Automap::Disabled).unwrap(); let inbox = folders.iter().find(|f| f.name == "INBOX").unwrap(); assert_eq!(inbox.role, Some("inbox")); let sent = folders.iter().find(|f| f.name == "Sent").unwrap(); @@ -488,7 +479,7 @@ mod tests { for sub in ["cur", "new", "tmp"] { fs::create_dir_all(p.join(sub)).unwrap(); } - let folders = discover(td.path(), true).unwrap(); + let folders = ResolvedFolder::discover(td.path(), Automap::Enabled).unwrap(); assert_eq!(folders.len(), 2); let sub = folders.iter().find(|f| f.name != "INBOX").unwrap(); assert!( @@ -506,7 +497,7 @@ mod tests { "Archive.2025.January".to_owned(), "Archive.2025".to_owned(), ]; - vanished_depth_sort(&mut names); + FolderPath::sort_children_first_by(&mut names, DELIMITER, String::as_str); assert_eq!( names, vec![ @@ -525,7 +516,7 @@ mod tests { td.path(), &[".&AOk-t&AOk-", ".R&-D", ".Plain", ".&ZeVnLIqe-.&U,BTFw-"], ); - let folders = discover(td.path(), true).unwrap(); + let folders = ResolvedFolder::discover(td.path(), Automap::Enabled).unwrap(); let by_name = |name: &str| folders.iter().find(|f| f.name == name).unwrap(); let ete = by_name("&AOk-t&AOk-"); @@ -539,7 +530,7 @@ mod tests { assert_eq!(nested.leaf, "台北"); assert_eq!(nested.parent_path.as_deref(), Some("&ZeVnLIqe-")); let parent = by_name("&ZeVnLIqe-"); - assert!(parent.ephemeral); + assert!(parent.is_ephemeral()); assert_eq!(parent.leaf, "日本語"); } @@ -547,9 +538,9 @@ mod tests { #[test] fn decode_keeps_literal_ampersands_and_utf8() { for name in ["R&D", "AT&T", "Tom & Jerry-2", "Été", "Sales&", "A&-B&C"] { - assert_eq!(decode_name(name), name); + assert_eq!(MaildirName(name).decoded(), name); } - assert_eq!(decode_name("R&D.&AOk-t&AOk-"), "R&D.été"); + assert_eq!(MaildirName("R&D.&AOk-t&AOk-").decoded(), "R&D.été"); } #[test] @@ -562,14 +553,14 @@ mod tests { parent_path: None, path: PathBuf::new(), role: None, - ephemeral: false, + presence: Presence::OnDisk, }); let filters = FolderFilters { include: Vec::new(), exclude: Vec::new(), explicit: vec!["été".to_owned()], }; - let kept = apply_filters(folders, &filters); + let kept = filters.apply(folders); assert_eq!(kept.len(), 1); assert_eq!(kept[0].name, "&AOk-t&AOk-"); } diff --git a/src/import/managesieve/control.rs b/src/import/managesieve/control.rs new file mode 100644 index 0000000..9efb9cc --- /dev/null +++ b/src/import/managesieve/control.rs @@ -0,0 +1,101 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::error::Error; +use crate::managesieve::SieveClient; +use crate::managesieve::error::SieveError; +use crate::managesieve::response::ListedScript; +use crate::managesieve::session::SieveSession; +use crate::net::backoff::{BackoffState, Fault, RetryError}; +use crate::net::http::RetryStats; + +pub(super) struct Control { + client: SieveClient, + session: SieveSession, + backoff: BackoffState, + max_retries: u32, +} + +impl Control { + pub fn new(client: SieveClient, session: SieveSession, max_retries: u32) -> Control { + Control { + client, + session, + backoff: BackoffState::new(), + max_retries, + } + } + + pub fn stats(&self) -> RetryStats { + self.backoff.stats() + } + + pub fn logout(&mut self) { + self.client.logout(); + } + + pub fn listscripts(&mut self) -> Result, Error> { + let session = &self.session; + let logger = session.logger(); + let block = self + .backoff + .run( + self.max_retries, + &mut self.client, + SieveClient::listscripts, + |client, e| { + *client = session.reopen()?; + logger.info(format_args!( + "Reconnected to ManageSieve after the connection dropped ({e})" + )); + Ok(()) + }, + |e| logger.progress(format_args!("LISTSCRIPTS failed ({e}); retrying")), + ) + .map_err(|e| match e { + RetryError::Exhausted { + error, + fault: Fault::TransportDrop, + } => Error::Connection(format!( + "LISTSCRIPTS transport drop, retries exhausted: {error}" + )), + RetryError::Exhausted { error, .. } => { + Error::Connection(format!("LISTSCRIPTS retries exhausted: {error}")) + } + RetryError::Reconnect(error) => { + Error::Connection(format!("LISTSCRIPTS reconnect failed: {error}")) + } + RetryError::Fatal(error) if error.is_referral() => Error::Connection(format!( + "LISTSCRIPTS referral: {error}; inbuxa-migrate does not follow referrals" + )), + RetryError::Fatal(error) => { + Error::Connection(format!("LISTSCRIPTS failed: {error}")) + } + })?; + ListedScript::parse_all(&block.data) + .map_err(|e| Error::Connection(format!("LISTSCRIPTS parse: {e}"))) + } + + pub fn getscript(&mut self, name: &str) -> Result, SieveError> { + let session = &self.session; + let logger = session.logger(); + self.backoff + .run( + self.max_retries, + &mut self.client, + |client| client.getscript(name), + |client, _| { + *client = session.reopen()?; + logger.info(format_args!( + "Reconnected to ManageSieve after the connection dropped during GETSCRIPT \"{name}\"" + )); + Ok(()) + }, + |e| logger.progress(format_args!("GETSCRIPT \"{name}\" failed ({e}); retrying")), + ) + .map_err(RetryError::into_inner) + } +} diff --git a/src/import/managesieve/mod.rs b/src/import/managesieve/mod.rs new file mode 100644 index 0000000..ec5439c --- /dev/null +++ b/src/import/managesieve/mod.rs @@ -0,0 +1,21 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +mod control; +mod plan; +mod run; + +use crate::archive::sources::SourceChange; +use crate::net::stream::Cleartext; + +pub use crate::net::sasl::SaslAuth; + +pub struct ManageSieveImport { + pub url: String, + pub auth: SaslAuth, + pub cleartext: Cleartext, + pub source_change: SourceChange, +} diff --git a/src/import/managesieve/plan.rs b/src/import/managesieve/plan.rs new file mode 100644 index 0000000..e30259d --- /dev/null +++ b/src/import/managesieve/plan.rs @@ -0,0 +1,431 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; + +use crate::archive::rows::SieveScriptRecord; +use crate::archive::{LocalId, SourceId, Store}; +use crate::error::Error; +use crate::sync::PlanCounts; + +pub(super) struct ServerScript { + pub name: String, + pub active: bool, + pub hash: [u8; 32], + pub bytes: Vec, +} + +impl ServerScript { + pub fn new(name: String, active: bool, bytes: Vec) -> ServerScript { + ServerScript { + name, + active, + hash: *blake3::hash(&bytes).as_bytes(), + bytes, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum PresentOutcome { + Unchanged, + ActiveOnly, + ContentUpdated, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct StoredScript { + pub hash: Option<[u8; 32]>, + pub active: bool, +} + +#[derive(Default)] +pub(super) struct LocalScripts { + ids: HashMap, + stored: HashMap, +} + +impl LocalScripts { + pub fn load(store: Store<'_>, source: SourceId) -> Result { + let ids = store.sieve_ids(source).all()?; + let stored = store + .sieve_scripts() + .managesieve_states(source)? + .into_iter() + .map(|state| { + ( + state.id, + StoredScript { + hash: state.hash, + active: state.is_active, + }, + ) + }) + .collect(); + Ok(LocalScripts { ids, stored }) + } + + fn stored(&self, local: LocalId) -> StoredScript { + self.stored.get(&local).copied().unwrap_or(StoredScript { + hash: None, + active: false, + }) + } +} + +#[derive(Clone, Copy)] +pub(super) enum Action<'p> { + New(&'p ServerScript), + Present { + script: &'p ServerScript, + local: LocalId, + stored: StoredScript, + outcome: PresentOutcome, + }, + Vanished { + name: &'p str, + local: LocalId, + }, +} + +pub(super) struct Plan<'p> { + pub actions: Vec>, +} + +impl<'p> Plan<'p> { + pub fn compute(server: &'p [ServerScript], local: &'p LocalScripts) -> Plan<'p> { + let mut actions = Vec::with_capacity(server.len()); + let mut server_names: HashSet<&str> = HashSet::with_capacity(server.len()); + for script in server { + server_names.insert(&script.name); + let action = match local.ids.get(&script.name) { + None => Action::New(script), + Some(&id) => { + let stored = local.stored(id); + let outcome = if stored.hash != Some(script.hash) { + PresentOutcome::ContentUpdated + } else if stored.active != script.active { + PresentOutcome::ActiveOnly + } else { + PresentOutcome::Unchanged + }; + Action::Present { + script, + local: id, + stored, + outcome, + } + } + }; + actions.push(action); + } + actions.extend( + local + .ids + .iter() + .filter(|(name, _)| !server_names.contains(name.as_str())) + .map(|(name, &id)| Action::Vanished { name, local: id }), + ); + Plan { actions } + } + + pub fn counts(&self) -> PlanCounts { + let mut plan = PlanCounts::default(); + for action in &self.actions { + match action { + Action::New(_) => plan.new += 1, + Action::Present { .. } => plan.present += 1, + Action::Vanished { .. } => plan.vanished += 1, + } + } + plan + } +} + +#[derive(Clone, Copy)] +pub(super) struct ScriptWriter<'a> { + pub store: Store<'a>, + pub source: SourceId, +} + +impl ScriptWriter<'_> { + pub fn insert(&self, script: &ServerScript) -> Result { + let blob = self.store.blobs().intern(&script.bytes)?; + let local = self.store.sieve_scripts().insert(&SieveScriptRecord { + name: Some(&script.name), + is_active: false, + blob, + })?; + self.store + .sieve_ids(self.source) + .insert(&script.name, local)?; + Ok(local) + } + + pub fn update_content( + &self, + local: LocalId, + stored: StoredScript, + script: &ServerScript, + ) -> Result<(), Error> { + let blob = self.store.blobs().intern(&script.bytes)?; + self.store.sieve_scripts().update( + local, + &SieveScriptRecord { + name: Some(&script.name), + is_active: stored.active, + blob, + }, + )?; + Ok(()) + } + + pub fn delete(&self, name: &str, local: LocalId) -> Result<(), Error> { + self.store.sieve_scripts().delete(local)?; + self.store.sieve_ids(self.source).delete(name)?; + Ok(()) + } + + pub fn assign_active(&self, active: Option) -> Result<(), Error> { + self.store.sieve_scripts().set_active(active)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use rusqlite::OptionalExtension; + + use super::*; + use crate::archive::Archive; + use crate::archive::sources::{SourceKey, SourceKind}; + + fn archive() -> (Archive, SourceId) { + let archive = Archive::open_in_memory().unwrap(); + let source = archive + .store() + .sources() + .upsert( + &SourceKey { + kind: SourceKind::ManageSieve, + url: "sieve://h:4190", + account_id: "a", + }, + None, + "a", + ) + .unwrap(); + (archive, source) + } + + fn script(name: &str, active: bool, bytes: &[u8]) -> ServerScript { + ServerScript::new(name.to_owned(), active, bytes.to_vec()) + } + + fn local(entries: &[(&str, i64, &[u8], bool)]) -> LocalScripts { + let mut local = LocalScripts::default(); + for &(name, id, bytes, active) in entries { + let id = LocalId::from_raw(id); + local.ids.insert(name.to_owned(), id); + local.stored.insert( + id, + StoredScript { + hash: Some(*blake3::hash(bytes).as_bytes()), + active, + }, + ); + } + local + } + + fn outcomes(plan: &Plan<'_>) -> Vec> { + plan.actions + .iter() + .map(|action| match action { + Action::Present { outcome, .. } => Some(*outcome), + Action::New(_) | Action::Vanished { .. } => None, + }) + .collect() + } + + fn active_id(archive: &Archive) -> Option { + archive + .connection() + .query_row( + "SELECT id FROM sieve_scripts WHERE is_active = 1", + [], + |r| r.get(0), + ) + .optional() + .unwrap() + } + + fn count(archive: &Archive, table: &str) -> i64 { + archive + .connection() + .query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0)) + .unwrap() + } + + #[test] + fn plan_classifies_new_present_unchanged_and_vanished() { + let server = [ + script("a", false, b"v"), + script("b", true, b"w"), + script("c", false, b"v"), + ]; + let local = local(&[ + ("b", 2, b"w", true), + ("c", 3, b"v", false), + ("d", 4, b"v", false), + ]); + let plan = Plan::compute(&server, &local); + assert_eq!( + plan.counts(), + PlanCounts { + new: 1, + vanished: 1, + present: 2, + } + ); + assert!(matches!( + plan.actions[3], + Action::Vanished { name: "d", .. } + )); + } + + #[test] + fn plan_flags_content_update_when_hash_differs() { + let server = [script("a", false, b"w")]; + let local = local(&[("a", 1, b"v", false)]); + let plan = Plan::compute(&server, &local); + assert_eq!(outcomes(&plan), [Some(PresentOutcome::ContentUpdated)]); + } + + #[test] + fn plan_flags_active_only_when_only_flag_differs() { + let server = [script("a", true, b"same")]; + let local = local(&[("a", 1, b"same", false)]); + let plan = Plan::compute(&server, &local); + assert_eq!(outcomes(&plan), [Some(PresentOutcome::ActiveOnly)]); + } + + #[test] + fn insert_writes_script_and_id_mapping() { + let (archive, source) = archive(); + let writer = ScriptWriter { + store: archive.store(), + source, + }; + let id = writer + .insert(&script("vacation", true, b"require[\"x\"];")) + .unwrap(); + assert_eq!( + archive + .store() + .sieve_ids(source) + .local_id("vacation") + .unwrap(), + Some(id) + ); + assert_eq!(count(&archive, "sieve_scripts"), 1); + assert_eq!(active_id(&archive), None); + } + + #[test] + fn content_update_keeps_the_active_flag_and_orphans_the_old_blob() { + let (archive, source) = archive(); + let writer = ScriptWriter { + store: archive.store(), + source, + }; + let id = writer.insert(&script("a", true, b"v1")).unwrap(); + writer.assign_active(Some(id)).unwrap(); + let stored = LocalScripts::load(archive.store(), source) + .unwrap() + .stored(id); + writer + .update_content(id, stored, &script("a", true, b"v2")) + .unwrap(); + let data: Vec = archive + .connection() + .query_row( + "SELECT b.data FROM sieve_scripts s JOIN blobs b ON b.id = s.blob_id WHERE s.id = ?1", + [id], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(data, b"v2"); + assert_eq!(active_id(&archive), Some(id)); + assert_eq!(count(&archive, "blobs"), 2); + archive.store().blobs().gc_orphans().unwrap(); + assert_eq!(count(&archive, "blobs"), 1); + } + + #[test] + fn assign_active_swaps_and_clears() { + let (archive, source) = archive(); + let writer = ScriptWriter { + store: archive.store(), + source, + }; + let a = writer.insert(&script("a", false, b"x")).unwrap(); + let b = writer.insert(&script("b", false, b"y")).unwrap(); + for target in [Some(a), Some(b), None] { + writer.assign_active(target).unwrap(); + assert_eq!(active_id(&archive), target); + } + } + + #[test] + fn delete_removes_an_active_script_and_its_mapping() { + let (archive, source) = archive(); + let writer = ScriptWriter { + store: archive.store(), + source, + }; + let id = writer.insert(&script("a", false, b"x")).unwrap(); + writer.assign_active(Some(id)).unwrap(); + writer.delete("a", id).unwrap(); + assert_eq!(count(&archive, "sieve_scripts"), 0); + assert!( + archive + .store() + .sieve_ids(source) + .local_id("a") + .unwrap() + .is_none() + ); + } + + #[test] + fn identical_script_bytes_share_one_blob() { + let (archive, source) = archive(); + let writer = ScriptWriter { + store: archive.store(), + source, + }; + let bytes = b"require [\"fileinto\"];\n"; + writer.insert(&script("a", false, bytes)).unwrap(); + writer.insert(&script("b", false, bytes)).unwrap(); + assert_eq!(count(&archive, "blobs"), 1); + } + + #[test] + fn partial_unique_index_rejects_two_active_rows() { + let (archive, source) = archive(); + let writer = ScriptWriter { + store: archive.store(), + source, + }; + let a = writer.insert(&script("a", false, b"x")).unwrap(); + let b = writer.insert(&script("b", false, b"y")).unwrap(); + let set = "UPDATE sieve_scripts SET is_active = 1 WHERE id = ?1"; + archive.connection().execute(set, [a]).unwrap(); + let err = archive.connection().execute(set, [b]).unwrap_err(); + assert!(err.to_string().contains("UNIQUE"), "{err}"); + } +} diff --git a/src/import/managesieve/run.rs b/src/import/managesieve/run.rs new file mode 100644 index 0000000..1fef684 --- /dev/null +++ b/src/import/managesieve/run.rs @@ -0,0 +1,228 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::archive::sources::{SourceKey, SourceKind}; +use crate::archive::{Archive, LocalId, SourceId}; +use crate::error::Error; +use crate::managesieve::session::{SIEVE, SieveSession}; +use crate::net::backoff::{Fault, Retryable}; +use crate::net::stream::{Connector, Endpoint}; +use crate::sync::run::{GcFailure, ImportRun, SourceClaim, SourceLabel}; +use crate::sync::{RunOptions, RunOutcome, Summary, TypeCounts}; +use crate::types::ObjectType; + +use super::ManageSieveImport; +use super::control::Control; +use super::plan::{Action, LocalScripts, Plan, PresentOutcome, ScriptWriter, ServerScript}; + +struct SieveRun { + run: ImportRun, + control: Control, + counts: TypeCounts, +} + +impl ManageSieveImport { + pub fn run(self, options: RunOptions) -> RunOutcome { + let failed = |error: Error| RunOutcome { + summary: Summary::default(), + error: Some(error), + }; + let logger = options.logger; + if options.threads > 1 { + logger.progress(format_args!( + "ManageSieve import runs on one thread; --threads is ignored" + )); + } + let archive = match Archive::open(&options.archive) { + Ok(archive) => archive, + Err(e) => return failed(e.into()), + }; + let ManageSieveImport { + url, + auth, + cleartext, + source_change, + } = self; + let session = match Endpoint::parse(&url, &SIEVE).and_then(|endpoint| { + let connector = Connector::new(options.tls).map_err(Error::connection)?; + Ok(SieveSession::new( + connector, endpoint, cleartext, auth, logger, + )) + }) { + Ok(session) => session, + Err(e) => return failed(e), + }; + let client = match session.open() { + Ok(client) => client, + Err(e) => return failed(e), + }; + let session_url = session.session_url(); + let account_id = session.account_id().to_owned(); + let key = SourceKey { + kind: SourceKind::ManageSieve, + url: &session_url, + account_id: &account_id, + }; + let label = SourceLabel { + name: Some(&account_id), + username: &account_id, + }; + let max_retries = options.max_retries; + let run = match ImportRun::start_on(archive, options, &key, label, source_change) { + Ok(run) => run, + Err(e) => return failed(e), + }; + let mut sieve = SieveRun { + run, + control: Control::new(client, session, max_retries), + counts: TypeCounts::default(), + }; + let result = sieve.execute(); + sieve.finish(result) + } +} + +impl SieveRun { + fn execute(&mut self) -> Result<(), Error> { + let server = self.fetch_scripts()?; + match *self.run.claim() { + SourceClaim::DryRun(source) => self.dry_run(source, &server), + SourceClaim::Registered(source) => self.apply(source, &server), + } + } + + fn fetch_scripts(&mut self) -> Result, Error> { + let listing = self.control.listscripts()?; + let logger = self.run.options().logger; + let mut server = Vec::with_capacity(listing.len()); + for listed in listing { + let name = listed.name; + match self.control.getscript(&name) { + Ok(bytes) => server.push(ServerScript::new(name, listed.active, bytes)), + Err(e) if e.is_referral() => { + return Err(Error::Connection(format!( + "ManageSieve referral on GETSCRIPT {name:?}: {e}; inbuxa-migrate does not follow referrals" + ))); + } + Err(e) if e.fault() == Fault::TransportDrop => { + return Err(Error::Connection(format!( + "GETSCRIPT {name:?} transport drop exhausted retries: {e}" + ))); + } + Err(e) => { + logger.warn(format_args!("GETSCRIPT \"{name}\" failed; skipped: {e}")); + self.counts.failed += 1; + } + } + } + Ok(server) + } + + fn dry_run(&mut self, source: Option, server: &[ServerScript]) -> Result<(), Error> { + let local = match source { + Some(source) => LocalScripts::load(self.run.store(), source)?, + None => LocalScripts::default(), + }; + *self.run.summary().plan(ObjectType::SieveScript) = Plan::compute(server, &local).counts(); + Ok(()) + } + + fn apply(&mut self, source: SourceId, server: &[ServerScript]) -> Result<(), Error> { + let logger = self.run.options().logger; + let local = LocalScripts::load(self.run.store(), source)?; + let mut active: Option = None; + for action in Plan::compute(server, &local).actions { + match action { + Action::New(script) => match self.write(source, |writer| writer.insert(script))? { + Ok(local) => { + if script.active { + active = Some(local); + } + self.counts.created += 1; + self.counts.fetched += 1; + } + Err(e) => { + logger.warn(format_args!( + "cannot store sieve script \"{}\": {e}", + script.name + )); + self.counts.failed += 1; + } + }, + Action::Present { + script, + local, + stored, + outcome, + } => { + match outcome { + PresentOutcome::Unchanged | PresentOutcome::ActiveOnly => { + self.counts.skipped += 1; + } + PresentOutcome::ContentUpdated => { + match self.write(source, |writer| { + writer.update_content(local, stored, script) + })? { + Ok(()) => self.counts.fetched += 1, + Err(e) => { + logger.warn(format_args!( + "cannot update sieve script \"{}\": {e}", + script.name + )); + self.counts.failed += 1; + } + } + } + } + if script.active { + active = Some(local); + } + } + Action::Vanished { name, local } => { + match self.write(source, |writer| writer.delete(name, local))? { + Ok(()) => self.counts.deleted += 1, + Err(e) => { + logger.warn(format_args!("cannot delete sieve script \"{name}\": {e}")); + self.counts.failed += 1; + } + } + } + } + } + self.write(source, |writer| writer.assign_active(active))? + } + + fn write( + &mut self, + source: SourceId, + op: impl FnOnce(ScriptWriter<'_>) -> Result, + ) -> Result, Error> { + let tx = self.run.archive().transaction()?; + let written = op(ScriptWriter { + store: tx.store(), + source, + }); + if written.is_ok() { + tx.commit()?; + } + Ok(written) + } + + fn finish(self, result: Result<(), Error>) -> RunOutcome { + let SieveRun { + mut run, + mut control, + counts, + } = self; + *run.counts(ObjectType::SieveScript) = counts; + let stats = control.stats(); + if let Err(error) = result { + return run.abort(error, stats); + } + control.logout(); + run.finish(stats, GcFailure::Abort) + } +} diff --git a/src/import/mod.rs b/src/import/mod.rs new file mode 100644 index 0000000..4d227cb --- /dev/null +++ b/src/import/mod.rs @@ -0,0 +1,51 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod dav; +pub mod ews; +pub mod graph; +pub mod imap; +pub mod jmap; +pub mod maildir; +pub mod managesieve; +pub mod takeout; + +use crate::sync::{RunOptions, RunOutcome}; + +macro_rules! imports { + ($($variant:ident($ty:ty),)+) => { + pub enum Import { + $($variant($ty),)+ + } + + impl Import { + pub fn run(self, options: RunOptions) -> RunOutcome { + match self { + $(Import::$variant(import) => import.run(options),)+ + } + } + } + + $( + impl From<$ty> for Import { + fn from(import: $ty) -> Import { + Import::$variant(import) + } + } + )+ + }; +} + +imports! { + Jmap(jmap::JmapImport), + Imap(imap::ImapImport), + Dav(dav::DavImport), + ManageSieve(managesieve::ManageSieveImport), + Maildir(maildir::MaildirImport), + Takeout(takeout::TakeoutImport), + Ews(ews::EwsImport), + Graph(graph::GraphImport), +} diff --git a/src/import/takeout/calendar.rs b/src/import/takeout/calendar.rs new file mode 100644 index 0000000..200274e --- /dev/null +++ b/src/import/takeout/calendar.rs @@ -0,0 +1,273 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; +use std::fmt::Write as _; +use std::fs; +use std::path::Path; + +use crate::archive::ids::takeout::TakeoutIds; +use crate::archive::rows::{CalendarEventRecord, CalendarRecord, Subscribed, Write}; +use crate::archive::{LocalId, Store}; +use crate::error::Error; +use crate::logging::Logger; +use crate::sync::TypeCounts; +use crate::sync::calcard::{CalcardError, ExtractedFields, JsCalendarEntry, SyntheticUid}; +use crate::types::ObjectType; + +const DEFAULT_CALENDAR: &str = "Imported"; + +pub struct CalendarFile { + entries: Vec, + metadata: CalendarMetadata, +} + +impl CalendarFile { + pub fn load( + path: &Path, + event_counts: &mut TypeCounts, + logger: Logger, + ) -> Option { + let text = match fs::read_to_string(path) { + Ok(s) => s, + Err(e) => { + logger.warn(format_args!("cannot read {}: {e}", path.display())); + event_counts.failed += 1; + return None; + } + }; + let entries = match JsCalendarEntry::parse_all(&text) { + Ok(v) => v, + Err(CalcardError::NoEntries) => { + logger.warn(format_args!("{} has no events", path.display())); + return None; + } + Err(e) => { + logger.warn(format_args!("cannot parse {}: {e}", path.display())); + event_counts.failed += 1; + return None; + } + }; + Some(CalendarFile { + entries, + metadata: CalendarMetadata::parse(&text), + }) + } +} + +pub struct CalendarWriter<'a> { + pub store: Store<'a>, + pub ids: TakeoutIds<'a>, + pub calendars: &'a mut HashMap, + pub calendar_counts: &'a mut TypeCounts, + pub event_counts: &'a mut TypeCounts, + pub logger: Logger, +} + +impl CalendarWriter<'_> { + pub fn write(&mut self, path: &Path, file: CalendarFile) -> Result<(), Error> { + let CalendarFile { entries, metadata } = file; + let calendar = self.ensure_calendar(&metadata)?; + let mut seed = String::new(); + for (ordinal, mut entry) in entries.into_iter().enumerate() { + if let Err(e) = self.write_event(calendar, path, ordinal, &mut entry, &mut seed) { + self.logger.warn(format_args!( + "cannot store event {ordinal} of {}: {e}", + path.display() + )); + self.event_counts.failed += 1; + } + } + Ok(()) + } + + fn ensure_calendar(&mut self, metadata: &CalendarMetadata) -> Result { + let name = metadata.x_wr_calname.as_deref().unwrap_or(DEFAULT_CALENDAR); + if let Some(&id) = self.calendars.get(name) { + return Ok(id); + } + if let Some(id) = self.ids.local_id(ObjectType::Calendar, name)? { + self.calendars.insert(name.to_owned(), id); + return Ok(id); + } + let id = self.store.calendars().insert(&CalendarRecord { + name, + description: metadata.x_wr_caldesc.as_deref(), + color: None, + sort_order: 0, + subscribed: Subscribed::Yes, + is_visible: true, + is_default: false, + include_in_availability: "all", + default_alerts_with_time: None, + default_alerts_without_time: None, + time_zone: metadata.x_wr_timezone.as_deref(), + })?; + self.ids.insert(ObjectType::Calendar, name, id)?; + self.calendars.insert(name.to_owned(), id); + self.calendar_counts.created += 1; + self.calendar_counts.fetched += 1; + Ok(id) + } + + fn write_event( + &mut self, + calendar: LocalId, + path: &Path, + ordinal: usize, + entry: &mut JsCalendarEntry, + seed: &mut String, + ) -> Result<(), Error> { + let ExtractedFields { + is_draft, + use_default_alerts, + uid, + } = entry.take_extracted(); + let uid = match uid.as_deref().map(str::trim).filter(|s| !s.is_empty()) { + Some(u) => u.to_owned(), + None => { + seed.clear(); + let _ = write!(seed, "{}#{ordinal}", path.to_string_lossy()); + SyntheticUid::derive(seed) + } + }; + let data = entry.data.to_string(); + let events = self.store.calendar_events(); + + if let Some(local) = self.ids.local_id(ObjectType::CalendarEvent, &uid)? { + let calendars = StoredCalendars::merged(self.store, local, calendar)?; + let record = CalendarEventRecord { + calendars: &calendars, + is_draft, + use_default_alerts, + data_type: entry.data_type, + data: &data, + }; + match events.update(local, &record)? { + Write::Changed => self.event_counts.fetched += 1, + Write::Unchanged => self.event_counts.skipped += 1, + } + return Ok(()); + } + + let local = events.insert(&CalendarEventRecord { + calendars: &[calendar], + is_draft, + use_default_alerts, + data_type: entry.data_type, + data: &data, + })?; + self.ids.insert(ObjectType::CalendarEvent, &uid, local)?; + self.event_counts.created += 1; + self.event_counts.fetched += 1; + Ok(()) + } +} + +struct StoredCalendars; + +impl StoredCalendars { + fn merged(store: Store<'_>, event: LocalId, calendar: LocalId) -> Result, Error> { + let mut stmt = store.connection().prepare_cached( + "SELECT j.value FROM calendar_events e, json_each(e.calendar_ids) j + WHERE e.id = ?1 AND json_type(e.calendar_ids) = 'array' AND j.type = 'integer'", + )?; + let mut ids = stmt + .query_map([event], |r| r.get(0))? + .collect::>>()?; + ids.push(calendar); + ids.sort_unstable(); + ids.dedup(); + Ok(ids) + } +} + +#[derive(Debug, Default, Clone)] +struct CalendarMetadata { + x_wr_calname: Option, + x_wr_caldesc: Option, + x_wr_timezone: Option, +} + +impl CalendarMetadata { + fn parse(text: &str) -> CalendarMetadata { + let mut out = CalendarMetadata::default(); + let mut inside_vevent = false; + for raw_line in text.lines() { + let line = raw_line.trim_end_matches('\r'); + if line.starts_with("BEGIN:VEVENT") + || line.starts_with("BEGIN:VTODO") + || line.starts_with("BEGIN:VJOURNAL") + { + inside_vevent = true; + continue; + } + if line.starts_with("END:VEVENT") + || line.starts_with("END:VTODO") + || line.starts_with("END:VJOURNAL") + { + inside_vevent = false; + continue; + } + if inside_vevent { + continue; + } + if let Some(rest) = line.strip_prefix("X-WR-CALNAME:") { + out.x_wr_calname = Some(rest.trim().to_owned()); + } else if let Some(rest) = line.strip_prefix("X-WR-CALDESC:") { + out.x_wr_caldesc = Some(rest.trim().to_owned()); + } else if let Some(rest) = line.strip_prefix("X-WR-TIMEZONE:") { + out.x_wr_timezone = Some(rest.trim().to_owned()); + } + } + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_metadata_finds_xwr_fields() { + let ical = "BEGIN:VCALENDAR\r\n\ + PRODID:-//Google Inc//Google Calendar 70.9054//EN\r\n\ + X-WR-CALNAME:My Calendar\r\n\ + X-WR-CALDESC:Some description\r\n\ + X-WR-TIMEZONE:Europe/Madrid\r\n\ + BEGIN:VEVENT\r\nUID:e1@google.com\r\nDTSTAMP:20250101T000000Z\r\n\ + END:VEVENT\r\nEND:VCALENDAR\r\n"; + let m = CalendarMetadata::parse(ical); + assert_eq!(m.x_wr_calname.as_deref(), Some("My Calendar")); + assert_eq!(m.x_wr_caldesc.as_deref(), Some("Some description")); + assert_eq!(m.x_wr_timezone.as_deref(), Some("Europe/Madrid")); + } + + #[test] + fn parse_metadata_ignores_xwr_inside_vevent() { + let ical = "BEGIN:VCALENDAR\r\n\ + BEGIN:VEVENT\r\nUID:e1\r\nX-WR-CALNAME:NOT THIS\r\nEND:VEVENT\r\n\ + END:VCALENDAR\r\n"; + let m = CalendarMetadata::parse(ical); + assert!(m.x_wr_calname.is_none()); + } + + #[test] + fn parse_metadata_empty_when_missing() { + let ical = "BEGIN:VCALENDAR\r\nEND:VCALENDAR\r\n"; + let m = CalendarMetadata::parse(ical); + assert!(m.x_wr_calname.is_none()); + assert!(m.x_wr_caldesc.is_none()); + assert!(m.x_wr_timezone.is_none()); + } + + #[test] + fn parse_metadata_handles_lf_only_line_endings() { + let ical = "BEGIN:VCALENDAR\nX-WR-CALNAME:Foo\nEND:VCALENDAR\n"; + let m = CalendarMetadata::parse(ical); + assert_eq!(m.x_wr_calname.as_deref(), Some("Foo")); + } +} diff --git a/src/import/takeout/contacts.rs b/src/import/takeout/contacts.rs new file mode 100644 index 0000000..ef78bbd --- /dev/null +++ b/src/import/takeout/contacts.rs @@ -0,0 +1,215 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fs; +use std::path::Path; + +use crate::archive::ids::takeout::TakeoutIds; +use crate::archive::rows::{AddressBookRecord, ContactCardRecord, Subscribed, Write}; +use crate::archive::{LocalId, Store}; +use crate::error::Error; +use crate::logging::Logger; +use crate::sync::TypeCounts; +use crate::sync::calcard::JsContact; +use crate::types::ObjectType; + +pub const DEFAULT_ADDRESS_BOOK: &str = "Imported"; + +pub struct VCardFile; + +impl VCardFile { + pub fn load(path: &Path, card_counts: &mut TypeCounts, logger: Logger) -> Vec { + match fs::read_to_string(path) { + Ok(text) => VCardFile::split(&text), + Err(e) => { + logger.warn(format_args!("cannot read {}: {e}", path.display())); + card_counts.failed += 1; + Vec::new() + } + } + } + + fn split(text: &str) -> Vec { + let mut chunks: Vec = Vec::new(); + let mut current: Option = None; + for raw_line in text.lines() { + let line = raw_line.trim_end_matches('\r'); + if VCardFile::starts_with_ignore_case(line, "BEGIN:VCARD") { + current = Some(String::new()); + } + if let Some(buf) = current.as_mut() { + buf.push_str(line); + buf.push('\n'); + } + if VCardFile::starts_with_ignore_case(line, "END:VCARD") + && let Some(buf) = current.take() + { + chunks.push(buf); + } + } + chunks + } + + fn starts_with_ignore_case(line: &str, prefix: &str) -> bool { + line.as_bytes() + .get(..prefix.len()) + .is_some_and(|head| head.eq_ignore_ascii_case(prefix.as_bytes())) + } +} + +pub struct ContactWriter<'a> { + pub store: Store<'a>, + pub ids: TakeoutIds<'a>, + pub book: &'a mut Option, + pub book_counts: &'a mut TypeCounts, + pub card_counts: &'a mut TypeCounts, + pub logger: Logger, +} + +impl ContactWriter<'_> { + pub fn write(&mut self, path: &Path, chunks: &[String]) -> Result<(), Error> { + let book = match *self.book { + Some(id) => id, + None => { + let id = self.ensure_default_address_book()?; + *self.book = Some(id); + id + } + }; + for (ordinal, chunk) in chunks.iter().enumerate() { + if let Err(e) = self.write_card(book, chunk) { + self.logger.warn(format_args!( + "cannot store vCard {ordinal} of {}: {e}", + path.display() + )); + self.card_counts.failed += 1; + } + } + Ok(()) + } + + fn ensure_default_address_book(&mut self) -> Result { + if let Some(id) = self + .ids + .local_id(ObjectType::AddressBook, DEFAULT_ADDRESS_BOOK)? + { + return Ok(id); + } + let id = self.store.address_books().insert(&AddressBookRecord { + name: DEFAULT_ADDRESS_BOOK, + description: None, + sort_order: 0, + is_default: false, + subscribed: Subscribed::Yes, + })?; + self.ids + .insert(ObjectType::AddressBook, DEFAULT_ADDRESS_BOOK, id)?; + self.book_counts.created += 1; + self.book_counts.fetched += 1; + Ok(id) + } + + fn write_card(&mut self, book: LocalId, chunk: &str) -> Result<(), Error> { + let card = JsContact::from_vcard(chunk, chunk) + .map_err(|e| Error::Partial(format!("vCard parse: {e}")))?; + let data = card.data.to_string(); + let record = ContactCardRecord { + uid: &card.uid, + address_books: &[book], + data: &data, + }; + let cards = self.store.contact_cards(); + if let Some(local) = self.ids.local_id(ObjectType::ContactCard, &card.uid)? { + match cards.update(local, &record)? { + Write::Changed => self.card_counts.fetched += 1, + Write::Unchanged => self.card_counts.skipped += 1, + } + return Ok(()); + } + let local = cards.insert(&record)?; + self.ids.insert(ObjectType::ContactCard, &card.uid, local)?; + self.card_counts.created += 1; + self.card_counts.fetched += 1; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn splits_single_card() { + let text = "BEGIN:VCARD\r\nVERSION:3.0\r\nFN:A\r\nEND:VCARD\r\n"; + let chunks = VCardFile::split(text); + assert_eq!(chunks.len(), 1); + assert!(chunks[0].contains("FN:A")); + } + + #[test] + fn splits_multiple_cards() { + let text = "BEGIN:VCARD\r\nVERSION:3.0\r\nFN:A\r\nEND:VCARD\r\n\ + BEGIN:VCARD\r\nVERSION:3.0\r\nFN:B\r\nEND:VCARD\r\n\ + BEGIN:VCARD\r\nVERSION:3.0\r\nFN:C\r\nEND:VCARD\r\n"; + let chunks = VCardFile::split(text); + assert_eq!(chunks.len(), 3); + assert!(chunks[0].contains("FN:A")); + assert!(chunks[1].contains("FN:B")); + assert!(chunks[2].contains("FN:C")); + } + + #[test] + fn skips_lines_outside_any_card() { + let text = "Some preamble garbage\r\n\ + BEGIN:VCARD\r\nVERSION:3.0\r\nFN:A\r\nEND:VCARD\r\n\ + garbage between cards\r\n\ + BEGIN:VCARD\r\nVERSION:3.0\r\nFN:B\r\nEND:VCARD\r\n\ + trailing garbage\r\n"; + let chunks = VCardFile::split(text); + assert_eq!(chunks.len(), 2); + assert!(!chunks[0].contains("garbage")); + } + + #[test] + fn preserves_folded_continuation_lines_inside_a_card() { + let text = "BEGIN:VCARD\r\nVERSION:3.0\r\n\ + ADR:;;1 Infinite Loop\\nCupertino\\, CA 95014\\nUnited States\r\n \ + ;;;;;\r\nEND:VCARD\r\n"; + let chunks = VCardFile::split(text); + assert_eq!(chunks.len(), 1); + assert!(chunks[0].contains("ADR:")); + assert!(chunks[0].contains(";;;;;")); + } + + #[test] + fn handles_lf_only_line_endings() { + let text = "BEGIN:VCARD\nVERSION:3.0\nFN:LF\nEND:VCARD\n"; + let chunks = VCardFile::split(text); + assert_eq!(chunks.len(), 1); + assert!(chunks[0].contains("FN:LF")); + } + + #[test] + fn empty_input_yields_no_chunks() { + assert!(VCardFile::split("").is_empty()); + assert!(VCardFile::split("not a vcard\n").is_empty()); + } + + #[test] + fn unterminated_card_is_dropped() { + let text = "BEGIN:VCARD\r\nVERSION:3.0\r\nFN:A\r\n"; + let chunks = VCardFile::split(text); + assert!(chunks.is_empty()); + } + + #[test] + fn case_insensitive_boundary_match() { + let text = "begin:vcard\r\nFN:CaseTest\r\nend:vcard\r\n"; + let chunks = VCardFile::split(text); + assert_eq!(chunks.len(), 1); + assert!(chunks[0].contains("FN:CaseTest")); + } +} diff --git a/src/sync/import_takeout/labels.rs b/src/import/takeout/labels.rs similarity index 51% rename from src/sync/import_takeout/labels.rs rename to src/import/takeout/labels.rs index fb9c135..fb741a5 100644 --- a/src/sync/import_takeout/labels.rs +++ b/src/import/takeout/labels.rs @@ -5,69 +5,102 @@ */ use std::borrow::Cow; -use std::collections::BTreeSet; +use std::str; -use mail_parser::HeaderValue; -use mail_parser::parsers::MessageStream; +use mail_parser::{HeaderForm, HeaderValue}; + +use crate::imap::automap::Automap; +use crate::sync::keywords::Keywords; #[derive(Debug, Clone, PartialEq, Eq)] pub struct LabelClassification { pub mailboxes: Vec, - pub keywords: BTreeSet, - pub had_explicit_seen_signal: bool, - pub had_unread_signal: bool, + pub keywords: Keywords, pub opened_won_over_unread: bool, } -impl LabelClassification { - pub fn keywords_sorted(&self) -> Vec { - self.keywords.iter().cloned().collect() +pub struct GmailLabels; + +impl GmailLabels { + pub fn parse(value: &str) -> Vec { + GmailLabels::decode_encoded_words(value) + .split(',') + .map(str::trim) + .filter(|t| !t.is_empty()) + .map(str::to_owned) + .collect() } - pub fn is_empty(&self) -> bool { - self.mailboxes.is_empty() && self.keywords.is_empty() + pub fn header(bytes: &[u8]) -> Option { + let mut joined: Option = None; + let mut current: Option = None; + for line in bytes.split(|&b| b == b'\n') { + let line = line.strip_suffix(b"\r").unwrap_or(line); + if line.is_empty() { + break; + } + if let [b' ' | b'\t', ..] = line { + if let Some(buf) = current.as_mut() { + let extra = str::from_utf8(line).unwrap_or("").trim_start(); + if !extra.is_empty() { + buf.push(' '); + buf.push_str(extra); + } + } + continue; + } + GmailLabels::append(&mut joined, current.take()); + let mut parts = line.splitn(2, |&b| b == b':'); + if let (Some(name), Some(value)) = (parts.next(), parts.next()) + && name.eq_ignore_ascii_case(b"X-Gmail-Labels") + { + current = Some(str::from_utf8(value).unwrap_or("").trim().to_owned()); + } + } + GmailLabels::append(&mut joined, current); + joined + } + + fn append(joined: &mut Option, value: Option) { + let Some(value) = value else { + return; + }; + match joined { + Some(out) => { + out.push(','); + out.push_str(&value); + } + None => *joined = Some(value), + } + } + + fn decode_encoded_words(value: &str) -> Cow<'_, str> { + if !value.contains("=?") { + return Cow::Borrowed(value); + } + match HeaderForm::Text.parse(value.as_bytes()).value() { + HeaderValue::Text(text) => Cow::Owned(text.to_owned()), + _ => Cow::Borrowed(value), + } + } + + pub fn role(canonical_path: &str, automap: Automap) -> Option<&'static str> { + if automap == Automap::Disabled || canonical_path.contains('/') { + return None; + } + hashify::map!(canonical_path.as_bytes(), Role, + "Inbox" => "inbox", + "Sent" => "sent", + "Drafts" => "drafts", + "Trash" => "trash", + "Spam" => "junk", + "Archive" => "archive", + ) + .copied() } } -#[derive(Debug, Clone, Copy)] -pub struct MappingOptions { - pub automap: bool, -} - -impl Default for MappingOptions { - fn default() -> Self { - Self { automap: true } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct MailboxAssignment { - pub canonical_path: String, - pub role: Option<&'static str>, -} - -pub fn parse_header(value: &str) -> Vec { - let decoded = decode_encoded_words(value); - decoded - .split(',') - .map(str::trim) - .filter(|t| !t.is_empty()) - .map(str::to_owned) - .collect() -} - -fn decode_encoded_words(value: &str) -> Cow<'_, str> { - if !value.contains("=?") { - return Cow::Borrowed(value); - } - let mut buf = Vec::with_capacity(value.len() + 1); - buf.extend_from_slice(value.as_bytes()); - buf.push(b'\n'); - match MessageStream::new(&buf).parse_unstructured() { - HeaderValue::Text(text) => Cow::Owned(text.into_owned()), - _ => Cow::Borrowed(value), - } -} +type Role = &'static str; #[derive(Debug, Clone, Copy)] enum SystemLabel { @@ -113,128 +146,118 @@ impl SystemLabel { } } -pub fn classify(tokens: &[String]) -> LabelClassification { - let mut mailboxes: Vec = Vec::new(); - let mut keywords: BTreeSet = BTreeSet::new(); - let mut had_opened = false; - let mut had_unread = false; +impl LabelClassification { + pub fn classify(tokens: &[String]) -> LabelClassification { + let mut mailboxes: Vec = Vec::new(); + let mut keywords = Keywords::new(); + let mut had_opened = false; + let mut had_unread = false; - for tok in tokens { - let mailbox = match SystemLabel::parse(tok) { - Some(SystemLabel::Category) => continue, - Some(SystemLabel::Starred) => { - keywords.insert("$flagged".to_owned()); - continue; + for tok in tokens { + let mailbox = match SystemLabel::parse(tok) { + Some(SystemLabel::Category) => continue, + Some(SystemLabel::Starred) => { + keywords.push("$flagged"); + continue; + } + Some(SystemLabel::Important) => { + keywords.push("$important"); + continue; + } + Some(SystemLabel::Opened) => { + had_opened = true; + continue; + } + Some(SystemLabel::Unread) => { + had_unread = true; + continue; + } + Some(SystemLabel::Mailbox(name)) => name, + None => tok.as_str(), + }; + if !mailboxes.iter().any(|m| m == mailbox) { + mailboxes.push(mailbox.to_owned()); } - Some(SystemLabel::Important) => { - keywords.insert("$important".to_owned()); - continue; - } - Some(SystemLabel::Opened) => { - had_opened = true; - continue; - } - Some(SystemLabel::Unread) => { - had_unread = true; - continue; - } - Some(SystemLabel::Mailbox(name)) => name, - None => tok.as_str(), - }; - if !mailboxes.iter().any(|m| m == mailbox) { - mailboxes.push(mailbox.to_owned()); + } + + if had_opened || (!had_unread && !tokens.is_empty()) { + keywords.push("$seen"); + } + keywords.sort(); + + LabelClassification { + mailboxes, + keywords, + opened_won_over_unread: had_opened && had_unread, } } - - if had_opened || (!had_unread && !tokens.is_empty()) { - keywords.insert("$seen".to_owned()); - } - - let opened_won_over_unread = had_opened && had_unread; - LabelClassification { - mailboxes, - keywords, - had_explicit_seen_signal: had_opened, - had_unread_signal: had_unread, - opened_won_over_unread, - } -} - -pub fn role_for_mailbox(canonical_path: &str, opts: MappingOptions) -> Option<&'static str> { - if !opts.automap { - return None; - } - if canonical_path.contains('/') { - return None; - } - match canonical_path { - "Inbox" => Some("inbox"), - "Sent" => Some("sent"), - "Drafts" => Some("drafts"), - "Trash" => Some("trash"), - "Spam" => Some("junk"), - "Archive" => Some("archive"), - _ => None, - } -} - -pub fn assignments_for( - classification: &LabelClassification, - opts: MappingOptions, -) -> Vec { - classification - .mailboxes - .iter() - .map(|p| MailboxAssignment { - canonical_path: p.clone(), - role: role_for_mailbox(p, opts), - }) - .collect() } #[cfg(test)] mod tests { use super::*; + struct Assignment<'a> { + canonical_path: &'a str, + role: Option<&'static str>, + } + + fn assignments_for(c: &LabelClassification, automap: Automap) -> Vec> { + c.mailboxes + .iter() + .map(|p| Assignment { + canonical_path: p, + role: GmailLabels::role(p, automap), + }) + .collect() + } + fn classify_str(s: &str) -> LabelClassification { - classify(&parse_header(s)) + LabelClassification::classify(&GmailLabels::parse(s)) } #[test] fn parse_header_splits_on_comma_and_trims() { - let v = parse_header("Inbox, Opened ,Starred,,Important"); + let v = GmailLabels::parse("Inbox, Opened ,Starred,,Important"); assert_eq!(v, vec!["Inbox", "Opened", "Starred", "Important"]); } #[test] fn parse_header_decodes_rfc2047_encoded_words() { - let v = parse_header("=?UTF-8?Q?=C3=85pnet,Kategori:_Reklame?="); + let v = GmailLabels::parse("=?UTF-8?Q?=C3=85pnet,Kategori:_Reklame?="); assert_eq!(v, vec!["Åpnet", "Kategori: Reklame"]); } #[test] fn parse_header_decodes_mixed_plain_and_encoded_tokens() { - let v = parse_header("Inbox,=?UTF-8?Q?=C3=85pnet?=,Github"); + let v = GmailLabels::parse("Inbox,=?UTF-8?Q?=C3=85pnet?=,Github"); assert_eq!(v, vec!["Inbox", "Åpnet", "Github"]); } + // A character split across two encoded words is joined before decoding. + #[test] + fn parse_header_joins_a_character_split_across_encoded_words() { + let v = GmailLabels::parse("=?utf-8?q?caf=C3?= =?UTF-8?q?=A9?=,Inbox"); + assert_eq!(v, vec!["café", "Inbox"]); + } + #[test] fn parse_header_leaves_plain_ascii_untouched() { - let v = parse_header("Inbox,Opened,Label_001/Label_002"); + let v = GmailLabels::parse("Inbox,Opened,Label_001/Label_002"); assert_eq!(v, vec!["Inbox", "Opened", "Label_001/Label_002"]); } #[test] fn parse_header_empty_yields_empty() { - assert!(parse_header("").is_empty()); - assert!(parse_header(",, ,, ").is_empty()); + assert!(GmailLabels::parse("").is_empty()); + assert!(GmailLabels::parse(",, ,, ").is_empty()); } #[test] fn inbox_token_becomes_inbox_mailbox_with_role() { let c = classify_str("Inbox"); assert_eq!(c.mailboxes, vec!["Inbox".to_owned()]); - let a = assignments_for(&c, MappingOptions::default()); + let a = assignments_for(&c, Automap::Enabled); assert_eq!(a[0].role, Some("inbox")); } @@ -248,7 +271,7 @@ mod tests { ("Archived", "archive"), ] { let c = classify_str(token); - let a = assignments_for(&c, MappingOptions::default()); + let a = assignments_for(&c, Automap::Enabled); assert_eq!(a[0].role, Some(role), "token {token}"); } let c = classify_str("Archived"); @@ -273,7 +296,6 @@ mod tests { fn unread_alone_means_no_seen() { let c = classify_str("Inbox,Unread"); assert!(!c.keywords.contains("$seen")); - assert!(c.had_unread_signal); } #[test] @@ -286,7 +308,7 @@ mod tests { #[test] fn no_labels_sets_no_keywords() { - assert!(classify(&[]).keywords.is_empty()); + assert!(LabelClassification::classify(&[]).keywords.is_empty()); } #[test] @@ -301,8 +323,11 @@ mod tests { "Inbox", "Sent", "Drafts", "Trash", "Archive", "Spam", "Clientes" ] ); - assert_eq!(c.keywords_sorted(), ["$flagged", "$important", "$seen"]); - let roles: Vec<_> = assignments_for(&c, MappingOptions::default()) + assert_eq!( + c.keywords.iter().collect::>(), + ["$flagged", "$important", "$seen"] + ); + let roles: Vec<_> = assignments_for(&c, Automap::Enabled) .into_iter() .map(|a| a.role) .collect(); @@ -324,7 +349,6 @@ mod tests { fn pt_br_unread_label_means_no_seen() { let c = classify_str("Caixa de entrada,Não lidas"); assert!(!c.keywords.contains("$seen")); - assert!(c.had_unread_signal); } #[test] @@ -348,7 +372,7 @@ mod tests { c.mailboxes .contains(&"Label_001/Label_002_under_Label_001".to_owned()) ); - let a = assignments_for(&c, MappingOptions::default()); + let a = assignments_for(&c, Automap::Enabled); let nested = a .iter() .find(|m| m.canonical_path == "Label_001/Label_002_under_Label_001") @@ -359,7 +383,7 @@ mod tests { #[test] fn custom_labels_get_no_role() { let c = classify_str("Inbox,Github,Newsletter"); - let a = assignments_for(&c, MappingOptions::default()); + let a = assignments_for(&c, Automap::Enabled); let github = a.iter().find(|m| m.canonical_path == "Github").unwrap(); assert_eq!(github.role, None); } @@ -378,15 +402,14 @@ mod tests { assert!(c.mailboxes.contains(&"Github".to_owned())); assert!(c.keywords.contains("$important")); assert!(!c.keywords.contains("$seen")); - assert!(c.had_unread_signal); } #[test] fn noautomap_suppresses_roles_but_keeps_names() { let c = classify_str("Inbox,Sent,Github"); - let a = assignments_for(&c, MappingOptions { automap: false }); + let a = assignments_for(&c, Automap::Disabled); assert!(a.iter().all(|m| m.role.is_none())); - let names: Vec<&str> = a.iter().map(|m| m.canonical_path.as_str()).collect(); + let names: Vec<&str> = a.iter().map(|m| m.canonical_path).collect(); assert!(names.contains(&"Inbox")); assert!(names.contains(&"Sent")); assert!(names.contains(&"Github")); @@ -395,15 +418,8 @@ mod tests { #[test] fn keywords_sorted_returns_deterministic_order() { let c = classify_str("Starred,Important,Opened"); - let k = c.keywords_sorted(); - assert_eq!( - k, - vec![ - "$flagged".to_owned(), - "$important".to_owned(), - "$seen".to_owned() - ] - ); + let k = c.keywords.iter().collect::>(); + assert_eq!(k, ["$flagged", "$important", "$seen"]); } #[test] @@ -416,4 +432,58 @@ mod tests { .count(); assert_eq!(count, 1); } + + #[test] + fn extract_labels_from_simple_message() { + let bytes = b"From: a@b\r\n\ + X-Gmail-Labels: Inbox,Opened\r\n\ + Subject: hi\r\n\ + \r\n\ + body\r\n"; + assert_eq!(GmailLabels::header(bytes), Some("Inbox,Opened".to_owned())); + } + + #[test] + fn extract_labels_handles_lf_only_line_endings() { + let bytes = b"From: a@b\nX-Gmail-Labels: Inbox\n\nbody\n"; + assert_eq!(GmailLabels::header(bytes), Some("Inbox".to_owned())); + } + + #[test] + fn extract_labels_handles_folded_continuation() { + let bytes = b"From: a@b\r\n\ + X-Gmail-Labels: Inbox,\r\n\ + \tImportant,Github\r\n\ + Subject: hi\r\n\ + \r\n\ + body\r\n"; + assert_eq!( + GmailLabels::header(bytes), + Some("Inbox, Important,Github".to_owned()) + ); + } + + #[test] + fn extract_labels_returns_none_when_header_absent() { + let bytes = b"From: a@b\r\nSubject: hi\r\n\r\nbody\r\n"; + assert_eq!(GmailLabels::header(bytes), None); + } + + #[test] + fn extract_labels_case_insensitive_name_match() { + let bytes = b"x-gmail-labels: Sent\r\n\r\n"; + assert_eq!(GmailLabels::header(bytes), Some("Sent".to_owned())); + } + + #[test] + fn extract_labels_concatenates_multiple_header_occurrences() { + let bytes = b"X-Gmail-Labels: Inbox\r\nX-Gmail-Labels: Github\r\n\r\nbody"; + assert_eq!(GmailLabels::header(bytes), Some("Inbox,Github".to_owned())); + } + + #[test] + fn extract_labels_stops_at_blank_line() { + let bytes = b"X-Gmail-Labels: Inbox\r\n\r\nX-Gmail-Labels: NOT-A-HEADER\r\n"; + assert_eq!(GmailLabels::header(bytes), Some("Inbox".to_owned())); + } } diff --git a/src/import/takeout/mail.rs b/src/import/takeout/mail.rs new file mode 100644 index 0000000..4dc3a80 --- /dev/null +++ b/src/import/takeout/mail.rs @@ -0,0 +1,362 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; +use std::io; +use std::path::Path; + +use crate::archive::ids::takeout::TakeoutIds; +use crate::archive::rows::{EmailRecord, MailboxRecord, Subscribed, Write}; +use crate::archive::{LocalId, Store}; +use crate::console::Counted; +use crate::error::Error; +use crate::imap::automap::Automap; +use crate::logging::Logger; +use crate::sync::TypeCounts; +use crate::sync::emailmeta::EmailMeta; +use crate::types::ObjectType; + +use super::labels::{GmailLabels, LabelClassification}; +use super::mbox::Message; + +const DEFAULT_MAILBOX: &str = "Imported"; +const SHORT_HASH: usize = 12; +const PROGRESS_TICK: u64 = 256; + +pub struct MailWriter<'a> { + pub store: Store<'a>, + pub ids: TakeoutIds<'a>, + pub automap: Automap, + pub fallback: &'a str, + pub mailboxes: &'a mut HashMap, + pub mailbox_counts: &'a mut TypeCounts, + pub email_counts: &'a mut TypeCounts, + pub logger: Logger, +} + +impl MailWriter<'_> { + pub fn fallback_mailbox(path: &Path) -> &str { + path.file_stem() + .and_then(|s| s.to_str()) + .unwrap_or(DEFAULT_MAILBOX) + } + + pub fn write_all(&mut self, path: &Path, messages: impl Iterator>) { + let logger = self.logger; + let mut processed: u64 = 0; + for message in messages { + let message = match message { + Ok(message) => message, + Err(e) => { + logger.warn(format_args!("cannot read {}: {e}", path.display())); + self.email_counts.failed += 1; + break; + } + }; + match self.write_message(&message) { + Ok(()) => { + processed += 1; + if processed.is_multiple_of(PROGRESS_TICK) { + logger.progress(format_args!( + "{}: {} processed", + path.display(), + Counted::new(processed, "message", "messages") + )); + } + } + Err(e) => { + logger.warn(format_args!( + "cannot import a message from {}: {e}", + path.display() + )); + self.email_counts.failed += 1; + } + } + } + } + + fn write_message(&mut self, message: &Message) -> Result<(), Error> { + let bytes = message.contents(); + let hash = blake3::hash(bytes).to_hex(); + let msg_hash = hash.as_str(); + let short = msg_hash.get(..SHORT_HASH).unwrap_or(msg_hash); + + let header = GmailLabels::header(bytes); + let tokens = header + .as_deref() + .map(GmailLabels::parse) + .unwrap_or_default(); + let mut classification = LabelClassification::classify(&tokens); + if classification.mailboxes.is_empty() { + let fallback = match self.fallback.trim() { + "" => DEFAULT_MAILBOX, + fallback => fallback, + }; + if header.is_some() { + self.logger.warn(format_args!( + "message {short} has an empty X-Gmail-Labels header; filed under \"{fallback}\"" + )); + } else { + self.logger.warn(format_args!( + "message {short} has no X-Gmail-Labels header; filed under \"{fallback}\"" + )); + } + classification.mailboxes.push(fallback.to_owned()); + } + if classification.opened_won_over_unread { + self.logger.warn(format_args!( + "message {short} is labelled both Opened and Unread; marked as seen" + )); + } + + let mut mailbox_ids: Vec = Vec::with_capacity(classification.mailboxes.len()); + for path in &classification.mailboxes { + mailbox_ids.push(self.ensure_mailbox(path)?); + } + mailbox_ids.sort_unstable(); + mailbox_ids.dedup(); + let keywords = classification.keywords.to_json(); + + let emails = self.store.emails(); + if let Some(local) = self.ids.local_id(ObjectType::Email, msg_hash)? { + match emails.update_placement(local, &mailbox_ids, &keywords)? { + Write::Changed => self.email_counts.fetched += 1, + Write::Unchanged => self.email_counts.skipped += 1, + } + return Ok(()); + } + + let blob = self.store.blobs().intern(bytes)?; + let EmailMeta { index, date } = EmailMeta::from_blob(bytes); + let message_match = index.to_json(); + let received_at = message.received_at(date); + let local = emails.insert(&EmailRecord { + blob, + received_at: &received_at, + mailboxes: &mailbox_ids, + keywords: &keywords, + message_match: Some(&message_match), + })?; + self.ids.insert(ObjectType::Email, msg_hash, local)?; + self.email_counts.created += 1; + self.email_counts.fetched += 1; + Ok(()) + } + + fn ensure_mailbox(&mut self, path: &str) -> Result { + if let Some(&id) = self.mailboxes.get(path) { + return Ok(id); + } + if let Some(id) = self.ids.local_id(ObjectType::Mailbox, path)? { + self.mailboxes.insert(path.to_owned(), id); + return Ok(id); + } + let (parent, leaf) = match path.rsplit_once('/') { + Some((parent, leaf)) => (Some(self.ensure_mailbox(parent)?), leaf), + None => (None, path), + }; + let id = self.store.mailboxes().insert(&MailboxRecord { + name: leaf, + parent, + role: GmailLabels::role(path, self.automap), + sort_order: 0, + subscribed: Subscribed::Yes, + })?; + self.ids.insert(ObjectType::Mailbox, path, id)?; + self.mailboxes.insert(path.to_owned(), id); + self.mailbox_counts.created += 1; + self.mailbox_counts.fetched += 1; + Ok(id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::Archive; + use crate::archive::sources::{SourceKey, SourceKind}; + use crate::logging::Level; + + struct Planned { + rows: Vec<(String, Option, Option)>, + ids: Vec, + created: u64, + mapped: u64, + } + + fn plan(paths: &[&str], automap: Automap) -> Planned { + let archive = Archive::open_in_memory().unwrap(); + let store = archive.store(); + let source = store + .sources() + .upsert( + &SourceKey { + kind: SourceKind::Takeout, + url: "file:///tmp/Takeout", + account_id: "/tmp/Takeout", + }, + Some("Takeout"), + "", + ) + .unwrap(); + let mut mailboxes = HashMap::new(); + let mut mailbox_counts = TypeCounts::default(); + let mut email_counts = TypeCounts::default(); + let mut writer = MailWriter { + store, + ids: store.takeout_ids(source), + automap, + fallback: "", + mailboxes: &mut mailboxes, + mailbox_counts: &mut mailbox_counts, + email_counts: &mut email_counts, + logger: Logger::new(Level::Quiet), + }; + let ids = paths + .iter() + .map(|path| writer.ensure_mailbox(path).unwrap()) + .collect(); + let mapped = store + .takeout_ids(source) + .count(ObjectType::Mailbox) + .unwrap(); + let rows = archive + .connection() + .prepare( + "SELECT m.name, p.name, m.role FROM mailboxes m + LEFT JOIN mailboxes p ON p.id = m.parent_id ORDER BY m.name", + ) + .unwrap() + .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?))) + .unwrap() + .collect::>() + .unwrap(); + Planned { + rows, + ids, + created: mailbox_counts.created, + mapped, + } + } + + fn row( + name: &str, + parent: Option<&str>, + role: Option<&str>, + ) -> (String, Option, Option) { + ( + name.to_owned(), + parent.map(str::to_owned), + role.map(str::to_owned), + ) + } + + #[test] + fn flat_paths_create_top_level_mailboxes_only() { + let p = plan(&["Inbox", "Sent", "Github"], Automap::Disabled); + assert_eq!( + p.rows, + [ + row("Github", None, None), + row("Inbox", None, None), + row("Sent", None, None) + ] + ); + assert_eq!((p.created, p.mapped), (3, 3)); + } + + #[test] + fn nested_path_creates_its_missing_parent() { + let p = plan(&["Label_001/Label_002_under_Label_001"], Automap::Disabled); + assert_eq!( + p.rows, + [ + row("Label_001", None, None), + row("Label_002_under_Label_001", Some("Label_001"), None) + ] + ); + assert_eq!((p.created, p.mapped), (2, 2)); + } + + #[test] + fn explicit_parent_is_reused_by_its_children() { + let p = plan( + &["Label_001", "Label_001/Sub", "Label_001/Other"], + Automap::Disabled, + ); + assert_eq!( + p.rows, + [ + row("Label_001", None, None), + row("Other", Some("Label_001"), None), + row("Sub", Some("Label_001"), None) + ] + ); + assert_eq!((p.created, p.mapped), (3, 3)); + } + + #[test] + fn three_level_path_creates_every_missing_ancestor() { + let p = plan(&["A/B/C"], Automap::Disabled); + assert_eq!( + p.rows, + [ + row("A", None, None), + row("B", Some("A"), None), + row("C", Some("B"), None) + ] + ); + assert_eq!(p.created, 3); + } + + #[test] + fn role_applies_to_an_auto_created_parent() { + let p = plan(&["Inbox/Sub"], Automap::Enabled); + assert_eq!( + p.rows, + [ + row("Inbox", None, Some("inbox")), + row("Sub", Some("Inbox"), None) + ] + ); + } + + #[test] + fn role_applies_to_an_explicit_top_level_mailbox() { + let p = plan(&["Inbox"], Automap::Enabled); + assert_eq!(p.rows, [row("Inbox", None, Some("inbox"))]); + } + + #[test] + fn duplicate_paths_share_one_mailbox() { + let p = plan(&["Inbox", "Inbox", "Inbox"], Automap::Disabled); + assert_eq!(p.rows.len(), 1); + assert!(p.ids.windows(2).all(|w| w[0] == w[1])); + assert_eq!((p.created, p.mapped), (1, 1)); + } + + #[test] + fn no_paths_create_no_mailboxes() { + let p = plan(&[], Automap::Enabled); + assert!(p.rows.is_empty()); + assert_eq!((p.created, p.mapped), (0, 0)); + } + + #[test] + fn fallback_mailbox_strips_extension() { + let p = Path::new("/some/dir/Github.mbox"); + assert_eq!(MailWriter::fallback_mailbox(p), "Github"); + } + + #[test] + fn fallback_mailbox_handles_dotted_basename() { + let p = Path::new("/x/All mail Including Spam and Trash.mbox"); + assert_eq!( + MailWriter::fallback_mailbox(p), + "All mail Including Spam and Trash" + ); + } +} diff --git a/src/import/takeout/mbox.rs b/src/import/takeout/mbox.rs new file mode 100644 index 0000000..3437161 --- /dev/null +++ b/src/import/takeout/mbox.rs @@ -0,0 +1,317 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fs::File; +use std::io::{self, BufRead, BufReader}; +use std::path::Path; +use std::str; + +use time::{Date, Month, OffsetDateTime, Time, UtcOffset}; + +use crate::error::Error; +use crate::sync::dates::{MissingDate, Rfc3339}; + +pub struct MessageIterator { + reader: T, + message: Option, +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct Message { + internal_date: u64, + contents: Vec, +} + +impl MessageIterator +where + T: BufRead, +{ + pub fn new(reader: T) -> MessageIterator { + MessageIterator { + reader, + message: None, + } + } +} + +impl MessageIterator> { + pub fn open(path: &Path) -> Result { + File::open(path) + .map(|file| MessageIterator::new(BufReader::new(file))) + .map_err(|e| Error::Partial(format!("open {path:?}: {e}"))) + } +} + +impl Iterator for MessageIterator +where + T: BufRead, +{ + type Item = io::Result; + + fn next(&mut self) -> Option { + let mut line = Vec::with_capacity(80); + + loop { + match self.reader.read_until(b'\n', &mut line) { + Ok(0) => return self.message.take().map(Ok), + Ok(_) => {} + Err(e) => return Some(Err(e)), + } + + if let Some(envelope) = line.strip_prefix(b"From ") { + let finished = self.message.take().map(Ok); + self.message = Some(Message::from_envelope(envelope)); + if finished.is_some() { + return finished; + } + line.clear(); + continue; + } + + if let Some(message) = &mut self.message { + match line.split_first() { + Some((b'>', unquoted)) if Message::is_quoted_from(unquoted) => { + message.contents.extend_from_slice(unquoted); + } + _ => message.contents.extend_from_slice(&line), + } + } + line.clear(); + } + } +} + +impl Message { + fn from_envelope(envelope: &[u8]) -> Self { + let envelope = str::from_utf8(envelope).unwrap_or(""); + let date = envelope.split_once(' ').map_or("", |(_, rest)| rest); + Self { + internal_date: EnvelopeDate::parse(date).unwrap_or(0), + contents: Vec::with_capacity(1024), + } + } + + fn is_quoted_from(mut line: &[u8]) -> bool { + while let [b'>', rest @ ..] = line { + line = rest; + } + line.starts_with(b"From ") + } + + pub fn received_at(&self, date_header: Option) -> String { + if self.internal_date > 0 { + return Rfc3339::from_unix(self.internal_date); + } + date_header.unwrap_or_else(|| MissingDate::Now.resolve()) + } + + pub fn contents(&self) -> &[u8] { + &self.contents + } +} + +struct EnvelopeDate; + +impl EnvelopeDate { + fn parse(s: &str) -> Option { + let mut parts = s.split_whitespace(); + let (month, day, time, offset, year) = match ( + parts.next(), + parts.next(), + parts.next(), + parts.next(), + parts.next(), + parts.next(), + parts.next(), + ) { + (Some(_), Some(month), Some(day), Some(time), Some(year), None, None) => { + (month, day, time, None, year) + } + (Some(_), Some(month), Some(day), Some(time), Some(offset), Some(year), None) => { + (month, day, time, Some(offset), year) + } + _ => return None, + }; + + let month = EnvelopeDate::month(month)?; + let day: u8 = day.parse().ok()?; + let (hour, minute, second) = EnvelopeDate::hms(time)?; + let year: i32 = year.parse().ok()?; + let offset = match offset { + Some(offset) => EnvelopeDate::offset(offset)?, + None => UtcOffset::UTC, + }; + + let date = Date::from_calendar_date(year, month, day).ok()?; + let time = Time::from_hms(hour, minute, second).ok()?; + let ts = OffsetDateTime::new_in_offset(date, time, offset).unix_timestamp(); + u64::try_from(ts).ok() + } + + fn month(s: &str) -> Option { + hashify::map_ignore_case!(s.as_bytes(), Month, + "jan" => Month::January, + "feb" => Month::February, + "mar" => Month::March, + "apr" => Month::April, + "may" => Month::May, + "jun" => Month::June, + "jul" => Month::July, + "aug" => Month::August, + "sep" => Month::September, + "oct" => Month::October, + "nov" => Month::November, + "dec" => Month::December, + ) + .copied() + } + + fn hms(s: &str) -> Option<(u8, u8, u8)> { + let mut it = s.split(':'); + let h: u8 = it.next()?.parse().ok()?; + let m: u8 = it.next()?.parse().ok()?; + let sec: u8 = it.next()?.parse().ok()?; + if it.next().is_some() { + return None; + } + Some((h, m, sec)) + } + + fn offset(s: &str) -> Option { + let (sign, digits) = match s.as_bytes() { + [b'+', digits @ ..] => (1i8, digits), + [b'-', digits @ ..] => (-1i8, digits), + _ => return None, + }; + let [h1, h2, m1, m2] = digits else { + return None; + }; + let hh: i8 = str::from_utf8(&[*h1, *h2]).ok()?.parse().ok()?; + let mm: i8 = str::from_utf8(&[*m1, *m2]).ok()?.parse().ok()?; + UtcOffset::from_hms(sign * hh, sign * mm, 0).ok() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classic_ctime_envelope_parses() { + let raw = b"From god@heaven.af.mil Sat Jan 3 01:05:34 1996\n\ + Message 1\n\ + \n\ + From cras@irccrew.org Tue Jul 23 19:39:23 2002\n\ + Message 2\n\ + \n"; + let mut it = MessageIterator::new(&raw[..]); + let m1 = it.next().unwrap().unwrap(); + assert_eq!(m1.internal_date, 820631134); + assert_eq!(m1.contents(), b"Message 1\n\n"); + let m2 = it.next().unwrap().unwrap(); + assert_eq!(m2.internal_date, 1027453163); + assert!(it.next().is_none()); + } + + #[test] + fn mboxrd_unescape_recovers_lines() { + let raw = b"From a@b Tue Aug 6 13:34:34 2002\n\ + Message 3\n\ + >From hello\n\ + >>From world\n\ + >>>From test\n\ + \n"; + let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap(); + assert_eq!( + m.contents(), + b"Message 3\nFrom hello\n>From world\n>>From test\n\n" + ); + } + + #[test] + fn mboxrd_leaves_non_from_lines_starting_with_gt_alone() { + let raw = b"From a@b Mon Jan 15 15:30:00 2018\n\ + Message 4\n\ + > From\n\ + >F\n"; + let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap(); + assert_eq!(m.contents(), b"Message 4\n> From\n>F\n"); + } + + #[test] + fn takeout_envelope_with_utc_offset_parses() { + let raw = b"From 1848707889910060830@xxx Thu Nov 13 20:20:33 +0000 2025\n\ + X-GM-THRID: 1848707889910060830\n\ + X-Gmail-Labels: Inbox,Opened\n\ + Subject: hi\n\ + \n\ + body\n"; + let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap(); + assert_eq!(m.internal_date, 1763065233); + assert!(m.contents().starts_with(b"X-GM-THRID:")); + } + + #[test] + fn takeout_envelope_with_negative_offset_applies_offset() { + let raw = b"From x@y Thu Nov 13 12:20:33 -0800 2025\n\ + body\n"; + let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap(); + assert_eq!(m.internal_date, 1763065233); + } + + #[test] + fn malformed_envelope_date_yields_zero_not_error() { + let raw = b"From who knows what\nbody\n"; + let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap(); + assert_eq!(m.internal_date, 0); + } + + #[test] + fn streams_multiple_takeout_format_messages() { + let raw = b"From 1@xxx Thu Nov 13 20:20:33 +0000 2025\n\ + X-GM-THRID: 1\n\ + X-Gmail-Labels: Inbox\n\ + Subject: A\n\ + \n\ + body a\n\ + From 2@xxx Thu Nov 13 21:00:00 +0000 2025\n\ + X-GM-THRID: 2\n\ + X-Gmail-Labels: Sent,Archived\n\ + Subject: B\n\ + \n\ + body b\n"; + let parsed: Vec<_> = MessageIterator::new(&raw[..]) + .collect::, _>>() + .unwrap(); + assert_eq!(parsed.len(), 2); + assert!(parsed[0].contents().starts_with(b"X-GM-THRID: 1")); + } + + #[test] + fn received_at_prefers_envelope_internal_date() { + let raw = b"From x@y Thu Nov 13 20:20:33 +0000 2025\nbody\n"; + let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap(); + let got = m.received_at(Some("2025-05-12T10:00:00+02:00".to_owned())); + assert_eq!(got, "2025-11-13T20:20:33Z"); + } + + #[test] + fn received_at_falls_back_to_date_header() { + let raw = b"From who knows what\nbody\n"; + let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap(); + let got = m.received_at(Some("2025-05-12T10:00:00+02:00".to_owned())); + assert_eq!(got, "2025-05-12T10:00:00+02:00"); + } + + #[test] + fn received_at_last_resort_is_now_in_rfc3339() { + let raw = b"From who knows what\nbody\n"; + let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap(); + let got = m.received_at(None); + assert!(got.ends_with("Z") || got.contains("+") || got.contains("-")); + assert!(got.contains("T")); + } +} diff --git a/src/import/takeout/mod.rs b/src/import/takeout/mod.rs new file mode 100644 index 0000000..8f53148 --- /dev/null +++ b/src/import/takeout/mod.rs @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +mod calendar; +mod contacts; +mod labels; +mod mail; +mod mbox; +mod run; +mod walk; + +use std::path::PathBuf; + +use crate::archive::sources::SourceChange; +use crate::imap::automap::Automap; + +#[derive(Debug)] +pub struct TakeoutImport { + pub takeout_root: PathBuf, + pub source_change: SourceChange, + pub automap: Automap, +} diff --git a/src/import/takeout/run.rs b/src/import/takeout/run.rs new file mode 100644 index 0000000..40a9ee1 --- /dev/null +++ b/src/import/takeout/run.rs @@ -0,0 +1,388 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; +use std::fmt; +use std::fs; +use std::path::{Path, PathBuf}; + +use url::Url; + +use crate::archive::sources::{SourceKey, SourceKind}; +use crate::archive::{Archive, LocalId, SourceId}; +use crate::console::{Counted, Grouped}; +use crate::error::Error; +use crate::imap::automap::Automap; +use crate::logging::Logger; +use crate::net::http::RetryStats; +use crate::sync::run::{GcFailure, ImportRun, SourceClaim, SourceLabel}; +use crate::sync::{RunOptions, RunOutcome, Summary, TypeCounts}; +use crate::types::ObjectType; + +use super::TakeoutImport; +use super::calendar::{CalendarFile, CalendarWriter}; +use super::contacts::{ContactWriter, DEFAULT_ADDRESS_BOOK, VCardFile}; +use super::mail::MailWriter; +use super::mbox::MessageIterator; +use super::walk::{FileKind, WalkResult}; + +const TYPES: [ObjectType; 6] = [ + ObjectType::Mailbox, + ObjectType::Email, + ObjectType::Calendar, + ObjectType::CalendarEvent, + ObjectType::AddressBook, + ObjectType::ContactCard, +]; + +#[derive(Default)] +struct Counts { + mailbox: TypeCounts, + email: TypeCounts, + calendar: TypeCounts, + event: TypeCounts, + book: TypeCounts, + card: TypeCounts, +} + +struct TakeoutRun { + run: ImportRun, + automap: Automap, + counts: Counts, +} + +impl TakeoutImport { + pub fn run(self, options: RunOptions) -> RunOutcome { + let failed = |error: Error| RunOutcome { + summary: Summary::default(), + error: Some(error), + }; + let logger = options.logger; + if options.threads > 1 { + logger.progress(format_args!( + "Takeout import runs on one thread; --threads is ignored" + )); + } + let TakeoutImport { + takeout_root, + source_change, + automap, + } = self; + let canonical = match TakeoutImport::scan_root(&takeout_root) { + Ok(canonical) => canonical, + Err(e) => return failed(e), + }; + let walk = match WalkResult::scan(&canonical, logger) { + Ok(walk) if walk.is_empty() => { + return failed(Error::Usage(format!( + "{canonical:?}: no .mbox / .ics / .vcf files found under this path" + ))); + } + Ok(walk) => walk, + Err(e) => return failed(Error::Usage(format!("walking {canonical:?}: {e}"))), + }; + logger.progress(format_args!( + "Found {} mbox, {} iCalendar and {} vCard files ({} unreadable, {} symlink cycles)", + Grouped(walk.by_kind(FileKind::Mbox).count() as u64), + Grouped(walk.by_kind(FileKind::Ics).count() as u64), + Grouped(walk.by_kind(FileKind::Vcf).count() as u64), + Grouped(walk.io_failures), + Grouped(walk.symlink_cycles) + )); + let archive = match Archive::open(&options.archive) { + Ok(archive) => archive, + Err(e) => return failed(e.into()), + }; + let session_url = match TakeoutImport::source_url(&canonical) { + Ok(url) => url, + Err(e) => return failed(e), + }; + let account_id = canonical.to_string_lossy(); + let key = SourceKey { + kind: SourceKind::Takeout, + url: &session_url, + account_id: &account_id, + }; + let label = SourceLabel { + name: canonical.file_name().and_then(|n| n.to_str()), + username: "", + }; + let run = match ImportRun::start_on(archive, options, &key, label, source_change) { + Ok(run) => run, + Err(e) => return failed(e), + }; + let mut takeout = TakeoutRun { + run, + automap, + counts: Counts::default(), + }; + let result = takeout.execute(&walk); + takeout.finish(result) + } + + fn scan_root(root: &Path) -> Result { + let canonical = + fs::canonicalize(root).map_err(|e| Error::Usage(format!("--path {root:?}: {e}")))?; + if !canonical.is_dir() { + return Err(Error::Usage(format!("not a directory: {canonical:?}"))); + } + Ok(canonical) + } + + fn source_url(path: &Path) -> Result { + Url::from_file_path(path).map(String::from).map_err(|_| { + Error::Usage(format!( + "cannot encode takeout path as file:// URL: {path:?}" + )) + }) + } +} + +impl TakeoutRun { + fn logger(&self) -> Logger { + self.run.options().logger + } + + fn execute(&mut self, walk: &WalkResult) -> Result<(), Error> { + let source = match *self.run.claim() { + SourceClaim::DryRun(source) => { + self.dry_run(source, walk); + return Ok(()); + } + SourceClaim::Registered(source) => source, + }; + self.import_mail(source, walk)?; + self.import_calendars(source, walk)?; + self.import_contacts(source, walk) + } + + fn finish(self, result: Result<(), Error>) -> RunOutcome { + let TakeoutRun { + mut run, counts, .. + } = self; + let Counts { + mailbox, + email, + calendar, + event, + book, + card, + } = counts; + for (ty, counts) in TYPES + .into_iter() + .zip([mailbox, email, calendar, event, book, card]) + { + *run.counts(ty) = counts; + } + match result { + Ok(()) => run.finish(RetryStats::default(), GcFailure::Abort), + Err(error) => run.abort(error, RetryStats::default()), + } + } + + fn import_mail(&mut self, source: SourceId, walk: &WalkResult) -> Result<(), Error> { + let logger = self.logger(); + let mut mailboxes = self + .run + .store() + .takeout_ids(source) + .all(ObjectType::Mailbox)?; + for file in walk.by_kind(FileKind::Mbox) { + let fallback = MailWriter::fallback_mailbox(&file.path); + logger.progress(format_args!( + "Reading mail from {} (unlabelled messages go to \"{fallback}\")", + FileName(&file.path) + )); + if let Err(e) = self.import_mbox(source, &file.path, fallback, &mut mailboxes) { + if e.aborts_run() { + return Err(e); + } + logger.warn(format_args!("cannot import {}: {e}", file.path.display())); + self.counts.email.failed += 1; + } + } + Ok(()) + } + + fn import_mbox( + &mut self, + source: SourceId, + path: &Path, + fallback: &str, + mailboxes: &mut HashMap, + ) -> Result<(), Error> { + let messages = MessageIterator::open(path)?; + let logger = self.logger(); + let automap = self.automap; + let tx = self.run.archive().transaction()?; + let store = tx.store(); + MailWriter { + store, + ids: store.takeout_ids(source), + automap, + fallback, + mailboxes, + mailbox_counts: &mut self.counts.mailbox, + email_counts: &mut self.counts.email, + logger, + } + .write_all(path, messages); + tx.commit()?; + Ok(()) + } + + fn import_calendars(&mut self, source: SourceId, walk: &WalkResult) -> Result<(), Error> { + let logger = self.logger(); + let mut calendars = self + .run + .store() + .takeout_ids(source) + .all(ObjectType::Calendar)?; + for file in walk.by_kind(FileKind::Ics) { + logger.progress(format_args!("Reading events from {}", FileName(&file.path))); + if let Err(e) = self.import_ics(source, &file.path, &mut calendars) { + if e.aborts_run() { + return Err(e); + } + logger.warn(format_args!("cannot import {}: {e}", file.path.display())); + self.counts.event.failed += 1; + } + } + Ok(()) + } + + fn import_ics( + &mut self, + source: SourceId, + path: &Path, + calendars: &mut HashMap, + ) -> Result<(), Error> { + let logger = self.logger(); + let Some(file) = CalendarFile::load(path, &mut self.counts.event, logger) else { + return Ok(()); + }; + let tx = self.run.archive().transaction()?; + let store = tx.store(); + CalendarWriter { + store, + ids: store.takeout_ids(source), + calendars, + calendar_counts: &mut self.counts.calendar, + event_counts: &mut self.counts.event, + logger, + } + .write(path, file)?; + tx.commit()?; + Ok(()) + } + + fn import_contacts(&mut self, source: SourceId, walk: &WalkResult) -> Result<(), Error> { + let logger = self.logger(); + let mut book = self + .run + .store() + .takeout_ids(source) + .local_id(ObjectType::AddressBook, DEFAULT_ADDRESS_BOOK)?; + for file in walk.by_kind(FileKind::Vcf) { + logger.progress(format_args!( + "Reading contacts from {}", + FileName(&file.path) + )); + if let Err(e) = self.import_vcf(source, &file.path, &mut book) { + if e.aborts_run() { + return Err(e); + } + logger.warn(format_args!("cannot import {}: {e}", file.path.display())); + self.counts.card.failed += 1; + } + } + Ok(()) + } + + fn import_vcf( + &mut self, + source: SourceId, + path: &Path, + book: &mut Option, + ) -> Result<(), Error> { + let logger = self.logger(); + let chunks = VCardFile::load(path, &mut self.counts.card, logger); + if chunks.is_empty() { + return Ok(()); + } + let tx = self.run.archive().transaction()?; + let store = tx.store(); + ContactWriter { + store, + ids: store.takeout_ids(source), + book, + book_counts: &mut self.counts.book, + card_counts: &mut self.counts.card, + logger, + } + .write(path, &chunks)?; + tx.commit()?; + Ok(()) + } + + fn dry_run(&self, source: Option, walk: &WalkResult) { + let ids = source.map(|source| self.run.store().takeout_ids(source)); + let existing = |ty: ObjectType| ids.map_or(0, |ids| ids.count(ty).unwrap_or(0)); + self.logger().note(format_args!( + "a Takeout dry run does not parse files; found {} mbox, {} iCalendar and {} vCard files, \ + and the archive holds {}, {} and {}", + Grouped(walk.by_kind(FileKind::Mbox).count() as u64), + Grouped(walk.by_kind(FileKind::Ics).count() as u64), + Grouped(walk.by_kind(FileKind::Vcf).count() as u64), + Counted::new(existing(ObjectType::Email), "email", "emails"), + Counted::new( + existing(ObjectType::CalendarEvent), + "calendar event", + "calendar events" + ), + Counted::new(existing(ObjectType::ContactCard), "contact card", "contact cards") + )); + } +} + +struct FileName<'p>(&'p Path); + +impl fmt::Display for FileName<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0.file_name() { + Some(name) => name.display().fmt(f), + None => self.0.display().fmt(f), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn file_url_encodes_unicode_path() { + let u = TakeoutImport::source_url(Path::new("/tmp/Año")).unwrap(); + assert!(u.starts_with("file://")); + assert!(u.contains("A%C3%B1o")); + } + + // Progress lines name a file by its last component, or the whole path when it has none. + #[test] + fn file_name_shows_the_last_component() { + assert_eq!( + FileName(Path::new("/t/Mail/All mail.mbox")).to_string(), + "All mail.mbox" + ); + assert_eq!(FileName(Path::new("/")).to_string(), "/"); + } + + #[test] + fn file_url_rejects_relative_path() { + let err = TakeoutImport::source_url(Path::new("./relative")).unwrap_err(); + assert!(matches!(err, Error::Usage(_))); + } +} diff --git a/src/sync/import_takeout/walk.rs b/src/import/takeout/walk.rs similarity index 61% rename from src/sync/import_takeout/walk.rs rename to src/import/takeout/walk.rs index 6832a48..ea95561 100644 --- a/src/sync/import_takeout/walk.rs +++ b/src/import/takeout/walk.rs @@ -19,14 +19,14 @@ pub enum FileKind { } impl FileKind { - fn from_extension(ext: &str) -> Option { - let lower = ext.to_ascii_lowercase(); - match lower.as_str() { - "mbox" => Some(FileKind::Mbox), - "ics" => Some(FileKind::Ics), - "vcf" => Some(FileKind::Vcf), - _ => None, - } + fn of(path: &Path) -> Option { + let ext = path.extension()?.to_str()?; + hashify::map_ignore_case!(ext.as_bytes(), FileKind, + "mbox" => FileKind::Mbox, + "ics" => FileKind::Ics, + "vcf" => FileKind::Vcf, + ) + .copied() } } @@ -51,105 +51,119 @@ impl WalkResult { pub fn is_empty(&self) -> bool { self.files.is_empty() } -} -pub fn walk(root: &Path) -> io::Result { - walk_with_logger(root, Logger::from_flags(true, 0)) -} - -pub fn walk_with_logger(root: &Path, logger: Logger) -> io::Result { - let mut result = WalkResult::default(); - let mut visited: HashSet = HashSet::new(); - walk_dir(root, &mut result, &mut visited, logger)?; - result.files.sort_by(|a, b| a.path.cmp(&b.path)); - Ok(result) -} - -fn walk_dir( - dir: &Path, - result: &mut WalkResult, - visited: &mut HashSet, - logger: Logger, -) -> io::Result<()> { - let canonical = fs::canonicalize(dir)?; - if !visited.insert(canonical) { - result.symlink_cycles += 1; - logger.warn(&format!( - "takeout walk: symlink cycle broken at {dir:?} (target already visited)" - )); - return Ok(()); + pub fn scan(root: &Path, logger: Logger) -> io::Result { + let mut walker = Walker { + result: WalkResult::default(), + visited: HashSet::new(), + logger, + }; + walker.walk_dir(root)?; + let mut result = walker.result; + result.files.sort_by(|a, b| a.path.cmp(&b.path)); + Ok(result) } +} - let entries = match fs::read_dir(dir) { - Ok(it) => it, - Err(e) => { - result.io_failures += 1; - logger.warn(&format!("takeout walk: read_dir {dir:?}: {e}")); +struct Walker { + result: WalkResult, + visited: HashSet, + logger: Logger, +} + +impl Walker { + fn walk_dir(&mut self, dir: &Path) -> io::Result<()> { + let logger = self.logger; + let canonical = fs::canonicalize(dir)?; + if !self.visited.insert(canonical) { + self.result.symlink_cycles += 1; + logger.warn(format_args!( + "symlink cycle at {}; not followed again", + dir.display() + )); return Ok(()); } - }; - for entry in entries { - let entry = match entry { - Ok(e) => e, + let entries = match fs::read_dir(dir) { + Ok(it) => it, Err(e) => { - result.io_failures += 1; - logger.warn(&format!("takeout walk: entry in {dir:?}: {e}")); - continue; + self.result.io_failures += 1; + logger.warn(format_args!("cannot read directory {}: {e}", dir.display())); + return Ok(()); } }; - let path = entry.path(); - let file_type = match entry.file_type() { - Ok(t) => t, - Err(e) => { - result.io_failures += 1; - logger.warn(&format!("takeout walk: file_type {path:?}: {e}")); - continue; - } - }; - if file_type.is_dir() { - walk_dir(&path, result, visited, logger)?; - continue; - } - if file_type.is_symlink() { - let target_meta = match fs::metadata(&path) { - Ok(m) => m, + + for entry in entries { + let entry = match entry { + Ok(e) => e, Err(e) => { - result.io_failures += 1; - logger.warn(&format!("takeout walk: symlink target {path:?}: {e}")); + self.result.io_failures += 1; + logger.warn(format_args!( + "cannot read an entry of {}: {e}", + dir.display() + )); continue; } }; - if target_meta.is_dir() { - walk_dir(&path, result, visited, logger)?; + let path = entry.path(); + let file_type = match entry.file_type() { + Ok(t) => t, + Err(e) => { + self.result.io_failures += 1; + logger.warn(format_args!( + "cannot read the file type of {}: {e}", + path.display() + )); + continue; + } + }; + if file_type.is_dir() { + self.walk_dir(&path)?; continue; } - if !target_meta.is_file() { + if file_type.is_symlink() { + let target_meta = match fs::metadata(&path) { + Ok(m) => m, + Err(e) => { + self.result.io_failures += 1; + logger.warn(format_args!( + "cannot resolve symlink {}: {e}", + path.display() + )); + continue; + } + }; + if target_meta.is_dir() { + self.walk_dir(&path)?; + continue; + } + if !target_meta.is_file() { + continue; + } + } else if !file_type.is_file() { continue; } - } else if !file_type.is_file() { - continue; - } - if let Some(kind) = classify(&path) { - result.files.push(DiscoveredFile { path, kind }); + if let Some(kind) = FileKind::of(&path) { + self.result.files.push(DiscoveredFile { path, kind }); + } } + Ok(()) } - Ok(()) -} - -fn classify(path: &Path) -> Option { - let ext = path.extension()?.to_str()?; - FileKind::from_extension(ext) } #[cfg(test)] mod tests { use super::*; + use crate::logging::Level; use std::fs; #[cfg(unix)] use std::os::unix::fs::symlink; use tempfile::TempDir; + fn walk(root: &Path) -> io::Result { + WalkResult::scan(root, Logger::new(Level::Quiet)) + } + fn touch(path: &Path) { if let Some(parent) = path.parent() { fs::create_dir_all(parent).unwrap(); diff --git a/src/inspect/list.rs b/src/inspect/list.rs index 2690b06..f2e332f 100644 --- a/src/inspect/list.rs +++ b/src/inspect/list.rs @@ -5,627 +5,635 @@ */ use std::collections::HashMap; +use std::fmt::{self, Write as _}; use std::io::Write; -use rusqlite::Connection; +use rusqlite::Row; use serde_json::Value; -use super::{blob_summary, format_count}; +use super::{Page, View}; +use crate::archive::BlobId; +use crate::console::{ByteSize, Column as TableColumn, Grouped, Initial, Table, Tone}; use crate::error::Error; use crate::types::ObjectType; -pub fn write_list( - conn: &Connection, - ty: ObjectType, - limit: Option, - offset: usize, - out: &mut impl Write, -) -> Result<(), Error> { - let table = crate::sync::table_name(ty); - let total: i64 = conn.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r.get(0))?; - let total = total as u64; - write_header(out, ty, total, limit, offset)?; - if total == 0 { - writeln!(out, "(no {} in archive)", ty.token())?; - return Ok(()); - } - let names = Names::load(conn, ty)?; - let mut written = 0u64; - match ty { - ObjectType::Identity => write_identities(conn, limit, offset, out, &mut written)?, - ObjectType::Email => write_emails(conn, &names, limit, offset, out, &mut written)?, - ObjectType::SieveScript => write_sieves(conn, limit, offset, out, &mut written)?, - ObjectType::AddressBook => write_address_books(conn, limit, offset, out, &mut written)?, - ObjectType::ContactCard => { - write_contact_cards(conn, &names, limit, offset, out, &mut written)? - } - ObjectType::Calendar => write_calendars(conn, limit, offset, out, &mut written)?, - ObjectType::CalendarEvent => { - write_calendar_events(conn, &names, limit, offset, out, &mut written)? - } - ObjectType::ParticipantIdentity => { - write_participant_identities(conn, limit, offset, out, &mut written)? - } - ObjectType::Mailbox | ObjectType::FileNode => unreachable!("tree types do not list"), - } - if let Some(lim) = limit { - let next_offset = (offset as u64) + written; - if next_offset < total { - writeln!( - out, - "... ({} more; --offset {} --limit {} to continue)", - format_count(total - next_offset), - next_offset, - lim - )?; +const NULL_TEXT: &str = "-"; +const DASH: &str = "-"; +const MISSING: &str = "missing"; +const EMAIL_COLUMNS: [TableColumn; 6] = [ + TableColumn::right("ID"), + TableColumn::right("Size"), + TableColumn::left("Received"), + TableColumn::left("Mailboxes"), + TableColumn::left("Keywords"), + TableColumn::left("Message-ID"), +]; +const EMAIL_BATCH_ROWS: usize = 256; +const BLOB_KEY: &str = "@blob"; +const MESSAGE_IDS_KEY: &str = "m"; + +struct LimitClause(Page); + +impl fmt::Display for LimitClause { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0.limit { + Some(n) => write!(f, " LIMIT {n} OFFSET {}", self.0.offset), + None if self.0.offset > 0 => write!(f, " LIMIT -1 OFFSET {}", self.0.offset), + None => Ok(()), } } - Ok(()) } -fn write_header( - out: &mut impl Write, - ty: ObjectType, - total: u64, - limit: Option, - offset: usize, -) -> Result<(), Error> { - match limit { - None if offset == 0 => writeln!(out, "{} ({})", ty.jmap_name(), format_count(total))?, - None => writeln!( - out, - "{} ({}, skipping first {})", - ty.jmap_name(), - format_count(total), - format_count(offset as u64) - )?, - Some(lim) => writeln!( - out, - "{} ({} total, showing up to {} from offset {})", - ty.jmap_name(), - format_count(total), - format_count(lim as u64), - format_count(offset as u64) - )?, +struct Column<'t>(&'t str); + +impl Column<'_> { + fn or_null(value: Option) -> String { + value.unwrap_or_else(|| NULL_TEXT.to_owned()) + } + + fn flag(value: i64) -> String { + (if value != 0 { "yes" } else { "no" }).to_owned() + } + + fn local_ids(&self) -> Vec { + serde_json::from_str::>(self.0) + .map(|arr| arr.iter().filter_map(Value::as_i64).collect()) + .unwrap_or_default() + } + + fn keyword_list(&self, out: &mut String) { + let names = serde_json::from_str::>(self.0).unwrap_or_default(); + for (n, name) in names.iter().enumerate() { + if n > 0 { + out.push_str(", "); + } + out.push_str(name); + } + } + + fn first_message_id(&self, out: &mut String) { + if let Some(id) = serde_json::from_str::(self.0) + .ok() + .as_ref() + .and_then(|v| v.get(MESSAGE_IDS_KEY)) + .and_then(|m| m.get(0)) + .and_then(Value::as_str) + { + out.push_str(id); + } + } + + fn redacted_json(&self) -> String { + let Ok(mut value) = serde_json::from_str::(self.0) else { + return self.0.to_owned(); + }; + Column::redact(&mut value); + serde_json::to_string_pretty(&value).unwrap_or_else(|_| self.0.to_owned()) + } + + fn redact(value: &mut Value) { + match value { + Value::Object(map) => { + if let Some(blob) = map.get(BLOB_KEY).and_then(Value::as_i64) { + map.clear(); + map.insert( + BLOB_KEY.to_owned(), + Value::String(format!("local #{blob} (omitted)")), + ); + return; + } + map.values_mut().for_each(Column::redact); + } + Value::Array(arr) => arr.iter_mut().for_each(Column::redact), + _ => {} + } } - writeln!(out)?; - Ok(()) } -pub(crate) struct Names { +struct Names { mailboxes: HashMap, address_books: HashMap, calendars: HashMap, } impl Names { - pub(crate) fn load(conn: &Connection, ty: ObjectType) -> Result { - let mailboxes = if matches!(ty, ObjectType::Email) { - load_name_map(conn, "SELECT id, name FROM mailboxes")? - } else { - HashMap::new() - }; - let address_books = if matches!(ty, ObjectType::ContactCard) { - load_name_map(conn, "SELECT id, name FROM address_books")? - } else { - HashMap::new() - }; - let calendars = if matches!(ty, ObjectType::CalendarEvent) { - load_name_map(conn, "SELECT id, name FROM calendars")? - } else { - HashMap::new() + fn load(view: View<'_>, ty: ObjectType) -> Result { + let load = |wanted: ObjectType, sql: &str| -> Result, Error> { + if ty != wanted { + return Ok(HashMap::new()); + } + let mut stmt = view.conn.prepare(sql)?; + let rows = stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?; + Ok(rows.collect::>()?) }; Ok(Names { - mailboxes, - address_books, - calendars, + mailboxes: load(ObjectType::Email, "SELECT id, name FROM mailboxes")?, + address_books: load( + ObjectType::ContactCard, + "SELECT id, name FROM address_books", + )?, + calendars: load(ObjectType::CalendarEvent, "SELECT id, name FROM calendars")?, }) } - fn render(&self, kind: ObjectType, ids: &[i64]) -> String { + fn join(&self, out: &mut String, kind: ObjectType, ids: &[i64]) { let map = match kind { ObjectType::Mailbox => &self.mailboxes, ObjectType::AddressBook => &self.address_books, ObjectType::Calendar => &self.calendars, - _ => return format!("{ids:?}"), + _ => return, }; - let parts: Vec = ids - .iter() - .map(|id| match map.get(id) { - Some(name) => format!("{name:?}"), - None => format!("#{id} (missing)"), - }) - .collect(); - format!("[{}]", parts.join(", ")) - } -} - -fn load_name_map(conn: &Connection, sql: &str) -> Result, Error> { - let mut stmt = conn.prepare(sql)?; - let rows = stmt.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)))?; - let mut out = HashMap::new(); - for row in rows { - let (id, name) = row?; - out.insert(id, name); - } - Ok(out) -} - -fn limit_clause(limit: Option, offset: usize) -> String { - match limit { - Some(n) => format!(" LIMIT {n} OFFSET {offset}"), - None if offset > 0 => format!(" LIMIT -1 OFFSET {offset}"), - None => String::new(), - } -} - -fn parse_local_ids(text: &str) -> Vec { - serde_json::from_str::(text) - .ok() - .and_then(|v| { - v.as_array() - .map(|arr| arr.iter().filter_map(|x| x.as_i64()).collect::>()) - }) - .unwrap_or_default() -} - -fn json_pretty_or_raw(text: &str) -> String { - match serde_json::from_str::(text) { - Ok(v) => serde_json::to_string_pretty(&v).unwrap_or_else(|_| text.to_owned()), - Err(_) => text.to_owned(), - } -} - -fn keywords_summary(text: &str) -> String { - let ids = serde_json::from_str::>(text).unwrap_or_default(); - if ids.is_empty() { - "(none)".to_owned() - } else { - format!("[{}]", ids.join(", ")) - } -} - -fn opt_or_null(s: Option) -> String { - s.unwrap_or_else(|| "(null)".to_owned()) -} - -fn bool_str(b: bool) -> String { - (if b { "true" } else { "false" }).to_owned() -} - -fn write_record(out: &mut impl Write, title: &str, fields: &[(&str, String)]) -> Result<(), Error> { - writeln!(out, "{title}")?; - let max_label = fields.iter().map(|(l, _)| l.len()).max().unwrap_or(0); - let pad = " ".repeat(max_label + 4); - for (label, value) in fields { - for (i, line) in value.split('\n').enumerate() { - if i == 0 { - writeln!(out, " {label: 0 { + out.push_str(", "); } + let _ = match map.get(id) { + Some(name) => write!(out, "{name}"), + None => write!(out, "#{id} (missing)"), + }; } } - writeln!(out)?; - Ok(()) -} -fn write_identities( - conn: &Connection, - limit: Option, - offset: usize, - out: &mut impl Write, - written: &mut u64, -) -> Result<(), Error> { - let sql = format!( - "SELECT id, name, email, reply_to, bcc, text_signature, html_signature - FROM identities ORDER BY id{}", - limit_clause(limit, offset) - ); - let mut stmt = conn.prepare(&sql)?; - let mut rows = stmt.query([])?; - while let Some(row) = rows.next()? { - let id: i64 = row.get(0)?; - let fields = vec![ - ("name", row.get::<_, String>(1)?), - ("email", row.get::<_, String>(2)?), - ("reply_to", opt_or_null(row.get(3)?)), - ("bcc", opt_or_null(row.get(4)?)), - ("text_signature", row.get::<_, String>(5)?), - ("html_signature", row.get::<_, String>(6)?), - ]; - write_record(out, &format!("identity #{id}"), &fields)?; - *written += 1; + fn render(&self, kind: ObjectType, ids: &[i64]) -> String { + let mut out = String::new(); + self.join(&mut out, kind, ids); + out } - Ok(()) } -fn write_emails( - conn: &Connection, - names: &Names, - limit: Option, - offset: usize, - out: &mut impl Write, - written: &mut u64, -) -> Result<(), Error> { - let sql = format!( - "SELECT id, blob_id, received_at, mailbox_ids, keywords, message_match - FROM emails ORDER BY id{}", - limit_clause(limit, offset) - ); - let mut stmt = conn.prepare(&sql)?; - let mut rows = stmt.query([])?; - while let Some(row) = rows.next()? { - let id: i64 = row.get(0)?; - let blob_id: i64 = row.get(1)?; - let mailbox_ids: String = row.get(3)?; - let keywords: String = row.get(4)?; - let message_match: String = row.get(5)?; - let locals = parse_local_ids(&mailbox_ids); - let fields = vec![ - ("blob", blob_summary(conn, blob_id)?), - ("received_at", row.get::<_, String>(2)?), - ("mailboxes", names.render(ObjectType::Mailbox, &locals)), - ("keywords", keywords_summary(&keywords)), - ("message_match", json_pretty_or_raw(&message_match)), - ]; - write_record(out, &format!("email #{id}"), &fields)?; - *written += 1; +struct Listing<'v, 'o, W> { + view: View<'v>, + page: Page, + out: &'o mut W, + written: u64, +} + +impl View<'_> { + pub(super) fn write_list( + self, + ty: ObjectType, + page: Page, + out: &mut impl Write, + ) -> Result<(), Error> { + let total = self.count(ty)?; + let mut listing = Listing { + view: self, + page, + out, + written: 0, + }; + listing.header(ty, total)?; + if total == 0 { + writeln!( + listing.out, + "{}", + self.styled.paint( + Tone::Dim, + format_args!("No {} in the archive", Initial::Lower(ty.label())) + ) + )?; + return Ok(()); + } + let names = Names::load(self, ty)?; + listing.records(ty, &names)?; + if let Some(lim) = page.limit { + let next_offset = (page.offset as u64) + listing.written; + if next_offset < total { + writeln!( + listing.out, + "{}", + self.styled.paint( + Tone::Dim, + format_args!( + "{} more; continue with --offset {next_offset} --limit {lim}", + Grouped(total - next_offset) + ) + ) + )?; + } + } + Ok(()) } - Ok(()) } -fn write_sieves( - conn: &Connection, - limit: Option, - offset: usize, - out: &mut impl Write, - written: &mut u64, -) -> Result<(), Error> { - let sql = format!( - "SELECT id, name, is_active, blob_id FROM sieve_scripts ORDER BY id{}", - limit_clause(limit, offset) - ); - let mut stmt = conn.prepare(&sql)?; - let mut rows = stmt.query([])?; - while let Some(row) = rows.next()? { - let id: i64 = row.get(0)?; - let name: Option = row.get(1)?; - let active: i64 = row.get(2)?; - let blob_id: i64 = row.get(3)?; - let fields = vec![ - ("name", opt_or_null(name)), - ("is_active", bool_str(active != 0)), - ("blob", blob_summary(conn, blob_id)?), - ]; - write_record(out, &format!("sievescript #{id}"), &fields)?; - *written += 1; +impl Listing<'_, '_, W> { + fn header(&mut self, ty: ObjectType, total: u64) -> Result<(), Error> { + let styled = self.view.styled; + let name = styled.paint(Tone::Title, ty.label()); + let first = Grouped(self.page.offset as u64 + 1); + match self.page.limit { + None if self.page.offset == 0 => writeln!(self.out, "{name} ({})", Grouped(total))?, + None => writeln!(self.out, "{name} ({}, from {first})", Grouped(total))?, + Some(lim) => writeln!( + self.out, + "{name} ({}, showing {first} to {})", + Grouped(total), + Grouped(total.min(self.page.offset as u64 + lim as u64)) + )?, + } + writeln!(self.out)?; + Ok(()) } - Ok(()) -} -fn write_address_books( - conn: &Connection, - limit: Option, - offset: usize, - out: &mut impl Write, - written: &mut u64, -) -> Result<(), Error> { - let sql = format!( - "SELECT id, name, description, sort_order, is_default, is_subscribed - FROM address_books ORDER BY id{}", - limit_clause(limit, offset) - ); - let mut stmt = conn.prepare(&sql)?; - let mut rows = stmt.query([])?; - while let Some(row) = rows.next()? { - let id: i64 = row.get(0)?; - let fields = vec![ - ("name", row.get::<_, String>(1)?), - ("description", opt_or_null(row.get(2)?)), - ("sort_order", row.get::<_, i64>(3)?.to_string()), - ("is_default", bool_str(row.get::<_, i64>(4)? != 0)), - ("is_subscribed", bool_str(row.get::<_, i64>(5)? != 0)), - ]; - write_record(out, &format!("addressbook #{id}"), &fields)?; - *written += 1; - } - Ok(()) -} - -fn write_contact_cards( - conn: &Connection, - names: &Names, - limit: Option, - offset: usize, - out: &mut impl Write, - written: &mut u64, -) -> Result<(), Error> { - let sql = format!( - "SELECT id, uid, address_book_ids, data FROM contact_cards ORDER BY id{}", - limit_clause(limit, offset) - ); - let mut stmt = conn.prepare(&sql)?; - let mut rows = stmt.query([])?; - while let Some(row) = rows.next()? { - let id: i64 = row.get(0)?; - let uid: String = row.get(1)?; - let ab_ids: String = row.get(2)?; - let data: String = row.get(3)?; - let locals = parse_local_ids(&ab_ids); - let fields = vec![ - ("uid", uid), - ( - "address_books", - names.render(ObjectType::AddressBook, &locals), + fn records(&mut self, ty: ObjectType, names: &Names) -> Result<(), Error> { + match ty { + ObjectType::Identity => self.each( + ObjectType::Identity, + "SELECT id, name, email, reply_to, bcc, text_signature, html_signature + FROM identities ORDER BY id", + |_, row| { + Ok(vec![ + ("Name", row.get(1)?), + ("Email", row.get(2)?), + ("Reply to", Column::or_null(row.get(3)?)), + ("Bcc", Column::or_null(row.get(4)?)), + ("Text signature", row.get(5)?), + ("HTML signature", row.get(6)?), + ]) + }, ), - ("data", redact_blobs_in_json(&data)), - ]; - write_record(out, &format!("contactcard #{id}"), &fields)?; - *written += 1; + ObjectType::Email => self.emails(names), + ObjectType::SieveScript => self.each( + ObjectType::SieveScript, + "SELECT id, name, is_active, blob_id FROM sieve_scripts ORDER BY id", + |view, row| { + Ok(vec![ + ("Name", Column::or_null(row.get(1)?)), + ("Active", Column::flag(row.get(2)?)), + ("Content", view.blob_summary(row.get::<_, BlobId>(3)?)?), + ]) + }, + ), + ObjectType::AddressBook => self.each( + ObjectType::AddressBook, + "SELECT id, name, description, sort_order, is_default, is_subscribed + FROM address_books ORDER BY id", + |_, row| { + Ok(vec![ + ("Name", row.get(1)?), + ("Description", Column::or_null(row.get(2)?)), + ("Sort order", row.get::<_, i64>(3)?.to_string()), + ("Default", Column::flag(row.get(4)?)), + ("Subscribed", Column::flag(row.get(5)?)), + ]) + }, + ), + ObjectType::ContactCard => self.each( + ObjectType::ContactCard, + "SELECT id, uid, address_book_ids, data FROM contact_cards ORDER BY id", + |_, row| { + Ok(vec![ + ("UID", row.get(1)?), + ( + "Address books", + names.render( + ObjectType::AddressBook, + &Column(&row.get::<_, String>(2)?).local_ids(), + ), + ), + ("Data", Column(&row.get::<_, String>(3)?).redacted_json()), + ]) + }, + ), + ObjectType::Calendar => self.each( + ObjectType::Calendar, + "SELECT id, name, description, color, sort_order, is_subscribed, is_visible, is_default, + include_in_availability, default_alerts_with_time, default_alerts_without_time, + time_zone + FROM calendars ORDER BY id", + |_, row| { + Ok(vec![ + ("Name", row.get(1)?), + ("Description", Column::or_null(row.get(2)?)), + ("Colour", Column::or_null(row.get(3)?)), + ("Sort order", row.get::<_, i64>(4)?.to_string()), + ("Subscribed", Column::flag(row.get(5)?)), + ("Visible", Column::flag(row.get(6)?)), + ("Default", Column::flag(row.get(7)?)), + ("Availability", row.get(8)?), + ("Default timed alerts", Column::or_null(row.get(9)?)), + ( + "Default all-day alerts", + Column::or_null(row.get(10)?), + ), + ("Time zone", Column::or_null(row.get(11)?)), + ]) + }, + ), + ObjectType::CalendarEvent => self.each( + ObjectType::CalendarEvent, + "SELECT id, calendar_ids, is_draft, use_default_alerts, data + FROM calendar_events ORDER BY id", + |_, row| { + Ok(vec![ + ( + "Calendars", + names.render( + ObjectType::Calendar, + &Column(&row.get::<_, String>(1)?).local_ids(), + ), + ), + ("Draft", Column::flag(row.get(2)?)), + ("Default alerts", Column::flag(row.get(3)?)), + ("Data", Column(&row.get::<_, String>(4)?).redacted_json()), + ]) + }, + ), + ObjectType::ParticipantIdentity => self.each( + ObjectType::ParticipantIdentity, + "SELECT id, name, calendar_address, is_default + FROM participant_identities ORDER BY id", + |_, row| { + Ok(vec![ + ("Name", row.get(1)?), + ("Calendar address", row.get(2)?), + ("Default", Column::flag(row.get(3)?)), + ]) + }, + ), + ObjectType::Mailbox | ObjectType::FileNode => Ok(()), + } } - Ok(()) -} -fn write_calendars( - conn: &Connection, - limit: Option, - offset: usize, - out: &mut impl Write, - written: &mut u64, -) -> Result<(), Error> { - let sql = format!( - "SELECT id, name, description, color, sort_order, is_subscribed, is_visible, is_default, - include_in_availability, default_alerts_with_time, default_alerts_without_time, - time_zone - FROM calendars ORDER BY id{}", - limit_clause(limit, offset) - ); - let mut stmt = conn.prepare(&sql)?; - let mut rows = stmt.query([])?; - while let Some(row) = rows.next()? { - let id: i64 = row.get(0)?; - let fields = vec![ - ("name", row.get::<_, String>(1)?), - ("description", opt_or_null(row.get(2)?)), - ("color", opt_or_null(row.get(3)?)), - ("sort_order", row.get::<_, i64>(4)?.to_string()), - ("is_subscribed", bool_str(row.get::<_, i64>(5)? != 0)), - ("is_visible", bool_str(row.get::<_, i64>(6)? != 0)), - ("is_default", bool_str(row.get::<_, i64>(7)? != 0)), - ("include_in_availability", row.get::<_, String>(8)?), - ("default_alerts_with_time", opt_or_null(row.get(9)?)), - ("default_alerts_without_time", opt_or_null(row.get(10)?)), - ("time_zone", opt_or_null(row.get(11)?)), - ]; - write_record(out, &format!("calendar #{id}"), &fields)?; - *written += 1; + fn each( + &mut self, + ty: ObjectType, + select: &str, + fields: impl Fn(View<'_>, &Row<'_>) -> Result, Error>, + ) -> Result<(), Error> { + let sql = format!("{select}{}", LimitClause(self.page)); + let mut stmt = self.view.conn.prepare(&sql)?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let id: i64 = row.get(0)?; + let fields = fields(self.view, row)?; + self.record(ty, id, &fields)?; + self.written += 1; + } + Ok(()) } - Ok(()) -} -fn write_calendar_events( - conn: &Connection, - names: &Names, - limit: Option, - offset: usize, - out: &mut impl Write, - written: &mut u64, -) -> Result<(), Error> { - let sql = format!( - "SELECT id, calendar_ids, is_draft, use_default_alerts, data - FROM calendar_events ORDER BY id{}", - limit_clause(limit, offset) - ); - let mut stmt = conn.prepare(&sql)?; - let mut rows = stmt.query([])?; - while let Some(row) = rows.next()? { - let id: i64 = row.get(0)?; - let cal_ids: String = row.get(1)?; - let data: String = row.get(4)?; - let locals = parse_local_ids(&cal_ids); - let fields = vec![ - ("calendars", names.render(ObjectType::Calendar, &locals)), - ("is_draft", bool_str(row.get::<_, i64>(2)? != 0)), - ("use_default_alerts", bool_str(row.get::<_, i64>(3)? != 0)), - ("data", redact_blobs_in_json(&data)), - ]; - write_record(out, &format!("calendarevent #{id}"), &fields)?; - *written += 1; - } - Ok(()) -} - -fn write_participant_identities( - conn: &Connection, - limit: Option, - offset: usize, - out: &mut impl Write, - written: &mut u64, -) -> Result<(), Error> { - let sql = format!( - "SELECT id, name, calendar_address, is_default - FROM participant_identities ORDER BY id{}", - limit_clause(limit, offset) - ); - let mut stmt = conn.prepare(&sql)?; - let mut rows = stmt.query([])?; - while let Some(row) = rows.next()? { - let id: i64 = row.get(0)?; - let fields = vec![ - ("name", row.get::<_, String>(1)?), - ("calendar_address", row.get::<_, String>(2)?), - ("is_default", bool_str(row.get::<_, i64>(3)? != 0)), - ]; - write_record(out, &format!("participantidentity #{id}"), &fields)?; - *written += 1; - } - Ok(()) -} - -fn redact_blobs_in_json(text: &str) -> String { - let mut value = match serde_json::from_str::(text) { - Ok(v) => v, - Err(_) => return text.to_owned(), - }; - redact_blobs(&mut value); - serde_json::to_string_pretty(&value).unwrap_or_else(|_| text.to_owned()) -} - -fn redact_blobs(value: &mut Value) { - match value { - Value::Object(map) => { - if let Some(bid) = map.get("@blob").and_then(|v| v.as_i64()) { - map.clear(); - map.insert( - "@blob".to_owned(), - Value::String(format!("local #{bid} (omitted)")), - ); - return; + fn record(&mut self, ty: ObjectType, id: i64, fields: &[(&str, String)]) -> Result<(), Error> { + let styled = self.view.styled; + writeln!( + self.out, + "{}", + styled.paint( + Tone::Heading, + format_args!("{} {id}", Initial::Upper(ty.noun())) + ) + )?; + let width = fields.iter().map(|(l, _)| l.len()).max().unwrap_or(0); + let pad = width + 4; + for (label, value) in fields { + let mut lines = value.split('\n'); + if let Some(first) = lines.next() { + writeln!( + self.out, + " {} {first}", + styled.paint(Tone::Dim, format_args!("{label: { - for v in arr.iter_mut() { - redact_blobs(v); + writeln!(self.out)?; + Ok(()) + } + + fn emails(&mut self, names: &Names) -> Result<(), Error> { + let sql = format!( + "SELECT e.id, length(b.data), e.received_at, e.mailbox_ids, e.keywords, e.message_match + FROM emails e LEFT JOIN blobs b ON b.id = e.blob_id ORDER BY e.id{}", + LimitClause(self.page) + ); + let mut stmt = self.view.conn.prepare(&sql)?; + let mut rows = stmt.query([])?; + let mut table = Table::new(&EMAIL_COLUMNS); + let mut text = String::new(); + while let Some(row) = rows.next()? { + table.cell(row.get::<_, i64>(0)?, Some(Tone::Dim)); + match row.get::<_, Option>(1)? { + Some(len) => table.cell(ByteSize(len.max(0) as u64), None), + None => table.cell(MISSING, Some(Tone::Bad)), + }; + table.cell(row.get::<_, String>(2)?, None); + text.clear(); + names.join( + &mut text, + ObjectType::Mailbox, + &Column(&row.get::<_, String>(3)?).local_ids(), + ); + table.cell(&text, None); + text.clear(); + Column(&row.get::<_, String>(4)?).keyword_list(&mut text); + Self::text_or_dash(&mut table, &text); + text.clear(); + Column(&row.get::<_, String>(5)?).first_message_id(&mut text); + Self::text_or_dash(&mut table, &text); + self.written += 1; + if table.rows() == EMAIL_BATCH_ROWS { + self.flush_table(&mut table)?; } } - _ => {} + if table.rows() > 0 { + self.flush_table(&mut table)?; + } + Ok(()) + } + + fn flush_table(&mut self, table: &mut Table<'_>) -> Result<(), Error> { + table.write(self.view.styled, self.out)?; + writeln!(self.out)?; + table.clear(); + Ok(()) + } + + fn text_or_dash(table: &mut Table<'_>, text: &str) { + if text.is_empty() { + table.cell(DASH, Some(Tone::Dim)); + } else { + table.cell(text, None); + } } } #[cfg(test)] mod tests { use super::*; - use crate::db::{blobs, init}; + use crate::archive::Archive; + use crate::console::Styled; use rusqlite::params; - fn mem() -> Connection { - let c = Connection::open_in_memory().unwrap(); - init::apply_schema(&c).unwrap(); - c + fn list(a: &Archive, ty: ObjectType, limit: Option) -> String { + let mut buf = Vec::new(); + View::new(a.connection(), Styled::Plain) + .write_list(ty, Page { limit, offset: 0 }, &mut buf) + .unwrap(); + String::from_utf8(buf).unwrap() } - fn add_mailbox(c: &Connection, name: &str) -> i64 { - c.execute("INSERT INTO mailboxes (name) VALUES (?1)", params![name]) + fn add_mailbox(a: &Archive, name: &str) -> i64 { + a.connection() + .execute("INSERT INTO mailboxes (name) VALUES (?1)", params![name]) .unwrap(); - c.last_insert_rowid() + a.connection().last_insert_rowid() } #[test] fn empty_table_prints_none_message() { - let c = mem(); - let mut buf = Vec::new(); - write_list(&c, ObjectType::Identity, None, 0, &mut buf).unwrap(); - let s = String::from_utf8(buf).unwrap(); - assert!(s.contains("Identity (0)")); - assert!(s.contains("(no identity in archive)")); + let a = Archive::open_in_memory().unwrap(); + let s = list(&a, ObjectType::Identity, None); + assert!(s.contains("Identities (0)")); + assert!(s.contains("No identities in the archive")); } #[test] fn identity_record_uses_aligned_fields_and_redacts_nothing() { - let c = mem(); - c.execute( - "INSERT INTO identities (name, email, text_signature) VALUES ('Alice', 'a@x.test', 'best,A')", - [], - ) - .unwrap(); - let mut buf = Vec::new(); - write_list(&c, ObjectType::Identity, None, 0, &mut buf).unwrap(); - let s = String::from_utf8(buf).unwrap(); - assert!(s.contains("identity #1")); - assert!(s.contains(" name Alice"), "got:\n{s}"); - assert!(s.contains(" email a@x.test")); - assert!(s.contains(" bcc (null)")); + let a = Archive::open_in_memory().unwrap(); + a.connection() + .execute( + "INSERT INTO identities (name, email, text_signature) VALUES ('Alice', 'a@x.test', 'best,A')", + [], + ) + .unwrap(); + let s = list(&a, ObjectType::Identity, None); + assert!(s.contains("Identity 1\n")); + assert!(s.contains(" Name Alice"), "got:\n{s}"); + assert!(s.contains(" Email a@x.test")); + assert!(s.contains(" Bcc -")); } #[test] fn email_resolves_mailbox_names_and_summarises_blob() { - let c = mem(); - let inbox = add_mailbox(&c, "Inbox"); - let other = add_mailbox(&c, "Personal"); - let blob = blobs::intern_blob(&c, b"From: a@x.test\r\nSubject: hi\r\n\r\nhello").unwrap(); - c.execute( - "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords) - VALUES (?1, '2024-01-01T00:00:00Z', ?2, ?3)", - params![ - blob, - format!("[{inbox},{other}]"), - "[\"$seen\",\"Important\"]" - ], - ) - .unwrap(); - let mut buf = Vec::new(); - write_list(&c, ObjectType::Email, None, 0, &mut buf).unwrap(); - let s = String::from_utf8(buf).unwrap(); - assert!(s.contains("email #1")); - assert!(s.contains("blob ")); - assert!(s.contains("blake3=")); - assert!(s.contains("mailboxes ")); - assert!(s.contains("\"Inbox\""), "got:\n{s}"); - assert!(s.contains("\"Personal\"")); - assert!(s.contains("keywords ")); - assert!(s.contains("[$seen, Important]")); + let a = Archive::open_in_memory().unwrap(); + let inbox = add_mailbox(&a, "Inbox"); + let other = add_mailbox(&a, "Personal"); + let blob = a + .store() + .blobs() + .intern(b"From: a@x.test\r\nSubject: hi\r\n\r\nhello") + .unwrap(); + a.connection() + .execute( + "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords) + VALUES (?1, '2024-01-01T00:00:00Z', ?2, ?3)", + params![ + blob, + format!("[{inbox},{other}]"), + "[\"$seen\",\"Important\"]" + ], + ) + .unwrap(); + let s = list(&a, ObjectType::Email, None); + assert!(s.contains(" ID Size Received"), "got:\n{s}"); + assert!( + s.contains(" 1 36 B 2024-01-01T00:00:00Z Inbox, Personal $seen, Important -"), + "got:\n{s}" + ); + } + + // The email table is written in batches, so a long listing streams with bounded memory. + #[test] + fn email_table_is_written_in_batches() { + let a = Archive::open_in_memory().unwrap(); + let blob = a.store().blobs().intern(b"Subject: x\r\n\r\nx").unwrap(); + let c = a.connection(); + for _ in 0..=EMAIL_BATCH_ROWS { + c.execute( + "INSERT INTO emails (blob_id, received_at, mailbox_ids) + VALUES (?1, '2024-01-01T00:00:00Z', '[]')", + params![blob], + ) + .unwrap(); + } + let s = list(&a, ObjectType::Email, None); + assert_eq!(s.matches(" ID Size").count(), 2, "got:\n{s}"); + let last = EMAIL_BATCH_ROWS + 1; + assert!(s.contains(&format!("\n {last} ")), "got:\n{s}"); } #[test] fn contact_card_redacts_blob_sentinel_in_data() { - let c = mem(); + let a = Archive::open_in_memory().unwrap(); + let c = a.connection(); c.execute("INSERT INTO address_books (name) VALUES ('Personal')", []) .unwrap(); c.execute( "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)", params![ "urn:uuid:1", - "[1]", + "[1,9]", r#"{"name":{"full":"Alice"},"photos":{"p":{"@blob":17}}}"# ], ) .unwrap(); - let mut buf = Vec::new(); - write_list(&c, ObjectType::ContactCard, None, 0, &mut buf).unwrap(); - let s = String::from_utf8(buf).unwrap(); - assert!(s.contains("contactcard #1")); - assert!(s.contains("\"Personal\"")); + let s = list(&a, ObjectType::ContactCard, None); + assert!(s.contains("Contact card 1\n")); + assert!(s.contains("Personal, #9 (missing)"), "got:\n{s}"); assert!(s.contains("local #17 (omitted)")); assert!(!s.contains("\"@blob\": 17")); + assert!( + s.contains(" Data {\n \"name\": {"), + "got:\n{s}" + ); } #[test] fn pagination_writes_continuation_hint_when_more_remain() { - let c = mem(); + let a = Archive::open_in_memory().unwrap(); for i in 0..5 { - c.execute( - "INSERT INTO identities (name, email) VALUES (?1, ?2)", - params![format!("u{i}"), format!("u{i}@x.test")], - ) - .unwrap(); + a.connection() + .execute( + "INSERT INTO identities (name, email) VALUES (?1, ?2)", + params![format!("u{i}"), format!("u{i}@x.test")], + ) + .unwrap(); } - let mut buf = Vec::new(); - write_list(&c, ObjectType::Identity, Some(2), 0, &mut buf).unwrap(); - let s = String::from_utf8(buf).unwrap(); - assert!(s.contains("Identity (5 total, showing up to 2 from offset 0)")); - assert!(s.contains("identity #1")); - assert!(s.contains("identity #2")); - assert!(!s.contains("identity #3")); - assert!(s.contains("... (3 more; --offset 2 --limit 2 to continue)")); + let s = list(&a, ObjectType::Identity, Some(2)); + assert!(s.contains("Identities (5, showing 1 to 2)")); + assert!(s.contains("Identity 1\n")); + assert!(s.contains("Identity 2\n")); + assert!(!s.contains("Identity 3\n")); + assert!(s.contains("3 more; continue with --offset 2 --limit 2")); } #[test] fn pagination_at_end_does_not_print_continuation() { - let c = mem(); + let a = Archive::open_in_memory().unwrap(); for i in 0..3 { - c.execute( - "INSERT INTO identities (name, email) VALUES (?1, ?2)", - params![format!("u{i}"), format!("u{i}@x.test")], - ) - .unwrap(); + a.connection() + .execute( + "INSERT INTO identities (name, email) VALUES (?1, ?2)", + params![format!("u{i}"), format!("u{i}@x.test")], + ) + .unwrap(); } - let mut buf = Vec::new(); - write_list(&c, ObjectType::Identity, Some(5), 0, &mut buf).unwrap(); - let s = String::from_utf8(buf).unwrap(); + let s = list(&a, ObjectType::Identity, Some(5)); + assert!(s.contains("Identities (3, showing 1 to 3)"), "got:\n{s}"); assert!(!s.contains("more;"), "got:\n{s}"); } + + #[test] + fn column_helpers_keep_the_old_texts() { + assert_eq!(Column("not json").redacted_json(), "not json"); + assert_eq!(Column("[1,\"x\",3]").local_ids(), vec![1, 3]); + assert_eq!(Column("{}").local_ids(), Vec::::new()); + assert_eq!( + LimitClause(Page { + limit: None, + offset: 3 + }) + .to_string(), + " LIMIT -1 OFFSET 3" + ); + assert_eq!(LimitClause(Page::default()).to_string(), ""); + let mut text = String::new(); + Column(r#"{"m":["",""]}"#).first_message_id(&mut text); + Column("[\"$seen\",\"$flagged\"]").keyword_list(&mut text); + assert_eq!(text, "$seen, $flagged"); + assert_eq!(Column::flag(0), "no"); + assert_eq!(Column::or_null(None), "-"); + } } diff --git a/src/inspect/mod.rs b/src/inspect/mod.rs index f09e273..70f32bf 100644 --- a/src/inspect/mod.rs +++ b/src/inspect/mod.rs @@ -4,108 +4,71 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use std::collections::HashMap; +mod list; +mod tree; + +use std::fmt::Display; use std::io::{self, Write}; use std::path::{Path, PathBuf}; use rusqlite::{Connection, OptionalExtension, params}; -use crate::db::init; +use crate::archive::sources::SourceKind; +use crate::archive::{Archive, BlobId}; +use crate::console::{ByteSize, Column, Console, Grouped, Initial, Severity, Styled, Table, Tone}; use crate::error::Error; use crate::types::ObjectType; -pub mod list; -pub mod tree; +const KEY_WIDTH: usize = 7; +const SUMMARY_COLUMNS: [Column; 2] = [Column::left("Type"), Column::right("Objects")]; -pub struct InspectConfig { +pub struct Inspect { pub archive: PathBuf, pub target: Option, + pub page: Page, + pub console: Console, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Page { pub limit: Option, pub offset: usize, } -pub fn run(config: InspectConfig) -> Result<(), Error> { - let conn = init::open(&config.archive)?; - let stdout = io::stdout(); - let mut out = stdout.lock(); - dispatch(&conn, &config, &mut out)?; - Ok(()) -} +impl Inspect { + pub fn run(self) -> Result<(), Error> { + let archive = Archive::open(&self.archive)?; + let stdout = io::stdout(); + let mut out = stdout.lock(); + self.write(View::new(archive.connection(), self.console.out), &mut out) + } -fn dispatch(conn: &Connection, cfg: &InspectConfig, out: &mut impl Write) -> Result<(), Error> { - match cfg.target { - None => write_summary(conn, &cfg.archive, out), - Some(ObjectType::Mailbox) => { - warn_if_paged(out, cfg, "mailbox")?; - tree::write_mailboxes(conn, out) + fn write(&self, view: View<'_>, out: &mut impl Write) -> Result<(), Error> { + match self.target { + None => view.write_summary(&self.archive, out), + Some(ObjectType::Mailbox) => { + self.page.note_ignored(self.console, ObjectType::Mailbox); + view.write_mailboxes(out) + } + Some(ObjectType::FileNode) => { + self.page.note_ignored(self.console, ObjectType::FileNode); + view.write_file_nodes(out) + } + Some(ty) => view.write_list(ty, self.page, out), } - Some(ObjectType::FileNode) => { - warn_if_paged(out, cfg, "filenode")?; - tree::write_file_nodes(conn, out) - } - Some(ty) => list::write_list(conn, ty, cfg.limit, cfg.offset, out), } } -fn warn_if_paged(out: &mut impl Write, cfg: &InspectConfig, token: &str) -> Result<(), Error> { - if cfg.limit.is_some() || cfg.offset > 0 { - writeln!( - out, - "note: --limit/--offset are ignored for {token} (tree view always shows the full hierarchy)" - )?; - } - Ok(()) -} - -fn write_summary(conn: &Connection, archive: &Path, out: &mut impl Write) -> Result<(), Error> { - writeln!(out, "Archive: {}", archive.display())?; - match read_source(conn)? { - Some(src) => { - let name = src.account_name.unwrap_or_else(|| "(unnamed)".to_owned()); - writeln!( - out, - "Source: {} {} (account {} / {})", - src.kind, src.session_url, src.account_id, name - )?; +impl Page { + fn note_ignored(self, console: Console, ty: ObjectType) { + if self.limit.is_some() || self.offset > 0 { + eprintln!( + "{} --limit and --offset are ignored for {}; the tree always shows the full hierarchy", + console.err.severity(Severity::Note), + Initial::Lower(ty.label()) + ); } - None => writeln!(out, "Source: (no source recorded)")?, } - writeln!(out)?; - let counts = read_type_counts(conn)?; - let label_width = ObjectType::ALL - .iter() - .map(|t| t.token().len()) - .max() - .unwrap_or(0); - let count_width = counts - .values() - .map(|n| format_count(*n).len()) - .max() - .unwrap_or(1) - .max("blobs".len()); - for ty in ObjectType::ALL { - let n = counts.get(&ty).copied().unwrap_or(0); - writeln!( - out, - " {:cw$}", - ty.token(), - format_count(n), - lw = label_width, - cw = count_width - )?; - } - let (blob_count, blob_bytes) = read_blob_totals(conn)?; - writeln!(out)?; - writeln!( - out, - " {:cw$} ({})", - "blobs", - format_count(blob_count), - format_bytes(blob_bytes), - lw = label_width, - cw = count_width - )?; - Ok(()) } struct SourceRow { @@ -115,171 +78,204 @@ struct SourceRow { account_name: Option, } -fn read_source(conn: &Connection) -> Result, Error> { - let row = conn - .query_row( - "SELECT kind, session_url, account_id, account_name FROM sources LIMIT 1", - [], - |r| { - Ok(SourceRow { - kind: r.get(0)?, - session_url: r.get(1)?, - account_id: r.get(2)?, - account_name: r.get(3)?, - }) - }, - ) - .optional()?; - Ok(row) +#[derive(Clone, Copy)] +struct View<'c> { + conn: &'c Connection, + styled: Styled, } -fn read_type_counts(conn: &Connection) -> Result, Error> { - let mut out = HashMap::new(); - for ty in ObjectType::ALL { - let sql = format!("SELECT COUNT(*) FROM {}", crate::sync::table_name(ty)); - let n: i64 = conn.query_row(&sql, [], |r| r.get(0))?; - out.insert(ty, n as u64); +impl<'c> View<'c> { + fn new(conn: &'c Connection, styled: Styled) -> View<'c> { + View { conn, styled } } - Ok(out) -} -fn read_blob_totals(conn: &Connection) -> Result<(u64, u64), Error> { - let (count, bytes): (i64, i64) = conn.query_row( - "SELECT COUNT(*), COALESCE(SUM(length(data)), 0) FROM blobs", - [], - |r| Ok((r.get(0)?, r.get(1)?)), - )?; - Ok((count as u64, bytes.max(0) as u64)) -} + fn key(self, out: &mut impl Write, key: &str, value: impl Display) -> io::Result<()> { + writeln!( + out, + "{} {value}", + self.styled + .paint(Tone::Heading, format_args!("{key: String { - let s = n.to_string(); - let bytes = s.as_bytes(); - let len = bytes.len(); - let mut out = String::with_capacity(len + len / 3); - for (i, ch) in bytes.iter().enumerate() { - if i > 0 && (len - i).is_multiple_of(3) { - out.push(','); + fn write_summary(self, archive: &Path, out: &mut impl Write) -> Result<(), Error> { + let styled = self.styled; + self.key(out, "Archive", archive.display())?; + match self.source()? { + Some(src) => { + let kind = match SourceKind::parse(&src.kind) { + Some(kind) => kind.label(), + None => src.kind.as_str(), + }; + self.key(out, "Source", format_args!("{kind}, {}", src.session_url))?; + match &src.account_name { + Some(name) => { + self.key(out, "Account", format_args!("{name} ({})", src.account_id))? + } + None => self.key(out, "Account", &src.account_id)?, + } + } + None => self.key(out, "Source", styled.paint(Tone::Dim, "none recorded"))?, } - out.push(*ch as char); + let (blob_count, blob_bytes) = self.blob_totals()?; + self.key( + out, + "Blobs", + format_args!("{}, {}", Grouped(blob_count), ByteSize(blob_bytes)), + )?; + writeln!(out)?; + let mut table = Table::new(&SUMMARY_COLUMNS); + for ty in ObjectType::ALL { + let n = self.count(ty)?; + table + .cell(ty.label(), (n == 0).then_some(Tone::Dim)) + .count(n, None); + } + table.write(styled, out)?; + Ok(()) } - out -} -pub(crate) fn format_bytes(n: u64) -> String { - const UNITS: &[&str] = &["B", "KB", "MB", "GB", "TB"]; - if n < 1024 { - return format!("{n} B"); + fn source(self) -> Result, Error> { + Ok(self + .conn + .query_row( + "SELECT kind, session_url, account_id, account_name FROM sources LIMIT 1", + [], + |r| { + Ok(SourceRow { + kind: r.get(0)?, + session_url: r.get(1)?, + account_id: r.get(2)?, + account_name: r.get(3)?, + }) + }, + ) + .optional()?) } - let mut value = n as f64; - let mut i = 0; - while value >= 1024.0 && i + 1 < UNITS.len() { - value /= 1024.0; - i += 1; - } - format!("{value:.1} {}", UNITS[i]) -} -pub(crate) fn blob_summary(conn: &Connection, blob_id: i64) -> Result { - let row = conn - .query_row( - "SELECT length(data), substr(lower(hex(hash)), 1, 12) FROM blobs WHERE id = ?1", - params![blob_id], - |r| { - let len: i64 = r.get(0)?; - let hex: String = r.get(1)?; - Ok((len, hex)) - }, - ) - .optional()?; - match row { - Some((len, hex)) => Ok(format!("{} blake3={hex}", format_bytes(len.max(0) as u64))), - None => Ok(format!("(missing blob #{blob_id})")), + fn count(self, ty: ObjectType) -> Result { + let n: i64 = + self.conn + .query_row(&format!("SELECT COUNT(*) FROM {}", ty.table()), [], |r| { + r.get(0) + })?; + Ok(n as u64) + } + + fn blob_totals(self) -> Result<(u64, u64), Error> { + let (count, bytes): (i64, i64) = self.conn.query_row( + "SELECT COUNT(*), COALESCE(SUM(length(data)), 0) FROM blobs", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + )?; + Ok((count as u64, bytes.max(0) as u64)) + } + + fn blob_summary(self, blob: BlobId) -> Result { + let row = self + .conn + .query_row( + "SELECT length(data), substr(lower(hex(hash)), 1, 12) FROM blobs WHERE id = ?1", + params![blob], + |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)), + ) + .optional()?; + Ok(match row { + Some((len, hex)) => format!("{} blake3={hex}", ByteSize(len.max(0) as u64)), + None => format!("(missing blob #{blob})"), + }) } } #[cfg(test)] mod tests { use super::*; - use crate::db::init; - - fn mem() -> Connection { - let c = Connection::open_in_memory().unwrap(); - init::apply_schema(&c).unwrap(); - c - } - - #[test] - fn format_count_inserts_grouping_commas() { - assert_eq!(format_count(0), "0"); - assert_eq!(format_count(42), "42"); - assert_eq!(format_count(999), "999"); - assert_eq!(format_count(1_000), "1,000"); - assert_eq!(format_count(12_345), "12,345"); - assert_eq!(format_count(1_234_567_890), "1,234,567,890"); - } - - #[test] - fn format_bytes_picks_appropriate_unit() { - assert_eq!(format_bytes(0), "0 B"); - assert_eq!(format_bytes(512), "512 B"); - assert_eq!(format_bytes(1024), "1.0 KB"); - assert_eq!(format_bytes(1_536), "1.5 KB"); - assert_eq!(format_bytes(10 * 1024 * 1024), "10.0 MB"); - } + use crate::archive::sources::{SourceKey, SourceKind}; #[test] fn blob_summary_reports_size_and_short_hash() { - let c = mem(); - let id = crate::db::blobs::intern_blob(&c, b"hello world").unwrap(); - let s = blob_summary(&c, id).unwrap(); + let a = Archive::open_in_memory().unwrap(); + let id = a.store().blobs().intern(b"hello world").unwrap(); + let s = View::new(a.connection(), Styled::Plain) + .blob_summary(id) + .unwrap(); assert!(s.starts_with("11 B blake3="), "got {s}"); assert!(s.len() > "11 B blake3=".len()); } #[test] fn blob_summary_handles_missing_id() { - let c = mem(); - let s = blob_summary(&c, 9_999).unwrap(); + let a = Archive::open_in_memory().unwrap(); + let s = View::new(a.connection(), Styled::Plain) + .blob_summary(BlobId::from_raw(9_999)) + .unwrap(); assert_eq!(s, "(missing blob #9999)"); } #[test] fn summary_lists_every_type_and_blob_totals() { - let c = mem(); - crate::db::sources::upsert_source( - &c, - &crate::db::sources::SourceKey { - kind: "jmap".to_owned(), - session_url: "https://example.test/jmap".to_owned(), - account_id: "u-1".to_owned(), - }, - Some("alice@example.test"), - "alice@example.test", - ) - .unwrap(); - c.execute("INSERT INTO mailboxes (name) VALUES (?1)", params!["Inbox"]) + let a = Archive::open_in_memory().unwrap(); + a.store() + .sources() + .upsert( + &SourceKey { + kind: SourceKind::Jmap, + url: "https://example.test/jmap", + account_id: "u-1", + }, + Some("alice@example.test"), + "alice@example.test", + ) .unwrap(); - crate::db::blobs::intern_blob(&c, b"some bytes").unwrap(); + a.connection() + .execute("INSERT INTO mailboxes (name) VALUES (?1)", params!["Inbox"]) + .unwrap(); + a.store().blobs().intern(b"some bytes").unwrap(); let mut buf = Vec::new(); - write_summary(&c, Path::new("test.sqlite"), &mut buf).unwrap(); + View::new(a.connection(), Styled::Plain) + .write_summary(Path::new("test.sqlite"), &mut buf) + .unwrap(); let s = String::from_utf8(buf).unwrap(); - assert!(s.contains("Archive: test.sqlite")); - assert!(s.contains("Source: jmap https://example.test/jmap")); - assert!(s.contains("alice@example.test")); + assert!(s.contains("Archive test.sqlite\n"), "got:\n{s}"); + assert!(s.contains("Source JMAP, https://example.test/jmap\n")); + assert!(s.contains("Account alice@example.test (u-1)\n")); + assert!(s.contains("Blobs 1, 10 B\n")); for ty in ObjectType::ALL { - assert!(s.contains(ty.token()), "missing {} in:\n{s}", ty.token()); + assert!(s.contains(ty.label()), "missing {} in:\n{s}", ty.label()); } - assert!(s.contains("blobs")); - assert!(s.contains("10 B")); } #[test] fn summary_without_source_reports_none() { - let c = mem(); + let a = Archive::open_in_memory().unwrap(); let mut buf = Vec::new(); - write_summary(&c, Path::new("x.sqlite"), &mut buf).unwrap(); + View::new(a.connection(), Styled::Plain) + .write_summary(Path::new("x.sqlite"), &mut buf) + .unwrap(); let s = String::from_utf8(buf).unwrap(); - assert!(s.contains("(no source recorded)")); + assert!(s.contains("Source none recorded")); + } + + #[test] + fn summary_table_aligns_labels_and_grouped_counts() { + // Pins the summary table: sentence-case labels, right-aligned grouped counts. + let a = Archive::open_in_memory().unwrap(); + for n in 0..1_234 { + a.connection() + .execute( + "INSERT INTO identities (name, email) VALUES (?1, ?1)", + params![n.to_string()], + ) + .unwrap(); + } + let mut buf = Vec::new(); + View::new(a.connection(), Styled::Plain) + .write_summary(Path::new("x.sqlite"), &mut buf) + .unwrap(); + let s = String::from_utf8(buf).unwrap(); + assert!( + s.contains(" Type Objects\n Mailboxes 0\n Emails 0\n Identities 1,234\n"), + "got:\n{s}" + ); } } diff --git a/src/inspect/tree.rs b/src/inspect/tree.rs index 5eed42f..8f58cae 100644 --- a/src/inspect/tree.rs +++ b/src/inspect/tree.rs @@ -4,235 +4,292 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +use std::cmp::Ordering; use std::collections::HashMap; +use std::fmt::Write as _; use std::io::Write; -use rusqlite::Connection; +use rusqlite::{OptionalExtension, params}; -use super::{blob_summary, format_count}; +use super::View; +use crate::archive::rows::NodeType; +use crate::archive::{BlobId, LocalId}; +use crate::console::{ByteSize, Counted, Grouped, Initial, Tone}; use crate::error::Error; +use crate::types::ObjectType; + +const DEFAULT_MEDIA_TYPE: &str = "application/octet-stream"; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Branch { + Middle, + Last, +} + +impl Branch { + fn connector(self) -> &'static str { + match self { + Branch::Middle => "├── ", + Branch::Last => "└── ", + } + } + + fn indent(self) -> &'static str { + match self { + Branch::Middle => "│ ", + Branch::Last => " ", + } + } +} struct MailboxNode { - id: i64, - parent_id: Option, + id: LocalId, + parent_id: Option, name: String, role: Option, sort_order: i64, msg_count: u64, } -pub fn write_mailboxes(conn: &Connection, out: &mut impl Write) -> Result<(), Error> { - let nodes = load_mailboxes(conn)?; - writeln!(out, "Mailboxes ({})", format_count(nodes.len() as u64))?; - if nodes.is_empty() { - writeln!(out, "(none)")?; - return Ok(()); - } - let by_id: HashMap = nodes.iter().map(|n| (n.id, n)).collect(); - let mut by_parent: HashMap, Vec> = HashMap::new(); - for n in &nodes { - by_parent.entry(n.parent_id).or_default().push(n.id); - } - for ids in by_parent.values_mut() { - ids.sort_by(|a, b| { - let na = by_id[a]; - let nb = by_id[b]; - na.sort_order - .cmp(&nb.sort_order) - .then_with(|| na.name.cmp(&nb.name)) - }); - } - let roots: Vec = by_parent.get(&None).cloned().unwrap_or_default(); - let last = roots.len().saturating_sub(1); - for (i, id) in roots.iter().enumerate() { - write_mailbox(out, &by_id, &by_parent, *id, "", i == last)?; - } - Ok(()) -} - -fn write_mailbox( - out: &mut impl Write, - by_id: &HashMap, - by_parent: &HashMap, Vec>, - id: i64, - prefix: &str, - is_last: bool, -) -> Result<(), Error> { - let node = by_id[&id]; - let connector = if is_last { "└── " } else { "├── " }; - let mut tags: Vec = Vec::new(); - if let Some(role) = &node.role { - tags.push(format!("role={role}")); - } - tags.push(format!("{} msgs", format_count(node.msg_count))); - writeln!( - out, - "{prefix}{connector}{} [{}]", - node.name, - tags.join(", ") - )?; - let child_prefix = format!("{prefix}{}", if is_last { " " } else { "│ " }); - if let Some(children) = by_parent.get(&Some(id)) { - let last = children.len() - 1; - for (i, child) in children.iter().enumerate() { - write_mailbox(out, by_id, by_parent, *child, &child_prefix, i == last)?; - } - } - Ok(()) -} - -fn load_mailboxes(conn: &Connection) -> Result, Error> { - let mut stmt = conn.prepare( - "SELECT m.id, m.parent_id, m.name, m.role, m.sort_order, - (SELECT COUNT(*) FROM emails e - WHERE EXISTS (SELECT 1 FROM json_each(e.mailbox_ids) j - WHERE CAST(j.value AS INTEGER) = m.id)) - FROM mailboxes m", - )?; - let rows = stmt.query_map([], |r| { - Ok(MailboxNode { - id: r.get(0)?, - parent_id: r.get(1)?, - name: r.get(2)?, - role: r.get(3)?, - sort_order: r.get(4)?, - msg_count: r.get::<_, i64>(5)? as u64, - }) - })?; - let mut out = Vec::new(); - for row in rows { - out.push(row?); - } - Ok(out) -} - -struct FileNodeRow { - id: i64, - parent_id: Option, - node_type: String, - blob_id: Option, +struct FileNode { + id: LocalId, + parent_id: Option, + node_type: Option, + blob_id: Option, name: String, media_type: Option, target: Option, child_count: u64, } -pub fn write_file_nodes(conn: &Connection, out: &mut impl Write) -> Result<(), Error> { - let nodes = load_file_nodes(conn)?; - writeln!(out, "File nodes ({})", format_count(nodes.len() as u64))?; - if nodes.is_empty() { - writeln!(out, "(none)")?; - return Ok(()); +impl FileNode { + fn is_directory(&self) -> bool { + self.node_type == Some(NodeType::Directory) } - let by_id: HashMap = nodes.iter().map(|n| (n.id, n)).collect(); - let mut by_parent: HashMap, Vec> = HashMap::new(); - for n in &nodes { - by_parent.entry(n.parent_id).or_default().push(n.id); - } - for ids in by_parent.values_mut() { - ids.sort_by(|a, b| { - let na = by_id[a]; - let nb = by_id[b]; - (na.node_type != "directory") - .cmp(&(nb.node_type != "directory")) - .then_with(|| na.name.cmp(&nb.name)) - }); - } - let roots: Vec = by_parent.get(&None).cloned().unwrap_or_default(); - let last = roots.len().saturating_sub(1); - for (i, id) in roots.iter().enumerate() { - write_file_node(conn, out, &by_id, &by_parent, *id, "", i == last)?; - } - Ok(()) } -fn write_file_node( - conn: &Connection, - out: &mut impl Write, - by_id: &HashMap, - by_parent: &HashMap, Vec>, - id: i64, - prefix: &str, - is_last: bool, -) -> Result<(), Error> { - let node = by_id[&id]; - let connector = if is_last { "└── " } else { "├── " }; - let display = match node.node_type.as_str() { - "directory" => format!("{}/", node.name), - _ => node.name.clone(), - }; - let detail = match node.node_type.as_str() { - "directory" => format!("({} children)", format_count(node.child_count)), - "symlink" => { - let target = node.target.as_deref().unwrap_or("[]"); - format!("symlink -> {target}") +trait TreeNode { + fn id(&self) -> LocalId; + fn parent(&self) -> Option; +} + +impl TreeNode for MailboxNode { + fn id(&self) -> LocalId { + self.id + } + + fn parent(&self) -> Option { + self.parent_id + } +} + +impl TreeNode for FileNode { + fn id(&self) -> LocalId { + self.id + } + + fn parent(&self) -> Option { + self.parent_id + } +} + +struct Forest<'n, N> { + children: HashMap, Vec<&'n N>>, +} + +impl<'n, N: TreeNode> Forest<'n, N> { + fn new(nodes: &'n [N], order: impl Fn(&&'n N, &&'n N) -> Ordering) -> Forest<'n, N> { + let mut children: HashMap, Vec<&'n N>> = HashMap::new(); + for n in nodes { + children.entry(n.parent()).or_default().push(n); } - _ => { - let media = node - .media_type - .as_deref() - .unwrap_or("application/octet-stream"); - match node.blob_id { - Some(bid) => format!("{media}, {}", blob_summary(conn, bid)?), - None => format!("{media}, (no blob)"), - } + for siblings in children.values_mut() { + siblings.sort_by(&order); } - }; - writeln!(out, "{prefix}{connector}{display} {detail}")?; - let child_prefix = format!("{prefix}{}", if is_last { " " } else { "│ " }); - if let Some(children) = by_parent.get(&Some(id)) { - let last = children.len() - 1; - for (i, child) in children.iter().enumerate() { - write_file_node( - conn, + Forest { children } + } + + fn walk( + &self, + parent: Option, + prefix: &mut String, + visit: &mut impl FnMut(&'n N, &str, Branch) -> Result<(), Error>, + ) -> Result<(), Error> { + let Some((last, rest)) = self.children.get(&parent).and_then(|c| c.split_last()) else { + return Ok(()); + }; + let siblings = rest + .iter() + .map(|n| (*n, Branch::Middle)) + .chain([(*last, Branch::Last)]); + for (node, branch) in siblings { + visit(node, prefix, branch)?; + let len = prefix.len(); + prefix.push_str(branch.indent()); + self.walk(Some(node.id()), prefix, visit)?; + prefix.truncate(len); + } + Ok(()) + } +} + +impl View<'_> { + pub(super) fn write_mailboxes(self, out: &mut impl Write) -> Result<(), Error> { + let styled = self.styled; + let nodes = self.mailboxes()?; + self.tree_header(out, ObjectType::Mailbox, nodes.len())?; + if nodes.is_empty() { + return Ok(()); + } + let forest = Forest::new(&nodes, |a, b| { + a.sort_order + .cmp(&b.sort_order) + .then_with(|| a.name.cmp(&b.name)) + }); + let mut meta = String::new(); + forest.walk(None, &mut String::new(), &mut |node, prefix, branch| { + meta.clear(); + let messages = Counted::new(node.msg_count, "message", "messages"); + let _ = match &node.role { + Some(role) => write!(meta, "({role}, {messages})"), + None => write!(meta, "({messages})"), + }; + writeln!( out, - by_id, - by_parent, - *child, - &child_prefix, - i == last, + "{}{} {}", + styled.paint(Tone::Dim, format_args!("{prefix}{}", branch.connector())), + node.name, + styled.paint(Tone::Dim, &meta) + )?; + Ok(()) + }) + } + + pub(super) fn write_file_nodes(self, out: &mut impl Write) -> Result<(), Error> { + let styled = self.styled; + let nodes = self.file_nodes()?; + self.tree_header(out, ObjectType::FileNode, nodes.len())?; + if nodes.is_empty() { + return Ok(()); + } + let forest = Forest::new(&nodes, |a, b| { + (!a.is_directory()) + .cmp(&!b.is_directory()) + .then_with(|| a.name.cmp(&b.name)) + }); + let mut meta = String::new(); + forest.walk(None, &mut String::new(), &mut |node, prefix, branch| { + meta.clear(); + match node.node_type { + Some(NodeType::Directory) => { + let _ = write!( + meta, + "({})", + Counted::new(node.child_count, "item", "items") + ); + } + Some(NodeType::SymLink) => { + let _ = write!(meta, "-> {}", node.target.as_deref().unwrap_or("?")); + } + Some(NodeType::File) | None => { + let media = node.media_type.as_deref().unwrap_or(DEFAULT_MEDIA_TYPE); + let _ = match node.blob_id.map(|b| self.blob_len(b)).transpose()? { + Some(Some(len)) => write!(meta, "({media}, {})", ByteSize(len)), + Some(None) => write!(meta, "({media}, missing content)"), + None => write!(meta, "({media}, no content)"), + }; + } + } + let slash = if node.is_directory() { "/" } else { "" }; + writeln!( + out, + "{}{}{slash} {}", + styled.paint(Tone::Dim, format_args!("{prefix}{}", branch.connector())), + node.name, + styled.paint(Tone::Dim, &meta) + )?; + Ok(()) + }) + } + + fn tree_header(self, out: &mut impl Write, ty: ObjectType, n: usize) -> Result<(), Error> { + writeln!( + out, + "{} ({})", + self.styled.paint(Tone::Title, ty.label()), + Grouped(n as u64) + )?; + if n == 0 { + writeln!( + out, + "{}", + self.styled.paint( + Tone::Dim, + format_args!("No {} in the archive", Initial::Lower(ty.label())) + ) )?; } + Ok(()) } - Ok(()) -} -fn load_file_nodes(conn: &Connection) -> Result, Error> { - let mut stmt = conn.prepare( - "SELECT f.id, f.parent_id, f.node_type, f.blob_id, f.name, f.media_type, f.target, - (SELECT COUNT(*) FROM file_nodes c WHERE c.parent_id = f.id) - FROM file_nodes f", - )?; - let rows = stmt.query_map([], |r| { - Ok(FileNodeRow { - id: r.get(0)?, - parent_id: r.get(1)?, - node_type: r.get(2)?, - blob_id: r.get(3)?, - name: r.get(4)?, - media_type: r.get(5)?, - target: r.get(6)?, - child_count: r.get::<_, i64>(7)? as u64, - }) - })?; - let mut out = Vec::new(); - for row in rows { - out.push(row?); + fn blob_len(self, blob: BlobId) -> Result, Error> { + Ok(self + .conn + .prepare_cached("SELECT length(data) FROM blobs WHERE id = ?1")? + .query_row(params![blob], |r| r.get::<_, i64>(0)) + .optional()? + .map(|len| len.max(0) as u64)) + } + + fn mailboxes(self) -> Result, Error> { + let mut stmt = self.conn.prepare( + "SELECT m.id, m.parent_id, m.name, m.role, m.sort_order, + (SELECT COUNT(*) FROM emails e + WHERE EXISTS (SELECT 1 FROM json_each(e.mailbox_ids) j + WHERE CAST(j.value AS INTEGER) = m.id)) + FROM mailboxes m", + )?; + let rows = stmt.query_map([], |r| { + Ok(MailboxNode { + id: r.get(0)?, + parent_id: r.get(1)?, + name: r.get(2)?, + role: r.get(3)?, + sort_order: r.get(4)?, + msg_count: r.get::<_, i64>(5)? as u64, + }) + })?; + Ok(rows.collect::>()?) + } + + fn file_nodes(self) -> Result, Error> { + let mut stmt = self.conn.prepare( + "SELECT f.id, f.parent_id, f.node_type, f.blob_id, f.name, f.media_type, f.target, + (SELECT COUNT(*) FROM file_nodes c WHERE c.parent_id = f.id) + FROM file_nodes f", + )?; + let rows = stmt.query_map([], |r| { + Ok(FileNode { + id: r.get(0)?, + parent_id: r.get(1)?, + node_type: NodeType::parse(r.get_ref(2)?.as_str()?), + blob_id: r.get(3)?, + name: r.get(4)?, + media_type: r.get(5)?, + target: r.get(6)?, + child_count: r.get::<_, i64>(7)? as u64, + }) + })?; + Ok(rows.collect::>()?) } - Ok(out) } #[cfg(test)] mod tests { use super::*; - use crate::db::{blobs, init}; - use rusqlite::params; - - fn mem() -> Connection { - let c = Connection::open_in_memory().unwrap(); - init::apply_schema(&c).unwrap(); - c - } + use crate::archive::Archive; + use crate::console::Styled; + use rusqlite::{Connection, params}; fn insert_mailbox( c: &Connection, @@ -250,58 +307,70 @@ mod tests { c.last_insert_rowid() } - fn insert_email_in(c: &Connection, mailbox_id: i64) { - let blob = blobs::intern_blob(c, format!("seed-{mailbox_id}").as_bytes()).unwrap(); - c.execute( - "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords) - VALUES (?1, '2024-01-01T00:00:00Z', ?2, '[]')", - params![blob, format!("[{mailbox_id}]")], - ) - .unwrap(); + fn insert_email_in(a: &Archive, mailbox_id: i64) { + let blob = a + .store() + .blobs() + .intern(format!("seed-{mailbox_id}").as_bytes()) + .unwrap(); + a.connection() + .execute( + "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords) + VALUES (?1, '2024-01-01T00:00:00Z', ?2, '[]')", + params![blob, format!("[{mailbox_id}]")], + ) + .unwrap(); } #[test] - fn mailbox_tree_draws_box_characters_and_msg_counts() { - let c = mem(); - let inbox = insert_mailbox(&c, "Inbox", None, Some("inbox"), 0); - let _archived = insert_mailbox(&c, "Archived", Some(inbox), None, 0); - let _y2024 = insert_mailbox(&c, "2024", Some(inbox), None, 1); - let _sent = insert_mailbox(&c, "Sent", None, Some("sent"), 1); - insert_email_in(&c, inbox); - insert_email_in(&c, inbox); + fn mailbox_tree_draws_box_characters_and_message_counts() { + let a = Archive::open_in_memory().unwrap(); + let c = a.connection(); + let inbox = insert_mailbox(c, "Inbox", None, Some("inbox"), 0); + let _archived = insert_mailbox(c, "Archived", Some(inbox), None, 0); + let _y2024 = insert_mailbox(c, "2024", Some(inbox), None, 1); + let _sent = insert_mailbox(c, "Sent", None, Some("sent"), 1); + insert_email_in(&a, inbox); + insert_email_in(&a, inbox); let mut buf = Vec::new(); - write_mailboxes(&c, &mut buf).unwrap(); + View::new(c, Styled::Plain) + .write_mailboxes(&mut buf) + .unwrap(); let s = String::from_utf8(buf).unwrap(); - assert!(s.contains("Mailboxes (4)"), "header missing: {s}"); - assert!(s.contains("├── Inbox [role=inbox, 2 msgs]"), "got:\n{s}"); - assert!(s.contains("│ ├── Archived"), "got:\n{s}"); - assert!(s.contains("│ └── 2024"), "got:\n{s}"); - assert!(s.contains("└── Sent [role=sent, 0 msgs]"), "got:\n{s}"); + assert_eq!( + s, + "Mailboxes (4)\n\ + ├── Inbox (inbox, 2 messages)\n\ + │ ├── Archived (0 messages)\n\ + │ └── 2024 (0 messages)\n\ + └── Sent (sent, 0 messages)\n" + ); } #[test] fn mailbox_tree_empty_says_none() { - let c = mem(); + let a = Archive::open_in_memory().unwrap(); let mut buf = Vec::new(); - write_mailboxes(&c, &mut buf).unwrap(); + View::new(a.connection(), Styled::Plain) + .write_mailboxes(&mut buf) + .unwrap(); let s = String::from_utf8(buf).unwrap(); - assert!(s.contains("Mailboxes (0)")); - assert!(s.contains("(none)")); + assert!(s.contains("Mailboxes (0)")); + assert!(s.contains("No mailboxes in the archive")); } #[test] fn file_node_tree_distinguishes_dirs_files_symlinks() { - let c = mem(); - let docs: i64 = { - c.execute( - "INSERT INTO file_nodes (parent_id, node_type, name, created) - VALUES (NULL, 'directory', 'Documents', '2024-01-01T00:00:00Z')", - [], - ) - .unwrap(); - c.last_insert_rowid() - }; - let blob = blobs::intern_blob(&c, b"hello world").unwrap(); + let a = Archive::open_in_memory().unwrap(); + let c = a.connection(); + c.execute( + "INSERT INTO file_nodes (parent_id, node_type, name, created) + VALUES (NULL, 'directory', 'Documents', '2024-01-01T00:00:00Z')", + [], + ) + .unwrap(); + let docs = c.last_insert_rowid(); + let blob = a.store().blobs().intern(b"hello world").unwrap(); c.execute( "INSERT INTO file_nodes (parent_id, node_type, blob_id, name, media_type, created) VALUES (?1, 'file', ?2, 'note.txt', 'text/plain', '2024-01-01T00:00:00Z')", @@ -315,13 +384,19 @@ mod tests { ) .unwrap(); let mut buf = Vec::new(); - write_file_nodes(&c, &mut buf).unwrap(); + View::new(c, Styled::Plain) + .write_file_nodes(&mut buf) + .unwrap(); let s = String::from_utf8(buf).unwrap(); - assert!(s.contains("File nodes (3)")); - assert!(s.contains("Documents/")); - assert!(s.contains("(2 children)")); - assert!(s.contains("note.txt")); - assert!(s.contains("text/plain, 11 B blake3=")); - assert!(s.contains("symlink -> [\"..\",\"etc\"]")); + assert!(s.contains("File nodes (3)")); + assert!(s.contains("└── Documents/ (2 items)"), "got:\n{s}"); + assert!( + s.contains(" ├── note.txt (text/plain, 11 B)"), + "got:\n{s}" + ); + assert!( + s.contains(" └── shortcut -> [\"..\",\"etc\"]"), + "got:\n{s}" + ); } } diff --git a/src/jmap/account.rs b/src/jmap/account.rs index ce43ac4..32e85d5 100644 --- a/src/jmap/account.rs +++ b/src/jmap/account.rs @@ -4,13 +4,15 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +use std::fmt::Write; + +use serde_json::{Value, json}; + use crate::error::Error; use crate::jmap::error::JmapError; -use crate::jmap::http::HttpClient; -use crate::jmap::request::{Request, check_method_error}; +use crate::jmap::http::JmapClient; +use crate::jmap::request::Request; use crate::jmap::session::Session; -use serde_json::{Value, json}; -use std::fmt::Write; #[derive(Debug, Clone)] pub enum AccountSelector { @@ -19,150 +21,159 @@ pub enum AccountSelector { } const PRINCIPALS_URN: &str = "urn:ietf:params:jmap:principals"; +const STATUS_BAD_REQUEST: u16 = 400; -pub fn resolve( - selector: &AccountSelector, - session: &Session, - client: &HttpClient, -) -> Result { - match selector { - AccountSelector::Id(id) => Ok(id.clone()), - AccountSelector::Name(name) => { - for (id, account) in &session.accounts { - if account.name == *name { - return Ok(id.clone()); - } - } - resolve_via_principal(name, session, client) - } - } -} - -fn resolve_via_principal( - name: &str, - session: &Session, - client: &HttpClient, -) -> Result { - if !session.capabilities.contains_key(PRINCIPALS_URN) { - return Err(unsupported_principals(name, session)); - } - - let mut req = Request::new(); - req.call( - "Principal/query", - json!({ "filter": { "name": name } }), - "q", - ); - req.call( - "Principal/get", - json!({ - "#ids": { "resultOf": "q", "name": "Principal/query", "path": "/ids" }, - "properties": ["id", "name", "accounts"] - }), - "g", - ); - let resp = match req.send(client, &session.api_url) { - Ok(resp) => resp, - Err(JmapError::HttpStatus { status: 400, body }) if body.contains("unknownCapability") => { - return Err(unsupported_principals(name, session)); - } - Err(e) => { - return Err(Error::Account(format!( - "principal resolution request failed: {e}" - ))); - } - }; - - let query = resp - .by_call_id("q") - .map_err(|e| Error::Account(e.to_string()))?; - check_method_error(query).map_err(|e| Error::Account(e.to_string()))?; - let get = resp - .by_call_id("g") - .map_err(|e| Error::Account(e.to_string()))?; - check_method_error(get).map_err(|e| Error::Account(e.to_string()))?; - - let list = get - .args - .get("list") - .and_then(Value::as_array) - .ok_or_else(|| Error::Account("Principal/get returned no list".to_owned()))?; - - let mut matches: Vec<&Value> = list - .iter() - .filter(|p| { - p.get("name") - .and_then(Value::as_str) - .map(|n| n == name) - .unwrap_or(false) - }) - .collect(); - - match matches.len() { - 0 => Err(Error::Account(format!( - "account name '{name}' not found via Principal/query" - ))), - 1 => extract_account_id(matches.remove(0), name), - _ => { - let ids: Vec = matches +impl AccountSelector { + pub fn resolve(&self, session: &Session, client: &JmapClient) -> Result { + match self { + AccountSelector::Id(id) => Ok(id.clone()), + AccountSelector::Name(name) => match session + .accounts .iter() - .filter_map(|p| p.get("id").and_then(Value::as_str).map(str::to_owned)) - .collect(); - Err(Error::Account(format!( - "account name '{name}' is ambiguous; candidate principals: {}", - ids.join(", ") - ))) + .find(|(_, account)| account.name == *name) + { + Some((id, _)) => Ok(id.clone()), + None => AccountSelector::resolve_via_principal(name, session, client), + }, } } -} -fn unsupported_principals(name: &str, session: &Session) -> Error { - let mut available = String::new(); - for (id, account) in &session.accounts { - if !available.is_empty() { - available.push_str(", "); + fn resolve_via_principal( + name: &str, + session: &Session, + client: &JmapClient, + ) -> Result { + if !session.capabilities.contains_key(PRINCIPALS_URN) { + return Err(AccountSelector::unsupported_principals(name, session)); } - let _ = write!(available, "{} ({id})", account.name); - } - if available.is_empty() { - available.push_str("(none)"); - } - Error::Account(format!( - "account name '{name}' is not enumerated in this session and the source server does not \ - support '{PRINCIPALS_URN}', which is required to resolve an account by name from an \ - administrator session. Accounts visible to these credentials: {available}. Pass \ - --account-id to use an account id verbatim (no principals lookup is performed), \ - authenticate directly as the target user, or import over IMAP instead." - )) -} -fn extract_account_id(principal: &Value, name: &str) -> Result { - if let Some(accounts) = principal.get("accounts").and_then(Value::as_object) { - let mut keys = accounts.keys(); - if let (Some(only), None) = (keys.next(), keys.next()) { - return Ok(only.clone()); - } - let named = accounts + let mut req = Request::new(); + req.call( + "Principal/query", + json!({ "filter": { "name": name } }), + "q", + ); + req.call( + "Principal/get", + json!({ + "#ids": { "resultOf": "q", "name": "Principal/query", "path": "/ids" }, + "properties": ["id", "name", "accounts"] + }), + "g", + ); + let resp = match req.send(client, &session.api_url) { + Ok(resp) => resp, + Err(JmapError::HttpStatus { + status: STATUS_BAD_REQUEST, + body, + }) if body.contains("unknownCapability") => { + return Err(AccountSelector::unsupported_principals(name, session)); + } + Err(e) => { + return Err(Error::Account(format!( + "principal resolution request failed: {e}" + ))); + } + }; + + let account_error = |e: JmapError| Error::Account(e.to_string()); + resp.by_call_id("q") + .and_then(|query| query.check_error()) + .map_err(account_error)?; + let get = resp.by_call_id("g").map_err(account_error)?; + get.check_error().map_err(account_error)?; + + let list = get + .args + .get("list") + .and_then(Value::as_array) + .ok_or_else(|| Error::Account("Principal/get returned no list".to_owned()))?; + + let mut matches = list .iter() - .find(|(_, account)| account.get("name").and_then(Value::as_str) == Some(name)); - if let Some((id, _)) = named { - return Ok(id.clone()); + .filter(|p| p.get("name").and_then(Value::as_str) == Some(name)); + match (matches.next(), matches.next()) { + (None, _) => Err(Error::Account(format!( + "account name '{name}' not found via Principal/query" + ))), + (Some(only), None) => Principal(only).account_id(name), + (Some(first), Some(second)) => { + let mut ids = String::new(); + for (i, id) in [first, second] + .into_iter() + .chain(matches) + .filter_map(|p| p.get("id").and_then(Value::as_str)) + .enumerate() + { + if i > 0 { + ids.push_str(", "); + } + ids.push_str(id); + } + Err(Error::Account(format!( + "account name '{name}' is ambiguous; candidate principals: {ids}" + ))) + } } } - principal - .get("id") - .and_then(Value::as_str) - .map(str::to_owned) - .ok_or_else(|| { - Error::Account(format!( - "principal for '{name}' has no resolvable data account id" - )) - }) + + fn unsupported_principals(name: &str, session: &Session) -> Error { + let mut available = String::new(); + for (id, account) in &session.accounts { + if !available.is_empty() { + available.push_str(", "); + } + let _ = write!(available, "{} ({id})", account.name); + } + if available.is_empty() { + available.push_str("(none)"); + } + Error::Account(format!( + "account name '{name}' is not enumerated in this session and the source server does not \ + support '{PRINCIPALS_URN}', which is required to resolve an account by name from an \ + administrator session. Accounts visible to these credentials: {available}. Pass \ + --account-id to use an account id verbatim (no principals lookup is performed), \ + authenticate directly as the target user, or import over IMAP instead." + )) + } +} + +struct Principal<'a>(&'a Value); + +impl Principal<'_> { + fn account_id(&self, name: &str) -> Result { + if let Some(accounts) = self.0.get("accounts").and_then(Value::as_object) { + let mut keys = accounts.keys(); + if let (Some(only), None) = (keys.next(), keys.next()) { + return Ok(only.clone()); + } + let named = accounts + .iter() + .find(|(_, account)| account.get("name").and_then(Value::as_str) == Some(name)); + if let Some((id, _)) = named { + return Ok(id.clone()); + } + } + self.0 + .get("id") + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| { + Error::Account(format!( + "principal for '{name}' has no resolvable data account id" + )) + }) + } } #[cfg(test)] mod tests { use super::*; + use crate::error::ExitStatus; + use crate::logging::{Level, Logger}; + use crate::net::auth::Auth; + use crate::net::http::{ClientOptions, TlsPolicy}; + use crate::net::retry::RetryPolicy; fn session_with(name: &str, id: &str) -> Session { let raw = serde_json::json!({ @@ -175,11 +186,14 @@ mod tests { serde_json::from_value(raw).unwrap() } - fn client() -> HttpClient { - HttpClient::new( - crate::jmap::http::Auth::Bearer { token: "t".into() }, - crate::jmap::http::RetryPolicy::new(0), - false, + fn client() -> JmapClient { + JmapClient::new( + Auth::Bearer { token: "t".into() }, + ClientOptions { + retry: RetryPolicy::new(0), + tls: TlsPolicy::Verify, + logger: Logger::new(Level::Default), + }, ) } @@ -187,7 +201,9 @@ mod tests { fn id_selector_is_verbatim() { let s = session_with("alice", "a"); assert_eq!( - resolve(&AccountSelector::Id("zzz".into()), &s, &client()).unwrap(), + AccountSelector::Id("zzz".into()) + .resolve(&s, &client()) + .unwrap(), "zzz" ); } @@ -196,12 +212,9 @@ mod tests { fn name_selector_exact_case_sensitive_match() { let s = session_with("alice@example.org", "w"); assert_eq!( - resolve( - &AccountSelector::Name("alice@example.org".into()), - &s, - &client() - ) - .unwrap(), + AccountSelector::Name("alice@example.org".into()) + .resolve(&s, &client()) + .unwrap(), "w" ); } @@ -221,19 +234,16 @@ mod tests { .filter(|p| p.get("name").and_then(Value::as_str) == Some("alice")) .collect(); assert_eq!(matches.len(), 1); - assert_eq!(extract_account_id(matches[0], "alice").unwrap(), "w"); + assert_eq!(Principal(matches[0]).account_id("alice").unwrap(), "w"); } #[test] fn missing_principals_capability_is_actionable_without_a_request() { let s = session_with("alice@example.org", "w"); - let err = resolve( - &AccountSelector::Name("bob@example.org".into()), - &s, - &client(), - ) - .unwrap_err(); - assert_eq!(err.exit_code(), 3); + let err = AccountSelector::Name("bob@example.org".into()) + .resolve(&s, &client()) + .unwrap_err(); + assert_eq!(err.exit_status(), ExitStatus::Account); let msg = err.to_string(); assert!(msg.contains(PRINCIPALS_URN)); assert!(msg.contains("--account-id")); @@ -244,7 +254,7 @@ mod tests { #[test] fn extract_falls_back_to_principal_id() { let p = json!({ "id": "acc-7", "name": "bob", "accounts": {} }); - assert_eq!(extract_account_id(&p, "bob").unwrap(), "acc-7"); + assert_eq!(Principal(&p).account_id("bob").unwrap(), "acc-7"); } // RFC 9670: the data account is the key of "accounts", not accountIdForPrincipal. @@ -262,7 +272,10 @@ mod tests { } } } }); - assert_eq!(extract_account_id(&p, "bob@example.org").unwrap(), "data-9"); + assert_eq!( + Principal(&p).account_id("bob@example.org").unwrap(), + "data-9" + ); } #[test] @@ -275,6 +288,6 @@ mod tests { "data-9": { "name": "bob" } } }); - assert_eq!(extract_account_id(&p, "bob").unwrap(), "data-9"); + assert_eq!(Principal(&p).account_id("bob").unwrap(), "data-9"); } } diff --git a/src/jmap/blob.rs b/src/jmap/blob.rs index 807ce70..337e6b1 100644 --- a/src/jmap/blob.rs +++ b/src/jmap/blob.rs @@ -8,6 +8,7 @@ use encodify::base64::STANDARD; use serde_json::{Map, Value}; use crate::jmap::error::JmapError; +use crate::net::encode::{Percent, SafeSet}; pub const SENTINEL_KEY: &str = "@blob"; @@ -30,149 +31,110 @@ impl InlineShape { } } -pub fn import_blob_ids(value: &mut Value, mut resolve: F) -> Result<(), BlobWalkError> -where - F: FnMut(&str) -> Result, -{ - walk_import(value, &mut resolve) -} +pub struct BlobRefs<'a>(pub &'a mut Value); -fn walk_import(value: &mut Value, resolve: &mut F) -> Result<(), BlobWalkError> -where - F: FnMut(&str) -> Result, -{ - match value { - Value::Object(map) => { - if let Some(Value::String(blob_id)) = map.remove("blobId") { - let local = resolve(&blob_id)?; - map.insert(SENTINEL_KEY.to_owned(), Value::from(local)); - } - for child in map.values_mut() { - walk_import(child, resolve)?; - } - } - Value::Array(items) => { - for item in items { - walk_import(item, resolve)?; - } - } - _ => {} +impl BlobRefs<'_> { + pub fn to_sentinels(self, mut resolve: F) -> Result<(), BlobWalkError> + where + F: FnMut(&str) -> Result, + { + BlobRefs::walk_import(self.0, &mut resolve) } - Ok(()) -} -pub fn inline_blob_data_uris( - value: &mut Value, - shape: InlineShape, - mut fetch: F, -) -> Result<(), BlobWalkError> -where - F: FnMut(i64) -> Result, BlobWalkError>, -{ - walk_inline(value, shape, &mut fetch) -} + pub fn to_data_uris(self, shape: InlineShape, mut fetch: F) -> Result<(), BlobWalkError> + where + F: FnMut(i64) -> Result, BlobWalkError>, + { + BlobRefs::walk_inline(self.0, shape, &mut fetch) + } -fn walk_inline(value: &mut Value, shape: InlineShape, fetch: &mut F) -> Result<(), BlobWalkError> -where - F: FnMut(i64) -> Result, BlobWalkError>, -{ - match value { - Value::Object(map) => { - if let Some(sentinel) = map.remove(SENTINEL_KEY) { - let local_id = sentinel.as_i64().ok_or(BlobWalkError::MalformedSentinel)?; - let bytes = fetch(local_id)?; - let uri = data_uri(map, &bytes); - map.insert(shape.uri_key().to_owned(), Value::String(uri)); + fn walk_import(value: &mut Value, resolve: &mut F) -> Result<(), BlobWalkError> + where + F: FnMut(&str) -> Result, + { + match value { + Value::Object(map) => { + if let Some(Value::String(blob_id)) = map.remove("blobId") { + let local = resolve(&blob_id)?; + map.insert(SENTINEL_KEY.to_owned(), Value::from(local)); + } + for child in map.values_mut() { + BlobRefs::walk_import(child, resolve)?; + } } - for child in map.values_mut() { - walk_inline(child, shape, fetch)?; + Value::Array(items) => { + for item in items { + BlobRefs::walk_import(item, resolve)?; + } } + _ => {} } - Value::Array(items) => { - for item in items { - walk_inline(item, shape, fetch)?; + Ok(()) + } + + fn walk_inline( + value: &mut Value, + shape: InlineShape, + fetch: &mut F, + ) -> Result<(), BlobWalkError> + where + F: FnMut(i64) -> Result, BlobWalkError>, + { + match value { + Value::Object(map) => { + if let Some(sentinel) = map.remove(SENTINEL_KEY) { + let local_id = sentinel.as_i64().ok_or(BlobWalkError::MalformedSentinel)?; + let bytes = fetch(local_id)?; + let uri = BlobRefs::data_uri(map, &bytes); + map.insert(shape.uri_key().to_owned(), Value::String(uri)); + } + for child in map.values_mut() { + BlobRefs::walk_inline(child, shape, fetch)?; + } } + Value::Array(items) => { + for item in items { + BlobRefs::walk_inline(item, shape, fetch)?; + } + } + _ => {} } - _ => {} + Ok(()) } - Ok(()) -} -fn data_uri(map: &Map, bytes: &[u8]) -> String { - let media_type = MEDIA_TYPE_KEYS - .iter() - .find_map(|k| map.get(*k).and_then(Value::as_str).and_then(uri_media_type)) - .unwrap_or_else(|| DEFAULT_MEDIA_TYPE.to_owned()); - let mut uri = String::with_capacity( - DATA_URI_OVERHEAD + media_type.len() + STANDARD.encoded_len(bytes.len()), - ); - uri.push_str("data:"); - uri.push_str(&media_type); - uri.push_str(";base64,"); - STANDARD.encode_append(bytes, &mut uri); - uri -} - -fn uri_media_type(raw: &str) -> Option { - let mut parts = raw.split(';').map(str::trim); - let essence = parts.next()?; - let (ty, subtype) = essence.split_once('/')?; - if ty.is_empty() || subtype.is_empty() || subtype.contains('/') { - return None; + fn data_uri(map: &Map, bytes: &[u8]) -> String { + let media_type = MEDIA_TYPE_KEYS + .iter() + .find_map(|k| { + map.get(*k) + .and_then(Value::as_str) + .and_then(BlobRefs::uri_media_type) + }) + .unwrap_or_else(|| DEFAULT_MEDIA_TYPE.to_owned()); + let mut uri = String::with_capacity( + DATA_URI_OVERHEAD + media_type.len() + STANDARD.encoded_len(bytes.len()), + ); + uri.push_str("data:"); + uri.push_str(&media_type); + uri.push_str(";base64,"); + STANDARD.encode_append(bytes, &mut uri); + uri } - let mut out = String::with_capacity(raw.len()); - escape_into(&mut out, essence); - for parameter in parts.filter(|p| !p.is_empty()) { - out.push(';'); - escape_into(&mut out, parameter); - } - Some(out) -} -fn escape_into(out: &mut String, part: &str) { - const HEX: &[u8; 16] = b"0123456789ABCDEF"; - for byte in part.bytes() { - if is_media_type_uri_byte(byte) { - out.push(byte as char); - } else { - out.push('%'); - out.push(HEX[(byte >> 4) as usize] as char); - out.push(HEX[(byte & 0x0F) as usize] as char); + fn uri_media_type(raw: &str) -> Option { + let mut parts = raw.split(';').map(str::trim); + let essence = parts.next()?; + let (ty, subtype) = essence.split_once('/')?; + if ty.is_empty() || subtype.is_empty() || subtype.contains('/') { + return None; } - } -} - -fn is_media_type_uri_byte(byte: u8) -> bool { - byte.is_ascii_alphanumeric() - || matches!( - byte, - b'!' | b'$' - | b'&' - | b'\'' - | b'(' - | b')' - | b'*' - | b'+' - | b'-' - | b'.' - | b'/' - | b':' - | b'=' - | b'@' - | b'_' - | b'~' - ) -} - -pub fn prepend_property(target: &mut Map, key: &str, value: Value) { - let existing: Vec<(String, Value)> = - target.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); - target.clear(); - target.insert(key.to_owned(), value); - for (k, v) in existing { - if k != key { - target.insert(k, v); + let mut out = String::with_capacity(raw.len()); + Percent::encode_into(&mut out, essence, SafeSet::MediaTypeUri); + for parameter in parts.filter(|p| !p.is_empty()) { + out.push(';'); + Percent::encode_into(&mut out, parameter, SafeSet::MediaTypeUri); } + Some(out) } } @@ -200,6 +162,7 @@ impl BlobWalkError { #[cfg(test)] mod tests { use super::*; + use crate::error::ExitStatus; use serde_json::json; fn fetch_ok(local_id: i64) -> Result, BlobWalkError> { @@ -237,7 +200,7 @@ mod tests { continue; } assert!( - is_media_type_uri_byte(b) || b == b';', + SafeSet::MediaTypeUri.contains(b) || b == b';', "byte {b:#04x} is not allowed unescaped in {uri}" ); i += 1; @@ -273,7 +236,9 @@ mod tests { "@blob": 7, "mediaType": "image/png" } } }); - inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap(); + BlobRefs(&mut card) + .to_data_uris(InlineShape::JsContactResource, fetch_ok) + .unwrap(); let photo = &card["media"]["photo"]; assert_eq!( decode(photo["uri"].as_str().unwrap(), "image/png"), @@ -293,7 +258,9 @@ mod tests { "@blob": 3, "contentType": "text/plain", "title": "note.txt" } } }); - inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap(); + BlobRefs(&mut event) + .to_data_uris(InlineShape::JsCalendarLink, fetch_ok) + .unwrap(); let link = &event["links"]["1"]; assert_eq!( decode(link["href"].as_str().unwrap(), "text/plain"), @@ -311,7 +278,9 @@ mod tests { "@type": "Card", "links": { "l1": { "@type": "Link", "@blob": 1, "mediaType": "application/pdf" } } }); - inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap(); + BlobRefs(&mut card) + .to_data_uris(InlineShape::JsContactResource, fetch_ok) + .unwrap(); let link = &card["links"]["l1"]; assert!(link.get("href").is_none(), "JSContact Link uses uri"); assert_eq!( @@ -323,7 +292,9 @@ mod tests { #[test] fn absent_media_type_defaults_to_octet_stream() { let mut card = json!({ "media": { "photo": { "@blob": 2 } } }); - inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap(); + BlobRefs(&mut card) + .to_data_uris(InlineShape::JsContactResource, fetch_ok) + .unwrap(); let uri = card["media"]["photo"]["uri"].as_str().unwrap(); assert_eq!(decode(uri, DEFAULT_MEDIA_TYPE), b"bytes-2"); } @@ -331,7 +302,9 @@ mod tests { #[test] fn empty_media_type_defaults_to_octet_stream() { let mut event = json!({ "links": { "1": { "@blob": 4, "contentType": "" } } }); - inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap(); + BlobRefs(&mut event) + .to_data_uris(InlineShape::JsCalendarLink, fetch_ok) + .unwrap(); let uri = event["links"]["1"]["href"].as_str().unwrap(); assert_eq!(decode(uri, DEFAULT_MEDIA_TYPE), b"bytes-4"); } @@ -341,7 +314,9 @@ mod tests { let mut card = json!({ "media": { "photo": { "@blob": 4, "mediaType": "", "contentType": "image/jpeg" } } }); - inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap(); + BlobRefs(&mut card) + .to_data_uris(InlineShape::JsContactResource, fetch_ok) + .unwrap(); let uri = card["media"]["photo"]["uri"].as_str().unwrap(); assert_eq!(decode(uri, "image/jpeg"), b"bytes-4"); } @@ -351,7 +326,9 @@ mod tests { let mut event = json!({ "links": { "1": { "@blob": 4, "contentType": "text/plain; charset=utf-8" } } }); - inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap(); + BlobRefs(&mut event) + .to_data_uris(InlineShape::JsCalendarLink, fetch_ok) + .unwrap(); let uri = event["links"]["1"]["href"].as_str().unwrap(); assert!(!uri.contains(' '), "{uri} carries a raw space"); assert_eq!(decode(uri, "text/plain;charset=utf-8"), b"bytes-4"); @@ -362,7 +339,9 @@ mod tests { let mut event = json!({ "links": { "1": { "@blob": 4, "contentType": " text/plain ;; charset=utf-8 ; format=flowed " } } }); - inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap(); + BlobRefs(&mut event) + .to_data_uris(InlineShape::JsCalendarLink, fetch_ok) + .unwrap(); let uri = event["links"]["1"]["href"].as_str().unwrap(); assert_eq!( decode(uri, "text/plain;charset=utf-8;format=flowed"), @@ -375,7 +354,9 @@ mod tests { let mut event = json!({ "links": { "1": { "@blob": 4, "contentType": "text/plain; name=\"rep ort,v2#1\u{e9}.txt\"" } } }); - inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap(); + BlobRefs(&mut event) + .to_data_uris(InlineShape::JsCalendarLink, fetch_ok) + .unwrap(); let uri = event["links"]["1"]["href"].as_str().unwrap(); assert_eq!( decode(uri, "text/plain;name=%22rep%20ort%2Cv2%231%C3%A9.txt%22"), @@ -388,7 +369,9 @@ mod tests { let mut card = json!({ "media": { "photo": { "@blob": 4, "mediaType": "image/png; note=100%" } } }); - inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap(); + BlobRefs(&mut card) + .to_data_uris(InlineShape::JsContactResource, fetch_ok) + .unwrap(); let uri = card["media"]["photo"]["uri"].as_str().unwrap(); assert_eq!(decode(uri, "image/png;note=100%25"), b"bytes-4"); } @@ -397,7 +380,9 @@ mod tests { fn a_media_type_that_is_not_type_slash_subtype_defaults_to_octet_stream() { for bogus in ["png", "image/", "/png", "image/png/extra", ";charset=utf-8"] { let mut card = json!({ "media": { "photo": { "@blob": 4, "mediaType": bogus } } }); - inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap(); + BlobRefs(&mut card) + .to_data_uris(InlineShape::JsContactResource, fetch_ok) + .unwrap(); let uri = card["media"]["photo"]["uri"].as_str().unwrap(); assert_eq!(decode(uri, DEFAULT_MEDIA_TYPE), b"bytes-4", "{bogus}"); } @@ -409,10 +394,9 @@ mod tests { let mut card = json!({ "media": { "photo": { "@blob": 1, "mediaType": "application/octet-stream; charset=binary" } } }); - inline_blob_data_uris(&mut card, InlineShape::JsContactResource, |_| { - Ok(payload.clone()) - }) - .unwrap(); + BlobRefs(&mut card) + .to_data_uris(InlineShape::JsContactResource, |_| Ok(payload.clone())) + .unwrap(); let uri = card["media"]["photo"]["uri"].as_str().unwrap(); assert_eq!( decode(uri, "application/octet-stream;charset=binary"), @@ -429,7 +413,9 @@ mod tests { ] } }, "links": { "top": { "@blob": 8, "contentType": "image/gif" } } }); - inline_blob_data_uris(&mut value, InlineShape::JsCalendarLink, fetch_ok).unwrap(); + BlobRefs(&mut value) + .to_data_uris(InlineShape::JsCalendarLink, fetch_ok) + .unwrap(); let arr = value["locations"]["loc1"]["links"].as_array().unwrap(); assert_eq!( decode(arr[0]["href"].as_str().unwrap(), "image/gif"), @@ -449,7 +435,8 @@ mod tests { #[test] fn malformed_sentinel_is_rejected() { let mut value = json!({ "media": { "photo": { "@blob": "not-an-id" } } }); - let err = inline_blob_data_uris(&mut value, InlineShape::JsContactResource, fetch_ok) + let err = BlobRefs(&mut value) + .to_data_uris(InlineShape::JsContactResource, fetch_ok) .expect_err("malformed sentinel"); assert!(matches!(err, BlobWalkError::MalformedSentinel), "{err:?}"); } @@ -457,27 +444,29 @@ mod tests { #[test] fn resolver_failure_propagates() { let mut value = json!({ "media": { "photo": { "@blob": 9 } } }); - let err = inline_blob_data_uris(&mut value, InlineShape::JsContactResource, |_| { - Err(BlobWalkError::resolver(JmapError::malformed("gone"))) - }) - .expect_err("resolver failure"); + let err = BlobRefs(&mut value) + .to_data_uris(InlineShape::JsContactResource, |_| { + Err(BlobWalkError::resolver(JmapError::malformed("gone"))) + }) + .expect_err("resolver failure"); assert!(matches!(err, BlobWalkError::Resolver(_)), "{err:?}"); } #[test] fn the_resolver_error_kind_survives_the_walk() { let mut value = json!({ "media": { "photo": { "@blob": 9 } } }); - let err = inline_blob_data_uris(&mut value, InlineShape::JsContactResource, |_| { - Err(BlobWalkError::resolver(JmapError::Sqlite( - rusqlite::Error::QueryReturnedNoRows, - ))) - }) - .expect_err("resolver failure") - .into_source(); + let err = BlobRefs(&mut value) + .to_data_uris(InlineShape::JsContactResource, |_| { + Err(BlobWalkError::resolver(JmapError::Sqlite( + rusqlite::Error::QueryReturnedNoRows, + ))) + }) + .expect_err("resolver failure") + .into_source(); assert!(matches!(err, JmapError::Sqlite(_)), "{err:?}"); let mapped = crate::error::Error::from(err); assert!(mapped.aborts_run(), "{mapped} must abort the run"); - assert_eq!(mapped.exit_code(), 7); + assert_eq!(mapped.exit_status(), ExitStatus::Archive); } #[test] @@ -487,7 +476,10 @@ mod tests { matches!(err, JmapError::Blob(BlobWalkError::MalformedSentinel)), "{err:?}" ); - assert_eq!(crate::error::Error::from(err).exit_code(), 5); + assert_eq!( + crate::error::Error::from(err).exit_status(), + ExitStatus::Partial + ); } #[test] @@ -499,10 +491,11 @@ mod tests { "nickNames": { "n1": { "name": "Jay" } } }); let mut value = original.clone(); - inline_blob_data_uris(&mut value, InlineShape::JsContactResource, |_| { - panic!("fetch must not be called") - }) - .unwrap(); + BlobRefs(&mut value) + .to_data_uris(InlineShape::JsContactResource, |_| { + panic!("fetch must not be called") + }) + .unwrap(); assert_eq!(value, original); } @@ -510,11 +503,12 @@ mod tests { fn import_walker_still_produces_the_sentinel() { let mut value = json!({ "media": { "photo": { "blobId": "B1", "mediaType": "image/png" } } }); - import_blob_ids(&mut value, |id| { - assert_eq!(id, "B1"); - Ok(11) - }) - .unwrap(); + BlobRefs(&mut value) + .to_sentinels(|id| { + assert_eq!(id, "B1"); + Ok(11) + }) + .unwrap(); assert_eq!(value["media"]["photo"][SENTINEL_KEY], json!(11)); assert!(value["media"]["photo"].get("blobId").is_none()); } diff --git a/src/jmap/blobxfer.rs b/src/jmap/blobxfer.rs index 353c532..ccc4435 100644 --- a/src/jmap/blobxfer.rs +++ b/src/jmap/blobxfer.rs @@ -6,35 +6,30 @@ use serde_json::Value; +use crate::jmap::connect::JmapAccount; use crate::jmap::error::JmapError; -use crate::jmap::http::HttpClient; -use crate::jmap::session::Session; use crate::jmap::wire::JmapId; -pub fn upload_bytes( - client: &HttpClient, - session: &Session, - account_id: &str, - content_type: &str, - bytes: &[u8], -) -> Result { - let url = session.upload_url_for(account_id); - let response = client.upload(&url, content_type, bytes)?; - response - .get("blobId") - .and_then(Value::as_str) - .map(|s| JmapId(s.to_owned())) - .ok_or_else(|| JmapError::malformed("upload response has no blobId")) -} +impl JmapAccount { + pub fn upload(&self, bytes: &[u8], content_type: &str) -> Result { + let url = self.session().upload_url_for(self.account_id()); + let response = self.client().upload(&url, content_type, bytes)?; + response + .get("blobId") + .and_then(Value::as_str) + .map(|s| JmapId(s.to_owned())) + .ok_or_else(|| JmapError::malformed("upload response has no blobId")) + } -pub fn download_bytes( - client: &HttpClient, - session: &Session, - account_id: &str, - blob_id: &str, - type_hint: &str, - name: &str, -) -> Result, JmapError> { - let url = session.download_url_for(account_id, blob_id, type_hint, name); - client.download(&url) + pub fn download( + &self, + blob_id: &str, + type_hint: &str, + name: &str, + ) -> Result, JmapError> { + let url = self + .session() + .download_url_for(self.account_id(), blob_id, type_hint, name); + self.client().download(&url) + } } diff --git a/src/jmap/connect.rs b/src/jmap/connect.rs index b9586d3..bdae19c 100644 --- a/src/jmap/connect.rs +++ b/src/jmap/connect.rs @@ -4,66 +4,106 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +use crate::console::Initial; use crate::error::Error; -use crate::jmap::account; +use crate::jmap::account::AccountSelector; +use crate::jmap::http::JmapClient; use crate::jmap::session::{Limits, Session}; -use crate::sync::{ConnectConfig, Context}; +use crate::net::auth::Auth; use crate::types::ObjectType; -pub struct Connected { - pub session: Session, - pub limits: Limits, - pub account_id: String, +#[derive(Debug, Clone)] +pub struct ConnectConfig { + pub url: String, + pub auth: Auth, + pub account: AccountSelector, +} + +#[derive(Clone)] +pub struct JmapAccount { + client: JmapClient, + session: Session, + account_id: String, + limits: Limits, supported: Vec, } -impl Connected { - pub fn supports(&self, ty: ObjectType) -> bool { - self.supported.contains(&ty) - } +impl JmapAccount { + pub fn connect(client: JmapClient, connect: &ConnectConfig) -> Result { + let logger = client.logger(); + let session = Session::discover(&client, &connect.url)?; + for mismatch in session.origin_mismatches(&connect.url) { + logger.warn(format_args!("{mismatch}")); + } + let limits = session.core_limits()?; + client.set_limits(&limits); + let account_id = connect.account.resolve(&session, &client)?; - pub fn supported_types(&self) -> &[ObjectType] { - &self.supported - } -} - -pub fn prepare(ctx: &Context, connect: &ConnectConfig) -> Result { - ctx.client.set_logger(ctx.common.logger); - - let session = Session::discover(&ctx.client, &connect.url)?; - for mismatch in session.origin_mismatches(&connect.url) { - ctx.common.logger.warn(&mismatch.to_string()); - } - let limits = session.core_limits()?; - ctx.client.set_limits(&limits); - let account_id = account::resolve(&connect.account, &session, &ctx.client)?; - - if session.account(&account_id).is_none() - && !matches!(connect.account, account::AccountSelector::Name(_)) - { - ctx.common.logger.warn(&format!( - "account id {account_id} is not enumerated in this session; capabilities cannot be gated and every type will be attempted" - )); - } - - let mut supported = Vec::new(); - for ty in ObjectType::ALL { - let urn = ty.capability_urn(); let known = session.account(&account_id).is_some(); - if !known || session.supports(&account_id, urn) { - supported.push(ty); - } else { - ctx.common.logger.warn(&format!( - "target account does not advertise {urn}; skipping type {}", - ty.jmap_name() + if !known && !matches!(connect.account, AccountSelector::Name(_)) { + logger.warn(format_args!( + "account id {account_id} is not enumerated in this session; capabilities cannot be gated and every type will be attempted" )); } + + let mut supported = Vec::with_capacity(ObjectType::ALL.len()); + for ty in ObjectType::ALL { + let urn = ty.capability_urn(); + if !known || session.supports(&account_id, urn) { + supported.push(ty); + } else { + logger.warn(format_args!( + "the target account does not advertise {urn}; skipping {}", + Initial::Lower(ty.label()) + )); + } + } + + Ok(JmapAccount { + client, + session, + account_id, + limits, + supported, + }) + } + + pub fn from_parts( + client: JmapClient, + session: Session, + account_id: String, + limits: Limits, + ) -> JmapAccount { + JmapAccount { + client, + session, + account_id, + limits, + supported: ObjectType::ALL.to_vec(), + } } - Ok(Connected { - session, - limits, - account_id, - supported, - }) + pub fn client(&self) -> &JmapClient { + &self.client + } + + pub fn session(&self) -> &Session { + &self.session + } + + pub fn account_id(&self) -> &str { + &self.account_id + } + + pub fn api_url(&self) -> &str { + &self.session.api_url + } + + pub fn limits(&self) -> &Limits { + &self.limits + } + + pub fn supports(&self, ty: ObjectType) -> bool { + self.supported.contains(&ty) + } } diff --git a/src/jmap/error.rs b/src/jmap/error.rs index 9147c2e..a2d9dc6 100644 --- a/src/jmap/error.rs +++ b/src/jmap/error.rs @@ -4,9 +4,10 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use crate::db::init::OpenError; +use crate::archive::ArchiveError; use crate::error::Error; use crate::jmap::blob::BlobWalkError; +use crate::net::http::TransportError; #[derive(Debug, thiserror::Error)] pub enum JmapError { @@ -60,6 +61,16 @@ pub enum JmapError { Blob(#[from] BlobWalkError), } +impl From for JmapError { + fn from(e: TransportError) -> JmapError { + if e.is_connect() { + JmapError::Connect(e.to_string()) + } else { + JmapError::Transport(e.to_string()) + } + } +} + impl JmapError { pub fn malformed(context: impl Into) -> JmapError { JmapError::Malformed(context.into()) @@ -71,7 +82,7 @@ impl From for Error { match value { JmapError::Connect(m) | JmapError::Transport(m) => Error::Connection(m), JmapError::Auth(m) => Error::Connection(format!("authentication rejected: {m}")), - JmapError::Sqlite(e) => Error::Db(OpenError::Sqlite(e)), + JmapError::Sqlite(e) => Error::Db(ArchiveError::Sqlite(e)), reached @ (JmapError::HttpStatus { .. } | JmapError::RetriesExhausted(_)) => { Error::Connection(reached.to_string()) } @@ -83,6 +94,19 @@ impl From for Error { #[cfg(test)] mod tests { use super::*; + use crate::error::ExitStatus; + + // the -vv transport log line and the error keep the transport or connection prefix + #[test] + fn transport_errors_keep_their_prefix() { + let io = TransportError::Io(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)); + let io_text = JmapError::from(io).to_string(); + assert!(io_text.starts_with("transport failure: io: "), "{io_text}"); + assert_eq!( + JmapError::from(TransportError::TooManyRedirects).to_string(), + format!("connection failure: {}", TransportError::TooManyRedirects) + ); + } #[test] fn unreachable_endpoint_aborts_the_run() { @@ -98,7 +122,7 @@ mod tests { ] { let mapped = Error::from(e); assert!(mapped.aborts_run(), "{mapped} must abort the run"); - assert_eq!(mapped.exit_code(), 2); + assert_eq!(mapped.exit_status(), ExitStatus::Connection); } } @@ -119,7 +143,7 @@ mod tests { ] { let mapped = Error::from(e); assert!(!mapped.aborts_run(), "{mapped} must not abort the run"); - assert_eq!(mapped.exit_code(), 5); + assert_eq!(mapped.exit_status(), ExitStatus::Partial); } } @@ -127,6 +151,6 @@ mod tests { fn archive_failures_are_local_io() { let mapped = Error::from(JmapError::Sqlite(rusqlite::Error::QueryReturnedNoRows)); assert!(mapped.aborts_run()); - assert_eq!(mapped.exit_code(), 7); + assert_eq!(mapped.exit_status(), ExitStatus::Archive); } } diff --git a/src/jmap/http.rs b/src/jmap/http.rs index 5543cf8..848df1e 100644 --- a/src/jmap/http.rs +++ b/src/jmap/http.rs @@ -4,178 +4,104 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use std::sync::Arc; -use std::sync::OnceLock; -use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; -use std::time::{Duration, Instant, SystemTime}; +use std::fmt; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, OnceLock}; +use std::thread; +use std::time::{Duration, Instant}; -use encodify::base64::STANDARD; use serde_json::Value; -use ureq::Agent; -use ureq::config::{Config, ConfigBuilder, RedirectAuthHeaders}; -use ureq::tls::{RootCerts, TlsConfig}; -use ureq::typestate::AgentScope; -use ureq::{ResponseExt, http::Uri}; +use ureq::ResponseExt; +use ureq::http::{HeaderMap, Uri}; use crate::jmap::error::JmapError; use crate::jmap::inflight::{Permit, Semaphore}; -use crate::jmap::retry::{self, Disposition, RateLimitState}; use crate::jmap::session::Limits; -use crate::logging::{HttpCall, LEVEL_BODIES, LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; +use crate::logging::{HttpCall, Level, Logger}; +use crate::net::auth::Auth; +use crate::net::http::{ + AgentProfile, BodyExcerpt, ClientOptions, HttpCore, Methods, Redirects, RetryStats, + TransportError, +}; +use crate::net::retry::{Disposition, RateLimit, RetryAfter, RetryPolicy, RetryWait}; -const MAX_BODY: u64 = 512 * 1024 * 1024; +const PROFILE: AgentProfile = AgentProfile { + redirects: Redirects::Follow, + methods: Methods::Standard, + max_body: 512 * 1024 * 1024, +}; const LONG_RETRY_THRESHOLD: Duration = Duration::from_secs(10); -const BASIC: &str = "Basic "; +const STATUS_UNAUTHORIZED: u16 = 401; +const STATUS_FORBIDDEN: u16 = 403; +const STATUS_PAYLOAD_TOO_LARGE: u16 = 413; +const PROTO: &str = "JMAP"; -const IO_TIMEOUT: Duration = Duration::from_secs(300); - -#[derive(Debug, Clone)] -pub enum Auth { - Basic { user: String, password: String }, - Bearer { token: String }, -} - -impl Auth { - pub fn header_value(&self) -> String { - match self { - Auth::Basic { user, password } => { - let credentials = format!("{user}:{password}"); - let mut header = - String::with_capacity(BASIC.len() + STANDARD.encoded_len(credentials.len())); - header.push_str(BASIC); - STANDARD.encode_append(credentials, &mut header); - header - } - Auth::Bearer { token } => format!("Bearer {token}"), - } - } -} - -#[derive(Debug, Clone, Copy)] -pub struct RetryPolicy { - pub max_retries: u32, - pub base: Duration, - pub cap: Duration, - pub io_timeout: Duration, -} - -impl RetryPolicy { - pub fn new(max_retries: u32) -> Self { - RetryPolicy { - max_retries, - base: Duration::from_millis(500), - cap: Duration::from_secs(30), - io_timeout: IO_TIMEOUT, - } - } - - pub fn apply_timeouts(&self, builder: ConfigBuilder) -> ConfigBuilder { - let timeout = Some(self.io_timeout); - builder - .timeout_connect(timeout) - .timeout_per_read(timeout) - .timeout_per_write(timeout) - } +struct Gates { + requests: Semaphore, + uploads: Semaphore, } struct Inner { - agent: Agent, + core: HttpCore, auth: Auth, - retry: RetryPolicy, - allow_invalid_certs: bool, - rate_limit: RateLimitState, - log_level: AtomicU8, - requests_gate: OnceLock, - uploads_gate: OnceLock, + gates: OnceLock, max_upload_bytes: AtomicU64, - retries_total: AtomicU64, - retry_after_sleeps: AtomicU64, } #[derive(Debug, Clone, Copy)] -enum Kind { +enum Lane { Api, Upload, } #[derive(Clone)] -pub struct HttpClient { +pub struct JmapClient { inner: Arc, } enum Attempt { - Ok { + Response { status: u16, body: Vec, retry_after: Option, - rate_limit_headers: Vec<(String, String)>, + rate_headers: RateHeaders, }, Transport(JmapError), } -impl HttpClient { - pub fn new(auth: Auth, retry: RetryPolicy, allow_invalid_certs: bool) -> Self { - let config: Config = retry - .apply_timeouts(Config::builder()) - .http_status_as_error(false) - .redirect_auth_headers(RedirectAuthHeaders::SameHost) - .tls_config( - TlsConfig::builder() - .unversioned_rustls_crypto_provider(std::sync::Arc::new( - rustls::crypto::aws_lc_rs::default_provider(), - )) - .root_certs(RootCerts::PlatformVerifier) - .disable_verification(allow_invalid_certs) - .build(), - ) - .build(); - HttpClient { +struct Exchange<'a> { + method: &'a str, + url: &'a str, + body: Option<&'a [u8]>, + content_type: Option<&'a str>, +} + +impl JmapClient { + pub fn new(auth: Auth, options: ClientOptions) -> JmapClient { + JmapClient { inner: Arc::new(Inner { - agent: config.new_agent(), + core: HttpCore::new(options, PROFILE), auth, - retry, - allow_invalid_certs, - rate_limit: RateLimitState::new(), - log_level: AtomicU8::new(LEVEL_DEFAULT), - requests_gate: OnceLock::new(), - uploads_gate: OnceLock::new(), + gates: OnceLock::new(), max_upload_bytes: AtomicU64::new(0), - retries_total: AtomicU64::new(0), - retry_after_sleeps: AtomicU64::new(0), }), } } pub fn set_limits(&self, limits: &Limits) { - let _ = self - .inner - .requests_gate - .set(Semaphore::new(limits.max_concurrent_requests)); - let _ = self - .inner - .uploads_gate - .set(Semaphore::new(limits.max_concurrent_upload)); + let _ = self.inner.gates.set(Gates { + requests: Semaphore::new(limits.max_concurrent_requests), + uploads: Semaphore::new(limits.max_concurrent_upload), + }); self.inner .max_upload_bytes .store(limits.max_size_upload, Ordering::Relaxed); } - pub fn retries_observed(&self) -> u64 { - self.inner.retries_total.load(Ordering::Relaxed) - } - - pub fn retry_after_sleeps(&self) -> u64 { - self.inner.retry_after_sleeps.load(Ordering::Relaxed) - } - - fn acquire_requests(&self) -> Option> { - self.inner.requests_gate.get().map(Semaphore::acquire) - } - - fn acquire_uploads(&self) -> Option> { - self.inner.uploads_gate.get().map(Semaphore::acquire) + pub fn stats(&self) -> RetryStats { + self.inner.core.stats().snapshot() } pub fn auth(&self) -> &Auth { @@ -183,44 +109,47 @@ impl HttpClient { } pub fn retry(&self) -> &RetryPolicy { - &self.inner.retry + self.inner.core.retry() } - pub fn allow_invalid_certs(&self) -> bool { - self.inner.allow_invalid_certs + pub fn rate_limit(&self) -> &RateLimit { + self.inner.core.rate_limit() } - pub fn rate_limit(&self) -> &RateLimitState { - &self.inner.rate_limit + pub fn logger(&self) -> Logger { + self.inner.core.logger() } - pub fn throttle_level(&self) -> u32 { - self.inner.rate_limit.level() - } - - pub fn set_logger(&self, logger: Logger) { - self.inner - .log_level - .store(logger.level(), Ordering::Relaxed); - } - - fn logger(&self) -> Logger { - Logger::new(self.inner.log_level.load(Ordering::Relaxed)) + pub fn backoff(&self, attempt: u32) -> Duration { + self.inner.core.backoff(attempt) } pub fn get(&self, url: &str) -> Result { - let body = self.execute(Kind::Api, "GET", url, None, None)?; + let body = self.execute( + Lane::Api, + &Exchange { + method: "GET", + url, + body: None, + content_type: None, + }, + )?; String::from_utf8(body).map_err(|e| JmapError::Malformed(format!("non-utf8 body: {e}"))) } pub fn post_json(&self, url: &str, body: &Value) -> Result { - let payload = serde_json::to_vec(body)?; + self.post_payload(url, &serde_json::to_vec(body)?) + } + + pub fn post_payload(&self, url: &str, payload: &[u8]) -> Result { let raw = self.execute( - Kind::Api, - "POST", - url, - Some(&payload), - Some("application/json"), + Lane::Api, + &Exchange { + method: "POST", + url, + body: Some(payload), + content_type: Some("application/json"), + }, )?; serde_json::from_slice(&raw) .map_err(|e| JmapError::Malformed(format!("response is not valid json: {e}"))) @@ -233,309 +162,275 @@ impl HttpClient { bytes: &[u8], ) -> Result { let cap = self.inner.max_upload_bytes.load(Ordering::Relaxed); - if cap > 0 && (bytes.len() as u64) > cap { + if cap > 0 && u64::try_from(bytes.len()).unwrap_or(u64::MAX) > cap { return Err(JmapError::SingleObjectTooLarge(format!( "blob upload of {} bytes exceeds maxSizeUpload ({cap})", bytes.len() ))); } let raw = self.execute( - Kind::Upload, - "POST", - upload_url, - Some(bytes), - Some(content_type), + Lane::Upload, + &Exchange { + method: "POST", + url: upload_url, + body: Some(bytes), + content_type: Some(content_type), + }, )?; serde_json::from_slice(&raw) .map_err(|e| JmapError::Malformed(format!("upload response is not valid json: {e}"))) } pub fn download(&self, url: &str) -> Result, JmapError> { - self.execute(Kind::Api, "GET", url, None, None) + self.execute( + Lane::Api, + &Exchange { + method: "GET", + url, + body: None, + content_type: None, + }, + ) } - fn execute( - &self, - kind: Kind, - method: &str, - url: &str, - body: Option<&[u8]>, - content_type: Option<&str>, - ) -> Result, JmapError> { - let policy = self.inner.retry; - let logger = self.logger(); + fn permits(&self, lane: Lane) -> (Option>, Option>) { + let Some(gates) = self.inner.gates.get() else { + return (None, None); + }; + let uploads = match lane { + Lane::Api => None, + Lane::Upload => Some(gates.uploads.acquire()), + }; + (Some(gates.requests.acquire()), uploads) + } + + fn execute(&self, lane: Lane, exchange: &Exchange<'_>) -> Result, JmapError> { + let core = &self.inner.core; + let max_retries = core.retry().max_retries; + let logger = core.logger(); let mut attempt: u32 = 0; loop { - self.inner.rate_limit.cooldown().wait(); - let attempt_outcome = { - let _r = self.acquire_requests(); - let _u = if matches!(kind, Kind::Upload) { - self.acquire_uploads() - } else { - None - }; - self.one_attempt(method, url, body, content_type) + core.rate_limit().cooldown().wait(); + let outcome = { + let _permits = self.permits(lane); + self.one_attempt(exchange) }; - match attempt_outcome { - Attempt::Ok { status, body, .. } if (200..300).contains(&status) => { - self.inner.rate_limit.on_success(); + match outcome { + Attempt::Response { status, body, .. } if (200..300).contains(&status) => { + core.rate_limit().on_success(); return Ok(body); } - Attempt::Ok { + Attempt::Response { status, body, retry_after, - rate_limit_headers, + rate_headers, } => { - let disposition = self.status_disposition(status, &body); - match disposition { + let problem = serde_json::from_slice::(&body).ok(); + match StatusOutcome::classify(status, problem.as_ref()) { StatusOutcome::Auth => { return Err(JmapError::Auth(format!( "server returned {status}: {}", - truncate(&body) + BodyExcerpt(&body) ))); } - StatusOutcome::RequestTooLarge => { - return Err(JmapError::RequestTooLarge); - } + StatusOutcome::RequestTooLarge => return Err(JmapError::RequestTooLarge), StatusOutcome::Fatal => { return Err(JmapError::HttpStatus { status, - body: truncate(&body), + body: BodyExcerpt(&body).to_string(), }); } - StatusOutcome::Retryable => { - attempt += 1; - self.inner.retries_total.fetch_add(1, Ordering::Relaxed); - let detail = problem_detail(&body); - let headers_blurb = format_rate_headers(&rate_limit_headers); - let reason = match (&detail, headers_blurb.as_deref()) { - (Some(d), Some(h)) => format!("http {status}: {d} [{h}]"), - (Some(d), None) => format!("http {status}: {d}"), - (None, Some(h)) => format!("http {status} [{h}]"), - (None, None) => format!("http {status}"), - }; - if attempt > policy.max_retries { - return Err(JmapError::RetriesExhausted(format!( - "{method} {url} kept returning {reason}" - ))); - } - if retry_after.is_some() { - self.inner - .retry_after_sleeps - .fetch_add(1, Ordering::Relaxed); - } - let delay = self.inner.rate_limit.on_throttle(&policy, retry_after); - if delay >= LONG_RETRY_THRESHOLD { - logger.warn(&format!( - "server rate-limited ({reason}); waiting {} before retry {}/{} (shared level {})", - format_retry_wait(delay), - attempt, - policy.max_retries, - self.inner.rate_limit.level(), - )); - } - self.log_retry( - &logger, - RetryLog { - method, - url, - attempt, - delay, - reason: &reason, - body: &body, - }, - ); - std::thread::sleep(delay); - } + StatusOutcome::Retryable => {} } + attempt += 1; + core.stats().note_retry(); + let reason = RetryReason { + status, + problem: problem.as_ref(), + rate_headers: &rate_headers, + } + .to_string(); + if attempt > max_retries { + return Err(JmapError::RetriesExhausted(format!( + "{} {} kept returning {reason}", + exchange.method, exchange.url + ))); + } + if retry_after.is_some() { + core.stats().note_retry_after(); + } + let delay = core.throttle(retry_after); + if delay >= LONG_RETRY_THRESHOLD { + logger.warn(format_args!( + "server rate-limited ({reason}); waiting {} before retry {attempt}/{max_retries} (shared level {})", + RetryWait(delay), + core.rate_limit().level(), + )); + } + self.log_retry(exchange, attempt, delay, &reason, &body); + thread::sleep(delay); } - Attempt::Transport(err) => match transport_disposition(&err) { - Disposition::Fatal => return Err(err), - Disposition::Retryable => { - attempt += 1; - self.inner.retries_total.fetch_add(1, Ordering::Relaxed); - if attempt > policy.max_retries { - return Err(JmapError::RetriesExhausted(format!( - "{method} {url}: {err}" - ))); - } - let delay = retry::backoff_delay(&policy, attempt); - if delay >= LONG_RETRY_THRESHOLD { - logger.warn(&format!( - "transient transport failure ({err}); waiting {} before retry {}/{}", - format_retry_wait(delay), - attempt, - policy.max_retries, - )); - } - self.log_retry( - &logger, - RetryLog { - method, - url, - attempt, - delay, - reason: &err.to_string(), - body: &[], - }, - ); - std::thread::sleep(delay); + Attempt::Transport(err) => { + if let JmapError::Connect(_) = err { + return Err(err); } - }, + attempt += 1; + core.stats().note_retry(); + if attempt > max_retries { + return Err(JmapError::RetriesExhausted(format!( + "{} {}: {err}", + exchange.method, exchange.url + ))); + } + let delay = core.backoff(attempt); + if delay >= LONG_RETRY_THRESHOLD { + logger.warn(format_args!( + "transient transport failure ({err}); waiting {} before retry {attempt}/{max_retries}", + RetryWait(delay), + )); + } + self.log_retry(exchange, attempt, delay, &err.to_string(), &[]); + thread::sleep(delay); + } } } } - fn one_attempt( - &self, - method: &str, - url: &str, - body: Option<&[u8]>, - content_type: Option<&str>, - ) -> Attempt { + fn one_attempt(&self, exchange: &Exchange<'_>) -> Attempt { + let core = &self.inner.core; let auth = self.inner.auth.header_value(); - let logger = self.logger(); + let logger = core.logger(); let started = Instant::now(); - let result = if let Some(payload) = body { - let mut req = self - .inner - .agent - .post(url) - .header("Authorization", auth) - .header("Accept", "application/json"); - if let Some(ct) = content_type { - req = req.header("Content-Type", ct); + let result = match exchange.body { + Some(payload) => { + let mut req = core + .agent() + .post(exchange.url) + .header("Authorization", auth) + .header("Accept", "application/json"); + if let Some(ct) = exchange.content_type { + req = req.header("Content-Type", ct); + } + req.send(payload) } - req.send(payload) - } else { - self.inner - .agent - .get(url) + None => core + .agent() + .get(exchange.url) .header("Authorization", auth) .header("Accept", "application/json") - .call() + .call(), }; - match result { - Ok(mut resp) => { - let status = resp.status().as_u16(); - let final_uri = resp.get_uri().clone(); - let response_type = resp - .headers() + let mut resp = match result { + Ok(resp) => resp, + Err(e) => { + let err = JmapError::from(TransportError::from(e)); + logger.trace_http_error( + PROTO, + exchange.method, + exchange.url, + &err.to_string(), + started.elapsed(), + ); + return Attempt::Transport(err); + } + }; + let status = resp.status().as_u16(); + let final_uri = resp.get_uri().clone(); + let headers = resp.headers(); + let response_type = logger + .enabled(Level::Bodies) + .then(|| { + headers .get("content-type") .and_then(|v| v.to_str().ok()) - .map(str::to_owned); - let retry_after = resp - .headers() - .get("retry-after") - .and_then(|v| v.to_str().ok()) - .and_then(retry_after_header); - let rate_limit_headers: Vec<(String, String)> = resp - .headers() - .iter() - .filter_map(|(name, value)| { - let lower = name.as_str().to_ascii_lowercase(); - if matches!( - lower.as_str(), - "retry-after" | "ratelimit" | "ratelimit-policy" | "content-type" - ) { - value.to_str().ok().map(|v| (lower, v.to_owned())) - } else { - None - } - }) - .collect(); - match resp.body_mut().with_config().limit(MAX_BODY).read_to_vec() { - Ok(bytes) => { - warn_on_redirect(&logger, method, url, &final_uri); - logger.trace_http(&HttpCall { - proto: "JMAP", - method, - url, - status, - elapsed: started.elapsed(), - note: None, - request: body, - request_type: content_type, - response: &bytes, - response_type: response_type.as_deref(), - }); - Attempt::Ok { - status, - body: bytes, - retry_after, - rate_limit_headers, - } - } - Err(e) => Attempt::Transport(JmapError::Transport(format!( - "reading response body: {e}" - ))), + .map(str::to_owned) + }) + .flatten(); + let retry_after = headers + .get("retry-after") + .and_then(|v| v.to_str().ok()) + .and_then(RetryAfter::parse_now); + let rate_headers = if (200..300).contains(&status) { + RateHeaders::default() + } else { + RateHeaders::collect(headers) + }; + match resp + .body_mut() + .with_config() + .limit(PROFILE.max_body) + .read_to_vec() + { + Ok(bytes) => { + self.warn_on_redirect(exchange, &final_uri); + logger.trace_http(&HttpCall { + proto: PROTO, + method: exchange.method, + url: exchange.url, + status, + elapsed: started.elapsed(), + note: None, + request: exchange.body, + request_type: exchange.content_type, + response: &bytes, + response_type: response_type.as_deref(), + }); + Attempt::Response { + status, + body: bytes, + retry_after, + rate_headers, } } Err(e) => { - let err = map_ureq_error(e); - logger.trace_http_error("JMAP", method, url, &err.to_string(), started.elapsed()); - Attempt::Transport(err) + Attempt::Transport(JmapError::Transport(format!("reading response body: {e}"))) } } } - fn status_disposition(&self, status: u16, body: &[u8]) -> StatusOutcome { - if status == 401 || status == 403 { - return StatusOutcome::Auth; + fn warn_on_redirect(&self, exchange: &Exchange<'_>, final_uri: &Uri) { + if final_uri == exchange.url { + return; } - if status == 413 { - return StatusOutcome::RequestTooLarge; - } - if let Ok(problem) = serde_json::from_slice::(body) - && let Some(kind) = problem.get("type").and_then(Value::as_str) - { - if kind.ends_with(":requestTooLarge") { - return StatusOutcome::RequestTooLarge; - } - if kind.ends_with(":limit") { - let which = problem.get("limit").and_then(Value::as_str).unwrap_or(""); - let subtype = problem.get("subType").and_then(Value::as_str).unwrap_or(""); - match which { - "maxSizeRequest" | "maxSizeUpload" => { - return StatusOutcome::RequestTooLarge; - } - "maxConcurrentRequests" | "maxConcurrentUpload" => { - return StatusOutcome::Retryable; - } - _ if subtype == "rateLimit" => { - return StatusOutcome::Retryable; - } - _ => {} - } - } - } - match retry::classify_http_status(status) { - Disposition::Retryable => StatusOutcome::Retryable, - Disposition::Fatal => StatusOutcome::Fatal, + let landed = final_uri.to_string(); + if HttpCore::is_cross_host(exchange.url, &landed) { + self.logger().warn(format_args!( + "{} {} was redirected across hosts to {landed}; a \ + JMAP endpoint must not redirect: verify the server's advertised base \ + URL and that the request host matches its configured hostname", + exchange.method, exchange.url + )); } } - fn log_retry(&self, logger: &Logger, r: RetryLog<'_>) { - if logger.enabled(LEVEL_BODIES) { - let snippet = if r.body.is_empty() { - String::new() + fn log_retry( + &self, + exchange: &Exchange<'_>, + attempt: u32, + delay: Duration, + reason: &str, + body: &[u8], + ) { + let logger = self.logger(); + let max_retries = self.retry().max_retries; + if logger.enabled(Level::Bodies) { + let (sep, excerpt) = if body.is_empty() { + ("", BodyExcerpt(&[])) } else { - format!(" body={}", truncate(r.body)) + (" body=", BodyExcerpt(body)) }; - eprintln!( - "retry {}/{} {} {} after {:?} ({}){snippet}", - r.attempt, self.inner.retry.max_retries, r.method, r.url, r.delay, r.reason, - ); - } else if logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "retry {}/{} ({})", - r.attempt, self.inner.retry.max_retries, r.reason - ); + logger.info(format_args!( + "retry {attempt}/{max_retries} {} {} after {delay:?} ({reason}){sep}{excerpt}", + exchange.method, exchange.url, + )); + } else { + logger.progress(format_args!("retry {attempt}/{max_retries} ({reason})")); } } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] enum StatusOutcome { Retryable, Fatal, @@ -543,121 +438,149 @@ enum StatusOutcome { RequestTooLarge, } -struct RetryLog<'a> { - method: &'a str, - url: &'a str, - attempt: u32, - delay: Duration, - reason: &'a str, - body: &'a [u8], -} - -fn warn_on_redirect(logger: &Logger, method: &str, requested: &str, final_uri: &Uri) { - let landed = final_uri.to_string(); - if cross_host(requested, &landed) { - logger.warn(&format!( - "{method} {requested} was redirected across hosts to {landed}; a \ - JMAP endpoint must not redirect: verify the server's advertised base \ - URL and that the request host matches its configured hostname" - )); - } -} - -pub fn cross_host(requested: &str, landed: &str) -> bool { - match (url::Url::parse(requested), url::Url::parse(landed)) { - (Ok(a), Ok(b)) => { - a.host_str() != b.host_str() || a.port_or_known_default() != b.port_or_known_default() +impl StatusOutcome { + fn classify(status: u16, problem: Option<&Value>) -> StatusOutcome { + if status == STATUS_UNAUTHORIZED || status == STATUS_FORBIDDEN { + return StatusOutcome::Auth; } - _ => false, - } -} - -fn transport_disposition(err: &JmapError) -> Disposition { - match err { - JmapError::Connect(_) => Disposition::Fatal, - _ => Disposition::Retryable, - } -} - -fn map_ureq_error(err: ureq::Error) -> JmapError { - match err { - ureq::Error::Io(e) => JmapError::Transport(format!("io: {e}")), - ureq::Error::Timeout(t) => JmapError::Transport(format!("timeout: {t}")), - ureq::Error::HostNotFound => JmapError::Transport("host not found".to_owned()), - ureq::Error::ConnectionFailed => JmapError::Transport("connection failed".to_owned()), - ureq::Error::BodyStalled => JmapError::Transport("body stalled".to_owned()), - ureq::Error::Tls(m) => JmapError::Transport(format!("tls: {m}")), - ureq::Error::TooManyRedirects => JmapError::Connect("too many redirects".to_owned()), - ureq::Error::RedirectFailed => JmapError::Connect("redirect failed".to_owned()), - ureq::Error::TlsRequired => { - JmapError::Connect("server requires TLS but transport is unsecured".to_owned()) + if status == STATUS_PAYLOAD_TOO_LARGE { + return StatusOutcome::RequestTooLarge; + } + if let Some(problem) = problem + && let Some(kind) = problem.get("type").and_then(Value::as_str) + { + if kind.ends_with(":requestTooLarge") { + return StatusOutcome::RequestTooLarge; + } + if kind.ends_with(":limit") { + let which = problem + .get("limit") + .and_then(Value::as_str) + .unwrap_or_default(); + if let Some(outcome) = hashify::map!( + which.as_bytes(), + StatusOutcome, + "maxSizeRequest" => StatusOutcome::RequestTooLarge, + "maxSizeUpload" => StatusOutcome::RequestTooLarge, + "maxConcurrentRequests" => StatusOutcome::Retryable, + "maxConcurrentUpload" => StatusOutcome::Retryable, + ) { + return *outcome; + } + if problem.get("subType").and_then(Value::as_str) == Some("rateLimit") { + return StatusOutcome::Retryable; + } + } + } + match Disposition::of_status(status) { + Disposition::Retry => StatusOutcome::Retryable, + Disposition::Fatal => StatusOutcome::Fatal, } - other => JmapError::Connect(other.to_string()), } } -fn truncate(body: &[u8]) -> String { - let text = String::from_utf8_lossy(body); - if text.len() <= 512 { - return text.into_owned(); - } - let end = text.floor_char_boundary(512); - format!("{}...", &text[..end]) -} +#[derive(Default)] +struct RateHeaders(Vec); -pub fn retry_after_header(value: &str) -> Option { - retry::parse_retry_after(value, SystemTime::now()) -} - -fn problem_detail(body: &[u8]) -> Option { - let value: Value = serde_json::from_slice(body).ok()?; - let title = value.get("title").and_then(Value::as_str); - let detail = value.get("detail").and_then(Value::as_str); - match (title, detail) { - (Some(t), Some(d)) => Some(format!("{t}: {d}")), - (Some(t), None) => Some(t.to_owned()), - (None, Some(d)) => Some(d.to_owned()), - (None, None) => None, +impl RateHeaders { + fn collect(headers: &HeaderMap) -> RateHeaders { + let mut pairs: Vec = headers + .iter() + .filter(|(name, _)| { + hashify::set!( + name.as_str().as_bytes(), + "retry-after", + "ratelimit", + "ratelimit-policy", + ) + }) + .filter_map(|(name, value)| { + value + .to_str() + .ok() + .map(|value| format!("{}={value}", name.as_str())) + }) + .collect(); + pairs.sort_unstable(); + RateHeaders(pairs) } } -fn format_rate_headers(headers: &[(String, String)]) -> Option { - if headers.is_empty() { - return None; +impl fmt::Display for RateHeaders { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut parts = self.0.iter(); + if let Some(first) = parts.next() { + f.write_str(first)?; + } + for part in parts { + f.write_str("; ")?; + f.write_str(part)?; + } + Ok(()) } - let mut parts: Vec = headers - .iter() - .filter(|(name, _)| name != "content-type") - .map(|(name, value)| format!("{name}={value}")) - .collect(); - if parts.is_empty() { - return None; - } - parts.sort(); - Some(parts.join("; ")) } -pub fn format_retry_wait(d: Duration) -> String { - let total = d.as_secs(); - if total == 0 { - format!("{}ms", d.as_millis()) - } else if total < 60 { - format!("{total}s") - } else if total < 3600 { - format!("{}m{:02}s", total / 60, total % 60) - } else { - format!( - "{}h{:02}m{:02}s", - total / 3600, - (total % 3600) / 60, - total % 60 - ) +struct ProblemDetail<'a>(&'a Value); + +impl ProblemDetail<'_> { + fn of(problem: Option<&Value>) -> Option> { + let problem = problem?; + (problem.get("title").and_then(Value::as_str).is_some() + || problem.get("detail").and_then(Value::as_str).is_some()) + .then_some(ProblemDetail(problem)) + } +} + +impl fmt::Display for ProblemDetail<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let title = self.0.get("title").and_then(Value::as_str); + let detail = self.0.get("detail").and_then(Value::as_str); + match (title, detail) { + (Some(t), Some(d)) => write!(f, "{t}: {d}"), + (Some(t), None) => f.write_str(t), + (None, Some(d)) => f.write_str(d), + (None, None) => Ok(()), + } + } +} + +struct RetryReason<'a> { + status: u16, + problem: Option<&'a Value>, + rate_headers: &'a RateHeaders, +} + +impl fmt::Display for RetryReason<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "http {}", self.status)?; + if let Some(detail) = ProblemDetail::of(self.problem) { + write!(f, ": {detail}")?; + } + if !self.rate_headers.0.is_empty() { + write!(f, " [{}]", self.rate_headers)?; + } + Ok(()) } } #[cfg(test)] mod tests { use super::*; + use crate::logging::Level; + use crate::net::http::TlsPolicy; + + fn client(retry: RetryPolicy) -> JmapClient { + JmapClient::new( + Auth::Bearer { + token: "t".to_owned(), + }, + ClientOptions { + retry, + tls: TlsPolicy::Verify, + logger: Logger::new(Level::Default), + }, + ) + } #[test] fn basic_header_matches_rfc7617_example() { @@ -684,13 +607,7 @@ mod tests { let holder = std::thread::spawn(move || listener.accept().map(|(stream, _)| stream)); let mut retry = RetryPolicy::new(0); retry.io_timeout = Duration::from_millis(200); - let client = HttpClient::new( - Auth::Bearer { - token: "t".to_owned(), - }, - retry, - false, - ); + let client = client(retry); let started = Instant::now(); let err = client.get(&url).unwrap_err(); assert!( @@ -703,7 +620,7 @@ mod tests { #[test] fn cross_host_detects_hostname_change() { - assert!(cross_host( + assert!(HttpCore::is_cross_host( "https://mail.example.com/jmap", "https://login.example.net/sso" )); @@ -711,7 +628,7 @@ mod tests { #[test] fn cross_host_ignores_same_host_path_change() { - assert!(!cross_host( + assert!(!HttpCore::is_cross_host( "https://mail.example.com/jmap", "https://mail.example.com/jmap/api" )); @@ -719,75 +636,36 @@ mod tests { #[test] fn cross_host_flags_port_change() { - assert!(cross_host( + assert!(HttpCore::is_cross_host( "https://mail.example.com/jmap", "https://mail.example.com:8443/jmap" )); } - #[test] - fn truncate_does_not_panic_on_multibyte_boundary() { - let mut body = vec![b'a'; 510]; - body.extend_from_slice("\u{1F4A9}".as_bytes()); - body.extend_from_slice(&[b'b'; 600]); - let s = truncate(&body); - assert!(s.ends_with("...")); - assert!(s.is_char_boundary(s.len())); - let inner = s.trim_end_matches('.'); - assert!(inner.is_char_boundary(inner.len())); - } - - #[test] - fn truncate_short_body_returns_verbatim() { - let s = truncate("héllo".as_bytes()); - assert_eq!(s, "héllo"); - } - #[test] fn limit_problem_unknown_kind_on_429_is_retried() { - let client = HttpClient::new( - Auth::Bearer { - token: "t".to_owned(), - }, - RetryPolicy::new(0), - false, - ); let body = br#"{"type":"urn:ietf:params:jmap:error:limit","limit":"someServerLimit"}"#; assert!(matches!( - client.status_disposition(429, body), + StatusOutcome::classify(429, serde_json::from_slice(body).ok().as_ref()), StatusOutcome::Retryable )); } #[test] fn limit_problem_max_size_request_is_resplit() { - let client = HttpClient::new( - Auth::Bearer { - token: "t".to_owned(), - }, - RetryPolicy::new(0), - false, - ); let body = br#"{"type":"urn:ietf:params:jmap:error:limit","limit":"maxSizeRequest"}"#; assert!(matches!( - client.status_disposition(400, body), + StatusOutcome::classify(400, serde_json::from_slice(body).ok().as_ref()), StatusOutcome::RequestTooLarge )); } #[test] fn limit_problem_max_concurrent_is_retried() { - let client = HttpClient::new( - Auth::Bearer { - token: "t".to_owned(), - }, - RetryPolicy::new(0), - false, - ); let body = br#"{"type":"urn:ietf:params:jmap:error:limit","limit":"maxConcurrentRequests"}"#; assert!(matches!( - client.status_disposition(429, body), + StatusOutcome::classify(429, serde_json::from_slice(body).ok().as_ref()), StatusOutcome::Retryable )); } @@ -806,13 +684,7 @@ mod tests { #[test] fn upload_rejects_oversize_before_sending() { - let client = HttpClient::new( - Auth::Bearer { - token: "t".to_owned(), - }, - RetryPolicy::new(0), - false, - ); + let client = client(RetryPolicy::new(0)); client.set_limits(&limits_with(10, 4, 4)); let err = client .upload( @@ -825,18 +697,12 @@ mod tests { matches!(err, JmapError::SingleObjectTooLarge(_)), "got {err:?}" ); - assert_eq!(client.retries_observed(), 0); + assert_eq!(client.stats().retries_observed, 0); } #[test] fn upload_within_max_size_is_not_short_circuited() { - let client = HttpClient::new( - Auth::Bearer { - token: "t".to_owned(), - }, - RetryPolicy::new(0), - false, - ); + let client = client(RetryPolicy::new(0)); client.set_limits(&limits_with(1024, 4, 4)); let err = client .upload( @@ -854,7 +720,9 @@ mod tests { #[test] fn problem_detail_extracts_title_and_detail() { let body = br#"{"type":"about:blank","status":429,"title":"Quota exceeded","detail":"You have exceeded the blob upload quota of 1000 files or 50000000 bytes."}"#; - let got = problem_detail(body).expect("title+detail"); + let got = ProblemDetail::of(serde_json::from_slice::(body).ok().as_ref()) + .map(|d| d.to_string()) + .expect("title+detail"); assert!(got.contains("Quota exceeded")); assert!(got.contains("50000000 bytes")); } @@ -862,34 +730,31 @@ mod tests { #[test] fn problem_detail_title_only() { let body = br#"{"title":"Too Many Requests"}"#; - assert_eq!(problem_detail(body).as_deref(), Some("Too Many Requests")); + assert_eq!( + ProblemDetail::of(serde_json::from_slice::(body).ok().as_ref()) + .map(|d| d.to_string()) + .as_deref(), + Some("Too Many Requests") + ); } #[test] fn problem_detail_returns_none_for_empty_body() { - assert_eq!(problem_detail(b"{}"), None); - assert_eq!(problem_detail(b"not json"), None); - } - - #[test] - fn format_retry_wait_short_medium_long() { - assert_eq!(format_retry_wait(Duration::from_millis(250)), "250ms"); - assert_eq!(format_retry_wait(Duration::from_secs(7)), "7s"); - assert_eq!(format_retry_wait(Duration::from_secs(67)), "1m07s"); - assert_eq!(format_retry_wait(Duration::from_secs(1427)), "23m47s"); - assert_eq!(format_retry_wait(Duration::from_secs(3661)), "1h01m01s"); + assert_eq!( + ProblemDetail::of(serde_json::from_slice::(b"{}").ok().as_ref()) + .map(|d| d.to_string()), + None + ); + assert_eq!( + ProblemDetail::of(serde_json::from_slice::(b"not json").ok().as_ref()) + .map(|d| d.to_string()), + None + ); } #[test] fn counters_start_at_zero() { - let client = HttpClient::new( - Auth::Bearer { - token: "t".to_owned(), - }, - RetryPolicy::new(0), - false, - ); - assert_eq!(client.retries_observed(), 0); - assert_eq!(client.retry_after_sleeps(), 0); + let client = client(RetryPolicy::new(0)); + assert_eq!(client.stats(), RetryStats::default()); } } diff --git a/src/jmap/inflight.rs b/src/jmap/inflight.rs index 67fb001..22d1572 100644 --- a/src/jmap/inflight.rs +++ b/src/jmap/inflight.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use std::sync::{Condvar, Mutex}; +use std::sync::{Condvar, Mutex, MutexGuard, PoisonError}; pub struct Semaphore { state: Mutex, @@ -20,21 +20,26 @@ impl Semaphore { } pub fn acquire(&self) -> Permit<'_> { - let mut guard = lock(&self.state); - loop { - if *guard > 0 { - *guard -= 1; - return Permit { sem: self }; - } - guard = wait(&self.cond, guard); + let mut guard = self.lock(); + while *guard == 0 { + guard = self + .cond + .wait(guard) + .unwrap_or_else(PoisonError::into_inner); } + *guard -= 1; + Permit { sem: self } } fn release(&self) { - let mut guard = lock(&self.state); + let mut guard = self.lock(); *guard = guard.saturating_add(1); self.cond.notify_one(); } + + fn lock(&self) -> MutexGuard<'_, u64> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } } pub struct Permit<'a> { @@ -47,20 +52,6 @@ impl Drop for Permit<'_> { } } -fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, u64> { - match m.lock() { - Ok(g) => g, - Err(p) => p.into_inner(), - } -} - -fn wait<'a>(cv: &Condvar, g: std::sync::MutexGuard<'a, u64>) -> std::sync::MutexGuard<'a, u64> { - match cv.wait(g) { - Ok(g) => g, - Err(p) => p.into_inner(), - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/jmap/mod.rs b/src/jmap/mod.rs index c99cd74..9ad5f66 100644 --- a/src/jmap/mod.rs +++ b/src/jmap/mod.rs @@ -12,6 +12,5 @@ pub mod error; pub mod http; pub mod inflight; pub mod request; -pub mod retry; pub mod session; pub mod wire; diff --git a/src/jmap/request.rs b/src/jmap/request.rs index cdd514e..775001e 100644 --- a/src/jmap/request.rs +++ b/src/jmap/request.rs @@ -4,34 +4,42 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +use std::thread; + use indexmap::IndexSet; use serde::de::DeserializeOwned; -use serde_json::{Map, Value, json}; +use serde::ser::{Serialize, SerializeMap, Serializer}; +use serde_json::{Map, Value}; +use crate::jmap::connect::JmapAccount; use crate::jmap::error::JmapError; -use crate::jmap::http::{HttpClient, format_retry_wait}; -use crate::jmap::retry::{Disposition, MethodCallKind, backoff_delay, jmap_method_disposition}; +use crate::jmap::http::JmapClient; use crate::jmap::session::Limits; use crate::jmap::wire::JmapId; -use crate::logging::Logger; +use crate::net::retry::{Disposition, RetryWait}; pub const URN_CORE: &str = "urn:ietf:params:jmap:core"; pub const URN_VACATION_RESPONSE: &str = "urn:ietf:params:jmap:vacationresponse"; -pub fn using_urn(method: &str) -> &'static str { - let prefix = method.split('/').next().unwrap_or(method); - match prefix { - "Mailbox" | "Email" => "urn:ietf:params:jmap:mail", - "Identity" => "urn:ietf:params:jmap:submission", - "SieveScript" => "urn:ietf:params:jmap:sieve", - "AddressBook" | "ContactCard" => "urn:ietf:params:jmap:contacts", - "Calendar" | "CalendarEvent" | "ParticipantIdentity" => "urn:ietf:params:jmap:calendars", - "FileNode" => "urn:ietf:params:jmap:filenode", - "Principal" => "urn:ietf:params:jmap:principals", - "VacationResponse" => URN_VACATION_RESPONSE, - "x:Account" | "x:Domain" => "urn:stalwart:jmap", - _ => URN_CORE, +const ANCHOR_RESTARTS: u32 = 2; + +type CapabilityUrn = &'static str; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MethodCallKind { + Read, + SingleObjectWrite, +} + +impl MethodCallKind { + pub fn disposition(self, error_type: &str) -> Disposition { + match (error_type, self) { + ("serverUnavailable", _) | ("serverPartialFail", MethodCallKind::Read) => { + Disposition::Retry + } + _ => Disposition::Fatal, + } } } @@ -42,11 +50,140 @@ pub struct MethodCall { pub call_id: String, } +impl Serialize for MethodCall { + fn serialize(&self, serializer: S) -> Result { + (&self.name, &self.args, &self.call_id).serialize(serializer) + } +} + +impl MethodCall { + pub fn check_error(&self) -> Result<(), JmapError> { + if self.name != "error" { + return Ok(()); + } + let error_type = self + .args + .get("type") + .and_then(Value::as_str) + .unwrap_or("unknown"); + if let Some(known) = hashify::map!( + error_type.as_bytes(), + KnownMethodError, + "anchorNotFound" => KnownMethodError::AnchorNotFound, + "cannotCalculateChanges" => KnownMethodError::CannotCalculateChanges, + "unknownMethod" => KnownMethodError::UnknownMethod, + ) { + return Err(known.error()); + } + Err(JmapError::Method { + call_id: self.call_id.clone(), + error_type: error_type.to_owned(), + description: self + .args + .get("description") + .and_then(Value::as_str) + .map(str::to_owned), + }) + } + + fn decode_get(&self, out: &mut GetResult) -> Result<(), JmapError> { + let list = self + .args + .get("list") + .and_then(Value::as_array) + .ok_or_else(|| JmapError::malformed("get response has no list array"))?; + out.list.reserve(list.len()); + for item in list { + out.list.push(T::deserialize(item)?); + } + if let Some(not_found) = self.args.get("notFound").and_then(Value::as_array) { + out.not_found.extend( + not_found + .iter() + .filter_map(Value::as_str) + .map(|s| JmapId(s.to_owned())), + ); + } + if let Some(state) = self.args.get("state").and_then(Value::as_str) { + out.state = Some(state.to_owned()); + } + Ok(()) + } + + fn decode_set(&self) -> SetOutcome { + let pairs = |key: &str| -> Vec<(String, Value)> { + self.args + .get(key) + .and_then(Value::as_object) + .map(|m| m.iter().map(|(k, v)| (k.clone(), v.clone())).collect()) + .unwrap_or_default() + }; + SetOutcome { + created: pairs("created"), + updated: self + .args + .get("updated") + .and_then(Value::as_object) + .map(|m| m.keys().cloned().collect()) + .unwrap_or_default(), + destroyed: self + .args + .get("destroyed") + .and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(|v| v.as_str().map(str::to_owned)) + .collect() + }) + .unwrap_or_default(), + not_created: pairs("notCreated"), + not_updated: pairs("notUpdated"), + not_destroyed: pairs("notDestroyed"), + } + } + + fn append_ids(&self, key: &str, out: &mut Vec) { + if let Some(arr) = self.args.get(key).and_then(Value::as_array) { + out.extend( + arr.iter() + .filter_map(Value::as_str) + .map(|s| JmapId(s.to_owned())), + ); + } + } +} + +#[derive(Clone, Copy)] +enum KnownMethodError { + AnchorNotFound, + CannotCalculateChanges, + UnknownMethod, +} + +impl KnownMethodError { + fn error(self) -> JmapError { + match self { + KnownMethodError::AnchorNotFound => JmapError::AnchorNotFound, + KnownMethodError::CannotCalculateChanges => JmapError::CannotCalculateChanges, + KnownMethodError::UnknownMethod => JmapError::UnknownMethod, + } + } +} + #[derive(Debug, Clone, Default)] pub struct Request { calls: Vec, } +impl Serialize for Request { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(Some(2))?; + map.serialize_entry("methodCalls", &self.calls)?; + map.serialize_entry("using", &self.using())?; + map.end() + } +} + impl Request { pub fn new() -> Request { Request { calls: Vec::new() } @@ -74,38 +211,66 @@ impl Request { self.calls.is_empty() } - pub fn using(&self) -> Vec { - let mut set: IndexSet = IndexSet::new(); - set.insert(URN_CORE.to_owned()); - for c in &self.calls { - set.insert(using_urn(&c.name).to_owned()); + pub fn using(&self) -> IndexSet<&'static str> { + let mut set = IndexSet::with_capacity(self.calls.len() + 1); + set.insert(URN_CORE); + for call in &self.calls { + set.insert(Request::capability_of(&call.name)); } - set.into_iter().collect() + set } - fn envelope(&self) -> Value { - let method_calls: Vec = self - .calls - .iter() - .map(|c| json!([c.name, c.args, c.call_id])) - .collect(); - json!({ "using": self.using(), "methodCalls": method_calls }) + pub fn capability_of(method: &str) -> &'static str { + let prefix = method.split_once('/').map_or(method, |(prefix, _)| prefix); + hashify::map!( + prefix.as_bytes(), + CapabilityUrn, + "Mailbox" => "urn:ietf:params:jmap:mail", + "Email" => "urn:ietf:params:jmap:mail", + "Identity" => "urn:ietf:params:jmap:submission", + "SieveScript" => "urn:ietf:params:jmap:sieve", + "AddressBook" => "urn:ietf:params:jmap:contacts", + "ContactCard" => "urn:ietf:params:jmap:contacts", + "Calendar" => "urn:ietf:params:jmap:calendars", + "CalendarEvent" => "urn:ietf:params:jmap:calendars", + "ParticipantIdentity" => "urn:ietf:params:jmap:calendars", + "FileNode" => "urn:ietf:params:jmap:filenode", + "Principal" => "urn:ietf:params:jmap:principals", + "VacationResponse" => URN_VACATION_RESPONSE, + "x:Account" => "urn:stalwart:jmap", + "x:Domain" => "urn:stalwart:jmap", + ) + .copied() + .unwrap_or(URN_CORE) + } + + fn encode(&self, limits: Option<&Limits>) -> Result, JmapError> { + if let Some(limits) = limits + && u64::try_from(self.calls.len()).unwrap_or(u64::MAX) > limits.max_calls_in_request + { + return Err(JmapError::RequestTooLarge); + } + let payload = serde_json::to_vec(self)?; + if let Some(limits) = limits + && u64::try_from(payload.len()).unwrap_or(u64::MAX) > limits.max_size_request + { + return Err(JmapError::RequestTooLarge); + } + Ok(payload) } pub fn fits(&self, limits: &Limits) -> Result<(), JmapError> { - if self.calls.len() as u64 > limits.max_calls_in_request { - return Err(JmapError::RequestTooLarge); - } - let size = serde_json::to_vec(&self.envelope())?.len() as u64; - if size > limits.max_size_request { - return Err(JmapError::RequestTooLarge); - } - Ok(()) + self.encode(Some(limits)).map(drop) } - pub fn send(&self, client: &HttpClient, api_url: &str) -> Result { - let value = client.post_json(api_url, &self.envelope())?; - Response::parse(value) + pub fn send(&self, client: &JmapClient, api_url: &str) -> Result { + let payload = self.encode(None)?; + Response::parse(client.post_payload(api_url, &payload)?) + } + + fn send_within(&self, account: &JmapAccount, limits: &Limits) -> Result { + let payload = self.encode(Some(limits))?; + Response::parse(account.client().post_payload(account.api_url(), &payload)?) } } @@ -115,33 +280,37 @@ pub struct Response { } impl Response { - fn parse(value: Value) -> Result { - let arr = value - .get("methodResponses") - .and_then(Value::as_array) - .ok_or_else(|| JmapError::malformed("response has no methodResponses array"))?; - let mut out = Vec::with_capacity(arr.len()); - for entry in arr { - let triple = entry.as_array().filter(|t| t.len() == 3).ok_or_else(|| { - JmapError::malformed("methodResponse is not a [name,args,id] triple") - })?; - let name = triple[0] - .as_str() - .ok_or_else(|| JmapError::malformed("methodResponse name is not a string"))? - .to_owned(); - let call_id = triple[2] - .as_str() - .ok_or_else(|| JmapError::malformed("methodResponse callId is not a string"))? - .to_owned(); - out.push(MethodCall { + fn parse(mut value: Value) -> Result { + let Some(Value::Array(entries)) = value.get_mut("methodResponses").map(Value::take) else { + return Err(JmapError::malformed( + "response has no methodResponses array", + )); + }; + let mut method_responses = Vec::with_capacity(entries.len()); + for entry in entries { + let Ok([name, args, call_id]) = <[Value; 3]>::try_from(match entry { + Value::Array(triple) => triple, + _ => Vec::new(), + }) else { + return Err(JmapError::malformed( + "methodResponse is not a [name,args,id] triple", + )); + }; + let Value::String(name) = name else { + return Err(JmapError::malformed("methodResponse name is not a string")); + }; + let Value::String(call_id) = call_id else { + return Err(JmapError::malformed( + "methodResponse callId is not a string", + )); + }; + method_responses.push(MethodCall { name, - args: triple[1].clone(), + args, call_id, }); } - Ok(Response { - method_responses: out, - }) + Ok(Response { method_responses }) } pub fn first(&self) -> Result<&MethodCall, JmapError> { @@ -158,156 +327,40 @@ impl Response { } } -pub fn check_method_error(mr: &MethodCall) -> Result<(), JmapError> { - if mr.name != "error" { - return Ok(()); - } - let error_type = mr - .args - .get("type") - .and_then(Value::as_str) - .unwrap_or("unknown") - .to_owned(); - if error_type == "anchorNotFound" { - return Err(JmapError::AnchorNotFound); - } - if error_type == "cannotCalculateChanges" { - return Err(JmapError::CannotCalculateChanges); - } - if error_type == "unknownMethod" { - return Err(JmapError::UnknownMethod); - } - let description = mr - .args - .get("description") - .and_then(Value::as_str) - .map(str::to_owned); - Err(JmapError::Method { - call_id: mr.call_id.clone(), - error_type, - description, - }) -} - -fn method_error_is_retryable(err: &JmapError, kind: MethodCallKind) -> bool { - match err { - JmapError::Method { error_type, .. } => { - jmap_method_disposition(error_type, kind) == Disposition::Retryable - } - _ => false, - } -} - -pub fn retry_method_call( - client: &HttpClient, - kind: MethodCallKind, - logger: &Logger, - mut attempt: impl FnMut() -> Result, -) -> Result { - let policy = *client.retry(); - let mut tries = 0u32; - loop { - let err = match attempt() { - Ok(value) => return Ok(value), - Err(e) => e, - }; - if tries >= policy.max_retries || !method_error_is_retryable(&err, kind) { - return Err(err); - } - tries += 1; - let delay = backoff_delay(&policy, tries); - client.rate_limit().cooldown().arm(delay); - logger.warn(&format!( - "{err}; retrying in {} ({tries}/{})", - format_retry_wait(delay), - policy.max_retries - )); - std::thread::sleep(delay); - } -} - -fn ids_array(ids: &[JmapId]) -> Value { - Value::Array(ids.iter().map(|i| Value::String(i.0.clone())).collect()) -} - -pub fn query_all_ids( - client: &HttpClient, - api_url: &str, - account_id: &str, - type_name: &str, - limits: &Limits, -) -> Result, JmapError> { - let mut restarts = 0u32; - loop { - match query_pages(client, api_url, account_id, type_name, limits) { - Ok(ids) => return Ok(ids), - Err(JmapError::AnchorNotFound) if restarts < 2 => { - restarts += 1; +impl JmapClient { + pub fn retry_method( + &self, + kind: MethodCallKind, + mut attempt: impl FnMut() -> Result, + ) -> Result { + let policy = *self.retry(); + let mut tries = 0u32; + loop { + let err = match attempt() { + Ok(value) => return Ok(value), + Err(e) => e, + }; + let retryable = matches!( + &err, + JmapError::Method { error_type, .. } + if kind.disposition(error_type) == Disposition::Retry + ); + if tries >= policy.max_retries || !retryable { + return Err(err); } - Err(e) => return Err(e), + tries += 1; + let delay = self.backoff(tries); + self.rate_limit().cooldown().arm(delay); + self.logger().warn(format_args!( + "{err}; retrying in {} ({tries}/{})", + RetryWait(delay), + policy.max_retries + )); + thread::sleep(delay); } } } -fn query_pages( - client: &HttpClient, - api_url: &str, - account_id: &str, - type_name: &str, - limits: &Limits, -) -> Result, JmapError> { - let limit = limits.max_objects_in_get.max(1); - let mut collected: Vec = Vec::new(); - let mut page_size: Option = None; - let mut anchor: Option = None; - - loop { - let mut args = Map::new(); - args.insert("accountId".to_owned(), Value::String(account_id.to_owned())); - args.insert("limit".to_owned(), Value::from(limit)); - if let Some(a) = &anchor { - args.insert("anchor".to_owned(), Value::String(a.clone())); - args.insert("anchorOffset".to_owned(), Value::from(1)); - } - let mut req = Request::new(); - req.call(format!("{type_name}/query"), Value::Object(args), "q"); - let resp = req.send(client, api_url)?; - let mr = resp.first()?; - check_method_error(mr)?; - let ids = mr - .args - .get("ids") - .and_then(Value::as_array) - .ok_or_else(|| JmapError::malformed("query response has no ids array"))?; - let this_len = ids.len(); - for v in ids { - let s = v - .as_str() - .ok_or_else(|| JmapError::malformed("query id is not a string"))?; - collected.push(JmapId(s.to_owned())); - } - match page_size { - None => { - if this_len == 0 { - break; - } - page_size = Some(this_len); - } - Some(full) => { - if this_len == 0 || this_len < full { - break; - } - } - } - let last = collected - .last() - .map(|i| i.0.clone()) - .ok_or_else(|| JmapError::malformed("non-empty page yielded no anchor"))?; - anchor = Some(last); - } - Ok(collected) -} - #[derive(Debug)] pub struct GetResult { pub list: Vec, @@ -325,131 +378,6 @@ impl Default for GetResult { } } -struct GetCtx<'a> { - client: &'a HttpClient, - api_url: &'a str, - account_id: &'a str, - type_name: &'a str, - properties: Option<&'a [&'a str]>, - limits: &'a Limits, -} - -pub fn get_objects( - client: &HttpClient, - api_url: &str, - account_id: &str, - type_name: &str, - ids: &[JmapId], - properties: Option<&[&str]>, - limits: &Limits, -) -> Result, JmapError> { - let ctx = GetCtx { - client, - api_url, - account_id, - type_name, - properties, - limits, - }; - let mut out = GetResult::default(); - let chunk = limits.max_objects_in_get.max(1) as usize; - let mut start = 0; - while start < ids.len() { - let end = (start + chunk).min(ids.len()); - get_chunk(&ctx, &ids[start..end], &mut out)?; - start = end; - } - Ok(out) -} - -fn get_chunk( - ctx: &GetCtx<'_>, - ids: &[JmapId], - out: &mut GetResult, -) -> Result<(), JmapError> { - let mut args = Map::new(); - args.insert( - "accountId".to_owned(), - Value::String(ctx.account_id.to_owned()), - ); - args.insert("ids".to_owned(), ids_array(ids)); - if let Some(props) = ctx.properties { - let mut p: Vec = props - .iter() - .map(|s| Value::String((*s).to_owned())) - .collect(); - p.push(Value::String("id".to_owned())); - args.insert("properties".to_owned(), Value::Array(p)); - } - let mut req = Request::new(); - req.call(format!("{}/get", ctx.type_name), Value::Object(args), "g"); - let send_result: Result<(), JmapError> = if req.fits(ctx.limits).is_err() { - Err(JmapError::RequestTooLarge) - } else { - match req.send(ctx.client, ctx.api_url) { - Ok(resp) => { - let mr = resp.first()?; - check_method_error(mr)?; - decode_get(mr, out) - } - Err(e) => Err(e), - } - }; - match send_result { - Ok(()) => Ok(()), - Err(JmapError::RequestTooLarge) => { - if ids.len() <= 1 { - return Err(JmapError::SingleObjectTooLarge(format!( - "{}/get of a single id exceeds maxSizeRequest", - ctx.type_name - ))); - } - let mid = ids.len() / 2; - get_chunk(ctx, &ids[..mid], out)?; - get_chunk(ctx, &ids[mid..], out) - } - Err(e) => Err(e), - } -} - -pub fn get_all( - client: &HttpClient, - api_url: &str, - account_id: &str, - type_name: &str, -) -> Result, JmapError> { - let mut args = Map::new(); - args.insert("accountId".to_owned(), Value::String(account_id.to_owned())); - args.insert("ids".to_owned(), Value::Null); - let mut req = Request::new(); - req.call(format!("{type_name}/get"), Value::Object(args), "g"); - let resp = req.send(client, api_url)?; - let mr = resp.first()?; - check_method_error(mr)?; - let mut out = GetResult::default(); - decode_get(mr, &mut out)?; - Ok(out) -} - -pub fn get_state( - client: &HttpClient, - api_url: &str, - account_id: &str, - type_name: &str, -) -> Result, JmapError> { - let mut args = Map::new(); - args.insert("accountId".to_owned(), Value::String(account_id.to_owned())); - args.insert("ids".to_owned(), Value::Array(Vec::new())); - let mut req = Request::new(); - req.call(format!("{type_name}/get"), Value::Object(args), "g"); - let resp = req.send(client, api_url)?; - let mr = resp.first()?; - check_method_error(mr)?; - let mut out: GetResult = GetResult::default(); - decode_get(mr, &mut out)?; - Ok(out.state) -} - #[derive(Debug, Default)] pub struct ChangesResult { pub created: Vec, @@ -458,95 +386,15 @@ pub struct ChangesResult { pub new_state: String, } -pub fn get_changes( - client: &HttpClient, - api_url: &str, - account_id: &str, - type_name: &str, - since_state: &str, - limits: &Limits, -) -> Result { - let mut out = ChangesResult::default(); - let mut since = since_state.to_owned(); - loop { - let mut args = Map::new(); - args.insert("accountId".to_owned(), Value::String(account_id.to_owned())); - args.insert("sinceState".to_owned(), Value::String(since.clone())); - args.insert( - "maxChanges".to_owned(), - Value::from(limits.max_objects_in_get.max(1)), - ); - let mut req = Request::new(); - req.call(format!("{type_name}/changes"), Value::Object(args), "c"); - let resp = req.send(client, api_url)?; - let mr = resp.first()?; - check_method_error(mr)?; - let new_state = mr - .args - .get("newState") - .and_then(Value::as_str) - .ok_or_else(|| JmapError::malformed("changes response has no newState"))? - .to_owned(); - append_id_array(&mr.args, "created", &mut out.created); - append_id_array(&mr.args, "updated", &mut out.updated); - append_id_array(&mr.args, "destroyed", &mut out.destroyed); - let has_more = mr - .args - .get("hasMoreChanges") - .and_then(Value::as_bool) - .unwrap_or(false); - out.new_state = new_state.clone(); - if !has_more || since == new_state { - break; - } - since = new_state; - } - dedup_ids(&mut out.created); - dedup_ids(&mut out.updated); - dedup_ids(&mut out.destroyed); - Ok(out) -} - -fn append_id_array(args: &Value, key: &str, out: &mut Vec) { - if let Some(arr) = args.get(key).and_then(Value::as_array) { - for v in arr { - if let Some(s) = v.as_str() { - out.push(JmapId(s.to_owned())); - } +impl ChangesResult { + fn dedup(&mut self) { + for ids in [&mut self.created, &mut self.updated, &mut self.destroyed] { + let mut seen = IndexSet::with_capacity(ids.len()); + ids.retain(|id| seen.insert(id.0.clone())); } } } -fn dedup_ids(ids: &mut Vec) { - let mut seen = IndexSet::new(); - ids.retain(|id| seen.insert(id.0.clone())); -} - -fn decode_get( - mr: &MethodCall, - out: &mut GetResult, -) -> Result<(), JmapError> { - let list = mr - .args - .get("list") - .and_then(Value::as_array) - .ok_or_else(|| JmapError::malformed("get response has no list array"))?; - for item in list { - out.list.push(serde_json::from_value(item.clone())?); - } - if let Some(nf) = mr.args.get("notFound").and_then(Value::as_array) { - for v in nf { - if let Some(s) = v.as_str() { - out.not_found.push(JmapId(s.to_owned())); - } - } - } - if let Some(s) = mr.args.get("state").and_then(Value::as_str) { - out.state = Some(s.to_owned()); - } - Ok(()) -} - #[derive(Debug, Default)] pub struct SetOutcome { pub created: Vec<(String, Value)>, @@ -583,198 +431,332 @@ enum Section { Destroy, } -fn collect_items(value: Option<&Value>, section: Section) -> Vec<(Section, String, Value)> { - match (section, value) { - (Section::Destroy, Some(Value::Array(ids))) => ids - .iter() - .filter_map(|v| v.as_str().map(|s| (section, s.to_owned(), Value::Null))) - .collect(), - (_, Some(Value::Object(m))) => m - .iter() - .map(|(k, v)| (section, k.clone(), v.clone())) - .collect(), - _ => Vec::new(), +struct SetItem { + section: Section, + key: String, + value: Value, +} + +impl SetItem { + fn collect(value: Option, section: Section, out: &mut Vec) { + match (section, value) { + (Section::Destroy, Some(Value::Array(ids))) => { + out.extend(ids.into_iter().filter_map(|v| match v { + Value::String(key) => Some(SetItem { + section, + key, + value: Value::Null, + }), + _ => None, + })); + } + (_, Some(Value::Object(m))) => out.extend(m.into_iter().map(|(key, value)| SetItem { + section, + key, + value, + })), + _ => {} + } } } -pub fn set_call( - client: &HttpClient, - api_url: &str, - account_id: &str, - type_name: &str, - body: SetRequest<'_>, - limits: &Limits, -) -> Result { - let mut items = collect_items(body.create.as_ref(), Section::Create); - items.extend(collect_items(body.update.as_ref(), Section::Update)); - items.extend(collect_items(body.destroy.as_ref(), Section::Destroy)); +impl JmapAccount { + pub fn send(&self, req: &Request) -> Result { + req.send(self.client(), self.api_url()) + } - if items.is_empty() { - return set_send( - client, - api_url, - account_id, - type_name, - &[], - body.extra_args, - limits, + fn single_call(&self, req: &Request) -> Result { + let resp = self.send(req)?; + resp.first()?.check_error()?; + Ok(resp) + } + + fn base_args(&self) -> Map { + let mut args = Map::new(); + args.insert( + "accountId".to_owned(), + Value::String(self.account_id().to_owned()), ); + args } - let chunk = limits.max_objects_in_set.max(1) as usize; - let mut outcome = SetOutcome::default(); - let mut start = 0; - while start < items.len() { - let end = (start + chunk).min(items.len()); - let part = set_send( - client, - api_url, - account_id, - type_name, - &items[start..end], - body.extra_args, - limits, - )?; - outcome.absorb(part); - start = end; - } - Ok(outcome) -} - -fn set_send( - client: &HttpClient, - api_url: &str, - account_id: &str, - type_name: &str, - items: &[(Section, String, Value)], - extra_args: &[(&str, Value)], - limits: &Limits, -) -> Result { - let mut args = Map::new(); - args.insert("accountId".to_owned(), Value::String(account_id.to_owned())); - let mut create = Map::new(); - let mut update = Map::new(); - let mut destroy: Vec = Vec::new(); - for (section, key, val) in items { - match section { - Section::Create => { - create.insert(key.clone(), val.clone()); + pub fn query_all_ids(&self, type_name: &str) -> Result, JmapError> { + let mut restarts = 0u32; + loop { + match self.query_pages(type_name) { + Err(JmapError::AnchorNotFound) if restarts < ANCHOR_RESTARTS => restarts += 1, + other => return other, } - Section::Update => { - update.insert(key.clone(), val.clone()); - } - Section::Destroy => destroy.push(Value::String(key.clone())), } } - if !create.is_empty() { - args.insert("create".to_owned(), Value::Object(create)); - } - if !update.is_empty() { - args.insert("update".to_owned(), Value::Object(update)); - } - if !destroy.is_empty() { - args.insert("destroy".to_owned(), Value::Array(destroy)); - } - for (k, v) in extra_args { - args.insert((*k).to_owned(), v.clone()); - } - let mut req = Request::new(); - req.call(format!("{type_name}/set"), Value::Object(args), "s"); - let oversize = req.fits(limits).is_err(); - let send_result = if oversize { - Err(JmapError::RequestTooLarge) - } else { - req.send(client, api_url).and_then(|resp| { + fn query_pages(&self, type_name: &str) -> Result, JmapError> { + let limit = self.limits().max_objects_in_get.max(1); + let method = format!("{type_name}/query"); + let mut collected: Vec = Vec::new(); + let mut page_size: Option = None; + let mut anchor: Option = None; + + loop { + let mut args = self.base_args(); + args.insert("limit".to_owned(), Value::from(limit)); + if let Some(a) = anchor.take() { + args.insert("anchor".to_owned(), Value::String(a)); + args.insert("anchorOffset".to_owned(), Value::from(1)); + } + let mut req = Request::new(); + req.call(method.as_str(), Value::Object(args), "q"); + let resp = self.single_call(&req)?; + let page = resp.first()?; + let ids = page + .args + .get("ids") + .and_then(Value::as_array) + .ok_or_else(|| JmapError::malformed("query response has no ids array"))?; + let this_len = ids.len(); + collected.reserve(this_len); + for v in ids { + let s = v + .as_str() + .ok_or_else(|| JmapError::malformed("query id is not a string"))?; + collected.push(JmapId(s.to_owned())); + } + match page_size { + None if this_len == 0 => break, + None => page_size = Some(this_len), + Some(full) if this_len == 0 || this_len < full => break, + Some(_) => {} + } + let last = collected + .last() + .map(|i| i.0.clone()) + .ok_or_else(|| JmapError::malformed("non-empty page yielded no anchor"))?; + anchor = Some(last); + } + Ok(collected) + } + + pub fn get( + &self, + type_name: &str, + ids: &[JmapId], + properties: Option<&[&str]>, + ) -> Result, JmapError> { + let mut out = GetResult::default(); + let chunk = usize::try_from(self.limits().max_objects_in_get.max(1)).unwrap_or(usize::MAX); + for part in ids.chunks(chunk) { + self.get_chunk(type_name, part, properties, &mut out)?; + } + Ok(out) + } + + fn get_chunk( + &self, + type_name: &str, + ids: &[JmapId], + properties: Option<&[&str]>, + out: &mut GetResult, + ) -> Result<(), JmapError> { + let mut args = self.base_args(); + args.insert( + "ids".to_owned(), + Value::Array(ids.iter().map(|i| Value::String(i.0.clone())).collect()), + ); + if let Some(props) = properties { + args.insert( + "properties".to_owned(), + Value::Array( + props + .iter() + .chain(&["id"]) + .map(|s| Value::String((*s).to_owned())) + .collect(), + ), + ); + } + let mut req = Request::new(); + req.call(format!("{type_name}/get"), Value::Object(args), "g"); + let sent = req.send_within(self, self.limits()).and_then(|resp| { let mr = resp.first()?; - check_method_error(mr)?; - Ok(decode_set(mr)) - }) - }; - - match send_result { - Ok(outcome) => Ok(outcome), - Err(JmapError::RequestTooLarge) => { - if items.len() <= 1 { - return Err(JmapError::SingleObjectTooLarge(format!( - "{type_name}/set of a single object exceeds maxSizeRequest" - ))); + mr.check_error()?; + mr.decode_get(out) + }); + match sent { + Err(JmapError::RequestTooLarge) if ids.len() <= 1 => { + Err(JmapError::SingleObjectTooLarge(format!( + "{type_name}/get of a single id exceeds maxSizeRequest" + ))) } - let mid = items.len() / 2; - let mut left = set_send( - client, - api_url, - account_id, - type_name, - &items[..mid], - extra_args, - limits, - )?; - let right = set_send( - client, - api_url, - account_id, - type_name, - &items[mid..], - extra_args, - limits, - )?; - left.absorb(right); - Ok(left) + Err(JmapError::RequestTooLarge) => { + let (head, tail) = ids.split_at(ids.len() / 2); + self.get_chunk(type_name, head, properties, out)?; + self.get_chunk(type_name, tail, properties, out) + } + other => other, } - Err(e) => Err(e), } -} -fn pairs(args: &Value, key: &str) -> Vec<(String, Value)> { - args.get(key) - .and_then(Value::as_object) - .map(|m| m.iter().map(|(k, v)| (k.clone(), v.clone())).collect()) - .unwrap_or_default() -} + pub fn get_all(&self, type_name: &str) -> Result, JmapError> { + let mut args = self.base_args(); + args.insert("ids".to_owned(), Value::Null); + let mut req = Request::new(); + req.call(format!("{type_name}/get"), Value::Object(args), "g"); + let resp = self.single_call(&req)?; + let mut out = GetResult::default(); + resp.first()?.decode_get(&mut out)?; + Ok(out) + } -fn keys(args: &Value, key: &str) -> Vec { - args.get(key) - .and_then(Value::as_array) - .map(|a| { - a.iter() - .filter_map(|v| v.as_str().map(str::to_owned)) - .collect() - }) - .unwrap_or_default() -} + pub fn state(&self, type_name: &str) -> Result, JmapError> { + let mut args = self.base_args(); + args.insert("ids".to_owned(), Value::Array(Vec::new())); + let mut req = Request::new(); + req.call(format!("{type_name}/get"), Value::Object(args), "g"); + let resp = self.single_call(&req)?; + let mut out: GetResult = GetResult::default(); + resp.first()?.decode_get(&mut out)?; + Ok(out.state) + } -fn decode_set(mr: &MethodCall) -> SetOutcome { - SetOutcome { - created: pairs(&mr.args, "created"), - updated: mr - .args - .get("updated") - .and_then(Value::as_object) - .map(|m| m.keys().cloned().collect()) - .unwrap_or_default(), - destroyed: keys(&mr.args, "destroyed"), - not_created: pairs(&mr.args, "notCreated"), - not_updated: pairs(&mr.args, "notUpdated"), - not_destroyed: pairs(&mr.args, "notDestroyed"), + pub fn changes(&self, type_name: &str, since_state: &str) -> Result { + let method = format!("{type_name}/changes"); + let max_changes = self.limits().max_objects_in_get.max(1); + let mut out = ChangesResult::default(); + let mut since = since_state.to_owned(); + loop { + let mut args = self.base_args(); + args.insert("sinceState".to_owned(), Value::String(since.clone())); + args.insert("maxChanges".to_owned(), Value::from(max_changes)); + let mut req = Request::new(); + req.call(method.as_str(), Value::Object(args), "c"); + let resp = self.single_call(&req)?; + let mr = resp.first()?; + let new_state = mr + .args + .get("newState") + .and_then(Value::as_str) + .ok_or_else(|| JmapError::malformed("changes response has no newState"))?; + mr.append_ids("created", &mut out.created); + mr.append_ids("updated", &mut out.updated); + mr.append_ids("destroyed", &mut out.destroyed); + let has_more = mr + .args + .get("hasMoreChanges") + .and_then(Value::as_bool) + .unwrap_or(false); + out.new_state.clear(); + out.new_state.push_str(new_state); + if !has_more || since == new_state { + break; + } + since.clear(); + since.push_str(new_state); + } + out.dedup(); + Ok(out) + } + + pub fn set(&self, type_name: &str, body: SetRequest<'_>) -> Result { + let mut items = Vec::new(); + SetItem::collect(body.create, Section::Create, &mut items); + SetItem::collect(body.update, Section::Update, &mut items); + SetItem::collect(body.destroy, Section::Destroy, &mut items); + + let method = format!("{type_name}/set"); + if items.is_empty() { + return self.set_send(type_name, &method, &[], body.extra_args); + } + + let chunk = usize::try_from(self.limits().max_objects_in_set.max(1)).unwrap_or(usize::MAX); + let mut outcome = SetOutcome::default(); + for part in items.chunks(chunk) { + outcome.absorb(self.set_send(type_name, &method, part, body.extra_args)?); + } + Ok(outcome) + } + + fn set_send( + &self, + type_name: &str, + method: &str, + items: &[SetItem], + extra_args: &[(&str, Value)], + ) -> Result { + let mut args = self.base_args(); + let mut create = Map::new(); + let mut update = Map::new(); + let mut destroy: Vec = Vec::new(); + for item in items { + match item.section { + Section::Create => { + create.insert(item.key.clone(), item.value.clone()); + } + Section::Update => { + update.insert(item.key.clone(), item.value.clone()); + } + Section::Destroy => destroy.push(Value::String(item.key.clone())), + } + } + if !create.is_empty() { + args.insert("create".to_owned(), Value::Object(create)); + } + if !update.is_empty() { + args.insert("update".to_owned(), Value::Object(update)); + } + if !destroy.is_empty() { + args.insert("destroy".to_owned(), Value::Array(destroy)); + } + for (k, v) in extra_args { + args.insert((*k).to_owned(), v.clone()); + } + let mut req = Request::new(); + req.call(method, Value::Object(args), "s"); + + let sent = req.send_within(self, self.limits()).and_then(|resp| { + let mr = resp.first()?; + mr.check_error()?; + Ok(mr.decode_set()) + }); + match sent { + Err(JmapError::RequestTooLarge) if items.len() <= 1 => { + Err(JmapError::SingleObjectTooLarge(format!( + "{type_name}/set of a single object exceeds maxSizeRequest" + ))) + } + Err(JmapError::RequestTooLarge) => { + let (head, tail) = items.split_at(items.len() / 2); + let mut left = self.set_send(type_name, method, head, extra_args)?; + left.absorb(self.set_send(type_name, method, tail, extra_args)?); + Ok(left) + } + other => other, + } } } #[cfg(test)] mod tests { + use crate::logging::Level; + use serde_json::json; use std::cell::Cell; use super::*; use crate::error::Error; - use crate::jmap::http::{Auth, RetryPolicy}; + use crate::logging::Logger; + use crate::net::auth::Auth; + use crate::net::http::{ClientOptions, TlsPolicy}; + use crate::net::retry::RetryPolicy; - fn test_client(max_retries: u32) -> HttpClient { - HttpClient::new( + fn test_client(max_retries: u32) -> JmapClient { + JmapClient::new( Auth::Bearer { token: "t".to_owned(), }, - RetryPolicy::new(max_retries), - false, + ClientOptions { + retry: RetryPolicy::new(max_retries), + tls: TlsPolicy::Verify, + logger: Logger::new(Level::Quiet), + }, ) } @@ -792,15 +774,10 @@ mod tests { ) -> (Result<&'static str, JmapError>, u32) { let client = test_client(max_retries); let calls = Cell::new(0u32); - let result = retry_method_call( - &client, - MethodCallKind::SingleObjectWrite, - &Logger::from_flags(true, 0), - || { - calls.set(calls.get() + 1); - outcome(calls.get()) - }, - ); + let result = client.retry_method(MethodCallKind::SingleObjectWrite, || { + calls.set(calls.get() + 1); + outcome(calls.get()) + }); (result, calls.get()) } @@ -859,23 +836,26 @@ mod tests { r.call("Mailbox/get", json!({}), "a"); r.call("Email/query", json!({}), "b"); let u = r.using(); - assert!(u.contains(&URN_CORE.to_owned())); - assert!(u.contains(&"urn:ietf:params:jmap:mail".to_owned())); - assert_eq!(u.iter().filter(|x| x.as_str() == URN_CORE).count(), 1); + assert!(u.contains(URN_CORE)); + assert!(u.contains("urn:ietf:params:jmap:mail")); + assert_eq!(u.iter().filter(|x| **x == URN_CORE).count(), 1); } #[test] fn using_maps_principal_and_stalwart() { assert_eq!( - using_urn("Principal/query"), + Request::capability_of("Principal/query"), "urn:ietf:params:jmap:principals" ); - assert_eq!(using_urn("x:Account/set"), "urn:stalwart:jmap"); + assert_eq!(Request::capability_of("x:Account/set"), "urn:stalwart:jmap"); assert_eq!( - using_urn("CalendarEvent/set"), + Request::capability_of("CalendarEvent/set"), "urn:ietf:params:jmap:calendars" ); - assert_eq!(using_urn("Identity/get"), "urn:ietf:params:jmap:submission"); + assert_eq!( + Request::capability_of("Identity/get"), + "urn:ietf:params:jmap:submission" + ); } #[test] @@ -905,8 +885,8 @@ mod tests { ] }); let r = Response::parse(v).unwrap(); - assert!(check_method_error(r.by_call_id("g").unwrap()).is_ok()); - let err = check_method_error(r.by_call_id("x").unwrap()).unwrap_err(); + assert!(r.by_call_id("g").unwrap().check_error().is_ok()); + let err = r.by_call_id("x").unwrap().check_error().unwrap_err(); assert!(matches!(err, JmapError::Method { .. })); } @@ -914,7 +894,27 @@ mod tests { fn anchor_not_found_is_classified() { let v = json!({ "methodResponses": [["error", {"type": "anchorNotFound"}, "q"]] }); let r = Response::parse(v).unwrap(); - let err = check_method_error(r.first().unwrap()).unwrap_err(); + let err = r.first().unwrap().check_error().unwrap_err(); assert!(matches!(err, JmapError::AnchorNotFound)); } + + #[test] + fn method_dispositions_depend_on_the_call_kind() { + assert_eq!( + MethodCallKind::Read.disposition("serverPartialFail"), + Disposition::Retry + ); + assert_eq!( + MethodCallKind::SingleObjectWrite.disposition("serverPartialFail"), + Disposition::Fatal + ); + assert_eq!( + MethodCallKind::SingleObjectWrite.disposition("serverUnavailable"), + Disposition::Retry + ); + assert_eq!( + MethodCallKind::Read.disposition("forbidden"), + Disposition::Fatal + ); + } } diff --git a/src/jmap/retry.rs b/src/jmap/retry.rs deleted file mode 100644 index 3ea86cd..0000000 --- a/src/jmap/retry.rs +++ /dev/null @@ -1,343 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::cell::Cell; -use std::sync::Mutex; -use std::sync::atomic::{AtomicU32, Ordering}; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; - -use crate::jmap::http::RetryPolicy; - -pub const THROTTLE_BASE: Duration = Duration::from_secs(1); -const THROTTLE_LEVEL_CAP: u32 = 16; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Disposition { - Retryable, - Fatal, -} - -pub fn classify_http_status(status: u16) -> Disposition { - match status { - 429 | 502 | 503 | 504 => Disposition::Retryable, - _ => Disposition::Fatal, - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum MethodCallKind { - Read, - SingleObjectWrite, -} - -pub fn jmap_method_disposition(error_type: &str, kind: MethodCallKind) -> Disposition { - match error_type { - "serverUnavailable" => Disposition::Retryable, - "serverPartialFail" => match kind { - MethodCallKind::Read => Disposition::Retryable, - MethodCallKind::SingleObjectWrite => Disposition::Fatal, - }, - _ => Disposition::Fatal, - } -} - -pub fn parse_retry_after(value: &str, now: SystemTime) -> Option { - let trimmed = value.trim(); - if let Ok(secs) = trimmed.parse::() { - return Some(Duration::from_secs(secs)); - } - let imf_fixdate = time::macros::format_description!( - "[weekday repr:short], [day] [month repr:short] [year] \ - [hour]:[minute]:[second] GMT" - ); - let parsed = time::PrimitiveDateTime::parse(trimmed, imf_fixdate).ok()?; - let target: SystemTime = parsed.assume_utc().into(); - target.duration_since(now).ok() -} - -thread_local! { - static RNG_STATE: Cell = Cell::new(seed()); -} - -fn seed() -> u64 { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_nanos() as u64) - .unwrap_or(0x9E37_79B9_7F4A_7C15); - let tid = { - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - std::hash::Hash::hash(&std::thread::current().id(), &mut hasher); - std::hash::Hasher::finish(&hasher) - }; - (nanos ^ tid).max(1) -} - -fn next_u64() -> u64 { - RNG_STATE.with(|cell| { - let mut x = cell.get(); - x ^= x << 13; - x ^= x >> 7; - x ^= x << 17; - cell.set(x); - x - }) -} - -fn jitter_uniform(span: Duration) -> Duration { - let nanos = span.as_nanos(); - if nanos == 0 { - return Duration::ZERO; - } - let pick = (next_u64() as u128) % (nanos + 1); - Duration::from_nanos(pick.min(u64::MAX as u128) as u64) -} - -pub fn backoff_delay(policy: &RetryPolicy, attempt: u32) -> Duration { - let shift = attempt.saturating_sub(1).min(32); - let scaled = policy - .base - .saturating_mul(1u32.checked_shl(shift).unwrap_or(u32::MAX)); - let capped = scaled.min(policy.cap); - jitter_uniform(capped) -} - -#[derive(Debug, Default)] -pub struct Cooldown { - until: Mutex>, -} - -impl Cooldown { - pub fn new() -> Cooldown { - Cooldown { - until: Mutex::new(None), - } - } - - pub fn arm(&self, delay: Duration) { - let target = Instant::now() + delay; - if let Ok(mut guard) = self.until.lock() { - let extend = guard.map(|cur| target > cur).unwrap_or(true); - if extend { - *guard = Some(target); - } - } - } - - pub fn remaining(&self) -> Option { - let guard = self.until.lock().ok()?; - let until = (*guard)?; - let now = Instant::now(); - if until > now { Some(until - now) } else { None } - } - - pub fn wait(&self) { - while let Some(remaining) = self.remaining() { - std::thread::sleep(remaining); - } - } -} - -#[derive(Debug, Default)] -pub struct RateLimitState { - cooldown: Cooldown, - level: AtomicU32, -} - -impl RateLimitState { - pub fn new() -> RateLimitState { - RateLimitState { - cooldown: Cooldown::new(), - level: AtomicU32::new(0), - } - } - - pub fn cooldown(&self) -> &Cooldown { - &self.cooldown - } - - pub fn level(&self) -> u32 { - self.level.load(Ordering::Relaxed) - } - - pub fn on_success(&self) { - self.level.store(0, Ordering::Relaxed); - } - - pub fn on_throttle(&self, policy: &RetryPolicy, retry_after: Option) -> Duration { - let n = self - .level - .fetch_add(1, Ordering::Relaxed) - .saturating_add(1) - .min(THROTTLE_LEVEL_CAP); - let delay = retry_after.unwrap_or_else(|| throttle_backoff(policy, n)); - self.cooldown.arm(delay); - delay - } -} - -pub fn throttle_backoff(policy: &RetryPolicy, level: u32) -> Duration { - let shift = level.saturating_sub(1).min(20); - let scaled = THROTTLE_BASE - .saturating_mul(1u32.checked_shl(shift).unwrap_or(u32::MAX)) - .min(policy.cap); - let half = scaled / 2; - half + jitter_uniform(scaled - half) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn policy() -> RetryPolicy { - RetryPolicy::new(5) - } - - #[test] - fn backoff_is_bounded_by_cap_with_full_jitter() { - let p = policy(); - for attempt in 1..=12 { - let d = backoff_delay(&p, attempt); - assert!(d <= p.cap, "attempt {attempt} delay {d:?} exceeds cap"); - } - } - - #[test] - fn backoff_grows_until_cap() { - let p = policy(); - let big = backoff_delay(&p, 30); - assert!(big <= p.cap); - } - - #[test] - fn retry_after_seconds() { - assert_eq!( - parse_retry_after("120", SystemTime::now()), - Some(Duration::from_secs(120)) - ); - } - - #[test] - fn retry_after_http_date_in_future() { - let now = SystemTime::UNIX_EPOCH + Duration::from_secs(784_111_777); - let d = parse_retry_after("Sun, 06 Nov 1994 08:49:47 GMT", now); - assert_eq!(d, Some(Duration::from_secs(10))); - } - - #[test] - fn retry_after_garbage_is_none() { - assert_eq!(parse_retry_after("soon", SystemTime::now()), None); - } - - #[test] - fn status_classification() { - assert_eq!(classify_http_status(429), Disposition::Retryable); - assert_eq!(classify_http_status(503), Disposition::Retryable); - assert_eq!(classify_http_status(400), Disposition::Fatal); - assert_eq!(classify_http_status(401), Disposition::Fatal); - } - - #[test] - fn method_classification() { - for kind in [MethodCallKind::Read, MethodCallKind::SingleObjectWrite] { - assert_eq!( - jmap_method_disposition("serverUnavailable", kind), - Disposition::Retryable, - "{kind:?}" - ); - assert_eq!( - jmap_method_disposition("invalidArguments", kind), - Disposition::Fatal, - "{kind:?}" - ); - assert_eq!( - jmap_method_disposition("forbidden", kind), - Disposition::Fatal, - "{kind:?}" - ); - } - } - - #[test] - fn partial_fail_is_never_retried_for_a_write() { - assert_eq!( - jmap_method_disposition("serverPartialFail", MethodCallKind::Read), - Disposition::Retryable - ); - assert_eq!( - jmap_method_disposition("serverPartialFail", MethodCallKind::SingleObjectWrite), - Disposition::Fatal - ); - } - - #[test] - fn cooldown_arms_and_clears() { - let c = Cooldown::new(); - assert!(c.remaining().is_none()); - c.arm(Duration::from_millis(40)); - assert!(c.remaining().is_some()); - std::thread::sleep(Duration::from_millis(60)); - assert!(c.remaining().is_none()); - } - - #[test] - fn throttle_backoff_doubles_then_clamps_to_cap() { - let p = RetryPolicy::new(8); - let mut prev = throttle_backoff(&p, 1); - for level in 2..=4 { - let d = throttle_backoff(&p, level); - assert!( - d >= prev / 2, - "level {level} ({d:?}) collapsed below half of prev ({prev:?})" - ); - prev = d; - } - assert!(throttle_backoff(&p, 20) <= p.cap); - } - - #[test] - fn rate_limit_state_escalates_across_concurrent_callers() { - let p = RetryPolicy::new(8); - let s = RateLimitState::new(); - let first = s.on_throttle(&p, None); - let second = s.on_throttle(&p, None); - let third = s.on_throttle(&p, None); - assert!( - third >= first, - "level 3 delay ({third:?}) should not be smaller than level 1 ({first:?})" - ); - assert!( - second >= first, - "level 2 delay ({second:?}) should not be smaller than level 1 ({first:?})" - ); - assert!(s.level() >= 3); - } - - #[test] - fn rate_limit_state_resets_on_success() { - let p = RetryPolicy::new(8); - let s = RateLimitState::new(); - let _ = s.on_throttle(&p, None); - let _ = s.on_throttle(&p, None); - let _ = s.on_throttle(&p, None); - assert!(s.level() >= 3); - s.on_success(); - assert_eq!(s.level(), 0); - let next = s.on_throttle(&p, None); - assert!( - next <= THROTTLE_BASE, - "first throttle after reset should be small ({next:?})" - ); - } - - #[test] - fn rate_limit_state_honours_retry_after_over_schedule() { - let p = RetryPolicy::new(8); - let s = RateLimitState::new(); - let d = s.on_throttle(&p, Some(Duration::from_millis(750))); - assert_eq!(d, Duration::from_millis(750)); - assert_eq!(s.level(), 1); - } -} diff --git a/src/jmap/session.rs b/src/jmap/session.rs index 3fac838..1626365 100644 --- a/src/jmap/session.rs +++ b/src/jmap/session.rs @@ -4,13 +4,17 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +use std::fmt; + use indexmap::IndexMap; use serde::Deserialize; use serde_json::Value; +use url::Url; use crate::error::Error; use crate::jmap::error::JmapError; -use crate::jmap::http::HttpClient; +use crate::jmap::http::JmapClient; +use crate::net::encode::{Percent, SafeSet}; #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")] @@ -42,83 +46,84 @@ pub struct Limits { pub max_size_upload: u64, } -fn parse_session(body: &str) -> Option { - let value: Value = serde_json::from_str(body).ok()?; - let has_shape = value.get("apiUrl").is_some() - && value.get("accounts").is_some() - && value.get("capabilities").is_some(); - if !has_shape { - return None; - } - serde_json::from_value(value).ok() +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Lane { + Api, + Upload, } -fn well_known_url(base: &str) -> String { - let trimmed = base.trim_end_matches('/'); - format!("{trimmed}/.well-known/jmap") +impl Limits { + pub fn workers(&self, requested: usize, lane: Lane) -> usize { + let server_cap = match lane { + Lane::Api => self.max_concurrent_requests, + Lane::Upload => self.max_concurrent_upload, + }; + requested.clamp(1, usize::try_from(server_cap.max(1)).unwrap_or(usize::MAX)) + } } -fn scheme_end(url: &str) -> Option { - let bytes = url.as_bytes(); - if !bytes.first()?.is_ascii_alphabetic() { - return None; +#[derive(Debug, Clone, Copy)] +pub struct SessionUrl<'a>(pub &'a str); + +impl<'a> SessionUrl<'a> { + fn split_scheme(self) -> Option<(&'a str, &'a str)> { + let (scheme, rest) = self.0.split_once(':')?; + let (first, tail) = scheme.as_bytes().split_first()?; + (first.is_ascii_alphabetic() + && tail + .iter() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'-' | b'.'))) + .then_some((scheme, rest)) } - for (index, byte) in bytes.iter().copied().enumerate() { - if byte == b':' { - return Some(index); + + pub fn resolve(self, reference: &str) -> String { + let base = self.0; + if reference.is_empty() { + return base.to_owned(); } - if !(byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'-' | b'.')) { - return None; + if SessionUrl(reference).split_scheme().is_some() { + return reference.to_owned(); } - } - None -} - -pub fn resolve_reference(base: &str, reference: &str) -> String { - if reference.is_empty() { - return base.to_owned(); - } - if scheme_end(reference).is_some() { - return reference.to_owned(); - } - let Some(base_scheme_end) = scheme_end(base) else { - return reference.to_owned(); - }; - let scheme = &base[..=base_scheme_end]; - let Some(authority_and_path) = base[base_scheme_end + 1..].strip_prefix("//") else { - return reference.to_owned(); - }; - let mut out = String::with_capacity(scheme.len() + 2 + base.len() + reference.len()); - out.push_str(scheme); - out.push_str("//"); - if let Some(rest) = reference.strip_prefix("//") { - out.push_str(rest); - return out; - } - let authority_len = authority_and_path - .find(['/', '?', '#']) - .unwrap_or(authority_and_path.len()); - out.push_str(&authority_and_path[..authority_len]); - if reference.starts_with('/') { + let Some((scheme, after_scheme)) = self.split_scheme() else { + return reference.to_owned(); + }; + let Some(authority_and_path) = after_scheme.strip_prefix("//") else { + return reference.to_owned(); + }; + let mut out = String::with_capacity(scheme.len() + 3 + base.len() + reference.len()); + out.push_str(scheme); + out.push_str("://"); + if let Some(rest) = reference.strip_prefix("//") { + out.push_str(rest); + return out; + } + let authority = authority_and_path + .split(['/', '?', '#']) + .next() + .unwrap_or_default(); + out.push_str(authority); + if reference.starts_with('/') { + out.push_str(reference); + return out; + } + let path = authority_and_path + .strip_prefix(authority) + .unwrap_or_default() + .split(['?', '#']) + .next() + .unwrap_or_default(); + if let Some((dir, _)) = path.rsplit_once('/') { + out.push_str(dir); + } + out.push('/'); out.push_str(reference); - return out; + out } - let path = &authority_and_path[authority_len..]; - let path = path.split(['?', '#']).next().unwrap_or(""); - match path.rfind('/') { - Some(last) => out.push_str(&path[..=last]), - None => out.push('/'), - } - out.push_str(reference); - out -} -pub fn origin_of(url: &str) -> Option { - let origin = url::Url::parse(url).ok()?.origin(); - if !origin.is_tuple() { - return None; + pub fn origin(self) -> Option { + let origin = Url::parse(self.0).ok()?.origin(); + origin.is_tuple().then(|| origin.ascii_serialization()) } - Some(origin.ascii_serialization()) } #[derive(Debug, Clone, PartialEq, Eq)] @@ -128,8 +133,8 @@ pub struct OriginMismatch { pub connect_origin: String, } -impl std::fmt::Display for OriginMismatch { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl fmt::Display for OriginMismatch { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str("session advertises ")?; for (index, field) in self.fields.iter().enumerate() { if index > 0 { @@ -147,20 +152,36 @@ impl std::fmt::Display for OriginMismatch { } impl Session { - pub fn discover(client: &HttpClient, url: &str) -> Result { + fn parse(body: &str) -> Option { + let value: Value = serde_json::from_str(body).ok()?; + let has_shape = value.get("apiUrl").is_some() + && value.get("accounts").is_some() + && value.get("capabilities").is_some(); + if !has_shape { + return None; + } + serde_json::from_value(value).ok() + } + + fn well_known_url(base: &str) -> String { + let trimmed = base.trim_end_matches('/'); + format!("{trimmed}/.well-known/jmap") + } + + pub fn discover(client: &JmapClient, url: &str) -> Result { match client.get(url) { Ok(direct) => { - if let Some(session) = parse_session(&direct) { + if let Some(session) = Session::parse(&direct) { return session.rebased_on(url).ensure_authenticated(); } } Err(JmapError::Auth(m)) => return Err(JmapError::Auth(m)), Err(_) => {} } - let well_known = well_known_url(url); + let well_known = Session::well_known_url(url); match client.get(&well_known) { Ok(body) => { - if let Some(session) = parse_session(&body) { + if let Some(session) = Session::parse(&body) { return session.rebased_on(&well_known).ensure_authenticated(); } Err(JmapError::Connect(format!( @@ -175,14 +196,14 @@ impl Session { } fn rebased_on(mut self, session_resource_url: &str) -> Session { - self.api_url = resolve_reference(session_resource_url, &self.api_url); - self.upload_url = resolve_reference(session_resource_url, &self.upload_url); - self.download_url = resolve_reference(session_resource_url, &self.download_url); + self.api_url = SessionUrl(session_resource_url).resolve(&self.api_url); + self.upload_url = SessionUrl(session_resource_url).resolve(&self.upload_url); + self.download_url = SessionUrl(session_resource_url).resolve(&self.download_url); self } pub fn origin_mismatches(&self, connect_url: &str) -> Vec { - let Some(connect_origin) = origin_of(connect_url) else { + let Some(connect_origin) = SessionUrl(connect_url).origin() else { return Vec::new(); }; let mut mismatches: Vec = Vec::new(); @@ -191,7 +212,7 @@ impl Session { ("uploadUrl", &self.upload_url), ("downloadUrl", &self.download_url), ] { - let Some(session_origin) = origin_of(value) else { + let Some(session_origin) = SessionUrl(value).origin() else { continue; }; if session_origin == connect_origin { @@ -257,8 +278,10 @@ impl Session { } pub fn upload_url_for(&self, account_id: &str) -> String { - self.upload_url - .replace("{accountId}", &encode_segment(account_id)) + self.upload_url.replace( + "{accountId}", + &Percent::encode(account_id, SafeSet::Unreserved), + ) } pub fn download_url_for( @@ -269,34 +292,37 @@ impl Session { name: &str, ) -> String { self.download_url - .replace("{accountId}", &encode_segment(account_id)) - .replace("{blobId}", &encode_segment(blob_id)) - .replace("{type}", &encode_segment(type_hint)) - .replace("{name}", &encode_segment(name)) + .replace( + "{accountId}", + &Percent::encode(account_id, SafeSet::Unreserved), + ) + .replace("{blobId}", &Percent::encode(blob_id, SafeSet::Unreserved)) + .replace("{type}", &Percent::encode(type_hint, SafeSet::Unreserved)) + .replace("{name}", &Percent::encode(name, SafeSet::Unreserved)) } } -fn encode_segment(input: &str) -> String { - const HEX: &[u8; 16] = b"0123456789ABCDEF"; - let mut out = String::with_capacity(input.len()); - for byte in input.as_bytes() { - let b = *byte; - let unreserved = b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.' | b'~'); - if unreserved { - out.push(b as char); - } else { - out.push('%'); - out.push(HEX[(b >> 4) as usize] as char); - out.push(HEX[(b & 0x0F) as usize] as char); - } - } - out -} - #[cfg(test)] mod tests { use super::*; + #[test] + fn worker_count_is_clamped_to_server_limits() { + let l = Limits { + max_objects_in_get: 500, + max_objects_in_set: 500, + max_calls_in_request: 16, + max_concurrent_requests: 4, + max_concurrent_upload: 2, + max_size_request: 10_000_000, + max_size_upload: 50_000_000, + }; + assert_eq!(l.workers(16, Lane::Api), 4); + assert_eq!(l.workers(16, Lane::Upload), 2); + assert_eq!(l.workers(1, Lane::Api), 1); + assert_eq!(l.workers(0, Lane::Api), 1); + } + fn raw_session() -> &'static str { r#"{ "apiUrl": "https://example.org/jmap/api", @@ -324,7 +350,7 @@ mod tests { #[test] fn parses_a_session_and_reads_limits() { - let session = parse_session(raw_session()).unwrap(); + let session = Session::parse(raw_session()).unwrap(); let limits = session.core_limits().unwrap(); assert_eq!(limits.max_objects_in_get, 500); assert_eq!(limits.max_size_upload, 50000000); @@ -333,7 +359,7 @@ mod tests { #[test] fn capability_gate_reads_account_capabilities() { - let session = parse_session(raw_session()).unwrap(); + let session = Session::parse(raw_session()).unwrap(); assert!(session.supports("w", "urn:ietf:params:jmap:mail")); assert!(!session.supports("w", "urn:ietf:params:jmap:filenode")); assert!(!session.supports("missing", "urn:ietf:params:jmap:mail")); @@ -342,7 +368,7 @@ mod tests { // Listed accounts use their own capabilities; unlisted ones fall back to the session's. #[test] fn may_support_falls_back_to_session_capabilities_for_unlisted_account() { - let session = parse_session(raw_session()).unwrap(); + let session = Session::parse(raw_session()).unwrap(); assert!(session.may_support("w", "urn:ietf:params:jmap:mail")); assert!(!session.may_support("w", "urn:ietf:params:jmap:core")); assert!(session.may_support("missing", "urn:ietf:params:jmap:core")); @@ -351,7 +377,7 @@ mod tests { #[test] fn templates_upload_and_download_urls_with_encoding() { - let session = parse_session(raw_session()).unwrap(); + let session = Session::parse(raw_session()).unwrap(); assert_eq!( session.upload_url_for("a c"), "https://example.org/jmap/upload/a%20c/" @@ -364,15 +390,15 @@ mod tests { #[test] fn rejects_body_that_is_not_a_session() { - assert!(parse_session("{\"hello\":true}").is_none()); - assert!(parse_session("not json").is_none()); + assert!(Session::parse("{\"hello\":true}").is_none()); + assert!(Session::parse("not json").is_none()); } #[test] fn anonymous_session_is_auth_failure() { let raw = r#"{"apiUrl":"x","uploadUrl":"u","downloadUrl":"d","capabilities":{},"accounts":{}}"#; - let session = parse_session(raw).unwrap(); + let session = Session::parse(raw).unwrap(); assert!(matches!( session.ensure_authenticated(), Err(JmapError::Auth(_)) @@ -382,21 +408,24 @@ mod tests { #[test] fn origin_treats_the_default_port_as_equivalent() { assert_eq!( - origin_of("https://mail.example/jmap"), - origin_of("https://mail.example:443/jmap") + SessionUrl("https://mail.example/jmap").origin(), + SessionUrl("https://mail.example:443/jmap").origin() ); assert_eq!( - origin_of("http://mail.example/jmap"), - origin_of("http://mail.example:80/jmap") + SessionUrl("http://mail.example/jmap").origin(), + SessionUrl("http://mail.example:80/jmap").origin() ); } #[test] fn origin_distinguishes_port_host_and_scheme() { - let base = origin_of("https://mail.example:10443/jmap"); - assert_ne!(base, origin_of("https://mail.example/jmap")); - assert_ne!(base, origin_of("https://other.example:10443/jmap")); - assert_ne!(base, origin_of("http://mail.example:10443/jmap")); + let base = SessionUrl("https://mail.example:10443/jmap").origin(); + assert_ne!(base, SessionUrl("https://mail.example/jmap").origin()); + assert_ne!( + base, + SessionUrl("https://other.example:10443/jmap").origin() + ); + assert_ne!(base, SessionUrl("http://mail.example:10443/jmap").origin()); } fn session_on(base: &str) -> Session { @@ -405,7 +434,7 @@ mod tests { "downloadUrl":"{base}/jmap/dl/{{accountId}}/{{blobId}}/{{type}}/{{name}}", "capabilities":{{}},"accounts":{{}}}}"# ); - parse_session(&raw).expect("session") + Session::parse(&raw).expect("session") } #[test] @@ -449,7 +478,7 @@ mod tests { "uploadUrl":"https://blobs.example/upload/{accountId}/", "downloadUrl":"https://blobs.example/dl/{accountId}/{blobId}/{type}/{name}", "capabilities":{},"accounts":{}}"#; - let session = parse_session(raw).expect("session"); + let session = Session::parse(raw).expect("session"); let mismatches = session.origin_mismatches("https://mail.example:10443"); assert_eq!(mismatches.len(), 1); assert_eq!(mismatches[0].fields, vec!["uploadUrl", "downloadUrl"]); @@ -459,10 +488,8 @@ mod tests { #[test] fn resolve_reference_keeps_absolute_urls_verbatim() { assert_eq!( - resolve_reference( - "https://h.example/.well-known/jmap", - "https://other.example:8443/jmap/api" - ), + SessionUrl("https://h.example/.well-known/jmap") + .resolve("https://other.example:8443/jmap/api"), "https://other.example:8443/jmap/api" ); } @@ -471,15 +498,15 @@ mod tests { fn resolve_reference_handles_scheme_relative_absolute_and_relative_paths() { let base = "https://h.example:10443/.well-known/jmap"; assert_eq!( - resolve_reference(base, "//other.example/jmap/api"), + SessionUrl(base).resolve("//other.example/jmap/api"), "https://other.example/jmap/api" ); assert_eq!( - resolve_reference(base, "/jmap/api"), + SessionUrl(base).resolve("/jmap/api"), "https://h.example:10443/jmap/api" ); assert_eq!( - resolve_reference(base, "api"), + SessionUrl(base).resolve("api"), "https://h.example:10443/.well-known/api" ); } @@ -487,10 +514,7 @@ mod tests { #[test] fn resolve_reference_preserves_uri_template_braces() { assert_eq!( - resolve_reference( - "https://h.example/.well-known/jmap", - "/jmap/upload/{accountId}/" - ), + SessionUrl("https://h.example/.well-known/jmap").resolve("/jmap/upload/{accountId}/"), "https://h.example/jmap/upload/{accountId}/" ); } @@ -500,7 +524,7 @@ mod tests { let raw = r#"{"apiUrl":"/jmap/","uploadUrl":"/jmap/upload/{accountId}/", "downloadUrl":"/jmap/download/{accountId}/{blobId}/{type}/{name}", "capabilities":{},"accounts":{}}"#; - let session = parse_session(raw) + let session = Session::parse(raw) .expect("session") .rebased_on("https://h.example:10443/.well-known/jmap"); assert_eq!(session.api_url, "https://h.example:10443/jmap/"); @@ -519,11 +543,11 @@ mod tests { #[test] fn well_known_appends_correctly() { assert_eq!( - well_known_url("https://h.example/"), + Session::well_known_url("https://h.example/"), "https://h.example/.well-known/jmap" ); assert_eq!( - well_known_url("https://h.example"), + Session::well_known_url("https://h.example"), "https://h.example/.well-known/jmap" ); } diff --git a/src/jmap/wire/address_book.rs b/src/jmap/wire/address_book.rs index 2e4e9cb..aab2f51 100644 --- a/src/jmap/wire/address_book.rs +++ b/src/jmap/wire/address_book.rs @@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize}; use super::JmapId; +use super::common::Lenient; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -25,10 +26,6 @@ pub struct AddressBook { #[serde(default)] pub is_default: bool, - #[serde(default = "default_true")] + #[serde(default = "Lenient::default_true")] pub is_subscribed: bool, } - -fn default_true() -> bool { - true -} diff --git a/src/jmap/wire/calendar.rs b/src/jmap/wire/calendar.rs index d59b658..5389996 100644 --- a/src/jmap/wire/calendar.rs +++ b/src/jmap/wire/calendar.rs @@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use super::JmapId; +use super::common::Lenient; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -27,16 +28,16 @@ pub struct Calendar { #[serde(default)] pub sort_order: u32, - #[serde(default = "default_true")] + #[serde(default = "Lenient::default_true")] pub is_subscribed: bool, - #[serde(default = "default_true")] + #[serde(default = "Lenient::default_true")] pub is_visible: bool, #[serde(default)] pub is_default: bool, - #[serde(default = "default_include_in_availability")] + #[serde(default = "Calendar::default_include_in_availability")] pub include_in_availability: String, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -60,7 +61,7 @@ impl Calendar { _ => None, }, Some(_) => None, - None => is_css3_color_name(trimmed).then_some(trimmed), + None => Calendar::is_css3_color_name(trimmed).then_some(trimmed), }; match kept { Some(kept) if kept.len() == color.len() => None, @@ -71,167 +72,163 @@ impl Calendar { None => self.color.take(), } } -} -fn is_css3_color_name(name: &str) -> bool { - hashify::set_ignore_case!( - name.as_bytes(), - "aliceblue", - "antiquewhite", - "aqua", - "aquamarine", - "azure", - "beige", - "bisque", - "black", - "blanchedalmond", - "blue", - "blueviolet", - "brown", - "burlywood", - "cadetblue", - "chartreuse", - "chocolate", - "coral", - "cornflowerblue", - "cornsilk", - "crimson", - "cyan", - "darkblue", - "darkcyan", - "darkgoldenrod", - "darkgray", - "darkgreen", - "darkgrey", - "darkkhaki", - "darkmagenta", - "darkolivegreen", - "darkorange", - "darkorchid", - "darkred", - "darksalmon", - "darkseagreen", - "darkslateblue", - "darkslategray", - "darkslategrey", - "darkturquoise", - "darkviolet", - "deeppink", - "deepskyblue", - "dimgray", - "dimgrey", - "dodgerblue", - "firebrick", - "floralwhite", - "forestgreen", - "fuchsia", - "gainsboro", - "ghostwhite", - "gold", - "goldenrod", - "gray", - "green", - "greenyellow", - "grey", - "honeydew", - "hotpink", - "indianred", - "indigo", - "ivory", - "khaki", - "lavender", - "lavenderblush", - "lawngreen", - "lemonchiffon", - "lightblue", - "lightcoral", - "lightcyan", - "lightgoldenrodyellow", - "lightgray", - "lightgreen", - "lightgrey", - "lightpink", - "lightsalmon", - "lightseagreen", - "lightskyblue", - "lightslategray", - "lightslategrey", - "lightsteelblue", - "lightyellow", - "lime", - "limegreen", - "linen", - "magenta", - "maroon", - "mediumaquamarine", - "mediumblue", - "mediumorchid", - "mediumpurple", - "mediumseagreen", - "mediumslateblue", - "mediumspringgreen", - "mediumturquoise", - "mediumvioletred", - "midnightblue", - "mintcream", - "mistyrose", - "moccasin", - "navajowhite", - "navy", - "oldlace", - "olive", - "olivedrab", - "orange", - "orangered", - "orchid", - "palegoldenrod", - "palegreen", - "paleturquoise", - "palevioletred", - "papayawhip", - "peachpuff", - "peru", - "pink", - "plum", - "powderblue", - "purple", - "red", - "rosybrown", - "royalblue", - "saddlebrown", - "salmon", - "sandybrown", - "seagreen", - "seashell", - "sienna", - "silver", - "skyblue", - "slateblue", - "slategray", - "slategrey", - "snow", - "springgreen", - "steelblue", - "tan", - "teal", - "thistle", - "tomato", - "turquoise", - "violet", - "wheat", - "white", - "whitesmoke", - "yellow", - "yellowgreen", - ) -} + fn default_include_in_availability() -> String { + "all".to_owned() + } -fn default_true() -> bool { - true -} - -fn default_include_in_availability() -> String { - "all".to_owned() + fn is_css3_color_name(name: &str) -> bool { + hashify::set_ignore_case!( + name.as_bytes(), + "aliceblue", + "antiquewhite", + "aqua", + "aquamarine", + "azure", + "beige", + "bisque", + "black", + "blanchedalmond", + "blue", + "blueviolet", + "brown", + "burlywood", + "cadetblue", + "chartreuse", + "chocolate", + "coral", + "cornflowerblue", + "cornsilk", + "crimson", + "cyan", + "darkblue", + "darkcyan", + "darkgoldenrod", + "darkgray", + "darkgreen", + "darkgrey", + "darkkhaki", + "darkmagenta", + "darkolivegreen", + "darkorange", + "darkorchid", + "darkred", + "darksalmon", + "darkseagreen", + "darkslateblue", + "darkslategray", + "darkslategrey", + "darkturquoise", + "darkviolet", + "deeppink", + "deepskyblue", + "dimgray", + "dimgrey", + "dodgerblue", + "firebrick", + "floralwhite", + "forestgreen", + "fuchsia", + "gainsboro", + "ghostwhite", + "gold", + "goldenrod", + "gray", + "green", + "greenyellow", + "grey", + "honeydew", + "hotpink", + "indianred", + "indigo", + "ivory", + "khaki", + "lavender", + "lavenderblush", + "lawngreen", + "lemonchiffon", + "lightblue", + "lightcoral", + "lightcyan", + "lightgoldenrodyellow", + "lightgray", + "lightgreen", + "lightgrey", + "lightpink", + "lightsalmon", + "lightseagreen", + "lightskyblue", + "lightslategray", + "lightslategrey", + "lightsteelblue", + "lightyellow", + "lime", + "limegreen", + "linen", + "magenta", + "maroon", + "mediumaquamarine", + "mediumblue", + "mediumorchid", + "mediumpurple", + "mediumseagreen", + "mediumslateblue", + "mediumspringgreen", + "mediumturquoise", + "mediumvioletred", + "midnightblue", + "mintcream", + "mistyrose", + "moccasin", + "navajowhite", + "navy", + "oldlace", + "olive", + "olivedrab", + "orange", + "orangered", + "orchid", + "palegoldenrod", + "palegreen", + "paleturquoise", + "palevioletred", + "papayawhip", + "peachpuff", + "peru", + "pink", + "plum", + "powderblue", + "purple", + "red", + "rosybrown", + "royalblue", + "saddlebrown", + "salmon", + "sandybrown", + "seagreen", + "seashell", + "sienna", + "silver", + "skyblue", + "slateblue", + "slategray", + "slategrey", + "snow", + "springgreen", + "steelblue", + "tan", + "teal", + "thistle", + "tomato", + "turquoise", + "violet", + "wheat", + "white", + "whitesmoke", + "yellow", + "yellowgreen", + ) + } } #[cfg(test)] diff --git a/src/jmap/wire/common.rs b/src/jmap/wire/common.rs index 41e56af..21c0055 100644 --- a/src/jmap/wire/common.rs +++ b/src/jmap/wire/common.rs @@ -4,7 +4,8 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; +use time::format_description::well_known::Rfc3339; #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(transparent)] @@ -32,33 +33,47 @@ pub type UtcDate = time::OffsetDateTime; pub type Date = time::OffsetDateTime; -pub fn bool_or_true<'de, D>(d: D) -> Result -where - D: serde::Deserializer<'de>, -{ - Ok(Option::::deserialize(d)?.unwrap_or(true)) -} +pub struct Lenient; -pub fn lenient_utc_date<'de, D>(d: D) -> Result, D::Error> -where - D: serde::Deserializer<'de>, -{ - Ok(Option::::deserialize(d)? - .as_deref() - .and_then(parse_utc_date)) -} +impl Lenient { + pub const fn default_true() -> bool { + true + } -pub fn parse_utc_date(raw: &str) -> Option { - use time::format_description::well_known::Rfc3339; - let trimmed = raw.trim(); - if let Ok(dt) = UtcDate::parse(trimmed, &Rfc3339) { - return Some(dt); + pub fn bool_or_true<'de, D>(d: D) -> Result + where + D: Deserializer<'de>, + { + Ok(Option::::deserialize(d)?.unwrap_or(true)) } - let separated = trimmed.replacen(' ', "T", 1); - if let Ok(dt) = UtcDate::parse(&separated, &Rfc3339) { - return Some(dt); + + pub fn bool_or_false<'de, D>(d: D) -> Result + where + D: Deserializer<'de>, + { + Ok(Option::::deserialize(d)?.unwrap_or_default()) + } + + pub fn utc_date<'de, D>(d: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + Ok(Option::::deserialize(d)? + .as_deref() + .and_then(Lenient::parse_utc_date)) + } + + pub fn parse_utc_date(raw: &str) -> Option { + let trimmed = raw.trim(); + if let Ok(dt) = UtcDate::parse(trimmed, &Rfc3339) { + return Some(dt); + } + let separated = trimmed.replacen(' ', "T", 1); + if let Ok(dt) = UtcDate::parse(&separated, &Rfc3339) { + return Some(dt); + } + UtcDate::parse(&format!("{separated}Z"), &Rfc3339).ok() } - UtcDate::parse(&format!("{separated}Z"), &Rfc3339).ok() } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -74,23 +89,23 @@ mod tests { #[test] fn strict_rfc3339_is_accepted() { - let dt = parse_utc_date("2026-08-22T09:42:52Z").expect("parsed"); + let dt = Lenient::parse_utc_date("2026-08-22T09:42:52Z").expect("parsed"); assert_eq!(dt.year(), 2026); - assert!(parse_utc_date("2026-08-22T09:42:52.123+02:00").is_some()); + assert!(Lenient::parse_utc_date("2026-08-22T09:42:52.123+02:00").is_some()); } #[test] fn space_separator_and_missing_zone_are_salvaged() { - assert!(parse_utc_date("2026-08-22 09:42:52Z").is_some()); - assert!(parse_utc_date("2026-08-22T09:42:52").is_some()); - assert!(parse_utc_date(" 2026-08-22T09:42:52Z ").is_some()); + assert!(Lenient::parse_utc_date("2026-08-22 09:42:52Z").is_some()); + assert!(Lenient::parse_utc_date("2026-08-22T09:42:52").is_some()); + assert!(Lenient::parse_utc_date(" 2026-08-22T09:42:52Z ").is_some()); } #[test] fn unrepresentable_dates_yield_none_instead_of_an_error() { - assert!(parse_utc_date("30828-09-14T02:48:05Z").is_none()); - assert!(parse_utc_date("10000-01-01T00:00:00Z").is_none()); - assert!(parse_utc_date("").is_none()); - assert!(parse_utc_date("not a date").is_none()); + assert!(Lenient::parse_utc_date("30828-09-14T02:48:05Z").is_none()); + assert!(Lenient::parse_utc_date("10000-01-01T00:00:00Z").is_none()); + assert!(Lenient::parse_utc_date("").is_none()); + assert!(Lenient::parse_utc_date("not a date").is_none()); } } diff --git a/src/jmap/wire/email.rs b/src/jmap/wire/email.rs index e84f795..fa6dbee 100644 --- a/src/jmap/wire/email.rs +++ b/src/jmap/wire/email.rs @@ -7,7 +7,7 @@ use indexmap::IndexMap; use serde::{Deserialize, Serialize}; -use super::common::lenient_utc_date; +use super::common::Lenient; use super::{JmapId, UtcDate}; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -20,7 +20,7 @@ pub struct Email { #[serde( default, - deserialize_with = "lenient_utc_date", + deserialize_with = "Lenient::utc_date", serialize_with = "time::serde::rfc3339::option::serialize", skip_serializing_if = "Option::is_none" )] diff --git a/src/jmap/wire/file_node.rs b/src/jmap/wire/file_node.rs index 9380ad5..d77f27e 100644 --- a/src/jmap/wire/file_node.rs +++ b/src/jmap/wire/file_node.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use super::common::bool_or_true; +use super::common::Lenient; use super::{JmapId, UtcDate}; use serde::{Deserialize, Serialize}; @@ -50,7 +50,7 @@ pub struct FileProps { #[serde(default, skip_serializing_if = "Option::is_none")] pub media_type: Option, - #[serde(default, deserialize_with = "bool_or_false")] + #[serde(default, deserialize_with = "Lenient::bool_or_false")] pub executable: bool, } @@ -60,7 +60,10 @@ pub struct DirectoryProps { #[serde(default, skip_serializing_if = "Option::is_none")] pub role: Option, - #[serde(default = "default_true", deserialize_with = "bool_or_true")] + #[serde( + default = "Lenient::default_true", + deserialize_with = "Lenient::bool_or_true" + )] pub is_subscribed: bool, } @@ -68,24 +71,3 @@ pub struct DirectoryProps { pub struct SymLinkProps { pub target: Vec, } - -impl NodeKind { - pub fn db_name(&self) -> &'static str { - match self { - NodeKind::File(_) => "file", - NodeKind::Directory(_) => "directory", - NodeKind::SymLink(_) => "symlink", - } - } -} - -fn bool_or_false<'de, D>(d: D) -> Result -where - D: serde::Deserializer<'de>, -{ - Ok(Option::::deserialize(d)?.unwrap_or_default()) -} - -fn default_true() -> bool { - true -} diff --git a/src/jmap/wire/mailbox.rs b/src/jmap/wire/mailbox.rs index 9699fdb..b4c3f65 100644 --- a/src/jmap/wire/mailbox.rs +++ b/src/jmap/wire/mailbox.rs @@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize}; use super::JmapId; +use super::common::Lenient; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -25,10 +26,6 @@ pub struct Mailbox { #[serde(default)] pub sort_order: u32, - #[serde(default = "default_true")] + #[serde(default = "Lenient::default_true")] pub is_subscribed: bool, } - -fn default_true() -> bool { - true -} diff --git a/src/lib.rs b/src/lib.rs index d20a878..fd375f4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,18 +4,20 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +pub mod archive; pub mod cli; +pub mod console; pub mod dav; -pub mod db; pub mod error; pub mod exchange; -pub mod exchange_ews; -pub mod exchange_graph; +pub mod export; pub mod imap; -pub mod inspect; +pub mod import; +pub(crate) mod inspect; pub mod jmap; pub mod logging; pub mod managesieve; -pub mod secret; +pub mod net; +pub(crate) mod secret; pub mod sync; pub mod types; diff --git a/src/logging.rs b/src/logging.rs index 1b01b9b..2460dbe 100644 --- a/src/logging.rs +++ b/src/logging.rs @@ -4,109 +4,215 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +use std::fmt; use std::time::Duration; -pub const LEVEL_QUIET: u8 = 0; -pub const LEVEL_DEFAULT: u8 = 1; -pub const LEVEL_PROGRESS: u8 = 2; -pub const LEVEL_METHOD: u8 = 3; -pub const LEVEL_BODIES: u8 = 4; +use crate::console::{Console, Severity, Tone}; + +const HTTP_ERROR_STATUS: u16 = 400; pub const TRACE_BODY_CAP: usize = 8192; +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Level { + Quiet, + Default, + Progress, + Method, + Bodies, +} + +const LEVELS: [Level; 5] = [ + Level::Quiet, + Level::Default, + Level::Progress, + Level::Method, + Level::Bodies, +]; + +impl Level { + pub fn from_index(index: u8) -> Level { + LEVELS + .get(usize::from(index)) + .copied() + .unwrap_or(Level::Bodies) + } + + pub fn index(self) -> u8 { + self as u8 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Verbosity { + Quiet, + Verbose(u8), +} + #[derive(Debug, Clone, Copy)] pub struct Logger { - level: u8, + level: Level, + console: Console, } impl Logger { - pub fn new(level: u8) -> Self { + pub fn new(level: Level) -> Self { Logger { - level: level.min(LEVEL_BODIES), + level, + console: Console::default(), } } - pub fn from_flags(quiet: bool, verbose: u8) -> Self { - let level = if quiet { - LEVEL_QUIET - } else { - LEVEL_DEFAULT.saturating_add(verbose) - }; - Logger::new(level) + pub fn with_console(self, console: Console) -> Self { + Logger { console, ..self } } - pub fn level(&self) -> u8 { + pub fn console(self) -> Console { + self.console + } + + pub fn from_verbosity(verbosity: Verbosity) -> Self { + match verbosity { + Verbosity::Quiet => Logger::new(Level::Quiet), + Verbosity::Verbose(extra) => Logger::new(Level::from_index( + Level::Default.index().saturating_add(extra), + )), + } + } + + pub fn level(self) -> Level { self.level } - pub fn enabled(&self, level: u8) -> bool { + pub fn enabled(self, level: Level) -> bool { self.level >= level } - pub fn warn(&self, message: &str) { - eprintln!("warning: {message}"); + pub fn info(self, args: fmt::Arguments<'_>) { + if self.enabled(Level::Default) { + eprintln!("{args}"); + } } - pub fn error(&self, message: &str) { - eprintln!("error: {message}"); + pub fn progress(self, args: fmt::Arguments<'_>) { + if self.enabled(Level::Progress) { + eprintln!("{args}"); + } } - pub fn trace_http(&self, call: &HttpCall<'_>) { - if !self.enabled(LEVEL_METHOD) { + pub fn note(self, args: fmt::Arguments<'_>) { + if self.enabled(Level::Default) { + eprintln!("{} {args}", self.console.err.severity(Severity::Note)); + } + } + + pub fn notice(self, args: fmt::Arguments<'_>) { + eprintln!("{args}"); + } + + pub fn warn(self, args: fmt::Arguments<'_>) { + eprintln!("{} {args}", self.console.err.severity(Severity::Warning)); + } + + pub fn error(self, args: fmt::Arguments<'_>) { + eprintln!("{} {args}", self.console.err.severity(Severity::Error)); + } + + pub fn trace_http(self, call: &HttpCall<'_>) { + if !self.enabled(Level::Method) { return; } - match call.note { - Some(note) => eprintln!( - "{} {} {} -> {} ({} ms) {note}", - call.proto, - call.method, - call.url, - call.status, - call.elapsed.as_millis() - ), - None => eprintln!( - "{} {} {} -> {} ({} ms)", - call.proto, - call.method, - call.url, - call.status, - call.elapsed.as_millis() - ), - } - if self.enabled(LEVEL_BODIES) { - if let Some(request) = call.request { - trace_body('>', request, call.request_type); + let styled = self.console.err; + let status_tone = if call.status < HTTP_ERROR_STATUS { + Tone::Good + } else { + Tone::Bad + }; + eprintln!( + "{} {} {} {} {} {}{}", + styled.paint(Tone::Heading, call.proto), + call.method, + call.url, + styled.paint(Tone::Dim, "->"), + styled.paint(status_tone, call.status), + styled.paint(Tone::Dim, Millis(call.elapsed)), + Note(call.note), + ); + if self.enabled(Level::Bodies) { + if let Some(body) = call.request { + WireBody { + body, + content_type: call.request_type, + } + .trace('>'); } - trace_body('<', call.response, call.response_type); + WireBody { + body: call.response, + content_type: call.response_type, + } + .trace('<'); } } - pub fn trace_cmd(&self, proto: &str, command: &str, status: &str, elapsed: Duration) { - if !self.enabled(LEVEL_METHOD) { + pub fn trace_cmd(self, proto: &str, command: &str, status: &str, elapsed: Duration) { + if !self.enabled(Level::Method) { return; } - let command = cap(&redact_wire(command), command.len()); - eprintln!("{proto} {command} -> {status} ({} ms)", elapsed.as_millis()); + let redacted = WireCommand(command).to_string(); + let command = Capped { + text: &redacted, + total_bytes: command.len(), + }; + let styled = self.console.err; + eprintln!( + "{} {command} {} {status} {}", + styled.paint(Tone::Heading, proto), + styled.paint(Tone::Dim, "->"), + styled.paint(Tone::Dim, Millis(elapsed)), + ); } pub fn trace_http_error( - &self, + self, proto: &str, method: &str, url: &str, error: &str, elapsed: Duration, ) { - if !self.enabled(LEVEL_METHOD) { + if !self.enabled(Level::Method) { return; } + let styled = self.console.err; eprintln!( - "{proto} {method} {url} -> transport error: {error} ({} ms)", - elapsed.as_millis() + "{} {method} {url} {} {} {}", + styled.paint(Tone::Heading, proto), + styled.paint(Tone::Dim, "->"), + styled.paint(Tone::Bad, format_args!("transport error: {error}")), + styled.paint(Tone::Dim, Millis(elapsed)), ); } } +struct Millis(Duration); + +impl fmt::Display for Millis { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "({} ms)", self.0.as_millis()) + } +} + +struct Note<'a>(Option<&'a str>); + +impl fmt::Display for Note<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0 { + Some(note) => write!(f, " {note}"), + None => Ok(()), + } + } +} + pub struct HttpCall<'a> { pub proto: &'a str, pub method: &'a str, @@ -120,117 +226,195 @@ pub struct HttpCall<'a> { pub response_type: Option<&'a str>, } -fn trace_body(marker: char, body: &[u8], content_type: Option<&str>) { - for line in body_for_log(body, content_type).lines() { - eprintln!(" {marker} {line}"); +#[derive(Clone, Copy)] +pub struct WireBody<'a> { + pub body: &'a [u8], + pub content_type: Option<&'a str>, +} + +impl<'a> WireBody<'a> { + fn trace(self, marker: char) { + for line in self.to_string().lines() { + eprintln!(" {marker} {line}"); + } + } + + fn is_json(self) -> bool { + self.content_type.is_some_and(|c| { + c.as_bytes() + .windows(JSON_MARKER.len()) + .any(|w| w.eq_ignore_ascii_case(JSON_MARKER)) + }) + } + + fn head(self) -> &'a [u8] { + let Some(prefix) = self.body.get(..=TRACE_BODY_CAP) else { + return self.body; + }; + prefix + .iter() + .rposition(|b| b & UTF8_CONTINUATION_MASK != UTF8_CONTINUATION) + .and_then(|end| self.body.get(..end)) + .unwrap_or_default() } } -pub fn body_for_log(body: &[u8], content_type: Option<&str>) -> String { - if body.is_empty() { - return "".to_owned(); +const JSON_MARKER: &[u8] = b"json"; +const UTF8_CONTINUATION_MASK: u8 = 0xC0; +const UTF8_CONTINUATION: u8 = 0x80; + +impl fmt::Display for WireBody<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let body = self.body; + if body.is_empty() { + return f.write_str(""); + } + if self.is_json() + && let Ok(value) = serde_json::from_slice::(body) + && let Ok(pretty) = serde_json::to_string_pretty(&value) + { + return Capped { + text: &pretty, + total_bytes: body.len(), + } + .fmt(f); + } + let head = self.head(); + match std::str::from_utf8(head) { + Ok(text) + if !text + .chars() + .any(|c| c.is_control() && !matches!(c, '\n' | '\r' | '\t')) => + { + f.write_str(text)?; + if body.len() > head.len() { + write!(f, "\n... <{} bytes total>", body.len())?; + } + Ok(()) + } + _ => write!( + f, + "<{} bytes, blake3={}>", + body.len(), + blake3::hash(body).to_hex() + ), + } } - let is_json = content_type.is_some_and(|c| c.to_ascii_lowercase().contains("json")); - if is_json - && let Ok(value) = serde_json::from_slice::(body) - && let Ok(pretty) = serde_json::to_string_pretty(&value) - { - return cap(&pretty, body.len()); +} + +struct Capped<'a> { + text: &'a str, + total_bytes: usize, +} + +impl fmt::Display for Capped<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.text.len() <= TRACE_BODY_CAP { + return f.write_str(self.text); + } + let end = self.text.floor_char_boundary(TRACE_BODY_CAP); + f.write_str(self.text.get(..end).unwrap_or_default())?; + write!(f, "\n... <{} bytes total>", self.total_bytes) } - let head = safe_head(body, TRACE_BODY_CAP); - match std::str::from_utf8(head) { - Ok(text) if is_texty(text) => { - if body.len() > head.len() { - format!("{text}\n... <{} bytes total>", body.len()) - } else { - text.to_owned() +} + +#[derive(Clone, Copy)] +enum Redact { + Arguments, + InitialResponse, +} + +#[derive(Clone, Copy)] +pub struct WireCommand<'a>(pub &'a str); + +impl fmt::Display for WireCommand<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let trimmed = self.0.trim_end_matches(['\r', '\n']); + let mut parts = trimmed.splitn(3, ' '); + let first = parts.next().unwrap_or_default(); + let redact = hashify::map_ignore_case!( + first.as_bytes(), + Redact, + "LOGIN" => Redact::Arguments, + "AUTHENTICATE" => Redact::InitialResponse, + ); + match redact { + None => f.write_str(trimmed), + Some(Redact::Arguments) => write!(f, "{first} "), + Some(Redact::InitialResponse) => { + let mechanism = parts.next().unwrap_or_default(); + match parts.next() { + Some(_) => write!(f, "{first} {mechanism} "), + None => f.write_str(trimmed.trim_end()), + } } } - _ => blob_summary(body), } } -fn safe_head(body: &[u8], max: usize) -> &[u8] { - if body.len() <= max { - return body; - } - let mut end = max; - while end > 0 && (body[end] & 0xC0) == 0x80 { - end -= 1; - } - &body[..end] -} - -fn is_texty(text: &str) -> bool { - !text - .chars() - .any(|c| c.is_control() && !matches!(c, '\n' | '\r' | '\t')) -} - -fn blob_summary(body: &[u8]) -> String { - format!( - "<{} bytes, blake3={}>", - body.len(), - blake3::hash(body).to_hex() - ) -} - -fn cap(text: &str, total_bytes: usize) -> String { - if text.len() <= TRACE_BODY_CAP { - return text.to_owned(); - } - let end = text.floor_char_boundary(TRACE_BODY_CAP); - format!("{}\n... <{total_bytes} bytes total>", &text[..end]) -} - -pub fn redact_wire(command: &str) -> String { - let trimmed = command.trim_end_matches(['\r', '\n']); - let mut parts = trimmed.splitn(3, ' '); - let first = parts.next().unwrap_or(""); - if first.eq_ignore_ascii_case("LOGIN") { - return format!("{first} "); - } - if first.eq_ignore_ascii_case("AUTHENTICATE") { - let mechanism = parts.next().unwrap_or(""); - return match parts.next() { - Some(_) => format!("{first} {mechanism} "), - None => format!("{first} {mechanism}").trim_end().to_owned(), - }; - } - trimmed.to_owned() -} - #[cfg(test)] mod tests { use super::*; + use crate::console::Styled; #[test] fn quiet_is_level_zero() { - assert_eq!(Logger::from_flags(true, 3).level(), LEVEL_QUIET); + assert_eq!( + Logger::from_verbosity(Verbosity::Quiet).level(), + Level::Quiet + ); } #[test] fn default_is_level_one() { - assert_eq!(Logger::from_flags(false, 0).level(), LEVEL_DEFAULT); + assert_eq!( + Logger::from_verbosity(Verbosity::Verbose(0)).level(), + Level::Default + ); } #[test] fn verbose_caps_at_four() { - assert_eq!(Logger::from_flags(false, 9).level(), LEVEL_BODIES); + assert_eq!( + Logger::from_verbosity(Verbosity::Verbose(9)).level(), + Level::Bodies + ); } #[test] fn enabled_is_a_superset_scale() { - let l = Logger::from_flags(false, 1); - assert!(l.enabled(LEVEL_DEFAULT)); - assert!(l.enabled(LEVEL_PROGRESS)); - assert!(!l.enabled(LEVEL_METHOD)); + let l = Logger::from_verbosity(Verbosity::Verbose(1)); + assert!(l.enabled(Level::Default)); + assert!(l.enabled(Level::Progress)); + assert!(!l.enabled(Level::Method)); + } + + // The console is plain until set, and setting it keeps the level. + #[test] + fn with_console_keeps_the_level() { + let logger = Logger::new(Level::Progress); + assert_eq!(logger.console(), Console::default()); + let console = Console { + out: Styled::Plain, + err: Styled::Ansi, + }; + let logger = logger.with_console(console); + assert_eq!(logger.console(), console); + assert_eq!(logger.level(), Level::Progress); + } + + // Trace pieces render the unstyled text unchanged. + #[test] + fn trace_parts_keep_the_plain_text() { + assert_eq!(Millis(Duration::from_millis(42)).to_string(), "(42 ms)"); + assert_eq!(Note(Some("retry 1")).to_string(), " retry 1"); + assert_eq!(Note(None).to_string(), ""); } #[test] fn redact_wire_hides_login_arguments() { assert_eq!( - redact_wire("LOGIN \"alice\" \"s3cret\""), + WireCommand("LOGIN \"alice\" \"s3cret\"").to_string(), "LOGIN " ); } @@ -238,42 +422,64 @@ mod tests { #[test] fn redact_wire_hides_authenticate_initial_response() { assert_eq!( - redact_wire("AUTHENTICATE PLAIN AGFsaWNlAHMzY3JldA=="), + WireCommand("AUTHENTICATE PLAIN AGFsaWNlAHMzY3JldA==").to_string(), "AUTHENTICATE PLAIN " ); } #[test] fn redact_wire_keeps_authenticate_without_initial_response() { - assert_eq!(redact_wire("AUTHENTICATE PLAIN"), "AUTHENTICATE PLAIN"); + assert_eq!( + WireCommand("AUTHENTICATE PLAIN").to_string(), + "AUTHENTICATE PLAIN" + ); } #[test] fn redact_wire_passes_through_ordinary_commands() { - assert_eq!(redact_wire("SELECT INBOX\r\n"), "SELECT INBOX"); + assert_eq!(WireCommand("SELECT INBOX\r\n").to_string(), "SELECT INBOX"); } #[test] fn body_for_log_pretty_prints_json() { - let out = body_for_log(br#"{"a":1}"#, Some("application/json")); + let out = WireBody { + body: br#"{"a":1}"#, + content_type: Some("application/json"), + } + .to_string(); assert!(out.contains("\"a\": 1"), "got {out}"); } #[test] fn body_for_log_summarises_binary_with_blake3() { - let out = body_for_log(&[0u8, 159, 146, 150], Some("application/octet-stream")); + let out = WireBody { + body: &[0u8, 159, 146, 150], + content_type: Some("application/octet-stream"), + } + .to_string(); assert!(out.starts_with("<4 bytes, blake3="), "got {out}"); } #[test] fn body_for_log_reports_empty() { - assert_eq!(body_for_log(&[], None), ""); + assert_eq!( + WireBody { + body: &[], + content_type: None + } + .to_string(), + "" + ); } #[test] fn body_for_log_caps_long_text_without_scanning_whole_blob() { let body = vec![b'a'; TRACE_BODY_CAP * 4]; - let out = body_for_log(&body, Some("text/plain")); + let out = WireBody { + body: &body, + content_type: Some("text/plain"), + } + .to_string(); assert!( out.contains("bytes total>"), "got tail {}", @@ -287,13 +493,17 @@ mod tests { let mut body = vec![b'a'; TRACE_BODY_CAP - 1]; body.extend_from_slice("\u{1F4A9}".as_bytes()); body.extend_from_slice(&[b'b'; 32]); - let out = body_for_log(&body, Some("text/plain")); + let out = WireBody { + body: &body, + content_type: Some("text/plain"), + } + .to_string(); assert!(out.contains("bytes total>"), "got {out}"); } #[test] fn trace_emit_paths_do_not_panic_at_max_verbosity() { - let logger = Logger::from_flags(false, 9); + let logger = Logger::from_verbosity(Verbosity::Verbose(9)); logger.trace_http(&HttpCall { proto: "JMAP", method: "POST", diff --git a/src/main.rs b/src/main.rs index bcb1baf..e64da96 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,118 +4,28 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use clap::Parser; +use std::{env, process}; use inbuxa_migrate::cli::{Action, Cli}; -use inbuxa_migrate::error::Error; -use inbuxa_migrate::inspect; -use inbuxa_migrate::sync::{self, RunOutcome, Summary}; +use inbuxa_migrate::console::{ColorChoice, Console}; +use inbuxa_migrate::error::ExitStatus; fn main() { - let code = run(); - std::process::exit(code); -} - -fn run() -> i32 { - let cli = match Cli::try_parse() { - Ok(cli) => cli, + let choice = ColorChoice::from_args(env::args_os()); + let console = Console::detect(choice); + let status = match Cli::try_parse_styled(choice, console) { + Ok(cli) => match cli.resolve() { + Ok(action) => action.with_console(console).run(), + Err(err) => Action::fail(console, &err), + }, Err(err) => { let _ = err.print(); - return if err.use_stderr() { 1 } else { 0 }; - } - }; - - let action = match cli.resolve() { - Ok(action) => action, - Err(err) => return fail(&err), - }; - - let (outcome, logger) = match action { - Action::Import(common, config) => { - let logger = common.logger; - (sync::import_jmap::run_reporting(common, config), logger) - } - Action::ImportImap(common, config) => { - let logger = common.logger; - (sync::import_imap::run_reporting(common, config), logger) - } - Action::ImportDav(common, config) => { - let logger = common.logger; - (sync::import_dav::run_reporting(common, config), logger) - } - Action::ImportManageSieve(common, config) => { - let logger = common.logger; - ( - RunOutcome::from_result(sync::import_managesieve::run(common, config)), - logger, - ) - } - Action::ImportMaildir(common, config) => { - let logger = common.logger; - (sync::import_maildir::run_reporting(common, config), logger) - } - Action::ImportTakeout(common, config) => { - let logger = common.logger; - (sync::import_takeout::run_reporting(common, config), logger) - } - Action::ImportExchangeEws(common, config) => { - let logger = common.logger; - ( - RunOutcome::from_result(sync::import_exchange_ews::run(common, config)), - logger, - ) - } - Action::ImportExchangeGraph(common, config) => { - let logger = common.logger; - ( - RunOutcome::from_result(sync::import_exchange_graph::run(common, config)), - logger, - ) - } - Action::Export(common, config) => { - let logger = common.logger; - ( - RunOutcome::from_result(sync::export::run(common, config)), - logger, - ) - } - Action::Inspect(config) => { - return match inspect::run(config) { - Ok(()) => 0, - Err(err) => fail(&err), - }; - } - }; - - report(&outcome.summary); - match outcome.error { - Some(err) => fail(&err), - None => { - if outcome.summary.any_failed() { - logger.error("some objects failed; the archive is consistent and resumable"); - 5 + if err.use_stderr() { + ExitStatus::Usage } else { - 0 + ExitStatus::Ok } } - } -} - -fn fail(err: &Error) -> i32 { - eprintln!("error: {err}"); - err.exit_code() -} - -fn report(summary: &Summary) { - for (type_name, counts) in &summary.per_type { - println!( - "{type_name}: created={} fetched={} updated={} deleted={} skipped={} failed={}", - counts.created, - counts.fetched, - counts.updated, - counts.deleted, - counts.skipped, - counts.failed - ); - } + }; + process::exit(status.code()); } diff --git a/src/managesieve/client.rs b/src/managesieve/client.rs index c75ee23..6c917be 100644 --- a/src/managesieve/client.rs +++ b/src/managesieve/client.rs @@ -4,34 +4,30 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use std::io::{BufRead, BufReader, Read, Write}; +use std::fmt::Write as _; +use std::io::{self, BufRead, BufReader, Write}; use std::time::Instant; use encodify::base64::STANDARD; -use crate::imap::transport::{Connector, ImapStream}; use crate::logging::Logger; +use crate::net::sasl::SaslPayload; +use crate::net::stream::{Cleartext, ConnectMode, Connector, Stream}; -use super::command; +use super::command::Command; use super::error::SieveError; -use super::response::{ - Capabilities, ResponseBlock, Status, StatusLine, Token, parse_capabilities, read_response, -}; +use super::response::{Capabilities, ResponseBlock, Status, StatusLine, Token}; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ConnectMode { - ImplicitTls, - Plain, - StartTls, -} +const LOGIN: &str = "LOGIN"; pub struct SieveClient { - reader: BufReader>, - pub host: String, - pub capabilities: Capabilities, + reader: BufReader, + host: String, + capabilities: Capabilities, closed: bool, fresh_post_auth_caps: bool, logger: Logger, + wire: String, } impl SieveClient { @@ -40,31 +36,24 @@ impl SieveClient { host: &str, port: u16, mode: ConnectMode, - allow_cleartext: bool, + cleartext: Cleartext, logger: Logger, ) -> Result { - let stream: Box = match mode { + let stream = match mode { ConnectMode::ImplicitTls => connector .connect_tls(host, port) .map_err(|e| SieveError::Tls(e.to_string()))?, ConnectMode::Plain | ConnectMode::StartTls => connector .connect_plain(host, port) - .map_err(|e| SieveError::Io(std::io::Error::other(e.to_string())))?, - }; - let mut client = SieveClient { - reader: BufReader::new(stream), - host: host.to_owned(), - capabilities: Capabilities::default(), - closed: false, - fresh_post_auth_caps: false, - logger, + .map_err(|e| SieveError::Io(io::Error::other(e.to_string())))?, }; + let mut client = SieveClient::from_stream(stream, host.to_owned(), logger); client.read_initial_capabilities()?; - if matches!(mode, ConnectMode::StartTls) { + if mode == ConnectMode::StartTls { if client.capabilities.starttls { client.start_tls(connector)?; client.read_initial_capabilities()?; - } else if !allow_cleartext { + } else if cleartext == Cleartext::Forbid { return Err(SieveError::Unsupported( "server does not advertise STARTTLS; pass --allow-cleartext to permit \ credentials over an unencrypted socket" @@ -75,19 +64,27 @@ impl SieveClient { Ok(client) } + fn from_stream(stream: Stream, host: String, logger: Logger) -> SieveClient { + SieveClient { + reader: BufReader::new(stream), + host, + capabilities: Capabilities::default(), + closed: false, + fresh_post_auth_caps: false, + logger, + wire: String::with_capacity(64), + } + } + pub fn capabilities(&self) -> &Capabilities { &self.capabilities } - pub fn is_closed(&self) -> bool { - self.closed - } - fn read_initial_capabilities(&mut self) -> Result<(), SieveError> { - let block = read_response(&mut self.reader)?; + let block = ResponseBlock::read(&mut self.reader)?; match block.status.status { Status::Ok => { - self.capabilities = parse_capabilities(&block.data); + self.capabilities = Capabilities::parse(&block.data); Ok(()) } Status::Bye => { @@ -99,9 +96,8 @@ impl SieveClient { } pub fn refresh_capabilities(&mut self) -> Result<(), SieveError> { - let block = self.run(&command::capability())?; - let caps = parse_capabilities(&block.data); - self.capabilities = caps; + let block = self.run(Command::Capability)?; + self.capabilities = Capabilities::parse(&block.data); self.fresh_post_auth_caps = true; Ok(()) } @@ -110,141 +106,94 @@ impl SieveClient { self.fresh_post_auth_caps } - fn consume_post_auth_caps(&mut self, data: &[Vec]) { + fn consume_post_auth_caps(&mut self, data: &[Vec]) { if data.is_empty() { return; } - let parsed = parse_capabilities(data); - let mentions_any = - !parsed.sasl.is_empty() || parsed.implementation.is_some() || parsed.version.is_some(); - if mentions_any { + let parsed = Capabilities::parse(data); + if !parsed.sasl.is_empty() || parsed.implementation.is_some() || parsed.version.is_some() { self.capabilities = parsed; self.fresh_post_auth_caps = true; } } fn start_tls(&mut self, connector: &Connector) -> Result<(), SieveError> { - let block = self.run(&command::starttls())?; - let _ = block; - let stream = self.reader.get_ref(); - if stream.is_tls() { + self.run(Command::StartTls)?; + if self.reader.get_ref().is_tls() { return Err(SieveError::Protocol( "STARTTLS issued but stream already TLS".into(), )); } - let buffered = std::mem::replace(&mut self.reader, BufReader::new(noop_stream())); - if !buffered.buffer().is_empty() { + if !self.reader.buffer().is_empty() { return Err(SieveError::Protocol( "buffered data found after STARTTLS OK; possible response injection".into(), )); } - let inner = buffered.into_inner(); - let upgraded = inner + let upgraded = self + .reader + .get_mut() + .take() .upgrade_tls(connector, &self.host) .map_err(|e| SieveError::Tls(e.to_string()))?; self.reader = BufReader::new(upgraded); Ok(()) } - pub fn run(&mut self, command: &str) -> Result { + fn run(&mut self, command: Command<'_>) -> Result { let started = Instant::now(); - self.write_all(command.as_bytes())?; - let block = read_response(&mut self.reader)?; + self.send(command)?; + let block = ResponseBlock::read(&mut self.reader)?; self.logger.trace_cmd( "SIEVE", - command, - &sieve_status(&block.status), + &self.wire, + &block.status.to_string(), started.elapsed(), ); - finish_block(block, &mut self.closed) - } - - pub fn run_raw(&mut self, bytes: &[u8]) -> Result { - let started = Instant::now(); - self.write_all(bytes)?; - let block = read_response(&mut self.reader)?; - self.logger.trace_cmd( - "SIEVE", - &String::from_utf8_lossy(bytes), - &sieve_status(&block.status), - started.elapsed(), - ); - finish_block(block, &mut self.closed) + self.finish(block) } pub fn listscripts(&mut self) -> Result { - self.run(&command::listscripts()) + self.run(Command::ListScripts) } - pub fn getscript(&mut self, name: &str) -> Result { - self.run(&command::getscript(name)) + pub fn getscript(&mut self, name: &str) -> Result, SieveError> { + self.run(Command::GetScript(name))?.into_script() } - pub fn noop(&mut self) -> Result<(), SieveError> { - let _ = self.run(&command::noop())?; - Ok(()) - } - - pub fn logout(&mut self) -> Result<(), SieveError> { - if self.closed { - return Ok(()); + pub fn logout(&mut self) { + if !self.closed { + let _ = self.send(Command::Logout); + self.closed = true; } - let _ = self.write_all(command::logout().as_bytes()); - self.closed = true; - Ok(()) } - pub fn authenticate_plain(&mut self, authcid: &str, password: &str) -> Result<(), SieveError> { - let mut payload: Vec = Vec::with_capacity(authcid.len() + password.len() + 3); - payload.push(0); - payload.extend_from_slice(authcid.as_bytes()); - payload.push(0); - payload.extend_from_slice(password.as_bytes()); - let encoded = STANDARD.encode(&payload); - self.send_authenticate("PLAIN", &encoded) + pub fn authenticate(&mut self, payload: SaslPayload<'_>) -> Result<(), SieveError> { + self.fresh_post_auth_caps = false; + let initial = payload.base64(); + self.send(Command::Authenticate { + mechanism: payload.mechanism(), + initial: Some(&initial), + })?; + let block = ResponseBlock::read(&mut self.reader)?; + self.finish_authentication(block) } pub fn authenticate_login(&mut self, authcid: &str, password: &str) -> Result<(), SieveError> { self.fresh_post_auth_caps = false; - self.write_all(command::authenticate("LOGIN").as_bytes())?; - let user_payload = STANDARD.encode(authcid); - self.expect_continuation()?; - self.write_all(command::continuation_payload(&user_payload).as_bytes())?; - let pass_payload = STANDARD.encode(password); - self.expect_continuation()?; - self.write_all(command::continuation_payload(&pass_payload).as_bytes())?; - let block = read_response(&mut self.reader)?; - match finish_block(block, &mut self.closed) { - Ok(block) => { - self.consume_post_auth_caps(&block.data); - Ok(()) - } - Err(SieveError::No(no)) if no.is_referral() => { - Err(SieveError::Referral(no.to_string())) - } - Err(other) => Err(other), + self.send(Command::Authenticate { + mechanism: LOGIN, + initial: None, + })?; + for secret in [authcid, password] { + self.expect_continuation()?; + self.send(Command::Continuation(&STANDARD.encode(secret)))?; } + let block = ResponseBlock::read(&mut self.reader)?; + self.finish_authentication(block) } - pub fn authenticate_oauthbearer(&mut self, user: &str, token: &str) -> Result<(), SieveError> { - let mut payload = String::with_capacity(user.len() + token.len() + 32); - payload.push_str("n,a="); - payload.push_str(user); - payload.push(','); - payload.push('\x01'); - payload.push_str("auth=Bearer "); - payload.push_str(token); - payload.push('\x01'); - payload.push('\x01'); - let encoded = STANDARD.encode(&payload); - self.send_authenticate("OAUTHBEARER", &encoded) - } - - fn send_authenticate(&mut self, mechanism: &str, b64: &str) -> Result<(), SieveError> { - self.fresh_post_auth_caps = false; - self.write_all(command::authenticate_with_initial(mechanism, b64).as_bytes())?; - let block = read_response(&mut self.reader)?; - match finish_block(block, &mut self.closed) { + fn finish_authentication(&mut self, block: ResponseBlock) -> Result<(), SieveError> { + match self.finish(block) { Ok(block) => { self.consume_post_auth_caps(&block.data); Ok(()) @@ -260,43 +209,60 @@ impl SieveClient { let mut line = Vec::new(); loop { line.clear(); - let n = self.reader.read_until(b'\n', &mut line)?; - if n == 0 { - return Err(SieveError::Io(std::io::Error::new( - std::io::ErrorKind::UnexpectedEof, + if self.reader.read_until(b'\n', &mut line)? == 0 { + return Err(SieveError::Io(io::Error::new( + io::ErrorKind::UnexpectedEof, "expected continuation", ))); } while matches!(line.last(), Some(b'\n') | Some(b'\r')) { line.pop(); } + let start = line + .iter() + .position(|b| !b.is_ascii_whitespace()) + .unwrap_or(line.len()); + let trimmed = line.get(start..).unwrap_or_default(); if line.is_empty() { continue; } - let s = String::from_utf8_lossy(&line); - let trimmed = s.trim_start(); - if trimmed.starts_with('"') || trimmed.starts_with('{') { + if trimmed.starts_with(b"\"") || trimmed.starts_with(b"{") { return Ok(()); } - if trimmed.starts_with("NO") || trimmed.starts_with("BYE") { - let line_tokens = tokens_from_text(&line)?; - let status = super::response::try_parse_status(&line_tokens) - .ok_or_else(|| SieveError::Parse(format!("bad status: {s:?}")))?; - let block = ResponseBlock { - data: Vec::new(), - status, - }; - return finish_block(block, &mut self.closed).map(|_| ()); + if trimmed.starts_with(b"NO") || trimmed.starts_with(b"BYE") { + let status = StatusLine::from_line(&line)?.ok_or_else(|| { + SieveError::Parse(format!("bad status: {:?}", String::from_utf8_lossy(&line))) + })?; + return self + .finish(ResponseBlock { + data: Vec::new(), + status, + }) + .map(drop); } return Err(SieveError::Parse(format!( - "unexpected line during AUTHENTICATE LOGIN: {s:?}" + "unexpected line during AUTHENTICATE LOGIN: {:?}", + String::from_utf8_lossy(&line) ))); } } - fn write_all(&mut self, bytes: &[u8]) -> Result<(), SieveError> { + fn finish(&mut self, block: ResponseBlock) -> Result { + match block.status.status { + Status::Ok => Ok(block), + Status::No => Err(SieveError::No(block.status.into_no_error())), + Status::Bye => { + self.closed = true; + Err(SieveError::Bye(block.status.text)) + } + } + } + + fn send(&mut self, command: Command<'_>) -> Result<(), SieveError> { + self.wire.clear(); + let _ = write!(self.wire, "{command}"); let stream = self.reader.get_mut(); - stream.write_all(bytes)?; + stream.write_all(self.wire.as_bytes())?; stream.flush()?; Ok(()) } @@ -304,168 +270,30 @@ impl SieveClient { impl Drop for SieveClient { fn drop(&mut self) { - if !self.closed { - let _ = self.logout(); - } - } -} - -impl SieveClient { - pub fn new_from_stream(stream: Box, host: String) -> Self { - SieveClient { - reader: BufReader::new(stream), - host, - capabilities: Capabilities::default(), - closed: false, - fresh_post_auth_caps: false, - logger: Logger::from_flags(true, 0), - } - } - - pub fn read_response_block(&mut self) -> Result { - let block = read_response(&mut self.reader)?; - finish_block(block, &mut self.closed) - } -} - -fn tokens_from_text(line: &[u8]) -> Result, SieveError> { - let mut buf: Vec = line.to_vec(); - buf.extend_from_slice(b"\r\n"); - let mut cursor = std::io::Cursor::new(buf); - let mut reader = BufReader::new(&mut cursor); - let block = read_response_with_eof_ok(&mut reader)?; - if let Some(line) = block.data.into_iter().next() { - return Ok(line); - } - Ok(Vec::new()) -} - -fn read_response_with_eof_ok( - reader: &mut R, -) -> Result { - match read_response(reader) { - Ok(b) => Ok(b), - Err(SieveError::Io(_)) => Ok(ResponseBlock::default()), - Err(e) => Err(e), - } -} - -fn finish_block(block: ResponseBlock, closed: &mut bool) -> Result { - match block.status.status { - Status::Ok => Ok(block), - Status::No => { - let no = StatusLine::into_no_error(block.status); - Err(SieveError::No(no)) - } - Status::Bye => { - *closed = true; - Err(SieveError::Bye(block.status.text)) - } - } -} - -fn sieve_status(status: &StatusLine) -> String { - let word = match status.status { - Status::Ok => "OK", - Status::No => "NO", - Status::Bye => "BYE", - }; - format!("{word} {}", status.text) -} - -fn noop_stream() -> Box { - Box::new(NoopStream) -} - -struct NoopStream; - -impl Read for NoopStream { - fn read(&mut self, _buf: &mut [u8]) -> std::io::Result { - Ok(0) - } -} - -impl Write for NoopStream { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - Ok(buf.len()) - } - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } -} - -impl ImapStream for NoopStream { - fn upgrade_tls( - self: Box, - _connector: &Connector, - _host: &str, - ) -> Result, crate::imap::ImapError> { - Ok(self) - } - fn is_tls(&self) -> bool { - false + self.logout(); } } #[cfg(test)] mod tests { use super::*; - use crate::imap::ImapError; - use std::io::Cursor; - - struct MockStream { - reader: Cursor>, - written: Vec, - } - - impl MockStream { - fn new(server_says: &[u8]) -> Self { - MockStream { - reader: Cursor::new(server_says.to_vec()), - written: Vec::new(), - } - } - } - - impl Read for MockStream { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - self.reader.read(buf) - } - } - - impl Write for MockStream { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - self.written.extend_from_slice(buf); - Ok(buf.len()) - } - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } - } - - impl ImapStream for MockStream { - fn upgrade_tls( - self: Box, - _c: &Connector, - _h: &str, - ) -> Result, ImapError> { - Ok(self) - } - fn is_tls(&self) -> bool { - false - } - } + use crate::logging::Level; + use crate::managesieve::response::ListedScript; fn client_with(server: &[u8]) -> SieveClient { - SieveClient::new_from_stream(Box::new(MockStream::new(server)), "test.example".to_owned()) + SieveClient::from_stream( + Stream::mock(server), + "test.example".to_owned(), + Logger::new(Level::Quiet), + ) } #[test] fn run_listscripts_returns_data_and_ok() { let server = b"\"a\"\r\n\"b\" ACTIVE\r\nOK\r\n"; let mut c = client_with(server); - let r = c.listscripts().unwrap(); - let scripts = super::super::response::parse_listscripts(&r.data).unwrap(); + let block = c.listscripts().unwrap(); + let scripts = ListedScript::parse_all(&block.data).unwrap(); assert_eq!(scripts.len(), 2); assert!(scripts[1].active); } @@ -478,7 +306,6 @@ mod tests { match err { SieveError::No(no) => { assert_eq!(no.code.as_deref(), Some("NONEXISTENT")); - assert!(no.is_nonexistent()); } other => panic!("expected No, got {other:?}"), } @@ -488,23 +315,31 @@ mod tests { fn bye_closes_client() { let server = b"BYE \"going away\"\r\n"; let mut c = client_with(server); - let err = c.noop().unwrap_err(); + let err = c.listscripts().unwrap_err(); assert!(matches!(err, SieveError::Bye(_))); - assert!(c.is_closed()); + assert!(c.closed); } #[test] fn authenticate_plain_sends_initial_response() { let server = b"OK\r\n"; let mut c = client_with(server); - c.authenticate_plain("alice", "p@ss").unwrap(); + c.authenticate(SaslPayload::Plain { + authcid: "alice", + password: "p@ss", + }) + .unwrap(); } #[test] fn authenticate_oauthbearer_sends_initial_response() { let server = b"OK\r\n"; let mut c = client_with(server); - c.authenticate_oauthbearer("alice@x", "tok").unwrap(); + c.authenticate(SaslPayload::OAuthBearer { + user: "alice@x", + token: "tok", + }) + .unwrap(); } #[test] @@ -518,7 +353,12 @@ mod tests { fn authenticate_no_response_propagates_as_no() { let server = b"NO \"bad creds\"\r\n"; let mut c = client_with(server); - let err = c.authenticate_plain("alice", "wrong").unwrap_err(); + let err = c + .authenticate(SaslPayload::Plain { + authcid: "alice", + password: "wrong", + }) + .unwrap_err(); assert!(matches!(err, SieveError::No(_)), "got {err:?}"); } @@ -526,7 +366,32 @@ mod tests { fn authenticate_referral_no_translates_to_referral() { let server = b"NO (REFERRAL \"sieve://other\") \"go\"\r\n"; let mut c = client_with(server); - let err = c.authenticate_plain("alice", "x").unwrap_err(); + let err = c + .authenticate(SaslPayload::Plain { + authcid: "alice", + password: "x", + }) + .unwrap_err(); assert!(matches!(err, SieveError::Referral(_)), "got {err:?}"); } + + #[test] + fn authenticate_login_no_at_a_continuation_is_a_no() { + // a NO in place of a LOGIN challenge surfaces as SieveError::No, not a parse error + let server = b"NO \"bad creds\"\r\n"; + let mut c = client_with(server); + let err = c.authenticate_login("alice", "wrong").unwrap_err(); + assert!(matches!(err, SieveError::No(_)), "got {err:?}"); + } + + #[test] + fn plain_initial_response_is_the_sasl_payload() { + let mut c = client_with(b"OK\r\n"); + c.authenticate(SaslPayload::Plain { + authcid: "alice", + password: "p@ss", + }) + .unwrap(); + assert_eq!(c.wire, "AUTHENTICATE \"PLAIN\" \"AGFsaWNlAHBAc3M=\"\r\n"); + } } diff --git a/src/managesieve/command.rs b/src/managesieve/command.rs index 94ac480..0a2dbec 100644 --- a/src/managesieve/command.rs +++ b/src/managesieve/command.rs @@ -4,80 +4,71 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use super::name::encode_argument; +use std::fmt; -pub fn capability() -> String { - "CAPABILITY\r\n".to_owned() +use super::name::SieveString; + +#[derive(Debug, Clone, Copy)] +pub enum Command<'a> { + Capability, + StartTls, + Logout, + ListScripts, + GetScript(&'a str), + Authenticate { + mechanism: &'a str, + initial: Option<&'a str>, + }, + Continuation(&'a str), } -pub fn starttls() -> String { - "STARTTLS\r\n".to_owned() +impl fmt::Display for Command<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + Command::Capability => f.write_str("CAPABILITY")?, + Command::StartTls => f.write_str("STARTTLS")?, + Command::Logout => f.write_str("LOGOUT")?, + Command::ListScripts => f.write_str("LISTSCRIPTS")?, + Command::GetScript(name) => write!(f, "GETSCRIPT {}", SieveString(name))?, + Command::Authenticate { + mechanism, + initial: None, + } => write!(f, "AUTHENTICATE {}", SieveString(mechanism))?, + Command::Authenticate { + mechanism, + initial: Some(initial), + } => write!( + f, + "AUTHENTICATE {} {}", + SieveString(mechanism), + SieveString(initial) + )?, + Command::Continuation(payload) => write!(f, "{}", SieveString(payload))?, + } + f.write_str("\r\n") + } } -pub fn noop() -> String { - "NOOP\r\n".to_owned() -} - -pub fn logout() -> String { - "LOGOUT\r\n".to_owned() -} - -pub fn listscripts() -> String { - "LISTSCRIPTS\r\n".to_owned() -} - -pub fn getscript(name: &str) -> String { - let mut out = String::with_capacity(name.len() + 16); - out.push_str("GETSCRIPT "); - encode_argument(name, &mut out); - out.push_str("\r\n"); - out -} - -pub fn authenticate(mechanism: &str) -> String { - let mut out = String::with_capacity(mechanism.len() + 20); - out.push_str("AUTHENTICATE "); - encode_argument(mechanism, &mut out); - out.push_str("\r\n"); - out -} - -pub fn authenticate_with_initial(mechanism: &str, initial_b64: &str) -> String { - let mut out = String::with_capacity(mechanism.len() + initial_b64.len() + 32); - out.push_str("AUTHENTICATE "); - encode_argument(mechanism, &mut out); - out.push(' '); - encode_argument(initial_b64, &mut out); - out.push_str("\r\n"); - out -} - -pub fn continuation_payload(initial_b64: &str) -> String { - let mut out = String::with_capacity(initial_b64.len() + 8); - encode_argument(initial_b64, &mut out); - out.push_str("\r\n"); - out -} - -pub use super::name::contains_literal; - #[cfg(test)] mod tests { use super::*; #[test] fn capability_is_uppercase_with_crlf() { - assert_eq!(capability(), "CAPABILITY\r\n"); + assert_eq!(Command::Capability.to_string(), "CAPABILITY\r\n"); } #[test] fn getscript_quoted_for_ascii_names() { - assert_eq!(getscript("vacation"), "GETSCRIPT \"vacation\"\r\n"); + assert_eq!( + Command::GetScript("vacation").to_string(), + "GETSCRIPT \"vacation\"\r\n" + ); } #[test] fn getscript_escapes_quotes_and_backslashes() { - let out = getscript("a\"b\\c"); + let out = Command::GetScript("a\"b\\c").to_string(); assert!(out.contains("\\\"")); assert!(out.contains("\\\\")); } @@ -85,7 +76,7 @@ mod tests { #[test] fn getscript_uses_literal_for_non_ascii() { let name = "fancy\u{00e7}y"; - let out = getscript(name); + let out = Command::GetScript(name).to_string(); let nbytes = name.len(); let expected = { let mut v: Vec = Vec::new(); @@ -104,7 +95,7 @@ mod tests { #[test] fn getscript_uses_literal_for_names_containing_crlf() { let name = "with\nlinebreak"; - let out = getscript(name); + let out = Command::GetScript(name).to_string(); assert!( out.contains("{") && out.contains("+}\r\n"), "expected literal for newline-bearing name: {out:?}" @@ -113,24 +104,26 @@ mod tests { #[test] fn authenticate_with_initial_emits_two_arguments() { - let out = authenticate_with_initial("PLAIN", "AGZvbwBiYXI="); + let out = Command::Authenticate { + mechanism: "PLAIN", + initial: Some("AGZvbwBiYXI="), + } + .to_string(); assert_eq!(out, "AUTHENTICATE \"PLAIN\" \"AGZvbwBiYXI=\"\r\n"); } #[test] fn authenticate_without_initial_emits_one_argument() { - let out = authenticate("LOGIN"); + let out = Command::Authenticate { + mechanism: "LOGIN", + initial: None, + } + .to_string(); assert_eq!(out, "AUTHENTICATE \"LOGIN\"\r\n"); } #[test] fn continuation_payload_quotes_simple_base64() { - assert_eq!(continuation_payload("Zm9v"), "\"Zm9v\"\r\n"); - } - - #[test] - fn contains_literal_detects_open_brace() { - assert!(contains_literal(b"GETSCRIPT {3+}\r\nabc\r\n")); - assert!(!contains_literal(b"GETSCRIPT \"abc\"\r\n")); + assert_eq!(Command::Continuation("Zm9v").to_string(), "\"Zm9v\"\r\n"); } } diff --git a/src/managesieve/error.rs b/src/managesieve/error.rs index f282cfc..6a675fe 100644 --- a/src/managesieve/error.rs +++ b/src/managesieve/error.rs @@ -34,6 +34,16 @@ pub enum SieveError { Referral(String), } +const TRANSIENT_MARKERS: &[&str] = &[ + "temp", + "try again", + "try later", + "unavailable", + "throttle", + "rate limit", + "too many", +]; + #[derive(Debug, Clone)] pub struct NoError { pub text: String, @@ -49,53 +59,35 @@ impl NoError { } pub fn is_transient(&self) -> bool { - if let Some(c) = &self.code { - let cu = c.to_ascii_uppercase(); - if cu == "TRYLATER" { + if let Some(code) = &self.code { + if code.eq_ignore_ascii_case("TRYLATER") { return true; } - if matches!( - cu.as_str(), - "QUOTA" - | "QUOTA/MAXSCRIPTS" - | "AUTH-TOO-WEAK" - | "ENCRYPT-NEEDED" - | "TRANSITION-NEEDED" - | "NONEXISTENT" - | "ALREADYEXISTS" - | "ACTIVE" - | "SASL" - | "TAG" - | "WARNINGS" + if hashify::set_ignore_case!( + code.as_bytes(), + "QUOTA", + "QUOTA/MAXSCRIPTS", + "AUTH-TOO-WEAK", + "ENCRYPT-NEEDED", + "TRANSITION-NEEDED", + "NONEXISTENT", + "ALREADYEXISTS", + "ACTIVE", + "SASL", + "TAG", + "WARNINGS", ) { return false; } } let lower = self.text.to_ascii_lowercase(); - const TRANSIENT_MARKERS: &[&str] = &[ - "temp", - "try again", - "try later", - "unavailable", - "throttle", - "rate limit", - "too many", - ]; TRANSIENT_MARKERS.iter().any(|m| lower.contains(m)) } pub fn is_referral(&self) -> bool { - matches!( - self.code.as_deref().map(str::to_ascii_uppercase).as_deref(), - Some("REFERRAL") - ) - } - - pub fn is_nonexistent(&self) -> bool { - matches!( - self.code.as_deref().map(str::to_ascii_uppercase).as_deref(), - Some("NONEXISTENT") - ) + self.code + .as_deref() + .is_some_and(|c| c.eq_ignore_ascii_case("REFERRAL")) } } @@ -152,12 +144,6 @@ mod tests { assert!(!NoError::new("x", None).is_referral()); } - #[test] - fn nonexistent_code_detected() { - assert!(NoError::new("x", Some("NONEXISTENT".into())).is_nonexistent()); - assert!(NoError::new("x", Some("nonexistent".into())).is_nonexistent()); - } - #[test] fn non_transient_no_not_flagged() { assert!(!NoError::new("Permission denied", None).is_transient()); diff --git a/src/managesieve/mod.rs b/src/managesieve/mod.rs index dfbebad..643d05e 100644 --- a/src/managesieve/mod.rs +++ b/src/managesieve/mod.rs @@ -10,10 +10,8 @@ pub mod error; pub mod name; pub mod response; pub mod retry; +pub mod session; -pub use client::{ConnectMode, SieveClient}; +pub use client::SieveClient; pub use error::{NoError, SieveError}; -pub use response::{ - Capabilities, ListedScript, ResponseBlock, Status, StatusLine, Token, parse_capabilities, - parse_getscript, parse_listscripts, read_response, try_parse_status, -}; +pub use response::{Capabilities, ListedScript, ResponseBlock, Status, StatusLine, Token}; diff --git a/src/managesieve/name.rs b/src/managesieve/name.rs index c774459..120c4b1 100644 --- a/src/managesieve/name.rs +++ b/src/managesieve/name.rs @@ -4,46 +4,50 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -pub fn encode_argument(s: &str, out: &mut String) { - let bytes = s.as_bytes(); - if can_be_quoted(bytes) { - out.push('"'); - for ch in s.chars() { - if ch == '"' || ch == '\\' { - out.push('\\'); +use std::fmt; + +const MAX_QUOTED_LEN: usize = 1024; + +#[derive(Debug, Clone, Copy)] +pub struct SieveString<'a>(pub &'a str); + +impl SieveString<'_> { + pub fn can_be_quoted(self) -> bool { + let bytes = self.0.as_bytes(); + bytes.len() <= MAX_QUOTED_LEN + && bytes + .iter() + .all(|&b| b == b'\t' || (0x20..=0x7e).contains(&b)) + } +} + +impl fmt::Display for SieveString<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.can_be_quoted() { + f.write_str("\"")?; + for part in self.0.split_inclusive(['"', '\\']) { + match part.strip_suffix(['"', '\\']) { + Some(head) => { + f.write_str(head)?; + f.write_str("\\")?; + f.write_str(part.get(head.len()..).unwrap_or_default())?; + } + None => f.write_str(part)?, + } } - out.push(ch); + f.write_str("\"") + } else { + write!(f, "{{{}+}}\r\n{}", self.0.len(), self.0) } - out.push('"'); - } else { - out.push('{'); - out.push_str(&bytes.len().to_string()); - out.push_str("+}\r\n"); - out.push_str(s); } } -pub fn can_be_quoted(bytes: &[u8]) -> bool { - if bytes.len() > 1024 { - return false; - } - bytes - .iter() - .all(|&b| b != b'\r' && b != b'\n' && b != 0 && (b == b'\t' || (0x20..=0x7e).contains(&b))) -} - -pub fn contains_literal(bytes: &[u8]) -> bool { - bytes.contains(&b'{') -} - #[cfg(test)] mod tests { use super::*; fn encoded(s: &str) -> String { - let mut out = String::new(); - encode_argument(s, &mut out); - out + SieveString(s).to_string() } #[test] @@ -101,23 +105,17 @@ mod tests { #[test] fn tab_is_allowed_in_quoted_string() { - assert!(can_be_quoted(b"abc\tdef")); + assert!(SieveString("abc\tdef").can_be_quoted()); } #[test] fn nul_byte_forces_literal() { - assert!(!can_be_quoted(b"a\0b")); + assert!(!SieveString("a\0b").can_be_quoted()); } #[test] fn names_over_quoted_threshold_use_literal() { - let long: Vec = (0..2048).map(|_| b'x').collect(); - assert!(!can_be_quoted(&long)); - } - - #[test] - fn contains_literal_detects_open_brace() { - assert!(contains_literal(b"GETSCRIPT {3+}\r\nabc\r\n")); - assert!(!contains_literal(b"GETSCRIPT \"abc\"\r\n")); + let long = "x".repeat(2048); + assert!(!SieveString(&long).can_be_quoted()); } } diff --git a/src/managesieve/response.rs b/src/managesieve/response.rs index f67032d..53841c0 100644 --- a/src/managesieve/response.rs +++ b/src/managesieve/response.rs @@ -4,10 +4,13 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use std::io::{self, BufRead}; +use std::fmt; +use std::io::{self, BufRead, Read}; use super::error::{NoError, SieveError}; +const LITERAL_RESERVE: usize = 1 << 20; + #[derive(Debug, Clone, PartialEq, Eq)] pub enum Token { Atom(String), @@ -18,31 +21,157 @@ pub enum Token { } impl Token { - pub fn as_string_bytes(&self) -> Option> { + pub fn as_string(&self) -> Option { match self { - Token::QStr(s) => Some(s.as_bytes().to_vec()), - Token::Literal(b) => Some(b.clone()), + Token::QStr(s) | Token::Atom(s) => Some(s.clone()), + Token::Literal(b) => Some(String::from_utf8_lossy(b).into_owned()), _ => None, } } - pub fn as_string(&self) -> Option { - match self { - Token::QStr(s) => Some(s.clone()), - Token::Literal(b) => Some(String::from_utf8_lossy(b).into_owned()), - Token::Atom(s) => Some(s.clone()), - _ => None, + fn read_line(reader: &mut R) -> Result, SieveError> { + let mut tokens: Vec = Vec::new(); + let mut physical = Vec::with_capacity(256); + Token::read_physical(reader, &mut physical)?; + while let Some(len) = Token::scan(&physical, &mut tokens)? { + tokens.push(Token::Literal(Token::read_literal(reader, len)?)); + Token::read_physical(reader, &mut physical)?; } + Ok(tokens) + } + + fn scan(line: &[u8], tokens: &mut Vec) -> Result, SieveError> { + let mut rest = line; + loop { + let skip = rest + .iter() + .take_while(|b| matches!(b, b' ' | b'\t')) + .count(); + rest = rest.get(skip..).unwrap_or_default(); + let Some((&first, tail)) = rest.split_first() else { + return Ok(None); + }; + match first { + b'"' => { + let (s, after) = Token::quoted(tail)?; + tokens.push(Token::QStr(s)); + rest = after; + } + b'{' => { + let (n, trailing) = Token::literal_marker(tail)?; + if trailing.iter().any(|b| !matches!(b, b' ' | b'\t')) { + return Err(SieveError::Parse(format!( + "literal marker {{{n}}} not at end of line: trailing {:?}", + String::from_utf8_lossy(trailing) + ))); + } + return Ok(Some(n)); + } + b'(' => { + tokens.push(Token::LParen); + rest = tail; + } + b')' => { + tokens.push(Token::RParen); + rest = tail; + } + _ => { + let len = rest + .iter() + .position(|b| matches!(b, b' ' | b'\t' | b'"' | b'{' | b'(' | b')')) + .unwrap_or(rest.len()); + let (atom, after) = rest.split_at_checked(len).unwrap_or((rest, &[])); + tokens.push(Token::Atom(String::from_utf8_lossy(atom).into_owned())); + rest = after; + } + } + } + } + + fn read_physical(reader: &mut R, buf: &mut Vec) -> Result<(), SieveError> { + buf.clear(); + let n = reader.read_until(b'\n', buf)?; + if n == 0 { + return Err(SieveError::Io(io::Error::new( + io::ErrorKind::UnexpectedEof, + "managesieve closed stream", + ))); + } + while matches!(buf.last(), Some(b'\n') | Some(b'\r')) { + buf.pop(); + } + Ok(()) + } + + fn read_literal(reader: &mut R, len: usize) -> Result, SieveError> { + let mut data = Vec::with_capacity(len.min(LITERAL_RESERVE)); + reader + .take(u64::try_from(len).unwrap_or(u64::MAX)) + .read_to_end(&mut data)?; + if data.len() != len { + return Err(SieveError::Io(io::Error::new( + io::ErrorKind::UnexpectedEof, + "failed to fill whole buffer", + ))); + } + Ok(data) + } + + fn quoted(buf: &[u8]) -> Result<(String, &[u8]), SieveError> { + let mut bytes: Vec = Vec::new(); + let mut iter = buf.iter().enumerate(); + while let Some((i, &b)) = iter.next() { + match b { + b'"' => { + let s = String::from_utf8_lossy(&bytes).into_owned(); + return Ok((s, buf.get(i + 1..).unwrap_or_default())); + } + b'\\' => match iter.next() { + Some((_, &escaped)) => bytes.push(escaped), + None => break, + }, + _ => bytes.push(b), + } + } + Err(SieveError::Parse("unterminated quoted-string".into())) + } + + fn literal_marker(buf: &[u8]) -> Result<(usize, &[u8]), SieveError> { + let digits = buf.iter().take_while(|b| b.is_ascii_digit()).count(); + let (number, rest) = buf.split_at_checked(digits).unwrap_or((buf, &[])); + if number.is_empty() { + return Err(SieveError::Parse("literal marker missing digits".into())); + } + let n = number.iter().try_fold(0usize, |n, d| { + n.checked_mul(10) + .and_then(|v| v.checked_add(usize::from(d - b'0'))) + .ok_or_else(|| SieveError::Parse("literal size overflow".into())) + })?; + let rest = rest.strip_prefix(b"+").unwrap_or(rest); + let rest = rest + .strip_prefix(b"}") + .ok_or_else(|| SieveError::Parse("literal marker missing closing brace".into()))?; + Ok((n, rest)) } } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Status { Ok, No, Bye, } +impl fmt::Display for Status { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Status::Ok => "OK", + Status::No => "NO", + Status::Bye => "BYE", + }) + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct StatusLine { pub status: Status, @@ -62,10 +191,91 @@ impl Default for StatusLine { } } +impl fmt::Display for StatusLine { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{} {}", self.status, self.text) + } +} + impl StatusLine { pub fn into_no_error(self) -> NoError { NoError::new(self.text, self.code) } + + pub fn from_line(line: &[u8]) -> Result, SieveError> { + let mut tokens = Vec::new(); + if Token::scan(line, &mut tokens)?.is_some() { + return Ok(None); + } + Ok(StatusLine::parse(&tokens)) + } + + pub fn parse(tokens: &[Token]) -> Option { + let (Token::Atom(word), mut rest) = tokens.split_first()? else { + return None; + }; + let status = *hashify::map!(word.as_bytes(), Status, + "OK" => Status::Ok, + "ok" => Status::Ok, + "Ok" => Status::Ok, + "NO" => Status::No, + "no" => Status::No, + "No" => Status::No, + "BYE" => Status::Bye, + "bye" => Status::Bye, + "Bye" => Status::Bye, + )?; + let mut code: Option = None; + let mut code_args: Option = None; + if let Some((Token::LParen, after)) = rest.split_first() { + let mut depth: u32 = 1; + let mut consumed = 0usize; + let mut inside: Vec<&Token> = Vec::new(); + for t in after { + consumed += 1; + match t { + Token::LParen => depth += 1, + Token::RParen => { + depth -= 1; + if depth == 0 { + break; + } + } + _ => {} + } + inside.push(t); + } + rest = after.get(consumed..).unwrap_or_default(); + if let Some((name, args)) = inside.split_first() { + code = name.as_string(); + let mut joined = String::new(); + for (i, s) in args + .iter() + .enumerate() + .filter_map(|(i, t)| t.as_string().map(|s| (i, s))) + { + if i > 0 { + joined.push(' '); + } + joined.push_str(&s); + } + if !joined.is_empty() { + code_args = Some(joined); + } + } + } + let text = match rest.first() { + Some(Token::QStr(s) | Token::Atom(s)) => s.clone(), + Some(Token::Literal(b)) => String::from_utf8_lossy(b).into_owned(), + _ => String::new(), + }; + Some(StatusLine { + status, + code, + code_args, + text, + }) + } } #[derive(Debug, Default)] @@ -74,206 +284,38 @@ pub struct ResponseBlock { pub status: StatusLine, } -pub fn read_response(reader: &mut R) -> Result { - let mut data: Vec> = Vec::new(); - loop { - let line = read_logical_line(reader)?; - if line.is_empty() { - continue; +impl ResponseBlock { + pub fn read(reader: &mut R) -> Result { + let mut data: Vec> = Vec::new(); + loop { + let line = Token::read_line(reader)?; + if line.is_empty() { + continue; + } + if let Some(status) = StatusLine::parse(&line) { + return Ok(ResponseBlock { data, status }); + } + data.push(line); } - if let Some(status) = try_parse_status(&line) { - return Ok(ResponseBlock { data, status }); - } - data.push(line); } -} -pub fn try_parse_status(tokens: &[Token]) -> Option { - let first = tokens.first()?; - let status = match first { - Token::Atom(s) => match s.as_str() { - "OK" | "ok" | "Ok" => Status::Ok, - "NO" | "no" | "No" => Status::No, - "BYE" | "bye" | "Bye" => Status::Bye, - _ => return None, - }, - _ => return None, - }; - let mut idx = 1usize; - let mut code: Option = None; - let mut code_args: Option = None; - if matches!(tokens.get(idx), Some(Token::LParen)) { - idx += 1; - let mut depth: i32 = 1; - let mut inside: Vec = Vec::new(); - while idx < tokens.len() && depth > 0 { - match &tokens[idx] { - Token::LParen => { - depth += 1; - inside.push(tokens[idx].clone()); - } - Token::RParen => { - depth -= 1; - if depth > 0 { - inside.push(tokens[idx].clone()); - } - } - t => inside.push(t.clone()), - } - idx += 1; - } - if !inside.is_empty() { - if let Some(name) = inside[0].as_string() { - code = Some(name); - } - if inside.len() > 1 { - let mut joined = String::new(); - for (i, t) in inside.iter().enumerate().skip(1) { - if let Some(s) = t.as_string() { - if i > 1 { - joined.push(' '); - } - joined.push_str(&s); - } - } - if !joined.is_empty() { - code_args = Some(joined); - } + pub fn into_script(self) -> Result, SieveError> { + let mut body: Option> = None; + for token in self.data.into_iter().flatten() { + let bytes = match token { + Token::Literal(b) => b, + Token::QStr(s) => s.into_bytes(), + _ => continue, + }; + if body.is_some() { + return Err(SieveError::Parse( + "GETSCRIPT returned more than one body token".into(), + )); } + body = Some(bytes); } + Ok(body.unwrap_or_default()) } - let text = match tokens.get(idx) { - Some(Token::QStr(s)) => s.clone(), - Some(Token::Literal(b)) => String::from_utf8_lossy(b).into_owned(), - Some(Token::Atom(s)) => s.clone(), - _ => String::new(), - }; - Some(StatusLine { - status, - code, - code_args, - text, - }) -} - -fn read_logical_line(reader: &mut R) -> Result, SieveError> { - let mut tokens: Vec = Vec::new(); - let mut physical = Vec::with_capacity(256); - read_physical_line(reader, &mut physical)?; - let mut pos = 0usize; - loop { - while pos < physical.len() && matches!(physical[pos], b' ' | b'\t') { - pos += 1; - } - if pos >= physical.len() { - return Ok(tokens); - } - match physical[pos] { - b'"' => { - let (s, new_pos) = parse_quoted_string(&physical, pos + 1)?; - tokens.push(Token::QStr(s)); - pos = new_pos; - } - b'{' => { - let (n, plus, new_pos) = parse_literal_marker(&physical, pos + 1)?; - let _ = plus; - let trailing = &physical[new_pos..]; - if trailing.iter().any(|b| !matches!(b, b' ' | b'\t')) { - return Err(SieveError::Parse(format!( - "literal marker {{{n}}} not at end of line: trailing {:?}", - String::from_utf8_lossy(trailing) - ))); - } - let mut data = vec![0u8; n]; - reader.read_exact(&mut data)?; - tokens.push(Token::Literal(data)); - physical.clear(); - read_physical_line(reader, &mut physical)?; - pos = 0; - } - b'(' => { - tokens.push(Token::LParen); - pos += 1; - } - b')' => { - tokens.push(Token::RParen); - pos += 1; - } - _ => { - let start = pos; - while pos < physical.len() - && !matches!(physical[pos], b' ' | b'\t' | b'"' | b'{' | b'(' | b')') - { - pos += 1; - } - let atom = String::from_utf8_lossy(&physical[start..pos]).into_owned(); - tokens.push(Token::Atom(atom)); - } - } - } -} - -fn read_physical_line(reader: &mut R, buf: &mut Vec) -> Result<(), SieveError> { - buf.clear(); - let n = reader.read_until(b'\n', buf)?; - if n == 0 { - return Err(SieveError::Io(io::Error::new( - io::ErrorKind::UnexpectedEof, - "managesieve closed stream", - ))); - } - while matches!(buf.last(), Some(b'\n') | Some(b'\r')) { - buf.pop(); - } - Ok(()) -} - -fn parse_quoted_string(buf: &[u8], start: usize) -> Result<(String, usize), SieveError> { - let mut bytes: Vec = Vec::new(); - let mut i = start; - while i < buf.len() { - match buf[i] { - b'"' => { - let s = String::from_utf8_lossy(&bytes).into_owned(); - return Ok((s, i + 1)); - } - b'\\' if i + 1 < buf.len() => { - bytes.push(buf[i + 1]); - i += 2; - } - _ => { - bytes.push(buf[i]); - i += 1; - } - } - } - Err(SieveError::Parse("unterminated quoted-string".into())) -} - -fn parse_literal_marker(buf: &[u8], start: usize) -> Result<(usize, bool, usize), SieveError> { - let mut i = start; - let mut n: usize = 0; - let digits_start = i; - while i < buf.len() && buf[i].is_ascii_digit() { - n = n - .checked_mul(10) - .and_then(|v| v.checked_add((buf[i] - b'0') as usize)) - .ok_or_else(|| SieveError::Parse("literal size overflow".into()))?; - i += 1; - } - if i == digits_start { - return Err(SieveError::Parse("literal marker missing digits".into())); - } - let plus = i < buf.len() && buf[i] == b'+'; - if plus { - i += 1; - } - if i >= buf.len() || buf[i] != b'}' { - return Err(SieveError::Parse( - "literal marker missing closing brace".into(), - )); - } - Ok((n, plus, i + 1)) } #[derive(Debug, Default, Clone)] @@ -282,97 +324,23 @@ pub struct ListedScript { pub active: bool, } -pub fn parse_listscripts(data: &[Vec]) -> Result, SieveError> { - let mut out = Vec::with_capacity(data.len()); - for line in data { - let mut iter = line.iter(); - let first = match iter.next() { - Some(t) => t, - None => continue, - }; - let name = first - .as_string() - .ok_or_else(|| SieveError::Parse(format!("LISTSCRIPTS line missing name: {line:?}")))?; - let active = iter.any(|t| matches!(t, Token::Atom(a) if a.eq_ignore_ascii_case("ACTIVE"))); - out.push(ListedScript { name, active }); - } - Ok(out) -} - -pub fn parse_getscript(data: &[Vec]) -> Result, SieveError> { - let mut buf: Vec = Vec::new(); - let mut found = false; - for line in data { - for token in line { - match token { - Token::Literal(b) => { - if found { - return Err(SieveError::Parse( - "GETSCRIPT returned more than one body token".into(), - )); - } - buf = b.clone(); - found = true; - } - Token::QStr(s) => { - if found { - return Err(SieveError::Parse( - "GETSCRIPT returned more than one body token".into(), - )); - } - buf = s.as_bytes().to_vec(); - found = true; - } - _ => {} - } - } - } - if !found { - return Ok(Vec::new()); - } - Ok(buf) -} - -pub fn parse_capabilities(data: &[Vec]) -> Capabilities { - let mut caps = Capabilities::default(); - for line in data { - if line.is_empty() { - continue; - } - let name = match line[0].as_string() { - Some(s) => s, - None => continue, - }; - let value = if line.len() > 1 { - line[1..] +impl ListedScript { + pub fn parse_all(data: &[Vec]) -> Result, SieveError> { + let mut out = Vec::with_capacity(data.len()); + for line in data { + let Some((first, rest)) = line.split_first() else { + continue; + }; + let name = first.as_string().ok_or_else(|| { + SieveError::Parse(format!("LISTSCRIPTS line missing name: {line:?}")) + })?; + let active = rest .iter() - .filter_map(|t| t.as_string()) - .collect::>() - .join(" ") - } else { - String::new() - }; - match name.to_ascii_uppercase().as_str() { - "IMPLEMENTATION" => caps.implementation = Some(value), - "VERSION" => caps.version = Some(value), - "SASL" => { - caps.sasl = value - .split_ascii_whitespace() - .map(|s| s.to_ascii_uppercase()) - .collect() - } - "SIEVE" => caps.sieve = Some(value), - "STARTTLS" => caps.starttls = true, - "MAXREDIRECTS" => caps.maxredirects = value.parse().ok(), - "NOTIFY" => caps.notify = Some(value), - "LANGUAGE" => caps.language = Some(value), - "OWNER" => caps.owner = Some(value), - other => { - caps.raw.push((other.to_ascii_uppercase(), value)); - } + .any(|t| matches!(t, Token::Atom(a) if a.eq_ignore_ascii_case("ACTIVE"))); + out.push(ListedScript { name, active }); } + Ok(out) } - caps } #[derive(Debug, Default, Clone, PartialEq, Eq)] @@ -389,7 +357,70 @@ pub struct Capabilities { pub raw: Vec<(String, String)>, } +#[derive(Clone, Copy)] +enum CapabilityName { + Implementation, + Version, + Sasl, + Sieve, + StartTls, + MaxRedirects, + Notify, + Language, + Owner, +} + impl Capabilities { + pub fn parse(data: &[Vec]) -> Capabilities { + let mut caps = Capabilities::default(); + for line in data { + let Some((first, values)) = line.split_first() else { + continue; + }; + let Some(name) = first.as_string() else { + continue; + }; + let mut value = String::new(); + for (i, s) in values.iter().filter_map(Token::as_string).enumerate() { + if i > 0 { + value.push(' '); + } + value.push_str(&s); + } + let Some(known) = hashify::map_ignore_case!(name.as_bytes(), CapabilityName, + "IMPLEMENTATION" => CapabilityName::Implementation, + "VERSION" => CapabilityName::Version, + "SASL" => CapabilityName::Sasl, + "SIEVE" => CapabilityName::Sieve, + "STARTTLS" => CapabilityName::StartTls, + "MAXREDIRECTS" => CapabilityName::MaxRedirects, + "NOTIFY" => CapabilityName::Notify, + "LANGUAGE" => CapabilityName::Language, + "OWNER" => CapabilityName::Owner, + ) else { + caps.raw.push((name.to_ascii_uppercase(), value)); + continue; + }; + match known { + CapabilityName::Implementation => caps.implementation = Some(value), + CapabilityName::Version => caps.version = Some(value), + CapabilityName::Sasl => { + caps.sasl = value + .split_ascii_whitespace() + .map(str::to_ascii_uppercase) + .collect(); + } + CapabilityName::Sieve => caps.sieve = Some(value), + CapabilityName::StartTls => caps.starttls = true, + CapabilityName::MaxRedirects => caps.maxredirects = value.parse().ok(), + CapabilityName::Notify => caps.notify = Some(value), + CapabilityName::Language => caps.language = Some(value), + CapabilityName::Owner => caps.owner = Some(value), + } + } + caps + } + pub fn has_sasl(&self, mech: &str) -> bool { self.sasl.iter().any(|m| m.eq_ignore_ascii_case(mech)) } @@ -409,10 +440,10 @@ mod tests { let input = b"\"IMPLEMENTATION\" \"Test Server\"\r\n\"VERSION\" \"1.0\"\r\n\"STARTTLS\"\r\n\"SASL\" \"PLAIN LOGIN\"\r\nOK \"hello\"\r\n"; let mut c = cursor(input); - let r = read_response(&mut c).unwrap(); + let r = ResponseBlock::read(&mut c).unwrap(); assert_eq!(r.status.status, Status::Ok); assert_eq!(r.status.text, "hello"); - let caps = parse_capabilities(&r.data); + let caps = Capabilities::parse(&r.data); assert_eq!(caps.implementation, Some("Test Server".to_owned())); assert_eq!(caps.version, Some("1.0".to_owned())); assert!(caps.starttls); @@ -425,9 +456,9 @@ mod tests { fn parses_listscripts_with_active_marker() { let input = b"\"vacation\"\r\n\"out-of-office\" ACTIVE\r\n\"forward\"\r\nOK\r\n"; let mut c = cursor(input); - let r = read_response(&mut c).unwrap(); + let r = ResponseBlock::read(&mut c).unwrap(); assert_eq!(r.status.status, Status::Ok); - let list = parse_listscripts(&r.data).unwrap(); + let list = ListedScript::parse_all(&r.data).unwrap(); assert_eq!(list.len(), 3); assert!(!list[0].active); assert!(list[1].active); @@ -446,9 +477,9 @@ mod tests { data.extend_from_slice(b" ACTIVE\r\n"); data.extend_from_slice(b"OK\r\n"); let mut c = cursor(&data); - let r = read_response(&mut c).unwrap(); + let r = ResponseBlock::read(&mut c).unwrap(); assert_eq!(r.status.status, Status::Ok); - let list = parse_listscripts(&r.data).unwrap(); + let list = ListedScript::parse_all(&r.data).unwrap(); assert_eq!(list.len(), 1); assert_eq!(list[0].name.as_bytes(), body); assert!(list[0].active); @@ -462,8 +493,8 @@ mod tests { data.extend_from_slice(script); data.extend_from_slice(b"\r\nOK\r\n"); let mut c = cursor(&data); - let r = read_response(&mut c).unwrap(); - let body = parse_getscript(&r.data).unwrap(); + let r = ResponseBlock::read(&mut c).unwrap(); + let body = r.into_script().unwrap(); assert_eq!(body, script); } @@ -471,8 +502,8 @@ mod tests { fn parses_getscript_quoted_one_line_body() { let data = b"\"stop;\"\r\nOK\r\n"; let mut c = cursor(data); - let r = read_response(&mut c).unwrap(); - let body = parse_getscript(&r.data).unwrap(); + let r = ResponseBlock::read(&mut c).unwrap(); + let body = r.into_script().unwrap(); assert_eq!(body, b"stop;"); } @@ -480,7 +511,7 @@ mod tests { fn parses_no_with_code_and_text() { let data = b"NO (NONEXISTENT) \"no such script\"\r\n"; let mut c = cursor(data); - let r = read_response(&mut c).unwrap(); + let r = ResponseBlock::read(&mut c).unwrap(); assert_eq!(r.status.status, Status::No); assert_eq!(r.status.code.as_deref(), Some("NONEXISTENT")); assert_eq!(r.status.text, "no such script"); @@ -490,7 +521,7 @@ mod tests { fn parses_no_without_code_and_quoted_text() { let data = b"NO \"failed\"\r\n"; let mut c = cursor(data); - let r = read_response(&mut c).unwrap(); + let r = ResponseBlock::read(&mut c).unwrap(); assert_eq!(r.status.status, Status::No); assert!(r.status.code.is_none()); assert_eq!(r.status.text, "failed"); @@ -500,7 +531,7 @@ mod tests { fn parses_bye_referral_with_code_args() { let data = b"BYE (REFERRAL \"sieve://other.example/\") \"go away\"\r\n"; let mut c = cursor(data); - let r = read_response(&mut c).unwrap(); + let r = ResponseBlock::read(&mut c).unwrap(); assert_eq!(r.status.status, Status::Bye); assert_eq!(r.status.code.as_deref(), Some("REFERRAL")); assert_eq!(r.status.text, "go away"); @@ -511,7 +542,7 @@ mod tests { fn parses_bare_ok_response() { let data = b"OK\r\n"; let mut c = cursor(data); - let r = read_response(&mut c).unwrap(); + let r = ResponseBlock::read(&mut c).unwrap(); assert_eq!(r.status.status, Status::Ok); assert!(r.status.text.is_empty()); assert!(r.data.is_empty()); @@ -524,7 +555,7 @@ mod tests { let mut buf: Vec = Vec::new(); buf.extend_from_slice(b"\"a \\\"b\\\\\"\r\nOK\r\n"); let mut c = cursor(&buf); - let r = read_response(&mut c).unwrap(); + let r = ResponseBlock::read(&mut c).unwrap(); assert_eq!(r.data.len(), 1); match &r.data[0][0] { Token::QStr(s) => assert_eq!(s, "a \"b\\"), @@ -536,7 +567,7 @@ mod tests { fn literal_size_overflow_is_parse_error() { let data = b"{99999999999999999999}\r\nbody\r\nOK\r\n"; let mut c = cursor(data); - let err = read_response(&mut c).unwrap_err(); + let err = ResponseBlock::read(&mut c).unwrap_err(); assert!(matches!(err, SieveError::Parse(_)), "got {err:?}"); } @@ -544,7 +575,7 @@ mod tests { fn unterminated_quoted_string_is_parse_error() { let data = b"\"unterminated\r\n"; let mut c = cursor(data); - let err = read_response(&mut c).unwrap_err(); + let err = ResponseBlock::read(&mut c).unwrap_err(); assert!(matches!(err, SieveError::Parse(_)), "got {err:?}"); } @@ -552,8 +583,8 @@ mod tests { fn capability_atoms_recognised_without_quotes() { let data = b"IMPLEMENTATION \"X\"\r\nSTARTTLS\r\nSASL PLAIN\r\nOK\r\n"; let mut c = cursor(data); - let r = read_response(&mut c).unwrap(); - let caps = parse_capabilities(&r.data); + let r = ResponseBlock::read(&mut c).unwrap(); + let caps = Capabilities::parse(&r.data); assert_eq!(caps.implementation.as_deref(), Some("X")); assert!(caps.starttls); assert!(caps.has_sasl("PLAIN")); @@ -563,8 +594,8 @@ mod tests { fn capability_sasl_as_multiple_unquoted_atoms_captures_every_mechanism() { let data = b"IMPLEMENTATION \"X\"\r\nSASL PLAIN LOGIN OAUTHBEARER\r\nOK\r\n"; let mut c = cursor(data); - let r = read_response(&mut c).unwrap(); - let caps = parse_capabilities(&r.data); + let r = ResponseBlock::read(&mut c).unwrap(); + let caps = Capabilities::parse(&r.data); assert!(caps.has_sasl("PLAIN"), "PLAIN missing: {:?}", caps.sasl); assert!(caps.has_sasl("LOGIN"), "LOGIN missing: {:?}", caps.sasl); assert!( @@ -578,8 +609,8 @@ mod tests { fn capability_sasl_as_single_quoted_string_captures_every_mechanism() { let data = b"\"SASL\" \"PLAIN LOGIN OAUTHBEARER\"\r\nOK\r\n"; let mut c = cursor(data); - let r = read_response(&mut c).unwrap(); - let caps = parse_capabilities(&r.data); + let r = ResponseBlock::read(&mut c).unwrap(); + let caps = Capabilities::parse(&r.data); assert!(caps.has_sasl("PLAIN")); assert!(caps.has_sasl("LOGIN")); assert!(caps.has_sasl("OAUTHBEARER")); @@ -594,7 +625,7 @@ mod tests { data.push(b'"'); data.extend_from_slice(b"\r\nOK\r\n"); let mut c = cursor(&data); - let r = read_response(&mut c).unwrap(); + let r = ResponseBlock::read(&mut c).unwrap(); assert_eq!(r.data.len(), 1); match &r.data[0][0] { Token::QStr(s) => assert_eq!(s.as_bytes(), name_bytes), @@ -607,8 +638,8 @@ mod tests { let mut data: Vec = Vec::new(); data.extend_from_slice(b"\"fun\xc3\xa7y\"\r\nOK\r\n"); let mut c = cursor(&data); - let r = read_response(&mut c).unwrap(); - let list = parse_listscripts(&r.data).unwrap(); + let r = ResponseBlock::read(&mut c).unwrap(); + let list = ListedScript::parse_all(&r.data).unwrap(); assert_eq!(list.len(), 1); assert_eq!(list[0].name.as_bytes(), b"fun\xc3\xa7y"); } @@ -617,7 +648,17 @@ mod tests { fn parses_two_logical_lines_with_intervening_blank_skipped() { let data = b"\"a\"\r\n\r\n\"b\"\r\nOK\r\n"; let mut c = cursor(data); - let r = read_response(&mut c).unwrap(); + let r = ResponseBlock::read(&mut c).unwrap(); assert_eq!(r.data.len(), 2); } + + #[test] + fn oversized_literal_header_fails_without_preallocating() { + // a hostile {n} must not allocate n bytes before the data arrives + let mut c = cursor(b"{4294967295}\r\nabc"); + assert!(matches!( + ResponseBlock::read(&mut c), + Err(SieveError::Io(_)) + )); + } } diff --git a/src/managesieve/retry.rs b/src/managesieve/retry.rs index 521e52c..fb280dd 100644 --- a/src/managesieve/retry.rs +++ b/src/managesieve/retry.rs @@ -4,54 +4,35 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +use crate::net::backoff::{Fault, Retryable}; + use super::error::SieveError; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Disposition { - TransportDrop, - Transient, - Permanent, - PerScriptRecoverable, - FeatureNegotiation, - Referral, -} +impl SieveError { + pub fn is_referral(&self) -> bool { + match self { + SieveError::No(no) => no.is_referral(), + SieveError::Referral(_) => true, + _ => false, + } + } -pub fn classify(err: &SieveError) -> Disposition { - match err { - SieveError::Io(_) => Disposition::TransportDrop, - SieveError::Tls(_) => Disposition::Permanent, - SieveError::Bye(_) => Disposition::TransportDrop, - SieveError::No(no) => { - if no.is_referral() { - Disposition::Referral - } else if no.is_transient() { - Disposition::Transient - } else { - Disposition::Permanent - } + pub fn is_negotiation_failure(&self) -> bool { + match self { + SieveError::No(no) => !no.is_transient() && !no.is_referral(), + SieveError::Unsupported(_) => true, + _ => false, } - SieveError::Protocol(_) | SieveError::Parse(_) | SieveError::Unsupported(_) => { - Disposition::Permanent - } - SieveError::Referral(_) => Disposition::Referral, } } -pub fn is_negotiation_failure(err: &SieveError) -> bool { - match err { - SieveError::No(no) => !no.is_transient() && !no.is_referral(), - SieveError::Unsupported(_) => true, - _ => false, - } -} - -pub fn script_disposition(err: &SieveError) -> Disposition { - let base = classify(err); - match base { - Disposition::Permanent | Disposition::FeatureNegotiation => { - Disposition::PerScriptRecoverable +impl Retryable for SieveError { + fn fault(&self) -> Fault { + match self { + SieveError::Io(_) | SieveError::Bye(_) => Fault::TransportDrop, + SieveError::No(no) if !no.is_referral() && no.is_transient() => Fault::Transient, + _ => Fault::Fatal, } - other => other, } } @@ -71,57 +52,52 @@ mod tests { ErrorKind::TimedOut, ] { let e = SieveError::Io(io::Error::new(k, "x")); - assert_eq!(classify(&e), Disposition::TransportDrop, "{k:?}"); + assert_eq!(e.fault(), Fault::TransportDrop, "{k:?}"); } } #[test] fn bye_is_transport_drop() { assert_eq!( - classify(&SieveError::Bye("going away".into())), - Disposition::TransportDrop + SieveError::Bye("going away".into()).fault(), + Fault::TransportDrop ); } #[test] fn transient_no_text_marked_transient() { let e = SieveError::No(NoError::new("Try again later", None)); - assert_eq!(classify(&e), Disposition::Transient); + assert_eq!(e.fault(), Fault::Transient); } #[test] fn trylater_code_marked_transient() { let e = SieveError::No(NoError::new("x", Some("TRYLATER".into()))); - assert_eq!(classify(&e), Disposition::Transient); + assert_eq!(e.fault(), Fault::Transient); } #[test] - fn referral_code_classified_as_referral() { - let e = SieveError::No(NoError::new("x", Some("REFERRAL".into()))); - assert_eq!(classify(&e), Disposition::Referral); + fn referral_is_fatal_and_flagged() { + let e = SieveError::No(NoError::new("temp", Some("REFERRAL".into()))); + assert_eq!(e.fault(), Fault::Fatal); + assert!(e.is_referral()); + assert!(SieveError::Referral("x".into()).is_referral()); } #[test] - fn nonexistent_no_is_permanent_but_per_script_demotes_it() { + fn nonexistent_no_and_tls_are_fatal() { let e = SieveError::No(NoError::new("no such script", Some("NONEXISTENT".into()))); - assert_eq!(classify(&e), Disposition::Permanent); - assert_eq!(script_disposition(&e), Disposition::PerScriptRecoverable); + assert_eq!(e.fault(), Fault::Fatal); + assert_eq!(SieveError::Tls("x".into()).fault(), Fault::Fatal); } #[test] fn negotiation_failure_matches_non_transient_no_and_unsupported() { - assert!(is_negotiation_failure(&SieveError::Unsupported("x".into()))); - assert!(is_negotiation_failure(&SieveError::No(NoError::new( - "no such mech", - None - )))); - assert!(!is_negotiation_failure(&SieveError::No(NoError::new( - "try again", - None - )))); - assert!(!is_negotiation_failure(&SieveError::No(NoError::new( - "x", - Some("REFERRAL".into()) - )))); + assert!(SieveError::Unsupported("x".into()).is_negotiation_failure()); + assert!(SieveError::No(NoError::new("no such mech", None)).is_negotiation_failure()); + assert!(!SieveError::No(NoError::new("try again", None)).is_negotiation_failure()); + assert!( + !SieveError::No(NoError::new("x", Some("REFERRAL".into()))).is_negotiation_failure() + ); } } diff --git a/src/managesieve/session.rs b/src/managesieve/session.rs new file mode 100644 index 0000000..f093522 --- /dev/null +++ b/src/managesieve/session.rs @@ -0,0 +1,205 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::error::Error; +use crate::logging::Logger; +use crate::net::sasl::{SaslAuth, SaslPayload}; +use crate::net::stream::{Cleartext, ConnectMode, Connector, Endpoint, Scheme, UrlTls}; + +use super::client::SieveClient; +use super::error::SieveError; + +pub const SIEVE: Scheme = Scheme { + plain: "sieve", + tls: "sieves", + plain_port: 4190, + tls_port: 4190, +}; + +#[derive(Debug)] +enum AuthFailure { + Refused(String), + NoMechanism(String), + Wire(SieveError), +} + +pub struct SieveSession { + connector: Connector, + endpoint: Endpoint, + mode: ConnectMode, + cleartext: Cleartext, + auth: SaslAuth, + logger: Logger, +} + +impl SieveSession { + pub fn new( + connector: Connector, + endpoint: Endpoint, + cleartext: Cleartext, + auth: SaslAuth, + logger: Logger, + ) -> SieveSession { + let mode = match endpoint.tls { + UrlTls::Implicit => ConnectMode::ImplicitTls, + UrlTls::Negotiated => ConnectMode::StartTls, + }; + SieveSession { + connector, + endpoint, + mode, + cleartext, + auth, + logger, + } + } + + pub fn session_url(&self) -> String { + self.endpoint.session_url(&SIEVE) + } + + pub fn account_id(&self) -> &str { + self.auth.user() + } + + pub fn logger(&self) -> Logger { + self.logger + } + + fn connect(&self) -> Result { + SieveClient::connect( + &self.connector, + &self.endpoint.host, + self.endpoint.port, + self.mode, + self.cleartext, + self.logger, + ) + } + + pub fn open(&self) -> Result { + let mut client = self.connect().map_err(Error::connection)?; + self.authenticate(&mut client).map_err(|e| match e { + AuthFailure::Refused(text) => Error::Connection(format!("auth failed: {text}")), + AuthFailure::NoMechanism(text) => Error::Connection(text), + AuthFailure::Wire(e) => Error::Connection(e.to_string()), + })?; + self.logger.progress(format_args!( + "Authenticated as {} on {}", + self.account_id(), + self.session_url() + )); + self.refresh_capabilities(&mut client, "post-auth"); + Ok(client) + } + + pub fn reopen(&self) -> Result { + let mut client = self.connect()?; + self.authenticate(&mut client).map_err(|e| match e { + AuthFailure::Wire(e) => e, + AuthFailure::Refused(text) => { + SieveError::Protocol(format!("auth refused on reconnect: {text}")) + } + AuthFailure::NoMechanism(text) => SieveError::Unsupported(text), + })?; + self.refresh_capabilities(&mut client, "post-reconnect"); + Ok(client) + } + + fn refresh_capabilities(&self, client: &mut SieveClient, when: &str) { + if !client.had_fresh_post_auth_caps() + && let Err(e) = client.refresh_capabilities() + { + self.logger.warn(format_args!( + "{when} CAPABILITY refresh failed: {e}; continuing" + )); + } + } + + fn authenticate(&self, client: &mut SieveClient) -> Result<(), AuthFailure> { + match &self.auth { + SaslAuth::Basic { user, password } => { + let plain = client.capabilities().has_sasl("PLAIN"); + let login = client.capabilities().has_sasl("LOGIN"); + let mut plain_error: Option = None; + if plain { + match client.authenticate(SaslPayload::Plain { + authcid: user, + password, + }) { + Ok(()) => return Ok(()), + Err(e) if e.is_negotiation_failure() => plain_error = Some(e.to_string()), + Err(e) => return Err(AuthFailure::Wire(e)), + } + } + if login { + return match client.authenticate_login(user, password) { + Ok(()) => Ok(()), + Err(e) if e.is_negotiation_failure() => { + Err(AuthFailure::Refused(match plain_error { + Some(p) => format!("LOGIN: {e}; PLAIN: {p}"), + None => format!("LOGIN: {e}"), + })) + } + Err(e) => Err(AuthFailure::Wire(e)), + }; + } + Err(match plain_error { + Some(p) => AuthFailure::Refused(format!("PLAIN: {p}")), + None => AuthFailure::NoMechanism(format!( + "server SASL list lacks PLAIN and LOGIN (advertised: {:?})", + client.capabilities().sasl + )), + }) + } + SaslAuth::Bearer { user, token } => { + if !client.capabilities().has_sasl("OAUTHBEARER") { + return Err(AuthFailure::NoMechanism(format!( + "server does not advertise OAUTHBEARER (advertised: {:?})", + client.capabilities().sasl + ))); + } + match client.authenticate(SaslPayload::OAuthBearer { user, token }) { + Ok(()) => Ok(()), + Err(e) if e.is_negotiation_failure() => { + Err(AuthFailure::Refused(format!("OAUTHBEARER: {e}"))) + } + Err(e) => Err(AuthFailure::Wire(e)), + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::error::Error; + + #[test] + fn sieve_urls_default_to_4190_and_keep_their_tls_mode() { + let e = Endpoint::parse("sieves://mail.example.com", &SIEVE).unwrap(); + assert_eq!( + (e.host.as_str(), e.port, e.tls), + ("mail.example.com", 4190, UrlTls::Implicit) + ); + let e = Endpoint::parse("sieve://mail.example.com", &SIEVE).unwrap(); + assert_eq!( + (e.host.as_str(), e.port, e.tls), + ("mail.example.com", 4190, UrlTls::Negotiated) + ); + let e = Endpoint::parse("sieve://mail.example.com:14190", &SIEVE).unwrap(); + assert_eq!((e.port, e.tls), (14190, UrlTls::Negotiated)); + assert!(matches!( + Endpoint::parse("imap://example.com", &SIEVE), + Err(Error::Usage(_)) + )); + assert!(matches!( + Endpoint::parse("sieve:///path", &SIEVE), + Err(Error::Usage(_)) + )); + } +} diff --git a/src/net/auth.rs b/src/net/auth.rs new file mode 100644 index 0000000..1787ac4 --- /dev/null +++ b/src/net/auth.rs @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use encodify::base64::STANDARD; + +const BASIC: &str = "Basic "; + +#[derive(Debug, Clone)] +pub enum Auth { + Basic { user: String, password: String }, + Bearer { token: String }, +} + +impl Auth { + pub fn header_value(&self) -> String { + match self { + Auth::Basic { user, password } => { + let credentials = format!("{user}:{password}"); + let mut header = + String::with_capacity(BASIC.len() + STANDARD.encoded_len(credentials.len())); + header.push_str(BASIC); + STANDARD.encode_append(credentials, &mut header); + header + } + Auth::Bearer { token } => format!("Bearer {token}"), + } + } +} diff --git a/src/net/backoff.rs b/src/net/backoff.rs new file mode 100644 index 0000000..7827f43 --- /dev/null +++ b/src/net/backoff.rs @@ -0,0 +1,287 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::sync::Arc; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::thread; +use std::time::Duration; + +use crate::net::http::RetryStats; +use crate::net::retry::Jitter; + +const SHARED_BASE: Duration = Duration::from_millis(1_000); +const TRANSPORT_BASE: Duration = Duration::from_millis(500); +const DELAY_CAP: Duration = Duration::from_millis(30_000); +const LEVEL_CAP: u32 = 16; +const SHIFT_CAP: u32 = 15; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Fault { + TransportDrop, + Transient, + Fatal, +} + +pub trait Retryable { + fn fault(&self) -> Fault; +} + +#[derive(Debug)] +pub enum RetryError { + Exhausted { error: E, fault: Fault }, + Fatal(E), + Reconnect(E), +} + +impl RetryError { + pub fn into_inner(self) -> E { + match self { + RetryError::Exhausted { error, .. } + | RetryError::Fatal(error) + | RetryError::Reconnect(error) => error, + } + } +} + +#[derive(Default, Clone)] +pub struct BackoffState { + inner: Arc, +} + +#[derive(Default)] +struct BackoffInner { + level: AtomicU32, + transient_retries: AtomicU32, + transport_retries: AtomicU32, + jitter: Jitter, +} + +impl BackoffState { + pub fn new() -> BackoffState { + BackoffState::default() + } + + pub fn reset(&self) { + self.inner.level.store(0, Ordering::Relaxed); + } + + pub fn current_level(&self) -> u32 { + self.inner.level.load(Ordering::Relaxed) + } + + pub fn transient_retries(&self) -> u32 { + self.inner.transient_retries.load(Ordering::Relaxed) + } + + pub fn transport_retries(&self) -> u32 { + self.inner.transport_retries.load(Ordering::Relaxed) + } + + pub fn stats(&self) -> RetryStats { + let transient = u64::from(self.transient_retries()); + RetryStats { + retries_observed: transient + u64::from(self.transport_retries()), + retry_after_sleeps: transient, + } + } + + pub fn next_shared_delay(&self) -> Duration { + let prev = self.inner.level.fetch_add(1, Ordering::Relaxed); + self.inner.transient_retries.fetch_add(1, Ordering::Relaxed); + let scaled = BackoffState::scaled(SHARED_BASE, prev.saturating_add(1)); + let half = scaled / 2; + half + self.inner.jitter.uniform(scaled - half) + } + + pub fn transport_delay(&self, attempt: u32) -> Duration { + self.inner.transport_retries.fetch_add(1, Ordering::Relaxed); + self.inner + .jitter + .uniform(BackoffState::scaled(TRANSPORT_BASE, attempt)) + } + + fn scaled(base: Duration, level: u32) -> Duration { + let shift = level.min(LEVEL_CAP).saturating_sub(1).min(SHIFT_CAP); + base.saturating_mul(1 << shift).min(DELAY_CAP) + } + + pub fn run( + &self, + max_retries: u32, + target: &mut C, + mut op: impl FnMut(&mut C) -> Result, + mut reconnect: impl FnMut(&mut C, &E) -> Result<(), E>, + mut on_transient: impl FnMut(&E), + ) -> Result> { + let mut transient_attempts: u32 = 0; + let mut transport_attempts: u32 = 0; + loop { + let error = match op(target) { + Ok(value) => { + self.reset(); + return Ok(value); + } + Err(error) => error, + }; + match error.fault() { + Fault::TransportDrop => { + if transport_attempts >= max_retries { + return Err(RetryError::Exhausted { + error, + fault: Fault::TransportDrop, + }); + } + transport_attempts += 1; + thread::sleep(self.transport_delay(transport_attempts)); + reconnect(target, &error).map_err(RetryError::Reconnect)?; + } + Fault::Transient => { + if transient_attempts >= max_retries { + return Err(RetryError::Exhausted { + error, + fault: Fault::Transient, + }); + } + transient_attempts += 1; + on_transient(&error); + thread::sleep(self.next_shared_delay()); + } + Fault::Fatal => return Err(RetryError::Fatal(error)), + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Debug, PartialEq)] + struct Fail(Fault); + + impl Retryable for Fail { + fn fault(&self) -> Fault { + self.0 + } + } + + #[test] + fn shared_delay_grows_and_caps() { + let s = BackoffState::new(); + let d1 = s.next_shared_delay(); + assert!(d1 >= SHARED_BASE / 2 && d1 <= SHARED_BASE); + s.next_shared_delay(); + let d3 = s.next_shared_delay(); + assert!(d3 >= d1, "{d1:?} -> {d3:?}"); + assert_eq!(s.transient_retries(), 3); + for _ in 0..40 { + assert!(s.next_shared_delay() <= DELAY_CAP); + } + let d = s.next_shared_delay(); + assert!(d >= DELAY_CAP / 2 && d <= DELAY_CAP); + } + + #[test] + fn reset_clears_level() { + let s = BackoffState::new(); + s.next_shared_delay(); + s.next_shared_delay(); + assert_eq!(s.current_level(), 2); + s.reset(); + assert_eq!(s.current_level(), 0); + } + + #[test] + fn transport_delay_is_local_and_bounded() { + let s = BackoffState::new(); + assert!(s.transport_delay(1) <= TRANSPORT_BASE); + assert!(s.transport_delay(6) <= DELAY_CAP); + assert_eq!(s.current_level(), 0); + assert_eq!(s.transport_retries(), 2); + } + + #[test] + fn counters_and_stats_are_shared_across_clones() { + let a = BackoffState::new(); + let b = a.clone(); + a.next_shared_delay(); + a.next_shared_delay(); + b.transport_delay(1); + assert_eq!(a.current_level(), b.current_level()); + assert_eq!( + b.stats(), + RetryStats { + retries_observed: 3, + retry_after_sleeps: 2 + } + ); + } + + #[test] + fn run_reconnects_on_transport_drop_then_succeeds() { + let s = BackoffState::new(); + let mut state = (0u32, 0u32); + let out = s.run( + 3, + &mut state, + |st| { + st.0 += 1; + if st.0 < 2 { + Err(Fail(Fault::TransportDrop)) + } else { + Ok(st.0) + } + }, + |st, _| { + st.1 += 1; + Ok(()) + }, + |_| {}, + ); + assert_eq!(out.unwrap(), 2); + assert_eq!(state.1, 1); + } + + #[test] + fn run_stops_on_fatal_and_on_exhaustion() { + let s = BackoffState::new(); + let fatal = s.run( + 3, + &mut (), + |_| Err::<(), _>(Fail(Fault::Fatal)), + |_, _| Ok(()), + |_| {}, + ); + assert!(matches!(fatal, Err(RetryError::Fatal(_)))); + let mut transients = 0; + let exhausted = s.run( + 0, + &mut (), + |_| Err::<(), _>(Fail(Fault::Transient)), + |_, _| Ok(()), + |_| transients += 1, + ); + assert!(matches!( + exhausted, + Err(RetryError::Exhausted { + fault: Fault::Transient, + .. + }) + )); + assert_eq!(transients, 0); + let reconnect = s.run( + 1, + &mut (), + |_| Err::<(), _>(Fail(Fault::TransportDrop)), + |_, _| Err(Fail(Fault::Fatal)), + |_| {}, + ); + assert!(matches!( + reconnect, + Err(RetryError::Reconnect(Fail(Fault::Fatal))) + )); + } +} diff --git a/src/net/encode.rs b/src/net/encode.rs new file mode 100644 index 0000000..f2e3b2e --- /dev/null +++ b/src/net/encode.rs @@ -0,0 +1,179 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +const HEX: &[u8; 16] = b"0123456789ABCDEF"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SafeSet { + Unreserved, + Path, + MediaTypeUri, +} + +impl SafeSet { + const UNRESERVED: u128 = SafeSet::mask(b"-_.~"); + const PATH: u128 = SafeSet::mask(b"-_.~/@"); + const MEDIA_TYPE_URI: u128 = SafeSet::mask(b"!$&'()*+-./:=@_~"); + + const fn mask(extra: &[u8]) -> u128 { + let mut mask = 0u128; + let mut b = b'0'; + while b <= b'9' { + mask |= 1 << b; + b += 1; + } + b = b'A'; + while b <= b'Z' { + mask |= 1 << b; + mask |= 1 << (b + 32); + b += 1; + } + let mut rest = extra; + while let [first, tail @ ..] = rest { + mask |= 1 << *first; + rest = tail; + } + mask + } + + fn bits(self) -> u128 { + match self { + SafeSet::Unreserved => SafeSet::UNRESERVED, + SafeSet::Path => SafeSet::PATH, + SafeSet::MediaTypeUri => SafeSet::MEDIA_TYPE_URI, + } + } + + pub fn contains(self, byte: u8) -> bool { + byte < 128 && self.bits() & (1 << byte) != 0 + } +} + +pub struct Percent; + +impl Percent { + pub fn encode_into(out: &mut String, input: &str, safe: SafeSet) { + out.reserve(input.len()); + for byte in input.bytes() { + if safe.contains(byte) { + out.push(char::from(byte)); + } else { + out.push('%'); + out.push(char::from(HEX[usize::from(byte >> 4)])); + out.push(char::from(HEX[usize::from(byte & 0x0F)])); + } + } + } + + pub fn encode(input: &str, safe: SafeSet) -> String { + let mut out = String::with_capacity(input.len()); + Percent::encode_into(&mut out, input, safe); + out + } +} + +#[derive(Debug, Default, Clone)] +pub struct FormBody(String); + +impl FormBody { + pub fn new() -> FormBody { + FormBody::default() + } + + pub fn from_pairs(pairs: &[(&str, &str)]) -> FormBody { + let mut body = FormBody::new(); + for (key, value) in pairs { + body.pair(key, value); + } + body + } + + pub fn pair(&mut self, key: &str, value: &str) -> &mut FormBody { + if !self.0.is_empty() { + self.0.push('&'); + } + Percent::encode_into(&mut self.0, key, SafeSet::Unreserved); + self.0.push('='); + Percent::encode_into(&mut self.0, value, SafeSet::Unreserved); + self + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn into_string(self) -> String { + self.0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn reference(s: &str, safe: impl Fn(u8) -> bool) -> String { + let mut out = String::new(); + for b in s.bytes() { + if safe(b) { + out.push(b as char); + } else { + out.push_str(&format!("%{b:02X}")); + } + } + out + } + + #[test] + fn sets_match_the_replaced_encoders_on_every_byte() { + let all: String = (1u8..128) + .map(char::from) + .chain("é€\u{10348}".chars()) + .collect(); + let unreserved = + |b: u8| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.' | b'~'); + let path = |b: u8| unreserved(b) || matches!(b, b'/' | b'@'); + let media = |b: u8| { + b.is_ascii_alphanumeric() + || matches!( + b, + b'!' | b'$' + | b'&' + | b'\'' + | b'(' + | b')' + | b'*' + | b'+' + | b'-' + | b'.' + | b'/' + | b':' + | b'=' + | b'@' + | b'_' + | b'~' + ) + }; + assert_eq!( + Percent::encode(&all, SafeSet::Unreserved), + reference(&all, unreserved) + ); + assert_eq!(Percent::encode(&all, SafeSet::Path), reference(&all, path)); + assert_eq!( + Percent::encode(&all, SafeSet::MediaTypeUri), + reference(&all, media) + ); + } + + #[test] + fn form_body_joins_encoded_pairs() { + let body = FormBody::from_pairs(&[("grant_type", "refresh_token"), ("scope", "a b+c/d")]); + assert_eq!( + body.as_str(), + "grant_type=refresh_token&scope=a%20b%2Bc%2Fd" + ); + assert_eq!(FormBody::new().into_string(), ""); + } +} diff --git a/src/net/http.rs b/src/net/http.rs new file mode 100644 index 0000000..2cc946c --- /dev/null +++ b/src/net/http.rs @@ -0,0 +1,454 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fmt; +use std::io; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use ureq::Agent; +use ureq::config::{Config, RedirectAuthHeaders}; +use ureq::tls::{RootCerts, TlsConfig}; +use url::Url; + +use crate::logging::Logger; +use crate::net::retry::{Jitter, RateLimit, RetryPolicy}; + +const USER_AGENT: &str = concat!("inbuxa-migrate/", env!("CARGO_PKG_VERSION")); +const EXCERPT_LIMIT: usize = 512; +const ELLIPSIS: &str = "..."; +const REPLACEMENT: &str = "\u{FFFD}"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TlsPolicy { + Verify, + AcceptInvalid, +} + +#[derive(Debug, Clone, Copy)] +pub struct ClientOptions { + pub retry: RetryPolicy, + pub tls: TlsPolicy, + pub logger: Logger, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Redirects { + Follow, + Manual, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Methods { + Standard, + Extended, +} + +#[derive(Debug, Clone, Copy)] +pub struct AgentProfile { + pub redirects: Redirects, + pub methods: Methods, + pub max_body: u64, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct RetryStats { + pub retries_observed: u64, + pub retry_after_sleeps: u64, +} + +#[derive(Debug, Default)] +pub struct TransferStats { + retries: AtomicU64, + retry_after_sleeps: AtomicU64, + requests: AtomicU64, +} + +impl TransferStats { + pub fn snapshot(&self) -> RetryStats { + RetryStats { + retries_observed: self.retries.load(Ordering::Relaxed), + retry_after_sleeps: self.retry_after_sleeps.load(Ordering::Relaxed), + } + } + + pub fn requests(&self) -> u64 { + self.requests.load(Ordering::Relaxed) + } + + pub fn note_request(&self) { + self.requests.fetch_add(1, Ordering::Relaxed); + } + + pub fn note_retry(&self) { + self.retries.fetch_add(1, Ordering::Relaxed); + } + + pub fn note_retry_after(&self) { + self.retry_after_sleeps.fetch_add(1, Ordering::Relaxed); + } +} + +pub struct HttpCore { + agent: Agent, + retry: RetryPolicy, + tls: TlsPolicy, + max_body: u64, + rate_limit: RateLimit, + jitter: Jitter, + stats: TransferStats, + logger: Logger, +} + +impl HttpCore { + pub fn new(options: ClientOptions, profile: AgentProfile) -> HttpCore { + let mut builder = options + .retry + .apply_timeouts(Config::builder()) + .http_status_as_error(false) + .redirect_auth_headers(RedirectAuthHeaders::SameHost) + .tls_config( + TlsConfig::builder() + .unversioned_rustls_crypto_provider(Arc::new( + rustls::crypto::aws_lc_rs::default_provider(), + )) + .root_certs(RootCerts::PlatformVerifier) + .disable_verification(options.tls == TlsPolicy::AcceptInvalid) + .build(), + ); + if profile.methods == Methods::Extended { + builder = builder.allow_non_standard_methods(true); + } + if profile.redirects == Redirects::Manual { + builder = builder.max_redirects(0); + } + HttpCore { + agent: builder.build().new_agent(), + retry: options.retry, + tls: options.tls, + max_body: profile.max_body, + rate_limit: RateLimit::new(), + jitter: Jitter::new(), + stats: TransferStats::default(), + logger: options.logger, + } + } + + pub fn agent(&self) -> &Agent { + &self.agent + } + + pub fn retry(&self) -> &RetryPolicy { + &self.retry + } + + pub fn tls(&self) -> TlsPolicy { + self.tls + } + + pub fn max_body(&self) -> u64 { + self.max_body + } + + pub fn rate_limit(&self) -> &RateLimit { + &self.rate_limit + } + + pub fn logger(&self) -> Logger { + self.logger + } + + pub fn stats(&self) -> &TransferStats { + &self.stats + } + + pub fn user_agent(&self) -> &'static str { + USER_AGENT + } + + pub fn backoff(&self, attempt: u32) -> Duration { + self.retry.backoff(attempt, &self.jitter) + } + + pub fn throttle(&self, retry_after: Option) -> Duration { + self.rate_limit + .on_throttle(&self.retry, &self.jitter, retry_after) + } + + pub fn is_cross_host(requested: &str, landed: &str) -> bool { + match (Url::parse(requested), Url::parse(landed)) { + (Ok(a), Ok(b)) => { + a.host_str() != b.host_str() + || a.port_or_known_default() != b.port_or_known_default() + } + _ => false, + } + } +} + +#[derive(Debug)] +pub enum TransportError { + Io(io::Error), + Timeout(ureq::Timeout), + HostNotFound, + ConnectionFailed, + BodyStalled, + Tls(&'static str), + TooManyRedirects, + RedirectFailed, + TlsRequired, + Other(ureq::Error), +} + +impl TransportError { + pub fn is_connect(&self) -> bool { + matches!( + self, + TransportError::TooManyRedirects + | TransportError::RedirectFailed + | TransportError::TlsRequired + | TransportError::Other(_) + ) + } +} + +impl From for TransportError { + fn from(err: ureq::Error) -> TransportError { + match err { + ureq::Error::Io(e) => TransportError::Io(e), + ureq::Error::Timeout(t) => TransportError::Timeout(t), + ureq::Error::HostNotFound => TransportError::HostNotFound, + ureq::Error::ConnectionFailed => TransportError::ConnectionFailed, + ureq::Error::BodyStalled => TransportError::BodyStalled, + ureq::Error::Tls(m) => TransportError::Tls(m), + ureq::Error::TooManyRedirects => TransportError::TooManyRedirects, + ureq::Error::RedirectFailed => TransportError::RedirectFailed, + ureq::Error::TlsRequired => TransportError::TlsRequired, + other => TransportError::Other(other), + } + } +} + +impl fmt::Display for TransportError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + TransportError::Io(e) => write!(f, "io: {e}"), + TransportError::Timeout(t) => write!(f, "timeout: {t}"), + TransportError::HostNotFound => f.write_str("host not found"), + TransportError::ConnectionFailed => f.write_str("connection failed"), + TransportError::BodyStalled => f.write_str("body stalled"), + TransportError::Tls(m) => write!(f, "tls: {m}"), + TransportError::TooManyRedirects => f.write_str("too many redirects"), + TransportError::RedirectFailed => f.write_str("redirect failed"), + TransportError::TlsRequired => { + f.write_str("server requires TLS but transport is unsecured") + } + TransportError::Other(e) => e.fmt(f), + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct BodyExcerpt<'a>(pub &'a [u8]); + +impl fmt::Display for BodyExcerpt<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut budget = EXCERPT_LIMIT; + for chunk in self.0.utf8_chunks() { + let valid = chunk.valid(); + if valid.len() > budget { + let cut = valid.floor_char_boundary(budget); + f.write_str(valid.get(..cut).unwrap_or_default())?; + return f.write_str(ELLIPSIS); + } + f.write_str(valid)?; + budget -= valid.len(); + if !chunk.invalid().is_empty() { + if REPLACEMENT.len() > budget { + return f.write_str(ELLIPSIS); + } + f.write_str(REPLACEMENT)?; + budget -= REPLACEMENT.len(); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::logging::Level; + + fn reference_truncate(body: &[u8]) -> String { + let text = String::from_utf8_lossy(body); + if text.len() <= 512 { + return text.into_owned(); + } + let end = text.floor_char_boundary(512); + format!("{}...", &text[..end]) + } + + fn options(tls: TlsPolicy) -> ClientOptions { + ClientOptions { + retry: RetryPolicy::new(3), + tls, + logger: Logger::new(Level::Quiet), + } + } + + #[test] + fn body_excerpt_matches_reference_truncate() { + let mut cases: Vec> = vec![ + Vec::new(), + b"short".to_vec(), + vec![b'a'; 512], + vec![b'a'; 513], + "é".repeat(300).into_bytes(), + "€".repeat(200).into_bytes(), + ]; + for prefix in 505..=515 { + let mut body = vec![b'x'; prefix]; + body.extend_from_slice(&[0xFF, 0xFE, b'z']); + body.extend_from_slice("ü".as_bytes()); + cases.push(body); + let mut body = vec![b'x'; prefix]; + body.extend_from_slice("€".as_bytes()); + body.extend_from_slice(&[b'y'; 10]); + cases.push(body); + let mut body = vec![b'x'; prefix]; + body.push(0x80); + cases.push(body); + } + for body in cases { + assert_eq!( + BodyExcerpt(&body).to_string(), + reference_truncate(&body), + "len {}", + body.len() + ); + } + } + + #[test] + fn transport_error_texts_are_unchanged() { + let cases = [ + (ureq::Error::HostNotFound, "host not found", false), + (ureq::Error::ConnectionFailed, "connection failed", false), + (ureq::Error::BodyStalled, "body stalled", false), + (ureq::Error::TooManyRedirects, "too many redirects", true), + (ureq::Error::RedirectFailed, "redirect failed", true), + ( + ureq::Error::TlsRequired, + "server requires TLS but transport is unsecured", + true, + ), + ]; + for (err, text, connect) in cases { + let mapped = TransportError::from(err); + assert_eq!(mapped.to_string(), text); + assert_eq!(mapped.is_connect(), connect, "{text}"); + } + let io = TransportError::from(ureq::Error::Io(io::Error::other("broken pipe"))); + assert_eq!(io.to_string(), "io: broken pipe"); + assert!(!io.is_connect()); + } + + #[test] + fn cross_host_detects_host_and_port_changes() { + assert!(HttpCore::is_cross_host( + "https://a.example.com/x", + "https://b.example.com/x" + )); + assert!(HttpCore::is_cross_host( + "https://a.example.com/x", + "https://a.example.com:8443/x" + )); + assert!(!HttpCore::is_cross_host( + "https://a.example.com/x", + "https://a.example.com:443/y" + )); + assert!(!HttpCore::is_cross_host( + "not a url", + "https://a.example.com/" + )); + } + + #[test] + fn core_keeps_options_and_profile() { + let core = HttpCore::new( + options(TlsPolicy::AcceptInvalid), + AgentProfile { + redirects: Redirects::Manual, + methods: Methods::Extended, + max_body: 1024, + }, + ); + assert_eq!(core.max_body(), 1024); + assert_eq!(core.tls(), TlsPolicy::AcceptInvalid); + assert_eq!(core.retry().max_retries, 3); + assert_eq!(core.logger().level(), Level::Quiet); + assert!(core.user_agent().starts_with("inbuxa-migrate/")); + assert_eq!(core.stats().snapshot(), RetryStats::default()); + assert!(core.backoff(1) <= core.retry().base); + } + + #[test] + fn throttle_arms_the_shared_cooldown() { + let core = HttpCore::new( + options(TlsPolicy::Verify), + AgentProfile { + redirects: Redirects::Follow, + methods: Methods::Standard, + max_body: 1, + }, + ); + let delay = core.throttle(Some(Duration::from_secs(5))); + assert_eq!(delay, Duration::from_secs(5)); + assert_eq!(core.rate_limit().level(), 1); + assert!(core.rate_limit().cooldown().remaining().is_some()); + } + + #[test] + fn transfer_stats_count() { + let stats = TransferStats::default(); + stats.note_request(); + stats.note_retry(); + stats.note_retry(); + stats.note_retry_after(); + assert_eq!(stats.requests(), 1); + assert_eq!( + stats.snapshot(), + RetryStats { + retries_observed: 2, + retry_after_sleeps: 1 + } + ); + } + + #[test] + fn redirect_limit_is_manual_only_when_asked() { + let follow = HttpCore::new( + options(TlsPolicy::Verify), + AgentProfile { + redirects: Redirects::Follow, + methods: Methods::Standard, + max_body: 1, + }, + ); + let manual = HttpCore::new( + options(TlsPolicy::Verify), + AgentProfile { + redirects: Redirects::Manual, + methods: Methods::Standard, + max_body: 1, + }, + ); + assert_ne!(follow.agent().config().max_redirects(), 0); + assert_eq!(manual.agent().config().max_redirects(), 0); + } +} diff --git a/src/net/mod.rs b/src/net/mod.rs new file mode 100644 index 0000000..956c274 --- /dev/null +++ b/src/net/mod.rs @@ -0,0 +1,14 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod auth; +pub mod backoff; +pub mod encode; +pub mod http; +pub mod retry; +pub mod sasl; +pub mod stream; +pub mod xml; diff --git a/src/net/retry.rs b/src/net/retry.rs new file mode 100644 index 0000000..b514c60 --- /dev/null +++ b/src/net/retry.rs @@ -0,0 +1,435 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fmt; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use time::PrimitiveDateTime; +use time::format_description::BorrowedFormatItem; +use time::macros::format_description; +use ureq::config::ConfigBuilder; +use ureq::typestate::AgentScope; + +const IO_TIMEOUT: Duration = Duration::from_secs(300); +const DEFAULT_BASE: Duration = Duration::from_millis(500); +const DEFAULT_CAP: Duration = Duration::from_secs(30); +pub const THROTTLE_BASE: Duration = Duration::from_secs(1); +const THROTTLE_LEVEL_CAP: u32 = 16; +const BACKOFF_SHIFT_CAP: u32 = 32; +const THROTTLE_SHIFT_CAP: u32 = 20; +const SEED_FALLBACK: u64 = 0x9E37_79B9_7F4A_7C15; + +pub const IMF_FIXDATE: &[BorrowedFormatItem<'static>] = format_description!( + "[weekday repr:short], [day] [month repr:short] [year] [hour]:[minute]:[second] GMT" +); + +const TOO_MANY_REQUESTS: u16 = 429; +const BAD_GATEWAY: u16 = 502; +const SERVICE_UNAVAILABLE: u16 = 503; +const GATEWAY_TIMEOUT: u16 = 504; + +#[derive(Debug, Clone, Copy)] +pub struct RetryPolicy { + pub max_retries: u32, + pub base: Duration, + pub cap: Duration, + pub io_timeout: Duration, +} + +impl RetryPolicy { + pub fn new(max_retries: u32) -> Self { + RetryPolicy { + max_retries, + base: DEFAULT_BASE, + cap: DEFAULT_CAP, + io_timeout: IO_TIMEOUT, + } + } + + pub fn apply_timeouts(&self, builder: ConfigBuilder) -> ConfigBuilder { + let timeout = Some(self.io_timeout); + builder + .timeout_connect(timeout) + .timeout_per_read(timeout) + .timeout_per_write(timeout) + } + + pub fn backoff(&self, attempt: u32, jitter: &Jitter) -> Duration { + let shift = attempt.saturating_sub(1).min(BACKOFF_SHIFT_CAP); + let scaled = self + .base + .saturating_mul(1u32.checked_shl(shift).unwrap_or(u32::MAX)); + jitter.uniform(scaled.min(self.cap)) + } + + pub fn throttle_backoff(&self, level: u32, jitter: &Jitter) -> Duration { + let shift = level.saturating_sub(1).min(THROTTLE_SHIFT_CAP); + let scaled = THROTTLE_BASE + .saturating_mul(1u32.checked_shl(shift).unwrap_or(u32::MAX)) + .min(self.cap); + let half = scaled / 2; + half + jitter.uniform(scaled - half) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Disposition { + Retry, + Fatal, +} + +impl Disposition { + pub fn of_status(status: u16) -> Disposition { + match status { + TOO_MANY_REQUESTS | BAD_GATEWAY | SERVICE_UNAVAILABLE | GATEWAY_TIMEOUT => { + Disposition::Retry + } + _ => Disposition::Fatal, + } + } +} + +#[derive(Debug)] +pub struct Jitter(AtomicU64); + +impl Jitter { + pub fn new() -> Jitter { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos() as u64) + .unwrap_or(SEED_FALLBACK); + Jitter::with_seed(nanos ^ u64::from(std::process::id()).rotate_left(32)) + } + + pub fn with_seed(seed: u64) -> Jitter { + Jitter(AtomicU64::new(seed.max(1))) + } + + pub fn next_u64(&self) -> u64 { + let mut current = self.0.load(Ordering::Relaxed); + loop { + let next = Jitter::step(current); + match self + .0 + .compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) + { + Ok(_) => return next, + Err(seen) => current = seen, + } + } + } + + fn step(mut x: u64) -> u64 { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x + } + + pub fn uniform(&self, span: Duration) -> Duration { + let nanos = span.as_nanos(); + if nanos == 0 { + return Duration::ZERO; + } + let pick = u128::from(self.next_u64()) % (nanos + 1); + Duration::from_nanos(u64::try_from(pick).unwrap_or(u64::MAX)) + } +} + +impl Default for Jitter { + fn default() -> Jitter { + Jitter::new() + } +} + +#[derive(Debug, Default)] +pub struct Cooldown { + until: Mutex>, +} + +impl Cooldown { + pub fn new() -> Cooldown { + Cooldown::default() + } + + pub fn arm(&self, delay: Duration) { + let target = Instant::now() + delay; + if let Ok(mut guard) = self.until.lock() + && guard.is_none_or(|current| target > current) + { + *guard = Some(target); + } + } + + pub fn remaining(&self) -> Option { + let until = (*self.until.lock().ok()?)?; + until + .checked_duration_since(Instant::now()) + .filter(|left| !left.is_zero()) + } + + pub fn wait(&self) { + while let Some(remaining) = self.remaining() { + thread::sleep(remaining); + } + } +} + +#[derive(Debug, Default)] +pub struct RateLimit { + cooldown: Cooldown, + level: AtomicU32, +} + +impl RateLimit { + pub fn new() -> RateLimit { + RateLimit::default() + } + + pub fn cooldown(&self) -> &Cooldown { + &self.cooldown + } + + pub fn level(&self) -> u32 { + self.level.load(Ordering::Relaxed) + } + + pub fn on_success(&self) { + self.level.store(0, Ordering::Relaxed); + } + + pub fn on_throttle( + &self, + policy: &RetryPolicy, + jitter: &Jitter, + retry_after: Option, + ) -> Duration { + let level = self + .level + .fetch_add(1, Ordering::Relaxed) + .saturating_add(1) + .min(THROTTLE_LEVEL_CAP); + let delay = retry_after.unwrap_or_else(|| policy.throttle_backoff(level, jitter)); + self.cooldown.arm(delay); + delay + } +} + +pub struct RetryAfter; + +impl RetryAfter { + pub fn parse(value: &str, now: SystemTime) -> Option { + let trimmed = value.trim(); + if let Ok(secs) = trimmed.parse::() { + return Some(Duration::from_secs(secs)); + } + let parsed = PrimitiveDateTime::parse(trimmed, IMF_FIXDATE).ok()?; + SystemTime::from(parsed.assume_utc()) + .duration_since(now) + .ok() + } + + pub fn parse_now(value: &str) -> Option { + RetryAfter::parse(value, SystemTime::now()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RetryWait(pub Duration); + +impl fmt::Display for RetryWait { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let total = self.0.as_secs(); + match total { + 0 => write!(f, "{}ms", self.0.as_millis()), + 1..60 => write!(f, "{total}s"), + 60..3600 => write!(f, "{}m{:02}s", total / 60, total % 60), + _ => write!( + f, + "{}h{:02}m{:02}s", + total / 3600, + (total % 3600) / 60, + total % 60 + ), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn policy() -> RetryPolicy { + RetryPolicy::new(5) + } + + #[test] + fn backoff_is_bounded_by_cap_with_full_jitter() { + let p = policy(); + let jitter = Jitter::with_seed(7); + for attempt in 1..=40 { + let d = p.backoff(attempt, &jitter); + assert!(d <= p.cap, "attempt {attempt} delay {d:?} exceeds cap"); + } + } + + #[test] + fn first_backoff_stays_within_base() { + let p = policy(); + let jitter = Jitter::with_seed(99); + for _ in 0..100 { + assert!(p.backoff(1, &jitter) <= p.base); + } + } + + #[test] + fn jitter_with_zero_seed_still_advances() { + let jitter = Jitter::with_seed(0); + let a = jitter.next_u64(); + let b = jitter.next_u64(); + assert_ne!(a, 0); + assert_ne!(a, b); + } + + #[test] + fn jitter_uniform_of_zero_span_is_zero() { + assert_eq!(Jitter::with_seed(3).uniform(Duration::ZERO), Duration::ZERO); + } + + #[test] + fn jitter_is_shared_across_threads() { + let jitter = Jitter::with_seed(11); + thread::scope(|s| { + for _ in 0..4 { + s.spawn(|| { + for _ in 0..1000 { + assert!( + jitter.uniform(Duration::from_millis(5)) <= Duration::from_millis(5) + ); + } + }); + } + }); + } + + #[test] + fn retry_after_seconds() { + assert_eq!( + RetryAfter::parse(" 120 ", SystemTime::now()), + Some(Duration::from_secs(120)) + ); + } + + #[test] + fn retry_after_http_date_in_future() { + let now = UNIX_EPOCH + Duration::from_secs(784_111_777); + let d = RetryAfter::parse("Sun, 06 Nov 1994 08:49:47 GMT", now); + assert_eq!(d, Some(Duration::from_secs(10))); + } + + #[test] + fn retry_after_http_date_in_past_is_none() { + let now = UNIX_EPOCH + Duration::from_secs(784_111_800); + assert_eq!( + RetryAfter::parse("Sun, 06 Nov 1994 08:49:47 GMT", now), + None + ); + } + + #[test] + fn retry_after_garbage_is_none() { + assert_eq!(RetryAfter::parse("soon", SystemTime::now()), None); + } + + #[test] + fn status_classification() { + for status in [429, 502, 503, 504] { + assert_eq!(Disposition::of_status(status), Disposition::Retry); + } + for status in [200, 400, 401, 404, 500] { + assert_eq!(Disposition::of_status(status), Disposition::Fatal); + } + } + + #[test] + fn cooldown_arms_and_clears() { + let c = Cooldown::new(); + assert!(c.remaining().is_none()); + c.arm(Duration::from_millis(40)); + assert!(c.remaining().is_some()); + thread::sleep(Duration::from_millis(60)); + assert!(c.remaining().is_none()); + } + + #[test] + fn cooldown_never_shortens() { + let c = Cooldown::new(); + c.arm(Duration::from_secs(60)); + c.arm(Duration::from_millis(1)); + assert!( + c.remaining() + .is_some_and(|left| left > Duration::from_secs(30)) + ); + } + + #[test] + fn throttle_backoff_doubles_then_clamps_to_cap() { + let p = RetryPolicy::new(8); + let jitter = Jitter::with_seed(5); + let mut prev = p.throttle_backoff(1, &jitter); + for level in 2..=4 { + let d = p.throttle_backoff(level, &jitter); + assert!( + d >= prev / 2, + "level {level} ({d:?}) below half of {prev:?}" + ); + prev = d; + } + assert!(p.throttle_backoff(20, &jitter) <= p.cap); + } + + #[test] + fn rate_limit_escalates_and_resets() { + let p = RetryPolicy::new(8); + let jitter = Jitter::with_seed(17); + let s = RateLimit::new(); + let first = s.on_throttle(&p, &jitter, None); + let second = s.on_throttle(&p, &jitter, None); + let third = s.on_throttle(&p, &jitter, None); + assert!(second >= first); + assert!(third >= first); + assert_eq!(s.level(), 3); + s.on_success(); + assert_eq!(s.level(), 0); + assert!(s.on_throttle(&p, &jitter, None) <= THROTTLE_BASE); + } + + #[test] + fn rate_limit_honours_retry_after_over_schedule() { + let p = RetryPolicy::new(8); + let s = RateLimit::new(); + let d = s.on_throttle(&p, &Jitter::with_seed(1), Some(Duration::from_millis(750))); + assert_eq!(d, Duration::from_millis(750)); + assert_eq!(s.level(), 1); + assert!(s.cooldown().remaining().is_some()); + } + + #[test] + fn retry_wait_formats_like_the_jmap_client() { + let cases = [ + (Duration::from_millis(250), "250ms"), + (Duration::from_secs(45), "45s"), + (Duration::from_secs(125), "2m05s"), + (Duration::from_secs(3725), "1h02m05s"), + ]; + for (d, want) in cases { + assert_eq!(RetryWait(d).to_string(), want); + } + } +} diff --git a/src/net/sasl.rs b/src/net/sasl.rs new file mode 100644 index 0000000..da2d4e1 --- /dev/null +++ b/src/net/sasl.rs @@ -0,0 +1,113 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use encodify::base64::STANDARD; + +const BEARER: &str = "auth=Bearer "; +const SEP: char = '\x01'; + +#[derive(Debug, Clone)] +pub enum SaslAuth { + Basic { user: String, password: String }, + Bearer { user: String, token: String }, +} + +impl SaslAuth { + pub fn user(&self) -> &str { + match self { + SaslAuth::Basic { user, .. } | SaslAuth::Bearer { user, .. } => user, + } + } +} + +#[derive(Clone, Copy)] +pub enum SaslPayload<'a> { + Plain { authcid: &'a str, password: &'a str }, + XOauth2 { user: &'a str, token: &'a str }, + OAuthBearer { user: &'a str, token: &'a str }, +} + +impl SaslPayload<'_> { + pub fn mechanism(self) -> &'static str { + match self { + SaslPayload::Plain { .. } => "PLAIN", + SaslPayload::XOauth2 { .. } => "XOAUTH2", + SaslPayload::OAuthBearer { .. } => "OAUTHBEARER", + } + } + + pub fn bytes(self) -> Vec { + match self { + SaslPayload::Plain { authcid, password } => { + let mut out = Vec::with_capacity(authcid.len() + password.len() + 2); + out.push(0); + out.extend_from_slice(authcid.as_bytes()); + out.push(0); + out.extend_from_slice(password.as_bytes()); + out + } + SaslPayload::XOauth2 { user, token } => { + let mut out = String::with_capacity(user.len() + token.len() + 32); + out.push_str("user="); + out.push_str(user); + out.push(SEP); + out.push_str(BEARER); + out.push_str(token); + out.push(SEP); + out.push(SEP); + out.into_bytes() + } + SaslPayload::OAuthBearer { user, token } => { + let mut out = String::with_capacity(user.len() + token.len() + 32); + out.push_str("n,a="); + out.push_str(user); + out.push(','); + out.push(SEP); + out.push_str(BEARER); + out.push_str(token); + out.push(SEP); + out.push(SEP); + out.into_bytes() + } + } + } + + pub fn base64(self) -> String { + STANDARD.encode(self.bytes()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn payloads_match_the_wire_formats() { + let plain = SaslPayload::Plain { + authcid: "foo", + password: "bar", + }; + assert_eq!(plain.mechanism(), "PLAIN"); + assert_eq!(plain.bytes(), b"\0foo\0bar"); + assert_eq!(plain.base64(), "AGZvbwBiYXI="); + assert_eq!( + SaslPayload::XOauth2 { + user: "u", + token: "t" + } + .bytes(), + b"user=u\x01auth=Bearer t\x01\x01" + ); + assert_eq!( + SaslPayload::OAuthBearer { + user: "u", + token: "t" + } + .bytes(), + b"n,a=u,\x01auth=Bearer t\x01\x01" + ); + } +} diff --git a/src/net/stream.rs b/src/net/stream.rs new file mode 100644 index 0000000..922a1a6 --- /dev/null +++ b/src/net/stream.rs @@ -0,0 +1,524 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fmt; +use std::io::{self, Read, Write}; +use std::mem; +use std::net::TcpStream; +use std::sync::Arc; +use std::time::Duration; + +use flate2::{Compress, Compression, Decompress, FlushCompress, FlushDecompress}; +use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; +use rustls::pki_types::{CertificateDer, ServerName, UnixTime}; +use rustls::{ClientConfig, ClientConnection, DigitallySignedStruct, SignatureScheme, StreamOwned}; +use rustls_platform_verifier::BuilderVerifierExt; +use url::Url; + +use crate::error::Error; +use crate::net::http::TlsPolicy; + +const SOCKET_TIMEOUT: Duration = Duration::from_secs(60); +const INFLATE_CHUNK: usize = 8192; +const INFLATE_RATIO: usize = 8; +const DEFLATE_SLACK: usize = 256; + +#[derive(Debug)] +pub enum StreamError { + Io(io::Error), + Tls(String), + CompressedStartTls, +} + +impl fmt::Display for StreamError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + StreamError::Io(e) => write!(f, "transport error: {e}"), + StreamError::Tls(m) => write!(f, "tls error: {m}"), + StreamError::CompressedStartTls => { + f.write_str("protocol error: cannot STARTTLS after COMPRESS DEFLATE") + } + } + } +} + +impl From for StreamError { + fn from(e: io::Error) -> StreamError { + StreamError::Io(e) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConnectMode { + ImplicitTls, + Plain, + StartTls, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Cleartext { + Forbid, + Allow, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UrlTls { + Implicit, + Negotiated, +} + +#[derive(Debug, Clone, Copy)] +pub struct Scheme { + pub plain: &'static str, + pub tls: &'static str, + pub plain_port: u16, + pub tls_port: u16, +} + +#[derive(Debug, Clone)] +pub struct Endpoint { + pub host: String, + pub port: u16, + pub tls: UrlTls, +} + +impl Endpoint { + pub fn parse(url: &str, scheme: &Scheme) -> Result { + let parsed = + Url::parse(url).map_err(|e| Error::Usage(format!("invalid --url {url:?}: {e}")))?; + let tls = match parsed.scheme() { + s if s == scheme.tls => UrlTls::Implicit, + s if s == scheme.plain => UrlTls::Negotiated, + other => { + return Err(Error::Usage(format!( + "--url scheme must be {} or {}, got {other}", + scheme.plain, scheme.tls + ))); + } + }; + let host = parsed + .host_str() + .ok_or_else(|| Error::Usage(format!("--url missing host: {url}")))? + .to_owned(); + let port = parsed.port().unwrap_or(match tls { + UrlTls::Implicit => scheme.tls_port, + UrlTls::Negotiated => scheme.plain_port, + }); + Ok(Endpoint { host, port, tls }) + } + + pub fn session_url(&self, scheme: &Scheme) -> String { + let name = match self.tls { + UrlTls::Implicit => scheme.tls, + UrlTls::Negotiated => scheme.plain, + }; + format!("{name}://{}:{}", self.host, self.port) + } +} + +pub struct Connector { + config: Arc, +} + +impl Connector { + pub fn new(tls: TlsPolicy) -> Result { + let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + let builder = ClientConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .map_err(|e| StreamError::Tls(format!("rustls protocol versions: {e}")))?; + let config = match tls { + TlsPolicy::AcceptInvalid => builder + .dangerous() + .with_custom_certificate_verifier(Arc::new(NoVerify)) + .with_no_client_auth(), + TlsPolicy::Verify => builder + .with_platform_verifier() + .map_err(|e| StreamError::Tls(format!("rustls platform verifier: {e}")))? + .with_no_client_auth(), + }; + Ok(Connector { + config: Arc::new(config), + }) + } + + pub fn connect_plain(&self, host: &str, port: u16) -> Result { + Ok(Stream::Tcp(Connector::tcp(host, port)?)) + } + + pub fn connect_tls(&self, host: &str, port: u16) -> Result { + let tcp = Connector::tcp(host, port)?; + self.wrap_tls(tcp, host) + } + + fn tcp(host: &str, port: u16) -> Result { + let tcp = TcpStream::connect((host, port))?; + tcp.set_read_timeout(Some(SOCKET_TIMEOUT))?; + tcp.set_write_timeout(Some(SOCKET_TIMEOUT))?; + tcp.set_nodelay(true)?; + Ok(tcp) + } + + fn wrap_tls(&self, tcp: TcpStream, host: &str) -> Result { + let server_name = ServerName::try_from(host.to_owned()) + .map_err(|e| StreamError::Tls(format!("invalid server name {host}: {e}")))?; + let conn = ClientConnection::new(self.config.clone(), server_name) + .map_err(|e| StreamError::Tls(format!("rustls client: {e}")))?; + Ok(Stream::Tls(Box::new(StreamOwned::new(conn, tcp)))) + } +} + +pub enum Stream { + Tcp(TcpStream), + Tls(Box>), + Deflate(Box), + Closed, + #[cfg(test)] + Mock(Box), +} + +impl Stream { + pub fn is_tls(&self) -> bool { + match self { + Stream::Tls(_) => true, + Stream::Deflate(deflate) => deflate.inner.is_tls(), + Stream::Tcp(_) | Stream::Closed => false, + #[cfg(test)] + Stream::Mock(_) => false, + } + } + + pub fn take(&mut self) -> Stream { + mem::replace(self, Stream::Closed) + } + + pub fn upgrade_tls(self, connector: &Connector, host: &str) -> Result { + match self { + Stream::Tcp(tcp) => connector.wrap_tls(tcp, host), + Stream::Deflate(_) => Err(StreamError::CompressedStartTls), + other => Ok(other), + } + } + + pub fn compress(self, primer: &[u8]) -> Stream { + Stream::Deflate(Box::new(Deflate { + inner: self, + decompress: Decompress::new(false), + compress: Compress::new(Compression::default(), false), + rx_raw: primer.to_vec(), + rx_raw_pos: 0, + rx_decoded: Vec::new(), + rx_decoded_pos: 0, + tx: Vec::new(), + })) + } +} + +impl Read for Stream { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + Stream::Tcp(tcp) => tcp.read(buf), + Stream::Tls(tls) => tls.read(buf), + Stream::Deflate(deflate) => deflate.read(buf), + Stream::Closed => Ok(0), + #[cfg(test)] + Stream::Mock(mock) => mock.reader.read(buf), + } + } +} + +impl Write for Stream { + fn write(&mut self, buf: &[u8]) -> io::Result { + match self { + Stream::Tcp(tcp) => tcp.write(buf), + Stream::Tls(tls) => tls.write(buf), + Stream::Deflate(deflate) => deflate.write(buf), + Stream::Closed => Ok(buf.len()), + #[cfg(test)] + Stream::Mock(mock) => { + mock.written.extend_from_slice(buf); + Ok(buf.len()) + } + } + } + + fn flush(&mut self) -> io::Result<()> { + match self { + Stream::Tcp(tcp) => tcp.flush(), + Stream::Tls(tls) => tls.flush(), + Stream::Deflate(deflate) => deflate.inner.flush(), + Stream::Closed => Ok(()), + #[cfg(test)] + Stream::Mock(_) => Ok(()), + } + } +} + +pub struct Deflate { + inner: Stream, + decompress: Decompress, + compress: Compress, + rx_raw: Vec, + rx_raw_pos: usize, + rx_decoded: Vec, + rx_decoded_pos: usize, + tx: Vec, +} + +impl Read for Deflate { + fn read(&mut self, out: &mut [u8]) -> io::Result { + loop { + if let Some(mut avail) = self.rx_decoded.get(self.rx_decoded_pos..) + && !avail.is_empty() + { + let n = avail.read(out)?; + self.rx_decoded_pos += n; + return Ok(n); + } + if self.rx_raw_pos >= self.rx_raw.len() { + self.rx_raw.clear(); + self.rx_raw_pos = 0; + self.rx_raw.resize(INFLATE_CHUNK, 0); + let n = self.inner.read(&mut self.rx_raw)?; + if n == 0 { + return Ok(0); + } + self.rx_raw.truncate(n); + } + self.rx_decoded.clear(); + self.rx_decoded_pos = 0; + self.rx_decoded + .resize(out.len().max(INFLATE_CHUNK) * INFLATE_RATIO, 0); + let in_before = self.decompress.total_in(); + let out_before = self.decompress.total_out(); + self.decompress + .decompress( + self.rx_raw.get(self.rx_raw_pos..).unwrap_or_default(), + &mut self.rx_decoded, + FlushDecompress::None, + ) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + let consumed = (self.decompress.total_in() - in_before) as usize; + let produced = (self.decompress.total_out() - out_before) as usize; + self.rx_raw_pos += consumed; + self.rx_decoded.truncate(produced); + if produced == 0 && consumed == 0 { + self.rx_raw.clear(); + self.rx_raw_pos = 0; + } + } + } +} + +impl Write for Deflate { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.tx.clear(); + self.tx.resize(buf.len() + DEFLATE_SLACK, 0); + let in_before = self.compress.total_in(); + let out_before = self.compress.total_out(); + self.compress + .compress(buf, &mut self.tx, FlushCompress::Sync) + .map_err(io::Error::other)?; + let consumed = (self.compress.total_in() - in_before) as usize; + let produced = (self.compress.total_out() - out_before) as usize; + self.inner + .write_all(self.tx.get(..produced).unwrap_or_default())?; + Ok(consumed) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +#[cfg(test)] +pub struct MockStream { + reader: io::Cursor>, + written: Vec, +} + +#[cfg(test)] +impl Stream { + pub fn mock(server_says: &[u8]) -> Stream { + Stream::Mock(Box::new(MockStream { + reader: io::Cursor::new(server_says.to_vec()), + written: Vec::new(), + })) + } + + pub fn written(&self) -> &[u8] { + match self { + Stream::Mock(mock) => &mock.written, + _ => &[], + } + } +} + +#[derive(Debug)] +struct NoVerify; + +impl ServerCertVerifier for NoVerify { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn supported_verify_schemes(&self) -> Vec { + vec![ + SignatureScheme::RSA_PKCS1_SHA256, + SignatureScheme::RSA_PKCS1_SHA384, + SignatureScheme::RSA_PKCS1_SHA512, + SignatureScheme::ECDSA_NISTP256_SHA256, + SignatureScheme::ECDSA_NISTP384_SHA384, + SignatureScheme::ECDSA_NISTP521_SHA512, + SignatureScheme::RSA_PSS_SHA256, + SignatureScheme::RSA_PSS_SHA384, + SignatureScheme::RSA_PSS_SHA512, + SignatureScheme::ED25519, + ] + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const IMAP: Scheme = Scheme { + plain: "imap", + tls: "imaps", + plain_port: 143, + tls_port: 993, + }; + + fn deflated(plaintext: &[u8]) -> Vec { + let mut compressor = Compress::new(Compression::default(), false); + let mut out = vec![0u8; plaintext.len() + 256]; + compressor + .compress(plaintext, &mut out, FlushCompress::Sync) + .unwrap(); + out.truncate(compressor.total_out() as usize); + out + } + + fn read_all(stream: &mut Stream, want: usize) -> Vec { + let mut got = vec![0u8; 256]; + let mut total = 0usize; + while total < want { + let n = stream.read(&mut got[total..]).unwrap(); + if n == 0 { + break; + } + total += n; + } + got.truncate(total); + got + } + + #[test] + fn connector_constructs_with_either_policy() { + Connector::new(TlsPolicy::Verify).unwrap(); + Connector::new(TlsPolicy::AcceptInvalid).unwrap(); + } + + #[test] + fn deflate_writes_round_trip_through_peer_decompressor() { + let mut stream = Stream::mock(b"").compress(b""); + stream.write_all(b"hello world").unwrap(); + stream.write_all(b" again").unwrap(); + stream.flush().unwrap(); + let Stream::Deflate(deflate) = &stream else { + panic!("not deflate"); + }; + let mut inflater = Decompress::new(false); + let mut plain = vec![0u8; 64]; + inflater + .decompress(deflate.inner.written(), &mut plain, FlushDecompress::Sync) + .unwrap(); + plain.truncate(inflater.total_out() as usize); + assert_eq!(plain, b"hello world again"); + } + + #[test] + fn deflate_reads_decompressed_payload_from_inner() { + let plaintext = b"* OK [CAPABILITY IMAP4rev2 COMPRESS=DEFLATE] hi\r\n"; + let mut stream = Stream::mock(&deflated(plaintext)).compress(b""); + assert_eq!(read_all(&mut stream, plaintext.len()), plaintext); + } + + #[test] + fn deflate_consumes_primer_before_reading_inner() { + let plaintext = b"primed payload\r\n"; + let mut stream = Stream::mock(b"").compress(&deflated(plaintext)); + assert_eq!(read_all(&mut stream, plaintext.len()), plaintext); + } + + #[test] + fn starttls_after_compress_is_refused() { + let connector = Connector::new(TlsPolicy::AcceptInvalid).unwrap(); + let err = Stream::mock(b"") + .compress(b"") + .upgrade_tls(&connector, "example.com") + .err() + .unwrap(); + assert_eq!( + err.to_string(), + "protocol error: cannot STARTTLS after COMPRESS DEFLATE" + ); + } + + #[test] + fn endpoint_defaults_port_per_scheme() { + let e = Endpoint::parse("imaps://mail.example.com", &IMAP).unwrap(); + assert_eq!( + (e.host.as_str(), e.port, e.tls), + ("mail.example.com", 993, UrlTls::Implicit) + ); + let e = Endpoint::parse("imap://mail.example.com", &IMAP).unwrap(); + assert_eq!( + (e.host.as_str(), e.port, e.tls), + ("mail.example.com", 143, UrlTls::Negotiated) + ); + let e = Endpoint::parse("imaps://mail.example.com:1993", &IMAP).unwrap(); + assert_eq!(e.port, 1993); + assert_eq!(e.session_url(&IMAP), "imaps://mail.example.com:1993"); + } + + #[test] + fn endpoint_rejects_other_schemes_and_missing_host() { + let err = Endpoint::parse("http://example.com", &IMAP).unwrap_err(); + assert!( + matches!(&err, Error::Usage(m) if m == "--url scheme must be imap or imaps, got http"), + "{err:?}" + ); + assert!(matches!( + Endpoint::parse("imap:///path", &IMAP), + Err(Error::Usage(_)) + )); + } +} diff --git a/src/net/xml.rs b/src/net/xml.rs new file mode 100644 index 0000000..e7d0e61 --- /dev/null +++ b/src/net/xml.rs @@ -0,0 +1,92 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fmt; + +use quick_xml::events::BytesRef; + +const SPECIAL: [char; 5] = ['&', '<', '>', '"', '\'']; + +#[derive(Debug, Clone, Copy)] +pub struct XmlEscape<'a>(pub &'a str); + +impl XmlEscape<'_> { + fn entity(special: u8) -> &'static str { + match special { + b'&' => "&", + b'<' => "<", + b'>' => ">", + b'"' => """, + _ => "'", + } + } +} + +impl fmt::Display for XmlEscape<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for piece in self.0.split_inclusive(SPECIAL) { + match (piece.strip_suffix(SPECIAL), piece.as_bytes().last()) { + (Some(text), Some(&special)) => { + f.write_str(text)?; + f.write_str(XmlEscape::entity(special))?; + } + _ => f.write_str(piece)?, + } + } + Ok(()) + } +} + +pub struct Entity; + +impl Entity { + pub fn resolve(name: &[u8]) -> Option { + hashify::map!( + name, + char, + "amp" => '&', + "lt" => '<', + "gt" => '>', + "quot" => '"', + "apos" => '\'', + ) + .copied() + } + + pub fn from_ref(reference: &BytesRef<'_>) -> Option { + match reference.resolve_char_ref() { + Ok(Some(c)) => Some(c), + _ => Entity::resolve(reference.as_bytes()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn escapes_all_five_entities() { + assert_eq!( + XmlEscape("a&bd\"e'f").to_string(), + "a&b<c>d"e'f" + ); + assert_eq!(XmlEscape("").to_string(), ""); + assert_eq!(XmlEscape("plain é").to_string(), "plain é"); + assert_eq!(XmlEscape("<<").to_string(), "<<"); + } + + #[test] + fn resolves_named_and_numeric_references() { + assert_eq!(Entity::resolve(b"amp"), Some('&')); + assert_eq!(Entity::resolve(b"apos"), Some('\'')); + assert_eq!(Entity::resolve(b"nbsp"), None); + assert_eq!(Entity::from_ref(&BytesRef::new("quot")), Some('"')); + assert_eq!(Entity::from_ref(&BytesRef::new("#x41")), Some('A')); + assert_eq!(Entity::from_ref(&BytesRef::new("#66")), Some('B')); + assert_eq!(Entity::from_ref(&BytesRef::new("bogus")), None); + } +} diff --git a/src/secret.rs b/src/secret.rs index 153427e..f0a67b1 100644 --- a/src/secret.rs +++ b/src/secret.rs @@ -4,60 +4,89 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use std::io::{BufRead, IsTerminal, Write}; -use std::process::Command; +use std::env; +use std::io::{self, BufRead, IsTerminal, Write}; +use std::process::{Command, Stdio}; use crate::error::Error; -pub fn resolve( - flag_value: Option<&str>, - env_var: &str, - prompt_label: &str, -) -> Result { - if let Some(value) = flag_value { - return Ok(value.to_owned()); +pub struct Secret(String); + +impl Secret { + pub fn resolve(flag: Option<&str>, env_var: &str, prompt: &str) -> Result { + if let Some(value) = flag { + return Ok(Secret(value.to_owned())); + } + if let Ok(value) = env::var(env_var) + && !value.is_empty() + { + return Ok(Secret(value)); + } + if io::stdin().is_terminal() { + return Secret::prompt(prompt); + } + Err(Error::Usage(format!( + "no {prompt} provided: pass the flag, set ${env_var}, or run on a TTY for an interactive prompt" + ))) } - if let Ok(value) = std::env::var(env_var) - && !value.is_empty() - { - return Ok(value); + + pub fn into_inner(self) -> String { + self.0 } - if std::io::stdin().is_terminal() { - return prompt_no_echo(prompt_label); + + fn prompt(label: &str) -> Result { + let mut stderr = io::stderr(); + write!(stderr, "{label}: ")?; + stderr.flush()?; + + let mut line = String::new(); + let read = { + let _echo = EchoGuard::disable(); + io::stdin().lock().read_line(&mut line) + }; + writeln!(stderr)?; + stderr.flush()?; + read?; + + let trimmed = line.trim_end_matches(['\r', '\n']); + if trimmed.is_empty() { + return Err(Error::Usage(format!("empty {label}"))); + } + line.truncate(trimmed.len()); + Ok(Secret(line)) } - Err(Error::Usage(format!( - "no {prompt_label} provided: pass the flag, set ${env_var}, or run on a TTY for an interactive prompt" - ))) } -fn prompt_no_echo(label: &str) -> Result { - let mut stderr = std::io::stderr(); - write!(stderr, "{label}: ")?; - stderr.flush()?; - - let echo_off = set_echo(false); - let mut line = String::new(); - let read = std::io::stdin().lock().read_line(&mut line); - if echo_off { - let _ = set_echo(true); - } - writeln!(stderr)?; - stderr.flush()?; - read?; - - let trimmed = line.trim_end_matches(['\r', '\n']); - if trimmed.is_empty() { - return Err(Error::Usage(format!("empty {label}"))); - } - Ok(trimmed.to_owned()) +#[derive(Clone, Copy)] +enum Echo { + On, + Off, } -fn set_echo(on: bool) -> bool { - let arg = if on { "echo" } else { "-echo" }; - Command::new("stty") - .arg(arg) - .stdin(std::process::Stdio::inherit()) - .status() - .map(|s| s.success()) - .unwrap_or(false) +impl Echo { + fn set(self) -> bool { + let arg = match self { + Echo::On => "echo", + Echo::Off => "-echo", + }; + Command::new("stty") + .arg(arg) + .stdin(Stdio::inherit()) + .status() + .is_ok_and(|s| s.success()) + } +} + +struct EchoGuard; + +impl EchoGuard { + fn disable() -> Option { + Echo::Off.set().then_some(EchoGuard) + } +} + +impl Drop for EchoGuard { + fn drop(&mut self) { + Echo::On.set(); + } } diff --git a/src/sync/calcard.rs b/src/sync/calcard.rs new file mode 100644 index 0000000..697ade5 --- /dev/null +++ b/src/sync/calcard.rs @@ -0,0 +1,424 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::borrow::Cow; +use std::fmt::Write; + +use calcard::Entry; +use calcard::icalendar::ICalendar; +use calcard::vcard::VCard; +use serde_json::{Map, Value}; + +use crate::archive::rows::EventDataType; + +const MAX_INVALID_LINE_CHARS: usize = 80; + +#[derive(Debug, thiserror::Error)] +pub enum CalcardError { + #[error("vcard parse failed: {0}")] + VCardParse(String), + #[error("icalendar parse failed: {0}")] + ICalParse(String), + #[error("jscontact serialisation: {0}")] + Json(#[from] serde_json::Error), + #[error("no parseable entries in iCalendar")] + NoEntries, + #[error("calcard output is not a JSON object")] + NotAnObject, +} + +#[derive(Debug)] +pub struct JsContact { + pub uid: String, + pub data: Value, +} + +impl JsContact { + pub fn from_vcard(text: &str, item_href: &str) -> Result { + let vcard = VCard::parse(LineBreaks::normalize(text)) + .map_err(|e| CalcardError::VCardParse(CalcardError::describe(e)))?; + let js = vcard.into_jscontact::(); + let value: Value = serde_json::from_str(&js.to_string_pretty())?; + let mut obj = match value { + Value::Object(map) => map, + _ => return Err(CalcardError::NotAnObject), + }; + let uid = match obj.remove("uid") { + Some(Value::String(uid)) if !uid.trim().is_empty() => { + if uid.trim().len() == uid.len() { + uid + } else { + uid.trim().to_owned() + } + } + _ => SyntheticUid::derive(item_href), + }; + Ok(JsContact { + uid, + data: Value::Object(obj), + }) + } +} + +pub struct SyntheticUid; + +impl SyntheticUid { + pub fn derive(seed: &str) -> String { + let hash = blake3::hash(seed.as_bytes()); + format!("inbuxa-migrate-syn-{}", hash.to_hex()) + } +} + +#[derive(Debug)] +pub struct JsCalendarEntry { + pub data: Value, + pub data_type: EventDataType, +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct ExtractedFields { + pub is_draft: bool, + pub use_default_alerts: bool, + pub uid: Option, +} + +#[derive(Clone, Copy)] +enum EntryKind { + Item(EventDataType), + Group, +} + +impl JsCalendarEntry { + pub fn parse_all(text: &str) -> Result, CalcardError> { + let ical = ICalendar::parse(LineBreaks::normalize(text)) + .map_err(|e| CalcardError::ICalParse(CalcardError::describe(e)))?; + let js = ical.into_jscalendar::(); + let value: Value = serde_json::from_str(&js.to_string_pretty())?; + let mut out = Vec::new(); + JsCalendarEntry::collect(value, &mut out); + if out.is_empty() { + return Err(CalcardError::NoEntries); + } + Ok(out) + } + + fn collect(value: Value, out: &mut Vec) { + match value { + Value::Object(mut map) => { + let kind = map + .get("@type") + .and_then(Value::as_str) + .and_then(|t| { + hashify::map!(t.as_bytes(), EntryKind, + "Event" => EntryKind::Item(EventDataType::Event), + "Task" => EntryKind::Item(EventDataType::Task), + "Note" => EntryKind::Item(EventDataType::Note), + "Group" => EntryKind::Group, + ) + }) + .copied(); + match kind { + Some(EntryKind::Item(data_type)) if JsCalendarEntry::has_uid(&map) => { + out.push(JsCalendarEntry { + data: Value::Object(map), + data_type, + }) + } + Some(EntryKind::Group) => { + if let Some(Value::Array(entries)) = map.remove("entries") { + for entry in entries { + JsCalendarEntry::collect(entry, out); + } + } + } + _ => {} + } + } + Value::Array(items) => { + for item in items { + JsCalendarEntry::collect(item, out); + } + } + _ => {} + } + } + + fn has_uid(map: &Map) -> bool { + map.get("uid") + .and_then(Value::as_str) + .is_some_and(|u| !u.trim().is_empty()) + } + + pub fn take_extracted(&mut self) -> ExtractedFields { + let Value::Object(map) = &mut self.data else { + return ExtractedFields::default(); + }; + let is_draft = map + .remove("isDraft") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + let use_default_alerts = map + .remove("useDefaultAlerts") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + for key in [ + "calendarIds", + "method", + "utcStart", + "utcEnd", + "isOrigin", + "baseEventId", + ] { + map.remove(key); + } + let uid = map.get("uid").and_then(Value::as_str).map(str::to_owned); + ExtractedFields { + is_draft, + use_default_alerts, + uid, + } + } +} + +struct LineBreaks; + +impl LineBreaks { + fn normalize(text: &str) -> Cow<'_, str> { + let mut after_cr = text.split('\r').skip(1).peekable(); + let mut has_bare_cr = false; + while let Some(rest) = after_cr.next() { + if LineBreaks::is_bare_cr(rest, after_cr.peek().is_some()) { + has_bare_cr = true; + break; + } + } + if !has_bare_cr { + return Cow::Borrowed(text); + } + let mut parts = text.split('\r').peekable(); + let mut out = String::with_capacity(text.len()); + out.push_str(parts.next().unwrap_or_default()); + while let Some(rest) = parts.next() { + if LineBreaks::is_bare_cr(rest, parts.peek().is_some()) { + out.push('\n'); + } + out.push_str(rest); + } + Cow::Owned(out) + } + + fn is_bare_cr(rest: &str, more: bool) -> bool { + let followed_by_cr = rest.is_empty() && more; + !followed_by_cr && !rest.starts_with('\n') + } +} + +impl CalcardError { + fn describe(entry: Entry) -> String { + let mut out = String::new(); + match entry { + Entry::InvalidLine(line) => { + out.push_str("invalid line "); + let mut chars = line.chars(); + let head: String = chars.by_ref().take(MAX_INVALID_LINE_CHARS).collect(); + let _ = write!(out, "{head:?}"); + if chars.next().is_some() { + out.push_str(" (truncated)"); + } + } + Entry::UnexpectedComponentEnd { expected, found } => { + let _ = write!( + out, + "found END:{} where END:{} was expected", + found.as_str(), + expected.as_str() + ); + } + Entry::UnterminatedComponent(name) => { + let _ = write!(out, "{name} is never closed"); + } + Entry::TooManyComponents => out.push_str("too many components"), + Entry::Eof => out.push_str("no BEGIN line found"), + Entry::VCard(_) => out.push_str("found a vCard where an iCalendar was expected"), + Entry::ICalendar(_) => out.push_str("found an iCalendar where a vCard was expected"), + _ => out.push_str("unrecognised content"), + } + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const SAMPLE_VCARD: &str = "BEGIN:VCARD\r\n\ +VERSION:4.0\r\n\ +UID:abc-123\r\n\ +FN:Alice Smith\r\n\ +EMAIL:alice@example.com\r\n\ +END:VCARD\r\n"; + + const SAMPLE_ICAL: &str = "BEGIN:VCALENDAR\r\n\ +VERSION:2.0\r\n\ +PRODID:-//Test//EN\r\n\ +BEGIN:VEVENT\r\n\ +UID:event-1@example.com\r\n\ +DTSTAMP:20260101T000000Z\r\n\ +DTSTART:20260101T090000Z\r\n\ +DTEND:20260101T100000Z\r\n\ +SUMMARY:Test Event\r\n\ +END:VEVENT\r\n\ +END:VCALENDAR\r\n"; + + #[test] + fn vcard_extracts_uid_into_separate_field() { + let jc = JsContact::from_vcard(SAMPLE_VCARD, "/dav/card/u/d/a.vcf").expect("parse"); + assert_eq!(jc.uid, "abc-123"); + let obj = jc.data.as_object().unwrap(); + assert!(!obj.contains_key("uid"), "uid stripped from data"); + } + + #[test] + fn ical_extracts_at_least_one_event() { + let events = JsCalendarEntry::parse_all(SAMPLE_ICAL).expect("parse"); + assert!(!events.is_empty()); + assert_eq!(events[0].data_type, EventDataType::Event); + let uid = events[0] + .data + .as_object() + .and_then(|m| m.get("uid")) + .and_then(Value::as_str) + .unwrap(); + assert!(uid.starts_with("event-1")); + } + + #[test] + fn vcard_without_uid_gets_synthetic_uid_from_href() { + let no_uid = "BEGIN:VCARD\r\nVERSION:4.0\r\nFN:Anon\r\nEND:VCARD\r\n"; + let href = "/dav/card/u/d/no-uid.vcf"; + let jc = JsContact::from_vcard(no_uid, href).expect("synthetic uid"); + assert_eq!(jc.uid, SyntheticUid::derive(href)); + assert!(jc.uid.starts_with("inbuxa-migrate-syn-")); + } + + #[test] + fn synthetic_uid_is_stable_per_href() { + assert_eq!(SyntheticUid::derive("/a"), SyntheticUid::derive("/a")); + assert_ne!(SyntheticUid::derive("/a"), SyntheticUid::derive("/b")); + } + + #[test] + fn ical_with_empty_calendar_fails_no_entries() { + let empty = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nEND:VCALENDAR\r\n"; + let err = JsCalendarEntry::parse_all(empty).unwrap_err(); + assert!(matches!(err, CalcardError::NoEntries)); + } + + #[test] + fn vcard_garbage_fails_parse() { + let err = JsContact::from_vcard("not a vcard", "/x").unwrap_err(); + assert!(matches!(err, CalcardError::VCardParse(_))); + } + + // Kerio Connect sends calendar-data with bare CR line breaks (issue #48). + #[test] + fn ical_with_bare_cr_line_breaks_parses() { + let ical = SAMPLE_ICAL.replace("\r\n", "\r"); + let entries = JsCalendarEntry::parse_all(&ical).expect("bare CR parses"); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].data["title"], "Test Event"); + } + + // A folded line split by a bare CR still unfolds. + #[test] + fn ical_with_bare_cr_unfolds_continuation_lines() { + let ical = "BEGIN:VCALENDAR\rVERSION:2.0\rBEGIN:VEVENT\rUID:k1@example\r\ + DTSTAMP:20260101T000000Z\rDTSTART:20260101T090000Z\r\ + SUMMARY:Quarterly\r review\rEND:VEVENT\rEND:VCALENDAR\r"; + let entries = JsCalendarEntry::parse_all(ical).expect("parse"); + assert_eq!(entries[0].data["title"], "Quarterly review"); + } + + #[test] + fn vcard_with_bare_cr_line_breaks_parses() { + let vcard = SAMPLE_VCARD.replace("\r\n", "\r"); + let jc = JsContact::from_vcard(&vcard, "/x").expect("bare CR parses"); + assert_eq!(jc.uid, "abc-123"); + } + + #[test] + fn line_break_normalization_borrows_unless_bare_cr() { + for text in ["A\nB\n", "A\r\nB\r\n", "A\r\r\nB\r\r\n", "no breaks"] { + assert!( + matches!(LineBreaks::normalize(text), Cow::Borrowed(_)), + "{text:?}" + ); + } + assert_eq!(LineBreaks::normalize("A\rB\r\nC\r"), "A\nB\nC\n"); + } + + #[test] + fn ical_parse_error_names_the_offending_line() { + let err = JsCalendarEntry::parse_all("Content-Type: text/calendar\r\nBEGIN:VCALENDAR\r\n") + .unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("invalid line"), "{msg}"); + assert!(msg.contains("Content-Type"), "{msg}"); + } + + #[test] + fn ical_vtodo_classified_as_task() { + let vtodo = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\n\ + BEGIN:VTODO\r\nUID:t1@example\r\nDTSTAMP:20260101T000000Z\r\n\ + SUMMARY:Buy milk\r\nEND:VTODO\r\nEND:VCALENDAR\r\n"; + let entries = JsCalendarEntry::parse_all(vtodo).expect("parse vtodo"); + assert_eq!(entries[0].data_type, EventDataType::Task); + } + + #[test] + fn ical_recurrence_overrides_collapse_to_single_event() { + let ical = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\n\ + BEGIN:VEVENT\r\nUID:r1@example\r\nDTSTAMP:20260101T000000Z\r\n\ + DTSTART:20260101T090000Z\r\nDTEND:20260101T100000Z\r\n\ + RRULE:FREQ=DAILY;COUNT=3\r\nSUMMARY:Standup\r\nEND:VEVENT\r\n\ + BEGIN:VEVENT\r\nUID:r1@example\r\nDTSTAMP:20260101T000000Z\r\n\ + RECURRENCE-ID:20260102T090000Z\r\nDTSTART:20260102T093000Z\r\nDTEND:20260102T103000Z\r\n\ + SUMMARY:Standup (moved)\r\nEND:VEVENT\r\n\ + END:VCALENDAR\r\n"; + let entries = JsCalendarEntry::parse_all(ical).expect("parse"); + assert_eq!(entries.len(), 1, "calcard merges overrides into master"); + let master = entries[0].data.as_object().unwrap(); + assert!(master.contains_key("recurrenceOverrides")); + } + + #[test] + fn strip_extracted_fields_pulls_isdraft_use_default_alerts() { + let data = serde_json::json!({ + "@type": "Event", + "uid": "x", + "title": "T", + "isDraft": true, + "useDefaultAlerts": true, + "calendarIds": {"a": true}, + "utcStart": "2026-01-01T09:00:00Z", + "method": "PUBLISH" + }); + let mut entry = JsCalendarEntry { + data, + data_type: EventDataType::Event, + }; + let fields = entry.take_extracted(); + assert!(fields.is_draft); + assert!(fields.use_default_alerts); + assert_eq!(fields.uid.as_deref(), Some("x")); + let m = entry.data.as_object().unwrap(); + assert!(!m.contains_key("calendarIds")); + assert!(!m.contains_key("isDraft")); + assert!(!m.contains_key("useDefaultAlerts")); + assert!(!m.contains_key("method")); + assert!(!m.contains_key("utcStart")); + } +} diff --git a/src/sync/dates.rs b/src/sync/dates.rs new file mode 100644 index 0000000..ab2a267 --- /dev/null +++ b/src/sync/dates.rs @@ -0,0 +1,79 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339 as Rfc3339Format; + +pub struct Rfc3339; + +impl Rfc3339 { + pub const EPOCH: &str = "1970-01-01T00:00:00Z"; + + pub fn now() -> String { + Rfc3339::format(OffsetDateTime::now_utc()) + } + + pub fn from_unix(secs: u64) -> String { + i64::try_from(secs) + .ok() + .and_then(|secs| OffsetDateTime::from_unix_timestamp(secs).ok()) + .map_or_else(|| Rfc3339::EPOCH.to_owned(), Rfc3339::format) + } + + fn format(when: OffsetDateTime) -> String { + when.format(&Rfc3339Format) + .unwrap_or_else(|_| Rfc3339::EPOCH.to_owned()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MissingDate { + Now, + Epoch, +} + +impl MissingDate { + pub fn resolve(self) -> String { + match self { + MissingDate::Now => Rfc3339::now(), + MissingDate::Epoch => Rfc3339::EPOCH.to_owned(), + } + } +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, UNIX_EPOCH}; + + use super::*; + + fn reference(secs: u64) -> String { + OffsetDateTime::from(UNIX_EPOCH + Duration::from_secs(secs)) + .format(&Rfc3339Format) + .unwrap() + } + + #[test] + fn from_unix_matches_the_replaced_helper() { + for secs in [0, 1, 784_111_777, 1_700_000_000, 253_402_300_799] { + assert_eq!(Rfc3339::from_unix(secs), reference(secs), "{secs}"); + } + assert_eq!(Rfc3339::from_unix(0), Rfc3339::EPOCH); + } + + #[test] + fn out_of_range_falls_back_to_epoch() { + assert_eq!(Rfc3339::from_unix(u64::MAX), Rfc3339::EPOCH); + } + + #[test] + fn missing_date_policies() { + assert_eq!(MissingDate::Epoch.resolve(), Rfc3339::EPOCH); + let now = MissingDate::Now.resolve(); + assert!(OffsetDateTime::parse(&now, &Rfc3339Format).is_ok()); + assert_ne!(now, Rfc3339::EPOCH); + } +} diff --git a/src/sync/diff.rs b/src/sync/diff.rs new file mode 100644 index 0000000..8cfbd1b --- /dev/null +++ b/src/sync/diff.rs @@ -0,0 +1,123 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; +use std::hash::Hash; + +use crate::archive::LocalId; +use crate::sync::PlanCounts; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IdDiff { + pub new: Vec, + pub present: Vec<(K, LocalId)>, + pub vanished: Vec<(K, LocalId)>, +} + +impl IdDiff { + pub fn compute(server: impl IntoIterator, local: &HashMap) -> IdDiff { + let server = server.into_iter(); + let mut seen = HashSet::with_capacity(server.size_hint().0); + let mut new = Vec::new(); + let mut present = Vec::with_capacity(local.len()); + for key in server { + if seen.contains(&key) { + continue; + } + match local.get(&key) { + Some(&id) => present.push((key.clone(), id)), + None => new.push(key.clone()), + } + seen.insert(key); + } + let mut vanished: Vec<(K, LocalId)> = local + .iter() + .filter(|(key, _)| !seen.contains(*key)) + .map(|(key, &id)| (key.clone(), id)) + .collect(); + vanished.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + IdDiff { + new, + present, + vanished, + } + } + + pub fn is_unchanged(&self) -> bool { + self.new.is_empty() && self.vanished.is_empty() + } + + pub fn plan(&self) -> PlanCounts { + PlanCounts { + new: self.new.len() as u64, + vanished: self.vanished.len() as u64, + present: self.present.len() as u64, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn local(pairs: &[(&str, i64)]) -> HashMap { + pairs + .iter() + .map(|(k, v)| ((*k).to_owned(), LocalId::from_raw(*v))) + .collect() + } + + fn server(ids: &[&str]) -> Vec { + ids.iter().map(|s| (*s).to_owned()).collect() + } + + #[test] + fn partitions_new_present_vanished() { + let d = IdDiff::compute( + server(&["a", "b", "c"]), + &local(&[("b", 2), ("c", 3), ("e", 5), ("d", 4)]), + ); + assert_eq!(d.new, ["a"]); + assert_eq!( + d.present, + [ + ("b".to_owned(), LocalId::from_raw(2)), + ("c".to_owned(), LocalId::from_raw(3)) + ] + ); + assert_eq!( + d.vanished, + [ + ("d".to_owned(), LocalId::from_raw(4)), + ("e".to_owned(), LocalId::from_raw(5)) + ] + ); + assert!(!d.is_unchanged()); + } + + #[test] + fn duplicate_server_ids_count_once_in_server_order() { + let d = IdDiff::compute(server(&["z", "a", "z", "a", "b"]), &local(&[("b", 1)])); + assert_eq!(d.new, ["z", "a"]); + assert_eq!(d.present.len(), 1); + assert!(d.vanished.is_empty()); + } + + #[test] + fn composite_keys_work() { + let mut map = HashMap::new(); + map.insert((7u32, 1u32), LocalId::from_raw(1)); + let d = IdDiff::compute([(7u32, 2u32)], &map); + assert_eq!(d.new, [(7, 2)]); + assert_eq!(d.vanished, [((7, 1), LocalId::from_raw(1))]); + } + + #[test] + fn empty_both_sides_is_unchanged() { + let d = IdDiff::::compute(Vec::new(), &HashMap::new()); + assert!(d.is_unchanged()); + } +} diff --git a/src/sync/emailmeta.rs b/src/sync/emailmeta.rs index 1c1945c..44180bc 100644 --- a/src/sync/emailmeta.rs +++ b/src/sync/emailmeta.rs @@ -4,179 +4,145 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use std::slice; +use mail_parser::{AddressList, HeaderName, HeaderValue, Message, MessageParser}; -use mail_parser::{Address, HeaderName, HeaderValue, Message, MessageParser}; +use crate::sync::keys::EmailIndex; +use crate::sync::keywords::Keywords; -use crate::sync::keys::{EmailIndex, email_index}; - -fn addrs(a: Option<&Address>) -> Vec { - match a { - Some(addr) => addr - .iter() - .filter_map(|x| x.address().map(|s| s.to_owned())) - .collect(), - None => Vec::new(), - } -} - -fn message_ids(value: &HeaderValue) -> Vec { - match value { - HeaderValue::Text(s) => vec![s.to_string()], - HeaderValue::TextList(l) => l.iter().map(|s| s.to_string()).collect(), - _ => Vec::new(), - } -} - -pub fn email_index_from_blob(bytes: &[u8]) -> EmailIndex { - email_meta_from_blob(bytes).0 -} - -const MAX_KEYWORD_LEN: usize = 255; const MAX_MIME_KEYWORDS: usize = 64; -pub fn email_index_and_mime_keywords(bytes: &[u8], keywords: &mut Vec) -> EmailIndex { - let Some(m) = MessageParser::default().parse(bytes) else { - return empty_meta().0; - }; - add_keywords_header(&m, keywords); - email_meta(&m).0 +#[derive(Debug, Clone)] +pub struct EmailMeta { + pub index: EmailIndex, + pub date: Option, } -pub fn add_mime_keywords(bytes: &[u8], keywords: &mut Vec) { - if let Some(m) = MessageParser::default().parse_headers(bytes) { - add_keywords_header(&m, keywords); +impl EmailMeta { + pub fn from_blob(bytes: &[u8]) -> EmailMeta { + MessageParser::default() + .parse(bytes) + .map_or_else(EmailMeta::empty, |m| EmailMeta::from_message(&m)) } -} -fn add_keywords_header(m: &Message<'_>, keywords: &mut Vec) { - let mut added = 0; - for value in m.header_values(HeaderName::Keywords) { - let phrases = match value { - HeaderValue::Text(phrase) => slice::from_ref(phrase), - HeaderValue::TextList(phrases) => phrases.as_slice(), - _ => &[], + pub fn index_with_mime_keywords(bytes: &[u8], keywords: &mut Keywords) -> EmailIndex { + let Some(m) = MessageParser::default().parse(bytes) else { + return EmailMeta::empty().index; }; - for keyword in phrases.iter().filter_map(|phrase| jmap_keyword(phrase)) { - if !keywords.iter().any(|k| k.eq_ignore_ascii_case(&keyword)) { - keywords.push(keyword); - added += 1; - if added == MAX_MIME_KEYWORDS { - return; + EmailMeta::push_keywords_header(&m, keywords); + EmailMeta::from_message(&m).index + } + + pub fn add_mime_keywords(bytes: &[u8], keywords: &mut Keywords) { + if let Some(m) = MessageParser::default().parse_headers(bytes) { + EmailMeta::push_keywords_header(&m, keywords); + } + } + + fn empty() -> EmailMeta { + EmailMeta { + index: EmailIndex::new(&[], &[], "", "", &[]), + date: None, + } + } + + fn from_message(m: &Message<'_>) -> EmailMeta { + let mut mids: Vec = Vec::new(); + for header in m.headers().all(HeaderName::MessageId) { + if let HeaderValue::TextList(ids) = header.value() { + mids.extend(ids.iter().map(str::to_owned)); + } + } + let date = m.date().map(|d| d.to_rfc3339()); + let index = EmailIndex::new( + &mids, + &EmailMeta::addresses(m.from()), + m.subject().unwrap_or(""), + date.as_deref().unwrap_or(""), + &EmailMeta::addresses(m.to()), + ); + EmailMeta { index, date } + } + + fn addresses(list: Option>) -> Vec { + list.map_or_else(Vec::new, |list| { + list.mailboxes() + .filter_map(|mailbox| mailbox.address().map(str::to_owned)) + .collect() + }) + } + + fn push_keywords_header(m: &Message<'_>, keywords: &mut Keywords) { + let mut added = 0; + for header in m.headers().all(HeaderName::Keywords) { + let HeaderValue::TextList(phrases) = header.value() else { + continue; + }; + for keyword in phrases.iter().filter_map(Keywords::from_phrase) { + if keywords.insert(keyword) { + added += 1; + if added == MAX_MIME_KEYWORDS { + return; + } } } } } } -pub(crate) fn jmap_keyword(phrase: &str) -> Option { - let mut keyword = String::with_capacity(phrase.len().min(MAX_KEYWORD_LEN)); - let mut separator = false; - for ch in phrase.chars() { - if ch.is_ascii_graphic() - && !matches!(ch, '(' | ')' | '{' | ']' | '%' | '*' | '"' | '\\') - && !(ch == '$' && keyword.is_empty()) - { - let sep = separator && !keyword.is_empty(); - if keyword.len() + usize::from(sep) >= MAX_KEYWORD_LEN { - break; - } - if sep { - keyword.push('_'); - } - keyword.push(ch.to_ascii_lowercase()); - separator = false; - } else { - separator = true; - } - } - (!keyword.is_empty()).then_some(keyword) -} - -fn empty_meta() -> (EmailIndex, Option) { - (email_index(&[], &[], "", "", &[]), None) -} - -pub fn email_meta_from_blob(bytes: &[u8]) -> (EmailIndex, Option) { - MessageParser::default() - .parse(bytes) - .map_or_else(empty_meta, |m| email_meta(&m)) -} - -fn email_meta(m: &Message<'_>) -> (EmailIndex, Option) { - let mut mids: Vec = Vec::new(); - for v in m.header_values(HeaderName::MessageId) { - mids.extend(message_ids(v)); - } - if mids.is_empty() - && let Some(single) = m.message_id() - { - mids.push(single.to_owned()); - } - let date = m.date().map(|d| d.to_rfc3339()); - let idx = email_index( - &mids, - &addrs(m.from()), - m.subject().unwrap_or(""), - date.as_deref().unwrap_or(""), - &addrs(m.to()), - ); - (idx, date) -} - #[cfg(test)] mod tests { use super::*; - use crate::sync::keys::{EmailKey, email_key}; + use crate::sync::keys::EmailKey; // Every Keywords header and every comma-separated phrase is mapped to a JMAP keyword. #[test] fn mime_keywords_are_read_from_every_keywords_header() { - let mut keywords = vec!["$seen".to_owned(), "VIP".to_owned()]; let raw = b"Message-ID: \r\nKeywords: bearbeitet von KL, Wiedervorlage\r\nKeywords: vip, =?UTF-8?Q?Pr=C3=BCfung?=\r\n\r\nbody"; - let idx = email_index_and_mime_keywords(raw, &mut keywords); + let mut keywords = Keywords::new(); + keywords.push("$seen"); + keywords.push("VIP"); + let idx = EmailMeta::index_with_mime_keywords(raw, &mut keywords); assert_eq!(idx.mids, vec!["k@h".to_owned()]); - assert_eq!( - keywords, - vec![ - "$seen", - "VIP", - "bearbeitet_von_kl", - "wiedervorlage", - "pr_fung" - ] - ); + assert!(keywords.iter().eq([ + "$seen", + "VIP", + "bearbeitet_von_kl", + "wiedervorlage", + "pr_fung" + ])); - let mut from_headers = vec!["$seen".to_owned(), "VIP".to_owned()]; - add_mime_keywords(raw, &mut from_headers); - assert_eq!(from_headers, keywords); + let mut typed_headers = Keywords::new(); + typed_headers.push("$seen"); + typed_headers.push("VIP"); + EmailMeta::add_mime_keywords(raw, &mut typed_headers); + assert_eq!(typed_headers, keywords); } // Phrases are reduced to the RFC 8621 keyword alphabet, without a leading "$", within 255 chars. #[test] fn jmap_keyword_follows_rfc8621_syntax() { assert_eq!( - jmap_keyword(" Review / Legal ").as_deref(), + Keywords::from_phrase(" Review / Legal ").as_deref(), Some("review_/_legal") ); assert_eq!( - jmap_keyword("a(b)c{d]e%f*g\"h\\i").as_deref(), + Keywords::from_phrase("a(b)c{d]e%f*g\"h\\i").as_deref(), Some("a_b_c_d_e_f_g_h_i") ); - assert_eq!(jmap_keyword("$seen").as_deref(), Some("seen")); - assert_eq!(jmap_keyword(" $$Junk").as_deref(), Some("junk")); - assert_eq!(jmap_keyword("a$b").as_deref(), Some("a$b")); + assert_eq!(Keywords::from_phrase("$seen").as_deref(), Some("seen")); + assert_eq!(Keywords::from_phrase(" $$Junk").as_deref(), Some("junk")); + assert_eq!(Keywords::from_phrase("a$b").as_deref(), Some("a$b")); assert_eq!( - jmap_keyword("Q4-2026.final_v2").as_deref(), + Keywords::from_phrase("Q4-2026.final_v2").as_deref(), Some("q4-2026.final_v2") ); - assert_eq!(jmap_keyword(" ( ) "), None); + assert_eq!(Keywords::from_phrase(" ( ) "), None); assert_eq!( - jmap_keyword("x".repeat(300).as_str()).map(|k| k.len()), + Keywords::from_phrase("x".repeat(300).as_str()).map(|k| k.len()), Some(255) ); assert_eq!( - jmap_keyword(&format!("{} b", "a".repeat(254))).map(|k| k.len()), + Keywords::from_phrase(&format!("{} b", "a".repeat(254))).map(|k| k.len()), Some(254) ); } @@ -184,16 +150,16 @@ mod tests { // Empty phrases, messages without the header and the keyword count cap. #[test] fn mime_keywords_skip_empty_and_are_capped() { - let mut keywords = Vec::new(); - add_mime_keywords(b"Keywords: , \r\n\r\nbody", &mut keywords); - add_mime_keywords(b"Subject: none\r\n\r\nbody", &mut keywords); + let mut keywords = Keywords::new(); + EmailMeta::add_mime_keywords(b"Keywords: , \r\n\r\nbody", &mut keywords); + EmailMeta::add_mime_keywords(b"Subject: none\r\n\r\nbody", &mut keywords); assert!(keywords.is_empty()); let many = (0..100) .map(|n| format!("k{n}")) .collect::>() .join(", "); - add_mime_keywords( + EmailMeta::add_mime_keywords( format!("Keywords: {many}\r\n\r\nbody").as_bytes(), &mut keywords, ); @@ -203,52 +169,56 @@ mod tests { #[test] fn parses_core_fields_and_keys_by_message_id() { let raw = b"From: Alice \r\nTo: Bob \r\nSubject: Hi there\r\nMessage-ID: \r\nDate: Mon, 12 May 2025 10:00:00 +0200\r\n\r\nbody\r\n"; - let idx = email_index_from_blob(raw); + let idx = EmailMeta::from_blob(raw).index; assert_eq!(idx.mids, vec!["abc-1@host".to_owned()]); - assert_eq!( - email_key(&idx), - EmailKey::MessageId("abc-1@host".to_owned()) - ); + assert_eq!(idx.key(), EmailKey::MessageId("abc-1@host".to_owned())); } #[test] fn sent_at_offset_is_preserved_then_z_for_utc() { - let off = email_index_from_blob( + let off = EmailMeta::from_blob( b"Message-ID: \r\nDate: Mon, 12 May 2025 10:00:00 +0200\r\n\r\nx", - ); - let utc = email_index_from_blob( + ) + .index; + let utc = EmailMeta::from_blob( b"Message-ID: \r\nDate: Mon, 12 May 2025 08:00:00 +0000\r\n\r\nx", - ); + ) + .index; assert_ne!(off.fb, utc.fb); } #[test] fn missing_message_id_falls_back_to_hash() { - let idx = email_index_from_blob(b"From: a@x\r\nSubject: only\r\n\r\nbody"); + let idx = EmailMeta::from_blob(b"From: a@x\r\nSubject: only\r\n\r\nbody").index; assert!(idx.mids.is_empty()); - assert!(matches!(email_key(&idx), EmailKey::Fallback(_))); + assert!(matches!(idx.key(), EmailKey::Fallback(_))); } #[test] fn multiple_message_id_values_are_all_captured() { let idx = - email_index_from_blob(b"Message-ID: \r\nMessage-ID: \r\nSubject: s\r\n\r\nx"); + EmailMeta::from_blob(b"Message-ID: \r\nMessage-ID: \r\nSubject: s\r\n\r\nx") + .index; assert_eq!(idx.mids, vec!["a@h".to_owned(), "b@h".to_owned()]); - assert_eq!( - email_key(&idx), - EmailKey::MessageId("a@h\u{1f}b@h".to_owned()) - ); + assert_eq!(idx.key(), EmailKey::MessageId("a@h\u{1f}b@h".to_owned())); + } + + // A multipart header with no body made mail-parser 0.11 panic. + #[test] + fn truncated_multipart_does_not_panic() { + let idx = EmailMeta::from_blob(b"d\nContent-Type:multipart;boundary=b5").index; + assert!(idx.mids.is_empty()); } #[test] fn same_message_different_received_metadata_same_key() { - let a = email_index_from_blob( + let a = EmailMeta::from_blob( b"From: a@x\r\nSubject: S\r\nMessage-ID: \r\nDate: Mon, 12 May 2025 10:00:00 +0200\r\n\r\nbody", - ); - let b = email_index_from_blob( + ).index; + let b = EmailMeta::from_blob( b"From: a@x\r\nSubject: S\r\nMessage-ID: \r\nDate: Tue, 13 May 2025 11:00:00 +0000\r\n\r\nbody", - ); - assert_eq!(email_key(&a), email_key(&b)); + ).index; + assert_eq!(a.key(), b.key()); } } diff --git a/src/sync/export.rs b/src/sync/export.rs deleted file mode 100644 index d5dc162..0000000 --- a/src/sync/export.rs +++ /dev/null @@ -1,605 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashMap; -use std::io::{IsTerminal, Write}; - -use rusqlite::Connection; -use serde_json::{Map, Value, json}; - -use crate::db; -use crate::error::Error; -use crate::jmap::blobxfer; -use crate::jmap::connect::{self, Connected}; -use crate::jmap::error::JmapError; -use crate::jmap::http::HttpClient; -use crate::jmap::request::{ - Request, SetRequest, URN_VACATION_RESPONSE, get_all, get_objects, query_all_ids, set_call, -}; -use crate::jmap::session::{Limits, Session}; -use crate::jmap::wire::JmapId; -use crate::jmap::wire::vacation_response; -use crate::logging::{LEVEL_DEFAULT, Logger}; -use crate::sync::import_jmap::mapping::{BlobBytes, TargetResolver}; -use crate::sync::{CommonConfig, Context, ExportConfig, Summary, TypeCounts}; -use crate::types::ObjectType; - -const EXPORT_ORDER: [ObjectType; 9] = [ - ObjectType::Mailbox, - ObjectType::AddressBook, - ObjectType::Calendar, - ObjectType::FileNode, - ObjectType::Identity, - ObjectType::SieveScript, - ObjectType::Email, - ObjectType::ContactCard, - ObjectType::CalendarEvent, -]; - -type IdMap = HashMap; - -#[derive(Default)] -struct Maps { - m: HashMap, -} - -impl Maps { - fn insert(&mut self, ty: ObjectType, local: i64, target: JmapId) { - self.m.entry(ty).or_default().insert(local, target); - } -} - -impl TargetResolver for Maps { - fn target(&self, ty: ObjectType, local_id: i64) -> Option { - self.m.get(&ty)?.get(&local_id).cloned() - } -} - -struct Uploader<'a> { - net: &'a Net, - conn: &'a Connection, - cache: HashMap, - touched: Vec, -} - -impl<'a> Uploader<'a> { - fn new(net: &'a Net, conn: &'a Connection) -> Uploader<'a> { - Uploader { - net, - conn, - cache: HashMap::new(), - touched: Vec::new(), - } - } - - fn upload_with(&mut self, local_id: i64, content_type: &str) -> Result { - self.touched.push(local_id); - if let Some(id) = self.cache.get(&local_id) { - return Ok(id.clone()); - } - let id = if self.net.dry_run { - let _exists = db::blobs::blob_bytes(self.conn, local_id)? - .ok_or_else(|| JmapError::malformed(format!("blob local id {local_id} missing")))?; - JmapId(format!("dryrun-blob-{local_id}")) - } else { - let bytes = db::blobs::blob_bytes(self.conn, local_id)? - .ok_or_else(|| JmapError::malformed(format!("blob local id {local_id} missing")))?; - blobxfer::upload_bytes( - &self.net.client, - &self.net.session, - &self.net.account, - content_type, - &bytes, - )? - }; - self.cache.insert(local_id, id.clone()); - Ok(id) - } - - fn touch(&mut self, local_id: i64) { - self.touched.push(local_id); - } - - fn cached(&self, local_id: i64) -> Option<&JmapId> { - self.cache.get(&local_id) - } - - fn store(&mut self, local_id: i64, id: JmapId) { - self.cache.insert(local_id, id); - } - - fn invalidate(&mut self, local_id: i64) { - self.cache.remove(&local_id); - } - - fn blob_len(&self, local_id: i64) -> Option { - db::blobs::blob_len(self.conn, local_id).ok().flatten() - } - - fn take_touched(&mut self) -> Vec { - std::mem::take(&mut self.touched) - } -} - -impl BlobBytes for Uploader<'_> { - fn bytes(&self, local_id: i64) -> Result, JmapError> { - db::blobs::blob_bytes(self.conn, local_id)? - .ok_or_else(|| JmapError::malformed(format!("blob local id {local_id} missing"))) - } -} - -#[derive(Clone)] -struct Net { - client: HttpClient, - api: String, - account: String, - limits: Limits, - session: Session, - dry_run: bool, - threads: usize, -} - -fn has_rows(conn: &Connection, ty: ObjectType) -> bool { - let table = crate::sync::table_name(ty); - conn.query_row(&format!("SELECT EXISTS(SELECT 1 FROM {table})"), [], |r| { - r.get::<_, i64>(0) - }) - .map(|n| n != 0) - .unwrap_or(false) -} - -pub fn run(common: CommonConfig, config: ExportConfig) -> Result { - let logger = common.logger; - let ctx = Context::open(common, &config.connect)?; - let connected = connect::prepare(&ctx, &config.connect)?; - - let net = Net { - client: ctx.client.clone(), - api: connected.session.api_url.clone(), - account: connected.account_id.clone(), - limits: connected.limits, - session: connected.session.clone(), - dry_run: ctx.dry_run(), - threads: ctx.common.threads, - }; - - let work = work_list(&ctx.conn, &config, &connected, &logger); - let mut maps = Maps::default(); - let mut summary = Summary::default(); - let mut dry_rows: Vec<(&'static str, u64, u64, u64)> = Vec::new(); - let mut plans: HashMap = HashMap::new(); - let mut counts_per_type: HashMap = HashMap::new(); - - for ty in &work { - if logger.enabled(LEVEL_DEFAULT) { - eprintln!("export: {} ...", ty.jmap_name()); - } - let mut counts = TypeCounts::default(); - let res = reconcile_type( - &ctx, - &net, - *ty, - &mut maps, - &logger, - &mut counts, - &mut dry_rows, - ); - let plan = match res { - Ok(p) => p, - Err(e) if e.aborts_run() => return Err(e), - Err(e) => { - logger.warn(&format!("type {} aborted: {e}", ty.jmap_name())); - counts.failed += 1; - Plan::default() - } - }; - plans.insert(*ty, plan); - counts_per_type.insert(*ty, counts); - } - - if config.prune { - prune_phase( - &ctx, - &net, - &work, - &plans, - &config, - &logger, - &mut counts_per_type, - )?; - } - - for ty in &work { - if let Some(counts) = counts_per_type.remove(ty) { - summary.per_type.push((ty.jmap_name(), counts)); - } - } - - if config - .objects - .as_ref() - .is_none_or(|s| s.contains(&ObjectType::SieveScript)) - && net.session.may_support(&net.account, URN_VACATION_RESPONSE) - { - let mut counts = TypeCounts::default(); - let present = match vacation::reconcile(&ctx, &net, &mut counts, &logger) { - Ok(present) => present, - Err(e) if e.aborts_run() => return Err(e), - Err(e) => { - logger.warn(&format!( - "type {} aborted: {e}", - vacation_response::TYPE_NAME - )); - counts.failed += 1; - true - } - }; - if present { - if ctx.dry_run() { - dry_rows.push((vacation_response::TYPE_NAME, counts.updated, 0, 0)); - } - summary - .per_type - .push((vacation_response::TYPE_NAME, counts)); - } - } - - if ctx.dry_run() { - print_dry_run(&dry_rows, config.prune); - return Ok(Summary::default()); - } - summary.retries_observed = ctx.client.retries_observed(); - summary.retry_after_sleeps = ctx.client.retry_after_sleeps(); - Ok(summary) -} - -fn prune_phase( - ctx: &Context, - net: &Net, - work: &[ObjectType], - plans: &HashMap, - config: &ExportConfig, - logger: &Logger, - counts_per_type: &mut HashMap, -) -> Result<(), Error> { - let totals: Vec<(ObjectType, &Plan)> = work - .iter() - .filter_map(|ty| plans.get(ty).map(|p| (*ty, p))) - .filter(|(_, p)| !p.prune_candidates.is_empty()) - .collect(); - if totals.is_empty() { - return Ok(()); - } - eprintln!("prune plan:"); - let total: usize = totals.iter().map(|(_, p)| p.prune_candidates.len()).sum(); - for (ty, p) in &totals { - eprintln!( - " {:<22} {:>6} candidate(s); sample: {}", - ty.jmap_name(), - p.prune_candidates.len(), - sample(&p.prune_candidates), - ); - } - eprintln!(" {:<22} {:>6} total", "(all types)", total); - if ctx.dry_run() { - return Ok(()); - } - if !config.yes && std::io::stdin().is_terminal() { - eprint!("destroy all {total} objects across all types? [y/N] "); - let _ = std::io::stderr().flush(); - let mut line = String::new(); - std::io::stdin() - .read_line(&mut line) - .map_err(|e| Error::Partial(e.to_string()))?; - if !matches!(line.trim(), "y" | "Y" | "yes") { - return Err(Error::PruneAborted); - } - } - for ty in work.iter().rev() { - if let Some(plan) = plans.get(ty) - && !plan.prune_candidates.is_empty() - && let Some(counts) = counts_per_type.get_mut(ty) - { - do_destroy(net, *ty, plan, logger, counts); - } - } - Ok(()) -} - -fn work_list( - conn: &Connection, - config: &ExportConfig, - connected: &Connected, - logger: &Logger, -) -> Vec { - let selected = config.objects.as_ref(); - let participant = ObjectType::ParticipantIdentity; - if selected.is_none_or(|s| s.contains(&participant)) - && has_rows(conn, participant) - && logger.enabled(LEVEL_DEFAULT) - { - eprintln!( - "export: {} not exported; the target creates participant identities from the account's addresses", - participant.jmap_name() - ); - } - EXPORT_ORDER - .into_iter() - .filter(|ty| selected.map(|s| s.contains(ty)).unwrap_or(true)) - .filter(|ty| has_rows(conn, *ty)) - .filter(|ty| { - if connected.supports(*ty) { - true - } else { - logger.warn(&format!( - "target does not support {}; skipping", - ty.jmap_name() - )); - false - } - }) - .collect() -} - -#[allow(clippy::too_many_arguments)] -fn reconcile_type( - ctx: &Context, - net: &Net, - ty: ObjectType, - maps: &mut Maps, - logger: &Logger, - counts: &mut TypeCounts, - dry_rows: &mut Vec<(&'static str, u64, u64, u64)>, -) -> Result { - let plan = match ty { - ObjectType::Mailbox | ObjectType::FileNode => { - tree::reconcile(ctx, net, ty, maps, counts, logger) - } - ObjectType::AddressBook | ObjectType::Calendar => { - flat::reconcile(ctx, net, ty, maps, counts, logger) - } - ObjectType::Identity => keyed::reconcile_identity(ctx, net, maps, counts, logger), - ObjectType::ParticipantIdentity => Ok(Plan::default()), - ObjectType::SieveScript => sieve::reconcile(ctx, net, maps, counts, logger), - ObjectType::ContactCard | ObjectType::CalendarEvent => { - uidtype::reconcile(ctx, net, ty, maps, counts, logger) - } - ObjectType::Email => email::reconcile(ctx, net, maps, counts, logger), - }?; - - if ctx.dry_run() { - dry_rows.push(( - ty.jmap_name(), - counts.created, - counts.skipped, - plan.prune_candidates.len() as u64, - )); - } - - Ok(plan) -} - -#[derive(Default)] -pub struct Plan { - pub prune_candidates: Vec, - pub active_sieve_target: Option, -} - -fn do_destroy(net: &Net, ty: ObjectType, plan: &Plan, logger: &Logger, counts: &mut TypeCounts) { - if ty == ObjectType::SieveScript && plan.active_sieve_target.is_none() { - deactivate_active_sieve_script(net, logger); - } - let destroy = Value::Array( - plan.prune_candidates - .iter() - .map(|s| Value::String(s.clone())) - .collect(), - ); - let extra = destroy_contents_arg(ty); - match set_call( - &net.client, - &net.api, - &net.account, - ty.jmap_name(), - SetRequest { - destroy: Some(destroy), - extra_args: &extra, - ..Default::default() - }, - &net.limits, - ) { - Ok(outcome) => { - counts.deleted += outcome.destroyed.len() as u64; - for (id, err) in &outcome.not_destroyed { - logger.warn(&format!( - "prune: {} {id} not destroyed: {err}", - ty.jmap_name() - )); - counts.skipped += 1; - } - } - Err(e) => { - logger.warn(&format!( - "prune {}: destroy request failed: {e}", - ty.jmap_name() - )); - counts.failed += plan.prune_candidates.len() as u64; - } - } -} - -fn deactivate_active_sieve_script(net: &Net, logger: &Logger) { - let mut req = Request::new(); - req.call( - "SieveScript/set", - json!({ "accountId": net.account, "onSuccessDeactivateScript": true }), - "d", - ); - let outcome = req.send(&net.client, &net.api).and_then(|resp| { - let mr = resp.first()?; - crate::jmap::request::check_method_error(mr) - }); - if let Err(e) = outcome { - logger.warn(&format!( - "prune: SieveScript deactivation failed before destroy: {e}" - )); - } -} - -fn destroy_contents_arg(ty: ObjectType) -> Vec<(&'static str, Value)> { - match ty { - ObjectType::AddressBook => vec![("onDestroyRemoveContents", Value::Bool(false))], - ObjectType::Calendar => vec![("onDestroyRemoveEvents", Value::Bool(false))], - ObjectType::FileNode => vec![("onDestroyRemoveChildren", Value::Bool(false))], - _ => Vec::new(), - } -} - -fn sample(ids: &[String]) -> String { - let n = ids.len().min(5); - ids[..n].join(", ") -} - -fn print_dry_run(rows: &[(&'static str, u64, u64, u64)], prune: bool) { - if prune { - println!( - "{:<22} {:>10} {:>10} {:>12}", - "TYPE", "CREATE", "MATCHED", "WOULD-DESTROY" - ); - for (ty, c, m, d) in rows { - println!("{ty:<22} {c:>10} {m:>10} {d:>12}"); - } - } else { - println!("{:<22} {:>10} {:>10}", "TYPE", "CREATE", "MATCHED"); - for (ty, c, m, _) in rows { - println!("{ty:<22} {c:>10} {m:>10}"); - } - } -} - -mod tree; - -mod flat; - -mod keyed; - -mod sieve; - -mod vacation; - -mod uidtype; - -mod email; - -mod blobpool; - -mod common { - use super::*; - - pub fn target_query_get( - net: &Net, - ty: ObjectType, - props: Option<&[&str]>, - ) -> Result, JmapError> { - let ids = query_all_ids( - &net.client, - &net.api, - &net.account, - ty.jmap_name(), - &net.limits, - )?; - if ids.is_empty() { - return Ok(Vec::new()); - } - let got = get_objects::( - &net.client, - &net.api, - &net.account, - ty.jmap_name(), - &ids, - props, - &net.limits, - )?; - Ok(got.list) - } - - pub fn target_get_all(net: &Net, ty: ObjectType) -> Result, JmapError> { - Ok(get_all::(&net.client, &net.api, &net.account, ty.jmap_name())?.list) - } - - pub fn jid(v: &Value) -> Option { - v.get("id").and_then(Value::as_str).map(str::to_owned) - } - - pub fn create_batch( - net: &Net, - ty: ObjectType, - creates: Vec<(String, Value)>, - ) -> Result { - if net.dry_run { - return Ok(synthesize_dry_run_outcome(ty, &creates)); - } - let mut map = Map::new(); - for (cid, obj) in creates { - map.insert(cid, obj); - } - set_call( - &net.client, - &net.api, - &net.account, - ty.jmap_name(), - SetRequest { - create: Some(Value::Object(map)), - ..Default::default() - }, - &net.limits, - ) - } - - fn blob_not_found(outcome: &crate::jmap::request::SetOutcome, cid: &str) -> bool { - outcome.not_created.iter().any(|(c, err)| { - c == cid && err.get("type").and_then(Value::as_str) == Some("blobNotFound") - }) - } - - pub fn retry_if_blob_missing( - net: &Net, - ty: ObjectType, - cid: &str, - uploader: &mut Uploader<'_>, - touched: Vec, - outcome: crate::jmap::request::SetOutcome, - mut rebuild: F, - ) -> Result - where - F: FnMut(&mut Uploader<'_>) -> Result, - { - if !blob_not_found(&outcome, cid) { - return Ok(outcome); - } - for id in &touched { - uploader.invalidate(*id); - } - let _ = uploader.take_touched(); - let wire = rebuild(uploader)?; - let _ = uploader.take_touched(); - create_batch(net, ty, vec![(cid.to_owned(), wire)]).map_err(Error::from) - } - - fn synthesize_dry_run_outcome( - ty: ObjectType, - creates: &[(String, Value)], - ) -> crate::jmap::request::SetOutcome { - let mut outcome = crate::jmap::request::SetOutcome::default(); - for (cid, _) in creates { - let synthetic = serde_json::json!({ - "id": format!("dryrun-{}-{cid}", ty.jmap_name()) - }); - outcome.created.push((cid.clone(), synthetic)); - } - outcome - } -} diff --git a/src/sync/export/email.rs b/src/sync/export/email.rs deleted file mode 100644 index b9a270c..0000000 --- a/src/sync/export/email.rs +++ /dev/null @@ -1,584 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::{HashMap, HashSet}; - -use serde_json::{Map, Value, json}; - -use super::blobpool::UploadPool; -use super::common::{jid, target_query_get}; -use super::{Maps, Net, Plan, Uploader}; -use crate::error::Error; -use crate::jmap::error::JmapError; -use crate::jmap::request::{Request, check_method_error, get_objects, retry_method_call}; -use crate::jmap::retry::MethodCallKind; -use crate::jmap::session::Limits; -use crate::jmap::wire::JmapId; -use crate::logging::Logger; -use crate::sync::import_jmap::mapping::{EMAIL_SELECT, EmailRow, TargetResolver, row_to_email}; -use crate::sync::keys::{EmailIndex, EmailKey, email_index, email_keys, index_from_json}; -use crate::sync::{Context, TypeCounts}; -use crate::types::ObjectType; - -fn server_index(v: &Value) -> EmailIndex { - let arr = |k: &str| { - v.get(k) - .and_then(Value::as_array) - .map(|a| { - a.iter() - .filter_map(|x| x.get("email").and_then(Value::as_str).map(str::to_owned)) - .collect::>() - }) - .unwrap_or_default() - }; - let mids: Vec = v - .get("messageId") - .and_then(Value::as_array) - .map(|a| { - a.iter() - .filter_map(|x| x.as_str().map(str::to_owned)) - .collect() - }) - .unwrap_or_default(); - email_index( - &mids, - &arr("from"), - v.get("subject").and_then(Value::as_str).unwrap_or(""), - v.get("sentAt").and_then(Value::as_str).unwrap_or(""), - &arr("to"), - ) -} - -pub fn reconcile( - ctx: &Context, - net: &Net, - maps: &mut Maps, - counts: &mut TypeCounts, - logger: &Logger, -) -> Result { - let ty = ObjectType::Email; - - let target_min = target_query_get(net, ty, Some(&["messageId"])).map_err(Error::from)?; - let mut indices: Vec = target_min.iter().map(server_index).collect(); - - let fallback_ids: Vec = target_min - .iter() - .zip(indices.iter()) - .filter(|(_, i)| i.mids.is_empty()) - .filter_map(|(v, _)| jid(v).map(JmapId)) - .collect(); - if !fallback_ids.is_empty() { - let got = get_objects::( - &net.client, - &net.api, - &net.account, - ty.jmap_name(), - &fallback_ids, - Some(&["messageId", "from", "subject", "sentAt", "to"]), - &net.limits, - ) - .map_err(Error::from)?; - let by_id: HashMap = got - .list - .iter() - .filter_map(|v| jid(v).map(|i| (i, v))) - .collect(); - for (v, slot) in target_min.iter().zip(indices.iter_mut()) { - if let Some(full) = jid(v).and_then(|i| by_id.get(&i)) { - *slot = server_index(full); - } - } - } - let target_keys: HashSet = email_keys(&indices).into_iter().collect(); - - let local: Vec<(i64, EmailRow)> = { - let mut stmt = ctx - .conn - .prepare(EMAIL_SELECT) - .map_err(|e| Error::Partial(e.to_string()))?; - stmt.query_map([], |row| { - let id: i64 = row.get(0)?; - Ok((id, row_to_email(row))) - }) - .and_then(|m| m.collect::, _>>()) - .map_err(|e| Error::Partial(e.to_string()))? - .into_iter() - .map(|(id, r)| Ok((id, r.map_err(Error::from)?))) - .collect::>()? - }; - - let local_indices: Vec = local - .iter() - .map(|(_, r)| index_from_json(&r.message_match)) - .collect(); - let local_keys = email_keys(&local_indices); - - let mut uploader = Uploader::new(net, &ctx.conn); - let pool = (!net.dry_run).then(|| UploadPool::new(net, net.threads, EMAIL_MIME)); - let unmatched: Vec<&(i64, EmailRow)> = local_keys - .iter() - .zip(local.iter()) - .filter_map(|(key, entry)| { - if target_keys.contains(key) { - counts.skipped += 1; - None - } else { - Some(entry) - } - }) - .collect(); - - let mut exporter = Exporter { - net, - uploader: &mut uploader, - pool: pool.as_ref(), - maps, - counts, - logger, - batch: import_batch_size(&net.limits), - }; - let mut rest = unmatched.as_slice(); - while !rest.is_empty() { - let (chunk, tail) = rest.split_at(exporter.batch.min(rest.len())); - exporter.export_chunk(chunk)?; - rest = tail; - } - - Ok(Plan::default()) -} - -const EMAIL_MIME: &str = "message/rfc822"; - -pub fn import_batch_size(limits: &Limits) -> usize { - usize::try_from(limits.max_objects_in_set.max(1)).unwrap_or(usize::MAX) -} - -pub fn reupload_batch_size(previous: usize) -> usize { - (previous / 2).max(1) -} - -fn build_mailbox_ids(row: &EmailRow, maps: &Maps) -> Option> { - let mut mids = Map::new(); - for ml in &row.mailbox_locals { - let t = maps.target(ObjectType::Mailbox, *ml)?; - mids.insert(t.0, Value::Bool(true)); - } - Some(mids) -} - -fn build_keywords(row: &EmailRow) -> Map { - let mut kw = Map::new(); - for k in &row.keywords { - kw.insert(k.clone(), Value::Bool(true)); - } - kw -} - -fn blob_hint(uploader: &Uploader, row: &EmailRow) -> String { - let idx = index_from_json(&row.message_match); - let mut s = match idx.mids.first() { - Some(mid) => format!("message-id <{mid}>"), - None => "no message-id".to_owned(), - }; - if let Some(len) = uploader.blob_len(row.blob_local_id) { - use std::fmt::Write; - let _ = write!(s, ", {}", crate::inspect::format_bytes(len)); - } - s -} - -fn size_note(e: &JmapError) -> &'static str { - if matches!( - e, - JmapError::RequestTooLarge | JmapError::SingleObjectTooLarge(_) - ) { - "; exceeds the target server size limit, so this message is skipped and re-running will not migrate it" - } else { - "" - } -} - -fn import_item( - blob: String, - mids: Map, - kw: Map, - received_at: &str, -) -> Value { - json!({ - "blobId": blob, - "mailboxIds": Value::Object(mids), - "keywords": Value::Object(kw), - "receivedAt": received_at, - }) -} - -struct Pending<'r> { - cid: String, - row: &'r EmailRow, - mids: Map, -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum Attempt { - First, - Reupload, - LastReupload, -} - -impl Attempt { - fn suffix(self) -> &'static str { - match self { - Attempt::First => "", - Attempt::Reupload | Attempt::LastReupload => " after blob re-upload", - } - } - - fn upload_verb(self) -> &'static str { - match self { - Attempt::First => "blob upload", - Attempt::Reupload | Attempt::LastReupload => "blob re-upload", - } - } -} - -struct Exporter<'a, 'u> { - net: &'a Net, - uploader: &'a mut Uploader<'u>, - pool: Option<&'a UploadPool>, - maps: &'a Maps, - counts: &'a mut TypeCounts, - logger: &'a Logger, - batch: usize, -} - -impl<'r> Exporter<'_, '_> { - fn export_chunk(&mut self, chunk: &[&'r (i64, EmailRow)]) -> Result<(), Error> { - let mut pending: Vec> = Vec::with_capacity(chunk.len()); - for (local_id, row) in chunk.iter().copied() { - let cid = format!("e{local_id}"); - match build_mailbox_ids(row, self.maps) { - Some(mids) => pending.push(Pending { cid, row, mids }), - None => { - self.logger.warn(&format!( - "Email/import {cid} ({}) skipped: mailbox not on target", - blob_hint(self.uploader, row) - )); - self.counts.failed += 1; - } - } - } - let mut size = pending.len(); - let mut retry = self.import(pending, Attempt::First)?; - if !retry.is_empty() { - self.batch = reupload_batch_size(self.batch); - } - while !retry.is_empty() { - size = reupload_batch_size(size); - let attempt = if size == 1 { - Attempt::LastReupload - } else { - Attempt::Reupload - }; - for p in &retry { - self.uploader.invalidate(p.row.blob_local_id); - } - let mut again = Vec::new(); - while !retry.is_empty() { - let tail = retry.split_off(size.min(retry.len())); - let head = std::mem::replace(&mut retry, tail); - again.extend(self.import(head, attempt)?); - } - retry = again; - } - Ok(()) - } - - fn import( - &mut self, - pending: Vec>, - attempt: Attempt, - ) -> Result>, Error> { - let mut failed = self.upload(&pending); - let mut ready: Vec> = Vec::with_capacity(pending.len()); - let mut items: Vec<(String, Value)> = Vec::with_capacity(pending.len()); - for p in pending { - let blob = match failed.remove(&p.row.blob_local_id) { - Some(e) => Err(e), - None => self - .uploader - .cached(p.row.blob_local_id) - .map(|b| b.0.clone()) - .ok_or_else(|| JmapError::malformed("blob upload did not complete")), - }; - let blob = match blob { - Ok(b) => b, - Err(e) => { - self.logger.warn(&format!( - "Email/import {} ({}) {} failed: {e}{}", - p.cid, - blob_hint(self.uploader, p.row), - attempt.upload_verb(), - size_note(&e) - )); - self.counts.failed += 1; - continue; - } - }; - if self.net.dry_run { - self.counts.created += 1; - continue; - } - let item = import_item( - blob, - p.mids.clone(), - build_keywords(p.row), - &p.row.received_at, - ); - items.push((p.cid.clone(), item)); - ready.push(p); - } - if items.is_empty() { - return Ok(Vec::new()); - } - - let mut results = Vec::with_capacity(items.len()); - if let Err(e) = send_import(self.net, &items, self.logger, &mut results) { - let mapped = Error::from(e); - if mapped.aborts_run() { - return Err(mapped); - } - for p in ready.iter().skip(results.len()) { - self.logger.warn(&format!( - "Email/import {} ({}) send failed{}: {mapped}", - p.cid, - blob_hint(self.uploader, p.row), - attempt.suffix() - )); - self.counts.failed += 1; - } - ready.truncate(results.len()); - } - - let mut retry = Vec::new(); - for (p, res) in ready.into_iter().zip(results) { - match res { - Ok(SingleImport::Created) => self.counts.created += 1, - Ok(SingleImport::Skipped) => self.counts.skipped += 1, - Ok(SingleImport::NotCreated { error_type, .. }) - if attempt != Attempt::LastReupload && error_type == "blobNotFound" => - { - retry.push(p); - } - Ok(SingleImport::NotCreated { detail, .. }) => { - self.logger.warn(&format!( - "Email/import {} ({}) failed{}: {detail}", - p.cid, - blob_hint(self.uploader, p.row), - attempt.suffix() - )); - self.counts.failed += 1; - } - Err(e) => { - self.logger.warn(&format!( - "Email/import {} ({}) send failed{}: {e}{}", - p.cid, - blob_hint(self.uploader, p.row), - attempt.suffix(), - size_note(&e) - )); - self.counts.failed += 1; - } - } - } - Ok(retry) - } - - fn upload(&mut self, pending: &[Pending<'_>]) -> HashMap { - let Some(pool) = self.pool else { - let mut failed = HashMap::new(); - for p in pending { - if let Err(e) = self.uploader.upload_with(p.row.blob_local_id, EMAIL_MIME) { - failed.insert(p.row.blob_local_id, e); - } - } - return failed; - }; - let mut seen: HashSet = HashSet::with_capacity(pending.len()); - let mut queue: Vec = Vec::with_capacity(pending.len()); - for p in pending { - let local_id = p.row.blob_local_id; - self.uploader.touch(local_id); - if self.uploader.cached(local_id).is_none() && seen.insert(local_id) { - queue.push(local_id); - } - } - pool.upload(self.uploader, &queue) - } -} - -enum SingleImport { - Created, - Skipped, - NotCreated { error_type: String, detail: String }, -} - -fn import_request(net: &Net, items: &[(String, Value)]) -> Request { - let emails: Map = items.iter().cloned().collect(); - let mut req = Request::new(); - req.call( - "Email/import", - json!({ "accountId": net.account, "emails": Value::Object(emails) }), - "i", - ); - req -} - -fn send_import( - net: &Net, - items: &[(String, Value)], - logger: &Logger, - out: &mut Vec>, -) -> Result<(), JmapError> { - let req = import_request(net, items); - let sent = match req.fits(&net.limits) { - Ok(()) => retry_method_call( - &net.client, - MethodCallKind::SingleObjectWrite, - logger, - || { - let resp = req.send(&net.client, &net.api)?; - let mr = resp.first()?; - check_method_error(mr)?; - Ok(items - .iter() - .map(|(cid, _)| item_outcome(&mr.args, cid)) - .collect::>()) - }, - ), - Err(e) => Err(e), - }; - match sent { - Ok(outcomes) => { - out.extend(outcomes.into_iter().map(Ok)); - Ok(()) - } - Err(JmapError::RequestTooLarge) if items.len() > 1 => { - let (left, right) = items.split_at(items.len() / 2); - send_import(net, left, logger, out)?; - send_import(net, right, logger, out) - } - Err(e @ JmapError::RequestTooLarge) => { - out.push(Err(e)); - Ok(()) - } - Err(e) => Err(e), - } -} - -fn item_outcome(args: &Value, cid: &str) -> SingleImport { - if let Some(err) = args - .get("notCreated") - .and_then(Value::as_object) - .and_then(|nc| nc.get(cid)) - { - let error_type = err - .get("type") - .and_then(Value::as_str) - .unwrap_or("") - .to_owned(); - if error_type == "alreadyExists" { - return SingleImport::Skipped; - } - return SingleImport::NotCreated { - error_type, - detail: err.to_string(), - }; - } - if args - .get("created") - .and_then(Value::as_object) - .is_some_and(|c| c.contains_key(cid)) - { - return SingleImport::Created; - } - SingleImport::NotCreated { - error_type: String::new(), - detail: format!("Email/import returned neither created nor notCreated for {cid}"), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn limits(set: u64) -> Limits { - Limits { - max_objects_in_get: 500, - max_objects_in_set: set, - max_calls_in_request: 16, - max_concurrent_requests: 4, - max_concurrent_upload: 4, - max_size_request: 10_000_000, - max_size_upload: 50_000_000, - } - } - - #[test] - fn batch_size_follows_max_objects_in_set() { - assert_eq!(import_batch_size(&limits(500)), 500); - assert_eq!(import_batch_size(&limits(7)), 7); - assert_eq!(import_batch_size(&limits(0)), 1); - } - - #[test] - fn reupload_batches_halve_down_to_one() { - let mut size = 500; - let mut steps = Vec::new(); - while size > 1 { - size = reupload_batch_size(size); - steps.push(size); - } - assert_eq!(steps, vec![250, 125, 62, 31, 15, 7, 3, 1]); - assert_eq!(reupload_batch_size(1), 1); - assert_eq!(reupload_batch_size(0), 1); - } - - #[test] - fn chunking_yields_ceil_n_over_limit_batches() { - let ids: Vec = (0..1001).collect(); - let sizes: Vec = ids - .chunks(import_batch_size(&limits(500))) - .map(<[u32]>::len) - .collect(); - assert_eq!(sizes, vec![500, 500, 1]); - } - - #[test] - fn item_outcome_reads_each_creation_id() { - let args = json!({ - "created": { "e1": { "id": "M1" } }, - "notCreated": { - "e2": { "type": "alreadyExists", "existingId": "M9" }, - "e3": { "type": "blobNotFound" }, - "e4": { "type": "overQuota" } - } - }); - assert!(matches!(item_outcome(&args, "e1"), SingleImport::Created)); - assert!(matches!(item_outcome(&args, "e2"), SingleImport::Skipped)); - assert!(matches!( - item_outcome(&args, "e3"), - SingleImport::NotCreated { ref error_type, .. } if error_type == "blobNotFound" - )); - assert!(matches!( - item_outcome(&args, "e4"), - SingleImport::NotCreated { ref error_type, .. } if error_type == "overQuota" - )); - assert!(matches!( - item_outcome(&args, "e5"), - SingleImport::NotCreated { ref error_type, .. } if error_type.is_empty() - )); - } -} diff --git a/src/sync/export/flat.rs b/src/sync/export/flat.rs deleted file mode 100644 index 68f9da5..0000000 --- a/src/sync/export/flat.rs +++ /dev/null @@ -1,164 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use rusqlite::params; -use serde_json::{Value, json}; - -use super::common::{create_batch, jid, target_get_all}; -use super::{Maps, Net, Plan}; -use crate::error::Error; -use crate::jmap::wire::JmapId; -use crate::logging::{LEVEL_PROGRESS, Logger}; -use crate::sync::import_jmap::mapping::{ - ADDRESS_BOOK_SELECT, CALENDAR_SELECT, row_to_address_book, row_to_calendar, -}; -use crate::sync::keys::fold_name; -use crate::sync::prune::{TargetObj, candidates}; -use crate::sync::{Context, TypeCounts}; -use crate::types::ObjectType; - -pub fn reconcile( - ctx: &Context, - net: &Net, - ty: ObjectType, - maps: &mut Maps, - counts: &mut TypeCounts, - logger: &Logger, -) -> Result { - let table = crate::sync::table_name(ty); - let select = if ty == ObjectType::AddressBook { - ADDRESS_BOOK_SELECT - } else { - CALENDAR_SELECT - }; - - let locals: Vec<(i64, String, bool)> = { - let mut stmt = ctx - .conn - .prepare(&format!("SELECT id, name, is_default FROM {table}")) - .map_err(|e| Error::Partial(e.to_string()))?; - stmt.query_map([], |r| { - Ok((r.get(0)?, r.get::<_, String>(1)?, r.get::<_, i64>(2)? != 0)) - }) - .and_then(|m| m.collect::, _>>()) - .map_err(|e| Error::Partial(e.to_string()))? - }; - - let targets = target_get_all(net, ty).map_err(Error::from)?; - let mut default_claimed = targets - .iter() - .any(|t| t.get("isDefault").and_then(Value::as_bool).unwrap_or(false)); - let mut tmatched = std::collections::HashSet::new(); - - let mut to_create: Vec<(i64, bool)> = Vec::new(); - for (local, name, is_default) in &locals { - let hit = targets.iter().find(|t| { - let tid = jid(t); - t.get("name") - .and_then(Value::as_str) - .map(|n| fold_name(n) == fold_name(name)) - .unwrap_or(false) - && tid.as_ref().map(|i| !tmatched.contains(i)).unwrap_or(false) - }); - match hit.and_then(jid) { - Some(tid) => { - tmatched.insert(tid.clone()); - maps.insert(ty, *local, JmapId(tid)); - counts.skipped += 1; - } - None => to_create.push((*local, *is_default)), - } - } - - if !to_create.is_empty() { - let mut batch = Vec::new(); - for (local, _) in &to_create { - let wire = ctx - .conn - .query_row(&format!("{select} WHERE id = ?1"), params![local], |row| { - Ok(if ty == ObjectType::AddressBook { - row_to_address_book(row).map(|w| serde_json::to_value(&w)) - } else { - row_to_calendar(row).map(|mut w| { - if let Some(dropped) = w.normalize_color() - && logger.enabled(LEVEL_PROGRESS) - { - eprintln!( - "Calendar {local}: dropping color {dropped:?}, not a CSS color name or #RGB/#RRGGBB value" - ); - } - serde_json::to_value(&w) - }) - }) - }) - .map_err(|e| Error::Partial(e.to_string()))? - .map_err(Error::from)? - .map_err(|e| Error::Partial(e.to_string()))?; - let mut wire = wire; - if let Value::Object(m) = &mut wire { - m.remove("isDefault"); - } - batch.push((format!("c{local}"), wire)); - } - let outcome = create_batch(net, ty, batch).map_err(Error::from)?; - for (cid, v) in &outcome.created { - if let Some(local) = cid.strip_prefix('c').and_then(|s| s.parse::().ok()) - && let Some(id) = jid(v) - { - maps.insert(ty, local, JmapId(id.clone())); - counts.created += 1; - if to_create.iter().any(|(l, d)| *l == local && *d) { - if default_claimed { - logger.warn(&format!( - "{} {}: the target account already has a default, leaving it alone", - ty.jmap_name(), - id - )); - } else { - let mut req = crate::jmap::request::Request::new(); - req.call( - format!("{}/set", ty.jmap_name()), - json!({ "accountId": net.account, "onSuccessSetIsDefault": id }), - "d", - ); - match req.send(&net.client, &net.api) { - Ok(_) => default_claimed = true, - Err(e) => { - logger.warn(&format!("{} isDefault not set: {e}", ty.jmap_name())); - } - } - } - } - } - } - for (cid, err) in &outcome.not_created { - logger.warn(&format!("{} {cid} not created: {err}", ty.jmap_name())); - counts.failed += 1; - } - } - - let objs: Vec = targets - .iter() - .filter_map(|t| { - let id = jid(t)?; - Some(TargetObj { - id: id.clone(), - matched: tmatched.contains(&id), - protected: t.get("isDefault").and_then(Value::as_bool).unwrap_or(false), - may_delete: t - .get("myRights") - .and_then(|r| r.get("mayDelete")) - .and_then(Value::as_bool) - .unwrap_or(true), - parent: None, - }) - }) - .collect(); - Ok(Plan { - prune_candidates: candidates(&objs, false), - active_sieve_target: None, - }) -} diff --git a/src/sync/export/keyed.rs b/src/sync/export/keyed.rs deleted file mode 100644 index 3d45853..0000000 --- a/src/sync/export/keyed.rs +++ /dev/null @@ -1,99 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashSet; - -use serde_json::Value; - -use super::common::{create_batch, target_get_all}; -use super::{Maps, Net, Plan}; -use crate::error::Error; -use crate::logging::Logger; -use crate::sync::import_jmap::mapping::{IDENTITY_SELECT, row_to_identity}; -use crate::sync::keys::identity_key; -use crate::sync::{Context, TypeCounts}; -use crate::types::ObjectType; - -fn run( - net: &Net, - ty: ObjectType, - counts: &mut TypeCounts, - logger: &Logger, - target_key: impl Fn(&Value) -> Option<[u8; 32]>, - local_rows: Vec<(i64, [u8; 32], Value)>, -) -> Result { - let targets = target_get_all(net, ty).map_err(Error::from)?; - let mut target_keys: HashSet<[u8; 32]> = HashSet::new(); - for t in &targets { - if let Some(k) = target_key(t) { - target_keys.insert(k); - } - } - - let mut seen: HashSet<[u8; 32]> = HashSet::new(); - let mut batch = Vec::new(); - for (local, key, wire) in local_rows { - if target_keys.contains(&key) || !seen.insert(key) { - counts.skipped += 1; - continue; - } - batch.push((format!("c{local}"), wire)); - } - if !batch.is_empty() { - let outcome = create_batch(net, ty, batch).map_err(Error::from)?; - counts.created += outcome.created.len() as u64; - for (cid, err) in &outcome.not_created { - logger.warn(&format!( - "{} {cid} not created (expected for non-owned addresses): {err}", - ty.jmap_name() - )); - counts.skipped += 1; - } - } - Ok(Plan::default()) -} - -pub fn reconcile_identity( - ctx: &Context, - net: &Net, - _maps: &mut Maps, - counts: &mut TypeCounts, - logger: &Logger, -) -> Result { - let mut stmt = ctx - .conn - .prepare(IDENTITY_SELECT) - .map_err(|e| Error::Partial(e.to_string()))?; - let rows: Vec<(i64, [u8; 32], Value)> = stmt - .query_map([], |row| { - let id: i64 = row.get(0)?; - Ok((id, row_to_identity(row))) - }) - .and_then(|m| m.collect::, _>>()) - .map_err(|e| Error::Partial(e.to_string()))? - .into_iter() - .map(|(id, w)| { - let w = w.map_err(Error::from)?; - let key = identity_key(&w.name, &w.email); - let v = serde_json::to_value(&w).map_err(|e| Error::Partial(e.to_string()))?; - Ok((id, key, v)) - }) - .collect::>()?; - drop(stmt); - run( - net, - ObjectType::Identity, - counts, - logger, - |t| { - Some(identity_key( - t.get("name").and_then(Value::as_str).unwrap_or(""), - t.get("email").and_then(Value::as_str).unwrap_or(""), - )) - }, - rows, - ) -} diff --git a/src/sync/export/sieve.rs b/src/sync/export/sieve.rs deleted file mode 100644 index f44125c..0000000 --- a/src/sync/export/sieve.rs +++ /dev/null @@ -1,157 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::{HashMap, HashSet}; - -use serde_json::{Value, json}; - -use super::common::{create_batch, jid, retry_if_blob_missing, target_get_all}; -use super::{Maps, Net, Plan, Uploader}; -use crate::db; -use crate::error::Error; -use crate::jmap::request::{Request, URN_VACATION_RESPONSE}; -use crate::jmap::wire::vacation_response::VacationResponse; -use crate::logging::Logger; -use crate::sync::import_jmap::mapping::{SIEVE_SELECT, row_to_sieve_script}; -use crate::sync::{Context, TypeCounts}; -use crate::types::ObjectType; - -pub fn reconcile( - ctx: &Context, - net: &Net, - _maps: &mut Maps, - counts: &mut TypeCounts, - logger: &Logger, -) -> Result { - let ty = ObjectType::SieveScript; - let targets = target_get_all(net, ty).map_err(Error::from)?; - - let mut target_by_name: HashMap = HashMap::new(); - for t in &targets { - let (Some(id), Some(name)) = (jid(t), t.get("name").and_then(Value::as_str)) else { - continue; - }; - target_by_name.insert(name.to_owned(), id); - } - - let target_has_vacation = net.session.may_support(&net.account, URN_VACATION_RESPONSE); - let vacation_exported = target_has_vacation - && db::vacation_response::exists(&ctx.conn).map_err(|e| Error::Partial(e.to_string()))?; - - let mut locals: Vec<(i64, Option, bool, i64)> = { - let mut stmt = ctx - .conn - .prepare(SIEVE_SELECT) - .map_err(|e| Error::Partial(e.to_string()))?; - stmt.query_map([], |row| { - let sr = row_to_sieve_script(row); - Ok((row.get::<_, i64>(0)?, sr)) - }) - .and_then(|m| m.collect::, _>>()) - .map_err(|e| Error::Partial(e.to_string()))? - .into_iter() - .map(|(id, sr)| { - let sr = sr.map_err(Error::from)?; - Ok((id, sr.name, sr.is_active, sr.blob_local_id)) - }) - .collect::>()? - }; - - if vacation_exported { - let before = locals.len(); - locals.retain(|(_, name, _, _)| { - !name - .as_deref() - .is_some_and(VacationResponse::is_reserved_script_name) - }); - counts.skipped += (before - locals.len()) as u64; - } - - let mut active_target: Option = None; - let mut deactivate = false; - let mut uploader = Uploader::new(net, &ctx.conn); - - for (local, name, is_active, blob_local) in &locals { - let matched = name.as_ref().and_then(|n| target_by_name.get(n)).cloned(); - let target_id = if let Some(id) = matched { - counts.skipped += 1; - id - } else { - let cid = format!("c{local}"); - let build = |up: &mut Uploader<'_>| -> Result { - let blob_id = up - .upload_with(*blob_local, "application/sieve") - .map_err(Error::from)?; - let mut obj = serde_json::Map::new(); - if let Some(n) = name { - obj.insert("name".to_owned(), Value::String(n.clone())); - } - obj.insert("blobId".to_owned(), Value::String(blob_id.0)); - Ok(Value::Object(obj)) - }; - let _ = uploader.take_touched(); - let wire = build(&mut uploader)?; - let touched = uploader.take_touched(); - let outcome = create_batch(net, ty, vec![(cid.clone(), wire)]).map_err(Error::from)?; - let outcome = - retry_if_blob_missing(net, ty, &cid, &mut uploader, touched, outcome, build)?; - match outcome.created.first().and_then(|(_, v)| jid(v)) { - Some(id) => { - counts.created += 1; - if let Some(n) = name { - target_by_name.insert(n.clone(), id.clone()); - } - id - } - None => { - for (cid, err) in &outcome.not_created { - logger.warn(&format!("SieveScript {cid} not created: {err}")); - } - counts.failed += 1; - continue; - } - } - }; - if *is_active { - active_target = Some(target_id); - } - } - - if active_target.is_none() && locals.iter().all(|(_, _, a, _)| !*a) { - deactivate = true; - } - - if !net.dry_run { - let mut req = Request::new(); - let args = if let Some(id) = &active_target { - json!({ "accountId": net.account, "onSuccessActivateScript": id }) - } else if deactivate { - json!({ "accountId": net.account, "onSuccessDeactivateScript": true }) - } else { - json!({ "accountId": net.account }) - }; - req.call("SieveScript/set", args, "a"); - if let Err(e) = req.send(&net.client, &net.api) { - logger.warn(&format!("SieveScript activation failed: {e}")); - } - } - - let local_names: HashSet = locals.iter().filter_map(|(_, n, _, _)| n.clone()).collect(); - let mut prune_candidates: Vec = target_by_name - .iter() - .filter(|(name, _)| !local_names.contains(*name)) - .filter(|(name, _)| { - !(target_has_vacation && VacationResponse::is_reserved_script_name(name)) - }) - .map(|(_, id)| id.clone()) - .collect(); - prune_candidates.sort(); - - Ok(Plan { - prune_candidates, - active_sieve_target: active_target, - }) -} diff --git a/src/sync/export/tree.rs b/src/sync/export/tree.rs deleted file mode 100644 index d83dec3..0000000 --- a/src/sync/export/tree.rs +++ /dev/null @@ -1,474 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::{HashMap, HashSet}; - -use rusqlite::params; -use serde_json::Value; - -use super::common::{create_batch, jid, retry_if_blob_missing, target_query_get}; -use super::{Net, Plan, Uploader}; -use crate::error::Error; -use crate::jmap::wire::JmapId; -use crate::jmap::wire::file_node::NodeKind; -use crate::logging::Logger; -use crate::sync::import_jmap::mapping::{ - FILE_NODE_SELECT, MAILBOX_SELECT, TargetResolver, row_to_file_node, row_to_mailbox, -}; -use crate::sync::keys::fold_name; -use crate::sync::{Context, TypeCounts}; -use crate::types::ObjectType; - -use super::Maps; - -struct LocalNode { - local: i64, - parent: Option, - name: String, - role: Option, -} - -struct TargetNode { - id: String, - parent: Option, - name: String, - role: Option, - may_delete: bool, -} - -fn load_local(ctx: &Context, ty: ObjectType) -> Result, Error> { - let table = crate::sync::table_name(ty); - let mut stmt = ctx - .conn - .prepare(&format!("SELECT id, parent_id, name, role FROM {table}")) - .map_err(|e| Error::Partial(e.to_string()))?; - let rows = stmt - .query_map([], |r| { - Ok(LocalNode { - local: r.get(0)?, - parent: r.get(1)?, - name: r.get(2)?, - role: r.get(3)?, - }) - }) - .and_then(|m| m.collect::, _>>()) - .map_err(|e| Error::Partial(e.to_string()))?; - Ok(rows) -} - -fn load_target(net: &Net, ty: ObjectType) -> Result, Error> { - let props: &[&str] = &["role", "name", "parentId", "myRights"]; - let list = target_query_get(net, ty, Some(props)).map_err(Error::from)?; - let mut own_rights: HashMap<&str, bool> = HashMap::with_capacity(list.len()); - for v in &list { - if let (Some(id), Some(may)) = ( - v.get("id").and_then(Value::as_str), - v.get("myRights") - .and_then(|r| r.get("mayDelete")) - .and_then(Value::as_bool), - ) { - own_rights.insert(id, may); - } - } - Ok(list - .iter() - .filter_map(|v| { - let id = jid(v)?; - let parent = v.get("parentId").and_then(Value::as_str); - let may_delete = own_rights - .get(id.as_str()) - .or_else(|| parent.and_then(|p| own_rights.get(p))) - .copied() - .unwrap_or(true); - Some(TargetNode { - id, - parent: parent.map(str::to_owned), - name: v.get("name").and_then(Value::as_str)?.to_owned(), - role: v.get("role").and_then(Value::as_str).map(str::to_owned), - may_delete, - }) - }) - .collect()) -} - -fn depth(local: i64, by: &HashMap>) -> usize { - let mut d = 0; - let mut cur = by.get(&local).copied().flatten(); - let mut seen = HashSet::new(); - while let Some(p) = cur { - if !seen.insert(p) { - break; - } - d += 1; - cur = by.get(&p).copied().flatten(); - } - d -} - -fn already_exists_id(err: &Value) -> Option { - if err.get("type").and_then(Value::as_str) == Some("alreadyExists") { - err.get("existingId") - .and_then(Value::as_str) - .map(str::to_owned) - } else { - None - } -} - -fn find_name_collision( - targets: &[TargetNode], - parent_target: Option<&str>, - name: &str, -) -> Option { - targets - .iter() - .find(|t| t.parent.as_deref() == parent_target && fold_name(&t.name) == fold_name(name)) - .map(|t| t.id.clone()) -} - -fn record_merge( - ty: ObjectType, - local: i64, - target_id: String, - reason: &str, - maps: &mut Maps, - tmatched: &mut HashSet, - logger: &Logger, -) { - logger.warn(&format!( - "{} local {local} merged into existing target {target_id}: {reason}", - ty.jmap_name() - )); - maps.insert(ty, local, JmapId(target_id.clone())); - tmatched.insert(target_id); -} - -pub fn reconcile( - ctx: &Context, - net: &Net, - ty: ObjectType, - maps: &mut Maps, - counts: &mut TypeCounts, - logger: &Logger, -) -> Result { - let locals = load_local(ctx, ty)?; - let targets = load_target(net, ty)?; - - let mut matched: HashMap = HashMap::new(); - let mut tmatched: HashSet = HashSet::new(); - - if ty == ObjectType::Mailbox { - for t in &targets { - if let Some(r) = &t.role - && let Some(l) = locals.iter().find(|l| { - l.role.as_deref() == Some(r.as_str()) && !matched.contains_key(&l.local) - }) - { - matched.insert(l.local, t.id.clone()); - tmatched.insert(t.id.clone()); - } - } - } - - let mut stack: Vec<(Option, Option)> = vec![(None, None)]; - for (l, tid) in matched.clone() { - stack.push((Some(l), Some(tid))); - } - while let Some((lp, tp)) = stack.pop() { - let lchildren: Vec<&LocalNode> = locals - .iter() - .filter(|n| n.parent == lp && !matched.contains_key(&n.local)) - .collect(); - for ln in lchildren { - if let Some(tn) = targets.iter().find(|t| { - t.parent.as_deref() == tp.as_deref() - && !tmatched.contains(&t.id) - && fold_name(&t.name) == fold_name(&ln.name) - }) { - matched.insert(ln.local, tn.id.clone()); - tmatched.insert(tn.id.clone()); - stack.push((Some(ln.local), Some(tn.id.clone()))); - } - } - } - - for (l, tid) in &matched { - maps.insert(ty, *l, JmapId(tid.clone())); - } - counts.skipped += matched.len() as u64; - - let by: HashMap> = locals.iter().map(|n| (n.local, n.parent)).collect(); - let mut to_create: Vec<&LocalNode> = locals - .iter() - .filter(|n| !matched.contains_key(&n.local)) - .collect(); - to_create.sort_by_key(|n| depth(n.local, &by)); - let max_depth = to_create - .iter() - .map(|n| depth(n.local, &by)) - .max() - .unwrap_or(0); - - let mut uploader = Uploader::new(net, &ctx.conn); - let mut taken_roles: HashSet = targets.iter().filter_map(|t| t.role.clone()).collect(); - let interleave = ty == ObjectType::FileNode; - for d in 0..=max_depth { - let level: Vec<&LocalNode> = to_create - .iter() - .copied() - .filter(|n| depth(n.local, &by) == d) - .collect(); - if interleave { - for n in &level { - let parent_target = match n.parent { - None => None, - Some(p) => match maps.target(ty, p) { - Some(t) => Some(t.0), - None => { - logger.warn(&format!( - "{} local {} skipped: parent {} not created", - ty.jmap_name(), - n.local, - p - )); - counts.failed += 1; - continue; - } - }, - }; - if let Some(tid) = find_name_collision(&targets, parent_target.as_deref(), &n.name) - { - record_merge( - ty, - n.local, - tid, - "name already present on target", - maps, - &mut tmatched, - logger, - ); - counts.skipped += 1; - continue; - } - let cid = format!("c{}", n.local); - let _ = uploader.take_touched(); - let obj = match build_create(ctx, ty, n.local, maps, &mut uploader) { - Ok(o) => o, - Err(e) => { - logger.warn(&format!( - "{} local {} skipped: {e}", - ty.jmap_name(), - n.local - )); - counts.failed += 1; - continue; - } - }; - let touched = uploader.take_touched(); - let outcome = - create_batch(net, ty, vec![(cid.clone(), obj)]).map_err(Error::from)?; - let outcome = match retry_if_blob_missing( - net, - ty, - &cid, - &mut uploader, - touched, - outcome, - |up| build_create(ctx, ty, n.local, maps, up), - ) { - Ok(o) => o, - Err(e) => { - logger.warn(&format!( - "{} local {} skipped: {e}", - ty.jmap_name(), - n.local - )); - counts.failed += 1; - continue; - } - }; - for (cid, v) in &outcome.created { - if let Some(local) = cid.strip_prefix('c').and_then(|s| s.parse::().ok()) - && let Some(id) = jid(v) - { - maps.insert(ty, local, JmapId(id)); - counts.created += 1; - } - } - for (cid, err) in &outcome.not_created { - let local = cid.strip_prefix('c').and_then(|s| s.parse::().ok()); - if let (Some(local), Some(existing)) = (local, already_exists_id(err)) { - record_merge( - ty, - local, - existing, - "alreadyExists: reusing existing target", - maps, - &mut tmatched, - logger, - ); - counts.skipped += 1; - continue; - } - logger.warn(&format!("{} {cid} not created: {err}", ty.jmap_name())); - counts.failed += 1; - } - } - continue; - } - let mut batch: Vec<(String, Value)> = Vec::new(); - for n in &level { - let parent_target = match n.parent { - None => None, - Some(p) => match maps.target(ty, p) { - Some(t) => Some(t.0), - None => { - logger.warn(&format!( - "{} local {} skipped: parent {} not created", - ty.jmap_name(), - n.local, - p - )); - counts.failed += 1; - continue; - } - }, - }; - if let Some(tid) = find_name_collision(&targets, parent_target.as_deref(), &n.name) { - record_merge( - ty, - n.local, - tid, - "name already present on target", - maps, - &mut tmatched, - logger, - ); - counts.skipped += 1; - continue; - } - match build_create(ctx, ty, n.local, maps, &mut uploader) { - Ok(mut obj) => { - if ty == ObjectType::Mailbox - && let Some(r) = n.role.as_deref() - { - if taken_roles.contains(r) { - if let Value::Object(m) = &mut obj { - m.remove("role"); - } - logger.warn(&format!( - "Mailbox local {}: role '{r}' already present on target, creating as a plain folder", - n.local - )); - } else { - taken_roles.insert(r.to_owned()); - } - } - batch.push((format!("c{}", n.local), obj)); - } - Err(e) => { - logger.warn(&format!( - "{} local {} skipped: {e}", - ty.jmap_name(), - n.local - )); - counts.failed += 1; - } - } - } - if batch.is_empty() { - continue; - } - let outcome = create_batch(net, ty, batch).map_err(Error::from)?; - for (cid, v) in &outcome.created { - if let Some(local) = cid.strip_prefix('c').and_then(|s| s.parse::().ok()) - && let Some(id) = jid(v) - { - maps.insert(ty, local, JmapId(id)); - counts.created += 1; - } - } - for (cid, err) in &outcome.not_created { - let local = cid.strip_prefix('c').and_then(|s| s.parse::().ok()); - if let (Some(local), Some(existing)) = (local, already_exists_id(err)) { - record_merge( - ty, - local, - existing, - "alreadyExists: reusing existing target", - maps, - &mut tmatched, - logger, - ); - counts.skipped += 1; - continue; - } - logger.warn(&format!("{} {cid} not created: {err}", ty.jmap_name())); - counts.failed += 1; - } - } - - let objs: Vec = targets - .iter() - .map(|t| crate::sync::prune::TargetObj { - id: t.id.clone(), - matched: tmatched.contains(&t.id), - protected: t.role.is_some(), - may_delete: t.may_delete, - parent: t.parent.clone(), - }) - .collect(); - Ok(Plan { - prune_candidates: crate::sync::prune::candidates(&objs, true), - active_sieve_target: None, - }) -} - -fn build_create( - ctx: &Context, - ty: ObjectType, - local: i64, - maps: &Maps, - uploader: &mut Uploader<'_>, -) -> Result { - if ty == ObjectType::Mailbox { - let wire = ctx - .conn - .query_row( - &format!("{MAILBOX_SELECT} WHERE id = ?1"), - params![local], - |row| Ok(row_to_mailbox(row, maps)), - ) - .map_err(|e| Error::Partial(e.to_string()))? - .map_err(Error::from)?; - return serde_json::to_value(&wire).map_err(|e| Error::Partial(e.to_string())); - } - let fnrow = ctx - .conn - .query_row( - &format!("{FILE_NODE_SELECT} WHERE id = ?1"), - params![local], - |row| Ok(row_to_file_node(row, maps)), - ) - .map_err(|e| Error::Partial(e.to_string()))? - .map_err(Error::from)?; - let mut wire = fnrow.wire; - if let (NodeKind::File(file), Some(blob_local)) = (&mut wire.kind, fnrow.blob_local_id) { - let ct = file - .media_type - .as_deref() - .unwrap_or("application/octet-stream"); - let id = uploader.upload_with(blob_local, ct).map_err(Error::from)?; - file.blob_id = Some(id); - } - let mut obj = serde_json::to_value(&wire).map_err(|e| Error::Partial(e.to_string()))?; - if let Value::Object(m) = &mut obj { - m.remove("created"); - m.remove("modified"); - m.remove("isSubscribed"); - } - Ok(obj) -} diff --git a/src/sync/export/uidtype.rs b/src/sync/export/uidtype.rs deleted file mode 100644 index fec9bf6..0000000 --- a/src/sync/export/uidtype.rs +++ /dev/null @@ -1,174 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashSet; -use std::fmt::Write as _; - -use serde_json::Value; - -use super::common::{create_batch, jid, target_query_get}; -use super::{Maps, Net, Plan, Uploader}; -use crate::error::Error; -use crate::logging::Logger; -use crate::sync::import_jmap::mapping::{ - CALENDAR_EVENT_SELECT, CONTACT_CARD_SELECT, calendar_event_to_wire, contact_card_to_wire, -}; -use crate::sync::prune::{TargetObj, candidates}; -use crate::sync::{Context, TypeCounts}; -use crate::types::ObjectType; - -fn target_uid(v: &Value) -> Option { - v.get("uid").and_then(Value::as_str).map(str::to_owned) -} - -fn describe(ty: ObjectType, local: i64, uid: &str) -> String { - let mut out = String::new(); - let _ = write!(out, "{} local {local}", ty.jmap_name()); - if !uid.is_empty() { - let _ = write!(out, " (uid {uid})"); - } - out -} - -pub fn reconcile( - ctx: &Context, - net: &Net, - ty: ObjectType, - maps: &mut Maps, - counts: &mut TypeCounts, - logger: &Logger, -) -> Result { - let targets = target_query_get(net, ty, None).map_err(Error::from)?; - let mut by_uid: std::collections::HashMap = std::collections::HashMap::new(); - for t in &targets { - if let (Some(uid), Some(id)) = (target_uid(t), jid(t)) { - by_uid.entry(uid).or_insert(id); - } - } - - let select = if ty == ObjectType::ContactCard { - CONTACT_CARD_SELECT - } else { - CALENDAR_EVENT_SELECT - }; - let rows: Vec<(i64, String)> = { - let mut stmt = ctx - .conn - .prepare(select) - .map_err(|e| Error::Partial(e.to_string()))?; - stmt.query_map([], |r| { - if ty == ObjectType::ContactCard { - Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)) - } else { - let data: String = r.get(4)?; - let uid = serde_json::from_str::(&data) - .ok() - .and_then(|v| v.get("uid").and_then(Value::as_str).map(str::to_owned)) - .unwrap_or_default(); - Ok((r.get::<_, i64>(0)?, uid)) - } - }) - .and_then(|m| m.collect::, _>>()) - .map_err(|e| Error::Partial(e.to_string()))? - }; - - let mut matched_uids: HashSet = HashSet::new(); - let blobs = Uploader::new(net, &ctx.conn); - for (local, uid) in &rows { - if let Some(tid) = by_uid.get(uid) { - maps.insert(ty, *local, crate::jmap::wire::JmapId(tid.clone())); - matched_uids.insert(uid.clone()); - counts.skipped += 1; - continue; - } - let cid = format!("c{local}"); - let wire = match build_wire(ctx, ty, *local, maps, &blobs) { - Ok(w) => w, - Err(e) if e.aborts_run() => return Err(e), - Err(e) => { - logger.warn(&format!("{} skipped: {e}", describe(ty, *local, uid))); - counts.failed += 1; - continue; - } - }; - let outcome = match create_batch(net, ty, vec![(cid.clone(), wire)]) { - Ok(o) => o, - Err(e) => { - let mapped = Error::from(e); - if mapped.aborts_run() { - return Err(mapped); - } - logger.warn(&format!( - "{} not created: {mapped}", - describe(ty, *local, uid) - )); - counts.failed += 1; - continue; - } - }; - for (cid, v) in &outcome.created { - if let Some(parsed) = cid.strip_prefix('c').and_then(|s| s.parse::().ok()) - && let Some(id) = jid(v) - { - maps.insert(ty, parsed, crate::jmap::wire::JmapId(id)); - counts.created += 1; - } - } - for (cid, err) in &outcome.not_created { - logger.warn(&format!("{} {cid} not created: {err}", ty.jmap_name())); - counts.failed += 1; - } - } - - let objs: Vec = targets - .iter() - .filter_map(|t| { - let id = jid(t)?; - let uid = target_uid(t); - Some(TargetObj { - id: id.clone(), - matched: uid.map(|u| matched_uids.contains(&u)).unwrap_or(false), - protected: false, - may_delete: true, - parent: None, - }) - }) - .collect(); - Ok(Plan { - prune_candidates: candidates(&objs, false), - active_sieve_target: None, - }) -} - -fn build_wire( - ctx: &Context, - ty: ObjectType, - local: i64, - maps: &Maps, - blobs: &Uploader<'_>, -) -> Result { - if ty == ObjectType::ContactCard { - let (uid, abids, data): (String, String, String) = ctx - .conn - .query_row( - &format!("{CONTACT_CARD_SELECT} WHERE id = ?1"), - rusqlite::params![local], - |r| Ok((r.get(1)?, r.get(2)?, r.get(3)?)), - ) - .map_err(|e| Error::Partial(e.to_string()))?; - contact_card_to_wire(&uid, &abids, &data, maps, blobs).map_err(Error::from) - } else { - let (cal, dr, ud, data): (String, i64, i64, String) = ctx - .conn - .query_row( - &format!("{CALENDAR_EVENT_SELECT} AND id = ?1"), - rusqlite::params![local], - |r| Ok((r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)), - ) - .map_err(|e| Error::Partial(e.to_string()))?; - calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, maps, blobs).map_err(Error::from) - } -} diff --git a/src/sync/export/vacation.rs b/src/sync/export/vacation.rs deleted file mode 100644 index 1c9035c..0000000 --- a/src/sync/export/vacation.rs +++ /dev/null @@ -1,50 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use serde_json::json; - -use super::Net; -use crate::db; -use crate::error::Error; -use crate::jmap::request::{SetRequest, set_call}; -use crate::jmap::wire::vacation_response::{SINGLETON_ID, TYPE_NAME}; -use crate::logging::Logger; -use crate::sync::{Context, TypeCounts}; - -pub fn reconcile( - ctx: &Context, - net: &Net, - counts: &mut TypeCounts, - logger: &Logger, -) -> Result { - let Some(vacation) = - db::vacation_response::load(&ctx.conn).map_err(|e| Error::Partial(e.to_string()))? - else { - return Ok(false); - }; - if net.dry_run { - counts.updated += 1; - return Ok(true); - } - let outcome = set_call( - &net.client, - &net.api, - &net.account, - TYPE_NAME, - SetRequest { - update: Some(json!({ SINGLETON_ID: vacation })), - ..Default::default() - }, - &net.limits, - ) - .map_err(Error::from)?; - counts.updated += outcome.updated.len() as u64; - for (id, err) in &outcome.not_updated { - logger.warn(&format!("{TYPE_NAME} {id} not updated: {err}")); - counts.failed += 1; - } - Ok(true) -} diff --git a/src/sync/import_dav.rs b/src/sync/import_dav.rs deleted file mode 100644 index 81779d1..0000000 --- a/src/sync/import_dav.rs +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -pub mod calcard; -pub mod collections; -pub mod coordinator; -pub mod items; -pub mod replace; -pub mod tree; - -pub use coordinator::{DavAuth, DavImportConfig, DavKindArg, SourceChange, run, run_reporting}; - -use crate::error::Error; -use crate::jmap::error::JmapError; - -pub(crate) fn per_collection_failure(err: JmapError) -> Error { - match err { - JmapError::Sqlite(e) => Error::Db(crate::db::init::OpenError::Sqlite(e)), - scoped => Error::Partial(scoped.to_string()), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn a_forbidden_collection_is_a_per_unit_failure() { - for e in [ - JmapError::Auth("server returned 403: forbidden".to_owned()), - JmapError::HttpStatus { - status: 405, - body: "method not allowed".to_owned(), - }, - JmapError::RetriesExhausted("PROPFIND kept returning 503".to_owned()), - JmapError::Transport("io: connection reset".to_owned()), - JmapError::Malformed("multistatus parse".to_owned()), - ] { - let mapped = per_collection_failure(e); - assert!( - !mapped.aborts_run(), - "{mapped} is scoped to one collection and must not abort the run" - ); - assert_eq!(mapped.exit_code(), 5); - } - } - - #[test] - fn an_archive_failure_still_aborts() { - let mapped = - per_collection_failure(JmapError::Sqlite(rusqlite::Error::QueryReturnedNoRows)); - assert!(mapped.aborts_run()); - assert_eq!(mapped.exit_code(), 7); - } -} diff --git a/src/sync/import_dav/calcard.rs b/src/sync/import_dav/calcard.rs deleted file mode 100644 index a7ccb1f..0000000 --- a/src/sync/import_dav/calcard.rs +++ /dev/null @@ -1,396 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::borrow::Cow; -use std::fmt::Write; - -use calcard::Entry; -use calcard::icalendar::ICalendar; -use calcard::vcard::VCard; -use serde_json::{Map, Value}; - -const MAX_INVALID_LINE_CHARS: usize = 80; - -#[derive(Debug, thiserror::Error)] -pub enum CalcardError { - #[error("vcard parse failed: {0}")] - VCardParse(String), - #[error("icalendar parse failed: {0}")] - ICalParse(String), - #[error("jscontact serialisation: {0}")] - Json(#[from] serde_json::Error), - #[error("no parseable entries in iCalendar")] - NoEntries, - #[error("calcard output is not a JSON object")] - NotAnObject, -} - -#[derive(Debug)] -pub struct JsContact { - pub uid: String, - pub data: Value, -} - -pub fn vcard_to_jscontact(text: &str, item_href: &str) -> Result { - let vcard = VCard::parse(normalize_line_breaks(text)) - .map_err(|e| CalcardError::VCardParse(describe_parse_error(e)))?; - let js = vcard.into_jscontact::(); - let value: Value = serde_json::from_str(&js.to_string_pretty())?; - let uid = value - .get("uid") - .and_then(Value::as_str) - .map(str::trim) - .filter(|s| !s.is_empty()) - .map(str::to_owned) - .unwrap_or_else(|| synthesise_uid(item_href)); - let mut obj = match value { - Value::Object(map) => map, - _ => return Err(CalcardError::NotAnObject), - }; - obj.remove("uid"); - Ok(JsContact { - uid, - data: Value::Object(obj), - }) -} - -pub fn synthesise_uid(item_href: &str) -> String { - let hash = blake3::hash(item_href.as_bytes()); - format!("inbuxa-migrate-syn-{}", hash.to_hex()) -} - -#[derive(Debug)] -pub struct JsCalendarEntry { - pub data: Value, - pub data_type: EntryType, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum EntryType { - Event, - Task, - Note, - Group, -} - -impl EntryType { - pub fn as_column(self) -> &'static str { - match self { - EntryType::Event => "Event", - EntryType::Task => "Task", - EntryType::Note => "Note", - EntryType::Group => "Group", - } - } -} - -pub fn ical_to_jscalendar_entries(text: &str) -> Result, CalcardError> { - let ical = ICalendar::parse(normalize_line_breaks(text)) - .map_err(|e| CalcardError::ICalParse(describe_parse_error(e)))?; - let js = ical.into_jscalendar::(); - let value: Value = serde_json::from_str(&js.to_string_pretty())?; - let mut out = Vec::new(); - collect_entries(value, &mut out); - if out.is_empty() { - return Err(CalcardError::NoEntries); - } - Ok(out) -} - -fn normalize_line_breaks(text: &str) -> Cow<'_, str> { - let mut after_cr = text.split('\r').skip(1).peekable(); - let mut has_bare_cr = false; - while let Some(rest) = after_cr.next() { - if is_bare_cr(rest, after_cr.peek().is_some()) { - has_bare_cr = true; - break; - } - } - if !has_bare_cr { - return Cow::Borrowed(text); - } - let mut parts = text.split('\r').peekable(); - let mut out = String::with_capacity(text.len()); - out.push_str(parts.next().unwrap_or_default()); - while let Some(rest) = parts.next() { - if is_bare_cr(rest, parts.peek().is_some()) { - out.push('\n'); - } - out.push_str(rest); - } - Cow::Owned(out) -} - -fn is_bare_cr(rest: &str, more: bool) -> bool { - let followed_by_cr = rest.is_empty() && more; - !followed_by_cr && !rest.starts_with('\n') -} - -fn describe_parse_error(entry: Entry) -> String { - let mut out = String::new(); - match entry { - Entry::InvalidLine(line) => { - out.push_str("invalid line "); - let mut chars = line.chars(); - let head: String = chars.by_ref().take(MAX_INVALID_LINE_CHARS).collect(); - let _ = write!(out, "{head:?}"); - if chars.next().is_some() { - out.push_str(" (truncated)"); - } - } - Entry::UnexpectedComponentEnd { expected, found } => { - let _ = write!( - out, - "found END:{} where END:{} was expected", - found.as_str(), - expected.as_str() - ); - } - Entry::UnterminatedComponent(name) => { - let _ = write!(out, "{name} is never closed"); - } - Entry::TooManyComponents => out.push_str("too many components"), - Entry::Eof => out.push_str("no BEGIN line found"), - Entry::VCard(_) => out.push_str("found a vCard where an iCalendar was expected"), - Entry::ICalendar(_) => out.push_str("found an iCalendar where a vCard was expected"), - _ => out.push_str("unrecognised content"), - } - out -} - -fn collect_entries(value: Value, out: &mut Vec) { - match value { - Value::Object(map) => { - let at_type = map.get("@type").and_then(Value::as_str).map(str::to_owned); - match at_type.as_deref() { - Some("Event") if has_nonempty_uid(&map) => out.push(JsCalendarEntry { - data: Value::Object(map), - data_type: EntryType::Event, - }), - Some("Task") if has_nonempty_uid(&map) => out.push(JsCalendarEntry { - data: Value::Object(map), - data_type: EntryType::Task, - }), - Some("Note") if has_nonempty_uid(&map) => out.push(JsCalendarEntry { - data: Value::Object(map), - data_type: EntryType::Note, - }), - Some("Group") => { - if let Some(entries) = map.get("entries").and_then(Value::as_array) { - for entry in entries { - collect_entries(entry.clone(), out); - } - } - } - _ => {} - } - } - Value::Array(items) => { - for item in items { - collect_entries(item, out); - } - } - _ => {} - } -} - -fn has_nonempty_uid(map: &Map) -> bool { - map.get("uid") - .and_then(Value::as_str) - .map(|u| !u.trim().is_empty()) - .unwrap_or(false) -} - -pub fn strip_extracted_fields_from_event(value: &mut Value) -> (bool, bool, Option) { - let Value::Object(map) = value else { - return (false, false, None); - }; - let is_draft = map - .remove("isDraft") - .and_then(|v| v.as_bool()) - .unwrap_or(false); - let use_default_alerts = map - .remove("useDefaultAlerts") - .and_then(|v| v.as_bool()) - .unwrap_or(false); - map.remove("calendarIds"); - map.remove("method"); - map.remove("utcStart"); - map.remove("utcEnd"); - map.remove("isOrigin"); - map.remove("baseEventId"); - let uid = map.get("uid").and_then(Value::as_str).map(str::to_owned); - (is_draft, use_default_alerts, uid) -} - -#[cfg(test)] -mod tests { - use super::*; - - const SAMPLE_VCARD: &str = "BEGIN:VCARD\r\n\ -VERSION:4.0\r\n\ -UID:abc-123\r\n\ -FN:Alice Smith\r\n\ -EMAIL:alice@example.com\r\n\ -END:VCARD\r\n"; - - const SAMPLE_ICAL: &str = "BEGIN:VCALENDAR\r\n\ -VERSION:2.0\r\n\ -PRODID:-//Test//EN\r\n\ -BEGIN:VEVENT\r\n\ -UID:event-1@example.com\r\n\ -DTSTAMP:20260101T000000Z\r\n\ -DTSTART:20260101T090000Z\r\n\ -DTEND:20260101T100000Z\r\n\ -SUMMARY:Test Event\r\n\ -END:VEVENT\r\n\ -END:VCALENDAR\r\n"; - - #[test] - fn vcard_extracts_uid_into_separate_field() { - let jc = vcard_to_jscontact(SAMPLE_VCARD, "/dav/card/u/d/a.vcf").expect("parse"); - assert_eq!(jc.uid, "abc-123"); - let obj = jc.data.as_object().unwrap(); - assert!(!obj.contains_key("uid"), "uid stripped from data"); - } - - #[test] - fn ical_extracts_at_least_one_event() { - let events = ical_to_jscalendar_entries(SAMPLE_ICAL).expect("parse"); - assert!(!events.is_empty()); - assert_eq!(events[0].data_type, EntryType::Event); - let uid = events[0] - .data - .as_object() - .and_then(|m| m.get("uid")) - .and_then(Value::as_str) - .unwrap(); - assert!(uid.starts_with("event-1")); - } - - #[test] - fn vcard_without_uid_gets_synthetic_uid_from_href() { - let no_uid = "BEGIN:VCARD\r\nVERSION:4.0\r\nFN:Anon\r\nEND:VCARD\r\n"; - let href = "/dav/card/u/d/no-uid.vcf"; - let jc = vcard_to_jscontact(no_uid, href).expect("synthetic uid"); - assert_eq!(jc.uid, synthesise_uid(href)); - assert!(jc.uid.starts_with("inbuxa-migrate-syn-")); - } - - #[test] - fn synthetic_uid_is_stable_per_href() { - assert_eq!(synthesise_uid("/a"), synthesise_uid("/a")); - assert_ne!(synthesise_uid("/a"), synthesise_uid("/b")); - } - - #[test] - fn ical_with_empty_calendar_fails_no_entries() { - let empty = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nEND:VCALENDAR\r\n"; - let err = ical_to_jscalendar_entries(empty).unwrap_err(); - assert!(matches!(err, CalcardError::NoEntries)); - } - - #[test] - fn vcard_garbage_fails_parse() { - let err = vcard_to_jscontact("not a vcard", "/x").unwrap_err(); - assert!(matches!(err, CalcardError::VCardParse(_))); - } - - // Kerio Connect sends calendar-data with bare CR line breaks (issue #48). - #[test] - fn ical_with_bare_cr_line_breaks_parses() { - let ical = SAMPLE_ICAL.replace("\r\n", "\r"); - let entries = ical_to_jscalendar_entries(&ical).expect("bare CR parses"); - assert_eq!(entries.len(), 1); - assert_eq!(entries[0].data["title"], "Test Event"); - } - - // A folded line split by a bare CR still unfolds. - #[test] - fn ical_with_bare_cr_unfolds_continuation_lines() { - let ical = "BEGIN:VCALENDAR\rVERSION:2.0\rBEGIN:VEVENT\rUID:k1@example\r\ - DTSTAMP:20260101T000000Z\rDTSTART:20260101T090000Z\r\ - SUMMARY:Quarterly\r review\rEND:VEVENT\rEND:VCALENDAR\r"; - let entries = ical_to_jscalendar_entries(ical).expect("parse"); - assert_eq!(entries[0].data["title"], "Quarterly review"); - } - - #[test] - fn vcard_with_bare_cr_line_breaks_parses() { - let vcard = SAMPLE_VCARD.replace("\r\n", "\r"); - let jc = vcard_to_jscontact(&vcard, "/x").expect("bare CR parses"); - assert_eq!(jc.uid, "abc-123"); - } - - #[test] - fn line_break_normalization_borrows_unless_bare_cr() { - for text in ["A\nB\n", "A\r\nB\r\n", "A\r\r\nB\r\r\n", "no breaks"] { - assert!( - matches!(normalize_line_breaks(text), Cow::Borrowed(_)), - "{text:?}" - ); - } - assert_eq!(normalize_line_breaks("A\rB\r\nC\r"), "A\nB\nC\n"); - } - - #[test] - fn ical_parse_error_names_the_offending_line() { - let err = ical_to_jscalendar_entries("Content-Type: text/calendar\r\nBEGIN:VCALENDAR\r\n") - .unwrap_err(); - let msg = err.to_string(); - assert!(msg.contains("invalid line"), "{msg}"); - assert!(msg.contains("Content-Type"), "{msg}"); - } - - #[test] - fn ical_vtodo_classified_as_task() { - let vtodo = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\n\ - BEGIN:VTODO\r\nUID:t1@example\r\nDTSTAMP:20260101T000000Z\r\n\ - SUMMARY:Buy milk\r\nEND:VTODO\r\nEND:VCALENDAR\r\n"; - let entries = ical_to_jscalendar_entries(vtodo).expect("parse vtodo"); - assert_eq!(entries[0].data_type, EntryType::Task); - } - - #[test] - fn ical_recurrence_overrides_collapse_to_single_event() { - let ical = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\n\ - BEGIN:VEVENT\r\nUID:r1@example\r\nDTSTAMP:20260101T000000Z\r\n\ - DTSTART:20260101T090000Z\r\nDTEND:20260101T100000Z\r\n\ - RRULE:FREQ=DAILY;COUNT=3\r\nSUMMARY:Standup\r\nEND:VEVENT\r\n\ - BEGIN:VEVENT\r\nUID:r1@example\r\nDTSTAMP:20260101T000000Z\r\n\ - RECURRENCE-ID:20260102T090000Z\r\nDTSTART:20260102T093000Z\r\nDTEND:20260102T103000Z\r\n\ - SUMMARY:Standup (moved)\r\nEND:VEVENT\r\n\ - END:VCALENDAR\r\n"; - let entries = ical_to_jscalendar_entries(ical).expect("parse"); - assert_eq!(entries.len(), 1, "calcard merges overrides into master"); - let master = entries[0].data.as_object().unwrap(); - assert!(master.contains_key("recurrenceOverrides")); - } - - #[test] - fn strip_extracted_fields_pulls_isdraft_use_default_alerts() { - let mut v = serde_json::json!({ - "@type": "Event", - "uid": "x", - "title": "T", - "isDraft": true, - "useDefaultAlerts": true, - "calendarIds": {"a": true}, - "utcStart": "2026-01-01T09:00:00Z", - "method": "PUBLISH" - }); - let (d, u, uid) = strip_extracted_fields_from_event(&mut v); - assert!(d); - assert!(u); - assert_eq!(uid.as_deref(), Some("x")); - let m = v.as_object().unwrap(); - assert!(!m.contains_key("calendarIds")); - assert!(!m.contains_key("isDraft")); - assert!(!m.contains_key("useDefaultAlerts")); - assert!(!m.contains_key("method")); - assert!(!m.contains_key("utcStart")); - } -} diff --git a/src/sync/import_dav/collections.rs b/src/sync/import_dav/collections.rs deleted file mode 100644 index 9507468..0000000 --- a/src/sync/import_dav/collections.rs +++ /dev/null @@ -1,374 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashMap; - -use rusqlite::{Connection, params}; - -use crate::dav::discover::DiscoveredCollection; -use crate::dav::href::{Href, last_path_component}; -use crate::db::dav_ids; -use crate::error::Error; -use crate::logging::{LEVEL_PROGRESS, Logger}; -use crate::sync::TypeCounts; - -pub fn reconcile_calendars( - conn: &mut Connection, - source_id: i64, - discovered: &[DiscoveredCollection], - counts: &mut TypeCounts, - logger: Logger, -) -> Result, Error> { - reconcile_calendar_like( - conn, - source_id, - discovered, - &RowOps { - type_name: dav_ids::CALENDAR, - upsert: upsert_calendar_row, - delete: delete_calendar_row, - }, - counts, - logger, - ) -} - -pub fn reconcile_address_books( - conn: &mut Connection, - source_id: i64, - discovered: &[DiscoveredCollection], - counts: &mut TypeCounts, - logger: Logger, -) -> Result, Error> { - reconcile_calendar_like( - conn, - source_id, - discovered, - &RowOps { - type_name: dav_ids::ADDRESS_BOOK, - upsert: upsert_address_book_row, - delete: delete_address_book_row, - }, - counts, - logger, - ) -} - -type RowUpsert = - fn(&Connection, &DiscoveredCollection, Option) -> Result; -type RowDelete = fn(&Connection, i64) -> Result<(), rusqlite::Error>; - -struct RowOps { - type_name: &'static str, - upsert: RowUpsert, - delete: RowDelete, -} - -fn reconcile_calendar_like( - conn: &mut Connection, - source_id: i64, - discovered: &[DiscoveredCollection], - ops: &RowOps, - counts: &mut TypeCounts, - logger: Logger, -) -> Result, Error> { - let type_name = ops.type_name; - let upsert_row = ops.upsert; - let delete_row = ops.delete; - let existing: HashMap = dav_ids::collections_of_type(conn, source_id, type_name) - .map_err(|e| Error::Partial(e.to_string()))?; - - let mut upserted: Vec<(String, i64)> = Vec::new(); - let server_hrefs: std::collections::HashSet = discovered - .iter() - .map(|c| c.href.as_str().to_owned()) - .collect(); - - for coll in discovered { - let collection_href = coll.href.as_str().to_owned(); - let existing_local = existing.get(&collection_href).copied(); - let tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - match upsert_row(&tx, coll, existing_local) { - Ok(local_id) => { - if existing_local.is_none() { - dav_ids::insert( - &tx, - source_id, - type_name, - &collection_href, - &collection_href, - "", - local_id, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - counts.created += 1; - } else { - counts.fetched += 1; - } - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - upserted.push((collection_href.clone(), local_id)); - if logger.enabled(LEVEL_PROGRESS) { - eprintln!("collection upserted: {collection_href} -> {local_id}"); - } - } - Err(e) => { - let _ = tx.rollback(); - logger.warn(&format!( - "collection upsert {collection_href:?} failed: {e}" - )); - counts.failed += 1; - } - } - } - - for (collection_href, local_id) in &existing { - if server_hrefs.contains(collection_href) { - continue; - } - let tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - let item_type = item_type_for(type_name); - if let Some(item_type) = item_type { - let _ = dav_ids::delete_collection(&tx, source_id, item_type, collection_href); - } - let _ = dav_ids::delete_collection(&tx, source_id, type_name, collection_href); - match delete_row(&tx, *local_id) { - Ok(()) => { - counts.deleted += 1; - } - Err(e) => { - logger.warn(&format!( - "vanished collection {collection_href:?} delete failed: {e}" - )); - counts.failed += 1; - } - } - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - } - - Ok(upserted) -} - -fn item_type_for(collection_type: &str) -> Option<&'static str> { - match collection_type { - dav_ids::CALENDAR => Some(dav_ids::CALENDAR_EVENT), - dav_ids::ADDRESS_BOOK => Some(dav_ids::CONTACT_CARD), - _ => None, - } -} - -fn upsert_calendar_row( - conn: &Connection, - coll: &DiscoveredCollection, - existing_local: Option, -) -> Result { - let name = display_or_fallback(&coll.props.displayname, &coll.href); - let description = coll.props.calendar_description.as_deref(); - let color = coll.props.calendar_color.as_deref(); - let sort_order = coll.props.calendar_order.unwrap_or(0).max(0); - let time_zone = parse_tzid_from_vtimezone(coll.props.calendar_timezone.as_deref()); - - if let Some(local) = existing_local { - conn.execute( - "UPDATE calendars SET name = ?1, description = ?2, color = ?3, sort_order = ?4, - time_zone = ?5 - WHERE id = ?6", - params![name, description, color, sort_order, time_zone, local], - )?; - Ok(local) - } else { - conn.execute( - "INSERT INTO calendars (name, description, color, sort_order, is_subscribed, - is_visible, is_default, include_in_availability, - default_alerts_with_time, default_alerts_without_time, time_zone) - VALUES (?1, ?2, ?3, ?4, 1, 1, 0, 'all', NULL, NULL, ?5)", - params![name, description, color, sort_order, time_zone], - )?; - Ok(conn.last_insert_rowid()) - } -} - -fn delete_calendar_row(conn: &Connection, local_id: i64) -> Result<(), rusqlite::Error> { - conn.execute( - "DELETE FROM calendar_events - WHERE EXISTS ( - SELECT 1 FROM json_each(calendar_ids) WHERE value = ?1 - )", - params![local_id], - )?; - conn.execute("DELETE FROM calendars WHERE id = ?1", params![local_id])?; - Ok(()) -} - -fn upsert_address_book_row( - conn: &Connection, - coll: &DiscoveredCollection, - existing_local: Option, -) -> Result { - let name = display_or_fallback(&coll.props.displayname, &coll.href); - let description = coll.props.addressbook_description.as_deref(); - - if let Some(local) = existing_local { - conn.execute( - "UPDATE address_books SET name = ?1, description = ?2 WHERE id = ?3", - params![name, description, local], - )?; - Ok(local) - } else { - conn.execute( - "INSERT INTO address_books (name, description, sort_order, is_default, is_subscribed) - VALUES (?1, ?2, 0, 0, 1)", - params![name, description], - )?; - Ok(conn.last_insert_rowid()) - } -} - -fn delete_address_book_row(conn: &Connection, local_id: i64) -> Result<(), rusqlite::Error> { - conn.execute( - "DELETE FROM contact_cards - WHERE EXISTS ( - SELECT 1 FROM json_each(address_book_ids) WHERE value = ?1 - )", - params![local_id], - )?; - conn.execute("DELETE FROM address_books WHERE id = ?1", params![local_id])?; - Ok(()) -} - -fn display_or_fallback(name: &Option, href: &Href) -> String { - if let Some(n) = name - && !n.trim().is_empty() - { - return n.clone(); - } - let comp = last_path_component(href); - if comp.is_empty() { - "Untitled".to_owned() - } else { - comp - } -} - -fn parse_tzid_from_vtimezone(vt: Option<&str>) -> Option { - let body = vt?; - for line in body.lines() { - let line = line.trim(); - if let Some(rest) = line.strip_prefix("TZID:") { - let v = rest.trim(); - if !v.is_empty() { - return Some(v.to_owned()); - } - } - } - None -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_tzid_from_vtimezone_extracts_first() { - let v = "BEGIN:VTIMEZONE\r\nTZID:Europe/Berlin\r\nEND:VTIMEZONE\r\n"; - assert_eq!( - parse_tzid_from_vtimezone(Some(v)).as_deref(), - Some("Europe/Berlin") - ); - } - - #[test] - fn parse_tzid_returns_none_when_absent() { - assert!(parse_tzid_from_vtimezone(Some("nothing here")).is_none()); - assert!(parse_tzid_from_vtimezone(None).is_none()); - } - - #[test] - fn display_or_fallback_uses_href_when_displayname_empty() { - let href = Href::from_normalised("/dav/cal/u/work/".to_owned()); - let name: Option = None; - assert_eq!(display_or_fallback(&name, &href), "work"); - } - - #[test] - fn display_or_fallback_uses_displayname_when_present() { - let href = Href::from_normalised("/dav/cal/u/default/".to_owned()); - let name = Some("Default".to_owned()); - assert_eq!(display_or_fallback(&name, &href), "Default"); - } - - #[test] - fn delete_calendar_row_cascades_events_and_sync_id_rows() { - use crate::db::init; - use crate::db::sources::{SourceKey, upsert_source}; - let conn = rusqlite::Connection::open_in_memory().unwrap(); - init::apply_schema(&conn).unwrap(); - let sid = upsert_source( - &conn, - &SourceKey { - kind: "caldav".to_owned(), - session_url: "https://x".to_owned(), - account_id: "u".to_owned(), - }, - Some("u"), - "u", - ) - .unwrap(); - conn.execute( - "INSERT INTO calendars (name, sort_order, is_subscribed, is_visible, is_default, - include_in_availability) - VALUES ('Work', 0, 1, 1, 0, 'all')", - [], - ) - .unwrap(); - let cal_local = conn.last_insert_rowid(); - conn.execute( - "INSERT INTO calendar_events (calendar_ids, data, data_type) - VALUES (?1, '{}', 'Event')", - rusqlite::params![format!("[{cal_local}]")], - ) - .unwrap(); - let evt_local = conn.last_insert_rowid(); - dav_ids::insert( - &conn, - sid, - dav_ids::CALENDAR, - "/cal/work/", - "/cal/work/", - "", - cal_local, - ) - .unwrap(); - dav_ids::insert( - &conn, - sid, - dav_ids::CALENDAR_EVENT, - "/cal/work/", - "/cal/work/e.ics", - "\"v\"", - evt_local, - ) - .unwrap(); - let _ = dav_ids::delete_collection(&conn, sid, dav_ids::CALENDAR_EVENT, "/cal/work/"); - let _ = dav_ids::delete_collection(&conn, sid, dav_ids::CALENDAR, "/cal/work/"); - delete_calendar_row(&conn, cal_local).unwrap(); - let cal_count: i64 = conn - .query_row("SELECT count(*) FROM calendars", [], |r| r.get(0)) - .unwrap(); - let evt_count: i64 = conn - .query_row("SELECT count(*) FROM calendar_events", [], |r| r.get(0)) - .unwrap(); - let dav_count: i64 = conn - .query_row("SELECT count(*) FROM sync_id_dav", [], |r| r.get(0)) - .unwrap(); - assert_eq!(cal_count, 0, "calendar row gone"); - assert_eq!(evt_count, 0, "event rows cascaded"); - assert_eq!(dav_count, 0, "sync_id_dav rows for both types cleared"); - } -} diff --git a/src/sync/import_dav/coordinator.rs b/src/sync/import_dav/coordinator.rs deleted file mode 100644 index c4336db..0000000 --- a/src/sync/import_dav/coordinator.rs +++ /dev/null @@ -1,632 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use rusqlite::Connection; - -use crate::dav::client::DavClient; -use crate::dav::discover::{DavKind, Discovery, DiscoveryError, discover}; -use crate::db; -use crate::db::sources::SourceKey; -use crate::error::Error; -use crate::jmap::http::{Auth, RetryPolicy}; -use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; -use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts}; - -use super::collections; -use super::items; -use super::replace::{self, Removed}; -use super::tree; - -#[derive(Debug, Clone, Copy)] -pub enum DavKindArg { - Caldav, - Carddav, - Webdav, -} - -impl DavKindArg { - pub fn kind(self) -> DavKind { - match self { - DavKindArg::Caldav => DavKind::Caldav, - DavKindArg::Carddav => DavKind::Carddav, - DavKindArg::Webdav => DavKind::Webdav, - } - } - - pub fn source_kind(self) -> &'static str { - match self { - DavKindArg::Caldav => "caldav", - DavKindArg::Carddav => "carddav", - DavKindArg::Webdav => "webdav", - } - } -} - -#[derive(Debug, Clone)] -pub enum DavAuth { - Basic { user: String, password: String }, - Bearer { token: String }, -} - -impl DavAuth { - pub fn to_jmap_auth(&self) -> Auth { - match self { - DavAuth::Basic { user, password } => Auth::Basic { - user: user.clone(), - password: password.clone(), - }, - DavAuth::Bearer { token } => Auth::Bearer { - token: token.clone(), - }, - } - } - - pub fn username(&self) -> String { - match self { - DavAuth::Basic { user, .. } => user.clone(), - DavAuth::Bearer { .. } => "(bearer)".to_owned(), - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SourceChange { - Reject, - Allow, - Replace, -} - -#[derive(Debug)] -pub struct DavImportConfig { - pub kind: DavKindArg, - pub url: String, - pub auth: DavAuth, - pub allow_cleartext: bool, - pub dav_connections: usize, - pub multiget_batch: usize, - pub source_change: SourceChange, -} - -pub fn run(common: CommonConfig, config: DavImportConfig) -> Result { - run_reporting(common, config).into_result() -} - -pub fn run_reporting(common: CommonConfig, config: DavImportConfig) -> RunOutcome { - let mut summary = Summary::default(); - let error = run_into(common, config, &mut summary).err(); - RunOutcome { summary, error } -} - -fn run_into( - common: CommonConfig, - config: DavImportConfig, - summary: &mut Summary, -) -> Result<(), Error> { - let logger = common.logger; - enforce_tls_policy(&config.url, config.allow_cleartext)?; - - let mut conn = db::init::open(&common.archive)?; - - let client = DavClient::new( - config.auth.to_jmap_auth(), - RetryPolicy::new(common.max_retries), - common.allow_invalid_certs, - ); - client.set_logger(logger); - - let discovery = - discover(&client, config.kind.kind(), &config.url).map_err(map_discovery_error)?; - - let session_url = normalise_base_url(&config.url); - let account_id_raw = discovery - .principal_url - .clone() - .unwrap_or_else(|| discovery.home_set_url.clone()); - let account_id = normalise_account_url(&account_id_raw); - - let key = SourceKey { - kind: config.kind.source_kind().to_owned(), - session_url: session_url.clone(), - account_id: account_id.clone(), - }; - - if !common.dry_run - && config.source_change == SourceChange::Reject - && let Some((url, acc)) = db::sources::conflicting_source( - &conn, - config.kind.source_kind(), - &session_url, - &account_id, - ) - .map_err(|e| Error::Partial(e.to_string()))? - { - return Err(Error::SourceChange(format!( - "archive already records {} source ({url}, account {acc}); \ - pass --allow-source-change to import a different account, \ - or --replace-source to drop the recorded one", - config.kind.source_kind() - ))); - } - - if logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "DAV discovery: principal={:?} home_set={} collections={}", - discovery.principal_url, - discovery.home_set_url, - discovery.collections.len() - ); - } - - if common.dry_run { - if config.source_change == SourceChange::Replace && logger.enabled(LEVEL_DEFAULT) { - for (_, url, acc) in - db::sources::other_sources(&conn, &key.kind, &session_url, &account_id)? - { - eprintln!( - "dry-run: would remove {} source {url} (account {acc})", - key.kind - ); - } - } - let result = run_dry_diff(&conn, &client, &discovery, &config, logger, summary); - summary.retries_observed = client.retries_observed(); - summary.retry_after_sleeps = client.retry_after_sleeps(); - return result; - } - - let removed = match config.source_change { - SourceChange::Replace => replace::remove_other_sources(&conn, &key, logger)?, - SourceChange::Reject | SourceChange::Allow => Removed::default(), - }; - - let username = config.auth.username(); - let source_id = db::sources::upsert_source(&conn, &key, Some(&account_id), &username) - .map_err(|e| Error::Partial(e.to_string()))?; - - let phase = match config.kind { - DavKindArg::Caldav => { - run_caldav(&mut conn, &client, source_id, &discovery, &config, logger) - } - DavKindArg::Carddav => { - run_carddav(&mut conn, &client, source_id, &discovery, &config, logger) - } - DavKindArg::Webdav => { - run_webdav(&mut conn, &client, source_id, &discovery, &config, logger) - } - }; - - *summary = phase.summary; - removed.add_to(summary); - summary.retries_observed = client.retries_observed(); - summary.retry_after_sleeps = client.retry_after_sleeps(); - if let Some(e) = phase.error { - return Err(e); - } - - if !summary.any_failed() - && let Err(e) = run_gc(&conn) - { - logger.warn(&format!("blob GC skipped: {e}")); - } - - Ok(()) -} - -type ReconcileCollections = fn( - &mut Connection, - i64, - &[crate::dav::discover::DiscoveredCollection], - &mut TypeCounts, - Logger, -) -> Result, Error>; - -type ReconcileItems = - fn(&mut Connection, &items::ItemRunCtx<'_>, &str, i64, &mut TypeCounts) -> Result<(), Error>; - -struct ItemPhase { - container_label: &'static str, - item_label: &'static str, - reconcile_collections: ReconcileCollections, - reconcile_items: ReconcileItems, -} - -fn run_collection_phase( - conn: &mut Connection, - client: &DavClient, - source_id: i64, - discovery: &Discovery, - config: &DavImportConfig, - logger: Logger, - phase: ItemPhase, -) -> RunOutcome { - let mut summary = Summary::default(); - let mut counts = PhaseCounts::default(); - let error = collection_phase_into( - conn, - client, - source_id, - PhaseInput { - discovery, - config, - logger, - phase: &phase, - }, - &mut counts, - ) - .err(); - - summary - .per_type - .push((phase.container_label, counts.container)); - summary.per_type.push((phase.item_label, counts.items)); - RunOutcome { summary, error } -} - -#[derive(Default)] -struct PhaseCounts { - container: TypeCounts, - items: TypeCounts, -} - -struct PhaseInput<'a> { - discovery: &'a Discovery, - config: &'a DavImportConfig, - logger: Logger, - phase: &'a ItemPhase, -} - -fn collection_phase_into( - conn: &mut Connection, - client: &DavClient, - source_id: i64, - input: PhaseInput<'_>, - counts: &mut PhaseCounts, -) -> Result<(), Error> { - let PhaseInput { - discovery, - config, - logger, - phase, - } = input; - - let upserted = (phase.reconcile_collections)( - conn, - source_id, - &discovery.collections, - &mut counts.container, - logger, - )?; - if logger.enabled(LEVEL_DEFAULT) { - eprintln!( - "import: {} done (upserted={} deleted={} failed={})", - phase.container_label, - counts.container.created + counts.container.fetched, - counts.container.deleted, - counts.container.failed - ); - } - - let ctx = items::ItemRunCtx { - client, - source_id, - base_url: &discovery.home_set_url, - multiget_batch: config.multiget_batch, - dav_connections: config.dav_connections, - logger, - }; - for (collection_href, local_id) in &upserted { - match (phase.reconcile_items)(conn, &ctx, collection_href, *local_id, &mut counts.items) { - Ok(()) => {} - Err(e) if e.aborts_run() => return Err(e), - Err(e) => { - logger.warn(&format!( - "{} {collection_href:?}: items failed: {e}", - phase.container_label - )); - counts.items.failed += 1; - } - } - } - Ok(()) -} - -fn run_dry_diff( - conn: &rusqlite::Connection, - client: &DavClient, - discovery: &Discovery, - config: &DavImportConfig, - logger: Logger, - summary: &mut Summary, -) -> Result<(), Error> { - use crate::dav::href::join_absolute; - use crate::dav::xml; - use crate::db::dav_ids; - let (container_label, item_label, container_type, item_type) = match config.kind { - DavKindArg::Caldav => ( - "calendar", - "calendarevent", - dav_ids::CALENDAR, - dav_ids::CALENDAR_EVENT, - ), - DavKindArg::Carddav => ( - "addressbook", - "contactcard", - dav_ids::ADDRESS_BOOK, - dav_ids::CONTACT_CARD, - ), - DavKindArg::Webdav => ( - "filenode", - "filenode", - dav_ids::FILE_NODE, - dav_ids::FILE_NODE, - ), - }; - - let mut container_counts = TypeCounts::default(); - let mut item_counts = TypeCounts::default(); - container_counts.created = discovery.collections.len() as u64; - if logger.enabled(LEVEL_DEFAULT) { - eprintln!( - "dry-run: {} new={} (no archive to diff against)", - container_label, - discovery.collections.len(), - ); - } - - if !matches!(config.kind, DavKindArg::Webdav) { - for coll in &discovery.collections { - let url = join_absolute(&discovery.home_set_url, coll.href.as_str()) - .map_err(|e| Error::Partial(e.to_string()))?; - match client - .propfind_responses(&url, 1, &xml::propfind_dav_items(), &url) - .map_err(super::per_collection_failure) - { - Ok(ms) => { - let new_count = ms - .responses - .iter() - .filter(|r| !r.props.is_collection) - .count(); - item_counts.created += new_count as u64; - if logger.enabled(LEVEL_DEFAULT) { - eprintln!(" {} items: {new_count}", coll.href.as_str()); - } - } - Err(e) if e.aborts_run() => return Err(e), - Err(e) => { - logger.warn(&format!( - "{container_label} {:?}: enumeration failed: {e}", - coll.href.as_str() - )); - item_counts.failed += 1; - } - } - } - } else if let Some(root) = discovery.collections.first() { - let url = join_absolute(&discovery.home_set_url, root.href.as_str()) - .map_err(|e| Error::Partial(e.to_string()))?; - match client - .propfind_responses(&url, 1, &xml::propfind_webdav_listing(), &url) - .map_err(super::per_collection_failure) - { - Ok(ms) => { - let new_count = ms - .responses - .iter() - .filter(|r| !r.props.is_collection) - .count(); - item_counts.created += new_count as u64; - if logger.enabled(LEVEL_DEFAULT) { - eprintln!(" root {} files: {new_count}", root.href.as_str()); - } - } - Err(e) if e.aborts_run() => return Err(e), - Err(e) => { - logger.warn(&format!( - "{container_label} {:?}: enumeration failed: {e}", - root.href.as_str() - )); - container_counts.failed += 1; - } - } - } - - let _ = container_type; - let _ = item_type; - let _ = conn; - summary.per_type.push((container_label, container_counts)); - if !matches!(config.kind, DavKindArg::Webdav) { - summary.per_type.push((item_label, item_counts)); - } - Ok(()) -} - -fn run_caldav( - conn: &mut Connection, - client: &DavClient, - source_id: i64, - discovery: &Discovery, - config: &DavImportConfig, - logger: Logger, -) -> RunOutcome { - run_collection_phase( - conn, - client, - source_id, - discovery, - config, - logger, - ItemPhase { - container_label: "calendar", - item_label: "calendarevent", - reconcile_collections: collections::reconcile_calendars, - reconcile_items: items::reconcile_calendar_events, - }, - ) -} - -fn run_carddav( - conn: &mut Connection, - client: &DavClient, - source_id: i64, - discovery: &Discovery, - config: &DavImportConfig, - logger: Logger, -) -> RunOutcome { - run_collection_phase( - conn, - client, - source_id, - discovery, - config, - logger, - ItemPhase { - container_label: "addressbook", - item_label: "contactcard", - reconcile_collections: collections::reconcile_address_books, - reconcile_items: items::reconcile_contact_cards, - }, - ) -} - -fn run_webdav( - conn: &mut Connection, - client: &DavClient, - source_id: i64, - discovery: &Discovery, - config: &DavImportConfig, - logger: Logger, -) -> RunOutcome { - let mut summary = Summary::default(); - let mut file_counts = TypeCounts::default(); - - if discovery.collections.is_empty() { - summary.per_type.push(("filenode", file_counts)); - return RunOutcome { - summary, - error: None, - }; - } - let root = &discovery.collections[0]; - let ctx = tree::WebDavCtx { - client, - source_id, - base_url: &discovery.home_set_url, - dav_connections: config.dav_connections, - logger, - }; - let error = tree::reconcile_filenodes(conn, &ctx, root, &mut file_counts).err(); - - summary.per_type.push(("filenode", file_counts)); - RunOutcome { summary, error } -} - -fn map_discovery_error(err: DiscoveryError) -> Error { - match err { - DiscoveryError::NotFound { url } => Error::Usage(format!( - "no DAV collections found under --url {url:?}; \ - check the URL and that the account has access" - )), - DiscoveryError::Transport(e) => Error::from(e), - DiscoveryError::Parse(e) => { - Error::Connection(format!("DAV discovery: malformed multistatus: {e}")) - } - DiscoveryError::Href(e) => Error::Connection(format!("DAV discovery: bad href: {e}")), - } -} - -fn run_gc(conn: &Connection) -> Result<(), Error> { - let tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - db::blobs::gc_orphan_blobs(&tx).map_err(|e| Error::Partial(e.to_string()))?; - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - Ok(()) -} - -fn enforce_tls_policy(url: &str, allow_cleartext: bool) -> Result<(), Error> { - let parsed = - url::Url::parse(url).map_err(|e| Error::Usage(format!("invalid --url {url:?}: {e}")))?; - match parsed.scheme() { - "https" => Ok(()), - "http" => { - if allow_cleartext { - Ok(()) - } else { - Err(Error::Connection( - "http:// URL requires --allow-cleartext".to_owned(), - )) - } - } - other => Err(Error::Usage(format!( - "--url scheme must be http or https, got {other}" - ))), - } -} - -fn normalise_base_url(url: &str) -> String { - let parsed = match url::Url::parse(url) { - Ok(u) => u, - Err(_) => return url.to_owned(), - }; - let scheme = parsed.scheme(); - let host = parsed.host_str().unwrap_or(""); - let port = parsed.port().map(|p| format!(":{p}")).unwrap_or_default(); - format!("{scheme}://{host}{port}") -} - -fn normalise_account_url(url: &str) -> String { - let mut parsed = match url::Url::parse(url) { - Ok(u) => u, - Err(_) => return url.to_owned(), - }; - parsed.set_query(None); - parsed.set_fragment(None); - let path = parsed.path().to_owned(); - if !path.ends_with('/') { - parsed.set_path(&format!("{path}/")); - } - parsed.to_string() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn https_url_is_accepted_without_allow_cleartext() { - assert!(enforce_tls_policy("https://x/", false).is_ok()); - } - - #[test] - fn http_url_requires_allow_cleartext() { - let err = enforce_tls_policy("http://x/", false).unwrap_err(); - assert!(matches!(err, Error::Connection(_))); - assert!(enforce_tls_policy("http://x/", true).is_ok()); - } - - #[test] - fn ftp_url_rejected_as_usage() { - let err = enforce_tls_policy("ftp://x/", false).unwrap_err(); - assert!(matches!(err, Error::Usage(_))); - } - - #[test] - fn normalise_base_url_strips_path() { - assert_eq!( - normalise_base_url("https://dav.example.com/cal/"), - "https://dav.example.com" - ); - } - - #[test] - fn normalise_base_url_keeps_explicit_port() { - assert_eq!( - normalise_base_url("https://dav.example.com:8443/cal/"), - "https://dav.example.com:8443" - ); - } -} diff --git a/src/sync/import_dav/items.rs b/src/sync/import_dav/items.rs deleted file mode 100644 index dac1ffc..0000000 --- a/src/sync/import_dav/items.rs +++ /dev/null @@ -1,653 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::{HashMap, HashSet}; - -use rusqlite::{Connection, params}; - -use crate::dav::client::{DavClient, MultiStatus}; -use crate::dav::href::{Href, join_absolute, normalise}; -use crate::dav::parse::DavResponse; -use crate::dav::xml; -use crate::db::dav_ids; -use crate::error::Error; -use crate::jmap::error::JmapError; -use crate::logging::{LEVEL_PROGRESS, Logger}; -use crate::sync::TypeCounts; -use crate::sync::import_jmap::pool::Pool; - -use super::calcard; - -pub struct ItemRunCtx<'a> { - pub client: &'a DavClient, - pub source_id: i64, - pub base_url: &'a str, - pub multiget_batch: usize, - pub dav_connections: usize, - pub logger: Logger, -} - -struct PerCollectionCtx<'a> { - run: &'a ItemRunCtx<'a>, - kind: ItemKind, - collection_href: &'a str, - container_local_id: i64, - absolute_url: &'a str, -} - -pub fn reconcile_calendar_events( - conn: &mut Connection, - ctx: &ItemRunCtx<'_>, - collection_href: &str, - calendar_local_id: i64, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let absolute = absolute_collection_url(ctx.base_url, collection_href)?; - let server_items = enumerate_items(ctx.client, &absolute)?; - let pcc = PerCollectionCtx { - run: ctx, - kind: ItemKind::CalendarEvent, - collection_href, - container_local_id: calendar_local_id, - absolute_url: &absolute, - }; - reconcile_items_generic(conn, &pcc, &server_items, counts) -} - -pub fn reconcile_contact_cards( - conn: &mut Connection, - ctx: &ItemRunCtx<'_>, - collection_href: &str, - address_book_local_id: i64, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let absolute = absolute_collection_url(ctx.base_url, collection_href)?; - let server_items = enumerate_items(ctx.client, &absolute)?; - let pcc = PerCollectionCtx { - run: ctx, - kind: ItemKind::ContactCard, - collection_href, - container_local_id: address_book_local_id, - absolute_url: &absolute, - }; - reconcile_items_generic(conn, &pcc, &server_items, counts) -} - -#[derive(Debug, Clone, Copy)] -enum ItemKind { - CalendarEvent, - ContactCard, -} - -impl ItemKind { - fn type_name(self) -> &'static str { - match self { - ItemKind::CalendarEvent => dav_ids::CALENDAR_EVENT, - ItemKind::ContactCard => dav_ids::CONTACT_CARD, - } - } - fn data_query(self) -> fn(&[Href]) -> String { - match self { - ItemKind::CalendarEvent => xml::calendar_multiget, - ItemKind::ContactCard => xml::addressbook_multiget, - } - } - fn data_field(self) -> &'static str { - match self { - ItemKind::CalendarEvent => "calendar_data", - ItemKind::ContactCard => "address_data", - } - } -} - -#[derive(Debug, Clone)] -struct ServerItem { - href: String, - etag: String, -} - -fn enumerate_items(client: &DavClient, url: &str) -> Result, Error> { - let body = xml::propfind_dav_items(); - let ms = client - .propfind_responses(url, 1, &body, url) - .map_err(super::per_collection_failure)?; - if ms.status >= 400 { - return Err(Error::Partial(format!( - "enumerate {url}: http {}", - ms.status - ))); - } - let self_href = normalise(url, "") - .map(|h| h.into_string()) - .unwrap_or_default(); - let mut out = Vec::new(); - for r in ms.responses { - if r.href.as_str() == self_href { - continue; - } - if r.props.is_collection { - continue; - } - out.push(ServerItem { - href: r.href.into_string(), - etag: r.props.etag.unwrap_or_default(), - }); - } - Ok(out) -} - -struct MultigetJob { - body: String, - hrefs: Vec, -} - -struct MultigetReply { - hrefs: Vec, - result: Result, -} - -struct GetJob { - url: String, - item: ServerItem, -} - -struct GetReply { - item: ServerItem, - result: Result, -} - -struct GetBody { - status: u16, - bytes: Vec, - etag: Option, -} - -fn reconcile_items_generic( - conn: &mut Connection, - pcc: &PerCollectionCtx<'_>, - server_items: &[ServerItem], - counts: &mut TypeCounts, -) -> Result<(), Error> { - let type_name = pcc.kind.type_name(); - let local_rows = - dav_ids::items_in_collection(conn, pcc.run.source_id, type_name, pcc.collection_href) - .map_err(|e| Error::Partial(e.to_string()))?; - let local_map: HashMap = local_rows - .iter() - .map(|r| (r.item_href.clone(), (r.etag.clone(), r.local_id))) - .collect(); - let server_set: HashSet<&str> = server_items.iter().map(|s| s.href.as_str()).collect(); - - let mut to_fetch: Vec = Vec::new(); - for s in server_items { - match local_map.get(&s.href) { - None => to_fetch.push(s.clone()), - Some((local_etag, _)) => { - if s.etag.is_empty() || s.etag != *local_etag { - to_fetch.push(s.clone()); - } - } - } - } - - let vanished: Vec = local_map - .keys() - .filter(|h| !server_set.contains(h.as_str())) - .cloned() - .collect(); - - if !to_fetch.is_empty() { - let fallback = run_multiget_pool(conn, pcc, &to_fetch, &local_map, counts)?; - if !fallback.is_empty() { - run_get_pool(conn, pcc, &fallback, &local_map, counts)?; - } - } - - for href in &vanished { - let local_id = match local_map.get(href) { - Some((_, id)) => *id, - None => continue, - }; - let tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - let table = match pcc.kind { - ItemKind::CalendarEvent => "calendar_events", - ItemKind::ContactCard => "contact_cards", - }; - tx.execute( - &format!("DELETE FROM {table} WHERE id = ?1"), - params![local_id], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - dav_ids::delete_item(&tx, pcc.run.source_id, type_name, href) - .map_err(|e| Error::Partial(e.to_string()))?; - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - counts.deleted += 1; - } - - if pcc.run.logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "items in {}: fetched={} deleted={} failed={}", - pcc.collection_href, counts.fetched, counts.deleted, counts.failed - ); - } - Ok(()) -} - -fn run_multiget_pool( - conn: &mut Connection, - pcc: &PerCollectionCtx<'_>, - to_fetch: &[ServerItem], - local_map: &HashMap, - counts: &mut TypeCounts, -) -> Result, Error> { - let logger = pcc.run.logger; - let absolute_url = pcc.absolute_url; - let batch_size = pcc.run.multiget_batch.max(1); - let workers = pcc.run.dav_connections.clamp(1, 8); - let by_href: HashMap = to_fetch - .iter() - .map(|s| (s.href.clone(), s.clone())) - .collect(); - let client_for_pool = pcc.run.client.clone(); - let url_for_pool = absolute_url.to_owned(); - let pool: Pool = Pool::new(workers, move |job: MultigetJob| { - let result = client_for_pool.report_responses(&url_for_pool, 1, &job.body, &url_for_pool); - MultigetReply { - hrefs: job.hrefs, - result, - } - }); - let mut submitted: usize = 0; - for chunk in to_fetch.chunks(batch_size) { - let hrefs: Vec = chunk - .iter() - .map(|s| Href::from_normalised(s.href.clone())) - .collect(); - let chunk_hrefs: Vec = chunk.iter().map(|s| s.href.clone()).collect(); - pool.submit(MultigetJob { - body: (pcc.kind.data_query())(&hrefs), - hrefs: chunk_hrefs, - }); - submitted += 1; - } - - let mut fallback: Vec = Vec::new(); - for _ in 0..submitted { - let reply = match pool.results().recv() { - Ok(r) => r, - Err(_) => break, - }; - match reply.result { - Err(JmapError::HttpStatus { status, .. }) if status == 405 || status == 501 => { - logger.warn(&format!( - "multiget {absolute_url} returned {status}; server may not implement multiget, falling back to per-item GET" - )); - fallback.extend(reply.hrefs.iter().filter_map(|h| by_href.get(h).cloned())); - } - Err(e) => { - logger.warn(&format!( - "multiget {absolute_url}: {e}; falling back to per-item GET" - )); - fallback.extend(reply.hrefs.iter().filter_map(|h| by_href.get(h).cloned())); - } - Ok(ms) if ms.status >= 400 => { - logger.warn(&format!( - "multiget {absolute_url} returned {}; chunk failed", - ms.status - )); - counts.failed += reply.hrefs.len() as u64; - } - Ok(ms) => { - let requested: HashSet<&str> = reply.hrefs.iter().map(String::as_str).collect(); - let mut tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - let mut pending: usize = 0; - for r in ms.responses { - if !requested.contains(r.href.as_str()) { - logger.warn(&format!( - "multiget {absolute_url}: server returned unsolicited href {}; skipping", - r.href.as_str() - )); - continue; - } - handle_multiget_response(&tx, pcc, &r, local_map, counts); - pending += 1; - if pending >= COMMIT_BATCH { - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - pending = 0; - } - } - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - } - } - } - drop(pool); - Ok(fallback) -} - -fn run_get_pool( - conn: &mut Connection, - pcc: &PerCollectionCtx<'_>, - items: &[ServerItem], - local_map: &HashMap, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let logger = pcc.run.logger; - let absolute_url = pcc.absolute_url; - let workers = pcc.run.dav_connections.clamp(1, 8); - let client_for_pool = pcc.run.client.clone(); - let pool: Pool = Pool::new(workers, move |job: GetJob| { - let r = client_for_pool.get(&job.url).map(|resp| GetBody { - status: resp.status, - bytes: resp.body, - etag: resp.etag, - }); - GetReply { - item: job.item, - result: r, - } - }); - - let mut submitted: usize = 0; - for item in items { - let url = match absolute_item_url(absolute_url, &item.href) { - Ok(u) => u, - Err(e) => { - logger.warn(&format!("item url {}: {e}", item.href)); - counts.failed += 1; - continue; - } - }; - pool.submit(GetJob { - url, - item: item.clone(), - }); - submitted += 1; - } - - let mut tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - let mut pending: usize = 0; - for _ in 0..submitted { - let reply = match pool.results().recv() { - Ok(r) => r, - Err(_) => break, - }; - match reply.result { - Err(e) => { - logger.warn(&format!("GET {}: {e}", reply.item.href)); - counts.failed += 1; - } - Ok(body) if body.status == 404 || body.status == 410 => {} - Ok(body) if body.status >= 400 => { - logger.warn(&format!("GET {}: http {}", reply.item.href, body.status)); - counts.failed += 1; - } - Ok(body) => { - let etag = body.etag.unwrap_or_else(|| reply.item.etag.clone()); - let raw = match std::str::from_utf8(&body.bytes) { - Ok(s) => s.to_owned(), - Err(e) => { - logger.warn(&format!("GET {}: non-utf8 body: {e}", reply.item.href)); - counts.failed += 1; - continue; - } - }; - if let Err(e) = - insert_or_update_tx(&tx, pcc, &reply.item.href, &etag, &raw, local_map) - { - logger.warn(&format!("item {}: {e}", reply.item.href)); - counts.failed += 1; - } else { - counts.fetched += 1; - pending += 1; - if pending >= COMMIT_BATCH { - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - pending = 0; - } - } - } - } - } - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - drop(pool); - Ok(()) -} - -const COMMIT_BATCH: usize = 64; - -fn handle_multiget_response( - tx: &rusqlite::Transaction<'_>, - pcc: &PerCollectionCtx<'_>, - response: &DavResponse, - local_map: &HashMap, - counts: &mut TypeCounts, -) { - let logger = pcc.run.logger; - if response_indicates_vanished(response) { - return; - } - let item_href = response.href.as_str().to_owned(); - let etag = response.props.etag.clone().unwrap_or_default(); - let data = match pcc.kind { - ItemKind::CalendarEvent => response.props.calendar_data.as_deref(), - ItemKind::ContactCard => response.props.address_data.as_deref(), - }; - let Some(raw) = data else { - logger.warn(&format!( - "multiget {item_href}: missing {} payload", - pcc.kind.data_field() - )); - counts.failed += 1; - return; - }; - if let Err(e) = insert_or_update_tx(tx, pcc, &item_href, &etag, raw, local_map) { - logger.warn(&format!("item {item_href}: {e}")); - counts.failed += 1; - } else { - counts.fetched += 1; - } -} - -fn insert_or_update_tx( - tx: &rusqlite::Transaction<'_>, - pcc: &PerCollectionCtx<'_>, - item_href: &str, - etag: &str, - raw: &str, - local_map: &HashMap, -) -> Result<(), Error> { - let existing = local_map.get(item_href).map(|(_, id)| *id); - match pcc.kind { - ItemKind::CalendarEvent => insert_or_update_event( - tx, - pcc.container_local_id, - existing, - raw, - item_href, - pcc.run.logger, - )?, - ItemKind::ContactCard => { - insert_or_update_card(tx, pcc.container_local_id, existing, raw, item_href)? - } - }; - if existing.is_none() { - let new_local = tx.last_insert_rowid(); - dav_ids::insert( - tx, - pcc.run.source_id, - pcc.kind.type_name(), - pcc.collection_href, - item_href, - etag, - new_local, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - } else { - dav_ids::update_etag(tx, pcc.run.source_id, pcc.kind.type_name(), item_href, etag) - .map_err(|e| Error::Partial(e.to_string()))?; - } - Ok(()) -} - -fn insert_or_update_event( - tx: &rusqlite::Transaction<'_>, - calendar_local: i64, - existing: Option, - raw: &str, - item_href: &str, - logger: Logger, -) -> Result<(), Error> { - let entries = calcard::ical_to_jscalendar_entries(raw) - .map_err(|e| Error::Partial(format!("iCalendar parse: {e}")))?; - if entries.len() > 1 && logger.enabled(crate::logging::LEVEL_DEFAULT) { - logger.warn(&format!( - "{item_href}: iCalendar resource has {} entries; storing the first ({}), \ - others dropped (recurrence overrides on the master are preserved)", - entries.len(), - entries[0].data_type.as_column(), - )); - } - let mut first = entries - .into_iter() - .next() - .ok_or_else(|| Error::Partial("iCalendar contained no parseable entries".to_owned()))?; - let (is_draft, use_default_alerts, _uid) = - calcard::strip_extracted_fields_from_event(&mut first.data); - let data_type = first.data_type.as_column(); - let calendar_ids = format!("[{calendar_local}]"); - if let Some(local) = existing { - tx.execute( - "UPDATE calendar_events SET calendar_ids = ?1, is_draft = ?2, - use_default_alerts = ?3, data = ?4, data_type = ?5 - WHERE id = ?6", - params![ - calendar_ids, - is_draft as i64, - use_default_alerts as i64, - first.data.to_string(), - data_type, - local, - ], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - } else { - tx.execute( - "INSERT INTO calendar_events (calendar_ids, is_draft, use_default_alerts, data, data_type) - VALUES (?1, ?2, ?3, ?4, ?5)", - params![ - calendar_ids, - is_draft as i64, - use_default_alerts as i64, - first.data.to_string(), - data_type, - ], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - } - Ok(()) -} - -fn insert_or_update_card( - tx: &rusqlite::Transaction<'_>, - address_book_local: i64, - existing: Option, - raw: &str, - item_href: &str, -) -> Result<(), Error> { - let card = calcard::vcard_to_jscontact(raw, item_href) - .map_err(|e| Error::Partial(format!("vCard parse: {e}")))?; - let address_book_ids = format!("[{address_book_local}]"); - if let Some(local) = existing { - tx.execute( - "UPDATE contact_cards SET uid = ?1, address_book_ids = ?2, data = ?3 - WHERE id = ?4", - params![card.uid, address_book_ids, card.data.to_string(), local], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - } else { - tx.execute( - "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)", - params![card.uid, address_book_ids, card.data.to_string()], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - } - Ok(()) -} - -fn response_indicates_vanished(response: &DavResponse) -> bool { - if let Some(status) = response.status - && (status == 404 || status == 410) - { - return true; - } - response - .propstat_errors - .iter() - .any(|s| *s == 404 || *s == 410) -} - -fn absolute_collection_url(base_url: &str, href: &str) -> Result { - join_absolute(base_url, href).map_err(|e| Error::Partial(e.to_string())) -} - -fn absolute_item_url(collection_url: &str, item_href: &str) -> Result { - join_absolute(collection_url, item_href).map_err(|e| Error::Partial(e.to_string())) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn absolute_collection_url_joins_path() { - let u = absolute_collection_url("https://x/dav/", "/dav/cal/u/d/").unwrap(); - assert_eq!(u, "https://x/dav/cal/u/d/"); - } - - #[test] - fn absolute_item_url_joins_relative() { - let u = absolute_item_url("https://x/dav/cal/u/d/", "/dav/cal/u/d/event.ics").unwrap(); - assert_eq!(u, "https://x/dav/cal/u/d/event.ics"); - } - - fn response(status: Option, propstat_errors: Vec) -> DavResponse { - DavResponse { - href: Href::from_normalised("/dav/cal/u/d/e.ics".to_owned()), - status, - props: crate::dav::parse::ResourceProps::default(), - propstat_errors, - } - } - - #[test] - fn response_level_404_is_vanished() { - assert!(response_indicates_vanished(&response(Some(404), vec![]))); - assert!(response_indicates_vanished(&response(Some(410), vec![]))); - } - - #[test] - fn per_propstat_404_or_410_is_vanished() { - assert!(response_indicates_vanished(&response(None, vec![404]))); - assert!(response_indicates_vanished(&response(None, vec![200, 410]))); - } - - #[test] - fn other_propstat_errors_are_not_vanished() { - assert!(!response_indicates_vanished(&response(None, vec![403]))); - assert!(!response_indicates_vanished(&response(None, vec![]))); - } -} diff --git a/src/sync/import_dav/replace.rs b/src/sync/import_dav/replace.rs deleted file mode 100644 index 4456ee2..0000000 --- a/src/sync/import_dav/replace.rs +++ /dev/null @@ -1,255 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use rusqlite::{Connection, params}; - -use crate::db::dav_ids::{ADDRESS_BOOK, CALENDAR, CALENDAR_EVENT, CONTACT_CARD, FILE_NODE}; -use crate::db::sources::{self, SourceKey}; -use crate::error::Error; -use crate::logging::{LEVEL_DEFAULT, Logger}; -use crate::sync::Summary; - -const DELETE_MAPPED: [(&str, &str); 4] = [ - ( - CALENDAR_EVENT, - "DELETE FROM calendar_events WHERE id IN - (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)", - ), - ( - CONTACT_CARD, - "DELETE FROM contact_cards WHERE id IN - (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)", - ), - ( - CALENDAR, - "DELETE FROM calendars WHERE id IN - (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)", - ), - ( - ADDRESS_BOOK, - "DELETE FROM address_books WHERE id IN - (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)", - ), -]; - -const DELETE_LEAF_FILE_NODES: &str = "DELETE FROM file_nodes WHERE id IN - (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2) - AND NOT EXISTS (SELECT 1 FROM file_nodes AS child WHERE child.parent_id = file_nodes.id)"; - -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -pub struct Removed { - pub calendar_events: u64, - pub contact_cards: u64, - pub calendars: u64, - pub address_books: u64, - pub file_nodes: u64, -} - -impl Removed { - fn slot(&mut self, type_name: &str) -> Option<&mut u64> { - match type_name { - CALENDAR_EVENT => Some(&mut self.calendar_events), - CONTACT_CARD => Some(&mut self.contact_cards), - CALENDAR => Some(&mut self.calendars), - ADDRESS_BOOK => Some(&mut self.address_books), - FILE_NODE => Some(&mut self.file_nodes), - _ => None, - } - } - - pub fn add_to(mut self, summary: &mut Summary) { - for (label, counts) in &mut summary.per_type { - if let Some(n) = self.slot(label) { - counts.deleted += std::mem::take(n); - } - } - } -} - -pub fn remove_other_sources( - conn: &Connection, - key: &SourceKey, - logger: Logger, -) -> Result { - let others = sources::other_sources(conn, &key.kind, &key.session_url, &key.account_id)?; - let mut removed = Removed::default(); - if others.is_empty() { - return Ok(removed); - } - let tx = conn.unchecked_transaction()?; - for (source_id, session_url, account_id) in &others { - remove_source(&tx, *source_id, &mut removed)?; - if logger.enabled(LEVEL_DEFAULT) { - eprintln!( - "import: removed {} source {session_url} (account {account_id})", - key.kind - ); - } - } - tx.commit()?; - Ok(removed) -} - -fn remove_source( - conn: &Connection, - source_id: i64, - removed: &mut Removed, -) -> Result<(), rusqlite::Error> { - for (type_name, sql) in DELETE_MAPPED { - let n = conn.execute(sql, params![source_id, type_name])?; - if let Some(slot) = removed.slot(type_name) { - *slot += n as u64; - } - } - let mut leaves = conn.prepare(DELETE_LEAF_FILE_NODES)?; - loop { - let n = leaves.execute(params![source_id, FILE_NODE])?; - if n == 0 { - break; - } - removed.file_nodes += n as u64; - } - sources::delete_source(conn, source_id) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::{dav_ids, init}; - use crate::sync::TypeCounts; - - fn mem() -> Connection { - let c = Connection::open_in_memory().unwrap(); - init::apply_schema(&c).unwrap(); - c.pragma_update(None, "foreign_keys", "ON").unwrap(); - c - } - - fn source(c: &Connection, kind: &str, url: &str) -> (SourceKey, i64) { - let key = SourceKey { - kind: kind.to_owned(), - session_url: url.to_owned(), - account_id: format!("{url}/u/"), - }; - let id = sources::upsert_source(c, &key, None, "u").unwrap(); - (key, id) - } - - fn file_node(c: &Connection, sid: i64, parent: Option, name: &str, href: &str) -> i64 { - c.execute( - "INSERT INTO file_nodes (parent_id, node_type, name, created) - VALUES (?1, 'directory', ?2, '2026-01-01T00:00:00Z')", - params![parent, name], - ) - .unwrap(); - let id = c.last_insert_rowid(); - dav_ids::insert(c, sid, FILE_NODE, href, href, "", id).unwrap(); - id - } - - fn count(c: &Connection, table: &str) -> i64 { - c.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0)) - .unwrap() - } - - #[test] - fn replacing_a_webdav_source_drops_the_old_tree_leaf_first() { - let c = mem(); - let (_, old) = source(&c, "webdav", "https://old"); - let a = file_node(&c, old, None, "a", "/a/"); - let b = file_node(&c, old, Some(a), "b", "/a/b/"); - file_node(&c, old, Some(b), "c", "/a/b/c/"); - let (new_key, new) = source(&c, "webdav", "https://new"); - file_node(&c, new, None, "a", "/a/"); - - let removed = remove_other_sources(&c, &new_key, Logger::from_flags(true, 0)).unwrap(); - - assert_eq!(removed.file_nodes, 3); - assert_eq!(count(&c, "file_nodes"), 1); - assert_eq!(count(&c, "sources"), 1); - assert_eq!(count(&c, "sync_id_dav"), 1); - let kept: i64 = c - .query_row("SELECT source_id FROM sync_id_dav", [], |r| r.get(0)) - .unwrap(); - assert_eq!(kept, new); - } - - #[test] - fn replacing_leaves_other_kinds_untouched() { - let c = mem(); - let (_, cal) = source(&c, "caldav", "https://old"); - c.execute("INSERT INTO calendars (name) VALUES ('Work')", []) - .unwrap(); - let cal_local = c.last_insert_rowid(); - dav_ids::insert(&c, cal, CALENDAR, "/c/", "/c/", "", cal_local).unwrap(); - c.execute( - "INSERT INTO calendar_events (calendar_ids, data) VALUES (?1, '{}')", - params![format!("[{cal_local}]")], - ) - .unwrap(); - let ev = c.last_insert_rowid(); - dav_ids::insert(&c, cal, CALENDAR_EVENT, "/c/", "/c/1.ics", "", ev).unwrap(); - let (_, old_web) = source(&c, "webdav", "https://old"); - file_node(&c, old_web, None, "a", "/a/"); - let (new_web, _) = source(&c, "webdav", "https://new"); - - let removed = remove_other_sources(&c, &new_web, Logger::from_flags(true, 0)).unwrap(); - - assert_eq!(removed.file_nodes, 1); - assert_eq!(removed.calendars + removed.calendar_events, 0); - assert_eq!(count(&c, "calendars"), 1); - assert_eq!(count(&c, "calendar_events"), 1); - assert_eq!(count(&c, "file_nodes"), 0); - } - - #[test] - fn replacing_a_caldav_source_drops_its_calendars_and_events() { - let c = mem(); - let (_, old) = source(&c, "caldav", "https://old"); - c.execute("INSERT INTO calendars (name) VALUES ('Work')", []) - .unwrap(); - let cal_local = c.last_insert_rowid(); - dav_ids::insert(&c, old, CALENDAR, "/c/", "/c/", "", cal_local).unwrap(); - c.execute( - "INSERT INTO calendar_events (calendar_ids, data) VALUES (?1, '{}')", - params![format!("[{cal_local}]")], - ) - .unwrap(); - let ev = c.last_insert_rowid(); - dav_ids::insert(&c, old, CALENDAR_EVENT, "/c/", "/c/1.ics", "", ev).unwrap(); - let (new_key, _) = source(&c, "caldav", "https://new"); - - let removed = remove_other_sources(&c, &new_key, Logger::from_flags(true, 0)).unwrap(); - - assert_eq!((removed.calendars, removed.calendar_events), (1, 1)); - assert_eq!(count(&c, "calendars"), 0); - assert_eq!(count(&c, "calendar_events"), 0); - assert_eq!(count(&c, "sync_id_dav"), 0); - } - - #[test] - fn no_other_source_is_a_no_op() { - let c = mem(); - let (key, _) = source(&c, "webdav", "https://only"); - let removed = remove_other_sources(&c, &key, Logger::from_flags(true, 0)).unwrap(); - assert_eq!(removed, Removed::default()); - assert_eq!(count(&c, "sources"), 1); - } - - #[test] - fn removed_counts_land_on_matching_summary_rows() { - let mut summary = Summary::default(); - summary.per_type.push((FILE_NODE, TypeCounts::default())); - Removed { - file_nodes: 4, - calendars: 9, - ..Removed::default() - } - .add_to(&mut summary); - assert_eq!(summary.per_type.len(), 1); - assert_eq!(summary.per_type[0].1.deleted, 4); - } -} diff --git a/src/sync/import_dav/tree.rs b/src/sync/import_dav/tree.rs deleted file mode 100644 index 1984f06..0000000 --- a/src/sync/import_dav/tree.rs +++ /dev/null @@ -1,681 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::{HashSet, VecDeque}; -use std::io::Read; - -use rusqlite::{Connection, OptionalExtension, params}; -use time::format_description::well_known::Rfc3339; - -use crate::dav::client::DavClient; -use crate::dav::discover::DiscoveredCollection; -use crate::dav::href::{Href, join_absolute, last_path_component}; -use crate::dav::parse::DavResponse; -use crate::dav::xml; -use crate::db; -use crate::db::dav_ids; -use crate::error::Error; -use crate::jmap::error::JmapError; -use crate::logging::{LEVEL_PROGRESS, Logger}; -use crate::sync::TypeCounts; -use crate::sync::import_jmap::pool::Pool; - -struct FilePlan { - item_href: String, - parent_href: String, - parent_local: Option, - existing_local: Option, - propfind_etag: String, - propfind_content_type: Option, - propfind_last_modified: Option, - propfind_creation_date: Option, - propfind_displayname: Option, - url: String, -} - -struct FileFetch { - plan: FilePlan, -} - -struct FileFetched { - plan: FilePlan, - result: Result, -} - -struct FetchedBody { - status: u16, - bytes: Vec, - etag: Option, - content_type: Option, - last_modified: Option, -} - -pub struct WebDavCtx<'a> { - pub client: &'a DavClient, - pub source_id: i64, - pub base_url: &'a str, - pub dav_connections: usize, - pub logger: Logger, -} - -pub fn reconcile_filenodes( - conn: &mut Connection, - ctx: &WebDavCtx<'_>, - root: &DiscoveredCollection, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let client = ctx.client; - let source_id = ctx.source_id; - let base_url = ctx.base_url; - let dav_connections = ctx.dav_connections; - let logger = ctx.logger; - let absolute_root = absolute(base_url, root.href.as_str())?; - - let known: Vec<(String, i64)> = - dav_ids::collections_of_type(conn, source_id, dav_ids::FILE_NODE) - .map_err(|e| Error::Partial(e.to_string()))? - .into_iter() - .collect(); - let mut seen: HashSet = HashSet::new(); - seen.insert(root.href.as_str().to_owned()); - - let mut visited: HashSet = HashSet::new(); - visited.insert(root.href.as_str().to_owned()); - let mut queue: VecDeque<(String, String, Option)> = VecDeque::new(); - queue.push_back((absolute_root, root.href.as_str().to_owned(), None)); - - let mut file_plans: Vec = Vec::new(); - - while let Some((url, parent_href, parent_local)) = queue.pop_front() { - match walk_one( - conn, - client, - source_id, - WalkPos { - url: &url, - parent_href: &parent_href, - parent_local, - }, - WalkState { - counts, - seen: &mut seen, - file_plans: &mut file_plans, - }, - logger, - ) { - Ok(children) => { - for (child_url, child_href, child_local) in children { - if visited.insert(child_href.clone()) { - queue.push_back((child_url, child_href, child_local)); - } else { - logger.warn(&format!("cycle detected at {child_href}; not recursing")); - } - } - } - Err(e) if e.aborts_run() => return Err(e), - Err(e) => { - logger.warn(&format!("PROPFIND {url}: {e}")); - counts.failed += 1; - } - } - } - - fetch_files_parallel( - conn, - client, - source_id, - dav_connections, - file_plans, - counts, - logger, - )?; - - delete_vanished(conn, source_id, &known, &seen, counts, logger)?; - - Ok(()) -} - -fn fetch_files_parallel( - conn: &mut Connection, - client: &DavClient, - source_id: i64, - dav_connections: usize, - plans: Vec, - counts: &mut TypeCounts, - logger: Logger, -) -> Result<(), Error> { - if plans.is_empty() { - return Ok(()); - } - let workers = dav_connections.clamp(1, 8); - let client_for_pool = client.clone(); - let pool: Pool = Pool::new(workers, move |job: FileFetch| { - let result = match client_for_pool.get_stream(&job.plan.url) { - Ok(mut s) => { - let mut bytes = Vec::new(); - match s.read_to_end(&mut bytes) { - Ok(_) => Ok(FetchedBody { - status: s.status, - bytes, - etag: s.etag, - content_type: s.content_type, - last_modified: s.last_modified, - }), - Err(e) => Err(JmapError::Transport(format!("read body: {e}"))), - } - } - Err(e) => Err(e), - }; - FileFetched { - plan: job.plan, - result, - } - }); - - let mut submitted: usize = 0; - for plan in plans { - pool.submit(FileFetch { plan }); - submitted += 1; - } - - const FILE_COMMIT_BATCH: usize = 16; - let mut tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - let mut pending: usize = 0; - for _ in 0..submitted { - let fetched = match pool.results().recv() { - Ok(r) => r, - Err(_) => break, - }; - match fetched.result { - Err(e) => { - logger.warn(&format!("file {}: {e}", fetched.plan.item_href)); - counts.failed += 1; - } - Ok(body) if body.status == 404 || body.status == 410 => {} - Ok(body) if body.status >= 400 => { - logger.warn(&format!( - "file {}: http {}", - fetched.plan.item_href, body.status - )); - counts.failed += 1; - } - Ok(body) => match commit_file(&tx, source_id, &fetched.plan, body, logger) { - Ok(()) => { - counts.fetched += 1; - pending += 1; - if pending >= FILE_COMMIT_BATCH { - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - pending = 0; - } - } - Err(e) => { - logger.warn(&format!("file {}: {e}", fetched.plan.item_href)); - counts.failed += 1; - } - }, - } - } - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - drop(pool); - Ok(()) -} - -fn commit_file( - tx: &rusqlite::Transaction<'_>, - source_id: i64, - plan: &FilePlan, - body: FetchedBody, - logger: Logger, -) -> Result<(), Error> { - let etag = body - .etag - .clone() - .unwrap_or_else(|| plan.propfind_etag.clone()); - let content_type = body - .content_type - .or_else(|| plan.propfind_content_type.clone()); - let last_modified = normalise_dav_date( - &body - .last_modified - .or_else(|| plan.propfind_last_modified.clone()), - ); - let created = format_or_now(&plan.propfind_creation_date)?; - let name = display_or_basename(plan, logger); - - let blob_local = - db::blobs::intern_blob(tx, &body.bytes).map_err(|e| Error::Partial(e.to_string()))?; - if let Some(local) = plan.existing_local { - tx.execute( - "UPDATE file_nodes SET parent_id = ?1, node_type = 'file', blob_id = ?2, - target = NULL, name = ?3, media_type = ?4, - created = ?5, modified = ?6 - WHERE id = ?7", - params![ - plan.parent_local, - blob_local, - name, - content_type, - created, - last_modified, - local, - ], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - dav_ids::update_etag(tx, source_id, dav_ids::FILE_NODE, &plan.item_href, &etag) - .map_err(|e| Error::Partial(e.to_string()))?; - } else { - tx.execute( - "INSERT INTO file_nodes (parent_id, node_type, blob_id, target, name, media_type, - created, modified, is_subscribed, role) - VALUES (?1, 'file', ?2, NULL, ?3, ?4, ?5, ?6, 1, NULL)", - params![ - plan.parent_local, - blob_local, - name, - content_type, - created, - last_modified, - ], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - let local = tx.last_insert_rowid(); - dav_ids::insert( - tx, - source_id, - dav_ids::FILE_NODE, - &plan.parent_href, - &plan.item_href, - &etag, - local, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - } - Ok(()) -} - -fn display_or_basename(plan: &FilePlan, logger: Logger) -> String { - let href = Href::from_normalised(plan.item_href.clone()); - let basename = decoded_basename(&href); - if let Some(n) = &plan.propfind_displayname - && !n.trim().is_empty() - && n.trim() != basename - && logger.enabled(crate::logging::LEVEL_BODIES) - { - eprintln!( - "webdav: {} displayname={:?} differs from basename={basename:?}; using basename", - plan.item_href, n - ); - } - basename -} - -fn delete_vanished( - conn: &mut Connection, - source_id: i64, - known: &[(String, i64)], - seen: &HashSet, - counts: &mut TypeCounts, - logger: Logger, -) -> Result<(), Error> { - let mut vanished: Vec<(String, i64)> = known - .iter() - .filter(|(href, _)| !seen.contains(href)) - .cloned() - .collect(); - if vanished.is_empty() { - return Ok(()); - } - vanished.sort_by(|a, b| { - let depth_a = a.0.matches('/').count(); - let depth_b = b.0.matches('/').count(); - depth_b.cmp(&depth_a).then_with(|| a.0.cmp(&b.0)) - }); - - let tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - for (href, local_id) in &vanished { - if let Err(e) = tx.execute("DELETE FROM file_nodes WHERE id = ?1", params![local_id]) { - logger.warn(&format!( - "vanished file_node {href:?} delete failed: {e}; skipping" - )); - counts.failed += 1; - continue; - } - if let Err(e) = dav_ids::delete_item(&tx, source_id, dav_ids::FILE_NODE, href) { - logger.warn(&format!( - "vanished file_node {href:?} sync_id delete failed: {e}" - )); - counts.failed += 1; - continue; - } - counts.deleted += 1; - } - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - if logger.enabled(LEVEL_PROGRESS) { - eprintln!("webdav: deleted {} vanished file_nodes", vanished.len()); - } - Ok(()) -} - -struct WalkPos<'a> { - url: &'a str, - parent_href: &'a str, - parent_local: Option, -} - -struct WalkState<'a> { - counts: &'a mut TypeCounts, - seen: &'a mut HashSet, - file_plans: &'a mut Vec, -} - -fn walk_one( - conn: &mut Connection, - client: &DavClient, - source_id: i64, - pos: WalkPos<'_>, - state: WalkState<'_>, - logger: Logger, -) -> Result)>, Error> { - let url = pos.url; - let parent_href = pos.parent_href; - let parent_local = pos.parent_local; - let counts = state.counts; - let seen = state.seen; - let file_plans = state.file_plans; - let body = xml::propfind_webdav_listing(); - let ms = client - .propfind_responses(url, 1, &body, url) - .map_err(super::per_collection_failure)?; - if ms.status >= 400 { - return Err(Error::Partial(format!("http {}", ms.status))); - } - let parent_norm = crate::dav::href::normalise(url, "") - .map(|h| h.into_string()) - .unwrap_or_default(); - let mut children = Vec::new(); - for r in ms.responses { - if r.href.as_str() == parent_norm || r.href.as_str() == parent_href { - continue; - } - seen.insert(r.href.as_str().to_owned()); - if r.props.is_collection { - let local = upsert_directory(conn, source_id, &r, parent_local, counts)?; - let abs = absolute(url, r.href.as_str())?; - children.push((abs, r.href.as_str().to_owned(), Some(local))); - } else { - match plan_file(conn, source_id, &r, url, parent_local, parent_href) { - Ok(Some(plan)) => file_plans.push(plan), - Ok(None) => {} - Err(e) => { - logger.warn(&format!("file {}: {e}", r.href.as_str())); - counts.failed += 1; - } - } - } - } - if logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "dir {url}: queued {} files (fetched-so-far={} failed-so-far={})", - file_plans.len(), - counts.fetched, - counts.failed - ); - } - Ok(children) -} - -fn plan_file( - conn: &Connection, - source_id: i64, - response: &DavResponse, - collection_url: &str, - parent_local: Option, - parent_href: &str, -) -> Result, Error> { - let item_href = response.href.as_str().to_owned(); - let propfind_etag = response.props.etag.clone().unwrap_or_default(); - let existing = dav_ids::local_for_item(conn, source_id, dav_ids::FILE_NODE, &item_href) - .map_err(|e| Error::Partial(e.to_string()))?; - if let Some(local) = existing { - let stored_etag: Option = conn - .query_row( - "SELECT etag FROM sync_id_dav - WHERE source_id = ?1 AND type_name = ?2 AND item_href = ?3", - params![source_id, dav_ids::FILE_NODE, item_href], - |r| r.get(0), - ) - .optional() - .map_err(|e| Error::Partial(e.to_string()))?; - if let Some(stored) = stored_etag.as_deref() - && !propfind_etag.is_empty() - && stored == propfind_etag - { - return Ok(None); - } - let url = absolute(collection_url, &item_href)?; - Ok(Some(FilePlan { - item_href, - parent_href: parent_href.to_owned(), - parent_local, - existing_local: Some(local), - propfind_etag, - propfind_content_type: response.props.content_type.clone(), - propfind_last_modified: response.props.last_modified.clone(), - propfind_creation_date: response.props.creation_date.clone(), - propfind_displayname: response.props.displayname.clone(), - url, - })) - } else { - let url = absolute(collection_url, &item_href)?; - Ok(Some(FilePlan { - item_href, - parent_href: parent_href.to_owned(), - parent_local, - existing_local: None, - propfind_etag, - propfind_content_type: response.props.content_type.clone(), - propfind_last_modified: response.props.last_modified.clone(), - propfind_creation_date: response.props.creation_date.clone(), - propfind_displayname: response.props.displayname.clone(), - url, - })) - } -} - -fn upsert_directory( - conn: &mut Connection, - source_id: i64, - response: &DavResponse, - parent_local: Option, - counts: &mut TypeCounts, -) -> Result { - let item_href = response.href.as_str().to_owned(); - let name = display_or_path(&response.props.displayname, &response.href); - let modified = normalise_dav_date(&response.props.last_modified); - let existing = dav_ids::local_for_item(conn, source_id, dav_ids::FILE_NODE, &item_href) - .map_err(|e| Error::Partial(e.to_string()))?; - let tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - if let Some(local) = existing { - tx.execute( - "UPDATE file_nodes SET parent_id = ?1, name = ?2, modified = ?3 - WHERE id = ?4", - params![parent_local, name, modified, local], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - counts.fetched += 1; - Ok(local) - } else { - let created = format_or_now(&response.props.creation_date)?; - tx.execute( - "INSERT INTO file_nodes (parent_id, node_type, blob_id, target, name, media_type, - created, modified, is_subscribed, role) - VALUES (?1, 'directory', NULL, NULL, ?2, NULL, ?3, ?4, 1, NULL)", - params![parent_local, name, created, modified], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - let local = tx.last_insert_rowid(); - let parent_href = parent_href_string(&response.href); - dav_ids::insert( - &tx, - source_id, - dav_ids::FILE_NODE, - &parent_href, - &item_href, - "", - local, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - counts.created += 1; - Ok(local) - } -} - -pub(super) fn parent_href_string(href: &Href) -> String { - let s = href.as_str(); - let trimmed = s.trim_end_matches('/'); - if let Some(slash) = trimmed.rfind('/') { - s[..=slash].to_owned() - } else { - "/".to_owned() - } -} - -fn absolute(base: &str, href: &str) -> Result { - join_absolute(base, href).map_err(|e| Error::Partial(e.to_string())) -} - -fn display_or_path(name: &Option, href: &Href) -> String { - if let Some(n) = name - && !n.trim().is_empty() - { - return n.clone(); - } - decoded_basename(href) -} - -fn decoded_basename(href: &Href) -> String { - let c = last_path_component(href); - if c.is_empty() { "_".to_owned() } else { c } -} - -fn normalise_dav_date(input: &Option) -> Option { - let raw = input.as_deref()?.trim(); - if raw.is_empty() { - return None; - } - if let Ok(t) = time::OffsetDateTime::parse(raw, &Rfc3339) { - return t.format(&Rfc3339).ok(); - } - let imf_fixdate = time::macros::format_description!( - "[weekday repr:short], [day] [month repr:short] [year] \ - [hour]:[minute]:[second] GMT" - ); - if let Ok(p) = time::PrimitiveDateTime::parse(raw, imf_fixdate) { - return p.assume_utc().format(&Rfc3339).ok(); - } - None -} - -fn format_or_now(input: &Option) -> Result { - if let Some(s) = normalise_dav_date(input) { - return Ok(s); - } - time::OffsetDateTime::now_utc() - .format(&Rfc3339) - .map_err(|e| Error::Partial(format!("clock: {e}"))) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parent_href_string_of_root_is_root() { - let h = Href::from_normalised("/".to_owned()); - assert_eq!(parent_href_string(&h), "/"); - } - - #[test] - fn parent_href_string_of_dir_strips_trailing_segment() { - let h = Href::from_normalised("/dav/file/u/work/".to_owned()); - assert_eq!(parent_href_string(&h), "/dav/file/u/"); - } - - #[test] - fn decoded_basename_handles_percent_encoding() { - let h = Href::from_normalised("/dav/file/u/work/note%201.txt".to_owned()); - assert_eq!(decoded_basename(&h), "note 1.txt"); - } - - fn plan_with(displayname: Option<&str>, href: &str) -> FilePlan { - FilePlan { - item_href: href.to_owned(), - parent_href: "/dav/file/u/".to_owned(), - parent_local: Some(1), - existing_local: None, - propfind_etag: String::new(), - propfind_content_type: None, - propfind_last_modified: None, - propfind_creation_date: None, - propfind_displayname: displayname.map(str::to_owned), - url: format!("https://x{href}"), - } - } - - #[test] - fn display_or_basename_prefers_url_basename_over_displayname() { - let plan = plan_with(Some("Pretty Name.txt"), "/dav/file/u/photo.jpg"); - let logger = Logger::from_flags(false, 0); - assert_eq!(display_or_basename(&plan, logger), "photo.jpg"); - } - - #[test] - fn display_or_basename_returns_basename_when_no_displayname() { - let plan = plan_with(None, "/dav/file/u/photo.jpg"); - let logger = Logger::from_flags(false, 0); - assert_eq!(display_or_basename(&plan, logger), "photo.jpg"); - } - - #[test] - fn normalise_dav_date_parses_imf_fixdate() { - let n = normalise_dav_date(&Some("Wed, 27 May 2026 07:59:07 GMT".to_owned())) - .expect("imf parse"); - assert!(n.starts_with("2026-05-27T07:59:07")); - } - - #[test] - fn normalise_dav_date_returns_none_on_garbage() { - assert!(normalise_dav_date(&Some("not a date".to_owned())).is_none()); - assert!(normalise_dav_date(&None).is_none()); - } - - #[test] - fn visited_hashset_breaks_recursion_on_cyclic_listing() { - let mut visited: HashSet = HashSet::new(); - visited.insert("/files/u/".to_owned()); - assert!(visited.insert("/files/u/a/".to_owned())); - assert!(visited.insert("/files/u/a/b/".to_owned())); - assert!( - !visited.insert("/files/u/".to_owned()), - "cycle back to root not re-recursed" - ); - assert!( - !visited.insert("/files/u/a/".to_owned()), - "cycle back to a/ not re-recursed" - ); - } -} diff --git a/src/sync/import_exchange_ews/attachments.rs b/src/sync/import_exchange_ews/attachments.rs deleted file mode 100644 index 6cd4ae0..0000000 --- a/src/sync/import_exchange_ews/attachments.rs +++ /dev/null @@ -1,79 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use encodify::base64::LENIENT; - -use crate::db::blobs; -use crate::error::Error; -use crate::exchange_ews::error::EwsError; -use crate::exchange_ews::parse::parse_get_attachment_inline; -use crate::exchange_ews::xml::get_attachment_body; - -use super::items::ItemRunCtx; - -#[derive(Debug, Clone)] -pub struct FetchedAttachment { - pub attachment_id: String, - pub bytes: Vec, - pub media_type: String, -} - -pub fn fetch_contact_photo( - conn: &rusqlite::Connection, - ctx: &ItemRunCtx<'_>, - attachment_id: &str, -) -> Result, Error> { - let Some(att) = fetch_attachment_bytes(ctx, attachment_id).map_err(Error::from)? else { - return Ok(None); - }; - let blob_id = intern_attachment(conn, &att.bytes)?; - Ok(Some((blob_id, att.media_type))) -} - -pub fn fetch_attachment_bytes( - ctx: &ItemRunCtx<'_>, - attachment_id: &str, -) -> Result, EwsError> { - let mut out = fetch_attachments(ctx, &[attachment_id])?; - Ok(out.pop()) -} - -pub fn fetch_attachments( - ctx: &ItemRunCtx<'_>, - attachment_ids: &[&str], -) -> Result, EwsError> { - if attachment_ids.is_empty() { - return Ok(Vec::new()); - } - let batch = ctx.attachment_batch.max(1); - let mut out: Vec = Vec::with_capacity(attachment_ids.len()); - for chunk in attachment_ids.chunks(batch) { - let body = get_attachment_body(chunk); - let resp = ctx.client.call(ctx.url, "GetAttachment", &body)?; - let inline = parse_get_attachment_inline(&resp.body)?; - for att in inline { - let bytes = LENIENT.decode(&att.content_base64).map_err(|e| { - EwsError::Malformed(format!("attachment {}: base64: {e}", att.attachment_id)) - })?; - if bytes.is_empty() { - continue; - } - let media_type = att - .content_type - .unwrap_or_else(|| "application/octet-stream".to_owned()); - out.push(FetchedAttachment { - attachment_id: att.attachment_id, - bytes, - media_type, - }); - } - } - Ok(out) -} - -pub fn intern_attachment(conn: &rusqlite::Connection, bytes: &[u8]) -> Result { - blobs::intern_blob(conn, bytes).map_err(|e| Error::Partial(e.to_string())) -} diff --git a/src/sync/import_exchange_ews/calendar.rs b/src/sync/import_exchange_ews/calendar.rs deleted file mode 100644 index fae845e..0000000 --- a/src/sync/import_exchange_ews/calendar.rs +++ /dev/null @@ -1,333 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use rusqlite::{Connection, params}; -use serde_json::Value; - -use crate::db::exchange_ews_ids; -use crate::error::Error; -use crate::exchange::date::is_outlook_no_date; -use crate::exchange_ews::calendar_map::to_jscalendar_with_exceptions; -use crate::exchange_ews::parse::{CalendarItemRaw, parse_calendar_item}; -use crate::exchange_ews::types::{CalendarItemType, ItemId}; -use crate::exchange_ews::xml::ItemShape; -use crate::logging::LEVEL_PROGRESS; -use crate::sync::TypeCounts; - -use super::attachments::{fetch_attachments, intern_attachment}; -use super::folders::FolderPlan; -use super::items::{ - EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, for_each_fetched_item, - get_items, plan_for, -}; - -pub fn reconcile_all( - conn: &mut Connection, - ctx: &ItemRunCtx<'_>, - plan: &FolderPlan, - counts: &mut TypeCounts, -) -> Result<(), Error> { - for folder in &plan.calendar { - let folder_id = &folder.folder.folder_id; - let local_folder_id = match exchange_ews_ids::local_for_item( - conn, - ctx.source_id, - exchange_ews_ids::CALENDAR, - &folder_id.id, - ) - .map_err(|e| Error::Partial(e.to_string()))? - { - Some(id) => id, - None => continue, - }; - if let Err(e) = reconcile_one( - conn, - ctx, - folder_id, - &folder.folder.display_name, - local_folder_id, - counts, - ) { - ctx.logger.warn(&format!( - "calendar folder {:?} failed: {}", - folder.folder.display_name, e - )); - counts.failed += 1; - } - } - Ok(()) -} - -fn reconcile_one( - conn: &mut Connection, - ctx: &ItemRunCtx<'_>, - folder: &crate::exchange_ews::types::FolderId, - folder_name: &str, - local_folder_id: i64, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let prior = exchange_ews_ids::get_sync_state( - conn, - ctx.source_id, - exchange_ews_ids::CALENDAR, - &folder.id, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - let mut outcome = enumerate_folder(ctx, folder, prior.as_deref()).map_err(Error::from)?; - outcome.items.retain(|s| { - matches!( - s.element.to_ascii_lowercase().as_str(), - "calendaritem" | "item" - ) - }); - let local = exchange_ews_ids::items_in_folder( - conn, - ctx.source_id, - exchange_ews_ids::CALENDAR_EVENT, - &folder.id, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - let plan = plan_for(&outcome, &local); - if ctx.logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "EWS calendar folder {:?}: new={} changed={} vanished={}", - folder_name, - plan.new.len(), - plan.present_changed.len(), - plan.vanished.len() - ); - } - let mut to_fetch: Vec = plan.new.clone(); - for (id, _) in &plan.present_changed { - to_fetch.push(id.clone()); - } - if !to_fetch.is_empty() { - let failed_items = for_each_fetched_item(ctx, ItemShape::CalendarItem, &to_fetch, |msg| { - if !msg.success { - if matches!( - msg.response_code, - crate::exchange_ews::types::ResponseCode::ItemNotFound - ) { - counts.skipped += 1; - } else { - counts.failed += 1; - ctx.logger - .warn(&format!("GetItem (calendar) error: {}", msg.response_code)); - } - return Ok(()); - } - let parsed = parse_calendar_item(&msg.inner_xml) - .map_err(|e| Error::Partial(format!("unparsable item: {e}")))?; - if parsed.id.id.is_empty() { - counts.failed += 1; - return Ok(()); - } - if matches!( - parsed.calendar_item_type, - Some(CalendarItemType::Occurrence) | Some(CalendarItemType::Exception) - ) || parsed.start.as_deref().is_some_and(is_outlook_no_date) - { - counts.skipped += 1; - return Ok(()); - } - let existing = plan - .present_changed - .iter() - .find(|(id, _)| id.id == parsed.id.id) - .map(|(_, local)| *local); - apply_event( - conn, - ctx, - &parsed, - local_folder_id, - &folder.id, - existing, - counts, - ) - })?; - counts.failed += failed_items; - } - delete_vanished( - conn, - ctx.source_id, - exchange_ews_ids::CALENDAR_EVENT, - "calendar_events", - &plan.vanished, - counts, - )?; - if let EnumerationMode::Delta { new_sync_state, .. } = &outcome.mode { - exchange_ews_ids::set_sync_state( - conn, - ctx.source_id, - exchange_ews_ids::CALENDAR, - &folder.id, - new_sync_state, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - } - Ok(()) -} - -fn apply_event( - conn: &mut Connection, - ctx: &ItemRunCtx<'_>, - raw: &CalendarItemRaw, - local_folder_id: i64, - folder_id: &str, - existing_local_id: Option, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let modified_full = fetch_modified_occurrences(ctx, raw); - let mut built = to_jscalendar_with_exceptions(raw, &modified_full, ctx.owner_smtp); - attach_calendar_links(conn, ctx, raw, &mut built.data)?; - let calendar_ids = serde_json::json!([local_folder_id]).to_string(); - let data = built.data.to_string(); - let is_draft = if built.is_draft { 1 } else { 0 }; - let use_default_alerts = if built.use_default_alerts { 1 } else { 0 }; - - let tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - match existing_local_id { - Some(id) => { - tx.execute( - "UPDATE calendar_events SET calendar_ids = ?1, is_draft = ?2, \ - use_default_alerts = ?3, data = ?4, data_type = 'Event' WHERE id = ?5", - params![calendar_ids, is_draft, use_default_alerts, data, id], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - exchange_ews_ids::update_change_key( - &tx, - ctx.source_id, - exchange_ews_ids::CALENDAR_EVENT, - &raw.id.id, - &raw.id.change_key, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - counts.fetched += 1; - } - None => { - tx.execute( - "INSERT INTO calendar_events (calendar_ids, is_draft, use_default_alerts, data, data_type) \ - VALUES (?1, ?2, ?3, ?4, 'Event')", - params![calendar_ids, is_draft, use_default_alerts, data], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - let new_id = tx.last_insert_rowid(); - exchange_ews_ids::insert( - &tx, - ctx.source_id, - exchange_ews_ids::CALENDAR_EVENT, - folder_id, - &raw.id.id, - &raw.id.change_key, - new_id, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - counts.created += 1; - } - } - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - Ok(()) -} - -fn fetch_modified_occurrences(ctx: &ItemRunCtx<'_>, raw: &CalendarItemRaw) -> Vec { - let ids: Vec = raw - .modified_occurrences - .iter() - .filter(|o| !o.item_id.id.is_empty()) - .map(|o| o.item_id.clone()) - .collect(); - if ids.is_empty() { - return Vec::new(); - } - match get_items(ctx, ItemShape::CalendarItem, &ids) { - Ok(outcome) => outcome - .messages - .iter() - .filter(|m| m.success) - .filter_map(|m| parse_calendar_item(&m.inner_xml).ok()) - .filter(|occ| !occ.id.id.is_empty()) - .collect(), - Err(e) => { - ctx.logger.warn(&format!( - "GetItem (calendar exceptions) failed: {e}; overrides keep time-only changes" - )); - Vec::new() - } - } -} - -fn attach_calendar_links( - conn: &mut Connection, - ctx: &ItemRunCtx<'_>, - raw: &CalendarItemRaw, - data: &mut Value, -) -> Result<(), Error> { - let refs: Vec<_> = raw - .attachments - .iter() - .filter(|a| !a.is_item_attachment && !a.attachment_id.is_empty()) - .collect(); - if refs.is_empty() { - return Ok(()); - } - let ids: Vec<&str> = refs.iter().map(|a| a.attachment_id.as_str()).collect(); - let fetched = match fetch_attachments(ctx, &ids) { - Ok(f) => f, - Err(e) => { - ctx.logger.warn(&format!( - "calendar attachments fetch failed: {e}; continuing without enclosures" - )); - return Ok(()); - } - }; - let mut links = serde_json::Map::new(); - for (idx, att) in (1u32..).zip(fetched) { - let blob_id = intern_attachment(conn, &att.bytes)?; - let declared = refs - .iter() - .find(|r| r.attachment_id == att.attachment_id) - .and_then(|r| r.content_type.as_deref()); - let name = refs - .iter() - .find(|r| r.attachment_id == att.attachment_id) - .and_then(|r| r.name.clone()); - let key = idx.to_string(); - let mut entry = serde_json::Map::new(); - entry.insert("@type".to_owned(), Value::String("Link".to_owned())); - entry.insert("@blob".to_owned(), Value::from(blob_id)); - entry.insert( - "contentType".to_owned(), - Value::String(content_type_or_from(declared, &att.media_type)), - ); - if let Some(n) = name { - entry.insert("title".to_owned(), Value::String(n)); - } - entry.insert("rel".to_owned(), Value::String("enclosure".to_owned())); - links.insert(key, Value::Object(entry)); - } - if links.is_empty() { - return Ok(()); - } - if let Some(obj) = data.as_object_mut() { - if let Some(existing) = obj.get_mut("links").and_then(Value::as_object_mut) { - for (k, v) in links { - existing.insert(k, v); - } - } else { - obj.insert("links".to_owned(), Value::Object(links)); - } - } - Ok(()) -} - -fn content_type_or_from(declared: Option<&str>, fallback: &str) -> String { - declared - .filter(|s| !s.is_empty()) - .map(str::to_owned) - .unwrap_or_else(|| fallback.to_owned()) -} diff --git a/src/sync/import_exchange_ews/contacts.rs b/src/sync/import_exchange_ews/contacts.rs deleted file mode 100644 index 6f672da..0000000 --- a/src/sync/import_exchange_ews/contacts.rs +++ /dev/null @@ -1,237 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use rusqlite::{Connection, params}; - -use crate::db::exchange_ews_ids; -use crate::error::Error; -use crate::exchange_ews::contact_map::{synthetic_uid, to_jscontact}; -use crate::exchange_ews::parse::{ContactItemRaw, parse_contact_item}; -use crate::exchange_ews::types::ItemId; -use crate::exchange_ews::xml::ItemShape; -use crate::logging::LEVEL_PROGRESS; -use crate::sync::TypeCounts; - -use super::attachments::fetch_contact_photo; -use super::folders::FolderPlan; -use super::items::{ - EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, for_each_fetched_item, plan_for, -}; - -pub fn reconcile_all( - conn: &mut Connection, - ctx: &ItemRunCtx<'_>, - plan: &FolderPlan, - counts: &mut TypeCounts, -) -> Result<(), Error> { - for folder in &plan.contacts { - let folder_id = &folder.folder.folder_id; - let local_folder_id = match exchange_ews_ids::local_for_item( - conn, - ctx.source_id, - exchange_ews_ids::ADDRESS_BOOK, - &folder_id.id, - ) - .map_err(|e| Error::Partial(e.to_string()))? - { - Some(id) => id, - None => continue, - }; - if let Err(e) = reconcile_one( - conn, - ctx, - folder_id, - &folder.folder.display_name, - local_folder_id, - counts, - ) { - ctx.logger.warn(&format!( - "contact folder {:?} failed: {}", - folder.folder.display_name, e - )); - counts.failed += 1; - } - } - Ok(()) -} - -fn reconcile_one( - conn: &mut Connection, - ctx: &ItemRunCtx<'_>, - folder: &crate::exchange_ews::types::FolderId, - folder_name: &str, - local_folder_id: i64, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let prior = exchange_ews_ids::get_sync_state( - conn, - ctx.source_id, - exchange_ews_ids::ADDRESS_BOOK, - &folder.id, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - let outcome = enumerate_folder(ctx, folder, prior.as_deref()).map_err(Error::from)?; - let local = exchange_ews_ids::items_in_folder( - conn, - ctx.source_id, - exchange_ews_ids::CONTACT_CARD, - &folder.id, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - let plan = plan_for(&outcome, &local); - if ctx.logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "EWS contacts folder {:?}: new={} changed={} vanished={}", - folder_name, - plan.new.len(), - plan.present_changed.len(), - plan.vanished.len() - ); - } - let mut to_fetch: Vec = plan.new.clone(); - for (id, _) in &plan.present_changed { - to_fetch.push(id.clone()); - } - if !to_fetch.is_empty() { - let failed_items = for_each_fetched_item(ctx, ItemShape::Contact, &to_fetch, |msg| { - if !msg.success { - if matches!( - msg.response_code, - crate::exchange_ews::types::ResponseCode::ItemNotFound - ) { - counts.skipped += 1; - } else { - counts.failed += 1; - ctx.logger - .warn(&format!("GetItem (contact) error: {}", msg.response_code)); - } - return Ok(()); - } - let parsed = parse_contact_item(&msg.inner_xml) - .map_err(|e| Error::Partial(format!("unparsable item: {e}")))?; - if parsed.id.id.is_empty() { - counts.failed += 1; - return Ok(()); - } - let existing = plan - .present_changed - .iter() - .find(|(id, _)| id.id == parsed.id.id) - .map(|(_, local)| *local); - apply_contact( - conn, - ctx, - &parsed, - local_folder_id, - &folder.id, - existing, - counts, - ) - })?; - counts.failed += failed_items; - } - delete_vanished( - conn, - ctx.source_id, - exchange_ews_ids::CONTACT_CARD, - "contact_cards", - &plan.vanished, - counts, - )?; - if let EnumerationMode::Delta { new_sync_state, .. } = &outcome.mode { - exchange_ews_ids::set_sync_state( - conn, - ctx.source_id, - exchange_ews_ids::ADDRESS_BOOK, - &folder.id, - new_sync_state, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - } - Ok(()) -} - -fn apply_contact( - conn: &mut Connection, - ctx: &ItemRunCtx<'_>, - item: &ContactItemRaw, - local_folder_id: i64, - folder_id: &str, - existing_local_id: Option, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let mut card = to_jscontact(item); - if let Some(photo_att) = item.attachments.iter().find(|a| a.is_contact_photo) { - match fetch_contact_photo(conn, ctx, &photo_att.attachment_id) { - Ok(Some((blob_id, media_type))) => { - let mut media = serde_json::Map::new(); - media.insert( - "photo".to_owned(), - serde_json::json!({ - "@type": "Media", - "kind": "photo", - "@blob": blob_id, - "mediaType": media_type, - }), - ); - if let Some(m) = card.as_object_mut() { - m.insert("media".to_owned(), serde_json::Value::Object(media)); - } - } - Ok(None) => {} - Err(e) => { - ctx.logger - .warn(&format!("contact photo fetch failed: {e}; continuing")); - } - } - } - let uid = synthetic_uid(&item.id.id); - let address_book_ids = serde_json::json!([local_folder_id]).to_string(); - let data = card.to_string(); - - let tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - match existing_local_id { - Some(id) => { - tx.execute( - "UPDATE contact_cards SET uid = ?1, address_book_ids = ?2, data = ?3 WHERE id = ?4", - params![uid, address_book_ids, data, id], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - exchange_ews_ids::update_change_key( - &tx, - ctx.source_id, - exchange_ews_ids::CONTACT_CARD, - &item.id.id, - &item.id.change_key, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - counts.fetched += 1; - } - None => { - tx.execute( - "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)", - params![uid, address_book_ids, data], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - let new_id = tx.last_insert_rowid(); - exchange_ews_ids::insert( - &tx, - ctx.source_id, - exchange_ews_ids::CONTACT_CARD, - folder_id, - &item.id.id, - &item.id.change_key, - new_id, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - counts.created += 1; - } - } - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - Ok(()) -} diff --git a/src/sync/import_exchange_ews/coordinator.rs b/src/sync/import_exchange_ews/coordinator.rs deleted file mode 100644 index cf5d26f..0000000 --- a/src/sync/import_exchange_ews/coordinator.rs +++ /dev/null @@ -1,541 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use rusqlite::Connection; - -use crate::db; -use crate::db::sources::SourceKey; -use crate::error::Error; -use crate::exchange_ews::EwsClient; -use crate::exchange_ews::autodiscover::{DiscoveryResult, discover}; -use crate::exchange_ews::oauth::{AcquiredToken, OAuthFlow, acquire}; -use crate::exchange_ews::types::MailboxKind; -use crate::jmap::http::{Auth, RetryPolicy}; -use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS}; -use crate::sync::{CommonConfig, Summary, TypeCounts}; - -use super::folders::{self, plan_folders}; -use super::{calendar, contacts, messages}; - -#[derive(Debug, Clone)] -pub enum EwsAuth { - Basic { user: String, password: String }, - Bearer { token: String }, - OAuth(OAuthFlow), -} - -#[derive(Debug, Clone)] -pub struct EwsImportConfig { - pub url: Option, - pub mailbox: Option, - pub mailbox_kind: MailboxKind, - pub auth: EwsAuth, - pub ews_connections: usize, - pub getitem_batch: usize, - pub attachment_batch: usize, - pub use_syncfolderitems: bool, - pub allow_source_change: bool, -} - -pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result { - let logger = common.logger; - let mut conn = db::init::open(&common.archive)?; - - let (auth, acquired) = resolve_auth(&config.auth, common.allow_invalid_certs)?; - let discovery = run_autodiscover(&config, &acquired, common.allow_invalid_certs)?; - if logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "EWS discovery: url={} source={:?}", - discovery.ews_url, discovery.source - ); - } - let mailbox = resolve_mailbox(&config, &acquired)?; - let account_id = synthetic_account_id(&mailbox, config.mailbox_kind); - let session_url = normalise_ews_url(&discovery.ews_url); - enforce_basic_auth_policy(&config.auth, &session_url)?; - let key = SourceKey { - kind: "exchange_ews".to_owned(), - session_url: session_url.clone(), - account_id: account_id.clone(), - }; - - if !common.dry_run - && let Some((url, acc)) = - db::sources::conflicting_source(&conn, "exchange_ews", &session_url, &account_id)? - && !config.allow_source_change - { - return Err(Error::SourceChange(format!( - "archive already records exchange_ews source ({url}, account {acc}); \ - pass --allow-source-change to import a different account" - ))); - } - - let client = EwsClient::new( - auth, - RetryPolicy::new(common.max_retries), - common.allow_invalid_certs, - ); - client.set_logger(logger); - if matches!(config.mailbox_kind, MailboxKind::PublicFolders) { - client.set_anchor_mailbox(None); - } else { - client.set_anchor_mailbox(Some(mailbox.clone())); - } - if let EwsAuth::OAuth(OAuthFlow::ClientCredentials { .. }) = &config.auth { - client.set_impersonation(Some(mailbox.clone())); - } - spawn_token_refresher( - &client, - &config.auth, - &acquired, - common.allow_invalid_certs, - logger, - ); - - let username = match &config.auth { - EwsAuth::Basic { user, .. } => user.clone(), - EwsAuth::Bearer { .. } | EwsAuth::OAuth(_) => acquired - .as_ref() - .and_then(|a| a.upn.clone()) - .unwrap_or_else(|| mailbox.clone()), - }; - - if common.dry_run { - let summary = run_dry(&conn, &client, &session_url, &config, logger)?; - return Ok(summary); - } - - let account_name = acquired.as_ref().and_then(|a| a.name.clone()); - let source_id = db::sources::upsert_source(&conn, &key, account_name.as_deref(), &username)?; - - let mut summary = Summary::default(); - let mut mailbox_counts = TypeCounts::default(); - let mut calendar_counts = TypeCounts::default(); - let mut addressbook_counts = TypeCounts::default(); - let mut email_counts = TypeCounts::default(); - let mut calendar_event_counts = TypeCounts::default(); - let mut contact_counts = TypeCounts::default(); - - let plan = plan_folders( - &client, - &session_url, - config.mailbox_kind, - config.mailbox.as_deref(), - logger, - ) - .map_err(Error::from)?; - - folders::reconcile( - &mut conn, - source_id, - &plan, - &mut folders::ReconcileCounts { - mailbox: &mut mailbox_counts, - calendar: &mut calendar_counts, - addressbook: &mut addressbook_counts, - email: &mut email_counts, - calendar_event: &mut calendar_event_counts, - contact: &mut contact_counts, - }, - logger, - )?; - - if logger.enabled(LEVEL_DEFAULT) { - eprintln!( - "import: mailbox={} calendar={} addressbook={}", - mailbox_counts.created + mailbox_counts.fetched, - calendar_counts.created + calendar_counts.fetched, - addressbook_counts.created + addressbook_counts.fetched, - ); - } - - let item_ctx = super::items::ItemRunCtx { - client: &client, - url: &session_url, - source_id, - owner_smtp: (!matches!(config.mailbox_kind, MailboxKind::PublicFolders)) - .then_some(mailbox.as_str()), - batch_size: config.getitem_batch.max(1), - attachment_batch: config.attachment_batch.max(1), - connections: config.ews_connections.clamp(1, 8), - use_syncfolderitems: config.use_syncfolderitems, - sync_batch: super::items::SYNC_BATCH_MAX, - logger, - }; - - messages::reconcile_all(&mut conn, &item_ctx, &plan, &mut email_counts)?; - contacts::reconcile_all(&mut conn, &item_ctx, &plan, &mut contact_counts)?; - calendar::reconcile_all(&mut conn, &item_ctx, &plan, &mut calendar_event_counts)?; - - summary.per_type.push(("mailbox", mailbox_counts)); - summary.per_type.push(("calendar", calendar_counts)); - summary.per_type.push(("addressbook", addressbook_counts)); - summary.per_type.push(("email", email_counts)); - summary - .per_type - .push(("calendarevent", calendar_event_counts)); - summary.per_type.push(("contactcard", contact_counts)); - - if !summary.any_failed() - && let Err(e) = run_gc(&conn) - { - logger.warn(&format!("blob GC skipped: {e}")); - } - - summary.retries_observed = client.retries_observed(); - summary.retry_after_sleeps = client.retry_after_sleeps(); - Ok(summary) -} - -fn run_dry( - _conn: &Connection, - client: &EwsClient, - session_url: &str, - config: &EwsImportConfig, - logger: crate::logging::Logger, -) -> Result { - let plan = plan_folders( - client, - session_url, - config.mailbox_kind, - config.mailbox.as_deref(), - logger, - ) - .map_err(Error::from)?; - let mut summary = Summary::default(); - let mut mailbox_counts = TypeCounts::default(); - let mut calendar_counts = TypeCounts::default(); - let mut addressbook_counts = TypeCounts::default(); - mailbox_counts.created = plan.mail.len() as u64; - calendar_counts.created = plan.calendar.len() as u64; - addressbook_counts.created = plan.contacts.len() as u64; - if logger.enabled(LEVEL_DEFAULT) { - eprintln!( - "dry-run: mailbox={} calendar={} addressbook={}", - mailbox_counts.created, calendar_counts.created, addressbook_counts.created - ); - } - summary.per_type.push(("mailbox", mailbox_counts)); - summary.per_type.push(("calendar", calendar_counts)); - summary.per_type.push(("addressbook", addressbook_counts)); - Ok(summary) -} - -fn resolve_auth( - auth: &EwsAuth, - allow_invalid_certs: bool, -) -> Result<(Auth, Option), Error> { - match auth { - EwsAuth::Basic { user, password } => Ok(( - Auth::Basic { - user: user.clone(), - password: password.clone(), - }, - None, - )), - EwsAuth::Bearer { token } => { - let acq = acquire( - &OAuthFlow::PreAcquired { - token: token.clone(), - }, - allow_invalid_certs, - ) - .map_err(Error::from)?; - Ok(( - Auth::Bearer { - token: acq.access_token.clone(), - }, - Some(acq), - )) - } - EwsAuth::OAuth(flow) => { - let acq = acquire(flow, allow_invalid_certs).map_err(Error::from)?; - Ok(( - Auth::Bearer { - token: acq.access_token.clone(), - }, - Some(acq), - )) - } - } -} - -fn run_autodiscover( - config: &EwsImportConfig, - acquired: &Option, - allow_invalid_certs: bool, -) -> Result { - let email = config - .mailbox - .clone() - .or_else(|| acquired.as_ref().and_then(|a| a.upn.clone())); - let result = discover( - config.url.as_deref(), - email.as_deref(), - None, - allow_invalid_certs, - ) - .map_err(Error::from)?; - Ok(result) -} - -fn resolve_mailbox( - config: &EwsImportConfig, - acquired: &Option, -) -> Result { - if let Some(mb) = config.mailbox.as_ref() { - return Ok(mb.clone()); - } - if let Some(acq) = acquired.as_ref() - && let Some(upn) = acq.upn.as_ref() - { - return Ok(upn.clone()); - } - if let EwsAuth::Basic { user, .. } = &config.auth - && user.contains('@') - { - return Ok(user.clone()); - } - Err(Error::Account( - "could not resolve mailbox SMTP; pass --mailbox".to_owned(), - )) -} - -pub fn synthetic_account_id(mailbox: &str, kind: MailboxKind) -> String { - match kind { - MailboxKind::Primary => mailbox.to_owned(), - MailboxKind::Archive => format!("{mailbox}#archive"), - MailboxKind::PublicFolders => { - let domain = mailbox.split('@').nth(1).unwrap_or(""); - format!("__public_folders__@{domain}") - } - } -} - -pub fn normalise_ews_url(raw: &str) -> String { - let Ok(parsed) = url::Url::parse(raw) else { - return raw.to_owned(); - }; - let scheme = parsed.scheme().to_ascii_lowercase(); - let host = parsed.host_str().unwrap_or("").to_ascii_lowercase(); - let default_port = match scheme.as_str() { - "https" => 443, - "http" => 80, - _ => 0, - }; - let port = parsed.port().filter(|p| *p != default_port); - let path = if parsed.path().is_empty() || parsed.path() == "/" { - "/EWS/Exchange.asmx".to_owned() - } else { - let last = parsed.path().rsplit('/').next().unwrap_or(""); - if last.eq_ignore_ascii_case("Exchange.asmx") { - let mut p = String::new(); - let mut parts: Vec<&str> = parsed.path().split('/').collect(); - if let Some(last_part) = parts.last_mut() { - *last_part = "Exchange.asmx"; - } - if let Some(second_last_idx) = parts.len().checked_sub(2) - && parts[second_last_idx].eq_ignore_ascii_case("EWS") - { - parts[second_last_idx] = "EWS"; - } - p.push_str(&parts.join("/")); - p - } else { - parsed.path().to_owned() - } - }; - match port { - Some(p) => format!("{scheme}://{host}:{p}{path}"), - None => format!("{scheme}://{host}{path}"), - } -} - -fn enforce_basic_auth_policy(auth: &EwsAuth, url: &str) -> Result<(), Error> { - if let EwsAuth::Basic { .. } = auth { - let parsed = url::Url::parse(url) - .map_err(|e| Error::Usage(format!("invalid EWS URL {url:?}: {e}")))?; - if parsed.scheme() != "https" { - return Err(Error::Connection( - "EWS requires https:// (basic auth refused on cleartext)".to_owned(), - )); - } - let host = parsed.host_str().unwrap_or("").to_ascii_lowercase(); - let blocked = [ - "outlook.office365.com", - "outlook.office.com", - "outlook.office365.us", - "office365.us", - ]; - if blocked.iter().any(|b| host == *b || host.ends_with(b)) { - return Err(Error::Connection( - "Basic auth is disabled on Exchange Online; use --auth-bearer".to_owned(), - )); - } - } - Ok(()) -} - -fn spawn_token_refresher( - client: &EwsClient, - auth: &EwsAuth, - initial: &Option, - allow_invalid_certs: bool, - logger: crate::logging::Logger, -) { - let flow = match auth { - EwsAuth::OAuth(flow) => flow.clone(), - _ => return, - }; - let exp = initial.as_ref().and_then(|t| { - crate::exchange_ews::oauth::decode_jwt_claims(&t.access_token).and_then(|c| c.exp) - }); - let Some(exp_secs) = exp else { - return; - }; - let client = client.clone(); - let initial_refresh = initial.as_ref().and_then(|t| t.refresh_token.clone()); - std::thread::Builder::new() - .name("inbuxa-migrate-ews-token-refresh".to_owned()) - .spawn(move || { - let mut next_exp = exp_secs; - let mut refresh_token = initial_refresh; - loop { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0); - let refresh_at = next_exp.saturating_sub(5 * 60); - if refresh_at > now { - std::thread::sleep(std::time::Duration::from_secs(refresh_at - now)); - } - let result = if let (Some(rt), OAuthFlow::DeviceCode { tenant, client_id }) = - (refresh_token.as_deref(), &flow) - { - crate::exchange_ews::oauth::refresh_with_token( - tenant, - client_id, - rt, - allow_invalid_certs, - ) - } else { - crate::exchange_ews::oauth::acquire(&flow, allow_invalid_certs) - }; - match result { - Ok(tok) => { - client.set_auth(crate::jmap::http::Auth::Bearer { - token: tok.access_token.clone(), - }); - if tok.refresh_token.is_some() { - refresh_token = tok.refresh_token.clone(); - } - if let Some(new_exp) = - crate::exchange_ews::oauth::decode_jwt_claims(&tok.access_token) - .and_then(|c| c.exp) - { - next_exp = new_exp; - } else { - next_exp = next_exp.saturating_add(50 * 60); - } - } - Err(e) => { - logger.warn(&format!( - "EWS token refresh failed: {e}; sleeping 60s before retry" - )); - std::thread::sleep(std::time::Duration::from_secs(60)); - } - } - } - }) - .ok(); -} - -fn run_gc(conn: &Connection) -> Result<(), Error> { - let tx = conn.unchecked_transaction()?; - db::blobs::gc_orphan_blobs(&tx)?; - tx.commit()?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn synthetic_account_id_uses_smtp_for_primary() { - assert_eq!( - synthetic_account_id("alice@contoso.com", MailboxKind::Primary), - "alice@contoso.com" - ); - } - - #[test] - fn synthetic_account_id_archive_suffix() { - assert_eq!( - synthetic_account_id("alice@contoso.com", MailboxKind::Archive), - "alice@contoso.com#archive" - ); - } - - #[test] - fn synthetic_account_id_public_folders_uses_tenant() { - assert_eq!( - synthetic_account_id("alice@contoso.com", MailboxKind::PublicFolders), - "__public_folders__@contoso.com" - ); - } - - #[test] - fn basic_against_office365_endpoint_is_refused() { - let auth = EwsAuth::Basic { - user: "alice@x".to_owned(), - password: "p".to_owned(), - }; - let res = - enforce_basic_auth_policy(&auth, "https://outlook.office365.com/EWS/Exchange.asmx"); - assert!(matches!(res, Err(Error::Connection(_)))); - } - - #[test] - fn basic_over_https_on_prem_is_allowed() { - let auth = EwsAuth::Basic { - user: "alice@x".to_owned(), - password: "p".to_owned(), - }; - assert!( - enforce_basic_auth_policy(&auth, "https://exchange.example.com/EWS/Exchange.asmx") - .is_ok() - ); - } - - #[test] - fn basic_over_http_is_refused() { - let auth = EwsAuth::Basic { - user: "alice".to_owned(), - password: "p".to_owned(), - }; - assert!( - enforce_basic_auth_policy(&auth, "http://exchange.example.com/EWS/Exchange.asmx") - .is_err() - ); - } - - #[test] - fn normalise_ews_url_lowercases_host_and_keeps_canonical_path() { - assert_eq!( - normalise_ews_url("https://OUTLOOK.OFFICE365.COM/EWS/Exchange.asmx"), - "https://outlook.office365.com/EWS/Exchange.asmx" - ); - assert_eq!( - normalise_ews_url("https://outlook.office365.com:443/ews/exchange.asmx"), - "https://outlook.office365.com/EWS/Exchange.asmx" - ); - assert_eq!( - normalise_ews_url("https://srv.example.com:8443/EWS/Exchange.asmx"), - "https://srv.example.com:8443/EWS/Exchange.asmx" - ); - } -} diff --git a/src/sync/import_exchange_ews/folders.rs b/src/sync/import_exchange_ews/folders.rs deleted file mode 100644 index 58fbf4c..0000000 --- a/src/sync/import_exchange_ews/folders.rs +++ /dev/null @@ -1,1024 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::{HashMap, HashSet}; - -use rusqlite::{Connection, params}; - -use crate::db::exchange_ews_ids; -use crate::error::Error; -use crate::exchange_ews::EwsClient; -use crate::exchange_ews::error::EwsError; -use crate::exchange_ews::parse::{ - FolderEntry, parse_find_folder_response, parse_find_item_response, parse_folder_inner, - parse_get_folder_response, parse_response_messages, -}; -use crate::exchange_ews::types::{DistinguishedFolderId, FolderClass, MailboxKind}; -use crate::exchange_ews::xml::{ - FolderRef, FolderShape, Traversal, find_folder_body, find_item_body, get_folder_body, -}; -use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; -use crate::sync::TypeCounts; - -pub struct FolderPlan { - pub mail: Vec, - pub calendar: Vec, - pub contacts: Vec, - pub well_known_roles: HashMap, - pub root_folder_id: String, -} - -#[derive(Debug, Clone)] -pub struct ClassifiedFolder { - pub folder: FolderEntry, - pub kind: FolderClass, -} - -pub fn plan_folders( - client: &EwsClient, - url: &str, - mailbox_kind: MailboxKind, - mailbox: Option<&str>, - logger: Logger, -) -> Result { - let mailbox = mailbox.filter(|_| !matches!(mailbox_kind, MailboxKind::PublicFolders)); - let root = mailbox_kind.distinguished_root(); - let request = find_folder_body(FolderRef::distinguished(root, mailbox), Traversal::Deep); - let resp = client.call(url, "FindFolder", &request)?; - let parsed = parse_find_folder_response(&resp.body)?; - if logger.enabled(LEVEL_PROGRESS) { - eprintln!("FindFolder returned {} folders", parsed.folders.len()); - } - let folders = drop_hidden_subtrees(parsed.folders); - let parents: HashMap = folders - .iter() - .filter_map(|f| Some((f.folder_id.id.clone(), f.parent_id.clone()?))) - .collect(); - let mut mail = Vec::new(); - let mut calendar = Vec::new(); - let mut contacts = Vec::new(); - for folder in folders { - let mut class = FolderClass::from_ipf(&folder.folder_class); - if class == FolderClass::Mail && FolderClass::is_mail_fallback(&folder.folder_class) { - let classless = folder.folder_class.trim().is_empty(); - if classless && !matches!(mailbox_kind, MailboxKind::Archive) { - class = kind_from_items(client, url, &folder, logger); - } - if logger.enabled(LEVEL_DEFAULT) { - let shown = if classless { "no" } else { "unrecognized" }; - let kind = match class { - FolderClass::Calendar => "calendar", - FolderClass::Contacts => "contacts", - FolderClass::Mail | FolderClass::Skipped => "mail", - }; - eprintln!( - "warning: EWS folder {:?} has {shown} FolderClass ({:?}); importing as {kind}", - folder.display_name, folder.folder_class - ); - } - } - match (mailbox_kind, class) { - (MailboxKind::Archive, FolderClass::Mail) => { - mail.push(ClassifiedFolder { - folder, - kind: class, - }); - } - (MailboxKind::Archive, _) => {} - (_, FolderClass::Mail) => mail.push(ClassifiedFolder { - folder, - kind: class, - }), - (_, FolderClass::Calendar) => { - calendar.push(ClassifiedFolder { - folder, - kind: class, - }); - } - (_, FolderClass::Contacts) => { - contacts.push(ClassifiedFolder { - folder, - kind: class, - }); - } - (_, FolderClass::Skipped) => {} - } - } - reparent_under_mail_ancestors(&mut mail, &parents); - let well_known_roles = resolve_well_known_roles(client, url, mailbox_kind, mailbox, logger)?; - let root_folder_id = resolve_root_folder_id(client, url, mailbox_kind, mailbox)?; - Ok(FolderPlan { - mail, - calendar, - contacts, - well_known_roles, - root_folder_id, - }) -} - -const CLASS_SAMPLE_SIZE: u32 = 10; - -fn kind_from_items( - client: &EwsClient, - url: &str, - folder: &FolderEntry, - logger: Logger, -) -> FolderClass { - let request = find_item_body( - FolderRef::Concrete(&folder.folder_id), - Traversal::Shallow, - 0, - CLASS_SAMPLE_SIZE, - ); - let sampled = client - .call(url, "FindItem", &request) - .and_then(|resp| parse_find_item_response(&resp.body)); - match sampled { - Ok(found) => FolderClass::from_item_classes( - found - .items - .iter() - .filter_map(|item| item.item_class.as_deref()), - ), - Err(e) => { - if logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "FindItem sample of {:?} failed: {e}; treating it as mail", - folder.display_name - ); - } - FolderClass::Mail - } - } -} - -fn drop_hidden_subtrees(mut folders: Vec) -> Vec { - if !folders.iter().any(|f| f.is_hidden) { - return folders; - } - let by_id: HashMap<&str, &FolderEntry> = folders - .iter() - .map(|f| (f.folder_id.id.as_str(), f)) - .collect(); - let hidden: HashSet = folders - .iter() - .filter(|f| { - let mut current = Some(*f); - for _ in 0..=by_id.len() { - let Some(folder) = current else { - return false; - }; - if folder.is_hidden { - return true; - } - current = folder - .parent_id - .as_deref() - .and_then(|p| by_id.get(p).copied()); - } - false - }) - .map(|f| f.folder_id.id.clone()) - .collect(); - folders.retain(|f| !hidden.contains(&f.folder_id.id)); - folders -} - -fn reparent_under_mail_ancestors(mail: &mut [ClassifiedFolder], parents: &HashMap) { - let mail_ids: HashSet = mail.iter().map(|f| f.folder.folder_id.id.clone()).collect(); - for folder in mail.iter_mut() { - let mut parent = folder.folder.parent_id.as_deref(); - for _ in 0..=parents.len() { - match parent { - Some(p) if !mail_ids.contains(p) => match parents.get(p) { - Some(grandparent) => parent = Some(grandparent), - None => break, - }, - _ => break, - } - } - let parent = parent.map(str::to_owned); - folder.folder.parent_id = parent; - } -} - -fn resolve_well_known_roles( - client: &EwsClient, - url: &str, - mailbox_kind: MailboxKind, - mailbox: Option<&str>, - logger: Logger, -) -> Result, EwsError> { - if !matches!(mailbox_kind, MailboxKind::Primary) { - return Ok(HashMap::new()); - } - let folders: Vec> = WELL_KNOWN_ROLES - .iter() - .map(|(d, _)| FolderRef::distinguished(*d, mailbox)) - .collect(); - let request = get_folder_body(&folders, FolderShape::IdOnly); - let resp = match client.call(url, "GetFolder", &request) { - Ok(r) => r, - Err(e) => { - if logger.enabled(LEVEL_PROGRESS) { - eprintln!("GetFolder for distinguished ids failed: {e}; roles will be empty"); - } - return Ok(HashMap::new()); - } - }; - let messages = - parse_response_messages(&resp.body, "GetFolderResponseMessage").unwrap_or_default(); - let mut map: HashMap = HashMap::new(); - for (msg, (_, role)) in messages.into_iter().zip(WELL_KNOWN_ROLES.iter()) { - if !msg.success { - continue; - } - let Ok(Some(entry)) = parse_folder_inner(&msg.inner_xml) else { - continue; - }; - if entry.folder_id.id.is_empty() { - continue; - } - if let Some(r) = role { - map.insert(entry.folder_id.id, r); - } - } - Ok(map) -} - -fn resolve_root_folder_id( - client: &EwsClient, - url: &str, - mailbox_kind: MailboxKind, - mailbox: Option<&str>, -) -> Result { - let root = mailbox_kind.distinguished_root(); - let body = get_folder_body( - &[FolderRef::distinguished(root, mailbox)], - FolderShape::IdOnly, - ); - let resp = client.call(url, "GetFolder", &body)?; - let entries = parse_get_folder_response(&resp.body)?; - Ok(entries - .into_iter() - .next() - .map(|e| e.folder_id.id) - .unwrap_or_default()) -} - -const WELL_KNOWN_ROLES: &[(DistinguishedFolderId, Option<&'static str>)] = &[ - (DistinguishedFolderId::Inbox, Some("inbox")), - (DistinguishedFolderId::SentItems, Some("sent")), - (DistinguishedFolderId::Drafts, Some("drafts")), - (DistinguishedFolderId::DeletedItems, Some("trash")), - (DistinguishedFolderId::JunkEmail, Some("junk")), - (DistinguishedFolderId::Archive, Some("archive")), - (DistinguishedFolderId::Outbox, None), - (DistinguishedFolderId::ConversationHistory, None), -]; - -struct DeleteTarget<'a> { - type_name: &'a str, - table: &'a str, - counts: &'a mut TypeCounts, -} - -pub struct ReconcileCounts<'a> { - pub mailbox: &'a mut TypeCounts, - pub calendar: &'a mut TypeCounts, - pub addressbook: &'a mut TypeCounts, - pub email: &'a mut TypeCounts, - pub calendar_event: &'a mut TypeCounts, - pub contact: &'a mut TypeCounts, -} - -pub fn reconcile( - conn: &mut Connection, - source_id: i64, - plan: &FolderPlan, - counts: &mut ReconcileCounts<'_>, - _logger: Logger, -) -> Result<(), Error> { - let local_mailbox: HashMap = - exchange_ews_ids::folders_of_type(conn, source_id, exchange_ews_ids::MAILBOX) - .map_err(|e| Error::Partial(e.to_string()))?; - let local_calendar: HashMap = - exchange_ews_ids::folders_of_type(conn, source_id, exchange_ews_ids::CALENDAR) - .map_err(|e| Error::Partial(e.to_string()))?; - let local_addressbook: HashMap = - exchange_ews_ids::folders_of_type(conn, source_id, exchange_ews_ids::ADDRESS_BOOK) - .map_err(|e| Error::Partial(e.to_string()))?; - - let mut mailbox_local_ids: HashMap = local_mailbox - .iter() - .map(|(id, row)| (id.clone(), row.local_id)) - .collect(); - - let ordered_mail = order_by_parent(&plan.mail); - let ordered_calendar = order_by_parent(&plan.calendar); - let ordered_contacts = order_by_parent(&plan.contacts); - - let mut server_ids_mail: Vec = Vec::new(); - let mut server_ids_calendar: Vec = Vec::new(); - let mut server_ids_addressbook: Vec = Vec::new(); - - for folder in &ordered_mail { - server_ids_mail.push(folder.folder.folder_id.id.clone()); - let role = plan - .well_known_roles - .get(&folder.folder.folder_id.id) - .copied(); - let mut ctx = UpsertCtx { - conn, - source_id, - local: &local_mailbox, - counts: &mut *counts.mailbox, - root_folder_id: &plan.root_folder_id, - }; - upsert_mailbox(&mut ctx, &mut mailbox_local_ids, folder, role)?; - } - for folder in &ordered_calendar { - server_ids_calendar.push(folder.folder.folder_id.id.clone()); - let mut ctx = UpsertCtx { - conn, - source_id, - local: &local_calendar, - counts: &mut *counts.calendar, - root_folder_id: &plan.root_folder_id, - }; - upsert_calendar(&mut ctx, folder)?; - } - for folder in &ordered_contacts { - server_ids_addressbook.push(folder.folder.folder_id.id.clone()); - let mut ctx = UpsertCtx { - conn, - source_id, - local: &local_addressbook, - counts: &mut *counts.addressbook, - root_folder_id: &plan.root_folder_id, - }; - upsert_address_book(&mut ctx, folder)?; - } - - delete_vanished( - conn, - source_id, - &local_mailbox, - &server_ids_mail, - DeleteTarget { - type_name: exchange_ews_ids::MAILBOX, - table: "mailboxes", - counts: counts.mailbox, - }, - Some(DeleteTarget { - type_name: exchange_ews_ids::EMAIL, - table: "emails", - counts: counts.email, - }), - )?; - delete_vanished( - conn, - source_id, - &local_calendar, - &server_ids_calendar, - DeleteTarget { - type_name: exchange_ews_ids::CALENDAR, - table: "calendars", - counts: counts.calendar, - }, - Some(DeleteTarget { - type_name: exchange_ews_ids::CALENDAR_EVENT, - table: "calendar_events", - counts: counts.calendar_event, - }), - )?; - delete_vanished( - conn, - source_id, - &local_addressbook, - &server_ids_addressbook, - DeleteTarget { - type_name: exchange_ews_ids::ADDRESS_BOOK, - table: "address_books", - counts: counts.addressbook, - }, - Some(DeleteTarget { - type_name: exchange_ews_ids::CONTACT_CARD, - table: "contact_cards", - counts: counts.contact, - }), - )?; - Ok(()) -} - -fn order_by_parent(folders: &[ClassifiedFolder]) -> Vec<&ClassifiedFolder> { - let by_id: HashMap<&str, &ClassifiedFolder> = folders - .iter() - .map(|f| (f.folder.folder_id.id.as_str(), f)) - .collect(); - let mut depths: HashMap<&str, usize> = HashMap::new(); - fn depth_of<'a>( - id: &'a str, - by_id: &HashMap<&'a str, &'a ClassifiedFolder>, - memo: &mut HashMap<&'a str, usize>, - seen: &mut std::collections::HashSet<&'a str>, - ) -> usize { - if let Some(d) = memo.get(id) { - return *d; - } - if !seen.insert(id) { - return 0; - } - let d = match by_id.get(id).and_then(|f| f.folder.parent_id.as_deref()) { - Some(parent) if by_id.contains_key(parent) => 1 + depth_of(parent, by_id, memo, seen), - _ => 0, - }; - memo.insert(id, d); - d - } - for f in folders { - let id = f.folder.folder_id.id.as_str(); - let mut seen: std::collections::HashSet<&str> = std::collections::HashSet::new(); - let d = depth_of(id, &by_id, &mut depths, &mut seen); - depths.insert(id, d); - } - let mut out: Vec<&ClassifiedFolder> = folders.iter().collect(); - out.sort_by_key(|f| { - depths - .get(f.folder.folder_id.id.as_str()) - .copied() - .unwrap_or(0) - }); - out -} - -struct UpsertCtx<'a> { - conn: &'a mut Connection, - source_id: i64, - local: &'a HashMap, - counts: &'a mut TypeCounts, - root_folder_id: &'a str, -} - -fn upsert_mailbox( - ctx: &mut UpsertCtx<'_>, - mailbox_local_ids: &mut HashMap, - folder: &ClassifiedFolder, - role: Option<&'static str>, -) -> Result<(), Error> { - let parent_local = parent_local_id(folder, mailbox_local_ids, ctx.root_folder_id); - let tx = ctx - .conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - let existing = ctx.local.get(&folder.folder.folder_id.id); - let local_id = if let Some(row) = existing { - let role = crate::db::roles::unique_role(&tx, role, Some(row.local_id)) - .map_err(|e| Error::Partial(e.to_string()))?; - tx.execute( - "UPDATE mailboxes SET name = ?1, parent_id = ?2, role = ?3 WHERE id = ?4", - params![folder.folder.display_name, parent_local, role, row.local_id], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - exchange_ews_ids::update_change_key( - &tx, - ctx.source_id, - exchange_ews_ids::MAILBOX, - &folder.folder.folder_id.id, - &folder.folder.folder_id.change_key, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - ctx.counts.fetched += 1; - row.local_id - } else { - let role = crate::db::roles::unique_role(&tx, role, None) - .map_err(|e| Error::Partial(e.to_string()))?; - tx.execute( - "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) \ - VALUES (?1, ?2, ?3, 0, 1)", - params![folder.folder.display_name, parent_local, role], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - let new_id = tx.last_insert_rowid(); - exchange_ews_ids::insert( - &tx, - ctx.source_id, - exchange_ews_ids::MAILBOX, - folder.folder.parent_id.as_deref().unwrap_or(""), - &folder.folder.folder_id.id, - &folder.folder.folder_id.change_key, - new_id, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - ctx.counts.created += 1; - new_id - }; - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - mailbox_local_ids.insert(folder.folder.folder_id.id.clone(), local_id); - Ok(()) -} - -fn upsert_calendar(ctx: &mut UpsertCtx<'_>, folder: &ClassifiedFolder) -> Result<(), Error> { - let tx = ctx - .conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - let existing = ctx.local.get(&folder.folder.folder_id.id); - if let Some(row) = existing { - tx.execute( - "UPDATE calendars SET name = ?1 WHERE id = ?2", - params![folder.folder.display_name, row.local_id], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - exchange_ews_ids::update_change_key( - &tx, - ctx.source_id, - exchange_ews_ids::CALENDAR, - &folder.folder.folder_id.id, - &folder.folder.folder_id.change_key, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - ctx.counts.fetched += 1; - } else { - tx.execute( - "INSERT INTO calendars (name, sort_order, is_subscribed, is_visible) \ - VALUES (?1, 0, 1, 1)", - params![folder.folder.display_name], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - let new_id = tx.last_insert_rowid(); - exchange_ews_ids::insert( - &tx, - ctx.source_id, - exchange_ews_ids::CALENDAR, - folder.folder.parent_id.as_deref().unwrap_or(""), - &folder.folder.folder_id.id, - &folder.folder.folder_id.change_key, - new_id, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - ctx.counts.created += 1; - } - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - Ok(()) -} - -fn upsert_address_book(ctx: &mut UpsertCtx<'_>, folder: &ClassifiedFolder) -> Result<(), Error> { - let tx = ctx - .conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - let existing = ctx.local.get(&folder.folder.folder_id.id); - if let Some(row) = existing { - tx.execute( - "UPDATE address_books SET name = ?1 WHERE id = ?2", - params![folder.folder.display_name, row.local_id], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - exchange_ews_ids::update_change_key( - &tx, - ctx.source_id, - exchange_ews_ids::ADDRESS_BOOK, - &folder.folder.folder_id.id, - &folder.folder.folder_id.change_key, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - ctx.counts.fetched += 1; - } else { - tx.execute( - "INSERT INTO address_books (name, sort_order, is_subscribed) \ - VALUES (?1, 0, 1)", - params![folder.folder.display_name], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - let new_id = tx.last_insert_rowid(); - exchange_ews_ids::insert( - &tx, - ctx.source_id, - exchange_ews_ids::ADDRESS_BOOK, - folder.folder.parent_id.as_deref().unwrap_or(""), - &folder.folder.folder_id.id, - &folder.folder.folder_id.change_key, - new_id, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - ctx.counts.created += 1; - } - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - Ok(()) -} - -fn parent_local_id( - folder: &ClassifiedFolder, - mailbox_local_ids: &HashMap, - root_folder_id: &str, -) -> Option { - let parent_id = folder.folder.parent_id.as_deref()?; - if parent_id == root_folder_id || parent_id.is_empty() { - return None; - } - mailbox_local_ids.get(parent_id).copied() -} - -fn delete_vanished( - conn: &mut Connection, - source_id: i64, - local: &HashMap, - server_ids: &[String], - target: DeleteTarget<'_>, - mut child: Option>, -) -> Result<(), Error> { - let server_set: std::collections::HashSet<&str> = - server_ids.iter().map(String::as_str).collect(); - let mut vanished: Vec<(&str, &exchange_ews_ids::FolderRow)> = local - .iter() - .filter(|(id, _)| !server_set.contains(id.as_str())) - .map(|(id, row)| (id.as_str(), row)) - .collect(); - if target.type_name == exchange_ews_ids::MAILBOX { - vanished.sort_by_key(|(_, row)| std::cmp::Reverse(folder_depth(row, local))); - } - for (item_id, row) in vanished { - if let Some(c) = child.as_mut() { - delete_folder_children(conn, source_id, item_id, c)?; - } - let tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - let result = tx.execute( - &format!("DELETE FROM {} WHERE id = ?1", target.table), - params![row.local_id], - ); - match result { - Ok(_) => { - exchange_ews_ids::delete_item(&tx, source_id, target.type_name, item_id) - .map_err(|e| Error::Partial(e.to_string()))?; - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - target.counts.deleted += 1; - } - Err(_) => { - let _ = tx.rollback(); - target.counts.failed += 1; - } - } - } - Ok(()) -} - -fn delete_folder_children( - conn: &mut Connection, - source_id: i64, - folder_item_id: &str, - child: &mut DeleteTarget<'_>, -) -> Result<(), Error> { - let kids = exchange_ews_ids::items_in_folder(conn, source_id, child.type_name, folder_item_id) - .map_err(|e| Error::Partial(e.to_string()))?; - for kid in kids { - let tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - let deleted = tx.execute( - &format!("DELETE FROM {} WHERE id = ?1", child.table), - params![kid.local_id], - ); - match deleted { - Ok(_) => { - exchange_ews_ids::delete_item(&tx, source_id, child.type_name, &kid.item_id) - .map_err(|e| Error::Partial(e.to_string()))?; - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - child.counts.deleted += 1; - } - Err(_) => { - let _ = tx.rollback(); - child.counts.failed += 1; - } - } - } - Ok(()) -} - -fn folder_depth( - row: &exchange_ews_ids::FolderRow, - local: &HashMap, -) -> usize { - let mut depth = 0; - let mut cursor = row.folder_id.as_str(); - let mut visited: std::collections::HashSet<&str> = std::collections::HashSet::new(); - while !cursor.is_empty() && visited.insert(cursor) { - match local.get(cursor) { - Some(parent) => { - depth += 1; - cursor = parent.folder_id.as_str(); - } - None => break, - } - } - depth -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn vanished_folder_cascades_deletion_of_its_child_items() { - let conn = rusqlite::Connection::open_in_memory().unwrap(); - crate::db::init::apply_schema(&conn).unwrap(); - let sid = crate::db::sources::upsert_source( - &conn, - &crate::db::sources::SourceKey { - kind: "exchange_ews".to_owned(), - session_url: "https://x/EWS/Exchange.asmx".to_owned(), - account_id: "u@d".to_owned(), - }, - Some("u"), - "u@d", - ) - .unwrap(); - conn.execute( - "INSERT INTO blobs (id, hash, data) VALUES (1, X'00', X'01')", - [], - ) - .unwrap(); - conn.execute("INSERT INTO mailboxes (id, name) VALUES (10, 'Tasks')", []) - .unwrap(); - conn.execute( - "INSERT INTO emails (id, blob_id, received_at, mailbox_ids) VALUES (20, 1, '', '[10]')", - [], - ) - .unwrap(); - exchange_ews_ids::insert( - &conn, - sid, - exchange_ews_ids::MAILBOX, - "", - "FOLDER", - "CK", - 10, - ) - .unwrap(); - exchange_ews_ids::insert( - &conn, - sid, - exchange_ews_ids::EMAIL, - "FOLDER", - "MAIL", - "CK", - 20, - ) - .unwrap(); - - let mut conn = conn; - let local = - exchange_ews_ids::folders_of_type(&conn, sid, exchange_ews_ids::MAILBOX).unwrap(); - let mut folder_counts = TypeCounts::default(); - let mut email_counts = TypeCounts::default(); - delete_vanished( - &mut conn, - sid, - &local, - &[], - DeleteTarget { - type_name: exchange_ews_ids::MAILBOX, - table: "mailboxes", - counts: &mut folder_counts, - }, - Some(DeleteTarget { - type_name: exchange_ews_ids::EMAIL, - table: "emails", - counts: &mut email_counts, - }), - ) - .unwrap(); - - assert_eq!(folder_counts.deleted, 1); - assert_eq!( - email_counts.deleted, 1, - "child email must be cascade-deleted" - ); - let mailboxes: i64 = conn - .query_row("SELECT count(*) FROM mailboxes", [], |r| r.get(0)) - .unwrap(); - let emails: i64 = conn - .query_row("SELECT count(*) FROM emails", [], |r| r.get(0)) - .unwrap(); - let orphan_idmap: i64 = conn - .query_row("SELECT count(*) FROM sync_id_exchange_ews", [], |r| { - r.get(0) - }) - .unwrap(); - assert_eq!(mailboxes, 0); - assert_eq!(emails, 0, "no orphaned email row may remain"); - assert_eq!(orphan_idmap, 0, "both idmap rows must be removed"); - } - - fn entry(id: &str, parent: &str, class: &str, hidden: bool) -> FolderEntry { - FolderEntry { - folder_id: crate::exchange_ews::types::FolderId { - id: id.to_owned(), - change_key: "CK".to_owned(), - }, - parent_id: Some(parent.to_owned()), - display_name: id.to_owned(), - folder_class: class.to_owned(), - is_hidden: hidden, - ..FolderEntry::default() - } - } - - fn classified(id: &str, parent: &str, kind: FolderClass) -> ClassifiedFolder { - ClassifiedFolder { - folder: entry(id, parent, "", false), - kind, - } - } - - #[test] - fn hidden_folders_are_dropped_with_their_descendants() { - let folders = vec![ - entry("CONTACTS", "ROOT", "IPF.Contact", false), - entry("RECIPIENT_CACHE", "CONTACTS", "IPF.Contact", true), - entry("YAMMER", "ROOT", "IPF.Note", true), - entry("YAMMER_FEEDS", "YAMMER", "IPF.Note", false), - entry("YAMMER_DEEP", "YAMMER_FEEDS", "IPF.Note", false), - entry("INBOX", "ROOT", "IPF.Note", false), - ]; - let kept: Vec = drop_hidden_subtrees(folders) - .into_iter() - .map(|f| f.folder_id.id) - .collect(); - assert_eq!(kept, ["CONTACTS", "INBOX"]); - } - - #[test] - fn mail_folder_is_filed_under_its_nearest_mail_ancestor() { - let parents: HashMap = [ - ("PROJECTS", "ROOT"), - ("PCAL", "PROJECTS"), - ("SUB", "PCAL"), - ("CAL", "ROOT"), - ("NOTES", "CAL"), - ("INBOX", "ROOT"), - ("A", "B"), - ("B", "A"), - ("LOOP", "A"), - ] - .into_iter() - .map(|(id, parent)| (id.to_owned(), parent.to_owned())) - .collect(); - let mut mail = vec![ - classified("PROJECTS", "ROOT", FolderClass::Mail), - classified("SUB", "PCAL", FolderClass::Mail), - classified("NOTES", "CAL", FolderClass::Mail), - classified("INBOX", "ROOT", FolderClass::Mail), - classified("LOOP", "A", FolderClass::Mail), - ]; - reparent_under_mail_ancestors(&mut mail, &parents); - let parent_of = |id: &str| { - mail.iter() - .find(|f| f.folder.folder_id.id == id) - .and_then(|f| f.folder.parent_id.clone()) - }; - assert_eq!(parent_of("SUB").as_deref(), Some("PROJECTS")); - assert_eq!(parent_of("NOTES").as_deref(), Some("ROOT")); - assert_eq!(parent_of("INBOX").as_deref(), Some("ROOT")); - assert_ne!(parent_of("LOOP").as_deref(), Some("PROJECTS")); - } - - #[test] - fn classless_folder_kind_comes_from_its_items() { - let mut server = mockito::Server::new(); - let url = format!("{}/EWS/Exchange.asmx", server.url()); - let _m = server - .mock("POST", "/EWS/Exchange.asmx") - .with_status(200) - .with_header("content-type", "text/xml; charset=utf-8") - .with_body( - "\ - \ - NoError\ - \ - IPM.Contact\ - \ - ", - ) - .create(); - let client = EwsClient::new( - crate::jmap::http::Auth::Bearer { - token: "t".to_owned(), - }, - crate::jmap::http::RetryPolicy::new(0), - false, - ); - let folder = entry("LOCAL", "ROOT", "", false); - assert_eq!( - kind_from_items(&client, &url, &folder, Logger::new(LEVEL_DEFAULT)), - FolderClass::Contacts - ); - } - - #[test] - fn rerun_does_not_take_a_calendar_local_id_as_mailbox_parent() { - let mut conn = rusqlite::Connection::open_in_memory().unwrap(); - crate::db::init::apply_schema(&conn).unwrap(); - let sid = crate::db::sources::upsert_source( - &conn, - &crate::db::sources::SourceKey { - kind: "exchange_ews".to_owned(), - session_url: "https://x/EWS/Exchange.asmx".to_owned(), - account_id: "u@d".to_owned(), - }, - Some("u"), - "u@d", - ) - .unwrap(); - let plan = FolderPlan { - mail: vec![ - classified("INBOX", "ROOT", FolderClass::Mail), - classified("NOTES", "CAL", FolderClass::Mail), - ], - calendar: vec![ - classified("CAL", "ROOT", FolderClass::Calendar), - classified("CAL2", "ROOT", FolderClass::Calendar), - ], - contacts: vec![], - well_known_roles: HashMap::new(), - root_folder_id: "ROOT".to_owned(), - }; - for _ in 0..2 { - let mut counts: [TypeCounts; 6] = Default::default(); - let [ - mailbox, - calendar, - addressbook, - email, - calendar_event, - contact, - ] = &mut counts; - reconcile( - &mut conn, - sid, - &plan, - &mut ReconcileCounts { - mailbox, - calendar, - addressbook, - email, - calendar_event, - contact, - }, - Logger::new(LEVEL_DEFAULT), - ) - .unwrap(); - let parent: Option = conn - .query_row( - "SELECT parent_id FROM mailboxes WHERE name = 'NOTES'", - [], - |r| r.get(0), - ) - .unwrap(); - assert_eq!(parent, None); - } - } - - fn row(id: &str, parent_ews_id: &str, local_id: i64) -> exchange_ews_ids::FolderRow { - exchange_ews_ids::FolderRow { - item_id: id.to_owned(), - folder_id: parent_ews_id.to_owned(), - change_key: String::new(), - local_id, - } - } - - #[test] - fn folder_depth_walks_parent_chain() { - let mut local: HashMap = HashMap::new(); - local.insert("ROOT".to_owned(), row("ROOT", "", 1)); - local.insert("CHILD".to_owned(), row("CHILD", "ROOT", 2)); - local.insert("GRAND".to_owned(), row("GRAND", "CHILD", 3)); - assert_eq!(folder_depth(local.get("ROOT").unwrap(), &local), 0); - assert_eq!(folder_depth(local.get("CHILD").unwrap(), &local), 1); - assert_eq!(folder_depth(local.get("GRAND").unwrap(), &local), 2); - } - - #[test] - fn folder_depth_handles_unknown_parent() { - let mut local: HashMap = HashMap::new(); - local.insert("X".to_owned(), row("X", "ORPHAN", 1)); - assert_eq!(folder_depth(local.get("X").unwrap(), &local), 0); - } - - #[test] - fn folder_depth_stops_on_cycle() { - let mut local: HashMap = HashMap::new(); - local.insert("A".to_owned(), row("A", "B", 1)); - local.insert("B".to_owned(), row("B", "A", 2)); - let d = folder_depth(local.get("A").unwrap(), &local); - assert!(d <= 2); - } -} diff --git a/src/sync/import_exchange_ews/items.rs b/src/sync/import_exchange_ews/items.rs deleted file mode 100644 index 04f56fc..0000000 --- a/src/sync/import_exchange_ews/items.rs +++ /dev/null @@ -1,628 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashMap; - -use crate::error::Error; -use crate::exchange_ews::EwsClient; -use crate::exchange_ews::error::EwsError; -use crate::exchange_ews::parse::{ - SyncChange, parse_find_item_response, parse_response_messages, parse_sync_folder_items_response, -}; -use crate::exchange_ews::types::{FolderId, ItemId, ResponseCode}; -use crate::exchange_ews::xml::{ - FolderRef, ItemShape, Traversal, find_item_body, get_item_body, sync_folder_items_body, -}; -use crate::logging::Logger; - -pub struct ItemRunCtx<'a> { - pub client: &'a EwsClient, - pub url: &'a str, - pub source_id: i64, - pub owner_smtp: Option<&'a str>, - pub batch_size: usize, - pub attachment_batch: usize, - pub connections: usize, - pub use_syncfolderitems: bool, - pub sync_batch: u32, - pub logger: Logger, -} - -pub const SYNC_BATCH_MAX: u32 = 512; -const SYNC_BATCH_MIN: u32 = 32; - -fn shrink_sync_batch(current: u32) -> u32 { - (current / 2).max(SYNC_BATCH_MIN) -} - -#[derive(Debug, Clone)] -pub struct EnumeratedItem { - pub element: String, - pub id: ItemId, - pub item_class: Option, -} - -#[derive(Debug, Clone)] -pub struct EnumerationOutcome { - pub items: Vec, - pub mode: EnumerationMode, -} - -#[derive(Debug, Clone)] -pub enum EnumerationMode { - Full, - Delta { - deletions: Vec, - new_sync_state: String, - }, -} - -pub fn enumerate_folder( - ctx: &ItemRunCtx<'_>, - folder: &FolderId, - prior_sync_state: Option<&str>, -) -> Result { - if ctx.use_syncfolderitems - && let Some(state) = prior_sync_state - && !state.is_empty() - && let Some(outcome) = try_sync_folder_items(ctx, folder, state)? - { - return Ok(outcome); - } - enumerate_via_find_item(ctx, folder).map(|items| EnumerationOutcome { - items, - mode: EnumerationMode::Full, - }) -} - -fn sync_fallback_codes(code: &ResponseCode) -> bool { - matches!( - code, - ResponseCode::InvalidIdMalformed | ResponseCode::AccessDenied | ResponseCode::Other(_) - ) -} - -fn try_sync_folder_items( - ctx: &ItemRunCtx<'_>, - folder: &FolderId, - prior_state: &str, -) -> Result, EwsError> { - let mut sync_state = prior_state.to_owned(); - let mut items: Vec = Vec::new(); - let mut deletions: Vec = Vec::new(); - let mut iters = 0; - let mut batch = ctx.sync_batch.clamp(SYNC_BATCH_MIN, SYNC_BATCH_MAX); - let version = ctx.client.server_version(); - loop { - let retries_before = ctx.client.retries_observed(); - let body = sync_folder_items_body(folder, &sync_state, batch, version); - let resp = match ctx.client.call(ctx.url, "SyncFolderItems", &body) { - Ok(r) => r, - Err(EwsError::SoapFault { - code: ResponseCode::InvalidSyncStateData, - .. - }) => { - return Ok(None); - } - Err(EwsError::SoapFault { code, .. }) if sync_fallback_codes(&code) => { - return Ok(None); - } - Err(e) => return Err(e), - }; - let parsed = parse_sync_folder_items_response(&resp.body)?; - for change in parsed.changes { - match change { - SyncChange::Create { - id, - element, - item_class, - } - | SyncChange::Update { - id, - element, - item_class, - } => { - items.push(EnumeratedItem { - element, - id, - item_class, - }); - } - SyncChange::Delete { id } => { - deletions.push(id.id); - } - SyncChange::ReadFlagChange { id, .. } => { - items.push(EnumeratedItem { - element: "Message".to_owned(), - id, - item_class: None, - }); - } - } - } - if ctx.client.retries_observed() > retries_before { - let shrunk = shrink_sync_batch(batch); - if shrunk != batch { - ctx.logger.warn(&format!( - "EWS SyncFolderItems throttled; shrinking change batch {batch} -> {shrunk}" - )); - batch = shrunk; - } - } - sync_state = parsed.sync_state; - if !parsed.more { - break; - } - iters += 1; - if iters > 200 { - break; - } - } - Ok(Some(EnumerationOutcome { - items, - mode: EnumerationMode::Delta { - deletions, - new_sync_state: sync_state, - }, - })) -} - -fn enumerate_via_find_item( - ctx: &ItemRunCtx<'_>, - folder: &FolderId, -) -> Result, EwsError> { - let mut items: Vec = Vec::new(); - let mut offset: u32 = 0; - let page_size: u32 = 500; - loop { - let body = find_item_body( - FolderRef::Concrete(folder), - Traversal::Shallow, - offset, - page_size, - ); - let resp = ctx.client.call(ctx.url, "FindItem", &body)?; - let parsed = parse_find_item_response(&resp.body)?; - let returned = parsed.items.len() as u32; - for entry in parsed.items { - items.push(EnumeratedItem { - element: entry.element, - id: entry.id, - item_class: entry.item_class, - }); - } - if !parsed.more { - break; - } - if returned == 0 { - break; - } - offset = offset.saturating_add(page_size); - if offset > 1_000_000 { - break; - } - } - Ok(items) -} - -#[derive(Debug, Clone)] -pub struct DiffPlan { - pub new: Vec, - pub vanished: Vec<(String, i64)>, - pub present_changed: Vec<(ItemId, i64)>, - pub present_unchanged: Vec<(String, i64)>, -} - -pub fn plan_for( - outcome: &EnumerationOutcome, - local: &[crate::db::exchange_ews_ids::ItemRow], -) -> DiffPlan { - match &outcome.mode { - EnumerationMode::Full => diff(&outcome.items, local), - EnumerationMode::Delta { deletions, .. } => diff_delta(&outcome.items, deletions, local), - } -} - -fn diff_delta( - server_changes: &[EnumeratedItem], - deletions: &[String], - local: &[crate::db::exchange_ews_ids::ItemRow], -) -> DiffPlan { - let local_map: HashMap<&str, &crate::db::exchange_ews_ids::ItemRow> = - local.iter().map(|r| (r.item_id.as_str(), r)).collect(); - let mut plan = DiffPlan { - new: Vec::new(), - vanished: Vec::new(), - present_changed: Vec::new(), - present_unchanged: Vec::new(), - }; - for s in server_changes { - match local_map.get(s.id.id.as_str()) { - None => plan.new.push(s.id.clone()), - Some(row) => plan.present_changed.push((s.id.clone(), row.local_id)), - } - } - for d in deletions { - if let Some(row) = local_map.get(d.as_str()) { - plan.vanished.push((d.clone(), row.local_id)); - } - } - plan -} - -pub fn diff(server: &[EnumeratedItem], local: &[crate::db::exchange_ews_ids::ItemRow]) -> DiffPlan { - let local_map: HashMap<&str, &crate::db::exchange_ews_ids::ItemRow> = - local.iter().map(|r| (r.item_id.as_str(), r)).collect(); - let server_map: HashMap<&str, &EnumeratedItem> = - server.iter().map(|s| (s.id.id.as_str(), s)).collect(); - let mut plan = DiffPlan { - new: Vec::new(), - vanished: Vec::new(), - present_changed: Vec::new(), - present_unchanged: Vec::new(), - }; - for s in server { - match local_map.get(s.id.id.as_str()) { - None => plan.new.push(s.id.clone()), - Some(row) => { - if s.id.change_key.is_empty() || s.id.change_key == row.change_key { - plan.present_unchanged - .push((row.item_id.clone(), row.local_id)); - } else { - plan.present_changed.push((s.id.clone(), row.local_id)); - } - } - } - } - for row in local { - if !server_map.contains_key(row.item_id.as_str()) { - plan.vanished.push((row.item_id.clone(), row.local_id)); - } - } - plan -} - -pub struct GetItemBatchOutcome { - pub messages: Vec, - pub failed_items: u64, -} - -fn is_per_batch_fault(err: &EwsError) -> bool { - matches!( - err, - EwsError::SoapFault { .. } | EwsError::HttpStatus { .. } | EwsError::Malformed(_) - ) -} - -pub fn get_items( - ctx: &ItemRunCtx<'_>, - shape: ItemShape, - ids: &[ItemId], -) -> Result { - let batch = ctx.batch_size.max(1); - let workers = ctx.connections.clamp(1, 8); - let version = ctx.client.server_version(); - let mut failed_items: u64 = 0; - if workers <= 1 || ids.len() <= batch { - let mut all = Vec::new(); - for chunk in ids.chunks(batch) { - let body = get_item_body(shape, chunk, version); - match ctx.client.call(ctx.url, "GetItem", &body) { - Ok(resp) => match parse_response_messages(&resp.body, "GetItemResponseMessage") { - Ok(mut msgs) => all.append(&mut msgs), - Err(e) if is_per_batch_fault(&e) => { - ctx.logger.warn(&format!( - "GetItem batch parse failed ({e}); {} ids left for retry next run", - chunk.len() - )); - failed_items += chunk.len() as u64; - } - Err(e) => return Err(e), - }, - Err(e) if is_per_batch_fault(&e) => { - ctx.logger.warn(&format!( - "GetItem batch failed ({e}); {} ids left for retry next run", - chunk.len() - )); - failed_items += chunk.len() as u64; - } - Err(e) => return Err(e), - } - } - return Ok(GetItemBatchOutcome { - messages: all, - failed_items, - }); - } - let client = ctx.client.clone(); - let url = ctx.url.to_owned(); - type BatchResult = ( - usize, - Result, EwsError>, - ); - let pool: crate::sync::import_jmap::pool::Pool, BatchResult> = - crate::sync::import_jmap::pool::Pool::new(workers, move |chunk: Vec| { - let body = get_item_body(shape, &chunk, version); - let n = chunk.len(); - let result = match client.call(&url, "GetItem", &body) { - Ok(resp) => parse_response_messages(&resp.body, "GetItemResponseMessage"), - Err(e) => Err(e), - }; - (n, result) - }); - let mut submitted = 0usize; - for chunk in ids.chunks(batch) { - pool.submit(chunk.to_vec()); - submitted += 1; - } - let mut all = Vec::new(); - let mut abort_err: Option = None; - for _ in 0..submitted { - match pool.results().recv() { - Ok((_, Ok(mut msgs))) => all.append(&mut msgs), - Ok((n, Err(e))) => { - if is_per_batch_fault(&e) { - ctx.logger.warn(&format!( - "GetItem batch failed ({e}); {n} ids left for retry next run" - )); - failed_items += n as u64; - } else if abort_err.is_none() { - abort_err = Some(e); - } - } - Err(_) => break, - } - } - if let Some(e) = abort_err { - return Err(e); - } - Ok(GetItemBatchOutcome { - messages: all, - failed_items, - }) -} - -pub fn for_each_fetched_item( - ctx: &ItemRunCtx<'_>, - shape: ItemShape, - ids: &[ItemId], - mut on_message: F, -) -> Result -where - F: FnMut(crate::exchange_ews::parse::ResponseMessage) -> Result<(), Error>, -{ - let batch = ctx.batch_size.max(1); - let workers = ctx.connections.clamp(1, 8); - let window = batch.saturating_mul(workers).max(batch); - let mut failed_items = 0u64; - for win in ids.chunks(window) { - let outcome = get_items(ctx, shape, win).map_err(Error::from)?; - failed_items = failed_items.saturating_add(outcome.failed_items); - for msg in outcome.messages { - if let Err(e) = on_message(msg) { - if e.aborts_run() { - return Err(e); - } - ctx.logger.warn(&format!("EWS item not imported: {e}")); - failed_items = failed_items.saturating_add(1); - } - } - } - Ok(failed_items) -} - -pub fn delete_vanished( - conn: &mut rusqlite::Connection, - source_id: i64, - type_name: &str, - table: &str, - vanished: &[(String, i64)], - counts: &mut crate::sync::TypeCounts, -) -> Result<(), Error> { - use rusqlite::params; - for (item_id, local_id) in vanished { - let tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - match tx.execute( - &format!("DELETE FROM {table} WHERE id = ?1"), - params![local_id], - ) { - Ok(_) => { - crate::db::exchange_ews_ids::delete_item(&tx, source_id, type_name, item_id) - .map_err(|e| Error::Partial(e.to_string()))?; - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - counts.deleted += 1; - } - Err(_) => { - let _ = tx.rollback(); - counts.failed += 1; - } - } - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::jmap::http::{Auth, RetryPolicy}; - use crate::logging::LEVEL_DEFAULT; - - const NS: &str = " xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\" \ - xmlns:t=\"http://schemas.microsoft.com/exchange/services/2006/types\" \ - xmlns:m=\"http://schemas.microsoft.com/exchange/services/2006/messages\""; - - fn fetch_three(fail_with: fn() -> Error) -> (Result, Vec) { - let mut server = mockito::Server::new(); - let url = format!("{}/EWS/Exchange.asmx", server.url()); - let messages: String = ["A", "B", "C"] - .iter() - .map(|id| { - format!( - "\ - NoError\ - \ - " - ) - }) - .collect(); - let _m = server - .mock("POST", "/EWS/Exchange.asmx") - .with_status(200) - .with_header("content-type", "text/xml; charset=utf-8") - .with_body(format!( - "\ - {messages}" - )) - .create(); - let client = EwsClient::new( - Auth::Bearer { - token: "t".to_owned(), - }, - RetryPolicy::new(0), - false, - ); - let ctx = ItemRunCtx { - client: &client, - url: &url, - source_id: 1, - owner_smtp: None, - batch_size: 10, - attachment_batch: 10, - connections: 1, - use_syncfolderitems: false, - sync_batch: SYNC_BATCH_MAX, - logger: Logger::new(LEVEL_DEFAULT), - }; - let ids: Vec = ["A", "B", "C"] - .iter() - .map(|id| ItemId::new(*id, "K")) - .collect(); - let mut seen = Vec::new(); - let result = for_each_fetched_item(&ctx, ItemShape::Message, &ids, |msg| { - let parsed = crate::exchange_ews::parse::parse_message_item(&msg.inner_xml) - .map_err(|e| Error::Partial(e.to_string()))?; - seen.push(parsed.id.id.clone()); - if parsed.id.id == "A" { - return Err(fail_with()); - } - Ok(()) - }); - (result, seen) - } - - #[test] - fn per_item_failure_does_not_abort_the_folder() { - let (result, seen) = fetch_three(|| Error::Partial("UNIQUE constraint failed".to_owned())); - assert_eq!(result.unwrap(), 1); - assert_eq!(seen, ["A", "B", "C"]); - } - - #[test] - fn run_aborting_failure_still_stops() { - let (result, seen) = fetch_three(|| Error::Connection("gone".to_owned())); - assert!(result.is_err()); - assert_eq!(seen, ["A"]); - } - - #[test] - fn sync_batch_shrinks_by_half_down_to_floor() { - assert_eq!(shrink_sync_batch(512), 256); - assert_eq!(shrink_sync_batch(256), 128); - assert_eq!(shrink_sync_batch(64), 32); - assert_eq!(shrink_sync_batch(32), 32); - assert_eq!(shrink_sync_batch(40), 32); - } - - #[test] - fn diff_delta_only_yields_changes_from_server_response() { - use crate::db::exchange_ews_ids::ItemRow; - let outcome = EnumerationOutcome { - items: vec![EnumeratedItem { - element: "Message".to_owned(), - id: ItemId::new("A", "ck-2"), - item_class: None, - }], - mode: EnumerationMode::Delta { - deletions: vec!["Z".to_owned()], - new_sync_state: "STATE2".to_owned(), - }, - }; - let local = vec![ - ItemRow { - item_id: "A".to_owned(), - change_key: "ck-1".to_owned(), - local_id: 1, - }, - ItemRow { - item_id: "B".to_owned(), - change_key: "ck-1".to_owned(), - local_id: 2, - }, - ItemRow { - item_id: "Z".to_owned(), - change_key: "ck-9".to_owned(), - local_id: 99, - }, - ]; - let plan = plan_for(&outcome, &local); - assert!(plan.new.is_empty()); - assert_eq!(plan.present_changed.len(), 1); - assert_eq!(plan.present_changed[0].0.id, "A"); - assert_eq!(plan.vanished.len(), 1); - assert_eq!(plan.vanished[0].0, "Z"); - assert_eq!(plan.present_unchanged.len(), 0); - } - - #[test] - fn diff_splits_into_new_unchanged_changed_vanished() { - use crate::db::exchange_ews_ids::ItemRow; - let server = vec![ - EnumeratedItem { - element: "Message".to_owned(), - id: ItemId::new("A", "ck-1"), - item_class: None, - }, - EnumeratedItem { - element: "Message".to_owned(), - id: ItemId::new("B", "ck-1"), - item_class: None, - }, - EnumeratedItem { - element: "Message".to_owned(), - id: ItemId::new("C", "ck-2"), - item_class: None, - }, - ]; - let local = vec![ - ItemRow { - item_id: "A".to_owned(), - change_key: "ck-1".to_owned(), - local_id: 1, - }, - ItemRow { - item_id: "B".to_owned(), - change_key: "ck-0".to_owned(), - local_id: 2, - }, - ItemRow { - item_id: "Z".to_owned(), - change_key: "ck-9".to_owned(), - local_id: 99, - }, - ]; - let plan = diff(&server, &local); - assert_eq!(plan.new.len(), 1); - assert_eq!(plan.new[0].id, "C"); - assert_eq!(plan.present_unchanged.len(), 1); - assert_eq!(plan.present_unchanged[0].0, "A"); - assert_eq!(plan.present_changed.len(), 1); - assert_eq!(plan.present_changed[0].0.id, "B"); - assert_eq!(plan.vanished.len(), 1); - assert_eq!(plan.vanished[0].0, "Z"); - } -} diff --git a/src/sync/import_exchange_ews/messages.rs b/src/sync/import_exchange_ews/messages.rs deleted file mode 100644 index b0637f4..0000000 --- a/src/sync/import_exchange_ews/messages.rs +++ /dev/null @@ -1,386 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::fmt::Write; - -use encodify::base64::LENIENT; -use rusqlite::{Connection, params}; -use serde_json::{Value, json}; - -use crate::db::{blobs, exchange_ews_ids}; -use crate::error::Error; -use crate::exchange_ews::parse::{MessageItem, parse_message_item}; -use crate::exchange_ews::types::{ItemId, is_non_mail_item_class}; -use crate::exchange_ews::xml::ItemShape; -use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS}; -use crate::sync::TypeCounts; -use crate::sync::emailmeta::{email_meta_from_blob, jmap_keyword}; -use crate::sync::keys::index_to_json; - -use super::folders::FolderPlan; -use super::items::{ - EnumeratedItem, EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, - for_each_fetched_item, plan_for, -}; - -pub fn reconcile_all( - conn: &mut Connection, - ctx: &ItemRunCtx<'_>, - plan: &FolderPlan, - counts: &mut TypeCounts, -) -> Result<(), Error> { - for folder in &plan.mail { - let folder_id = &folder.folder.folder_id; - let local_folder_id = match exchange_ews_ids::local_for_item( - conn, - ctx.source_id, - exchange_ews_ids::MAILBOX, - &folder_id.id, - ) - .map_err(|e| Error::Partial(e.to_string()))? - { - Some(id) => id, - None => continue, - }; - if let Err(e) = reconcile_one_folder( - conn, - ctx, - folder_id, - &folder.folder.display_name, - local_folder_id, - counts, - ) { - ctx.logger.warn(&format!( - "email folder {:?} failed: {}", - folder.folder.display_name, e - )); - counts.failed += 1; - } - } - Ok(()) -} - -fn reconcile_one_folder( - conn: &mut Connection, - ctx: &ItemRunCtx<'_>, - folder: &crate::exchange_ews::types::FolderId, - folder_name: &str, - local_folder_id: i64, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let prior = exchange_ews_ids::get_sync_state( - conn, - ctx.source_id, - exchange_ews_ids::MAILBOX, - &folder.id, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - let mut outcome = enumerate_folder(ctx, folder, prior.as_deref()).map_err(Error::from)?; - let skipped = drop_non_mail_items(&mut outcome.items); - if !skipped.is_empty() { - let total: u64 = skipped.iter().map(|(_, n)| n).sum(); - counts.skipped += total; - if ctx.logger.enabled(LEVEL_DEFAULT) { - let mut summary = String::new(); - for (class, n) in &skipped { - if !summary.is_empty() { - summary.push_str(", "); - } - let _ = write!(summary, "{class}={n}"); - } - eprintln!("EWS folder {folder_name:?}: skipped {total} non-mail items ({summary})"); - } - } - let local = - exchange_ews_ids::items_in_folder(conn, ctx.source_id, exchange_ews_ids::EMAIL, &folder.id) - .map_err(|e| Error::Partial(e.to_string()))?; - let plan = plan_for(&outcome, &local); - if ctx.logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "EWS folder {:?}: new={} changed={} vanished={} unchanged={}", - folder_name, - plan.new.len(), - plan.present_changed.len(), - plan.vanished.len(), - plan.present_unchanged.len() - ); - } - let mut to_fetch: Vec = plan.new.clone(); - for (id, _local_id) in &plan.present_changed { - to_fetch.push(id.clone()); - } - if !to_fetch.is_empty() { - let failed_items = for_each_fetched_item(ctx, ItemShape::Message, &to_fetch, |msg| { - if !msg.success { - if matches!( - msg.response_code, - crate::exchange_ews::types::ResponseCode::ItemNotFound - ) { - counts.skipped += 1; - } else { - counts.failed += 1; - ctx.logger.warn(&format!( - "GetItem (message) error: {} {}", - msg.response_code, msg.message_text - )); - } - return Ok(()); - } - let parsed = parse_message_item(&msg.inner_xml) - .map_err(|e| Error::Partial(format!("unparsable item: {e}")))?; - if parsed.id.id.is_empty() { - counts.failed += 1; - return Ok(()); - } - let existing = plan - .present_changed - .iter() - .find(|(id, _)| id.id == parsed.id.id) - .map(|(_, local)| *local); - apply_message( - conn, - ctx, - &parsed, - local_folder_id, - &folder.id, - existing, - counts, - ) - })?; - counts.failed += failed_items; - } - delete_vanished( - conn, - ctx.source_id, - exchange_ews_ids::EMAIL, - "emails", - &plan.vanished, - counts, - )?; - if let EnumerationMode::Delta { new_sync_state, .. } = &outcome.mode { - exchange_ews_ids::set_sync_state( - conn, - ctx.source_id, - exchange_ews_ids::MAILBOX, - &folder.id, - new_sync_state, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - } - Ok(()) -} - -fn apply_message( - conn: &mut Connection, - ctx: &ItemRunCtx<'_>, - item: &MessageItem, - local_folder_id: i64, - folder_id: &str, - existing_local_id: Option, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let Some(mime_b64) = item.mime_content.as_ref() else { - counts.skipped += 1; - ctx.logger.warn(&format!( - "message {} has no MimeContent; skipping", - item.id.id - )); - return Ok(()); - }; - let bytes = match LENIENT.decode(mime_b64) { - Ok(b) => b, - Err(e) => { - counts.failed += 1; - ctx.logger.warn(&format!( - "message {} MimeContent base64 decode failed: {e}", - item.id.id - )); - return Ok(()); - } - }; - let (idx, _date_header) = email_meta_from_blob(&bytes); - let message_match = index_to_json(&idx); - let received_at = item - .date_time_received - .clone() - .unwrap_or_else(|| "1970-01-01T00:00:00Z".to_owned()); - let keywords = keyword_array(item); - let mailbox_ids = json!([local_folder_id]).to_string(); - - let tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - let blob_id = blobs::intern_blob(&tx, &bytes).map_err(|e| Error::Partial(e.to_string()))?; - match existing_local_id { - Some(id) => { - tx.execute( - "UPDATE emails SET blob_id = ?1, received_at = ?2, mailbox_ids = ?3, \ - keywords = ?4, message_match = ?5 WHERE id = ?6", - params![ - blob_id, - received_at, - mailbox_ids, - keywords, - message_match, - id - ], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - exchange_ews_ids::update_change_key( - &tx, - ctx.source_id, - exchange_ews_ids::EMAIL, - &item.id.id, - &item.id.change_key, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - counts.fetched += 1; - } - None => { - tx.execute( - "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match) \ - VALUES (?1, ?2, ?3, ?4, ?5)", - params![blob_id, received_at, mailbox_ids, keywords, message_match], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - let new_id = tx.last_insert_rowid(); - exchange_ews_ids::insert( - &tx, - ctx.source_id, - exchange_ews_ids::EMAIL, - folder_id, - &item.id.id, - &item.id.change_key, - new_id, - ) - .map_err(|e| Error::Partial(e.to_string()))?; - counts.created += 1; - } - } - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - Ok(()) -} - -fn drop_non_mail_items(items: &mut Vec) -> Vec<(String, u64)> { - let mut skipped: Vec<(String, u64)> = Vec::new(); - items.retain(|item| { - let Some(class) = item - .item_class - .as_deref() - .filter(|c| is_non_mail_item_class(c)) - else { - return true; - }; - match skipped - .iter_mut() - .find(|(c, _)| c.eq_ignore_ascii_case(class)) - { - Some((_, n)) => *n += 1, - None => skipped.push((class.to_owned(), 1)), - } - false - }); - skipped -} - -fn keyword_array(item: &MessageItem) -> String { - let mut kws: Vec = Vec::new(); - if matches!(item.is_read, Some(true)) { - kws.push("$seen".to_owned()); - } - if matches!(item.is_draft, Some(true)) { - kws.push("$draft".to_owned()); - } - if matches!(item.flag_status.as_deref(), Some("Flagged")) { - kws.push("$flagged".to_owned()); - } - for keyword in item.categories.iter().filter_map(|cat| jmap_keyword(cat)) { - if !kws.contains(&keyword) { - kws.push(keyword); - } - } - let value: Value = Value::Array(kws.into_iter().map(Value::String).collect()); - value.to_string() -} - -#[cfg(test)] -mod tests { - use super::{drop_non_mail_items, keyword_array}; - use crate::exchange_ews::parse::MessageItem; - use crate::exchange_ews::types::ItemId; - use crate::sync::import_exchange_ews::items::EnumeratedItem; - - #[test] - fn non_mail_items_are_dropped_and_counted_per_class() { - let mut items: Vec = [ - ("M1", Some("IPM.Note")), - ("C1", Some("IPM.Contact")), - ("R1", Some("IPM.Schedule.Meeting.Request")), - ("C2", Some("IPM.Contact")), - ("T1", Some("IPM.Task")), - ("P1", Some("IPM.Post")), - ("X1", None), - ] - .into_iter() - .map(|(id, class)| EnumeratedItem { - element: "Item".to_owned(), - id: ItemId::new(id, "K"), - item_class: class.map(str::to_owned), - }) - .collect(); - let skipped = drop_non_mail_items(&mut items); - let kept: Vec<&str> = items.iter().map(|i| i.id.id.as_str()).collect(); - assert_eq!(kept, ["M1", "R1", "P1", "X1"]); - assert_eq!( - skipped, - [("IPM.Contact".to_owned(), 2), ("IPM.Task".to_owned(), 1)] - ); - } - - #[test] - fn read_draft_flagged_and_categories_map_to_jmap_keywords() { - let item = MessageItem { - is_read: Some(true), - is_draft: Some(true), - flag_status: Some("Flagged".to_owned()), - categories: vec!["Red Category".to_owned(), "VIP".to_owned()], - ..MessageItem::default() - }; - let kws = keyword_array(&item); - for expected in ["$seen", "$draft", "$flagged", "red_category", "vip"] { - assert!(kws.contains(expected), "missing {expected} in {kws}"); - } - } - - #[test] - fn categories_become_valid_deduplicated_keywords() { - let item = MessageItem { - categories: vec![ - "Catégorie Rouge".to_owned(), - "catégorie rouge".to_owned(), - "$seen".to_owned(), - "( )".to_owned(), - ], - ..MessageItem::default() - }; - assert_eq!(keyword_array(&item), r#"["cat_gorie_rouge","seen"]"#); - } - - #[test] - fn unread_unflagged_message_has_no_seen_or_flagged() { - let item = MessageItem { - is_read: Some(false), - flag_status: Some("NotFlagged".to_owned()), - ..MessageItem::default() - }; - let kws = keyword_array(&item); - assert!(!kws.contains("$seen"), "unread must not be $seen: {kws}"); - assert!( - !kws.contains("$flagged"), - "only FlagStatus=Flagged maps to $flagged: {kws}" - ); - } -} diff --git a/src/sync/import_exchange_graph/calendar.rs b/src/sync/import_exchange_graph/calendar.rs deleted file mode 100644 index 4b3683e..0000000 --- a/src/sync/import_exchange_graph/calendar.rs +++ /dev/null @@ -1,807 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashMap; - -use encodify::base64::STANDARD; -use rusqlite::{Connection, Transaction, params}; -use serde_json::{Value, json}; - -use crate::db::exchange_graph_ids; -use crate::error::Error; -use crate::exchange::date::{all_day_date, is_outlook_no_date}; -use crate::exchange::jscalendar::override_patch_from_event; -use crate::exchange_graph::api::{self, PREFER_BODY_HTML, PREFER_BODY_TEXT, PREFER_TIMEZONE_UTC}; -use crate::exchange_graph::calendar_map::{ - ConvertedEvent, EventType, all_day_key, classify_event_type, convert_event, - windows_or_iana_to_iana, -}; -use crate::exchange_graph::error::GraphError; -use crate::exchange_graph::types::EventBodyFormat; -use crate::logging::LEVEL_PROGRESS; -use crate::sync::TypeCounts; -use crate::sync::import_jmap::pool::Pool; - -use super::coordinator::{CHUNK_SIZE, GraphCoordinator}; -use super::folders::CalendarFolder; - -pub type EventAttachment = (String, String, Vec); - -struct Occurrence { - start: String, - zone: Option, -} - -fn fetch_event_attachments( - client: &crate::exchange_graph::client::GraphClient, - endpoints: &crate::exchange_graph::api::Endpoints, - event_id: &str, -) -> Vec { - let url = endpoints.event_attachments(event_id); - let Ok(body) = client.get_json_with_prefer(&url, &[]) else { - return Vec::new(); - }; - let Some(items) = body.get("value").and_then(Value::as_array) else { - return Vec::new(); - }; - let mut out = Vec::new(); - for item in items { - if item.get("@odata.type").and_then(Value::as_str) - != Some("#microsoft.graph.fileAttachment") - { - continue; - } - let Some(encoded) = item.get("contentBytes").and_then(Value::as_str) else { - continue; - }; - let Ok(bytes) = STANDARD.decode(encoded) else { - continue; - }; - if bytes.is_empty() { - continue; - } - let name = item - .get("name") - .and_then(Value::as_str) - .unwrap_or("attachment") - .to_owned(); - let content_type = item - .get("contentType") - .and_then(Value::as_str) - .unwrap_or("application/octet-stream") - .to_owned(); - out.push((name, content_type, bytes)); - } - out -} - -pub fn reconcile_all( - conn: &mut Connection, - ctx: &GraphCoordinator<'_>, - calendars: &[CalendarFolder], - counts: &mut TypeCounts, -) -> Result<(), Error> { - let local: HashMap = - exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::CALENDAR_EVENT)?; - let mut server_total: std::collections::HashSet = std::collections::HashSet::new(); - let mut planned: std::collections::HashSet = std::collections::HashSet::new(); - let mut any_failure = false; - - for cal in calendars { - let url = ctx.endpoints.calendar_events_ids(&cal.graph_id, ctx.top); - let stubs = match api::collect_all_values(ctx.client, &url, &[PREFER_TIMEZONE_UTC]) { - Ok(v) => v, - Err(e) => { - ctx.logger - .warn(&format!("calendar {} stub fetch failed: {e}", cal.graph_id)); - counts.failed += 1; - any_failure = true; - continue; - } - }; - let mut want_ids: Vec = Vec::new(); - let mut occurrence_count = 0usize; - let mut series_masters: Vec = Vec::new(); - for stub in &stubs { - match classify_event_type(stub) { - EventType::Occurrence => occurrence_count += 1, - _ => { - if matches!(classify_event_type(stub), EventType::SeriesMaster) - && let Some(id) = stub.get("id").and_then(Value::as_str) - { - series_masters.push(id.to_owned()); - } - if let Some(id) = stub.get("id").and_then(Value::as_str) { - server_total.insert(id.to_owned()); - if local.contains_key(id) { - counts.fetched += 1; - } else if planned.insert(id.to_owned()) { - want_ids.push(id.to_owned()); - } - } - } - } - } - if ctx.logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "graph calendar {} events: stubs={} new={} occurrences_skipped={}", - cal.graph_id, - stubs.len(), - want_ids.len(), - occurrence_count - ); - } - let fetched = fetch_events(ctx, &want_ids); - let mut masters: Vec<(String, ConvertedEvent)> = Vec::new(); - let mut attachments_by_id: HashMap> = HashMap::new(); - for (graph_id, result, attachments) in fetched { - match result { - Ok(raw) if starts_on_outlook_no_date(&raw) => counts.skipped += 1, - Ok(raw) => match convert_event(&raw, None) { - Ok(c) => { - attachments_by_id.insert(graph_id.clone(), attachments); - masters.push((graph_id, c)); - } - Err(e) => { - counts.failed += 1; - ctx.logger - .warn(&format!("graph event {graph_id} convert failed: {e}")); - } - }, - Err(GraphError::Vanished) => counts.skipped += 1, - Err(e) => { - counts.failed += 1; - ctx.logger - .warn(&format!("graph event {graph_id} fetch failed: {e}")); - } - } - } - - let master_by_graph_id: HashMap = masters.into_iter().collect(); - let pairs: Vec<(String, ConvertedEvent)> = master_by_graph_id.into_iter().collect(); - insert_events_chunked(conn, ctx, cal.local_id, &pairs, &attachments_by_id, counts)?; - - if !series_masters.is_empty() { - apply_series_expansion(conn, ctx, &cal.graph_id, &series_masters, counts)?; - } - } - - if any_failure { - ctx.logger.warn( - "graph event vanished-cleanup skipped: one or more calendars failed to enumerate; \ - a clean re-run will reconcile deletions", - ); - } else { - delete_vanished(conn, ctx.source_id, &local, &server_total, counts)?; - } - Ok(()) -} - -fn fetch_events( - ctx: &GraphCoordinator<'_>, - ids: &[String], -) -> Vec<(String, Result, Vec)> { - type R = (String, Result, Vec); - let client = ctx.client.clone(); - let endpoints: crate::exchange_graph::api::Endpoints = (*ctx.endpoints).clone(); - let body_prefer = match ctx.event_body_format { - EventBodyFormat::Text => PREFER_BODY_TEXT, - EventBodyFormat::Html => PREFER_BODY_HTML, - }; - let prefer: Vec = vec![PREFER_TIMEZONE_UTC.to_owned(), body_prefer.to_owned()]; - let want_attachments = ctx.event_attachments; - let pool: Pool = Pool::new(ctx.workers, move |id: String| { - let url = endpoints.event(&id); - let prefer_refs: Vec<&str> = prefer.iter().map(String::as_str).collect(); - let result = client.get_json_with_prefer(&url, &prefer_refs); - let attachments = match (&result, want_attachments) { - (Ok(raw), true) if raw.get("hasAttachments").and_then(Value::as_bool) == Some(true) => { - fetch_event_attachments(&client, &endpoints, &id) - } - _ => Vec::new(), - }; - (id, result, attachments) - }); - for id in ids { - pool.submit(id.clone()); - } - let mut out = Vec::with_capacity(ids.len()); - for _ in 0..ids.len() { - if let Ok(r) = pool.results().recv() { - out.push(r); - } - } - out -} - -fn starts_on_outlook_no_date(raw: &Value) -> bool { - raw.get("start") - .and_then(|s| s.get("dateTime")) - .and_then(Value::as_str) - .is_some_and(is_outlook_no_date) -} - -fn exception_windows(years: i32) -> Vec<(String, String)> { - let today = chrono::Utc::now().date_naive(); - let span = chrono::Duration::days( - i64::from(years.max(1)) - .saturating_mul(365) - .min(api::EXCEPTION_WINDOW_MAX_DAYS), - ); - let fmt = |d: chrono::NaiveDate| d.format("%Y-%m-%dT00:00:00").to_string(); - vec![ - (fmt(today - span), fmt(today)), - (fmt(today), fmt(today + span)), - ] -} - -fn fetch_calendar_exceptions( - ctx: &GraphCoordinator<'_>, - calendar_id: &str, - counts: &mut TypeCounts, -) -> Vec<(String, ConvertedEvent)> { - let windows = exception_windows(ctx.exception_window_years); - let mut out = Vec::new(); - let mut seen: std::collections::HashSet = std::collections::HashSet::new(); - for (from, to) in windows { - let url = ctx - .endpoints - .calendar_exceptions(calendar_id, &from, &to, ctx.top); - match api::collect_all_values(ctx.client, &url, &[PREFER_TIMEZONE_UTC]) { - Ok(values) => { - if ctx.logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "graph calendar {calendar_id} exceptions in {from}..{to}: {}", - values.len() - ); - } - for raw in values { - let Some(ex_id) = raw.get("id").and_then(Value::as_str).map(str::to_owned) - else { - continue; - }; - if !seen.insert(ex_id.clone()) { - continue; - } - match convert_event(&raw, None) { - Ok(c) => out.push((ex_id, c)), - Err(e) => { - counts.failed += 1; - ctx.logger - .warn(&format!("graph exception {ex_id} convert failed: {e}")); - } - } - } - } - Err(e) => { - counts.failed += 1; - ctx.logger.warn(&format!( - "graph calendar {calendar_id} exception lookup {from}..{to} failed: {e}" - )); - } - } - } - out -} - -fn apply_series_expansion( - conn: &mut Connection, - ctx: &GraphCoordinator<'_>, - calendar_id: &str, - series_masters: &[String], - counts: &mut TypeCounts, -) -> Result<(), Error> { - let exceptions = fetch_calendar_exceptions(ctx, calendar_id, counts); - let occurrences = fetch_calendar_occurrences(ctx, calendar_id, counts); - let windows = exception_windows(ctx.exception_window_years); - let ids: HashMap = - exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::CALENDAR_EVENT)?; - - let mut by_master: HashMap<&str, Vec<&ConvertedEvent>> = HashMap::new(); - for (_, ex) in &exceptions { - if let Some(master) = ex.series_master_id.as_deref() { - by_master.entry(master).or_default().push(ex); - } - } - - for master_id in series_masters { - let Some(local_id) = ids.get(master_id).copied() else { - continue; - }; - let stored: String = match conn.query_row( - "SELECT data FROM calendar_events WHERE id = ?1", - params![local_id], - |row| row.get(0), - ) { - Ok(v) => v, - Err(_) => continue, - }; - let Ok(mut card) = serde_json::from_str::(&stored) else { - continue; - }; - let before = card.clone(); - - for ex in by_master.get(master_id.as_str()).into_iter().flatten() { - let Some(raw_key) = ex.original_start.as_deref() else { - continue; - }; - let key = occurrence_key(&card, raw_key, ex.zone.as_deref()); - if let Some(map) = card.as_object_mut() { - let entry = map - .entry("recurrenceOverrides".to_owned()) - .or_insert_with(|| Value::Object(serde_json::Map::new())); - if let Some(overrides) = entry.as_object_mut() { - overrides.insert(key, override_patch_from_event(&ex.data)); - } - } - } - - let cancelled = cancelled_occurrence_keys( - &card, - master_id, - &occurrences, - by_master - .get(master_id.as_str()) - .map(Vec::as_slice) - .unwrap_or(&[]), - &windows, - ); - if !cancelled.is_empty() - && let Some(map) = card.as_object_mut() - { - let entry = map - .entry("recurrenceOverrides".to_owned()) - .or_insert_with(|| Value::Object(serde_json::Map::new())); - if let Some(overrides) = entry.as_object_mut() { - for key in cancelled { - overrides - .entry(key) - .or_insert_with(|| json!({"excluded": true})); - } - } - } - - if card != before { - let tx = conn.unchecked_transaction()?; - tx.execute( - "UPDATE calendar_events SET data = ?1 WHERE id = ?2", - params![card.to_string(), local_id], - )?; - tx.commit()?; - counts.updated += 1; - } - } - Ok(()) -} - -fn cancelled_occurrence_keys( - card: &Value, - master_id: &str, - occurrences: &HashMap>, - exceptions: &[&ConvertedEvent], - windows: &[(String, String)], -) -> Vec { - let Some(rule) = card.get("recurrenceRule") else { - return Vec::new(); - }; - let Some(start) = card.get("start").and_then(Value::as_str) else { - return Vec::new(); - }; - let (start_date, start_time) = match start.split_once('T') { - Some((d, t)) => (d, t), - None => (start, "00:00:00"), - }; - let Ok(series_start) = chrono::NaiveDate::parse_from_str(start_date, "%Y-%m-%d") else { - return Vec::new(); - }; - let Some(pattern) = - crate::exchange_graph::expand::graph_pattern_from_jscalendar(rule, series_start) - else { - return Vec::new(); - }; - let mut actual: std::collections::HashSet = std::collections::HashSet::new(); - for occ in occurrences.get(master_id).into_iter().flatten() { - actual.insert(local_date_of(card, &occ.start, occ.zone.as_deref())); - } - for ex in exceptions { - if let Some(raw) = ex.original_start.as_deref() { - actual.insert(local_date_of(card, raw, ex.zone.as_deref())); - } - } - - let mut out = Vec::new(); - for (from, to) in windows { - let (Ok(w0), Ok(w1)) = ( - chrono::NaiveDate::parse_from_str(&from[..10], "%Y-%m-%d"), - chrono::NaiveDate::parse_from_str(&to[..10], "%Y-%m-%d"), - ) else { - continue; - }; - let Some(expected) = crate::exchange_graph::expand::expected_dates(&pattern, w0, w1) else { - return Vec::new(); - }; - for date in expected { - let key = date.format("%Y-%m-%d").to_string(); - if !actual.contains(&key) { - out.push(format!("{key}T{start_time}")); - } - } - } - out.sort(); - out.dedup(); - out -} - -fn local_date_of(master: &Value, utc: &str, zone: Option<&str>) -> String { - let mut key = occurrence_key(master, utc, zone); - if let Some((date, _)) = key.split_once('T') { - let len = date.len(); - key.truncate(len); - } - key -} - -fn occurrence_key(master: &Value, utc: &str, zone: Option<&str>) -> String { - if master.get("showWithoutTime").and_then(Value::as_bool) == Some(true) - && let Some(date) = all_day_date(utc, zone) - { - return all_day_key(date); - } - normalise_override_key(utc, master.get("timeZone").and_then(Value::as_str)) -} - -fn fetch_calendar_occurrences( - ctx: &GraphCoordinator<'_>, - calendar_id: &str, - counts: &mut TypeCounts, -) -> HashMap> { - let mut out: HashMap> = HashMap::new(); - for (from, to) in exception_windows(ctx.exception_window_years) { - let url = ctx - .endpoints - .calendar_occurrences(calendar_id, &from, &to, ctx.top); - match api::collect_all_values(ctx.client, &url, &[PREFER_TIMEZONE_UTC]) { - Ok(values) => { - for raw in values { - let Some(master) = raw.get("seriesMasterId").and_then(Value::as_str) else { - continue; - }; - let Some(start) = raw - .get("start") - .and_then(|s| s.get("dateTime")) - .and_then(Value::as_str) - else { - continue; - }; - let zone = raw - .get("originalStartTimeZone") - .and_then(Value::as_str) - .and_then(windows_or_iana_to_iana); - out.entry(master.to_owned()).or_default().push(Occurrence { - start: start.to_owned(), - zone, - }); - } - } - Err(e) => { - counts.failed += 1; - ctx.logger.warn(&format!( - "graph calendar {calendar_id} occurrence sweep {from}..{to} failed: {e}; \ - deleted occurrences cannot be detected for this calendar" - )); - } - } - } - out -} - -pub fn normalise_override_key(raw: &str, master_tz: Option<&str>) -> String { - if let Some(tz_name) = master_tz - && let Some(local) = utc_offset_to_local(raw, tz_name) - { - return local; - } - strip_offset_and_fractional(raw) -} - -fn utc_offset_to_local(raw: &str, tz_name: &str) -> Option { - use chrono::{DateTime, NaiveDateTime, TimeZone}; - use chrono_tz::Tz; - let tz: Tz = tz_name.parse().ok()?; - let parsed: DateTime = if let Ok(dt) = DateTime::parse_from_rfc3339(raw) { - dt - } else if let Ok(naive) = NaiveDateTime::parse_from_str(raw, "%Y-%m-%dT%H:%M:%S%.f") { - return Some(format_local(naive)); - } else if let Ok(naive) = NaiveDateTime::parse_from_str(raw, "%Y-%m-%dT%H:%M:%S") { - return Some(format_local(naive)); - } else { - return None; - }; - let local = tz.from_utc_datetime(&parsed.naive_utc()).naive_local(); - Some(format_local(local)) -} - -fn format_local(dt: chrono::NaiveDateTime) -> String { - dt.format("%Y-%m-%dT%H:%M:%S").to_string() -} - -fn strip_offset_and_fractional(raw: &str) -> String { - let no_offset = if let Some(idx) = raw.find('Z') { - &raw[..idx] - } else if let Some(idx) = raw.find('+') { - &raw[..idx] - } else if let Some(idx) = raw.rfind('-') - && idx > 10 - { - &raw[..idx] - } else { - raw - }; - let trimmed = no_offset.split('.').next().unwrap_or(no_offset); - if trimmed.contains('T') { - trimmed.to_owned() - } else { - format!("{trimmed}T00:00:00") - } -} - -fn insert_events_chunked( - conn: &mut Connection, - ctx: &GraphCoordinator<'_>, - calendar_local_id: i64, - pairs: &[(String, ConvertedEvent)], - attachments: &HashMap>, - counts: &mut TypeCounts, -) -> Result<(), Error> { - for chunk in pairs.chunks(CHUNK_SIZE) { - let tx = conn.unchecked_transaction()?; - for (graph_id, event) in chunk { - let files = attachments.get(graph_id).map(Vec::as_slice).unwrap_or(&[]); - insert_event_in_tx(&tx, ctx, calendar_local_id, graph_id, event, files, counts)?; - } - tx.commit()?; - } - Ok(()) -} - -fn insert_event_in_tx( - tx: &Transaction<'_>, - ctx: &GraphCoordinator<'_>, - calendar_local_id: i64, - graph_id: &str, - event: &ConvertedEvent, - attachments: &[EventAttachment], - counts: &mut TypeCounts, -) -> Result<(), Error> { - let calendar_ids = json!([calendar_local_id]).to_string(); - let mut card = event.data.clone(); - attach_links(tx, &mut card, attachments)?; - let data = card.to_string(); - tx.execute( - "INSERT INTO calendar_events (calendar_ids, is_draft, use_default_alerts, data, data_type) - VALUES (?1, ?2, ?3, ?4, 'Event')", - params![ - calendar_ids, - event.is_draft as i64, - event.use_default_alerts as i64, - data, - ], - )?; - let new_id = tx.last_insert_rowid(); - exchange_graph_ids::insert( - tx, - ctx.source_id, - exchange_graph_ids::CALENDAR_EVENT, - graph_id, - new_id, - )?; - counts.created += 1; - Ok(()) -} - -fn attach_links( - tx: &Transaction<'_>, - card: &mut Value, - attachments: &[EventAttachment], -) -> Result<(), Error> { - if attachments.is_empty() { - return Ok(()); - } - let mut links = serde_json::Map::new(); - for (idx, (name, content_type, bytes)) in (1u32..).zip(attachments) { - let blob_id = crate::db::blobs::intern_blob(tx, bytes)?; - links.insert( - idx.to_string(), - json!({ - "@type": "Link", - "@blob": blob_id, - "contentType": content_type, - "title": name, - "rel": "enclosure", - }), - ); - } - if let Some(map) = card.as_object_mut() { - match map.get_mut("links").and_then(Value::as_object_mut) { - Some(existing) => existing.extend(links), - None => { - map.insert("links".to_owned(), Value::Object(links)); - } - } - } - Ok(()) -} - -fn delete_vanished( - conn: &mut Connection, - source_id: i64, - local: &HashMap, - server: &std::collections::HashSet, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let vanished: Vec<(&String, &i64)> = local - .iter() - .filter(|(graph_id, _)| !server.contains(graph_id.as_str())) - .collect(); - for chunk in vanished.chunks(CHUNK_SIZE) { - let tx = conn.unchecked_transaction()?; - for (graph_id, local_id) in chunk { - let result = tx.execute( - "DELETE FROM calendar_events WHERE id = ?1", - params![local_id], - ); - match result { - Ok(_) => { - exchange_graph_ids::delete( - &tx, - source_id, - exchange_graph_ids::CALENDAR_EVENT, - graph_id, - )?; - counts.deleted += 1; - } - Err(_) => { - counts.failed += 1; - } - } - } - tx.commit()?; - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn all_day_daily_master() -> Value { - json!({ - "@type": "Event", - "start": "2026-11-12T00:00:00", - "duration": "P1D", - "showWithoutTime": true, - "recurrenceRule": {"@type": "RecurrenceRule", "frequency": "daily", "count": 3} - }) - } - - #[test] - fn all_day_override_key_takes_the_date_in_the_event_zone() { - let master = all_day_daily_master(); - let paris = Some("Europe/Paris"); - assert_eq!( - occurrence_key(&master, "2026-11-12T23:00:00Z", paris), - "2026-11-13T00:00:00" - ); - assert_eq!( - occurrence_key(&master, "2026-11-13T00:00:00Z", paris), - "2026-11-13T00:00:00" - ); - assert_eq!( - local_date_of(&master, "2026-11-13T23:00:00.0000000", paris), - "2026-11-14" - ); - } - - #[test] - fn all_day_occurrences_east_of_utc_are_not_reported_cancelled() { - let master = all_day_daily_master(); - let zone = Some("Europe/Paris".to_owned()); - let occurrences = HashMap::from([( - "M".to_owned(), - ["2026-11-11T23:00:00.0000000", "2026-11-13T23:00:00.0000000"] - .into_iter() - .map(|start| Occurrence { - start: start.to_owned(), - zone: zone.clone(), - }) - .collect(), - )]); - let windows = vec![( - "2026-11-01T00:00:00".to_owned(), - "2026-11-30T00:00:00".to_owned(), - )]; - let cancelled = cancelled_occurrence_keys(&master, "M", &occurrences, &[], &windows); - assert_eq!(cancelled, vec!["2026-11-13T00:00:00".to_owned()]); - } - - #[test] - fn exception_windows_never_exceed_the_graph_limit() { - for requested in [1, 5, 50] { - let windows = exception_windows(requested); - assert_eq!(windows.len(), 2, "one window behind today, one ahead"); - for (from, to) in windows { - let start = chrono::NaiveDate::parse_from_str(&from[..10], "%Y-%m-%d").unwrap(); - let end = chrono::NaiveDate::parse_from_str(&to[..10], "%Y-%m-%d").unwrap(); - let days = (end - start).num_days(); - assert!(days > 0, "window runs forwards: {from}..{to}"); - assert!( - days <= api::EXCEPTION_WINDOW_MAX_DAYS, - "requested {requested}y produced {days} days, over Graph's cap" - ); - } - } - } - - #[test] - fn cancelled_occurrence_keys_drop_the_oid_prefix() { - let raw = serde_json::json!({ - "id": "M", - "start": {"dateTime": "2026-09-21T11:00:00.0000000", "timeZone": "UTC"}, - "end": {"dateTime": "2026-09-21T11:30:00.0000000", "timeZone": "UTC"}, - "recurrence": { - "pattern": {"type": "daily", "interval": 1}, - "range": {"type": "numbered", "startDate": "2026-09-21", - "numberOfOccurrences": 6} - }, - "cancelledOccurrences": ["OID.AAMkAGI2TGuLAAA=.2026-09-24"] - }); - let converted = convert_event(&raw, None).unwrap(); - let overrides = converted.data["recurrenceOverrides"].as_object().unwrap(); - assert!( - overrides.contains_key("2026-09-24T11:00:00"), - "beta cancelledOccurrences are OID..; the key must be a LocalDateTime, \ - got {:?}", - overrides.keys().collect::>() - ); - assert_eq!(overrides["2026-09-24T11:00:00"]["excluded"], true); - } - - #[test] - fn override_key_utc_converts_to_master_local_time() { - let local = normalise_override_key("2026-05-11T15:00:00Z", Some("America/New_York")); - assert_eq!(local, "2026-05-11T11:00:00"); - } - - #[test] - fn override_key_no_tz_strips_z_only() { - let local = normalise_override_key("2026-05-11T15:00:00Z", None); - assert_eq!(local, "2026-05-11T15:00:00"); - } - - #[test] - fn override_key_with_explicit_offset_converts() { - let local = normalise_override_key("2026-05-11T15:00:00+00:00", Some("Europe/London")); - assert_eq!(local, "2026-05-11T16:00:00"); - } - - #[test] - fn override_key_naive_localdatetime_passes_through() { - let local = normalise_override_key("2026-05-11T15:00:00", Some("America/New_York")); - assert_eq!(local, "2026-05-11T15:00:00"); - } - - #[test] - fn override_key_unknown_timezone_strips_offset_only() { - let local = normalise_override_key("2026-05-11T15:00:00Z", Some("Not/A_Zone")); - assert_eq!(local, "2026-05-11T15:00:00"); - } - - #[test] - fn override_key_fractional_seconds_stripped() { - let local = normalise_override_key("2026-05-11T15:00:00.1234567Z", None); - assert_eq!(local, "2026-05-11T15:00:00"); - } -} diff --git a/src/sync/import_exchange_graph/contacts.rs b/src/sync/import_exchange_graph/contacts.rs deleted file mode 100644 index 6d7b3ad..0000000 --- a/src/sync/import_exchange_graph/contacts.rs +++ /dev/null @@ -1,289 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashMap; - -use rusqlite::{Connection, Transaction, params}; -use serde_json::{Value, json}; - -use crate::db::exchange_graph_ids; -use crate::error::Error; -use crate::exchange_graph::api; -use crate::exchange_graph::contact_map::{ConvertedContact, convert_contact}; -use crate::exchange_graph::error::GraphError; -use crate::logging::LEVEL_PROGRESS; -use crate::sync::TypeCounts; -use crate::sync::import_jmap::pool::Pool; - -use super::coordinator::{CHUNK_SIZE, GraphCoordinator}; -use super::folders::ContactFolder; - -fn fetch_photo( - client: &crate::exchange_graph::client::GraphClient, - endpoints: &crate::exchange_graph::api::Endpoints, - id: &str, -) -> Option { - let url = endpoints.contact_photo(id); - let resp = client - .get_with_prefer(&url, crate::exchange_graph::client::Accept::Binary, &[]) - .ok()?; - if resp.body.is_empty() { - return None; - } - let media_type = sniff_image_type(&resp.body) - .map(str::to_owned) - .or_else(|| resp.content_type.clone()) - .unwrap_or_else(|| "application/octet-stream".to_owned()); - Some((resp.body, media_type)) -} - -fn sniff_image_type(bytes: &[u8]) -> Option<&'static str> { - if bytes.starts_with(b"\x89PNG\r\n\x1a\n") { - return Some("image/png"); - } - if bytes.starts_with(&[0xFF, 0xD8, 0xFF]) { - return Some("image/jpeg"); - } - if bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a") { - return Some("image/gif"); - } - if bytes.len() >= 12 && bytes.starts_with(b"RIFF") && &bytes[8..12] == b"WEBP" { - return Some("image/webp"); - } - if bytes.starts_with(b"BM") { - return Some("image/bmp"); - } - None -} - -pub fn reconcile_all( - conn: &mut Connection, - ctx: &GraphCoordinator<'_>, - books: &[ContactFolder], - counts: &mut TypeCounts, -) -> Result<(), Error> { - let local: HashMap = - exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::CONTACT_CARD)?; - let mut server_total: std::collections::HashSet = std::collections::HashSet::new(); - let mut planned: std::collections::HashSet = std::collections::HashSet::new(); - let mut any_failure = false; - - for book in books { - let url = ctx - .endpoints - .contact_folder_contacts_ids(&book.graph_id, ctx.top); - let ids = match api::collect_all_ids(ctx.client, &url, &[]) { - Ok(v) => v, - Err(e) => { - ctx.logger.warn(&format!( - "contact folder {} enumeration failed: {e}", - book.graph_id - )); - counts.failed += 1; - any_failure = true; - continue; - } - }; - if ctx.logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "graph contact folder {} contacts: {}", - book.graph_id, - ids.len() - ); - } - for id in &ids { - server_total.insert(id.clone()); - } - let mut new_ids: Vec = Vec::new(); - for id in ids { - if local.contains_key(&id) { - counts.fetched += 1; - } else if planned.insert(id.clone()) { - new_ids.push(id); - } - } - if new_ids.is_empty() { - continue; - } - let fetched = fetch_contacts(ctx, &new_ids); - let mut converted: Vec<(String, ConvertedContact, Option)> = Vec::new(); - for (graph_id, result, photo) in fetched { - match result { - Ok(raw) => match convert_contact(&raw) { - Ok(c) => converted.push((graph_id, c, photo)), - Err(e) => { - counts.failed += 1; - ctx.logger - .warn(&format!("graph contact {graph_id} convert failed: {e}")); - } - }, - Err(GraphError::Vanished) => counts.skipped += 1, - Err(e) => { - counts.failed += 1; - ctx.logger - .warn(&format!("graph contact {graph_id} fetch failed: {e}")); - } - } - } - insert_contacts_chunked(conn, ctx, book.local_id, &converted, counts)?; - } - - if any_failure { - ctx.logger.warn( - "graph contact vanished-cleanup skipped: one or more contact folders failed to enumerate; \ - a clean re-run will reconcile deletions", - ); - } else { - delete_vanished(conn, ctx.source_id, &local, &server_total, counts)?; - } - Ok(()) -} - -pub type ContactPhoto = (Vec, String); - -fn fetch_contacts( - ctx: &GraphCoordinator<'_>, - ids: &[String], -) -> Vec<(String, Result, Option)> { - type R = (String, Result, Option); - let client = ctx.client.clone(); - let endpoints: crate::exchange_graph::api::Endpoints = (*ctx.endpoints).clone(); - let want_photos = ctx.contact_photos; - let pool: Pool = Pool::new(ctx.workers, move |id: String| { - let url = endpoints.contact(&id); - let contact = client.get_json_with_prefer(&url, &[]); - let photo = if want_photos && contact.is_ok() { - fetch_photo(&client, &endpoints, &id) - } else { - None - }; - (id, contact, photo) - }); - for id in ids { - pool.submit(id.clone()); - } - let mut out = Vec::with_capacity(ids.len()); - for _ in 0..ids.len() { - if let Ok(r) = pool.results().recv() { - out.push(r); - } - } - out -} - -fn insert_contacts_chunked( - conn: &mut Connection, - ctx: &GraphCoordinator<'_>, - book_local_id: i64, - pairs: &[(String, ConvertedContact, Option)], - counts: &mut TypeCounts, -) -> Result<(), Error> { - for chunk in pairs.chunks(CHUNK_SIZE) { - let tx = conn.unchecked_transaction()?; - for (graph_id, contact, photo) in chunk { - insert_contact_in_tx(&tx, ctx, book_local_id, graph_id, contact, photo, counts)?; - } - tx.commit()?; - } - Ok(()) -} - -fn insert_contact_in_tx( - tx: &Transaction<'_>, - ctx: &GraphCoordinator<'_>, - book_local_id: i64, - graph_id: &str, - converted: &ConvertedContact, - photo: &Option, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let book_ids = json!([book_local_id]).to_string(); - let mut card = converted.data.clone(); - if let Some((bytes, media_type)) = photo { - let blob_id = crate::db::blobs::intern_blob(tx, bytes)?; - if let Some(map) = card.as_object_mut() { - map.insert( - "media".to_owned(), - json!({ - "photo": { - "@type": "Media", - "kind": "photo", - "@blob": blob_id, - "mediaType": media_type, - } - }), - ); - } - } - let data = card.to_string(); - tx.execute( - "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)", - params![converted.uid, book_ids, data], - )?; - let new_id = tx.last_insert_rowid(); - exchange_graph_ids::insert( - tx, - ctx.source_id, - exchange_graph_ids::CONTACT_CARD, - graph_id, - new_id, - )?; - counts.created += 1; - Ok(()) -} - -fn delete_vanished( - conn: &mut Connection, - source_id: i64, - local: &HashMap, - server: &std::collections::HashSet, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let vanished: Vec<(&String, &i64)> = local - .iter() - .filter(|(graph_id, _)| !server.contains(graph_id.as_str())) - .collect(); - for chunk in vanished.chunks(CHUNK_SIZE) { - let tx = conn.unchecked_transaction()?; - for (graph_id, local_id) in chunk { - let result = tx.execute("DELETE FROM contact_cards WHERE id = ?1", params![local_id]); - match result { - Ok(_) => { - exchange_graph_ids::delete( - &tx, - source_id, - exchange_graph_ids::CONTACT_CARD, - graph_id, - )?; - counts.deleted += 1; - } - Err(_) => { - counts.failed += 1; - } - } - } - tx.commit()?; - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::sniff_image_type; - - #[test] - fn photo_type_comes_from_the_bytes_not_the_header() { - let png = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR"; - assert_eq!(sniff_image_type(png), Some("image/png")); - assert_eq!( - sniff_image_type(&[0xFF, 0xD8, 0xFF, 0xE0]), - Some("image/jpeg") - ); - assert_eq!(sniff_image_type(b"GIF89a..."), Some("image/gif")); - assert_eq!(sniff_image_type(b"RIFF____WEBPVP8 "), Some("image/webp")); - assert_eq!(sniff_image_type(b"not an image"), None); - } -} diff --git a/src/sync/import_exchange_graph/coordinator.rs b/src/sync/import_exchange_graph/coordinator.rs deleted file mode 100644 index be71758..0000000 --- a/src/sync/import_exchange_graph/coordinator.rs +++ /dev/null @@ -1,529 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use rusqlite::Connection; -use serde_json::Value; - -use crate::db; -use crate::db::sources::SourceKey; -use crate::error::Error; -use crate::exchange_graph::api::{DEFAULT_API_BASE, Endpoints}; -use crate::exchange_graph::client::GraphClient; -use crate::exchange_graph::error::GraphError; -use crate::exchange_graph::oauth::{ - AcquiredToken, OAuthFlow, acquire, default_authority, refresh_access_token, -}; -use crate::exchange_graph::types::{EventBodyFormat, MailboxKind, Surfaces, synthetic_account_id}; -use crate::jmap::http::RetryPolicy; -use crate::logging::LEVEL_DEFAULT; -use crate::sync::{CommonConfig, Summary, TypeCounts}; - -#[derive(Debug, Clone)] -pub enum GraphAuth { - PreAcquired { - token: String, - }, - DeviceCode { - authority: String, - client_id: String, - }, -} - -#[derive(Debug, Clone)] -pub struct GraphImportConfig { - pub auth: GraphAuth, - pub api_base: String, - pub user_target: Option, - pub mailbox_kind: MailboxKind, - pub surfaces: Surfaces, - pub event_body_format: EventBodyFormat, - pub graph_connections: usize, - pub top: usize, - pub exception_window_years: i32, - pub contact_photos: bool, - pub event_attachments: bool, - pub allow_source_change: bool, -} - -pub const CHUNK_SIZE: usize = 100; - -pub struct GraphCoordinator<'a> { - pub client: &'a GraphClient, - pub endpoints: &'a Endpoints, - pub source_id: i64, - pub top: usize, - pub workers: usize, - pub logger: crate::logging::Logger, - pub event_body_format: EventBodyFormat, - pub exception_window_years: i32, - pub contact_photos: bool, - pub event_attachments: bool, -} - -pub fn run(common: CommonConfig, config: GraphImportConfig) -> Result { - let logger = common.logger; - let mut conn = db::init::open(&common.archive)?; - - let acquired = acquire_with_flow(&config.auth, common.allow_invalid_certs)?; - let client = GraphClient::new( - acquired.access_token.clone(), - RetryPolicy::new(common.max_retries), - common.allow_invalid_certs, - ); - client.set_logger(logger); - - let endpoints = resolve_endpoints(&config, &client)?; - let principal = resolve_principal(&client, &endpoints, acquired.upn.as_deref())?; - - let session_url = canonical_session_url(&config); - let account_id = synthetic_account_id(&principal.id, config.mailbox_kind); - - if !common.dry_run - && let Some((url, acc)) = - db::sources::conflicting_source(&conn, "exchange_graph", &session_url, &account_id)? - && !config.allow_source_change - { - return Err(Error::SourceChange(format!( - "archive already records exchange_graph source ({url}, account {acc}); \ - pass --allow-source-change to import a different account" - ))); - } - - if matches!(config.mailbox_kind, MailboxKind::Archive) { - logger.warn( - "--mailbox-kind archive: Microsoft Online Archive holds only mail; \ - calendar and contact surfaces are skipped for this run", - ); - } - - if common.dry_run { - return run_dry( - &client, - &endpoints, - config.mailbox_kind, - &logger, - config.top, - ); - } - - let source_id = db::sources::upsert_source( - &conn, - &SourceKey { - kind: "exchange_graph".to_owned(), - session_url: session_url.clone(), - account_id: account_id.clone(), - }, - Some(&principal.user_principal_name), - &principal.user_principal_name, - )?; - - let _refresher = spawn_token_refresher( - &client, - &config.auth, - &acquired, - common.allow_invalid_certs, - logger, - ); - - let mut summary = Summary::default(); - let mut mailbox_counts = TypeCounts::default(); - let mut email_counts = TypeCounts::default(); - let mut calendar_counts = TypeCounts::default(); - let mut event_counts = TypeCounts::default(); - let mut addressbook_counts = TypeCounts::default(); - let mut contact_counts = TypeCounts::default(); - let mut file_counts = TypeCounts::default(); - - let ctx = GraphCoordinator { - client: &client, - endpoints: &endpoints, - source_id, - top: config.top.clamp(1, 1000), - workers: config.graph_connections.clamp(1, 16), - logger, - event_body_format: config.event_body_format, - exception_window_years: config.exception_window_years, - contact_photos: config.contact_photos, - event_attachments: config.event_attachments, - }; - - let primary = !matches!(config.mailbox_kind, MailboxKind::Archive); - let want_mail = config.surfaces.mail; - let want_calendar = config.surfaces.calendar && primary; - let want_contacts = config.surfaces.contacts && primary; - let want_files = config.surfaces.files && primary; - - if want_mail { - let folders = super::folders::reconcile_mail( - &mut conn, - &ctx, - config.mailbox_kind, - &mut mailbox_counts, - )?; - if logger.enabled(LEVEL_DEFAULT) { - eprintln!( - "graph mail folders: created={} fetched={} deleted={}", - mailbox_counts.created, mailbox_counts.fetched, mailbox_counts.deleted - ); - } - super::messages::reconcile_all(&mut conn, &ctx, &folders, &mut email_counts)?; - } - - if want_calendar { - let calendars = super::folders::reconcile_calendars(&mut conn, &ctx, &mut calendar_counts)?; - super::calendar::reconcile_all(&mut conn, &ctx, &calendars, &mut event_counts)?; - } - - if want_contacts { - let books = - super::folders::reconcile_address_books(&mut conn, &ctx, &mut addressbook_counts)?; - super::contacts::reconcile_all(&mut conn, &ctx, &books, &mut contact_counts)?; - } - - if want_files { - super::files::reconcile_all(&mut conn, &ctx, &mut file_counts)?; - } - - summary.per_type.push(("mailbox", mailbox_counts)); - summary.per_type.push(("email", email_counts)); - summary.per_type.push(("calendar", calendar_counts)); - summary.per_type.push(("calendarevent", event_counts)); - summary.per_type.push(("addressbook", addressbook_counts)); - summary.per_type.push(("contactcard", contact_counts)); - summary.per_type.push(("filenode", file_counts)); - - if !summary.any_failed() - && let Err(e) = run_gc(&conn) - { - logger.warn(&format!("blob GC skipped: {e}")); - } - - summary.retries_observed = client.retries_observed(); - summary.retry_after_sleeps = client.retry_after_sleeps(); - Ok(summary) -} - -fn run_dry( - client: &GraphClient, - endpoints: &Endpoints, - mailbox_kind: MailboxKind, - logger: &crate::logging::Logger, - top: usize, -) -> Result { - let mut mailbox_counts = TypeCounts::default(); - let mut calendar_counts = TypeCounts::default(); - let mut addressbook_counts = TypeCounts::default(); - let folders = - enumerate_mail_folders(client, endpoints, mailbox_kind, top).map_err(Error::from)?; - mailbox_counts.created = folders.len() as u64; - if !matches!(mailbox_kind, MailboxKind::Archive) { - let calendars = - crate::exchange_graph::api::collect_all_values(client, &endpoints.calendars(top), &[]) - .map_err(Error::from)?; - calendar_counts.created = calendars.len() as u64; - let books = crate::exchange_graph::api::collect_all_values( - client, - &endpoints.contact_folders(top), - &[], - ) - .map_err(Error::from)?; - addressbook_counts.created = books.len() as u64; - } - if logger.enabled(LEVEL_DEFAULT) { - eprintln!( - "dry-run: mailbox={} calendar={} addressbook={}", - mailbox_counts.created, calendar_counts.created, addressbook_counts.created - ); - } - let mut summary = Summary::default(); - summary.per_type.push(("mailbox", mailbox_counts)); - summary.per_type.push(("calendar", calendar_counts)); - summary.per_type.push(("addressbook", addressbook_counts)); - Ok(summary) -} - -pub fn enumerate_mail_folders( - client: &GraphClient, - endpoints: &Endpoints, - mailbox_kind: MailboxKind, - top: usize, -) -> Result, GraphError> { - let mut all = Vec::new(); - let mut frontier: Vec = Vec::new(); - let mut seen: std::collections::HashSet = std::collections::HashSet::new(); - let initial = endpoints.mail_folders_root(mailbox_kind, top); - let level = crate::exchange_graph::api::collect_all_values(client, &initial, &[])?; - for folder in level { - let Some(id) = folder.get("id").and_then(Value::as_str) else { - continue; - }; - if seen.insert(id.to_owned()) { - frontier.push(id.to_owned()); - all.push(folder); - } - } - while let Some(parent) = frontier.pop() { - let url = endpoints.mail_folder_child_folders(&parent, top); - let children = crate::exchange_graph::api::collect_all_values(client, &url, &[])?; - for child in children { - let Some(id) = child.get("id").and_then(Value::as_str) else { - continue; - }; - if seen.insert(id.to_owned()) { - frontier.push(id.to_owned()); - all.push(child); - } - } - } - Ok(all) -} - -fn acquire_with_flow(auth: &GraphAuth, allow_invalid_certs: bool) -> Result { - let flow = match auth { - GraphAuth::PreAcquired { token } => OAuthFlow::PreAcquired { - token: token.clone(), - }, - GraphAuth::DeviceCode { - authority, - client_id, - } => OAuthFlow::DeviceCode { - authority: authority.clone(), - client_id: client_id.clone(), - }, - }; - acquire(&flow, allow_invalid_certs).map_err(Error::from) -} - -fn resolve_endpoints(config: &GraphImportConfig, client: &GraphClient) -> Result { - if let Some(target) = config.user_target.as_deref() { - let resolved = if looks_like_uuid(target) { - target.to_owned() - } else { - resolve_user_id(client, &config.api_base, target) - .map_err(|e| Error::Connection(format!("user resolution: {e}")))? - }; - Ok(Endpoints::for_user(&config.api_base, &resolved)) - } else { - Ok(Endpoints::for_me(&config.api_base)) - } -} - -fn looks_like_uuid(s: &str) -> bool { - s.len() == 36 - && s.chars().enumerate().all(|(i, c)| match i { - 8 | 13 | 18 | 23 => c == '-', - _ => c.is_ascii_hexdigit(), - }) -} - -fn resolve_user_id( - client: &GraphClient, - api_base: &str, - upn_or_id: &str, -) -> Result { - let url = format!( - "{}/users/{}?$select=id,userPrincipalName", - api_base.trim_end_matches('/'), - upn_or_id - ); - let body = client.get_json(&url)?; - let id = body - .get("id") - .and_then(Value::as_str) - .ok_or_else(|| GraphError::Malformed("user resolution missing id".to_owned()))? - .to_owned(); - Ok(id) -} - -fn resolve_principal( - client: &GraphClient, - endpoints: &Endpoints, - fallback_upn: Option<&str>, -) -> Result { - let url = endpoints.me_select_id_upn(); - let value = client.get_json(&url).map_err(Error::from)?; - let id = value - .get("id") - .and_then(Value::as_str) - .ok_or_else(|| Error::Connection("graph principal has no id".to_owned()))? - .to_owned(); - let upn = value - .get("userPrincipalName") - .and_then(Value::as_str) - .map(str::to_owned) - .or_else(|| fallback_upn.map(str::to_owned)) - .unwrap_or_default(); - Ok(crate::exchange_graph::ResolvedPrincipal { - id, - user_principal_name: upn, - }) -} - -fn canonical_session_url(config: &GraphImportConfig) -> String { - let authority = match &config.auth { - GraphAuth::DeviceCode { authority, .. } => authority.clone(), - GraphAuth::PreAcquired { .. } => default_authority("common"), - }; - format!( - "{}|{}", - authority.trim_end_matches('/'), - config.api_base.trim_end_matches('/') - ) -} - -fn spawn_token_refresher( - client: &GraphClient, - auth: &GraphAuth, - initial: &AcquiredToken, - allow_invalid_certs: bool, - logger: crate::logging::Logger, -) -> Option { - let (authority, client_id) = match auth { - GraphAuth::DeviceCode { - authority, - client_id, - } => (authority.clone(), client_id.clone()), - GraphAuth::PreAcquired { .. } => return None, - }; - let refresh = initial.refresh_token.clone()?; - let mut deadline_unix = initial_deadline_unix(initial)?; - let client = client.clone(); - let mut refresh_token = refresh; - let shutdown = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let shutdown_signal = shutdown.clone(); - let handle = std::thread::Builder::new() - .name("inbuxa-migrate-graph-token-refresh".to_owned()) - .spawn(move || { - while !shutdown_signal.load(std::sync::atomic::Ordering::Relaxed) { - let now = unix_now(); - let refresh_at = deadline_unix.saturating_sub(60); - if refresh_at > now { - let wait = std::time::Duration::from_secs(refresh_at - now); - let chunk = std::time::Duration::from_secs(2); - let mut left = wait; - while left > std::time::Duration::ZERO - && !shutdown_signal.load(std::sync::atomic::Ordering::Relaxed) - { - let step = chunk.min(left); - std::thread::sleep(step); - left = left.saturating_sub(step); - } - if shutdown_signal.load(std::sync::atomic::Ordering::Relaxed) { - break; - } - } - match refresh_access_token( - &authority, - &client_id, - &refresh_token, - allow_invalid_certs, - ) { - Ok(tok) => { - client.set_bearer(tok.access_token.clone()); - if let Some(new_refresh) = tok.refresh_token { - refresh_token = new_refresh; - } - deadline_unix = unix_now() - + tok.expires_in.unwrap_or_else(|| { - crate::exchange_graph::oauth::decode_jwt_claims(&tok.access_token) - .and_then(|c| c.exp) - .and_then(|exp| exp.checked_sub(unix_now())) - .unwrap_or(50 * 60) - }); - } - Err(e) => { - logger.warn(&format!( - "graph token refresh failed: {e}; sleeping 60s before retry" - )); - std::thread::sleep(std::time::Duration::from_secs(60)); - } - } - } - }) - .ok()?; - Some(TokenRefresher { - shutdown, - handle: Some(handle), - }) -} - -pub struct TokenRefresher { - shutdown: std::sync::Arc, - handle: Option>, -} - -impl Drop for TokenRefresher { - fn drop(&mut self) { - self.shutdown - .store(true, std::sync::atomic::Ordering::Relaxed); - if let Some(h) = self.handle.take() { - let _ = h.join(); - } - } -} - -fn unix_now() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0) -} - -fn initial_deadline_unix(initial: &AcquiredToken) -> Option { - if let Some(expires_in) = initial.expires_in { - return Some(unix_now() + expires_in); - } - crate::exchange_graph::oauth::decode_jwt_claims(&initial.access_token).and_then(|c| c.exp) -} - -fn run_gc(conn: &Connection) -> Result<(), Error> { - let tx = conn.unchecked_transaction()?; - db::blobs::gc_orphan_blobs(&tx)?; - tx.commit()?; - Ok(()) -} - -pub fn default_api_base() -> String { - DEFAULT_API_BASE.to_owned() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn canonical_session_url_concatenates_authority_and_base() { - let config = GraphImportConfig { - auth: GraphAuth::DeviceCode { - authority: "https://login.microsoftonline.com/common".to_owned(), - client_id: "uuid".to_owned(), - }, - api_base: "https://graph.microsoft.com/v1.0".to_owned(), - user_target: None, - mailbox_kind: MailboxKind::Primary, - surfaces: Surfaces::ALL, - event_body_format: EventBodyFormat::Text, - graph_connections: 4, - top: 100, - exception_window_years: 5, - contact_photos: true, - event_attachments: true, - allow_source_change: false, - }; - let url = canonical_session_url(&config); - assert_eq!( - url, - "https://login.microsoftonline.com/common|https://graph.microsoft.com/v1.0" - ); - } - - #[test] - fn looks_like_uuid_only_matches_well_formed() { - assert!(looks_like_uuid("12345678-1234-1234-1234-123456789abc")); - assert!(!looks_like_uuid("alice@x.com")); - assert!(!looks_like_uuid("12345678123412341234123456789abc")); - } -} diff --git a/src/sync/import_exchange_graph/files.rs b/src/sync/import_exchange_graph/files.rs deleted file mode 100644 index dcc0a0b..0000000 --- a/src/sync/import_exchange_graph/files.rs +++ /dev/null @@ -1,438 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashMap; - -use rusqlite::{Connection, Transaction, params}; -use serde_json::Value; - -use crate::db::{blobs, exchange_graph_ids}; -use crate::error::Error; -use crate::exchange_graph::api; -use crate::exchange_graph::client::Accept; -use crate::exchange_graph::error::GraphError; -use crate::logging::LEVEL_PROGRESS; -use crate::sync::TypeCounts; -use crate::sync::import_jmap::pool::Pool; - -use super::coordinator::{CHUNK_SIZE, GraphCoordinator}; - -#[derive(Debug, Clone)] -struct DriveNode { - graph_id: String, - name: String, - is_folder: bool, - media_type: Option, - created: String, - modified: Option, - parent_graph_id: Option, -} - -pub fn reconcile_all( - conn: &mut Connection, - ctx: &GraphCoordinator<'_>, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let Some(root_id) = drive_root_id(ctx) else { - ctx.logger - .warn("graph drive unavailable for this account; file import skipped"); - return Ok(()); - }; - - let nodes = match enumerate_drive(ctx, &root_id) { - Ok(v) => v, - Err(e) => { - counts.failed += 1; - ctx.logger - .warn(&format!("graph drive enumeration failed: {e}")); - return Ok(()); - } - }; - if ctx.logger.enabled(LEVEL_PROGRESS) { - eprintln!("graph drive items enumerated: {}", nodes.len()); - } - - let local: HashMap = - exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::FILE_NODE)?; - let mut by_graph_id: HashMap = HashMap::new(); - let mut server_ids: Vec = Vec::new(); - - let (folders, files): (Vec, Vec) = - nodes.into_iter().partition(|n| n.is_folder); - - for node in order_by_depth(folders) { - server_ids.push(node.graph_id.clone()); - let parent_local = node - .parent_graph_id - .as_deref() - .and_then(|p| by_graph_id.get(p)) - .copied(); - let local_id = upsert_directory(conn, ctx, &node, parent_local, &local, counts)?; - by_graph_id.insert(node.graph_id.clone(), local_id); - } - - let mut wanted: Vec = Vec::new(); - for node in files { - server_ids.push(node.graph_id.clone()); - if local.contains_key(&node.graph_id) { - counts.fetched += 1; - } else { - wanted.push(node); - } - } - - download_and_insert(conn, ctx, &wanted, &by_graph_id, counts)?; - delete_vanished(conn, ctx.source_id, &local, &server_ids, counts, ctx)?; - Ok(()) -} - -fn drive_root_id(ctx: &GraphCoordinator<'_>) -> Option { - let value = ctx.client.get_json(&ctx.endpoints.drive_root()).ok()?; - value.get("id").and_then(Value::as_str).map(str::to_owned) -} - -fn enumerate_drive( - ctx: &GraphCoordinator<'_>, - root_id: &str, -) -> Result, GraphError> { - let mut out: Vec = Vec::new(); - let mut seen: std::collections::HashSet = std::collections::HashSet::new(); - let mut frontier: Vec = vec![root_id.to_owned()]; - seen.insert(root_id.to_owned()); - - while let Some(parent) = frontier.pop() { - let url = ctx.endpoints.drive_children(&parent, ctx.top); - let children = api::collect_all_values(ctx.client, &url, &[])?; - for child in &children { - let Some(node) = parse_node(child, &parent) else { - continue; - }; - if !seen.insert(node.graph_id.clone()) { - continue; - } - if node.is_folder { - frontier.push(node.graph_id.clone()); - } - out.push(node); - } - } - Ok(out) -} - -fn parse_node(value: &Value, parent_graph_id: &str) -> Option { - let graph_id = value.get("id").and_then(Value::as_str)?.to_owned(); - let name = value.get("name").and_then(Value::as_str)?.to_owned(); - let is_folder = value.get("folder").is_some_and(|f| !f.is_null()); - let is_file = value.get("file").is_some_and(|f| !f.is_null()); - if !is_folder && !is_file { - return None; - } - Some(DriveNode { - graph_id, - name, - is_folder, - media_type: value - .get("file") - .and_then(|f| f.get("mimeType")) - .and_then(Value::as_str) - .map(str::to_owned), - created: value - .get("createdDateTime") - .and_then(Value::as_str) - .unwrap_or("1970-01-01T00:00:00Z") - .to_owned(), - modified: value - .get("lastModifiedDateTime") - .and_then(Value::as_str) - .map(str::to_owned), - parent_graph_id: Some(parent_graph_id.to_owned()), - }) -} - -fn order_by_depth(folders: Vec) -> Vec { - let parents: HashMap> = folders - .iter() - .map(|n| (n.graph_id.clone(), n.parent_graph_id.clone())) - .collect(); - let depth = |start: &str| -> usize { - let mut d = 0; - let mut id = start.to_owned(); - let mut guard = 0; - while let Some(Some(parent)) = parents.get(&id) { - d += 1; - id = parent.clone(); - guard += 1; - if guard > 256 { - break; - } - } - d - }; - let mut ordered: Vec<(usize, DriveNode)> = folders - .iter() - .map(|n| (depth(n.graph_id.as_str()), n.clone())) - .collect(); - ordered.sort_by_key(|(d, _)| *d); - ordered.into_iter().map(|(_, n)| n).collect() -} - -fn upsert_directory( - conn: &mut Connection, - ctx: &GraphCoordinator<'_>, - node: &DriveNode, - parent_local: Option, - local: &HashMap, - counts: &mut TypeCounts, -) -> Result { - let tx = conn.unchecked_transaction()?; - let id = if let Some(existing) = local.get(&node.graph_id).copied() { - tx.execute( - "UPDATE file_nodes SET parent_id = ?1, name = ?2, modified = ?3 WHERE id = ?4", - params![parent_local, node.name, node.modified, existing], - )?; - counts.fetched += 1; - existing - } else { - tx.execute( - "INSERT INTO file_nodes (parent_id, node_type, blob_id, target, name, media_type, - created, modified, is_subscribed, role) - VALUES (?1, 'directory', NULL, NULL, ?2, NULL, ?3, ?4, 1, NULL)", - params![parent_local, node.name, node.created, node.modified], - )?; - let new_id = tx.last_insert_rowid(); - exchange_graph_ids::insert( - &tx, - ctx.source_id, - exchange_graph_ids::FILE_NODE, - &node.graph_id, - new_id, - )?; - counts.created += 1; - new_id - }; - tx.commit()?; - Ok(id) -} - -fn download_and_insert( - conn: &mut Connection, - ctx: &GraphCoordinator<'_>, - nodes: &[DriveNode], - by_graph_id: &HashMap, - counts: &mut TypeCounts, -) -> Result<(), Error> { - if nodes.is_empty() { - return Ok(()); - } - let node_by_id: HashMap<&str, &DriveNode> = - nodes.iter().map(|n| (n.graph_id.as_str(), n)).collect(); - let client = ctx.client.clone(); - let endpoints: crate::exchange_graph::api::Endpoints = (*ctx.endpoints).clone(); - type FetchResult = (String, Result, GraphError>); - let pool: Pool = Pool::new(ctx.workers, move |id: String| { - let url = endpoints.drive_item_content(&id); - match client.get_with_prefer(&url, Accept::Binary, &[]) { - Ok(resp) => (id, Ok(resp.body)), - Err(e) => (id, Err(e)), - } - }); - for node in nodes { - pool.submit(node.graph_id.clone()); - } - - let mut tx_opt: Option> = None; - let mut in_batch = 0usize; - for _ in 0..nodes.len() { - let Ok((graph_id, result)) = pool.results().recv() else { - break; - }; - let Some(node) = node_by_id.get(graph_id.as_str()).copied() else { - continue; - }; - match result { - Ok(bytes) => { - if tx_opt.is_none() { - tx_opt = Some(conn.unchecked_transaction()?); - } - let tx = tx_opt.as_mut().expect("tx is Some"); - let parent_local = node - .parent_graph_id - .as_deref() - .and_then(|p| by_graph_id.get(p)) - .copied(); - insert_file_in_tx(tx, ctx, node, parent_local, &bytes, counts)?; - in_batch += 1; - if in_batch >= CHUNK_SIZE { - if let Some(t) = tx_opt.take() { - t.commit()?; - } - in_batch = 0; - } - } - Err(GraphError::Vanished) => counts.skipped += 1, - Err(e) => { - counts.failed += 1; - ctx.logger - .warn(&format!("graph drive item {graph_id} download failed: {e}")); - } - } - } - if let Some(t) = tx_opt.take() { - t.commit()?; - } - Ok(()) -} - -fn insert_file_in_tx( - tx: &Transaction<'_>, - ctx: &GraphCoordinator<'_>, - node: &DriveNode, - parent_local: Option, - bytes: &[u8], - counts: &mut TypeCounts, -) -> Result<(), Error> { - let blob_id = blobs::intern_blob(tx, bytes)?; - tx.execute( - "INSERT INTO file_nodes (parent_id, node_type, blob_id, target, name, media_type, - created, modified, is_subscribed, role) - VALUES (?1, 'file', ?2, NULL, ?3, ?4, ?5, ?6, 1, NULL)", - params![ - parent_local, - blob_id, - node.name, - node.media_type, - node.created, - node.modified, - ], - )?; - let new_id = tx.last_insert_rowid(); - exchange_graph_ids::insert( - tx, - ctx.source_id, - exchange_graph_ids::FILE_NODE, - &node.graph_id, - new_id, - )?; - counts.created += 1; - Ok(()) -} - -fn delete_vanished( - conn: &mut Connection, - source_id: i64, - local: &HashMap, - server_ids: &[String], - counts: &mut TypeCounts, - ctx: &GraphCoordinator<'_>, -) -> Result<(), Error> { - let server: std::collections::HashSet<&str> = server_ids.iter().map(String::as_str).collect(); - let mut vanished: Vec<(&String, &i64)> = local - .iter() - .filter(|(graph_id, _)| !server.contains(graph_id.as_str())) - .collect(); - let depths = node_depths(conn)?; - vanished.sort_by_key(|(_, local_id)| std::cmp::Reverse(*depths.get(*local_id).unwrap_or(&0))); - - for (graph_id, local_id) in vanished { - let tx = conn.unchecked_transaction()?; - match tx.execute("DELETE FROM file_nodes WHERE id = ?1", params![local_id]) { - Ok(_) => { - exchange_graph_ids::delete( - &tx, - source_id, - exchange_graph_ids::FILE_NODE, - graph_id, - )?; - tx.commit()?; - counts.deleted += 1; - } - Err(e) => { - let _ = tx.rollback(); - ctx.logger.warn(&format!( - "file node {graph_id} (local id {local_id}) could not be deleted: {e}" - )); - counts.failed += 1; - } - } - } - Ok(()) -} - -fn node_depths(conn: &Connection) -> Result, Error> { - let mut parents: HashMap> = HashMap::new(); - { - let mut stmt = conn.prepare("SELECT id, parent_id FROM file_nodes")?; - let mut rows = stmt.query([])?; - while let Some(row) = rows.next()? { - parents.insert(row.get(0)?, row.get(1)?); - } - } - let mut depths = HashMap::new(); - for id in parents.keys().copied() { - let mut d = 0; - let mut cursor = id; - let mut seen: std::collections::HashSet = std::collections::HashSet::new(); - while seen.insert(cursor) { - match parents.get(&cursor).copied().flatten() { - Some(p) => { - d += 1; - cursor = p; - } - None => break, - } - } - depths.insert(id, d); - } - Ok(depths) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn parse_node_rejects_items_without_file_or_folder_facet() { - let vault = json!({"id": "V", "name": "Personal Vault", "remoteItem": {}}); - assert!(parse_node(&vault, "root").is_none()); - } - - #[test] - fn parse_node_reads_folder_and_file_facets() { - let folder = json!({"id": "F", "name": "Docs", "folder": {"childCount": 2}}); - let parsed = parse_node(&folder, "root").expect("folder parses"); - assert!(parsed.is_folder); - - let file = json!({ - "id": "X", "name": "a.txt", "size": 12, - "file": {"mimeType": "text/plain"}, - "createdDateTime": "2026-01-01T00:00:00Z" - }); - let parsed = parse_node(&file, "root").expect("file parses"); - assert!(!parsed.is_folder); - assert_eq!(parsed.media_type.as_deref(), Some("text/plain")); - } - - #[test] - fn order_by_depth_puts_parents_before_children() { - let node = |id: &str, parent: Option<&str>| DriveNode { - graph_id: id.to_owned(), - name: id.to_owned(), - is_folder: true, - media_type: None, - created: "2026-01-01T00:00:00Z".to_owned(), - modified: None, - parent_graph_id: parent.map(str::to_owned), - }; - let ordered = order_by_depth(vec![ - node("C", Some("B")), - node("B", Some("A")), - node("A", None), - ]); - let names: Vec<&str> = ordered.iter().map(|n| n.graph_id.as_str()).collect(); - assert_eq!(names, ["A", "B", "C"]); - } -} diff --git a/src/sync/import_exchange_graph/folders.rs b/src/sync/import_exchange_graph/folders.rs deleted file mode 100644 index e9f6dce..0000000 --- a/src/sync/import_exchange_graph/folders.rs +++ /dev/null @@ -1,581 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashMap; - -use rusqlite::{Connection, params}; -use serde_json::Value; - -use crate::db::exchange_graph_ids; -use crate::error::Error; -use crate::exchange_graph::api; -use crate::exchange_graph::calendar_map::{graph_calendar_color_to_hex, windows_or_iana_to_iana}; -use crate::exchange_graph::types::MailboxKind; -use crate::logging::LEVEL_PROGRESS; -use crate::sync::TypeCounts; -use crate::types::ObjectType; - -use super::coordinator::{CHUNK_SIZE, GraphCoordinator, enumerate_mail_folders}; - -#[derive(Debug, Clone)] -pub struct MailFolder { - pub graph_id: String, - pub parent_graph_id: Option, - pub display_name: String, - pub is_hidden: bool, - pub local_id: i64, -} - -#[derive(Debug, Clone)] -pub struct CalendarFolder { - pub graph_id: String, - pub local_id: i64, -} - -#[derive(Debug, Clone)] -pub struct ContactFolder { - pub graph_id: String, - pub local_id: i64, -} - -pub fn reconcile_mail( - conn: &mut Connection, - ctx: &GraphCoordinator<'_>, - mailbox_kind: MailboxKind, - counts: &mut TypeCounts, -) -> Result, Error> { - let server = enumerate_mail_folders(ctx.client, ctx.endpoints, mailbox_kind, ctx.top) - .map_err(Error::from)?; - if ctx.logger.enabled(LEVEL_PROGRESS) { - eprintln!("graph mailFolders enumerated: {}", server.len()); - } - let well_known = match mailbox_kind { - MailboxKind::Primary => resolve_well_known_roles(ctx), - MailboxKind::Archive => HashMap::new(), - }; - let local: HashMap = - exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::MAILBOX)?; - - let entries = order_by_parent(server); - let mut by_id: HashMap = HashMap::new(); - let mut out: Vec = Vec::new(); - let mut server_ids: Vec = Vec::new(); - - for chunk in entries.chunks(CHUNK_SIZE) { - let tx = conn.unchecked_transaction()?; - for value in chunk { - let Some(graph_id) = value.get("id").and_then(Value::as_str) else { - continue; - }; - let parent_graph_id = value - .get("parentFolderId") - .and_then(Value::as_str) - .map(str::to_owned); - let display_name = value - .get("displayName") - .and_then(Value::as_str) - .unwrap_or("Unnamed") - .to_owned(); - let is_hidden = value - .get("isHidden") - .and_then(Value::as_bool) - .unwrap_or(false); - let role = well_known.get(graph_id).copied(); - let parent_local_id = parent_graph_id - .as_deref() - .and_then(|p| by_id.get(p)) - .copied(); - let existing = local.get(graph_id).copied(); - server_ids.push(graph_id.to_owned()); - - let local_id = if let Some(id) = existing { - let role = crate::db::roles::unique_role(&tx, role, Some(id))?; - tx.execute( - "UPDATE mailboxes SET name = ?1, parent_id = ?2, role = ?3, is_subscribed = ?4 - WHERE id = ?5", - params![display_name, parent_local_id, role, !is_hidden as i64, id,], - )?; - counts.fetched += 1; - id - } else { - let role = crate::db::roles::unique_role(&tx, role, None)?; - tx.execute( - "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) - VALUES (?1, ?2, ?3, 0, ?4)", - params![display_name, parent_local_id, role, !is_hidden as i64], - )?; - let new_id = tx.last_insert_rowid(); - exchange_graph_ids::insert( - &tx, - ctx.source_id, - exchange_graph_ids::MAILBOX, - graph_id, - new_id, - )?; - counts.created += 1; - new_id - }; - by_id.insert(graph_id.to_owned(), local_id); - out.push(MailFolder { - graph_id: graph_id.to_owned(), - parent_graph_id, - display_name, - is_hidden, - local_id, - }); - } - tx.commit()?; - } - - delete_vanished_mailboxes( - conn, - ctx.source_id, - &local, - &server_ids, - counts, - &ctx.logger, - )?; - - Ok(out) -} - -pub fn reconcile_calendars( - conn: &mut Connection, - ctx: &GraphCoordinator<'_>, - counts: &mut TypeCounts, -) -> Result, Error> { - let server = api::collect_all_values(ctx.client, &ctx.endpoints.calendars(ctx.top), &[]) - .map_err(Error::from)?; - let mailbox_tz = mailbox_timezone(ctx); - let local: HashMap = - exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::CALENDAR)?; - let mut out: Vec = Vec::new(); - let mut server_ids: Vec = Vec::new(); - - for chunk in server.chunks(CHUNK_SIZE) { - let tx = conn.unchecked_transaction()?; - for value in chunk { - let Some(graph_id) = value.get("id").and_then(Value::as_str) else { - continue; - }; - server_ids.push(graph_id.to_owned()); - let name = value - .get("name") - .and_then(Value::as_str) - .unwrap_or("Calendar") - .to_owned(); - let color = value - .get("hexColor") - .and_then(Value::as_str) - .map(str::to_owned) - .or_else(|| { - value - .get("color") - .and_then(Value::as_str) - .and_then(graph_calendar_color_to_hex) - .map(str::to_owned) - }); - let is_default = value - .get("isDefaultCalendar") - .and_then(Value::as_bool) - .unwrap_or(false); - let existing = local.get(graph_id).copied(); - let tz = mailbox_tz.clone(); - let local_id = if let Some(id) = existing { - tx.execute( - "UPDATE calendars SET name = ?1, color = ?2, is_default = ?3, time_zone = ?4 - WHERE id = ?5", - params![name, color, is_default as i64, tz, id], - )?; - counts.fetched += 1; - id - } else { - tx.execute( - "INSERT INTO calendars (name, color, sort_order, is_subscribed, is_visible, - is_default, include_in_availability, time_zone) - VALUES (?1, ?2, 0, 1, 1, ?3, 'all', ?4)", - params![name, color, is_default as i64, tz], - )?; - let new_id = tx.last_insert_rowid(); - exchange_graph_ids::insert( - &tx, - ctx.source_id, - exchange_graph_ids::CALENDAR, - graph_id, - new_id, - )?; - counts.created += 1; - new_id - }; - out.push(CalendarFolder { - graph_id: graph_id.to_owned(), - local_id, - }); - } - tx.commit()?; - } - - delete_vanished_flat( - conn, - ctx.source_id, - exchange_graph_ids::CALENDAR, - "calendars", - &local, - &server_ids, - counts, - )?; - Ok(out) -} - -pub fn reconcile_address_books( - conn: &mut Connection, - ctx: &GraphCoordinator<'_>, - counts: &mut TypeCounts, -) -> Result, Error> { - let mut server = - api::collect_all_values(ctx.client, &ctx.endpoints.contact_folders(ctx.top), &[]) - .map_err(Error::from)?; - let mut seen: std::collections::HashSet = server - .iter() - .filter_map(|f| f.get("id").and_then(Value::as_str)) - .map(str::to_owned) - .collect(); - let mut default_id: Option = None; - if let Some(default) = default_contact_folder(ctx) - && let Some(id) = default.get("id").and_then(Value::as_str) - { - default_id = Some(id.to_owned()); - if seen.insert(id.to_owned()) { - server.insert(0, default); - } - } - let mut frontier: Vec = seen.iter().cloned().collect(); - while let Some(parent) = frontier.pop() { - let url = ctx.endpoints.contact_folder_children(&parent, ctx.top); - let children = api::collect_all_values(ctx.client, &url, &[]).map_err(Error::from)?; - for child in children { - let Some(id) = child.get("id").and_then(Value::as_str) else { - continue; - }; - if seen.insert(id.to_owned()) { - frontier.push(id.to_owned()); - server.push(child); - } - } - } - - let local: HashMap = - exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::ADDRESS_BOOK)?; - let mut out: Vec = Vec::new(); - let mut server_ids: Vec = Vec::new(); - for chunk in server.chunks(CHUNK_SIZE) { - let tx = conn.unchecked_transaction()?; - for value in chunk { - let Some(graph_id) = value.get("id").and_then(Value::as_str) else { - continue; - }; - server_ids.push(graph_id.to_owned()); - let name = value - .get("displayName") - .and_then(Value::as_str) - .unwrap_or("Contacts") - .to_owned(); - let is_default = default_id.as_deref() == Some(graph_id); - let existing = local.get(graph_id).copied(); - let local_id = if let Some(id) = existing { - let is_default = crate::db::defaults::unique_default( - &tx, - ObjectType::AddressBook, - is_default, - Some(id), - )?; - tx.execute( - "UPDATE address_books SET name = ?1, is_default = ?2 WHERE id = ?3", - params![name, is_default as i64, id], - )?; - counts.fetched += 1; - id - } else { - let is_default = crate::db::defaults::unique_default( - &tx, - ObjectType::AddressBook, - is_default, - None, - )?; - tx.execute( - "INSERT INTO address_books (name, sort_order, is_default, is_subscribed) - VALUES (?1, 0, ?2, 1)", - params![name, is_default as i64], - )?; - let new_id = tx.last_insert_rowid(); - exchange_graph_ids::insert( - &tx, - ctx.source_id, - exchange_graph_ids::ADDRESS_BOOK, - graph_id, - new_id, - )?; - counts.created += 1; - new_id - }; - out.push(ContactFolder { - graph_id: graph_id.to_owned(), - local_id, - }); - } - tx.commit()?; - } - - delete_vanished_flat( - conn, - ctx.source_id, - exchange_graph_ids::ADDRESS_BOOK, - "address_books", - &local, - &server_ids, - counts, - )?; - Ok(out) -} - -fn default_contact_folder(ctx: &GraphCoordinator<'_>) -> Option { - match ctx.client.get_json(&ctx.endpoints.default_contact_folder()) { - Ok(value) if value.get("id").and_then(Value::as_str).is_some() => return Some(value), - Ok(_) => {} - Err(e) => ctx.logger.warn(&format!( - "graph default contact folder lookup failed ({e}); \ - falling back to deriving it from an existing contact" - )), - } - let probe = ctx - .client - .get_json(&ctx.endpoints.any_contact_parent()) - .ok()?; - let parent = probe - .get("value") - .and_then(Value::as_array)? - .first()? - .get("parentFolderId") - .and_then(Value::as_str)?; - ctx.client - .get_json(&ctx.endpoints.contact_folder(parent)) - .ok() -} - -fn mailbox_timezone(ctx: &GraphCoordinator<'_>) -> Option { - let url = ctx.endpoints.mailbox_settings_timezone(); - let value = ctx.client.get_json(&url).ok()?; - let tz = value.get("timeZone").and_then(Value::as_str)?; - windows_or_iana_to_iana(tz) -} - -fn resolve_well_known_roles(ctx: &GraphCoordinator<'_>) -> HashMap { - let mapping: &[(&str, &str)] = &[ - ("inbox", "inbox"), - ("drafts", "drafts"), - ("sentitems", "sent"), - ("deleteditems", "trash"), - ("junkemail", "junk"), - ("archive", "archive"), - ]; - let mut out = HashMap::new(); - for (short_name, role) in mapping { - let url = ctx.endpoints.well_known_folder(short_name); - match ctx.client.get_json(&url) { - Ok(value) => { - if let Some(id) = value.get("id").and_then(Value::as_str) { - out.insert(id.to_owned(), *role); - } - } - Err(_) => continue, - } - } - out -} - -fn order_by_parent(server: Vec) -> Vec { - let mut parents: HashMap> = HashMap::new(); - for v in &server { - if let Some(id) = v.get("id").and_then(Value::as_str) { - let parent = v - .get("parentFolderId") - .and_then(Value::as_str) - .map(str::to_owned); - parents.insert(id.to_owned(), parent); - } - } - fn depth_of( - id: &str, - parents: &HashMap>, - memo: &mut HashMap, - seen: &mut std::collections::HashSet, - ) -> usize { - if let Some(d) = memo.get(id) { - return *d; - } - if !seen.insert(id.to_owned()) { - return 0; - } - let parent = parents.get(id).and_then(|p| p.as_deref()); - let d = match parent { - Some(p) if parents.contains_key(p) => 1 + depth_of(p, parents, memo, seen), - _ => 0, - }; - memo.insert(id.to_owned(), d); - d - } - let mut memo: HashMap = HashMap::new(); - let mut entries: Vec<(usize, Value)> = server - .into_iter() - .map(|v| { - let id = v.get("id").and_then(Value::as_str).unwrap_or("").to_owned(); - let mut seen = std::collections::HashSet::new(); - let d = depth_of(&id, &parents, &mut memo, &mut seen); - (d, v) - }) - .collect(); - entries.sort_by_key(|(d, _)| *d); - entries.into_iter().map(|(_, v)| v).collect() -} - -fn delete_vanished_flat( - conn: &mut Connection, - source_id: i64, - type_name: &str, - table: &str, - local: &HashMap, - server_ids: &[String], - counts: &mut TypeCounts, -) -> Result<(), Error> { - let server_set: std::collections::HashSet<&str> = - server_ids.iter().map(String::as_str).collect(); - let vanished: Vec<(&String, &i64)> = local - .iter() - .filter(|(graph_id, _)| !server_set.contains(graph_id.as_str())) - .collect(); - for chunk in vanished.chunks(CHUNK_SIZE) { - let tx = conn.unchecked_transaction()?; - for (graph_id, local_id) in chunk { - let result = tx.execute( - &format!("DELETE FROM {table} WHERE id = ?1"), - params![local_id], - ); - match result { - Ok(_) => { - exchange_graph_ids::delete(&tx, source_id, type_name, graph_id)?; - counts.deleted += 1; - } - Err(_) => { - counts.failed += 1; - } - } - } - tx.commit()?; - } - Ok(()) -} - -fn delete_vanished_mailboxes( - conn: &mut Connection, - source_id: i64, - local: &HashMap, - server_ids: &[String], - counts: &mut TypeCounts, - logger: &crate::logging::Logger, -) -> Result<(), Error> { - let server_set: std::collections::HashSet<&str> = - server_ids.iter().map(String::as_str).collect(); - let mut vanished: Vec<(String, i64)> = local - .iter() - .filter(|(id, _)| !server_set.contains(id.as_str())) - .map(|(id, lid)| (id.clone(), *lid)) - .collect(); - let depths = mailbox_depths(conn, &vanished)?; - vanished.sort_by_key(|(_, local_id)| std::cmp::Reverse(*depths.get(local_id).unwrap_or(&0))); - - for (graph_id, local_id) in vanished { - let tx = conn.unchecked_transaction()?; - let result = tx.execute("DELETE FROM mailboxes WHERE id = ?1", params![local_id]); - match result { - Ok(_) => { - exchange_graph_ids::delete(&tx, source_id, exchange_graph_ids::MAILBOX, &graph_id)?; - tx.commit()?; - counts.deleted += 1; - } - Err(e) => { - let _ = tx.rollback(); - logger.warn(&format!( - "mailbox {graph_id} (local id {local_id}) could not be deleted (live children?): {e}" - )); - counts.failed += 1; - } - } - } - Ok(()) -} - -fn mailbox_depths(conn: &Connection, rows: &[(String, i64)]) -> Result, Error> { - let mut parents: HashMap> = HashMap::new(); - { - let mut stmt = conn.prepare("SELECT id, parent_id FROM mailboxes")?; - let mut iter = stmt.query([])?; - while let Some(row) = iter.next()? { - let id: i64 = row.get(0)?; - let parent: Option = row.get(1)?; - parents.insert(id, parent); - } - } - let mut depths: HashMap = HashMap::new(); - for (_, local_id) in rows { - depths.insert(*local_id, depth_of(*local_id, &parents)); - } - Ok(depths) -} - -fn depth_of(id: i64, parents: &HashMap>) -> usize { - let mut depth = 0; - let mut cursor = id; - let mut seen: std::collections::HashSet = std::collections::HashSet::new(); - while seen.insert(cursor) { - match parents.get(&cursor).copied().flatten() { - Some(p) => { - depth += 1; - cursor = p; - } - None => break, - } - } - depth -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn order_by_parent_returns_parents_first() { - let input = vec![ - json!({"id": "C", "parentFolderId": "B"}), - json!({"id": "B", "parentFolderId": "A"}), - json!({"id": "A"}), - ]; - let out = order_by_parent(input); - assert_eq!(out[0]["id"], "A"); - assert_eq!(out[1]["id"], "B"); - assert_eq!(out[2]["id"], "C"); - } - - #[test] - fn order_by_parent_tolerates_missing_parent() { - let input = vec![ - json!({"id": "Orphan", "parentFolderId": "Missing"}), - json!({"id": "A"}), - ]; - let out = order_by_parent(input); - assert_eq!(out.len(), 2); - } -} diff --git a/src/sync/import_exchange_graph/messages.rs b/src/sync/import_exchange_graph/messages.rs deleted file mode 100644 index 041a834..0000000 --- a/src/sync/import_exchange_graph/messages.rs +++ /dev/null @@ -1,246 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use rusqlite::{Connection, Transaction, params}; -use serde_json::{Value, json}; - -use crate::db::{blobs, exchange_graph_ids}; -use crate::error::Error; -use crate::exchange_graph::api; -use crate::exchange_graph::client::Accept; -use crate::exchange_graph::error::GraphError; -use crate::logging::LEVEL_PROGRESS; -use crate::sync::TypeCounts; -use crate::sync::emailmeta::{email_meta_from_blob, jmap_keyword}; -use crate::sync::import_jmap::pool::Pool; -use crate::sync::keys::index_to_json; - -use super::coordinator::{CHUNK_SIZE, GraphCoordinator}; -use super::folders::MailFolder; - -pub fn reconcile_all( - conn: &mut Connection, - ctx: &GraphCoordinator<'_>, - folders: &[MailFolder], - counts: &mut TypeCounts, -) -> Result<(), Error> { - let local: std::collections::HashMap = - exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::EMAIL)?; - let mut server_total: std::collections::HashSet = std::collections::HashSet::new(); - let mut planned: std::collections::HashSet = std::collections::HashSet::new(); - let mut any_failure = false; - - for folder in folders { - let url = ctx.endpoints.folder_messages_ids(&folder.graph_id, ctx.top); - let stubs = match api::collect_all_values(ctx.client, &url, &[]) { - Ok(v) => v, - Err(e) => { - ctx.logger.warn(&format!( - "folder {} message enumeration failed: {e}", - folder.graph_id - )); - counts.failed += 1; - any_failure = true; - continue; - } - }; - if ctx.logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "graph folder {} enumerated {} messages", - folder.graph_id, - stubs.len() - ); - } - let mut new_ids: Vec<(String, String)> = Vec::new(); - for stub in &stubs { - let Some(id) = stub.get("id").and_then(Value::as_str) else { - continue; - }; - server_total.insert(id.to_owned()); - if local.contains_key(id) { - counts.fetched += 1; - } else if planned.insert(id.to_owned()) { - new_ids.push((id.to_owned(), keyword_array(stub))); - } - } - if new_ids.is_empty() { - continue; - } - fetch_and_insert(conn, ctx, folder, &new_ids, counts)?; - } - - if any_failure { - ctx.logger.warn( - "graph message vanished-cleanup skipped: one or more folders failed to enumerate; \ - a clean re-run will reconcile deletions", - ); - } else { - delete_vanished(conn, ctx.source_id, &local, &server_total, counts)?; - } - Ok(()) -} - -fn fetch_and_insert( - conn: &mut Connection, - ctx: &GraphCoordinator<'_>, - folder: &MailFolder, - ids: &[(String, String)], - counts: &mut TypeCounts, -) -> Result<(), Error> { - let keywords_by_id: std::collections::HashMap<&str, &str> = ids - .iter() - .map(|(id, kw)| (id.as_str(), kw.as_str())) - .collect(); - let client = ctx.client.clone(); - let endpoints: crate::exchange_graph::api::Endpoints = (*ctx.endpoints).clone(); - type FetchResult = (String, Result, GraphError>); - let pool: Pool = Pool::new(ctx.workers, move |id: String| { - let url = endpoints.message_mime(&id); - match client.get_with_prefer(&url, Accept::Text, &[]) { - Ok(resp) => (id, Ok(resp.body)), - Err(e) => (id, Err(e)), - } - }); - for (id, _) in ids { - pool.submit(id.clone()); - } - let mut tx_opt: Option> = None; - let mut in_batch: usize = 0; - for _ in 0..ids.len() { - let Ok((graph_id, result)) = pool.results().recv() else { - break; - }; - match result { - Ok(bytes) => { - if tx_opt.is_none() { - tx_opt = Some(conn.unchecked_transaction()?); - } - let tx = tx_opt.as_mut().expect("tx is Some"); - let keywords = keywords_by_id - .get(graph_id.as_str()) - .copied() - .unwrap_or("[]"); - apply_message_in_tx(tx, ctx, folder, &graph_id, &bytes, keywords, counts)?; - in_batch += 1; - if in_batch >= CHUNK_SIZE { - if let Some(t) = tx_opt.take() { - t.commit()?; - } - in_batch = 0; - } - } - Err(GraphError::Vanished) => { - counts.skipped += 1; - } - Err(e) => { - counts.failed += 1; - ctx.logger - .warn(&format!("graph message {graph_id} fetch failed: {e}")); - } - } - } - if let Some(t) = tx_opt.take() { - t.commit()?; - } - Ok(()) -} - -fn apply_message_in_tx( - tx: &Transaction<'_>, - ctx: &GraphCoordinator<'_>, - folder: &MailFolder, - graph_id: &str, - bytes: &[u8], - keywords: &str, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let (idx, date_header) = email_meta_from_blob(bytes); - let message_match = index_to_json(&idx); - let received_at = date_header.unwrap_or_else(|| "1970-01-01T00:00:00Z".to_owned()); - let mailbox_ids = json!([folder.local_id]).to_string(); - - let blob_id = blobs::intern_blob(tx, bytes)?; - tx.execute( - "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match) - VALUES (?1, ?2, ?3, ?4, ?5)", - params![blob_id, received_at, mailbox_ids, keywords, message_match], - )?; - let new_id = tx.last_insert_rowid(); - exchange_graph_ids::insert( - tx, - ctx.source_id, - exchange_graph_ids::EMAIL, - graph_id, - new_id, - )?; - counts.created += 1; - Ok(()) -} - -fn keyword_array(stub: &Value) -> String { - let mut kws: Vec = Vec::new(); - if stub.get("isRead").and_then(Value::as_bool) == Some(true) { - kws.push("$seen".to_owned()); - } - if stub.get("isDraft").and_then(Value::as_bool) == Some(true) { - kws.push("$draft".to_owned()); - } - if stub - .get("flag") - .and_then(|f| f.get("flagStatus")) - .and_then(Value::as_str) - == Some("flagged") - { - kws.push("$flagged".to_owned()); - } - if let Some(cats) = stub.get("categories").and_then(Value::as_array) { - for keyword in cats - .iter() - .filter_map(Value::as_str) - .filter_map(jmap_keyword) - { - if !kws.contains(&keyword) { - kws.push(keyword); - } - } - } - Value::Array(kws.into_iter().map(Value::String).collect()).to_string() -} - -fn delete_vanished( - conn: &mut Connection, - source_id: i64, - local: &std::collections::HashMap, - server: &std::collections::HashSet, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let vanished: Vec<(&String, &i64)> = local - .iter() - .filter(|(graph_id, _)| !server.contains(graph_id.as_str())) - .collect(); - for chunk in vanished.chunks(CHUNK_SIZE) { - let tx = conn.unchecked_transaction()?; - for (graph_id, local_id) in chunk { - let result = tx.execute("DELETE FROM emails WHERE id = ?1", params![local_id]); - match result { - Ok(_) => { - exchange_graph_ids::delete( - &tx, - source_id, - exchange_graph_ids::EMAIL, - graph_id, - )?; - counts.deleted += 1; - } - Err(_) => { - counts.failed += 1; - } - } - } - tx.commit()?; - } - Ok(()) -} diff --git a/src/sync/import_imap.rs b/src/sync/import_imap.rs deleted file mode 100644 index 21cc892..0000000 --- a/src/sync/import_imap.rs +++ /dev/null @@ -1,16 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -pub mod fetch; -pub mod folders; -pub mod internaldate; -pub mod keywords; -pub mod messages; -pub mod pool; - -pub mod coordinator; - -pub use coordinator::{ImapAuth, ImapImportConfig, run, run_reporting}; diff --git a/src/sync/import_imap/coordinator.rs b/src/sync/import_imap/coordinator.rs deleted file mode 100644 index 851eeee..0000000 --- a/src/sync/import_imap/coordinator.rs +++ /dev/null @@ -1,1294 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::{HashMap, HashSet}; - -use crossbeam_channel::RecvTimeoutError; -use regex::Regex; -use rusqlite::{Connection, OptionalExtension, params}; -use serde_json::Value; -use time::OffsetDateTime; -use time::format_description::well_known::Rfc3339; -use url::Url; - -use crate::db; -use crate::db::sources::SourceKey; -use crate::error::Error; -use crate::imap::client::{ConnectMode, ImapClient}; -use crate::imap::command; -use crate::imap::error::ImapError; -use crate::imap::response::{NamespaceEntry, Untagged}; -use crate::imap::retry::{ - BackoffState, Disposition, RetryPolicy, classify, is_negotiation_failure, -}; -use crate::imap::transport::Connector; -use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; -use crate::sync::emailmeta::{ - add_mime_keywords, email_index_and_mime_keywords, email_index_from_blob, -}; -use crate::sync::keys::index_to_json; -use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts}; - -use super::fetch; -use super::folders::{ - FolderFilters, ResolvedFolder, apply_filters, collect_from_list, sort_by_depth, - vanished_depth_sort, vanished_folders, -}; -use super::internaldate::imap_internaldate_to_rfc3339; -use super::keywords::translate_flags; -use super::messages::{chunks, diff_uids, select_uids}; -use super::pool::{FetchEvent, FetchJob, WorkerArgs, WorkerPool}; - -const EMAIL_TYPE: &str = "email"; - -#[derive(Clone, Copy)] -pub(super) struct RunOpts { - source_id: i64, - fetch_batch: usize, - include_deleted: bool, - mime_keywords: bool, - logger: Logger, -} - -#[derive(Clone)] -pub(super) struct ControlCtx { - pub connector: std::sync::Arc, - pub endpoint: std::sync::Arc, - pub mode: ConnectMode, - pub auth: ImapAuth, - pub compress: bool, - pub policy: RetryPolicy, - pub backoff: BackoffState, - pub logger: Logger, -} - -pub(super) fn control_run_collect( - client: &mut ImapClient, - ctx: &ControlCtx, - command: &str, -) -> Result { - let mut transient_attempts: u32 = 0; - let mut transport_attempts: u32 = 0; - loop { - match client.run_collect(command) { - Ok(r) => { - ctx.backoff.reset(); - return Ok(r); - } - Err(e) => match classify(&e) { - Disposition::TransportDrop => { - if transport_attempts >= ctx.policy.max_retries { - return Err(Error::Connection(format!("control connection lost: {e}"))); - } - transport_attempts += 1; - std::thread::sleep(ctx.backoff.transport_delay(transport_attempts)); - match reconnect_control(client, ctx) { - Ok(()) => { - log_at(ctx.logger, LEVEL_DEFAULT, "control connection reconnected"); - } - Err(e2) => { - return Err(Error::Connection(format!( - "control reconnect failed: {e2}" - ))); - } - } - } - Disposition::Transient => { - if transient_attempts >= ctx.policy.max_retries { - return Err(Error::Partial(format!( - "{command:?}: {e} (max retries exhausted)" - ))); - } - transient_attempts += 1; - std::thread::sleep(ctx.backoff.next_shared_delay()); - } - _ => { - return Err(Error::Partial(format!("{command:?}: {e}"))); - } - }, - } - } -} - -pub(super) fn select_folder( - client: &mut ImapClient, - ctx: &ControlCtx, - folder: &ResolvedFolder, -) -> Result { - control_run_collect(client, ctx, &command::select(&folder.wire_name)) -} - -pub(super) fn call_with_retry( - client: &mut ImapClient, - ctx: &ControlCtx, - mut attempt: F, -) -> Result -where - F: FnMut(&mut ImapClient) -> Result, -{ - let mut transient_attempts: u32 = 0; - let mut transport_attempts: u32 = 0; - loop { - match attempt(client) { - Ok(v) => { - ctx.backoff.reset(); - return Ok(v); - } - Err(e) => match classify(&e) { - Disposition::TransportDrop => { - if transport_attempts >= ctx.policy.max_retries { - return Err(e); - } - transport_attempts += 1; - std::thread::sleep(ctx.backoff.transport_delay(transport_attempts)); - reconnect_control(client, ctx)?; - } - Disposition::Transient => { - if transient_attempts >= ctx.policy.max_retries { - return Err(e); - } - transient_attempts += 1; - std::thread::sleep(ctx.backoff.next_shared_delay()); - } - _ => return Err(e), - }, - } - } -} - -fn reconnect_control(client: &mut ImapClient, ctx: &ControlCtx) -> Result<(), ImapError> { - let new_client = ImapClient::connect( - &ctx.connector, - &ctx.endpoint.host, - ctx.endpoint.port, - ctx.mode, - ctx.logger, - )?; - *client = new_client; - authenticate_client(client, &ctx.auth).map_err(|e| ImapError::AuthFailed(e.to_string()))?; - let _ = client.refresh_capabilities(); - if ctx.compress && client.has_capability("COMPRESS=DEFLATE") { - client.compress_deflate()?; - } - if client.has_capability("ENABLE") && client.has_capability("UTF8=ACCEPT") { - let _ = client.enable(&["UTF8=ACCEPT"]); - } - Ok(()) -} - -fn log_at(logger: Logger, level: u8, msg: &str) { - if logger.enabled(level) { - println!("{msg}"); - } -} - -pub struct ImapImportConfig { - pub url: String, - pub auth: ImapAuth, - pub allow_cleartext: bool, - pub compress: bool, - pub include: Vec, - pub exclude: Vec, - pub exclude_special: Vec, - pub folder: Vec, - pub subscribed_only: bool, - pub automap: bool, - pub include_deleted: bool, - pub mime_keywords: bool, - pub fetch_batch: usize, - pub imap_connections: usize, - pub allow_source_change: bool, -} - -#[derive(Debug, Clone)] -pub enum ImapAuth { - Basic { user: String, password: String }, - Bearer { user: String, token: String }, -} - -pub fn run(common: CommonConfig, config: ImapImportConfig) -> Result { - run_reporting(common, config).into_result() -} - -pub fn run_reporting(common: CommonConfig, config: ImapImportConfig) -> RunOutcome { - let mut summary = Summary::default(); - let error = run_into(common, config, &mut summary).err(); - RunOutcome { summary, error } -} - -fn run_into( - common: CommonConfig, - config: ImapImportConfig, - summary: &mut Summary, -) -> Result<(), Error> { - let logger = common.logger; - let mut conn = db::init::open(&common.archive)?; - - let endpoint = parse_endpoint(&config.url)?; - let session_url = format!( - "{}://{}:{}", - if endpoint.implicit_tls { - "imaps" - } else { - "imap" - }, - endpoint.host, - endpoint.port - ); - - let connector = std::sync::Arc::new( - Connector::new(common.allow_invalid_certs).map_err(|e| Error::Connection(e.to_string()))?, - ); - let endpoint = std::sync::Arc::new(endpoint); - let mode = if endpoint.implicit_tls { - ConnectMode::ImplicitTls - } else if config.allow_cleartext { - ConnectMode::Plain - } else { - ConnectMode::StartTls - }; - let mut client = ImapClient::connect(&connector, &endpoint.host, endpoint.port, mode, logger) - .map_err(|e| Error::Connection(e.to_string()))?; - - let account_id = authenticate(&mut client, &config.auth)?; - log_at( - logger, - LEVEL_PROGRESS, - &format!("authenticated as {account_id} on {session_url}"), - ); - client - .refresh_capabilities() - .map_err(|e| Error::Connection(format!("post-auth CAPABILITY: {e}")))?; - - if config.compress { - if client.has_capability("COMPRESS=DEFLATE") { - client - .compress_deflate() - .map_err(|e| Error::Connection(format!("COMPRESS DEFLATE: {e}")))?; - log_at(logger, LEVEL_PROGRESS, "COMPRESS DEFLATE enabled"); - } else { - log_at( - logger, - LEVEL_PROGRESS, - "--compress requested but server does not advertise COMPRESS=DEFLATE; ignored", - ); - } - } - - if client.has_capability("ENABLE") && client.has_capability("UTF8=ACCEPT") { - match client.enable(&["UTF8=ACCEPT"]) { - Ok(_) => log_at(logger, LEVEL_PROGRESS, "ENABLE UTF8=ACCEPT"), - Err(e) => log_at( - logger, - LEVEL_PROGRESS, - &format!("ENABLE UTF8=ACCEPT failed, falling back to modified UTF-7: {e}"), - ), - } - } - - let source_key = SourceKey { - kind: "imap".to_owned(), - session_url: session_url.clone(), - account_id: account_id.clone(), - }; - if let Some((existing_url, existing_account)) = - db::sources::conflicting_source(&conn, "imap", &session_url, &account_id)? - && !config.allow_source_change - { - return Err(Error::SourceChange(format!( - "archive already records imap source {existing_url} / {existing_account}; \ - re-run with --allow-source-change or use a fresh archive" - ))); - } - let source_id = if common.dry_run { - -1 - } else { - db::sources::upsert_source(&conn, &source_key, Some(&account_id), &account_id)? - }; - - let backoff = BackoffState::new(); - let policy = RetryPolicy::new(common.max_retries); - let control_ctx = ControlCtx { - connector: connector.clone(), - endpoint: endpoint.clone(), - mode, - auth: config.auth.clone(), - compress: config.compress, - policy, - backoff: backoff.clone(), - logger, - }; - - let namespace_prefix = discover_namespace(&mut client, logger)?; - - let extended_supported = client.has_capability("LIST-EXTENDED"); - let special_use_supported = client.has_capability("SPECIAL-USE"); - let list_status_supported = extended_supported && client.has_capability("LIST-STATUS"); - let list_cmd = if extended_supported { - let mut return_opts: Vec<&str> = Vec::new(); - if special_use_supported { - return_opts.push("SPECIAL-USE"); - } - return_opts.push("SUBSCRIBED"); - if list_status_supported { - return_opts.push("CHILDREN"); - return_opts.push("STATUS (UIDVALIDITY UIDNEXT MESSAGES)"); - } - command::list_extended("", &["*"], &[], &return_opts) - } else { - command::list("", "*") - }; - let list_resp = control_run_collect(&mut client, &control_ctx, &list_cmd)?; - let mut all_responses = list_resp.untagged; - if (!extended_supported || !list_status_supported) - && let Ok(lsub_resp) = - control_run_collect(&mut client, &control_ctx, &command::lsub("", "*")) - { - all_responses.extend(lsub_resp.untagged); - } - - let discovered = collect_from_list(&all_responses, client.utf8_accept()) - .map_err(|e| Error::Partial(format!("LIST parse: {e}")))?; - let filters = FolderFilters { - include: config.include, - exclude: config.exclude, - exclude_special: config.exclude_special, - explicit: config.folder, - subscribed_only: config.subscribed_only, - automap_enabled: config.automap, - namespace_prefix, - }; - let mut resolved = apply_filters(discovered, &filters); - sort_by_depth(&mut resolved); - - if common.dry_run { - let existing_source = db::sources::find_source(&conn, &source_key)?; - *summary = dry_run_summary( - &conn, - existing_source, - &mut client, - &control_ctx, - &resolved, - logger, - )?; - return Ok(()); - } - - let mut mailbox_counts = TypeCounts::default(); - let mut email_counts = TypeCounts::default(); - - let local_mailboxes = db::imap_ids::mailbox_folders(&conn, source_id)?; - let server_folder_set: HashSet = resolved.iter().map(|f| f.name.clone()).collect(); - - upsert_mailboxes(&mut conn, source_id, &resolved, &mut mailbox_counts, logger)?; - - let server_delimiter = resolved.iter().find_map(|f| f.delimiter).unwrap_or('/'); - - let mut vanished = vanished_folders(&local_mailboxes, &server_folder_set); - vanished_depth_sort(&mut vanished, server_delimiter); - if !vanished.is_empty() { - delete_vanished_folders( - &mut conn, - source_id, - &vanished, - &mut mailbox_counts, - &mut email_counts, - logger, - )?; - } - - let opts = RunOpts { - source_id, - fetch_batch: config.fetch_batch.max(1), - include_deleted: config.include_deleted, - mime_keywords: config.mime_keywords, - logger, - }; - - let pool = WorkerPool::start( - WorkerArgs { - connector: connector.clone(), - endpoint: endpoint.clone(), - mode, - auth: config.auth.clone(), - compress: config.compress, - policy, - backoff: backoff.clone(), - logger, - }, - config.imap_connections.max(1), - ) - .map_err(|e| Error::Connection(format!("worker pool: {e}")))?; - - for (i, folder) in resolved.iter().filter(|f| f.selectable).enumerate() { - if i > 0 { - let _ = control_run_collect(&mut client, &control_ctx, "NOOP"); - } - match reconcile_folder( - &mut conn, - &mut client, - &control_ctx, - &pool, - folder, - opts, - &mut email_counts, - ) { - Ok(()) => {} - Err(e) if e.aborts_run() => { - pool.shutdown(); - let _ = client.logout(); - *summary = Summary { - per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)], - retries_observed: backoff.total_retries(), - retry_after_sleeps: backoff.transient_retries() as u64, - }; - return Err(e); - } - Err(e) => { - log_at( - logger, - LEVEL_DEFAULT, - &format!("folder {:?}: {e}", folder.name), - ); - email_counts.failed += 1; - } - } - } - - pool.shutdown(); - let _ = client.logout(); - - *summary = Summary { - per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)], - retries_observed: backoff.total_retries(), - retry_after_sleeps: backoff.transient_retries() as u64, - }; - Ok(()) -} - -#[derive(Debug, Clone)] -pub struct Endpoint { - pub host: String, - pub port: u16, - pub implicit_tls: bool, -} - -fn parse_endpoint(url: &str) -> Result { - let parsed = - Url::parse(url).map_err(|e| Error::Usage(format!("invalid --url {url:?}: {e}")))?; - let scheme = parsed.scheme(); - let implicit_tls = match scheme { - "imaps" => true, - "imap" => false, - other => { - return Err(Error::Usage(format!( - "--url scheme must be imap or imaps, got {other}" - ))); - } - }; - let host = parsed - .host_str() - .ok_or_else(|| Error::Usage(format!("--url missing host: {url}")))? - .to_owned(); - let port = parsed - .port() - .unwrap_or(if implicit_tls { 993 } else { 143 }); - Ok(Endpoint { - host, - port, - implicit_tls, - }) -} - -pub(super) fn authenticate_client(client: &mut ImapClient, auth: &ImapAuth) -> Result<(), Error> { - authenticate(client, auth).map(|_| ()) -} - -fn authenticate(client: &mut ImapClient, auth: &ImapAuth) -> Result { - match auth { - ImapAuth::Basic { user, password } => { - if client.has_capability("AUTH=PLAIN") { - match client.authenticate_plain(user, password) { - Ok(()) => return Ok(user.clone()), - Err(e) if is_auth_failed(&e) => { - return Err(Error::Connection(format!("auth failed: {e}"))); - } - Err(e) if is_negotiation_failure(&e) => {} - Err(e) => return Err(Error::Connection(e.to_string())), - } - } - match client.login(user, password) { - Ok(()) => Ok(user.clone()), - Err(e) if is_auth_failed(&e) => Err(Error::Connection(format!("auth failed: {e}"))), - Err(e) => Err(Error::Connection(format!("LOGIN failed: {e}"))), - } - } - ImapAuth::Bearer { user, token } => { - let has_xoauth2 = client.has_capability("AUTH=XOAUTH2"); - let has_oauthbearer = client.has_capability("AUTH=OAUTHBEARER"); - if !has_xoauth2 && !has_oauthbearer { - return Err(Error::Connection( - "server does not advertise AUTH=XOAUTH2 or AUTH=OAUTHBEARER".into(), - )); - } - if has_xoauth2 { - match client.authenticate_xoauth2(user, token) { - Ok(()) => return Ok(user.clone()), - Err(e) if is_auth_failed(&e) => { - return Err(Error::Connection(format!("auth failed: {e}"))); - } - Err(e) if is_negotiation_failure(&e) && has_oauthbearer => {} - Err(e) => return Err(Error::Connection(format!("XOAUTH2 failed: {e}"))), - } - } - client - .authenticate_oauthbearer(user, token) - .map_err(|e| Error::Connection(format!("OAUTHBEARER failed: {e}")))?; - Ok(user.clone()) - } - } -} - -fn is_auth_failed(e: &ImapError) -> bool { - match e { - ImapError::AuthFailed(_) => true, - ImapError::No(no) => no.is_auth_failed(), - _ => false, - } -} - -fn discover_namespace(client: &mut ImapClient, logger: Logger) -> Result { - if !client.has_capability("NAMESPACE") { - return Ok(String::new()); - } - let resp = match client.run_collect(command::namespace()) { - Ok(r) => r, - Err(e) => { - log_at( - logger, - LEVEL_PROGRESS, - &format!("NAMESPACE failed, ignoring: {e}"), - ); - return Ok(String::new()); - } - }; - for u in resp.untagged { - if let Untagged::Namespace { personal, .. } = u - && let Some(NamespaceEntry { prefix, .. }) = personal.into_iter().next() - { - return Ok(prefix); - } - } - Ok(String::new()) -} - -fn upsert_mailboxes( - conn: &mut Connection, - source_id: i64, - folders: &[ResolvedFolder], - counts: &mut TypeCounts, - logger: Logger, -) -> Result<(), Error> { - let tx = conn.transaction()?; - let mut local_ids: HashMap = HashMap::new(); - for folder in folders { - let parent_local = match &folder.parent_path { - Some(p) => { - let found = match local_ids.get(p.as_str()) { - Some(id) => Some(*id), - None => db::imap_ids::local_for_mailbox(&tx, source_id, p)?, - }; - if found.is_none() { - log_at( - logger, - LEVEL_DEFAULT, - &format!( - "folder {:?}: parent {p:?} could not be resolved; storing it at the top level", - folder.name - ), - ); - } - found - } - None => None, - }; - let existing = db::imap_ids::local_for_mailbox(&tx, source_id, &folder.name)?; - let id = if let Some(id) = existing { - let role = db::roles::unique_role(&tx, folder.role, Some(id))?; - tx.execute( - "UPDATE mailboxes SET name = ?1, parent_id = ?2, role = ?3, - is_subscribed = ?4 WHERE id = ?5", - params![ - folder.leaf, - parent_local, - role, - folder.subscribed as i64, - id - ], - )?; - id - } else { - let role = db::roles::unique_role(&tx, folder.role, None)?; - tx.execute( - "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) - VALUES (?1, ?2, ?3, 0, ?4)", - params![folder.leaf, parent_local, role, folder.subscribed as i64], - )?; - let new_id = tx.last_insert_rowid(); - db::imap_ids::insert_mailbox(&tx, source_id, &folder.name, new_id)?; - counts.created += 1; - counts.fetched += 1; - new_id - }; - local_ids.insert(folder.name.clone(), id); - } - tx.commit()?; - Ok(()) -} - -fn delete_vanished_folders( - conn: &mut Connection, - source_id: i64, - folders: &[String], - mailbox_counts: &mut TypeCounts, - email_counts: &mut TypeCounts, - logger: Logger, -) -> Result<(), Error> { - let tx = conn.transaction()?; - for name in folders { - let local_id = match db::imap_ids::local_for_mailbox(&tx, source_id, name)? { - Some(id) => id, - None => continue, - }; - let surviving_child: Option = tx - .query_row( - "SELECT id FROM mailboxes WHERE parent_id = ?1 LIMIT 1", - params![local_id], - |row| row.get(0), - ) - .optional()?; - if surviving_child.is_some() { - log_at( - logger, - LEVEL_DEFAULT, - &format!( - "folder {name:?}: vanished from server but still has child mailboxes in the archive; skipping delete (parent_id RESTRICT)" - ), - ); - mailbox_counts.failed += 1; - continue; - } - let email_ids: Vec = tx - .prepare( - "SELECT local_id FROM sync_id_imap - WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3", - )? - .query_map(params![source_id, EMAIL_TYPE, name], |row| row.get(0))? - .collect::, _>>()?; - for eid in &email_ids { - tx.execute("DELETE FROM emails WHERE id = ?1", params![eid])?; - email_counts.deleted += 1; - } - db::imap_ids::delete_all_emails_in_folder(&tx, source_id, name)?; - db::imap_state::delete(&tx, source_id, name)?; - tx.execute("DELETE FROM mailboxes WHERE id = ?1", params![local_id])?; - db::imap_ids::delete_mailbox(&tx, source_id, name)?; - mailbox_counts.deleted += 1; - } - tx.commit()?; - Ok(()) -} - -fn reconcile_folder( - conn: &mut Connection, - client: &mut ImapClient, - control_ctx: &ControlCtx, - pool: &WorkerPool, - folder: &ResolvedFolder, - opts: RunOpts, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let RunOpts { - source_id, - fetch_batch, - include_deleted: _, - mime_keywords: _, - logger, - } = opts; - - if let Some(status) = folder.status.as_ref() - && status.messages == Some(0) - && let Some(uv) = status.uidvalidity - && let Some(un) = status.uidnext - { - let local = db::imap_ids::email_uids_in_folder(conn, source_id, &folder.name)?; - if local.is_empty() { - let now = OffsetDateTime::now_utc() - .format(&Rfc3339) - .unwrap_or_else(|_| String::from("1970-01-01T00:00:00Z")); - db::imap_state::upsert(conn, source_id, &folder.name, uv as u32, un as u32, &now)?; - log_at( - logger, - LEVEL_PROGRESS, - &format!("folder {:?}: empty (LIST-STATUS), skipped", folder.name), - ); - return Ok(()); - } - } - - let resp = select_folder(client, control_ctx, folder)?; - let mut uidvalidity: u32 = 0; - let mut uidnext: u32 = 0; - for u in &resp.untagged { - if let Untagged::StatusLine(line) = u - && let Some(code) = &line.code - && let Some(args) = &line.code_args - { - match code.as_str() { - "UIDVALIDITY" => { - if let Ok(n) = args.parse() { - uidvalidity = n; - } - } - "UIDNEXT" => { - if let Ok(n) = args.parse() { - uidnext = n; - } - } - _ => {} - } - } - } - if uidvalidity == 0 { - return Err(Error::Partial(format!( - "SELECT {:?}: server gave no UIDVALIDITY", - folder.name - ))); - } - - if let Some(prev) = db::imap_state::get(conn, source_id, &folder.name)? - && prev.uidvalidity != uidvalidity - { - log_at( - logger, - LEVEL_DEFAULT, - &format!( - "folder {:?}: UIDVALIDITY changed {} -> {} (wiping local rows)", - folder.name, prev.uidvalidity, uidvalidity - ), - ); - wipe_folder_emails(conn, source_id, &folder.name, counts)?; - } - - let server_uids = call_with_retry(client, control_ctx, select_uids) - .map_err(|e| Error::Partial(format!("UID enum {:?}: {e}", folder.name)))?; - let local_uids = - db::imap_ids::email_uids_at_validity(conn, source_id, &folder.name, uidvalidity)?; - let diff = diff_uids(&local_uids, &server_uids); - - log_at( - logger, - LEVEL_PROGRESS, - &format!( - "folder {:?}: new={} vanished={} present={}", - folder.name, - diff.new.len(), - diff.vanished.len(), - diff.present.len() - ), - ); - - if !diff.vanished.is_empty() { - delete_vanished_emails( - conn, - source_id, - &folder.name, - uidvalidity, - &diff.vanished, - counts, - )?; - } - - if !diff.new.is_empty() { - let mailbox_local = db::imap_ids::local_for_mailbox(conn, source_id, &folder.name)? - .ok_or_else(|| { - Error::Partial(format!( - "mailbox {:?} missing from sync_id_imap", - folder.name - )) - })?; - let batches: Vec<&[u32]> = chunks(&diff.new, fetch_batch); - let n_batches = batches.len(); - for batch in &batches { - pool.submit(FetchJob { - folder: folder.name.clone(), - wire_name: folder.wire_name.clone(), - uidvalidity, - uids: batch.to_vec(), - }); - } - let keepalive_interval = std::time::Duration::from_secs(45); - let mut chunks_done: usize = 0; - let tx = conn.transaction()?; - let target = FetchTarget { - folder: folder.name.as_str(), - uidvalidity, - mailbox_local, - }; - while chunks_done < n_batches { - let event = loop { - match pool.recv_timeout(keepalive_interval) { - Ok(r) => break r, - Err(RecvTimeoutError::Timeout) => { - let _ = control_run_collect(client, control_ctx, "NOOP"); - } - Err(RecvTimeoutError::Disconnected) => { - return Err(Error::Partial("worker channel disconnected".to_owned())); - } - } - }; - match event { - FetchEvent::Item { attrs, .. } => { - insert_single_message(&tx, &target, &attrs, opts, counts)?; - } - FetchEvent::ChunkDone { - folder: chunk_folder, - outcome, - uids_requested, - .. - } => { - chunks_done += 1; - if let Err(e) = outcome { - log_at( - logger, - LEVEL_DEFAULT, - &format!("UID FETCH chunk failed for folder {chunk_folder:?}: {e}",), - ); - counts.failed += uids_requested.len() as u64; - } - } - } - } - tx.commit()?; - } - - if !diff.present.is_empty() { - counts.updated += refresh_present_flags( - conn, - client, - control_ctx, - &folder.name, - uidvalidity, - &diff.present, - opts, - )?; - } - - let now = OffsetDateTime::now_utc() - .format(&Rfc3339) - .unwrap_or_else(|_| String::from("1970-01-01T00:00:00Z")); - db::imap_state::upsert(conn, source_id, &folder.name, uidvalidity, uidnext, &now)?; - Ok(()) -} - -fn wipe_folder_emails( - conn: &mut Connection, - source_id: i64, - folder: &str, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let tx = conn.transaction()?; - let ids: Vec = tx - .prepare( - "SELECT local_id FROM sync_id_imap - WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3", - )? - .query_map(params![source_id, EMAIL_TYPE, folder], |row| row.get(0))? - .collect::, _>>()?; - for id in &ids { - tx.execute("DELETE FROM emails WHERE id = ?1", params![id])?; - counts.deleted += 1; - } - db::imap_ids::delete_all_emails_in_folder(&tx, source_id, folder)?; - tx.commit()?; - Ok(()) -} - -fn delete_vanished_emails( - conn: &mut Connection, - source_id: i64, - folder: &str, - uidvalidity: u32, - uids: &[u32], - counts: &mut TypeCounts, -) -> Result<(), Error> { - let tx = conn.transaction()?; - for &uid in uids { - if let Some(local_id) = - db::imap_ids::local_for_email(&tx, source_id, folder, uidvalidity, uid)? - { - tx.execute("DELETE FROM emails WHERE id = ?1", params![local_id])?; - db::imap_ids::delete_email(&tx, source_id, folder, uidvalidity, uid)?; - counts.deleted += 1; - } - } - tx.commit()?; - Ok(()) -} - -pub(super) struct FetchTarget<'a> { - pub folder: &'a str, - pub uidvalidity: u32, - pub mailbox_local: i64, -} - -fn insert_single_message( - tx: &rusqlite::Transaction<'_>, - target: &FetchTarget<'_>, - attrs: &fetch::FetchAttrs, - opts: RunOpts, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let RunOpts { - source_id, - fetch_batch: _, - include_deleted, - mime_keywords, - logger, - } = opts; - let folder = target.folder; - let uidvalidity = target.uidvalidity; - let mailbox_local = target.mailbox_local; - - let Some(uid) = attrs.uid else { - return Ok(()); - }; - let Some(body) = attrs.body.as_ref() else { - counts.skipped += 1; - return Ok(()); - }; - if let Some(declared_size) = attrs.size - && (declared_size as usize) != body.len() - { - log_at( - logger, - LEVEL_DEFAULT, - &format!( - "folder {folder:?} uid {uid}: BODY[] {} bytes vs RFC822.SIZE {declared_size}, importing the fetched literal", - body.len() - ), - ); - } - let translation = translate_flags(&attrs.flags, include_deleted); - if translation.has_deleted_flag && !include_deleted { - counts.skipped += 1; - return Ok(()); - } - let received_at = match attrs.internaldate.as_deref() { - Some(s) => imap_internaldate_to_rfc3339(s)?, - None => OffsetDateTime::now_utc() - .format(&Rfc3339) - .unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_owned()), - }; - let blob_id = db::blobs::intern_blob(tx, body)?; - let mut keywords = translation.keywords; - let index = if mime_keywords { - email_index_and_mime_keywords(body, &mut keywords) - } else { - email_index_from_blob(body) - }; - let message_match = index_to_json(&index); - let mailbox_ids = Value::Array(vec![Value::from(mailbox_local)]); - let keywords = Value::Array(keywords.into_iter().map(Value::String).collect()); - tx.execute( - "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match) - VALUES (?1, ?2, ?3, ?4, ?5)", - params![ - blob_id, - received_at, - mailbox_ids.to_string(), - keywords.to_string(), - message_match, - ], - )?; - let email_local = tx.last_insert_rowid(); - db::imap_ids::insert_email(tx, source_id, folder, uidvalidity, uid, email_local)?; - counts.created += 1; - counts.fetched += 1; - Ok(()) -} - -fn refresh_present_flags( - conn: &mut Connection, - client: &mut ImapClient, - control_ctx: &ControlCtx, - folder: &str, - uidvalidity: u32, - present: &[u32], - opts: RunOpts, -) -> Result { - let RunOpts { - source_id, - fetch_batch, - include_deleted, - mime_keywords, - logger: _, - } = opts; - let mut updated: u64 = 0; - let tx = conn.transaction()?; - for batch in chunks(present, fetch_batch.max(1)) { - let set = command::format_uid_set(batch, true); - let resp = control_run_collect( - client, - control_ctx, - &command::uid_fetch(&set, &["UID", "FLAGS"]), - )?; - for u in &resp.untagged { - let Some(attrs) = fetch::extract(u) else { - continue; - }; - let Some(uid) = attrs.uid else { - continue; - }; - let translation = translate_flags(&attrs.flags, include_deleted); - if translation.has_deleted_flag && !include_deleted { - continue; - } - let Some(stored) = load_email_keywords(&tx, source_id, folder, uidvalidity, uid)? - else { - continue; - }; - let mut keywords = translation.keywords; - if mime_keywords && let Some(blob) = db::blobs::blob_bytes(&tx, stored.blob_id)? { - add_mime_keywords(&blob, &mut keywords); - } - if keyword_set_differs(&stored.keywords, &keywords) { - let json = Value::Array(keywords.into_iter().map(Value::String).collect()); - tx.execute( - "UPDATE emails SET keywords = ?1 WHERE id = ?2", - params![json.to_string(), stored.local_id], - )?; - updated += 1; - } - } - } - tx.commit()?; - Ok(updated) -} - -struct StoredKeywords { - local_id: i64, - blob_id: i64, - keywords: Vec, -} - -fn load_email_keywords( - tx: &rusqlite::Transaction<'_>, - source_id: i64, - folder: &str, - uidvalidity: u32, - uid: u32, -) -> Result, Error> { - let Some(local_id) = db::imap_ids::local_for_email(tx, source_id, folder, uidvalidity, uid)? - else { - return Ok(None); - }; - let (blob_id, kw_json): (i64, String) = tx.query_row( - "SELECT blob_id, keywords FROM emails WHERE id = ?1", - params![local_id], - |r| Ok((r.get(0)?, r.get(1)?)), - )?; - let keywords = serde_json::from_str(&kw_json).unwrap_or_default(); - Ok(Some(StoredKeywords { - local_id, - blob_id, - keywords, - })) -} - -fn keyword_set_differs(a: &[String], b: &[String]) -> bool { - let sa: HashSet<&str> = a.iter().map(String::as_str).collect(); - let sb: HashSet<&str> = b.iter().map(String::as_str).collect(); - sa != sb -} - -fn dry_run_summary( - conn: &Connection, - source_id: Option, - client: &mut ImapClient, - control_ctx: &ControlCtx, - folders: &[ResolvedFolder], - logger: Logger, -) -> Result { - let mut mailbox = TypeCounts::default(); - let mut email = TypeCounts::default(); - - let server_set: HashSet = folders.iter().map(|f| f.name.clone()).collect(); - let local_mailboxes = match source_id { - Some(sid) => db::imap_ids::mailbox_folders(conn, sid)?, - None => HashMap::new(), - }; - let new_folders: Vec<&str> = folders - .iter() - .map(|f| f.name.as_str()) - .filter(|n| !local_mailboxes.contains_key(*n)) - .collect(); - let vanished_folder_names: Vec<&str> = local_mailboxes - .keys() - .filter(|n| !server_set.contains(n.as_str())) - .map(|s| s.as_str()) - .collect(); - mailbox.created += new_folders.len() as u64; - mailbox.deleted += vanished_folder_names.len() as u64; - mailbox.fetched += new_folders.len() as u64; - if !new_folders.is_empty() { - log_at( - logger, - LEVEL_PROGRESS, - &format!( - "dry-run: folders new={} vanished={}", - new_folders.len(), - vanished_folder_names.len() - ), - ); - } - - for folder in folders.iter().filter(|f| f.selectable) { - let select_resp = match select_folder(client, control_ctx, folder) { - Ok(r) => r, - Err(e) => { - log_at( - logger, - LEVEL_DEFAULT, - &format!("dry-run SELECT {:?}: {e}", folder.name), - ); - email.failed += 1; - continue; - } - }; - let mut server_uidvalidity: u32 = 0; - for u in &select_resp.untagged { - if let Untagged::StatusLine(line) = u - && line.code.as_deref() == Some("UIDVALIDITY") - && let Some(args) = &line.code_args - && let Ok(n) = args.parse() - { - server_uidvalidity = n; - break; - } - } - let server_uids = match call_with_retry(client, control_ctx, select_uids) { - Ok(v) => v, - Err(e) => { - log_at( - logger, - LEVEL_DEFAULT, - &format!("dry-run UID enum {:?}: {e}", folder.name), - ); - email.failed += 1; - continue; - } - }; - let local_pairs = match source_id { - Some(sid) => db::imap_ids::email_uids_in_folder(conn, sid, &folder.name)?, - None => HashMap::new(), - }; - let prev_state = match source_id { - Some(sid) => db::imap_state::get(conn, sid, &folder.name)?, - None => None, - }; - let uv_changed = prev_state - .as_ref() - .is_some_and(|p| p.uidvalidity != server_uidvalidity && server_uidvalidity != 0); - let local_uids: Vec = if uv_changed { - Vec::new() - } else { - local_pairs - .keys() - .filter(|(v, _)| server_uidvalidity == 0 || *v == server_uidvalidity) - .map(|(_, u)| *u) - .collect() - }; - let diff = diff_uids(&local_uids, &server_uids); - email.created += diff.new.len() as u64; - email.deleted += diff.vanished.len() as u64; - email.fetched += diff.new.len() as u64; - let uv_note = if uv_changed { - format!( - " (UIDVALIDITY {} -> {})", - prev_state.as_ref().map_or(0, |p| p.uidvalidity), - server_uidvalidity, - ) - } else { - String::new() - }; - log_at( - logger, - LEVEL_PROGRESS, - &format!( - "dry-run: {:?} new={} vanished={} present={}{uv_note}", - folder.name, - diff.new.len(), - diff.vanished.len(), - diff.present.len(), - ), - ); - } - for name in &vanished_folder_names { - if let Some(sid) = source_id { - let n = db::imap_ids::email_uids_in_folder(conn, sid, name)?.len() as u64; - email.deleted += n; - log_at( - logger, - LEVEL_PROGRESS, - &format!( - "dry-run: folder {:?} vanished ({n} emails would be removed)", - name - ), - ); - } - } - - let _ = client.logout(); - Ok(Summary { - per_type: vec![("mailbox", mailbox), ("email", email)], - retries_observed: control_ctx.backoff.total_retries(), - retry_after_sleeps: control_ctx.backoff.transient_retries() as u64, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_endpoint_imaps_defaults_to_993() { - let e = parse_endpoint("imaps://mail.example.com").unwrap(); - assert_eq!(e.host, "mail.example.com"); - assert_eq!(e.port, 993); - assert!(e.implicit_tls); - } - - #[test] - fn parse_endpoint_imap_defaults_to_143() { - let e = parse_endpoint("imap://mail.example.com").unwrap(); - assert_eq!(e.host, "mail.example.com"); - assert_eq!(e.port, 143); - assert!(!e.implicit_tls); - } - - #[test] - fn parse_endpoint_explicit_port() { - let e = parse_endpoint("imaps://mail.example.com:1993").unwrap(); - assert_eq!(e.port, 1993); - } - - #[test] - fn parse_endpoint_rejects_other_schemes() { - let err = parse_endpoint("http://example.com").unwrap_err(); - assert!(matches!(err, Error::Usage(_))); - } -} diff --git a/src/sync/import_imap/internaldate.rs b/src/sync/import_imap/internaldate.rs deleted file mode 100644 index 039a7dd..0000000 --- a/src/sync/import_imap/internaldate.rs +++ /dev/null @@ -1,152 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use crate::error::Error; - -pub fn imap_internaldate_to_rfc3339(s: &str) -> Result { - let s = s.trim_start_matches(' ').trim_end(); - let (day, rest) = s - .split_once('-') - .ok_or_else(|| Error::Partial(format!("INTERNALDATE: missing day in {s:?}")))?; - let (mon, rest) = rest - .split_once('-') - .ok_or_else(|| Error::Partial(format!("INTERNALDATE: missing month in {s:?}")))?; - let (year, rest) = rest - .split_once(' ') - .ok_or_else(|| Error::Partial(format!("INTERNALDATE: missing year in {s:?}")))?; - let (time, zone) = rest - .split_once(' ') - .ok_or_else(|| Error::Partial(format!("INTERNALDATE: missing zone in {s:?}")))?; - let (h, ms) = time - .split_once(':') - .ok_or_else(|| Error::Partial(format!("INTERNALDATE: bad time {time:?}")))?; - let (m, sec) = ms - .split_once(':') - .ok_or_else(|| Error::Partial(format!("INTERNALDATE: bad time {time:?}")))?; - let day: u32 = day - .trim() - .parse() - .map_err(|e| Error::Partial(format!("INTERNALDATE day {day:?}: {e}")))?; - let mon_idx = month_to_num(mon)?; - let year: i32 = year - .parse() - .map_err(|e| Error::Partial(format!("INTERNALDATE year {year:?}: {e}")))?; - let h: u32 = h - .parse() - .map_err(|e| Error::Partial(format!("INTERNALDATE hour {h:?}: {e}")))?; - let m: u32 = m - .parse() - .map_err(|e| Error::Partial(format!("INTERNALDATE minute {m:?}: {e}")))?; - let sec: u32 = sec - .parse() - .map_err(|e| Error::Partial(format!("INTERNALDATE second {sec:?}: {e}")))?; - let (sign, hh, mm) = parse_zone(zone)?; - let mut out = String::with_capacity(32); - use std::fmt::Write; - let _ = write!( - &mut out, - "{year:04}-{mon_idx:02}-{day:02}T{h:02}:{m:02}:{sec:02}" - ); - if hh == 0 && mm == 0 { - out.push('Z'); - } else { - let _ = write!(&mut out, "{sign}{hh:02}:{mm:02}"); - } - Ok(out) -} - -fn month_to_num(s: &str) -> Result { - let m = match s { - "Jan" => 1, - "Feb" => 2, - "Mar" => 3, - "Apr" => 4, - "May" => 5, - "Jun" => 6, - "Jul" => 7, - "Aug" => 8, - "Sep" => 9, - "Oct" => 10, - "Nov" => 11, - "Dec" => 12, - other => return Err(Error::Partial(format!("INTERNALDATE month {other:?}"))), - }; - Ok(m) -} - -fn parse_zone(s: &str) -> Result<(char, u32, u32), Error> { - let s = s.trim(); - if s.len() != 5 { - return Err(Error::Partial(format!("INTERNALDATE zone {s:?}"))); - } - let sign = s.as_bytes()[0] as char; - if sign != '+' && sign != '-' { - return Err(Error::Partial(format!("INTERNALDATE zone sign {sign}"))); - } - let hh: u32 = s[1..3] - .parse() - .map_err(|e| Error::Partial(format!("INTERNALDATE zone hour: {e}")))?; - let mm: u32 = s[3..5] - .parse() - .map_err(|e| Error::Partial(format!("INTERNALDATE zone min: {e}")))?; - Ok((sign, hh, mm)) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn utc_zone_becomes_z() { - assert_eq!( - imap_internaldate_to_rfc3339("17-Jul-1996 02:44:25 +0000").unwrap(), - "1996-07-17T02:44:25Z" - ); - } - - #[test] - fn positive_offset_preserved() { - assert_eq!( - imap_internaldate_to_rfc3339("17-Jul-1996 02:44:25 +0200").unwrap(), - "1996-07-17T02:44:25+02:00" - ); - } - - #[test] - fn negative_offset_preserved() { - assert_eq!( - imap_internaldate_to_rfc3339("17-Jul-1996 02:44:25 -0700").unwrap(), - "1996-07-17T02:44:25-07:00" - ); - } - - #[test] - fn single_digit_day_padded() { - assert_eq!( - imap_internaldate_to_rfc3339("5-Jan-2024 09:05:01 +0000").unwrap(), - "2024-01-05T09:05:01Z" - ); - } - - #[test] - fn leading_space_single_digit_day() { - assert_eq!( - imap_internaldate_to_rfc3339(" 5-Jan-2024 09:05:01 +0000").unwrap(), - "2024-01-05T09:05:01Z" - ); - } - - #[test] - fn invalid_month_errors() { - assert!(imap_internaldate_to_rfc3339("1-XYZ-2024 00:00:00 +0000").is_err()); - } - - #[test] - fn malformed_input_errors() { - assert!(imap_internaldate_to_rfc3339("garbage").is_err()); - assert!(imap_internaldate_to_rfc3339("17-Jul-1996").is_err()); - } -} diff --git a/src/sync/import_imap/keywords.rs b/src/sync/import_imap/keywords.rs deleted file mode 100644 index 9dcc74f..0000000 --- a/src/sync/import_imap/keywords.rs +++ /dev/null @@ -1,173 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum FlagAction { - Keep(&'static str), - Drop, - Verbatim, - Deleted, -} - -pub fn translate_system_flag(flag: &str) -> FlagAction { - if !flag.starts_with('\\') { - return FlagAction::Verbatim; - } - match &flag[1..].to_ascii_lowercase()[..] { - "seen" => FlagAction::Keep("$seen"), - "flagged" => FlagAction::Keep("$flagged"), - "answered" => FlagAction::Keep("$answered"), - "draft" => FlagAction::Keep("$draft"), - "recent" => FlagAction::Drop, - "deleted" => FlagAction::Deleted, - _ => FlagAction::Verbatim, - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Translation { - pub keywords: Vec, - pub has_deleted_flag: bool, -} - -pub fn translate_flags(imap_flags: &[String], include_deleted: bool) -> Translation { - let mut out: Vec = Vec::with_capacity(imap_flags.len()); - let mut has_deleted = false; - for f in imap_flags { - if f.is_empty() { - continue; - } - match translate_system_flag(f) { - FlagAction::Keep(k) => { - if !out.iter().any(|x| x.eq_ignore_ascii_case(k)) { - out.push(k.to_owned()); - } - } - FlagAction::Drop => {} - FlagAction::Verbatim => { - if !out.iter().any(|x| x.eq_ignore_ascii_case(f)) { - out.push(f.clone()); - } - } - FlagAction::Deleted => { - has_deleted = true; - if include_deleted && !out.iter().any(|x| x.eq_ignore_ascii_case("$deleted")) { - out.push("$deleted".to_owned()); - } - } - } - } - Translation { - keywords: out, - has_deleted_flag: has_deleted, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn flags(s: &[&str]) -> Vec { - s.iter().map(|x| (*x).to_owned()).collect() - } - - #[test] - fn system_flag_mapping() { - assert!(matches!( - translate_system_flag("\\Seen"), - FlagAction::Keep("$seen") - )); - assert!(matches!( - translate_system_flag("\\Flagged"), - FlagAction::Keep("$flagged") - )); - assert!(matches!( - translate_system_flag("\\Answered"), - FlagAction::Keep("$answered") - )); - assert!(matches!( - translate_system_flag("\\Draft"), - FlagAction::Keep("$draft") - )); - assert!(matches!( - translate_system_flag("\\Recent"), - FlagAction::Drop - )); - assert!(matches!( - translate_system_flag("\\Deleted"), - FlagAction::Deleted - )); - } - - #[test] - fn unknown_system_flag_passes_verbatim() { - assert!(matches!( - translate_system_flag("\\Forwarded"), - FlagAction::Verbatim - )); - assert!(matches!( - translate_system_flag("\\MDNSent"), - FlagAction::Verbatim - )); - } - - #[test] - fn custom_keywords_pass_verbatim_case() { - let t = translate_flags(&flags(&["$Junk", "NotJunk", "MyLabel"]), false); - assert_eq!(t.keywords, vec!["$Junk", "NotJunk", "MyLabel"]); - assert!(!t.has_deleted_flag); - } - - #[test] - fn translates_seen_and_flagged() { - let t = translate_flags(&flags(&["\\Seen", "\\Flagged"]), false); - assert_eq!(t.keywords, vec!["$seen", "$flagged"]); - } - - #[test] - fn drops_recent() { - let t = translate_flags(&flags(&["\\Seen", "\\Recent"]), false); - assert_eq!(t.keywords, vec!["$seen"]); - } - - #[test] - fn deleted_flag_without_include_marks_only() { - let t = translate_flags(&flags(&["\\Deleted", "\\Seen"]), false); - assert!(t.has_deleted_flag); - assert_eq!(t.keywords, vec!["$seen"]); - } - - #[test] - fn deleted_flag_with_include_adds_dollar_deleted() { - let t = translate_flags(&flags(&["\\Deleted", "\\Seen"]), true); - assert!(t.has_deleted_flag); - assert_eq!(t.keywords, vec!["$deleted", "$seen"]); - } - - #[test] - fn duplicate_keywords_deduped_case_insensitively() { - let t = translate_flags(&flags(&["mykey", "MYKEY", "MyKey"]), false); - assert_eq!(t.keywords, vec!["mykey"]); - } - - #[test] - fn empty_flag_strings_ignored() { - let t = translate_flags(&flags(&["", "\\Seen", ""]), false); - assert_eq!(t.keywords, vec!["$seen"]); - } - - #[test] - fn order_preserved() { - let t = translate_flags( - &flags(&["MyLabel", "\\Seen", "$Important", "\\Flagged"]), - false, - ); - assert_eq!( - t.keywords, - vec!["MyLabel", "$seen", "$Important", "$flagged"] - ); - } -} diff --git a/src/sync/import_imap/messages.rs b/src/sync/import_imap/messages.rs deleted file mode 100644 index 95249ca..0000000 --- a/src/sync/import_imap/messages.rs +++ /dev/null @@ -1,158 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashSet; - -use crate::imap::client::ImapClient; -use crate::imap::command; -use crate::imap::error::ImapError; -use crate::imap::response::Untagged; -use crate::imap::retry::is_negotiation_failure; - -pub fn select_uids(client: &mut ImapClient) -> Result, ImapError> { - if client.has_capability("ESEARCH") { - match client.run_collect(command::uid_search_esearch_all()) { - Ok(resp) => { - for u in &resp.untagged { - if let Untagged::Esearch { all, .. } = u { - return Ok(all.clone()); - } - } - } - Err(e) if is_negotiation_failure(&e) => {} - Err(e) => return Err(e), - } - } - match client.run_collect(command::uid_search_all()) { - Ok(resp) => { - for u in &resp.untagged { - if let Untagged::Search(uids) = u { - return Ok(uids.clone()); - } - } - Ok(Vec::new()) - } - Err(e) if is_negotiation_failure(&e) => uids_via_fetch(client), - Err(e) => Err(e), - } -} - -fn uids_via_fetch(client: &mut ImapClient) -> Result, ImapError> { - let resp = client.run_collect(command::uid_fetch_all_uids())?; - let mut out = Vec::new(); - for u in resp.untagged { - if let Untagged::Fetch { items, .. } = u { - for (n, v) in items { - if n == "UID" - && let Some(num) = v.as_number() - { - out.push(num as u32); - } - } - } - } - Ok(out) -} - -#[derive(Debug, Clone, Default)] -pub struct UidDiff { - pub new: Vec, - pub vanished: Vec, - pub present: Vec, -} - -pub fn diff_uids(local: &[u32], server: &[u32]) -> UidDiff { - let local_set: HashSet = local.iter().copied().collect(); - let server_set: HashSet = server.iter().copied().collect(); - let mut new = Vec::new(); - let mut present = Vec::new(); - for &uid in server { - if local_set.contains(&uid) { - present.push(uid); - } else { - new.push(uid); - } - } - let mut vanished: Vec = local - .iter() - .copied() - .filter(|u| !server_set.contains(u)) - .collect(); - new.sort_unstable(); - new.dedup(); - vanished.sort_unstable(); - vanished.dedup(); - present.sort_unstable(); - present.dedup(); - UidDiff { - new, - vanished, - present, - } -} - -pub fn chunks(uids: &[u32], chunk_size: usize) -> Vec<&[u32]> { - if chunk_size == 0 { - return vec![uids]; - } - uids.chunks(chunk_size).collect() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn diff_basic() { - let local = vec![1, 2, 5]; - let server = vec![2, 3, 5, 7]; - let d = diff_uids(&local, &server); - assert_eq!(d.new, vec![3, 7]); - assert_eq!(d.vanished, vec![1]); - assert_eq!(d.present, vec![2, 5]); - } - - #[test] - fn diff_empty_local_is_all_new() { - let d = diff_uids(&[], &[1, 2, 3]); - assert_eq!(d.new, vec![1, 2, 3]); - assert!(d.vanished.is_empty()); - assert!(d.present.is_empty()); - } - - #[test] - fn diff_empty_server_is_all_vanished() { - let d = diff_uids(&[1, 2, 3], &[]); - assert_eq!(d.vanished, vec![1, 2, 3]); - assert!(d.new.is_empty()); - assert!(d.present.is_empty()); - } - - #[test] - fn diff_dedups_inputs() { - let d = diff_uids(&[1, 1, 2, 2], &[2, 2, 3, 3]); - assert_eq!(d.new, vec![3]); - assert_eq!(d.vanished, vec![1]); - assert_eq!(d.present, vec![2]); - } - - #[test] - fn chunks_splits_evenly() { - let v: Vec = (1..=10).collect(); - let c = chunks(&v, 3); - assert_eq!(c.len(), 4); - assert_eq!(c[0], &[1, 2, 3][..]); - assert_eq!(c[3], &[10][..]); - } - - #[test] - fn chunks_zero_returns_whole() { - let v: Vec = vec![1, 2, 3]; - let c = chunks(&v, 0); - assert_eq!(c.len(), 1); - assert_eq!(c[0], &v[..]); - } -} diff --git a/src/sync/import_imap/pool.rs b/src/sync/import_imap/pool.rs deleted file mode 100644 index ff9900f..0000000 --- a/src/sync/import_imap/pool.rs +++ /dev/null @@ -1,258 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::panic::{AssertUnwindSafe, catch_unwind}; -use std::sync::Arc; -use std::thread; - -use crossbeam_channel::{Receiver, Sender, unbounded}; - -use crate::imap::client::{ConnectMode, ImapClient}; -use crate::imap::command; -use crate::imap::error::ImapError; -use crate::imap::response::Untagged; -use crate::imap::retry::{BackoffState, Disposition, RetryPolicy, classify}; -use crate::imap::transport::Connector; -use crate::logging::Logger; - -use super::coordinator::{Endpoint, ImapAuth, authenticate_client}; -use super::fetch::FetchAttrs; - -pub const HARD_CAP: usize = 8; - -pub struct FetchJob { - pub folder: String, - pub wire_name: String, - pub uidvalidity: u32, - pub uids: Vec, -} - -pub enum FetchEvent { - Item { - folder: String, - uidvalidity: u32, - attrs: FetchAttrs, - }, - ChunkDone { - folder: String, - uidvalidity: u32, - uids_requested: Vec, - outcome: Result<(), ImapError>, - }, -} - -pub struct WorkerArgs { - pub connector: Arc, - pub endpoint: Arc, - pub mode: ConnectMode, - pub auth: ImapAuth, - pub compress: bool, - pub policy: RetryPolicy, - pub backoff: BackoffState, - pub logger: Logger, -} - -pub struct WorkerPool { - job_tx: Sender, - event_rx: Receiver, - handles: Vec>, -} - -impl WorkerPool { - pub fn start(args: WorkerArgs, pool_size: usize) -> Result { - let size = pool_size.clamp(1, HARD_CAP); - let (job_tx, job_rx) = unbounded::(); - let (event_tx, event_rx) = unbounded::(); - let mut handles = Vec::with_capacity(size); - let args = Arc::new(args); - - for _ in 0..size { - let args = args.clone(); - let job_rx = job_rx.clone(); - let event_tx = event_tx.clone(); - let handle = thread::spawn(move || { - worker_loop(args, job_rx, event_tx); - }); - handles.push(handle); - } - - Ok(WorkerPool { - job_tx, - event_rx, - handles, - }) - } - - pub fn submit(&self, job: FetchJob) { - let _ = self.job_tx.send(job); - } - - pub fn recv(&self) -> Result { - self.event_rx.recv() - } - - pub fn recv_timeout( - &self, - timeout: std::time::Duration, - ) -> Result { - self.event_rx.recv_timeout(timeout) - } - - pub fn shutdown(self) { - drop(self.job_tx); - for h in self.handles { - let _ = h.join(); - } - } -} - -fn worker_loop(args: Arc, job_rx: Receiver, event_tx: Sender) { - let mut client: Option = None; - let mut current_folder: Option = None; - while let Ok(job) = job_rx.recv() { - let job_folder = job.folder.clone(); - let job_uv = job.uidvalidity; - let job_uids = job.uids.clone(); - let event_tx_for_job = event_tx.clone(); - let outcome = match catch_unwind(AssertUnwindSafe(|| { - run_job_with_retry( - &args, - &mut client, - &mut current_folder, - &job, - &event_tx_for_job, - ) - })) { - Ok(r) => r, - Err(_) => { - client = None; - current_folder = None; - Err(ImapError::Protocol("worker thread panicked".into())) - } - }; - let _ = event_tx.send(FetchEvent::ChunkDone { - folder: job_folder, - uidvalidity: job_uv, - uids_requested: job_uids, - outcome, - }); - } -} - -fn run_job_with_retry( - args: &WorkerArgs, - client_slot: &mut Option, - current_folder: &mut Option, - job: &FetchJob, - event_tx: &Sender, -) -> Result<(), ImapError> { - let mut transient_attempts: u32 = 0; - let mut transport_attempts: u32 = 0; - loop { - if client_slot.is_none() { - match connect_and_auth(args) { - Ok(c) => { - *client_slot = Some(c); - *current_folder = None; - } - Err(e) => { - let disp = classify(&e); - if disp == Disposition::TransportDrop - && transport_attempts < args.policy.max_retries - { - transport_attempts += 1; - std::thread::sleep(args.backoff.transport_delay(transport_attempts)); - continue; - } - return Err(e); - } - } - } - let Some(client) = client_slot.as_mut() else { - return Err(ImapError::Protocol( - "worker pool: client slot empty after connect".into(), - )); - }; - match run_one_job(client, current_folder, job, event_tx) { - Ok(()) => { - args.backoff.reset(); - return Ok(()); - } - Err(e) => match classify(&e) { - Disposition::TransportDrop => { - *client_slot = None; - *current_folder = None; - if transport_attempts >= args.policy.max_retries { - return Err(e); - } - transport_attempts += 1; - std::thread::sleep(args.backoff.transport_delay(transport_attempts)); - } - Disposition::Transient => { - if transient_attempts >= args.policy.max_retries { - return Err(e); - } - transient_attempts += 1; - std::thread::sleep(args.backoff.next_shared_delay()); - } - _ => return Err(e), - }, - } - } -} - -fn connect_and_auth(args: &WorkerArgs) -> Result { - let mut client = ImapClient::connect( - &args.connector, - &args.endpoint.host, - args.endpoint.port, - args.mode, - args.logger, - )?; - authenticate_client(&mut client, &args.auth) - .map_err(|e| ImapError::AuthFailed(e.to_string()))?; - let _ = client.refresh_capabilities(); - if args.compress && client.has_capability("COMPRESS=DEFLATE") { - client.compress_deflate()?; - } - if client.has_capability("ENABLE") && client.has_capability("UTF8=ACCEPT") { - let _ = client.enable(&["UTF8=ACCEPT"]); - } - Ok(client) -} - -fn run_one_job( - client: &mut ImapClient, - current_folder: &mut Option, - job: &FetchJob, - event_tx: &Sender, -) -> Result<(), ImapError> { - if current_folder.as_deref() != Some(job.folder.as_str()) { - client.run_collect(&command::select(&job.wire_name))?; - *current_folder = Some(job.folder.clone()); - } - let set = command::format_uid_set(&job.uids, true); - let folder = job.folder.clone(); - let uv = job.uidvalidity; - client.run_streamed( - &command::uid_fetch( - &set, - &["UID", "FLAGS", "INTERNALDATE", "RFC822.SIZE", "BODY.PEEK[]"], - ), - |u| { - if let Untagged::Fetch { .. } = &u - && let Some(attrs) = super::fetch::extract(&u) - { - let _ = event_tx.send(FetchEvent::Item { - folder: folder.clone(), - uidvalidity: uv, - attrs, - }); - } - }, - )?; - Ok(()) -} diff --git a/src/sync/import_jmap.rs b/src/sync/import_jmap.rs deleted file mode 100644 index d15714f..0000000 --- a/src/sync/import_jmap.rs +++ /dev/null @@ -1,1370 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -pub mod diff; -pub mod mapping; -pub mod pool; -pub mod tree; - -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; - -use rusqlite::Connection; -use serde_json::Value; - -use self::diff::diff; -use self::mapping::{BlobIntern, LocalResolver}; -use self::pool::{Pool, effective_workers}; -use self::tree::topo_order; -use crate::db; -use crate::db::sources::SourceKey; -use crate::error::Error; -use crate::jmap::blobxfer; -use crate::jmap::connect::{self, Connected}; -use crate::jmap::error::JmapError; -use crate::jmap::http::{Auth, HttpClient}; -use crate::jmap::request::{ - URN_VACATION_RESPONSE, get_all, get_changes, get_objects, get_state, query_all_ids, -}; -use crate::jmap::session::{Limits, Session}; -use crate::jmap::wire::JmapId; -use crate::jmap::wire::common::parse_utc_date; -use crate::jmap::wire::email::Email; -use crate::jmap::wire::file_node::{FileNode, FileProps, NodeKind}; -use crate::jmap::wire::sieve_script::SieveScript; -use crate::jmap::wire::vacation_response::{self, VacationResponse}; -use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; -use crate::sync::emailmeta::email_meta_from_blob; -use crate::sync::keys::index_to_json; -use crate::sync::{CommonConfig, Context, ImportConfig, RunOutcome, Summary, TypeCounts}; -use crate::types::ObjectType; - -const IMPORT_ORDER: [ObjectType; 10] = [ - ObjectType::Mailbox, - ObjectType::AddressBook, - ObjectType::Calendar, - ObjectType::FileNode, - ObjectType::Identity, - ObjectType::SieveScript, - ObjectType::ParticipantIdentity, - ObjectType::Email, - ObjectType::ContactCard, - ObjectType::CalendarEvent, -]; - -fn is_queryless(ty: ObjectType) -> bool { - matches!( - ty, - ObjectType::Identity - | ObjectType::AddressBook - | ObjectType::Calendar - | ObjectType::ParticipantIdentity - ) -} - -fn get_props(ty: ObjectType) -> Option<&'static [&'static str]> { - match ty { - ObjectType::Mailbox => Some(&["name", "parentId", "role", "sortOrder", "isSubscribed"]), - ObjectType::Email => Some(&["blobId", "receivedAt", "mailboxIds", "keywords"]), - ObjectType::SieveScript => Some(&["name", "isActive", "blobId"]), - ObjectType::FileNode => Some(&[ - "@type", - "parentId", - "blobId", - "target", - "name", - "mediaType", - "executable", - "created", - "modified", - "isSubscribed", - "role", - ]), - _ => None, - } -} - -type GetMsg = Result, JmapError>; -type BlobRef = (String, String, String); -type DlMsg = (String, Result, JmapError>); - -#[derive(Clone)] -struct Net { - client: HttpClient, - api: String, - account: String, - limits: Limits, - session: Session, -} - -struct DbResolver<'a> { - conn: &'a Connection, - source_id: i64, -} - -impl LocalResolver for DbResolver<'_> { - fn local(&self, ty: ObjectType, jmap_id: &str) -> Option { - db::ids::local_for(self.conn, self.source_id, ty, jmap_id) - .ok() - .flatten() - } -} - -struct EmailMeta { - match_key: String, - date_header: Option, -} - -struct StoredBlob { - local_id: i64, - email: Option, -} - -#[derive(Default)] -struct StoredBlobs(HashMap); - -impl StoredBlobs { - fn get(&self, jmap_blob_id: &str, kind: &str) -> Result<&StoredBlob, JmapError> { - self.0 - .get(jmap_blob_id) - .ok_or_else(|| JmapError::malformed(format!("{kind} blob missing"))) - } -} - -impl BlobIntern for &StoredBlobs { - fn intern(&mut self, jmap_blob_id: &str) -> Result { - self.0 - .get(jmap_blob_id) - .map(|b| b.local_id) - .ok_or_else(|| JmapError::malformed(format!("blob {jmap_blob_id} was not prefetched"))) - } -} - -fn collect_blob_ids(value: &Value, out: &mut Vec) { - match value { - Value::Object(map) => { - if let Some(Value::String(b)) = map.get("blobId") { - out.push(b.clone()); - } - for v in map.values() { - collect_blob_ids(v, out); - } - } - Value::Array(items) => { - for v in items { - collect_blob_ids(v, out); - } - } - _ => {} - } -} - -fn username_of(auth: &Auth) -> String { - match auth { - Auth::Basic { user, .. } => user.clone(), - Auth::Bearer { .. } => "(bearer)".to_owned(), - } -} - -pub fn run(common: CommonConfig, config: ImportConfig) -> Result { - run_reporting(common, config).into_result() -} - -pub fn run_reporting(common: CommonConfig, config: ImportConfig) -> RunOutcome { - let mut summary = Summary::default(); - let error = run_into(common, config, &mut summary).err(); - RunOutcome { summary, error } -} - -fn run_into( - common: CommonConfig, - config: ImportConfig, - summary: &mut Summary, -) -> Result<(), Error> { - let logger = common.logger; - let ctx = Context::open(common, &config.connect)?; - let connected = connect::prepare(&ctx, &config.connect)?; - - let work = work_list(&config, &connected); - let key = SourceKey { - kind: "jmap".to_owned(), - session_url: config.connect.url.clone(), - account_id: connected.account_id.clone(), - }; - - if !ctx.dry_run() - && let Some((url, acc)) = - db::sources::conflicting_source(&ctx.conn, "jmap", &key.session_url, &key.account_id) - .map_err(|e| Error::Partial(e.to_string()))? - && !config.allow_source_change - { - return Err(Error::SourceChange(format!( - "archive already records JMAP source ({url}, account {acc}); \ - pass --allow-source-change to import a different account" - ))); - } - - let source_id = if ctx.dry_run() { - db::sources::find_source(&ctx.conn, &key).map_err(|e| Error::Partial(e.to_string()))? - } else { - let account_name = match &config.connect.account { - crate::jmap::account::AccountSelector::Name(n) => Some(n.clone()), - crate::jmap::account::AccountSelector::Id(_) => None, - }; - Some( - db::sources::upsert_source( - &ctx.conn, - &key, - account_name.as_deref(), - &username_of(ctx.client.auth()), - ) - .map_err(|e| Error::Partial(e.to_string()))?, - ) - }; - - let net = Net { - client: ctx.client.clone(), - api: connected.session.api_url.clone(), - account: connected.account_id.clone(), - limits: connected.limits, - session: connected.session.clone(), - }; - - let mut dry_rows: Vec<(&'static str, u64, u64, u64)> = Vec::new(); - let threads = ctx.common.threads; - - for ty in work { - if logger.enabled(LEVEL_DEFAULT) { - eprintln!("import: {} ...", ty.jmap_name()); - } - let mut counts = TypeCounts::default(); - match reconcile_type( - &ctx, - &net, - ty, - source_id, - threads, - &logger, - &mut counts, - &mut dry_rows, - ) { - Ok(()) => {} - Err(e) if e.aborts_run() => return Err(e), - Err(e) => { - logger.warn(&format!( - "type {} aborted: {e}; continuing (run will exit 5)", - ty.jmap_name() - )); - counts.failed += 1; - } - } - summary.per_type.push((ty.jmap_name(), counts)); - } - - if let Some(source_id) = source_id - && !ctx.dry_run() - && config - .objects - .as_ref() - .is_none_or(|s| s.contains(&ObjectType::SieveScript)) - && net.session.may_support(&net.account, URN_VACATION_RESPONSE) - { - if logger.enabled(LEVEL_DEFAULT) { - eprintln!("import: {} ...", vacation_response::TYPE_NAME); - } - let mut counts = TypeCounts::default(); - match import_vacation_response(&ctx.conn, &net, source_id) { - Ok(fetched) => counts.fetched = fetched, - Err(e) if e.aborts_run() => return Err(e), - Err(e) => { - logger.warn(&format!( - "type {} aborted: {e}; continuing (run will exit 5)", - vacation_response::TYPE_NAME - )); - counts.failed += 1; - } - } - summary - .per_type - .push((vacation_response::TYPE_NAME, counts)); - } - - if ctx.dry_run() { - print_dry_run(&dry_rows); - *summary = Summary::default(); - return Ok(()); - } - - if !summary.any_failed() - && let Err(e) = run_gc(&ctx.conn) - { - logger.warn(&format!("blob GC skipped: {e}")); - } - - summary.retries_observed = ctx.client.retries_observed(); - summary.retry_after_sleeps = ctx.client.retry_after_sleeps(); - Ok(()) -} - -fn import_vacation_response(conn: &Connection, net: &Net, source_id: i64) -> Result { - let got = get_all::( - &net.client, - &net.api, - &net.account, - vacation_response::TYPE_NAME, - ) - .map_err(Error::from)?; - match got.list.into_iter().find(|v| !v.is_unset()) { - Some(vacation) => { - db::vacation_response::store(conn, source_id, &vacation) - .map_err(|e| Error::Partial(e.to_string()))?; - Ok(1) - } - None => { - db::vacation_response::clear(conn, source_id) - .map_err(|e| Error::Partial(e.to_string()))?; - Ok(0) - } - } -} - -fn work_list(config: &ImportConfig, connected: &Connected) -> Vec { - let selected = config.objects.as_ref(); - IMPORT_ORDER - .into_iter() - .filter(|ty| connected.supports(*ty)) - .filter(|ty| selected.map(|s| s.contains(ty)).unwrap_or(true)) - .collect() -} - -#[allow(clippy::too_many_arguments)] -fn reconcile_type( - ctx: &Context, - net: &Net, - ty: ObjectType, - source_id: Option, - threads: usize, - logger: &Logger, - counts: &mut TypeCounts, - dry_rows: &mut Vec<(&'static str, u64, u64, u64)>, -) -> Result<(), Error> { - let local_map: HashMap = match source_id { - Some(sid) => { - db::ids::jmap_to_local(&ctx.conn, sid, ty).map_err(|e| Error::Partial(e.to_string()))? - } - None => HashMap::new(), - }; - let local_ids: HashSet = local_map.keys().cloned().collect(); - - let (server_ids, preloaded, enum_state): (Vec, Option>, Option) = - if is_queryless(ty) { - let got = get_all::(&net.client, &net.api, &net.account, ty.jmap_name()) - .map_err(Error::from)?; - let ids = got - .list - .iter() - .filter_map(|v| { - v.get("id") - .and_then(Value::as_str) - .map(|s| JmapId(s.to_owned())) - }) - .collect(); - (ids, Some(got.list), got.state) - } else { - let ids = query_all_ids( - &net.client, - &net.api, - &net.account, - ty.jmap_name(), - &net.limits, - ) - .map_err(Error::from)?; - (ids, None, None) - }; - - let d = diff(&server_ids, &local_ids); - - if ctx.dry_run() { - dry_rows.push(( - ty.jmap_name(), - d.new.len() as u64, - d.vanished.len() as u64, - d.present.len() as u64, - )); - return Ok(()); - } - - let source_id = source_id.expect("source_id present outside dry-run"); - - let cursor = db::sync_state_jmap::get(&ctx.conn, source_id, ty) - .map_err(|e| Error::Partial(e.to_string()))?; - let run_state = if !supports_changes(ty) { - None - } else if is_queryless(ty) { - enum_state - } else if cursor.is_none() { - get_state(&net.client, &net.api, &net.account, ty.jmap_name()).map_err(Error::from)? - } else { - None - }; - - if !d.new.is_empty() { - let objects = match preloaded { - Some(list) => { - let want: HashSet<&str> = d.new.iter().map(|i| i.0.as_str()).collect(); - list.into_iter() - .filter(|v| { - v.get("id") - .and_then(Value::as_str) - .map(|s| want.contains(s)) - .unwrap_or(false) - }) - .collect() - } - None => fetch_objects(net, ty, &d.new, get_props(ty), threads, logger, counts), - }; - insert_objects(ctx, net, ty, source_id, objects, threads, logger, counts)?; - } - - if !d.vanished.is_empty() { - delete_vanished(&ctx.conn, ty, source_id, &d.vanished, &local_map, logger)?; - counts.deleted += d.vanished.len() as u64; - } - - reconcile_updates( - ctx, net, ty, source_id, &d.present, &local_map, cursor, run_state, threads, logger, counts, - )?; - - if logger.enabled(LEVEL_DEFAULT) { - eprintln!( - "import: {} done (fetched={} deleted={} skipped={} failed={})", - ty.jmap_name(), - counts.fetched, - counts.deleted, - counts.skipped, - counts.failed - ); - } - Ok(()) -} - -fn fetch_objects( - net: &Net, - ty: ObjectType, - new_ids: &[JmapId], - props: Option<&'static [&'static str]>, - threads: usize, - logger: &Logger, - counts: &mut TypeCounts, -) -> Vec { - let chunk = net.limits.max_objects_in_get.max(1) as usize; - let workers = effective_workers(threads, &net.limits, false); - let net = Arc::new(net.clone()); - let pool: Pool, GetMsg> = Pool::new(workers, { - let net = net.clone(); - move |ids: Vec| { - get_objects::( - &net.client, - &net.api, - &net.account, - ty.jmap_name(), - &ids, - props, - &net.limits, - ) - .map(|r| r.list) - } - }); - - let mut batches = 0u64; - for c in new_ids.chunks(chunk) { - pool.submit(c.to_vec()); - batches += 1; - } - let mut out = Vec::with_capacity(new_ids.len()); - let mut done = 0u64; - for res in pool.finish() { - match res { - Ok(list) => out.extend(list), - Err(e) => { - logger.warn(&format!("{} /get chunk failed: {e}", ty.jmap_name())); - counts.failed += 1; - } - } - done += 1; - } - if logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "import: {} fetched {}/{} chunks", - ty.jmap_name(), - done, - batches - ); - } - out -} - -#[allow(clippy::too_many_arguments)] -fn insert_objects( - ctx: &Context, - net: &Net, - ty: ObjectType, - source_id: i64, - objects: Vec, - threads: usize, - logger: &Logger, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let existing_parents: HashSet = - if matches!(ty, ObjectType::Mailbox | ObjectType::FileNode) { - db::ids::jmap_to_local(&ctx.conn, source_id, ty) - .map(|m| m.into_keys().collect()) - .map_err(|e| Error::Partial(e.to_string()))? - } else { - HashSet::new() - }; - let (ordered, warnings) = order_objects(ty, &objects, &existing_parents); - for w in &warnings { - logger.warn(w); - } - - for batch in ordered.chunks(200) { - let blob_refs = blob_references(ty, batch.iter().map(|&i| &objects[i])); - let blobs = if blob_refs.is_empty() { - StoredBlobs::default() - } else { - download_blobs(&ctx.conn, net, ty, blob_refs, threads, logger)? - }; - let tx = ctx - .conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - for &idx in batch { - let obj = &objects[idx]; - let jmap_id = match obj.get("id").and_then(Value::as_str) { - Some(s) => s.to_owned(), - None => { - counts.failed += 1; - continue; - } - }; - match insert_one(&tx, ty, source_id, obj, &blobs, logger) { - Ok(local_id) => { - db::ids::insert(&tx, source_id, ty, &jmap_id, local_id) - .map_err(|e| Error::Partial(e.to_string()))?; - counts.fetched += 1; - } - Err(e) => { - logger.warn(&format!("{} {jmap_id} skipped: {e}", ty.jmap_name())); - counts.failed += 1; - } - } - } - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - } - Ok(()) -} - -fn blob_references<'a>(ty: ObjectType, objects: impl Iterator) -> Vec { - let mut seen: HashSet = HashSet::new(); - let mut refs = Vec::new(); - match ty { - ObjectType::Email => { - for o in objects { - if let Some(b) = o.get("blobId").and_then(Value::as_str) - && seen.insert(b.to_owned()) - { - refs.push(( - b.to_owned(), - "message/rfc822".to_owned(), - "email".to_owned(), - )); - } - } - } - ObjectType::SieveScript => { - for o in objects { - if let Some(b) = o.get("blobId").and_then(Value::as_str) - && seen.insert(b.to_owned()) - { - refs.push(( - b.to_owned(), - "application/sieve".to_owned(), - "script".to_owned(), - )); - } - } - } - ObjectType::FileNode => { - for o in objects { - if let Some(b) = o.get("blobId").and_then(Value::as_str) - && seen.insert(b.to_owned()) - { - let mt = o - .get("mediaType") - .and_then(Value::as_str) - .unwrap_or("application/octet-stream") - .to_owned(); - let name = o - .get("name") - .and_then(Value::as_str) - .unwrap_or("file") - .to_owned(); - refs.push((b.to_owned(), mt, name)); - } - } - } - ObjectType::ContactCard | ObjectType::CalendarEvent => { - for o in objects { - let mut ids = Vec::new(); - collect_blob_ids(o, &mut ids); - for b in ids { - if seen.insert(b.clone()) { - refs.push((b, "application/octet-stream".to_owned(), "blob".to_owned())); - } - } - } - } - _ => {} - } - refs -} - -fn download_blobs( - conn: &Connection, - net: &Net, - ty: ObjectType, - refs: Vec, - threads: usize, - logger: &Logger, -) -> Result { - let workers = effective_workers(threads, &net.limits, true); - let net_arc = Arc::new(net.clone()); - let pool: Pool = Pool::new(workers, { - let net = net_arc.clone(); - move |(blob_id, type_hint, name): (String, String, String)| { - let r = blobxfer::download_bytes( - &net.client, - &net.session, - &net.account, - &blob_id, - &type_hint, - &name, - ); - (blob_id, r) - } - }); - let total = refs.len(); - for r in refs { - pool.submit(r); - } - let mut out = HashMap::with_capacity(total); - for _ in 0..total { - let Ok((blob_id, res)) = pool.results().recv() else { - break; - }; - match res { - Ok(bytes) => { - let local_id = db::blobs::intern_blob(conn, &bytes) - .map_err(|e| Error::Partial(e.to_string()))?; - let email = (ty == ObjectType::Email).then(|| { - let (index, date_header) = email_meta_from_blob(&bytes); - EmailMeta { - match_key: index_to_json(&index), - date_header, - } - }); - out.insert(blob_id, StoredBlob { local_id, email }); - } - Err(e) => { - logger.warn(&format!("blob {blob_id} download failed: {e}")); - } - } - } - if logger.enabled(LEVEL_PROGRESS) { - eprintln!("import: downloaded {}/{} blobs", out.len(), total); - } - Ok(StoredBlobs(out)) -} - -fn order_objects( - ty: ObjectType, - objects: &[Value], - existing_parents: &HashSet, -) -> (Vec, Vec) { - if !matches!(ty, ObjectType::Mailbox | ObjectType::FileNode) { - return ((0..objects.len()).collect(), Vec::new()); - } - let items: Vec<(String, Option)> = objects - .iter() - .map(|o| { - ( - o.get("id") - .and_then(Value::as_str) - .unwrap_or_default() - .to_owned(), - o.get("parentId").and_then(Value::as_str).map(str::to_owned), - ) - }) - .collect(); - let result = topo_order(&items, existing_parents); - let mut warnings = Vec::new(); - for i in &result.orphans { - let (id, parent) = &items[*i]; - warnings.push(format!( - "{} {id}: parent {} not present in batch or archive; inserting with NULL parent", - ty.jmap_name(), - parent.as_deref().unwrap_or(""), - )); - } - for i in &result.cycle_roots { - warnings.push(format!( - "{} {}: parent cycle detected; broken at this node", - ty.jmap_name(), - items[*i].0, - )); - } - (result.order, warnings) -} - -fn insert_one( - conn: &Connection, - ty: ObjectType, - source_id: i64, - obj: &Value, - blobs: &StoredBlobs, - logger: &Logger, -) -> Result { - let resolver = DbResolver { conn, source_id }; - match ty { - ObjectType::Mailbox => { - let w = serde_json::from_value(obj.clone())?; - mapping::insert_mailbox(conn, &w, &resolver) - } - ObjectType::Identity => { - let w = serde_json::from_value(obj.clone())?; - mapping::insert_identity(conn, &w) - } - ObjectType::AddressBook => { - let w = serde_json::from_value(obj.clone())?; - mapping::insert_address_book(conn, &w) - } - ObjectType::Calendar => { - let w = serde_json::from_value(obj.clone())?; - mapping::insert_calendar(conn, &w) - } - ObjectType::ParticipantIdentity => { - let w = serde_json::from_value(obj.clone())?; - mapping::insert_participant_identity(conn, &w) - } - ObjectType::Email => { - let mut w: Email = serde_json::from_value(obj.clone())?; - let stored = blobs.get(&w.blob_id.0, "email")?; - let meta = stored - .email - .as_ref() - .ok_or_else(|| JmapError::malformed("email blob missing"))?; - if w.received_at.is_none() { - w.received_at = meta.date_header.as_deref().and_then(parse_utc_date); - let id = obj.get("id").and_then(Value::as_str).unwrap_or("?"); - let raw = obj - .get("receivedAt") - .and_then(Value::as_str) - .unwrap_or(""); - logger.warn(&format!( - "Email {id}: unusable receivedAt {raw:?}; falling back to {}", - match &w.received_at { - Some(_) => "the Date header", - None => "1970-01-01T00:00:00Z", - } - )); - } - mapping::insert_email(conn, &w, stored.local_id, &meta.match_key, &resolver) - } - ObjectType::SieveScript => { - let w: SieveScript = serde_json::from_value(obj.clone())?; - let blob_local = blobs.get(&w.blob_id.0, "sieve")?.local_id; - mapping::insert_sieve_script(conn, &w, blob_local) - } - ObjectType::FileNode => { - let w: FileNode = serde_json::from_value(obj.clone())?; - let blob_local = match &w.kind { - NodeKind::File(FileProps { - blob_id: Some(b), .. - }) => Some(blobs.get(&b.0, "file")?.local_id), - _ => None, - }; - mapping::insert_file_node(conn, &w, blob_local, &resolver) - } - ObjectType::ContactCard => { - let w = serde_json::from_value(obj.clone())?; - let mut bi = blobs; - mapping::insert_contact_card(conn, &w, &resolver, &mut bi) - } - ObjectType::CalendarEvent => { - let w = serde_json::from_value(obj.clone())?; - let mut bi = blobs; - mapping::insert_calendar_event(conn, &w, &resolver, &mut bi) - } - } -} - -fn supports_changes(ty: ObjectType) -> bool { - !matches!(ty, ObjectType::SieveScript) -} - -fn update_props(ty: ObjectType) -> Option<&'static [&'static str]> { - match ty { - ObjectType::Email => Some(&["mailboxIds", "keywords"]), - other => get_props(other), - } -} - -#[allow(clippy::too_many_arguments)] -fn reconcile_updates( - ctx: &Context, - net: &Net, - ty: ObjectType, - source_id: i64, - present: &[JmapId], - local_map: &HashMap, - cursor: Option, - run_state: Option, - threads: usize, - logger: &Logger, - counts: &mut TypeCounts, -) -> Result<(), Error> { - if supports_changes(ty) - && let Some(since) = cursor - { - match get_changes( - &net.client, - &net.api, - &net.account, - ty.jmap_name(), - &since, - &net.limits, - ) { - Ok(ch) => { - let present_set: HashSet<&str> = present.iter().map(|j| j.0.as_str()).collect(); - let updated: Vec = ch - .updated - .into_iter() - .filter(|j| present_set.contains(j.0.as_str())) - .collect(); - let clean = fetch_and_update( - ctx, net, ty, source_id, &updated, local_map, threads, logger, counts, - )?; - advance_state(ctx, ty, source_id, &ch.new_state, clean, logger)?; - } - Err(err) - if matches!( - err, - JmapError::CannotCalculateChanges | JmapError::UnknownMethod - ) => - { - let reason = if matches!(err, JmapError::UnknownMethod) { - "server does not implement /changes" - } else { - "server cannot calculate changes from stored state" - }; - logger.warn(&format!( - "{}: {reason}; refreshing all present objects", - ty.jmap_name() - )); - let captured = get_state(&net.client, &net.api, &net.account, ty.jmap_name()) - .map_err(Error::from)?; - let clean = fetch_and_update( - ctx, net, ty, source_id, present, local_map, threads, logger, counts, - )?; - if let Some(s) = captured { - advance_state(ctx, ty, source_id, &s, clean, logger)?; - } - } - Err(e) => return Err(Error::from(e)), - } - return Ok(()); - } - - let clean = fetch_and_update( - ctx, net, ty, source_id, present, local_map, threads, logger, counts, - )?; - if let Some(s) = run_state { - advance_state(ctx, ty, source_id, &s, clean, logger)?; - } - Ok(()) -} - -fn advance_state( - ctx: &Context, - ty: ObjectType, - source_id: i64, - state: &str, - clean: bool, - logger: &Logger, -) -> Result<(), Error> { - if !clean { - logger.warn(&format!( - "{}: holding sync state; some updates failed and will be retried on the next run", - ty.jmap_name() - )); - return Ok(()); - } - db::sync_state_jmap::upsert(&ctx.conn, source_id, ty, state) - .map_err(|e| Error::Partial(e.to_string())) -} - -#[allow(clippy::too_many_arguments)] -fn fetch_and_update( - ctx: &Context, - net: &Net, - ty: ObjectType, - source_id: i64, - ids: &[JmapId], - local_map: &HashMap, - threads: usize, - logger: &Logger, - counts: &mut TypeCounts, -) -> Result { - if ids.is_empty() { - return Ok(true); - } - let failed_before = counts.failed; - let objects = fetch_objects(net, ty, ids, update_props(ty), threads, logger, counts); - update_objects( - ctx, net, ty, source_id, objects, local_map, threads, logger, counts, - )?; - Ok(counts.failed == failed_before) -} - -#[allow(clippy::too_many_arguments)] -fn update_objects( - ctx: &Context, - net: &Net, - ty: ObjectType, - source_id: i64, - objects: Vec, - local_map: &HashMap, - threads: usize, - logger: &Logger, - counts: &mut TypeCounts, -) -> Result<(), Error> { - for batch in objects.chunks(200) { - let blob_refs = blob_references(ty, batch.iter()); - let blobs = if blob_refs.is_empty() { - StoredBlobs::default() - } else { - download_blobs(&ctx.conn, net, ty, blob_refs, threads, logger)? - }; - let tx = ctx - .conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - for obj in batch { - let jmap_id = match obj.get("id").and_then(Value::as_str) { - Some(s) => s.to_owned(), - None => { - counts.failed += 1; - continue; - } - }; - let Some(&local_id) = local_map.get(&jmap_id) else { - continue; - }; - match update_one(&tx, ty, source_id, local_id, obj, &blobs) { - Ok(true) => counts.updated += 1, - Ok(false) => {} - Err(e) => { - logger.warn(&format!("{} {jmap_id} update skipped: {e}", ty.jmap_name())); - counts.failed += 1; - } - } - } - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - } - Ok(()) -} - -fn update_one( - conn: &Connection, - ty: ObjectType, - source_id: i64, - local_id: i64, - obj: &Value, - blobs: &StoredBlobs, -) -> Result { - let resolver = DbResolver { conn, source_id }; - match ty { - ObjectType::Mailbox => { - let w = serde_json::from_value(obj.clone())?; - mapping::update_mailbox(conn, local_id, &w, &resolver) - } - ObjectType::Identity => { - let w = serde_json::from_value(obj.clone())?; - mapping::update_identity(conn, local_id, &w) - } - ObjectType::AddressBook => { - let w = serde_json::from_value(obj.clone())?; - mapping::update_address_book(conn, local_id, &w) - } - ObjectType::Calendar => { - let w = serde_json::from_value(obj.clone())?; - mapping::update_calendar(conn, local_id, &w) - } - ObjectType::ParticipantIdentity => { - let w = serde_json::from_value(obj.clone())?; - mapping::update_participant_identity(conn, local_id, &w) - } - ObjectType::Email => mapping::update_email(conn, local_id, obj, &resolver), - ObjectType::SieveScript => { - let w: SieveScript = serde_json::from_value(obj.clone())?; - let blob_local = blobs.get(&w.blob_id.0, "sieve")?.local_id; - mapping::update_sieve_script(conn, local_id, &w, blob_local) - } - ObjectType::FileNode => { - let w: FileNode = serde_json::from_value(obj.clone())?; - let blob_local = match &w.kind { - NodeKind::File(FileProps { - blob_id: Some(b), .. - }) => Some(blobs.get(&b.0, "file")?.local_id), - _ => None, - }; - mapping::update_file_node(conn, local_id, &w, blob_local, &resolver) - } - ObjectType::ContactCard => { - let w = serde_json::from_value(obj.clone())?; - let mut bi = blobs; - mapping::update_contact_card(conn, local_id, &w, &resolver, &mut bi) - } - ObjectType::CalendarEvent => { - let w = serde_json::from_value(obj.clone())?; - let mut bi = blobs; - mapping::update_calendar_event(conn, local_id, &w, &resolver, &mut bi) - } - } -} - -fn delete_vanished( - conn: &Connection, - ty: ObjectType, - source_id: i64, - vanished: &[JmapId], - local_map: &HashMap, - logger: &Logger, -) -> Result<(), Error> { - let locals: Vec = vanished - .iter() - .filter_map(|j| local_map.get(&j.0).copied()) - .collect(); - if locals.is_empty() { - return Ok(()); - } - let tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - let vanished_set: HashSet = locals.iter().copied().collect(); - - match ty { - ObjectType::Mailbox | ObjectType::FileNode => { - let table = if ty == ObjectType::Mailbox { - "mailboxes" - } else { - "file_nodes" - }; - let mut stmt = tx - .prepare(&format!("SELECT id, parent_id FROM {table}")) - .map_err(|e| Error::Partial(e.to_string()))?; - let rows: Vec<(i64, Option)> = stmt - .query_map([], |r| Ok((r.get(0)?, r.get(1)?))) - .and_then(|m| m.collect()) - .map_err(|e| Error::Partial(e.to_string()))?; - drop(stmt); - for (id, parent) in &rows { - if let Some(p) = parent - && vanished_set.contains(p) - && !vanished_set.contains(id) - { - tx.execute( - &format!("UPDATE {table} SET parent_id = NULL WHERE id = ?1"), - rusqlite::params![id], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - } - } - let by: HashMap> = rows.iter().copied().collect(); - let depth = |mut id: i64| -> usize { - let mut d = 0; - let mut seen = HashSet::new(); - while let Some(Some(p)) = by.get(&id) { - if !seen.insert(id) || !vanished_set.contains(p) { - break; - } - d += 1; - id = *p; - } - d - }; - let depths: HashMap = locals.iter().map(|id| (*id, depth(*id))).collect(); - let mut ordered = locals.clone(); - ordered.sort_by(|a, b| depths[b].cmp(&depths[a]).then(a.cmp(b))); - for id in ordered { - tx.execute( - &format!("DELETE FROM {table} WHERE id = ?1"), - rusqlite::params![id], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - } - } - _ => { - let table = crate::sync::table_name(ty); - for id in &locals { - tx.execute( - &format!("DELETE FROM {table} WHERE id = ?1"), - rusqlite::params![id], - ) - .map_err(|e| Error::Partial(e.to_string()))?; - } - } - } - - if let Some((table, column)) = cross_ref_target(ty) { - prune_id_arrays(&tx, table, column, &vanished_set) - .map_err(|e| Error::Partial(e.to_string()))?; - } - - for j in vanished { - db::ids::delete(&tx, source_id, ty, &j.0).map_err(|e| Error::Partial(e.to_string()))?; - } - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - if logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "import: {} deleted {} vanished", - ty.jmap_name(), - locals.len() - ); - } - Ok(()) -} - -fn cross_ref_target(ty: ObjectType) -> Option<(&'static str, &'static str)> { - match ty { - ObjectType::Mailbox => Some(("emails", "mailbox_ids")), - ObjectType::Calendar => Some(("calendar_events", "calendar_ids")), - ObjectType::AddressBook => Some(("contact_cards", "address_book_ids")), - _ => None, - } -} - -fn prune_id_arrays( - conn: &Connection, - table: &str, - column: &str, - removed: &HashSet, -) -> Result<(), rusqlite::Error> { - let mut stmt = conn.prepare(&format!("SELECT id, {column} FROM {table}"))?; - let rows: Vec<(i64, String)> = stmt - .query_map([], |r| Ok((r.get(0)?, r.get(1)?)))? - .collect::>()?; - drop(stmt); - for (id, json) in rows { - let arr: Vec = serde_json::from_str(&json).unwrap_or_default(); - if arr.iter().any(|v| removed.contains(v)) { - let kept: Vec = arr.into_iter().filter(|v| !removed.contains(v)).collect(); - let new_json = serde_json::to_string(&kept).unwrap_or_else(|_| "[]".to_owned()); - conn.execute( - &format!("UPDATE {table} SET {column} = ?1 WHERE id = ?2"), - rusqlite::params![new_json, id], - )?; - } - } - Ok(()) -} - -fn run_gc(conn: &Connection) -> Result<(), Error> { - let tx = conn - .unchecked_transaction() - .map_err(|e| Error::Partial(e.to_string()))?; - db::blobs::gc_orphan_blobs(&tx).map_err(|e| Error::Partial(e.to_string()))?; - tx.commit().map_err(|e| Error::Partial(e.to_string()))?; - Ok(()) -} - -fn print_dry_run(rows: &[(&'static str, u64, u64, u64)]) { - println!( - "{:<22} {:>8} {:>10} {:>9}", - "TYPE", "NEW", "VANISHED", "PRESENT" - ); - for (ty, new, vanished, present) in rows { - println!("{ty:<22} {new:>8} {vanished:>10} {present:>9}"); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::init; - use crate::db::sources::{SourceKey, upsert_source}; - use crate::jmap::wire::JmapId; - use crate::logging::Logger; - - fn mem() -> (Connection, i64) { - let c = Connection::open_in_memory().unwrap(); - init::apply_schema(&c).unwrap(); - let sid = upsert_source( - &c, - &SourceKey { - kind: "jmap".to_owned(), - session_url: "u".to_owned(), - account_id: "w".to_owned(), - }, - None, - "u", - ) - .unwrap(); - (c, sid) - } - - #[test] - fn vanished_tree_deletes_leaf_first_and_nulls_true_orphan() { - let (c, sid) = mem(); - c.execute( - "INSERT INTO mailboxes (id,name,parent_id) VALUES (1,'root',NULL)", - [], - ) - .unwrap(); - c.execute( - "INSERT INTO mailboxes (id,name,parent_id) VALUES (2,'mid',1)", - [], - ) - .unwrap(); - c.execute( - "INSERT INTO mailboxes (id,name,parent_id) VALUES (3,'leaf',2)", - [], - ) - .unwrap(); - c.execute( - "INSERT INTO mailboxes (id,name,parent_id) VALUES (4,'survivor',2)", - [], - ) - .unwrap(); - for (jid, local) in [("R", 1), ("M", 2), ("L", 3), ("S", 4)] { - db::ids::insert(&c, sid, ObjectType::Mailbox, jid, local).unwrap(); - } - let local_map: HashMap = [ - ("R".to_owned(), 1), - ("M".to_owned(), 2), - ("L".to_owned(), 3), - ] - .into_iter() - .collect(); - let vanished = vec![ - JmapId("R".to_owned()), - JmapId("M".to_owned()), - JmapId("L".to_owned()), - ]; - delete_vanished( - &c, - ObjectType::Mailbox, - sid, - &vanished, - &local_map, - &Logger::from_flags(true, 0), - ) - .unwrap(); - let remaining: i64 = c - .query_row("SELECT count(*) FROM mailboxes", [], |r| r.get(0)) - .unwrap(); - assert_eq!(remaining, 1, "only the surviving child remains"); - let parent: Option = c - .query_row("SELECT parent_id FROM mailboxes WHERE id=4", [], |r| { - r.get(0) - }) - .unwrap(); - assert_eq!(parent, None, "true orphan's parent was nulled"); - assert!( - db::ids::local_for(&c, sid, ObjectType::Mailbox, "L") - .unwrap() - .is_none(), - "vanished sync_id rows removed" - ); - } - - #[test] - fn vanished_mailbox_is_pruned_from_email_mailbox_ids() { - let (c, sid) = mem(); - c.execute( - "INSERT INTO mailboxes (id,name,parent_id) VALUES (1,'a',NULL)", - [], - ) - .unwrap(); - c.execute( - "INSERT INTO mailboxes (id,name,parent_id) VALUES (2,'b',NULL)", - [], - ) - .unwrap(); - let blob = db::blobs::intern_blob(&c, b"msg").unwrap(); - c.execute( - "INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords) - VALUES (?1,'2020-01-01T00:00:00Z','[1,2]','[]')", - rusqlite::params![blob], - ) - .unwrap(); - db::ids::insert(&c, sid, ObjectType::Mailbox, "A", 1).unwrap(); - db::ids::insert(&c, sid, ObjectType::Mailbox, "B", 2).unwrap(); - let local_map: HashMap = [("A".to_owned(), 1)].into_iter().collect(); - delete_vanished( - &c, - ObjectType::Mailbox, - sid, - &[JmapId("A".to_owned())], - &local_map, - &Logger::from_flags(true, 0), - ) - .unwrap(); - let ids: String = c - .query_row("SELECT mailbox_ids FROM emails", [], |r| r.get(0)) - .unwrap(); - assert_eq!( - ids, "[2]", - "deleted mailbox id removed from referring email" - ); - } - - #[test] - fn order_objects_does_not_warn_when_parent_already_in_archive() { - let objects = vec![serde_json::json!({"id":"child","parentId":"existing"})]; - let existing: HashSet = ["existing".to_owned()].into_iter().collect(); - let (ordered, warnings) = order_objects(ObjectType::Mailbox, &objects, &existing); - assert_eq!(ordered, vec![0]); - assert!( - warnings.is_empty(), - "no orphan warning expected: {warnings:?}" - ); - } - - #[test] - fn order_objects_warns_on_orphan_parent() { - let objects = vec![serde_json::json!({"id":"orphan","parentId":"ghost"})]; - let (ordered, warnings) = order_objects(ObjectType::Mailbox, &objects, &HashSet::new()); - assert_eq!(ordered, vec![0]); - assert_eq!(warnings.len(), 1); - assert!(warnings[0].contains("ghost"), "msg was: {}", warnings[0]); - } - - #[test] - fn order_objects_warns_on_cycle() { - let objects = vec![ - serde_json::json!({"id":"a","parentId":"b"}), - serde_json::json!({"id":"b","parentId":"a"}), - ]; - let (ordered, warnings) = order_objects(ObjectType::Mailbox, &objects, &HashSet::new()); - assert_eq!(ordered.len(), 2); - assert!( - warnings.iter().any(|w| w.contains("cycle")), - "expected a cycle warning, got: {warnings:?}" - ); - } -} diff --git a/src/sync/import_jmap/diff.rs b/src/sync/import_jmap/diff.rs deleted file mode 100644 index 5b6dd7a..0000000 --- a/src/sync/import_jmap/diff.rs +++ /dev/null @@ -1,88 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashSet; - -use crate::jmap::wire::JmapId; - -pub struct IdSetDiff { - pub new: Vec, - pub vanished: Vec, - pub present: Vec, -} - -pub fn diff(server_ids: &[JmapId], local_ids: &HashSet) -> IdSetDiff { - let server_set: HashSet<&str> = server_ids.iter().map(|i| i.0.as_str()).collect(); - let mut seen: HashSet<&str> = HashSet::new(); - let mut new = Vec::new(); - let mut present = Vec::new(); - for id in server_ids { - if !seen.insert(id.0.as_str()) { - continue; - } - if local_ids.contains(&id.0) { - present.push(id.clone()); - } else { - new.push(id.clone()); - } - } - let mut vanished = Vec::new(); - for local in local_ids { - if !server_set.contains(local.as_str()) { - vanished.push(JmapId(local.clone())); - } - } - vanished.sort_by(|a, b| a.0.cmp(&b.0)); - IdSetDiff { - new, - vanished, - present, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn ids(v: &[&str]) -> Vec { - v.iter().map(|s| JmapId((*s).to_owned())).collect() - } - - fn locals(v: &[&str]) -> HashSet { - v.iter().map(|s| (*s).to_owned()).collect() - } - - #[test] - fn partitions_new_vanished_present() { - let d = diff(&ids(&["a", "b", "c"]), &locals(&["b", "c", "d"])); - assert_eq!(d.new, ids(&["a"])); - assert_eq!(d.present, ids(&["b", "c"])); - assert_eq!(d.vanished, ids(&["d"])); - } - - #[test] - fn empty_local_means_all_new() { - let d = diff(&ids(&["x", "y"]), &locals(&[])); - assert_eq!(d.new, ids(&["x", "y"])); - assert!(d.vanished.is_empty()); - assert!(d.present.is_empty()); - } - - #[test] - fn empty_server_means_all_vanished() { - let d = diff(&ids(&[]), &locals(&["p", "q"])); - assert!(d.new.is_empty()); - assert_eq!(d.vanished, ids(&["p", "q"])); - } - - #[test] - fn duplicate_server_ids_are_collapsed() { - let d = diff(&ids(&["a", "a", "b"]), &locals(&["a"])); - assert_eq!(d.new, ids(&["b"])); - assert_eq!(d.present, ids(&["a"])); - assert!(d.vanished.is_empty()); - } -} diff --git a/src/sync/import_jmap/mapping.rs b/src/sync/import_jmap/mapping.rs deleted file mode 100644 index ce9327f..0000000 --- a/src/sync/import_jmap/mapping.rs +++ /dev/null @@ -1,1844 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use indexmap::IndexMap; -use rusqlite::{Connection, Row, params}; -use serde_json::{Map, Value}; - -use crate::db::defaults::unique_default; -use crate::jmap::blob::{BlobWalkError, InlineShape, import_blob_ids, inline_blob_data_uris}; -use crate::jmap::error::JmapError; -use crate::jmap::wire::JmapId; -use crate::jmap::wire::address_book::AddressBook; -use crate::jmap::wire::calendar::Calendar; -use crate::jmap::wire::calendar_event::CalendarEvent; -use crate::jmap::wire::contact_card::ContactCard; -use crate::jmap::wire::email::Email; -use crate::jmap::wire::file_node::{DirectoryProps, FileNode, FileProps, NodeKind, SymLinkProps}; -use crate::jmap::wire::identity::Identity; -use crate::jmap::wire::mailbox::Mailbox; -use crate::jmap::wire::participant_identity::ParticipantIdentity; -use crate::jmap::wire::sieve_script::SieveScript; -use crate::types::ObjectType; - -pub trait LocalResolver { - fn local(&self, ty: ObjectType, jmap_id: &str) -> Option; -} - -pub trait TargetResolver { - fn target(&self, ty: ObjectType, local_id: i64) -> Option; -} - -pub trait BlobIntern { - fn intern(&mut self, jmap_blob_id: &str) -> Result; -} - -pub trait BlobBytes { - fn bytes(&self, local_id: i64) -> Result, JmapError>; -} - -fn translate_in( - ids: &IndexMap, - ty: ObjectType, - resolver: &impl LocalResolver, -) -> Result, JmapError> { - let mut out = Vec::with_capacity(ids.len()); - for (id, _) in ids { - match resolver.local(ty, &id.0) { - Some(local) => out.push(local), - None => { - return Err(JmapError::malformed(format!( - "unresolved {} reference {} on insert", - ty.jmap_name(), - id.0 - ))); - } - } - } - Ok(out) -} - -fn translate_out( - locals: &[i64], - ty: ObjectType, - resolver: &impl TargetResolver, -) -> Result, JmapError> { - let mut out = IndexMap::new(); - for local in locals { - match resolver.target(ty, *local) { - Some(target) => { - out.insert(target, true); - } - None => { - return Err(JmapError::malformed(format!( - "unresolved local {} id {local} on export", - ty.jmap_name() - ))); - } - } - } - Ok(out) -} - -fn id_array_json(ids: &[i64]) -> String { - let vals: Vec = ids.iter().map(|i| Value::from(*i)).collect(); - Value::Array(vals).to_string() -} - -fn parse_local_id_array(text: &str) -> Result, JmapError> { - let v: Value = serde_json::from_str(text)?; - v.as_array() - .ok_or_else(|| JmapError::malformed("id column is not a JSON array"))? - .iter() - .map(|x| { - x.as_i64() - .ok_or_else(|| JmapError::malformed("id array element is not an integer")) - }) - .collect() -} - -fn opt_parent( - resolver: &impl LocalResolver, - ty: ObjectType, - parent: &Option, -) -> Option { - parent.as_ref().and_then(|p| resolver.local(ty, &p.0)) -} - -pub fn insert_mailbox( - conn: &Connection, - wire: &Mailbox, - resolver: &impl LocalResolver, -) -> Result { - let parent = opt_parent(resolver, ObjectType::Mailbox, &wire.parent_id); - let role = crate::db::roles::unique_role(conn, wire.role.as_deref(), None)?; - conn.execute( - "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) - VALUES (?1, ?2, ?3, ?4, ?5)", - params![ - wire.name, - parent, - role, - wire.sort_order, - wire.is_subscribed as i64 - ], - )?; - Ok(conn.last_insert_rowid()) -} - -pub const MAILBOX_SELECT: &str = - "SELECT id, name, parent_id, role, sort_order, is_subscribed FROM mailboxes"; - -pub fn row_to_mailbox(row: &Row, resolver: &impl TargetResolver) -> Result { - let parent_local: Option = row.get(2)?; - let parent_id = match parent_local { - Some(p) => Some( - resolver - .target(ObjectType::Mailbox, p) - .ok_or_else(|| JmapError::malformed("unresolved mailbox parent on export"))?, - ), - None => None, - }; - Ok(Mailbox { - id: None, - name: row.get(1)?, - parent_id, - role: row.get(3)?, - sort_order: row.get::<_, i64>(4)? as u32, - is_subscribed: row.get::<_, i64>(5)? != 0, - }) -} - -pub fn insert_email( - conn: &Connection, - wire: &Email, - blob_local_id: i64, - message_match: &str, - resolver: &impl LocalResolver, -) -> Result { - let mailbox_locals = translate_in(&wire.mailbox_ids, ObjectType::Mailbox, resolver)?; - if mailbox_locals.is_empty() { - return Err(JmapError::malformed("email has no resolvable mailbox")); - } - let keywords: Vec = wire.keywords.keys().cloned().collect(); - conn.execute( - "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match) - VALUES (?1, ?2, ?3, ?4, ?5)", - params![ - blob_local_id, - format_utc(&wire.received_at.unwrap_or(time::OffsetDateTime::UNIX_EPOCH))?, - id_array_json(&mailbox_locals), - Value::Array(keywords.iter().map(|k| Value::from(k.as_str())).collect()).to_string(), - message_match - ], - )?; - Ok(conn.last_insert_rowid()) -} - -#[derive(Debug, Clone)] -pub struct EmailRow { - pub blob_local_id: i64, - pub received_at: String, - pub mailbox_locals: Vec, - pub keywords: Vec, - pub message_match: String, -} - -pub const EMAIL_SELECT: &str = - "SELECT id, blob_id, received_at, mailbox_ids, keywords, message_match FROM emails"; - -pub fn row_to_email(row: &Row) -> Result { - let mailbox_text: String = row.get(3)?; - let keyword_text: String = row.get(4)?; - let keywords: Vec = serde_json::from_str(&keyword_text)?; - Ok(EmailRow { - blob_local_id: row.get(1)?, - received_at: row.get(2)?, - mailbox_locals: parse_local_id_array(&mailbox_text)?, - keywords, - message_match: row.get(5)?, - }) -} - -pub fn insert_identity(conn: &Connection, wire: &Identity) -> Result { - conn.execute( - "INSERT INTO identities (name, email, reply_to, bcc, text_signature, html_signature) - VALUES (?1, ?2, ?3, ?4, ?5, ?6)", - params![ - wire.name, - wire.email, - opt_json(&wire.reply_to)?, - opt_json(&wire.bcc)?, - wire.text_signature, - wire.html_signature - ], - )?; - Ok(conn.last_insert_rowid()) -} - -pub const IDENTITY_SELECT: &str = - "SELECT id, name, email, reply_to, bcc, text_signature, html_signature FROM identities"; - -pub fn row_to_identity(row: &Row) -> Result { - Ok(Identity { - id: None, - name: row.get(1)?, - email: row.get(2)?, - reply_to: from_opt_json(row.get::<_, Option>(3)?)?, - bcc: from_opt_json(row.get::<_, Option>(4)?)?, - text_signature: row.get(5)?, - html_signature: row.get(6)?, - }) -} - -pub fn insert_sieve_script( - conn: &Connection, - wire: &SieveScript, - blob_local_id: i64, -) -> Result { - conn.execute( - "INSERT INTO sieve_scripts (name, is_active, blob_id) VALUES (?1, ?2, ?3)", - params![wire.name, wire.is_active as i64, blob_local_id], - )?; - Ok(conn.last_insert_rowid()) -} - -pub const SIEVE_SELECT: &str = "SELECT id, name, is_active, blob_id FROM sieve_scripts"; - -pub fn insert_address_book(conn: &Connection, wire: &AddressBook) -> Result { - let is_default = unique_default(conn, ObjectType::AddressBook, wire.is_default, None)?; - conn.execute( - "INSERT INTO address_books (name, description, sort_order, is_default, is_subscribed) - VALUES (?1, ?2, ?3, ?4, ?5)", - params![ - wire.name, - wire.description, - wire.sort_order, - is_default as i64, - wire.is_subscribed as i64 - ], - )?; - Ok(conn.last_insert_rowid()) -} - -pub const ADDRESS_BOOK_SELECT: &str = - "SELECT id, name, description, sort_order, is_default, is_subscribed FROM address_books"; - -pub fn insert_calendar(conn: &Connection, wire: &Calendar) -> Result { - let is_default = unique_default(conn, ObjectType::Calendar, wire.is_default, None)?; - conn.execute( - "INSERT INTO calendars (name, description, color, sort_order, is_subscribed, is_visible, - is_default, include_in_availability, default_alerts_with_time, - default_alerts_without_time, time_zone) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", - params![ - wire.name, - wire.description, - wire.color, - wire.sort_order, - wire.is_subscribed as i64, - wire.is_visible as i64, - is_default as i64, - wire.include_in_availability, - opt_json(&wire.default_alerts_with_time)?, - opt_json(&wire.default_alerts_without_time)?, - wire.time_zone - ], - )?; - Ok(conn.last_insert_rowid()) -} - -pub const CALENDAR_SELECT: &str = "SELECT id, name, description, color, sort_order, is_subscribed, \ - is_visible, is_default, include_in_availability, default_alerts_with_time, \ - default_alerts_without_time, time_zone FROM calendars"; - -pub fn insert_participant_identity( - conn: &Connection, - wire: &ParticipantIdentity, -) -> Result { - let is_default = unique_default(conn, ObjectType::ParticipantIdentity, wire.is_default, None)?; - conn.execute( - "INSERT INTO participant_identities (name, calendar_address, is_default) - VALUES (?1, ?2, ?3)", - params![wire.name, wire.calendar_address, is_default as i64], - )?; - Ok(conn.last_insert_rowid()) -} - -pub const PARTICIPANT_IDENTITY_SELECT: &str = - "SELECT id, name, calendar_address, is_default FROM participant_identities"; - -pub fn insert_file_node( - conn: &Connection, - wire: &FileNode, - blob_local_id: Option, - resolver: &impl LocalResolver, -) -> Result { - let parent = opt_parent(resolver, ObjectType::FileNode, &wire.parent_id); - let cols = FileNodeColumns::new(&wire.kind)?; - conn.execute( - "INSERT INTO file_nodes (parent_id, node_type, blob_id, target, name, media_type, - created, modified, is_subscribed, role, executable) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", - params![ - parent, - wire.kind.db_name(), - blob_local_id, - cols.target, - wire.name, - cols.media_type, - format_utc(&wire.created)?, - opt_format_utc(&wire.modified)?, - cols.is_subscribed as i64, - cols.role, - cols.executable as i64 - ], - )?; - Ok(conn.last_insert_rowid()) -} - -struct FileNodeColumns<'a> { - target: Option, - media_type: Option<&'a str>, - is_subscribed: bool, - role: Option<&'a str>, - executable: bool, -} - -impl<'a> FileNodeColumns<'a> { - fn new(kind: &'a NodeKind) -> Result { - let mut cols = FileNodeColumns { - target: None, - media_type: None, - is_subscribed: true, - role: None, - executable: false, - }; - match kind { - NodeKind::File(f) => { - cols.media_type = f.media_type.as_deref(); - cols.executable = f.executable; - } - NodeKind::Directory(d) => { - cols.role = d.role.as_deref(); - cols.is_subscribed = d.is_subscribed; - } - NodeKind::SymLink(l) => cols.target = Some(serde_json::to_string(&l.target)?), - } - Ok(cols) - } -} - -pub const FILE_NODE_SELECT: &str = "SELECT id, parent_id, node_type, blob_id, target, name, \ - media_type, created, modified, is_subscribed, role, executable FROM file_nodes"; - -pub fn insert_contact_card( - conn: &Connection, - wire: &ContactCard, - resolver: &impl LocalResolver, - blobs: &mut impl BlobIntern, -) -> Result { - let address_books = translate_in(&wire.address_book_ids, ObjectType::AddressBook, resolver)?; - let mut data = Value::Object(value_map(&wire.rest)); - let uid = take_string(&mut data, "uid") - .ok_or_else(|| JmapError::malformed("ContactCard has no uid"))?; - rewrite_blobs_in(&mut data, blobs)?; - conn.execute( - "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)", - params![uid, id_array_json(&address_books), data.to_string()], - )?; - Ok(conn.last_insert_rowid()) -} - -pub const CONTACT_CARD_SELECT: &str = "SELECT id, uid, address_book_ids, data FROM contact_cards"; - -pub fn insert_calendar_event( - conn: &Connection, - wire: &CalendarEvent, - resolver: &impl LocalResolver, - blobs: &mut impl BlobIntern, -) -> Result { - let calendars = translate_in(&wire.calendar_ids, ObjectType::Calendar, resolver)?; - let mut data = Value::Object(value_map(&wire.rest)); - for drop_key in ["method", "utcStart", "utcEnd", "isOrigin", "baseEventId"] { - if let Value::Object(m) = &mut data { - m.remove(drop_key); - } - } - rewrite_blobs_in(&mut data, blobs)?; - conn.execute( - "INSERT INTO calendar_events (calendar_ids, is_draft, use_default_alerts, data) - VALUES (?1, ?2, ?3, ?4)", - params![ - id_array_json(&calendars), - wire.is_draft as i64, - wire.use_default_alerts as i64, - data.to_string() - ], - )?; - Ok(conn.last_insert_rowid()) -} - -pub const CALENDAR_EVENT_SELECT: &str = "SELECT id, calendar_ids, is_draft, use_default_alerts, data FROM calendar_events \ - WHERE data_type = 'Event'"; - -pub fn update_mailbox( - conn: &Connection, - local_id: i64, - wire: &Mailbox, - resolver: &impl LocalResolver, -) -> Result { - let parent = opt_parent(resolver, ObjectType::Mailbox, &wire.parent_id); - let role = crate::db::roles::unique_role(conn, wire.role.as_deref(), Some(local_id))?; - let n = conn.execute( - "UPDATE mailboxes SET name = ?1, parent_id = ?2, role = ?3, sort_order = ?4, - is_subscribed = ?5 - WHERE id = ?6 AND (name IS NOT ?1 OR parent_id IS NOT ?2 OR role IS NOT ?3 - OR sort_order IS NOT ?4 OR is_subscribed IS NOT ?5)", - params![ - wire.name, - parent, - role, - wire.sort_order, - wire.is_subscribed as i64, - local_id - ], - )?; - Ok(n > 0) -} - -pub fn update_email( - conn: &Connection, - local_id: i64, - obj: &Value, - resolver: &impl LocalResolver, -) -> Result { - let mailbox_ids: IndexMap = match obj.get("mailboxIds") { - Some(v) => serde_json::from_value(v.clone())?, - None => IndexMap::new(), - }; - let mailbox_locals = translate_in(&mailbox_ids, ObjectType::Mailbox, resolver)?; - if mailbox_locals.is_empty() { - return Err(JmapError::malformed( - "email update has no resolvable mailbox", - )); - } - let keywords: IndexMap = match obj.get("keywords") { - Some(v) => serde_json::from_value(v.clone())?, - None => IndexMap::new(), - }; - let kw: Vec = keywords.keys().cloned().collect(); - let n = conn.execute( - "UPDATE emails SET mailbox_ids = ?1, keywords = ?2 - WHERE id = ?3 AND (mailbox_ids IS NOT ?1 OR keywords IS NOT ?2)", - params![ - id_array_json(&mailbox_locals), - Value::Array(kw.iter().map(|k| Value::from(k.as_str())).collect()).to_string(), - local_id - ], - )?; - Ok(n > 0) -} - -pub fn update_identity( - conn: &Connection, - local_id: i64, - wire: &Identity, -) -> Result { - let n = conn.execute( - "UPDATE identities SET name = ?1, email = ?2, reply_to = ?3, bcc = ?4, - text_signature = ?5, html_signature = ?6 - WHERE id = ?7 AND (name IS NOT ?1 OR email IS NOT ?2 OR reply_to IS NOT ?3 - OR bcc IS NOT ?4 OR text_signature IS NOT ?5 OR html_signature IS NOT ?6)", - params![ - wire.name, - wire.email, - opt_json(&wire.reply_to)?, - opt_json(&wire.bcc)?, - wire.text_signature, - wire.html_signature, - local_id - ], - )?; - Ok(n > 0) -} - -pub fn update_sieve_script( - conn: &Connection, - local_id: i64, - wire: &SieveScript, - blob_local_id: i64, -) -> Result { - let n = conn.execute( - "UPDATE sieve_scripts SET name = ?1, is_active = ?2, blob_id = ?3 - WHERE id = ?4 AND (name IS NOT ?1 OR is_active IS NOT ?2 OR blob_id IS NOT ?3)", - params![wire.name, wire.is_active as i64, blob_local_id, local_id], - )?; - Ok(n > 0) -} - -pub fn update_address_book( - conn: &Connection, - local_id: i64, - wire: &AddressBook, -) -> Result { - let is_default = unique_default( - conn, - ObjectType::AddressBook, - wire.is_default, - Some(local_id), - )?; - let n = conn.execute( - "UPDATE address_books SET name = ?1, description = ?2, sort_order = ?3, is_default = ?4, - is_subscribed = ?5 - WHERE id = ?6 AND (name IS NOT ?1 OR description IS NOT ?2 OR sort_order IS NOT ?3 - OR is_default IS NOT ?4 OR is_subscribed IS NOT ?5)", - params![ - wire.name, - wire.description, - wire.sort_order, - is_default as i64, - wire.is_subscribed as i64, - local_id - ], - )?; - Ok(n > 0) -} - -pub fn update_calendar( - conn: &Connection, - local_id: i64, - wire: &Calendar, -) -> Result { - let is_default = unique_default(conn, ObjectType::Calendar, wire.is_default, Some(local_id))?; - let n = conn.execute( - "UPDATE calendars SET name = ?1, description = ?2, color = ?3, sort_order = ?4, - is_subscribed = ?5, is_visible = ?6, is_default = ?7, include_in_availability = ?8, - default_alerts_with_time = ?9, default_alerts_without_time = ?10, time_zone = ?11 - WHERE id = ?12 AND (name IS NOT ?1 OR description IS NOT ?2 OR color IS NOT ?3 - OR sort_order IS NOT ?4 OR is_subscribed IS NOT ?5 OR is_visible IS NOT ?6 - OR is_default IS NOT ?7 OR include_in_availability IS NOT ?8 - OR default_alerts_with_time IS NOT ?9 OR default_alerts_without_time IS NOT ?10 - OR time_zone IS NOT ?11)", - params![ - wire.name, - wire.description, - wire.color, - wire.sort_order, - wire.is_subscribed as i64, - wire.is_visible as i64, - is_default as i64, - wire.include_in_availability, - opt_json(&wire.default_alerts_with_time)?, - opt_json(&wire.default_alerts_without_time)?, - wire.time_zone, - local_id - ], - )?; - Ok(n > 0) -} - -pub fn update_participant_identity( - conn: &Connection, - local_id: i64, - wire: &ParticipantIdentity, -) -> Result { - let is_default = unique_default( - conn, - ObjectType::ParticipantIdentity, - wire.is_default, - Some(local_id), - )?; - let n = conn.execute( - "UPDATE participant_identities SET name = ?1, calendar_address = ?2, is_default = ?3 - WHERE id = ?4 AND (name IS NOT ?1 OR calendar_address IS NOT ?2 OR is_default IS NOT ?3)", - params![ - wire.name, - wire.calendar_address, - is_default as i64, - local_id - ], - )?; - Ok(n > 0) -} - -pub fn update_file_node( - conn: &Connection, - local_id: i64, - wire: &FileNode, - blob_local_id: Option, - resolver: &impl LocalResolver, -) -> Result { - let parent = opt_parent(resolver, ObjectType::FileNode, &wire.parent_id); - let cols = FileNodeColumns::new(&wire.kind)?; - let n = conn.execute( - "UPDATE file_nodes SET parent_id = ?1, node_type = ?2, blob_id = ?3, target = ?4, - name = ?5, media_type = ?6, created = ?7, modified = ?8, is_subscribed = ?9, - role = ?10, executable = ?11 - WHERE id = ?12 AND (parent_id IS NOT ?1 OR node_type IS NOT ?2 OR blob_id IS NOT ?3 - OR target IS NOT ?4 OR name IS NOT ?5 OR media_type IS NOT ?6 OR created IS NOT ?7 - OR modified IS NOT ?8 OR is_subscribed IS NOT ?9 OR role IS NOT ?10 - OR executable IS NOT ?11)", - params![ - parent, - wire.kind.db_name(), - blob_local_id, - cols.target, - wire.name, - cols.media_type, - format_utc(&wire.created)?, - opt_format_utc(&wire.modified)?, - cols.is_subscribed as i64, - cols.role, - cols.executable as i64, - local_id - ], - )?; - Ok(n > 0) -} - -pub fn update_contact_card( - conn: &Connection, - local_id: i64, - wire: &ContactCard, - resolver: &impl LocalResolver, - blobs: &mut impl BlobIntern, -) -> Result { - let address_books = translate_in(&wire.address_book_ids, ObjectType::AddressBook, resolver)?; - let mut data = Value::Object(value_map(&wire.rest)); - let uid = take_string(&mut data, "uid") - .ok_or_else(|| JmapError::malformed("ContactCard has no uid"))?; - rewrite_blobs_in(&mut data, blobs)?; - let n = conn.execute( - "UPDATE contact_cards SET uid = ?1, address_book_ids = ?2, data = ?3 - WHERE id = ?4 AND (uid IS NOT ?1 OR address_book_ids IS NOT ?2 OR data IS NOT ?3)", - params![ - uid, - id_array_json(&address_books), - data.to_string(), - local_id - ], - )?; - Ok(n > 0) -} - -pub fn update_calendar_event( - conn: &Connection, - local_id: i64, - wire: &CalendarEvent, - resolver: &impl LocalResolver, - blobs: &mut impl BlobIntern, -) -> Result { - let calendars = translate_in(&wire.calendar_ids, ObjectType::Calendar, resolver)?; - let mut data = Value::Object(value_map(&wire.rest)); - for drop_key in ["method", "utcStart", "utcEnd", "isOrigin", "baseEventId"] { - if let Value::Object(m) = &mut data { - m.remove(drop_key); - } - } - rewrite_blobs_in(&mut data, blobs)?; - let n = conn.execute( - "UPDATE calendar_events SET calendar_ids = ?1, is_draft = ?2, use_default_alerts = ?3, - data = ?4 - WHERE id = ?5 AND (calendar_ids IS NOT ?1 OR is_draft IS NOT ?2 - OR use_default_alerts IS NOT ?3 OR data IS NOT ?4)", - params![ - id_array_json(&calendars), - wire.is_draft as i64, - wire.use_default_alerts as i64, - data.to_string(), - local_id - ], - )?; - Ok(n > 0) -} - -pub fn contact_card_to_wire( - uid: &str, - address_book_ids: &str, - data: &str, - resolver: &impl TargetResolver, - blobs: &impl BlobBytes, -) -> Result { - let mut value: Value = serde_json::from_str(data)?; - inline_blobs_in(&mut value, InlineShape::JsContactResource, blobs)?; - let locals = parse_local_id_array(address_book_ids)?; - let abids = translate_out(&locals, ObjectType::AddressBook, resolver)?; - if let Value::Object(map) = &mut value { - map.insert("uid".to_owned(), Value::from(uid)); - map.insert("addressBookIds".to_owned(), id_bool_map(&abids)); - } - Ok(value) -} - -pub fn calendar_event_to_wire( - calendar_ids: &str, - is_draft: bool, - use_default_alerts: bool, - data: &str, - resolver: &impl TargetResolver, - blobs: &impl BlobBytes, -) -> Result { - let mut value: Value = serde_json::from_str(data)?; - inline_blobs_in(&mut value, InlineShape::JsCalendarLink, blobs)?; - let locals = parse_local_id_array(calendar_ids)?; - let calids = translate_out(&locals, ObjectType::Calendar, resolver)?; - if let Value::Object(map) = &mut value { - map.insert("calendarIds".to_owned(), id_bool_map(&calids)); - map.insert("isDraft".to_owned(), Value::Bool(is_draft)); - map.insert( - "useDefaultAlerts".to_owned(), - Value::Bool(use_default_alerts), - ); - } - Ok(value) -} - -fn id_bool_map(ids: &IndexMap) -> Value { - let mut m = Map::new(); - for (id, _) in ids { - m.insert(id.0.clone(), Value::Bool(true)); - } - Value::Object(m) -} - -fn value_map(rest: &IndexMap) -> Map { - let mut m = Map::new(); - for (k, v) in rest { - m.insert(k.clone(), v.clone()); - } - m -} - -fn take_string(value: &mut Value, key: &str) -> Option { - if let Value::Object(map) = value - && let Some(Value::String(s)) = map.remove(key) - { - return Some(s); - } - None -} - -fn rewrite_blobs_in(data: &mut Value, blobs: &mut impl BlobIntern) -> Result<(), JmapError> { - import_blob_ids(data, |jmap_blob_id| { - blobs.intern(jmap_blob_id).map_err(BlobWalkError::resolver) - }) - .map_err(BlobWalkError::into_source) -} - -fn inline_blobs_in( - data: &mut Value, - shape: InlineShape, - blobs: &impl BlobBytes, -) -> Result<(), JmapError> { - inline_blob_data_uris(data, shape, |local_id| { - blobs.bytes(local_id).map_err(BlobWalkError::resolver) - }) - .map_err(BlobWalkError::into_source) -} - -fn opt_json(value: &Option) -> Result, JmapError> { - match value { - Some(v) => Ok(Some(serde_json::to_string(v)?)), - None => Ok(None), - } -} - -fn from_opt_json( - text: Option, -) -> Result, JmapError> { - match text { - Some(t) => Ok(Some(serde_json::from_str(&t)?)), - None => Ok(None), - } -} - -pub fn row_to_address_book(row: &Row) -> Result { - Ok(AddressBook { - id: None, - name: row.get(1)?, - description: row.get(2)?, - sort_order: row.get::<_, i64>(3)? as u32, - is_default: row.get::<_, i64>(4)? != 0, - is_subscribed: row.get::<_, i64>(5)? != 0, - }) -} - -pub fn row_to_calendar(row: &Row) -> Result { - Ok(Calendar { - id: None, - name: row.get(1)?, - description: row.get(2)?, - color: row.get(3)?, - sort_order: row.get::<_, i64>(4)? as u32, - is_subscribed: row.get::<_, i64>(5)? != 0, - is_visible: row.get::<_, i64>(6)? != 0, - is_default: row.get::<_, i64>(7)? != 0, - include_in_availability: row.get(8)?, - default_alerts_with_time: from_opt_json(row.get::<_, Option>(9)?)?, - default_alerts_without_time: from_opt_json(row.get::<_, Option>(10)?)?, - time_zone: row.get(11)?, - }) -} - -pub fn row_to_participant_identity(row: &Row) -> Result { - Ok(ParticipantIdentity { - id: None, - name: row.get(1)?, - calendar_address: row.get(2)?, - is_default: row.get::<_, i64>(3)? != 0, - }) -} - -#[derive(Debug, Clone)] -pub struct SieveRow { - pub name: Option, - pub is_active: bool, - pub blob_local_id: i64, -} - -pub fn row_to_sieve_script(row: &Row) -> Result { - Ok(SieveRow { - name: row.get(1)?, - is_active: row.get::<_, i64>(2)? != 0, - blob_local_id: row.get(3)?, - }) -} - -#[derive(Debug, Clone)] -pub struct FileNodeRow { - pub wire: FileNode, - pub blob_local_id: Option, -} - -pub fn row_to_file_node( - row: &Row, - resolver: &impl TargetResolver, -) -> Result { - let parent_local: Option = row.get(1)?; - let parent_id = match parent_local { - Some(p) => Some( - resolver - .target(ObjectType::FileNode, p) - .ok_or_else(|| JmapError::malformed("unresolved file_node parent on export"))?, - ), - None => None, - }; - let node_type: String = row.get(2)?; - let kind = match node_type.as_str() { - "file" => NodeKind::File(FileProps { - blob_id: None, - media_type: row.get(6)?, - executable: row.get::<_, i64>(11)? != 0, - }), - "directory" => NodeKind::Directory(DirectoryProps { - role: row.get(10)?, - is_subscribed: row.get::<_, i64>(9)? != 0, - }), - "symlink" => NodeKind::SymLink(SymLinkProps { - target: from_opt_json(row.get::<_, Option>(4)?)?.unwrap_or_default(), - }), - other => { - return Err(JmapError::malformed(format!( - "unknown file_node type {other}" - ))); - } - }; - Ok(FileNodeRow { - wire: FileNode { - id: None, - parent_id, - name: row.get(5)?, - created: parse_utc(&row.get::<_, String>(7)?)?, - modified: opt_parse_utc(row.get::<_, Option>(8)?)?, - kind, - }, - blob_local_id: row.get(3)?, - }) -} - -fn parse_utc(text: &str) -> Result { - time::OffsetDateTime::parse(text, &time::format_description::well_known::Rfc3339) - .map_err(|e| JmapError::malformed(format!("cannot parse date {text}: {e}"))) -} - -fn opt_parse_utc(text: Option) -> Result, JmapError> { - match text { - Some(t) => Ok(Some(parse_utc(&t)?)), - None => Ok(None), - } -} - -fn format_utc(value: &time::OffsetDateTime) -> Result { - value - .format(&time::format_description::well_known::Rfc3339) - .map_err(|e| JmapError::malformed(format!("cannot format date: {e}"))) -} - -fn opt_format_utc(value: &Option) -> Result, JmapError> { - match value { - Some(v) => Ok(Some(format_utc(v)?)), - None => Ok(None), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::init; - use encodify::base64::STANDARD; - use std::collections::HashMap; - - struct MapResolver { - to_local: HashMap<(ObjectType, String), i64>, - to_target: HashMap<(ObjectType, i64), String>, - } - impl LocalResolver for MapResolver { - fn local(&self, ty: ObjectType, jmap_id: &str) -> Option { - self.to_local.get(&(ty, jmap_id.to_owned())).copied() - } - } - impl TargetResolver for MapResolver { - fn target(&self, ty: ObjectType, local_id: i64) -> Option { - self.to_target - .get(&(ty, local_id)) - .map(|s| JmapId(s.clone())) - } - } - - struct FakeBlobs; - impl BlobIntern for FakeBlobs { - fn intern(&mut self, _jmap_blob_id: &str) -> Result { - Ok(42) - } - } - impl BlobBytes for FakeBlobs { - fn bytes(&self, local_id: i64) -> Result, JmapError> { - Ok(format!("payload-{local_id}").into_bytes()) - } - } - - fn decode_data_uri(value: &Value, media_type: &str) -> Vec { - let uri = value.as_str().unwrap_or_else(|| panic!("{value} is a URI")); - let prefix = format!("data:{media_type};base64,"); - let payload = uri - .strip_prefix(&prefix) - .unwrap_or_else(|| panic!("{uri} does not start with {prefix}")); - STANDARD.decode(payload).expect("base64 payload") - } - - fn assert_no_blob_id(value: &Value) { - match value { - Value::Object(map) => { - assert!(map.get("blobId").is_none(), "blobId present in {value}"); - assert!(map.get("@blob").is_none(), "@blob present in {value}"); - for child in map.values() { - assert_no_blob_id(child); - } - } - Value::Array(items) => items.iter().for_each(assert_no_blob_id), - _ => {} - } - } - - fn mem() -> Connection { - let c = Connection::open_in_memory().unwrap(); - init::apply_schema(&c).unwrap(); - c - } - - #[test] - fn mailbox_roundtrips_with_parent_translation() { - let c = mem(); - let root: Mailbox = serde_json::from_value(serde_json::json!({ - "id": "M1", "name": "Root", "role": "archive", "sortOrder": 3 - })) - .unwrap(); - let empty = MapResolver { - to_local: HashMap::new(), - to_target: HashMap::new(), - }; - let root_local = insert_mailbox(&c, &root, &empty).unwrap(); - - let child: Mailbox = serde_json::from_value(serde_json::json!({ - "id": "M2", "name": "Child", "parentId": "M1" - })) - .unwrap(); - let mut res = MapResolver { - to_local: HashMap::new(), - to_target: HashMap::new(), - }; - res.to_local - .insert((ObjectType::Mailbox, "M1".to_owned()), root_local); - let child_local = insert_mailbox(&c, &child, &res).unwrap(); - - res.to_target - .insert((ObjectType::Mailbox, root_local), "TGT1".to_owned()); - let wire = c - .query_row( - &format!("{MAILBOX_SELECT} WHERE id = ?1"), - params![child_local], - |row| Ok(row_to_mailbox(row, &res)), - ) - .unwrap() - .unwrap(); - assert_eq!(wire.name, "Child"); - assert_eq!(wire.parent_id, Some(JmapId("TGT1".to_owned()))); - } - - #[test] - fn email_keywords_keep_verbatim_case_and_mailbox_translation() { - let c = mem(); - let mut res = MapResolver { - to_local: HashMap::new(), - to_target: HashMap::new(), - }; - res.to_local - .insert((ObjectType::Mailbox, "MB".to_owned()), 7); - let email: Email = serde_json::from_value(serde_json::json!({ - "id": "E1", - "blobId": "B1", - "receivedAt": "2021-05-04T10:00:00Z", - "mailboxIds": { "MB": true }, - "keywords": { "$Seen": true, "MyTag": true } - })) - .unwrap(); - let blob_id = crate::db::blobs::intern_blob(&c, b"rfc5322 bytes").unwrap(); - let local = insert_email(&c, &email, blob_id, "{}", &res).unwrap(); - let got = c - .query_row( - &format!("{EMAIL_SELECT} WHERE id = ?1"), - params![local], - |row| Ok(row_to_email(row)), - ) - .unwrap() - .unwrap(); - assert_eq!(got.blob_local_id, blob_id); - assert_eq!(got.mailbox_locals, vec![7]); - assert_eq!(got.keywords, vec!["$Seen".to_owned(), "MyTag".to_owned()]); - } - - #[test] - fn unresolved_foreign_key_on_insert_is_malformed() { - let c = mem(); - let empty = MapResolver { - to_local: HashMap::new(), - to_target: HashMap::new(), - }; - let email: Email = serde_json::from_value(serde_json::json!({ - "id": "E1", - "blobId": "B1", - "receivedAt": "2021-05-04T10:00:00Z", - "mailboxIds": { "UNKNOWN": true }, - "keywords": {} - })) - .unwrap(); - let err = insert_email(&c, &email, 1, "{}", &empty).unwrap_err(); - assert!(matches!(err, JmapError::Malformed(_)), "got {err:?}"); - } - - #[test] - fn unresolved_target_on_export_is_malformed() { - let c = mem(); - let root: Mailbox = serde_json::from_value(serde_json::json!({ - "id": "M1", "name": "Root", "parentId": "P9" - })) - .unwrap(); - let mut res = MapResolver { - to_local: HashMap::new(), - to_target: HashMap::new(), - }; - res.to_local - .insert((ObjectType::Mailbox, "P9".to_owned()), 1); - let local = insert_mailbox(&c, &root, &res).unwrap(); - let err = c - .query_row( - &format!("{MAILBOX_SELECT} WHERE id = ?1"), - params![local], - |row| Ok(row_to_mailbox(row, &res)), - ) - .unwrap() - .unwrap_err(); - assert!(matches!(err, JmapError::Malformed(_)), "got {err:?}"); - } - - #[test] - fn contact_card_strips_uid_and_inlines_media_as_data_uri() { - let c = mem(); - let res = MapResolver { - to_local: HashMap::from([((ObjectType::AddressBook, "AB".to_owned()), 1)]), - to_target: HashMap::from([((ObjectType::AddressBook, 1), "TAB".to_owned())]), - }; - let card: ContactCard = serde_json::from_value(serde_json::json!({ - "id": "C1", - "addressBookIds": { "AB": true }, - "uid": "urn:uuid:42", - "name": { "full": "Jane" }, - "media": { "photo": { - "@type": "Media", "kind": "photo", - "blobId": "PB", "mediaType": "image/png" - } } - })) - .unwrap(); - let mut blobs = FakeBlobs; - let local = insert_contact_card(&c, &card, &res, &mut blobs).unwrap(); - let (uid, abids, data): (String, String, String) = c - .query_row( - &format!("{CONTACT_CARD_SELECT} WHERE id = ?1"), - params![local], - |r| Ok((r.get(1)?, r.get(2)?, r.get(3)?)), - ) - .unwrap(); - assert_eq!(uid, "urn:uuid:42"); - let stored: Value = serde_json::from_str(&data).unwrap(); - assert!(stored.get("uid").is_none()); - assert_eq!(stored["media"]["photo"]["@blob"], Value::from(42)); - - let up = FakeBlobs; - let wire = contact_card_to_wire(&uid, &abids, &data, &res, &up).unwrap(); - assert_eq!(wire["uid"], Value::from("urn:uuid:42")); - assert_eq!(wire["addressBookIds"]["TAB"], Value::Bool(true)); - assert_eq!( - decode_data_uri(&wire["media"]["photo"]["uri"], "image/png"), - b"payload-42" - ); - assert_eq!( - wire["media"]["photo"]["mediaType"], - Value::from("image/png") - ); - assert!(wire["media"]["photo"].get("href").is_none()); - assert_no_blob_id(&wire); - } - - #[test] - fn calendar_event_inlines_link_enclosure_as_data_uri() { - let c = mem(); - let res = MapResolver { - to_local: HashMap::from([((ObjectType::Calendar, "CAL".to_owned()), 5)]), - to_target: HashMap::from([((ObjectType::Calendar, 5), "TCAL".to_owned())]), - }; - let ev: CalendarEvent = serde_json::from_value(serde_json::json!({ - "id": "EV1", - "calendarIds": { "CAL": true }, - "uid": "ev-with-enclosure", - "title": "Review", - "@type": "Event", - "links": { "1": { - "@type": "Link", "rel": "enclosure", - "blobId": "AB", "contentType": "text/plain", "title": "agenda.txt" - } } - })) - .unwrap(); - let mut blobs = FakeBlobs; - let local = insert_calendar_event(&c, &ev, &res, &mut blobs).unwrap(); - let (cal, dr, ud, data): (String, i64, i64, String) = c - .query_row( - &format!("{CALENDAR_EVENT_SELECT} AND id = ?1"), - params![local], - |r| Ok((r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)), - ) - .unwrap(); - let stored: Value = serde_json::from_str(&data).unwrap(); - assert_eq!(stored["links"]["1"]["@blob"], Value::from(42)); - - let up = FakeBlobs; - let wire = calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, &res, &up).unwrap(); - assert_eq!( - decode_data_uri(&wire["links"]["1"]["href"], "text/plain"), - b"payload-42" - ); - assert_eq!(wire["links"]["1"]["contentType"], Value::from("text/plain")); - assert_eq!(wire["links"]["1"]["rel"], Value::from("enclosure")); - assert!(wire["links"]["1"].get("uri").is_none()); - assert_no_blob_id(&wire); - } - - #[test] - fn contact_card_media_without_media_type_defaults_to_octet_stream() { - let c = mem(); - let res = MapResolver { - to_local: HashMap::from([((ObjectType::AddressBook, "AB".to_owned()), 1)]), - to_target: HashMap::from([((ObjectType::AddressBook, 1), "TAB".to_owned())]), - }; - c.execute( - "INSERT INTO contact_cards (id,uid,address_book_ids,data) - VALUES (1,'u-1','[1]',?1)", - params![ - serde_json::json!({ "@type": "Card", "media": { "photo": { "@blob": 9 } } }) - .to_string() - ], - ) - .unwrap(); - let (uid, abids, data): (String, String, String) = c - .query_row(&format!("{CONTACT_CARD_SELECT} WHERE id = 1"), [], |r| { - Ok((r.get(1)?, r.get(2)?, r.get(3)?)) - }) - .unwrap(); - let up = FakeBlobs; - let wire = contact_card_to_wire(&uid, &abids, &data, &res, &up).unwrap(); - assert_eq!( - decode_data_uri(&wire["media"]["photo"]["uri"], "application/octet-stream"), - b"payload-9" - ); - assert_no_blob_id(&wire); - } - - #[test] - fn an_archive_read_failure_while_inlining_is_an_archive_error_not_a_unit_failure() { - struct BrokenBlobs; - impl BlobBytes for BrokenBlobs { - fn bytes(&self, _local_id: i64) -> Result, JmapError> { - Err(JmapError::Sqlite(rusqlite::Error::QueryReturnedNoRows)) - } - } - let res = MapResolver { - to_local: HashMap::new(), - to_target: HashMap::from([ - ((ObjectType::AddressBook, 1), "TAB".to_owned()), - ((ObjectType::Calendar, 1), "TCAL".to_owned()), - ]), - }; - let card = serde_json::json!({ "@type": "Card", "media": { "photo": { "@blob": 9 } } }) - .to_string(); - let event = - serde_json::json!({ "@type": "Event", "links": { "1": { "@blob": 9 } } }).to_string(); - let failures = [ - contact_card_to_wire("u-1", "[1]", &card, &res, &BrokenBlobs).expect_err("card"), - calendar_event_to_wire("[1]", false, false, &event, &res, &BrokenBlobs) - .expect_err("event"), - ]; - for err in failures { - assert!(matches!(err, JmapError::Sqlite(_)), "{err:?}"); - let mapped = crate::error::Error::from(err); - assert!(mapped.aborts_run(), "{mapped} must abort the run"); - assert_eq!(mapped.exit_code(), 7); - } - } - - #[test] - fn calendar_event_extracts_columns_and_drops_method() { - let c = mem(); - let res = MapResolver { - to_local: HashMap::from([((ObjectType::Calendar, "CAL".to_owned()), 5)]), - to_target: HashMap::from([((ObjectType::Calendar, 5), "TCAL".to_owned())]), - }; - let ev: CalendarEvent = serde_json::from_value(serde_json::json!({ - "id": "EV1", - "calendarIds": { "CAL": true }, - "isDraft": true, - "useDefaultAlerts": false, - "method": "request", - "utcStart": "2020-01-01T00:00:00Z", - "title": "Sprint", - "@type": "Event" - })) - .unwrap(); - let mut blobs = FakeBlobs; - let local = insert_calendar_event(&c, &ev, &res, &mut blobs).unwrap(); - let (cal, dr, ud, data): (String, i64, i64, String) = c - .query_row( - &format!("{CALENDAR_EVENT_SELECT} AND id = ?1"), - params![local], - |r| Ok((r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)), - ) - .unwrap(); - assert_eq!(dr, 1); - assert_eq!(ud, 0); - let stored: Value = serde_json::from_str(&data).unwrap(); - assert!(stored.get("method").is_none()); - assert!(stored.get("utcStart").is_none()); - assert!(stored.get("calendarIds").is_none()); - assert_eq!(stored["title"], Value::from("Sprint")); - - let up = FakeBlobs; - let wire = calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, &res, &up).unwrap(); - assert_eq!(wire["calendarIds"]["TCAL"], Value::Bool(true)); - assert_eq!(wire["isDraft"], Value::Bool(true)); - assert_eq!(wire["title"], Value::from("Sprint")); - } - - #[test] - fn identity_json_columns_roundtrip() { - let c = mem(); - let id: Identity = serde_json::from_value(serde_json::json!({ - "id": "I1", - "name": "Alice", - "email": "a@x.test", - "replyTo": [ { "name": "A", "email": "a@x.test" } ] - })) - .unwrap(); - let local = insert_identity(&c, &id).unwrap(); - let wire = c - .query_row( - &format!("{IDENTITY_SELECT} WHERE id = ?1"), - params![local], - |row| Ok(row_to_identity(row)), - ) - .unwrap() - .unwrap(); - assert_eq!(wire.email, "a@x.test"); - assert_eq!(wire.reply_to.unwrap()[0].email, "a@x.test"); - assert!(wire.bcc.is_none()); - } - - #[test] - fn file_node_enum_and_target_roundtrip() { - let c = mem(); - let empty = MapResolver { - to_local: HashMap::new(), - to_target: HashMap::new(), - }; - let node: FileNode = serde_json::from_value(serde_json::json!({ - "id": "F1", - "@type": "SymLink", - "target": ["..", "etc", "hosts"], - "name": "link", - "created": "2022-02-02T02:02:02Z" - })) - .unwrap(); - let local = insert_file_node(&c, &node, None, &empty).unwrap(); - let (nt, tgt): (String, Option) = c - .query_row( - &format!("{FILE_NODE_SELECT} WHERE id = ?1"), - params![local], - |r| Ok((r.get(2)?, r.get(4)?)), - ) - .unwrap(); - assert_eq!(nt, "symlink"); - assert_eq!( - serde_json::from_str::>(&tgt.unwrap()).unwrap(), - vec!["..", "etc", "hosts"] - ); - } - - #[test] - fn address_book_and_calendar_row_to_wire_roundtrip() { - let c = mem(); - let ab: AddressBook = serde_json::from_value(serde_json::json!({ - "id": "A1", "name": "Personal", "description": "d", "sortOrder": 2, - "isDefault": true, "isSubscribed": false - })) - .unwrap(); - let abl = insert_address_book(&c, &ab).unwrap(); - let got = c - .query_row( - &format!("{ADDRESS_BOOK_SELECT} WHERE id = ?1"), - params![abl], - |r| Ok(row_to_address_book(r)), - ) - .unwrap() - .unwrap(); - assert_eq!(got.name, "Personal"); - assert!(got.is_default); - assert!(!got.is_subscribed); - - let cal: Calendar = serde_json::from_value(serde_json::json!({ - "id": "C1", "name": "Work", "color": "#ff0000", "timeZone": "Europe/Rome", - "isVisible": false - })) - .unwrap(); - let cl = insert_calendar(&c, &cal).unwrap(); - let gc = c - .query_row( - &format!("{CALENDAR_SELECT} WHERE id = ?1"), - params![cl], - |r| Ok(row_to_calendar(r)), - ) - .unwrap() - .unwrap(); - assert_eq!(gc.name, "Work"); - assert_eq!(gc.color.as_deref(), Some("#ff0000")); - assert_eq!(gc.time_zone.as_deref(), Some("Europe/Rome")); - assert!(!gc.is_visible); - } - - #[test] - fn participant_identity_and_sieve_row_to_wire_roundtrip() { - let c = mem(); - let pi: ParticipantIdentity = serde_json::from_value(serde_json::json!({ - "id": "P1", "name": "Me", "calendarAddress": "mailto:me@x.test", - "isDefault": true - })) - .unwrap(); - let pil = insert_participant_identity(&c, &pi).unwrap(); - let gp = c - .query_row( - &format!("{PARTICIPANT_IDENTITY_SELECT} WHERE id = ?1"), - params![pil], - |r| Ok(row_to_participant_identity(r)), - ) - .unwrap() - .unwrap(); - assert_eq!(gp.calendar_address, "mailto:me@x.test"); - assert!(gp.is_default); - - let ss: SieveScript = serde_json::from_value(serde_json::json!({ - "id": "S1", "name": "main", "isActive": true, "blobId": "B1" - })) - .unwrap(); - let blob = crate::db::blobs::intern_blob(&c, b"keep;").unwrap(); - let ssl = insert_sieve_script(&c, &ss, blob).unwrap(); - let gs = c - .query_row( - &format!("{SIEVE_SELECT} WHERE id = ?1"), - params![ssl], - |r| Ok(row_to_sieve_script(r)), - ) - .unwrap() - .unwrap(); - assert_eq!(gs.name.as_deref(), Some("main")); - assert!(gs.is_active); - assert_eq!(gs.blob_local_id, blob); - } - - #[test] - fn file_node_row_to_wire_translates_parent() { - let c = mem(); - let mut res = MapResolver { - to_local: HashMap::new(), - to_target: HashMap::new(), - }; - let dir: FileNode = serde_json::from_value(serde_json::json!({ - "id": "D1", "@type": "Directory", "name": "Docs", - "created": "2022-02-02T02:02:02Z" - })) - .unwrap(); - let dirl = insert_file_node(&c, &dir, None, &res).unwrap(); - res.to_local - .insert((ObjectType::FileNode, "D1".to_owned()), dirl); - let file: FileNode = serde_json::from_value(serde_json::json!({ - "id": "F1", "@type": "File", "parentId": "D1", "name": "a.bin", - "mediaType": "application/octet-stream", "created": "2022-02-02T02:02:02Z" - })) - .unwrap(); - let blob = crate::db::blobs::intern_blob(&c, b"bytes").unwrap(); - let fl = insert_file_node(&c, &file, Some(blob), &res).unwrap(); - res.to_target - .insert((ObjectType::FileNode, dirl), "TGTD".to_owned()); - let got = c - .query_row( - &format!("{FILE_NODE_SELECT} WHERE id = ?1"), - params![fl], - |r| Ok(row_to_file_node(r, &res)), - ) - .unwrap() - .unwrap(); - assert_eq!(got.wire.name, "a.bin"); - assert_eq!(got.wire.parent_id, Some(JmapId("TGTD".to_owned()))); - assert_eq!(got.blob_local_id, Some(blob)); - } - - // Each @type serializes only the properties the draft defines for it. - #[test] - fn file_node_row_to_wire_emits_only_type_specific_properties() { - let c = mem(); - let mut res = MapResolver { - to_local: HashMap::new(), - to_target: HashMap::new(), - }; - let nodes = [ - serde_json::json!({"id": "D1", "@type": "Directory", "name": "Docs", - "role": "documents", "isSubscribed": false, - "created": "2022-02-02T02:02:02Z"}), - serde_json::json!({"id": "F1", "@type": "File", "name": "run.sh", - "blobId": "B1", "mediaType": "text/x-sh", "executable": true, "size": 3, - "created": "2022-02-02T02:02:02Z"}), - serde_json::json!({"id": "L1", "@type": "SymLink", "name": "latest", - "target": ["", "Docs"], "created": "2022-02-02T02:02:02Z"}), - ]; - let blob = crate::db::blobs::intern_blob(&c, b"#!\n").unwrap(); - let mut out = Vec::new(); - for (i, n) in nodes.into_iter().enumerate() { - let w: FileNode = serde_json::from_value(n).unwrap(); - let blob_local = matches!(w.kind, NodeKind::File(_)).then_some(blob); - let l = insert_file_node(&c, &w, blob_local, &res).unwrap(); - res.to_target - .insert((ObjectType::FileNode, l), format!("T{i}")); - let row = c - .query_row( - &format!("{FILE_NODE_SELECT} WHERE id = ?1"), - params![l], - |r| Ok(row_to_file_node(r, &res)), - ) - .unwrap() - .unwrap(); - out.push(serde_json::to_value(&row.wire).unwrap()); - } - let keys = |v: &Value| { - let mut k: Vec = v.as_object().unwrap().keys().cloned().collect(); - k.sort_unstable(); - k - }; - assert_eq!( - keys(&out[0]), - ["@type", "created", "isSubscribed", "name", "role"] - ); - assert_eq!(out[0]["@type"], "Directory"); - assert_eq!(out[0]["isSubscribed"], false); - assert_eq!( - keys(&out[1]), - ["@type", "created", "executable", "mediaType", "name"] - ); - assert_eq!(out[1]["executable"], true); - assert_eq!(out[1]["mediaType"], "text/x-sh"); - assert_eq!(keys(&out[2]), ["@type", "created", "name", "target"]); - assert_eq!(out[2]["@type"], "SymLink"); - assert_eq!(out[2]["target"], serde_json::json!(["", "Docs"])); - } - - #[test] - fn file_node_rejects_unknown_type() { - let r: Result = serde_json::from_value(serde_json::json!({ - "id": "X1", "@type": "Socket", "name": "s", "created": "2022-02-02T02:02:02Z" - })); - assert!(r.is_err()); - } - - fn one_row(c: &Connection, table: &str) -> i64 { - c.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0)) - .unwrap() - } - - #[test] - fn delta_update_mailbox_in_place_preserves_id() { - let c = mem(); - let empty = MapResolver { - to_local: HashMap::new(), - to_target: HashMap::new(), - }; - let m: Mailbox = - serde_json::from_value(serde_json::json!({"id":"M1","name":"Personal"})).unwrap(); - let local = insert_mailbox(&c, &m, &empty).unwrap(); - let changed: Mailbox = serde_json::from_value( - serde_json::json!({"id":"M1","name":"PersonalRenamed","role":"archive","sortOrder":4}), - ) - .unwrap(); - assert!( - update_mailbox(&c, local, &changed, &empty).unwrap(), - "a real change reports changed=true" - ); - assert!( - !update_mailbox(&c, local, &changed, &empty).unwrap(), - "re-applying identical values is a no-op (changed=false), so re-runs converge" - ); - assert_eq!(one_row(&c, "mailboxes"), 1); - let (name, role, sort): (String, Option, i64) = c - .query_row( - "SELECT name, role, sort_order FROM mailboxes WHERE id=?1", - params![local], - |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), - ) - .unwrap(); - assert_eq!(name, "PersonalRenamed"); - assert_eq!(role.as_deref(), Some("archive")); - assert_eq!(sort, 4); - } - - #[test] - fn delta_update_email_changes_keywords_and_mailboxes_keeps_blob() { - let c = mem(); - let mut res = MapResolver { - to_local: HashMap::new(), - to_target: HashMap::new(), - }; - res.to_local - .insert((ObjectType::Mailbox, "MB".to_owned()), 7); - res.to_local - .insert((ObjectType::Mailbox, "MB2".to_owned()), 8); - let email: Email = serde_json::from_value(serde_json::json!({ - "id":"E1","blobId":"B1","receivedAt":"2021-05-04T10:00:00Z", - "mailboxIds":{"MB":true},"keywords":{"$seen":true} - })) - .unwrap(); - let blob = crate::db::blobs::intern_blob(&c, b"rfc5322").unwrap(); - let local = insert_email(&c, &email, blob, "{}", &res).unwrap(); - - let changed = serde_json::json!({ - "id":"E1","mailboxIds":{"MB2":true},"keywords":{"$seen":true,"$flagged":true} - }); - update_email(&c, local, &changed, &res).unwrap(); - assert_eq!(one_row(&c, "emails"), 1); - let got = c - .query_row( - &format!("{EMAIL_SELECT} WHERE id=?1"), - params![local], - |row| Ok(row_to_email(row)), - ) - .unwrap() - .unwrap(); - assert_eq!(got.blob_local_id, blob, "immutable body blob is untouched"); - assert_eq!(got.mailbox_locals, vec![8], "mailbox membership moved"); - assert!(got.keywords.contains(&"$flagged".to_owned())); - assert!(got.keywords.contains(&"$seen".to_owned())); - } - - #[test] - fn delta_update_identity_in_place() { - let c = mem(); - let id: Identity = - serde_json::from_value(serde_json::json!({"id":"I1","name":"Old","email":"a@x.test"})) - .unwrap(); - let local = insert_identity(&c, &id).unwrap(); - let changed: Identity = serde_json::from_value( - serde_json::json!({"id":"I1","name":"New Name","email":"a@x.test","textSignature":"sig"}), - ) - .unwrap(); - update_identity(&c, local, &changed).unwrap(); - assert_eq!(one_row(&c, "identities"), 1); - let got = c - .query_row( - &format!("{IDENTITY_SELECT} WHERE id=?1"), - params![local], - |row| Ok(row_to_identity(row)), - ) - .unwrap() - .unwrap(); - assert_eq!(got.name, "New Name"); - assert_eq!(got.text_signature, "sig"); - } - - #[test] - fn delta_update_address_book_in_place() { - let c = mem(); - let ab: AddressBook = - serde_json::from_value(serde_json::json!({"id":"A1","name":"Old"})).unwrap(); - let local = insert_address_book(&c, &ab).unwrap(); - let changed: AddressBook = serde_json::from_value( - serde_json::json!({"id":"A1","name":"Renamed","description":"d2","sortOrder":3}), - ) - .unwrap(); - update_address_book(&c, local, &changed).unwrap(); - assert_eq!(one_row(&c, "address_books"), 1); - let got = c - .query_row( - &format!("{ADDRESS_BOOK_SELECT} WHERE id=?1"), - params![local], - |r| Ok(row_to_address_book(r)), - ) - .unwrap() - .unwrap(); - assert_eq!(got.name, "Renamed"); - assert_eq!(got.description.as_deref(), Some("d2")); - } - - #[test] - fn delta_update_calendar_in_place() { - let c = mem(); - let cal: Calendar = - serde_json::from_value(serde_json::json!({"id":"C1","name":"Old","color":"#000"})) - .unwrap(); - let local = insert_calendar(&c, &cal).unwrap(); - let changed: Calendar = serde_json::from_value( - serde_json::json!({"id":"C1","name":"Renamed","color":"#abcdef","timeZone":"Europe/Rome"}), - ) - .unwrap(); - update_calendar(&c, local, &changed).unwrap(); - assert_eq!(one_row(&c, "calendars"), 1); - let got = c - .query_row( - &format!("{CALENDAR_SELECT} WHERE id=?1"), - params![local], - |r| Ok(row_to_calendar(r)), - ) - .unwrap() - .unwrap(); - assert_eq!(got.name, "Renamed"); - assert_eq!(got.color.as_deref(), Some("#abcdef")); - assert_eq!(got.time_zone.as_deref(), Some("Europe/Rome")); - } - - #[test] - fn delta_update_participant_identity_in_place() { - let c = mem(); - let pi: ParticipantIdentity = serde_json::from_value( - serde_json::json!({"id":"P1","name":"Old","calendarAddress":"mailto:me@x.test"}), - ) - .unwrap(); - let local = insert_participant_identity(&c, &pi).unwrap(); - let changed: ParticipantIdentity = serde_json::from_value( - serde_json::json!({"id":"P1","name":"New","calendarAddress":"mailto:me@x.test"}), - ) - .unwrap(); - update_participant_identity(&c, local, &changed).unwrap(); - assert_eq!(one_row(&c, "participant_identities"), 1); - let got = c - .query_row( - &format!("{PARTICIPANT_IDENTITY_SELECT} WHERE id=?1"), - params![local], - |r| Ok(row_to_participant_identity(r)), - ) - .unwrap() - .unwrap(); - assert_eq!(got.name, "New"); - } - - #[test] - fn delta_update_sieve_script_swaps_blob_and_active() { - let c = mem(); - let ss: SieveScript = serde_json::from_value( - serde_json::json!({"id":"S1","name":"main","isActive":false,"blobId":"B1"}), - ) - .unwrap(); - let blob1 = crate::db::blobs::intern_blob(&c, b"keep;").unwrap(); - let local = insert_sieve_script(&c, &ss, blob1).unwrap(); - let blob2 = crate::db::blobs::intern_blob(&c, b"discard;").unwrap(); - let changed: SieveScript = serde_json::from_value( - serde_json::json!({"id":"S1","name":"main","isActive":true,"blobId":"B2"}), - ) - .unwrap(); - update_sieve_script(&c, local, &changed, blob2).unwrap(); - assert_eq!(one_row(&c, "sieve_scripts"), 1); - let got = c - .query_row( - &format!("{SIEVE_SELECT} WHERE id=?1"), - params![local], - |r| Ok(row_to_sieve_script(r)), - ) - .unwrap() - .unwrap(); - assert!(got.is_active); - assert_eq!(got.blob_local_id, blob2, "content blob swapped"); - } - - #[test] - fn delta_update_file_node_changes_blob_and_name() { - let c = mem(); - let res = MapResolver { - to_local: HashMap::new(), - to_target: HashMap::new(), - }; - let file: FileNode = serde_json::from_value(serde_json::json!({ - "id":"F1","@type":"File","name":"a.bin", - "mediaType":"application/octet-stream","created":"2022-02-02T02:02:02Z" - })) - .unwrap(); - let blob1 = crate::db::blobs::intern_blob(&c, b"v1").unwrap(); - let local = insert_file_node(&c, &file, Some(blob1), &res).unwrap(); - let blob2 = crate::db::blobs::intern_blob(&c, b"v2").unwrap(); - let changed: FileNode = serde_json::from_value(serde_json::json!({ - "id":"F1","@type":"File","name":"renamed.bin", - "mediaType":"text/plain","executable":true,"created":"2022-02-02T02:02:02Z" - })) - .unwrap(); - update_file_node(&c, local, &changed, Some(blob2), &res).unwrap(); - assert_eq!(one_row(&c, "file_nodes"), 1); - let (name, mt, b, x): (String, Option, Option, i64) = c - .query_row( - "SELECT name, media_type, blob_id, executable FROM file_nodes WHERE id=?1", - params![local], - |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)), - ) - .unwrap(); - assert_eq!(name, "renamed.bin"); - assert_eq!(mt.as_deref(), Some("text/plain")); - assert_eq!(b, Some(blob2)); - assert_eq!(x, 1); - } - - #[test] - fn delta_update_contact_card_rewrites_data_and_columns() { - let c = mem(); - let res = MapResolver { - to_local: HashMap::from([((ObjectType::AddressBook, "AB".to_owned()), 1)]), - to_target: HashMap::new(), - }; - let card: ContactCard = serde_json::from_value(serde_json::json!({ - "id":"C1","addressBookIds":{"AB":true},"uid":"u-1","name":{"full":"Old"} - })) - .unwrap(); - let mut blobs = FakeBlobs; - let local = insert_contact_card(&c, &card, &res, &mut blobs).unwrap(); - let changed: ContactCard = serde_json::from_value(serde_json::json!({ - "id":"C1","addressBookIds":{"AB":true},"uid":"u-1","name":{"full":"New Name"} - })) - .unwrap(); - update_contact_card(&c, local, &changed, &res, &mut blobs).unwrap(); - assert_eq!(one_row(&c, "contact_cards"), 1); - let (uid, data): (String, String) = c - .query_row( - &format!("{CONTACT_CARD_SELECT} WHERE id=?1"), - params![local], - |r| Ok((r.get(1)?, r.get(3)?)), - ) - .unwrap(); - assert_eq!(uid, "u-1"); - let stored: Value = serde_json::from_str(&data).unwrap(); - assert_eq!(stored["name"]["full"], Value::from("New Name")); - } - - #[test] - fn delta_update_calendar_event_rewrites_data_and_columns() { - let c = mem(); - let res = MapResolver { - to_local: HashMap::from([((ObjectType::Calendar, "CAL".to_owned()), 5)]), - to_target: HashMap::new(), - }; - let ev: CalendarEvent = serde_json::from_value(serde_json::json!({ - "id":"EV1","calendarIds":{"CAL":true},"isDraft":false, - "useDefaultAlerts":false,"title":"Old","@type":"Event" - })) - .unwrap(); - let mut blobs = FakeBlobs; - let local = insert_calendar_event(&c, &ev, &res, &mut blobs).unwrap(); - let changed: CalendarEvent = serde_json::from_value(serde_json::json!({ - "id":"EV1","calendarIds":{"CAL":true},"isDraft":true, - "useDefaultAlerts":false,"title":"Rescheduled","@type":"Event" - })) - .unwrap(); - update_calendar_event(&c, local, &changed, &res, &mut blobs).unwrap(); - assert_eq!(one_row(&c, "calendar_events"), 1); - let (dr, data): (i64, String) = c - .query_row( - &format!("{CALENDAR_EVENT_SELECT} AND id=?1"), - params![local], - |r| Ok((r.get(2)?, r.get(4)?)), - ) - .unwrap(); - assert_eq!(dr, 1, "isDraft column updated"); - let stored: Value = serde_json::from_str(&data).unwrap(); - assert_eq!(stored["title"], Value::from("Rescheduled")); - } -} diff --git a/src/sync/import_jmap/tree.rs b/src/sync/import_jmap/tree.rs deleted file mode 100644 index 77cb554..0000000 --- a/src/sync/import_jmap/tree.rs +++ /dev/null @@ -1,131 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::{HashMap, HashSet}; - -pub struct TreeOrder { - pub order: Vec, - pub orphans: Vec, - pub cycle_roots: Vec, -} - -pub fn topo_order( - items: &[(String, Option)], - existing_parents: &HashSet, -) -> TreeOrder { - let index: HashMap<&str, usize> = items - .iter() - .enumerate() - .map(|(i, (id, _))| (id.as_str(), i)) - .collect(); - - let mut order = Vec::with_capacity(items.len()); - let mut orphans = Vec::new(); - let mut cycle_roots = Vec::new(); - let mut placed: HashSet = HashSet::new(); - - loop { - let mut progressed = false; - for (i, (_, parent)) in items.iter().enumerate() { - if placed.contains(&i) { - continue; - } - let ready = match parent { - None => true, - Some(p) => match index.get(p.as_str()) { - Some(&pi) => placed.contains(&pi), - None => { - if !existing_parents.contains(p) { - orphans.push(i); - } - true - } - }, - }; - if ready { - order.push(i); - placed.insert(i); - progressed = true; - } - } - if placed.len() == items.len() { - break; - } - if !progressed { - let mut remaining: Vec = - (0..items.len()).filter(|i| !placed.contains(i)).collect(); - remaining.sort_by(|&a, &b| items[a].0.cmp(&items[b].0)); - let root = remaining[0]; - cycle_roots.push(root); - order.push(root); - placed.insert(root); - } - } - - TreeOrder { - order, - orphans, - cycle_roots, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn item(id: &str, parent: Option<&str>) -> (String, Option) { - (id.to_owned(), parent.map(str::to_owned)) - } - - fn empty() -> HashSet { - HashSet::new() - } - - fn pos(order: &[usize], items: &[(String, Option)], id: &str) -> usize { - order - .iter() - .position(|&i| items[i].0 == id) - .expect("id in order") - } - - #[test] - fn parents_precede_children() { - let items = vec![item("c", Some("b")), item("b", Some("a")), item("a", None)]; - let t = topo_order(&items, &empty()); - assert_eq!(t.order.len(), 3); - assert!(pos(&t.order, &items, "a") < pos(&t.order, &items, "b")); - assert!(pos(&t.order, &items, "b") < pos(&t.order, &items, "c")); - assert!(t.orphans.is_empty()); - assert!(t.cycle_roots.is_empty()); - } - - #[test] - fn parent_not_fetched_and_not_existing_is_orphan() { - let items = vec![item("x", Some("missing"))]; - let t = topo_order(&items, &empty()); - assert_eq!(t.order, vec![0]); - assert_eq!(t.orphans, vec![0]); - } - - #[test] - fn parent_already_in_db_is_not_orphan() { - let items = vec![item("x", Some("p"))]; - let existing: HashSet = ["p".to_owned()].into_iter().collect(); - let t = topo_order(&items, &existing); - assert_eq!(t.order, vec![0]); - assert!(t.orphans.is_empty()); - } - - #[test] - fn cycle_is_broken_by_smallest_id() { - let items = vec![item("b", Some("a")), item("a", Some("b"))]; - let t = topo_order(&items, &empty()); - assert_eq!(t.order.len(), 2); - assert_eq!(t.cycle_roots.len(), 1); - assert_eq!(items[t.cycle_roots[0]].0, "a"); - assert!(pos(&t.order, &items, "a") < pos(&t.order, &items, "b")); - } -} diff --git a/src/sync/import_maildir/coordinator.rs b/src/sync/import_maildir/coordinator.rs deleted file mode 100644 index 2eaafe2..0000000 --- a/src/sync/import_maildir/coordinator.rs +++ /dev/null @@ -1,527 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::{HashMap, HashSet}; -use std::path::{Path, PathBuf}; - -use regex::Regex; -use rusqlite::{Connection, OptionalExtension, params}; -use url::Url; - -use crate::db; -use crate::db::sources::SourceKey; -use crate::error::Error; -use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; -use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts}; - -use super::messages; -use super::tree; -use super::tree::{DiscoverError, FolderFilters, ResolvedFolder}; - -pub struct MaildirImportConfig { - pub maildir: PathBuf, - pub include: Vec, - pub exclude: Vec, - pub folder: Vec, - pub automap: bool, - pub include_deleted: bool, - pub allow_source_change: bool, -} - -pub fn run(common: CommonConfig, config: MaildirImportConfig) -> Result { - run_reporting(common, config).into_result() -} - -pub fn run_reporting(common: CommonConfig, config: MaildirImportConfig) -> RunOutcome { - let mut summary = Summary::default(); - let error = run_into(common, config, &mut summary).err(); - RunOutcome { summary, error } -} - -fn run_into( - common: CommonConfig, - config: MaildirImportConfig, - summary: &mut Summary, -) -> Result<(), Error> { - let logger = common.logger; - if common.threads > 1 { - log_at( - logger, - LEVEL_PROGRESS, - "maildir importer is single-threaded; --threads value will be ignored", - ); - } - - let canonical = std::fs::canonicalize(&config.maildir) - .map_err(|e| Error::Usage(format!("--maildir {:?}: {e}", config.maildir)))?; - - warn_if_looks_like_subfolder(&canonical, logger); - - let discovered = match tree::discover(&canonical, config.automap) { - Ok(v) => v, - Err(DiscoverError::NotFound(p)) => { - return Err(Error::Usage(format!("maildir path does not exist: {p:?}"))); - } - Err(DiscoverError::NotADirectory(p)) => { - return Err(Error::Usage(format!("not a directory: {p:?}"))); - } - Err(DiscoverError::NotAMaildir(p)) => { - return Err(Error::Usage(format!( - "not a Maildir: {p:?} (missing cur/ subdirectory)" - ))); - } - Err(DiscoverError::NotMaildirPlus(name)) => { - return Err(Error::Usage(format!( - "only Maildir++ layout is supported; found non-prefixed subfolder {name:?} \ - (subfolders must be named with a leading '.', e.g. '.{name}')" - ))); - } - Err(DiscoverError::Io(p, e)) => { - return Err(Error::Usage(format!("walking {p:?}: {e}"))); - } - }; - - let include_deleted = config.include_deleted; - let filters = FolderFilters { - include: config.include, - exclude: config.exclude, - explicit: config.folder, - }; - if !filters.explicit.is_empty() && (!filters.include.is_empty() || !filters.exclude.is_empty()) - { - return Err(Error::Usage( - "--folder is mutually exclusive with --include/--exclude".to_owned(), - )); - } - let mut resolved = tree::apply_filters(discovered, &filters); - tree::restore_ephemeral_parents(&mut resolved, &canonical); - let run_flags = RunFlags { include_deleted }; - - let mut conn = db::init::open(&common.archive)?; - let session_url = file_url_for(&canonical)?; - let account_id = canonical.to_string_lossy().into_owned(); - let account_name = canonical - .file_name() - .and_then(|n| n.to_str()) - .map(str::to_owned); - - if let Some((existing_url, existing_account)) = - db::sources::conflicting_source(&conn, "maildir", &session_url, &account_id)? - && !config.allow_source_change - { - return Err(Error::SourceChange(format!( - "archive already records maildir source {existing_url} / {existing_account}; \ - re-run with --allow-source-change or use a fresh archive" - ))); - } - let source_key = SourceKey { - kind: "maildir".to_owned(), - session_url: session_url.clone(), - account_id: account_id.clone(), - }; - let source_id = if common.dry_run { - db::sources::find_source(&conn, &source_key)?.unwrap_or(-1) - } else { - db::sources::upsert_source(&conn, &source_key, account_name.as_deref(), "")? - }; - - log_at( - logger, - LEVEL_PROGRESS, - &format!("maildir source {session_url}"), - ); - - if common.dry_run { - *summary = build_dry_run_summary(&conn, source_id, &resolved, logger)?; - return Ok(()); - } - - let mut mailbox_counts = TypeCounts::default(); - let mut email_counts = TypeCounts::default(); - - let local_mailboxes = db::maildir_ids::mailbox_folders(&conn, source_id)?; - let mailbox_local_ids = upsert_mailboxes(&mut conn, source_id, &resolved, &mut mailbox_counts)?; - - let server_names: HashSet<&str> = resolved.iter().map(|f| f.name.as_str()).collect(); - let mut vanished: Vec = local_mailboxes - .keys() - .filter(|n| !server_names.contains(n.as_str())) - .cloned() - .collect(); - tree::vanished_depth_sort(&mut vanished); - if !vanished.is_empty() { - delete_vanished_folders( - &mut conn, - source_id, - &vanished, - &mut mailbox_counts, - &mut email_counts, - logger, - )?; - } - - for folder in &resolved { - if folder.ephemeral { - continue; - } - let mailbox_local = match mailbox_local_ids.get(&folder.name).copied() { - Some(id) => id, - None => { - log_at( - logger, - LEVEL_DEFAULT, - &format!("folder {:?}: missing from mailbox id map", folder.name), - ); - email_counts.failed += 1; - continue; - } - }; - if let Err(e) = reconcile_folder( - &mut conn, - source_id, - folder, - mailbox_local, - &run_flags, - &mut email_counts, - logger, - ) { - if e.aborts_run() { - *summary = Summary { - per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)], - retries_observed: 0, - retry_after_sleeps: 0, - }; - return Err(e); - } - log_at( - logger, - LEVEL_DEFAULT, - &format!("folder {:?}: {e}", folder.name), - ); - email_counts.failed += 1; - } - } - - if email_counts.failed == 0 && mailbox_counts.failed == 0 { - let tx = conn.unchecked_transaction()?; - db::blobs::gc_orphan_blobs(&tx)?; - tx.commit()?; - } - - log_at( - logger, - LEVEL_DEFAULT, - &format!( - "maildir: mailbox created={} deleted={} skipped={} failed={}; \ - email created={} fetched={} deleted={} skipped={} failed={}", - mailbox_counts.created, - mailbox_counts.deleted, - mailbox_counts.skipped, - mailbox_counts.failed, - email_counts.created, - email_counts.fetched, - email_counts.deleted, - email_counts.skipped, - email_counts.failed - ), - ); - - *summary = Summary { - per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)], - retries_observed: 0, - retry_after_sleeps: 0, - }; - Ok(()) -} - -struct RunFlags { - include_deleted: bool, -} - -fn file_url_for(p: &Path) -> Result { - Url::from_file_path(p).map(|u| u.to_string()).map_err(|_| { - Error::Usage(format!( - "cannot encode maildir path as a file:// URL: {p:?}" - )) - }) -} - -fn warn_if_looks_like_subfolder(canonical: &Path, logger: Logger) { - let Some(basename) = canonical.file_name().and_then(|n| n.to_str()) else { - return; - }; - if !basename.starts_with('.') { - return; - } - let Some(parent) = canonical.parent() else { - return; - }; - if !parent.join("cur").is_dir() { - return; - } - log_at( - logger, - LEVEL_DEFAULT, - &format!( - "warning: {canonical:?} looks like a Maildir++ subfolder of {parent:?} \ - (basename starts with '.' and the parent has its own cur/); proceeding, \ - but you may have meant to import the parent" - ), - ); -} - -fn upsert_mailboxes( - conn: &mut Connection, - source_id: i64, - folders: &[ResolvedFolder], - counts: &mut TypeCounts, -) -> Result, Error> { - let tx = conn.transaction()?; - let mut local_ids: HashMap = HashMap::new(); - for folder in folders { - let parent_local = match folder.parent_path.as_deref() { - Some(p) => local_ids.get(p).copied().or_else(|| { - db::maildir_ids::local_for_mailbox(&tx, source_id, p) - .ok() - .flatten() - }), - None => None, - }; - let is_subscribed: i64 = if folder.ephemeral { 0 } else { 1 }; - let existing = db::maildir_ids::local_for_mailbox(&tx, source_id, &folder.name)?; - let id = if let Some(id) = existing { - tx.execute( - "UPDATE mailboxes SET name = ?1, parent_id = ?2, role = ?3, is_subscribed = ?4 - WHERE id = ?5", - params![folder.leaf, parent_local, folder.role, is_subscribed, id], - )?; - id - } else { - tx.execute( - "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) - VALUES (?1, ?2, ?3, 0, ?4)", - params![folder.leaf, parent_local, folder.role, is_subscribed], - )?; - let new_id = tx.last_insert_rowid(); - db::maildir_ids::insert_mailbox(&tx, source_id, &folder.name, new_id)?; - counts.created += 1; - counts.fetched += 1; - new_id - }; - local_ids.insert(folder.name.clone(), id); - } - tx.commit()?; - Ok(local_ids) -} - -fn delete_vanished_folders( - conn: &mut Connection, - source_id: i64, - folders: &[String], - mailbox_counts: &mut TypeCounts, - email_counts: &mut TypeCounts, - logger: Logger, -) -> Result<(), Error> { - let tx = conn.transaction()?; - for name in folders { - let local_id = match db::maildir_ids::local_for_mailbox(&tx, source_id, name)? { - Some(id) => id, - None => continue, - }; - let surviving_child: Option = tx - .query_row( - "SELECT id FROM mailboxes WHERE parent_id = ?1 LIMIT 1", - params![local_id], - |row| row.get(0), - ) - .optional()?; - if surviving_child.is_some() { - log_at( - logger, - LEVEL_DEFAULT, - &format!( - "folder {name:?}: vanished from disk but still has child mailboxes in the \ - archive; skipping delete (parent_id RESTRICT)" - ), - ); - mailbox_counts.failed += 1; - continue; - } - let email_ids: Vec = tx - .prepare( - "SELECT local_id FROM sync_id_maildir - WHERE source_id = ?1 AND type_name = 'email' AND folder = ?2", - )? - .query_map(params![source_id, name], |row| row.get(0))? - .collect::, _>>()?; - for eid in &email_ids { - tx.execute("DELETE FROM emails WHERE id = ?1", params![eid])?; - email_counts.deleted += 1; - } - db::maildir_ids::delete_all_emails_in_folder(&tx, source_id, name)?; - tx.execute("DELETE FROM mailboxes WHERE id = ?1", params![local_id])?; - db::maildir_ids::delete_mailbox(&tx, source_id, name)?; - mailbox_counts.deleted += 1; - } - tx.commit()?; - Ok(()) -} - -fn reconcile_folder( - conn: &mut Connection, - source_id: i64, - folder: &ResolvedFolder, - mailbox_local: i64, - flags: &RunFlags, - counts: &mut TypeCounts, - logger: Logger, -) -> Result<(), Error> { - let listing = match messages::list_folder(&folder.path) { - Ok(l) => l, - Err(e) => { - return Err(Error::Partial(format!( - "list {folder:?}: {e}", - folder = folder.path - ))); - } - }; - if listing.io_failures > 0 { - counts.failed += listing.io_failures; - } - let local = db::maildir_ids::email_ids_in_folder(conn, source_id, &folder.name)?; - let diff = messages::diff(listing.entries, &local); - - log_at( - logger, - LEVEL_PROGRESS, - &format!( - "folder {:?}: new={} present={} vanished={}", - folder.name, - diff.new.len(), - diff.present.len(), - diff.vanished.len() - ), - ); - - let ctx = messages::InsertContext { - source_id, - folder: &folder.name, - mailbox_local, - include_deleted: flags.include_deleted, - }; - messages::apply_folder(conn, ctx, diff, counts, logger)?; - Ok(()) -} - -fn build_dry_run_summary( - conn: &Connection, - source_id: i64, - folders: &[ResolvedFolder], - logger: Logger, -) -> Result { - let mut mailbox = TypeCounts::default(); - let mut email = TypeCounts::default(); - let local_mailboxes = if source_id < 0 { - HashMap::new() - } else { - db::maildir_ids::mailbox_folders(conn, source_id)? - }; - let server_set: HashSet<&str> = folders.iter().map(|f| f.name.as_str()).collect(); - let new_folders: Vec<&str> = folders - .iter() - .map(|f| f.name.as_str()) - .filter(|n| !local_mailboxes.contains_key(*n)) - .collect(); - let vanished_folder_names: Vec<&str> = local_mailboxes - .keys() - .filter(|n| !server_set.contains(n.as_str())) - .map(|s| s.as_str()) - .collect(); - mailbox.created += new_folders.len() as u64; - mailbox.fetched += new_folders.len() as u64; - mailbox.deleted += vanished_folder_names.len() as u64; - - for folder in folders { - if folder.ephemeral { - continue; - } - let listing = match messages::list_folder(&folder.path) { - Ok(l) => l, - Err(e) => { - logger.warn(&format!("dry-run list {:?}: {e}", folder.name)); - email.failed += 1; - continue; - } - }; - email.failed += listing.io_failures; - let disk_ids: HashSet = listing - .entries - .iter() - .map(|e| e.unique_id.clone()) - .collect(); - let local = if source_id < 0 { - HashMap::new() - } else { - db::maildir_ids::email_ids_in_folder(conn, source_id, &folder.name)? - }; - let local_keys: HashSet<&String> = local.keys().collect(); - let new_count = disk_ids.iter().filter(|u| !local.contains_key(*u)).count() as u64; - let vanished_count = local_keys - .iter() - .filter(|k| !disk_ids.contains(k.as_str())) - .count() as u64; - let present_count = disk_ids.iter().filter(|u| local.contains_key(*u)).count() as u64; - email.created += new_count; - email.fetched += new_count; - email.deleted += vanished_count; - email.skipped += present_count; - } - for name in &vanished_folder_names { - if source_id >= 0 { - let n = db::maildir_ids::email_ids_in_folder(conn, source_id, name)?.len() as u64; - email.deleted += n; - } - } - - Ok(Summary { - per_type: vec![("mailbox", mailbox), ("email", email)], - retries_observed: 0, - retry_after_sleeps: 0, - }) -} - -fn log_at(logger: Logger, level: u8, msg: &str) { - if logger.enabled(level) { - eprintln!("{msg}"); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn file_url_for_ascii_path_is_passthrough() { - let p = std::path::Path::new("/home/alice/Maildir"); - let url = file_url_for(p).unwrap(); - assert_eq!(url, "file:///home/alice/Maildir"); - } - - #[test] - fn file_url_for_percent_encodes_spaces_and_unicode() { - let p = std::path::Path::new("/home/alice/My Maildir/Sübfolder"); - let url = file_url_for(p).unwrap(); - assert!(url.starts_with("file:///home/alice/My%20Maildir/")); - assert!(url.contains("S%C3%BCbfolder")); - } - - #[test] - fn file_url_for_rejects_relative_paths() { - let p = std::path::Path::new("./relative"); - let err = file_url_for(p).unwrap_err(); - assert!(matches!(err, Error::Usage(_))); - } -} diff --git a/src/sync/import_maildir/keywords.rs b/src/sync/import_maildir/keywords.rs deleted file mode 100644 index 5afb158..0000000 --- a/src/sync/import_maildir/keywords.rs +++ /dev/null @@ -1,200 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use mail_parser::mailbox::maildir::Flag; - -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct Translation { - pub keywords: Vec, - pub has_trashed_flag: bool, -} - -pub fn translate_flags(flags: &[Flag], include_deleted: bool) -> Translation { - let mut out: Vec = Vec::new(); - let mut has_trashed = false; - for f in flags { - match f { - Flag::Passed => push_unique(&mut out, "$forwarded"), - Flag::Replied => push_unique(&mut out, "$answered"), - Flag::Seen => push_unique(&mut out, "$seen"), - Flag::Draft => push_unique(&mut out, "$draft"), - Flag::Flagged => push_unique(&mut out, "$flagged"), - Flag::Trashed => { - has_trashed = true; - if include_deleted { - push_unique(&mut out, "$deleted"); - } - } - } - } - out.sort(); - Translation { - keywords: out, - has_trashed_flag: has_trashed, - } -} - -fn push_unique(out: &mut Vec, k: &str) { - if !out.iter().any(|x| x == k) { - out.push(k.to_owned()); - } -} - -pub fn flags_from_filename(filename: &str) -> Vec { - let Some(part) = filename.rsplit_once("2,").map(|(_, s)| s) else { - return Vec::new(); - }; - let mut out = Vec::new(); - for &ch in part.as_bytes() { - match ch { - b'P' => out.push(Flag::Passed), - b'R' => out.push(Flag::Replied), - b'S' => out.push(Flag::Seen), - b'T' => out.push(Flag::Trashed), - b'D' => out.push(Flag::Draft), - b'F' => out.push(Flag::Flagged), - other => { - if !other.is_ascii_alphanumeric() { - break; - } - } - } - } - out -} - -pub fn unique_id_from_filename(filename: &str) -> &str { - match filename.split_once(':') { - Some((unique, _)) => unique, - None => filename, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn empty_flags_yields_empty_keywords() { - let t = translate_flags(&[], false); - assert!(t.keywords.is_empty()); - assert!(!t.has_trashed_flag); - } - - #[test] - fn each_flag_maps_to_expected_jmap_keyword() { - let t = translate_flags( - &[ - Flag::Passed, - Flag::Replied, - Flag::Seen, - Flag::Draft, - Flag::Flagged, - ], - false, - ); - let mut expected = vec!["$forwarded", "$answered", "$seen", "$draft", "$flagged"]; - expected.sort(); - let expected: Vec = expected.into_iter().map(str::to_owned).collect(); - assert_eq!(t.keywords, expected); - assert!(!t.has_trashed_flag); - } - - #[test] - fn trashed_dropped_by_default_marks_flag() { - let t = translate_flags(&[Flag::Trashed, Flag::Seen], false); - assert!(t.has_trashed_flag); - assert_eq!(t.keywords, vec!["$seen".to_owned()]); - } - - #[test] - fn trashed_with_include_deleted_adds_dollar_deleted() { - let t = translate_flags(&[Flag::Trashed, Flag::Seen], true); - assert!(t.has_trashed_flag); - let mut expected = vec!["$seen", "$deleted"]; - expected.sort(); - let expected: Vec = expected.into_iter().map(str::to_owned).collect(); - assert_eq!(t.keywords, expected); - } - - #[test] - fn order_independent_keywords_are_stable() { - let a = translate_flags(&[Flag::Seen, Flag::Flagged], false); - let b = translate_flags(&[Flag::Flagged, Flag::Seen], false); - assert_eq!(a.keywords, b.keywords); - } - - #[test] - fn duplicate_flags_dedup() { - let t = translate_flags(&[Flag::Seen, Flag::Seen, Flag::Seen], false); - assert_eq!(t.keywords, vec!["$seen".to_owned()]); - } - - #[test] - fn unique_id_strips_at_first_colon() { - assert_eq!( - unique_id_from_filename("1739471123.M001P01234V0.host:2,RS"), - "1739471123.M001P01234V0.host" - ); - } - - #[test] - fn unique_id_no_colon_is_whole_basename() { - assert_eq!( - unique_id_from_filename("1739471123.M001P0.host"), - "1739471123.M001P0.host" - ); - } - - #[test] - fn unique_id_strips_legacy_one_form() { - assert_eq!( - unique_id_from_filename("1234.M5.host:1,experimental"), - "1234.M5.host" - ); - } - - #[test] - fn flags_from_filename_recognises_full_set() { - let mut flags = flags_from_filename("name:2,DFPRST"); - flags.sort(); - let mut want = vec![ - Flag::Draft, - Flag::Flagged, - Flag::Passed, - Flag::Replied, - Flag::Seen, - Flag::Trashed, - ]; - want.sort(); - assert_eq!(flags, want); - } - - #[test] - fn flags_from_filename_no_info_section_yields_empty() { - assert!(flags_from_filename("plain").is_empty()); - assert!(flags_from_filename("name:").is_empty()); - assert!(flags_from_filename("name:1,X").is_empty()); - } - - #[test] - fn flags_from_filename_dovecot_extension_metadata_passes_through() { - let mut flags = flags_from_filename("uid,S=1234,W=1300:2,RS"); - flags.sort(); - let mut want = vec![Flag::Replied, Flag::Seen]; - want.sort(); - assert_eq!(flags, want); - } - - #[test] - fn flags_from_filename_unknown_alpha_chars_skipped() { - let mut flags = flags_from_filename("uid:2,SaT"); - flags.sort(); - let mut want = vec![Flag::Seen, Flag::Trashed]; - want.sort(); - assert_eq!(flags, want); - } -} diff --git a/src/sync/import_maildir/messages.rs b/src/sync/import_maildir/messages.rs deleted file mode 100644 index c223b0a..0000000 --- a/src/sync/import_maildir/messages.rs +++ /dev/null @@ -1,801 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::time::{Duration, UNIX_EPOCH}; - -use mail_parser::mailbox::maildir::Flag; -use rusqlite::{Connection, params}; -use serde_json::Value; -use time::OffsetDateTime; -use time::format_description::well_known::Rfc3339; - -use crate::db; -use crate::sync::TypeCounts; -use crate::sync::emailmeta::email_meta_from_blob; -use crate::sync::keys::index_to_json; - -use super::keywords::{Translation, flags_from_filename, translate_flags, unique_id_from_filename}; - -#[derive(Debug, Clone)] -pub struct DiskEntry { - pub unique_id: String, - pub filename: String, - pub path: PathBuf, - pub flags: Vec, - pub mtime_unix: u64, -} - -#[derive(Debug, Clone, Default)] -pub struct DiskListing { - pub entries: Vec, - pub io_failures: u64, -} - -pub fn list_folder(folder_path: &Path) -> std::io::Result { - let mut out = DiskListing::default(); - let mut seen: std::collections::HashMap = std::collections::HashMap::new(); - for sub in ["cur", "new"] { - let dir = folder_path.join(sub); - if !dir.is_dir() { - continue; - } - let entries = std::fs::read_dir(&dir)?; - for entry in entries { - let entry = match entry { - Ok(e) => e, - Err(_) => { - out.io_failures += 1; - continue; - } - }; - let path = entry.path(); - let file_type = match entry.file_type() { - Ok(t) => t, - Err(_) => { - out.io_failures += 1; - continue; - } - }; - if !file_type.is_file() { - continue; - } - let Some(raw_name) = path.file_name() else { - continue; - }; - let filename: String = raw_name.to_string_lossy().into_owned(); - if filename.starts_with('.') { - continue; - } - let unique_id = unique_id_from_filename(&filename).to_owned(); - if unique_id.is_empty() { - continue; - } - if seen.contains_key(&unique_id) { - continue; - } - let flags = flags_from_filename(&filename); - let mtime_unix = match entry.metadata().and_then(|m| m.modified()).and_then(|t| { - t.duration_since(UNIX_EPOCH).map_err(|e| { - std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()) - }) - }) { - Ok(d) => d.as_secs(), - Err(_) => 0, - }; - seen.insert(unique_id.clone(), out.entries.len()); - out.entries.push(DiskEntry { - unique_id, - filename, - path, - flags, - mtime_unix, - }); - } - } - Ok(out) -} - -#[derive(Debug, Clone, Default)] -pub struct FolderDiff { - pub new: Vec, - pub present: Vec<(DiskEntry, i64)>, - pub vanished: Vec<(String, i64)>, -} - -pub fn diff(disk: Vec, local: &HashMap) -> FolderDiff { - let mut diff = FolderDiff::default(); - let mut seen: std::collections::HashSet = std::collections::HashSet::new(); - for entry in disk { - match local.get(&entry.unique_id) { - None => { - seen.insert(entry.unique_id.clone()); - diff.new.push(entry); - } - Some(&local_id) => { - seen.insert(entry.unique_id.clone()); - diff.present.push((entry, local_id)); - } - } - } - for (uid, local_id) in local { - if !seen.contains(uid) { - diff.vanished.push((uid.clone(), *local_id)); - } - } - diff.vanished.sort_by(|a, b| a.0.cmp(&b.0)); - diff -} - -#[derive(Clone, Copy)] -pub struct InsertContext<'a> { - pub source_id: i64, - pub folder: &'a str, - pub mailbox_local: i64, - pub include_deleted: bool, -} - -pub fn insert_new( - tx: &rusqlite::Transaction<'_>, - ctx: InsertContext<'_>, - entry: &DiskEntry, -) -> Result, InsertError> { - let bytes = std::fs::read(&entry.path).map_err(InsertError::Io)?; - let translation = translate_flags(&entry.flags, ctx.include_deleted); - if translation.has_trashed_flag && !ctx.include_deleted { - return Ok(None); - } - let (index, date_header) = email_meta_from_blob(&bytes); - let received_at = pick_received_at(&entry.filename, entry.mtime_unix, date_header.as_deref()); - let blob_id = db::blobs::intern_blob(tx, &bytes)?; - let message_match = index_to_json(&index); - let mailbox_ids = Value::Array(vec![Value::from(ctx.mailbox_local)]); - let keywords = keywords_json(&translation); - tx.execute( - "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match) - VALUES (?1, ?2, ?3, ?4, ?5)", - params![ - blob_id, - received_at, - mailbox_ids.to_string(), - keywords, - message_match - ], - )?; - let local_id = tx.last_insert_rowid(); - db::maildir_ids::insert_email(tx, ctx.source_id, ctx.folder, &entry.unique_id, local_id)?; - Ok(Some(local_id)) -} - -pub fn delivery_time_from_filename(filename: &str) -> Option { - let digits: String = filename.chars().take_while(char::is_ascii_digit).collect(); - if digits.is_empty() || digits.len() > 12 { - return None; - } - let secs: u64 = digits.parse().ok()?; - if secs == 0 { - return None; - } - let now = OffsetDateTime::now_utc().unix_timestamp().max(0) as u64; - if secs > now.saturating_add(86_400) { - return None; - } - Some(secs) -} - -pub fn pick_received_at(filename: &str, mtime_unix: u64, date_header: Option<&str>) -> String { - if let Some(secs) = delivery_time_from_filename(filename) { - return format_unix_rfc3339(secs); - } - if let Some(d) = date_header { - return d.to_owned(); - } - if mtime_unix > 0 { - return format_unix_rfc3339(mtime_unix); - } - OffsetDateTime::now_utc() - .format(&Rfc3339) - .unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_owned()) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PresentOutcome { - Unchanged, - - KeywordsUpdated, - - DeleteRequested, -} - -pub fn refresh_flags( - tx: &rusqlite::Transaction<'_>, - local_id: i64, - entry: &DiskEntry, - stored_keywords: &str, - include_deleted: bool, -) -> Result { - let translation = translate_flags(&entry.flags, include_deleted); - if translation.has_trashed_flag && !include_deleted { - return Ok(PresentOutcome::DeleteRequested); - } - let expected = keywords_json(&translation); - if stored_keywords == expected { - return Ok(PresentOutcome::Unchanged); - } - tx.execute( - "UPDATE emails SET keywords = ?1 WHERE id = ?2", - params![expected, local_id], - )?; - Ok(PresentOutcome::KeywordsUpdated) -} - -pub fn load_present_keywords( - conn: &Connection, - source_id: i64, - folder: &str, -) -> Result, rusqlite::Error> { - let mut stmt = conn.prepare( - "SELECT e.id, e.keywords FROM emails e - JOIN sync_id_maildir m ON m.local_id = e.id AND m.type_name = 'email' - WHERE m.source_id = ?1 AND m.folder = ?2", - )?; - let rows = stmt.query_map(params![source_id, folder], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) - })?; - let mut out = HashMap::new(); - for r in rows { - let (id, kw) = r?; - out.insert(id, kw); - } - Ok(out) -} - -pub fn delete_vanished( - tx: &rusqlite::Transaction<'_>, - source_id: i64, - folder: &str, - unique_id: &str, - local_id: i64, -) -> Result<(), rusqlite::Error> { - tx.execute("DELETE FROM emails WHERE id = ?1", params![local_id])?; - db::maildir_ids::delete_email(tx, source_id, folder, unique_id)?; - Ok(()) -} - -const PROGRESS_TICK: u64 = 1000; - -pub fn apply_folder( - conn: &mut Connection, - ctx: InsertContext<'_>, - diff: FolderDiff, - counts: &mut TypeCounts, - logger: crate::logging::Logger, -) -> Result<(), crate::error::Error> { - let stored_keywords = - load_present_keywords(conn, ctx.source_id, ctx.folder).unwrap_or_default(); - let tx = conn.transaction()?; - let total_new = diff.new.len() as u64; - let mut inserted: u64 = 0; - for entry in &diff.new { - match insert_new(&tx, ctx, entry) { - Ok(Some(_)) => { - counts.created += 1; - counts.fetched += 1; - inserted += 1; - if inserted.is_multiple_of(PROGRESS_TICK) - && logger.enabled(crate::logging::LEVEL_PROGRESS) - { - eprintln!( - "folder {folder:?}: inserted {inserted}/{total_new}", - folder = ctx.folder - ); - } - } - Ok(None) => { - counts.skipped += 1; - } - Err(e) => { - logger.warn(&format!( - "maildir {folder:?}/{name}: {e}", - folder = ctx.folder, - name = entry.filename - )); - counts.failed += 1; - } - } - } - for (entry, local_id) in &diff.present { - let stored = stored_keywords - .get(local_id) - .map(String::as_str) - .unwrap_or("[]"); - match refresh_flags(&tx, *local_id, entry, stored, ctx.include_deleted) { - Ok(PresentOutcome::Unchanged) => counts.skipped += 1, - Ok(PresentOutcome::KeywordsUpdated) => counts.fetched += 1, - Ok(PresentOutcome::DeleteRequested) => { - match delete_vanished(&tx, ctx.source_id, ctx.folder, &entry.unique_id, *local_id) { - Ok(()) => counts.deleted += 1, - Err(e) => { - logger.warn(&format!( - "maildir {folder:?}/{name}: T-flag delete failed: {e}", - folder = ctx.folder, - name = entry.filename - )); - counts.failed += 1; - } - } - } - Err(e) => { - logger.warn(&format!( - "maildir {folder:?}/{name}: flag update failed: {e}", - folder = ctx.folder, - name = entry.filename - )); - counts.failed += 1; - } - } - } - for (unique_id, local_id) in &diff.vanished { - if let Err(e) = delete_vanished(&tx, ctx.source_id, ctx.folder, unique_id, *local_id) { - logger.warn(&format!( - "maildir {folder:?}/{unique_id}: delete failed: {e}", - folder = ctx.folder - )); - counts.failed += 1; - } else { - counts.deleted += 1; - } - } - tx.commit()?; - Ok(()) -} - -fn keywords_json(translation: &Translation) -> String { - Value::Array( - translation - .keywords - .iter() - .map(|k| Value::String(k.clone())) - .collect(), - ) - .to_string() -} - -fn format_unix_rfc3339(secs: u64) -> String { - let when = UNIX_EPOCH + Duration::from_secs(secs); - OffsetDateTime::from(when) - .format(&Rfc3339) - .unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_owned()) -} - -#[derive(Debug, thiserror::Error)] -pub enum InsertError { - #[error("io: {0}")] - Io(#[from] std::io::Error), - #[error("db: {0}")] - Db(#[from] rusqlite::Error), -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::init; - use crate::db::sources::{SourceKey, upsert_source}; - use std::fs; - - fn fresh_archive() -> (Connection, i64) { - let c = Connection::open_in_memory().unwrap(); - init::apply_schema(&c).unwrap(); - let sid = upsert_source( - &c, - &SourceKey { - kind: "maildir".to_owned(), - session_url: "file:///tmp/Maildir".to_owned(), - account_id: "/tmp/Maildir".to_owned(), - }, - Some("Maildir"), - "", - ) - .unwrap(); - (c, sid) - } - - fn write_maildir_message( - folder_path: &Path, - sub: &str, - filename: &str, - body: &[u8], - ) -> PathBuf { - let dir = folder_path.join(sub); - fs::create_dir_all(&dir).unwrap(); - let p = dir.join(filename); - fs::write(&p, body).unwrap(); - p - } - - fn ensure_folder_skel(folder_path: &Path) { - for sub in ["cur", "new", "tmp"] { - fs::create_dir_all(folder_path.join(sub)).unwrap(); - } - } - - #[test] - fn list_folder_returns_cur_and_new_skipping_tmp() { - let td = tempfile::tempdir().unwrap(); - ensure_folder_skel(td.path()); - write_maildir_message(td.path(), "cur", "1.M0.host:2,S", b"a"); - write_maildir_message(td.path(), "new", "2.M0.host", b"b"); - write_maildir_message(td.path(), "tmp", "3.M0.host", b"c"); - let listing = list_folder(td.path()).unwrap(); - assert_eq!(listing.entries.len(), 2); - assert!(listing.entries.iter().any(|e| e.unique_id == "1.M0.host")); - assert!(listing.entries.iter().any(|e| e.unique_id == "2.M0.host")); - } - - #[test] - fn list_folder_skips_dotfiles() { - let td = tempfile::tempdir().unwrap(); - ensure_folder_skel(td.path()); - write_maildir_message(td.path(), "cur", ".uidvalidity", b"meta"); - write_maildir_message(td.path(), "cur", ".dovecot-uidlist", b"meta"); - write_maildir_message(td.path(), "cur", "1.M0.host:2,S", b"a"); - let listing = list_folder(td.path()).unwrap(); - assert_eq!(listing.entries.len(), 1); - assert_eq!(listing.entries[0].unique_id, "1.M0.host"); - } - - #[test] - fn list_folder_parses_dovecot_extension_filename() { - let td = tempfile::tempdir().unwrap(); - ensure_folder_skel(td.path()); - write_maildir_message( - td.path(), - "cur", - "1739471123.M001P01234V0000I0000abcd.host,S=42,W=44:2,RS", - b"a", - ); - let listing = list_folder(td.path()).unwrap(); - assert_eq!(listing.entries.len(), 1); - let e = &listing.entries[0]; - assert_eq!( - e.unique_id, - "1739471123.M001P01234V0000I0000abcd.host,S=42,W=44" - ); - assert!(e.flags.contains(&Flag::Replied)); - assert!(e.flags.contains(&Flag::Seen)); - } - - #[cfg(target_os = "linux")] - #[test] - fn list_folder_renders_non_utf8_filename_lossy() { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - let td = tempfile::tempdir().unwrap(); - ensure_folder_skel(td.path()); - let mut bytes = b"1.M0.host".to_vec(); - bytes.extend_from_slice(&[0xff, 0xfe]); - bytes.extend_from_slice(b":2,S"); - let bad_name = OsStr::from_bytes(&bytes); - let p = td.path().join("cur").join(bad_name); - fs::write(&p, b"body").unwrap(); - let listing = list_folder(td.path()).unwrap(); - assert_eq!(listing.entries.len(), 1); - let e = &listing.entries[0]; - assert!( - e.unique_id.contains('\u{FFFD}'), - "lossy substitution: {:?}", - e.unique_id - ); - assert!(e.unique_id.starts_with("1.M0.host")); - assert!(e.flags.contains(&Flag::Seen)); - } - - #[test] - fn diff_partitions_new_present_vanished() { - let entries = vec![ - DiskEntry { - unique_id: "a".into(), - filename: "a".into(), - path: PathBuf::new(), - flags: vec![], - mtime_unix: 0, - }, - DiskEntry { - unique_id: "b".into(), - filename: "b".into(), - path: PathBuf::new(), - flags: vec![], - mtime_unix: 0, - }, - ]; - let mut local = HashMap::new(); - local.insert("b".to_owned(), 10); - local.insert("c".to_owned(), 11); - let d = diff(entries, &local); - assert_eq!(d.new.len(), 1); - assert_eq!(d.new[0].unique_id, "a"); - assert_eq!(d.present.len(), 1); - assert_eq!(d.present[0].0.unique_id, "b"); - assert_eq!(d.present[0].1, 10); - assert_eq!(d.vanished, vec![("c".to_owned(), 11)]); - } - - #[test] - fn insert_new_writes_blob_email_and_sync_row() { - let (mut c, sid) = fresh_archive(); - c.execute( - "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) - VALUES ('INBOX', NULL, 'inbox', 0, 1)", - [], - ) - .unwrap(); - let mailbox_local: i64 = c.last_insert_rowid(); - db::maildir_ids::insert_mailbox(&c, sid, "INBOX", mailbox_local).unwrap(); - - let td = tempfile::tempdir().unwrap(); - ensure_folder_skel(td.path()); - let body = b"From: a@x\r\nSubject: hi\r\nMessage-ID: \r\n\r\nhi"; - write_maildir_message(td.path(), "cur", "uid1.M0.host:2,S", body); - let listing = list_folder(td.path()).unwrap(); - let entry = listing.entries.into_iter().next().unwrap(); - - let tx = c.transaction().unwrap(); - let inserted = insert_new( - &tx, - InsertContext { - source_id: sid, - folder: "INBOX", - mailbox_local, - include_deleted: false, - }, - &entry, - ) - .unwrap(); - tx.commit().unwrap(); - assert!(inserted.is_some()); - - let email_count: i64 = c - .query_row("SELECT count(*) FROM emails", [], |r| r.get(0)) - .unwrap(); - assert_eq!(email_count, 1); - let local = db::maildir_ids::local_for_email(&c, sid, "INBOX", "uid1.M0.host") - .unwrap() - .unwrap(); - assert_eq!(local, inserted.unwrap()); - let kws: String = c - .query_row( - "SELECT keywords FROM emails WHERE id = ?1", - params![local], - |r| r.get(0), - ) - .unwrap(); - assert!(kws.contains("$seen")); - } - - #[test] - fn insert_new_skips_trashed_without_include_deleted() { - let (mut c, sid) = fresh_archive(); - c.execute( - "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) - VALUES ('INBOX', NULL, 'inbox', 0, 1)", - [], - ) - .unwrap(); - let mailbox_local: i64 = c.last_insert_rowid(); - db::maildir_ids::insert_mailbox(&c, sid, "INBOX", mailbox_local).unwrap(); - - let td = tempfile::tempdir().unwrap(); - ensure_folder_skel(td.path()); - write_maildir_message(td.path(), "cur", "uid.host:2,T", b"x"); - let listing = list_folder(td.path()).unwrap(); - let entry = listing.entries.into_iter().next().unwrap(); - let tx = c.transaction().unwrap(); - let inserted = insert_new( - &tx, - InsertContext { - source_id: sid, - folder: "INBOX", - mailbox_local, - include_deleted: false, - }, - &entry, - ) - .unwrap(); - tx.commit().unwrap(); - assert!(inserted.is_none()); - let email_count: i64 = c - .query_row("SELECT count(*) FROM emails", [], |r| r.get(0)) - .unwrap(); - assert_eq!(email_count, 0); - } - - #[test] - fn refresh_flags_updates_only_when_changed() { - let (mut c, sid) = fresh_archive(); - c.execute( - "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) - VALUES ('INBOX', NULL, 'inbox', 0, 1)", - [], - ) - .unwrap(); - let mailbox_local = c.last_insert_rowid(); - db::maildir_ids::insert_mailbox(&c, sid, "INBOX", mailbox_local).unwrap(); - let td = tempfile::tempdir().unwrap(); - ensure_folder_skel(td.path()); - write_maildir_message(td.path(), "cur", "u.host:2,", b"hi"); - let entry = list_folder(td.path()).unwrap().entries.remove(0); - let tx = c.transaction().unwrap(); - let local = insert_new( - &tx, - InsertContext { - source_id: sid, - folder: "INBOX", - mailbox_local, - include_deleted: false, - }, - &entry, - ) - .unwrap() - .unwrap(); - tx.commit().unwrap(); - - let stored = load_present_keywords(&c, sid, "INBOX").unwrap(); - let stored_for_local = stored - .get(&local) - .cloned() - .unwrap_or_else(|| "[]".to_owned()); - - let tx = c.transaction().unwrap(); - assert_eq!( - refresh_flags(&tx, local, &entry, &stored_for_local, false).unwrap(), - PresentOutcome::Unchanged - ); - tx.commit().unwrap(); - - fs::rename(&entry.path, entry.path.with_file_name("u.host:2,S")).unwrap(); - let entry2 = list_folder(td.path()).unwrap().entries.remove(0); - let tx = c.transaction().unwrap(); - assert_eq!( - refresh_flags(&tx, local, &entry2, &stored_for_local, false).unwrap(), - PresentOutcome::KeywordsUpdated - ); - tx.commit().unwrap(); - let kws: String = c - .query_row( - "SELECT keywords FROM emails WHERE id = ?1", - params![local], - |r| r.get(0), - ) - .unwrap(); - assert!(kws.contains("$seen")); - } - - #[test] - fn refresh_flags_returns_delete_when_trashed_added_and_include_deleted_off() { - let (mut c, sid) = fresh_archive(); - c.execute( - "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) - VALUES ('INBOX', NULL, 'inbox', 0, 1)", - [], - ) - .unwrap(); - let mailbox_local = c.last_insert_rowid(); - db::maildir_ids::insert_mailbox(&c, sid, "INBOX", mailbox_local).unwrap(); - let td = tempfile::tempdir().unwrap(); - ensure_folder_skel(td.path()); - write_maildir_message(td.path(), "cur", "u.host:2,", b"hi"); - let entry = list_folder(td.path()).unwrap().entries.remove(0); - let tx = c.transaction().unwrap(); - let local = insert_new( - &tx, - InsertContext { - source_id: sid, - folder: "INBOX", - mailbox_local, - include_deleted: false, - }, - &entry, - ) - .unwrap() - .unwrap(); - tx.commit().unwrap(); - - fs::rename(&entry.path, entry.path.with_file_name("u.host:2,T")).unwrap(); - let entry2 = list_folder(td.path()).unwrap().entries.remove(0); - let tx = c.transaction().unwrap(); - assert_eq!( - refresh_flags(&tx, local, &entry2, "[]", false).unwrap(), - PresentOutcome::DeleteRequested - ); - tx.commit().unwrap(); - } - - #[test] - fn refresh_flags_keeps_trashed_with_include_deleted_on() { - let (mut c, sid) = fresh_archive(); - c.execute( - "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) - VALUES ('INBOX', NULL, 'inbox', 0, 1)", - [], - ) - .unwrap(); - let mailbox_local = c.last_insert_rowid(); - db::maildir_ids::insert_mailbox(&c, sid, "INBOX", mailbox_local).unwrap(); - let td = tempfile::tempdir().unwrap(); - ensure_folder_skel(td.path()); - write_maildir_message(td.path(), "cur", "u.host:2,T", b"hi"); - let entry = list_folder(td.path()).unwrap().entries.remove(0); - let tx = c.transaction().unwrap(); - let local = insert_new( - &tx, - InsertContext { - source_id: sid, - folder: "INBOX", - mailbox_local, - include_deleted: true, - }, - &entry, - ) - .unwrap() - .unwrap(); - tx.commit().unwrap(); - - let stored = load_present_keywords(&c, sid, "INBOX").unwrap(); - let stored_for_local = stored.get(&local).cloned().unwrap(); - let tx = c.transaction().unwrap(); - assert_eq!( - refresh_flags(&tx, local, &entry, &stored_for_local, true).unwrap(), - PresentOutcome::Unchanged - ); - tx.commit().unwrap(); - } - - #[test] - fn filename_delivery_time_wins_over_mtime() { - let got = pick_received_at( - "1763065233.M12345P678.host,S=1234:2,S", - 99, - Some("2025-05-12T10:00:00+02:00"), - ); - assert_eq!(got, "2025-11-13T20:20:33Z"); - } - - #[test] - fn date_header_used_when_the_filename_carries_no_timestamp() { - let got = pick_received_at( - "M12345P678.host:2,S", - 1763065233, - Some("2025-05-12T10:00:00+02:00"), - ); - assert_eq!(got, "2025-05-12T10:00:00+02:00"); - } - - #[test] - fn mtime_is_the_last_resort_before_now() { - let got = pick_received_at("M12345P678.host:2,S", 1763065233, None); - assert_eq!(got, "2025-11-13T20:20:33Z"); - } - - #[test] - fn implausible_filename_timestamps_are_rejected() { - assert_eq!(delivery_time_from_filename("0.M1P1.host"), None); - assert_eq!(delivery_time_from_filename("M1P1.host"), None); - assert_eq!( - delivery_time_from_filename("99999999999999.M1P1.host"), - None - ); - let far_future = (OffsetDateTime::now_utc().unix_timestamp() as u64) + 400_000; - assert_eq!( - delivery_time_from_filename(&format!("{far_future}.M1P1.host")), - None - ); - assert_eq!( - delivery_time_from_filename("1763065233.M12345P678.host"), - Some(1763065233) - ); - } -} diff --git a/src/sync/import_managesieve/coordinator.rs b/src/sync/import_managesieve/coordinator.rs deleted file mode 100644 index e92d740..0000000 --- a/src/sync/import_managesieve/coordinator.rs +++ /dev/null @@ -1,639 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::sync::Arc; - -use url::Url; - -use crate::db; -use crate::db::sources::SourceKey; -use crate::error::Error; -use crate::imap::retry::BackoffState; -use crate::imap::transport::Connector; -use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; -use crate::managesieve::error::SieveError; -use crate::managesieve::response::parse_listscripts; -use crate::managesieve::retry::{Disposition, classify, is_negotiation_failure}; -use crate::managesieve::{ConnectMode, SieveClient, parse_getscript}; -use crate::sync::keys::blake3_bytes; -use crate::sync::{CommonConfig, Summary, TypeCounts}; - -use super::reconcile::{ - Action, PresentOutcome, apply_active_assignment, apply_content_update, apply_delete, apply_new, - load_local_state, plan, -}; - -pub struct ManageSieveImportConfig { - pub url: String, - pub auth: ManageSieveAuth, - pub allow_cleartext: bool, - pub allow_source_change: bool, -} - -#[derive(Debug, Clone)] -pub enum ManageSieveAuth { - Basic { user: String, password: String }, - Bearer { user: String, token: String }, -} - -#[derive(Debug, Clone)] -struct Endpoint { - host: String, - port: u16, - implicit_tls: bool, -} - -struct ControlCtx { - connector: Arc, - host: String, - port: u16, - mode: ConnectMode, - allow_cleartext: bool, - auth: ManageSieveAuth, - backoff: BackoffState, - max_retries: u32, - logger: Logger, -} - -fn reconnect(client: &mut SieveClient, ctx: &ControlCtx) -> Result<(), SieveError> { - let new = SieveClient::connect( - &ctx.connector, - &ctx.host, - ctx.port, - ctx.mode, - ctx.allow_cleartext, - ctx.logger, - )?; - *client = new; - match do_authenticate(client, &ctx.auth) { - Ok(()) => {} - Err(SieveAuthError::Wire(e)) => return Err(e), - Err(SieveAuthError::TerminallyRefused(text)) => { - return Err(SieveError::Protocol(format!( - "auth refused on reconnect: {text}" - ))); - } - Err(SieveAuthError::NoUsableMechanism(text)) => { - return Err(SieveError::Unsupported(text)); - } - } - if !client.had_fresh_post_auth_caps() - && let Err(e) = client.refresh_capabilities() - { - ctx.logger.warn(&format!( - "post-reconnect CAPABILITY refresh failed: {e}; continuing" - )); - } - Ok(()) -} - -pub fn run(common: CommonConfig, config: ManageSieveImportConfig) -> Result { - let logger = common.logger; - if common.threads > 1 { - log_at( - logger, - LEVEL_PROGRESS, - "managesieve importer is single-threaded; --threads value will be ignored", - ); - } - - let mut conn = db::init::open(&common.archive)?; - - let endpoint = parse_endpoint(&config.url)?; - let session_url = format!( - "{}://{}:{}", - if endpoint.implicit_tls { - "sieves" - } else { - "sieve" - }, - endpoint.host, - endpoint.port - ); - - let connector = Arc::new( - Connector::new(common.allow_invalid_certs).map_err(|e| Error::Connection(e.to_string()))?, - ); - let mode = if endpoint.implicit_tls { - ConnectMode::ImplicitTls - } else { - ConnectMode::StartTls - }; - - let mut client = SieveClient::connect( - &connector, - &endpoint.host, - endpoint.port, - mode, - config.allow_cleartext, - logger, - ) - .map_err(|e| Error::Connection(e.to_string()))?; - - let account_id = authenticate(&mut client, &config.auth)?; - log_at( - logger, - LEVEL_PROGRESS, - &format!("authenticated as {account_id} on {session_url}"), - ); - if !client.had_fresh_post_auth_caps() - && let Err(e) = client.refresh_capabilities() - { - logger.warn(&format!( - "post-auth CAPABILITY refresh failed: {e}; continuing" - )); - } - - let source_key = SourceKey { - kind: "managesieve".to_owned(), - session_url: session_url.clone(), - account_id: account_id.clone(), - }; - if let Some((existing_url, existing_account)) = - db::sources::conflicting_source(&conn, "managesieve", &session_url, &account_id)? - && !config.allow_source_change - { - return Err(Error::SourceChange(format!( - "archive already records managesieve source {existing_url} / {existing_account}; \ - re-run with --allow-source-change or use a fresh archive" - ))); - } - let source_id = if common.dry_run { - db::sources::find_source(&conn, &source_key)?.unwrap_or(-1) - } else { - db::sources::upsert_source(&conn, &source_key, Some(&account_id), &account_id)? - }; - - let backoff = BackoffState::new(); - let control_ctx = ControlCtx { - connector: connector.clone(), - host: endpoint.host.clone(), - port: endpoint.port, - mode, - allow_cleartext: config.allow_cleartext, - auth: config.auth.clone(), - backoff: backoff.clone(), - max_retries: common.max_retries, - logger, - }; - let server_listing = listscripts_with_retry(&mut client, &control_ctx)?; - - let mut server_with_bytes: Vec<(String, bool, [u8; 32], Vec)> = - Vec::with_capacity(server_listing.len()); - let mut counts = TypeCounts::default(); - for (name, active) in &server_listing { - match getscript_with_retry(&mut client, &control_ctx, name) { - Ok(bytes) => { - let hash = blake3_bytes(&bytes); - server_with_bytes.push((name.clone(), *active, hash, bytes)); - } - Err(e) => match classify(&e) { - Disposition::Permanent - | Disposition::PerScriptRecoverable - | Disposition::Transient => { - logger.warn(&format!("GETSCRIPT {name:?} failed, skipping: {e}")); - counts.failed += 1; - } - Disposition::Referral => { - return Err(Error::Connection(format!( - "ManageSieve referral on GETSCRIPT {name:?}: {e}; inbuxa-migrate does not follow referrals" - ))); - } - Disposition::TransportDrop => { - return Err(Error::Connection(format!( - "GETSCRIPT {name:?} transport drop exhausted retries: {e}" - ))); - } - Disposition::FeatureNegotiation => { - return Err(Error::Connection(format!("GETSCRIPT {name:?}: {e}"))); - } - }, - } - } - - if common.dry_run { - let summary = - build_dry_run_summary(&conn, source_id, &server_with_bytes, counts, &backoff)?; - let _ = client.logout(); - return Ok(summary); - } - - let local_map = db::managesieve_ids::all_names(&conn, source_id)?; - let (local_hashes, local_active) = load_local_state(&conn, source_id)?; - let server_keyed: Vec<(String, bool, [u8; 32])> = server_with_bytes - .iter() - .map(|(n, a, h, _)| (n.clone(), *a, *h)) - .collect(); - let plan = plan(&server_keyed, &local_map, &local_hashes, &local_active); - - let bytes_by_name: std::collections::HashMap> = server_with_bytes - .iter() - .map(|(n, _, _, b)| (n.clone(), b)) - .collect(); - - let mut active_local: Option = None; - - for action in &plan.actions { - match action { - Action::New { name, active } => { - let bytes = bytes_by_name.get(name).cloned().unwrap(); - let tx = conn.transaction()?; - match apply_new(&tx, source_id, name, bytes) { - Ok(local_id) => { - tx.commit()?; - if *active { - active_local = Some(local_id); - } - counts.created += 1; - counts.fetched += 1; - } - Err(e) => { - let _ = tx.rollback(); - logger.warn(&format!("INSERT sieve_scripts {name:?} failed: {e}")); - counts.failed += 1; - } - } - } - Action::Present { - name, - local_id, - outcome, - active, - } => { - match outcome { - PresentOutcome::Unchanged => { - counts.skipped += 1; - } - PresentOutcome::ActiveOnly => { - counts.skipped += 1; - } - PresentOutcome::ContentUpdated => { - let bytes = bytes_by_name.get(name).cloned().unwrap(); - let tx = conn.transaction()?; - match apply_content_update(&tx, *local_id, bytes) { - Ok(()) => { - tx.commit()?; - counts.fetched += 1; - } - Err(e) => { - let _ = tx.rollback(); - logger.warn(&format!("UPDATE sieve_scripts {name:?} failed: {e}")); - counts.failed += 1; - } - } - } - } - if *active { - active_local = Some(*local_id); - } - } - Action::Vanished { name, local_id } => { - let tx = conn.transaction()?; - match apply_delete(&tx, source_id, name, *local_id) { - Ok(()) => { - tx.commit()?; - counts.deleted += 1; - } - Err(e) => { - let _ = tx.rollback(); - logger.warn(&format!("DELETE sieve_scripts {name:?} failed: {e}")); - counts.failed += 1; - } - } - } - } - } - - apply_active_assignment(&mut conn, active_local)?; - - let _ = client.logout(); - - if counts.failed == 0 { - let tx = conn.unchecked_transaction()?; - db::blobs::gc_orphan_blobs(&tx)?; - tx.commit()?; - } - - log_at( - logger, - LEVEL_DEFAULT, - &format!( - "managesieve: created={} fetched={} deleted={} skipped={} failed={}", - counts.created, counts.fetched, counts.deleted, counts.skipped, counts.failed - ), - ); - - Ok(Summary { - per_type: vec![("sievescript", counts)], - retries_observed: backoff.total_retries(), - retry_after_sleeps: backoff.transient_retries() as u64, - }) -} - -fn build_dry_run_summary( - conn: &rusqlite::Connection, - source_id: i64, - server: &[(String, bool, [u8; 32], Vec)], - mut counts: TypeCounts, - backoff: &BackoffState, -) -> Result { - let server_keyed: Vec<(String, bool, [u8; 32])> = server - .iter() - .map(|(n, a, h, _)| (n.clone(), *a, *h)) - .collect(); - let (local_map, local_hashes, local_active) = if source_id < 0 { - ( - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - ) - } else { - let local_map = db::managesieve_ids::all_names(conn, source_id)?; - let (h, a) = load_local_state(conn, source_id)?; - (local_map, h, a) - }; - let plan = plan(&server_keyed, &local_map, &local_hashes, &local_active); - for action in &plan.actions { - match action { - Action::New { .. } => { - counts.created += 1; - counts.fetched += 1; - } - Action::Present { - outcome: PresentOutcome::ContentUpdated, - .. - } => counts.fetched += 1, - Action::Present { .. } => counts.skipped += 1, - Action::Vanished { .. } => counts.deleted += 1, - } - } - Ok(Summary { - per_type: vec![("sievescript", counts)], - retries_observed: backoff.total_retries(), - retry_after_sleeps: backoff.transient_retries() as u64, - }) -} - -fn parse_endpoint(url: &str) -> Result { - let parsed = - Url::parse(url).map_err(|e| Error::Usage(format!("invalid --url {url:?}: {e}")))?; - let scheme = parsed.scheme(); - let implicit_tls = match scheme { - "sieves" => true, - "sieve" => false, - other => { - return Err(Error::Usage(format!( - "--url scheme must be sieve or sieves, got {other}" - ))); - } - }; - let host = parsed - .host_str() - .ok_or_else(|| Error::Usage(format!("--url missing host: {url}")))? - .to_owned(); - let port = parsed.port().unwrap_or(4190); - Ok(Endpoint { - host, - port, - implicit_tls, - }) -} - -fn authenticate(client: &mut SieveClient, auth: &ManageSieveAuth) -> Result { - do_authenticate(client, auth).map_err(|e| match e { - SieveAuthError::TerminallyRefused(text) => { - Error::Connection(format!("auth failed: {text}")) - } - SieveAuthError::NoUsableMechanism(text) => Error::Connection(text), - SieveAuthError::Wire(e) => Error::Connection(e.to_string()), - })?; - Ok(account_id_for(auth)) -} - -fn account_id_for(auth: &ManageSieveAuth) -> String { - match auth { - ManageSieveAuth::Basic { user, .. } => user.clone(), - ManageSieveAuth::Bearer { user, .. } => user.clone(), - } -} - -#[derive(Debug)] -enum SieveAuthError { - TerminallyRefused(String), - - NoUsableMechanism(String), - - Wire(SieveError), -} - -fn do_authenticate(client: &mut SieveClient, auth: &ManageSieveAuth) -> Result<(), SieveAuthError> { - match auth { - ManageSieveAuth::Basic { user, password } => { - let caps = client.capabilities().clone(); - let mut last_plain_error: Option = None; - if caps.has_sasl("PLAIN") { - match client.authenticate_plain(user, password) { - Ok(()) => return Ok(()), - Err(e) if is_negotiation_failure(&e) => { - last_plain_error = Some(e.to_string()); - } - Err(e) => return Err(SieveAuthError::Wire(e)), - } - } - if caps.has_sasl("LOGIN") { - match client.authenticate_login(user, password) { - Ok(()) => return Ok(()), - Err(e) if is_negotiation_failure(&e) => { - let combined = match last_plain_error { - Some(p) => format!("LOGIN: {e}; PLAIN: {p}"), - None => format!("LOGIN: {e}"), - }; - return Err(SieveAuthError::TerminallyRefused(combined)); - } - Err(e) => return Err(SieveAuthError::Wire(e)), - } - } - if let Some(p) = last_plain_error { - return Err(SieveAuthError::TerminallyRefused(format!("PLAIN: {p}"))); - } - Err(SieveAuthError::NoUsableMechanism(format!( - "server SASL list lacks PLAIN and LOGIN (advertised: {:?})", - caps.sasl - ))) - } - ManageSieveAuth::Bearer { user, token } => { - if !client.capabilities().has_sasl("OAUTHBEARER") { - return Err(SieveAuthError::NoUsableMechanism(format!( - "server does not advertise OAUTHBEARER (advertised: {:?})", - client.capabilities().sasl - ))); - } - match client.authenticate_oauthbearer(user, token) { - Ok(()) => Ok(()), - Err(e) if is_negotiation_failure(&e) => Err(SieveAuthError::TerminallyRefused( - format!("OAUTHBEARER: {e}"), - )), - Err(e) => Err(SieveAuthError::Wire(e)), - } - } - } -} - -fn listscripts_with_retry( - client: &mut SieveClient, - ctx: &ControlCtx, -) -> Result, Error> { - let mut transient_attempts: u32 = 0; - let mut transport_attempts: u32 = 0; - loop { - match client.listscripts() { - Ok(block) => { - ctx.backoff.reset(); - let listed = parse_listscripts(&block.data) - .map_err(|e| Error::Connection(format!("LISTSCRIPTS parse: {e}")))?; - return Ok(listed.into_iter().map(|s| (s.name, s.active)).collect()); - } - Err(e) => match classify(&e) { - Disposition::Transient => { - if transient_attempts >= ctx.max_retries { - return Err(Error::Connection(format!( - "LISTSCRIPTS retries exhausted: {e}" - ))); - } - transient_attempts += 1; - log_at( - ctx.logger, - LEVEL_PROGRESS, - &format!("LISTSCRIPTS transient: {e}; retrying"), - ); - std::thread::sleep(ctx.backoff.next_shared_delay()); - } - Disposition::TransportDrop => { - if transport_attempts >= ctx.max_retries { - return Err(Error::Connection(format!( - "LISTSCRIPTS transport drop, retries exhausted: {e}" - ))); - } - transport_attempts += 1; - std::thread::sleep(ctx.backoff.transport_delay(transport_attempts)); - match reconnect(client, ctx) { - Ok(()) => { - log_at( - ctx.logger, - LEVEL_DEFAULT, - &format!("managesieve reconnected after transport drop ({e})"), - ); - } - Err(e2) => { - return Err(Error::Connection(format!( - "LISTSCRIPTS reconnect failed: {e2}" - ))); - } - } - } - Disposition::Referral => { - return Err(Error::Connection(format!( - "LISTSCRIPTS referral: {e}; inbuxa-migrate does not follow referrals" - ))); - } - _ => return Err(Error::Connection(format!("LISTSCRIPTS failed: {e}"))), - }, - } - } -} - -fn getscript_with_retry( - client: &mut SieveClient, - ctx: &ControlCtx, - name: &str, -) -> Result, SieveError> { - let mut transient_attempts: u32 = 0; - let mut transport_attempts: u32 = 0; - loop { - match client.getscript(name) { - Ok(block) => { - ctx.backoff.reset(); - return parse_getscript(&block.data); - } - Err(e) => match classify(&e) { - Disposition::Transient => { - if transient_attempts >= ctx.max_retries { - return Err(e); - } - transient_attempts += 1; - log_at( - ctx.logger, - LEVEL_PROGRESS, - &format!("GETSCRIPT {name:?} transient: {e}; retrying"), - ); - std::thread::sleep(ctx.backoff.next_shared_delay()); - } - Disposition::TransportDrop => { - if transport_attempts >= ctx.max_retries { - return Err(e); - } - transport_attempts += 1; - std::thread::sleep(ctx.backoff.transport_delay(transport_attempts)); - reconnect(client, ctx)?; - log_at( - ctx.logger, - LEVEL_DEFAULT, - &format!( - "managesieve reconnected mid-GETSCRIPT {name:?} after transport drop" - ), - ); - } - _ => return Err(e), - }, - } - } -} - -fn log_at(logger: Logger, level: u8, msg: &str) { - if logger.enabled(level) { - eprintln!("{msg}"); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_endpoint_sieves_defaults_to_4190() { - let e = parse_endpoint("sieves://mail.example.com").unwrap(); - assert_eq!(e.host, "mail.example.com"); - assert_eq!(e.port, 4190); - assert!(e.implicit_tls); - } - - #[test] - fn parse_endpoint_sieve_defaults_to_4190() { - let e = parse_endpoint("sieve://mail.example.com").unwrap(); - assert_eq!(e.host, "mail.example.com"); - assert_eq!(e.port, 4190); - assert!(!e.implicit_tls); - } - - #[test] - fn parse_endpoint_explicit_port() { - let e = parse_endpoint("sieve://mail.example.com:14190").unwrap(); - assert_eq!(e.port, 14190); - assert!(!e.implicit_tls); - } - - #[test] - fn parse_endpoint_rejects_other_schemes() { - let err = parse_endpoint("imap://example.com").unwrap_err(); - assert!(matches!(err, Error::Usage(_))); - } - - #[test] - fn parse_endpoint_rejects_missing_host() { - let err = parse_endpoint("sieve:///path").unwrap_err(); - assert!(matches!(err, Error::Usage(_))); - } -} diff --git a/src/sync/import_managesieve/reconcile.rs b/src/sync/import_managesieve/reconcile.rs deleted file mode 100644 index 184b603..0000000 --- a/src/sync/import_managesieve/reconcile.rs +++ /dev/null @@ -1,480 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::{HashMap, HashSet}; - -use rusqlite::{Connection, OptionalExtension, params}; - -use crate::db; -use crate::error::Error; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PresentOutcome { - Unchanged, - ActiveOnly, - ContentUpdated, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Action { - New { - name: String, - active: bool, - }, - Present { - name: String, - local_id: i64, - outcome: PresentOutcome, - active: bool, - }, - Vanished { - name: String, - local_id: i64, - }, -} - -pub struct Plan { - pub actions: Vec, -} - -pub fn plan( - server: &[(String, bool, [u8; 32])], - local: &HashMap, - local_hashes: &HashMap, - local_active: &HashMap, -) -> Plan { - let mut actions = Vec::new(); - let mut server_names: HashSet<&str> = HashSet::new(); - for (name, active, hash) in server { - server_names.insert(name.as_str()); - match local.get(name) { - None => actions.push(Action::New { - name: name.clone(), - active: *active, - }), - Some(local_id) => { - let cur_hash = local_hashes.get(local_id); - let cur_active = local_active.get(local_id).copied().unwrap_or(false); - let outcome = if cur_hash != Some(hash) { - PresentOutcome::ContentUpdated - } else if cur_active != *active { - PresentOutcome::ActiveOnly - } else { - PresentOutcome::Unchanged - }; - actions.push(Action::Present { - name: name.clone(), - local_id: *local_id, - outcome, - active: *active, - }); - } - } - } - for (name, local_id) in local { - if !server_names.contains(name.as_str()) { - actions.push(Action::Vanished { - name: name.clone(), - local_id: *local_id, - }); - } - } - Plan { actions } -} - -pub type LocalState = (HashMap, HashMap); - -pub fn load_local_state(conn: &Connection, source_id: i64) -> Result { - let mut stmt = conn.prepare( - "SELECT s.id, b.hash, s.is_active - FROM sieve_scripts s JOIN blobs b ON b.id = s.blob_id - JOIN sync_id_managesieve m ON m.local_id = s.id - WHERE m.source_id = ?1", - )?; - let rows = stmt.query_map(params![source_id], |row| { - let id: i64 = row.get(0)?; - let hash: Vec = row.get(1)?; - let active: i64 = row.get(2)?; - Ok((id, hash, active != 0)) - })?; - let mut hashes = HashMap::new(); - let mut active = HashMap::new(); - for r in rows { - let (id, hash, a) = r?; - if hash.len() == 32 { - let mut arr = [0u8; 32]; - arr.copy_from_slice(&hash); - hashes.insert(id, arr); - } - active.insert(id, a); - } - Ok((hashes, active)) -} - -pub fn apply_new( - tx: &rusqlite::Transaction<'_>, - source_id: i64, - name: &str, - bytes: &[u8], -) -> Result { - let blob_id = db::blobs::intern_blob(tx, bytes)?; - tx.execute( - "INSERT INTO sieve_scripts (name, is_active, blob_id) VALUES (?1, 0, ?2)", - params![name, blob_id], - )?; - let local_id = tx.last_insert_rowid(); - db::managesieve_ids::insert(tx, source_id, name, local_id)?; - Ok(local_id) -} - -pub fn apply_content_update( - tx: &rusqlite::Transaction<'_>, - local_id: i64, - bytes: &[u8], -) -> Result<(), Error> { - let blob_id = db::blobs::intern_blob(tx, bytes)?; - tx.execute( - "UPDATE sieve_scripts SET blob_id = ?1 WHERE id = ?2", - params![blob_id, local_id], - )?; - Ok(()) -} - -pub fn apply_delete( - tx: &rusqlite::Transaction<'_>, - source_id: i64, - name: &str, - local_id: i64, -) -> Result<(), Error> { - tx.execute("DELETE FROM sieve_scripts WHERE id = ?1", params![local_id])?; - db::managesieve_ids::delete(tx, source_id, name)?; - Ok(()) -} - -pub fn apply_active_assignment( - conn: &mut Connection, - new_active_local: Option, -) -> Result<(), Error> { - let tx = conn.transaction()?; - let existing_active: Option = tx - .query_row( - "SELECT id FROM sieve_scripts WHERE is_active = 1", - [], - |r| r.get(0), - ) - .optional()?; - if existing_active != new_active_local { - if existing_active.is_some() { - tx.execute( - "UPDATE sieve_scripts SET is_active = 0 WHERE is_active = 1", - [], - )?; - } - if let Some(id) = new_active_local { - tx.execute( - "UPDATE sieve_scripts SET is_active = 1 WHERE id = ?1", - params![id], - )?; - } - } - tx.commit()?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::init; - use crate::db::sources::{SourceKey, upsert_source}; - use blake3::Hasher; - - fn hash(b: &[u8]) -> [u8; 32] { - let mut h = Hasher::new(); - h.update(b); - h.finalize().into() - } - - fn mem() -> (Connection, i64) { - let c = Connection::open_in_memory().unwrap(); - init::apply_schema(&c).unwrap(); - let sid = upsert_source( - &c, - &SourceKey { - kind: "managesieve".to_owned(), - session_url: "sieve://h:4190".to_owned(), - account_id: "a".to_owned(), - }, - None, - "a", - ) - .unwrap(); - (c, sid) - } - - #[test] - fn plan_classifies_new_present_unchanged_and_vanished() { - let h_v = hash(b"v"); - let h_w = hash(b"w"); - let server: Vec<(String, bool, [u8; 32])> = vec![ - ("a".into(), false, h_v), - ("b".into(), true, h_w), - ("c".into(), false, h_v), - ]; - let mut local: HashMap = HashMap::new(); - local.insert("b".into(), 2); - local.insert("c".into(), 3); - local.insert("d".into(), 4); - let mut lh: HashMap = HashMap::new(); - lh.insert(2, h_w); - lh.insert(3, h_v); - lh.insert(4, h_v); - let mut la: HashMap = HashMap::new(); - la.insert(2, true); - la.insert(3, false); - la.insert(4, false); - let p = plan(&server, &local, &lh, &la); - let mut new_count = 0; - let mut unchanged = 0; - let mut content_updated = 0; - let mut active_only = 0; - let mut vanished = 0; - for a in p.actions { - match a { - Action::New { .. } => new_count += 1, - Action::Present { - outcome: PresentOutcome::Unchanged, - .. - } => unchanged += 1, - Action::Present { - outcome: PresentOutcome::ContentUpdated, - .. - } => content_updated += 1, - Action::Present { - outcome: PresentOutcome::ActiveOnly, - .. - } => active_only += 1, - Action::Vanished { .. } => vanished += 1, - } - } - assert_eq!(new_count, 1); - assert_eq!(unchanged, 2); - assert_eq!(content_updated, 0); - assert_eq!(active_only, 0); - assert_eq!(vanished, 1); - } - - #[test] - fn plan_flags_content_update_when_hash_differs() { - let h_v = hash(b"v"); - let h_w = hash(b"w"); - let server: Vec<(String, bool, [u8; 32])> = vec![("a".into(), false, h_w)]; - let mut local: HashMap = HashMap::new(); - local.insert("a".into(), 1); - let mut lh = HashMap::new(); - lh.insert(1, h_v); - let mut la = HashMap::new(); - la.insert(1, false); - let p = plan(&server, &local, &lh, &la); - assert!(matches!( - p.actions[0], - Action::Present { - outcome: PresentOutcome::ContentUpdated, - .. - } - )); - } - - #[test] - fn plan_flags_active_only_when_only_flag_differs() { - let h = hash(b"same"); - let server: Vec<(String, bool, [u8; 32])> = vec![("a".into(), true, h)]; - let mut local: HashMap = HashMap::new(); - local.insert("a".into(), 1); - let mut lh = HashMap::new(); - lh.insert(1, h); - let mut la = HashMap::new(); - la.insert(1, false); - let p = plan(&server, &local, &lh, &la); - assert!(matches!( - p.actions[0], - Action::Present { - outcome: PresentOutcome::ActiveOnly, - .. - } - )); - } - - #[test] - fn apply_new_inserts_script_and_id_mapping() { - let (mut c, sid) = mem(); - let tx = c.transaction().unwrap(); - let id = apply_new(&tx, sid, "vacation", b"require[\"x\"];").unwrap(); - tx.commit().unwrap(); - assert!(id > 0); - let local = db::managesieve_ids::local_for(&c, sid, "vacation") - .unwrap() - .unwrap(); - assert_eq!(local, id); - let count: i64 = c - .query_row("SELECT count(*) FROM sieve_scripts", [], |r| r.get(0)) - .unwrap(); - assert_eq!(count, 1); - } - - #[test] - fn apply_content_update_swaps_blob_pointer() { - let (mut c, sid) = mem(); - let tx = c.transaction().unwrap(); - let id = apply_new(&tx, sid, "vacation", b"v1").unwrap(); - tx.commit().unwrap(); - let tx = c.transaction().unwrap(); - apply_content_update(&tx, id, b"v2").unwrap(); - tx.commit().unwrap(); - let blob: Vec = c - .query_row( - "SELECT b.data FROM sieve_scripts s JOIN blobs b ON b.id=s.blob_id WHERE s.id=?1", - params![id], - |r| r.get(0), - ) - .unwrap(); - assert_eq!(blob, b"v2"); - } - - #[test] - fn apply_active_assignment_swaps_active_in_one_transaction() { - let (mut c, sid) = mem(); - let tx = c.transaction().unwrap(); - let id_a = apply_new(&tx, sid, "a", b"x").unwrap(); - let id_b = apply_new(&tx, sid, "b", b"y").unwrap(); - tx.commit().unwrap(); - apply_active_assignment(&mut c, Some(id_a)).unwrap(); - let cur: Option = c - .query_row( - "SELECT id FROM sieve_scripts WHERE is_active = 1", - [], - |r| r.get(0), - ) - .optional() - .unwrap(); - assert_eq!(cur, Some(id_a)); - apply_active_assignment(&mut c, Some(id_b)).unwrap(); - let cur: Option = c - .query_row( - "SELECT id FROM sieve_scripts WHERE is_active = 1", - [], - |r| r.get(0), - ) - .optional() - .unwrap(); - assert_eq!(cur, Some(id_b)); - apply_active_assignment(&mut c, None).unwrap(); - let cur: Option = c - .query_row( - "SELECT id FROM sieve_scripts WHERE is_active = 1", - [], - |r| r.get(0), - ) - .optional() - .unwrap(); - assert_eq!(cur, None); - } - - #[test] - fn apply_delete_clears_active_before_destroy() { - let (mut c, sid) = mem(); - let tx = c.transaction().unwrap(); - let id = apply_new(&tx, sid, "a", b"x").unwrap(); - tx.commit().unwrap(); - apply_active_assignment(&mut c, Some(id)).unwrap(); - let tx = c.transaction().unwrap(); - apply_delete(&tx, sid, "a", id).unwrap(); - tx.commit().unwrap(); - let count: i64 = c - .query_row("SELECT count(*) FROM sieve_scripts", [], |r| r.get(0)) - .unwrap(); - assert_eq!(count, 0); - assert!( - db::managesieve_ids::local_for(&c, sid, "a") - .unwrap() - .is_none() - ); - } - - #[test] - fn apply_new_dedups_identical_script_bytes_at_blob_level() { - let (mut c, sid) = mem(); - let tx = c.transaction().unwrap(); - let id_a = apply_new(&tx, sid, "a", b"require [\"fileinto\"];\n").unwrap(); - let id_b = apply_new(&tx, sid, "b", b"require [\"fileinto\"];\n").unwrap(); - tx.commit().unwrap(); - let blob_a: i64 = c - .query_row( - "SELECT blob_id FROM sieve_scripts WHERE id = ?1", - params![id_a], - |r| r.get(0), - ) - .unwrap(); - let blob_b: i64 = c - .query_row( - "SELECT blob_id FROM sieve_scripts WHERE id = ?1", - params![id_b], - |r| r.get(0), - ) - .unwrap(); - assert_eq!(blob_a, blob_b, "identical bytes must share a blob row"); - let blob_count: i64 = c - .query_row("SELECT count(*) FROM blobs", [], |r| r.get(0)) - .unwrap(); - assert_eq!(blob_count, 1, "blob table must contain a single row"); - } - - #[test] - fn apply_content_update_orphans_old_blob_for_gc() { - let (mut c, sid) = mem(); - let tx = c.transaction().unwrap(); - let id = apply_new(&tx, sid, "a", b"v1").unwrap(); - tx.commit().unwrap(); - let tx = c.transaction().unwrap(); - apply_content_update(&tx, id, b"v2").unwrap(); - tx.commit().unwrap(); - let blob_count_before_gc: i64 = c - .query_row("SELECT count(*) FROM blobs", [], |r| r.get(0)) - .unwrap(); - assert_eq!(blob_count_before_gc, 2, "old blob still present pre-GC"); - let tx = c.unchecked_transaction().unwrap(); - crate::db::blobs::gc_orphan_blobs(&tx).unwrap(); - tx.commit().unwrap(); - let blob_count_after_gc: i64 = c - .query_row("SELECT count(*) FROM blobs", [], |r| r.get(0)) - .unwrap(); - assert_eq!(blob_count_after_gc, 1, "orphan blob reclaimed by GC"); - } - - #[test] - fn partial_unique_index_rejects_two_active_rows() { - let (mut c, sid) = mem(); - let tx = c.transaction().unwrap(); - let id_a = apply_new(&tx, sid, "a", b"x").unwrap(); - let id_b = apply_new(&tx, sid, "b", b"y").unwrap(); - tx.commit().unwrap(); - c.execute( - "UPDATE sieve_scripts SET is_active = 1 WHERE id = ?1", - params![id_a], - ) - .unwrap(); - let err = c - .execute( - "UPDATE sieve_scripts SET is_active = 1 WHERE id = ?1", - params![id_b], - ) - .unwrap_err(); - let msg = err.to_string(); - assert!( - msg.contains("UNIQUE") || msg.contains("unique"), - "expected unique-constraint failure, got {msg}" - ); - } -} diff --git a/src/sync/import_takeout.rs b/src/sync/import_takeout.rs deleted file mode 100644 index 7693eb7..0000000 --- a/src/sync/import_takeout.rs +++ /dev/null @@ -1,16 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -pub mod calendar; -pub mod contacts; -pub mod coordinator; -pub mod labels; -pub mod mail; -pub mod mbox; -pub mod tree; -pub mod walk; - -pub use coordinator::{TakeoutImportConfig, run, run_reporting}; diff --git a/src/sync/import_takeout/calendar.rs b/src/sync/import_takeout/calendar.rs deleted file mode 100644 index bb6dd95..0000000 --- a/src/sync/import_takeout/calendar.rs +++ /dev/null @@ -1,309 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashMap; -use std::path::Path; - -use rusqlite::{Connection, OptionalExtension, Transaction, params}; -use serde_json::Value; - -use crate::db::takeout_ids; -use crate::error::Error; -use crate::logging::Logger; -use crate::sync::TypeCounts; -use crate::sync::import_dav::calcard; - -const DEFAULT_CALENDAR: &str = "Imported"; - -pub fn process_file( - conn: &mut Connection, - file_path: &Path, - source_id: i64, - calendar_cache: &mut HashMap, - calendar_counts: &mut TypeCounts, - event_counts: &mut TypeCounts, - logger: Logger, -) -> Result<(), Error> { - let text = match std::fs::read_to_string(file_path) { - Ok(s) => s, - Err(e) => { - logger.warn(&format!("{file_path:?}: read: {e}")); - event_counts.failed += 1; - return Ok(()); - } - }; - - let entries = match calcard::ical_to_jscalendar_entries(&text) { - Ok(v) => v, - Err(calcard::CalcardError::NoEntries) => { - logger.warn(&format!("{file_path:?}: iCalendar contains no events")); - return Ok(()); - } - Err(e) => { - logger.warn(&format!("{file_path:?}: iCalendar parse: {e}")); - event_counts.failed += 1; - return Ok(()); - } - }; - - let metadata = parse_calendar_metadata(&text); - let container_name = metadata - .x_wr_calname - .clone() - .unwrap_or_else(|| DEFAULT_CALENDAR.to_owned()); - - let tx = conn.transaction()?; - let calendar_local = ensure_calendar( - &tx, - source_id, - &container_name, - &metadata, - calendar_cache, - calendar_counts, - )?; - - for (ordinal, mut entry) in entries.into_iter().enumerate() { - match insert_event( - &tx, - source_id, - calendar_local, - file_path, - ordinal, - &mut entry, - event_counts, - ) { - Ok(()) => {} - Err(e) => { - logger.warn(&format!("{file_path:?}#{ordinal}: event insert: {e}")); - event_counts.failed += 1; - } - } - } - tx.commit()?; - Ok(()) -} - -#[derive(Debug, Default, Clone)] -struct CalendarMetadata { - x_wr_calname: Option, - x_wr_caldesc: Option, - x_wr_timezone: Option, -} - -fn ensure_calendar( - tx: &Transaction<'_>, - source_id: i64, - name: &str, - metadata: &CalendarMetadata, - cache: &mut HashMap, - counts: &mut TypeCounts, -) -> Result { - if let Some(&id) = cache.get(name) { - return Ok(id); - } - if let Some(id) = takeout_ids::local_for(tx, source_id, takeout_ids::CALENDAR, name)? { - cache.insert(name.to_owned(), id); - return Ok(id); - } - tx.execute( - "INSERT INTO calendars - (name, description, time_zone, sort_order, is_subscribed, is_visible, is_default) - VALUES (?1, ?2, ?3, 0, 1, 1, 0)", - params![name, metadata.x_wr_caldesc, metadata.x_wr_timezone], - )?; - let id = tx.last_insert_rowid(); - takeout_ids::insert(tx, source_id, takeout_ids::CALENDAR, name, id)?; - cache.insert(name.to_owned(), id); - counts.created += 1; - counts.fetched += 1; - Ok(id) -} - -fn insert_event( - tx: &Transaction<'_>, - source_id: i64, - calendar_local: i64, - file_path: &Path, - ordinal: usize, - entry: &mut calcard::JsCalendarEntry, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let (is_draft, use_default_alerts, uid_field) = - calcard::strip_extracted_fields_from_event(&mut entry.data); - let uid = match uid_field - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()) - { - Some(u) => u.to_owned(), - None => calcard::synthesise_uid(&format!("{}#{ordinal}", file_path.to_string_lossy())), - }; - let data_type = entry.data_type.as_column(); - let data_json = entry.data.to_string(); - - let existing = takeout_ids::local_for(tx, source_id, takeout_ids::CALENDAR_EVENT, &uid)?; - - if let Some(local) = existing { - let calendar_ids = - merge_container_id(tx, "calendar_events", "calendar_ids", local, calendar_local)?; - let current: (String, i64, i64, String, String) = tx.query_row( - "SELECT calendar_ids, is_draft, use_default_alerts, data, data_type - FROM calendar_events WHERE id = ?1", - params![local], - |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)), - )?; - let want = ( - calendar_ids.clone(), - is_draft as i64, - use_default_alerts as i64, - data_json.clone(), - data_type.to_owned(), - ); - if current == want { - counts.skipped += 1; - return Ok(()); - } - tx.execute( - "UPDATE calendar_events SET calendar_ids = ?1, is_draft = ?2, - use_default_alerts = ?3, data = ?4, data_type = ?5 - WHERE id = ?6", - params![ - calendar_ids, - is_draft as i64, - use_default_alerts as i64, - data_json, - data_type, - local, - ], - )?; - counts.fetched += 1; - return Ok(()); - } - - let calendar_ids = format!("[{calendar_local}]"); - tx.execute( - "INSERT INTO calendar_events - (calendar_ids, is_draft, use_default_alerts, data, data_type) - VALUES (?1, ?2, ?3, ?4, ?5)", - params![ - calendar_ids, - is_draft as i64, - use_default_alerts as i64, - data_json, - data_type, - ], - )?; - let local_id = tx.last_insert_rowid(); - takeout_ids::insert(tx, source_id, takeout_ids::CALENDAR_EVENT, &uid, local_id)?; - counts.created += 1; - counts.fetched += 1; - Ok(()) -} - -fn merge_container_id( - tx: &Transaction<'_>, - table: &str, - column: &str, - local_id: i64, - new_id: i64, -) -> Result { - let current: String = tx - .query_row( - &format!("SELECT {column} FROM {table} WHERE id = ?1"), - params![local_id], - |r| r.get(0), - ) - .optional()? - .unwrap_or_else(|| "[]".to_owned()); - let mut ids: Vec = match serde_json::from_str::(¤t) { - Ok(Value::Array(arr)) => arr.into_iter().filter_map(|v| v.as_i64()).collect(), - _ => Vec::new(), - }; - if !ids.contains(&new_id) { - ids.push(new_id); - } - ids.sort_unstable(); - ids.dedup(); - Ok(Value::Array(ids.into_iter().map(Value::from).collect()).to_string()) -} - -fn parse_calendar_metadata(text: &str) -> CalendarMetadata { - let mut out = CalendarMetadata::default(); - let mut inside_vevent = false; - for raw_line in text.lines() { - let line = raw_line.trim_end_matches('\r'); - if line.starts_with("BEGIN:VEVENT") - || line.starts_with("BEGIN:VTODO") - || line.starts_with("BEGIN:VJOURNAL") - { - inside_vevent = true; - continue; - } - if line.starts_with("END:VEVENT") - || line.starts_with("END:VTODO") - || line.starts_with("END:VJOURNAL") - { - inside_vevent = false; - continue; - } - if inside_vevent { - continue; - } - if let Some(rest) = line.strip_prefix("X-WR-CALNAME:") { - out.x_wr_calname = Some(rest.trim().to_owned()); - } else if let Some(rest) = line.strip_prefix("X-WR-CALDESC:") { - out.x_wr_caldesc = Some(rest.trim().to_owned()); - } else if let Some(rest) = line.strip_prefix("X-WR-TIMEZONE:") { - out.x_wr_timezone = Some(rest.trim().to_owned()); - } - } - out -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_metadata_finds_xwr_fields() { - let ical = "BEGIN:VCALENDAR\r\n\ - PRODID:-//Google Inc//Google Calendar 70.9054//EN\r\n\ - X-WR-CALNAME:My Calendar\r\n\ - X-WR-CALDESC:Some description\r\n\ - X-WR-TIMEZONE:Europe/Madrid\r\n\ - BEGIN:VEVENT\r\nUID:e1@google.com\r\nDTSTAMP:20250101T000000Z\r\n\ - END:VEVENT\r\nEND:VCALENDAR\r\n"; - let m = parse_calendar_metadata(ical); - assert_eq!(m.x_wr_calname.as_deref(), Some("My Calendar")); - assert_eq!(m.x_wr_caldesc.as_deref(), Some("Some description")); - assert_eq!(m.x_wr_timezone.as_deref(), Some("Europe/Madrid")); - } - - #[test] - fn parse_metadata_ignores_xwr_inside_vevent() { - let ical = "BEGIN:VCALENDAR\r\n\ - BEGIN:VEVENT\r\nUID:e1\r\nX-WR-CALNAME:NOT THIS\r\nEND:VEVENT\r\n\ - END:VCALENDAR\r\n"; - let m = parse_calendar_metadata(ical); - assert!(m.x_wr_calname.is_none()); - } - - #[test] - fn parse_metadata_empty_when_missing() { - let ical = "BEGIN:VCALENDAR\r\nEND:VCALENDAR\r\n"; - let m = parse_calendar_metadata(ical); - assert!(m.x_wr_calname.is_none()); - assert!(m.x_wr_caldesc.is_none()); - assert!(m.x_wr_timezone.is_none()); - } - - #[test] - fn parse_metadata_handles_lf_only_line_endings() { - let ical = "BEGIN:VCALENDAR\nX-WR-CALNAME:Foo\nEND:VCALENDAR\n"; - let m = parse_calendar_metadata(ical); - assert_eq!(m.x_wr_calname.as_deref(), Some("Foo")); - } -} diff --git a/src/sync/import_takeout/contacts.rs b/src/sync/import_takeout/contacts.rs deleted file mode 100644 index 475261f..0000000 --- a/src/sync/import_takeout/contacts.rs +++ /dev/null @@ -1,253 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::path::Path; - -use rusqlite::{Connection, Transaction, params}; - -use crate::db::takeout_ids; -use crate::error::Error; -use crate::logging::Logger; -use crate::sync::TypeCounts; -use crate::sync::import_dav::calcard; - -const DEFAULT_ADDRESS_BOOK: &str = "Imported"; - -pub fn process_file( - conn: &mut Connection, - file_path: &Path, - source_id: i64, - address_book_local: &mut Option, - book_counts: &mut TypeCounts, - card_counts: &mut TypeCounts, - logger: Logger, -) -> Result<(), Error> { - let text = match std::fs::read_to_string(file_path) { - Ok(s) => s, - Err(e) => { - logger.warn(&format!("{file_path:?}: read: {e}")); - card_counts.failed += 1; - return Ok(()); - } - }; - - let chunks = split_vcards(&text); - if chunks.is_empty() { - return Ok(()); - } - - let tx = conn.transaction()?; - let book_local = match *address_book_local { - Some(id) => id, - None => { - let id = ensure_default_address_book(&tx, source_id, book_counts)?; - *address_book_local = Some(id); - id - } - }; - - for (ordinal, chunk) in chunks.into_iter().enumerate() { - match insert_card(&tx, source_id, book_local, &chunk, card_counts) { - Ok(()) => {} - Err(e) => { - logger.warn(&format!("{file_path:?}#{ordinal}: vCard: {e}")); - card_counts.failed += 1; - } - } - } - tx.commit()?; - Ok(()) -} - -fn ensure_default_address_book( - tx: &Transaction<'_>, - source_id: i64, - counts: &mut TypeCounts, -) -> Result { - if let Some(id) = takeout_ids::local_for( - tx, - source_id, - takeout_ids::ADDRESS_BOOK, - DEFAULT_ADDRESS_BOOK, - )? { - return Ok(id); - } - tx.execute( - "INSERT INTO address_books - (name, description, sort_order, is_default, is_subscribed) - VALUES (?1, NULL, 0, 0, 1)", - params![DEFAULT_ADDRESS_BOOK], - )?; - let id = tx.last_insert_rowid(); - takeout_ids::insert( - tx, - source_id, - takeout_ids::ADDRESS_BOOK, - DEFAULT_ADDRESS_BOOK, - id, - )?; - counts.created += 1; - counts.fetched += 1; - Ok(id) -} - -fn insert_card( - tx: &Transaction<'_>, - source_id: i64, - book_local: i64, - chunk: &str, - counts: &mut TypeCounts, -) -> Result<(), Error> { - let card = calcard::vcard_to_jscontact(chunk, chunk) - .map_err(|e| Error::Partial(format!("vCard parse: {e}")))?; - let data_json = card.data.to_string(); - let address_book_ids = format!("[{book_local}]"); - - let existing = takeout_ids::local_for(tx, source_id, takeout_ids::CONTACT_CARD, &card.uid)?; - - if let Some(local) = existing { - let current: (String, String, String) = tx.query_row( - "SELECT uid, address_book_ids, data FROM contact_cards WHERE id = ?1", - params![local], - |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), - )?; - let want = ( - card.uid.clone(), - address_book_ids.clone(), - data_json.clone(), - ); - if current == want { - counts.skipped += 1; - return Ok(()); - } - tx.execute( - "UPDATE contact_cards SET uid = ?1, address_book_ids = ?2, data = ?3 - WHERE id = ?4", - params![card.uid, address_book_ids, data_json, local], - )?; - counts.fetched += 1; - return Ok(()); - } - - tx.execute( - "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)", - params![card.uid, address_book_ids, data_json], - )?; - let local_id = tx.last_insert_rowid(); - takeout_ids::insert( - tx, - source_id, - takeout_ids::CONTACT_CARD, - &card.uid, - local_id, - )?; - counts.created += 1; - counts.fetched += 1; - Ok(()) -} - -fn split_vcards(text: &str) -> Vec { - let mut chunks: Vec = Vec::new(); - let mut current: Option = None; - for raw_line in text.lines() { - let line = raw_line.trim_end_matches('\r'); - let bytes = line.as_bytes(); - if starts_with_ignore_ascii_case(bytes, b"BEGIN:VCARD") { - current = Some(String::new()); - } - if let Some(buf) = current.as_mut() { - buf.push_str(line); - buf.push('\n'); - } - if starts_with_ignore_ascii_case(bytes, b"END:VCARD") - && let Some(buf) = current.take() - { - chunks.push(buf); - } - } - chunks -} - -fn starts_with_ignore_ascii_case(haystack: &[u8], needle: &[u8]) -> bool { - haystack.len() >= needle.len() && haystack[..needle.len()].eq_ignore_ascii_case(needle) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn splits_single_card() { - let text = "BEGIN:VCARD\r\nVERSION:3.0\r\nFN:A\r\nEND:VCARD\r\n"; - let chunks = split_vcards(text); - assert_eq!(chunks.len(), 1); - assert!(chunks[0].contains("FN:A")); - } - - #[test] - fn splits_multiple_cards() { - let text = "BEGIN:VCARD\r\nVERSION:3.0\r\nFN:A\r\nEND:VCARD\r\n\ - BEGIN:VCARD\r\nVERSION:3.0\r\nFN:B\r\nEND:VCARD\r\n\ - BEGIN:VCARD\r\nVERSION:3.0\r\nFN:C\r\nEND:VCARD\r\n"; - let chunks = split_vcards(text); - assert_eq!(chunks.len(), 3); - assert!(chunks[0].contains("FN:A")); - assert!(chunks[1].contains("FN:B")); - assert!(chunks[2].contains("FN:C")); - } - - #[test] - fn skips_lines_outside_any_card() { - let text = "Some preamble garbage\r\n\ - BEGIN:VCARD\r\nVERSION:3.0\r\nFN:A\r\nEND:VCARD\r\n\ - garbage between cards\r\n\ - BEGIN:VCARD\r\nVERSION:3.0\r\nFN:B\r\nEND:VCARD\r\n\ - trailing garbage\r\n"; - let chunks = split_vcards(text); - assert_eq!(chunks.len(), 2); - assert!(!chunks[0].contains("garbage")); - } - - #[test] - fn preserves_folded_continuation_lines_inside_a_card() { - let text = "BEGIN:VCARD\r\nVERSION:3.0\r\n\ - ADR:;;1 Infinite Loop\\nCupertino\\, CA 95014\\nUnited States\r\n \ - ;;;;;\r\nEND:VCARD\r\n"; - let chunks = split_vcards(text); - assert_eq!(chunks.len(), 1); - assert!(chunks[0].contains("ADR:")); - assert!(chunks[0].contains(";;;;;")); - } - - #[test] - fn handles_lf_only_line_endings() { - let text = "BEGIN:VCARD\nVERSION:3.0\nFN:LF\nEND:VCARD\n"; - let chunks = split_vcards(text); - assert_eq!(chunks.len(), 1); - assert!(chunks[0].contains("FN:LF")); - } - - #[test] - fn empty_input_yields_no_chunks() { - assert!(split_vcards("").is_empty()); - assert!(split_vcards("not a vcard\n").is_empty()); - } - - #[test] - fn unterminated_card_is_dropped() { - let text = "BEGIN:VCARD\r\nVERSION:3.0\r\nFN:A\r\n"; - let chunks = split_vcards(text); - assert!(chunks.is_empty()); - } - - #[test] - fn case_insensitive_boundary_match() { - let text = "begin:vcard\r\nFN:CaseTest\r\nend:vcard\r\n"; - let chunks = split_vcards(text); - assert_eq!(chunks.len(), 1); - assert!(chunks[0].contains("FN:CaseTest")); - } -} diff --git a/src/sync/import_takeout/coordinator.rs b/src/sync/import_takeout/coordinator.rs deleted file mode 100644 index f6f8743..0000000 --- a/src/sync/import_takeout/coordinator.rs +++ /dev/null @@ -1,420 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashMap; -use std::path::{Path, PathBuf}; - -use rusqlite::{Connection, params}; -use url::Url; - -use crate::db; -use crate::db::sources::SourceKey; -use crate::db::takeout_ids; -use crate::error::Error; -use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; -use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts}; - -use super::calendar; -use super::contacts; -use super::labels::MappingOptions; -use super::mail; -use super::walk::{self, FileKind}; - -#[derive(Debug)] -pub struct TakeoutImportConfig { - pub takeout_root: PathBuf, - pub allow_source_change: bool, - pub automap: bool, -} - -pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result { - run_reporting(common, config).into_result() -} - -pub fn run_reporting(common: CommonConfig, config: TakeoutImportConfig) -> RunOutcome { - let mut summary = Summary::default(); - let error = run_into(common, config, &mut summary).err(); - RunOutcome { summary, error } -} - -fn run_into( - common: CommonConfig, - config: TakeoutImportConfig, - summary: &mut Summary, -) -> Result<(), Error> { - let logger = common.logger; - if common.threads > 1 && logger.enabled(LEVEL_PROGRESS) { - eprintln!("takeout importer is single-threaded; --threads value will be ignored"); - } - - let canonical = std::fs::canonicalize(&config.takeout_root) - .map_err(|e| Error::Usage(format!("--path {:?}: {e}", config.takeout_root)))?; - if !canonical.is_dir() { - return Err(Error::Usage(format!("not a directory: {canonical:?}"))); - } - - let walk_result = walk::walk_with_logger(&canonical, logger) - .map_err(|e| Error::Usage(format!("walking {canonical:?}: {e}")))?; - if walk_result.is_empty() { - return Err(Error::Usage(format!( - "{canonical:?}: no .mbox / .ics / .vcf files found under this path" - ))); - } - - if logger.enabled(LEVEL_PROGRESS) { - let mbox_n = walk_result.by_kind(FileKind::Mbox).count(); - let ics_n = walk_result.by_kind(FileKind::Ics).count(); - let vcf_n = walk_result.by_kind(FileKind::Vcf).count(); - eprintln!( - "takeout discovery: {} .mbox, {} .ics, {} .vcf (io failures: {}, symlink cycles: {})", - mbox_n, ics_n, vcf_n, walk_result.io_failures, walk_result.symlink_cycles - ); - } - - let mut conn = db::init::open(&common.archive)?; - let session_url = file_url_for(&canonical)?; - let account_id = canonical.to_string_lossy().into_owned(); - let account_name = canonical - .file_name() - .and_then(|n| n.to_str()) - .map(str::to_owned); - - if let Some((existing_url, existing_account)) = - db::sources::conflicting_source(&conn, "takeout", &session_url, &account_id)? - && !config.allow_source_change - { - return Err(Error::SourceChange(format!( - "archive already records takeout source {existing_url} / {existing_account}; \ - re-run with --allow-source-change or use a fresh archive" - ))); - } - - let source_key = SourceKey { - kind: "takeout".to_owned(), - session_url: session_url.clone(), - account_id: account_id.clone(), - }; - let source_id = if common.dry_run { - db::sources::find_source(&conn, &source_key)?.unwrap_or(-1) - } else { - db::sources::upsert_source(&conn, &source_key, account_name.as_deref(), "")? - }; - - if common.dry_run { - *summary = dry_run_summary(&conn, source_id, &walk_result, logger); - return Ok(()); - } - - let options = MappingOptions { - automap: config.automap, - }; - - let mut mailbox_counts = TypeCounts::default(); - let mut email_counts = TypeCounts::default(); - let mut calendar_counts = TypeCounts::default(); - let mut calendar_event_counts = TypeCounts::default(); - let mut book_counts = TypeCounts::default(); - let mut card_counts = TypeCounts::default(); - - let mut mailbox_cache: HashMap = - takeout_ids::all_for_type(&conn, source_id, takeout_ids::MAILBOX)?; - - let mut aborted: Option = None; - if let Err(e) = process_mbox_files( - &mut conn, - source_id, - &walk_result, - options, - &mut mailbox_cache, - &mut mailbox_counts, - &mut email_counts, - logger, - ) { - aborted = Some(e); - } - - if aborted.is_none() - && let Err(e) = process_ics_files( - &mut conn, - source_id, - &walk_result, - &mut calendar_counts, - &mut calendar_event_counts, - logger, - ) - { - aborted = Some(e); - } - - if aborted.is_none() - && let Err(e) = process_vcf_files( - &mut conn, - source_id, - &walk_result, - &mut book_counts, - &mut card_counts, - logger, - ) - { - aborted = Some(e); - } - - let no_failures = aborted.is_none() - && mailbox_counts.failed == 0 - && email_counts.failed == 0 - && calendar_counts.failed == 0 - && calendar_event_counts.failed == 0 - && book_counts.failed == 0 - && card_counts.failed == 0; - if no_failures { - let tx = conn.unchecked_transaction()?; - db::blobs::gc_orphan_blobs(&tx)?; - tx.commit()?; - } - - if logger.enabled(LEVEL_DEFAULT) { - eprintln!( - "takeout: mailbox c={} f={} | email c={} u={} f={} | calendar c={} f={} | \ - event c={} u={} f={} | addressbook c={} | card c={} u={} f={}", - mailbox_counts.created, - mailbox_counts.failed, - email_counts.created, - email_counts.fetched.saturating_sub(email_counts.created), - email_counts.failed, - calendar_counts.created, - calendar_counts.failed, - calendar_event_counts.created, - calendar_event_counts - .fetched - .saturating_sub(calendar_event_counts.created), - calendar_event_counts.failed, - book_counts.created, - card_counts.created, - card_counts.fetched.saturating_sub(card_counts.created), - card_counts.failed, - ); - } - - *summary = Summary { - per_type: vec![ - ("mailbox", mailbox_counts), - ("email", email_counts), - ("calendar", calendar_counts), - ("calendarevent", calendar_event_counts), - ("addressbook", book_counts), - ("contactcard", card_counts), - ], - retries_observed: 0, - retry_after_sleeps: 0, - }; - match aborted { - Some(e) => Err(e), - None => Ok(()), - } -} - -#[allow(clippy::too_many_arguments)] -fn process_mbox_files( - conn: &mut Connection, - source_id: i64, - walk_result: &walk::WalkResult, - options: MappingOptions, - mailbox_cache: &mut HashMap, - mailbox_counts: &mut TypeCounts, - email_counts: &mut TypeCounts, - logger: Logger, -) -> Result<(), Error> { - for file in walk_result.by_kind(FileKind::Mbox) { - let fallback = mailbox_name_from_filename(&file.path); - if logger.enabled(LEVEL_PROGRESS) { - eprintln!( - "mbox: processing {:?} (fallback mailbox: {:?})", - file.path, fallback - ); - } - let ctx = mail::InsertContext { - source_id, - fallback_mailbox: &fallback, - options, - mailbox_cache, - }; - match mail::process_file(conn, &file.path, ctx, mailbox_counts, email_counts, logger) { - Ok(()) => {} - Err(e) if e.aborts_run() => return Err(e), - Err(e) => { - logger.warn(&format!("mbox {:?}: {e}", file.path)); - email_counts.failed += 1; - } - } - } - Ok(()) -} - -fn process_ics_files( - conn: &mut Connection, - source_id: i64, - walk_result: &walk::WalkResult, - calendar_counts: &mut TypeCounts, - event_counts: &mut TypeCounts, - logger: Logger, -) -> Result<(), Error> { - let mut cache: HashMap = - takeout_ids::all_for_type(conn, source_id, takeout_ids::CALENDAR)?; - for file in walk_result.by_kind(FileKind::Ics) { - if logger.enabled(LEVEL_PROGRESS) { - eprintln!("ics: processing {:?}", file.path); - } - if let Err(e) = calendar::process_file( - conn, - &file.path, - source_id, - &mut cache, - calendar_counts, - event_counts, - logger, - ) { - if e.aborts_run() { - return Err(e); - } - logger.warn(&format!("ics {:?}: {e}", file.path)); - event_counts.failed += 1; - } - } - Ok(()) -} - -fn process_vcf_files( - conn: &mut Connection, - source_id: i64, - walk_result: &walk::WalkResult, - book_counts: &mut TypeCounts, - card_counts: &mut TypeCounts, - logger: Logger, -) -> Result<(), Error> { - let mut book_local: Option = - takeout_ids::local_for(conn, source_id, takeout_ids::ADDRESS_BOOK, "Imported")?; - for file in walk_result.by_kind(FileKind::Vcf) { - if logger.enabled(LEVEL_PROGRESS) { - eprintln!("vcf: processing {:?}", file.path); - } - if let Err(e) = contacts::process_file( - conn, - &file.path, - source_id, - &mut book_local, - book_counts, - card_counts, - logger, - ) { - if e.aborts_run() { - return Err(e); - } - logger.warn(&format!("vcf {:?}: {e}", file.path)); - card_counts.failed += 1; - } - } - Ok(()) -} - -fn mailbox_name_from_filename(path: &Path) -> String { - path.file_stem() - .and_then(|s| s.to_str()) - .map(str::to_owned) - .unwrap_or_else(|| "Imported".to_owned()) -} - -fn file_url_for(p: &Path) -> Result { - Url::from_file_path(p) - .map(|u| u.to_string()) - .map_err(|_| Error::Usage(format!("cannot encode takeout path as file:// URL: {p:?}"))) -} - -fn dry_run_summary( - conn: &Connection, - source_id: i64, - walk_result: &walk::WalkResult, - logger: Logger, -) -> Summary { - let mbox_n = walk_result.by_kind(FileKind::Mbox).count(); - let ics_n = walk_result.by_kind(FileKind::Ics).count(); - let vcf_n = walk_result.by_kind(FileKind::Vcf).count(); - let local_emails = if source_id < 0 { - 0 - } else { - count_rows(conn, source_id, takeout_ids::EMAIL) - }; - let local_events = if source_id < 0 { - 0 - } else { - count_rows(conn, source_id, takeout_ids::CALENDAR_EVENT) - }; - let local_cards = if source_id < 0 { - 0 - } else { - count_rows(conn, source_id, takeout_ids::CONTACT_CARD) - }; - if logger.enabled(LEVEL_DEFAULT) { - eprintln!("dry-run: mbox files={mbox_n} ics files={ics_n} vcf files={vcf_n}"); - eprintln!( - " existing in archive: emails={local_emails} events={local_events} cards={local_cards}" - ); - } - Summary { - per_type: vec![ - ("mailbox", TypeCounts::default()), - ("email", TypeCounts::default()), - ("calendar", TypeCounts::default()), - ("calendarevent", TypeCounts::default()), - ("addressbook", TypeCounts::default()), - ("contactcard", TypeCounts::default()), - ], - retries_observed: 0, - retry_after_sleeps: 0, - } -} - -fn count_rows(conn: &Connection, source_id: i64, type_name: &str) -> u64 { - conn.query_row( - "SELECT COUNT(*) FROM sync_id_takeout WHERE source_id = ?1 AND type_name = ?2", - params![source_id, type_name], - |row| row.get::<_, i64>(0), - ) - .unwrap_or(0) as u64 -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn file_url_encodes_unicode_path() { - let p = Path::new("/tmp/Año"); - let u = file_url_for(p).unwrap(); - assert!(u.starts_with("file://")); - assert!(u.contains("A%C3%B1o")); - } - - #[test] - fn file_url_rejects_relative_path() { - let err = file_url_for(Path::new("./relative")).unwrap_err(); - assert!(matches!(err, Error::Usage(_))); - } - - #[test] - fn mailbox_name_from_filename_strips_extension() { - let p = Path::new("/some/dir/Github.mbox"); - assert_eq!(mailbox_name_from_filename(p), "Github"); - } - - #[test] - fn mailbox_name_from_filename_handles_dotted_basename() { - let p = Path::new("/x/All mail Including Spam and Trash.mbox"); - assert_eq!( - mailbox_name_from_filename(p), - "All mail Including Spam and Trash" - ); - } -} diff --git a/src/sync/import_takeout/mail.rs b/src/sync/import_takeout/mail.rs deleted file mode 100644 index e34321b..0000000 --- a/src/sync/import_takeout/mail.rs +++ /dev/null @@ -1,416 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::HashMap; -use std::fs::File; -use std::io::BufReader; -use std::path::Path; -use std::time::{Duration, UNIX_EPOCH}; - -use rusqlite::{Connection, OptionalExtension, Transaction, params}; -use serde_json::Value; -use time::OffsetDateTime; -use time::format_description::well_known::Rfc3339; - -use crate::db; -use crate::db::takeout_ids; -use crate::error::Error; -use crate::logging::{LEVEL_PROGRESS, Logger}; -use crate::sync::TypeCounts; -use crate::sync::emailmeta::email_meta_from_blob; -use crate::sync::keys::index_to_json; - -use super::labels::{self, MappingOptions}; -use super::mbox::{Message, MessageIterator}; - -pub struct InsertContext<'a> { - pub source_id: i64, - pub fallback_mailbox: &'a str, - pub options: MappingOptions, - pub mailbox_cache: &'a mut HashMap, -} - -pub fn process_file( - conn: &mut Connection, - file_path: &Path, - ctx: InsertContext<'_>, - mailbox_counts: &mut TypeCounts, - email_counts: &mut TypeCounts, - logger: Logger, -) -> Result<(), Error> { - let file = - File::open(file_path).map_err(|e| Error::Partial(format!("open {file_path:?}: {e}")))?; - let reader = BufReader::new(file); - let mut iter = MessageIterator::new(reader); - - let InsertContext { - source_id, - fallback_mailbox, - options, - mailbox_cache, - } = ctx; - let tx = conn.transaction()?; - let mut commit_pending: u64 = 0; - - loop { - match iter.next() { - None => break, - Some(Err(e)) => { - logger.warn(&format!("{file_path:?}: mbox io: {e}")); - email_counts.failed += 1; - break; - } - Some(Ok(msg)) => { - match process_message( - &tx, - source_id, - fallback_mailbox, - options, - &msg, - mailbox_cache, - mailbox_counts, - email_counts, - logger, - ) { - Ok(()) => { - commit_pending += 1; - if commit_pending.is_multiple_of(PROGRESS_TICK) - && logger.enabled(LEVEL_PROGRESS) - { - eprintln!("{file_path:?}: processed {commit_pending} messages"); - } - } - Err(e) => { - logger.warn(&format!("{file_path:?}: message: {e}")); - email_counts.failed += 1; - } - } - } - } - } - tx.commit()?; - Ok(()) -} - -const PROGRESS_TICK: u64 = 256; - -#[allow(clippy::too_many_arguments)] -fn process_message( - tx: &Transaction<'_>, - source_id: i64, - fallback_mailbox: &str, - options: MappingOptions, - msg: &Message, - cache: &mut HashMap, - mailbox_counts: &mut TypeCounts, - email_counts: &mut TypeCounts, - logger: Logger, -) -> Result<(), Error> { - let bytes = msg.contents(); - let msg_hash = blake3::hash(bytes).to_hex().to_string(); - - let raw_header = extract_x_gmail_labels(bytes); - let had_labels_header = raw_header.is_some(); - let tokens = match &raw_header { - Some(s) => labels::parse_header(s), - None => Vec::new(), - }; - let mut classification = labels::classify(&tokens); - if classification.mailboxes.is_empty() { - let fallback = fallback_mailbox.trim(); - if fallback.is_empty() { - classification.mailboxes.push("Imported".to_owned()); - } else { - classification.mailboxes.push(fallback.to_owned()); - } - if had_labels_header { - logger.warn(&format!( - "message {msg_hash}: X-Gmail-Labels present but empty after parse; \ - placing in fallback mailbox {:?}", - classification.mailboxes[0] - )); - } else { - logger.warn(&format!( - "message {msg_hash}: no X-Gmail-Labels header; placing in fallback mailbox {:?}", - classification.mailboxes[0] - )); - } - } - if classification.opened_won_over_unread { - logger.warn(&format!( - "message {msg_hash}: X-Gmail-Labels carried both 'Opened' and 'Unread'; \ - treating as $seen (Opened wins)" - )); - } - - let mut mailbox_ids: Vec = Vec::with_capacity(classification.mailboxes.len()); - for path in &classification.mailboxes { - let id = ensure_mailbox(tx, source_id, path, options, cache, mailbox_counts)?; - mailbox_ids.push(id); - } - mailbox_ids.sort_unstable(); - mailbox_ids.dedup(); - - let keywords_json = json_string_array(&classification.keywords_sorted()); - let mailbox_ids_json = json_int_array(&mailbox_ids); - - if let Some(local_id) = takeout_ids::local_for(tx, source_id, takeout_ids::EMAIL, &msg_hash)? { - let current: (String, String) = tx - .query_row( - "SELECT mailbox_ids, keywords FROM emails WHERE id = ?1", - params![local_id], - |r| Ok((r.get(0)?, r.get(1)?)), - ) - .optional()? - .unwrap_or_default(); - if current.0 != mailbox_ids_json || current.1 != keywords_json { - tx.execute( - "UPDATE emails SET mailbox_ids = ?1, keywords = ?2 WHERE id = ?3", - params![mailbox_ids_json, keywords_json, local_id], - )?; - email_counts.fetched += 1; - } else { - email_counts.skipped += 1; - } - return Ok(()); - } - - let blob_id = db::blobs::intern_blob(tx, bytes)?; - let (index, date_rfc3339) = email_meta_from_blob(bytes); - let message_match = index_to_json(&index); - let received_at = pick_received_at(msg.internal_date(), date_rfc3339.as_deref()); - - tx.execute( - "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match) - VALUES (?1, ?2, ?3, ?4, ?5)", - params![ - blob_id, - received_at, - mailbox_ids_json, - keywords_json, - message_match - ], - )?; - let local_id = tx.last_insert_rowid(); - takeout_ids::insert(tx, source_id, takeout_ids::EMAIL, &msg_hash, local_id)?; - email_counts.created += 1; - email_counts.fetched += 1; - Ok(()) -} - -fn ensure_mailbox( - tx: &Transaction<'_>, - source_id: i64, - path: &str, - options: MappingOptions, - cache: &mut HashMap, - counts: &mut TypeCounts, -) -> Result { - if let Some(&id) = cache.get(path) { - return Ok(id); - } - if let Some(id) = takeout_ids::local_for(tx, source_id, takeout_ids::MAILBOX, path)? { - cache.insert(path.to_owned(), id); - return Ok(id); - } - let (parent_id, leaf) = match path.rsplit_once('/') { - Some((parent, leaf)) => ( - Some(ensure_mailbox( - tx, source_id, parent, options, cache, counts, - )?), - leaf, - ), - None => (None, path), - }; - let role = labels::role_for_mailbox(path, options); - tx.execute( - "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) - VALUES (?1, ?2, ?3, 0, 1)", - params![leaf, parent_id, role], - )?; - let id = tx.last_insert_rowid(); - takeout_ids::insert(tx, source_id, takeout_ids::MAILBOX, path, id)?; - cache.insert(path.to_owned(), id); - counts.created += 1; - counts.fetched += 1; - Ok(id) -} - -fn json_string_array(values: &[String]) -> String { - Value::Array(values.iter().map(|s| Value::String(s.clone())).collect()).to_string() -} - -fn json_int_array(values: &[i64]) -> String { - Value::Array(values.iter().map(|&i| Value::from(i)).collect()).to_string() -} - -fn pick_received_at(internal_date: u64, date_rfc3339: Option<&str>) -> String { - if internal_date > 0 { - return format_unix_rfc3339(internal_date); - } - if let Some(d) = date_rfc3339 { - return d.to_owned(); - } - OffsetDateTime::now_utc() - .format(&Rfc3339) - .unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_owned()) -} - -fn format_unix_rfc3339(secs: u64) -> String { - let when = UNIX_EPOCH + Duration::from_secs(secs); - OffsetDateTime::from(when) - .format(&Rfc3339) - .unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_owned()) -} - -fn extract_x_gmail_labels(bytes: &[u8]) -> Option { - let mut values: Vec = Vec::new(); - let mut current: Option = None; - let mut i = 0; - while i < bytes.len() { - let nl = bytes[i..].iter().position(|&b| b == b'\n'); - let (line, advance) = match nl { - Some(p) => (&bytes[i..i + p], p + 1), - None => (&bytes[i..], bytes.len() - i), - }; - let line = line.strip_suffix(b"\r").unwrap_or(line); - if line.is_empty() { - break; - } - let is_continuation = line - .first() - .map(|b| *b == b' ' || *b == b'\t') - .unwrap_or(false); - if is_continuation { - if let Some(buf) = current.as_mut() { - let extra = std::str::from_utf8(line).unwrap_or("").trim_start(); - if !extra.is_empty() { - buf.push(' '); - buf.push_str(extra); - } - } - } else { - if let Some(buf) = current.take() { - values.push(buf); - } - if let Some(colon) = line.iter().position(|&b| b == b':') { - let name = &line[..colon]; - if name.eq_ignore_ascii_case(b"X-Gmail-Labels") { - let value = &line[colon + 1..]; - let value = std::str::from_utf8(value).unwrap_or("").trim(); - current = Some(value.to_owned()); - } - } - } - i += advance; - } - if let Some(buf) = current.take() { - values.push(buf); - } - if values.is_empty() { - None - } else { - Some(values.join(",")) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn extract_labels_from_simple_message() { - let bytes = b"From: a@b\r\n\ - X-Gmail-Labels: Inbox,Opened\r\n\ - Subject: hi\r\n\ - \r\n\ - body\r\n"; - assert_eq!( - extract_x_gmail_labels(bytes), - Some("Inbox,Opened".to_owned()) - ); - } - - #[test] - fn extract_labels_handles_lf_only_line_endings() { - let bytes = b"From: a@b\nX-Gmail-Labels: Inbox\n\nbody\n"; - assert_eq!(extract_x_gmail_labels(bytes), Some("Inbox".to_owned())); - } - - #[test] - fn extract_labels_handles_folded_continuation() { - let bytes = b"From: a@b\r\n\ - X-Gmail-Labels: Inbox,\r\n\ - \tImportant,Github\r\n\ - Subject: hi\r\n\ - \r\n\ - body\r\n"; - assert_eq!( - extract_x_gmail_labels(bytes), - Some("Inbox, Important,Github".to_owned()) - ); - } - - #[test] - fn extract_labels_returns_none_when_header_absent() { - let bytes = b"From: a@b\r\nSubject: hi\r\n\r\nbody\r\n"; - assert_eq!(extract_x_gmail_labels(bytes), None); - } - - #[test] - fn extract_labels_case_insensitive_name_match() { - let bytes = b"x-gmail-labels: Sent\r\n\r\n"; - assert_eq!(extract_x_gmail_labels(bytes), Some("Sent".to_owned())); - } - - #[test] - fn extract_labels_concatenates_multiple_header_occurrences() { - let bytes = b"X-Gmail-Labels: Inbox\r\nX-Gmail-Labels: Github\r\n\r\nbody"; - assert_eq!( - extract_x_gmail_labels(bytes), - Some("Inbox,Github".to_owned()) - ); - } - - #[test] - fn extract_labels_stops_at_blank_line() { - let bytes = b"X-Gmail-Labels: Inbox\r\n\r\nX-Gmail-Labels: NOT-A-HEADER\r\n"; - assert_eq!(extract_x_gmail_labels(bytes), Some("Inbox".to_owned())); - } - - #[test] - fn json_int_array_emits_compact_form() { - assert_eq!(json_int_array(&[1, 2, 3]), "[1,2,3]"); - assert_eq!(json_int_array(&[]), "[]"); - } - - #[test] - fn json_string_array_quotes_each_value() { - assert_eq!( - json_string_array(&["$seen".to_owned(), "$flagged".to_owned()]), - "[\"$seen\",\"$flagged\"]" - ); - } - - #[test] - fn pick_received_at_prefers_envelope_internal_date() { - let got = pick_received_at(1763065233, Some("2025-05-12T10:00:00+02:00")); - assert_eq!(got, "2025-11-13T20:20:33Z"); - } - - #[test] - fn pick_received_at_falls_back_to_date_header() { - let got = pick_received_at(0, Some("2025-05-12T10:00:00+02:00")); - assert_eq!(got, "2025-05-12T10:00:00+02:00"); - } - - #[test] - fn pick_received_at_last_resort_is_now_in_rfc3339() { - let got = pick_received_at(0, None); - assert!(got.ends_with("Z") || got.contains("+") || got.contains("-")); - assert!(got.contains("T")); - } -} diff --git a/src/sync/import_takeout/mbox.rs b/src/sync/import_takeout/mbox.rs deleted file mode 100644 index 33548e7..0000000 --- a/src/sync/import_takeout/mbox.rs +++ /dev/null @@ -1,278 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::io::BufRead; - -use time::{Date, Month, OffsetDateTime, Time, UtcOffset}; - -pub struct MessageIterator { - reader: T, - message: Option, -} - -#[derive(Debug, PartialEq, Eq, Clone)] -pub struct Message { - internal_date: u64, - envelope_from: String, - contents: Vec, -} - -impl MessageIterator -where - T: BufRead, -{ - pub fn new(reader: T) -> MessageIterator { - MessageIterator { - reader, - message: None, - } - } -} - -impl Iterator for MessageIterator -where - T: BufRead, -{ - type Item = std::io::Result; - - fn next(&mut self) -> Option { - let mut line = Vec::with_capacity(80); - - loop { - match self.reader.read_until(b'\n', &mut line) { - Ok(0) => return self.message.take().map(Ok), - Ok(_) => {} - Err(e) => return Some(Err(e)), - } - - if line.starts_with(b"From ") { - let finished = self.message.take().map(Ok); - let header = std::str::from_utf8(&line).unwrap_or(""); - self.message = Some(Message::from_envelope(header)); - if finished.is_some() { - return finished; - } - line.clear(); - continue; - } - - if let Some(message) = &mut self.message { - if line.first() != Some(&b'>') { - message.contents.extend_from_slice(&line); - line.clear(); - continue; - } - let non_gt = line.iter().position(|&c| c != b'>').unwrap_or(line.len()); - if line[non_gt..].starts_with(b"From ") { - message.contents.extend_from_slice(&line[1..]); - } else { - message.contents.extend_from_slice(&line); - } - } - line.clear(); - } - } -} - -impl Message { - fn from_envelope(header: &str) -> Self { - let trimmed = header.strip_prefix("From ").unwrap_or(header); - let (envelope_from, date_str) = match trimmed.split_once(' ') { - Some((from, rest)) => (from.trim().to_owned(), rest), - None => (String::new(), ""), - }; - let internal_date = parse_envelope_date(date_str).unwrap_or(0); - Self { - internal_date, - envelope_from, - contents: Vec::with_capacity(1024), - } - } - - pub fn internal_date(&self) -> u64 { - self.internal_date - } - - pub fn envelope_from(&self) -> &str { - &self.envelope_from - } - - pub fn contents(&self) -> &[u8] { - &self.contents - } - - pub fn unwrap_contents(self) -> Vec { - self.contents - } -} - -fn parse_envelope_date(s: &str) -> Option { - let parts: Vec<&str> = s.split_whitespace().collect(); - let (month_pos, day_pos, time_pos, offset_pos, year_pos) = match parts.len() { - 5 => (1, 2, 3, None, 4), - 6 => (1, 2, 3, Some(4), 5), - _ => return None, - }; - - let month = parse_month(parts[month_pos])?; - let day: u8 = parts[day_pos].parse().ok()?; - let (hour, minute, second) = parse_hms(parts[time_pos])?; - let year: i32 = parts[year_pos].parse().ok()?; - let offset = match offset_pos { - Some(p) => parse_offset(parts[p])?, - None => UtcOffset::UTC, - }; - - let date = Date::from_calendar_date(year, month, day).ok()?; - let time = Time::from_hms(hour, minute, second).ok()?; - let ts = OffsetDateTime::new_in_offset(date, time, offset).unix_timestamp(); - u64::try_from(ts).ok() -} - -fn parse_month(s: &str) -> Option { - Some(match s { - x if x.eq_ignore_ascii_case("jan") => Month::January, - x if x.eq_ignore_ascii_case("feb") => Month::February, - x if x.eq_ignore_ascii_case("mar") => Month::March, - x if x.eq_ignore_ascii_case("apr") => Month::April, - x if x.eq_ignore_ascii_case("may") => Month::May, - x if x.eq_ignore_ascii_case("jun") => Month::June, - x if x.eq_ignore_ascii_case("jul") => Month::July, - x if x.eq_ignore_ascii_case("aug") => Month::August, - x if x.eq_ignore_ascii_case("sep") => Month::September, - x if x.eq_ignore_ascii_case("oct") => Month::October, - x if x.eq_ignore_ascii_case("nov") => Month::November, - x if x.eq_ignore_ascii_case("dec") => Month::December, - _ => return None, - }) -} - -fn parse_hms(s: &str) -> Option<(u8, u8, u8)> { - let mut it = s.split(':'); - let h: u8 = it.next()?.parse().ok()?; - let m: u8 = it.next()?.parse().ok()?; - let sec: u8 = it.next()?.parse().ok()?; - if it.next().is_some() { - return None; - } - Some((h, m, sec)) -} - -fn parse_offset(s: &str) -> Option { - let bytes = s.as_bytes(); - if bytes.len() != 5 { - return None; - } - let sign: i8 = match bytes[0] { - b'+' => 1, - b'-' => -1, - _ => return None, - }; - let hh: i8 = std::str::from_utf8(&bytes[1..3]).ok()?.parse().ok()?; - let mm: i8 = std::str::from_utf8(&bytes[3..5]).ok()?.parse().ok()?; - UtcOffset::from_hms(sign * hh, sign * mm, 0).ok() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn classic_ctime_envelope_parses() { - let raw = b"From god@heaven.af.mil Sat Jan 3 01:05:34 1996\n\ - Message 1\n\ - \n\ - From cras@irccrew.org Tue Jul 23 19:39:23 2002\n\ - Message 2\n\ - \n"; - let mut it = MessageIterator::new(&raw[..]); - let m1 = it.next().unwrap().unwrap(); - assert_eq!(m1.envelope_from(), "god@heaven.af.mil"); - assert_eq!(m1.internal_date(), 820631134); - assert_eq!(m1.contents(), b"Message 1\n\n"); - let m2 = it.next().unwrap().unwrap(); - assert_eq!(m2.envelope_from(), "cras@irccrew.org"); - assert_eq!(m2.internal_date(), 1027453163); - assert!(it.next().is_none()); - } - - #[test] - fn mboxrd_unescape_recovers_lines() { - let raw = b"From a@b Tue Aug 6 13:34:34 2002\n\ - Message 3\n\ - >From hello\n\ - >>From world\n\ - >>>From test\n\ - \n"; - let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap(); - assert_eq!( - m.contents(), - b"Message 3\nFrom hello\n>From world\n>>From test\n\n" - ); - } - - #[test] - fn mboxrd_leaves_non_from_lines_starting_with_gt_alone() { - let raw = b"From a@b Mon Jan 15 15:30:00 2018\n\ - Message 4\n\ - > From\n\ - >F\n"; - let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap(); - assert_eq!(m.contents(), b"Message 4\n> From\n>F\n"); - } - - #[test] - fn takeout_envelope_with_utc_offset_parses() { - let raw = b"From 1848707889910060830@xxx Thu Nov 13 20:20:33 +0000 2025\n\ - X-GM-THRID: 1848707889910060830\n\ - X-Gmail-Labels: Inbox,Opened\n\ - Subject: hi\n\ - \n\ - body\n"; - let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap(); - assert_eq!(m.envelope_from(), "1848707889910060830@xxx"); - assert_eq!(m.internal_date(), 1763065233); - assert!(m.contents().starts_with(b"X-GM-THRID:")); - } - - #[test] - fn takeout_envelope_with_negative_offset_applies_offset() { - let raw = b"From x@y Thu Nov 13 12:20:33 -0800 2025\n\ - body\n"; - let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap(); - assert_eq!(m.internal_date(), 1763065233); - } - - #[test] - fn malformed_envelope_date_yields_zero_not_error() { - let raw = b"From who knows what\nbody\n"; - let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap(); - assert_eq!(m.internal_date(), 0); - } - - #[test] - fn streams_multiple_takeout_format_messages() { - let raw = b"From 1@xxx Thu Nov 13 20:20:33 +0000 2025\n\ - X-GM-THRID: 1\n\ - X-Gmail-Labels: Inbox\n\ - Subject: A\n\ - \n\ - body a\n\ - From 2@xxx Thu Nov 13 21:00:00 +0000 2025\n\ - X-GM-THRID: 2\n\ - X-Gmail-Labels: Sent,Archived\n\ - Subject: B\n\ - \n\ - body b\n"; - let parsed: Vec<_> = MessageIterator::new(&raw[..]) - .collect::, _>>() - .unwrap(); - assert_eq!(parsed.len(), 2); - assert_eq!(parsed[0].envelope_from(), "1@xxx"); - assert_eq!(parsed[1].envelope_from(), "2@xxx"); - assert!(parsed[0].contents().starts_with(b"X-GM-THRID: 1")); - } -} diff --git a/src/sync/import_takeout/tree.rs b/src/sync/import_takeout/tree.rs deleted file mode 100644 index fdba6b8..0000000 --- a/src/sync/import_takeout/tree.rs +++ /dev/null @@ -1,210 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::BTreeSet; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PlannedMailbox { - pub canonical_path: String, - pub leaf: String, - pub parent_path: Option, - pub role: Option<&'static str>, - pub ephemeral: bool, -} - -pub fn assemble(input: &[String], role_for: F) -> Vec -where - F: Fn(&str) -> Option<&'static str>, -{ - let explicit: BTreeSet = input.iter().cloned().collect(); - let mut all_paths: BTreeSet = BTreeSet::new(); - for p in &explicit { - for prefix in prefixes(p) { - all_paths.insert(prefix); - } - } - - let mut planned: Vec = Vec::with_capacity(all_paths.len()); - for path in all_paths { - let (parent_path, leaf) = split_path(&path); - let ephemeral = !explicit.contains(&path); - let role = role_for(&path); - planned.push(PlannedMailbox { - canonical_path: path, - leaf, - parent_path, - role, - ephemeral, - }); - } - planned -} - -pub fn vanished_depth_sort(paths: &mut [String]) { - paths.sort_by(|a, b| depth_of(b).cmp(&depth_of(a)).then_with(|| a.cmp(b))); -} - -fn prefixes(path: &str) -> Vec { - let parts: Vec<&str> = path.split('/').collect(); - let mut out = Vec::with_capacity(parts.len()); - for i in 1..=parts.len() { - out.push(parts[..i].join("/")); - } - out -} - -fn split_path(path: &str) -> (Option, String) { - match path.rsplit_once('/') { - Some((parent, leaf)) => (Some(parent.to_owned()), leaf.to_owned()), - None => (None, path.to_owned()), - } -} - -fn depth_of(path: &str) -> usize { - path.bytes().filter(|b| *b == b'/').count() + 1 -} - -#[cfg(test)] -mod tests { - use super::*; - - fn no_roles(_: &str) -> Option<&'static str> { - None - } - - fn inbox_role(p: &str) -> Option<&'static str> { - if p == "Inbox" { Some("inbox") } else { None } - } - - #[test] - fn flat_paths_pass_through_unchanged() { - let r = assemble( - &["Inbox".to_owned(), "Sent".to_owned(), "Github".to_owned()], - no_roles, - ); - let names: Vec<&str> = r.iter().map(|m| m.canonical_path.as_str()).collect(); - assert_eq!(names, vec!["Github", "Inbox", "Sent"]); - assert!(r.iter().all(|m| !m.ephemeral)); - assert!(r.iter().all(|m| m.parent_path.is_none())); - } - - #[test] - fn nested_path_creates_ephemeral_parent() { - let r = assemble( - &["Label_001/Label_002_under_Label_001".to_owned()], - no_roles, - ); - assert_eq!(r.len(), 2); - assert_eq!(r[0].canonical_path, "Label_001"); - assert!(r[0].ephemeral); - assert_eq!(r[0].parent_path, None); - assert_eq!(r[1].canonical_path, "Label_001/Label_002_under_Label_001"); - assert!(!r[1].ephemeral); - assert_eq!(r[1].parent_path, Some("Label_001".to_owned())); - assert_eq!(r[1].leaf, "Label_002_under_Label_001"); - } - - #[test] - fn explicit_parent_is_not_marked_ephemeral() { - let r = assemble( - &[ - "Label_001".to_owned(), - "Label_001/Sub".to_owned(), - "Label_001/Other".to_owned(), - ], - no_roles, - ); - let parent = r.iter().find(|m| m.canonical_path == "Label_001").unwrap(); - assert!(!parent.ephemeral); - } - - #[test] - fn three_level_deep_path_creates_all_missing_ancestors() { - let r = assemble(&["A/B/C".to_owned()], no_roles); - let names: Vec<&str> = r.iter().map(|m| m.canonical_path.as_str()).collect(); - assert_eq!(names, vec!["A", "A/B", "A/B/C"]); - assert!(r[0].ephemeral); - assert!(r[1].ephemeral); - assert!(!r[2].ephemeral); - assert_eq!(r[2].parent_path, Some("A/B".to_owned())); - } - - #[test] - fn role_closure_consulted_for_ephemeral_parents_too() { - let r = assemble(&["Inbox/Sub".to_owned()], inbox_role); - let inbox = r.iter().find(|m| m.canonical_path == "Inbox").unwrap(); - assert!(inbox.ephemeral, "auto-created from the nested label"); - assert_eq!( - inbox.role, - Some("inbox"), - "an auto-created Inbox is still the inbox" - ); - } - - #[test] - fn role_closure_applies_to_explicit_top_level() { - let r = assemble(&["Inbox".to_owned()], inbox_role); - assert_eq!(r[0].role, Some("inbox")); - } - - #[test] - fn order_is_parents_before_children() { - let r = assemble( - &[ - "Zeta/Alpha".to_owned(), - "Alpha".to_owned(), - "Beta/Gamma/Delta".to_owned(), - ], - no_roles, - ); - let names: Vec<&str> = r.iter().map(|m| m.canonical_path.as_str()).collect(); - assert_eq!( - names, - vec![ - "Alpha", - "Beta", - "Beta/Gamma", - "Beta/Gamma/Delta", - "Zeta", - "Zeta/Alpha", - ] - ); - } - - #[test] - fn duplicate_inputs_dedup() { - let r = assemble( - &["Inbox".to_owned(), "Inbox".to_owned(), "Inbox".to_owned()], - no_roles, - ); - assert_eq!(r.len(), 1); - } - - #[test] - fn vanished_depth_sort_orders_leaves_first() { - let mut v = vec![ - "Foo".to_owned(), - "Foo/Bar/Baz".to_owned(), - "Other".to_owned(), - "Foo/Bar".to_owned(), - ]; - vanished_depth_sort(&mut v); - assert_eq!(v, vec!["Foo/Bar/Baz", "Foo/Bar", "Foo", "Other",]); - } - - #[test] - fn vanished_depth_sort_handles_ties_lexicographically() { - let mut v = vec!["B".to_owned(), "A".to_owned(), "C".to_owned()]; - vanished_depth_sort(&mut v); - assert_eq!(v, vec!["A", "B", "C"]); - } - - #[test] - fn empty_input_yields_empty_output() { - let r = assemble(&[], no_roles); - assert!(r.is_empty()); - } -} diff --git a/src/sync/jmap_map/address_book.rs b/src/sync/jmap_map/address_book.rs new file mode 100644 index 0000000..83bad43 --- /dev/null +++ b/src/sync/jmap_map/address_book.rs @@ -0,0 +1,46 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::Row; + +use crate::archive::rows::{AddressBookRecord, Subscribed, Write}; +use crate::archive::{LocalId, Store}; +use crate::jmap::error::JmapError; +use crate::jmap::wire::address_book::AddressBook; + +impl AddressBook { + pub(crate) const SELECT: &str = + "SELECT id, name, description, sort_order, is_default, is_subscribed FROM address_books"; + + fn record(&self) -> AddressBookRecord<'_> { + AddressBookRecord { + name: &self.name, + description: self.description.as_deref(), + sort_order: i64::from(self.sort_order), + is_default: self.is_default, + subscribed: Subscribed::from(self.is_subscribed), + } + } + + pub(crate) fn insert(&self, store: Store<'_>) -> Result { + Ok(store.address_books().insert(&self.record())?) + } + + pub(crate) fn update(&self, store: Store<'_>, id: LocalId) -> Result { + Ok(store.address_books().update(id, &self.record())?) + } + + pub(crate) fn from_row(row: &Row<'_>) -> Result { + Ok(AddressBook { + id: None, + name: row.get(1)?, + description: row.get(2)?, + sort_order: row.get::<_, i64>(3)? as u32, + is_default: row.get::<_, i64>(4)? != 0, + is_subscribed: row.get::<_, i64>(5)? != 0, + }) + } +} diff --git a/src/sync/jmap_map/calendar.rs b/src/sync/jmap_map/calendar.rs new file mode 100644 index 0000000..8c4745f --- /dev/null +++ b/src/sync/jmap_map/calendar.rs @@ -0,0 +1,65 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::Row; + +use super::JsonColumn; +use crate::archive::rows::{CalendarRecord, Subscribed, Write}; +use crate::archive::{LocalId, Store}; +use crate::jmap::error::JmapError; +use crate::jmap::wire::calendar::Calendar; + +impl Calendar { + pub(crate) const SELECT: &str = "SELECT id, name, description, color, sort_order, is_subscribed, \ + is_visible, is_default, include_in_availability, default_alerts_with_time, \ + default_alerts_without_time, time_zone FROM calendars"; + + fn with_record( + &self, + write: impl FnOnce(&CalendarRecord<'_>) -> rusqlite::Result, + ) -> Result { + let with_time = JsonColumn::encode(self.default_alerts_with_time.as_ref())?; + let without_time = JsonColumn::encode(self.default_alerts_without_time.as_ref())?; + Ok(write(&CalendarRecord { + name: &self.name, + description: self.description.as_deref(), + color: self.color.as_deref(), + sort_order: i64::from(self.sort_order), + subscribed: Subscribed::from(self.is_subscribed), + is_visible: self.is_visible, + is_default: self.is_default, + include_in_availability: &self.include_in_availability, + default_alerts_with_time: with_time.as_deref(), + default_alerts_without_time: without_time.as_deref(), + time_zone: self.time_zone.as_deref(), + })?) + } + + pub(crate) fn insert(&self, store: Store<'_>) -> Result { + self.with_record(|r| store.calendars().insert(r)) + } + + pub(crate) fn update(&self, store: Store<'_>, id: LocalId) -> Result { + self.with_record(|r| store.calendars().update(id, r)) + } + + pub(crate) fn from_row(row: &Row<'_>) -> Result { + Ok(Calendar { + id: None, + name: row.get(1)?, + description: row.get(2)?, + color: row.get(3)?, + sort_order: row.get::<_, i64>(4)? as u32, + is_subscribed: row.get::<_, i64>(5)? != 0, + is_visible: row.get::<_, i64>(6)? != 0, + is_default: row.get::<_, i64>(7)? != 0, + include_in_availability: row.get(8)?, + default_alerts_with_time: JsonColumn::decode(row.get(9)?)?, + default_alerts_without_time: JsonColumn::decode(row.get(10)?)?, + time_zone: row.get(11)?, + }) + } +} diff --git a/src/sync/jmap_map/calendar_event.rs b/src/sync/jmap_map/calendar_event.rs new file mode 100644 index 0000000..eee75c5 --- /dev/null +++ b/src/sync/jmap_map/calendar_event.rs @@ -0,0 +1,78 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::Value; + +use super::{IdRefs, JsData, LocalIds}; +use crate::archive::rows::{CalendarEventRecord, EventDataType, Write}; +use crate::archive::{BlobId, LocalId, Store}; +use crate::jmap::blob::InlineShape; +use crate::jmap::error::JmapError; +use crate::jmap::wire::JmapId; +use crate::jmap::wire::calendar_event::CalendarEvent; +use crate::types::ObjectType; + +const SERVER_SET: [&str; 5] = ["method", "utcStart", "utcEnd", "isOrigin", "baseEventId"]; + +impl CalendarEvent { + pub(crate) const SELECT: &str = "SELECT id, calendar_ids, is_draft, use_default_alerts, data FROM calendar_events \ + WHERE data_type = 'Event'"; + + fn with_record( + self, + resolve: &impl Fn(ObjectType, &str) -> Option, + intern: &mut impl FnMut(&str) -> Result, + write: impl FnOnce(&CalendarEventRecord<'_>) -> rusqlite::Result, + ) -> Result { + let calendars = IdRefs(&self.calendar_ids).to_local(ObjectType::Calendar, resolve)?; + let mut data = JsData::from_rest(self.rest); + data.remove_all(&SERVER_SET); + data.intern_blobs(intern)?; + Ok(write(&CalendarEventRecord { + calendars: &calendars, + is_draft: self.is_draft, + use_default_alerts: self.use_default_alerts, + data_type: EventDataType::Event, + data: &data.text(), + })?) + } + + pub(crate) fn insert( + self, + store: Store<'_>, + resolve: &impl Fn(ObjectType, &str) -> Option, + intern: &mut impl FnMut(&str) -> Result, + ) -> Result { + self.with_record(resolve, intern, |r| store.calendar_events().insert(r)) + } + + pub(crate) fn update( + self, + store: Store<'_>, + id: LocalId, + resolve: &impl Fn(ObjectType, &str) -> Option, + intern: &mut impl FnMut(&str) -> Result, + ) -> Result { + self.with_record(resolve, intern, |r| store.calendar_events().update(id, r)) + } + + pub(crate) fn export_json( + calendar_ids: &str, + is_draft: bool, + use_default_alerts: bool, + data: &str, + resolve: &impl Fn(ObjectType, LocalId) -> Option, + bytes: &impl Fn(BlobId) -> Result, JmapError>, + ) -> Result { + let mut value = JsData::parse(data)?; + value.inline_blobs(InlineShape::JsCalendarLink, bytes)?; + let calendars = LocalIds::parse(calendar_ids)?.to_target(ObjectType::Calendar, resolve)?; + value.insert("calendarIds", calendars); + value.insert("isDraft", Value::Bool(is_draft)); + value.insert("useDefaultAlerts", Value::Bool(use_default_alerts)); + Ok(value.0) + } +} diff --git a/src/sync/jmap_map/contact_card.rs b/src/sync/jmap_map/contact_card.rs new file mode 100644 index 0000000..09a2f4e --- /dev/null +++ b/src/sync/jmap_map/contact_card.rs @@ -0,0 +1,75 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::Value; + +use super::{IdRefs, JsData, LocalIds}; +use crate::archive::rows::{ContactCardRecord, Write}; +use crate::archive::{BlobId, LocalId, Store}; +use crate::jmap::blob::InlineShape; +use crate::jmap::error::JmapError; +use crate::jmap::wire::JmapId; +use crate::jmap::wire::contact_card::ContactCard; +use crate::types::ObjectType; + +impl ContactCard { + pub(crate) const SELECT: &str = "SELECT id, uid, address_book_ids, data FROM contact_cards"; + + fn with_record( + self, + resolve: &impl Fn(ObjectType, &str) -> Option, + intern: &mut impl FnMut(&str) -> Result, + write: impl FnOnce(&ContactCardRecord<'_>) -> rusqlite::Result, + ) -> Result { + let address_books = + IdRefs(&self.address_book_ids).to_local(ObjectType::AddressBook, resolve)?; + let mut data = JsData::from_rest(self.rest); + let uid = data + .take_string("uid") + .ok_or_else(|| JmapError::malformed("ContactCard has no uid"))?; + data.intern_blobs(intern)?; + Ok(write(&ContactCardRecord { + uid: &uid, + address_books: &address_books, + data: &data.text(), + })?) + } + + pub(crate) fn insert( + self, + store: Store<'_>, + resolve: &impl Fn(ObjectType, &str) -> Option, + intern: &mut impl FnMut(&str) -> Result, + ) -> Result { + self.with_record(resolve, intern, |r| store.contact_cards().insert(r)) + } + + pub(crate) fn update( + self, + store: Store<'_>, + id: LocalId, + resolve: &impl Fn(ObjectType, &str) -> Option, + intern: &mut impl FnMut(&str) -> Result, + ) -> Result { + self.with_record(resolve, intern, |r| store.contact_cards().update(id, r)) + } + + pub(crate) fn export_json( + uid: &str, + address_book_ids: &str, + data: &str, + resolve: &impl Fn(ObjectType, LocalId) -> Option, + bytes: &impl Fn(BlobId) -> Result, JmapError>, + ) -> Result { + let mut value = JsData::parse(data)?; + value.inline_blobs(InlineShape::JsContactResource, bytes)?; + let books = + LocalIds::parse(address_book_ids)?.to_target(ObjectType::AddressBook, resolve)?; + value.insert("uid", Value::from(uid)); + value.insert("addressBookIds", books); + Ok(value.0) + } +} diff --git a/src/sync/jmap_map/email.rs b/src/sync/jmap_map/email.rs new file mode 100644 index 0000000..8132255 --- /dev/null +++ b/src/sync/jmap_map/email.rs @@ -0,0 +1,93 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use indexmap::IndexMap; +use rusqlite::Row; +use serde::Deserialize; +use serde_json::Value; + +use super::{IdRefs, KeywordNames, LocalIds, UtcText}; +use crate::archive::rows::{EmailRecord, Write}; +use crate::archive::{BlobId, LocalId, Store}; +use crate::jmap::error::JmapError; +use crate::jmap::wire::JmapId; +use crate::jmap::wire::common::UtcDate; +use crate::jmap::wire::email::Email; +use crate::types::ObjectType; + +#[derive(Debug, Clone)] +pub(crate) struct EmailRow { + pub(crate) blob: BlobId, + pub(crate) received_at: String, + pub(crate) mailboxes: LocalIds, + pub(crate) keywords: Vec, + pub(crate) message_match: String, +} + +impl EmailRow { + pub(crate) const SELECT: &str = + "SELECT id, blob_id, received_at, mailbox_ids, keywords, message_match FROM emails"; + + pub(crate) fn from_row(row: &Row<'_>) -> Result { + let mailbox_text: String = row.get(3)?; + let keyword_text: String = row.get(4)?; + Ok(EmailRow { + blob: row.get(1)?, + received_at: row.get(2)?, + mailboxes: LocalIds::parse(&mailbox_text)?, + keywords: serde_json::from_str(&keyword_text)?, + message_match: row.get(5)?, + }) + } +} + +impl Email { + pub(crate) fn insert( + &self, + store: Store<'_>, + blob: BlobId, + message_match: &str, + resolve: &impl Fn(ObjectType, &str) -> Option, + ) -> Result { + let mailboxes = IdRefs(&self.mailbox_ids).to_local(ObjectType::Mailbox, resolve)?; + if mailboxes.is_empty() { + return Err(JmapError::malformed("email has no resolvable mailbox")); + } + let received_at = UtcText::format(&self.received_at.unwrap_or(UtcDate::UNIX_EPOCH))?; + let keywords = KeywordNames(&self.keywords).to_json()?; + Ok(store.emails().insert(&EmailRecord { + blob, + received_at: &received_at, + mailboxes: &mailboxes, + keywords: &keywords, + message_match: Some(message_match), + })?) + } + + pub(crate) fn update_placement( + store: Store<'_>, + id: LocalId, + obj: &Value, + resolve: &impl Fn(ObjectType, &str) -> Option, + ) -> Result { + let mailbox_ids = match obj.get("mailboxIds") { + Some(v) => IndexMap::::deserialize(v)?, + None => IndexMap::new(), + }; + let mailboxes = IdRefs(&mailbox_ids).to_local(ObjectType::Mailbox, resolve)?; + if mailboxes.is_empty() { + return Err(JmapError::malformed( + "email update has no resolvable mailbox", + )); + } + let keywords = match obj.get("keywords") { + Some(v) => IndexMap::::deserialize(v)?, + None => IndexMap::new(), + }; + let keywords = KeywordNames(&keywords).to_json()?; + Ok(store.emails().update_placement(id, &mailboxes, &keywords)?) + } +} diff --git a/src/sync/jmap_map/file_node.rs b/src/sync/jmap_map/file_node.rs new file mode 100644 index 0000000..a17ff66 --- /dev/null +++ b/src/sync/jmap_map/file_node.rs @@ -0,0 +1,136 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::Row; + +use super::{IdRefs, JsonColumn, UtcText}; +use crate::archive::rows::{FileNodeRecord, NodeType, Subscribed, Write}; +use crate::archive::{BlobId, LocalId, Store}; +use crate::jmap::error::JmapError; +use crate::jmap::wire::JmapId; +use crate::jmap::wire::file_node::{DirectoryProps, FileNode, FileProps, NodeKind, SymLinkProps}; +use crate::types::ObjectType; + +#[derive(Debug, Clone)] +pub(crate) struct FileNodeRow { + pub(crate) wire: FileNode, + pub(crate) blob: Option, +} + +impl FileNodeRow { + pub(crate) const SELECT: &str = "SELECT id, parent_id, node_type, blob_id, target, name, \ + media_type, created, modified, is_subscribed, role, executable FROM file_nodes"; + + pub(crate) fn from_row( + row: &Row<'_>, + resolve: &impl Fn(ObjectType, LocalId) -> Option, + ) -> Result { + let parent_id = row + .get::<_, Option>(1)? + .map(|p| { + resolve(ObjectType::FileNode, p) + .ok_or_else(|| JmapError::malformed("unresolved file_node parent on export")) + }) + .transpose()?; + let node_type: String = row.get(2)?; + let kind = match NodeType::parse(&node_type) { + Some(NodeType::File) => NodeKind::File(FileProps { + blob_id: None, + media_type: row.get(6)?, + executable: row.get::<_, i64>(11)? != 0, + }), + Some(NodeType::Directory) => NodeKind::Directory(DirectoryProps { + role: row.get(10)?, + is_subscribed: row.get::<_, i64>(9)? != 0, + }), + Some(NodeType::SymLink) => NodeKind::SymLink(SymLinkProps { + target: JsonColumn::decode(row.get(4)?)?.unwrap_or_default(), + }), + None => { + return Err(JmapError::malformed(format!( + "unknown file_node type {node_type}" + ))); + } + }; + Ok(FileNodeRow { + wire: FileNode { + id: None, + parent_id, + name: row.get(5)?, + created: UtcText::parse(&row.get::<_, String>(7)?)?, + modified: UtcText::parse_opt(row.get(8)?)?, + kind, + }, + blob: row.get(3)?, + }) + } +} + +impl FileNode { + fn with_record( + &self, + blob: Option, + resolve: &impl Fn(ObjectType, &str) -> Option, + write: impl FnOnce(&FileNodeRecord<'_>) -> rusqlite::Result, + ) -> Result { + let parent = IdRefs::parent(self.parent_id.as_ref(), ObjectType::FileNode, resolve); + let target = match &self.kind { + NodeKind::SymLink(l) => Some(serde_json::to_string(&l.target)?), + NodeKind::File(_) | NodeKind::Directory(_) => None, + }; + let created = UtcText::format(&self.created)?; + let modified = UtcText::format_opt(self.modified.as_ref())?; + let (node_type, media_type, executable, role, subscribed) = match &self.kind { + NodeKind::File(f) => ( + NodeType::File, + f.media_type.as_deref(), + f.executable, + None, + Subscribed::Yes, + ), + NodeKind::Directory(d) => ( + NodeType::Directory, + None, + false, + d.role.as_deref(), + Subscribed::from(d.is_subscribed), + ), + NodeKind::SymLink(_) => (NodeType::SymLink, None, false, None, Subscribed::Yes), + }; + Ok(write(&FileNodeRecord { + parent, + node_type, + blob, + target: target.as_deref(), + name: &self.name, + media_type, + created: &created, + modified: modified.as_deref(), + subscribed, + role, + executable, + })?) + } + + pub(crate) fn insert( + &self, + store: Store<'_>, + blob: Option, + resolve: &impl Fn(ObjectType, &str) -> Option, + ) -> Result { + self.with_record(blob, resolve, |r| store.file_nodes().insert(r)) + } + + pub(crate) fn update( + &self, + store: Store<'_>, + id: LocalId, + blob: Option, + resolve: &impl Fn(ObjectType, &str) -> Option, + ) -> Result { + self.with_record(blob, resolve, |r| store.file_nodes().update(id, r)) + } +} diff --git a/src/sync/jmap_map/identity.rs b/src/sync/jmap_map/identity.rs new file mode 100644 index 0000000..49d2135 --- /dev/null +++ b/src/sync/jmap_map/identity.rs @@ -0,0 +1,54 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::Row; + +use super::JsonColumn; +use crate::archive::rows::{IdentityRecord, Write}; +use crate::archive::{LocalId, Store}; +use crate::jmap::error::JmapError; +use crate::jmap::wire::identity::Identity; + +impl Identity { + pub(crate) const SELECT: &str = + "SELECT id, name, email, reply_to, bcc, text_signature, html_signature FROM identities"; + + fn with_record( + &self, + write: impl FnOnce(&IdentityRecord<'_>) -> rusqlite::Result, + ) -> Result { + let reply_to = JsonColumn::encode(self.reply_to.as_ref())?; + let bcc = JsonColumn::encode(self.bcc.as_ref())?; + Ok(write(&IdentityRecord { + name: &self.name, + email: &self.email, + reply_to: reply_to.as_deref(), + bcc: bcc.as_deref(), + text_signature: &self.text_signature, + html_signature: &self.html_signature, + })?) + } + + pub(crate) fn insert(&self, store: Store<'_>) -> Result { + self.with_record(|r| store.identities().insert(r)) + } + + pub(crate) fn update(&self, store: Store<'_>, id: LocalId) -> Result { + self.with_record(|r| store.identities().update(id, r)) + } + + pub(crate) fn from_row(row: &Row<'_>) -> Result { + Ok(Identity { + id: None, + name: row.get(1)?, + email: row.get(2)?, + reply_to: JsonColumn::decode(row.get(3)?)?, + bcc: JsonColumn::decode(row.get(4)?)?, + text_signature: row.get(5)?, + html_signature: row.get(6)?, + }) + } +} diff --git a/src/sync/jmap_map/mailbox.rs b/src/sync/jmap_map/mailbox.rs new file mode 100644 index 0000000..9cb7754 --- /dev/null +++ b/src/sync/jmap_map/mailbox.rs @@ -0,0 +1,68 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::Row; + +use super::IdRefs; +use crate::archive::rows::{MailboxRecord, Subscribed, Write}; +use crate::archive::{LocalId, Store}; +use crate::jmap::error::JmapError; +use crate::jmap::wire::JmapId; +use crate::jmap::wire::mailbox::Mailbox; +use crate::types::ObjectType; + +impl Mailbox { + pub(crate) const SELECT: &str = + "SELECT id, name, parent_id, role, sort_order, is_subscribed FROM mailboxes"; + + fn record(&self, resolve: &impl Fn(ObjectType, &str) -> Option) -> MailboxRecord<'_> { + MailboxRecord { + name: &self.name, + parent: IdRefs::parent(self.parent_id.as_ref(), ObjectType::Mailbox, resolve), + role: self.role.as_deref(), + sort_order: i64::from(self.sort_order), + subscribed: Subscribed::from(self.is_subscribed), + } + } + + pub(crate) fn insert( + &self, + store: Store<'_>, + resolve: &impl Fn(ObjectType, &str) -> Option, + ) -> Result { + Ok(store.mailboxes().insert(&self.record(resolve))?) + } + + pub(crate) fn update( + &self, + store: Store<'_>, + id: LocalId, + resolve: &impl Fn(ObjectType, &str) -> Option, + ) -> Result { + Ok(store.mailboxes().update(id, &self.record(resolve))?) + } + + pub(crate) fn from_row( + row: &Row<'_>, + resolve: &impl Fn(ObjectType, LocalId) -> Option, + ) -> Result { + let parent_id = row + .get::<_, Option>(2)? + .map(|p| { + resolve(ObjectType::Mailbox, p) + .ok_or_else(|| JmapError::malformed("unresolved mailbox parent on export")) + }) + .transpose()?; + Ok(Mailbox { + id: None, + name: row.get(1)?, + parent_id, + role: row.get(3)?, + sort_order: row.get::<_, i64>(4)? as u32, + is_subscribed: row.get::<_, i64>(5)? != 0, + }) + } +} diff --git a/src/sync/jmap_map/mod.rs b/src/sync/jmap_map/mod.rs new file mode 100644 index 0000000..ae896da --- /dev/null +++ b/src/sync/jmap_map/mod.rs @@ -0,0 +1,220 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +mod address_book; +mod calendar; +mod calendar_event; +mod contact_card; +mod email; +mod file_node; +mod identity; +mod mailbox; +mod participant_identity; +mod sieve_script; +mod vacation; + +#[cfg(test)] +mod tests; + +use indexmap::IndexMap; +use serde::Serialize; +use serde::de::DeserializeOwned; +use serde_json::{Map, Value}; +use time::format_description::well_known::Rfc3339; + +use crate::archive::{BlobId, LocalId}; +use crate::jmap::blob::{BlobRefs, BlobWalkError, InlineShape}; +use crate::jmap::error::JmapError; +use crate::jmap::wire::JmapId; +use crate::jmap::wire::common::UtcDate; +use crate::types::ObjectType; + +pub(crate) use self::email::EmailRow; +pub(crate) use self::file_node::FileNodeRow; +pub(crate) use self::sieve_script::SieveRow; + +pub(crate) struct IdRefs<'a>(&'a IndexMap); + +impl IdRefs<'_> { + fn to_local( + &self, + ty: ObjectType, + resolve: &impl Fn(ObjectType, &str) -> Option, + ) -> Result, JmapError> { + self.0 + .keys() + .map(|id| { + resolve(ty, &id.0).ok_or_else(|| { + JmapError::malformed(format!( + "unresolved {} reference {} on insert", + ty.jmap_name(), + id.0 + )) + }) + }) + .collect() + } + + fn parent( + parent: Option<&JmapId>, + ty: ObjectType, + resolve: &impl Fn(ObjectType, &str) -> Option, + ) -> Option { + parent.and_then(|p| resolve(ty, &p.0)) + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct LocalIds(Vec); + +impl LocalIds { + pub(crate) fn parse(text: &str) -> Result { + let v: Value = serde_json::from_str(text)?; + v.as_array() + .ok_or_else(|| JmapError::malformed("id column is not a JSON array"))? + .iter() + .map(|x| { + x.as_i64() + .map(LocalId::from_raw) + .ok_or_else(|| JmapError::malformed("id array element is not an integer")) + }) + .collect::, _>>() + .map(LocalIds) + } + + pub(crate) fn as_slice(&self) -> &[LocalId] { + &self.0 + } + + fn to_target( + &self, + ty: ObjectType, + resolve: &impl Fn(ObjectType, LocalId) -> Option, + ) -> Result { + let mut out = Map::new(); + for &local in &self.0 { + let Some(target) = resolve(ty, local) else { + return Err(JmapError::malformed(format!( + "unresolved local {} id {local} on export", + ty.jmap_name() + ))); + }; + out.insert(target.0, Value::Bool(true)); + } + Ok(Value::Object(out)) + } +} + +struct UtcText; + +impl UtcText { + fn parse(text: &str) -> Result { + UtcDate::parse(text, &Rfc3339) + .map_err(|e| JmapError::malformed(format!("cannot parse date {text}: {e}"))) + } + + fn parse_opt(text: Option) -> Result, JmapError> { + text.as_deref().map(UtcText::parse).transpose() + } + + fn format(value: &UtcDate) -> Result { + value + .format(&Rfc3339) + .map_err(|e| JmapError::malformed(format!("cannot format date: {e}"))) + } + + fn format_opt(value: Option<&UtcDate>) -> Result, JmapError> { + value.map(UtcText::format).transpose() + } +} + +struct JsonColumn; + +impl JsonColumn { + fn encode(value: Option<&T>) -> Result, JmapError> { + Ok(value.map(serde_json::to_string).transpose()?) + } + + fn decode(text: Option) -> Result, JmapError> { + Ok(text.as_deref().map(serde_json::from_str).transpose()?) + } +} + +struct KeywordNames<'a>(&'a IndexMap); + +impl KeywordNames<'_> { + fn to_json(&self) -> Result { + Ok(serde_json::to_string(self)?) + } +} + +impl Serialize for KeywordNames<'_> { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq(self.0.keys()) + } +} + +struct JsData(Value); + +impl JsData { + fn from_rest(rest: IndexMap) -> JsData { + JsData(Value::Object(rest.into_iter().collect())) + } + + fn parse(text: &str) -> Result { + Ok(JsData(serde_json::from_str(text)?)) + } + + fn take_string(&mut self, key: &str) -> Option { + match self.0.as_object_mut()?.remove(key) { + Some(Value::String(s)) => Some(s), + _ => None, + } + } + + fn remove_all(&mut self, keys: &[&str]) { + if let Value::Object(map) = &mut self.0 { + for key in keys { + map.remove(*key); + } + } + } + + fn insert(&mut self, key: &str, value: Value) { + if let Value::Object(map) = &mut self.0 { + map.insert(key.to_owned(), value); + } + } + + fn intern_blobs( + &mut self, + intern: &mut impl FnMut(&str) -> Result, + ) -> Result<(), JmapError> { + BlobRefs(&mut self.0) + .to_sentinels(|jmap_blob_id| { + intern(jmap_blob_id) + .map(BlobId::get) + .map_err(BlobWalkError::resolver) + }) + .map_err(BlobWalkError::into_source) + } + + fn inline_blobs( + &mut self, + shape: InlineShape, + bytes: &impl Fn(BlobId) -> Result, JmapError>, + ) -> Result<(), JmapError> { + BlobRefs(&mut self.0) + .to_data_uris(shape, |local_id| { + bytes(BlobId::from_raw(local_id)).map_err(BlobWalkError::resolver) + }) + .map_err(BlobWalkError::into_source) + } + + fn text(&self) -> String { + self.0.to_string() + } +} diff --git a/src/sync/jmap_map/participant_identity.rs b/src/sync/jmap_map/participant_identity.rs new file mode 100644 index 0000000..767fcda --- /dev/null +++ b/src/sync/jmap_map/participant_identity.rs @@ -0,0 +1,28 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::archive::rows::{ParticipantIdentityRecord, Write}; +use crate::archive::{LocalId, Store}; +use crate::jmap::error::JmapError; +use crate::jmap::wire::participant_identity::ParticipantIdentity; + +impl ParticipantIdentity { + fn record(&self) -> ParticipantIdentityRecord<'_> { + ParticipantIdentityRecord { + name: &self.name, + calendar_address: &self.calendar_address, + is_default: self.is_default, + } + } + + pub(crate) fn insert(&self, store: Store<'_>) -> Result { + Ok(store.participant_identities().insert(&self.record())?) + } + + pub(crate) fn update(&self, store: Store<'_>, id: LocalId) -> Result { + Ok(store.participant_identities().update(id, &self.record())?) + } +} diff --git a/src/sync/jmap_map/sieve_script.rs b/src/sync/jmap_map/sieve_script.rs new file mode 100644 index 0000000..22a9f32 --- /dev/null +++ b/src/sync/jmap_map/sieve_script.rs @@ -0,0 +1,54 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::Row; + +use crate::archive::rows::{SieveScriptRecord, Write}; +use crate::archive::{BlobId, LocalId, Store}; +use crate::jmap::error::JmapError; +use crate::jmap::wire::sieve_script::SieveScript; + +#[derive(Debug, Clone)] +pub(crate) struct SieveRow { + pub(crate) name: Option, + pub(crate) is_active: bool, + pub(crate) blob: BlobId, +} + +impl SieveRow { + pub(crate) const SELECT: &str = "SELECT id, name, is_active, blob_id FROM sieve_scripts"; + + pub(crate) fn from_row(row: &Row<'_>) -> Result { + Ok(SieveRow { + name: row.get(1)?, + is_active: row.get::<_, i64>(2)? != 0, + blob: row.get(3)?, + }) + } +} + +impl SieveScript { + fn record(&self, blob: BlobId) -> SieveScriptRecord<'_> { + SieveScriptRecord { + name: self.name.as_deref(), + is_active: self.is_active, + blob, + } + } + + pub(crate) fn insert(&self, store: Store<'_>, blob: BlobId) -> Result { + Ok(store.sieve_scripts().insert(&self.record(blob))?) + } + + pub(crate) fn update( + &self, + store: Store<'_>, + id: LocalId, + blob: BlobId, + ) -> Result { + Ok(store.sieve_scripts().update(id, &self.record(blob))?) + } +} diff --git a/src/sync/jmap_map/tests.rs b/src/sync/jmap_map/tests.rs new file mode 100644 index 0000000..c8a1867 --- /dev/null +++ b/src/sync/jmap_map/tests.rs @@ -0,0 +1,1247 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use encodify::base64::STANDARD; +use rusqlite::{Connection, Row, params}; +use serde_json::Value; + +use super::{EmailRow, FileNodeRow, SieveRow}; +use crate::archive::rows::Write; +use crate::archive::{Archive, BlobId, LocalId, Store}; +use crate::error::ExitStatus; +use crate::jmap::error::JmapError; +use crate::jmap::wire::JmapId; +use crate::jmap::wire::address_book::AddressBook; +use crate::jmap::wire::calendar::Calendar; +use crate::jmap::wire::calendar_event::CalendarEvent; +use crate::jmap::wire::contact_card::ContactCard; +use crate::jmap::wire::email::Email; +use crate::jmap::wire::file_node::{FileNode, NodeKind}; +use crate::jmap::wire::identity::Identity; +use crate::jmap::wire::mailbox::Mailbox; +use crate::jmap::wire::participant_identity::ParticipantIdentity; +use crate::jmap::wire::sieve_script::SieveScript; +use crate::types::ObjectType; + +const MAILBOX_SELECT: &str = Mailbox::SELECT; +const EMAIL_SELECT: &str = EmailRow::SELECT; +const IDENTITY_SELECT: &str = Identity::SELECT; +const SIEVE_SELECT: &str = SieveRow::SELECT; +const ADDRESS_BOOK_SELECT: &str = AddressBook::SELECT; +const CALENDAR_SELECT: &str = Calendar::SELECT; +const PARTICIPANT_IDENTITY_SELECT: &str = + "SELECT id, name, calendar_address, is_default FROM participant_identities"; +const FILE_NODE_SELECT: &str = FileNodeRow::SELECT; +const CONTACT_CARD_SELECT: &str = ContactCard::SELECT; +const CALENDAR_EVENT_SELECT: &str = CalendarEvent::SELECT; + +struct MapResolver { + to_local: HashMap<(ObjectType, String), i64>, + to_target: HashMap<(ObjectType, i64), String>, +} + +impl MapResolver { + fn local(&self) -> impl Fn(ObjectType, &str) -> Option + '_ { + |ty, id| { + self.to_local + .get(&(ty, id.to_owned())) + .copied() + .map(LocalId::from_raw) + } + } + + fn target(&self) -> impl Fn(ObjectType, LocalId) -> Option + '_ { + |ty, id| { + self.to_target + .get(&(ty, id.get())) + .map(|s| JmapId(s.clone())) + } + } +} + +trait BlobIntern { + fn intern(&mut self, jmap_blob_id: &str) -> Result; +} + +trait BlobBytes { + fn bytes(&self, local_id: i64) -> Result, JmapError>; +} + +struct FakeBlobs; + +impl BlobIntern for FakeBlobs { + fn intern(&mut self, _jmap_blob_id: &str) -> Result { + Ok(42) + } +} + +impl BlobBytes for FakeBlobs { + fn bytes(&self, local_id: i64) -> Result, JmapError> { + Ok(format!("payload-{local_id}").into_bytes()) + } +} + +fn changed(write: Result) -> Result { + write.map(|w| w == Write::Changed) +} + +fn intern_blob(c: &Connection, bytes: &[u8]) -> rusqlite::Result { + Store::new(c).blobs().intern(bytes).map(BlobId::get) +} + +fn blob_id(id: i64) -> BlobId { + BlobId::from_raw(id) +} + +fn interner(blobs: &mut impl BlobIntern) -> impl FnMut(&str) -> Result + '_ { + |id| blobs.intern(id).map(BlobId::from_raw) +} + +fn reader(blobs: &impl BlobBytes) -> impl Fn(BlobId) -> Result, JmapError> + '_ { + |id| blobs.bytes(id.get()) +} + +fn insert_mailbox(c: &Connection, w: &Mailbox, r: &MapResolver) -> Result { + w.insert(Store::new(c), &r.local()).map(LocalId::get) +} + +fn update_mailbox( + c: &Connection, + id: i64, + w: &Mailbox, + r: &MapResolver, +) -> Result { + changed(w.update(Store::new(c), LocalId::from_raw(id), &r.local())) +} + +fn row_to_mailbox(row: &Row<'_>, r: &MapResolver) -> Result { + Mailbox::from_row(row, &r.target()) +} + +fn insert_email( + c: &Connection, + w: &Email, + blob: i64, + message_match: &str, + r: &MapResolver, +) -> Result { + w.insert(Store::new(c), blob_id(blob), message_match, &r.local()) + .map(LocalId::get) +} + +fn update_email(c: &Connection, id: i64, obj: &Value, r: &MapResolver) -> Result { + changed(Email::update_placement( + Store::new(c), + LocalId::from_raw(id), + obj, + &r.local(), + )) +} + +fn row_to_email(row: &Row<'_>) -> Result { + EmailRow::from_row(row) +} + +fn insert_identity(c: &Connection, w: &Identity) -> Result { + w.insert(Store::new(c)).map(LocalId::get) +} + +fn update_identity(c: &Connection, id: i64, w: &Identity) -> Result { + changed(w.update(Store::new(c), LocalId::from_raw(id))) +} + +fn row_to_identity(row: &Row<'_>) -> Result { + Identity::from_row(row) +} + +fn insert_sieve_script(c: &Connection, w: &SieveScript, blob: i64) -> Result { + w.insert(Store::new(c), blob_id(blob)).map(LocalId::get) +} + +fn update_sieve_script( + c: &Connection, + id: i64, + w: &SieveScript, + blob: i64, +) -> Result { + changed(w.update(Store::new(c), LocalId::from_raw(id), blob_id(blob))) +} + +fn row_to_sieve_script(row: &Row<'_>) -> Result { + SieveRow::from_row(row) +} + +fn insert_address_book(c: &Connection, w: &AddressBook) -> Result { + w.insert(Store::new(c)).map(LocalId::get) +} + +fn update_address_book(c: &Connection, id: i64, w: &AddressBook) -> Result { + changed(w.update(Store::new(c), LocalId::from_raw(id))) +} + +fn row_to_address_book(row: &Row<'_>) -> Result { + AddressBook::from_row(row) +} + +fn insert_calendar(c: &Connection, w: &Calendar) -> Result { + w.insert(Store::new(c)).map(LocalId::get) +} + +fn update_calendar(c: &Connection, id: i64, w: &Calendar) -> Result { + changed(w.update(Store::new(c), LocalId::from_raw(id))) +} + +fn row_to_calendar(row: &Row<'_>) -> Result { + Calendar::from_row(row) +} + +fn insert_participant_identity(c: &Connection, w: &ParticipantIdentity) -> Result { + w.insert(Store::new(c)).map(LocalId::get) +} + +fn update_participant_identity( + c: &Connection, + id: i64, + w: &ParticipantIdentity, +) -> Result { + changed(w.update(Store::new(c), LocalId::from_raw(id))) +} + +fn row_to_participant_identity(row: &Row<'_>) -> Result { + Ok(ParticipantIdentity { + id: None, + name: row.get(1)?, + calendar_address: row.get(2)?, + is_default: row.get::<_, i64>(3)? != 0, + }) +} + +fn insert_file_node( + c: &Connection, + w: &FileNode, + blob: Option, + r: &MapResolver, +) -> Result { + w.insert(Store::new(c), blob.map(blob_id), &r.local()) + .map(LocalId::get) +} + +fn update_file_node( + c: &Connection, + id: i64, + w: &FileNode, + blob: Option, + r: &MapResolver, +) -> Result { + changed(w.update( + Store::new(c), + LocalId::from_raw(id), + blob.map(blob_id), + &r.local(), + )) +} + +fn row_to_file_node(row: &Row<'_>, r: &MapResolver) -> Result { + FileNodeRow::from_row(row, &r.target()) +} + +fn insert_contact_card( + c: &Connection, + w: &ContactCard, + r: &MapResolver, + blobs: &mut impl BlobIntern, +) -> Result { + w.clone() + .insert(Store::new(c), &r.local(), &mut interner(blobs)) + .map(LocalId::get) +} + +fn update_contact_card( + c: &Connection, + id: i64, + w: &ContactCard, + r: &MapResolver, + blobs: &mut impl BlobIntern, +) -> Result { + changed(w.clone().update( + Store::new(c), + LocalId::from_raw(id), + &r.local(), + &mut interner(blobs), + )) +} + +fn contact_card_to_wire( + uid: &str, + address_book_ids: &str, + data: &str, + r: &MapResolver, + blobs: &impl BlobBytes, +) -> Result { + ContactCard::export_json(uid, address_book_ids, data, &r.target(), &reader(blobs)) +} + +fn insert_calendar_event( + c: &Connection, + w: &CalendarEvent, + r: &MapResolver, + blobs: &mut impl BlobIntern, +) -> Result { + w.clone() + .insert(Store::new(c), &r.local(), &mut interner(blobs)) + .map(LocalId::get) +} + +fn update_calendar_event( + c: &Connection, + id: i64, + w: &CalendarEvent, + r: &MapResolver, + blobs: &mut impl BlobIntern, +) -> Result { + changed(w.clone().update( + Store::new(c), + LocalId::from_raw(id), + &r.local(), + &mut interner(blobs), + )) +} + +fn calendar_event_to_wire( + calendar_ids: &str, + is_draft: bool, + use_default_alerts: bool, + data: &str, + r: &MapResolver, + blobs: &impl BlobBytes, +) -> Result { + CalendarEvent::export_json( + calendar_ids, + is_draft, + use_default_alerts, + data, + &r.target(), + &reader(blobs), + ) +} + +fn decode_data_uri(value: &Value, media_type: &str) -> Vec { + let uri = value.as_str().unwrap_or_else(|| panic!("{value} is a URI")); + let prefix = format!("data:{media_type};base64,"); + let payload = uri + .strip_prefix(&prefix) + .unwrap_or_else(|| panic!("{uri} does not start with {prefix}")); + STANDARD.decode(payload).expect("base64 payload") +} + +fn assert_no_blob_id(value: &Value) { + match value { + Value::Object(map) => { + assert!(map.get("blobId").is_none(), "blobId present in {value}"); + assert!(map.get("@blob").is_none(), "@blob present in {value}"); + for child in map.values() { + assert_no_blob_id(child); + } + } + Value::Array(items) => items.iter().for_each(assert_no_blob_id), + _ => {} + } +} + +fn mem() -> Archive { + Archive::open_in_memory().unwrap() +} + +#[test] +fn mailbox_roundtrips_with_parent_translation() { + let archive = mem(); + let c = archive.connection(); + let root: Mailbox = serde_json::from_value(serde_json::json!({ + "id": "M1", "name": "Root", "role": "archive", "sortOrder": 3 + })) + .unwrap(); + let empty = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + let root_local = insert_mailbox(c, &root, &empty).unwrap(); + + let child: Mailbox = serde_json::from_value(serde_json::json!({ + "id": "M2", "name": "Child", "parentId": "M1" + })) + .unwrap(); + let mut res = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + res.to_local + .insert((ObjectType::Mailbox, "M1".to_owned()), root_local); + let child_local = insert_mailbox(c, &child, &res).unwrap(); + + res.to_target + .insert((ObjectType::Mailbox, root_local), "TGT1".to_owned()); + let wire = c + .query_row( + &format!("{MAILBOX_SELECT} WHERE id = ?1"), + params![child_local], + |row| Ok(row_to_mailbox(row, &res)), + ) + .unwrap() + .unwrap(); + assert_eq!(wire.name, "Child"); + assert_eq!(wire.parent_id, Some(JmapId("TGT1".to_owned()))); +} + +#[test] +fn email_keywords_keep_verbatim_case_and_mailbox_translation() { + let archive = mem(); + let c = archive.connection(); + let mut res = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + res.to_local + .insert((ObjectType::Mailbox, "MB".to_owned()), 7); + let email: Email = serde_json::from_value(serde_json::json!({ + "id": "E1", + "blobId": "B1", + "receivedAt": "2021-05-04T10:00:00Z", + "mailboxIds": { "MB": true }, + "keywords": { "$Seen": true, "MyTag": true } + })) + .unwrap(); + let blob_id = intern_blob(c, b"rfc5322 bytes").unwrap(); + let local = insert_email(c, &email, blob_id, "{}", &res).unwrap(); + let got = c + .query_row( + &format!("{EMAIL_SELECT} WHERE id = ?1"), + params![local], + |row| Ok(row_to_email(row)), + ) + .unwrap() + .unwrap(); + assert_eq!(got.blob.get(), blob_id); + assert_eq!( + got.mailboxes + .as_slice() + .iter() + .map(|l| l.get()) + .collect::>(), + vec![7] + ); + assert_eq!(got.keywords, vec!["$Seen".to_owned(), "MyTag".to_owned()]); +} + +#[test] +fn unresolved_foreign_key_on_insert_is_malformed() { + let archive = mem(); + let c = archive.connection(); + let empty = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + let email: Email = serde_json::from_value(serde_json::json!({ + "id": "E1", + "blobId": "B1", + "receivedAt": "2021-05-04T10:00:00Z", + "mailboxIds": { "UNKNOWN": true }, + "keywords": {} + })) + .unwrap(); + let err = insert_email(c, &email, 1, "{}", &empty).unwrap_err(); + assert!(matches!(err, JmapError::Malformed(_)), "got {err:?}"); +} + +#[test] +fn unresolved_target_on_export_is_malformed() { + let archive = mem(); + let c = archive.connection(); + let root: Mailbox = serde_json::from_value(serde_json::json!({ + "id": "M1", "name": "Root", "parentId": "P9" + })) + .unwrap(); + let mut res = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + res.to_local + .insert((ObjectType::Mailbox, "P9".to_owned()), 1); + let local = insert_mailbox(c, &root, &res).unwrap(); + let err = c + .query_row( + &format!("{MAILBOX_SELECT} WHERE id = ?1"), + params![local], + |row| Ok(row_to_mailbox(row, &res)), + ) + .unwrap() + .unwrap_err(); + assert!(matches!(err, JmapError::Malformed(_)), "got {err:?}"); +} + +#[test] +fn contact_card_strips_uid_and_inlines_media_as_data_uri() { + let archive = mem(); + let c = archive.connection(); + let res = MapResolver { + to_local: HashMap::from([((ObjectType::AddressBook, "AB".to_owned()), 1)]), + to_target: HashMap::from([((ObjectType::AddressBook, 1), "TAB".to_owned())]), + }; + let card: ContactCard = serde_json::from_value(serde_json::json!({ + "id": "C1", + "addressBookIds": { "AB": true }, + "uid": "urn:uuid:42", + "name": { "full": "Jane" }, + "media": { "photo": { + "@type": "Media", "kind": "photo", + "blobId": "PB", "mediaType": "image/png" + } } + })) + .unwrap(); + let mut blobs = FakeBlobs; + let local = insert_contact_card(c, &card, &res, &mut blobs).unwrap(); + let (uid, abids, data): (String, String, String) = c + .query_row( + &format!("{CONTACT_CARD_SELECT} WHERE id = ?1"), + params![local], + |r| Ok((r.get(1)?, r.get(2)?, r.get(3)?)), + ) + .unwrap(); + assert_eq!(uid, "urn:uuid:42"); + let stored: Value = serde_json::from_str(&data).unwrap(); + assert!(stored.get("uid").is_none()); + assert_eq!(stored["media"]["photo"]["@blob"], Value::from(42)); + + let up = FakeBlobs; + let wire = contact_card_to_wire(&uid, &abids, &data, &res, &up).unwrap(); + assert_eq!(wire["uid"], Value::from("urn:uuid:42")); + assert_eq!(wire["addressBookIds"]["TAB"], Value::Bool(true)); + assert_eq!( + decode_data_uri(&wire["media"]["photo"]["uri"], "image/png"), + b"payload-42" + ); + assert_eq!( + wire["media"]["photo"]["mediaType"], + Value::from("image/png") + ); + assert!(wire["media"]["photo"].get("href").is_none()); + assert_no_blob_id(&wire); +} + +#[test] +fn calendar_event_inlines_link_enclosure_as_data_uri() { + let archive = mem(); + let c = archive.connection(); + let res = MapResolver { + to_local: HashMap::from([((ObjectType::Calendar, "CAL".to_owned()), 5)]), + to_target: HashMap::from([((ObjectType::Calendar, 5), "TCAL".to_owned())]), + }; + let ev: CalendarEvent = serde_json::from_value(serde_json::json!({ + "id": "EV1", + "calendarIds": { "CAL": true }, + "uid": "ev-with-enclosure", + "title": "Review", + "@type": "Event", + "links": { "1": { + "@type": "Link", "rel": "enclosure", + "blobId": "AB", "contentType": "text/plain", "title": "agenda.txt" + } } + })) + .unwrap(); + let mut blobs = FakeBlobs; + let local = insert_calendar_event(c, &ev, &res, &mut blobs).unwrap(); + let (cal, dr, ud, data): (String, i64, i64, String) = c + .query_row( + &format!("{CALENDAR_EVENT_SELECT} AND id = ?1"), + params![local], + |r| Ok((r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)), + ) + .unwrap(); + let stored: Value = serde_json::from_str(&data).unwrap(); + assert_eq!(stored["links"]["1"]["@blob"], Value::from(42)); + + let up = FakeBlobs; + let wire = calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, &res, &up).unwrap(); + assert_eq!( + decode_data_uri(&wire["links"]["1"]["href"], "text/plain"), + b"payload-42" + ); + assert_eq!(wire["links"]["1"]["contentType"], Value::from("text/plain")); + assert_eq!(wire["links"]["1"]["rel"], Value::from("enclosure")); + assert!(wire["links"]["1"].get("uri").is_none()); + assert_no_blob_id(&wire); +} + +#[test] +fn contact_card_media_without_media_type_defaults_to_octet_stream() { + let archive = mem(); + let c = archive.connection(); + let res = MapResolver { + to_local: HashMap::from([((ObjectType::AddressBook, "AB".to_owned()), 1)]), + to_target: HashMap::from([((ObjectType::AddressBook, 1), "TAB".to_owned())]), + }; + c.execute( + "INSERT INTO contact_cards (id,uid,address_book_ids,data) + VALUES (1,'u-1','[1]',?1)", + params![ + serde_json::json!({ "@type": "Card", "media": { "photo": { "@blob": 9 } } }) + .to_string() + ], + ) + .unwrap(); + let (uid, abids, data): (String, String, String) = c + .query_row(&format!("{CONTACT_CARD_SELECT} WHERE id = 1"), [], |r| { + Ok((r.get(1)?, r.get(2)?, r.get(3)?)) + }) + .unwrap(); + let up = FakeBlobs; + let wire = contact_card_to_wire(&uid, &abids, &data, &res, &up).unwrap(); + assert_eq!( + decode_data_uri(&wire["media"]["photo"]["uri"], "application/octet-stream"), + b"payload-9" + ); + assert_no_blob_id(&wire); +} + +#[test] +fn an_archive_read_failure_while_inlining_is_an_archive_error_not_a_unit_failure() { + struct BrokenBlobs; + impl BlobBytes for BrokenBlobs { + fn bytes(&self, _local_id: i64) -> Result, JmapError> { + Err(JmapError::Sqlite(rusqlite::Error::QueryReturnedNoRows)) + } + } + let res = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::from([ + ((ObjectType::AddressBook, 1), "TAB".to_owned()), + ((ObjectType::Calendar, 1), "TCAL".to_owned()), + ]), + }; + let card = + serde_json::json!({ "@type": "Card", "media": { "photo": { "@blob": 9 } } }).to_string(); + let event = + serde_json::json!({ "@type": "Event", "links": { "1": { "@blob": 9 } } }).to_string(); + let failures = [ + contact_card_to_wire("u-1", "[1]", &card, &res, &BrokenBlobs).expect_err("card"), + calendar_event_to_wire("[1]", false, false, &event, &res, &BrokenBlobs).expect_err("event"), + ]; + for err in failures { + assert!(matches!(err, JmapError::Sqlite(_)), "{err:?}"); + let mapped = crate::error::Error::from(err); + assert!(mapped.aborts_run(), "{mapped} must abort the run"); + assert_eq!(mapped.exit_status(), ExitStatus::Archive); + } +} + +#[test] +fn calendar_event_extracts_columns_and_drops_method() { + let archive = mem(); + let c = archive.connection(); + let res = MapResolver { + to_local: HashMap::from([((ObjectType::Calendar, "CAL".to_owned()), 5)]), + to_target: HashMap::from([((ObjectType::Calendar, 5), "TCAL".to_owned())]), + }; + let ev: CalendarEvent = serde_json::from_value(serde_json::json!({ + "id": "EV1", + "calendarIds": { "CAL": true }, + "isDraft": true, + "useDefaultAlerts": false, + "method": "request", + "utcStart": "2020-01-01T00:00:00Z", + "title": "Sprint", + "@type": "Event" + })) + .unwrap(); + let mut blobs = FakeBlobs; + let local = insert_calendar_event(c, &ev, &res, &mut blobs).unwrap(); + let (cal, dr, ud, data): (String, i64, i64, String) = c + .query_row( + &format!("{CALENDAR_EVENT_SELECT} AND id = ?1"), + params![local], + |r| Ok((r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)), + ) + .unwrap(); + assert_eq!(dr, 1); + assert_eq!(ud, 0); + let stored: Value = serde_json::from_str(&data).unwrap(); + assert!(stored.get("method").is_none()); + assert!(stored.get("utcStart").is_none()); + assert!(stored.get("calendarIds").is_none()); + assert_eq!(stored["title"], Value::from("Sprint")); + + let up = FakeBlobs; + let wire = calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, &res, &up).unwrap(); + assert_eq!(wire["calendarIds"]["TCAL"], Value::Bool(true)); + assert_eq!(wire["isDraft"], Value::Bool(true)); + assert_eq!(wire["title"], Value::from("Sprint")); +} + +#[test] +fn identity_json_columns_roundtrip() { + let archive = mem(); + let c = archive.connection(); + let id: Identity = serde_json::from_value(serde_json::json!({ + "id": "I1", + "name": "Alice", + "email": "a@x.test", + "replyTo": [ { "name": "A", "email": "a@x.test" } ] + })) + .unwrap(); + let local = insert_identity(c, &id).unwrap(); + let wire = c + .query_row( + &format!("{IDENTITY_SELECT} WHERE id = ?1"), + params![local], + |row| Ok(row_to_identity(row)), + ) + .unwrap() + .unwrap(); + assert_eq!(wire.email, "a@x.test"); + assert_eq!(wire.reply_to.unwrap()[0].email, "a@x.test"); + assert!(wire.bcc.is_none()); +} + +#[test] +fn file_node_enum_and_target_roundtrip() { + let archive = mem(); + let c = archive.connection(); + let empty = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + let node: FileNode = serde_json::from_value(serde_json::json!({ + "id": "F1", + "@type": "SymLink", + "target": ["..", "etc", "hosts"], + "name": "link", + "created": "2022-02-02T02:02:02Z" + })) + .unwrap(); + let local = insert_file_node(c, &node, None, &empty).unwrap(); + let (nt, tgt): (String, Option) = c + .query_row( + &format!("{FILE_NODE_SELECT} WHERE id = ?1"), + params![local], + |r| Ok((r.get(2)?, r.get(4)?)), + ) + .unwrap(); + assert_eq!(nt, "symlink"); + assert_eq!( + serde_json::from_str::>(&tgt.unwrap()).unwrap(), + vec!["..", "etc", "hosts"] + ); +} + +#[test] +fn address_book_and_calendar_row_to_wire_roundtrip() { + let archive = mem(); + let c = archive.connection(); + let ab: AddressBook = serde_json::from_value(serde_json::json!({ + "id": "A1", "name": "Personal", "description": "d", "sortOrder": 2, + "isDefault": true, "isSubscribed": false + })) + .unwrap(); + let abl = insert_address_book(c, &ab).unwrap(); + let got = c + .query_row( + &format!("{ADDRESS_BOOK_SELECT} WHERE id = ?1"), + params![abl], + |r| Ok(row_to_address_book(r)), + ) + .unwrap() + .unwrap(); + assert_eq!(got.name, "Personal"); + assert!(got.is_default); + assert!(!got.is_subscribed); + + let cal: Calendar = serde_json::from_value(serde_json::json!({ + "id": "C1", "name": "Work", "color": "#ff0000", "timeZone": "Europe/Rome", + "isVisible": false + })) + .unwrap(); + let cl = insert_calendar(c, &cal).unwrap(); + let gc = c + .query_row( + &format!("{CALENDAR_SELECT} WHERE id = ?1"), + params![cl], + |r| Ok(row_to_calendar(r)), + ) + .unwrap() + .unwrap(); + assert_eq!(gc.name, "Work"); + assert_eq!(gc.color.as_deref(), Some("#ff0000")); + assert_eq!(gc.time_zone.as_deref(), Some("Europe/Rome")); + assert!(!gc.is_visible); +} + +#[test] +fn participant_identity_and_sieve_row_to_wire_roundtrip() { + let archive = mem(); + let c = archive.connection(); + let pi: ParticipantIdentity = serde_json::from_value(serde_json::json!({ + "id": "P1", "name": "Me", "calendarAddress": "mailto:me@x.test", + "isDefault": true + })) + .unwrap(); + let pil = insert_participant_identity(c, &pi).unwrap(); + let gp = c + .query_row( + &format!("{PARTICIPANT_IDENTITY_SELECT} WHERE id = ?1"), + params![pil], + |r| Ok(row_to_participant_identity(r)), + ) + .unwrap() + .unwrap(); + assert_eq!(gp.calendar_address, "mailto:me@x.test"); + assert!(gp.is_default); + + let ss: SieveScript = serde_json::from_value(serde_json::json!({ + "id": "S1", "name": "main", "isActive": true, "blobId": "B1" + })) + .unwrap(); + let blob = intern_blob(c, b"keep;").unwrap(); + let ssl = insert_sieve_script(c, &ss, blob).unwrap(); + let gs = c + .query_row( + &format!("{SIEVE_SELECT} WHERE id = ?1"), + params![ssl], + |r| Ok(row_to_sieve_script(r)), + ) + .unwrap() + .unwrap(); + assert_eq!(gs.name.as_deref(), Some("main")); + assert!(gs.is_active); + assert_eq!(gs.blob.get(), blob); +} + +#[test] +fn file_node_row_to_wire_translates_parent() { + let archive = mem(); + let c = archive.connection(); + let mut res = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + let dir: FileNode = serde_json::from_value(serde_json::json!({ + "id": "D1", "@type": "Directory", "name": "Docs", + "created": "2022-02-02T02:02:02Z" + })) + .unwrap(); + let dirl = insert_file_node(c, &dir, None, &res).unwrap(); + res.to_local + .insert((ObjectType::FileNode, "D1".to_owned()), dirl); + let file: FileNode = serde_json::from_value(serde_json::json!({ + "id": "F1", "@type": "File", "parentId": "D1", "name": "a.bin", + "mediaType": "application/octet-stream", "created": "2022-02-02T02:02:02Z" + })) + .unwrap(); + let blob = intern_blob(c, b"bytes").unwrap(); + let fl = insert_file_node(c, &file, Some(blob), &res).unwrap(); + res.to_target + .insert((ObjectType::FileNode, dirl), "TGTD".to_owned()); + let got = c + .query_row( + &format!("{FILE_NODE_SELECT} WHERE id = ?1"), + params![fl], + |r| Ok(row_to_file_node(r, &res)), + ) + .unwrap() + .unwrap(); + assert_eq!(got.wire.name, "a.bin"); + assert_eq!(got.wire.parent_id, Some(JmapId("TGTD".to_owned()))); + assert_eq!(got.blob.map(BlobId::get), Some(blob)); +} + +// Each @type serializes only the properties the draft defines for it. +#[test] +fn file_node_row_to_wire_emits_only_type_specific_properties() { + let archive = mem(); + let c = archive.connection(); + let mut res = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + let nodes = [ + serde_json::json!({"id": "D1", "@type": "Directory", "name": "Docs", + "role": "documents", "isSubscribed": false, + "created": "2022-02-02T02:02:02Z"}), + serde_json::json!({"id": "F1", "@type": "File", "name": "run.sh", + "blobId": "B1", "mediaType": "text/x-sh", "executable": true, "size": 3, + "created": "2022-02-02T02:02:02Z"}), + serde_json::json!({"id": "L1", "@type": "SymLink", "name": "latest", + "target": ["", "Docs"], "created": "2022-02-02T02:02:02Z"}), + ]; + let blob = intern_blob(c, b"#!\n").unwrap(); + let mut out = Vec::new(); + for (i, n) in nodes.into_iter().enumerate() { + let w: FileNode = serde_json::from_value(n).unwrap(); + let blob_local = matches!(w.kind, NodeKind::File(_)).then_some(blob); + let l = insert_file_node(c, &w, blob_local, &res).unwrap(); + res.to_target + .insert((ObjectType::FileNode, l), format!("T{i}")); + let row = c + .query_row( + &format!("{FILE_NODE_SELECT} WHERE id = ?1"), + params![l], + |r| Ok(row_to_file_node(r, &res)), + ) + .unwrap() + .unwrap(); + out.push(serde_json::to_value(&row.wire).unwrap()); + } + let keys = |v: &Value| { + let mut k: Vec = v.as_object().unwrap().keys().cloned().collect(); + k.sort_unstable(); + k + }; + assert_eq!( + keys(&out[0]), + ["@type", "created", "isSubscribed", "name", "role"] + ); + assert_eq!(out[0]["@type"], "Directory"); + assert_eq!(out[0]["isSubscribed"], false); + assert_eq!( + keys(&out[1]), + ["@type", "created", "executable", "mediaType", "name"] + ); + assert_eq!(out[1]["executable"], true); + assert_eq!(out[1]["mediaType"], "text/x-sh"); + assert_eq!(keys(&out[2]), ["@type", "created", "name", "target"]); + assert_eq!(out[2]["@type"], "SymLink"); + assert_eq!(out[2]["target"], serde_json::json!(["", "Docs"])); +} + +#[test] +fn file_node_rejects_unknown_type() { + let r: Result = serde_json::from_value(serde_json::json!({ + "id": "X1", "@type": "Socket", "name": "s", "created": "2022-02-02T02:02:02Z" + })); + assert!(r.is_err()); +} + +fn one_row(c: &Connection, table: &str) -> i64 { + c.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0)) + .unwrap() +} + +#[test] +fn delta_update_mailbox_in_place_preserves_id() { + let archive = mem(); + let c = archive.connection(); + let empty = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + let m: Mailbox = + serde_json::from_value(serde_json::json!({"id":"M1","name":"Personal"})).unwrap(); + let local = insert_mailbox(c, &m, &empty).unwrap(); + let changed: Mailbox = serde_json::from_value( + serde_json::json!({"id":"M1","name":"PersonalRenamed","role":"archive","sortOrder":4}), + ) + .unwrap(); + assert!( + update_mailbox(c, local, &changed, &empty).unwrap(), + "a real change reports changed=true" + ); + assert!( + !update_mailbox(c, local, &changed, &empty).unwrap(), + "re-applying identical values is a no-op (changed=false), so re-runs converge" + ); + assert_eq!(one_row(c, "mailboxes"), 1); + let (name, role, sort): (String, Option, i64) = c + .query_row( + "SELECT name, role, sort_order FROM mailboxes WHERE id=?1", + params![local], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .unwrap(); + assert_eq!(name, "PersonalRenamed"); + assert_eq!(role.as_deref(), Some("archive")); + assert_eq!(sort, 4); +} + +#[test] +fn delta_update_email_changes_keywords_and_mailboxes_keeps_blob() { + let archive = mem(); + let c = archive.connection(); + let mut res = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + res.to_local + .insert((ObjectType::Mailbox, "MB".to_owned()), 7); + res.to_local + .insert((ObjectType::Mailbox, "MB2".to_owned()), 8); + let email: Email = serde_json::from_value(serde_json::json!({ + "id":"E1","blobId":"B1","receivedAt":"2021-05-04T10:00:00Z", + "mailboxIds":{"MB":true},"keywords":{"$seen":true} + })) + .unwrap(); + let blob = intern_blob(c, b"rfc5322").unwrap(); + let local = insert_email(c, &email, blob, "{}", &res).unwrap(); + + let changed = serde_json::json!({ + "id":"E1","mailboxIds":{"MB2":true},"keywords":{"$seen":true,"$flagged":true} + }); + update_email(c, local, &changed, &res).unwrap(); + assert_eq!(one_row(c, "emails"), 1); + let got = c + .query_row( + &format!("{EMAIL_SELECT} WHERE id=?1"), + params![local], + |row| Ok(row_to_email(row)), + ) + .unwrap() + .unwrap(); + assert_eq!(got.blob.get(), blob, "immutable body blob is untouched"); + assert_eq!( + got.mailboxes + .as_slice() + .iter() + .map(|l| l.get()) + .collect::>(), + vec![8], + "mailbox membership moved" + ); + assert!(got.keywords.contains(&"$flagged".to_owned())); + assert!(got.keywords.contains(&"$seen".to_owned())); +} + +#[test] +fn delta_update_identity_in_place() { + let archive = mem(); + let c = archive.connection(); + let id: Identity = + serde_json::from_value(serde_json::json!({"id":"I1","name":"Old","email":"a@x.test"})) + .unwrap(); + let local = insert_identity(c, &id).unwrap(); + let changed: Identity = serde_json::from_value( + serde_json::json!({"id":"I1","name":"New Name","email":"a@x.test","textSignature":"sig"}), + ) + .unwrap(); + update_identity(c, local, &changed).unwrap(); + assert_eq!(one_row(c, "identities"), 1); + let got = c + .query_row( + &format!("{IDENTITY_SELECT} WHERE id=?1"), + params![local], + |row| Ok(row_to_identity(row)), + ) + .unwrap() + .unwrap(); + assert_eq!(got.name, "New Name"); + assert_eq!(got.text_signature, "sig"); +} + +#[test] +fn delta_update_address_book_in_place() { + let archive = mem(); + let c = archive.connection(); + let ab: AddressBook = + serde_json::from_value(serde_json::json!({"id":"A1","name":"Old"})).unwrap(); + let local = insert_address_book(c, &ab).unwrap(); + let changed: AddressBook = serde_json::from_value( + serde_json::json!({"id":"A1","name":"Renamed","description":"d2","sortOrder":3}), + ) + .unwrap(); + update_address_book(c, local, &changed).unwrap(); + assert_eq!(one_row(c, "address_books"), 1); + let got = c + .query_row( + &format!("{ADDRESS_BOOK_SELECT} WHERE id=?1"), + params![local], + |r| Ok(row_to_address_book(r)), + ) + .unwrap() + .unwrap(); + assert_eq!(got.name, "Renamed"); + assert_eq!(got.description.as_deref(), Some("d2")); +} + +#[test] +fn delta_update_calendar_in_place() { + let archive = mem(); + let c = archive.connection(); + let cal: Calendar = + serde_json::from_value(serde_json::json!({"id":"C1","name":"Old","color":"#000"})).unwrap(); + let local = insert_calendar(c, &cal).unwrap(); + let changed: Calendar = serde_json::from_value( + serde_json::json!({"id":"C1","name":"Renamed","color":"#abcdef","timeZone":"Europe/Rome"}), + ) + .unwrap(); + update_calendar(c, local, &changed).unwrap(); + assert_eq!(one_row(c, "calendars"), 1); + let got = c + .query_row( + &format!("{CALENDAR_SELECT} WHERE id=?1"), + params![local], + |r| Ok(row_to_calendar(r)), + ) + .unwrap() + .unwrap(); + assert_eq!(got.name, "Renamed"); + assert_eq!(got.color.as_deref(), Some("#abcdef")); + assert_eq!(got.time_zone.as_deref(), Some("Europe/Rome")); +} + +#[test] +fn delta_update_participant_identity_in_place() { + let archive = mem(); + let c = archive.connection(); + let pi: ParticipantIdentity = serde_json::from_value( + serde_json::json!({"id":"P1","name":"Old","calendarAddress":"mailto:me@x.test"}), + ) + .unwrap(); + let local = insert_participant_identity(c, &pi).unwrap(); + let changed: ParticipantIdentity = serde_json::from_value( + serde_json::json!({"id":"P1","name":"New","calendarAddress":"mailto:me@x.test"}), + ) + .unwrap(); + update_participant_identity(c, local, &changed).unwrap(); + assert_eq!(one_row(c, "participant_identities"), 1); + let got = c + .query_row( + &format!("{PARTICIPANT_IDENTITY_SELECT} WHERE id=?1"), + params![local], + |r| Ok(row_to_participant_identity(r)), + ) + .unwrap() + .unwrap(); + assert_eq!(got.name, "New"); +} + +#[test] +fn delta_update_sieve_script_swaps_blob_and_active() { + let archive = mem(); + let c = archive.connection(); + let ss: SieveScript = serde_json::from_value( + serde_json::json!({"id":"S1","name":"main","isActive":false,"blobId":"B1"}), + ) + .unwrap(); + let blob1 = intern_blob(c, b"keep;").unwrap(); + let local = insert_sieve_script(c, &ss, blob1).unwrap(); + let blob2 = intern_blob(c, b"discard;").unwrap(); + let changed: SieveScript = serde_json::from_value( + serde_json::json!({"id":"S1","name":"main","isActive":true,"blobId":"B2"}), + ) + .unwrap(); + update_sieve_script(c, local, &changed, blob2).unwrap(); + assert_eq!(one_row(c, "sieve_scripts"), 1); + let got = c + .query_row( + &format!("{SIEVE_SELECT} WHERE id=?1"), + params![local], + |r| Ok(row_to_sieve_script(r)), + ) + .unwrap() + .unwrap(); + assert!(got.is_active); + assert_eq!(got.blob.get(), blob2, "content blob swapped"); +} + +#[test] +fn delta_update_file_node_changes_blob_and_name() { + let archive = mem(); + let c = archive.connection(); + let res = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + let file: FileNode = serde_json::from_value(serde_json::json!({ + "id":"F1","@type":"File","name":"a.bin", + "mediaType":"application/octet-stream","created":"2022-02-02T02:02:02Z" + })) + .unwrap(); + let blob1 = intern_blob(c, b"v1").unwrap(); + let local = insert_file_node(c, &file, Some(blob1), &res).unwrap(); + let blob2 = intern_blob(c, b"v2").unwrap(); + let changed: FileNode = serde_json::from_value(serde_json::json!({ + "id":"F1","@type":"File","name":"renamed.bin", + "mediaType":"text/plain","executable":true,"created":"2022-02-02T02:02:02Z" + })) + .unwrap(); + update_file_node(c, local, &changed, Some(blob2), &res).unwrap(); + assert_eq!(one_row(c, "file_nodes"), 1); + let (name, mt, b, x): (String, Option, Option, i64) = c + .query_row( + "SELECT name, media_type, blob_id, executable FROM file_nodes WHERE id=?1", + params![local], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)), + ) + .unwrap(); + assert_eq!(name, "renamed.bin"); + assert_eq!(mt.as_deref(), Some("text/plain")); + assert_eq!(b, Some(blob2)); + assert_eq!(x, 1); +} + +#[test] +fn delta_update_contact_card_rewrites_data_and_columns() { + let archive = mem(); + let c = archive.connection(); + let res = MapResolver { + to_local: HashMap::from([((ObjectType::AddressBook, "AB".to_owned()), 1)]), + to_target: HashMap::new(), + }; + let card: ContactCard = serde_json::from_value(serde_json::json!({ + "id":"C1","addressBookIds":{"AB":true},"uid":"u-1","name":{"full":"Old"} + })) + .unwrap(); + let mut blobs = FakeBlobs; + let local = insert_contact_card(c, &card, &res, &mut blobs).unwrap(); + let changed: ContactCard = serde_json::from_value(serde_json::json!({ + "id":"C1","addressBookIds":{"AB":true},"uid":"u-1","name":{"full":"New Name"} + })) + .unwrap(); + update_contact_card(c, local, &changed, &res, &mut blobs).unwrap(); + assert_eq!(one_row(c, "contact_cards"), 1); + let (uid, data): (String, String) = c + .query_row( + &format!("{CONTACT_CARD_SELECT} WHERE id=?1"), + params![local], + |r| Ok((r.get(1)?, r.get(3)?)), + ) + .unwrap(); + assert_eq!(uid, "u-1"); + let stored: Value = serde_json::from_str(&data).unwrap(); + assert_eq!(stored["name"]["full"], Value::from("New Name")); +} + +#[test] +fn delta_update_calendar_event_rewrites_data_and_columns() { + let archive = mem(); + let c = archive.connection(); + let res = MapResolver { + to_local: HashMap::from([((ObjectType::Calendar, "CAL".to_owned()), 5)]), + to_target: HashMap::new(), + }; + let ev: CalendarEvent = serde_json::from_value(serde_json::json!({ + "id":"EV1","calendarIds":{"CAL":true},"isDraft":false, + "useDefaultAlerts":false,"title":"Old","@type":"Event" + })) + .unwrap(); + let mut blobs = FakeBlobs; + let local = insert_calendar_event(c, &ev, &res, &mut blobs).unwrap(); + let changed: CalendarEvent = serde_json::from_value(serde_json::json!({ + "id":"EV1","calendarIds":{"CAL":true},"isDraft":true, + "useDefaultAlerts":false,"title":"Rescheduled","@type":"Event" + })) + .unwrap(); + update_calendar_event(c, local, &changed, &res, &mut blobs).unwrap(); + assert_eq!(one_row(c, "calendar_events"), 1); + let (dr, data): (i64, String) = c + .query_row( + &format!("{CALENDAR_EVENT_SELECT} AND id=?1"), + params![local], + |r| Ok((r.get(2)?, r.get(4)?)), + ) + .unwrap(); + assert_eq!(dr, 1, "isDraft column updated"); + let stored: Value = serde_json::from_str(&data).unwrap(); + assert_eq!(stored["title"], Value::from("Rescheduled")); +} diff --git a/src/sync/jmap_map/vacation.rs b/src/sync/jmap_map/vacation.rs new file mode 100644 index 0000000..c528933 --- /dev/null +++ b/src/sync/jmap_map/vacation.rs @@ -0,0 +1,34 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::archive::rows::{StoredVacation, VacationRecord}; +use crate::jmap::wire::vacation_response::VacationResponse; + +impl VacationResponse { + pub(crate) fn record(&self) -> VacationRecord<'_> { + VacationRecord { + is_enabled: self.is_enabled, + from_date: self.from_date.as_deref(), + to_date: self.to_date.as_deref(), + subject: self.subject.as_deref(), + text_body: self.text_body.as_deref(), + html_body: self.html_body.as_deref(), + } + } +} + +impl From for VacationResponse { + fn from(stored: StoredVacation) -> VacationResponse { + VacationResponse { + is_enabled: stored.is_enabled, + from_date: stored.from_date, + to_date: stored.to_date, + subject: stored.subject, + text_body: stored.text_body, + html_body: stored.html_body, + } + } +} diff --git a/src/sync/keys.rs b/src/sync/keys.rs index 6d80604..3f68ccd 100644 --- a/src/sync/keys.rs +++ b/src/sync/keys.rs @@ -4,173 +4,247 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ -use serde_json::{Value, json}; +use std::fmt; +use std::str; -pub fn normalize_message_id(raw: &str) -> String { - let t = raw.trim(); - let t = t.strip_prefix('<').unwrap_or(t); - let t = t.strip_suffix('>').unwrap_or(t); - t.trim().to_lowercase() -} +use serde::ser::{Serialize, Serializer}; +use serde_json::Value; -pub fn fold_name(name: &str) -> String { - name.trim().to_lowercase() -} +const FIELD_SEPARATOR: &[u8] = b"\x1f"; +const MESSAGE_ID_SEPARATOR: &str = "\x1f"; +const KEY_LEN: usize = 32; -pub fn blake3_fields(fields: &[&str]) -> [u8; 32] { - let mut hasher = blake3::Hasher::new(); - for (i, f) in fields.iter().enumerate() { - if i > 0 { - hasher.update(b"\x1f"); +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub struct Key32(pub [u8; KEY_LEN]); + +impl Key32 { + pub const ZERO: Key32 = Key32([0; KEY_LEN]); + + pub fn of_fields(fields: &[&str]) -> Key32 { + let mut hasher = blake3::Hasher::new(); + let mut fields = fields.iter(); + if let Some(first) = fields.next() { + hasher.update(first.as_bytes()); } - hasher.update(f.as_bytes()); + for field in fields { + hasher.update(FIELD_SEPARATOR); + hasher.update(field.as_bytes()); + } + Key32(*hasher.finalize().as_bytes()) + } + + pub fn identity(name: &str, email: &str) -> Key32 { + Key32::of_fields(&[name, &email.to_lowercase()]) + } + + pub fn participant_identity(calendar_address: &str, name: &str) -> Key32 { + Key32::of_fields(&[&calendar_address.to_lowercase(), name]) + } + + pub fn from_hex(text: &str) -> Option { + let bytes = text.as_bytes(); + if bytes.len() != KEY_LEN * 2 { + return None; + } + let mut out = [0u8; KEY_LEN]; + for (slot, pair) in out.iter_mut().zip(bytes.as_chunks::<2>().0) { + *slot = u8::from_str_radix(str::from_utf8(pair).ok()?, 16).ok()?; + } + Some(Key32(out)) } - *hasher.finalize().as_bytes() } -pub fn blake3_bytes(bytes: &[u8]) -> [u8; 32] { - *blake3::hash(bytes).as_bytes() +impl fmt::Display for Key32 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for byte in &self.0 { + write!(f, "{byte:02x}")?; + } + Ok(()) + } } -pub fn identity_key(name: &str, email: &str) -> [u8; 32] { - blake3_fields(&[name, &email.to_lowercase()]) +impl Serialize for Key32 { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(self) + } } -pub fn participant_identity_key(calendar_address: &str, name: &str) -> [u8; 32] { - blake3_fields(&[&calendar_address.to_lowercase(), name]) +#[derive(Debug, Clone, Copy)] +pub struct MessageId<'a>(pub &'a str); + +impl MessageId<'_> { + pub fn normalized(self) -> String { + let t = self.0.trim(); + let t = t.strip_prefix('<').unwrap_or(t); + let t = t.strip_suffix('>').unwrap_or(t); + t.trim().to_lowercase() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct FoldedName(String); + +impl FoldedName { + pub fn new(name: &str) -> FoldedName { + FoldedName(name.trim().to_lowercase()) + } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum EmailKey { MessageId(String), - Fallback([u8; 32]), + Fallback(Key32), } #[derive(Debug, Clone, PartialEq, Eq)] pub struct EmailIndex { pub mids: Vec, - - pub fb: [u8; 32], + pub fb: Key32, } -fn join_addrs(addrs: &[String]) -> String { - addrs - .iter() - .map(|a| a.to_lowercase()) - .collect::>() - .join(",") +#[derive(serde::Serialize)] +struct StoredIndex<'a> { + f: Key32, + m: &'a [String], } -pub fn email_index( - message_ids: &[String], - from: &[String], - subject: &str, - sent_at: &str, - to: &[String], -) -> EmailIndex { - let mut mids: Vec = message_ids - .iter() - .map(|m| normalize_message_id(m)) - .filter(|s| !s.is_empty()) - .collect(); - mids.sort(); - mids.dedup(); - let fb = blake3_fields(&[&join_addrs(from), subject, sent_at, &join_addrs(to)]); - EmailIndex { mids, fb } -} +impl EmailIndex { + pub fn new( + message_ids: &[String], + from: &[String], + subject: &str, + sent_at: &str, + to: &[String], + ) -> EmailIndex { + let mut mids: Vec = message_ids + .iter() + .map(|m| MessageId(m).normalized()) + .filter(|s| !s.is_empty()) + .collect(); + mids.sort_unstable(); + mids.dedup(); + let fb = Key32::of_fields(&[ + &EmailIndex::address_list(from), + subject, + sent_at, + &EmailIndex::address_list(to), + ]); + EmailIndex { mids, fb } + } -pub fn index_to_json(idx: &EmailIndex) -> String { - json!({ "m": idx.mids, "f": hex(&idx.fb) }).to_string() -} + fn address_list(addrs: &[String]) -> String { + let mut out = String::with_capacity(addrs.iter().map(|a| a.len() + 1).sum()); + for (i, addr) in addrs.iter().enumerate() { + if i > 0 { + out.push(','); + } + out.push_str(&addr.to_lowercase()); + } + out + } -pub fn index_from_json(text: &str) -> EmailIndex { - let v: Value = serde_json::from_str(text).unwrap_or(Value::Null); - let mids = v - .get("m") - .and_then(Value::as_array) - .map(|a| { - a.iter() - .filter_map(|x| x.as_str().map(str::to_owned)) - .collect() + pub fn to_json(&self) -> String { + serde_json::to_string(&StoredIndex { + f: self.fb, + m: &self.mids, }) - .unwrap_or_default(); - let fb = v - .get("f") - .and_then(Value::as_str) - .and_then(unhex) - .unwrap_or([0u8; 32]); - EmailIndex { mids, fb } -} - -fn hex(bytes: &[u8; 32]) -> String { - let mut s = String::with_capacity(64); - for b in bytes { - use std::fmt::Write; - let _ = write!(s, "{b:02x}"); + .unwrap_or_default() } - s -} -fn unhex(s: &str) -> Option<[u8; 32]> { - if s.len() != 64 { - return None; + pub fn from_json(text: &str) -> EmailIndex { + let v: Value = serde_json::from_str(text).unwrap_or(Value::Null); + let mids = v + .get("m") + .and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(|x| x.as_str().map(str::to_owned)) + .collect() + }) + .unwrap_or_default(); + let fb = v + .get("f") + .and_then(Value::as_str) + .and_then(Key32::from_hex) + .unwrap_or(Key32::ZERO); + EmailIndex { mids, fb } } - let mut out = [0u8; 32]; - for (i, slot) in out.iter_mut().enumerate() { - *slot = u8::from_str_radix(&s[i * 2..i * 2 + 2], 16).ok()?; - } - Some(out) -} -pub fn email_key(idx: &EmailIndex) -> EmailKey { - if idx.mids.is_empty() { - EmailKey::Fallback(idx.fb) - } else { - EmailKey::MessageId(idx.mids.join("\x1f")) + pub fn key(&self) -> EmailKey { + if self.mids.is_empty() { + EmailKey::Fallback(self.fb) + } else { + EmailKey::MessageId(self.mids.join(MESSAGE_ID_SEPARATOR)) + } } } -pub fn email_keys(indices: &[EmailIndex]) -> Vec { - indices.iter().map(email_key).collect() -} - #[cfg(test)] mod tests { use super::*; + // address_list matches the old lowercase-and-join, empty entries included + #[test] + fn address_list_matches_the_old_join() { + let reference = |addrs: &[String]| { + addrs + .iter() + .map(|a| a.to_lowercase()) + .collect::>() + .join(",") + }; + let cases: [&[&str]; 7] = [ + &[], + &[""], + &["", ""], + &["", "B@x"], + &["a@X", ""], + &["A@x", "", "c@Y"], + &["ΣΑΣ@x"], + ]; + for case in cases { + let addrs: Vec = case.iter().map(|a| (*a).to_owned()).collect(); + assert_eq!( + EmailIndex::address_list(&addrs), + reference(&addrs), + "{case:?}" + ); + } + } + #[test] fn message_id_normalization_strips_brackets_and_lowercases() { - assert_eq!(normalize_message_id(" "), "abc@host.com"); - assert_eq!(normalize_message_id("plain@x"), "plain@x"); + assert_eq!(MessageId(" ").normalized(), "abc@host.com"); + assert_eq!(MessageId("plain@x").normalized(), "plain@x"); } #[test] fn name_fold_is_case_insensitive() { - assert_eq!(fold_name(" Sales "), fold_name("sales")); - assert_ne!(fold_name("Sales"), fold_name("sale")); + assert_eq!(FoldedName::new(" Sales "), FoldedName::new("sales")); + assert_ne!(FoldedName::new("Sales"), FoldedName::new("sale")); } #[test] fn identity_collapse_ignores_signature() { assert_eq!( - identity_key("Alice", "A@x.test"), - identity_key("Alice", "a@x.test") + Key32::identity("Alice", "A@x.test"), + Key32::identity("Alice", "a@x.test") ); } #[test] fn participant_identity_key_lowercases_address_only() { - let a = participant_identity_key("MailTo:User@Host", "Bob"); - let b = participant_identity_key("mailto:user@host", "Bob"); + let a = Key32::participant_identity("MailTo:User@Host", "Bob"); + let b = Key32::participant_identity("mailto:user@host", "Bob"); assert_eq!(a, b); - assert_ne!(a, participant_identity_key("mailto:user@host", "bob")); + assert_ne!(a, Key32::participant_identity("mailto:user@host", "bob")); } fn idx(mids: &[&str], from: &[&str], subj: &str, sent: &str, to: &[&str]) -> EmailIndex { let m: Vec = mids.iter().map(|s| (*s).to_owned()).collect(); let f: Vec = from.iter().map(|s| (*s).to_owned()).collect(); let t: Vec = to.iter().map(|s| (*s).to_owned()).collect(); - email_index(&m, &f, subj, sent, &t) + EmailIndex::new(&m, &f, subj, sent, &t) } #[test] @@ -183,7 +257,7 @@ mod tests { #[test] fn unique_single_message_id_uses_message_id_key() { - let keys = email_keys(&[ + let keys: Vec = [ idx( &[""], &["a@x"], @@ -198,14 +272,17 @@ mod tests { "2020-01-02T00:00:00Z", &["b@y"], ), - ]); + ] + .iter() + .map(EmailIndex::key) + .collect(); assert_eq!(keys[0], EmailKey::MessageId("m1@h".to_owned())); assert_eq!(keys[1], EmailKey::MessageId("m2@h".to_owned())); } #[test] fn shared_message_id_matches_regardless_of_other_fields() { - let keys = email_keys(&[ + let keys: Vec = [ idx( &[""], &["a@x"], @@ -220,14 +297,17 @@ mod tests { "2020-01-02T00:00:00Z", &["d@y"], ), - ]); + ] + .iter() + .map(EmailIndex::key) + .collect(); assert_eq!(keys[0], EmailKey::MessageId("dup@h".to_owned())); assert_eq!(keys[0], keys[1], "same Message-ID is the same message"); } #[test] fn only_absent_message_id_falls_back_multi_is_still_message_id() { - let keys = email_keys(&[ + let keys: Vec = [ idx(&[], &["a@x"], "S", "2020-01-01T00:00:00Z", &["b@y"]), idx( &["", ""], @@ -236,7 +316,10 @@ mod tests { "2020-01-01T00:00:00Z", &["b@y"], ), - ]); + ] + .iter() + .map(EmailIndex::key) + .collect(); assert!(matches!(keys[0], EmailKey::Fallback(_))); assert_eq!(keys[1], EmailKey::MessageId("a@h\u{1f}b@h".to_owned())); } @@ -253,16 +336,31 @@ mod tests { #[test] fn index_json_roundtrips() { let i = idx(&[""], &["a@x"], "S", "2020-01-01T00:00:00Z", &["b@y"]); - let j = index_to_json(&i); - let back = index_from_json(&j); + let j = i.to_json(); + let back = EmailIndex::from_json(&j); assert_eq!(i, back); - assert_eq!(email_keys(&[i]), email_keys(&[back])); + assert_eq!(i.key(), back.key()); } #[test] fn malformed_stored_json_degrades_to_empty_fallback() { - let back = index_from_json("{}"); + let back = EmailIndex::from_json("{}"); assert!(back.mids.is_empty()); - assert_eq!(back.fb, [0u8; 32]); + assert_eq!(back.fb, Key32::ZERO); + } + + // A stored fallback key with multibyte text degrades to the zero key instead of panicking. + #[test] + fn non_ascii_fallback_hex_is_rejected_without_panicking() { + let text = format!("{{\"f\":\"a{}a\",\"m\":[]}}", "\u{e9}".repeat(31)); + assert_eq!(EmailIndex::from_json(&text).fb, Key32::ZERO); + } + + #[test] + fn stored_json_layout_is_unchanged() { + let i = idx(&[""], &[], "", "", &[]); + let hex = i.fb.to_string(); + assert_eq!(i.to_json(), format!("{{\"f\":\"{hex}\",\"m\":[\"m@h\"]}}")); + assert_eq!(Key32::from_hex(&hex), Some(i.fb)); } } diff --git a/src/sync/keywords.rs b/src/sync/keywords.rs new file mode 100644 index 0000000..33380af --- /dev/null +++ b/src/sync/keywords.rs @@ -0,0 +1,196 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +const MAX_KEYWORD_LEN: usize = 255; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SystemKeyword { + Seen, + Draft, + Flagged, + Answered, + Forwarded, + Deleted, +} + +impl SystemKeyword { + pub fn as_str(self) -> &'static str { + match self { + SystemKeyword::Seen => "$seen", + SystemKeyword::Draft => "$draft", + SystemKeyword::Flagged => "$flagged", + SystemKeyword::Answered => "$answered", + SystemKeyword::Forwarded => "$forwarded", + SystemKeyword::Deleted => "$deleted", + } + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Keywords(Vec>); + +impl Keywords { + pub fn new() -> Keywords { + Keywords::default() + } + + pub fn with_capacity(capacity: usize) -> Keywords { + Keywords(Vec::with_capacity(capacity)) + } + + pub fn contains(&self, keyword: &str) -> bool { + self.0.iter().any(|k| k.eq_ignore_ascii_case(keyword)) + } + + pub fn push(&mut self, keyword: &str) { + if !self.contains(keyword) { + self.0.push(keyword.into()); + } + } + + pub fn insert(&mut self, keyword: String) -> bool { + let fresh = !self.contains(&keyword); + if fresh { + self.0.push(keyword.into_boxed_str()); + } + fresh + } + + pub fn push_system(&mut self, keyword: SystemKeyword) { + self.push(keyword.as_str()); + } + + pub fn from_phrase(phrase: &str) -> Option { + let mut keyword = String::with_capacity(phrase.len().min(MAX_KEYWORD_LEN)); + let mut separator = false; + for ch in phrase.chars() { + if ch.is_ascii_graphic() + && !matches!(ch, '(' | ')' | '{' | ']' | '%' | '*' | '"' | '\\') + && !(ch == '$' && keyword.is_empty()) + { + let sep = separator && !keyword.is_empty(); + if keyword.len() + usize::from(sep) >= MAX_KEYWORD_LEN { + break; + } + if sep { + keyword.push('_'); + } + keyword.push(ch.to_ascii_lowercase()); + separator = false; + } else { + separator = true; + } + } + (!keyword.is_empty()).then_some(keyword) + } + + pub fn push_category(&mut self, phrase: &str) { + if let Some(keyword) = Keywords::from_phrase(phrase) + && !self.contains(&keyword) + { + self.0.push(keyword.into_boxed_str()); + } + } + + pub fn sort(&mut self) { + self.0.sort_unstable(); + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn iter(&self) -> impl Iterator { + self.0.iter().map(AsRef::as_ref) + } + + pub fn to_json(&self) -> String { + serde_json::to_string(&self.0).unwrap_or_else(|_| "[]".to_owned()) + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct MessageFlags { + pub read: bool, + pub draft: bool, + pub flagged: bool, +} + +impl MessageFlags { + pub fn keywords(self) -> Keywords { + let mut keywords = Keywords::with_capacity(3); + for (set, keyword) in [ + (self.read, SystemKeyword::Seen), + (self.draft, SystemKeyword::Draft), + (self.flagged, SystemKeyword::Flagged), + ] { + if set { + keywords.push_system(keyword); + } + } + keywords + } +} + +#[cfg(test)] +mod tests { + use serde_json::Value; + + use super::*; + + #[test] + fn push_keeps_order_and_dedups_case_insensitively() { + let mut k = Keywords::new(); + k.push("$seen"); + k.push("Work"); + k.push("$SEEN"); + k.push("work"); + k.push_system(SystemKeyword::Deleted); + assert_eq!(k.iter().collect::>(), ["$seen", "Work", "$deleted"]); + } + + #[test] + fn sort_orders_bytewise() { + let mut k = Keywords::new(); + for kw in ["$seen", "$answered", "$flagged"] { + k.push(kw); + } + k.sort(); + assert_eq!(k.to_json(), r#"["$answered","$flagged","$seen"]"#); + } + + #[test] + fn json_matches_the_value_array_encoding() { + let mut k = Keywords::new(); + k.push("a\"b"); + k.push("ü"); + let reference = + Value::Array(k.iter().map(|s| Value::String(s.to_owned())).collect()).to_string(); + assert_eq!(k.to_json(), reference); + assert_eq!(Keywords::new().to_json(), "[]"); + } + + #[test] + fn message_flags_then_categories() { + let mut k = MessageFlags { + read: true, + draft: false, + flagged: true, + } + .keywords(); + k.push_category("Review / Legal"); + k.push_category("review / legal"); + k.push_category(" "); + assert_eq!( + k.iter().collect::>(), + ["$seen", "$flagged", "review_/_legal"] + ); + } +} diff --git a/src/sync/mod.rs b/src/sync/mod.rs index e01de3b..643799e 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -4,71 +4,49 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +pub mod calcard; +pub mod dates; +pub mod diff; pub mod emailmeta; -pub mod export; -pub mod import_dav; -pub mod import_exchange_ews; -pub mod import_exchange_graph; -pub mod import_imap; -pub mod import_jmap; -pub mod import_maildir; -pub mod import_managesieve; -pub mod import_takeout; +pub(crate) mod jmap_map; pub mod keys; -pub mod prune; +pub mod keywords; +pub mod pool; +pub mod run; +pub mod tree; +pub mod vanish; use std::path::PathBuf; -use rusqlite::Connection; - -use crate::db; use crate::error::Error; - -pub(crate) fn table_name(ty: ObjectType) -> &'static str { - match ty { - ObjectType::Mailbox => "mailboxes", - ObjectType::Email => "emails", - ObjectType::Identity => "identities", - ObjectType::SieveScript => "sieve_scripts", - ObjectType::AddressBook => "address_books", - ObjectType::ContactCard => "contact_cards", - ObjectType::Calendar => "calendars", - ObjectType::CalendarEvent => "calendar_events", - ObjectType::ParticipantIdentity => "participant_identities", - ObjectType::FileNode => "file_nodes", - } -} -use crate::jmap::account::AccountSelector; -use crate::jmap::http::{Auth, HttpClient, RetryPolicy}; use crate::logging::Logger; +use crate::net::http::{ClientOptions, RetryStats, TlsPolicy}; +use crate::net::retry::RetryPolicy; use crate::types::ObjectType; -pub struct CommonConfig { +pub struct RunOptions { pub archive: PathBuf, pub threads: usize, - pub dry_run: bool, + pub mode: RunMode, pub max_retries: u32, - pub allow_invalid_certs: bool, + pub tls: TlsPolicy, pub logger: Logger, } -pub struct ConnectConfig { - pub url: String, - pub auth: Auth, - pub account: AccountSelector, +impl RunOptions { + pub fn client_options(&self) -> ClientOptions { + ClientOptions { + retry: RetryPolicy::new(self.max_retries), + tls: self.tls, + logger: self.logger, + } + } } -pub struct ImportConfig { - pub connect: ConnectConfig, - pub objects: Option>, - pub allow_source_change: bool, -} - -pub struct ExportConfig { - pub connect: ConnectConfig, - pub objects: Option>, - pub prune: bool, - pub yes: bool, +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RunMode { + Apply, + DryRun, } #[derive(Debug, Default, Clone)] @@ -81,17 +59,103 @@ pub struct TypeCounts { pub failed: u64, } +impl TypeCounts { + pub fn add(&mut self, other: &TypeCounts) { + self.created += other.created; + self.fetched += other.fetched; + self.updated += other.updated; + self.deleted += other.deleted; + self.skipped += other.skipped; + self.failed += other.failed; + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum SummaryType { + Object(ObjectType), + VacationResponse, +} + +impl SummaryType { + pub const fn label(self) -> &'static str { + match self { + SummaryType::Object(ty) => ty.label(), + SummaryType::VacationResponse => "Vacation response", + } + } + + pub const fn noun(self) -> &'static str { + match self { + SummaryType::Object(ty) => ty.noun(), + SummaryType::VacationResponse => "vacation response", + } + } +} + +impl From for SummaryType { + fn from(ty: ObjectType) -> SummaryType { + SummaryType::Object(ty) + } +} + +impl PartialEq for SummaryType { + fn eq(&self, other: &ObjectType) -> bool { + *self == SummaryType::Object(*other) + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct PlanCounts { + pub new: u64, + pub vanished: u64, + pub present: u64, +} + #[derive(Debug, Default, Clone)] pub struct Summary { - pub per_type: Vec<(&'static str, TypeCounts)>, + pub per_type: Vec<(SummaryType, TypeCounts)>, + pub plan: Vec<(SummaryType, PlanCounts)>, pub retries_observed: u64, pub retry_after_sleeps: u64, } impl Summary { + pub fn counts(&mut self, ty: impl Into) -> &mut TypeCounts { + Summary::slot(&mut self.per_type, ty.into()) + } + + pub fn get(&self, ty: impl Into) -> Option<&TypeCounts> { + let ty = ty.into(); + self.per_type + .iter() + .find_map(|(t, counts)| (*t == ty).then_some(counts)) + } + + pub fn plan(&mut self, ty: impl Into) -> &mut PlanCounts { + Summary::slot(&mut self.plan, ty.into()) + } + + fn slot(rows: &mut Vec<(SummaryType, T)>, ty: SummaryType) -> &mut T { + let index = rows.iter().position(|(t, _)| *t == ty); + let (_, value) = match index { + Some(index) => &mut rows[index], + None => rows.push_mut((ty, T::default())), + }; + value + } + + pub fn set_retry_stats(&mut self, stats: RetryStats) { + self.retries_observed = stats.retries_observed; + self.retry_after_sleeps = stats.retry_after_sleeps; + } + pub fn any_failed(&self) -> bool { self.per_type.iter().any(|(_, c)| c.failed > 0) } + + pub fn failed(&self) -> u64 { + self.per_type.iter().map(|(_, c)| c.failed).sum() + } } #[derive(Debug)] @@ -101,16 +165,10 @@ pub struct RunOutcome { } impl RunOutcome { - pub fn from_result(result: Result) -> RunOutcome { - match result { - Ok(summary) => RunOutcome { - summary, - error: None, - }, - Err(error) => RunOutcome { - summary: Summary::default(), - error: Some(error), - }, + pub fn failed(error: Error) -> RunOutcome { + RunOutcome { + summary: Summary::default(), + error: Some(error), } } @@ -122,52 +180,89 @@ impl RunOutcome { } } -pub struct Context { - pub conn: Connection, - pub client: HttpClient, - pub common: CommonConfig, -} - -impl Context { - pub fn open(common: CommonConfig, connect: &ConnectConfig) -> Result { - let conn = db::init::open(&common.archive)?; - let client = HttpClient::new( - connect.auth.clone(), - RetryPolicy::new(common.max_retries), - common.allow_invalid_certs, - ); - Ok(Context { - conn, - client, - common, - }) - } - - pub fn dry_run(&self) -> bool { - self.common.dry_run - } -} - #[cfg(test)] mod tests { use super::*; + use crate::error::ExitStatus; #[test] fn an_aborted_run_keeps_its_error_and_its_partial_counts() { let mut summary = Summary::default(); - summary.per_type.push(("Mailbox", TypeCounts::default())); + summary.counts(ObjectType::Mailbox); let outcome = RunOutcome { summary, error: Some(Error::Connection("http status 404".to_owned())), }; assert_eq!(outcome.summary.per_type.len(), 1); let err = outcome.into_result().expect_err("aborted"); - assert_eq!(err.exit_code(), 2); + assert_eq!(err.exit_status(), ExitStatus::Connection); + } + + #[test] + fn counts_are_insertion_ordered_and_reused() { + let mut summary = Summary::default(); + summary.counts(ObjectType::Mailbox).created += 1; + summary.counts(ObjectType::Email).failed += 2; + summary.counts(SummaryType::VacationResponse).fetched += 1; + summary.counts(ObjectType::Mailbox).created += 1; + let types: Vec = summary.per_type.iter().map(|(t, _)| *t).collect(); + assert_eq!( + types, + [ + SummaryType::Object(ObjectType::Mailbox), + SummaryType::Object(ObjectType::Email), + SummaryType::VacationResponse + ] + ); + assert_eq!(summary.get(ObjectType::Mailbox).map(|c| c.created), Some(2)); + assert!(summary.get(ObjectType::Calendar).is_none()); + assert!(summary.any_failed()); + assert_eq!(summary.failed(), 2); + summary.set_retry_stats(RetryStats { + retries_observed: 4, + retry_after_sleeps: 1, + }); + assert_eq!( + (summary.retries_observed, summary.retry_after_sleeps), + (4, 1) + ); + } + + // Plan rows are kept apart from the run counts, one row per type. + #[test] + fn plan_rows_are_separate_from_counts() { + let mut summary = Summary::default(); + summary.plan(ObjectType::Email).new = 3; + summary.plan(ObjectType::Email).present += 2; + assert!(summary.per_type.is_empty()); + assert_eq!( + summary.plan, + [( + SummaryType::Object(ObjectType::Email), + PlanCounts { + new: 3, + vanished: 0, + present: 2 + } + )] + ); + } + + // Labels and nouns come from the object type, with the vacation response added. + #[test] + fn summary_type_vocabulary() { + let event = SummaryType::from(ObjectType::CalendarEvent); + assert_eq!(event.label(), "Calendar events"); + assert_eq!(event.noun(), "calendar event"); + assert_eq!(SummaryType::VacationResponse.label(), "Vacation response"); + assert_eq!(SummaryType::VacationResponse.noun(), "vacation response"); + assert!(event == ObjectType::CalendarEvent); + assert!(SummaryType::VacationResponse != ObjectType::SieveScript); } #[test] fn a_result_without_a_native_outcome_reports_nothing_on_abort() { - let outcome = RunOutcome::from_result(Err(Error::Partial("one object".to_owned()))); + let outcome = RunOutcome::failed(Error::Partial("one object".to_owned())); assert!(outcome.summary.per_type.is_empty()); assert!(outcome.error.is_some()); } diff --git a/src/sync/import_jmap/pool.rs b/src/sync/pool.rs similarity index 77% rename from src/sync/import_jmap/pool.rs rename to src/sync/pool.rs index bb58a12..f19df30 100644 --- a/src/sync/import_jmap/pool.rs +++ b/src/sync/pool.rs @@ -8,18 +8,6 @@ use std::thread::JoinHandle; use crossbeam_channel::{Receiver, Sender, bounded, unbounded}; -use crate::jmap::session::Limits; - -pub fn effective_workers(requested: usize, limits: &Limits, upload: bool) -> usize { - let server_cap = if upload { - limits.max_concurrent_upload - } else { - limits.max_concurrent_requests - }; - let cap = server_cap.max(1).min(usize::MAX as u64) as usize; - requested.clamp(1, cap) -} - pub struct Pool where J: Send + 'static, @@ -101,27 +89,6 @@ where mod tests { use super::*; - fn limits(req: u64, up: u64) -> Limits { - Limits { - max_objects_in_get: 500, - max_objects_in_set: 500, - max_calls_in_request: 16, - max_concurrent_requests: req, - max_concurrent_upload: up, - max_size_request: 10_000_000, - max_size_upload: 50_000_000, - } - } - - #[test] - fn worker_count_is_clamped_to_server_limits() { - let l = limits(4, 2); - assert_eq!(effective_workers(16, &l, false), 4); - assert_eq!(effective_workers(16, &l, true), 2); - assert_eq!(effective_workers(1, &l, false), 1); - assert_eq!(effective_workers(0, &l, false), 1); - } - #[test] fn pool_processes_all_jobs_across_workers() { let pool: Pool = Pool::new(4, |n| n * n); diff --git a/src/sync/prune.rs b/src/sync/prune.rs deleted file mode 100644 index 61eedaf..0000000 --- a/src/sync/prune.rs +++ /dev/null @@ -1,124 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - */ - -use std::collections::{HashMap, HashSet}; - -#[derive(Debug, Clone)] -pub struct TargetObj { - pub id: String, - pub matched: bool, - pub protected: bool, - pub may_delete: bool, - pub parent: Option, -} - -fn protected_with_ancestors(objs: &[TargetObj]) -> HashSet { - let by_id: HashMap<&str, &TargetObj> = objs.iter().map(|o| (o.id.as_str(), o)).collect(); - let mut protected: HashSet = HashSet::new(); - for o in objs { - if o.protected { - let mut cur = Some(o.id.clone()); - while let Some(id) = cur { - if !protected.insert(id.clone()) { - break; - } - cur = by_id - .get(id.as_str()) - .and_then(|x| x.parent.clone()) - .filter(|p| by_id.contains_key(p.as_str())); - } - } - } - protected -} - -fn depth(id: &str, by_id: &HashMap<&str, &TargetObj>) -> usize { - let mut d = 0; - let mut cur = by_id.get(id).and_then(|o| o.parent.clone()); - let mut seen: HashSet = HashSet::new(); - while let Some(p) = cur { - if !seen.insert(p.clone()) || !by_id.contains_key(p.as_str()) { - break; - } - d += 1; - cur = by_id.get(p.as_str()).and_then(|o| o.parent.clone()); - } - d -} - -pub fn candidates(objs: &[TargetObj], tree: bool) -> Vec { - let protected = protected_with_ancestors(objs); - let by_id: HashMap<&str, &TargetObj> = objs.iter().map(|o| (o.id.as_str(), o)).collect(); - let mut out: Vec<&TargetObj> = objs - .iter() - .filter(|o| !o.matched && o.may_delete && !protected.contains(&o.id)) - .collect(); - if tree { - out.sort_by(|a, b| { - depth(&b.id, &by_id) - .cmp(&depth(&a.id, &by_id)) - .then_with(|| a.id.cmp(&b.id)) - }); - } else { - out.sort_by(|a, b| a.id.cmp(&b.id)); - } - out.into_iter().map(|o| o.id.clone()).collect() -} - -#[cfg(test)] -mod tests { - use super::*; - - fn o(id: &str, matched: bool, protected: bool, parent: Option<&str>) -> TargetObj { - TargetObj { - id: id.to_owned(), - matched, - protected, - may_delete: true, - parent: parent.map(str::to_owned), - } - } - - #[test] - fn unmatched_unprotected_are_candidates() { - let objs = vec![ - o("a", true, false, None), - o("b", false, false, None), - o("c", false, true, None), - ]; - assert_eq!(candidates(&objs, false), vec!["b".to_owned()]); - } - - #[test] - fn protected_excludes_ancestors() { - let objs = vec![ - o("root", false, false, None), - o("mid", false, false, Some("root")), - o("leaf", false, true, Some("mid")), - ]; - assert!(candidates(&objs, true).is_empty()); - } - - #[test] - fn may_delete_false_is_skipped() { - let mut x = o("x", false, false, None); - x.may_delete = false; - assert!(candidates(&[x], false).is_empty()); - } - - #[test] - fn tree_destroy_is_leaf_first() { - let objs = vec![ - o("root", false, false, None), - o("child", false, false, Some("root")), - o("grand", false, false, Some("child")), - ]; - assert_eq!( - candidates(&objs, true), - vec!["grand".to_owned(), "child".to_owned(), "root".to_owned()] - ); - } -} diff --git a/src/sync/run.rs b/src/sync/run.rs new file mode 100644 index 0000000..6080214 --- /dev/null +++ b/src/sync/run.rs @@ -0,0 +1,429 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::archive::sources::{SourceChange, SourceKey, SourceKind}; +use crate::archive::{Archive, SourceId, Store}; +use crate::error::Error; +use crate::net::http::RetryStats; +use crate::sync::{RunMode, RunOptions, RunOutcome, Summary, SummaryType, TypeCounts}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceClaim { + Registered(SourceId), + DryRun(Option), +} + +impl SourceClaim { + pub fn id(&self) -> Option { + match *self { + SourceClaim::Registered(id) => Some(id), + SourceClaim::DryRun(id) => id, + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct SourceLabel<'a> { + pub name: Option<&'a str>, + pub username: &'a str, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GcFailure { + Abort, + Warn, + WarnAsPartial, +} + +pub struct ImportRun { + archive: Archive, + claim: SourceClaim, + options: RunOptions, + summary: Summary, +} + +impl ImportRun { + pub fn start( + options: RunOptions, + key: &SourceKey<'_>, + label: SourceLabel<'_>, + change: SourceChange, + ) -> Result { + let archive = Archive::open(&options.archive)?; + ImportRun::start_on(archive, options, key, label, change) + } + + pub fn start_on( + archive: Archive, + options: RunOptions, + key: &SourceKey<'_>, + label: SourceLabel<'_>, + change: SourceChange, + ) -> Result { + let sources = archive.store().sources(); + let sql_error = |e| ImportRun::source_sql_error(key.kind, e); + if change == SourceChange::Reject + && let Some((url, account)) = sources.conflicting(key).map_err(sql_error)? + { + return Err(ImportRun::source_changed(key.kind, &url, &account)); + } + let claim = match options.mode { + RunMode::Apply => SourceClaim::Registered( + sources + .upsert(key, label.name, label.username) + .map_err(sql_error)?, + ), + RunMode::DryRun => SourceClaim::DryRun(sources.find(key).map_err(sql_error)?), + }; + Ok(ImportRun { + archive, + claim, + options, + summary: Summary::default(), + }) + } + + #[cold] + fn source_sql_error(kind: SourceKind, e: rusqlite::Error) -> Error { + match kind { + SourceKind::Jmap | SourceKind::CalDav | SourceKind::CardDav | SourceKind::WebDav => { + Error::partial(e) + } + SourceKind::Imap + | SourceKind::Maildir + | SourceKind::ManageSieve + | SourceKind::Takeout + | SourceKind::ExchangeEws + | SourceKind::ExchangeGraph => Error::from(e), + } + } + + #[cold] + fn source_changed(kind: SourceKind, url: &str, account: &str) -> Error { + Error::SourceChange(match kind { + SourceKind::Imap + | SourceKind::Maildir + | SourceKind::ManageSieve + | SourceKind::Takeout => { + format!( + "archive already records {kind} source {url} / {account}; \ + re-run with --allow-source-change or use a fresh archive" + ) + } + SourceKind::CalDav | SourceKind::CardDav | SourceKind::WebDav => format!( + "archive already records {kind} source ({url}, account {account}); \ + pass --allow-source-change to import a different account, \ + or --replace-source to drop the recorded one" + ), + SourceKind::Jmap => format!( + "archive already records JMAP source ({url}, account {account}); \ + pass --allow-source-change to import a different account" + ), + SourceKind::ExchangeEws | SourceKind::ExchangeGraph => format!( + "archive already records {kind} source ({url}, account {account}); \ + pass --allow-source-change to import a different account" + ), + }) + } + + pub fn archive(&mut self) -> &mut Archive { + &mut self.archive + } + + pub fn store(&self) -> Store<'_> { + self.archive.store() + } + + pub fn claim(&self) -> &SourceClaim { + &self.claim + } + + pub fn options(&self) -> &RunOptions { + &self.options + } + + pub fn summary(&mut self) -> &mut Summary { + &mut self.summary + } + + pub fn counts(&mut self, ty: impl Into) -> &mut TypeCounts { + self.summary.counts(ty) + } + + pub fn finish(mut self, stats: RetryStats, gc: GcFailure) -> RunOutcome { + self.summary.set_retry_stats(stats); + let mut error = None; + if self.options.mode == RunMode::Apply + && !self.summary.any_failed() + && let Err(e) = self.collect_garbage() + { + match gc { + GcFailure::Abort => error = Some(Error::from(e)), + GcFailure::Warn => self + .options + .logger + .warn(format_args!("blob GC skipped: {}", Error::from(e))), + GcFailure::WarnAsPartial => self + .options + .logger + .warn(format_args!("blob GC skipped: {}", Error::partial(e))), + } + } + RunOutcome { + summary: self.summary, + error, + } + } + + pub fn abort(mut self, error: Error, stats: RetryStats) -> RunOutcome { + self.summary.set_retry_stats(stats); + RunOutcome { + summary: self.summary, + error: Some(error), + } + } + + fn collect_garbage(&self) -> rusqlite::Result { + let tx = self.archive.store().begin()?; + let removed = tx.store().blobs().gc_orphans()?; + tx.commit()?; + Ok(removed) + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::*; + use crate::error::ExitStatus; + use crate::logging::{Level, Logger}; + use crate::net::http::TlsPolicy; + use crate::types::ObjectType; + + fn options(mode: RunMode) -> RunOptions { + RunOptions { + archive: PathBuf::new(), + threads: 1, + mode, + max_retries: 0, + tls: TlsPolicy::Verify, + logger: Logger::new(Level::Quiet), + } + } + + fn key(kind: SourceKind, account_id: &str) -> SourceKey<'_> { + SourceKey { + kind, + url: "imap://mail.example.com", + account_id, + } + } + + const LABEL: SourceLabel<'static> = SourceLabel { + name: Some("alice"), + username: "alice", + }; + + fn blob_rows(archive: &Archive) -> i64 { + archive + .connection() + .query_row("SELECT count(*) FROM blobs", [], |r| r.get(0)) + .unwrap() + } + + fn seeded(kind: SourceKind) -> Archive { + let archive = Archive::open_in_memory().unwrap(); + archive + .store() + .sources() + .upsert(&key(kind, "alice"), Some("alice"), "alice") + .unwrap(); + archive + } + + #[test] + fn apply_registers_the_source() { + let run = ImportRun::start_on( + Archive::open_in_memory().unwrap(), + options(RunMode::Apply), + &key(SourceKind::Imap, "alice"), + LABEL, + SourceChange::Reject, + ) + .unwrap(); + assert!(matches!(run.claim(), SourceClaim::Registered(_))); + assert!(run.claim().id().is_some()); + } + + #[test] + fn dry_run_finds_without_registering() { + let run = ImportRun::start_on( + Archive::open_in_memory().unwrap(), + options(RunMode::DryRun), + &key(SourceKind::Imap, "alice"), + LABEL, + SourceChange::Reject, + ) + .unwrap(); + assert_eq!(*run.claim(), SourceClaim::DryRun(None)); + let run = ImportRun::start_on( + seeded(SourceKind::Imap), + options(RunMode::DryRun), + &key(SourceKind::Imap, "alice"), + LABEL, + SourceChange::Reject, + ) + .unwrap(); + assert!(matches!(run.claim(), SourceClaim::DryRun(Some(_)))); + } + + #[test] + fn guard_rejects_another_account_in_both_modes() { + for mode in [RunMode::Apply, RunMode::DryRun] { + let err = ImportRun::start_on( + seeded(SourceKind::Imap), + options(mode), + &key(SourceKind::Imap, "bob"), + LABEL, + SourceChange::Reject, + ) + .err() + .unwrap(); + assert_eq!(err.exit_status(), ExitStatus::SourceChange, "{mode:?}"); + } + } + + // a failing source query keeps each protocol's old exit class + #[test] + fn source_sql_failure_keeps_the_exit_class_per_kind() { + for (kind, status) in [ + (SourceKind::Jmap, ExitStatus::Partial), + (SourceKind::CardDav, ExitStatus::Partial), + (SourceKind::Imap, ExitStatus::Archive), + ] { + for (mode, change) in [ + (RunMode::Apply, SourceChange::Reject), + (RunMode::Apply, SourceChange::Allow), + (RunMode::DryRun, SourceChange::Allow), + ] { + let archive = Archive::open_in_memory().unwrap(); + archive + .connection() + .execute_batch("PRAGMA foreign_keys = OFF; DROP TABLE sources;") + .unwrap(); + let err = + ImportRun::start_on(archive, options(mode), &key(kind, "alice"), LABEL, change) + .err() + .unwrap(); + assert_eq!(err.exit_status(), status, "{kind:?} {mode:?} {change:?}"); + } + } + } + + #[test] + fn guard_allows_when_asked() { + for change in [SourceChange::Allow, SourceChange::Replace] { + assert!( + ImportRun::start_on( + seeded(SourceKind::CalDav), + options(RunMode::Apply), + &key(SourceKind::CalDav, "bob"), + LABEL, + change, + ) + .is_ok() + ); + } + } + + #[test] + fn guard_messages_are_unchanged_per_kind() { + let text = |kind| ImportRun::source_changed(kind, "u", "a").to_string(); + assert_eq!( + text(SourceKind::Imap), + "source-change protection: archive already records imap source u / a; \ + re-run with --allow-source-change or use a fresh archive" + ); + assert_eq!( + text(SourceKind::Jmap), + "source-change protection: archive already records JMAP source (u, account a); \ + pass --allow-source-change to import a different account" + ); + assert_eq!( + text(SourceKind::ExchangeGraph), + "source-change protection: archive already records exchange_graph source (u, account a); \ + pass --allow-source-change to import a different account" + ); + assert_eq!( + text(SourceKind::CardDav), + "source-change protection: archive already records carddav source (u, account a); \ + pass --allow-source-change to import a different account, \ + or --replace-source to drop the recorded one" + ); + } + + #[test] + fn finish_collects_garbage_only_after_a_clean_apply() { + for (mode, failed, left) in [ + (RunMode::Apply, 0, 0), + (RunMode::Apply, 1, 1), + (RunMode::DryRun, 0, 1), + ] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("archive.sqlite"); + Archive::open(&path) + .unwrap() + .store() + .blobs() + .intern(b"orphan") + .unwrap(); + let mut opts = options(mode); + opts.archive = path.clone(); + let mut run = ImportRun::start( + opts, + &key(SourceKind::Maildir, "alice"), + LABEL, + SourceChange::Reject, + ) + .unwrap(); + run.counts(ObjectType::Email).failed = failed; + let outcome = run.finish( + RetryStats { + retries_observed: 2, + retry_after_sleeps: 1, + }, + GcFailure::Abort, + ); + assert!(outcome.error.is_none()); + assert_eq!(outcome.summary.retries_observed, 2); + assert_eq!( + blob_rows(&Archive::open(&path).unwrap()), + left, + "{mode:?} failed={failed}" + ); + } + } + + #[test] + fn abort_keeps_counts() { + let mut run = ImportRun::start_on( + Archive::open_in_memory().unwrap(), + options(RunMode::Apply), + &key(SourceKind::Takeout, "alice"), + LABEL, + SourceChange::Reject, + ) + .unwrap(); + run.counts(ObjectType::Mailbox).created = 3; + let outcome = run.abort(Error::Connection("gone".to_owned()), RetryStats::default()); + assert_eq!( + outcome.summary.get(ObjectType::Mailbox).map(|c| c.created), + Some(3) + ); + assert!(outcome.into_result().is_err()); + } +} diff --git a/src/sync/tree.rs b/src/sync/tree.rs new file mode 100644 index 0000000..0a86bff --- /dev/null +++ b/src/sync/tree.rs @@ -0,0 +1,199 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::cmp::Reverse; +use std::collections::HashMap; +use std::hash::Hash; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DanglingParent { + Counts, + Ignored, +} + +#[derive(Debug, Clone)] +pub struct ParentMap { + parents: HashMap>, + dangling: DanglingParent, +} + +impl ParentMap { + pub fn new(dangling: DanglingParent) -> ParentMap { + ParentMap { + parents: HashMap::new(), + dangling, + } + } + + pub fn with_capacity(dangling: DanglingParent, capacity: usize) -> ParentMap { + ParentMap { + parents: HashMap::with_capacity(capacity), + dangling, + } + } + + pub fn insert(&mut self, child: K, parent: Option) { + self.parents.insert(child, parent); + } + + pub fn parent(&self, child: &K) -> Option<&K> { + self.parents.get(child).and_then(Option::as_ref) + } + + pub fn depth(&self, key: &K) -> usize { + self.depth_with(key, &mut Vec::new()) + } + + fn depth_with<'s>(&'s self, key: &K, seen: &mut Vec<&'s K>) -> usize { + seen.clear(); + let mut depth = 0; + let mut next = self.parent(key); + while let Some(parent) = next { + if self.dangling == DanglingParent::Ignored && !self.parents.contains_key(parent) { + break; + } + depth += 1; + if parent == key || seen.contains(&parent) { + break; + } + seen.push(parent); + next = self.parent(parent); + } + depth + } + + pub fn parents_first(&self, items: &mut [T], key: impl Fn(&T) -> &K) { + let mut seen = Vec::new(); + items.sort_by_cached_key(|item| self.depth_with(key(item), &mut seen)); + } + + pub fn children_first(&self, items: &mut [T], key: impl Fn(&T) -> &K) { + let mut seen = Vec::new(); + items.sort_by_cached_key(|item| Reverse(self.depth_with(key(item), &mut seen))); + } +} +impl Extend<(K, Option)> for ParentMap { + fn extend)>>(&mut self, iter: I) { + self.parents.extend(iter); + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FolderPath<'a> { + path: &'a str, + delimiter: char, +} + +impl<'a> FolderPath<'a> { + pub fn new(path: &'a str, delimiter: char) -> FolderPath<'a> { + FolderPath { path, delimiter } + } + + pub fn depth(self) -> usize { + self.path.matches(self.delimiter).count() + } + + pub fn parent(self) -> Option<&'a str> { + self.path + .rsplit_once(self.delimiter) + .map(|(parent, _)| parent) + } + + pub fn leaf(self) -> &'a str { + self.path + .rsplit_once(self.delimiter) + .map_or(self.path, |(_, leaf)| leaf) + } + + pub fn sort_children_first(paths: &mut [String], delimiter: char) { + FolderPath::sort_children_first_by(paths, delimiter, String::as_str); + } + + pub fn sort_children_first_by(items: &mut [T], delimiter: char, path: impl Fn(&T) -> &str) { + items.sort_by(|a, b| { + let (a, b) = (path(a), path(b)); + let depth_a = FolderPath::new(a, delimiter).depth(); + let depth_b = FolderPath::new(b, delimiter).depth(); + depth_b.cmp(&depth_a).then_with(|| a.cmp(b)) + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn map(dangling: DanglingParent, edges: &[(i64, Option)]) -> ParentMap { + let mut m = ParentMap::new(dangling); + m.extend(edges.iter().copied()); + m + } + + #[test] + fn depth_walks_to_the_root() { + let m = map( + DanglingParent::Counts, + &[(1, None), (2, Some(1)), (3, Some(2))], + ); + assert_eq!(m.depth(&1), 0); + assert_eq!(m.depth(&3), 2); + assert_eq!(m.depth(&99), 0); + } + + #[test] + fn dangling_parent_policy() { + let edges = [(1, Some(50)), (2, Some(1))]; + assert_eq!(map(DanglingParent::Counts, &edges).depth(&2), 2); + assert_eq!(map(DanglingParent::Ignored, &edges).depth(&2), 1); + } + + #[test] + fn cycles_terminate() { + let m = map( + DanglingParent::Counts, + &[(1, Some(2)), (2, Some(1)), (3, Some(3))], + ); + assert_eq!(m.depth(&1), 2); + assert_eq!(m.depth(&3), 1); + } + + #[test] + fn orders_items_by_depth_stably() { + let m = map( + DanglingParent::Counts, + &[(1, None), (2, Some(1)), (3, Some(2)), (4, None)], + ); + let mut items = vec![3, 1, 2, 4]; + m.parents_first(&mut items, |i| i); + assert_eq!(items, [1, 4, 2, 3]); + m.children_first(&mut items, |i| i); + assert_eq!(items, [3, 2, 1, 4]); + } + + #[test] + fn folder_path_parts() { + let p = FolderPath::new("A/B/C", '/'); + assert_eq!(p.depth(), 2); + assert_eq!(p.parent(), Some("A/B")); + assert_eq!(p.leaf(), "C"); + let root = FolderPath::new("INBOX", '.'); + assert_eq!(root.depth(), 0); + assert_eq!(root.parent(), None); + assert_eq!(root.leaf(), "INBOX"); + } + + #[test] + fn sort_children_first_deepest_then_name() { + let mut names = vec![ + "A".to_owned(), + "A/B/C".to_owned(), + "Z/Y".to_owned(), + "A/B".to_owned(), + ]; + FolderPath::sort_children_first(&mut names, '/'); + assert_eq!(names, ["A/B/C", "A/B", "Z/Y", "A"]); + } +} diff --git a/src/sync/vanish.rs b/src/sync/vanish.rs new file mode 100644 index 0000000..538f2ca --- /dev/null +++ b/src/sync/vanish.rs @@ -0,0 +1,326 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::params; + +use crate::archive::{LocalId, Store}; +use crate::sync::TypeCounts; +use crate::types::ObjectType; + +const SAVEPOINT: &str = "SAVEPOINT vanished_row"; +const RELEASE: &str = "RELEASE vanished_row"; +const ROLLBACK: &str = "ROLLBACK TO vanished_row; RELEASE vanished_row"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OnRowError { + Abort, + Count, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Children { + Restrict, + DetachSurvivors, + RefuseSubfolders, +} + +#[derive(Debug)] +pub enum RowFailure<'e> { + Sql(&'e rusqlite::Error), + LiveChildren, +} + +enum RowOutcome { + Removed, + LiveChildren, +} + +struct Statements { + delete: String, + children: Option, +} + +#[derive(Clone, Copy)] +pub struct VanishedSweep<'a> { + store: Store<'a>, + on_row_error: OnRowError, + children: Children, +} + +impl<'a> VanishedSweep<'a> { + pub fn new( + store: Store<'a>, + on_row_error: OnRowError, + children: Children, + ) -> VanishedSweep<'a> { + VanishedSweep { + store, + on_row_error, + children, + } + } + + pub fn run( + &self, + ty: ObjectType, + vanished: &[(K, LocalId)], + counts: &mut TypeCounts, + forget: impl FnMut(Store<'_>, &K, LocalId) -> rusqlite::Result<()>, + report: impl FnMut(&K, LocalId, RowFailure<'_>), + ) -> rusqlite::Result<()> { + self.run_then(ty, vanished, counts, forget, report, |_| Ok(())) + } + + pub fn run_then( + &self, + ty: ObjectType, + vanished: &[(K, LocalId)], + counts: &mut TypeCounts, + mut forget: impl FnMut(Store<'_>, &K, LocalId) -> rusqlite::Result<()>, + mut report: impl FnMut(&K, LocalId, RowFailure<'_>), + epilogue: impl FnOnce(Store<'_>) -> rusqlite::Result<()>, + ) -> rusqlite::Result<()> { + if vanished.is_empty() { + return Ok(()); + } + let table = ty.table(); + let statements = Statements { + delete: format!("DELETE FROM {table} WHERE id = ?1"), + children: match self.children { + Children::Restrict => None, + Children::DetachSurvivors => Some(format!( + "UPDATE {table} SET parent_id = NULL WHERE parent_id = ?1" + )), + Children::RefuseSubfolders => Some(format!( + "SELECT EXISTS (SELECT 1 FROM {table} WHERE parent_id = ?1)" + )), + }, + }; + let tx = self.store.begin()?; + let store = tx.store(); + let conn = store.connection(); + for (key, id) in vanished { + let outcome = match self.on_row_error { + OnRowError::Abort => self.remove_row(store, &statements, key, *id, &mut forget)?, + OnRowError::Count => { + conn.execute_batch(SAVEPOINT)?; + match self.remove_row(store, &statements, key, *id, &mut forget) { + Ok(outcome) => { + conn.execute_batch(RELEASE)?; + outcome + } + Err(e) => { + conn.execute_batch(ROLLBACK)?; + counts.failed += 1; + report(key, *id, RowFailure::Sql(&e)); + continue; + } + } + } + }; + match outcome { + RowOutcome::Removed => counts.deleted += 1, + RowOutcome::LiveChildren => { + counts.failed += 1; + report(key, *id, RowFailure::LiveChildren); + } + } + } + epilogue(store)?; + tx.commit() + } + + fn remove_row( + &self, + store: Store<'_>, + statements: &Statements, + key: &K, + id: LocalId, + forget: &mut impl FnMut(Store<'_>, &K, LocalId) -> rusqlite::Result<()>, + ) -> rusqlite::Result { + let conn = store.connection(); + if let Some(sql) = &statements.children { + match self.children { + Children::RefuseSubfolders => { + let live: bool = conn + .prepare_cached(sql)? + .query_row(params![id], |row| row.get(0))?; + if live { + return Ok(RowOutcome::LiveChildren); + } + } + Children::DetachSurvivors => { + conn.prepare_cached(sql)?.execute(params![id])?; + } + Children::Restrict => {} + } + } + forget(store, key, id)?; + conn.prepare_cached(&statements.delete)? + .execute(params![id])?; + Ok(RowOutcome::Removed) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::Archive; + use crate::archive::rows::{MailboxRecord, Subscribed}; + + fn mailbox(archive: &Archive, name: &str, parent: Option) -> LocalId { + archive + .store() + .mailboxes() + .insert(&MailboxRecord { + name, + parent, + role: None, + sort_order: 0, + subscribed: Subscribed::Yes, + }) + .unwrap() + } + + fn names(archive: &Archive) -> Vec { + let conn = archive.connection(); + let mut stmt = conn + .prepare("SELECT name FROM mailboxes ORDER BY name") + .unwrap(); + stmt.query_map([], |r| r.get(0)) + .unwrap() + .collect::>() + .unwrap() + } + + fn sweep(archive: &Archive, on: OnRowError, children: Children) -> VanishedSweep<'_> { + VanishedSweep::new(archive.store(), on, children) + } + + #[test] + fn restrict_counts_a_parent_with_live_children_as_failed() { + let archive = Archive::open_in_memory().unwrap(); + let parent = mailbox(&archive, "A", None); + mailbox(&archive, "B", Some(parent)); + let lone = mailbox(&archive, "C", None); + let mut counts = TypeCounts::default(); + let mut forgotten = Vec::new(); + let mut failures = Vec::new(); + sweep(&archive, OnRowError::Count, Children::Restrict) + .run( + ObjectType::Mailbox, + &[("a", parent), ("c", lone)], + &mut counts, + |_, key, _| { + forgotten.push(*key); + Ok(()) + }, + |key, _, failure| failures.push((*key, matches!(failure, RowFailure::Sql(_)))), + ) + .unwrap(); + assert_eq!((counts.deleted, counts.failed), (1, 1)); + assert_eq!(failures, [("a", true)]); + assert_eq!(names(&archive), ["A", "B"]); + assert_eq!(forgotten, ["a", "c"]); + } + + #[test] + fn a_failed_row_rolls_back_its_forget() { + let archive = Archive::open_in_memory().unwrap(); + let parent = mailbox(&archive, "A", None); + mailbox(&archive, "B", Some(parent)); + let mut counts = TypeCounts::default(); + sweep(&archive, OnRowError::Count, Children::Restrict) + .run( + ObjectType::Mailbox, + &[("a", parent)], + &mut counts, + |store, _, _| { + store + .connection() + .execute( + "INSERT INTO identities (name, email) VALUES ('x', 'x@x')", + [], + ) + .map(drop) + }, + |_, _, _| {}, + ) + .unwrap(); + let n: i64 = archive + .connection() + .query_row("SELECT count(*) FROM identities", [], |r| r.get(0)) + .unwrap(); + assert_eq!(n, 0); + } + + #[test] + fn detach_survivors_keeps_children_as_roots() { + let archive = Archive::open_in_memory().unwrap(); + let parent = mailbox(&archive, "A", None); + let child = mailbox(&archive, "B", Some(parent)); + let mut counts = TypeCounts::default(); + sweep(&archive, OnRowError::Abort, Children::DetachSurvivors) + .run( + ObjectType::Mailbox, + &[("a", parent)], + &mut counts, + |_, _, _| Ok(()), + |_, _, _| {}, + ) + .unwrap(); + assert_eq!(counts.deleted, 1); + let parent_of_child: Option = archive + .connection() + .query_row( + "SELECT parent_id FROM mailboxes WHERE id = ?1", + [child], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(parent_of_child, None); + } + + #[test] + fn refuse_subfolders_skips_and_continues() { + let archive = Archive::open_in_memory().unwrap(); + let parent = mailbox(&archive, "A", None); + mailbox(&archive, "B", Some(parent)); + let lone = mailbox(&archive, "C", None); + let mut counts = TypeCounts::default(); + let mut refused = Vec::new(); + sweep(&archive, OnRowError::Abort, Children::RefuseSubfolders) + .run( + ObjectType::Mailbox, + &[("a", parent), ("c", lone)], + &mut counts, + |_, _, _| Ok(()), + |key, _, failure| refused.push((*key, matches!(failure, RowFailure::LiveChildren))), + ) + .unwrap(); + assert_eq!((counts.deleted, counts.failed), (1, 1)); + assert_eq!(refused, [("a", true)]); + assert_eq!(names(&archive), ["A", "B"]); + } + + #[test] + fn abort_propagates_and_rolls_back_everything() { + let archive = Archive::open_in_memory().unwrap(); + let lone = mailbox(&archive, "C", None); + let parent = mailbox(&archive, "A", None); + mailbox(&archive, "B", Some(parent)); + let mut counts = TypeCounts::default(); + let result = sweep(&archive, OnRowError::Abort, Children::Restrict).run( + ObjectType::Mailbox, + &[("c", lone), ("a", parent)], + &mut counts, + |_, _, _| Ok(()), + |_, _, _| {}, + ); + assert!(result.is_err()); + assert_eq!(names(&archive), ["A", "B", "C"]); + } +} diff --git a/src/types.rs b/src/types.rs index 82f8df9..513c575 100644 --- a/src/types.rs +++ b/src/types.rs @@ -4,106 +4,132 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +use std::ops::Deref; +use std::str::FromStr; + use crate::error::Error; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum ObjectType { - Mailbox, - Email, - Identity, - SieveScript, - AddressBook, - ContactCard, - Calendar, - CalendarEvent, - ParticipantIdentity, - FileNode, +macro_rules! object_types { + ($($variant:ident => $token:literal, $jmap:literal, $table:literal, $urn:literal, $label:literal, $noun:literal;)+) => { + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub enum ObjectType { + $($variant,)+ + } + + impl ObjectType { + pub const ALL: [ObjectType; 10] = [$(ObjectType::$variant,)+]; + + pub const fn token(self) -> &'static str { + match self { + $(ObjectType::$variant => $token,)+ + } + } + + pub const fn jmap_name(self) -> &'static str { + match self { + $(ObjectType::$variant => $jmap,)+ + } + } + + pub const fn label(self) -> &'static str { + match self { + $(ObjectType::$variant => $label,)+ + } + } + + pub const fn noun(self) -> &'static str { + match self { + $(ObjectType::$variant => $noun,)+ + } + } + + pub const fn table(self) -> &'static str { + match self { + $(ObjectType::$variant => $table,)+ + } + } + + pub const fn capability_urn(self) -> &'static str { + match self { + $(ObjectType::$variant => $urn,)+ + } + } + + fn from_token(token: &str) -> Option { + hashify::map_ignore_case!( + token.as_bytes(), + ObjectType, + $($token => ObjectType::$variant,)+ + ) + .copied() + } + } + }; } -impl ObjectType { - pub const ALL: [ObjectType; 10] = [ - ObjectType::Mailbox, - ObjectType::Email, - ObjectType::Identity, - ObjectType::SieveScript, - ObjectType::AddressBook, - ObjectType::ContactCard, - ObjectType::Calendar, - ObjectType::CalendarEvent, - ObjectType::ParticipantIdentity, - ObjectType::FileNode, - ]; +object_types! { + Mailbox => "mailbox", "Mailbox", "mailboxes", "urn:ietf:params:jmap:mail", "Mailboxes", "mailbox"; + Email => "email", "Email", "emails", "urn:ietf:params:jmap:mail", "Emails", "email"; + Identity => "identity", "Identity", "identities", "urn:ietf:params:jmap:submission", "Identities", "identity"; + SieveScript => "sievescript", "SieveScript", "sieve_scripts", "urn:ietf:params:jmap:sieve", "Sieve scripts", "sieve script"; + AddressBook => "addressbook", "AddressBook", "address_books", "urn:ietf:params:jmap:contacts", "Address books", "address book"; + ContactCard => "contactcard", "ContactCard", "contact_cards", "urn:ietf:params:jmap:contacts", "Contact cards", "contact card"; + Calendar => "calendar", "Calendar", "calendars", "urn:ietf:params:jmap:calendars", "Calendars", "calendar"; + CalendarEvent => "calendarevent", "CalendarEvent", "calendar_events", "urn:ietf:params:jmap:calendars", "Calendar events", "calendar event"; + ParticipantIdentity => "participantidentity", "ParticipantIdentity", "participant_identities", "urn:ietf:params:jmap:calendars", "Participant identities", "participant identity"; + FileNode => "filenode", "FileNode", "file_nodes", "urn:ietf:params:jmap:filenode", "File nodes", "file node"; +} - pub fn token(self) -> &'static str { - match self { - ObjectType::Mailbox => "mailbox", - ObjectType::Email => "email", - ObjectType::Identity => "identity", - ObjectType::SieveScript => "sievescript", - ObjectType::AddressBook => "addressbook", - ObjectType::ContactCard => "contactcard", - ObjectType::Calendar => "calendar", - ObjectType::CalendarEvent => "calendarevent", - ObjectType::ParticipantIdentity => "participantidentity", - ObjectType::FileNode => "filenode", - } - } +impl FromStr for ObjectType { + type Err = Error; - pub fn jmap_name(self) -> &'static str { - match self { - ObjectType::Mailbox => "Mailbox", - ObjectType::Email => "Email", - ObjectType::Identity => "Identity", - ObjectType::SieveScript => "SieveScript", - ObjectType::AddressBook => "AddressBook", - ObjectType::ContactCard => "ContactCard", - ObjectType::Calendar => "Calendar", - ObjectType::CalendarEvent => "CalendarEvent", - ObjectType::ParticipantIdentity => "ParticipantIdentity", - ObjectType::FileNode => "FileNode", - } - } - - pub fn capability_urn(self) -> &'static str { - match self { - ObjectType::Mailbox | ObjectType::Email => "urn:ietf:params:jmap:mail", - ObjectType::Identity => "urn:ietf:params:jmap:submission", - ObjectType::SieveScript => "urn:ietf:params:jmap:sieve", - ObjectType::AddressBook | ObjectType::ContactCard => "urn:ietf:params:jmap:contacts", - ObjectType::Calendar | ObjectType::CalendarEvent | ObjectType::ParticipantIdentity => { - "urn:ietf:params:jmap:calendars" - } - ObjectType::FileNode => "urn:ietf:params:jmap:filenode", - } - } - - pub fn parse(token: &str) -> Result { - let normalized = token.trim().to_ascii_lowercase(); - ObjectType::ALL - .into_iter() - .find(|t| t.token() == normalized) + fn from_str(token: &str) -> Result { + ObjectType::from_token(token.trim()) .ok_or_else(|| Error::Usage(format!("unknown object type: {token}"))) } } -pub fn parse_object_list(list: &str) -> Result, Error> { - let mut out = Vec::new(); - for token in list.split(',') { - let token = token.trim(); - if token.is_empty() { - continue; +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ObjectTypes(Vec); + +impl FromStr for ObjectTypes { + type Err = Error; + + fn from_str(list: &str) -> Result { + let mut out = Vec::new(); + for token in list.split(',').map(str::trim).filter(|t| !t.is_empty()) { + let parsed = token.parse::()?; + if !out.contains(&parsed) { + out.push(parsed); + } } - let parsed = ObjectType::parse(token)?; - if !out.contains(&parsed) { - out.push(parsed); + if out.is_empty() { + return Err(Error::Usage( + "--objects given but resolved to an empty type list".to_owned(), + )); } + Ok(ObjectTypes(out)) } - if out.is_empty() { - return Err(Error::Usage( - "--objects given but resolved to an empty type list".to_owned(), - )); +} + +impl From> for ObjectTypes { + fn from(types: Vec) -> ObjectTypes { + let mut out = Vec::with_capacity(types.len()); + for ty in types { + if !out.contains(&ty) { + out.push(ty); + } + } + ObjectTypes(out) + } +} + +impl Deref for ObjectTypes { + type Target = [ObjectType]; + + fn deref(&self) -> &[ObjectType] { + &self.0 } - Ok(out) } #[cfg(test)] @@ -113,29 +139,45 @@ mod tests { #[test] fn token_roundtrip() { for t in ObjectType::ALL { - assert_eq!(ObjectType::parse(t.token()).unwrap(), t); + assert_eq!(t.token().parse::().unwrap(), t); } } #[test] fn parse_list_dedups_and_trims() { - let parsed = parse_object_list(" mailbox, email ,mailbox").unwrap(); - assert_eq!(parsed, vec![ObjectType::Mailbox, ObjectType::Email]); + let parsed: ObjectTypes = " mailbox, email ,mailbox".parse().unwrap(); + assert_eq!(*parsed, [ObjectType::Mailbox, ObjectType::Email]); } #[test] fn parse_list_case_insensitive() { - let parsed = parse_object_list("Mailbox,EMAIL").unwrap(); - assert_eq!(parsed, vec![ObjectType::Mailbox, ObjectType::Email]); + let parsed: ObjectTypes = "Mailbox,EMAIL".parse().unwrap(); + assert_eq!(*parsed, [ObjectType::Mailbox, ObjectType::Email]); } #[test] fn parse_list_rejects_unknown() { - assert!(parse_object_list("mailbox,bogus").is_err()); + assert!("mailbox,bogus".parse::().is_err()); } #[test] fn parse_list_rejects_empty() { - assert!(parse_object_list(" , ").is_err()); + assert!(" , ".parse::().is_err()); + } + + #[test] + fn tables_match_schema_names() { + assert_eq!(ObjectType::Mailbox.table(), "mailboxes"); + assert_eq!( + ObjectType::ParticipantIdentity.table(), + "participant_identities" + ); + assert_eq!(ObjectType::FileNode.table(), "file_nodes"); + } + + #[test] + fn unknown_type_reports_the_raw_token() { + let err = " Bogus ".parse::().unwrap_err(); + assert_eq!(err.to_string(), "unknown object type: Bogus "); } } diff --git a/tests/common/line_server.rs b/tests/common/line_server.rs new file mode 100644 index 0000000..5bc12cc --- /dev/null +++ b/tests/common/line_server.rs @@ -0,0 +1,93 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::{self, BufRead, BufReader, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::Duration; + +const IO_TIMEOUT: Duration = Duration::from_secs(5); + +pub type Script = Box io::Result<()> + Send + 'static>; + +pub struct MockConn { + pub reader: BufReader, + pub writer: TcpStream, +} + +impl MockConn { + pub fn write_line(&mut self, s: &str) -> io::Result<()> { + self.writer.write_all(s.as_bytes())?; + self.writer.write_all(b"\r\n")?; + self.writer.flush() + } + + pub fn write_raw(&mut self, bytes: &[u8]) -> io::Result<()> { + self.writer.write_all(bytes)?; + self.writer.flush() + } + + pub fn read_line(&mut self) -> io::Result { + let mut line = String::new(); + if self.reader.read_line(&mut line)? == 0 { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "client closed", + )); + } + Ok(line.trim_end_matches(['\r', '\n']).to_owned()) + } +} + +pub struct LineServer { + pub addr: String, + pub port: u16, + _thread: thread::JoinHandle<()>, +} + +impl LineServer { + pub fn start(scripts: Vec