commit 576073f8c9fb6f01585944ccb84c517d917b90a0 Author: Maurus Decimus <11444311+mdecimus@users.noreply.github.com> Date: Fri May 29 18:02:15 2026 +0200 Initial commit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..091f3af --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,335 @@ +name: Release +permissions: + "contents": "write" + +on: + pull_request: + push: + tags: + - '**[0-9]+.[0-9]+.[0-9]+*' + +jobs: + # Run 'dist plan' (or host) to determine what tasks we need to do + plan: + runs-on: "ubuntu-22.04" + outputs: + val: ${{ steps.plan.outputs.manifest }} + tag: ${{ !github.event.pull_request && github.ref_name || '' }} + tag-flag: ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }} + publishing: ${{ !github.event.pull_request }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + submodules: recursive + - name: Install dist + shell: bash + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.31.0/cargo-dist-installer.sh | sh" + - name: Cache dist + uses: actions/upload-artifact@v7 + with: + name: cargo-dist-cache + path: ~/.cargo/bin/dist + - id: plan + run: | + dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json + echo "dist ran successfully" + cat plan-dist-manifest.json + echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT" + - name: "Upload dist-manifest.json" + uses: actions/upload-artifact@v7 + with: + name: artifacts-plan-dist-manifest + path: plan-dist-manifest.json + + build-local-artifacts: + name: build-local-artifacts (${{ join(matrix.targets, ', ') }}) + needs: + - plan + if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }} + runs-on: ${{ matrix.runner }} + container: ${{ matrix.container && matrix.container.image || null }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json + AWS_LC_SYS_PREBUILT_NASM: 1 + permissions: + "attestations": "write" + "contents": "read" + "id-token": "write" + steps: + - name: enable windows longpaths + run: | + git config --global core.longpaths true + - uses: actions/checkout@v6 + with: + persist-credentials: false + submodules: recursive + - name: Install Rust non-interactively if not already installed + if: ${{ matrix.container }} + run: | + if ! command -v cargo > /dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + fi + - name: Install dist + run: ${{ matrix.install_dist.run }} + - name: Fetch local artifacts + uses: actions/download-artifact@v8 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + - name: Install dependencies + run: | + ${{ matrix.packages_install }} + - name: Build artifacts + run: | + # Actually do builds and make zips and whatnot + dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json + echo "dist ran successfully" + - name: Attest + uses: actions/attest-build-provenance@v4 + with: + subject-path: "target/distrib/*${{ join(matrix.targets, ', ') }}*" + - id: cargo-dist + name: Post-build + shell: bash + run: | + # Parse out what we just built and upload it to scratch storage + echo "paths<> "$GITHUB_OUTPUT" + dist print-upload-files-from-manifest --manifest dist-manifest.json >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + cp dist-manifest.json "$BUILD_MANIFEST_NAME" + - name: "Upload artifacts" + uses: actions/upload-artifact@v7 + with: + name: artifacts-build-local-${{ join(matrix.targets, '_') }} + path: | + ${{ steps.cargo-dist.outputs.paths }} + ${{ env.BUILD_MANIFEST_NAME }} + + build-global-artifacts: + needs: + - plan + - build-local-artifacts + runs-on: "ubuntu-22.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + submodules: recursive + - name: Install cached dist + uses: actions/download-artifact@v8 + with: + name: cargo-dist-cache + path: ~/.cargo/bin/ + - run: chmod +x ~/.cargo/bin/dist + - name: Fetch local artifacts + uses: actions/download-artifact@v8 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + - id: cargo-dist + shell: bash + run: | + dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json + echo "dist ran successfully" + + # Parse out what we just built and upload it to scratch storage + echo "paths<> "$GITHUB_OUTPUT" + jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + cp dist-manifest.json "$BUILD_MANIFEST_NAME" + - name: "Upload artifacts" + uses: actions/upload-artifact@v7 + with: + name: artifacts-build-global + path: | + ${{ steps.cargo-dist.outputs.paths }} + ${{ env.BUILD_MANIFEST_NAME }} + + host: + needs: + - plan + - build-local-artifacts + - build-global-artifacts + if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + runs-on: "ubuntu-22.04" + outputs: + val: ${{ steps.host.outputs.manifest }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + submodules: recursive + - name: Install cached dist + uses: actions/download-artifact@v8 + with: + name: cargo-dist-cache + path: ~/.cargo/bin/ + - run: chmod +x ~/.cargo/bin/dist + - name: Fetch artifacts + uses: actions/download-artifact@v8 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + - id: host + shell: bash + run: | + dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json + echo "artifacts uploaded and released successfully" + cat dist-manifest.json + echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT" + - name: "Upload dist-manifest.json" + uses: actions/upload-artifact@v7 + with: + name: artifacts-dist-manifest + path: dist-manifest.json + - name: "Download GitHub Artifacts" + uses: actions/download-artifact@v8 + with: + pattern: artifacts-* + path: artifacts + merge-multiple: true + - name: Cleanup + run: | + # Remove the granular manifests + rm -f artifacts/*-dist-manifest.json + - name: Create GitHub Release + env: + PRERELEASE_FLAG: "${{ fromJson(steps.host.outputs.manifest).announcement_is_prerelease && '--prerelease' || '' }}" + RELEASE_COMMIT: "${{ github.sha }}" + run: | + awk '/^## \[/{c++} c==1' CHANGELOG.md > $RUNNER_TEMP/notes.txt + + # Tag-push releases are created as drafts; the `announce` job un-drafts + # them only after the build succeeds, so watcher notifications don't + # fire on broken builds. + gh release create "${{ needs.plan.outputs.tag }}" --target "$RELEASE_COMMIT" $PRERELEASE_FLAG --draft --title "${{ needs.plan.outputs.tag }}" --notes-file "$RUNNER_TEMP/notes.txt" artifacts/* + + publish-homebrew-formula: + needs: + - plan + - host + runs-on: "ubuntu-22.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PLAN: ${{ needs.plan.outputs.val }} + GITHUB_USER: "axo bot" + GITHUB_EMAIL: "admin+bot@axo.dev" + if: ${{ !fromJson(needs.plan.outputs.val).announcement_is_prerelease || fromJson(needs.plan.outputs.val).publish_prereleases }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: true + repository: "stalwartlabs/homebrew-tap" + token: ${{ secrets.HOMEBREW_TAP_TOKEN }} + - name: Fetch homebrew formulae + uses: actions/download-artifact@v8 + with: + pattern: artifacts-* + path: Formula/ + merge-multiple: true + - name: Commit formula files + run: | + git config --global user.name "${GITHUB_USER}" + git config --global user.email "${GITHUB_EMAIL}" + + for release in $(echo "$PLAN" | jq --compact-output '.releases[] | select([.artifacts[] | endswith(".rb")] | any)'); do + filename=$(echo "$release" | jq '.artifacts[] | select(endswith(".rb"))' --raw-output) + name=$(echo "$filename" | sed "s/\.rb$//") + version=$(echo "$release" | jq .app_version --raw-output) + + export PATH="/home/linuxbrew/.linuxbrew/bin:$PATH" + brew update + # We avoid reformatting user-provided data such as the app description and homepage. + brew style --except-cops FormulaAudit/Homepage,FormulaAudit/Desc,FormulaAuditStrict --fix "Formula/${filename}" || true + + git add "Formula/${filename}" + git commit -m "${name} ${version}" + done + git push + + publish-npm: + needs: + - plan + - host + runs-on: "ubuntu-22.04" + if: ${{ !fromJson(needs.plan.outputs.val).announcement_is_prerelease || fromJson(needs.plan.outputs.val).publish_prereleases }} + permissions: + "contents": "read" + "id-token": "write" + steps: + - uses: actions/setup-node@v4 + with: + node-version: '20' + - name: Fetch npm tarball + uses: actions/download-artifact@v8 + with: + pattern: artifacts-* + path: artifacts/ + merge-multiple: true + - name: Upgrade npm for trusted publishing + run: npm install -g npm@latest + - name: Publish to npm + env: + IS_PRERELEASE: ${{ fromJson(needs.plan.outputs.val).announcement_is_prerelease }} + run: | + TAG_FLAG="" + if [ "$IS_PRERELEASE" = "true" ]; then + TAG_FLAG="--tag next" + fi + npm publish ./artifacts/vandelay-npm-package.tar.gz --provenance --access public $TAG_FLAG + + announce: + needs: + - plan + - host + - publish-homebrew-formula + - publish-npm + if: ${{ always() && needs.host.result == 'success' && (needs.publish-homebrew-formula.result == 'skipped' || needs.publish-homebrew-formula.result == 'success') && (needs.publish-npm.result == 'skipped' || needs.publish-npm.result == 'success') }} + runs-on: "ubuntu-22.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + submodules: recursive + - name: Un-draft release + run: gh release edit "${{ needs.plan.outputs.tag }}" --draft=false --repo "${{ github.repository }}" + + cleanup: + name: Cleanup failed release + needs: + - plan + - build-local-artifacts + - build-global-artifacts + - host + - publish-homebrew-formula + - publish-npm + - announce + if: ${{ failure() && needs.plan.outputs.publishing == 'true' }} + runs-on: "ubuntu-22.04" + permissions: + contents: write + steps: + - name: Delete draft release and tag + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh release delete "${{ needs.plan.outputs.tag }}" --yes --cleanup-tag --repo "${{ github.repository }}" || true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..eac15d5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +debug +target +**/*.rs.bk +**/mutants.out*/ +.env +*.sqlite +*.md +!README.md +!CHANGELOG.md +specs/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..c82f09f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +# Change Log + +All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). + +## [1.0.0] - 2026-05-29 + +### Added +- Initial release. + +### Changed + +### Fixed diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..8ae62e4 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,3586 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "astral-tokio-tar" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb50a7aae84a03bf55b067832bc376f4961b790c97e64d3eacee97d389b90277" +dependencies = [ + "filetime", + "futures-core", + "libc", + "portable-atomic", + "rustc-hash", + "tokio", + "tokio-stream", + "xattr", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", +] + +[[package]] +name = "bollard" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee04c4c84f1f811b017f2fbb7dd8815c976e7ca98593de9c1e2afad0f636bff4" +dependencies = [ + "async-stream", + "base64", + "bitflags", + "bollard-buildkit-proto", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "home", + "http", + "http-body-util", + "hyper", + "hyper-named-pipe", + "hyper-rustls", + "hyper-util", + "hyperlocal", + "log", + "num", + "pin-project-lite", + "rand 0.9.4", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_derive", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.18", + "time", + "tokio", + "tokio-stream", + "tokio-util", + "tonic", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-buildkit-proto" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a885520bf6249ab931a764ffdb87b0ceef48e6e7d807cfdb21b751e086e1ad" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tonic-prost", + "ureq", +] + +[[package]] +name = "bollard-stubs" +version = "1.52.1-rc.29.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f0a8ca8799131c1837d1282c3f81f31e76ceb0ce426e04a7fe1ccee3287c066" +dependencies = [ + "base64", + "bollard-buildkit-proto", + "bytes", + "prost", + "serde", + "serde_json", + "serde_repr", + "time", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "calcard" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6d4f7ea9ab08310fba58c0010e3c203aaa2b6464f1d4886d36855d99b6b6036" +dependencies = [ + "ahash", + "chrono", + "chrono-tz", + "hashify", + "jmap-tools", + "mail-builder", + "mail-parser", + "serde", + "serde_json", + "uuid", +] + +[[package]] +name = "cc" +version = "1.2.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "colored" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206" +dependencies = [ + "cookie", + "document-features", + "idna", + "indexmap 2.14.0", + "log", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", + "serde_core", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "docker_credential" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d" +dependencies = [ + "base64", + "serde", + "serde_json", +] + +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "ferroid" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee93edf3c501f0035bbeffeccfed0b79e14c311f12195ec0e661e114a0f60da4" +dependencies = [ + "portable-atomic", + "rand 0.10.1", + "web-time", +] + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasip2", + "wasip3", +] + +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.14.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hashify" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd1246c0e5493286aeb2dde35b1f4eb9c4ce00e628641210a5e553fc001a1f26" +dependencies = [ + "indexmap 2.14.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "hashlink" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea0b22561a9c04a7cb1a302c013e0259cd3b4bb619f145b32f72b8b4bcbed230" +dependencies = [ + "hashbrown 0.16.1", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "http" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb92f162bf56536459fc83c79b974bb12837acfed43d6bc370a7916d0ae15ecc" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-named-pipe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +dependencies = [ + "hex", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "libc", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jmap-tools" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "751dc449b93fc24f460c346ac2e0a87f410cd3f78971709c4e7b80236432aa30" +dependencies = [ + "hashify", + "serde", + "serde_json", +] + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys 0.4.1", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libsqlite3-sys" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a76001fb4daed01e5f2b518aac0b4dc592e7c734da63dbffcf0c64fa612a8d0c" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" + +[[package]] +name = "mail-builder" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900998f307338c4013a28ab14d760b784067324b164448c6d98a89e44810473b" + +[[package]] +name = "mail-parser" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2420e9ce11c2b0583ca97ddff7ab2398c8a613154e9b72e3bafdbf767f1d7" +dependencies = [ + "hashify", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "mockito" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90820618712cab19cfc46b274c6c22546a82affcb3c3bdf0f29e3db8e1bb92c0" +dependencies = [ + "assert-json-diff", + "bytes", + "colored", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "log", + "pin-project-lite", + "rand 0.9.4", + "regex", + "serde_json", + "serde_urlencoded", + "similar", + "tokio", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "parse-display" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax", +] + +[[package]] +name = "parse-display-derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax", + "structmeta", + "syn", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + +[[package]] +name = "quick-xml" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" +dependencies = [ + "memchr", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.18", +] + +[[package]] +name = "rusqlite" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b3492ea85308705c3a5cc24fb9b9cf77273d30590349070db42991202b214c4" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "aws-lc-rs", + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni 0.21.1", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni 0.22.4", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" +dependencies = [ + "base64", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.0", + "schemars 0.9.0", + "schemars 1.2.1", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn", +] + +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "testcontainers" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfd5785b5483672915ed5fe3cddf9f546802779fc1eceff0a6fb7321fac81c1e" +dependencies = [ + "astral-tokio-tar", + "async-trait", + "bollard", + "bytes", + "docker_credential", + "either", + "etcetera", + "ferroid", + "futures", + "http", + "itertools", + "log", + "memchr", + "parse-display", + "pin-project-lite", + "serde", + "serde_json", + "serde_with", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tokio-util", + "url", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tonic" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-prost" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 2.14.0", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +dependencies = [ + "base64", + "cookie_store", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier 0.6.2", + "serde", + "serde_json", + "ureq-proto", + "utf8-zero", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +dependencies = [ + "sha1_smol", +] + +[[package]] +name = "vandelay" +version = "1.0.0" +dependencies = [ + "base64", + "blake3", + "calcard", + "chrono", + "chrono-tz", + "clap", + "crossbeam-channel", + "flate2", + "indexmap 2.14.0", + "mail-parser", + "mockito", + "num_cpus", + "quick-xml", + "regex", + "rusqlite", + "rustls", + "rustls-platform-verifier 0.7.0", + "serde", + "serde_json", + "tempfile", + "testcontainers", + "thiserror 2.0.18", + "time", + "ureq", + "url", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap 2.14.0", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap 2.14.0", + "semver", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap 2.14.0", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap 2.14.0", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.14.0", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bce33a6288fa3f072a8c2c7d0f2fdbb90e28298f0135c1f99b96c3db2efcc60b" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd425244944f4ab65ccff928e7323354c5a018c75838362fdce749dfad2ee1e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..b425262 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,53 @@ +[package] +name = "vandelay" +description = "JMAP account migration utility" +version = "1.0.0" +authors = ["Stalwart Labs LLC "] +license = "Apache-2.0 OR MIT" +repository = "https://github.com/stalwartlabs/vandelay" +homepage = "https://github.com/stalwartlabs/vandelay" +edition = "2024" +readme = "README.md" + +[package.metadata.wix] +upgrade-guid = "5C8148E7-34A9-4216-A802-FFB8345AECC4" +path-guid = "FA9E97EB-5E5C-4809-9018-9DC1EDB0E93D" +license = false +eula = false + +[[bin]] +name = "vandelay" +path = "src/main.rs" + +[dependencies] +serde = { version = "1.0.228", features = ["derive"] } +serde_json = "1.0" +indexmap = { version = "2.14.0", features = ["serde"] } +rusqlite = { version = "0.40", features = ["bundled", "blob"] } +blake3 = "1.8.5" +time = { version = "0.3.47", features = ["serde-well-known", "macros"] } +url= "2.5.8" +thiserror = "2.0.18" +ureq = { version = "3.3.0", features = ["json", "rustls-no-provider", "platform-verifier"] } +clap = { version = "4.6.1", features = ["derive"] } +crossbeam-channel = "0.5.15" +num_cpus = "1.17.0" +base64 = "0.22.1" +mail-parser = "0.11.3" +calcard = "0.3" +rustls = { version = "0.23.40", default-features = false, features = ["aws-lc-rs", "std", "tls12", "logging"] } +rustls-platform-verifier = "0.7" +regex = "1.11" +flate2 = "1.1.9" +quick-xml = "0.40" +chrono = { version = "0.4", default-features = false, features = ["std"] } +chrono-tz = { version = "0.10", default-features = false } + +[dev-dependencies] +mockito = "1.7.2" +tempfile = "3.13" +testcontainers = { version = "0.27", features = ["blocking"] } + +[profile.dist] +inherits = "release" +lto = "thin" diff --git a/LICENSES/Apache-2.0.txt b/LICENSES/Apache-2.0.txt new file mode 100644 index 0000000..16fe87b --- /dev/null +++ b/LICENSES/Apache-2.0.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/LICENSES/MIT.txt b/LICENSES/MIT.txt new file mode 100644 index 0000000..31aa793 --- /dev/null +++ b/LICENSES/MIT.txt @@ -0,0 +1,23 @@ +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..3a110c4 --- /dev/null +++ b/README.md @@ -0,0 +1,260 @@ +

Vandelay

+ +

+ The JMAP importer-exporter +
+ One-shot account migration and backup across JMAP, IMAP, CalDAV, CardDAV, WebDAV, ManageSieve, Maildir, Google Takeout and Microsoft Exchange. +

+ +
+ +

+ crates.io +   + build +   + docs.rs +   + license +

+ +
+ +He's an importer-exporter. + +- Jerry: Well, what does *he* do? +- George: He's an importer. +- Jerry: Just imports, no exports? +- George: He's an importer/exporter, okay? + +
+  + +## About + +Vandelay is a one-shot account-migration utility for JMAP, the JMAP analogue of `imapsync` generalised to every JMAP data type (mail, contacts, calendars, identities, sieve scripts, file storage). It imports an account from a wide range of source protocols into a local SQLite "archive", then exports that archive into a target JMAP server. Import and export never talk to each other, only to the SQLite archive. One archive holds exactly one account. + +Because the archive is a self-contained SQLite file that fully describes one account, vandelay doubles as a per-account backup tool: run an import on a schedule to capture a fresh snapshot, keep the resulting SQLite file as your backup, and restore it later by running an export against a JMAP target. + +## Features + +- **Many source protocols:** + - JMAP + - IMAP + - CalDAV + - CardDAV + - WebDAV + - ManageSieve + - Maildir++ + - Google Takeout + - Microsoft Exchange via EWS (*experimental*) + - Microsoft Exchange Online via Graph (*experimental*) +- **One target protocol:** JMAP, with type-by-type stateless re-matching on every run. +- **Convergent:** Re-running an interrupted import or export picks up where it left off without bookkeeping flags. +- **Multi-threaded, no async runtime:** Blocking HTTP with per-server concurrency caps respected automatically. +- **Content-addressed blobs:** Emails, sieve scripts and file-storage payloads are stored once by BLAKE3 hash and deduplicated across the archive. +- **Dry-run everywhere:** Every command supports `--dry-run` to compute the full plan without writing. +- **Source-change protection:** An archive remembers which account it was filled from; pointing it at a different one fails unless explicitly permitted. +- **Read-only inspection:** A built-in `inspect` command dumps any object type from an archive for verification. + +## CLI quick reference + +``` +vandelay [args...] +``` + +All actions accept a set of global flags (verbosity, worker pool size, retry policy, TLS handling) in addition to action-specific ones. The most useful are: + +| Flag | Purpose | +| --- | --- | +| `-j, --threads ` | Worker pool size (default: logical CPUs). | +| `--dry-run` | Compute the full plan; perform no writes. | +| `-v`, `-vv`, `-vvv` | Increase log verbosity. | +| `-q, --quiet` | Warnings and errors only. | +| `--max-retries ` | Max retries per request on transient failures (default 5). | +| `--allow-invalid-certs` | Accept self-signed / invalid TLS certs. | + +Credentials should be supplied via the `VANDELAY_PASSWORD` / `VANDELAY_TOKEN` / `VANDELAY_EWS_CLIENT_SECRET` / `VANDELAY_GRAPH_TOKEN` environment variables, or via an interactive prompt; passing them on the command line is supported but not recommended. + +### Import + +``` +vandelay import [source-args...] +``` + +Reads a source account into the local SQLite `ARCHIVE` (created if absent). Every importer accepts `--allow-source-change` to override the archive's source-identity guard. + +#### JMAP + +``` +vandelay import jmap \ + --url \ + (--auth-basic [--auth-password ] | --auth-bearer [TOKEN]) \ + (--account-id | --account-name ) \ + [--objects ] \ + +``` + +Imports a single JMAP account. `--objects` accepts a comma-separated list of object tokens (`mailbox,email,calendar,calendarevent,addressbook,contactcard,identity,sievescript,participantidentity,filenode`); default is everything the server advertises. + +#### IMAP + +``` +vandelay import imap \ + --url imap(s)://host[:port] \ + (--auth-basic [--auth-password ] | --auth-bearer [TOKEN] --auth-user ) \ + [--include ...] [--exclude ...] [--exclude-special ...] \ + [--folder ...] [--subscribed-only] [--noautomap] \ + [--include-deleted] [--allow-cleartext] [--compress] \ + [--fetch-batch ] [--imap-connections <1..8>] \ + +``` + +Imports mail (and only mail) from any IMAP server. Folder selection is via `--include`/`--exclude` regexes (mutually exclusive with the exact-match `--folder`); `--exclude-special` drops by SPECIAL-USE role. + +#### CalDAV + +``` +vandelay import caldav \ + --url \ + (--auth-basic [--auth-password ] | --auth-bearer [TOKEN]) \ + [--allow-cleartext] [--dav-connections <1..8>] [--multiget-batch ] \ + +``` + +Discovers the user's CalDAV principal (or accepts a URL pointing straight at a calendar-home or calendar), then imports calendars and events. + +#### CardDAV + +``` +vandelay import carddav \ + --url \ + (--auth-basic [--auth-password ] | --auth-bearer [TOKEN]) \ + [--allow-cleartext] [--dav-connections <1..8>] [--multiget-batch ] \ + +``` + +Same shape as `caldav`, but for address books and contacts. + +#### WebDAV + +``` +vandelay import webdav \ + --url \ + (--auth-basic [--auth-password ] | --auth-bearer [TOKEN]) \ + [--allow-cleartext] [--dav-connections <1..8>] [--multiget-batch ] \ + +``` + +Imports a plain WebDAV file collection as a JMAP `FileNode` tree. + +#### ManageSieve + +``` +vandelay import managesieve \ + --url sieve(s)://host[:port] \ + (--auth-basic [--auth-password ] | --auth-bearer [TOKEN] --auth-user ) \ + [--allow-cleartext] \ + +``` + +Imports sieve scripts only. Each script is content-addressed in the blob table; the active script is recorded. + +#### Maildir + +``` +vandelay import maildir \ + [--include ...] [--exclude ...] [--folder ...] \ + [--noautomap] [--include-deleted] +``` + +Reads a local Maildir++ tree (a directory with `cur/`, `new/`, `tmp/`). No network. Folder selection mirrors the IMAP importer. + +#### Google Takeout + +``` +vandelay import takeout [--noautomap] +``` + +Scans a directory tree recursively for `.mbox`, `.ics` and `.vcf` files and imports them. Tailored to Google Takeout layouts but works on any such tree; system-label role assignment can be disabled with `--noautomap`. + +#### Microsoft Exchange (EWS) + +``` +vandelay import exchange-ews \ + [--url ] [--mailbox ] \ + [--mailbox-kind primary|archive|public-folders] \ + (--auth-basic [--auth-password ] \ + | --auth-bearer [TOKEN] [--ews-tenant --ews-client-id \ + (--ews-device-code | --ews-client-secret )]) \ + [--ews-connections <1..8>] [--ews-getitem-batch ] [--ews-attachment-batch ] \ + [--ews-no-syncfolderitems] \ + +``` + +Imports a mailbox via EWS, against either on-prem Exchange Server or Exchange Online. Autodiscover is used when `--url` is omitted (a `--mailbox` SMTP address is then required). Supports Basic, pre-acquired bearer, interactive device-code OAuth, and app-only client-credentials OAuth. + +#### Microsoft Exchange (Graph) + +``` +vandelay import exchange-graph \ + (--client-id [--tenant ] | --access-token [TOKEN]) \ + [--user ] \ + [--mailbox-kind primary|archive] \ + [--objects mail,calendar,contacts] \ + [--event-body-format text|html] \ + [--graph-connections <1..16>] [--top <1..1000>] \ + +``` + +Imports a mailbox from Exchange Online via Microsoft Graph. Without `--access-token`, the interactive device-code flow is used. `public-folders` is rejected here (use `exchange-ews` instead). + +### Export + +``` +vandelay export \ + --url \ + (--auth-basic [--auth-password ] | --auth-bearer [TOKEN]) \ + (--account-id | --account-name ) \ + [--objects ] [--prune [--yes]] \ + +``` + +Stateless re-export of `ARCHIVE` into a target JMAP server account. The default behaviour is upsert-only: matched items are updated, unmatched local items are created, but pre-existing target items not covered by the archive are left alone. + +`--prune` enables destructive reconciliation: target objects that do not match anything in the archive are deleted. The confirmation prompt can be skipped with `--yes` for automation. Export speaks JMAP only; no other target protocols are currently supported. + +### Inspect + +``` +vandelay inspect [TYPE] [--limit ] [--offset ] +``` + +Read-only dump of a local archive. This command never opens a network connection and never writes to the archive. + +- Omit `TYPE` for a per-type summary (counts of every object kind plus blob storage stats). +- Pass an object type to dump it: `mailbox`, `email`, `identity`, `sievescript`, `addressbook`, `contactcard`, `calendar`, `calendarevent`, `participantidentity`, `filenode`. +- `mailbox` and `filenode` render as a tree (`--limit`/`--offset` are ignored); all other types use a paginated list and respect `--limit` and `--offset`. + +## Testing + +```sh +cargo build +cargo clippy --all-targets +cargo test +``` + +Live integration tests against a Stalwart server and container-based smoke tests against third-party servers (Dovecot, Cyrus, Radicale, Baikal, Apache `mod_dav`) are gated behind `--ignored` and require Docker plus the seeder fixtures. + +## License + +Licensed under either of + + * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) + +at your option. + +## Copyright + +Copyright (C) 2020, Stalwart Labs LLC diff --git a/assets/importer-exporter.jpg b/assets/importer-exporter.jpg new file mode 100644 index 0000000..1fa96d2 Binary files /dev/null and b/assets/importer-exporter.jpg differ diff --git a/dist-workspace.toml b/dist-workspace.toml new file mode 100644 index 0000000..0ff93c9 --- /dev/null +++ b/dist-workspace.toml @@ -0,0 +1,15 @@ +[workspace] +members = ["cargo:."] + +[dist] +cargo-dist-version = "0.31.0" +ci = "github" +installers = ["shell", "powershell", "npm", "homebrew", "msi"] +tap = "stalwartlabs/homebrew-tap" +npm-scope = "@stalwartlabs" +targets = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "aarch64-unknown-linux-musl", "armv7-unknown-linux-gnueabihf", "armv7-unknown-linux-musleabihf", "arm-unknown-linux-gnueabihf", "arm-unknown-linux-musleabihf", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl", "x86_64-pc-windows-msvc"] +install-path = "CARGO_HOME" +publish-jobs = ["homebrew", "npm"] +install-updater = false +github-attestations = true +allow-dirty = ["ci"] diff --git a/resources/icals/000.ics b/resources/icals/000.ics new file mode 100644 index 0000000..0945dcf --- /dev/null +++ b/resources/icals/000.ics @@ -0,0 +1,40 @@ +BEGIN:VCALENDAR +METHOD:PUBLISH +X-WR-TIMEZONE:America/Vancouver +PRODID:-//Apple Inc.//iCal 3.0//EN +CALSCALE:GREGORIAN +X-WR-CALNAME:weekly +VERSION:2.0 +X-WR-RELCALID:967F55F6-968E-49EF-A050-DAF6DB680D39 +X-APPLE-CALENDAR-COLOR:#F57802 +BEGIN:VTIMEZONE +TZID:America/Vancouver +BEGIN:DAYLIGHT +TZOFFSETFROM:-0800 +TZOFFSETTO:-0700 +DTSTART:20070311T020000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU +TZNAME:PDT +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:-0700 +TZOFFSETTO:-0800 +DTSTART:20071104T020000 +RRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU +TZNAME:PST +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +SEQUENCE:7 +DESCRIPTION:It will pay the bills. +UID:CFE00EC6-AB4A-4FCE-A32F-6076BA1D8578 +TRANSP:OPAQUE +URL;VALUE=URI:http://www.worklessparty.org/ +DTSTART;TZID=America/Vancouver:20090309T090000 +DTSTAMP:20090310T034303Z +SUMMARY:Go to work. +CREATED:20090310T033824Z +DTEND;TZID=America/Vancouver:20090309T170000 +RRULE:FREQ=WEEKLY;INTERVAL=1 +END:VEVENT +END:VCALENDAR diff --git a/resources/icals/001.ics b/resources/icals/001.ics new file mode 100644 index 0000000..6c286c9 --- /dev/null +++ b/resources/icals/001.ics @@ -0,0 +1,20 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Apple Inc.//iCal 3.0//EN +CALSCALE:GREGORIAN +BEGIN:VEVENT +SEQUENCE:1 +DESCRIPTION:Sunday - Partly cloudy with a 20 percent chance of snow show + ers. Highs in the lower to mid 40s.\nMore Information +UID:1205712057-8-1055761864 +TRANSP:TRANSPARENT +STATUS:CONFIRMED +DTSTART;VALUE=DATE:20080316 +DTSTAMP:20080310T010500Z +SUMMARY:Partly Cloudy 48F / F | 9C / C +CREATED:20080316T213048Z +DTEND;VALUE=DATE:20080317 +LOCATION:Leavenworth\, WA +END:VEVENT +END:VCALENDAR diff --git a/resources/icals/002.ics b/resources/icals/002.ics new file mode 100644 index 0000000..fd642c0 --- /dev/null +++ b/resources/icals/002.ics @@ -0,0 +1,22 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Apple Inc.//iCal 3.0//EN +CALSCALE:GREGORIAN +BEGIN:VEVENT +SEQUENCE:1 +DESCRIPTION:Saturday - Mostly cloudy. A chance of snow showers in the mo + rning...then a chance of rain or snow showers in the afternoon. Highs in + the mid 40s to lower 50s. Chance of precipitation 40 percent. \nMore Information + +UID:1205647257-7--150403793 +TRANSP:TRANSPARENT +STATUS:CONFIRMED +DTSTART;VALUE=DATE:20080315 +DTSTAMP:20080310T010500Z +SUMMARY:Partly Cloudy 48F / 25F | 9C / -4C +CREATED:20080316T213048Z +DTEND;VALUE=DATE:20080316 +LOCATION:Leavenworth\, WA +END:VEVENT +END:VCALENDAR diff --git a/resources/icals/003.ics b/resources/icals/003.ics new file mode 100644 index 0000000..d27b6b6 --- /dev/null +++ b/resources/icals/003.ics @@ -0,0 +1,32 @@ +BEGIN:VCALENDAR +PRODID:-//ABC Corporation/NONSGML iCal4j 1.0//EN +CALSCALE:GREGORIAN +VERSION:2.0 +BEGIN:VEVENT +DTSTAMP:20160504T175305Z +DTSTART;TZID=America/New_York:20160514T130000 +SUMMARY:Simple event +DURATION:PT1H +UID:20160504T175305Z-uidGen@fe80:0:0:0:c8d8:72ff:fe94:1087%awdl0 +LOCATION:Mo's bar - back room +DESCRIPTION:We'll continue with the unfinished business from last time\,\ + nin particular:\nCan names start wih a number?\nWhat if they are all num + eric?\nReuse of names - is it valid\nI remind the attendees we have spen + t 3 months on these subjects. We need closure!!! +END:VEVENT +BEGIN:VEVENT +DTSTAMP:20160504T175311Z +DTSTART;TZID=America/New_York:20160514T130000 +SUMMARY:Simple Meeting +DURATION:PT1H +UID:20160504T175311Z-uidGen@fe80:0:0:0:c8d8:72ff:fe94:1087%awdl0 +ATTENDEE;ROLE=REQ-PARTICIPANT;CN=Person 1:mailto:person1@example.com +ATTENDEE;ROLE=OPT-PARTICIPANT;CN=Person 2:mailto:person2@example.com +ORGANIZER;CN=Person 1:mailto:person1@example.com +LOCATION:Mo's bar - back room +DESCRIPTION:We'll continue with the unfinished business from last time\,\ + nin particular:\nCan names start wih a number?\nWhat if they are all num + eric?\nReuse of names - is it valid\nI remind the attendees we have spen + t 3 months on these subjects. We need closure!!! +END:VEVENT +END:VCALENDAR diff --git a/resources/icals/010.ics b/resources/icals/010.ics new file mode 100644 index 0000000..da39cd5 --- /dev/null +++ b/resources/icals/010.ics @@ -0,0 +1,44 @@ +BEGIN:VCALENDAR +METHOD:PUBLISH +PRODID:Microsoft Exchange Server 2010 +VERSION:2.0 +X-WR-CALNAME:Calendar +BEGIN:VTIMEZONE +TZID:Customized Time Zone +BEGIN:STANDARD +DTSTART:16010101T020000 +TZOFFSETFROM:-0400 +TZOFFSETTO:-0500 +RRULE:FREQ=YEARLY;INTERVAL=1;BYDAY=1SU;BYMONTH=11 +END:STANDARD +BEGIN:DAYLIGHT +DTSTART:16010101T020000 +TZOFFSETFROM:-0500 +TZOFFSETTO:-0400 +RRULE:FREQ=YEARLY;INTERVAL=1;BYDAY=2SU;BYMONTH=3 +END:DAYLIGHT +END:VTIMEZONE +BEGIN:VEVENT +DESCRIPTION:[private] +RRULE:FREQ=WEEKLY;UNTIL=20201217T153500Z;INTERVAL=1;BYDAY=TU,TH;WKST=SU +UID:1173422081SZR322 +SUMMARY:[private] +DTSTART;TZID=Customized Time Zone:20200825T103500 +DTEND;TZID=Customized Time Zone:20200825T115000 +CLASS:PUBLIC +PRIORITY:5 +DTSTAMP:20200901T175645Z +TRANSP:OPAQUE +STATUS:CONFIRMED +SEQUENCE:0 +LOCATION: [private] +X-MICROSOFT-CDO-APPT-SEQUENCE:0 +X-MICROSOFT-CDO-BUSYSTATUS:BUSY +X-MICROSOFT-CDO-INTENDEDSTATUS:BUSY +X-MICROSOFT-CDO-ALLDAYEVENT:FALSE +X-MICROSOFT-CDO-IMPORTANCE:1 +X-MICROSOFT-CDO-INSTTYPE:1 +X-MICROSOFT-DONOTFORWARDMEETING:FALSE +X-MICROSOFT-DISALLOW-COUNTER:FALSE +END:VEVENT +END:VCALENDAR \ No newline at end of file diff --git a/resources/icals/011.ics b/resources/icals/011.ics new file mode 100644 index 0000000..4f72c0e --- /dev/null +++ b/resources/icals/011.ics @@ -0,0 +1,40 @@ +BEGIN:VCALENDAR +PRODID:-//Google Inc//Google Calendar 70.9054//EN +VERSION:2.0 +CALSCALE:GREGORIAN +METHOD:PUBLISH +X-WR-CALNAME:ical +X-WR-TIMEZONE:US/Central +X-WR-CALDESC: +BEGIN:VEVENT +UID:98765432-ABCD-DCBB-999A-987765432123 +DTSTART;TZID=US/Central:20170601T090000 +DTEND;TZID=US/Central:20170601T170000 +DTSTAMP:20170727T044436Z +EXDATE;TZID=US/Central:20170706T090000,20170713T090000,20170720T090000,20 + 170803T090000 +LAST-MODIFIED:20170727T044435Z +RRULE:FREQ=WEEKLY;WKST=SU;UNTIL=20170814T045959Z;BYDAY=TH +SEQUENCE:0 +SUMMARY:Recurring weekly meeting from June 1 - Aug 14 (except July 6, July 13, July 20, Aug 3) +END:VEVENT +BEGIN:VEVENT +UID:98765432-ABCD-DCBB-999A-987765432123 +RECURRENCE-ID;TZID=US/Central:20170629T090000 +DTSTART;TZID=US/Central:20170703T090000 +DTEND;TZID=US/Central:20170703T120000 +DTSTAMP:20170727T044436Z +LAST-MODIFIED:20170216T143445Z +SEQUENCE:0 +SUMMARY:Last meeting in June moved to Monday July 3 and shortened to half day +END:VEVENT +BEGIN:VEVENT +UID:12354454-ABCD-DCBB-999A-2349872354897 +DTSTART;TZID=US/Central:20171201T130000 +DTEND;TZID=US/Central:20171201T150000 +DTSTAMP:20170727T044436Z +LAST-MODIFIED:20170727T044435Z +SEQUENCE:0 +SUMMARY:Single event on Dec 1 +END:VEVENT +END:VCALENDAR \ No newline at end of file diff --git a/resources/icals/012.ics b/resources/icals/012.ics new file mode 100644 index 0000000..04eb089 --- /dev/null +++ b/resources/icals/012.ics @@ -0,0 +1,15 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +DTSTART;TZID=Europe/Germany:20241022T230000 +DTEND;TZID=TZID=Europe/Germany:20241023T000000 +RRULE:FREQ=DAILY;WKST=MO;COUNT=4 +DTSTAMP:20241009T153220Z +UID:2m6mt1p89l2anl74915ur3hsgm@google.com +CREATED:20241009T153058Z +LAST-MODIFIED:20241009T153205Z +SEQUENCE:0 +STATUS:CONFIRMED +SUMMARY:TestCal_AllDayRepeatingEvent +TRANSP:TRANSPARENT +END:VEVENT +END:VCALENDAR \ No newline at end of file diff --git a/resources/icals/014.ics b/resources/icals/014.ics new file mode 100644 index 0000000..ff8cc07 --- /dev/null +++ b/resources/icals/014.ics @@ -0,0 +1,78 @@ +BEGIN:VCALENDAR +PRODID:-//lanyrd.com//Lanyrd//EN +X-ORIGINAL-URL:http://lanyrd.com/topics/nodejs/nodejs.ics +X-WR-CALNAME;CHARSET=utf-8:Node.js conferences +VERSION:2.0 +METHOD:PUBLISH +BEGIN:VEVENT +SUMMARY;CHARSET=utf-8:Dyncon 2011 +LOCATION;CHARSET=utf-8:Stockholm, Sweden +URL:http://lanyrd.com/2011/dyncon/ +UID:d4c826dfb701f611416d69b4df81caf9ff80b03a +DTSTART:20110312T200000Z +DTEND;VALUE=DATE:20110314 +END:VEVENT +BEGIN:VEVENT +SUMMARY;CHARSET=utf-8:[Async]: Everything Express +LOCATION;CHARSET=utf-8:Brighton, United Kingdom +URL:http://lanyrd.com/2011/asyncjs-express/ +UID:480a3ad48af5ed8965241f14920f90524f533c18 +DTSTART;VALUE=DATE:20110324 +DTEND;VALUE=DATE:20110325 +END:VEVENT +BEGIN:VEVENT +SUMMARY;CHARSET=utf-8:JSConf US 2011 +LOCATION;CHARSET=utf-8:Portland, United States +URL:http://lanyrd.com/2011/jsconf/ +UID:ed334cc85db5ebdff5ff5a630a7a48631a677dbe +DTSTART;VALUE=DATE:20110502 +DTEND;VALUE=DATE:20110504 +END:VEVENT +BEGIN:VEVENT +SUMMARY;CHARSET=utf-8:NodeConf 2011 +LOCATION;CHARSET=utf-8:Portland, United States +URL:http://lanyrd.com/2011/nodeconf/ +UID:25169a7b1ba5c248278f47120a40878055dc8c15 +DTSTART;VALUE=DATE:20110505 +DTEND;VALUE=DATE:20110506 +END:VEVENT +BEGIN:VEVENT +SUMMARY;CHARSET=utf-8:BrazilJS +LOCATION;CHARSET=utf-8:Fortaleza, Brazil +URL:http://lanyrd.com/2011/braziljs/ +UID:dafee3be83624f3388c5635662229ff11766bb9c +DTSTART;VALUE=DATE:20110513 +DTEND;VALUE=DATE:20110515 +END:VEVENT +BEGIN:VEVENT +SUMMARY;CHARSET=utf-8:Falsy Values +LOCATION;CHARSET=utf-8:Warsaw, Poland +URL:http://lanyrd.com/2011/falsy-values/ +UID:73cad6a09ac4e7310979c6130f871d17d990b5ad +DTSTART;VALUE=DATE:20110518 +DTEND;VALUE=DATE:20110521 +END:VEVENT +BEGIN:VEVENT +SUMMARY;CHARSET=utf-8:nodecamp.eu +LOCATION;CHARSET=utf-8:Cologne, Germany +URL:http://lanyrd.com/2011/nodecampde/ +UID:b728a5fdb5f292b6293e4a2fd97a1ccfc69e9d6f +DTSTART;VALUE=DATE:20110611 +DTEND;VALUE=DATE:20110613 +END:VEVENT +BEGIN:VEVENT +SUMMARY;CHARSET=utf-8:Rich Web Experience 2011 +LOCATION;CHARSET=utf-8:Fort Lauderdale, United States +URL:http://lanyrd.com/2011/rich-web-experience/ +UID:47f6ea3f28af2986a2192fa39a91fa7d60d26b76 +DTSTART;VALUE=DATE:20111129 +DTEND;VALUE=DATE:20111203 +END:VEVENT +BEGIN:VEVENT +SUMMARY;CHARSET=utf-8:Foobar +UID:sdfkf09fsd0 +DTSTART;VALUE=DATE:Next Year +DTEND;VALUE=DATE:20111203 +END:VEVENT + +END:VCALENDAR diff --git a/resources/icals/024.ics b/resources/icals/024.ics new file mode 100644 index 0000000..e019cc5 --- /dev/null +++ b/resources/icals/024.ics @@ -0,0 +1,16 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +UID:eb9e1bd2-ceba-499f-be77-f02773954c72 +SUMMARY:Event with an alarm +DESCRIPTION:This is an event with an alarm. +ORGANIZER;CN=stomlinson@mozilla.com:"mailto:stomlinson@mozilla.com" +DTSTART;TZID="America/Los_Angeles":20130418T110000 +DTEND;TZID="America/Los_Angeles":20130418T120000 +STATUS:CONFIRMED +CLASS:PUBLIC +TRANSP:OPAQUE +LAST-MODIFIED:20130418T175632Z +DTSTAMP:20130418T175632Z +SEQUENCE:3 +END:VEVENT +END:VCALENDAR diff --git a/resources/icals/026.ics b/resources/icals/026.ics new file mode 100644 index 0000000..c6b3f5b --- /dev/null +++ b/resources/icals/026.ics @@ -0,0 +1,75 @@ +BEGIN:VCALENDAR +METHOD:PUBLISH +VERSION:2.0 +X-WR-CALNAME:Test +PRODID:-//Apple Inc.//macOS 11.6.7//EN +X-WR-TIMEZONE:America/Vancouver +CALSCALE:GREGORIAN +BEGIN:VEVENT +TRANSP:OPAQUE +DTEND:20220704T235000Z +X-APPLE-TRAVEL-ADVISORY-BEHAVIOR:AUTOMATIC +ORGANIZER;SCHEDULE-AGENT=CLIENT:mailto:tentative@example.com +UID:31a1ffc9-9b76-465b-ae4a-cadb694c9d37 +DTSTAMP:20220704T225759Z +DESCRIPTION: +STATUS:TENTATIVE +SEQUENCE:0 +SUMMARY:Tentative +LAST-MODIFIED:20220704T230434Z +DTSTART:20220704T230000Z +CREATED:20220704T225437Z +ATTENDEE;CN="tentative@example.com";CUTYPE=INDIVIDUAL;EMAIL="tentative@e + xample.com";PARTSTAT=TENTATIVE;ROLE=REQ-PARTICIPANT;X-NUM-GUESTS=0:mailt + o:tentative@example.com +END:VEVENT +BEGIN:VEVENT +TRANSP:OPAQUE +DTEND;TZID=America/Vancouver:20220705T100000 +X-APPLE-TRAVEL-ADVISORY-BEHAVIOR:AUTOMATIC +ORGANIZER;SCHEDULE-AGENT=CLIENT:mailto:confirmed@example.com +UID:F00F3710-BF4D-46D3-9A2C-1037AB24C6AC +DTSTAMP:20220704T231538Z +DESCRIPTION: +STATUS:CONFIRMED +SEQUENCE:0 +SUMMARY:Confirmed +LAST-MODIFIED:20220704T231538Z +DTSTART;TZID=America/Vancouver:20220705T090000 +CREATED:20220704T231522Z +ATTENDEE;CN="confirmed@example.com";CUTYPE=INDIVIDUAL;EMAIL="confirmed@e + xample.com";PARTSTAT=CONFIRMED;ROLE=REQ-PARTICIPANT;X-NUM-GUESTS=0:mailt + o:confirmed@example.com +END:VEVENT +BEGIN:VEVENT +TRANSP:OPAQUE +DTEND;TZID=America/Vancouver:20220706T140000 +X-APPLE-TRAVEL-ADVISORY-BEHAVIOR:AUTOMATIC +ORGANIZER;SCHEDULE-AGENT=CLIENT:mailto:cancelled@example.com +UID:8CAB46C5-669F-4249-B1AD-52BFA72F4E0A +DTSTAMP:20220704T231547Z +DESCRIPTION: +STATUS:CANCELLED +SEQUENCE:0 +SUMMARY:Cancelled +LAST-MODIFIED:20220704T231559Z +DTSTART;TZID=America/Vancouver:20220706T130000 +CREATED:20220704T231539Z +ATTENDEE;CN="cancelled@example.com";CUTYPE=INDIVIDUAL;EMAIL="cancelled@e + xample.com";PARTSTAT=cancelled;ROLE=REQ-PARTICIPANT;X-NUM-GUESTS=0:mailt + o:cancelled@example.com +END:VEVENT +BEGIN:VEVENT +TRANSP:OPAQUE +DTEND;TZID=America/Vancouver:20220707T100000 +X-APPLE-TRAVEL-ADVISORY-BEHAVIOR:AUTOMATIC +UID:99F615DE-82C6-4CEF-97B8-CD0D3E1EE0D3 +DTSTAMP:20220704T234828Z +DESCRIPTION: +SEQUENCE:0 +SUMMARY:No Status +LAST-MODIFIED:20220704T234828Z +DTSTART;TZID=America/Vancouver:20220707T090000 +CREATED:20220704T234823Z +END:VEVENT +END:VCALENDAR diff --git a/resources/icals/041.ics b/resources/icals/041.ics new file mode 100644 index 0000000..c845cd3 --- /dev/null +++ b/resources/icals/041.ics @@ -0,0 +1,66 @@ +BEGIN:VCALENDAR +BEGIN:VEVENT +CREATED:20231214T150120Z +LAST-MODIFIED:20231214T150224Z +DTSTAMP:20231214T150224Z +UID:df21da35-3a0b-434a-9211-da9f9095fb16 +SUMMARY:test export import +CATEGORIES:Professionnel +DTSTART;TZID=Europe/Paris:20231215T150000 +DTEND;TZID=Europe/Paris:20231215T160000 +TRANSP:OPAQUE +LOCATION:dijon +DESCRIPTION;ALTREP="data:text/html,test":test +END:VEVENT +BEGIN:VTIMEZONE +TZID: 1 +BEGIN:STANDARD +DTSTART:16010101T030000 +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +RRULE:FREQ=YEARLY;INTERVAL=1;BYDAY=-1SU;BYMONTH=10 +END:STANDARD +BEGIN:DAYLIGHT +DTSTART:16010101T020000 +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +RRULE:FREQ=YEARLY;INTERVAL=1;BYDAY=-1SU;BYMONTH=3 +END:DAYLIGHT +END:VTIMEZONE +BEGIN:VEVENT +CREATED:20240229T160626Z +LAST-MODIFIED:20231221T141000Z +DTSTAMP:20231221T141000Z +UID:040000008200E00074C5B7101A82E008000000008D1F7DAB8632DA01000000000000000 + 0100000008EF89A92C90D1342BF1F2CB714276CCC +SUMMARY:Atelier groupes portail/Nuage +PRIORITY:5 +STATUS:CONFIRMED +DTSTART;TZID= 1:20240108T110000 +DTEND;TZID= 1:20240108T120000 +CLASS:PUBLIC +DESCRIPTION;LANGUAGE=fr-FR:Désolé pour ces changements de date\, j'espèr + e que ce dernier créneau conviendra à tout le monde. \n\nJe vous propose + cette date pour un atelier sur les soucis avec les commandes à passer sur + les serveurs nextcloud pour la mise en place des groupes. Dites moi si ça + ne convient pas. Merci !\n +TRANSP:OPAQUE +SEQUENCE:3 +LOCATION;LANGUAGE=fr-FR:https://visio-agents.education.fr/ +X-MICROSOFT-CDO-APPT-SEQUENCE:3 +X-MICROSOFT-CDO-OWNERAPPTID:2122095757 +X-MICROSOFT-CDO-BUSYSTATUS:TENTATIVE +X-MICROSOFT-CDO-INTENDEDSTATUS:BUSY +X-MICROSOFT-CDO-ALLDAYEVENT:FALSE +X-MICROSOFT-CDO-IMPORTANCE:1 +X-MICROSOFT-CDO-INSTTYPE:0 +X-MICROSOFT-DONOTFORWARDMEETING:FALSE +X-MICROSOFT-DISALLOW-COUNTER:FALSE +X-MICROSOFT-LOCATIONDISPLAYNAME:https://visio-agents.education.fr/ +X-MICROSOFT-LOCATIONSOURCE:None +X-MICROSOFT-LATITUDE:0 +X-MICROSOFT-LONGITUDE:0 +X-MOZ-RECEIVED-SEQUENCE:3 +X-MOZ-RECEIVED-DTSTAMP:20231221T141000Z +END:VEVENT +END:VCALENDAR diff --git a/resources/icals/070.ics b/resources/icals/070.ics new file mode 100644 index 0000000..bbee664 --- /dev/null +++ b/resources/icals/070.ics @@ -0,0 +1,235 @@ +BEGIN:VCALENDAR +PRODID:-//Offline Calendar//iCal Import/Export 2.8.1//EN +VERSION:2.0 +METHOD:PUBLISH +CALSCALE:GREGORIAN +BEGIN:VTIMEZONE +TZID:Europe/London +TZURL:http://tzurl.org/zoneinfo/Europe/London +X-LIC-LOCATION:Europe/London +BEGIN:DAYLIGHT +TZOFFSETFROM:+0000 +TZOFFSETTO:+0100 +TZNAME:BST +DTSTART:19810329T010000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:+0100 +TZOFFSETTO:+0000 +TZNAME:GMT +DTSTART:19961027T020000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU +END:STANDARD +BEGIN:STANDARD +TZOFFSETFROM:-000115 +TZOFFSETTO:+0000 +TZNAME:GMT +DTSTART:18471201T000115 +RDATE:18471201T000115 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETFROM:+0000 +TZOFFSETTO:+0100 +TZNAME:BST +DTSTART:19160521T020000 +RDATE:19160521T020000 +RDATE:19170408T020000 +RDATE:19180324T020000 +RDATE:19190330T020000 +RDATE:19200328T020000 +RDATE:19210403T020000 +RDATE:19220326T020000 +RDATE:19230422T020000 +RDATE:19240413T020000 +RDATE:19250419T020000 +RDATE:19260418T020000 +RDATE:19270410T020000 +RDATE:19280422T020000 +RDATE:19290421T020000 +RDATE:19300413T020000 +RDATE:19310419T020000 +RDATE:19320417T020000 +RDATE:19330409T020000 +RDATE:19340422T020000 +RDATE:19350414T020000 +RDATE:19360419T020000 +RDATE:19370418T020000 +RDATE:19380410T020000 +RDATE:19390416T020000 +RDATE:19400225T020000 +RDATE:19460414T020000 +RDATE:19470316T020000 +RDATE:19480314T020000 +RDATE:19490403T020000 +RDATE:19500416T020000 +RDATE:19510415T020000 +RDATE:19520420T020000 +RDATE:19530419T020000 +RDATE:19540411T020000 +RDATE:19550417T020000 +RDATE:19560422T020000 +RDATE:19570414T020000 +RDATE:19580420T020000 +RDATE:19590419T020000 +RDATE:19600410T020000 +RDATE:19610326T020000 +RDATE:19620325T020000 +RDATE:19630331T020000 +RDATE:19640322T020000 +RDATE:19650321T020000 +RDATE:19660320T020000 +RDATE:19670319T020000 +RDATE:19680218T020000 +RDATE:19720319T020000 +RDATE:19730318T020000 +RDATE:19740317T020000 +RDATE:19750316T020000 +RDATE:19760321T020000 +RDATE:19770320T020000 +RDATE:19780319T020000 +RDATE:19790318T020000 +RDATE:19800316T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:+0100 +TZOFFSETTO:+0000 +TZNAME:GMT +DTSTART:19161001T030000 +RDATE:19161001T030000 +RDATE:19170917T030000 +RDATE:19180930T030000 +RDATE:19190929T030000 +RDATE:19201025T030000 +RDATE:19211003T030000 +RDATE:19221008T030000 +RDATE:19230916T030000 +RDATE:19240921T030000 +RDATE:19251004T030000 +RDATE:19261003T030000 +RDATE:19271002T030000 +RDATE:19281007T030000 +RDATE:19291006T030000 +RDATE:19301005T030000 +RDATE:19311004T030000 +RDATE:19321002T030000 +RDATE:19331008T030000 +RDATE:19341007T030000 +RDATE:19351006T030000 +RDATE:19361004T030000 +RDATE:19371003T030000 +RDATE:19381002T030000 +RDATE:19391119T030000 +RDATE:19451007T030000 +RDATE:19461006T030000 +RDATE:19471102T030000 +RDATE:19481031T030000 +RDATE:19491030T030000 +RDATE:19501022T030000 +RDATE:19511021T030000 +RDATE:19521026T030000 +RDATE:19531004T030000 +RDATE:19541003T030000 +RDATE:19551002T030000 +RDATE:19561007T030000 +RDATE:19571006T030000 +RDATE:19581005T030000 +RDATE:19591004T030000 +RDATE:19601002T030000 +RDATE:19611029T030000 +RDATE:19621028T030000 +RDATE:19631027T030000 +RDATE:19641025T030000 +RDATE:19651024T030000 +RDATE:19661023T030000 +RDATE:19671029T030000 +RDATE:19711031T030000 +RDATE:19721029T030000 +RDATE:19731028T030000 +RDATE:19741027T030000 +RDATE:19751026T030000 +RDATE:19761024T030000 +RDATE:19771023T030000 +RDATE:19781029T030000 +RDATE:19791028T030000 +RDATE:19801026T030000 +RDATE:19811025T020000 +RDATE:19821024T020000 +RDATE:19831023T020000 +RDATE:19841028T020000 +RDATE:19851027T020000 +RDATE:19861026T020000 +RDATE:19871025T020000 +RDATE:19881023T020000 +RDATE:19891029T020000 +RDATE:19901028T020000 +RDATE:19911027T020000 +RDATE:19921025T020000 +RDATE:19931024T020000 +RDATE:19941023T020000 +RDATE:19951022T020000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +TZNAME:BDST +DTSTART:19410504T010000 +RDATE:19410504T010000 +RDATE:19420405T010000 +RDATE:19430404T010000 +RDATE:19440402T010000 +RDATE:19450402T010000 +RDATE:19470413T010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +TZNAME:BST +DTSTART:19410810T030000 +RDATE:19410810T030000 +RDATE:19420809T030000 +RDATE:19430815T030000 +RDATE:19440917T030000 +RDATE:19450715T030000 +RDATE:19470810T030000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:+0100 +TZOFFSETTO:+0100 +TZNAME:BST +DTSTART:19681026T230000 +RDATE:19681026T230000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETFROM:+0000 +TZOFFSETTO:+0000 +TZNAME:GMT +DTSTART:19960101T000000 +RDATE:19960101T000000 +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +DTSTAMP:20241005T113036Z +UID:17281276213728ad54d03afa44d1ca60b8c52afaece9e@sufficientlysecure.org +SUMMARY:event with alarms android +STATUS:CONFIRMED +DTSTART;TZID=Europe/London:20241005T130000 +DTEND:20241005T130000Z +LAST-MODIFIED:20241005T112701Z +BEGIN:VALARM +TRIGGER:-PT30M +ACTION:DISPLAY +DESCRIPTION:event with alarms android +END:VALARM +BEGIN:VALARM +TRIGGER:-PT25M +ACTION:DISPLAY +DESCRIPTION:event with alarms android +END:VALARM +BEGIN:VALARM +TRIGGER:-PT5M +ACTION:DISPLAY +DESCRIPTION:event with alarms android +END:VALARM +END:VEVENT +END:VCALENDAR diff --git a/resources/icals/075.ics b/resources/icals/075.ics new file mode 100644 index 0000000..851a8db --- /dev/null +++ b/resources/icals/075.ics @@ -0,0 +1,626 @@ +BEGIN:VCALENDAR +PRODID:-//Mozilla.org/NONSGML Mozilla Calendar V1.1//EN +VERSION:2.0 +BEGIN:VTIMEZONE +TZID:Europe/London +X-TZINFO:Europe/London[2024a] +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:-000115 +TZNAME:Europe/London(STD) +DTSTART:18471201T000000 +RDATE:18471201T000000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19160521T020000 +RDATE:19160521T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19161001T030000 +RDATE:19161001T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19170408T020000 +RDATE:19170408T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19170917T030000 +RDATE:19170917T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19180324T020000 +RDATE:19180324T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19180930T030000 +RRULE:FREQ=YEARLY;BYMONTH=9;BYDAY=-1MO;UNTIL=19190929T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19190330T020000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU;UNTIL=19200328T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19201025T030000 +RDATE:19201025T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19210403T020000 +RDATE:19210403T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19211003T030000 +RDATE:19211003T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19220326T020000 +RDATE:19220326T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19221008T030000 +RDATE:19221008T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19230422T020000 +RDATE:19230422T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19240413T020000 +RDATE:19240413T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19230916T030000 +RRULE:FREQ=YEARLY;BYMONTH=9;BYDAY=3SU;UNTIL=19240921T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19250419T020000 +RRULE:FREQ=YEARLY;BYMONTH=4;BYDAY=3SU;UNTIL=19260418T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19270410T020000 +RDATE:19270410T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19280422T020000 +RDATE:19280422T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19290421T020000 +RDATE:19290421T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19300413T020000 +RDATE:19300413T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19310419T020000 +RRULE:FREQ=YEARLY;BYMONTH=4;BYDAY=3SU;UNTIL=19320417T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19251004T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=1SU;UNTIL=19321002T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19330409T020000 +RDATE:19330409T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19331008T030000 +RDATE:19331008T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19340422T020000 +RDATE:19340422T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19350414T020000 +RDATE:19350414T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19360419T020000 +RRULE:FREQ=YEARLY;BYMONTH=4;BYDAY=3SU;UNTIL=19370418T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19380410T020000 +RDATE:19380410T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19341007T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=1SU;UNTIL=19381002T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19390416T020000 +RDATE:19390416T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19400225T020000 +RDATE:19400225T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+020000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(DST) +DTSTART:19410504T020000 +RDATE:19410504T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+020000 +TZNAME:Europe/London(DST) +DTSTART:19410810T030000 +RDATE:19410810T030000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+020000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(DST) +DTSTART:19420405T020000 +RDATE:19420405T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+020000 +TZNAME:Europe/London(DST) +DTSTART:19420809T030000 +RDATE:19420809T030000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+020000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(DST) +DTSTART:19430404T020000 +RDATE:19430404T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+020000 +TZNAME:Europe/London(DST) +DTSTART:19430815T030000 +RDATE:19430815T030000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+020000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(DST) +DTSTART:19440402T020000 +RDATE:19440402T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+020000 +TZNAME:Europe/London(DST) +DTSTART:19440917T030000 +RDATE:19440917T030000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+020000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(DST) +DTSTART:19450402T020000 +RDATE:19450402T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19391119T030000 +RDATE:19391119T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+020000 +TZNAME:Europe/London(DST) +DTSTART:19450715T030000 +RDATE:19450715T030000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19460414T020000 +RDATE:19460414T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19470316T020000 +RDATE:19470316T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+020000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(DST) +DTSTART:19470413T020000 +RDATE:19470413T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19451007T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=1SU;UNTIL=19461006T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+020000 +TZNAME:Europe/London(DST) +DTSTART:19470810T030000 +RDATE:19470810T030000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19471102T030000 +RDATE:19471102T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19480314T020000 +RDATE:19480314T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19490403T020000 +RDATE:19490403T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19481031T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU;UNTIL=19491030T030000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19501022T030000 +RDATE:19501022T030000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19511021T030000 +RDATE:19511021T030000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19521026T030000 +RDATE:19521026T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19500416T020000 +RRULE:FREQ=YEARLY;BYMONTH=4;BYDAY=3SU;UNTIL=19530419T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19540411T020000 +RDATE:19540411T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19550417T020000 +RDATE:19550417T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19560422T020000 +RDATE:19560422T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19570414T020000 +RDATE:19570414T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19580420T020000 +RRULE:FREQ=YEARLY;BYMONTH=4;BYDAY=3SU;UNTIL=19590419T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19600410T020000 +RDATE:19600410T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19531004T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=1SU;UNTIL=19601002T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19610326T020000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU;UNTIL=19630331T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19640322T020000 +RDATE:19640322T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19611029T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU;UNTIL=19641025T030000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19651024T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=4SU;UNTIL=19661023T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19650321T020000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=3SU;UNTIL=19670319T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19671029T030000 +RDATE:19671029T030000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+010000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19681027T000000 +RDATE:19681027T000000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19680218T020000 +RDATE:19680218T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19711031T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU;UNTIL=19751026T030000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19761024T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=4SU;UNTIL=19771023T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19720319T020000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=3SU;UNTIL=19800316T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19781029T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU;UNTIL=19801026T030000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19811025T020000 +RDATE:19811025T020000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19821024T020000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=4SU;UNTIL=19831023T020000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19841028T020000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU;UNTIL=19871025T020000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19881023T020000 +RDATE:19881023T020000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19891029T020000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU;UNTIL=19921025T020000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19931024T020000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=4SU;UNTIL=19951022T020000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19810329T010000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU;UNTIL=19960331T010000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19961027T020000 +RDATE:19961027T020000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:(DST) +DTSTART:19970330T010000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:(STD) +DTSTART:19971026T020000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +CREATED:20241023T173412Z +LAST-MODIFIED:20241023T173630Z +DTSTAMP:20241023T173630Z +UID:731b9b91-cf72-499b-bbc9-c53c28e21fc7 +SUMMARY:event +X-MOZ-LASTACK:20241023T173630Z +DTSTART;TZID=Europe/London:20241023T190000 +DTEND;TZID=Europe/London:20241023T200000 +TRANSP:OPAQUE +X-MOZ-GENERATION:3 +X-MOZ-SNOOZE-TIME:20241023T174130Z +BEGIN:VALARM +ACTION:DISPLAY +TRIGGER:-PT1M +DESCRIPTION:Mozilla Standardbeschreibung +END:VALARM +BEGIN:VALARM +ACTION:DISPLAY +TRIGGER:-PT24M +DESCRIPTION:Mozilla Standardbeschreibung +END:VALARM +END:VEVENT +END:VCALENDAR diff --git a/resources/icals/077.ics b/resources/icals/077.ics new file mode 100644 index 0000000..524f069 --- /dev/null +++ b/resources/icals/077.ics @@ -0,0 +1,625 @@ +BEGIN:VCALENDAR +PRODID:-//Mozilla.org/NONSGML Mozilla Calendar V1.1//EN +VERSION:2.0 +BEGIN:VTIMEZONE +TZID:Europe/London +X-TZINFO:Europe/London[2024a] +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:-000115 +TZNAME:Europe/London(STD) +DTSTART:18471201T000000 +RDATE:18471201T000000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19160521T020000 +RDATE:19160521T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19161001T030000 +RDATE:19161001T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19170408T020000 +RDATE:19170408T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19170917T030000 +RDATE:19170917T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19180324T020000 +RDATE:19180324T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19180930T030000 +RRULE:FREQ=YEARLY;BYMONTH=9;BYDAY=-1MO;UNTIL=19190929T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19190330T020000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU;UNTIL=19200328T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19201025T030000 +RDATE:19201025T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19210403T020000 +RDATE:19210403T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19211003T030000 +RDATE:19211003T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19220326T020000 +RDATE:19220326T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19221008T030000 +RDATE:19221008T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19230422T020000 +RDATE:19230422T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19240413T020000 +RDATE:19240413T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19230916T030000 +RRULE:FREQ=YEARLY;BYMONTH=9;BYDAY=3SU;UNTIL=19240921T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19250419T020000 +RRULE:FREQ=YEARLY;BYMONTH=4;BYDAY=3SU;UNTIL=19260418T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19270410T020000 +RDATE:19270410T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19280422T020000 +RDATE:19280422T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19290421T020000 +RDATE:19290421T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19300413T020000 +RDATE:19300413T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19310419T020000 +RRULE:FREQ=YEARLY;BYMONTH=4;BYDAY=3SU;UNTIL=19320417T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19251004T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=1SU;UNTIL=19321002T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19330409T020000 +RDATE:19330409T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19331008T030000 +RDATE:19331008T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19340422T020000 +RDATE:19340422T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19350414T020000 +RDATE:19350414T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19360419T020000 +RRULE:FREQ=YEARLY;BYMONTH=4;BYDAY=3SU;UNTIL=19370418T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19380410T020000 +RDATE:19380410T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19341007T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=1SU;UNTIL=19381002T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19390416T020000 +RDATE:19390416T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19400225T020000 +RDATE:19400225T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+020000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(DST) +DTSTART:19410504T020000 +RDATE:19410504T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+020000 +TZNAME:Europe/London(DST) +DTSTART:19410810T030000 +RDATE:19410810T030000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+020000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(DST) +DTSTART:19420405T020000 +RDATE:19420405T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+020000 +TZNAME:Europe/London(DST) +DTSTART:19420809T030000 +RDATE:19420809T030000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+020000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(DST) +DTSTART:19430404T020000 +RDATE:19430404T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+020000 +TZNAME:Europe/London(DST) +DTSTART:19430815T030000 +RDATE:19430815T030000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+020000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(DST) +DTSTART:19440402T020000 +RDATE:19440402T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+020000 +TZNAME:Europe/London(DST) +DTSTART:19440917T030000 +RDATE:19440917T030000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+020000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(DST) +DTSTART:19450402T020000 +RDATE:19450402T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19391119T030000 +RDATE:19391119T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+020000 +TZNAME:Europe/London(DST) +DTSTART:19450715T030000 +RDATE:19450715T030000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19460414T020000 +RDATE:19460414T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19470316T020000 +RDATE:19470316T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+020000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(DST) +DTSTART:19470413T020000 +RDATE:19470413T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19451007T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=1SU;UNTIL=19461006T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+020000 +TZNAME:Europe/London(DST) +DTSTART:19470810T030000 +RDATE:19470810T030000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19471102T030000 +RDATE:19471102T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19480314T020000 +RDATE:19480314T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19490403T020000 +RDATE:19490403T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19481031T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU;UNTIL=19491030T030000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19501022T030000 +RDATE:19501022T030000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19511021T030000 +RDATE:19511021T030000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19521026T030000 +RDATE:19521026T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19500416T020000 +RRULE:FREQ=YEARLY;BYMONTH=4;BYDAY=3SU;UNTIL=19530419T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19540411T020000 +RDATE:19540411T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19550417T020000 +RDATE:19550417T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19560422T020000 +RDATE:19560422T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19570414T020000 +RDATE:19570414T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19580420T020000 +RRULE:FREQ=YEARLY;BYMONTH=4;BYDAY=3SU;UNTIL=19590419T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19600410T020000 +RDATE:19600410T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19531004T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=1SU;UNTIL=19601002T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19610326T020000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU;UNTIL=19630331T020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19640322T020000 +RDATE:19640322T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19611029T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU;UNTIL=19641025T030000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19651024T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=4SU;UNTIL=19661023T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19650321T020000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=3SU;UNTIL=19670319T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19671029T030000 +RDATE:19671029T030000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+010000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19681027T000000 +RDATE:19681027T000000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19680218T020000 +RDATE:19680218T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19711031T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU;UNTIL=19751026T030000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19761024T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=4SU;UNTIL=19771023T030000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19720319T020000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=3SU;UNTIL=19800316T020000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19781029T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU;UNTIL=19801026T030000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19811025T020000 +RDATE:19811025T020000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19821024T020000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=4SU;UNTIL=19831023T020000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19841028T020000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU;UNTIL=19871025T020000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19881023T020000 +RDATE:19881023T020000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19891029T020000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU;UNTIL=19921025T020000 +END:STANDARD +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19931024T020000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=4SU;UNTIL=19951022T020000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:Europe/London(DST) +DTSTART:19810329T010000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU;UNTIL=19960331T010000 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:Europe/London(STD) +DTSTART:19961027T020000 +RDATE:19961027T020000 +END:STANDARD +BEGIN:DAYLIGHT +TZOFFSETTO:+010000 +TZOFFSETFROM:+000000 +TZNAME:(DST) +DTSTART:19970330T010000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETTO:+000000 +TZOFFSETFROM:+010000 +TZNAME:(STD) +DTSTART:19971026T020000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +CREATED:20241023T131035Z +LAST-MODIFIED:20241023T141941Z +DTSTAMP:20241023T141941Z +UID:b9a23b47-f109-4e7a-908c-75e925b27def +SUMMARY:event with alarms +X-MOZ-LASTACK:20241023T141941Z +DTSTART;TZID=Europe/London:20241023T150000 +DTEND;TZID=Europe/London:20241023T160000 +TRANSP:OPAQUE +X-MOZ-GENERATION:6 +BEGIN:VALARM +ACTION:DISPLAY +TRIGGER:-PT15M +DESCRIPTION:Mozilla Standardbeschreibung +END:VALARM +BEGIN:VALARM +ACTION:DISPLAY +TRIGGER:-PT45M +DESCRIPTION:Mozilla Standardbeschreibung +END:VALARM +END:VEVENT +END:VCALENDAR diff --git a/resources/icals/088.ics b/resources/icals/088.ics new file mode 100644 index 0000000..9c5288a --- /dev/null +++ b/resources/icals/088.ics @@ -0,0 +1,55 @@ +BEGIN:VCALENDAR +X-SOURCE-URL:https://github.com/pimutils/khal/issues/152#issuecomment-933635248 +VERSION:2.0 +PRODID:-//PIMUTILS.ORG//NONSGML khal / icalendar //EN +BEGIN:VTIMEZONE +TZID:Western/Central Europe +BEGIN:STANDARD +DTSTART:19501029T020000 +RRULE:FREQ=YEARLY;BYMINUTE=0;BYHOUR=2;BYDAY=-1SU;BYMONTH=10 +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +END:STANDARD +BEGIN:DAYLIGHT +DTSTART:19500326T020000 +RRULE:FREQ=YEARLY;BYMINUTE=0;BYHOUR=2;BYDAY=-1SU;BYMONTH=3 +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +END:DAYLIGHT +END:VTIMEZONE +BEGIN:VEVENT +SUMMARY:(omitted) +DTSTART;TZID="Western/Central Europe";VALUE=DATE-TIME:20211101T160000 +DTEND;TZID="Western/Central Europe";VALUE=DATE-TIME:20211101T163000 +DTSTAMP:20211004T150245Z +UID:BF5109494E67AAE20025875100566D31-Lotus_Notes_Generated +RECURRENCE-ID;RANGE=THISANDFUTURE:20211101T150000Z +SEQUENCE:0 +RDATE;TZID="Western/Central Europe";VALUE=PERIOD:20211101T160000/20211101T + 163000,20211206T160000/20211206T163000,20220103T160000/20220103T163000,202 + 20207T160000/20220207T163000 +ATTENDEE;CN="(omitted)";PARTSTAT=ACCEPTED;ROLE=CHAIR;RSVP=FAL + SE:mailto:omitted@example.com +CLASS:PUBLIC +TRANSP:OPAQUE +X-LOTUS-APPTTYPE:3 +X-LOTUS-AUDIOVIDEOFLAGS:0 +X-LOTUS-BROADCAST:FALSE +X-LOTUS-CHANGE-INST-DATES:20211101T150000Z\,20211206T150000Z\,20220103T150 + 000Z\,20220207T150000Z +X-LOTUS-CHILD-UID:567EFBAF6CBD07FC0025875100566D3B +X-LOTUS-INITIAL-RDATES:20211101T150000Z\,20211206T150000Z\,20220103T150000 + Z\,20220207T150000Z +X-LOTUS-LASTALL-RDATES;TZID="Western/Central Europe":20211101T160000\,2021 + 1206T160000\,20220103T160000\,20220207T160000 +X-LOTUS-NOTESVERSION:2 +X-LOTUS-NOTICETYPE:I +X-LOTUS-RECURID;RANGE=THISANDFUTURE:20211101T150000Z +X-LOTUS-UPDATE-SEQ:2 +X-LOTUS-UPDATE-WISL:$W:1\;$O:1\;$M:1\;RequiredAttendees:1\;INetRequiredNam + es:1\;AltRequiredNames:1\;StorageRequiredNames:1\;OptionalAttendees:1\;INe + tOptionalNames:1\;AltOptionalNames:1\;StorageOptionalNames:1\;ApptUNIDURL: + 1\;STUnyteConferenceURL:1\;STUnyteConferenceID:1\;SametimeType:1\;WhiteBoa + rdContent:1\;STRoomName:1\;$S:2\;$B:2\;$L:2\;$E:2\;$R:2 +END:VEVENT +END:VCALENDAR diff --git a/resources/icals/159.ics b/resources/icals/159.ics new file mode 100644 index 0000000..3a22d08 --- /dev/null +++ b/resources/icals/159.ics @@ -0,0 +1,436 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Last.fm Limited Event Feeds//NONSGML//EN +X-WR-CALNAME:Last.fm Events +X-WR-CALDESC:Event listing - supplied by http://www.Last.fm + +BEGIN:VEVENT +UID:LFMEVENT-1407206 +URL;VALUE=URI:http://www.last.fm/event/1407206+Gerard+van+Maasakkers+en+de+vaste+mannen+-+Deze+Jongen +DTSTAMP:20100210T203000Z +DTSTART:20100210T203000 +DTEND:20100210T235900 +SUMMARY:Gerard van Maasakkers en de vaste mannen - Deze Jongen +DESCRIPTION:Gerard van Maasakkers + \n\nhttp://www.last.fm/event/1407206+Gerard+van+Maasakkers+en+de+vaste+mannen+-+Deze+Jongen +LOCATION:Deventer Schouwburg\, Netherlands +GEO:6.161613;52.255862 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1361665 +URL;VALUE=URI:http://www.last.fm/event/1361665+Millennium+Jazz+Orchestra+en+Frank+Marocco +DTSTAMP:20100212T200000Z +DTSTART:20100212T200000 +DTEND:20100212T235900 +SUMMARY:Millennium Jazz Orchestra en Frank Marocco +DESCRIPTION:Millennium Jazz Orchestra\, Frank Marocco + \n\nhttp://www.last.fm/event/1361665+Millennium+Jazz+Orchestra+en+Frank+Marocco +LOCATION:Deventer Schouwburg\, Netherlands +GEO:6.161613;52.255862 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1358028 +URL;VALUE=URI:http://www.last.fm/event/1358028+Zu+at+Burgerweeshuis+on+12+February+2010 +DTSTAMP:20100212T210000Z +DTSTART:20100212T210000 +DTEND:20100212T235900 +SUMMARY:Zu + The New Earth Group at Burgerweeshuis +DESCRIPTION:Zu\, The New Earth Group + \n\nhttp://www.last.fm/event/1358028+Zu+at+Burgerweeshuis+on+12+February+2010 +LOCATION:Burgerweeshuis\, Netherlands +GEO:6.155862;52.254341 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1319333 +URL;VALUE=URI:http://www.last.fm/event/1319333+Fink+at+Burgerweeshuis+on+13+February+2010 +DTSTAMP:20100213T210000Z +DTSTART:20100213T210000 +DTEND:20100213T235900 +SUMMARY:Fink at Burgerweeshuis +DESCRIPTION:Fink + \n\nhttp://www.last.fm/event/1319333+Fink+at+Burgerweeshuis+on+13+February+2010 +LOCATION:Burgerweeshuis\, Netherlands +GEO:6.155862;52.254341 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1315921 +URL;VALUE=URI:http://www.last.fm/event/1315921+Love+For+Lunch +DTSTAMP:20100219T093200Z +DTSTART:20100219T000000 +DTEND:20100219T235900 +SUMMARY:Love For Lunch +DESCRIPTION:Gare du Nord + \n\nhttp://www.last.fm/event/1315921+Love+For+Lunch +LOCATION:Deventer Schouwburg\, Netherlands +GEO:6.161613;52.255862 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1361649 +URL;VALUE=URI:http://www.last.fm/event/1361649+Amsterdam+Klezmer+Band+at+Theater+Bouwkunde+on+19+February+2010 +DTSTAMP:20100219T203000Z +DTSTART:20100219T203000 +DTEND:20100219T235900 +SUMMARY:Amsterdam Klezmer Band at Theater Bouwkunde +DESCRIPTION:Amsterdam Klezmer Band + \n\nhttp://www.last.fm/event/1361649+Amsterdam+Klezmer+Band+at+Theater+Bouwkunde+on+19+February+2010 +LOCATION:Theater Bouwkunde\, Netherlands +GEO:6.153074;52.253485 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1334619 +URL;VALUE=URI:http://www.last.fm/event/1334619+Bettie+Serveert+at+Burgerweeshuis+on+19+February+2010 +DTSTAMP:20100219T210000Z +DTSTART:20100219T210000 +DTEND:20100219T235900 +SUMMARY:Bettie Serveert + The Monroes at Burgerweeshuis +DESCRIPTION:Bettie Serveert\, The Monroes + \n\nhttp://www.last.fm/event/1334619+Bettie+Serveert+at+Burgerweeshuis+on+19+February+2010 +LOCATION:Burgerweeshuis\, Netherlands +GEO:6.155862;52.254341 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1358751 +URL;VALUE=URI:http://www.last.fm/event/1358751+Ethisch+Reveil +DTSTAMP:20100219T210000Z +DTSTART:20100219T210000 +DTEND:20100219T235900 +SUMMARY:Ethisch Reveil +DESCRIPTION:Gentle Veincut\, Fine China Superbone + \n\nhttp://www.last.fm/event/1358751+Ethisch+Reveil +LOCATION:Havenkwartier\, Netherlands +GEO:6.168775;52.24939 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1392314 +URL;VALUE=URI:http://www.last.fm/event/1392314+The+Monolith+Deathcult+at+Burgerweeshuis+on+26+February+2010 +DTSTAMP:20100226T150100Z +DTSTART:20100226T000000 +DTEND:20100226T235900 +SUMMARY:The Monolith Deathcult + Erebus at Burgerweeshuis +DESCRIPTION:The Monolith Deathcult\, Erebus + \n\nhttp://www.last.fm/event/1392314+The+Monolith+Deathcult+at+Burgerweeshuis+on+26+February+2010 +LOCATION:Burgerweeshuis\, Netherlands +GEO:6.155862;52.254341 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1264138 +URL;VALUE=URI:http://www.last.fm/event/1264138+Loefah+at+Burgerweeshuis+on+27+February+2010 +DTSTAMP:20100227T163800Z +DTSTART:20100227T000000 +DTEND:20100227T235900 +SUMMARY:Loefah + 2562 + more at Burgerweeshuis +DESCRIPTION:Loefah\, 2562\, Cavemen + \n\nhttp://www.last.fm/event/1264138+Loefah+at+Burgerweeshuis+on+27+February+2010 +LOCATION:Burgerweeshuis\, Netherlands +GEO:6.155862;52.254341 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1391634 +URL;VALUE=URI:http://www.last.fm/event/1391634+JW+Roy+weet+het+zeker +DTSTAMP:20100227T203000Z +DTSTART:20100227T203000 +DTEND:20100227T235900 +SUMMARY:JW Roy weet het zeker +DESCRIPTION:JW Roy + \n\nhttp://www.last.fm/event/1391634+JW+Roy+weet+het+zeker +LOCATION:theater bouwkunde\, Netherlands +GEO:6.153073;52.253485 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1361670 +URL;VALUE=URI:http://www.last.fm/event/1361670+Millennium+Jazz+Orchestra+at+Theater+Bouwkunde+on+3+March+2010 +DTSTAMP:20100303T210000Z +DTSTART:20100303T210000 +DTEND:20100303T235900 +SUMMARY:Millennium Jazz Orchestra + Grant Stewart at Theater Bouwkunde +DESCRIPTION:Millennium Jazz Orchestra\, Grant Stewart + \n\nhttp://www.last.fm/event/1361670+Millennium+Jazz+Orchestra+at+Theater+Bouwkunde+on+3+March+2010 +LOCATION:Theater Bouwkunde\, Netherlands +GEO:6.153074;52.253485 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1283011 +URL;VALUE=URI:http://www.last.fm/event/1283011+Suimasen+at+Burgerweeshuis+on+6+March+2010 +DTSTAMP:20100306T103300Z +DTSTART:20100306T000000 +DTEND:20100306T235900 +SUMMARY:Suimasen at Burgerweeshuis +DESCRIPTION:Suimasen + \n\nhttp://www.last.fm/event/1283011+Suimasen+at+Burgerweeshuis+on+6+March+2010 +LOCATION:Burgerweeshuis\, Netherlands +GEO:6.155862;52.254341 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1302136 +URL;VALUE=URI:http://www.last.fm/event/1302136+El+Pino+and+the+Volunteers+at+Burgerweeshuis+on+6+March+2010 +DTSTAMP:20100306T203000Z +DTSTART:20100306T203000 +DTEND:20100306T235900 +SUMMARY:El Pino and the Volunteers + Coparck + more at Burgerweeshuis +DESCRIPTION:El Pino and the Volunteers\, Coparck\, Suimasen + \n\nhttp://www.last.fm/event/1302136+El+Pino+and+the+Volunteers+at+Burgerweeshuis+on+6+March+2010 +LOCATION:Burgerweeshuis\, Netherlands +GEO:6.155862;52.254341 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1323437 +URL;VALUE=URI:http://www.last.fm/event/1323437+Wouter+Hamel+theatertour +DTSTAMP:20100311T200000Z +DTSTART:20100311T200000 +DTEND:20100311T235900 +SUMMARY:Wouter Hamel theatertour +DESCRIPTION:Wouter Hamel + \n\nhttp://www.last.fm/event/1323437+Wouter+Hamel+theatertour +LOCATION:Deventer Schouwburg\, Netherlands +GEO:6.161613;52.255862 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1400318 +URL;VALUE=URI:http://www.last.fm/event/1400318+Dobet+Gnahor%C3%A9 +DTSTAMP:20100312T192400Z +DTSTART:20100312T000000 +DTEND:20100312T235900 +SUMMARY:Dobet Gnahoré +DESCRIPTION:Dobet Gnahoré + \n\nhttp://www.last.fm/event/1400318+Dobet+Gnahor%C3%A9 +LOCATION:Burgerweeshuis\, Netherlands +GEO:6.155862;52.254341 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1347202 +URL;VALUE=URI:http://www.last.fm/event/1347202+Waylon +DTSTAMP:20100319T203000Z +DTSTART:20100319T203000 +DTEND:20100319T235900 +SUMMARY:Waylon +DESCRIPTION:Waylon + \n\nhttp://www.last.fm/event/1347202+Waylon +LOCATION:Burgerweeshuis\, Netherlands +GEO:6.155862;52.254341 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1359365 +URL;VALUE=URI:http://www.last.fm/event/1359365+NO+blues+%40+Bouwkunde +DTSTAMP:20100320T113800Z +DTSTART:20100320T000000 +DTEND:20100320T235900 +SUMMARY:NO blues @ Bouwkunde +DESCRIPTION:No Blues + \n\nhttp://www.last.fm/event/1359365+NO+blues+%40+Bouwkunde +LOCATION:Theater Bouwkunde\, Netherlands +GEO:6.153074;52.253485 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1394328 +URL;VALUE=URI:http://www.last.fm/event/1394328+Schwarzblut+at+Burgerweeshuis+on+20+March+2010 +DTSTAMP:20100320T200000Z +DTSTART:20100320T200000 +DTEND:20100320T235900 +SUMMARY:Schwarzblut + Deadcell + more at Burgerweeshuis +DESCRIPTION:Schwarzblut\, Deadcell\, animAdverto + \n\nhttp://www.last.fm/event/1394328+Schwarzblut+at+Burgerweeshuis+on+20+March+2010 +LOCATION:Burgerweeshuis\, Netherlands +GEO:6.155862;52.254341 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1352883 +URL;VALUE=URI:http://www.last.fm/event/1352883+King+Me+-+Cords+Live +DTSTAMP:20100326T200000Z +DTSTART:20100326T200000 +DTEND:20100326T235900 +SUMMARY:King Me & Cords Live +DESCRIPTION:King Me\, Cords + \n\nhttp://www.last.fm/event/1352883+King+Me+-+Cords+Live +LOCATION:Burgerweeshuis\, Netherlands +GEO:6.155862;52.254341 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1322037 +URL;VALUE=URI:http://www.last.fm/event/1322037+The+Opposites +DTSTAMP:20100403T203000Z +DTSTART:20100403T203000 +DTEND:20100403T235900 +SUMMARY:The Opposites +DESCRIPTION:The Opposites + \n\nhttp://www.last.fm/event/1322037+The+Opposites +LOCATION:Burgerweeshuis\, Netherlands +GEO:; +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1387725 +URL;VALUE=URI:http://www.last.fm/event/1387725+Isbells+at+Burgerweeshuis+on+17+April+2010 +DTSTAMP:20100417T210000Z +DTSTART:20100417T210000 +DTEND:20100417T235900 +SUMMARY:Isbells at Burgerweeshuis +DESCRIPTION:Isbells + \n\nhttp://www.last.fm/event/1387725+Isbells+at+Burgerweeshuis+on+17+April+2010 +LOCATION:Burgerweeshuis\, Netherlands +GEO:6.155862;52.254341 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1326470 +URL;VALUE=URI:http://www.last.fm/event/1326470+Anathema+at+Burgerweeshuis+on+22+April+2010 +DTSTAMP:20100422T210000Z +DTSTART:20100422T210000 +DTEND:20100422T235900 +SUMMARY:Anathema at Burgerweeshuis +DESCRIPTION:Anathema + \n\nhttp://www.last.fm/event/1326470+Anathema+at+Burgerweeshuis+on+22+April+2010 +LOCATION:Burgerweeshuis\, Netherlands +GEO:6.155862;52.254341 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1400310 +URL;VALUE=URI:http://www.last.fm/event/1400310+Michiel+Borstlap+Eldorado +DTSTAMP:20100423T210000Z +DTSTART:20100423T210000 +DTEND:20100423T235900 +SUMMARY:Michiel Borstlap Eldorado +DESCRIPTION:Michiel Borstlap + \n\nhttp://www.last.fm/event/1400310+Michiel+Borstlap+Eldorado +LOCATION:Burgerweeshuis\, Netherlands +GEO:6.155862;52.254341 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1067247 +URL;VALUE=URI:http://www.last.fm/event/1067247+Heideroosjes+at+Burgerweeshuis+on+9+May+2010 +DTSTAMP:20100509T203000Z +DTSTART:20100509T203000 +DTEND:20100509T235900 +SUMMARY:Heideroosjes at Burgerweeshuis +DESCRIPTION:Heideroosjes + \n\nhttp://www.last.fm/event/1067247+Heideroosjes+at+Burgerweeshuis+on+9+May+2010 +LOCATION:Burgerweeshuis\, Netherlands +GEO:6.155862;52.254341 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1334621 +URL;VALUE=URI:http://www.last.fm/event/1334621+Roosbeef+at+Deventer+Schouwburg+on+11+May+2010 +DTSTAMP:20100511T200000Z +DTSTART:20100511T200000 +DTEND:20100511T235900 +SUMMARY:Roosbeef at Deventer Schouwburg +DESCRIPTION:Roosbeef + \n\nhttp://www.last.fm/event/1334621+Roosbeef+at+Deventer+Schouwburg+on+11+May+2010 +LOCATION:Deventer Schouwburg\, Netherlands +GEO:6.161613;52.255862 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1136119 +URL;VALUE=URI:http://www.last.fm/event/1136119+Heideroosjes +DTSTAMP:20100519T203000Z +DTSTART:20100519T203000 +DTEND:20100519T235900 +SUMMARY:Heideroosjes +DESCRIPTION:De Heideroosjes + \n\nhttp://www.last.fm/event/1136119+Heideroosjes +LOCATION:Deventer Schouwburg\, Netherlands +GEO:6.161613;52.255862 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1327278 +URL;VALUE=URI:http://www.last.fm/event/1327278+Aesun+at+Burgerweeshuis+on+9+June+2010 +DTSTAMP:20100609T112900Z +DTSTART:20100609T000000 +DTEND:20100609T235900 +SUMMARY:Aesun at Burgerweeshuis +DESCRIPTION:Aesun + \n\nhttp://www.last.fm/event/1327278+Aesun+at+Burgerweeshuis+on+9+June+2010 +LOCATION:Burgerweeshuis\, Netherlands +GEO:6.155862;52.254341 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1334624 +URL;VALUE=URI:http://www.last.fm/event/1334624+Peter+Pan+Speedrock+at+Burgerweeshuis+on+11+June+2010 +DTSTAMP:20100611T203000Z +DTSTART:20100611T203000 +DTEND:20100611T235900 +SUMMARY:Peter Pan Speedrock + Paceshifters at Burgerweeshuis +DESCRIPTION:Peter Pan Speedrock\, Paceshifters + \n\nhttp://www.last.fm/event/1334624+Peter+Pan+Speedrock+at+Burgerweeshuis+on+11+June+2010 +LOCATION:Burgerweeshuis\, Netherlands +GEO:6.155862;52.254341 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1405016 +URL;VALUE=URI:http://www.last.fm/event/1405016+Ground+Zero+Festival +DTSTAMP:20100703T133200Z +DTSTART:20100703T000000 +DTEND:20100703T235900 +SUMMARY:Ground Zero Festival +DESCRIPTION:TBA + \n\nhttp://www.last.fm/event/1405016+Ground+Zero+Festival +LOCATION:Recreatieplas Bussloo\, Netherlands +GEO:6.1333333;52.2 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1398450 +URL;VALUE=URI:http://www.last.fm/event/1398450+Xnoizz+Flevo+Festival +DTSTAMP:20100819T191900Z +DTSTART:20100819T000000 +DTEND:20100822T235900 +SUMMARY:Xnoizz Flevo Festival +DESCRIPTION:Nog Onbekend + \n\nhttp://www.last.fm/event/1398450+Xnoizz+Flevo+Festival +LOCATION:Recreatieplas Bussloo\, Netherlands +GEO:6.1333333;52.2 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1135861 +URL;VALUE=URI:http://www.last.fm/event/1135861+A+Campingflight+to+Lowlands+Paradise+2010 +DTSTAMP:20100820T160500Z +DTSTART:20100820T000000 +DTEND:20100822T235900 +SUMMARY:A Campingflight to Lowlands Paradise 2010 +DESCRIPTION:Kees van Hondt + \n\nhttp://www.last.fm/event/1135861+A+Campingflight+to+Lowlands+Paradise+2010 +LOCATION:Evenemententerrein Walibi World\, Netherlands +GEO:6.227748;52.24453 +END:VEVENT + +BEGIN:VEVENT +UID:LFMEVENT-1397948 +URL;VALUE=URI:http://www.last.fm/event/1397948+Clock+live%40+Kajak +DTSTAMP:20100917T141500Z +DTSTART:20100917T000000 +DTEND:20100917T235900 +SUMMARY:Clock live@ Kajak +DESCRIPTION:Clockrockz + \n\nhttp://www.last.fm/event/1397948+Clock+live%40+Kajak +LOCATION:Kajak\, Netherlands +GEO:6.092868;52.236304 +END:VEVENT + +END:VCALENDAR \ No newline at end of file diff --git a/resources/icals/162.ics b/resources/icals/162.ics new file mode 100644 index 0000000..dc5c31c --- /dev/null +++ b/resources/icals/162.ics @@ -0,0 +1,558 @@ +BEGIN:VCALENDAR +CALSCALE:GREGORIAN +PRODID:-//Ximian//NONSGML Evolution Calendar//EN +VERSION:2.0 +BEGIN:VEVENT +UID:20060425T220628Z-5616-1000-1-25@embrace +DTSTART;VALUE=DATE:20061004 +DTEND;VALUE=DATE:20061005 +SEQUENCE:3 +SUMMARY:GNOME 2.16.1 Stable Release +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +END:VEVENT +BEGIN:VEVENT +UID:20060425T220833Z-5616-1000-1-90@embrace +DTSTART;VALUE=DATE:20070108 +DTEND;VALUE=DATE:20070109 +SEQUENCE:4 +SUMMARY:GNOME 2.17.5 Tarballs Due +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +DESCRIPTION:GNOME 2.17.5 Tarballs Due +TRANSP:TRANSPARENT +BEGIN:VALARM +X-EVOLUTION-ALARM-UID:20060825T154249Z-17150-100-8562-140@embrace +DESCRIPTION:GNOME 2.17.5 Tarballs Due +ACTION:DISPLAY +TRIGGER;VALUE=DURATION;RELATED=START:-P1D +END:VALARM +END:VEVENT +BEGIN:VEVENT +UID:20060425T221010Z-5616-1000-1-48@embrace +DTSTART;VALUE=DATE:20070110 +DTEND;VALUE=DATE:20070111 +SEQUENCE:4 +SUMMARY:GNOME 2.17.5 Development Release +CLASS:PUBLIC +LAST-MODIFIED:20060830T041844 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +END:VEVENT +BEGIN:VEVENT +UID:20060425T220746Z-5616-1000-1-32@embrace +DTSTART;VALUE=DATE:20061204 +DTEND;VALUE=DATE:20061205 +SEQUENCE:4 +SUMMARY:GNOME 2.17.3 Tarballs Due +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +DESCRIPTION:GNOME 2.17.3 Tarballs Due +TRANSP:TRANSPARENT +BEGIN:VALARM +X-EVOLUTION-ALARM-UID:20060825T154209Z-17150-100-8562-122@embrace +DESCRIPTION:GNOME 2.17.3 Tarballs Due +ACTION:DISPLAY +TRIGGER;VALUE=DURATION;RELATED=START:-P1D +END:VALARM +END:VEVENT +BEGIN:VEVENT +UID:20060425T220728Z-5616-1000-1-30@embrace +DTSTART;VALUE=DATE:20061120 +DTEND;VALUE=DATE:20061121 +SEQUENCE:4 +SUMMARY:GNOME 2.16.2 Tarballs Due +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +DESCRIPTION:GNOME 2.16.2 Tarballs Due +TRANSP:TRANSPARENT +BEGIN:VALARM +X-EVOLUTION-ALARM-UID:20060825T154149Z-17150-100-8562-113@embrace +DESCRIPTION:GNOME 2.16.2 Tarballs Due +ACTION:DISPLAY +TRIGGER;VALUE=DURATION;RELATED=START:-P1D +END:VALARM +END:VEVENT +BEGIN:VEVENT +UID:20060425T220857Z-5616-1000-1-38@embrace +DTSTART;VALUE=DATE:20070122 +DTEND;VALUE=DATE:20070123 +SEQUENCE:4 +SUMMARY:GNOME 2.18.0 Beta 1 (2.17.90) Tarballs Due +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +DESCRIPTION:GNOME 2.18.0 Beta 1 (2.17.90) Tarballs Due +TRANSP:TRANSPARENT +BEGIN:VALARM +X-EVOLUTION-ALARM-UID:20060825T154337Z-17150-100-8562-161@embrace +DESCRIPTION:GNOME 2.18.0 Beta 1 (2.17.90) Tarballs Due +ACTION:DISPLAY +TRIGGER;VALUE=DURATION;RELATED=START:-P1D +END:VALARM +END:VEVENT +BEGIN:VEVENT +UID:20060425T221522Z-5616-1000-1-53@embrace +DTSTART;VALUE=DATE:20061218 +DTEND;VALUE=DATE:20061219 +SEQUENCE:5 +SUMMARY:UI Change Announcement Period +CLASS:PUBLIC +LAST-MODIFIED:20060830T041923 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +DESCRIPTION:UI Change Announcement Period: All user interface changes + must be announced to gnome-doc-list@. +END:VEVENT +BEGIN:VEVENT +UID:20060425T221023Z-5616-1000-1-45@embrace +DTSTART;VALUE=DATE:20070228 +DTEND;VALUE=DATE:20070301 +SEQUENCE:3 +SUMMARY:GNOME 2.18.0 Release Candidate (2.17.92) +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +END:VEVENT +BEGIN:VEVENT +UID:20060425T221110Z-5616-1000-1-48@embrace +DTSTART;VALUE=DATE:20070314 +DTEND;VALUE=DATE:20070315 +SEQUENCE:3 +SUMMARY:GNOME 2.18.0 Final Release! +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +END:VEVENT +BEGIN:VEVENT +UID:20060826T130552Z-26732-100-1-50@embrace +DTSTAMP:20060826T130552Z +DTSTART;VALUE=DATE:20070411 +DTEND;VALUE=DATE:20070412 +TRANSP:TRANSPARENT +SEQUENCE:3 +SUMMARY:Gnome 2.18.1 Stable Release +CLASS:PUBLIC +CREATED:20060830T041843 +LAST-MODIFIED:20060830T041843 +END:VEVENT +BEGIN:VEVENT +UID:20060425T220931Z-5616-1000-1-39@embrace +DTSTART;VALUE=DATE:20070124 +DTEND;VALUE=DATE:20070125 +SEQUENCE:3 +SUMMARY:GNOME 2.18.0 Beta 1 (2.17.90) Release +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +END:VEVENT +BEGIN:VEVENT +UID:20060425T220950Z-5616-1000-1-41@embrace +DTSTART;VALUE=DATE:20070131 +DTEND;VALUE=DATE:20070201 +SEQUENCE:3 +SUMMARY:GNOME 2.16.3 Stable Release +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +END:VEVENT +BEGIN:VEVENT +UID:20060425T220719Z-5616-1000-1-29@embrace +DTSTART;VALUE=DATE:20061108 +DTEND;VALUE=DATE:20061109 +SEQUENCE:3 +SUMMARY:GNOME 2.17.2 Development Release +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +END:VEVENT +BEGIN:VEVENT +UID:20060825T153859Z-23493-100-1-52@embrace +DTSTAMP:20060825T153859Z +DTSTART;VALUE=DATE-TIME; + TZID=/softwarestudio.org/Olson_20011030_5/America/New_York: + 20061007 +DTEND;VALUE=DATE-TIME; + TZID=/softwarestudio.org/Olson_20011030_5/America/New_York: + 20061009T235900 +SUMMARY:Boston Summit 2006 +CREATED:20060830T041843 +LAST-MODIFIED:20060830T041843 +CLASS:PUBLIC +TRANSP:TRANSPARENT +SEQUENCE:2 +DESCRIPTION:http://live.gnome.org/Boston2006 +END:VEVENT +BEGIN:VEVENT +UID:20060425T221353Z-5616-1000-1-51@embrace +DTSTART;VALUE=DATE:20061016 +DTEND;VALUE=DATE:20061017 +SEQUENCE:3 +SUMMARY:End of new (app) modules proposal period +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +END:VEVENT +BEGIN:VEVENT +UID:20060425T222220Z-5616-1000-1-58@embrace +DTSTART;VALUE=DATE:20070122 +DTEND;VALUE=DATE:20070123 +SEQUENCE:3 +SUMMARY:UI Freeze +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +DESCRIPTION:UI Freeze: No UI changes may be made without approval from + the release-team and notification to the GDP. +TRANSP:TRANSPARENT +END:VEVENT +BEGIN:VEVENT +UID:20060425T221534Z-5616-1000-1-54@embrace +DTSTART;VALUE=DATE:20061218 +DTEND;VALUE=DATE:20061219 +SEQUENCE:5 +SUMMARY:Module inclusion discussion heats up. +CLASS:PUBLIC +LAST-MODIFIED:20060830T041915 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +END:VEVENT +BEGIN:VEVENT +UID:20060425T220717Z-5616-1000-1-29@embrace +DTSTART;VALUE=DATE:20070214 +DTEND;VALUE=DATE:20070215 +SEQUENCE:2 +SUMMARY:GNOME 2.18.0 Beta 2 (2.17.91) +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +END:VEVENT +BEGIN:VEVENT +UID:20060425T221117Z-5616-1000-1-49@embrace +DTSTART;VALUE=DATE:20070409 +DTEND;VALUE=DATE:20070410 +SEQUENCE:4 +SUMMARY:GNOME 2.18.1 Tarballs Due +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +DESCRIPTION:GNOME 2.18.1 Tarballs Due +TRANSP:TRANSPARENT +END:VEVENT +BEGIN:VEVENT +UID:20060425T220832Z-5616-1000-1-36@embrace +DTSTART;VALUE=DATE:20061218 +DTEND;VALUE=DATE:20061219 +SEQUENCE:4 +SUMMARY:GNOME 2.17.4 Tarballs Due +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +DESCRIPTION:GNOME 2.17.4 Tarballs Due +TRANSP:TRANSPARENT +BEGIN:VALARM +X-EVOLUTION-ALARM-UID:20060825T154230Z-17150-100-8562-131@embrace +DESCRIPTION:GNOME 2.17.4 Tarballs Due +ACTION:DISPLAY +TRIGGER;VALUE=DURATION;RELATED=START:-P1D +END:VALARM +END:VEVENT +BEGIN:VEVENT +UID:20060425T220644Z-5616-1000-1-26@embrace +DTSTART;VALUE=DATE:20061016 +DTEND;VALUE=DATE:20061017 +SEQUENCE:6 +SUMMARY:GNOME 2.17.1 Tarballs Due +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +DESCRIPTION:GNOME 2.17.1 Tarballs Due +TRANSP:TRANSPARENT +BEGIN:VALARM +X-EVOLUTION-ALARM-UID:20060825T153408Z-17150-100-8562-82@embrace +DESCRIPTION:GNOME 2.17.1 Tarballs Due +ACTION:DISPLAY +TRIGGER;VALUE=DURATION;RELATED=START:-P1D +END:VALARM +END:VEVENT +BEGIN:VEVENT +UID:20060425T220752Z-5616-1000-1-33@embrace +DTSTART;VALUE=DATE:20061206 +DTEND;VALUE=DATE:20061207 +SEQUENCE:3 +SUMMARY:GNOME 2.17.3 Development Release +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +END:VEVENT +BEGIN:VEVENT +UID:20060425T220658Z-5616-1000-1-27@embrace +DTSTART;VALUE=DATE:20061018 +DTEND;VALUE=DATE:20061019 +SEQUENCE:3 +SUMMARY:GNOME 2.17.1 Development Release +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +END:VEVENT +BEGIN:VEVENT +UID:20060425T222230Z-5616-1000-1-59@embrace +DTSTART;VALUE=DATE:20070115 +DTEND;VALUE=DATE:20070116 +SEQUENCE:3 +SUMMARY:New APIs must be fully documented +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +END:VEVENT +BEGIN:VEVENT +UID:20060425T220736Z-5616-1000-1-31@embrace +DTSTART;VALUE=DATE:20061122 +DTEND;VALUE=DATE:20061123 +SEQUENCE:3 +SUMMARY:GNOME 2.16.2 Stable Release +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +END:VEVENT +BEGIN:VEVENT +UID:20060425T222150Z-5616-1000-1-56@embrace +DTSTART;VALUE=DATE:20070108 +DTEND;VALUE=DATE:20070109 +SEQUENCE:4 +SUMMARY:API/ABI Freeze +CLASS:PUBLIC +LAST-MODIFIED:20060830T041946 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +DESCRIPTION:API/ABI Freeze: developer APIs should be frozen at this + point. +END:VEVENT +BEGIN:VEVENT +UID:20060425T221019Z-5616-1000-1-44@embrace +DTSTART;VALUE=DATE:20070226 +DTEND;VALUE=DATE:20070227 +SEQUENCE:4 +SUMMARY:GNOME 2.18.0 Release Candidate (2.17.92) Tarballs Due +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +DESCRIPTION:GNOME 2.18.0 Release Candidate (2.17.92) Tarballs Due +TRANSP:TRANSPARENT +BEGIN:VALARM +X-EVOLUTION-ALARM-UID:20060825T154458Z-17150-100-8562-197@embrace +DESCRIPTION:GNOME 2.18.0 Release Candidate (2.17.92) Tarballs Due +ACTION:DISPLAY +TRIGGER;VALUE=DURATION;RELATED=START:-P1D +END:VALARM +END:VEVENT +BEGIN:VEVENT +UID:20060425T221031Z-5616-1000-1-46@embrace +DTSTART;VALUE=DATE:20070305 +DTEND;VALUE=DATE:20070306 +SEQUENCE:4 +SUMMARY:Hard Code Freeze +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +DESCRIPTION:Hard Code Freeze: no source code changes can be made without + approval from the release-team. Translation and documentation can + continue. +TRANSP:TRANSPARENT +END:VEVENT +BEGIN:VEVENT +UID:20060425T220955Z-5616-1000-1-42@embrace +DTSTART;VALUE=DATE:20070212 +DTEND;VALUE=DATE:20070213 +SEQUENCE:4 +SUMMARY:GNOME 2.18.0 Beta 2 (2.17.91) Tarballs Due +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +DESCRIPTION:GNOME 2.18.0 Beta 2 (2.17.91) Tarballs Due +TRANSP:TRANSPARENT +BEGIN:VALARM +X-EVOLUTION-ALARM-UID:20060825T154441Z-17150-100-8562-188@embrace +DESCRIPTION:GNOME 2.18.0 Beta 2 (2.17.91) Tarballs Due +ACTION:DISPLAY +TRIGGER;VALUE=DURATION;RELATED=START:-P1D +END:VALARM +END:VEVENT +BEGIN:VEVENT +UID:20060425T221051Z-5616-1000-1-47@embrace +DTSTART;VALUE=DATE:20070312 +DTEND;VALUE=DATE:20070313 +SEQUENCE:6 +SUMMARY:GNOME 2.18.0 Tarballs Due +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +DESCRIPTION:Hard Code Freeze ends\, but other freezes remain in effect + for the stable branch. +BEGIN:VALARM +X-EVOLUTION-ALARM-UID:20060825T155041Z-17150-100-8562-228@embrace +DESCRIPTION:GNOME 2.18.0 Tarballs Due +ACTION:DISPLAY +TRIGGER;VALUE=DURATION;RELATED=START:-P1D +END:VALARM +END:VEVENT +BEGIN:VEVENT +UID:20060425T220850Z-5616-1000-1-37@embrace +DTSTART;VALUE=DATE:20061220 +DTEND;VALUE=DATE:20061221 +SEQUENCE:3 +SUMMARY:GNOME 2.17.4 Development Release +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +END:VEVENT +BEGIN:VEVENT +UID:20060425T222207Z-5616-1000-1-57@embrace +DTSTART;VALUE=DATE:20070108 +DTEND;VALUE=DATE:20070109 +SEQUENCE:4 +SUMMARY:Feature and Module Freeze +CLASS:PUBLIC +LAST-MODIFIED:20060830T041953 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +DESCRIPTION:Feature and Module Freeze: new modules and functionality are + chosen now. +END:VEVENT +BEGIN:VEVENT +UID:20060425T220941Z-5616-1000-1-40@embrace +DTSTART;VALUE=DATE:20070129 +DTEND;VALUE=DATE:20070130 +SEQUENCE:4 +SUMMARY:GNOME 2.16.3 Tarballs Due +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +DESCRIPTION:GNOME 2.16.3 Tarballs Due +TRANSP:TRANSPARENT +BEGIN:VALARM +X-EVOLUTION-ALARM-UID:20060825T154417Z-17150-100-8562-179@embrace +DESCRIPTION:GNOME 2.16.3 Tarballs Due +ACTION:DISPLAY +TRIGGER;VALUE=DURATION;RELATED=START:-P1D +END:VALARM +END:VEVENT +BEGIN:VEVENT +UID:20060425T221456Z-5616-1000-1-52@embrace +DTSTART;VALUE=DATE:20061218 +DTEND;VALUE=DATE:20061219 +SEQUENCE:5 +SUMMARY:String Change Announcement Period +CLASS:PUBLIC +LAST-MODIFIED:20060830T041858 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +TRANSP:TRANSPARENT +DESCRIPTION:String Change Announcement Period: All string changes must be + announced to both gnome-i18n@ and gnome-doc-list@. +END:VEVENT +BEGIN:VEVENT +UID:20060425T221625Z-5616-1000-1-55@embrace +DTSTART;VALUE=DATE:20070212 +DTEND;VALUE=DATE:20070213 +SEQUENCE:3 +SUMMARY:String Freeze +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +DESCRIPTION:String Freeze: no string changes may be made without + confirmation from the l10n team (gnome-i18n@) and notification to both + the release team and the GDP (gnome-doc-list@). +TRANSP:TRANSPARENT +END:VEVENT +BEGIN:VEVENT +UID:20060425T220717Z-5616-1000-1-28@embrace +DTSTART;VALUE=DATE:20061002 +DTEND;VALUE=DATE:20061003 +SEQUENCE:5 +SUMMARY:GNOME 2.16.1 Tarballs Due +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +X-EVOLUTION-ALARM-UID:20060425T221751Z-9858-1000-1-227@embrace +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +DESCRIPTION:GNOME 2.16.1 Tarballs Due +TRANSP:TRANSPARENT +BEGIN:VALARM +X-EVOLUTION-ALARM-UID:20060825T154129Z-17150-100-8562-104@embrace +DESCRIPTION:GNOME 2.16.1 Tarballs Due +ACTION:DISPLAY +TRIGGER;VALUE=DURATION;RELATED=START:-P1D +END:VALARM +END:VEVENT +BEGIN:VEVENT +UID:20060425T220714Z-5616-1000-1-28@embrace +DTSTART;VALUE=DATE:20061106 +DTEND;VALUE=DATE:20061107 +SEQUENCE:4 +SUMMARY:GNOME 2.17.2 Tarballs Due +CLASS:PUBLIC +LAST-MODIFIED:20060830T041843 +X-EVOLUTION-ALARM-UID:20060425T221750Z-9858-1000-1-227@embrace +DTSTAMP:20060825T152756Z +CREATED:20060830T041843 +DESCRIPTION:GNOME 2.17.2 Tarballs Due +TRANSP:TRANSPARENT +BEGIN:VALARM +X-EVOLUTION-ALARM-UID:20060825T154128Z-17150-100-8562-104@embrace +DESCRIPTION:GNOME 2.17.2 Tarballs Due +ACTION:DISPLAY +TRIGGER;VALUE=DURATION;RELATED=START:-P1D +END:VALARM +END:VEVENT +END:VCALENDAR diff --git a/resources/icals/178.ics b/resources/icals/178.ics new file mode 100644 index 0000000..310812c --- /dev/null +++ b/resources/icals/178.ics @@ -0,0 +1,190 @@ +BEGIN:VCALENDAR +VERSION:2.0 +X-WR-CALNAME:Belgische feestdagen +PRODID:-//Apple Computer\, Inc//iCal 1.5//EN +X-WR-RELCALID:440DCB50-039E-11D8-88DC-000A957CAA38-CALP +X-WR-TIMEZONE:Europe/Brussels +CALSCALE:GREGORIAN +X-WR-CALDESC:Alle officiële Belgische feestdagen van 2001 tot en met 200 + 5. +BEGIN:VEVENT +DTSTART;VALUE=DATE:20041101 +DTEND;VALUE=DATE:20041102 +SUMMARY:Allerheiligen +UID:8684B909-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20050101 +DTEND;VALUE=DATE:20050102 +SUMMARY:Nieuwjaar +UID:868590B6-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040520 +DTEND;VALUE=DATE:20040521 +SUMMARY:O.-L.-Heer Hemelvaart +UID:8684D7A0-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20050711 +DTEND;VALUE=DATE:20050712 +SUMMARY:Feest van de Vlaamse Gemeenschap +UID:86850D80-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20041115 +DTEND;VALUE=DATE:20041116 +SUMMARY:Feest van de Dynastie +UID:86856160-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040721 +DTEND;VALUE=DATE:20040722 +SUMMARY:Nationale Feestdag +UID:86857F3C-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040501 +DTEND;VALUE=DATE:20040502 +SUMMARY:Feest van de Arbeid +UID:86857095-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20050721 +DTEND;VALUE=DATE:20050722 +SUMMARY:Nationale Feestdag +UID:86858207-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20041225 +DTEND;VALUE=DATE:20041226 +SUMMARY:Christmas Day +UID:8685BD6D-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20051115 +DTEND;VALUE=DATE:20051116 +SUMMARY:Feest van de Dynastie +UID:8685648E-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040711 +DTEND;VALUE=DATE:20040712 +SUMMARY:Feest van de Vlaamse Gemeenschap +UID:86850AA0-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20050501 +DTEND;VALUE=DATE:20050502 +SUMMARY:Feest van de Arbeid +UID:86857376-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20051101 +DTEND;VALUE=DATE:20051102 +SUMMARY:Allerheiligen +UID:8684BC36-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20050327 +DTEND;VALUE=DATE:20050328 +SUMMARY:Pasen +UID:8685327B-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040531 +DTEND;VALUE=DATE:20040601 +SUMMARY:Pinkstermaandag +UID:86859C8A-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20051111 +DTEND;VALUE=DATE:20051112 +SUMMARY:Wapenstilstand +UID:8684CB8D-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20051225 +DTEND;VALUE=DATE:20051226 +SUMMARY:Kerstmis +UID:8684FEAD-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20041225 +DTEND;VALUE=DATE:20041226 +SUMMARY:Kerstmis +UID:8684FB9D-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20050927 +DTEND;VALUE=DATE:20050928 +SUMMARY:Feest van de Waalse Gemeenschap +UID:868523A1-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20050516 +DTEND;VALUE=DATE:20050517 +SUMMARY:Pinkstermaandag +UID:86859F92-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20050505 +DTEND;VALUE=DATE:20050506 +SUMMARY:O.-L.-Heer Hemelvaart +UID:8684DA8A-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040411 +DTEND;VALUE=DATE:20040412 +SUMMARY:Pasen +UID:86852F70-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040927 +DTEND;VALUE=DATE:20040928 +SUMMARY:Feest van de Waalse Gemeenschap +UID:86852086-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20050328 +DTEND;VALUE=DATE:20050329 +SUMMARY:Paasmaandag +UID:868542CD-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20041111 +DTEND;VALUE=DATE:20041112 +SUMMARY:Wapenstilstand +UID:8684C881-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040412 +DTEND;VALUE=DATE:20040413 +SUMMARY:Paasmaandag +UID:86853FA8-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20050815 +DTEND;VALUE=DATE:20050816 +SUMMARY:Maria Tenhemelopneming +UID:8684EE84-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20050515 +DTEND;VALUE=DATE:20050516 +SUMMARY:Pinksteren +UID:8685B0B5-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040530 +DTEND;VALUE=DATE:20040531 +SUMMARY:Pinksteren +UID:8685AD9E-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040815 +DTEND;VALUE=DATE:20040816 +SUMMARY:Maria Tenhemelopneming +UID:8684E6C3-1F33-11D7-B218-000393A91E82-RID +END:VEVENT +END:VCALENDAR diff --git a/resources/icals/182.ics b/resources/icals/182.ics new file mode 100644 index 0000000..239bb0f --- /dev/null +++ b/resources/icals/182.ics @@ -0,0 +1,331 @@ +BEGIN:VCALENDAR +VERSION:2.0 +X-WR-CALNAME:Dryway +PRODID:-//Apple Computer\, Inc//iCal 1.5//EN +X-WR-RELCALID:9618E652-9A12-11D9-B9F6-003065D7F79A-CALP +X-WR-TIMEZONE:US/Eastern +CALSCALE:GREGORIAN +BEGIN:VTIMEZONE +TZID:US/Eastern +LAST-MODIFIED:20050420T000902Z +BEGIN:DAYLIGHT +DTSTART:20050403T070000 +TZOFFSETTO:-0400 +TZOFFSETFROM:+0000 +TZNAME:EDT +END:DAYLIGHT +BEGIN:STANDARD +DTSTART:20051030T020000 +TZOFFSETTO:-0500 +TZOFFSETFROM:-0400 +TZNAME:EST +END:STANDARD +BEGIN:DAYLIGHT +DTSTART:20060402T010000 +TZOFFSETTO:-0400 +TZOFFSETFROM:-0500 +TZNAME:EDT +END:DAYLIGHT +END:VTIMEZONE +BEGIN:VEVENT +DTSTAMP:20050321T140804Z +UID:9529B04C-9A10-11D9-B9F6-003065D7F79A-RID +DTSTART;TZID=US/Eastern:20050528T100000 +SUMMARY:Dry Way +DTEND;TZID=US/Eastern:20050528T150000 +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +RRULE:FREQ=DAILY;INTERVAL=1;COUNT=2 +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050624T100000 +SUMMARY:Dry Way +UID:CE57543A-9A5D-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050321T230707Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050612T100000 +SUMMARY:Dryway +UID:DF840B5C-9A5D-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050322T020207Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050618T100000 +SUMMARY:Dryway +UID:E9403F63-9A5D-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050322T020156Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050619T100000 +SUMMARY:Dryway +UID:EF7BAD31-9A5D-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050322T020211Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050625T100000 +SUMMARY:Dryway +UID:F34776C8-9A5D-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050322T020201Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050708T100000 +SUMMARY:Dry Way +UID:F4910F82-9A5D-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050321T230737Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050626T100000 +SUMMARY:Dryway +UID:F8A5E477-9A5D-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050322T020215Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050702T100000 +SUMMARY:Dryway +UID:FA4FEEB4-9A5D-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050322T020223Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050703T100000 +SUMMARY:Dryway +UID:02390F02-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050322T020220Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050710T100000 +SUMMARY:Dry Way +UID:052543DA-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050321T230741Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050807T100000 +SUMMARY:Dry Way +UID:088A8B2D-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050321T230902Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050717T100000 +SUMMARY:Dry Way +UID:0AE6CC6A-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050321T230748Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050729T100000 +SUMMARY:Dry Way +UID:0FBDCD3D-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050321T230756Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050730T100000 +SUMMARY:Dry Way +UID:119B40AD-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050321T230759Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050731T100000 +SUMMARY:Dry Way +UID:1AD30FA1-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050321T230815Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050806T100000 +SUMMARY:Dry Way +UID:33CAF8EE-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050321T230857Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050812T100000 +SUMMARY:Dry Way +UID:377F37E6-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050321T230906Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050814T100000 +SUMMARY:Dry Way +UID:3B8484C7-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050321T230915Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050820T100000 +SUMMARY:Dry Way +UID:3FB197BA-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050321T230919Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050821T100000 +SUMMARY:Dry Way +UID:417A378D-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050321T230922Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050904T100000 +SUMMARY:Dryway +UID:439E54E7-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050322T015827Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050826T100000 +SUMMARY:Dry Way +UID:468221CC-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050321T230928Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050819T100000 +SUMMARY:Dry Way +UID:48966612-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050322T020128Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050828T100000 +SUMMARY:Dry Way +UID:4B4CE621-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050321T230936Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050903T100000 +SUMMARY:Dryway +UID:4EB4963F-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050322T015805Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050917T100000 +SUMMARY:Dryway +UID:527DD9D5-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050322T015833Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20051009T100000 +SUMMARY:Dry Way +UID:544182EE-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050321T231026Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20050918T100000 +SUMMARY:Dryway +UID:56D4C4F4-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050322T015840Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=US/Eastern:20051008T100000 +SUMMARY:Dry Way +UID:670DB0B6-9A5E-11D9-ACB4-003065D7F79A-RID +DTSTAMP:20050321T231023Z +DESCRIPTION:Friday releases are for four hours and begin at 11 a.m. Satu + rday releases last five hours and begin at 10 a.m. Sunday releases are f + our hours beginning at 10 a.m. Flows vary from 900-1100 cfs. +DURATION:PT5H +END:VEVENT +END:VCALENDAR diff --git a/resources/icals/184.ics b/resources/icals/184.ics new file mode 100644 index 0000000..fbee29e --- /dev/null +++ b/resources/icals/184.ics @@ -0,0 +1,116 @@ +BEGIN:VCALENDAR +VERSION:2.0 +X-WR-CALNAME:EXDATE Example +PRODID:-//Apple Computer\, Inc//iCal 2.0//EN +X-WR-RELCALID:6E3F1EC4-081F-4DBF-8261-8938E9B32A70 +X-WR-TIMEZONE:Australia/Brisbane +CALSCALE:GREGORIAN +METHOD:PUBLISH +BEGIN:VTIMEZONE +TZID:Australia/Brisbane +LAST-MODIFIED:20051209T055354Z +BEGIN:STANDARD +DTSTART:19321213T204552 +TZOFFSETTO:+1000 +TZOFFSETFROM:+0000 +TZNAME:EST +END:STANDARD +BEGIN:DAYLIGHT +DTSTART:19420101T030000 +TZOFFSETTO:+1100 +TZOFFSETFROM:+1000 +TZNAME:EST +END:DAYLIGHT +BEGIN:STANDARD +DTSTART:19420329T020000 +TZOFFSETTO:+1000 +TZOFFSETFROM:+1100 +TZNAME:EST +END:STANDARD +BEGIN:DAYLIGHT +DTSTART:19420927T030000 +TZOFFSETTO:+1100 +TZOFFSETFROM:+1000 +TZNAME:EST +END:DAYLIGHT +BEGIN:STANDARD +DTSTART:19430328T020000 +TZOFFSETTO:+1000 +TZOFFSETFROM:+1100 +TZNAME:EST +END:STANDARD +BEGIN:DAYLIGHT +DTSTART:19431003T030000 +TZOFFSETTO:+1100 +TZOFFSETFROM:+1000 +TZNAME:EST +END:DAYLIGHT +BEGIN:STANDARD +DTSTART:19440326T020000 +TZOFFSETTO:+1000 +TZOFFSETFROM:+1100 +TZNAME:EST +END:STANDARD +BEGIN:DAYLIGHT +DTSTART:19711031T030000 +TZOFFSETTO:+1100 +TZOFFSETFROM:+1000 +TZNAME:EST +END:DAYLIGHT +BEGIN:STANDARD +DTSTART:19720227T030000 +TZOFFSETTO:+1000 +TZOFFSETFROM:+1100 +TZNAME:EST +END:STANDARD +BEGIN:DAYLIGHT +DTSTART:19891029T030000 +TZOFFSETTO:+1100 +TZOFFSETFROM:+1000 +TZNAME:EST +END:DAYLIGHT +BEGIN:STANDARD +DTSTART:19900304T030000 +TZOFFSETTO:+1000 +TZOFFSETFROM:+1100 +TZNAME:EST +END:STANDARD +BEGIN:DAYLIGHT +DTSTART:19901028T030000 +TZOFFSETTO:+1100 +TZOFFSETFROM:+1000 +TZNAME:EST +END:DAYLIGHT +BEGIN:STANDARD +DTSTART:19910303T030000 +TZOFFSETTO:+1000 +TZOFFSETFROM:+1100 +TZNAME:EST +END:STANDARD +BEGIN:DAYLIGHT +DTSTART:19911027T030000 +TZOFFSETTO:+1100 +TZOFFSETFROM:+1000 +TZNAME:EST +END:DAYLIGHT +BEGIN:STANDARD +DTSTART:19920301T030000 +TZOFFSETTO:+1000 +TZOFFSETFROM:+1100 +TZNAME:EST +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +DURATION:PT1H +DTSTAMP:20051209T055346Z +EXDATE;TZID=Australia/Brisbane:20060103T120000 +EXDATE;TZID=Australia/Brisbane:20060104T120000 +UID:DDC5B090-1B46-4C1D-8A3C-D305516A0B90 +SEQUENCE:9 +DTSTART;TZID=Australia/Brisbane:20060101T120000 +SUMMARY:Repeating Event +DESCRIPTION:This is a simple repeating event with a couple of days exclu + ded. +RRULE:FREQ=DAILY;INTERVAL=1;COUNT=7 +END:VEVENT +END:VCALENDAR diff --git a/resources/icals/195.ics b/resources/icals/195.ics new file mode 100644 index 0000000..5236373 --- /dev/null +++ b/resources/icals/195.ics @@ -0,0 +1,220 @@ +BEGIN:VCALENDAR +VERSION:2.0 +X-WR-CALNAME:2005 Chicago Bears Schedule +PRODID:-//Apple Computer\, Inc//iCal 1.5//EN +X-WR-RELCALID:202B7038-AFA2-11D9-A2CD-000A95AD128C +X-WR-TIMEZONE:Canada/Mountain +CALSCALE:GREGORIAN +METHOD:PUBLISH +BEGIN:VTIMEZONE +TZID:Canada/Mountain +LAST-MODIFIED:20050418T003804Z +BEGIN:DAYLIGHT +DTSTART:20050403T090000 +TZOFFSETTO:-0600 +TZOFFSETFROM:+0000 +TZNAME:MDT +END:DAYLIGHT +BEGIN:STANDARD +DTSTART:20051030T020000 +TZOFFSETTO:-0700 +TZOFFSETFROM:-0600 +TZNAME:MST +END:STANDARD +BEGIN:DAYLIGHT +DTSTART:20060402T010000 +TZOFFSETTO:-0600 +TZOFFSETFROM:-0700 +TZNAME:MDT +END:DAYLIGHT +END:VTIMEZONE +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20050808T180000 +DTEND;TZID=Canada/Mountain:20050808T210000 +SUMMARY:Dolphins @ Bears +UID:202B352E-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:2 +DTSTAMP:20050418T000046Z +DESCRIPTION:Preseason +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20050812T180000 +DTEND;TZID=Canada/Mountain:20050812T210000 +SUMMARY:Bears @ Rams +UID:202B39C2-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:2 +DTSTAMP:20050418T000056Z +DESCRIPTION:Preseason +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20050820T180000 +DTEND;TZID=Canada/Mountain:20050820T210000 +SUMMARY:Bears @ Colts +UID:202B3BCB-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:2 +DTSTAMP:20050418T000107Z +DESCRIPTION:Preseason +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20050826T180000 +DTEND;TZID=Canada/Mountain:20050826T210000 +SUMMARY:Bills @ Bears +UID:202B3DA8-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:3 +DTSTAMP:20050418T000120Z +DESCRIPTION:Preseason +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20050901T180000 +DTEND;TZID=Canada/Mountain:20050901T210000 +SUMMARY:Browns @ Bears +UID:202B3F83-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:2 +DTSTAMP:20050418T000134Z +DESCRIPTION:Preseason +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20050911T110000 +DTEND;TZID=Canada/Mountain:20050911T140000 +SUMMARY:Bears @ Redskins +UID:202B4156-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:4 +DTSTAMP:20050418T000400Z +DESCRIPTION:Week #1 +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20050918T110000 +DTEND;TZID=Canada/Mountain:20050918T140000 +SUMMARY:Lions @ Bears +UID:202B4354-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:4 +DTSTAMP:20050418T000354Z +DESCRIPTION:Week #2 +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20050925T110000 +DTEND;TZID=Canada/Mountain:20050925T140000 +SUMMARY:Bengals @ Bears +UID:202B4522-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:4 +DTSTAMP:20050418T000509Z +DESCRIPTION:Week #3 +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20051009T110000 +DTEND;TZID=Canada/Mountain:20051009T140000 +SUMMARY:Bears @ Browns +UID:202B46E9-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:4 +DTSTAMP:20050418T000809Z +DESCRIPTION:Week #4 +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20051016T110000 +DTEND;TZID=Canada/Mountain:20051016T140000 +SUMMARY:Vikings @ Bears +UID:202B48CF-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:4 +DTSTAMP:20050418T000821Z +DESCRIPTION:Week #5 +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20051023T141500 +DTEND;TZID=Canada/Mountain:20051023T171500 +SUMMARY:Ravens @ Bears +UID:202B4AA6-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:4 +DTSTAMP:20050418T000838Z +DESCRIPTION:Week #6 +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20051030T110000 +DTEND;TZID=Canada/Mountain:20051030T140000 +SUMMARY:Bears @ Lions +UID:202B4C7A-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:4 +DTSTAMP:20050418T001042Z +DESCRIPTION:Week #7 +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20051106T110000 +DTEND;TZID=Canada/Mountain:20051106T140000 +SUMMARY:Bears @ Saints +UID:202B4E54-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:4 +DTSTAMP:20050418T001115Z +DESCRIPTION:Week #8 +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20051113T110000 +DTEND;TZID=Canada/Mountain:20051113T140000 +SUMMARY:49ers @ Bears +UID:202B5047-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:4 +DTSTAMP:20050418T001149Z +DESCRIPTION:Week #9 +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20051120T110000 +DTEND;TZID=Canada/Mountain:20051120T140000 +SUMMARY:Panthers @ Bears +UID:202B5215-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:4 +DTSTAMP:20050418T001229Z +DESCRIPTION:Week #10 +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20051127T110000 +DTEND;TZID=Canada/Mountain:20051127T140000 +SUMMARY:Bears @ Buccaneers +UID:202B53E0-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:4 +DTSTAMP:20050418T001322Z +DESCRIPTION:Week #11 +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20051204T110000 +DTEND;TZID=Canada/Mountain:20051204T140000 +SUMMARY:Packers @ Bears +UID:202B55AB-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:4 +DTSTAMP:20050418T001408Z +DESCRIPTION:Week #12 +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20051211T110000 +DTEND;TZID=Canada/Mountain:20051211T140000 +SUMMARY:Bears @ Steelers +UID:202B5795-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:4 +DTSTAMP:20050418T001555Z +DESCRIPTION:Week #13 +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20051218T193000 +DTEND;TZID=Canada/Mountain:20051218T223000 +SUMMARY:Falcons @ Bears +UID:202B5963-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:8 +DTSTAMP:20050418T001943Z +DESCRIPTION:Week #14 +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20051225T150000 +DTEND;TZID=Canada/Mountain:20051225T180000 +SUMMARY:Bears @ Packers +UID:202B5B31-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:3 +DTSTAMP:20050418T001737Z +DESCRIPTION:Week #15 +END:VEVENT +BEGIN:VEVENT +DTSTART;TZID=Canada/Mountain:20060101T110000 +DTEND;TZID=Canada/Mountain:20060101T140000 +SUMMARY:Bears @ Vikings +UID:202B5D2A-AFA2-11D9-A2CD-000A95AD128C +SEQUENCE:4 +DTSTAMP:20050418T001843Z +DESCRIPTION:Week #16 +END:VEVENT +END:VCALENDAR diff --git a/resources/icals/224.ics b/resources/icals/224.ics new file mode 100644 index 0000000..039b144 --- /dev/null +++ b/resources/icals/224.ics @@ -0,0 +1,341 @@ +BEGIN:VCALENDAR +VERSION:2.0 +X-WR-CALNAME;VALUE=TEXT:Lord of the Rings +PRODID:-//Apple Computer\, Inc//iCal 1.0//EN +X-WR-RELCALID;VALUE=TEXT:B4B8ABF0-1ECC-11D7-A916-00306553216E +X-WR-TIMEZONE;VALUE=TEXT:America/New_York +CALSCALE:GREGORIAN +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040105 +DTEND;VALUE=DATE:20040106 +SUMMARY:Fellowship enters Moria +UID:C1C7837D-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=1 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040109 +DTEND;VALUE=DATE:20040110 +SUMMARY:Fellowship reaches Lorien +UID:C1C78BF1-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=1 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040117 +DTEND;VALUE=DATE:20040118 +SUMMARY:Passing of Gandalf +UID:C1C79008-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=1 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040207 +DTEND;VALUE=DATE:20040208 +SUMMARY:Fellowship leaves Lorien +UID:C1C793E2-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=2 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040217 +DTEND;VALUE=DATE:20040218 +SUMMARY:Death of Boromir +UID:C1C797C6-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=2 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040220 +DTEND;VALUE=DATE:20040221 +SUMMARY:Meriadoc & Pippin meet Treebeard +UID:C1C79D1B-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=2 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040222 +DTEND;VALUE=DATE:20040223 +SUMMARY:Passing of King Ellesar +UID:C1C7A154-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=2 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040224 +DTEND;VALUE=DATE:20040225 +SUMMARY:Ents destroy Isengard +UID:C1C7A52F-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=2 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040226 +DTEND;VALUE=DATE:20040227 +SUMMARY:Aragorn takes the Paths of the Dead +UID:C1C7A939-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=2 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040305 +DTEND;VALUE=DATE:20040306 +SUMMARY:Frodo & Samwise encounter Shelob +UID:C1C7AD08-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=3 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040308 +DTEND;VALUE=DATE:20040309 +SUMMARY:Deaths of Denethor & Theoden +UID:C1C7B0EF-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=3 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040318 +DTEND;VALUE=DATE:20040319 +SUMMARY:Destruction of the Ring +UID:C1C7B4A3-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;UNTIL=20030317;BYMONTH=3 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040329 +DTEND;VALUE=DATE:20040330 +SUMMARY:Flowering of the Mallorn +UID:C1C7B8A3-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=3 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040404 +DTEND;VALUE=DATE:20040405 +SUMMARY:Gandalf visits Bilbo +UID:C1C7BD03-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=4 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040417 +DTEND;VALUE=DATE:20040418 +SUMMARY:An unexpected party +UID:C1C7C0EF-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=4 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040423 +DTEND;VALUE=DATE:20040424 +SUMMARY:Crowning of King Ellesar +UID:C1C7C4DA-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=4 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040519 +DTEND;VALUE=DATE:20040520 +SUMMARY:Arwen leaves Lorian to wed King Ellesar +UID:C1C7C8A0-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=5 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040611 +DTEND;VALUE=DATE:20040612 +SUMMARY:Sauron attacks Osgilliath +UID:C1C7CC66-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=6 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040613 +DTEND;VALUE=DATE:20040614 +SUMMARY:Bilbo returns to Bag End +UID:C1C7D01C-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=6 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040623 +DTEND;VALUE=DATE:20040624 +SUMMARY:Wedding of Ellesar & Arwen +UID:C1C7D3CE-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=6 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040704 +DTEND;VALUE=DATE:20040705 +SUMMARY:Gandalf imprisoned by Saruman +UID:C1C7D799-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=7 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040724 +DTEND;VALUE=DATE:20040725 +SUMMARY:The ring comes to Bilbo +UID:C1C7DB68-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=7 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040726 +DTEND;VALUE=DATE:20040727 +SUMMARY:Bilbo rescued from Wargs by Eagles +UID:C1C7DF1F-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=7 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040803 +DTEND;VALUE=DATE:20040804 +SUMMARY:Funeral of King Theoden +UID:C1C7E2CC-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=8 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040829 +DTEND;VALUE=DATE:20040830 +SUMMARY:Saruman enters the Shire +UID:C1C7E6A6-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=8 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040910 +DTEND;VALUE=DATE:20040911 +SUMMARY:Gandalf escapes from Orthanc +UID:C1C7EA5F-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=9 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040914 +DTEND;VALUE=DATE:20040915 +SUMMARY:Frodo & Bilbo's birthday +UID:C1C7EE1A-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=9 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040915 +DTEND;VALUE=DATE:20040916 +SUMMARY:Black riders enter the Shire +UID:C1C7F1FC-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=9 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040918 +DTEND;VALUE=DATE:20040919 +SUMMARY:Frodo and company rescued by Bombadil +UID:C1C7F5C4-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=9 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20040928 +DTEND;VALUE=DATE:20040929 +SUMMARY:Frodo wounded at Weathertop +UID:C1C7F99E-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=9 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20041016 +DTEND;VALUE=DATE:20041017 +SUMMARY:Boromir reaches Rivendell +UID:C1C7FD5C-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=10 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20041017 +DTEND;VALUE=DATE:20041018 +SUMMARY:Council of Elrond +UID:C1C80115-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=10 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20041025 +DTEND;VALUE=DATE:20041026 +SUMMARY:End of War of the Ring +UID:C1C804F4-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=10 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20041116 +DTEND;VALUE=DATE:20041117 +SUMMARY:Bilbo reaches the Lonely Mountain +UID:C1C808BE-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=11 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20041205 +DTEND;VALUE=DATE:20041206 +SUMMARY:Death of Smaug +UID:C1C80C80-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=12 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20041216 +DTEND;VALUE=DATE:20041217 +SUMMARY:Fellowship begins Quest +UID:C1C8107A-1ECC-11D7-A916-00306553216E +DTSTAMP:20040103T033800Z +SEQUENCE:1 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=12 +END:VEVENT +BEGIN:VEVENT +DTSTART;VALUE=DATE:20030325 +DTEND;VALUE=DATE:20030326 +SUMMARY:Destruction of the Ring +UID:5EAE2BA6-D5D6-11D7-BA10-000393D17D20 +DTSTAMP:20030824T015752Z +SEQUENCE:2 +RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=3 +END:VEVENT +END:VCALENDAR diff --git a/resources/mailbox.gz b/resources/mailbox.gz new file mode 100644 index 0000000..0a07e9a Binary files /dev/null and b/resources/mailbox.gz differ diff --git a/resources/vcards/029-a.vcf b/resources/vcards/029-a.vcf new file mode 100644 index 0000000..0860c8a --- /dev/null +++ b/resources/vcards/029-a.vcf @@ -0,0 +1,6 @@ +BEGIN:VCARD +VERSION:2.1 +UID:urn:uuid:37ca3098-8a4a-5bc3-8069-e2d958d71dbb +EMAIL;PREF:john.doe@company.com +CATEGORIES:My Contacts +END:VCARD diff --git a/resources/vcards/029-b.vcf b/resources/vcards/029-b.vcf new file mode 100644 index 0000000..3997322 --- /dev/null +++ b/resources/vcards/029-b.vcf @@ -0,0 +1,6 @@ +BEGIN:VCARD +VERSION:2.1 +UID:urn:uuid:a426dfa3-8dde-5a29-9d07-9f025640b257 +EMAIL;PREF:jane.doe@company.com +CATEGORIES:My Contacts +END:VCARD diff --git a/resources/vcards/029-c.vcf b/resources/vcards/029-c.vcf new file mode 100644 index 0000000..d9e758c --- /dev/null +++ b/resources/vcards/029-c.vcf @@ -0,0 +1,8 @@ +BEGIN:VCARD +VERSION:2.1 +UID:urn:uuid:8ed76e29-9e7d-5a94-bc69-3ac49f5f0fc7 +N;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:=C3=91=20=C3=91=20=C3=91=20=C3=91=20;;;; +FN;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20 +TEL;CELL;PREF:123456789 +CATEGORIES:My Contacts +END:VCARD diff --git a/resources/vcards/029-d.vcf b/resources/vcards/029-d.vcf new file mode 100644 index 0000000..139f709 --- /dev/null +++ b/resources/vcards/029-d.vcf @@ -0,0 +1,19 @@ +BEGIN:VCARD +VERSION:2.1 +UID:urn:uuid:35ad7acc-830e-53e3-abdd-4e3586afcf94 +N;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91= +=20=C3=91=20=C3=91=20=C3=91;;;; +FN;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91= +=20=C3=91=20=C3=91=20=C3=91 +TEL;CELL;PREF:123456 +TEL;HOME:234567 +TEL;CELL:3456789 +TEL;HOME:45678901 +CATEGORIES:My Contacts +NOTE;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91= +=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91= +=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20 +NOTE;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91= +=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91= +=C3=91=20=C3=91=20=C3=91=20=C3=91=20=C3=91=20 +END:VCARD diff --git a/resources/vcards/029-e.vcf b/resources/vcards/029-e.vcf new file mode 100644 index 0000000..e0033f4 --- /dev/null +++ b/resources/vcards/029-e.vcf @@ -0,0 +1,36 @@ +BEGIN:VCARD +VERSION:2.1 +UID:urn:uuid:1a9dc94d-733b-50a3-8875-3d587274afc8 +N;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:=C3=91=20=C3=91=20;=C3=91=20=C3=91=20=C3=91=20;;; +FN;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:=C3=91=20=C3=91=20=C3=91=20=C3=91=20 +TEL;CELL;PREF:123456 +TEL;WORK:123456 +TEL;WORK;FAX:123456 +EMAIL;PREF;WORK:bob@company.com +EMAIL;PREF;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91= +=C3=91=C3=91=C3=91 +ORG;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91= +=C3=91 +ORG;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91= +=C3=91 +URL:www.company.com +URL:http://www.company.com +PHOTO;ENCODING=BASE64;JPEG:/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAIBAQEBAQIBAQE + CAgICAgQDAgICAgUEBAMEBgUGBgYFBgYGBwkIBgcJBwYGCAsICQoKCgoKBggLDAsKDAkKCgr/ + 2wBDAQICAgICAgUDAwUKBwYHCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKC + b2wxBqZRuiZQ8z9VmhGMY/wA803yB/dr4Z8a/tiftAaX8DPhLYzeM9U0a98WeCtH0tvFv9mW8 + 39ra1qWnWcseqSLIirBYWkn2gXc8TP5L3EI8qTcNnrOp/tsa7P8AseD9prxf8N7zwesGtiwuN + Ni1D7Tv/eFVmhuJoofOjO5VJKrh1dRu2ZM+zZKg2fO+g/8ABEP45eC/Dmn+HPBX7ZWhab/Zt2 + 9xAyfDS2kWF3WRWeNXlPlsVkKkrjhn5+Yg/Xn7HPwP+PHwC+GV74L/AGiv2kLn4natc6kLm21 + u6077L9ni8qNPJEe9wVDozgZwN5AA6n1ouMnJpkpDPk9cD+ZqnVqTiqUn6GtStiffUqnNzO+/ + /B8zlvG/wN+D3xF+IXhL4u+O/AdnqfibwFNdS+D9WuVYyaW1yiRzmIbsDesaA5B+6OlX/EXg3 + wH4t1fT/EPizwRDqF7pyXENhczMzNCk8kbyqPRWaCIsvIOxfSttSxIKjpSPdvEyBEyd+1/xP/ + 1qcXpyvdC5tDmdH+CHwg8K/EGb4w6B8PtKs/E95pkljJrUK4nFqxhJhz0K/wCj2/H/AExX0pu + jfBH4TaR8ZtX/AGgvDfw7tbXxd4g09LPVvXXKt9qnhQQqI3OcY220Axxwi/jxv7S/7eH7Kn7H + R0lP2g/iBeaJJrOuRaVYNa6PcXXm3MymRYz5UbhV2jJY4xxz1r0fwR498K+PtGbXfCGr/a7ZJ + 9j7w3CvijVraDxdbyWg/4lFlOVN2WP8AZ7GUhnJKv5Ifacule3Qz+QTKD3HanDUELEk9T6UKS + juVGVjxGe31HUdS1/4da/8Ask6APDvgrV4W8GzHT1mi1DzhPLK8UL2IS1O9V3PE8py+Tgnnn/ + AmoQfD/wAeab4J8F/8E3tC0qz1LTJJ9V1/RdO+zR2p8yTFviLTVWUsYYGO50+/nB2Lv+jnuoZ + l2S9ByOKYLrys+SP0p+0iPnP/2Q== + +END:VCARD diff --git a/resources/vcards/029-f.vcf b/resources/vcards/029-f.vcf new file mode 100644 index 0000000..6afed90 --- /dev/null +++ b/resources/vcards/029-f.vcf @@ -0,0 +1,24 @@ +BEGIN:VCARD +VERSION:2.1 +UID:urn:uuid:3fe8d47e-cc86-59da-a2b1-065c17b83016 +N;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:=C3=91=C3=91=C3=91=C3=91;;;; +FN;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:=C3=91=C3=91=C3=91=C3=91 +TEL;CELL;PREF:55556666 +EMAIL;PREF:henry@company.com +ORG;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91= +=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91= +=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91= +=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91= + +ORG;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91= +=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91= +=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91= +=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91= +=80 +ORG;CHARSET=UTF-8;ENCODING=QUOTED-PRINTABLE:=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91= +=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91= +=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91= +=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91=C3=91= + +CATEGORIES:My Contacts +END:VCARD diff --git a/resources/vcards/030.vcf b/resources/vcards/030.vcf new file mode 100644 index 0000000..d29d43f --- /dev/null +++ b/resources/vcards/030.vcf @@ -0,0 +1,11 @@ +BEGIN:VCARD +VERSION:2.1 +UID:urn:uuid:2f122946-71f4-5ab8-8529-6f5fe0b51e61 +FN:John Doe +N:Doe;john;;; +ORG:Acme Solutions +TEL;TYPE=CELL:+96123456789 +PHOTO;ENCODING=BASE64:/9j/4QFaRXhpZgAASUkqAAgAAAAAABABAgABAAAAAAAAABIBAwABAAAAAQAAABoBBQABAAAAhgAAABsBBQABAAAAjgAAACgBAwABAAAAAgAAADEBAgAWAAAAlgAAADIBAgABAAAAAAAAABMCAwABAAAAAQAAAGmHBAABAAAArAAAAAAAAABIAAAAAQAAAEgAAAABAAAAUmltIEV4aWYgVmVyc2lvbjEuMDBhAAwAmoIFAAEAAABCAQAAAJAHAAQAAAAwMjIwA5ACAAEAAAAAAAAAAZEHAAQAAAABAgMABpIFAAEAAABKAQAACJIDAAEAAAAAAAAACZIDAAEAAAAgAAAAfJIHAAAAAAAAAAAAAaADAAEAAAABAAAAAqADAAEAAABCAAAAA6ADAAEAAABkAAAAC6QHAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAEAAAD/2wCEABIMDRANCxIQDhAUExIVGywdGxgYGzYnKSAsQDlEQz85Pj1HUGZXR0thTT0+WXlaYWltcnNyRVV9hnxvhWZwcm4BExQUGxcbNB0dNG5JPklubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubv/EAaIAAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKCwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoLEAACAQMDAgQDBQUEBAAAAX0BAgMABBEFEiExQQYTUWEHInEUMoGRoQgjQrHBFVLR8CQzYnKCCQoWFxgZGiUmJygpKjQ1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4eLj5OXm5+jp6vHy8/T19vf4+foRAAIBAgQEAwQHBQQEAAECdwABAgMRBAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/AABEIAGQAQgMBEQACEQEDEQH/2gAMAwEAAhEDEQA/AOGoAKACgBQCTgdaAHmFlHIoAYaAAEjpQAlABQAUAFABQB0mh6MHtvPn4L8gegoAl1G0tUhb5gD2oA5mRQrkA5oAZQAUAFABQAUAS20XmzKCGK5G7b1xQB3s1tJHpymzTduUbR6UAczqOl6wsZluIn8v2xgUAYjAgkHOfegBKACgAoAKACgC/o09xDeYtnRGcYZnAIA68/lQBq6j4quHIisyI40GC2PvH19qAJpnv7/TP7Q88mNlIeJJCQGHHTt9KAOYdtzZIwaAGkYoASgAoAKACgDp5tKWHwtayxOsZufmlaU4zzwPpQBiS6fcRywx7Qxmx5ZU5D5OODQB0GpwPoFiLaNiN4y/fJxzQBy8rB2yOvegBlACUAFABQAo6igD0a1jnurJGk8n7OsKJFG65xgD5vr1oAybCyvbTUoZt4mUPzzu2j2oAt+NkW4sUlH3lHNAHCUAFACUAFABQAo6igD0qyfzdCiS7SMK0QGNp6fnQBy4W10/URNHK6orcLnAIoAm13VBcbViIMLKpz6UAczJjzGx0zQA2gBKAFoAKAFXgjNAHZ6hrNuNNjWJvmKAcHp/nigDkZpWduSaAGCVguM8UAMoASgAoAKAHqOM0ANNADg7KBzkDsaALWba7ljjjh8h3IUncSufX1/nQBZvfDt/ZffRXHqhzQBmPGyHDAg0ANKkdQaAEoAKAH5wuKAFXnk0AMbrQAqEqQwOCDkfWgD0TT9Xs9WsI98sYnCgSIx2898Z6igDL1b7HArHfEh9iCf8aAOWup1lc+X09aAK4GRmgA/GgBQMtg0APXk80AMYfNQAh64FADvagBMUALjH1oAQ+goAdt96ADHGWoAXkDNADDwOepoAaOtADyeaAFxt/wB7+VAAqknmgCWC2lnfbGhZsgYHv0pqLlsJtLcGgkRinyMLoiz61elAEJ5OaAEBe3tQAuOc0AShQBxx70ASafqD2E3mois2QRuVyeQ+ZIzkLliSeta8zJ5Uf/9k= + +NOTE: +END:VCARD diff --git a/resources/vcards/031.vcf b/resources/vcards/031.vcf new file mode 100644 index 0000000..d0e4f41 --- /dev/null +++ b/resources/vcards/031.vcf @@ -0,0 +1,42 @@ +BEGIN:VCARD +VERSION:3.0 +X-COUCHDB-APPLICATION-ANNOTATIONS:{"Evolution":{"revision":"2012-03-05T13:3 + 2:54Z"}} +X-AIM;TYPE=HOME;X-COUCHDB-UUID="cb9e11fc-bb97-4222-9cd8-99820c1de454":johnn + y5@aol.com +URL;X-COUCHDB-UUID="0abc9b8d-0845-47d0-9a91-3db5bb74620d":http://www.ibm.co + m +TEL;X-COUCHDB-UUID="c2fa1caa-2926-4087-8971-609cfc7354ce";TYPE=CELL:905-666 + -1234 +TEL;X-COUCHDB-UUID="fbfb2722-4fd8-4dbf-9abd-eeb24072fd8e";TYPE=WORK,VOICE:9 + 05-555-1234 +UID:477343c8e6bf375a9bac1f96a5000837 +N:Doe;John;Richter\, James;Mr.;Sr. +X-EVOLUTION-FILE-AS:Doe\, John +FN:Mr. John Richter\, James Doe Sr. +NICKNAME:Johny +X-EVOLUTION-SPOUSE:Maria +ORG:IBM;Accounting;Dungeon +TITLE:Money Counter +X-EVOLUTION-MANAGER:Big Blue +X-EVOLUTION-ASSISTANT:Little Red +CATEGORIES:VIP +NOTE:THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "A + S IS" AND ANY EXPRESS OR IMPLIED WARRANTIES\, INCLUDING\, BUT NOT LIMITED + TO\, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULA + R PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRI + BUTORS BE LIABLE FOR ANY DIRECT\, INDIRECT\, INCIDENTAL\, SPECIAL\, EXEMPL + ARY\, OR CONSEQUENTIAL DAMAGES (INCLUDING\, BUT NOT LIMITED TO\, PROCUREME + NT OF SUBSTITUTE GOODS OR SERVICES\; LOSS OF USE\, DATA\, OR PROFITS\; OR + BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY\, WHE + THER IN CONTRACT\, STRICT LIABILITY\, OR TORT (INCLUDING NEGLIGENCE OR OTH + ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE\, EVEN IF ADVIS + ED OF THE POSSIBILITY OF SUCH DAMAGE. +EMAIL;TYPE=WORK;X-COUCHDB-UUID="83a75a5d-2777-45aa-bab5-76a4bd972490":john. + doe@ibm.com +ADR;TYPE=HOME:ASB-123;;15 Crescent moon drive;Albaney;New York;12345;United + States of America +BDAY:1980-03-22 +X-EVOLUTION-ANNIVERSARY:1980-03-22 +REV:2012-03-05T13:32:54Z +END:VCARD \ No newline at end of file diff --git a/resources/vcards/034.vcf b/resources/vcards/034.vcf new file mode 100644 index 0000000..9c43776 --- /dev/null +++ b/resources/vcards/034.vcf @@ -0,0 +1,178 @@ +BEGIN:VCARD +VERSION:3.0 +PRODID:-//Apple Inc.//Address Book 6.1//EN +N:Doe;John;Johny;Mr.;I +FN:Mr. Doe John I Johny +NICKNAME:Johny\,JayJay +ORG:IBM;SUN +TITLE:Generic Accountant +EMAIL;type=INTERNET;type=WORK;type=pref:john.doe@ibm.com +EMAIL;type=INTERNET;type=WORK:billy_bob@gmail.com +TEL;type=CELL;type=VOICE;type=pref:+1 (212) 204-34456 +TEL;type=WORK;type=FAX:00-1-212-555-7777 +item1.ADR;type=HOME;type=pref:;;25334\nSouth cresent drive\, Building 5\, 3rd floo r;New York;New York;NYC887;U.S.A. +NOTE:THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"\nAND ANY EXPRESS OR IMPLIED WARRANTIES\, INCLUDING\, BUT NOT LIMITED TO \, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR P URPOSE\nARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTOR S BE\nLIABLE FOR ANY DIRECT\, INDIRECT\, INCIDENTAL\, SPECIAL\, EXEMPLARY\, OR\nCONSEQUENTIAL DAMAGES (INCLUDING\, BUT NOT LIMITED TO\, PROCUREMENT OF\n SUBSTITUTE GOODS OR SERVICES\; LOSS OF USE\, DATA\, OR PROFITS\; OR BUSINESS \nINTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY\, WHETHER IN\n CONTRACT\, STRICT LIABILITY\, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)\nA RISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE\, EVEN IF ADVISED OF THE\n POSSIBILITY OF SUCH DAMAGE. +item2.URL;type=pref:http://www.sun.com +item2.X-ABLabel:_$!!$_ +BDAY;value=date:1980-05-21 +PHOTO;ENCODING=b;TYPE=JPEG:/9j/4AAQSkZJRgABAQAAAQABAAD/4QBARXhpZgAATU0AKgAA + AAgAAYdpAAQAAAABAAAAGgAAAAAAAqACAAQAAAABAAAAyKADAAQAAAABAAAAyAAAAAD/2wBDAA + IBAQIBAQICAQICAgICAwUDAwMDAwYEBAMFBwYHBwcGBgYHCAsJBwgKCAYGCQ0JCgsLDAwMBwkN + Dg0MDgsMDAv/2wBDAQICAgMCAwUDAwULCAYICwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCw + sLCwsLCwsLCwsLCwsLCwsLCwsLCwv/wAARCADIAMgDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEA + AAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhBy + JxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZ + WmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxM + XGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAA + AAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFE + KRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVm + Z2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyM + nK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD9/KKKKAIN59T+dG8+ + p/OkorkuzMXefU/nRvPqfzpKKLsBd59T+dG8+p/OkqC+u/skZYkDAzyaLsCUyNk/MfzrkPihq9 + 3p9s5sLq4gIjQ5jkK878djWd4p+L8Wg3TI91aJgnhp0XufUe1eY/FX9oSCWCQfbrD/AFaj/j5j + /v8A0q0xn5C/8Fm/20vjH8LvGjRfDP4s/Ezw7Fx8ml+KL6zX/V2Z6RSju7/99H1NflT4i/4Ktf + tRwaxMsP7Sfx9RRtwF+IOrgD5R/wBPNfcv/BajxwniLxozLNC/T7sit/yzs/T6V+TviPD6zMcj + nb/6CK6YbHZBaHuH/D1/9qX/AKOV+P8A/wCHC1f/AOSa2vAH/BVT9qC98SRJeftIfHuVDjKv8Q + NWYH5l7G4r5nZdtbnw7k8vxNET7f8Aoa1omg5T+hz/AIIe/tYfFP4s+V/wtT4leP8AxNnbn+1f + EN3e5/48v+esjf33/wC+j6mv2a+Hl7Ne6buvZpZm8qM5dyxyQc9a/n4/4IYfEZPDXk7ri3j+79 + +VV/58vUe1ftR8Nf2h4V07Bv7D/VRf8vMXofasJbmM9z6Aorh/CHxUj8Rz7VubaT/cmRvX0HtX + awTCZMgg/Q1NyAlYjOCagMjZPzH86ml71AQcnitqRtHYVZGLcsfzp+8+p/Oo1B3U+rY7IcrEsM + k9fWpqgT74+tT1EjOYUUUVJAUUUUAQ7DSFCBT6R/u1lZGTQyiiisWSKBuNcV8Zdfk8P6HPJA20 + rAzdf9l/celdsn3q80/aTJHhW7x1+yv/AOgSVdhrU/HX/gpL/wAFIPEfwl+I+oWmk6hLEsTyAA + Tyr0luB2nX+4K+GPFX/BY7xhf3c6SatOQGK/8AH1P2Y/8AT1Wt/wAFqJ7hfjNq/lSzL+9m6MR/ + y3vK/MjU7q7/ALVucz3P+tb+NvU1eljVHun7SX7VOqfGbUvO1m5kmJ7mR27RjvI39wV4LesLi5 + Z26nH8qQySsP3ryN9STThCXPAJNZuTR30o3RXkG081No982n3wljOCP8Qf6VoWvhebUMbEkOfQ + H/CrX/Cu7oDIjnx/un/4mqVVdWU4nv8A+y3+2trXwU2/2LeSQ4x0lkX/AJ5f3ZV/55ivp/wZ/w + AFk/GVveRxLq8+GXH/AB9T9lP/AE9V+at/o11o3Rp0/Ej+ntT/AAzqV0uuQg3E/wDF/wAtD/dN + PmuYSjqf0ff8Evf+CgGv/GLxQsOsX0soPrNI38Fye8zf3BX68/D/AFd9V0svK2SP8W/wr+cv/g + hbeTS+O18yWRuvVif+Wd7X9EnwgJOhNn2/9Cen5mEjrmO7rSbB70tFaRdkJNoaUx0ptSHoajrR + O5SYqffH1qeoE++PrU9KQpBRRRUkhRRRQAmwe9J5Yp1FLlQWIivPNG2nHqaSiyFZBjFea/tGoG + 8L3ef+fV//AECSvSz1rzb9on/kWLr/AK9n/wDQJKTWhB/N1/wWX0pJvjFq5YDmWb/0feV+aera + FH/aVycL/rW7e59q/Tz/AILHDPxh1bP/AD1m/wDR93X5tanFv1K4AHWVv5mvNqVGmdUYqyORvt + PSBsADFP0PT1ubsq+MD/A1vS+E5r85hOB9B/j7Vf8AC/w6uhekluPoPQ/7VS6mm56NKOiPa/gH + 8EdO8TtbfbI4zvRicqv/AD1A7oa92b9kvQ/7G8wQwZx/cj/u/wDXOqH7Kfg5rdrLzxnEb/8Ao8 + e9fUT6BGvho/J+p/ufWvnMXjJwnZM6lTTVz83f2pPg/YeDt32BEXr0Cj/nr6KP7or54sIfI8Rw + hf7zj9DX2r+3pYLb79o9e/8A13r4wX/kZoP9+T+Rr6PLajqU02cNaNmfrz/wQk+bx4ue2f8A0X + e1/RX8IEH9hN+H/oT1/Op/wQjH/FeL+P8A6Lva/os+EH/ICb8P/QnrvZwVNGdbsFGwUtFOLMxC + gNJ5Qp1FaxYXGiMA06iimAUUUUAFFFFADfMNHmGm0UANZ/mo3/Wkf71JV2RdkS59a82/aKP/AB + TF1/17P/6BJXpFecftDxmTwxdAf8+r/wDoElRIiyP5x/8AgshJj4wat/11m/8AR93X536Zp41H + Vpw3edx/Wv0c/wCCxGgS3Xxd1Uof+Ws3p/z3u/evgTwl4SmOsTZbn7RJ2Hp9a8mtHVnRDQ+r/w + BkX9jGy+KWkCW6HJ92/vSjtKP7lfUGjf8ABLXS4LdJAOue7+pH/PxW1/wTB8IA+GAZ1yfr/t3P + vX6CWPg6H+yIsxc8/wAR/vfWvmcXiakJuKPVo25T4V8Mfsf2ngLb5Ax5YK9W7nd/z1NaviP4ep + ZaI6oOBn1/un/ar6Z8d+Egok8pdvI75/h+teY+N/CzrpcmPf8Ak3vXlSnOcrs3ukj8tv8Agor4 + eFnv2+//ALce9fB8tvs8UQdfvy/yNfov/wAFMtEeEybvf/2596/PW6sWXxTBz/HL/I19jlcmqa + PLry97c/WT/ghWvl+Ol/H/ANF3tf0SfCCY/wBhN+H/AKE9fzuf8EOXFt44Unnr/wCi72v6FfhF + q6Lobgj0/wDQnr1k7nDJ3Z3qSlm5qSqNrqCyyDHerobcMitoK5ItFFFabEthRRRQNBRRRQMKKK + KAI6KKKuxdhpTJo8v3p1MmuFgGZDTGPrifjRpv9o6DcJ6wMP8Ax1/8a6C58bWFqxE0mCP9pf8A + GuV+IXj7TbnTZVSUEmMj7yejf7VZ7mZ+Lf8AwUw/Zg/4TL4k6jNt+/JIenrLcn/nmf71fIHhf9 + h4Jqcp2Hmdz09v+uNfql+2Bo8Ov+L7uS3XcGZ+wP8AHL7H1rwzw78PsXzHyv8Alq38Pt/u1jKi + pGiZ61/wTs/Zb/sPw6FKfp/tXH/TP3r7i074C7tMjBXpnt7/AO5Xnf7EvhlbTQhvTH4f7U3tX1 + tpunRCxQY9ew9TXh4jAwlNto6YVpJHzf4i/Ze/tHd8vUjt7Y/551w3jz9kHGiSNtPft/st/wBM + q+zm0iFuq/oP8Kw/HuhQN4elwvPPYf3W9q5Hl9PsX7dn8+v/AAVv/Z8/4R3zePXt/wBfX+wPSv + y81v4e/ZvE0Jx0kl/l/u1+5H/BaPwg935v2RP73Rf+vz0FfkV4p8A3g8RRkRdJJf4W/wDia9fD + U1TikjlqVG3qfY//AASLv/8AhEfFqv06/wDoF39P71ftv8Lfjz5GllS3X/Fv9uvxA/4J7Wknh7 + xCGuxsH0I/hn9cetfp38PviDbRWu1pef8AeHq3+1XbBGDZ9q+Cvix/bOoRJn73/wAUB/ePrXqG + mX32qIH1Gf5V8ofBTxfDfa9arHJnOP4h/wA9F96+o/C8wms0I/uD+QreKBGsZOKb59B6Go60Su + UkmSibJHvT6gT74+tT0NWBpIKKKKkQUUUUAVPtH1oFxk96iorQ6uRFkScVxXxZ8df8Inp80mcb + MfqUHqPWuzX7teNftTzPFoV15Qz9z/0KKgztqfnl+0j/AMFfv+Fba1fw+bjyHjH3/WHf/wA/Ir + xbQv8AguL/AMJlqrWnm53S+V9/3A/5+z/er4d/4KIeK9Wt/GOuC2g3ASwY+dB/y5j1FfLXwK8Y + 61P46AktsD7fj/WR/wDPRPamopmVj+iL4I/8ZQ6JDqY+b7SFb1+8qN/t/wDPX1/+v6/on7EBRI + 5Nn3xv6eo/6415L/wR3spNW+EOlPqC7GMUOeQf+WFp6fU1+kmk+FIBptsc/wDLJex9B71LRLk0 + ecfA74Mf8IRYhNuP/wBb/wCwP71evW9r5UIXHSm2WmpajCVbAxWEqMW7sFNorNFiqHiKw+3aY8 + Z75/ka036VDcIHjIasHQiS6sj4V/bv/Y3/AOFyb/lznPQZ/wCe/wD0yb+/XxH4q/4JG/vTP5f3 + HY/d9SP+nav2R8eeF4dT/wBYcZ+v+17+9ee+KvhnanSJju5yOx/vD/apcqiTz33Pxh+JXwu/4Z + KtvtZG3Hrx3Uf3U/57f57+ZW//AAVP/sLV47cOPmz/ABf7Of8An4HrX0z/AMFj9LHhzwczW3zH + j2/jtPXPrX4d+LPiBdW/jO2AXj5u4/55j/ZrWIXuz98/2D/29/8AhY/xD0i1L/6/H8X/AE9Rr/ + z2PrX64/CLXf7b0SCTP3oVb/x1Pc+tfzW/8ElvHjT/ABm8OrcnaDtz3/5f4fRa/oq/Zz8UWjeG + LXzJufsqfwN/cj9q2iaI9YPQ1HUEWtW1xxDLkn/ZP+FTBgw4rSJpHYcn3x9anqBPvj61PSkKQU + UUVJIUUUUAUfL96UR81LSMwXrU8503YKMDmuA+NXg1/FOmzxxdX2/zT3HpXdtexIcM36GobiSC + 6GGfr7GjnZnqfiZ+1Z/wSk1b4h+INSmtxxO8ZHKdoAve5FeL/DL/AIIp63oPiUTyLx9q8zrH/f + U/8/Z9K/fu++H1lqEjFn+8fRvp/eqCH4UWUL7lbvno3/xVaKZJ85fsA/s13Pwc8BWNneDDRJGD + 07RwDtI39yvr2yHlWcKn+FFH6Vn6NoEWlxhYTkAe/t7+1aQGBVGLZNEcg06mQ/dp9Q9xETDIqN + huFTP96k61m9RNGJremm5+7/nrXM+KPDrnRZzn+7/6EPeu7lQNmsvxPCP7Dn/4D/6EK5pSsydj + 8Y/+C3mkNB4IYt7f+jLKv5//AIjN9j8WQux+7u/9AWv6Gv8Aguam3wM34f8Aoyyr+eH4vnGvL+ + P/AKClVGVwufTX7D37UFr8JPiDpV7dn5bfGev/AD8xv2jb+7X6t/CH/gtromg6fawSNz5aR9JP + RB/z6H0r+eiz1+XS2Dw8lfp659K19H+LV9FqVsFT/lqvdfUf7NdEHdGsHdH9Vn7Kn/BUbS/jJ4 + ptbOzbJldB0fu8Q726/wB+vvDwN4vTxHDCY/8AlpGr/mpPoK/mT/4I0fEK61b4w6Qky4Blh7j/ + AJ72f+z71/Rv+zxOZrKwLd7WE/8AkI1adjWL0PW0++PrU9QJ98fWp6G7ikFFFFIkKKKKAKHnVn + 6trqWSN5pA2/T1x61Pv9/1rjfHt20UU+0njHf/AG6+V/tSR3qmcf4t/aO0nQ7xo7mWIEY6snoP + 9v3rO0b9qTRdQlAimhJP+0nv/wBNPavkH4/eMLm28RTCNp+NvQn+7H71x3w28bXUt7EHa4PXuf + RverjmTua/V1a5+o/hbxTBrcMb25B3gngjs2PWujjhLAHnnmvHfgPqTXOj2Rdjkq/U8/62vZra + QGCM5H3R/KvQhjG0cU42dh0cZFP2GnKQRxilrqjinYx5EIsnljBpftAqKb71MrpjNyVyGkifzN + 2TSbj7Uxfu0tRKRDeoMeuazPE//IEn/wCA/wDoQrRc1neJ/wDkBz/8B/8AQhXDVqalKNz8fP8A + guhj/hBWxnt/6Msq/ng+L4/4ny8+v/oKV/Q9/wAFz/8AkRm/D/0ZZV/PB8X/APkPL+P/AKClbU + ZXRDVjj5RuUgU3S7c/2rb5/wCeq/zFSYzVjSk/4mltx/y1Xt7iuhSsa01ofpd/wRVgK/GbSCP+ + e0P/AKPs6/pO/ZzQ/YrD/r1h/wDRTV/N1/wRZXHxm0fj/lrD/wCj7Ov6SP2dDiysM/8APrD/AO + imrjr4t03ZGqietxqfMX61aqBCN4571PW+DruvFtimrBRRRXYQFFFFAGD5orkvHMBmhnwM5x/6 + HXT1g+J13xS/h/6FX8//AOsNU9qMLs+CPjr4HnvPEExRCQcdj/dT2rjvAPgK4s7uMsjcZ7H0b2 + r3/wCKWmiTV5DsU9Ow/urXLaFpgilXCLx7D3rpp8QVG0erDDJxPon4M6xHpel2iytgqrZ/7+Zr + 1qy8a27RoPMHQDqP8a8F8ITeVbxAEjAPT/ersdMviSnzN2HU+1fT4bOJySOKpgYns2maot5GGj + OQf/rVeEmetcr4Ln3WCHJPA/kK6RWJUcnpX0eHxkpxTZ5FWmoysOmcbqZvHrRN9+mV7lGreKOZ + xJVkGOtKZRUNFaORHKSs4bpWf4n/AOQHP/wH/wBCFXY+9UvE/wDyA5/+A/8AoQrx8TWcZ2NIwV + j8fP8AgugMeBWz7f8Aoyyr+eD4vKTrq49//QUr+h//AILpH/ih3/D/ANGWVfzxfFtSdeTn1/8A + QUrvwk3KNy1RTOYt7NpnAUVq6P4flbUbc7D/AK1e3uPaotEj3TqOp/8Ar12OhW3+m23A++vb3F + Y4nEuk7I6qWHVj9Av+CM2lSW3xj0guv/LWH/0fZ1/Rv+z1IEsrDPa1h/8ARRr+eX/gkBDs+Lmk + nA/1kP8A6OtK/oT+ADYtLLP/AD7Rf+izXymZZtOnIcqKR7DDODKo9SKvVkWz5uY/94fzrXr3uF + sZLGUqjl0a/I5a8eVoKKKK+pMAooooA8d/4Wpaf3/0b/4msnXfiXaTRvh+uOx9f92vmf8A4S2T + +/8AoP8ACmv4pkkGN/8A46P8K/jJZxI++WVJdTpPHOrpeag7JznHY+i+1c9pt0I3Gc/5zVCa+M + 7ZJyTUXnFCDmvRo5m9GdSwqirXPVtC8VxQQICegPY+v0rVl+JNtpyBpXwAM9D/APE+1eJDWGiJ + +b9P/rVy3xM8Uy22lTFX6RMeg9G9q+tweavRWOeeFTPftS/bS0LwfIYb662MnBHlSHpx2iP92k + 03/goT4au2wl7yCR/qZfT/AK4V+JX7bnx1utF8a3iRz7QHf+AH/lpMP7ntXg/gH9pO8l1Ng1z/ + AMt5B/q1/u/9c6/Ssur89JM8Gvg1zM/qc+D/AMcbD4h2vm6XL5g/3GHd/VB/dNeowEzRBlBwa/ + ML/gkJ8QJvEnhFWmk3df4QP47v/ZHpX6daA/maRET3z/6Ea+loVfdR41aHJJol2H0oZCB0NWV+ + b71MuQFiJFdPtLoxsZd9qi2mfNOPw+tY3iDxLE+lTLu5OOx/vD2rO+Il2Yhwcf5auB1HUi0bAt + 1Pp7183jsTyzZ20qPMj4Z/4K5fDu48f+EWj05N5OO4H8dr6sP7tfij8Sv2J9dvdYV47X1/5ax/ + 3V/6a1/R18ZfC0fiO0KzLv8AxI7p7j0rwnVvgNazzgm36f8ATQ+3+3XnS4glhvdSO6nhU0fhTp + f7EevQyqWtOn/TWP1/6610mkfsca1DdwM1rwHXP72P1H/TWv2kl+AdqqnNvj/gZ/8Ai6h/4Uba + p/yx6f7Z/wDi68rE8SSk9jrp4VHxv/wTk+DV98PviLYT6jFsVHjP3lPSW3PZz/dNfsR8Ivihaa + Vb2gmkwVgjU8HshH92vkiz8CxeGnEkMe3bz94n+p9K0Y/Fctlwr4A46D/CvncZmjryuW8GmfeX + hr4v2Op+I9PtopcvcXMcSja3JZgP7vvXrtfm78DPGkt18bfB0TPkSa5ZKeB3nQelfpFX6JwBV9 + rh6z/vL8jxszo+ylFBRRRX355gUUUUAfmjk+opwORVTcfU1JG2Rzmv4gVCx+rJ3Lsa+vWlZC1S + W67gOlTmIdq6Iy5ClG5mSRZB4rk/iVYPcaVMEQsTE3QH0au5aAZqjrWmpdQMJFDZXHI9jXqUcy + 9m1oN4e6Pym/bU+FGpaz4zu3tLGaRWd+RHIf45vQe9eGeAfgTrEWpMW0y4H7+Q/wCql/u/7tfr + H8Svg3p+uanI9zZRSFieTCp7t6qfWsDw3+zzpUNxuGm24+dj/wAe6en+5X6Dl3EXLCMbHl1sFf + W57T/wSI8HXegeEVW9t3iPP3lYfx3fqPev048P/LpEIP8Atf8AoRr5O/Yt8DW3h3RglrAkY9kA + /im9APWvq7TmKWiDsM9PrX6Ll2Ye2pKVj4/HUeSo0aG4etV9TuVgsmZ2AHr+Bpdx9TWL48uWtv + D0rISCM9/9lq9eNbmRxch4Z+018ZNL8Hbv7U1CCDGfvSRL/wA9P7zD+6a+dNV/a78NRz7W12zy + WYD/AEi3/wDi6+dv+C1Hxz1LwP5v9m309vjd92Zk/wCfz0cf3RX5C+Kv21vEMfiKJBrl7gySj/ + j8l7f9ta56uX/WPePSoR0P6BLP48aP4tOzTdTt52/2ZYj/AOgsfQ1oLqCXIzGwI/Cvym/4J8/t + E6r418QBNQ1K4nHo1w7fwz+rn+7X6a+Cb9rrTy0hJx7+7V8FnVH6tV5T1qFO6OhlYHkVXkXdnF + OD8806NQzD614ipe1N37pm3+nPdRkIpYn0BrKm8JXD9LeQ5/2W/wAK9F8MaYl5dKroCCR1HuK7 + ez8GQOFzBGeP7g/wo+o8/UXOeUfA3wrcW/xr8HyPA4Eet2TE7W4xOntX6P18r/DzwfBa+P8AQ5 + UhQGPULdgdg4xIp9K+qK/TOBaHsKFZf3l+R4Oby5px9Aooor7o8cKKKKAPz0/4V1d/8+0v/ftv + /iacnw4u+1tL/wB+2/8Aia+r/wDhX9l/z6x/98J/hQfAFkOlrHn/AHE/wr+O3hT7lZn5HzFbfD + +7CD/R34/6Zn/4mm3vg25tUJaFxj1Q/wCFfUI8C2gH/HtH/wB8p/hXM+PvCdtbWkhigQYx/Cvq + vtXJUo8pvTzLmaVj5S1fWE0+R1ldVK+px2z61lTeLoJcqsqHt94f41z/AMZtVew1S7WJyoDIOp + /5557V5tpniieW+ZWmcjzMdW9R7140q/LOx9TRj7SNz1e8hOpyF4huB9Of89asaJoEolBMbdT/ + AAn0+lWfhnEt/psbTDeSB1+i+v1r0TRtEhIX92vPsPSvpcFitEc1WkeyfszWjW2nAMpH4e8le/ + 2b4t1/H+dePfBC1WGxyigfQe7167bORCK/ZshrXw8fQ+AzKFqrRcEnFY/jqM3Ph+VEBJOe3+y1 + aQf071HeRi4tysgyD/hX1FOfu3PN5T8c/wDgs9+z/qnxCMv9ladcXGd33IHf/n89EP8AeH51+R + 3ir9g3xNJr8cg0DUCBJKc/YZe//bGv6pvjB8ENI8df8hjTre5/34om/v8A95T/AHj+deTar+xV + 4UkVnbw9Ykg5z9mte5/3Kwq519X92x0058uh+Ln7Bv7O+q/DnXhJq+m3Nsvq9u6fwz+qD+8Pzr + 9HPBmsx2dkUlkAJ/2h6n3rr/jJ8CNI8BWZk0XTLe2b1SKJe6f3VH94/nXiE+ry2eoIiSMoOeMn + 0r4TOcb9brcx7uD96Nz3DQ5Rq1wiW/zlvTnviuusPAd1cKpW3c554jP+FcR+z2/9o+JLNLk7w2 + OvP/LVR3r7K8GeCrOexjL28ZJQH7q+g9qWDpc6OfFVvZux4n4S8CXUF4pe3cAY/wCWZ9R7V39h + 4YmVRmJ+g/hP+Feo2XguzifK26D/AICv+FaUfhu3Uf6lP++V/wAK9alhbo4njDzbwjoEsPizS3 + aNgEu4iTtPGHHtXutc3Y6HDDewusSAq6kHaOOa6SvuuF6XsqVRea/I8zG1fayTCiiivqDiCiii + gDl6KKK/ktnuEdch8Tv+PJ/w/mtFFebW2Z1UPjR+dPx+/wCQve/7yf8AoqvJtF/5CJ/66/1FFF + fJ1vjP0XB/w0fRnwa/5A8X0X/0FK9X0j/Vp/ntRRX0GC2RNbZnuvwc/wCPH/Pq9ep2/wDqhRRX + 7ZkH+7R9D88zT+PIl/u049DRRX1lP4TzBF/pUGp/8ejfh/MUUV8tmX8Rh1Plv9sj/kCn/P8AFD + Xw5rH/ACGE/H/0EUUV8zifiPocB/DPeP2Xv+RvsPw/9HpX3v4C/wCQbH/1zH8loor18u2PPzH4 + joovv1NRRX0FHY8mQ6L/AFq/UVeoor7Hh7+HP1Oer0CiiivoTE//2Q== +UID:0e7602cc-443e-4b82-b4b1-90f62f99a199 +X-ABUID:0E7602CC-443E-4B82-B4B1-90F62F99A199:ABPerson +GEO:-2.600000;3.400000 +CLASS:Public +PROFILE:VCard +TZ:1:00 +LABEL;TYPE=HOME,PARCEL,PREF:John Doe\nNew York\, NewYork\,\nSouth Crecent Dr + ive\,\nBuilding 5\, floor 3\,\nUSA +SORT-STRING:JOHN +ROLE:Counting Money +X-GENERATOR:Cardme Generator +SOURCE:Whatever +MAILER:Mozilla Thunderbird +NAME:VCard for John Doe +X-LONG-STRING:12345678901234567890123456789012345678901234567890123456789012 + 34567890123456789012345678901234567890 +END:VCARD diff --git a/resources/vcards/035.vcf b/resources/vcards/035.vcf new file mode 100644 index 0000000..5b78c66 --- /dev/null +++ b/resources/vcards/035.vcf @@ -0,0 +1,353 @@ +BEGIN:VCARD +VERSION:3.0 +UID:urn:uuid:ddc3671b-3ed4-594c-b23e-fc37bd245a7c +N:Doe;John;Richter\,James;Mr.;Sr. +FN:Mr. John Richter\,James Doe Sr. +NICKNAME:Johny +X-PHONETIC-FIRST-NAME:Jon +X-PHONETIC-LAST-NAME:Dow +ORG:IBM;Accounting +TITLE:Money Counter +EMAIL;type=INTERNET;type=WORK;type=pref:john.doe@ibm.com +TEL;type=WORK;type=pref:905-777-1234 +TEL;type=HOME:905-666-1234 +TEL;type=CELL:905-555-1234 +TEL;type=HOME;type=FAX:905-888-1234 +TEL;type=WORK;type=FAX:905-999-1234 +TEL;type=PAGER:905-111-1234 +item1.TEL:905-222-1234 +item1.X-ABLabel:AssistantPhone +item2.ADR;type=HOME;type=pref:;;Silicon Alley 5\,;New York;New York;12345;United States of America +item2.X-ABADR:Silicon Alley +item3.ADR;type=WORK:;;Street4\nBuilding 6\nFloor 8;New York;;12345;USA +item3.X-ABADR:Street 4, Building 6,\nFloor 8\nNew York\nUSA +NOTE:THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES\, INCLUDING\, BUT NOT LIMITED TO\, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT\, INDIRECT\, INCIDENTAL\, SPECIAL\, EXEMPLARY\, OR CONSEQUENTIAL DAMAGES (INCLUDING\, BUT NOT LIMITED TO\, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES\; LOSS OF USE\, DATA\, OR PROFITS\; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY\, WHETHER IN CONTRACT\, STRICT LIABILITY\, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE\, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\nFavotire Color\: Blue +item4.URL;type=pref:http\://www.ibm.com +item4.X-ABLabel:_$!!$_ +BDAY;value=date:2012-06-06 +PHOTO;BASE64: + /9j/4AAQSkZJRgABAQAAAQABAAD/4QBARXhpZgAATU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAA + AqACAAQAAAABAAABAKADAAQAAAABAAABAAAAAAD/2wBDAAIBAQIBAQICAQICAgICAwUDAwMDAwYE + BAMFBwYHBwcGBgYHCAsJBwgKCAYGCQ0JCgsLDAwMBwkNDg0MDgsMDAv/2wBDAQICAgMCAwUDAwUL + CAYICwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwv/wAAR + CAEAAQADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgED + AwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRol + JicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWW + l5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3 + +Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3 + AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5 + OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaan + qKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIR + AxEAPwD9/KKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKK + KACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAoooo + AKKKKACiiigAooooAKKKKACiiigAooooAKKK8t/bU/aq0X9iX9l/xf8AE/x5bzXth4WtBMtpC22S + 9nd1ihhDEHbvlkjUtg7QScHGKaTk0kJtRV2epUV/PP4k/wCDrP8AaGv9buZfDPhn4W6dYPITBbya + Zd3DxJngNJ9qXccdTtGfQdK29B/4OQ/2t/E/wr17xponhf4Uz+HfDV7aafqFyNKuAYZ7rzfIQIb3 + c27yJOVBA284yK7Xl9Vb2+85frtN9z9/aK/nq1T/AIOe/wBqrQ7mWHWvCnw5s5oEWSSOfw5fRtGr + EBWYNdggEkYJ65pk/wDwdB/tTWrlbnwv8No2AckN4dvQRs+/1u/4e/p3o/s+r5feH12n5n9DFFfi + N8N/+CzX7d3xc+Ful+L/AId+Bvg3qWma6lzJpdsiquo6mtuzJMbaxbUhPNsZWBCITxwDXkY/4Oev + 2q200Xq+E/hybMzfZhP/AMI5feUZc48vd9rxuz/D1pLA1HomvvG8ZBbp/cf0K0V/PNrf/B0N+1L4 + au1g8R+GPhrp87LvEdz4evYnK5IyA12DjIPPtXvX/BOr/g6A8Q/FL486L4N/bZ8OeF9P0rxNdxWF + rr2hpLarps8jbUNzFLLIGiLFQXVlKZJIYdFLAVYq9gjjKcnY/Z+iiiuM6gooooAKKKKACiiigAoo + ooAKKKKACiiigAooooAKKKKACiiigAooooAK+Q/+C8ng2Txv/wAEmvjDb2qb5LPT7bUQPRbe9t5m + P4IjGtH/AIKy/wDBUnQf+CXPwS0zXtX0Z/E3iXxNdvZaJpAuPs6TMihpZpZdrFY4wyZwpJZ0Axks + Px8/aK/4Ocfiz+0f8FPGPgLxR4B+GtnovjLSrnR7iSCO9a4ginjaMsjNc7d6hsglcZA4rsw2GqTc + akVpc5cRXpxTg3rY/N2vsP8AYw/a9+F37IPwV0q38WahqnijW/E3jfRdd1bS7bSCq+H7bT/tg81Z + ZW2XEubmKREAAyhBIr4685fWkMiE5OM17tSCqKzZ5EJuDuj7v1r/AIKLfD3wx8Hrrwv4q1K9+Omv + Q6Q1s2teI7K9t49TMutW14lsWSdLkJbRQyyKWcAySMoBXBbrvjZ/wVY+EnxC0H4gaP8AY9VuV1LT + /F9z4c1BdPKS2N9qU10kUDhjnyZ4LiJmPJR7eL/ar84g6A8Y5o8xPasfq0e5p7d9j7T+Bf7Vfwb8 + J/C39mrXfil4r8S2nif4BXuo6rJ4d03RXll1ueTUPtdvGt2zrHFGdiq7HcQGICmuq8If8FNfAf8A + ZWm694o8QeIrG2uPD9tod18OodNdtMtL8aol3LqqSBvLIADTAhfO8w7Puc18Al0b7200F0PXbQ8O + m73D27StY9//AOCk/wC0JoP7TX7UN54p+G2unXNKurdliJi1GM2wNxM6xkX88r5CuCRGViBPyovN + eE2V01jeRTwHDwuHX6g5FVxIgPGBS+cvrW0IqEVEylLmdz+zb4ZeKYvHPw28Pa3YP5kGsaZbX0bf + 3lliVwfyYVuV/Oz8Gv8Ag6L+MvwW+EXhbwdongj4aX1l4U0i10e3uLuG+M88dvCsSvIVugCxVATg + AZ7Cul/4i1/jf/0T34Uf9+L/AP8AkuvEeX1b6fmessbTtqf0A0V/P9/xFr/G/wD6J78KP+/F/wD/ + ACXR/wARa/xv/wCie/Cj/vxf/wDyXS/s+t2/Ef12kf0A0V/P9/xFr/G//onvwo/78X//AMl0f8Ra + /wAb/wDonvwo/wC/F/8A/JdH9n1u34h9dpH9ANFfz/f8Ra/xv/6J78KP+/F//wDJdH/EWv8AG/8A + 6J78KP8Avxf/APyXR/Z9bt+IfXaR/QDRX8/3/EWv8b/+ie/Cj/vxf/8AyXR/xFr/ABv/AOie/Cj/ + AL8X/wD8l0f2fW7fiH12kf0A0V/P9/xFr/G//onvwo/78X//AMl0f8Ra/wAb/wDonvwo/wC/F/8A + /JdH9n1u34h9dpH9ANFfz/f8Ra/xv/6J78KP+/F//wDJdH/EWv8AG/8A6J78KP8Avxf/APyXR/Z9 + bt+IfXaR/QDRX8/3/EWv8b/+ie/Cj/vxf/8AyXR/xFr/ABv/AOie/Cj/AL8X/wD8l0f2fW7fiH12 + kf0A0V+AEf8AwdsfG0SKZvh58KmTPzAQ34JHsftRx+Vfpv8A8Efv+Cumkf8ABUz4feIGn0D/AIRT + xj4PeBdV05Ln7RbyxzB/LngkIDbSYpAUYZUgcnINZ1cJUpR5pLQuniadV8qep9kUUUVzHQH1rgda + /ao+Gvh3VpbHXfHvhK1vIH8uSGTVYQ6N/dI3cH2rh/8Ago38TL74Y/st6tN4cllt7rVp49NE0bbW + iSTcz4PuqMv/AAKvxy8Y+KHsNTSeOXHkNv3BsZ5A4/MUPRCvqe2/8HRFppP7U3wc+FOsfs56rY+N + tU8NavfW11Z6LMt7NHBcQxsZWWMkhVa2Vc+sgr8ZoP2aviVcput/Avit1HGV0yU/+y+4r9KbPxib + XTILQKZPPuC6EfxK4UKeR6qf5VpaX4vtbHTl8p5N0108IlLYHlsmBISORyytjHQV00cdOlHlSOer + hI1ZczZ+YY/Zr+JRCkeBfFR3HA/4lkvP/jvuKmsP2W/ihqkmzTvAHi6dgpbCaXKTgcE/d96/S3R7 + +4kBt5I5fMV9jxtxuIz8oYdCTjB9QPSugtbPVrK9W40RrzMIUSblP7tuOH69OPwx7VbzWS3SIWXJ + 9T8spf2XPihDE0k3gDxcqIcMx0qYBfr8vFVX/Z2+Isb7X8E+KA3TB02XP/oNfrfqlrfXXiCK5s7B + 5Ib1V86FG2q5YYbZ6HPPA55I6Gsjxdp1xaz3AJtreK2Yoq+X8zAnABxz+J/OpWbS7Ir+zY9z8ph+ + z18QznHgrxPx1/4l0vH/AI7UkX7N/wASJyBD4H8UsT0xpsp/9lr9KNTiNvo0s7zFZo35jLYYk5HT + 0HHP+FZOg+NGiufKdzvU9M9a0WZSfQh4CK6nwb4Z/YX+N3jOXZ4T+FPj7UnH8Nvok8h/ILW+P+CY + n7SbH5fgX8VT/wBy1df/ABFftt/wTe8S3/jfVvsGi39u1zbRlhb3BJO3cpJVh/D0yhDdARg8j9Dv + DFjJeWiPdo8Nyo2yxk52sPQ9x6GnHMZvoZywaTsmfye/8Ovv2lv+iEfFf/wmLv8A+N0H/gl/+0qO + vwJ+K/8A4TN3/wDG6/rcXTpMdf0pp0+Xtj8qv+0JdhfU0fySH/gmD+0oOvwK+K3/AITN3/8AG6Q/ + 8ExP2kx1+BfxV/8ACauv/jdf1tPYSDqqflSGxb/nnGaPr8uwvqiP5Jf+HY37SX/RC/ip/wCE1df/ + ABFH/Dsb9pL/AKIX8VP/AAmrr/4iv61zYHPMSflQbAn/AJYr+VH1+XYPqnmfyUH/AIJj/tJf9EL+ + Kv8A4TV1/wDG6T/h2R+0j/0Qv4qf+E1df/G6/rXOnkj/AFK/lR/Z3/TBfyo+vy7C+qeZ/JR/w7J/ + aR/6IZ8VP/Cauv8A43R/w7K/aQ/6IZ8VP/Cauv8A43X9azaYp6wD8qb/AGYuflg/Sj6/LsH1TzP5 + Kz/wTJ/aRHX4GfFT/wAJq6/+N0n/AA7M/aQ/6Ib8U/8Awmrr/wCIr+tUabjrAPypn9loWO6DOfQU + fX5dhfVT+S0/8EzP2jx1+BvxT/8ACauv/iKQ/wDBM79o/v8AA74pf+E3df8AxFf1oyaTEcboCPwo + bSICP9Saf1+XYPqvmfyX/wDDs39o/wD6Id8U/wDwm7r/AOIoH/BMz9o89Pgb8U//AAm7r/4iv6zv + 7Gt2XmI0z+w7bfyhFH1+XYX1Y/k1/wCHZX7SB/5ob8U//Cbuv/iK/Wn/AINav2SvHH7Nmu/GLW/2 + kPB/iPwReanBpljpqa3YS2TXUatcvMYxIBuAPk59Miv1iGgWxHKkfhSf8I/bKDhWrOri5VYuDW5p + TounJSRvN4gsUGXvLcD3kFW1YOoKkEHkEd65Y+HrZudrVreGVFqkttFnZFh1z23Z4H4qT+NcTjod + kJybs0fO3/BW65Np+yPM6gEjVIevb93NzX41eMrtvENneW2nKWlguBLHgcvG2AR+GFP4+9fs9/wV + e0KXXv2RL1LU4aDVLSQn0G4r/wCz1+bXhj9n20j1GCXUCjg/dYIQcnHGc81wYrFKg0md1Gj7RN3P + ENG8MavefZZLVZpreIKsWcnylOXAHp8xYkdmPvXf+GvglrWqMg0qGRra5X7hGAhPYf3uma+qPBnw + m8PeF7FJtRsYLlX+ZcrkZ9fY/hXoXhW2sJrSIWdtBGpOVVV7deK86eOc9Ebxo26Hg3wn/ZJ1PVQk + etWMc4mXaF3hHQ8HcpGc49Mda+m/h/8AsrppdnBcarDZSXKJsYFQQ/u3HB5PHvXqfwn06OJ0lt41 + A24OwYHP8+1dvJB9pLFQJVBwSn+fatoXnG7M5RVz538Sfs0WjwFreGO2QNvCIAdpzyFJ6dBx/KvJ + vjN+zxpfh/RbyezSedo4yJJQm9n3JweR1Bxzjjk8GvtBgsN0Q4yD2PSud8S+GbfxHp97YzIscU6M + pCrgknkHPb6d6yas9GU46H4z6/52sXGqxRKRPp4BZQcsy7goIz2yVGP9v2AHAvdT21/CSkse3Egf + HG0nr68Ej86+lPjR4ft/gV8c9UsdahN/Be75lWJCJbeONlYtz94Yibp0CnpXkXxJ8B3fh6+e6uYp + Q5ZLaBSwIjBd0X5SP+maKB23DPFdtKvpqZTs2ej/ALCfxyPw7+Nuj3LzyAxXCFtuQSpIDL1H8JYV + +5Hhe+g1nR7a90+VJ4rmIOkinIcHkV/Oh8DPt+mfEDRb0RSNbpcRQzN0LK5xyOOCp6+x9Dj97P2T + r+QfDS0sLiQy/YFEccnVZUwCpB9wQefWuqEk3oYyVj1fYaPLoL8UCTJ6GtiLgUz3pPLNL5mDyDS7 + 80BcTZx2oEfrj8qUtxmgN/ewKBieX9Pyo8vjoPyoMq56jNIbqNfvOg+poFcXygeoH5UnlD2/KmPq + ECcvNEB/vCon12zQ4a5gB/3xRYXNFdSwYR7UeSPaqv8AwkViTxdRH6HNMPiiwHW5jP0p2E5x7l3y + BnkCl8kHqBVJfElm33Zg30BpzeILUDPm/pSDmi+pa8lfSj7MnpVP/hI7X/nqPypf+Ehte8v6GgOa + L6lv7On90UhtkPaqh8R2gODMufTFQ3njTTdO2/b7uOIP93dxmnZg5w7ml9nT0FQwRiPU5toxmJP5 + vWYnxI0WVsRX8LH0FX9J1K31iSW402VZUwsZI7EZP/swp7JiU4SdkzwP/gqrr0Xhj9h/xZf3m/y7 + eWyJKAbhm7iGRn61+eHw7+JNhq+geYbnGwb2WQbMrz36dutfoL/wV40o6v8A8E8fiIi4zFFZyjJw + DtvYDX5KfDXWJr7w3M1miuVt2VVJ5Q7ec8e30r5nOpuFSNu36nsYFJwfqfY3gPVx4m0oeXKskUoK + Aqdw4A6Hv2/Gum8ICTTLoxuruUY7+cdu34HFfPPwbvZbGxAFyYxGd2xR93PQeuec/hXfeFfG9xeS + r9iuC4mJJdjnOCBkDsPvenSvJhiNrnTKnZ2R9e/DPXzbyQxWYFwpZC54GxC33seuO3/6q9O+1JbW + rMRtJXAzknv+v+NeD/s9wSalax3N+kRG3C5QHjHykdhkDpye/evX5NRRkVCoyCOFByeB0xz3/l61 + 72GqNwTZw1IpSsQ3MgiQHc+eDn0HvnrWfbXYW9aRirqSM8jtVrWbuO1sHjVDhl288Ajjuea5eHWV + MbgOFJ7/AKVnUmotalxi5I+Wf+CnfwghvltfF+ggJfaUHedYowWnQoQQSegClufU18r/ABc8e2Xx + BOuXEafv5rCVFiycrcxTFl2+gcAcDHJH4fbn7a3iPT7n4bajb6neW8KToRKWfadhDfLg98n06gH6 + /mL4w+MGnaZZyu81o/25r1UcyoXBaRYl2jODxsbpngHPWnCeuhySWrRraJ8X4dd8a+GZ/DcQj8y9 + SPy2UKgiViGyM4yUOecYyelfsh8BPiVe+G/AmweWk3yMvyAKYyoKFfbHHoNuK/DP4D39vrPxv0ey + 0eBZ9MmvUuI0jO5pAWQqu4HnKzHOODjiv3J1fQJNH0G1itkEQgtY02AcLhRx+Ferg1zts8vMakqS + XKd7D8ctTkQEyoM/7Iqzp3xk1We8WOCSBmkIUeaAFGffjH1zXjlrLfGNfLxV23kv4jkjODXo8p4y + xdRbtnvbeLvEXlgxrpDk+kq4/wDQ6ifxd4m3KBBpHJ5/fLx/4/Xltp44lgtVim8P6VIqjG5vMy3u + fmpw8cgtx4a0jj/rp/8AFVHL5HX9bT+0/wAf8j1IeKvEpYBoNIPuZV/+LrH8UfEPWNMUQ6gmmp5w + IzFhnH0wxxXDHx75YO3w5pAJ9fMx/wCh1zeqapd3N1JLDbxwK7bvLjJ2r7DJpqN+hFTGWXuydztJ + vGF02c3Eh+r0yHxJKY8tM3X+9Xn1zqN4F+6fXrVvQFur2MPKWAzjGarlOT28mztTrk9wQFdiPrVm + 2+0SY8xm596q6ZpixLHk+ma2yixqAW4pXsdFOLlq2WNPiYpksc9KsmIiT2ospkMI+YdOxoa+UEjP + zHsBUtnXGKSLkEoRfmOPenz3/lWrPHHJKACQEXczfQDqa8f/AGiv2g9I+BXgq61jxLZXeozrIttZ + 2MMxDXszHCoB0GT1OOBk1+e37Xf/AAV++KXwUh3WOlLp8Fy6rH9jjVbO1k67GlZWaVxwCAQvBHY1 + MpqO44y5tEfpN4m+MGsW961voXhy4hK8Ga+fy1HvsXJ/UVseGfifNd2hXXRapKkZctbJIckdvmz/ + AEr4V/Zk/wCCqfjLxFazX/xz0+1vNFW++wzTRW4hkicD5zHgDIRvlIIPI6ivt/RNVTXrGPU/D9xH + c6ff2Rlglj5V1OCCKqMlLY55c6le+hyHiT4qS+MbOG80C6nsI4bo24ldSxkbjoq56e+BXc6V8OFu + Ykfxdf3GoSY65K/1Nea/sa2cl38PtRGs4uHGrTEeYM46YxmvcYwcjI9qpK+pnQj7T35dTK1LQ9M8 + PaZu0q0AbI+Ykk9fU11vwRvDeaFett2hbrAH/AFrmfF2bfRHIzyy9unIrofgAd/hG7fnJvXH5IlO + XwnZh1aukjlP+Chfhb/hMv2J/iTYYB3aLLMM/wDTIrL/AOyV+RfgDwRc6L4CupJkGY4JnfcOqhGJ + JPYcE+p/l+0v7SGlDXP2fPHFoQG8/Qr1cHof3D1+Rfg7Uk+IXjG+8PzE/wBmRWphuRGQDKXj+ZSc + Hs5GPr7V8tnkbygz6XAy3Rwvhn4kreReT4edGiJVHkkGPNcYPHGQuTjPfAPevdfgz4OutSurWeUq + xvXEa5yAmB8wJ9MkivkT4H/8VH4qm+1YEdtcNFMExhSvylh7bh+NfoH+zRqFkt5DFevGrLjYCMoD + jDfTIwfwFeFQpc8kmenWagtD6E+HOhSaFoCJG9vGpwNvlHK5H+91/wA810Mvm200bs8beWu4koeg + +jY7/TrXN2utpY3EUt9NDBFEpbOf3bjBGdx44PXp/jIP2hvAtjq40zxBrmmQ3Mh2hHnG8++0Hof6 + mvoYJWS2PLbuy54hluLm3dU8tsjGc4z/AMBJOOe+e/0rkpLkySKAzbeWZemDwMf/AFq7vUb/AEnV + 7GF9DuIp4JkbZIr7l4Gfz/KuJ1KAG4Kwr91sZOOfb8v6Vx4qEro3pSTR4H+3R8NtQ8a/De4m0Azl + LdTLMsYXAjVSWJB68hTjvivxS/agin0DxjdwDa0FvIm2MncxO77renRfl9vXp/Qr4y0ZdW8GajaX + QIS4tXQk8kZB6evP8q/AX9vnwr/wh3xLvre+Kkw3AiyB8u3fuJwP4huPX+nKw6s9TOp8Wh6P/wAE + gfhJd/G/9sTwdaaNaC9ttLvI7yXa2PMRAzSLnuP3OAD6/jX7/a74d1e9sXfVdMvIJJVPCoWUYHqK + /Ef/AIN9I/Elj+0LqGp+HrK+fTNJup1e7hBQFZP4BnG7pIfYE561+0nxK+I1/wD8K5eePUb2F05c + rKQ64IPY+lfQYGpHWKep4Ga0ZRfPJPlscX8TPGuhfAbw0NU+Lms2Xh+yAwJLycR7z/dVerH2AJrE + +Dn7WHw3+OWqfYPhr400XU79vu2wuPKnb/djk2s34CvmX4B/s+af/wAFBvF7fEz9p261HxAdVuJl + 0vR5ZW+yaXarIVjRUXG4lVBYnqSeM10H7U//AAS0+HOi2f8AafwhtrvwZ4hssT2d/YzSIIpVwysV + LYPIHTB9663iYp2sccMsco8x916b8N9SuoQ0UEjAjjHPFOn+H17bOBcRSIT61y//AAT4+IutftCf + smeEvE3jHVrxNYnhktb7y8BXmgleB3A/2jFu/wCBV2Wt+IJtKvXGoX7NFbTyoZJH2jaucE9u1a8x + nOhCCu0ysvwu1O6UGKByPY1m6n8NtVtOHgkBY8fNXX+EAfFXh+1udP8AEBeSaFJHWEqyozKCRwfe + uG+Lk+oaPK9rJfTXCrc2Tq5yrDdcKGBweeB+tO5FWjCEeaz+8G+D2uXPzQ2khX3YDNWtL+GmqaPY + Mt/bSKwJJOQQKhfU74MQLqdV9BIeP1rmfih8a9H+Cvwy1XW/ijqM0ek26yCQ+YS0hOflXnrjcfYA + ntTb5TKEKbezOoXwz4j1LUYls4I4bdJsEj5mdBjuemee1ZPjrwP468Q3M9toEc9jAs4jEqsoxHu5 + Zeck4ya+FT/wXi0sfE/StO8HeGJdQtbwsfI+2Ot55YbG4fLtDHBIVhkj619X/s4/E62+P9jr2v8A + gnV7m40rXZ0lgEkhY2p2ndE6Z+VlPBGfccEVCkp7F8iS6/f/AMA9q8B/D6/0GyniL3N15svmb5ZN + 7fdUEZ+oP51g6R8LvFVh8e9X1bVZriXQLuwhS3hMn7uGRT82Fz1PBzjvW58P/NtPDkKySOW56Ej8 + q1ZLycvzM5CjONx4pWOulNKCsj84v+Cs3xwudA+JIsLcKzaDGotoTlg95OzJHkDnhQDj0J9a634B + f8EafB/xH07Sde/a88W+LvG2vhVvJLGfVPs+k2U2QwjjgiVcqh+Xk4bBO0A4Hh/7a2sJ40/bja12 + G9ez8UWm+LBIZI0QdBnJBL9j96vs/wDZu/aC8SePdWubPVPAt/o2kIQLW+nZ1ecFN+WjdBs42jGd + 2TgqMHHBObdRnThYws79TyX49f8ABMD4d6KNR1j4Qtq/h7Xt811HL9ukurV5mJY+ZBKWG1mJzt2n + nIIrqf8Agjf+0mfjH4G8TeBfEjpHrHhOeQLbhwxiXfslRT3QPtYH/bPpW34t+InxB8UePptKvfC8 + K6JfArDdQogNshDf6wtcbnb5cErGANwwGHzV8i/sGXOofs2/8FZtbRoLmPw94tv59PD7CIpJJIw5 + AbodkrYOK0ou0iq3LdOx+ovwj+H0/gy1v7ZbZ1jkuGmU7epJOf0AruEs5EUCSJxXJ+CfE8+oC9dL + h2CXLKp3HjGOK6AaxdEHdNIcerV3K5xxcIrQTxdpc15oxS3hmcll4VSe4rovgvpL6R4LC3CsjzXE + khVhgjkL/wCy1yPifxlqGkaSz2FzLG4dBke5Fdz8KdSuNZ8B2N3qkhlnn8xnYjr+8YD9AKJv3TXD + OMqztvYv+NdOXWPB2rWk2Sl1ZTQtj0ZGH9a/Bj/gmv8AFNPHPiHXpZZt0/8AaMhRScMVJOByfX0r + 9+WUMpDDIPBHrX8wP7H9vffCf9v/AMa+EUnZI9B8S6hpki7ziTybt0wMf7h/CvBzmnemqnb9T6PL + 1zOcV2PZ/CHww8ZfCL4oeMY9T0W8jW91G5uLKQxlo5VaRmAU4+bGeR6fhnyz4/ftj/HP4IXd/deD + rQ2WnxoC8tzaZjt8Yxhj06gZJPbjjn9w9C8N+HfFfgexiubK1uC8KM5dASWx1x61+eX/AAV8/ZFk + 8S6dB4gnab/hGbFwkOl2qhTe3JOQWyR8iIrMzH+8oHIrxVS9lLnvdHbCqqs0pqx+WXir/gqD8efi + nJJH4j8XTLbsrBLdVWGHBAB4JC9ARzn1wT07j9nP46eKtdvZP7W8W+E0a7dbWZrrU0jMZYnCiaQh + FZ9pGWJ3Ed8Gu4/ZZ+FHhFfEt+njjSnuzqsclkbdNNEhhRwQPLkDHBGcZwOhzX0N+yL/AMEvtF1v + xrBqF9c/afD0l3ZXt1pJsLhE1VoC/lm4gkKxIcM4+UEgFhkFufThUo1o9jnxMZUZ8sY6dyx8C59f + uviDHoH/AAkd3puv+Uskdt5eJJIssVaJujxks37xWZeBjkDP6Gfs/wCi+KNB0Bbfxvqd5qnyKVku + NpZOvcdeo7ngfgOW8Af8EvfDUM9trWkx/wBkNZXJutPtYdwjsnLEnyRuUQB/4kQoh9M5J98sfB0+ + g2gjvRK0kagNIyfL06cE/q3fvXJ7OS3WhN43vEzdWMdvp8y3X7xZEZWX1GDX4nf8FEfhxa/ED9oB + rbS5Gee5ufs06OxywzujkyehCEqe+PL64Br9q9Vk/wBGYAkup4BPB/D9a/NL9ob4ZXPjv/gopbeH + fDmlw6i16JNRlgVTvHlQxMVOPu+YzoM+meM9c4TtJ2NKkLWZ9kfCf9kjT/2ef2GNS0LwbNNpxtPD + RSXULcCO5mfynbzWdcHOWboenevPf+Cbni3V/HX7M2q2OrSXNyYbpWgEj5YhlDMo3HAzg8epr1L4 + PftdD42aLeeB/iZYQeHriWA6VcWKqQyKV2cZA49+1Y/wR+G0/wCzt+zRZQagsUF5cXjSTrkq6qkg + VQv5d+xrqy5qtiIyh0TuLOJrDZfUp1lq3G3/AADQ/Zu/Zom8CfDuy1O41CGKPw/FDb7EimW4jlTa + p2OkihFLbiQUJJPJxxXWfGL4V3PxDlvtf+1YNlOYZxeS3InQLgr5OJRGuBuypjO716isPVvilc+H + L+70WJbG+0htSzdQyB5DgklXwgYtgnBGCQazb34skPremaFY6fZ2OoTxxNLG8oe8kAb5sSop44G4 + +voK9iSdzxoSpuCtsfQH7A2jL8P/ANnz+yYo3j+yanfygFSFCy3Mkq7c9tsi/jmvALebX/23/ip4 + 10v+0YtE0fw7q89jHBLCZhLgsu/AZcMSGbcc44AHWvorwQmseE/DMNnZ3enFdnJktndyfdhIAfTp + 0ArivBvw5tvhz428Q31hM9vc69J9ub7MNm+RVORznqccA56+uK6K/MoI4qE4VKnL0PC/H3wFuv2D + /Hlr8QPB/jC9V4JkhvrKZ3FtfRs4xHsDbMsWC8rxkEYIr6T+KXjO38Z6HBq/hySO4g1IaZcWzZ+W + QPMrLz6HIrz39rn4VwftD/Ca58OWlzerqet3KvG08hf7J5cwcOEY/J91GK8HA6ZPOingv/hUnwY8 + O6NLdiaHRf7LsEl2lSUiliXcxJOSeTmjDc1nfYzzJKC5Yljwd4l8e3/i4x+OtK0Cy0ghsSW11I8o + P8OQygfWvm/9tQ3/AMX/ANo/Rfh1aTwW+gWcUd1e3kxX7NbtKwaRpCx2nEQYLn/loYh0Jr6ouPHu + jzusVvqNm7v8qjzQST6da8L+G3h/S/GWueMtM+KtnPO15ej7TdJOY3eIyziNRtwV2tFJnJIKsvpT + rzUo2TOLBwUp2Zofto/sR+DfG3wHTVvhP4U0aTxv4YtRNoskQWJrkIARB5q44cAgE5w2018Of8ET + v2pfGHh/9qW/8Ba3Df2tpq2pype6ZdJmWB/LkYltwDK6GM5PGRuznNffHjf4feCfid4UPhbUxdab + pGkRmS1ntLkoIWV8qYmViG+UDoCACQcZIr5YtfhND4X/AG1tB8f/AA2tbywlEscKnb+6vodk6JIx + A5kZ7edCuQdkR4IYVjTqJXPTxVPaVtT9LPCHjO1uNUbSIlb7TBAJ2YY2bScYz68Vp6pr9tpyXTXL + ECCIyHjoAK5f4XavZ6xaXUmlzea9nO9nONpHlyIRuXnrjiuJ/bi+P2lfs3fs9674l8QXCRyxW5jt + 41bEkzEgYUevP612OSSbOOMZRilLc/PnVPE39m/8FD7i7uRHNLcalf3UqPjb5sCxcA/UV+hFx8Qp + V8J6fqPgXw9da1PHL/pGn2c0FvO+UILAzMinacZGc855r8G7X9pHW/FnxRu9fd2e4isNYnEi/wAU + lxA4Vsn0cgg9cAV+iHwR/aI07w5pmkeHf+Cia2PhXxC1klzYa9LHE+na/bsinzI5JFKBxkb43AKs + cjg15nU9DD3grM+19V+Id5ofwrtk1WG3t9XaH9/GkwmMDddpcABsZ6gAV8Afs/fGibxR+17Yw+IX + tGt9N1mc2Pk5O4C+kZ5XyT87MZBleNsajtW78df2l9M8ZW8vw9/4J6Qw67r2txNPc6tZwolpp8Gd + rTvIqhT/AHUA4zj0wfN/gT+zf4i+BHxN0rSfiLZy23iLw8ixXOTuDsWkYsrfxDezHd3DCqUrWbJx + D5j9Rf2atefVPC2qzXOSy6jMo564Nekw6j5kJcAjPr2ryf8AZ0tJdH8NajHOD+81CWUAj1xXoKaj + vLocYJI6160FeKPHjOy18yLxvqxXSSrYyZI/5ivY/hjALf4e6MqDbutUcj3Ybj+pNeAeOrsi0A3f + xoK+jvDNkdN8OafbOQzW9tHGSO+1QP6UqmiO7L3zVJvyL1fzjfGv4Wy/B3/gsh8ZGv1EEDeLrvUA + meWS7b7ShHP92YE1/RzX4gf8FUfg7NpP/BYvXrojbb+KdK03UowP4gIVt2OfXfbNXh5y39Wdu6Pp + stdqrv2Ptn4Baw8vgqweBCI2hAXDZzj2PSrPxo+Ett8ZfCMun3aI7Kd0Zkzx6j6EZ9qvfBCztofh + RotvDhJ7ODY5XGDzn867XR/3KgTLg54xzn/PpXjU4uyT6o6aiUtT418FfsEeEvB+rSNc2p0yd7jP + mRkiPBH3dmfl6HkY7cen1R8F/hx4f8J+UHijuGIwHErfKOx+Zjkc1N460q1unIgtgZ0YbQ7DkY/T + IxXN6UkumarbwI+woW+7IZAD9eMY/XmrjahLRGbg6urZ9BxXVhFasliXwy4AB69emeO1c54sMy6P + JLA4ZMjcCM7R7HJ7Y/n7VpeHtFL6Kjap5J2jnqcDj3wDz09vpXO/ETxJEbL7LpzAxtwxA7E+v1r0 + Ks+WF5GVON5aHGay3nrI4xuwOBx36f59a+e/+CfulaX4/wD2oPij43usSNokr6dDdEghFVv3hB6j + BRBu6HZj+Hn6GcsNJnlPO1CRx1x9Pxrk/wBn74d6N4c8Ma5HouniH+3z5V60B8trgFGDEdMEbzx2 + +ua8un8Sdjpq6tRPK7H4Pan8QP2ovEHjqSBNJ8LLdGaF9wMk6Jj51QdAxHGcGvUPjVbzeJfBU0uo + jy7c48rcwQDLLk/l3r0uy8JaL4U8Kx2MEbR29lH829uNo5O4mvi74k/FS++JXi+aW+vXuLOOR/st + uOEiXOFwo49BnBJ4/D1sqw/s3NvrueXn8njI043sl+SOY+M3h/xN478f634g/Zq8SWOg6vaXe6SO + 4ga5stQiXaiyvGMMrAKh3r/dwQcDHLSfDv4h3MT65+054y0vWIIctZ6bokMltb3Tdd8rN+8kBbtw + vBznNU/iz4/1X4UNFL4Zg8tbm6D/AGtJFH2VW/55j724khcHg565IB0/Bk/ibxh4Fsr34vTWovp1 + MksNmzF2VjlFkc8gKrKpIADFTjgYPqOD5r3PLjZRSse4/Eb/AIKA+L4/AuiJ8HPDi3Mk9irX2q3Z + DiCZflcR2ync2SCQxOB6V8xH9vzxZ4T8cR+Lfiz4s1kw2rNZxssKSWkDM2B51llFdCVUMFZJADlX + BGD65p0X2PTl+xxFVtUURxrn7uSuPbp169fWuW+PvwO8MfHrwa+j/EK3mMU7fu7m3byrmBlAIZXx + hhk/dYMPbPNbqalpJaGSp8jUo7jPEn/BRTQfHyQsnjGPxd471j/iX6bHoGlT6PpHh5JMLLc4ndpp + J9pc79zDChQEBZm9U8K/tjalr2kWWi/FVftOnXF9CqaiDtuI1jlVsyoBhsheowR3zX502X7Oj/sc + fFiHUPEN/wD2po1xE39lXfl7N7kYZZByFdVPTODuBB6ge7W/j631XQdLuLdy4FwsjYP95mX+oreN + GnFLk2ZnWk6r94/Ry3+DegTvBe2E1zKOJUIlBVu4IwOQa1fDHg6wbRPEurlbaK9lt3lMs0ghRGtv + PZS8h4VSXwxPbntXin7OPx507Q/hutj40vGgntLmSKAOjtmIhXXkA8fORXjP7YH7dd1p2j3/AMOf + A0Nwjanv+236DCmKR2ZYQM7sspGTjvjvXLVUKMXoYYKm51+SJ3/g34teKvjj4/bR/ht8ItT8GJFG + 8F74m1qdF09Ic8m3CMzSlv7ihQe5Uc14n8Fv+ChEvwl/a11nw9K9v4q+Gmh3g0+GU2cBuZZU81nu + 45SuSVmurkoAQCrEdwRa/bK/bKuf2e/hs/w6+Ecb/wDCU21sPtt5M7PDYQNCJBlSQWlVWRdpG3kH + rwvx98G9FuU1eygaYSn5TK5TDytnJJPck5OfevOnP2bSifRYfD+2TlU10P6BvgzoeieLvC0es/Cu + dJ9J1uRrz7TtYea7HDHawyDkHivnz/gp1/wTI1b9r3SrJfCWr/ZwESK7SSV2eWIOGKxqxEacqOmC + 2TkjGa+qfgn8NLDwX8F/DmgpBFNaaNBFAQ65y3lgliPXLPmty9+Hfh/VXVdS0mxnBO8B4gQD9Ole + lKPNGzOBU4y2PyO+J3/BGe3+EPh3SNMRbS4k1a1SGdoJCRF5kycFwo81ykMgz8qrvwMjk/Vfxe/Z + b8K/GT4a2vhf4v8Ah6y13SLIKkMU6keXhQFaN1IZGwMZUg9q+rde8MWmr30enapZW1uLWMnTWiXC + OgXlAOishJ47ggjuF5i+8O/aY8aahMRX5gAMryOefWuSdLldzenG1zxj9kb9jHwb8GNTttC+Dfh2 + 30nTfOSS8kUvI8oX5trSSFmIUE4BOAW96+gPi5+zZ4f+K3iSK8120/0q0fzEkj+V34wQfUEY4P8A + dFaPgcL4VtprXw3bf2jrMp/e44gscjIEsnQHnJUZY8cY5rfEWoeHpVMUkmsQ8BlYqJ0OOcE4DD2O + D7npW1KHLHUJKL0see6l4MPhx7h9HXEbyl2XHKk9iKyXuip/fwqfXjmvVNU+zeKLWSbTjLDPD8si + yRNG6+zIwBx7/lXH6jpsJLxuUaWP7w712QlpY8vEYfld0edeKNSgvdc0rTFkjS61C8jjjiY4LZ4/ + nivq1VCqAvAHAr5ktdMTUfjD4VtlhjY/2qkmSPuhI3c/olfTlTV6GmWRa535/wBfmFfmz/wV7+FD + TftpfDfxJbIhbVdBl0/33W1wX6+/2pQPpX6TV8a/8Fj9Kh0v4f8AgLxVKwWXSNcfTwW4GLmEt19d + 1suPqa8vM4e0w01/W572Ely1Ym58ENFi0z4bQG6lVWkUMAh9ABgDA9+fr7VpajKYpdu1/L6BmH3f + rXFfCf4v6X4e+F9hLr9uzShQUYuShOB91Ryx471BbftheHPFnjGHw1Y2niK41e73GJG8P3kUbDuw + keJUx75xXj88PZxVzvUZOTdh3jD4u2B8SyWWoBIpI1+9nk9e+PYfQ12Pg+w03WpzfIUkdHGwlcMp + JHODxnOee+a+Vv26/BuseA9cs/E+giZYXkUXkYBZV6YPoB0Gf51137PPxSl8Q6DCEYplCSC3APAI + OBz1PeuRYhwqctRGjp3jeLPqHxH4zNrZG3gcAqu18dW+vHbJrz++1B9U1HGwhQMAZP8An1qOKaTU + XBdgCVw2DlfXHP8AKrtvbmG6VSScYwcdO3860rVpVnYmEVTRneKLg2fhW7eIujPF8p44P0NcdceJ + B8NfD2m6hNcTypPcOSgAJfKod+c4GNuPfdmul8d3Hm2LJPGZoeF2jjk9z6+uPbNcNFf6J41s/wCw + Ib5JdWsoN4gIwsyHJDxnoTg/MOCCD25rqwNKFWpy1NjzsfWqUVzU9zxT9rL9t/XPFGl3uh+FrdtL + sW/4+5WbdPPHnlSeiAgN6578V5B4K8fSXF75XliLABKd1AG5ensM8e+O9ei/GH4WCZb0qu9Cynno + ACd388fjXzRrvie48C6ndhhJ5caMItx+bKqMEnqOoI9MEd6+jhSjSVorQ8KWInVlzSdzvPjfeLr3 + iHSLG3JeGPZOxORghsgeg5RSPdT7V6V8LFW9+GunwXJaWaENGGII2xpIx4J46FM180n4rR6h4su8 + yBmLiPOMFcAZHI9vT36Y3e9/BzxNFH4LtVmBP2hrjccZOMoScDPZSOc/zpXuzd/Cjs9C1swXSxHa + fPmUnJzhRlsfrVXX9Rhg0d0nkjQQeWqHPOGZcg9uQ/H4DnFUbqc6XqZmkISMEiNkBJz6Ljr16n8K + 5Dxx4wjij0gXM5jEk3nyHO1pAiPgADjkk8H0Jz0qkrGbdhnx9+G9p8X/AIW6joJUfbbpDNp7MQNl + wp/dsCcY3HCEekhr5++B0xudOs7W5yDGRG6t1BB6H8RXvfg3xBc67q9rNdZO5jhMYUKWG0L7/Lz7 + 15zoHwV1ey+Omsw6fHusJ9Qa5i2rgIsh37R7Ddj8K0g+hnJ3R9ifA7wdZeKdIkj1SASGOON0OeTw + Qf6V8F/tl+JW8K/to+KoLFzcWttdwiGFWztKxRhlz25U/rX6XfBDwjN4Y0GIsAZWhwB6nHH6ivxi + +M/ijU/+F1eJbjxp5qapJfzfaVlBDq5c5BB6Vx4/4VY2yeMfrE5vsevftYfG9P2i/jJrniSxhewt + 78wRRREDesUSIhViOpYI2f8Ae9qf+y5oE/jr4saJpelpun1C/htYx6l3VR/OvDW+I1jaxBbl8n69 + a+rf+CQdvD47/bL8CJtLxJqIuSAM4EatJ/7LXn04uc1dH0VarCFJ2fQ/fC+I8MW16rlfL8pJxj6E + Nx9VX86etxvgidOwBz7Yqv43bFt/pSIYVjeJwWzlGH+IFZfgG4VPDFjbGeW6NvAkPnSkGSXaAu5i + MDccZPHWvdktDwKcrM6K9sYNUs1F0hcxOJo2Q4dHHQqfXqPcEg8E1zmmW0EerzGeB2iQF0VlKFyT + 8q4PQk10lo22LA7U5LONbxrg/M7BRz2xnp+ZrFxvubp9UWYIV0zS1SJUDKvIUYBb6fWqkkRRAyy4 + YnHPRj1qa8uAoXPOTnFch4l8aXdrIx0izjkSM/NJNL5YP0wCT+OKpK5M5KKNrXPEJhto4z/rXJ3D + 2HfP5fnXmV7qjR+MN00h/eq2f7p4J/mKtyfECbxJqvk3VokBgjYuyybw2SMdAPQ1yniPWPs+qW9w + Y53VS64jQuTkHsPxraKsefiKt1dM6n4Vab/b3xrsJgRjTBNdHvuzEYhj/v6D+Fe+V4T+yNqS+JvE + /iC8S0v7YWcMUINxCYw/mMxOM9ceUPzr3as5u7OjLl+65u7f+QV8tf8ABZ34Y3vxS/4J0+PYPDFs + t1qmji11i2Ug5XyLmN5WBHIIh87n0zX1LWL8SPB8PxC+Hmu6Df5EOtafPYuQASBJGyZAPGRuzWNS + HtIuL6o9GMnFqS6H5NfsPftvHxP4G0zSWtYEuNIiWGXLbHBBKsxZs98entz1+rfAvxGt57tprm3W + O6Y4eXdu3j1Zuw5zxX51fsz/AAl0fwV8f/EviKPXGk0qa5KpbW42ujgkMXXjadw9Pzr7U8O+PPAV + /YfZ4brULUDlfOAZQ3cjnPXIx7DGO3ytKpybvY9SUJs9X8b3ulfELw7NpWu26Tw3EZjcuwPXjr26 + 57dq8Y+GfwtPw11ieO0YtbpJgDPVcjJXnp0GD0x7VhfHP436B8PLGxm0rW9OJupFjjQSjexyTwpx + xjcfofwruPAfi8eMrW0uYQp3orBw+CSc8EYGO2MVFaUajv1CDcXZnpOjSI2wwqQNoznv9BTNS1TN + tMIATICc4B+b9OnvVex1D7LY3Dytt8n7p7Dvk/nVDTL2eS7nY4VCMsGkw4IPUEZG3p+fOKzWrSKm + 7alHxbfTy2ciQvAkmPk3s21n4AwB26/hXxV8d/ibqnw0/bW8Pz+ELmeGC7h0+aSynUpku2GIHYnr + nHOfevsXW7o6pcrb6JG7PeTIkbId2AxwpIA+UD1x29K+Of20tLjm/wCCuvww8M2cJCajb6YyJtwr + RxTSFxkccLGa2s1b1M4L2jfofan7VvwPTw1qKXumwCOz1NeUUYWNyMkfj1/Cvgv9pP4Iy3N7JJpM + Lu65LlRgDPv9cHvjAx61+yvxc+HafEHwHLaGMNMqBosjkOBx/h+NfIPjD9n67uxKl9YkckkEDgiv + sYu61PlK9KVKei0Z+RmjfDPVLbxS7uhRHmZyQfVs9/8AP6Y+nfgxG0Pgq2ivYzG1qHKl22/elI69 + +M17b4q/ZdW3mfzLRVJOe3FR6T8IE03TEhSLywzYJUZIIJP9ajksy3N2R5frmrtDarE0p8oqPmHV + Tyev1z0/DvXn3jHWNV1zX47DRtPN6kUHmSuinERPCjOOOVcke/brXuHir4SNDK8umL5E6n5TjKuB + 2ZRwfT1HrXHa34HuYbWV4beCGZX824iQZW44x83T+EdD6YqkiOZMxfBPh2TQY4Ddyo8gEKZQgndt + yxHTIJYD321798CfA0Wt3E12YB5kDryRnKnOP5GvKvCmhyssMc8SqUYPwC2wAjA/D0r6l/Yv8BXH + ivxNqWnaKA7C1WYtIcAlWxj2+/Vrcid5xstzrtLsHtYEGwDaOw6V+X3/AAW+8OeFNK+Nnh680LS7 + Zdfv7Rjq8sLBfNYEeWXXGd4XvkEgj0r9ePjF4On+CXw71LxH4vWL7HpcPmOkThpJDkAKvuSQK/Dv + 9qiz1P43/GHVdd1UNIt3dSTIjc+WCxwoPsMCsayc42ReX03Sq81TQ+bNZ8MW15Ahiyp9CTx+Pev0 + P/4IJ+DVtP2sPDE06kqsd1hhzt/0aQA18hN8HZbq7ijEBALAcV+mv/BFrwDB8N/imNS1S1uJvLsJ + IovKKL5bNtBcl2UABS2ec89DXJhlLnSke1jJwdO6P0p+JupXUPhy9XSojd3yQuI1A4Y7TgmvLf2e + /jePHOiJ9pIS/hJiuINhRopV+8hB6Ee9eySx2utanAYfOjDEHKxEo3TO6UAqx9sivmnxz4JufhL8 + VLnxJ4AtxI0V00Graf8Ad/tGJchZUPQTKpwCfvDg9Bj1FaTPHnzQtK59Nabqy3FpE6/xLnmrhuxG + TzkjFcZ4V8Sxarolnd2IkENzEsqq4wwDDPI7HnmtfUNWEEDbgYznkHrWbVzqjPS4eJPGdvYZFzIF + 2ryTxiuC8SfFHS3t5Wt/Mu5YkLJEpCh2xwMsQBn1ql8TNZ8N6Jatf/EPVxp8TH92nnndIfRU5yfZ + Rk1w0l3qXj+3aL4Y+Ep7XTgCov8AVkMDN6FYz+8bPuBVxskctScjp9H1q5urSW81lolvJkAMUbZS + Dk4UHvwTk9zntioJNT8zY5ywVsnv3rS074bXHhnw1bRzRzS3O398xU43Y7eg61nyaDcICqRSAE/3 + TxVI4Kzlax7X+zbpptvBl1cyKAbu7Yq2PvKqhR/49v8A1r0Oue+FWk/2L8O9JhOdzQCVs9cv85/9 + Croawnuz3cNDkpRj5BRTJZxEPmNUrnX4bf77j86k3Pxl/ae/ZUvPhl+2R8R9P0PxFPZRX+uz6vDG + sIULFdj7T5WdwGweZjjB+Qc8nPWfBH/gnF4fnvrjxH8TvGfiHxJHdfvnsLrVJWso+hA8kEIBk9Gy + fl4Nd/8A8Fs21/wF8TPC/jn4baXb6nY6tbf2bfOd26C4ifdFkgj76MQOn+pbk5ArkP2dfiXq/iTw + bBLrVstuFODGpG1TxngZx3H4V8zi37CrKLWn+Z7NGrF01JPU1PiT/wAEyvhL8TPD121t4XsodTd1 + eDU4v3Mlq6klPK2gHGcA5znuKtfsteAtQ+DNw3hnxBez3TWi5gnmO55AOBk+mOMj8a9h0rxLLFGQ + ct8h+QZJGM+n8vpWZb6q2ueJYHvIwEjG7zMcjqD069uT71xT9+zQpz7s6jV5Vh0djcNjzsB+RwPo + fy7da56/1iTw7pjveS/ZohGCxk5bPQ5x+X4jNbusX9mnOsyIsaguSRjbjqR9Dj/Irk9KuIfiJrrX + iLL/AGXBKPL8z5ftWOrAf3N2evXAI4A3XCDbuznqSvsdR8HLCZrqfVNQbCupWJCvJ5YmQn1+Yge3 + 1r538ZeCYPiX/wAFzPhW9piR/D3hO41C6A52YmkjTPp/rifpX1T4s8WaR8O/BNzq3iCaK3s7CFpC + CRk4BOBXzT/wR90m6+Pv7Z3xT+MPiZWYTLHpGnZ58iNfmKD22+T07g120qTnUijSm/Z05zv0t95+ + nyJmHC8YFfOX7Q82oeFvEsphlk+z3Cl0x2PQj/PrX0fIrGHAIHFeffGP4ZN440kqWUPGCUOOh9K+ + li7M8HFQlUj7m58l6rr0+oufNkJ+oFYl1DI8QCuTnJX65Ndt4o+Eep6NeypsDBT1BrJg8CapBGS1 + uxzyMjpWm+qPO5ZrRnKnSft0YVxlgemKyL/wGjyMTGJWmB3Iw4b6+vpXpUOiTwQn7VZygjg4XNX9 + M8LNqgdoIWDRgE71wTzjA9f/AK1MLM8qsvhSLRl+zoYkyCEVyQMHjA6L+Fez/A/SrzwL5l1pbSW0 + jxeX8jYOMg/0FXdE8CTTyputz6DjpXYW3hGW3gwsZHHTFNK5lUbtoeW/tOeJNe8WeFLixl33trOP + KcSEsozxyM/qPSvirW/2bmmnciFgSc9K/RXWvAM2oRSLglXGCMVzNz8C1lJIi6+q0uVEwqOO5+f2 + k/s2StqkZ8oud3TbX2x+wz8G7rwBqUGo6l5Ygxs8vq3POfQdK3LX4IfYrwP5P3favRvDdo/hvQY/ + KQKfNC4x7GhQSNVWlPRnvWk3dpdSRmONyzAHcST/AFryr4y2P2vxXrUMCL9qhVJoxj/WIVBH4hlb + 8x61ueHvE7WNzCJ8hAQPm/8Ar1zHxZ1waf43ub5Tua0lVZf9qB1Ab8ihb/gNSlZnbOadNEXw81BL + vwvai1bMa5VcdgCRirOrak9zIsOnyGQKMhuT+HPNY+kzR6NPOtoAkUs5kVRwMsBn9a2NOMczS3K/ + KJXOBnO3Bxj8wat6mSlpa5laF4DtrTUDqF3bQS6jNy13KoaRB/dQnlVHoOpyepp3ivxe+mw7tMcR + 2enOJJXHWaRTkKPYHkn147GrGu6rKjJaaWxNzc5G88+Sn8T49ugHckds1z/xHt0tvA93HbhwixhQ + Aeeo/WhbmM5cqbR0EXx0v5NKjk8uybcu4ZrQ8IftBXOtaza6c2mWZmvpUt1cMflLEKD+ua8x8K6a + knhy2VlkYiMZB6jNdl8G9Ft7f4lWM0mEWwDXMpbI42lUHHQliCM9QjUWVrs4qNevWqRjGW59Logj + QKvAUYFLWbbeIoLn/VuD+NXorhZR8pBrnPqDO18yeW3k5zXmPjd9RCv9l3Z9q9fmgEo+YZqjdeHY + LkfvEB/CgD5A+KfhvUfGWmXGneIbT7bZT/fikGQccgj0IPcV5fZfB7VvCoKaDBbeSpyiy2hZl65G + VdRg5PYV9/XHw+s52y0S/lVd/hbp8n3oEP4VlUoU63xxuVGbjsz4H8SaZ4vvLYw20dtboM7WiglD + AH3831APT2rHjl8Z6N8ttZWbx4UMDbvl8DAyfM+o6V+hUnwh01+tvH+VQSfBTS5PvW0fP+zWf1Si + vsjc5Pdn5s+J7rxzq08Pn2sX2aI7nhSJwsx56nfwD6DHT8Ks23xb8baDj7N4ZsX2cjiQZ4x61+i8 + nwI0eQ/NaxH/AICKrv8As96K/wB60i/75FP6pR/lJ5mflX+0H4o+Ivxt0X+zr/TotPsjkvHDG7eY + fclqvfsh/HTxt+xh4Ll0fwn4Js9W8+Z55Z5pZImkZmJyQoPQYH4Cv0/f9m7Q362cP/fIqF/2YtAf + 71nD/wB8itIUoQ+FA25Llb0Ph25/4KyfFW3XCfDPTTjpi+mH/stZ95/wV9+LSIyr8L9NYH1vpf8A + 4ivux/2VvDz9bKA/8AFRP+yZ4cfrYwf98CtuZmXso/02fnL4g/4Kj/FbVrgvP8L9MwTkqLuTn/xy + s9v+CoXxLhOG+EemOB2OoSj/ANp1+kz/ALInhputhb/98Co2/Y98Mt/zD7f/AL4FHOw9lH+mz82v + +HqfxJhP/JG9Lb2/tKX/AON1bsf+CuXxL09iY/gtpRz/ANRSb/41X6Ln9jnwz/0D7f8A74FIf2N/ + DH/QPt/++BRzMfs49j4C0v8A4LRfE7TnyvwO0dz33arNz/5CrYj/AOC43xPYc/AjQR/3FJv/AIzX + 3N/wxv4Y/wCgfbf98Cgfsc+Gf+gfb/8AfApczHyI+HR/wWy+JV0f3vwJ0A/9xOX/AONVND/wWY+I + s4+b4F+Hx/3EZv8A41X2+v7Hvhkf8w+3/wC+BUi/sieGl6WFv/3wKfMxezj2PiJP+Cufj67+/wDB + DQVJ/wCojMf/AGnVXXP+Cl/xA8U6c1ufhPpNoGIYNHfTZH/jtfdafsm+HF6WMH/fAqZP2VvDydLK + D/vgUc7E6MH0Pz/s/wBtz4gywCNPh9pyDjrczNj3HHB9xTNC/au+Ld1qmsS+LtE03U4dSmMlvGYH + iazQ4/d7g37wfKDkjJJbnmv0Ij/Zi0CP7tnD/wB8ip4/2btDQ8WcP/fIp+0kR9Xp9vzPhS0/aX8e + 3enLbT+FbHaE2bt8u7HQc569Oa6TQf2mPHlvZxQnwzp7CJQu5zKSfcndyfevs1P2e9FT7tpF/wB8 + irEfwI0eP7ttF/3yKPaSBYamuh8iaZ8fvGjTPI/hrS98uMkrKeB0H3unJP41s/8AC5/Fur25hvvD + mjmF/vL5cuT+O+vqiP4K6XH922iH/ARU8fwh01OlvH+VHOx+wp9j5p0/4neIYdPEOmeGNCtmIwZD + DM5A9h5oGfrke1HhRfECXjyT+a0kzb5HIwXP4cD2A4HQV9Op8LdPT7sCflU8Hw8soGysKflUuTZU + aUIO8VY848EPqRCfat/416h4eMnlr52antfDcFt/q0A/Cr0NssX3RikaEtFFFABRRRQAUUUUAFFF + FABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUU + AFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQB + /9k= +item5.X-ABRELATEDNAMES;type=pref:Jenny +item5.X-ABLabel:Spouse +X-ABUID:6B29A774-D124-4822-B8D0-2780EC117F60\:ABPerson +END:VCARD diff --git a/resources/vcards/036.vcf b/resources/vcards/036.vcf new file mode 100644 index 0000000..b0e2526 --- /dev/null +++ b/resources/vcards/036.vcf @@ -0,0 +1,47 @@ +BEGIN:VCARD +VERSION:2.1 +UID:urn:uuid:a04eb0c4-ef5a-538d-970b-c4b204cf8d2d +N;LANGUAGE=en-us:Doe;John;Richter,James;Mr.;Sr. +FN:Mr. John Richter James Doe Sr. +NICKNAME:Johny +ORG:IBM;Accounting +TITLE:Money Counter +NOTE:THIS SOFTWARE IS PROVIDED BY GEORGE EL-HADDAD ''AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GEORGE EL-HADDAD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +TEL;WORK;VOICE:(905) 555-1234 +TEL;HOME;VOICE:(905) 666-1234 +ADR;WORK;PREF:;;Cresent moon drive;Albaney;New York;12345;United States of America +LABEL;WORK;PREF;ENCODING=QUOTED-PRINTABLE:Cresent moon drive=0D=0A= +Albaney, New York 12345 +ADR;HOME:;;Silicon Alley 5,;New York;New York;12345;United States of America +LABEL;HOME;ENCODING=QUOTED-PRINTABLE:Silicon Alley 5,=0D=0A= +New York, New York 12345 +X-MS-OL-DEFAULT-POSTAL-ADDRESS:2 +URL;WORK:http://www.ibm.com +ROLE:Counting Money +BDAY:19800322 +X-MS-ANNIVERSARY:20110113 +EMAIL;PREF;INTERNET:john.doe@ibm.cm +X-MS-IMADDRESS:johny5@aol.com +PHOTO;TYPE=JPEG;ENCODING=BASE64: + /9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAYEBQYFBAYGBQYHBwYIChAKCgkJChQODwwQFxQY + GBcUFhYaHSUfGhsjHBYWICwgIyYnKSopGR8tMC0oMCUoKSj/2wBDAQcHBwoIChMKChMoGhYa + KCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wAAR + CAAaACADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAA + AgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkK + FhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWG + h4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl + 5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREA + AgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYk + NOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOE + hYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk + 5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDzS20gtj5a1R4feMqJYnQlQwDAjII4Irub + PQTx8ldNYacfs6213F59sM7R0eL1Kn+nT+dZyk1sQ2zyUaBkfc/SpB4ZOPuc/Su61vSo5tTf + SrZySEDKWQgyFgSowSBgEDdz7Z71S8ITNB4luPD2szQvdvM4t/KYsoGNwUNgZwAecCpVWLfK + NO56ba2sSqpIGD0Na1vDBxwKwYifU1dhJx1NJq5m1cw/Enw9Gt6ydRbXLmNljMcUW0FYgSCc + YI9KXwX4EXwz4hm1STUhdM8RiCtGM4JByWz7DpXQ72/vH86idm/vH86ShZ3uNXtY/9k= + +X-MS-OL-DESIGN;CHARSET=utf-8: +X-MS-MANAGER:Big Blue +X-MS-ASSISTANT:Jenny +REV:20120305T131933Z +END:VCARD diff --git a/resources/vcards/037.vcf b/resources/vcards/037.vcf new file mode 100644 index 0000000..8807116 --- /dev/null +++ b/resources/vcards/037.vcf @@ -0,0 +1,80 @@ +BEGIN:VCARD +VERSION:4.0 +UID:urn:uuid:44e43a63-2b48-575a-a5d7-933fc3eb2a9e +N:LastName;FirstName;MiddleName;Prefix;Suffix +FN:Prefix FirstName MiddleName LastName Suffix +TEL;TYPE=home,voice:555-555-1111 +TEL;TYPE=work,voice:555-555-1112 +TEL;TYPE=cell,voice:555-555-1113 +TEL;TYPE=cell,voice:555-555-1114 +TEL;TYPE=voice:555-555-1115 +TEL;TYPE=home,fax:555-555-1116 +TEL;TYPE=work,fax:555-555-1117 +TEL;TYPE=voice:555-555-1118 +TEL;TYPE=voice:555-555-1119 +EMAIL;TYPE=home:home@example.com +EMAIL;TYPE=work:work@example.com +EMAIL;TYPE=school:school@example.com +EMAIL;TYPE=other:other@example.com +EMAIL;TYPE=customtype:custom@example.com +PHOTO:https://d3m0kzytmr41b1.cloudfront.net/c335e945d1b60edd9d75eb4837c432f + 637e95c8a +PHOTO:https://d3m0kzytmr41b1.cloudfront.net/c335e945d1b60edd9d75eb4837c432f + 637e95c8a +PHOTO:https://d2ojpxxtu63wzl.cloudfront.net/static/aa915d1f29f19baf560e5491 + decdd30a_67c95da9133249fde8b0da7ceebc298bf680117e6f52054f7f5f7a95e8377238 +ORG:Organization1;Department1 +ORG:Organization2;Department2 +TITLE:Title1 +TITLE:Title2 +BDAY;ALTID=1:20160801 +BDAY;ALTID=1;VALUE=text:2016-08-01 +GENDER:M +X-GENDER:male +X-ID:14f9aba0c9422da9ae376fe28bd89c2a.0 +X-ETAG:fffffea9056d8166e2b7a427977e570c87dd51279d11d9b137c593eb +X-FC-TAGS:579c773f-736d-11e6-8dff-0ac8448704fb +X-FC-LIST-ID:8ad23200aa3e1984736b11e688dc0add41994b95 +X-FCENCODED-582D46432D52656C617465644E616D65733A4D6F74686572:Mother +X-FCENCODED-582D46432D52656C617465644E616D65733A466174686572:Father +X-FCENCODED-582D46432D52656C617465644E616D65733A506172656E74:Parent +X-FCENCODED-582D46432D52656C617465644E616D65733A42726F74686572:Brother +X-FCENCODED-582D46432D52656C617465644E616D65733A536973746572:Sister +X-FCENCODED-582D46432D52656C617465644E616D65733A4368696C64:Child +X-FCENCODED-582D46432D52656C617465644E616D65733A467269656E64:Friend +X-FCENCODED-582D46432D52656C617465644E616D65733A53706F757365:Spouse +X-FCENCODED-582D46432D52656C617465644E616D65733A4669616E63C3A9:Fiance +X-FCENCODED-582D46432D52656C617465644E616D65733A506172746E6572:Partner +X-FCENCODED-582D46432D52656C617465644E616D65733A417373697374616E74:Assistan + t +X-FCENCODED-582D46432D52656C617465644E616D65733A4D616E61676572:Manager +X-FCENCODED-582D46432D52656C617465644E616D65733A4F74686572:Other +X-FCENCODED-582D46432D52656C617465644E616D65733A437573746F6D54595045:Custom +X-FCENCODED-582D46432D4F7468657244617465733A416E6E6976657273617279:2016-08- + 02 +X-FCENCODED-582D46432D4F7468657244617465733A4F74686572:2016-08-03 +X-FCENCODED-582D46432D4F7468657244617465733A437573746F6D54595045:2016-08-04 +NOTE:Notes line 1\nNotes line 2 +URL:http://www.homepage.com +URL:http://www.blog.com +URL:http://www.other.com +URL:http://www.custom.com +ADR;TYPE=home:;HomeExtended;HomeStreet;HomeCity;HomeState;HomePostal;HomeCo + untry +ADR;TYPE=work:;WorkExtended;WorkStreet;WorkCity;WorkState;WorkPostal;WorkCo + untry +ADR;TYPE=other:;OtherExtended;OtherStreet;OtherCity;OtherState;OtherPostal; + OtherCountry +ADR;TYPE=customtype:;CustomExtended;CustomStreet;CustomCity;CustomState;Cus + tomPostal;CustomCountry +NICKNAME:NickName +IMPP;X-SERVICE-TYPE=GTalk:xmpp:gtalk +IMPP;X-SERVICE-TYPE=Skype:skype:skype +IMPP;X-SERVICE-TYPE=Yahoo:ymsgr:yahoo +IMPP;X-SERVICE-TYPE=AIM:aim:aim +IMPP;X-SERVICE-TYPE=Jabber:xmpp:jabber +IMPP;X-SERVICE-TYPE=Other:other:other +IMPP;X-SERVICE-TYPE=CustomTYPE:customtype:custom +CATEGORIES:Tag +PRODID:ez-vcard 0.9.14-fc +END:VCARD diff --git a/resources/vcards/040.vcf b/resources/vcards/040.vcf new file mode 100644 index 0000000..d5e62a2 --- /dev/null +++ b/resources/vcards/040.vcf @@ -0,0 +1,92 @@ +BEGIN:VCARD +VERSION:3.0 +UID:urn:uuid:887ee0fa-fccf-520e-af1a-ec028982989c +FN:VCard Test +N:Test;VCard;;; +NICKNAME:TheNickname +X-PHONETIC-FIRST-NAME:ThePhoneticFirstName +X-PHONETIC-LAST-NAME:ThePhoneticLastName +EMAIL;TYPE=INTERNET:email@example.com +EMAIL;TYPE=INTERNET;TYPE=HOME:homeemail@example.com +EMAIL;TYPE=INTERNET;TYPE=WORK:workemail@example.com +EMAIL;TYPE=INTERNET:otheremail@example.com +item1.EMAIL;TYPE=INTERNET:customcategory@example.com +item1.X-ABLabel:CustomEmailCategory +X-GTALK:IM2 +X-AIM:IM3 +X-YAHOO:IM4 +X-SKYPE:IM5 +X-QQ:IM6 +X-MSN:IM7 +X-ICQ:IM8 +X-JABBER:IM9 +TEL:5555551111 +TEL;TYPE=HOME:5555551112 +TEL;TYPE=WORK:5555551113 +TEL:5555551114 +TEL;TYPE=CELL:5555551115 +TEL;TYPE=MAIN:5555551116 +TEL;TYPE=HOME;TYPE=FAX:5555551117 +TEL;TYPE=WORK;TYPE=FAX:5555551118 +item2.TEL:5555551119 +item2.X-ABLabel:GRAND_CENTRAL +TEL;TYPE=PAGER:5555551120 +item3.TEL:5555551121 +item3.X-ABLabel:CustomePhoneCategory +ADR:;;111 Main St;NY;New York;10011; +ADR;TYPE=HOME:;;112 Main St;NY;New York;10011; +ADR;TYPE=WORK:;;113 Main St;NY;New York;10011; +ADR:;;114 Main St;NY;New York;10011; +item4.ADR:;;115 Main St;NY;New York;10011; +item4.X-ABLabel:CustomAddressCategory +ORG:TheCompany +TITLE:TheJobTitle +BDAY:1912-06-23 +URL:http\://www.example1.com +item5.URL:http\://www.example2.com +item5.X-ABLabel:PROFILE +item6.URL:http\://www.example3.com +item6.X-ABLabel:BLOG +item7.URL:http\://www.example4.com +item7.X-ABLabel:_$!!$_ +URL;TYPE=WORK:http\://www.example5.com +item8.URL:http\://www.example6.com +item8.X-ABLabel:CustomWebsiteCategory +item9.X-ABDATE:1930-03-20 +item9.X-ABLabel:_$!!$_ +X-ABDATE:1776-07-04 +item10.X-ABDATE:2000-09-12 +item10.X-ABLabel:CustomDateCategory +X-ABRELATEDNAMES:Name1 +item11.X-ABRELATEDNAMES:Name2 +item11.X-ABLabel:_$!!$_ +item12.X-ABRELATEDNAMES:Name3 +item12.X-ABLabel:_$!!$_ +item13.X-ABRELATEDNAMES:Name4 +item13.X-ABLabel:_$!!$_ +item14.X-ABRELATEDNAMES:Name5 +item14.X-ABLabel:_$!!$_ +item15.X-ABRELATEDNAMES:Name6 +item15.X-ABLabel:_$!!$_ +item16.X-ABRELATEDNAMES:Name7 +item16.X-ABLabel:_$!!$_ +item17.X-ABRELATEDNAMES:Name8 +item17.X-ABLabel:_$!!$_ +item18.X-ABRELATEDNAMES:Name9 +item18.X-ABLabel:_$!!$_ +item19.X-ABRELATEDNAMES:Name10 +item19.X-ABLabel:RELATIVE +item20.X-ABRELATEDNAMES:Name11 +item20.X-ABLabel:_$!!$_ +item21.X-ABRELATEDNAMES:Name12 +item21.X-ABLabel:_$!!$_ +item22.X-ABRELATEDNAMES:Name13 +item22.X-ABLabel:REFERRED_BY +item23.X-ABRELATEDNAMES:Name14 +item23.X-ABLabel:_$!!$_ +item24.X-ABRELATEDNAMES:Name15 +item24.X-ABLabel:DOMESTIC_PARTNER +item25.X-ABRELATEDNAMES:Name16 +item25.X-ABLabel:CustomRelationCategory +NOTE:note line 1\nnote line 2\nCustomField: field value +END:VCARD diff --git a/resources/vcards/042.vcf b/resources/vcards/042.vcf new file mode 100644 index 0000000..d22e984 --- /dev/null +++ b/resources/vcards/042.vcf @@ -0,0 +1,94 @@ +BEGIN:VCARD +VERSION:2.1 +UID:urn:uuid:e5421786-42af-5192-b326-011c7e68d2e7 +N;LANGUAGE=en-us:Angstadt;Michael;;Mr.;Jr. +FN:Mr. Michael Angstadt Jr. +NICKNAME:Mike +ORG:TheCompany;TheDepartment +TITLE:TheJobTitle +NOTE;CHARSET=us-ascii;ENCODING=QUOTED-PRINTABLE:This is the NOTE field =0D=0A= +I assume it encodes this text inside a NOTE vCard type.=0D=0A= +But I'm not sure because there's text formatting going on here.=0D=0A= +It does not preserve the formatting +TEL;WORK;VOICE:(111) 555-1111 +TEL;HOME;VOICE:(111) 555-2222 +TEL;CELL;VOICE:(111) 555-4444 +TEL;WORK;FAX:(111) 555-3333 +X-MS-TEL;VOICE;CALLBACK:(111) 555-4444 +ADR;WORK;PREF:;TheOffice;222 Broadway;New York;NY;99999;USA +LABEL;WORK;PREF;ENCODING=QUOTED-PRINTABLE:222 Broadway=0D=0A= +New York, NY 99999=0D=0A= +USA +X-MS-OL-DEFAULT-POSTAL-ADDRESS:2 +URL;HOME:http://mikeangstadt.name +URL;WORK:http://mikeangstadt.name +ROLE:TheProfession +BDAY:19220310 +X-MS-ANNIVERSARY:20120801 +KEY;X509;ENCODING=BASE64: + MIIB/jCCAWugAwIBAgIQDdkWkvA2cqtGkw2P4zAoZDAJBgUrDgMCHQUAMBMxETAPBgNVBAMT + CG1hbmdzdGFkMCAXDTExMDYxNzIwMTgwM1oYDzIxMTEwNTI0MjAxODAzWjATMREwDwYDVQQD + EwhtYW5nc3RhZDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAmfYuhkT693IXf/iSESbw + 9cmeM3p2AWDC8oGDiCDtenNIGdITa8e/kwx9JIFl7a02cMbHtG6BmbGBINzzgepsY6lE+Pcx + ns4FVpgWr1hn3YVg2LKqgRkKLp+ddqL+LWCg3qvcQxnZJoooRLhpurr/T9TaE83nAr6aqoSt + jYh+KtECAwEAAaNZMFcwFQYDVR0lBA4wDAYKKwYBBAGCNwoDBDAzBgNVHREELDAqoCgGCisG + AQQBgjcUAgOgGgwYbWFuZ3N0YWRAcGVubm11dHVhbC5jb20AMAkGA1UdEwQCMAAwCQYFKw4D + Ah0FAAOBgQA9/txKYVIj6/UoDRYAp23NoRjaeZ0cTqcpKtxlUi8z7TtczAROgTr4Pi+VaKR0 + Y+RzgVxPN8wvNoVogCWLTpVxhoe8R5geoS4L1w1O1HYwu2KlG57SOR4R/Lv5bgi6CNTlS1N4 + zIDBOSyypTmzxC+JPmyn/8vZleIz8CYnwmfBEg== + +EMAIL;PREF;INTERNET:mike.angstadt@gmail.com +X-MS-IMADDRESS:im@aim.com +PHOTO;TYPE=JPEG;ENCODING=BASE64: + /9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAYEBQYFBAYGBQYHBwYIChAKCgkJChQODwwQFxQY + GBcUFhYaHSUfGhsjHBYWICwgIyYnKSopGR8tMC0oMCUoKSj/2wBDAQcHBwoIChMKChMoGhYa + KCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wAAR + CABWAEgDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAA + AgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkK + FhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWG + h4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl + 5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREA + AgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYk + NOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOE + hYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk + 5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD6pooqlrOpQaTplxe3TBYolLH3PYUAQeIN + e0/QLM3Op3CxJztXqzH0Arx3xN8Zb5ndNEtBDHztlkj3tx04yB/OuM8Y6zqHiXU3ubhyImbC + J6JnGB6cE/lXJT2rbJHaZiioSrZPIBxk/oB7UhXOsvPir4ofJbU2jyeP3axgZHqM96NP+NXi + awZGmuIruI9FnjHzDnJyMfTr2rzq7WdI2wR5YVcg+5yBWTPIjxsI8K27lD0b5hhR/ntSBH1h + 4H+NWg6+Y7fUz/Zt43A8w/u3Ps3b8a9UVg6hlIKkZBHevzp80oeCyyLwc9f/ANXFfS37OnxF + lv2Hh3V53klC/wCjO5zjH8Oadyj6BooopiCvK/jXqjbbTTI2wp/eyDHX0r1SvAvHkp1/4hG0 + hkGwN5e7HQL1xUydlcaTk7I4o2tzds0dnA0rnPygdeO3tk/yqC/8L6skC5s5zBgAsyc4A546 + 5J56ete22NjDp9msVpGqKPTq3uT3qpqjMbYk4/GuZ130R2wwcerPm/Ura4RvKu0MaZ8xw4OC + ewAI5rmNSbICP8owWbjJyc8V7l4kMLRSJLGHX35ryDxHbRRXDeQDtLbwg7npjNXTrc+jM6+F + 9lqndHOM2X8lyBKn3HyMZ5PPrWr4Y1ObR9atL62ZlmgcMoHqOucVjSKZJJA7DzAeGHdj1/Kp + reTcVckZJ2tx39a2OdH6FeGdUTW9AsNSi4W5hWTHoSOlFcH+zrqDX3w3tY3JJtpGiGfTqP0N + FNEs9G1CcWthcTkgeXGzZPsK+TPtc+o+I7q7F41p5KtNNOoyRk19TeK4TP4a1OIAsWt3AAOM + 8V84/B23S+vNaaYCVflRtwzz1rGvOysdOGp80r9iz4E1u/vtbeFNTury0U4Pnw7cfjTfir4g + udPnj06znKTufvDtXfSWVrpQadUVMAscKB/KvINVjj17X5p5XyfM+U1yc2p6Sg+U4+aa/eDc + /iKLzmBIidTyPr0rLjW4uInjumBlzlHXoa9dvtGt722WN4oYpVTZkRDp6Vx17oUGlqQpJ54F + XGomZTotbnnEhJlZnG2VXKsvp9KjdWS6YEYV1zx6itTXjFb6tGUPMq4YD17Gqd0jEcFTIgDj + aMgV1xd1c82ceWVj6q/ZWvDN4T1G2OB5VwGA9mUf4UVl/spQSeTrNwM+QyxIAePm+Y9PoRRV + rYhnumvME0W/YjIED8fga+U/hh4nttA1W5jvTstbnqcdGHevrHVE36bdr6xMP0NfBmqyG3nO + 5mVzJtIC8nk1zYhNtHdg2knc+g/GPjLRYdInkWdLhimUVDkn8K8CsvEbnW1uLaGZWaUHyivB + Gaz55jLIBCHlRDmRxk7eOO3FZ812sLh4pBCeTgZLD0/kaxgu51ylbbY+lNUlgXT0nUAbkz9K + 8c8Ta8vny/NnbnAri7vxJqw06ANf3DIzHhnOMVm3FwTqPlyOxjde/XPvVQptasipWTVkSKzX + DXF1cN+9dsIPat6ztlXT4Ccl8nfk5OD0rmdPR5NqLkoGJGfSvafhR8M9V8YtDcTh7fRww3zN + xuA6hfU9q6I7nBVd0j3H9m3THsPh1HPIuDdzNKOP4R8o/lRXpmmWMGm6fb2Vmgjt4EEaKOwF + Fao5ywyhlKsMgjBFfFXxd8NT6J4vuYpo2gjkuDNGV6MpJIwfSvs37V+7d9nCvsrjfix4Mt/G + WhNHEY01G3+aGQ/+gn2NZ1YcyuuhvQqckrPZnx9pGt32kia4sJ1jEpCyBD1xkc5Hb15qjr2s + 3FyIZbqKAzOx5ZRxjqff/wCtVjxHot5pNpc6dcwNBNHk8qQTg8/596w9ahVoE2BR5JyyA56/ + 5Nc0Y3Z6E5tIg1Zla4tosRgL8xUdvas2QA3UzZLbVwKluiGuY5UUsWHAHOK6Lw7oB8z7RfYA + J3bD3HvWq91HO/ekeofs8eAtK8RXWNajlaPy/ORAQNwVhwfY9/pX15bQRW0EcFvGkUMahVRB + gKB2FfNHwf1iDSPFtm07JFbSKYSTwFyOPwzivpW2ure6TfbTxTL6xuGH6VdF3RhiI2kiaiii + tjnKT27CZhuHkk7yO+aoi4Ky7VXJZWGWPeiikwMPxPp8F/Z2r3ltbyuj4yyA8Y6dK8q8WeBt + LmilNva28A9FX8aKK5anxHbQfuHkV/oiaddOgKlQeMCkB5CrwDRRSZrHc2LNWYRxocMxCgmu + 00nUbzQZ4nsbmRHgQkkH7x9xRRVUu5liHsjv9L+LksMSLq2niUjhpIG2k++Dx/Kiiit7s5Gj + /9k= + +X-MS-OL-DESIGN;CHARSET=utf-8: +X-MS-MANAGER:TheManagerName +X-MS-ASSISTANT:TheAssistantName +X-MS-SPOUSE:TheSpouse +FBURL:http://website.com/mycal +REV:20120801T184631Z +END:VCARD diff --git a/resources/vcards/045.vcf b/resources/vcards/045.vcf new file mode 100644 index 0000000..9d65d23 --- /dev/null +++ b/resources/vcards/045.vcf @@ -0,0 +1,204 @@ +BEGIN:VCARD +VERSION:3.0 +UID:urn:uuid:8c11cf62-42b4-5513-9d35-1318e6cb0540 +N;CHARSET=UTF-8:Doe;John +FN;CHARSET=UTF-8:John Doe +ORG;CHARSET=UTF-8:TheOrganization;TheDepartment +NICKNAME;CHARSET=UTF-8:Johnny +ADR;TYPE=WORK,POSTAL;CHARSET=UTF-8:;222 Broadway;Suite 100;New York;NY;98765;USA +ADR;TYPE=HOME,POSTAL;CHARSET=UTF-8:;123 Main St;Apt 10;Austin;TX;12345;USA +TEL;TYPE=WORK,VOICE:555-555-1111 +TEL;TYPE=HOME,VOICE:555-555-2222 +TEL;TYPE=CELL,VOICE:555-555-5555 +TEL;TYPE=FAX:555-555-3333 +TEL;TYPE=PAGER:555-555-4444 +EMAIL;TYPE=PREF,INTERNET:doe.john@hotmail.com +EMAIL;TYPE=INTERNET:additional-email@company.com +EMAIL;TYPE=INTERNET:additional-email1@company.com +EMAIL;TYPE=INTERNET:additional-email2@company.com +EMAIL;TYPE=INTERNET:additional-email3@company.com +URL;TYPE=HOME:http://www.private-webpage.com +TITLE;CHARSET=UTF-8:TheTitle +URL;TYPE=WORK:http://www.work-webpage.com +CATEGORIES;CHARSET=UTF-8:category1\, category2\, category3 +X-SPOUSE:TheSpouse +BDAY:1970-09-21 +X-ANNIVERSARY:1990-04-30 +NOTE;CHARSET=UTF-8:This is the notes field.\nSecond Line\n\nFourth Line\nYou can put anything in the "note" field\; even curse words. +PHOTO;ENCODING=b;TYPE=JPEG:/9j/4AAQSkZJRgABAQEAYABgAAD/4QBoRXhpZgAASUk + qAAgAAAAEABoBBQABAAAAPgAAABsBBQABAAAARgAAACgBAwABAAAAAgAAADEBAgASAAA + ATgAAAAAAAABgAAAAAQAAAGAAAAABAAAAUGFpbnQuTkVUIHYzLjUuMTAA/9sAQwACAQE + BAQECAQEBAgICAgIEAwICAgIFBAQDBAYFBgYGBQYGBgcJCAYHCQcGBggLCAkKCgoKCgY + ICwwLCgwJCgoK/9sAQwECAgICAgIFAwMFCgcGBwoKCgoKCgoKCgoKCgoKCgoKCgoKCgo + KCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoK/8AAEQgAhABkAwEiAAIRAQMRAf/EAB8 + AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMFBQQEAAABfQE + CAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkqNDU + 2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpe + YmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fL + z9PX29/j5+v/EAB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQ + EAwQHBQQEAAECdwABAgMRBAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDT + hJfEXGBkaJicoKSo1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoK + DhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19j + Z2uLj5OXm5+jp6vLz9PX29/j5+v/aAAwDAQACEQMRAD8A+PNL8D3+jRfZbPxprEUiE/v + EmBHX0AH8605Lj4hWURGlfEO7LjAT7TDvGff5jXvP7I37F+v/ALXN5r/9ieNoNIGjPDv + 8+xMol80PjG1htxs9+orA/ak/Zn8Tfss/Ea3+H3ijX7LUnubKK8huLGJ1XYzsoBDDIOU + J79a/K/ZYqVL2zXu/I+yWNTq+z5te1jjfgH4x8T+NvDd/f+Jb7z5INSaCNhGq4UIp/hA + zyTXYzFhyTyPSuM/ZmtmXwLdu6BS+qyHl92fkT/P0xXV+KvEWheE9PbVPEF8lvEoyA2c + tg8/Qc8noOppYbDVsXW9nSV29l/XQ3xNSnQk5SdkieOSWSTykBLEdBUdzqOi2+ow6LqG + twpdzjcLRWBl2cZbDEDAz1z1rx7xj8VvFmt3el6Tp1pYLHOHlv/sl4rCydPnTcVfL5GA + cYxzwc1xOh/FOC7+LWpnR/HA0u6trcxNCLcGJCSp3pnB5G3cSMEgA4GK+7wPCVKEebFS + u30Wi/wA3+B85iM5nJ2oqy7vc+k5Phtp3jnVX0rw/4mJnWEP5M1oS6knADCIvgcg57bg + SMEGuV8S/Crxd4WtG1m9sxLYG6kt1vYTuTzEOCCPvIfZgD19DXlfw2+Mfi2P4zIbf4ia + jFeWEkZZb1C1pcnjPyebtETAhcbtoJJBXBNfVvw9+JB17U/GvwO0bw+l3dT6kX0WKCHY + izJicKEkKHby0YWPI4HybCcejW4OyzE0v3LcJet1+P+ZyU89xlCovaWkvuPCdT1a28Oa + Hda1dY2W0LOQT1PYficCvn74d6Tq3jX4iS/ErUJ2NrY6xaxwuR/rZpZhn8AufzWvrP9t + b9nTxJ4u+GY+In7N2l3GpaHd28t5qenxW7rPa+UVWVUQjLKrOPlxvQEFhjmvEvAHhq68 + LfCfS9FvlUXMXjG2juVHXzBIgYe+CP0r4mvl2KydyjVjZvRenVr1PrsLi6GNoqVJ37+X + kz6X/AGGFN3+0Z43mz8sOj2iD8z/jX1f4l8PWvivwvfeGb2V0ivrSSB3Q4KhlIyPcZzX + yx/wTwtGufjb8RLljkLHbxfkTX11JEYSVzXzeJcoV01pa35FVPjZ8Rfs5/wDBQ+7+DHh + jWfAdh8MNN8QC18S3Ql1G9iMTs6rGjKFYBto2cZAPOMcUV9B+Lf8AgmB8KPG2p/8ACbP + 4l+IOlvrBkvJbTwvYWktsHeaQs2XAO4nJxzgbRmivuo4yFWKnor26mEauHguV308v+Cd + X/wAEfv2qvhh+zRpfjeH4j2JlOt3ln9nIkQYESy7hh8A/6wd688/4K2/HLwF+0B+0dpn + jn4c27x6fH4atrcrIiKTIk0xY/ISCORzXyb4u+JR8CrbRLMo+0zSEKSRjaEHYj1rQHii + 58W6Ha+ILiYMn2UiIJ/CFkcEcn1GfxrwJ1av9jQjbRvfTuzzKdOP9puV9bfojR/Z+1G1 + 0n4Xy3rw4dtQmLIo+++1ckZ7cfpXk3xs+IniHXNMjm8YeDH1Cz1DXBbx2lqJA8kG5djP + sI+XAY4yeWA9617rXn0XTNO8HanDM1nb7rzeuQJ3MTTLGoGAQqY3ZJyByADz5n43vLzS + IPCus6Zrpu447sPeTPLv86BpGYBE3buPlHyg7sL7ivv8AIMuhl+AU2vfkrt+T2X4/eed + muKeKxkkvhT0/zGa3rFhr3xkDajfT21/ZaXJE9tHd5EivgI2QPlyzBlHJwBnHFc5o/ia + LTvHOo2vjHSN7WcKJbXF9AZGeNsb/ADGcfvMZxjnuRjFN8Y3VrL4yh1TWrZrTVZI3W3t + 9OQOxjJ2iFXBI3FGPOCRmsy40i4fVFjfTNPigX5bR7qQySxScF1YKSZG9cnHava529Ue + bY6DwJ8TNM0f4nNa+MtNt10lRmzntrON4I4jyWKkAnGSMffx0wcV9F/s2eKbDxl+1t9k + 1K7ewuNWmhto0tdQfydW09bfy1IZ96s7QFwu1hkHAKtgn5i0OyufFmtOdGWC6trdkCxy + 6Vvkt/LyXQAZCq2WxyK9g+HXg/wD4Tvx/pF8YNL1Pw/pt5Bc2+sPqDafJb6fG/wC9WTa + ylcOwAYnjg5VcgdVCpUStvqY1YRer7H2x+y/420/Sfgh4l8WaBevrFn4I8Y2cthrv2R1 + +2adJcfZyLqNwzszBkjYqTkMW42ZGd/wUF/ZVX4eW9j8R/BfhhNOsZ/Gdg2raVAVb7M0 + rAx3PyAARzYYg4HzKwwOg574MeI/E974t+Jfgr4eXsWn2PizQL3UNZ8IXe6HUrCS2MTx + rGzKYpkmljAIAyUuZB8zKC32N8IYPBHxR02H4aR3ln4l8G6v8PbddSupLQRix1K3uihL + ykKGG6JwNqqI3iTGFyBGa5bSzTBunL4lrF9mPA42pl+JVSO3VeR8h/wDBNmET/ET4m34 + bO3WIovy3cV9kxWMUkefKBJ6kivmn9jr4Wav8D/jJ8W/hn4osng1Ky8XsJFkwPMQcK6+ + qYIIPfNfUGnMhUKRnivw/G0/ZYuUGtrL5pI++jLnjzLrqc5qOm3JuSjM2FGFBY8Ciu1G + l2Mw3yW4Y+pJorj9k/wCYd0fjV8ffDl54i13S7O1uQhiS4Yg85yyAdj6V3fwp8PPZeAt + P0PUJVYiJllaMcfNI+T+tGrwaudeMli4ZFhkYo0auMhz6g46dq2fDVyw8Pz65rDYSK2M + kx8rhVCseAo/lX1FCjOtgsPTt7rmvXdnhurGONqtPVL9EcJ8TvE9r4dvZLW/sovs1zLc + JYG1kUGPMS7VYscbmfKjnAG0Y5wfJX0eHSNMubOyOoWzWVysUctpbN5sX7zKJH/z0HLM + SB2VcAYrd8U6xF4i12bRLTxC3lRSyS6cwiWQW0PmbWk2EcRjYFVs8BieBiuXtdV134sa + /H4M+G0t3Ik14xnuHHzPGvyqBgcR4GexYkZ+6MfpGJq0cPTc5u0UePhqFXE1VCCvJkC6 + lb2WpLpHhSykvbu2tzG0agGFpWOWZnHLMD1xgEnHQc6mifBj4ma8GlP2fTnl5ne0jCM+ + eoJAzjgcZ7V718Nv2aLHwVZK+prGZyoLMw5JrubPwtYWpGxQSvHSvhsXxNiJzccPol95 + +g4HhLDQpqWId5dlsfOUP7J3jG8ihupPGVwpjYFUSR9oP/fXX3rVsfBHx9+Epu5dG11t + Tsb3T3tNQsbiITRT27csjo33hkA8EEEAqQQDX0Va2EIj8oMvykc4wRXTeEfBuleJdUj0 + q+uUiErBQzjI5rjo8Q5lSkpOd/uOqtw3lk6bShb5s4b9mn9sLwb438Kp8Cfi5OnhC9v7 + 9Fk1qS+n8mUR7Ht0WabzWs5FmjX5iDGUJHyYyfubRfCTaBrcHijw1NY6XdaF4e16DxBf + rpZtE1YSsY5ZGmhUxNIbtBKUUkhSzj5mwfnP44/8ABKvxF458D3HjD4fRQXt/ZEMqWaE + zEdc7Ryy8duRWp/wTd/au8f6h8RdT/ZP/AGk/GGpf2ld36P4Uu9RkCvHcRtmWFpthKGS + MjazAhzuUlTIWr9ByPPYZjD2dRWn+Z+e51kU8vfPSd4/kfUXxL0Lw5dalY/FLwuJ7h/E + WkWMuq6pMq5u7gW6EOzKACSjL0wMg4Awag0qY/Lk+netvW9F12y8MeIdG8T3llAH1z+1 + vDulrFJb3UKoTaXlu0D9YoStv5cijDRFWON4A53SdxIGa/O+LsH9WzybitJ2kv1/FHrZ + NXdbAxTesdPuOhicmMcA/U0UttGPJHy0V89yRW56Vz4d/ZW/aJ8F/BrxO/hvxB8A9A8Y + zateKrT65okd0kC7yAqu8beWckk9OCM1yn/BRDxboE/jDxLfeB/AFj4Os5f7Ptho2jxL + Db2yNbR75DtG1QSS/TBJ6V434p8et4B+I9r4pifdJYpFOsM0JeFyLh+HGRke3eug+OHx + L1n4weGr74wT6bYwC4cXV7pun6dsidUiCIscWSB86pwSc5PXNfYZTVUKuGhUty30eu9n + 12PmKmGh9drVIX5mte266HzL8ZNXi8OWC+ENH1ONrpjH/AGlHbx7SIjkxW6k87SuCw6d + +9e1fsz+D9V+Enwxn8b6b4VkvdX1pMxMUH7mPPOOmM4Jz6YrxWw+FniDS/ifoviD4qSN + c2j6pCZJkOfOjcrmRsDgEOGyey9sV9uav8OYtX0RdEi1+SHQPlVzZMFkEf91WHQfTmsu + Is7o4tqlSleF+nddD7nh3JKmDi6teNpPb0/4J4D4w/aF+JWjztFq2r6ZbOJMNBNdxhlH + 03ZJrd+G/xZ17xNLH9ont5t3QwzA7vyrlfj7+z14KTxPFP4I1aCK2tdwhi8gb9pYn5mC + 5kbk8tk9OaZ8Kfhl4s1f4k2d/pGnNDAksfmmCIpEFXjnJ5JFeWoYOWH5oy19D2qdTH0s + Vyzh7vTW/6H0Fa22q3enStErLLtDbeefrXl3i740eK/AGupdnxfZQNHIP3JDE8cc4561 + 9ufsw/AfSvjL8YLP4WTSG2k1SxZIZRHkLJ5fBPtnrXx9+0t+wz8VPg58XNY8A/ELQryO + OS8P+lfZ2/eruO11OCcEf54rDCKFVc09InTi6tRP2cEnJq6V7H09/wTv/AOCjviXTPiL + pL3+uwas/niIaetxlZlOBkK2055I79Rzxiui/4LwfBnTNH+I2gftj/CXQ7jTJNTsoZdY + FghhNvcKV+ZSo+R8YbPXINT/8En/g3+yX4OttQ8M/HfR73W3urYxaNLPayLJbzFlw8J4 + KyAKvzDkDI6E5+9P2u/2dfhhY/sH+NvBviDxDd6jpuuaKw0241edHeB/l2AylV2AHHzN + yB3Nerh6/1e9WlL3Y6+at/mfN4uip1YUqsffn7vk79n5bnzx+y38YNc/aj+GHhn4y+GU + vLrUbi3OieJH1e3Wa1t9QjNsgumeJQ8X2u3DQtIMKGZcqS3zPtY1ivZIlBCrIwUegzXP + fsN/Am6+AXiDWdR07SUHhvXvDtq00NvqgUW00J82PAYFJmSTKq3zsVKMSfmLdXNpl9pu + u3On6igWeG4dJdrZG8MQ2D0PII49KvPs8weeYbD1KXxLmv5bHj0skxOS4qpCfwuzX9eR + sWkWYQQpNFWLJVFuuVP50V877qNNT8Qf2gfDnifxF4pDaTqlmkX2OMNBPq1vASwkkOSJ + HU4w3p1r134QeEoda+FVp4Z1+fbCkaC4azuFO4LtJAkQkEEjHXHJ5rnfE3wA1D4q6j/b + MVzZW8SpHEJLlXJbBY44I459a7vRtLf4dfDH/AIRK1t4ryW3R4ftAkChmUkBivX14Gf6 + 1rmOOlQyylyy1vp5ad9Tq4ewEMbnEk1p1+/seU/tA6OdW082trZrHOtptiEMYZd0Ue8A + kH+6rrjnJYV71+x/45g+IPwasr/V0UypA0TxsM7mHBOD2Brxa6ubqSc6J4siDEuhWGzi + w0oTOYyxUZB+bJHcjB5ArN+G/xA8W/CZ9U0jRY45NPnlT90spV7SZo97pkkbgcEjGSSO + nWvn8LViqDhLpr9+/6fcfquLwynNOPa33bfqe/wDjiz8H6feyTSRwDu6t3/wqf4L+J/C + /iXxDcaLpS7ZbUAyMi/KhPQH3r5z8XfEfWPFv+hQW14qy43PkKdpP3s84H19OcVf/AGf + vjR4o+BGo6w83ga3vrS6ljur6a71GKJrbGQNrMw84n+6OTg8HmvWVSE4cqetu/wDwTjq + 0lBbfM/SP9jnxla+AP2nPC3jPW8BLK6CSKGwSGBXv0619k/8ABWP42+Cfg/4W8L67qXw + zS/m8TeYLXVHhBSBlVWKM2OpX5hkjhWPQGvyI8CftMN4zv4fH+nMbG2S4iMkt1II4kd2 + baoJwCT5b4UEnCMegOPpv9u7/AIKSaz+0r+zBof7Ob/D8m0sr21kvvEmp2k8azywK2xL + fAXhhuy+9SVzjPOO3D4/D4XBTpVZWb206nk4zKKuIzLD4iC5lG6lrbR/5M9i/ZC+I/wA + HfiX420y88W6JYWk0km60aKJFUkErngccqRmvYv8Agsq+reKPgX4S+FHgPUoWi8X67b2 + QgSNnJEbCaTzNnJUrHjPY4z1r8uvgF8QtC8CeNbXRU+KUkVzZ3Mj2ravYuIioZvMV5FJ + KbCCCxULu3ncF5r7Lk8Y2vxy8Q+G/iL4r8dDUdH0a0dtAsbbzIjD5xaF7qQ7lDbXREZN + ucuAMgsw8mtn1GOBqUeX3pap7bM662QyWZ0sVGXuwvpvuvzPRPg/4Y8I3fha9ttPj1N4 + rVDFYJ9uWXyISyrJtkiKrtZSwGVPA2561la54X1LQdUlGqWEgmRwZJVcFXB+6+OThuDn + OMnGBxXq3gK10dtFE+niNbu6iO2XTjtlim3eYvTaARv65GcE5Hbzv4k+MLNr97XxzqkF + rdWbtbpdSzr5z7XACvGAGZc4JO3cM5xgGvKyypClH33ZHkcTV6cE5zfKl1l/wSKzfMA3 + MfzorwP8Aav8AjL4o8A/Ei00D4YXp1TTRokErahYM5imlZ5CxAZVK44UgjPy+9FFXPKU + KrUYNrvf/AIB+SVeLMJSqyhGLaT37nxJo974xs47uy0/Wlt7e6jUKPs5LxsM5IOMDI4/ + L3rVu4Li38LXFxfXbyyRsMKke3JZ8KoJwMnOM9uOvWmWUVzjCW7n6Iear+NoZ4fDN0k4 + nQTIkatHHk5Mi4wD/ADHI7V7WKh7ahyy6LsfdZRXeFxilH7TVzkvHel6jpV0k1z4eJcW + SiN1yWVSpbAyQv97n0Irzzxl5UqRWm64V0uhI1tDaK4U853bhg42nGOeOCe/0t428K+H + NS8MRXttqfmNBCpbZt6bCc5BwMnA/PrXzx408O6oGfWZLOedXlaZ7gMQyqCvyqcj7zc+ + gyPevEw81CpufqFOt9aocyOQuNI8S6XaT3U+kXAt4LeRLVmhOwjeQRgYG3JBAxxg9Mki + TXz4TsvhbqEur+NPK1YTwW9pZ21oyGUOCWPfciKuGdtrb5I9vG6p/FnxCuI7hLSSzkka + 3V1F07KkiIyHK7025XJ49D0xkBeLa0mtE/te6gjmiuGWbyIRuKqygd88YAzjnkc5zj38 + LBTtKrpqnp1PMxNarBuMNf0+R38vxn0G78KReEodAGmw3ekyS7rC8uMTXavtV2ikcpGC + ERGIXftAOcnNbFl8QbHxr4LlXX/El0ojmQ20MskaGHavzY3vvK/IvQclckDcK88g1nU3 + gijstDimE8h3sqnavlAquN2M/IRtPXqOtSDwr4w8eatBp2n6JPZtMgbcynaC2VIC8Ekg + 9M/XrWlXDYXWUnypdbr+uhFHF10lGC5n2sz7t+AfwY8MfGH4TaR4U1uK6a+vLKVNE1y3 + tzJbXCGNz5Duoba5KbcsUP3OucD3/APZX1i9+GGg6t4O8SW9tFcWmnJbtFJIRPJsd2lJ + 3JlYdskBSTLBsrjggj5I/Y/8AH2qfAvQNM0y71q61vw2upwzoumkeal/uAKAFSSDHEqn + B24fjlTX2F4LX/hK/jBceIfCHjm5fSvEkYhu7S7jQ+TcwyRsUj2ll2Mixt8xKhuOqgn4 + 3FtQcknfX+v8Ago9+PtZ251ZNfdbW1z6k+EGsXV74Ju9XXw1Pa3PkG4+1XYcBwUABUR+ + g3cBtucDkZx5h8Wta+H/xKvbDw14e00PqdzOuZdLAad5XIYzJuU8fMVdcEfIHz1r1/Tv + EV98F9DjuF8XaTcW09vJBJbzq8lwspX5ZotpO7GQW5I4IJGFNfDX7YWqftVBdMOm/CKa + C90WSV9X8a6BaecNQjkj25dIcBFZGwzOilwqnjLA9H9nZlSwkaioycbN/C2rK3VXt+B+ + UcaYuhipygrNX6q/y1tb8V2PevCPhHwboWltYzeMbTUj57MLi9uoZWXoNqsiBSoIOMep + 5or5C+DXxg/YUs/Dl3F+1B4E8VnxGdRYwnw1eGC2+y+XGF/d5+V94kzjrwepNFccOHFi + 4qs6sY82trtW8j8Zq5JXq1ZTi0k3troebfte6ZL4B0yw+Jdh4yu9Ntby4W1u7SzRfLEr + ZKvhvu8DB7cZwOa8p1bVtH06Ean4v8fPeRrGWWO6hiIKkZyNq7vy4rnP2pP2x7/4q3sn + hPSIg/h62uAYVFuskkrjgOzt0Ppjpnqa8P17xDqV1o1zNcjY8aqyQu+Tt3AFj6n9P0r9 + bp8NqpZ1ZW7pJXfz6fifqP9rKlG1NXa69D23RPjlc+O/ET+H/AAiYbLT5JY7OGe4izPd + ysxJ2KvYKAD1xuBJHSuw8TqNR+06Jp9nNeMLgRfuZgzTAKAWxnhQAMepyeMcfH/gjx9r + ngjUk8R+GZ1ivyssQleMFkWTIYr6Ngnn/ABNet/s8/Fm2j8TrPqesF5BZrJP5UhYxbHj + wBtYfL87H0yxHbA8riHhv2MPbUVaMVst36vf8fuPp+GeI/e9hV3k93+FjU1TwxqXiHfY + 3dhLDFbvuiuHYgsUfBUcgMcjoAfXjknGn0C0tLiW61Y+Wlyi28928xDRqnPmD0UEAHsM + nkYzXvGv+DtPvLyfXYb15LSwt3u7dtoUSByu3oMnC5I4ycdu3Fa3baHZeFBrWraGrvYQ + loLFmCpPG65UbRn5QpXJ6556mvlaOOnFqFtNrf18j7yVGlVXOt+5zOn6pq2sWd54aiuf + st3bSGSMG2V3ZEG6RIwxBVSFLHBycnnpXZ+ENEXxrLaXnhTUkDLcqihZ8eTMATtYkKPm + CnCsTkrgelWNAXw9qniR9UGmW+pS6joaTC1RJIpbfVXcICrMCc+Q6vkHByoyChA9N+CH + w60eewm8c30dykE1t9nuNJFsxa4dNyjcYzncGKBm6rvjOR8xGWJq0kuz3+b3X4f1c1w7 + l8T1R7t+zv8KrG50C08ObrOw1Se+SWO4uLZUEjRsA3PITiTLcqCMZ7g/UPws0zwR8KfD + dlY+PNMexnjd7l9TukEZsYljLLC3J3K3UZ5VgQchiDt/sq/B2LXfAsOoeIPDqTXKRML3 + 7cVe5dJIdsgViD8xWQjIOfmyDyRXgn7b/AMY/sl74r8N6P4llngGpWujeHhqs7sZGS3h + tGl8xQfLcyq0gb7v3SR1FXw/ktXN8Trolr8rr+vI8XPc8pYOLin/w/wDX3nrfjL4oeHI + fHfw40DxJeM0viLTkeOYSKYL+QxSYliZCyHaM9MZDY4K5Hnv7MP7Ul1r/AMede/ZV+Ie + qEatoU80GkyPhZGRSGhO4AMSANvXowr5R+Pn7SU+qeEvhR8TLfV2VfC3xLutGtbgTKVa + 2iSFGYFMKwO0sSB8xYnvXO/8ABRvxtr/7Pv7cnhf9pnwJcNENV0601KOSEkLK0bFJEOO + xUL+DCv6Ew8YYehFNbJa/I/E8RJ1asvO/5n1T8ff2jJvh98Rrjw9q8cepTCJZDPeaXFd + MoJYBA7gkAY6epJ70VhePvhFqv7Wd/Y/tDfBy7updJ8UaVBdyx2cylbe5wRLEQc4IYdP + Qiiun2VN68q+441y9Wfj3p2uhISsy7iH35Y+lUItWkvBqNzPJkywMCc8gZBrNu72a0gm + O75i2AfSq0Urw6RNyQZVC/hnmvCsjdTdrFjw3ILnWLe3Iyvmf+O96j8JeKrnwXqt7fWQ + LXDp5MSn7u3zFds9+iYAH97ORjBd4ZZINTiI6gHJ/A1i3QP21plbqxIz9TROMK14y1T3 + KhUlStKO6Ptj9nj9qfSfiNYxeBfHixxXc7NJpv2ZV/wBSoJMR9wC7fQkckYr0jxJ4Z8K + arof2WFIRFJp7gfJkOzHchypyCQgPbrnvX5/+B/F8/hLUxcNbh4nO51CgOxUEqAw5Ubt + pxnGQDgkDHqPww/ap8deH7iHR9X1F7mymQi83gbg2XywIHTy2wfbPsR+f5vwnKVZ1MKr + KOtv8v6/yP0DJ+LVCmqeJlq3v9259OfDP4darqmoObzV0jSxjZdNkKgHzEG5QSBk/c4y + ST0Gcivtf4WaH8OtFu4rPR7+F7K5v2WKOISyC+kyRKxVx/rcOqjIyVkXIbBNfCPw7+N+ + n6bdaLY3V9OLd7ppdRjtY9zRK8WwMpiJkyAx4wOQPStfQP2sPGaai+h+FtOdby0tr24u + kliV7d71Ek8plUENhkVAG5O5QcEDj5mPDmZYirZQflofVVs/wUaV+dL5n6I/tC/t3+G/ + 2XvB2l+CfD6HWNb8Q6qNDyZpUeCGWISNIeMxzmLCRq5wWCk5Csx+Cv2w/itaeGvCiWFr + q91MfD9kiWBvrgzTPcuGCMzHlmUZlJ7MOmTmqPwXl1j4s+I/+FqfE+ObUPEtlaraPe3x + MjLbwMfJbL85CMqZ4OI8kenhf7YnjibxJrUkCXAdIWVRIvRjySTx6kgegFfrPD2R/2Lg + XOp/Ef4H5dnWbf2jieWHwL8ToPFMjXn/BOPwlqm8+ZZ/Ea9w4/wBuDP8A7JXb/tSa1qf + xu/YZ+FXxTJe6uNFurnSNQlUbip2x7dx9xHXnf2ua/wD+CbFrbqAy2fxCBxj+9BNz+ld + d+wJ4r074u/Cnxl+yF4ouFT+1bX7f4eklfAiu4+Rj6kYP1r34tufJ3R4bnpfszy7QP2s + v2h/gdp6/D/4d/ErUdM063xJ9lgl+QSMo3EemcCiuG+MmjahoHxK1XRtVtlhuLa5Mcse + M7WHWip9rUi7XJervY801Ni90UY8bzxTGbOnMSBw2KKK5Z/D9wluTaCNkzyr95YmwfwN + ZtwijJA70UVk/iZfQMkHaCR71bt5XMIcnnODjvRRQgjsTfaLi5cNPcSOVztLOSRkknr7 + kn8a6b4L6xqeneObWewvHhklPlu8ZwSp6jPvRRWlHRoJao+5vh/dSaH8B/Euq6eAs4vF + tRI3J2Mrbj/vHA5/Svkv4u3UtxDcSyEbgFOfU56/rRRXfim1TRnD4md/4MuJW/wCCdes + QMcqnxAtiuR0zby/41wH7MniHVfDP7QPhfUdIuDHK2rRRMfVXYKwP4E0UVzybVeHovzB + bM6j9tHTra3/aa8WRRKQP7Qzz7qD/AFoooqppc7COx//Z +END:VCARD diff --git a/resources/vcards/046.vcf b/resources/vcards/046.vcf new file mode 100644 index 0000000..173bff2 --- /dev/null +++ b/resources/vcards/046.vcf @@ -0,0 +1,14 @@ +BEGIN:VCARD +VERSION:4.0 +FN:Dummy\, Dummy +N:Dummy;Dummy;;; +ORG:Dummy GmbH +TEL;TYPE=cell;PREF=1:+49 1234 56789 +TEL;TYPE=work:+49 9876 54321 +EMAIL;TYPE=home:dummy.dummy@dummy.com +ADR;TYPE=work;LABEL=Dummy-Dummy-Strasse 1 61352 Bad Homburg^nGERMANY^': BH + G01:^n61352 Bad Homburg^nGERMANY:61352 Bad Homburg\nGERMANY:;BHG01:;Dumm + y-Dummy-Strasse 1;Bad Homburg;;61352;Germany +REV;VALUE=DATE-AND-OR-TIME:20210314T092838Z +UID:8b574c60-fd7f-4e99-b584-c5db131ae687 +END:VCARD diff --git a/resources/vcards/048.vcf b/resources/vcards/048.vcf new file mode 100644 index 0000000..4d7f4b9 --- /dev/null +++ b/resources/vcards/048.vcf @@ -0,0 +1,55 @@ +BEGIN:VCARD +VERSION:3.0 +UID:urn:uuid:7511bfa7-f1f2-53cf-8b7d-b5362dcc0dd7 +FN:Mr. John Q. Public\, Esq. +N:Public;John;Quinlan;Mr.;Esq. +N:Stevenson;John;Philip,Paul;Dr.;Jr.,M.D.,A.C.P. +NICKNAME:Robbie +NICKNAME:Jim,Jimmie +PHOTO;VALUE=uri:http://www.abc.com/pub/photos + /jqpublic.gif +PHOTO;ENCODING=b;TYPE=JPEG:SGVsbG8sIHRoaXMgaXMgbm90IGEgcmVhbCBp + bWFnZSBqdXN0IGEgdGVzdC4= +BDAY:1996-04-15 +BDAY:1953-10-15T23:10:00Z +BDAY:1987-09-27T08:30:00-06:00 +ADR;TYPE=dom,home,postal,parcel:;;123 Main + Street;Any Town;CA;91921-1234 +LABEL;TYPE=dom,home,postal,parcel:Mr.John Q. Public\, Esq.\n + Mail Drop: TNE QB\n123 Main Street\nAny Town\, CA 91921-1234 + \nU.S.A. +TEL;TYPE=work,voice,pref,msg:+1-213-555-1234 +EMAIL;TYPE=internet:jqpublic@xyz.dom1.com +EMAIL;TYPE=internet:jdoe@isp.net +EMAIL;TYPE=internet,pref:jane_doe@abc.com +MAILER:PigeonMail 2.1 +TZ:-05:00 +TZ;VALUE=text:-05:00\; EST\; Raleigh/North America + \;This example has a single value\, not a structure text value. +GEO:37.386013;-122.082932 +TITLE:Director\, Research and Development +ROLE:Programmer +LOGO;VALUE=uri:http://www.abc.com/pub/logos/abccorp.jpg +LOGO;ENCODING=b;TYPE=JPEG:SGVsbG8sIHRoaXMgaXMgbm90IGEgcmVhbCBp + bWFnZSBqdXN0IGEgdGVzdC4= +AGENT;VALUE=uri: + CID:JQPUBLIC.part3.960129T083020.xyzMail@host3.com +AGENT:BEGIN:VCARD\nFN:Susan Thomas\nTEL:+1-919-555- + 1234\nEMAIL\;INTERNET:sthomas@host.com\nEND:VCARD\n +ORG:ABC\, Inc.;North American Division;Marketing +CATEGORIES:TRAVEL AGENT +CATEGORIES:INTERNET,IETF,INDUSTRY,INFORMATION TECHNOLOGY +NOTE:This fax number is operational 0800 to 1715 + EST\, Mon-Fri. +PRODID:-//ONLINE DIRECTORY//NONSGML Version 1//EN +REV:1995-10-31T22:27:10Z +REV:1997-11-15 +SORT-STRING:Harten +SOUND;TYPE=BASIC;VALUE=uri:CID:JOHNQPUBLIC.part8. + 19960229T080000.xyzMail@host1.com +SOUND;TYPE=BASIC;ENCODING=b:VGhlcmUgaXMgbm8gc291bmQgaW4gc3BhY2U= +CLASS:PUBLIC +CLASS:PRIVATE +CLASS:CONFIDENTIAL +KEY;ENCODING=b:Tm90IHRoZSBrZXkgeW91IGFyZSBsb29raW5nIGZvcg== +END:VCARD diff --git a/resources/vcards/051.vcf b/resources/vcards/051.vcf new file mode 100644 index 0000000..04d7189 --- /dev/null +++ b/resources/vcards/051.vcf @@ -0,0 +1,23 @@ +BEGIN:VCARD +VERSION:3.0 +PRODID:-//Sabre//Sabre VObject 4.1.6//EN +UID:ad612c16-fe12-4ec5-abf6-49998ee5ab88 +FN:First Last NextCloud +ADR;TYPE=HOME:PO;Street 2;Street;City;AL;zip;Germany +ADR;TYPE=WORK:PO W;Street 2 W;Street W;City Work;AL work;zip Work;Germany w + ork +EMAIL;TYPE=HOME:home@gmx.de +EMAIL;TYPE=WORK:work@gmx.de +TEL;TYPE="HOME,VOICE":11111111 +TEL;TYPE=CELL:205333 +TEL;TYPE="WORK,FAX":205246\;\;\,\; +TEL;TYPE="WORK,VOICE":222222 +ORG:Firma +BDAY:20190210T000033 +NICKNAME:Hugo +ITEM1.X-ABDATE;VALUE=DATE-AND-OR-TIME:20190220T000035 +ITEM1.X-ABLABEL:_$!!$_ +X-ANNIVERSARY;VALUE=DATE-AND-OR-TIME:20190220T000035 +CATEGORIES:Test-Kontakte +REV:20191008T170514Z +END:VCARD diff --git a/resources/vcards/066.vcf b/resources/vcards/066.vcf new file mode 100644 index 0000000..dd811af --- /dev/null +++ b/resources/vcards/066.vcf @@ -0,0 +1,41 @@ +BEGIN:VCARD +VERSION:3.0 +REV:20080409T095515Z +X-YAHOO;TYPE=HOME:yahoo.john +X-GOOGLE TALK;TYPE=WORK:gtalk.john +X-SAMETIME;TYPE=WORK:sametime.john +X-SKYPE;TYPE=WORK:skype.john +X-MSN;TYPE=WORK:msn.john +X-JABBER;TYPE=WORK:jabber.john +N:Doe;John;;; +ADR;TYPE=WORK:;;456 Grandview Building\, Wide Street;San Diego;CA;90204; + United States +ADR;TYPE=HOME:;;123 Sweet Home\, Narrow Street;New York;NY;91102;United + States +URL;TYPE=OTHER:http\://www.homepage.com +URL;TYPE=HOME:http\://www.home.com +URL;TYPE=WORK:http\://www.work.com +URL;TYPE=OTHER:http\://www.other.com +URL;TYPE=OTHER:http\://www.custom.com +ORG:John Doe & Partners Limited;; +TEL;TYPE=WORK:11111111 +TEL;TYPE=CELL:22222222 +TEL;TYPE=HOME:33333333 +TEL;TYPE=OTHER:44444444 +TEL;TYPE=FAX:55555555 +TEL;TYPE=FAX:66666666 +TEL;TYPE=PAGER:77777777 +TEL;TYPE=OTHER:88888888 +TEL;TYPE=OTHER:99999999 +UID:cc548e11-569e-3bf5-a9aa-722de4571f4a +X-ICQ;TYPE=HOME:icq.john +EMAIL;TYPE=WORK,INTERNET:john.doe@work.com +EMAIL;TYPE=HOME,INTERNET:john.doe@home.com +EMAIL;TYPE=OTHER,INTERNET:john.doe@other.com +EMAIL;TYPE=OTHER,INTERNET:john.doe@custom.com +TITLE:Sales Manager +X-OTHER;TYPE=WORK:other.john +X-AIM;TYPE=WORK:aim.john +X-QQ;TYPE=WORK:qq.john +FN:John Doe +END:VCARD diff --git a/resources/vcards/rfc-a.vcf b/resources/vcards/rfc-a.vcf new file mode 100644 index 0000000..484efc3 --- /dev/null +++ b/resources/vcards/rfc-a.vcf @@ -0,0 +1,62 @@ +BEGIN:VCARD +VERSION:4.0 +NOTE:Mythical Manager\nHyjinx Software Division\n + BabsCo\, Inc.\n +ROLE;LANGUAGE=tr:hoca +TITLE;ALTID=2;LANGUAGE=en:Chief vCard Evangelist +N;ALTID=1;LANGUAGE=en:Yamada;Taro;;; +N;SORT-AS="Harten,Rene":van der Harten;Rene,J.;Sir;R.D.O.N. +N;SORT-AS="Pozo,Oscar":del Pozo Triscon;Oscar;; +SOURCE:ldap://ldap.example.com/cn=Babs%20Jensen,%20o=Babsco,%20c=US +SOURCE:http://directory.example.com/addressbooks/jdoe/ + Jean%20Dupont.vcf +KIND:individual +N:Mr. John Q. Public\, Esq. +N:Stevenson;John;Philip,Paul;Dr.;Jr.,M.D.,A.C.P. +NICKNAME;TYPE=work:Boss +PHOTO:data:image/jpeg;base64,MIICajCCAdOgAwIBAgICBEUwDQYJKoZIhv + AQEEBQAwdzELMAkGA1UEBhMCVVMxLDAqBgNVBAoTI05ldHNjYXBlIENvbW11bm + ljYXRpb25zIENvcnBvcmF0aW9uMRwwGgYDVQQLExNJbmZvcm1hdGlvbiBTeXN0 +BDAY:19960415 +BDAY:--0415 +BDAY:19531015T231000Z +BDAY;VALUE=text:circa 1800 +GENDER:M +GENDER:F +GENDER:M;Fellow +GENDER:F;grrrl +GENDER:O;intersex +GENDER:;it's complicated +ADR;GEO="geo:12.3457,78.910";LABEL="Mr. John Q. Public, Esq.\n + Mail Drop: TNE QB\n123 Main Street\nAny Town, CA 91921-1234\n + U.S.A.":;;123 Main Street;Any Town;CA;91921-1234;U.S.A. +TEL;VALUE=uri;PREF=1;TYPE="voice,home":tel:+1-555-555-5555;ext=5555 +TEL;VALUE=uri;TYPE=home:tel:+33-01-23-45-67 +EMAIL;TYPE=work:jqpublic@xyz.example.com +IMPP;PREF=1:xmpp:alice@example.com +LANG;TYPE=work;PREF=1:en +LANG;TYPE=work;PREF=2:fr +TZ;VALUE=utc-offset:-0500 +GEO:geo:37.386013,-122.082932 +LOGO:http://www.example.com/pub/logos/abccorp.jpg +ORG:ABC\, Inc.;North American Division;Marketing +RELATED;TYPE=friend:urn:uuid:f81d4fae-7dec-11d0-a765-00a0c91e6bf6 +RELATED;TYPE=contact:http://example.com/directory/jdoe.vcf +RELATED;TYPE=co-worker;VALUE=text:Please contact my assistant Jane + Doe for any inquiries. +CATEGORIES:TRAVEL AGENT +CATEGORIES:INTERNET,IETF,INDUSTRY,INFORMATION TECHNOLOGY +NOTE:This fax number is operational 0800 to 1715 + EST\, Mon-Fri. +PRODID:-//ONLINE DIRECTORY//NONSGML Version 1//EN +REV:19951031T222710Z +SOUND:CID:JOHNQPUBLIC.part8.19960229T080000.xyzMail@example.com +UID:urn:uuid:f81d4fae-7dec-11d0-a765-00a0c91e6bf6 +TEL;PID=3.1,4.2;VALUE=uri:tel:+1-555-555-5555 +EMAIL;PID=4.1,5.2:jdoe@example.com +CLIENTPIDMAP:1;urn:uuid:3df403f4-5924-4bb7-b077-3c711d9eb34b +CLIENTPIDMAP:2;urn:uuid:d89c9c7a-2e1b-4832-82de-7e992d95faa5 +KEY;MEDIATYPE=application/pgp-keys:ftp://example.com/keys/jdoe +FBURL;MEDIATYPE=text/calendar:ftp://example.com/busy/project-a.ifb +CALADRURI;PREF=1:mailto:janedoe@example.com +END:VCARD diff --git a/resources/vcards/rfc-b.vcf b/resources/vcards/rfc-b.vcf new file mode 100644 index 0000000..10c2ccc --- /dev/null +++ b/resources/vcards/rfc-b.vcf @@ -0,0 +1,7 @@ +BEGIN:VCARD +VERSION:4.0 +UID:urn:uuid:3c380a90-9a23-59bc-b228-64e45cc23475 +EMAIL;PID=4.2,5.1:jdoe@example.com +CLIENTPIDMAP:1;urn:uuid:3eef374e-7179-4196-a914-27358c3e6527 +CLIENTPIDMAP:2;urn:uuid:42bcd5a7-1699-4514-87b4-056edf68e9cc +END:VCARD diff --git a/resources/vcards/rfc-c.vcf b/resources/vcards/rfc-c.vcf new file mode 100644 index 0000000..3145a9b --- /dev/null +++ b/resources/vcards/rfc-c.vcf @@ -0,0 +1,7 @@ +BEGIN:VCARD +VERSION:4.0 +UID:urn:uuid:9f501a4e-a0df-5d90-8ff9-64e811b8ec15 +EMAIL;PID=5.1,5.2:john@example.com +CLIENTPIDMAP:1;urn:uuid:0c75c629-6a8d-4d5e-a07f-1bb35846854d +CLIENTPIDMAP:2;urn:uuid:3eef374e-7179-4196-a914-27358c3e6527 +END:VCARD diff --git a/resources/vcards/rfc-d.vcf b/resources/vcards/rfc-d.vcf new file mode 100644 index 0000000..361a4fe --- /dev/null +++ b/resources/vcards/rfc-d.vcf @@ -0,0 +1,9 @@ +BEGIN:VCARD +VERSION:4.0 +UID:urn:uuid:4fbe8971-0bc3-424c-9c26-36c3e1eff6b1 +FN;PID=1.1:J. Doe +N:Doe;J.;;; +EMAIL;PID=1.1:jdoe@example.com +TEL;PID=1.1;VALUE=uri:tel:+1-555-555-5555 +CLIENTPIDMAP:1;urn:uuid:53e374d9-337e-4727-8803-a1e9c14e0556 +END:VCARD diff --git a/resources/vcards/rfc-e.vcf b/resources/vcards/rfc-e.vcf new file mode 100644 index 0000000..daf92bf --- /dev/null +++ b/resources/vcards/rfc-e.vcf @@ -0,0 +1,12 @@ +BEGIN:VCARD +VERSION:4.0 +UID:urn:uuid:d133e5b7-baa2-5520-9766-965ca3835985 +FN;PID=1.1:J. Doe +N:Doe;J.;;; +EMAIL;PID=1.1:jdoe@example.com +EMAIL;PID=2.2:ceo@example.com +TEL;PID=1.1;VALUE=uri:tel:+1-555-555-5555 +TEL;PID=2.2;VALUE=uri:tel:+1-666-666-6666 +CLIENTPIDMAP:1;urn:uuid:53e374d9-337e-4727-8803-a1e9c14e0556 +CLIENTPIDMAP:2;urn:uuid:1f762d2b-03c4-4a83-9a03-75ff658a6eee +END:VCARD diff --git a/resources/vcards/rfc-f.vcf b/resources/vcards/rfc-f.vcf new file mode 100644 index 0000000..6921898 --- /dev/null +++ b/resources/vcards/rfc-f.vcf @@ -0,0 +1,8 @@ +BEGIN:VCARD +VERSION:4.0 +UID:urn:uuid:7c9ba2ad-f8ab-5747-9738-ed156bad9270 +KIND:group +FN:The Doe family +MEMBER:urn:uuid:03a0e51f-d1aa-4385-8a53-e29025acd8af +MEMBER:urn:uuid:b8767877-b4a1-4c70-9acc-505d3819e519 +END:VCARD diff --git a/resources/vcards/rfc-g.vcf b/resources/vcards/rfc-g.vcf new file mode 100644 index 0000000..7988f42 --- /dev/null +++ b/resources/vcards/rfc-g.vcf @@ -0,0 +1,5 @@ +BEGIN:VCARD +VERSION:4.0 +FN:John Doe +UID:urn:uuid:03a0e51f-d1aa-4385-8a53-e29025acd8af +END:VCARD diff --git a/resources/vcards/rfc-h.vcf b/resources/vcards/rfc-h.vcf new file mode 100644 index 0000000..e5f9dbe --- /dev/null +++ b/resources/vcards/rfc-h.vcf @@ -0,0 +1,10 @@ +BEGIN:VCARD +VERSION:4.0 +UID:urn:uuid:e5b9689c-02bd-5c95-a857-bad3b5bd8cf4 +KIND:group +FN:Funky distribution list +MEMBER:mailto:subscriber1@example.com +MEMBER:xmpp:subscriber2@example.com +MEMBER:sip:subscriber3@example.com +MEMBER:tel:+1-418-555-5555 +END:VCARD diff --git a/resources/vcards/rfc-i.vcf b/resources/vcards/rfc-i.vcf new file mode 100644 index 0000000..ed4290c --- /dev/null +++ b/resources/vcards/rfc-i.vcf @@ -0,0 +1,22 @@ +BEGIN:VCARD +VERSION:4.0 +UID:urn:uuid:61bbee4f-102c-5d99-8ed2-5543c8f44b8d +FN:Simon Perreault +N:Perreault;Simon;;;ing. jr,M.Sc. +BDAY:--0203 +ANNIVERSARY:20090808T1430-0500 +GENDER:M +LANG;PREF=1:fr +LANG;PREF=2:en +ORG;TYPE=work:Viagenie +ADR;TYPE=work:;Suite D2-630;2875 Laurier; + Quebec;QC;G1V 2M2;Canada +TEL;VALUE=uri;TYPE=work,voice;PREF=1:tel:+1-418-656-9254;ext=102 +TEL;VALUE=uri;TYPE=work,cell,voice,video,text:tel:+1-418-262-6501 +EMAIL;TYPE=work:simon.perreault@viagenie.ca +GEO;TYPE=work:geo:46.772673,-71.282945 +KEY;TYPE=work;VALUE=uri: + http://www.viagenie.ca/simon.perreault/simon.asc +TZ:-0500 +URL;TYPE=home:http://nomis80.org +END:VCARD diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..ccccf87 --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,1364 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::path::PathBuf; + +use clap::{ArgAction, Args, Parser, Subcommand}; +use regex::Regex; + +use crate::error::Error; +use crate::exchange_ews::oauth::OAuthFlow; +use crate::exchange_ews::types::MailboxKind; +use crate::exchange_graph::types::{EventBodyFormat, MailboxKind as GraphMailboxKind}; +use crate::inspect::InspectConfig; +use crate::jmap::account::AccountSelector; +use crate::jmap::http::Auth; +use crate::logging::Logger; +use crate::secret; +use crate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; +use crate::sync::import_exchange_ews::{EwsAuth, EwsImportConfig}; +use crate::sync::import_exchange_graph::{GraphAuth, GraphImportConfig}; +use crate::sync::import_imap::{ImapAuth, ImapImportConfig}; +use crate::sync::import_maildir::MaildirImportConfig; +use crate::sync::import_managesieve::{ManageSieveAuth, ManageSieveImportConfig}; +use crate::sync::import_takeout::TakeoutImportConfig; +use crate::sync::{CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; +use crate::types::{ObjectType, parse_object_list}; + +#[derive(Parser)] +#[command( + name = "vandelay", + version, + about = "Vandelay: the JMAP importer-exporter" +)] +pub struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + #[command(about = "Read a source account into a local archive")] + Import { + #[command(subcommand)] + source: Source, + }, + #[command(about = "Push a local archive into a target JMAP server")] + Export(Box), + #[command(about = "Show the contents of a local archive (read-only)")] + Inspect(Box), +} + +#[derive(Subcommand)] +enum Source { + #[command(about = "JMAP server")] + Jmap(Box), + #[command(about = "IMAP server")] + Imap(Box), + #[command(about = "CalDAV server")] + Caldav(Box), + #[command(about = "CardDAV server")] + Carddav(Box), + #[command(about = "WebDAV server (plain files)")] + Webdav(Box), + #[command(about = "ManageSieve server (Sieve scripts only)")] + Managesieve(Box), + #[command(about = "Local Maildir++ tree")] + Maildir(Box), + #[command(about = "Google Takeout (or any mbox/ics/vcf directory tree)")] + Takeout(Box), + #[command(about = "MS Exchange via EWS (SOAP/XML)", name = "exchange-ews")] + ExchangeEws(Box), + #[command( + about = "MS Exchange Online via Microsoft Graph", + name = "exchange-graph" + )] + ExchangeGraph(Box), +} + +#[derive(Args)] +struct GlobalArgs { + #[arg( + short = 'j', + long, + value_name = "N", + help = "Worker pool size (default: logical CPUs)" + )] + threads: Option, + + #[arg(long, help = "Compute and report the full plan; perform no writes")] + dry_run: bool, + + #[arg( + short = 'v', + long, + action = ArgAction::Count, + conflicts_with = "quiet", + help = "Increase log verbosity (repeatable, max -vvv)", + long_help = "Increase log verbosity (repeatable, max -vvv).\n \ + (default) per-type start/finish lines and a final summary\n \ + -v per-chunk progress, account resolution, retry notices\n \ + -vv every JMAP call (method, accountId, args, status, timing)\n \ + -vvv full request/response bodies and backoff delays (stderr)" + )] + verbose: u8, + + #[arg( + short = 'q', + long, + help = "Quiet: warnings and errors only (verbosity level 0)" + )] + quiet: bool, + + #[arg( + long, + value_name = "N", + default_value_t = 5, + help = "Max retry attempts per request on transient failures" + )] + max_retries: u32, + + #[arg(long, help = "Accept self-signed / invalid TLS certificates")] + allow_invalid_certs: bool, +} + +#[derive(Args)] +#[command(group(clap::ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer"])))] +#[command(group(clap::ArgGroup::new("account").required(true).args(["account_id", "account_name"])))] +struct JmapImportArgs { + #[arg(long, value_name = "URL", help = "Base URL or full JMAP session URL")] + url: String, + + #[arg(long, value_name = "USER", help = "HTTP Basic user")] + auth_basic: Option, + + #[arg( + long, + value_name = "PASS", + help = "Basic password (prefer $VANDELAY_PASSWORD or prompt)" + )] + auth_password: Option, + + #[arg( + long, + value_name = "TOKEN", + num_args = 0..=1, + help = "Bearer token (value optional: falls back to $VANDELAY_TOKEN or prompt)" + )] + auth_bearer: Option>, + + #[arg(long, value_name = "ID", help = "Source account id")] + account_id: Option, + + #[arg(long, value_name = "NAME", help = "Source account name")] + account_name: Option, + + #[arg( + long, + value_name = "LIST", + help = "Comma-separated type list (default: all supported)" + )] + objects: Option, + + #[arg( + long, + help = "Permit importing when the archive records a different JMAP source" + )] + allow_source_change: bool, + + #[command(flatten)] + global: GlobalArgs, + + #[arg( + value_name = "ARCHIVE", + help = "Local SQLite archive (created if absent)" + )] + archive: PathBuf, +} + +#[derive(Args)] +#[command(group(clap::ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer"])))] +struct ImapImportArgs { + #[arg( + long, + value_name = "URL", + help = "imap://host[:port] or imaps://host[:port]" + )] + url: String, + + #[arg( + long, + value_name = "USER", + help = "PLAIN (preferred) with LOGIN fallback" + )] + auth_basic: Option, + + #[arg( + long, + value_name = "PASS", + help = "Basic password (prefer $VANDELAY_PASSWORD or prompt)" + )] + auth_password: Option, + + #[arg( + long, + value_name = "TOKEN", + num_args = 0..=1, + help = "Bearer token for XOAUTH2 (value optional: falls back to $VANDELAY_TOKEN or prompt)" + )] + auth_bearer: Option>, + + #[arg( + long, + value_name = "USER", + help = "XOAUTH2 identity (required with --auth-bearer)" + )] + auth_user: Option, + + #[arg(long, help = "Permit credentials on imap:// without STARTTLS")] + allow_cleartext: bool, + + #[arg(long, help = "Enable COMPRESS=DEFLATE post-auth (off by default)")] + compress: bool, + + #[arg( + long, + value_name = "REGEX", + action = ArgAction::Append, + help = "Folder include filter (repeatable)" + )] + include: Vec, + + #[arg( + long, + value_name = "REGEX", + action = ArgAction::Append, + help = "Folder exclude filter (repeatable)" + )] + exclude: Vec, + + #[arg( + long, + value_name = "ROLE", + action = ArgAction::Append, + help = "Drop folders matching a role (repeatable)" + )] + exclude_special: Vec, + + #[arg( + long, + value_name = "NAME", + action = ArgAction::Append, + help = "Exact include; mutually exclusive with --include/--exclude" + )] + folder: Vec, + + #[arg(long, help = "Keep only folders returned by LSUB")] + subscribed_only: bool, + + #[arg(long, help = "Disable SPECIAL-USE name-heuristic fallback")] + noautomap: bool, + + #[arg(long, help = "Import \\Deleted messages with the $deleted keyword")] + include_deleted: bool, + + #[arg( + long, + value_name = "N", + default_value_t = 256, + help = "UIDs per metadata FETCH chunk" + )] + fetch_batch: usize, + + #[arg( + long, + value_name = "N", + help = "Parallel IMAP connections for body fetch (1..=8). Default: min(--threads, 8)" + )] + imap_connections: Option, + + #[arg( + long, + help = "Permit importing when the archive records a different IMAP source" + )] + allow_source_change: bool, + + #[command(flatten)] + global: GlobalArgs, + + #[arg( + value_name = "ARCHIVE", + help = "Local SQLite archive (created if absent)" + )] + archive: PathBuf, +} + +#[derive(Args)] +#[command(group(clap::ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer"])))] +#[command(group(clap::ArgGroup::new("account").required(true).args(["account_id", "account_name"])))] +struct ExportArgs { + #[arg(long, value_name = "URL", help = "Base URL or full JMAP session URL")] + url: String, + + #[arg(long, value_name = "USER", help = "HTTP Basic user")] + auth_basic: Option, + + #[arg( + long, + value_name = "PASS", + help = "Basic password (prefer $VANDELAY_PASSWORD or prompt)" + )] + auth_password: Option, + + #[arg( + long, + value_name = "TOKEN", + num_args = 0..=1, + help = "Bearer token (value optional: falls back to $VANDELAY_TOKEN or prompt)" + )] + auth_bearer: Option>, + + #[arg(long, value_name = "ID", help = "Target account id")] + account_id: Option, + + #[arg(long, value_name = "NAME", help = "Target account name")] + account_name: Option, + + #[arg( + long, + value_name = "LIST", + help = "Types to export (default: all present in archive)" + )] + objects: Option, + + #[arg(long, help = "Delete target objects that match nothing in the archive")] + prune: bool, + + #[arg(long, help = "Skip the interactive --prune confirmation")] + yes: bool, + + #[command(flatten)] + global: GlobalArgs, + + #[arg(value_name = "ARCHIVE", help = "Local SQLite archive")] + archive: PathBuf, +} + +#[derive(Args)] +struct InspectArgs { + #[arg(value_name = "ARCHIVE", help = "Local SQLite archive")] + archive: PathBuf, + + #[arg( + value_name = "TYPE", + help = "Object type to dump (omit for a summary of all types)" + )] + target: Option, + + #[arg( + long, + value_name = "N", + help = "Maximum rows to show (omit to dump every row)" + )] + limit: Option, + + #[arg( + long, + value_name = "N", + default_value_t = 0, + help = "Skip the first N rows" + )] + offset: usize, +} + +pub enum Action { + Import(CommonConfig, ImportConfig), + ImportImap(CommonConfig, ImapImportConfig), + ImportDav(CommonConfig, DavImportConfig), + ImportManageSieve(CommonConfig, ManageSieveImportConfig), + ImportMaildir(CommonConfig, MaildirImportConfig), + ImportTakeout(CommonConfig, TakeoutImportConfig), + ImportExchangeEws(CommonConfig, EwsImportConfig), + ImportExchangeGraph(CommonConfig, GraphImportConfig), + Export(CommonConfig, ExportConfig), + Inspect(InspectConfig), +} + +#[derive(Args)] +#[command(group(clap::ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer"])))] +pub struct ManageSieveImportArgs { + #[arg( + long, + value_name = "URL", + help = "sieve://host[:port] or sieves://host[:port]" + )] + url: String, + + #[arg( + long, + value_name = "USER", + help = "SASL PLAIN (preferred) with LOGIN fallback" + )] + auth_basic: Option, + + #[arg( + long, + value_name = "PASS", + help = "Basic password (prefer $VANDELAY_PASSWORD or prompt)" + )] + auth_password: Option, + + #[arg( + long, + value_name = "TOKEN", + num_args = 0..=1, + help = "Bearer token for OAUTHBEARER (value optional: $VANDELAY_TOKEN or prompt)" + )] + auth_bearer: Option>, + + #[arg( + long, + value_name = "USER", + help = "OAUTHBEARER identity (required with --auth-bearer)" + )] + auth_user: Option, + + #[arg(long, help = "Permit credentials on sieve:// without STARTTLS")] + allow_cleartext: bool, + + #[arg( + long, + help = "Permit importing when the archive records a different ManageSieve source" + )] + allow_source_change: bool, + + #[command(flatten)] + global: GlobalArgs, + + #[arg( + value_name = "ARCHIVE", + help = "Local SQLite archive (created if absent)" + )] + archive: PathBuf, +} + +#[derive(Args)] +#[command(group(clap::ArgGroup::new("auth").required(true).args(["auth_basic", "auth_bearer"])))] +pub struct DavImportArgs { + #[arg(long, value_name = "URL", help = "http(s)://host[:port][/path]")] + url: String, + + #[arg(long, value_name = "USER", help = "HTTP Basic user")] + auth_basic: Option, + + #[arg( + long, + value_name = "PASS", + help = "Basic password (prefer $VANDELAY_PASSWORD or prompt)" + )] + auth_password: Option, + + #[arg( + long, + value_name = "TOKEN", + num_args = 0..=1, + help = "Bearer token (value optional: $VANDELAY_TOKEN or prompt)" + )] + auth_bearer: Option>, + + #[arg(long, help = "Permit credentials on http:// (no TLS)")] + allow_cleartext: bool, + + #[arg( + long, + value_name = "N", + default_value_t = 4, + help = "Parallel DAV worker connections (hard cap 8)" + )] + dav_connections: usize, + + #[arg( + long, + value_name = "N", + default_value_t = 50, + help = "Hrefs per calendar-/addressbook-multiget REPORT batch" + )] + multiget_batch: usize, + + #[arg( + long, + help = "Permit importing when the archive records a different DAV source" + )] + allow_source_change: bool, + + #[command(flatten)] + global: GlobalArgs, + + #[arg( + value_name = "ARCHIVE", + help = "Local SQLite archive (created if absent)" + )] + archive: PathBuf, +} + +impl Cli { + pub fn resolve(self) -> Result { + match self.command { + Command::Import { source } => resolve_import(source), + Command::Export(args) => resolve_export(*args), + Command::Inspect(args) => resolve_inspect(*args), + } + } +} + +fn resolve_import(source: Source) -> Result { + match source { + Source::Jmap(args) => resolve_jmap_import(*args), + Source::Imap(args) => resolve_imap_import(*args), + Source::Caldav(args) => resolve_dav_import(*args, DavKindArg::Caldav), + Source::Carddav(args) => resolve_dav_import(*args, DavKindArg::Carddav), + Source::Webdav(args) => resolve_dav_import(*args, DavKindArg::Webdav), + Source::Managesieve(args) => resolve_managesieve_import(*args), + Source::Maildir(args) => resolve_maildir_import(*args), + Source::Takeout(args) => resolve_takeout_import(*args), + Source::ExchangeEws(args) => resolve_exchange_ews_import(*args), + Source::ExchangeGraph(args) => resolve_exchange_graph_import(*args), + } +} + +fn resolve_managesieve_import(args: ManageSieveImportArgs) -> Result { + let common = common_config(&args.global, args.archive); + let auth = resolve_managesieve_auth( + args.auth_basic.as_deref(), + args.auth_password.as_deref(), + args.auth_bearer.as_ref(), + args.auth_user.as_deref(), + )?; + Ok(Action::ImportManageSieve( + common, + ManageSieveImportConfig { + url: args.url, + auth, + allow_cleartext: args.allow_cleartext, + allow_source_change: args.allow_source_change, + }, + )) +} + +fn resolve_managesieve_auth( + auth_basic: Option<&str>, + auth_password: Option<&str>, + auth_bearer: Option<&Option>, + auth_user: Option<&str>, +) -> Result { + if let Some(user) = auth_basic { + if auth_user.is_some() { + return Err(Error::Usage( + "--auth-user is only valid with --auth-bearer".to_owned(), + )); + } + let password = secret::resolve(auth_password, "VANDELAY_PASSWORD", "password")?; + return Ok(ManageSieveAuth::Basic { + user: user.to_owned(), + password, + }); + } + if auth_password.is_some() { + return Err(Error::Usage( + "--auth-password is only valid together with --auth-basic".to_owned(), + )); + } + let bearer = auth_bearer.ok_or_else(|| { + Error::Usage("exactly one of --auth-basic / --auth-bearer is required".to_owned()) + })?; + let user = auth_user + .ok_or_else(|| Error::Usage("--auth-user is required with --auth-bearer".to_owned()))? + .to_owned(); + let token = secret::resolve(bearer.as_deref(), "VANDELAY_TOKEN", "bearer token")?; + Ok(ManageSieveAuth::Bearer { user, token }) +} + +#[derive(Args)] +pub struct MaildirImportArgs { + #[arg( + value_name = "MAILDIR", + help = "Path to the Maildir++ root (a directory containing cur/ new/ tmp/)" + )] + maildir: PathBuf, + + #[arg( + value_name = "ARCHIVE", + help = "Local SQLite archive (created if absent)" + )] + archive: PathBuf, + + #[arg( + long, + value_name = "REGEX", + action = ArgAction::Append, + help = "Folder include filter against canonical name (repeatable)" + )] + include: Vec, + + #[arg( + long, + value_name = "REGEX", + action = ArgAction::Append, + help = "Folder exclude filter against canonical name (repeatable)" + )] + exclude: Vec, + + #[arg( + long, + value_name = "NAME", + action = ArgAction::Append, + help = "Exact include; mutually exclusive with --include/--exclude" + )] + folder: Vec, + + #[arg(long, help = "Disable the name-heuristic role assignment")] + noautomap: bool, + + #[arg(long, help = "Import T-flagged messages with the $deleted keyword")] + include_deleted: bool, + + #[arg( + long, + help = "Permit importing when the archive records a different maildir path" + )] + allow_source_change: bool, + + #[command(flatten)] + global: GlobalArgs, +} + +fn resolve_maildir_import(args: MaildirImportArgs) -> Result { + let common = common_config(&args.global, args.archive); + if !args.folder.is_empty() && (!args.include.is_empty() || !args.exclude.is_empty()) { + return Err(Error::Usage( + "--folder is mutually exclusive with --include/--exclude".to_owned(), + )); + } + let include = compile_regexes(&args.include, "--include")?; + let exclude = compile_regexes(&args.exclude, "--exclude")?; + Ok(Action::ImportMaildir( + common, + MaildirImportConfig { + maildir: args.maildir, + include, + exclude, + folder: args.folder, + automap: !args.noautomap, + include_deleted: args.include_deleted, + allow_source_change: args.allow_source_change, + }, + )) +} + +#[derive(Args)] +pub struct TakeoutImportArgs { + #[arg( + value_name = "PATH", + help = "Directory tree to scan recursively for .mbox / .ics / .vcf files" + )] + path: PathBuf, + + #[arg( + value_name = "ARCHIVE", + help = "Local SQLite archive (created if absent)" + )] + archive: PathBuf, + + #[arg(long, help = "Disable the system-label role assignment")] + noautomap: bool, + + #[arg( + long, + help = "Permit importing when the archive records a takeout source at a different path" + )] + allow_source_change: bool, + + #[command(flatten)] + global: GlobalArgs, +} + +fn resolve_takeout_import(args: TakeoutImportArgs) -> Result { + let common = common_config(&args.global, args.archive); + Ok(Action::ImportTakeout( + common, + TakeoutImportConfig { + takeout_root: args.path, + allow_source_change: args.allow_source_change, + automap: !args.noautomap, + }, + )) +} + +fn resolve_dav_import(args: DavImportArgs, kind: DavKindArg) -> Result { + let common = common_config(&args.global, args.archive); + let auth = resolve_dav_auth( + args.auth_basic.as_deref(), + args.auth_password.as_deref(), + args.auth_bearer.as_ref(), + )?; + let dav_connections = args.dav_connections.clamp(1, 8).min(common.threads.max(1)); + let multiget_batch = args.multiget_batch.max(1); + Ok(Action::ImportDav( + common, + DavImportConfig { + kind, + url: args.url, + auth, + allow_cleartext: args.allow_cleartext, + dav_connections, + multiget_batch, + allow_source_change: args.allow_source_change, + }, + )) +} + +fn resolve_dav_auth( + auth_basic: Option<&str>, + auth_password: Option<&str>, + auth_bearer: Option<&Option>, +) -> Result { + if let Some(user) = auth_basic { + let password = secret::resolve(auth_password, "VANDELAY_PASSWORD", "password")?; + return Ok(DavAuth::Basic { + user: user.to_owned(), + password, + }); + } + if auth_password.is_some() { + return Err(Error::Usage( + "--auth-password is only valid together with --auth-basic".to_owned(), + )); + } + let bearer = auth_bearer.ok_or_else(|| { + Error::Usage("exactly one of --auth-basic / --auth-bearer is required".to_owned()) + })?; + let token = secret::resolve(bearer.as_deref(), "VANDELAY_TOKEN", "bearer token")?; + Ok(DavAuth::Bearer { token }) +} + +fn resolve_jmap_import(args: JmapImportArgs) -> Result { + let common = common_config(&args.global, args.archive); + let auth = resolve_auth( + args.auth_basic.as_deref(), + args.auth_password.as_deref(), + args.auth_bearer.as_ref(), + )?; + let account = resolve_account(args.account_id, args.account_name)?; + let objects = args.objects.as_deref().map(parse_object_list).transpose()?; + + Ok(Action::Import( + common, + ImportConfig { + connect: ConnectConfig { + url: args.url, + auth, + account, + }, + objects, + allow_source_change: args.allow_source_change, + }, + )) +} + +fn resolve_imap_import(args: ImapImportArgs) -> Result { + let common = common_config(&args.global, args.archive); + let auth = resolve_imap_auth( + args.auth_basic.as_deref(), + args.auth_password.as_deref(), + args.auth_bearer.as_ref(), + args.auth_user.as_deref(), + )?; + if !args.folder.is_empty() && (!args.include.is_empty() || !args.exclude.is_empty()) { + return Err(Error::Usage( + "--folder is mutually exclusive with --include/--exclude".to_owned(), + )); + } + let include = compile_regexes(&args.include, "--include")?; + let exclude = compile_regexes(&args.exclude, "--exclude")?; + let imap_connections = args.imap_connections.unwrap_or(common.threads).clamp(1, 8); + + Ok(Action::ImportImap( + common, + ImapImportConfig { + url: args.url, + auth, + allow_cleartext: args.allow_cleartext, + compress: args.compress, + include, + exclude, + exclude_special: args.exclude_special, + folder: args.folder, + subscribed_only: args.subscribed_only, + automap: !args.noautomap, + include_deleted: args.include_deleted, + fetch_batch: args.fetch_batch, + imap_connections, + allow_source_change: args.allow_source_change, + }, + )) +} + +fn compile_regexes(raw: &[String], flag: &str) -> Result, Error> { + let mut out = Vec::with_capacity(raw.len()); + for r in raw { + let re = + Regex::new(r).map_err(|e| Error::Usage(format!("{flag} {r:?}: invalid regex: {e}")))?; + out.push(re); + } + Ok(out) +} + +fn resolve_imap_auth( + auth_basic: Option<&str>, + auth_password: Option<&str>, + auth_bearer: Option<&Option>, + auth_user: Option<&str>, +) -> Result { + if let Some(user) = auth_basic { + if auth_user.is_some() { + return Err(Error::Usage( + "--auth-user is only valid with --auth-bearer".to_owned(), + )); + } + let password = secret::resolve(auth_password, "VANDELAY_PASSWORD", "password")?; + return Ok(ImapAuth::Basic { + user: user.to_owned(), + password, + }); + } + if auth_password.is_some() { + return Err(Error::Usage( + "--auth-password is only valid together with --auth-basic".to_owned(), + )); + } + let bearer = auth_bearer.ok_or_else(|| { + Error::Usage("exactly one of --auth-basic / --auth-bearer is required".to_owned()) + })?; + let user = auth_user + .ok_or_else(|| Error::Usage("--auth-user is required with --auth-bearer".to_owned()))? + .to_owned(); + let token = secret::resolve(bearer.as_deref(), "VANDELAY_TOKEN", "bearer token")?; + Ok(ImapAuth::Bearer { user, token }) +} + +fn resolve_export(args: ExportArgs) -> Result { + let common = common_config(&args.global, args.archive); + let auth = resolve_auth( + args.auth_basic.as_deref(), + args.auth_password.as_deref(), + args.auth_bearer.as_ref(), + )?; + let account = resolve_account(args.account_id, args.account_name)?; + let objects = args.objects.as_deref().map(parse_object_list).transpose()?; + + Ok(Action::Export( + common, + ExportConfig { + connect: ConnectConfig { + url: args.url, + auth, + account, + }, + objects, + prune: args.prune, + yes: args.yes, + }, + )) +} + +fn resolve_inspect(args: InspectArgs) -> Result { + let target = match args.target.as_deref() { + Some(token) => Some(ObjectType::parse(token)?), + None => None, + }; + if let Some(limit) = args.limit + && limit == 0 + { + return Err(Error::Usage("--limit must be greater than zero".to_owned())); + } + Ok(Action::Inspect(InspectConfig { + archive: args.archive, + target, + limit: args.limit, + offset: args.offset, + })) +} + +#[derive(Args)] +#[command(group(clap::ArgGroup::new("ews_auth").required(true).args([ + "auth_basic", "auth_bearer", +])))] +pub struct ExchangeEwsImportArgs { + #[arg(long, value_name = "URL", help = "Fully-qualified EWS endpoint URL")] + url: Option, + + #[arg( + long, + value_name = "EMAIL", + help = "Target mailbox SMTP address (required for autodiscover and app-only OAuth)" + )] + mailbox: Option, + + #[arg( + long, + value_name = "KIND", + default_value = "primary", + help = "Mailbox surface: primary | archive | public-folders" + )] + mailbox_kind: String, + + #[arg(long, value_name = "USER", help = "Basic auth user (on-prem only)")] + auth_basic: Option, + + #[arg( + long, + value_name = "PASS", + help = "Basic password (prefer $VANDELAY_PASSWORD or prompt)" + )] + auth_password: Option, + + #[arg( + long, + value_name = "TOKEN", + num_args = 0..=1, + help = "Bearer token (value optional: $VANDELAY_TOKEN or device-code flow)" + )] + auth_bearer: Option>, + + #[arg( + long, + value_name = "TENANT", + help = "Entra tenant id or .onmicrosoft.com short name" + )] + ews_tenant: Option, + + #[arg(long, value_name = "ID", help = "Entra app (client) id")] + ews_client_id: Option, + + #[arg( + long, + value_name = "SECRET", + help = "OAuth client secret (app-only flow; prefer $VANDELAY_EWS_CLIENT_SECRET)" + )] + ews_client_secret: Option, + + #[arg(long, help = "Use the OAuth device-code interactive flow")] + ews_device_code: bool, + + #[arg( + long, + value_name = "N", + default_value_t = 4, + help = "Parallel EWS worker connections (hard cap 8)" + )] + ews_connections: usize, + + #[arg( + long, + value_name = "N", + default_value_t = 20, + help = "ItemIds per GetItem batch" + )] + ews_getitem_batch: usize, + + #[arg( + long, + value_name = "N", + default_value_t = 5, + help = "AttachmentIds per GetAttachment batch" + )] + ews_attachment_batch: usize, + + #[arg(long, help = "Force FindItem even when SyncFolderItems would work")] + ews_no_syncfolderitems: bool, + + #[arg( + long, + help = "Permit importing when the archive records a different EWS source" + )] + allow_source_change: bool, + + #[command(flatten)] + global: GlobalArgs, + + #[arg( + value_name = "ARCHIVE", + help = "Local SQLite archive (created if absent)" + )] + archive: PathBuf, +} + +fn resolve_exchange_ews_import(args: ExchangeEwsImportArgs) -> Result { + let archive = args.archive.clone(); + let common = common_config(&args.global, archive); + let mailbox_kind = match args.mailbox_kind.as_str() { + "primary" => MailboxKind::Primary, + "archive" => MailboxKind::Archive, + "public-folders" => MailboxKind::PublicFolders, + other => { + return Err(Error::Usage(format!( + "--mailbox-kind must be primary | archive | public-folders, got {other:?}" + ))); + } + }; + let auth = resolve_exchange_ews_auth(&args)?; + let ews_connections = args.ews_connections.clamp(1, 8).min(common.threads.max(1)); + let config = EwsImportConfig { + url: args.url, + mailbox: args.mailbox, + mailbox_kind, + auth, + ews_connections, + getitem_batch: args.ews_getitem_batch.max(1), + attachment_batch: args.ews_attachment_batch.max(1), + use_syncfolderitems: !args.ews_no_syncfolderitems, + allow_source_change: args.allow_source_change, + }; + Ok(Action::ImportExchangeEws(common, config)) +} + +fn resolve_exchange_ews_auth(args: &ExchangeEwsImportArgs) -> Result { + if let Some(user) = args.auth_basic.as_deref() { + let password = secret::resolve( + args.auth_password.as_deref(), + "VANDELAY_PASSWORD", + "password", + )?; + return Ok(EwsAuth::Basic { + user: user.to_owned(), + password, + }); + } + if args.auth_password.is_some() { + return Err(Error::Usage( + "--auth-password is only valid with --auth-basic".to_owned(), + )); + } + let Some(bearer) = args.auth_bearer.as_ref() else { + return Err(Error::Usage( + "exactly one of --auth-basic / --auth-bearer is required".to_owned(), + )); + }; + if let Some(token) = bearer.clone() { + return Ok(EwsAuth::Bearer { token }); + } + let tenant = args + .ews_tenant + .clone() + .unwrap_or_else(|| "common".to_owned()); + let client_id = args.ews_client_id.clone().ok_or_else(|| { + Error::Usage( + "--ews-client-id is required when --auth-bearer is used without an inline token" + .to_owned(), + ) + })?; + if args.ews_device_code { + return Ok(EwsAuth::OAuth(OAuthFlow::DeviceCode { tenant, client_id })); + } + if let Some(secret) = args.ews_client_secret.clone() { + let resolved = secret::resolve( + Some(secret.as_str()), + "VANDELAY_EWS_CLIENT_SECRET", + "client secret", + )?; + return Ok(EwsAuth::OAuth(OAuthFlow::ClientCredentials { + tenant, + client_id, + client_secret: resolved, + })); + } + if let Ok(env_secret) = std::env::var("VANDELAY_EWS_CLIENT_SECRET") + && !env_secret.is_empty() + { + return Ok(EwsAuth::OAuth(OAuthFlow::ClientCredentials { + tenant, + client_id, + client_secret: env_secret, + })); + } + let token = secret::resolve(None, "VANDELAY_TOKEN", "bearer token")?; + Ok(EwsAuth::Bearer { token }) +} + +#[derive(Args)] +pub struct ExchangeGraphImportArgs { + #[arg( + long, + value_name = "UUID", + help = "Entra app (client) id (required unless --access-token is given)" + )] + client_id: Option, + + #[arg( + long, + value_name = "ID", + default_value = "common", + help = "Entra authority tenant" + )] + tenant: String, + + #[arg( + long, + value_name = "UPN|UUID", + help = "Target a different mailbox the signed-in account can read" + )] + user: Option, + + #[arg( + long, + value_name = "KIND", + default_value = "primary", + value_parser = parse_graph_mailbox_kind, + help = "Mailbox surface: primary | archive (public-folders is rejected: use exchange-ews)" + )] + mailbox_kind: GraphMailboxKind, + + #[arg( + long, + value_name = "TOKEN", + num_args = 0..=1, + help = "Pre-acquired bearer token (skip device-code flow). Resolves: flag value -> $VANDELAY_GRAPH_TOKEN -> prompt." + )] + access_token: Option>, + + #[arg( + long, + value_name = "LIST", + help = "Comma-separated surface list (mail,calendar,contacts; default all)" + )] + objects: Option, + + #[arg( + long, + value_name = "FMT", + default_value = "text", + value_parser = parse_event_body_format, + help = "outlook.body-content-type for event GETs: text | html" + )] + event_body_format: EventBodyFormat, + + #[arg( + long, + value_name = "N", + default_value_t = 4, + help = "Parallel Graph worker connections (hard cap 16)" + )] + graph_connections: usize, + + #[arg( + long, + value_name = "N", + default_value_t = 100, + help = "Per-page size ($top query parameter, range [1, 1000])" + )] + top: usize, + + #[arg( + long, + value_name = "URL", + default_value = "https://graph.microsoft.com/v1.0", + help = "Microsoft Graph API base URL (override for national clouds or tests)", + hide = true + )] + api_base: String, + + #[arg( + long, + help = "Permit importing when the archive records a different exchange_graph source" + )] + allow_source_change: bool, + + #[command(flatten)] + global: GlobalArgs, + + #[arg( + value_name = "ARCHIVE", + help = "Local SQLite archive (created if absent)" + )] + archive: PathBuf, +} + +fn parse_graph_mailbox_kind(s: &str) -> Result { + GraphMailboxKind::parse(s).map_err(|e| e.to_string()) +} + +fn parse_event_body_format(s: &str) -> Result { + EventBodyFormat::parse(s).map_err(|e| e.to_string()) +} + +fn resolve_exchange_graph_import(args: ExchangeGraphImportArgs) -> Result { + let common = common_config(&args.global, args.archive.clone()); + let mailbox_kind = args.mailbox_kind; + let event_body_format = args.event_body_format; + let auth = resolve_graph_auth( + args.access_token.as_ref(), + args.client_id.as_deref(), + &args.tenant, + )?; + let objects = args + .objects + .as_deref() + .map(crate::types::parse_object_list) + .transpose()?; + let top = args.top.clamp(1, 1000); + let graph_connections = args + .graph_connections + .clamp(1, 16) + .min(common.threads.max(1)); + Ok(Action::ImportExchangeGraph( + common, + GraphImportConfig { + auth, + api_base: args.api_base, + user_target: args.user, + mailbox_kind, + objects, + event_body_format, + graph_connections, + top, + allow_source_change: args.allow_source_change, + }, + )) +} + +fn resolve_graph_auth( + access_token: Option<&Option>, + client_id: Option<&str>, + tenant: &str, +) -> Result { + if let Some(slot) = access_token { + let token = secret::resolve(slot.as_deref(), "VANDELAY_GRAPH_TOKEN", "bearer token")?; + return Ok(GraphAuth::PreAcquired { token }); + } + let client_id = client_id + .ok_or_else(|| { + Error::Usage( + "--client-id is required for exchange-graph (unless --access-token is given)" + .to_owned(), + ) + })? + .to_owned(); + let authority = crate::exchange_graph::oauth::default_authority(tenant); + Ok(GraphAuth::DeviceCode { + authority, + client_id, + }) +} + +fn common_config(global: &GlobalArgs, archive: PathBuf) -> CommonConfig { + let threads = global + .threads + .filter(|n| *n > 0) + .unwrap_or_else(num_cpus::get) + .max(1); + CommonConfig { + archive, + threads, + dry_run: global.dry_run, + max_retries: global.max_retries, + allow_invalid_certs: global.allow_invalid_certs, + logger: Logger::from_flags(global.quiet, global.verbose), + } +} + +fn resolve_auth( + auth_basic: Option<&str>, + auth_password: Option<&str>, + auth_bearer: Option<&Option>, +) -> Result { + if let Some(user) = auth_basic { + let password = secret::resolve(auth_password, "VANDELAY_PASSWORD", "password")?; + return Ok(Auth::Basic { + user: user.to_owned(), + password, + }); + } + if auth_password.is_some() { + return Err(Error::Usage( + "--auth-password is only valid together with --auth-basic".to_owned(), + )); + } + let bearer = auth_bearer.ok_or_else(|| { + Error::Usage("exactly one of --auth-basic / --auth-bearer is required".to_owned()) + })?; + let token = secret::resolve(bearer.as_deref(), "VANDELAY_TOKEN", "bearer token")?; + Ok(Auth::Bearer { token }) +} + +fn resolve_account( + account_id: Option, + account_name: Option, +) -> Result { + match (account_id, account_name) { + (Some(id), None) => Ok(AccountSelector::Id(id)), + (None, Some(name)) => Ok(AccountSelector::Name(name)), + _ => Err(Error::Usage( + "exactly one of --account-id / --account-name is required".to_owned(), + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::env; + + fn lock() -> std::sync::MutexGuard<'static, ()> { + static M: std::sync::Mutex<()> = std::sync::Mutex::new(()); + M.lock().unwrap_or_else(|e| e.into_inner()) + } + + #[test] + fn auth_password_without_basic_is_usage_error() { + let _g = lock(); + unsafe { + env::remove_var("VANDELAY_TOKEN"); + env::remove_var("VANDELAY_PASSWORD"); + } + let bearer: Option = Some("t".to_owned()); + let r = resolve_auth(None, Some("hunter2"), Some(&bearer)); + match r { + Err(Error::Usage(m)) => { + assert!(m.contains("auth-password"), "msg was: {m}"); + } + other => panic!("expected Usage error, got {other:?}"), + } + } + + #[test] + fn auth_basic_with_password_resolves() { + let _g = lock(); + unsafe { + env::remove_var("VANDELAY_TOKEN"); + env::remove_var("VANDELAY_PASSWORD"); + } + let auth = resolve_auth(Some("alice"), Some("pw"), None).unwrap(); + match auth { + Auth::Basic { user, password } => { + assert_eq!(user, "alice"); + assert_eq!(password, "pw"); + } + _ => panic!("expected Basic"), + } + } + + #[test] + fn auth_bearer_inline_token_resolves() { + let _g = lock(); + unsafe { + env::remove_var("VANDELAY_TOKEN"); + env::remove_var("VANDELAY_PASSWORD"); + } + let bearer: Option = Some("t".to_owned()); + let auth = resolve_auth(None, None, Some(&bearer)).unwrap(); + assert!(matches!(auth, Auth::Bearer { token } if token == "t")); + } +} diff --git a/src/dav/client.rs b/src/dav/client.rs new file mode 100644 index 0000000..290cc68 --- /dev/null +++ b/src/dav/client.rs @@ -0,0 +1,898 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::{BufReader, Read}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::time::Duration; + +use ureq::Agent; +use ureq::Body; +use ureq::config::{Config, RedirectAuthHeaders}; +use ureq::http::{Method, Request, Response}; +use ureq::tls::{RootCerts, TlsConfig}; + +use crate::dav::parse::{ControlStrippingReader, DavResponse, parse_multistatus}; +use crate::dav::retry::{DavOutcome, classify}; +use crate::jmap::error::JmapError; +use crate::jmap::http::{Auth, RetryPolicy, retry_after_header}; +use crate::jmap::retry::{self, RateLimitState}; +use crate::logging::{LEVEL_BODIES, LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; + +const MAX_BODY: u64 = 512 * 1024 * 1024; +const LONG_RETRY_THRESHOLD: Duration = Duration::from_secs(10); +const MAX_DAV_REDIRECTS: u32 = 5; + +const ACCEPT_DAV: &str = "application/xml, text/xml;q=0.9, */*;q=0.5"; +const ACCEPT_BINARY: &str = "*/*"; + +#[derive(Debug)] +pub struct DavHttpResponse { + pub status: u16, + pub body: Vec, + pub etag: Option, + pub content_type: Option, + pub last_modified: Option, +} + +#[derive(Debug)] +pub struct MultiStatus { + pub status: u16, + pub responses: Vec, + pub final_url: String, +} + +struct Inner { + agent: Agent, + auth: Auth, + retry: RetryPolicy, + rate_limit: RateLimitState, + log_level: AtomicU8, + retries_total: AtomicU64, + retry_after_sleeps: AtomicU64, + user_agent: String, +} + +#[derive(Clone)] +pub struct DavClient { + inner: Arc, +} + +impl DavClient { + pub fn new(auth: Auth, retry: RetryPolicy, allow_invalid_certs: bool) -> Self { + let config: Config = Config::builder() + .http_status_as_error(false) + .allow_non_standard_methods(true) + .max_redirects(0) + .redirect_auth_headers(RedirectAuthHeaders::SameHost) + .tls_config( + TlsConfig::builder() + .root_certs(RootCerts::PlatformVerifier) + .disable_verification(allow_invalid_certs) + .build(), + ) + .build(); + DavClient { + inner: Arc::new(Inner { + agent: config.new_agent(), + auth, + retry, + rate_limit: RateLimitState::new(), + log_level: AtomicU8::new(LEVEL_DEFAULT), + retries_total: AtomicU64::new(0), + retry_after_sleeps: AtomicU64::new(0), + user_agent: format!("vandelay/{}", env!("CARGO_PKG_VERSION")), + }), + } + } + + pub fn set_logger(&self, logger: Logger) { + self.inner + .log_level + .store(logger.level(), Ordering::Relaxed); + } + + pub fn logger(&self) -> Logger { + Logger::new(self.inner.log_level.load(Ordering::Relaxed)) + } + + pub fn rate_limit(&self) -> &RateLimitState { + &self.inner.rate_limit + } + + pub fn retries_observed(&self) -> u64 { + self.inner.retries_total.load(Ordering::Relaxed) + } + + pub fn retry_after_sleeps(&self) -> u64 { + self.inner.retry_after_sleeps.load(Ordering::Relaxed) + } + + pub fn propfind(&self, url: &str, depth: u8, body: &str) -> Result { + self.execute( + "PROPFIND", + url, + Some(body.as_bytes()), + Some("application/xml; charset=utf-8"), + Some(depth), + ACCEPT_DAV, + ) + } + + pub fn report(&self, url: &str, depth: u8, body: &str) -> Result { + self.execute( + "REPORT", + url, + Some(body.as_bytes()), + Some("application/xml; charset=utf-8"), + Some(depth), + ACCEPT_DAV, + ) + } + + pub fn get(&self, url: &str) -> Result { + self.execute("GET", url, None, None, None, ACCEPT_BINARY) + } + + pub fn propfind_responses( + &self, + url: &str, + depth: u8, + body: &str, + base_url: &str, + ) -> Result { + self.stream_multistatus("PROPFIND", url, Some(body.as_bytes()), depth, base_url) + } + + pub fn report_responses( + &self, + url: &str, + depth: u8, + body: &str, + base_url: &str, + ) -> Result { + self.stream_multistatus("REPORT", url, Some(body.as_bytes()), depth, base_url) + } + + pub fn get_stream(&self, url: &str) -> Result { + let logger = self.logger(); + let policy = self.inner.retry; + let mut attempt: u32 = 0; + let original_url = url.to_owned(); + let mut current_url = url.to_owned(); + let mut redirects: u32 = 0; + let mut include_auth = true; + loop { + self.inner.rate_limit.cooldown().wait(); + let outcome = self.one_attempt_stream(¤t_url, include_auth); + match outcome { + AttemptStream::Ok { + status, + body_reader, + etag, + content_type, + last_modified, + .. + } if (200..300).contains(&status) => { + self.inner.rate_limit.on_success(); + return Ok(DavStream { + status, + body_reader, + etag, + content_type, + last_modified, + }); + } + AttemptStream::Ok { status, .. } if status == 404 || status == 410 => { + self.inner.rate_limit.on_success(); + return Ok(DavStream { + status, + body_reader: Box::new(std::io::empty()), + etag: None, + content_type: None, + last_modified: None, + }); + } + AttemptStream::Ok { + status, + location: Some(loc), + .. + } if is_redirect_status(status) => { + if redirects >= MAX_DAV_REDIRECTS { + return Err(JmapError::Connect(format!( + "too many redirects ({redirects}) starting from {url}" + ))); + } + redirects += 1; + let next = resolve_redirect(¤t_url, &loc)?; + if include_auth && !same_host(&original_url, &next) { + include_auth = false; + if logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "DAV redirect leaves origin host {original_url}; dropping Authorization for {next}" + ); + } + } + if logger.enabled(LEVEL_PROGRESS) { + eprintln!("DAV {status}: {current_url} -> {next}"); + } + current_url = next; + } + AttemptStream::Ok { + status, + body_reader, + retry_after, + .. + } => { + let mut body_bytes = Vec::new(); + let mut limited = body_reader.take(64 * 1024); + let _ = limited.read_to_end(&mut body_bytes); + self.handle_status( + &logger, + status, + &body_bytes, + retry_after, + &mut attempt, + &policy, + )?; + } + AttemptStream::Transport(err) => { + self.handle_transport(&logger, err, &mut attempt, &policy)?; + } + } + } + } + + fn execute( + &self, + method: &str, + url: &str, + body: Option<&[u8]>, + content_type: Option<&str>, + depth: Option, + accept: &str, + ) -> Result { + let logger = self.logger(); + let policy = self.inner.retry; + let mut attempt: u32 = 0; + let original_url = url.to_owned(); + let mut current_url = url.to_owned(); + let mut redirects: u32 = 0; + let mut include_auth = true; + loop { + self.inner.rate_limit.cooldown().wait(); + let outcome = self.one_attempt(WireRequest { + method, + url: ¤t_url, + body, + content_type, + depth, + accept, + include_auth, + }); + match outcome { + Attempt::Ok { + status, + body, + etag, + content_type, + last_modified, + .. + } if (200..300).contains(&status) => { + self.inner.rate_limit.on_success(); + return Ok(DavHttpResponse { + status, + body, + etag, + content_type, + last_modified, + }); + } + Attempt::Ok { + status, + location: Some(loc), + .. + } if is_redirect_status(status) => { + if redirects >= MAX_DAV_REDIRECTS { + return Err(JmapError::Connect(format!( + "too many redirects ({redirects}) starting from {url}" + ))); + } + redirects += 1; + let next = resolve_redirect(¤t_url, &loc)?; + if include_auth && !same_host(&original_url, &next) { + include_auth = false; + if logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "DAV redirect leaves origin host {original_url}; dropping Authorization for {next}" + ); + } + } + if logger.enabled(LEVEL_PROGRESS) { + eprintln!("DAV {status}: {current_url} -> {next}"); + } + current_url = next; + } + Attempt::Ok { status, body, .. } if is_redirect_status(status) => { + return Err(JmapError::Connect(format!( + "{method} {current_url} returned {status} without Location: {}", + truncate(&body) + ))); + } + Attempt::Ok { + status, + body, + retry_after, + etag: _, + content_type: _, + last_modified: _, + location: _, + } => { + let disposition = classify(status, &body); + match disposition { + DavOutcome::Vanished => { + return Ok(DavHttpResponse { + status, + body, + etag: None, + content_type: None, + last_modified: None, + }); + } + DavOutcome::Auth => { + return Err(JmapError::Auth(format!( + "server returned {status}: {}", + truncate(&body) + ))); + } + DavOutcome::Fatal => { + return Err(JmapError::HttpStatus { + status, + body: truncate(&body), + }); + } + DavOutcome::Retryable => { + attempt += 1; + self.inner.retries_total.fetch_add(1, Ordering::Relaxed); + if attempt > policy.max_retries { + return Err(JmapError::RetriesExhausted(format!( + "{method} {current_url} kept returning {status}" + ))); + } + if retry_after.is_some() { + self.inner + .retry_after_sleeps + .fetch_add(1, Ordering::Relaxed); + } + let delay = self.inner.rate_limit.on_throttle(&policy, retry_after); + if delay >= LONG_RETRY_THRESHOLD { + logger.warn(&format!( + "DAV {method} rate-limited (http {status}); waiting {}s before retry {}/{}", + delay.as_secs(), + attempt, + policy.max_retries, + )); + } + if logger.enabled(LEVEL_BODIES) { + eprintln!( + "retry {}/{} {method} {current_url} after {:?} (http {status}) body={}", + attempt, + policy.max_retries, + delay, + truncate(&body), + ); + } else if logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "retry {}/{} (http {status})", + attempt, policy.max_retries + ); + } + std::thread::sleep(delay); + } + DavOutcome::Success => unreachable!(), + } + } + Attempt::Transport(err) => { + self.handle_transport(&logger, err, &mut attempt, &policy)?; + } + } + } + } + + fn stream_multistatus( + &self, + method: &str, + url: &str, + body: Option<&[u8]>, + depth: u8, + base_url: &str, + ) -> Result { + let logger = self.logger(); + let policy = self.inner.retry; + let mut attempt: u32 = 0; + let original_url = url.to_owned(); + let mut current_url = url.to_owned(); + let mut redirects: u32 = 0; + let mut include_auth = true; + loop { + self.inner.rate_limit.cooldown().wait(); + let send = self.build_and_send(WireRequest { + method, + url: ¤t_url, + body, + content_type: Some("application/xml; charset=utf-8"), + depth: Some(depth), + accept: ACCEPT_DAV, + include_auth, + }); + match send { + Ok(mut resp) => { + let status = resp.status().as_u16(); + let retry_after = resp + .headers() + .get("retry-after") + .and_then(|v| v.to_str().ok()) + .and_then(retry_after_header); + let location = resp + .headers() + .get("location") + .and_then(|v| v.to_str().ok()) + .map(str::to_owned); + if is_redirect_status(status) { + let Some(loc) = location else { + return Err(JmapError::Connect(format!( + "{method} {current_url} returned {status} without Location" + ))); + }; + if redirects >= MAX_DAV_REDIRECTS { + return Err(JmapError::Connect(format!( + "too many redirects ({redirects}) starting from {url}" + ))); + } + redirects += 1; + let next = resolve_redirect(¤t_url, &loc)?; + if include_auth && !same_host(&original_url, &next) { + include_auth = false; + if logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "DAV redirect leaves origin host {original_url}; dropping Authorization for {next}" + ); + } + } + if logger.enabled(LEVEL_PROGRESS) { + eprintln!("DAV {status}: {current_url} -> {next}"); + } + current_url = next; + continue; + } + if (200..300).contains(&status) { + self.inner.rate_limit.on_success(); + let parse_base = if current_url == url { + base_url + } else { + ¤t_url + }; + let raw = resp.into_body().into_reader(); + let stripped = ControlStrippingReader::new(raw); + let reader = BufReader::new(stripped); + let responses = parse_multistatus(reader, parse_base) + .map_err(|e| JmapError::Malformed(format!("multistatus parse: {e}")))?; + return Ok(MultiStatus { + status, + responses, + final_url: current_url.clone(), + }); + } + let body_bytes = + match resp.body_mut().with_config().limit(64 * 1024).read_to_vec() { + Ok(b) => b, + Err(e) => { + return Err(JmapError::Transport(format!( + "reading error body: {e}" + ))); + } + }; + let disposition = classify(status, &body_bytes); + match disposition { + DavOutcome::Vanished => { + return Ok(MultiStatus { + status, + responses: Vec::new(), + final_url: current_url.clone(), + }); + } + DavOutcome::Auth => { + return Err(JmapError::Auth(format!( + "server returned {status}: {}", + truncate(&body_bytes) + ))); + } + DavOutcome::Fatal => { + return Err(JmapError::HttpStatus { + status, + body: truncate(&body_bytes), + }); + } + DavOutcome::Retryable => { + attempt += 1; + self.inner.retries_total.fetch_add(1, Ordering::Relaxed); + if attempt > policy.max_retries { + return Err(JmapError::RetriesExhausted(format!( + "{method} {url} kept returning {status}" + ))); + } + if retry_after.is_some() { + self.inner + .retry_after_sleeps + .fetch_add(1, Ordering::Relaxed); + } + let delay = self.inner.rate_limit.on_throttle(&policy, retry_after); + if delay >= LONG_RETRY_THRESHOLD { + logger.warn(&format!( + "DAV {method} rate-limited (http {status}); waiting {}s before retry {}/{}", + delay.as_secs(), + attempt, + policy.max_retries, + )); + } + std::thread::sleep(delay); + } + DavOutcome::Success => unreachable!(), + } + } + Err(e) => { + let err = map_ureq_error(e); + self.handle_transport(&logger, err, &mut attempt, &policy)?; + } + } + } + } + + fn handle_status( + &self, + logger: &Logger, + status: u16, + body: &[u8], + retry_after: Option, + attempt: &mut u32, + policy: &RetryPolicy, + ) -> Result<(), JmapError> { + let disposition = classify(status, body); + match disposition { + DavOutcome::Vanished => Err(JmapError::HttpStatus { + status, + body: truncate(body), + }), + DavOutcome::Auth => Err(JmapError::Auth(format!("server returned {status}"))), + DavOutcome::Fatal => Err(JmapError::HttpStatus { + status, + body: truncate(body), + }), + DavOutcome::Retryable => { + *attempt += 1; + self.inner.retries_total.fetch_add(1, Ordering::Relaxed); + if *attempt > policy.max_retries { + return Err(JmapError::RetriesExhausted(format!( + "GET stream kept returning {status}" + ))); + } + if retry_after.is_some() { + self.inner + .retry_after_sleeps + .fetch_add(1, Ordering::Relaxed); + } + let delay = self.inner.rate_limit.on_throttle(policy, retry_after); + if delay >= LONG_RETRY_THRESHOLD { + logger.warn(&format!( + "DAV stream rate-limited (http {status}); waiting {}s", + delay.as_secs() + )); + } + std::thread::sleep(delay); + Ok(()) + } + DavOutcome::Success => Ok(()), + } + } + + fn handle_transport( + &self, + logger: &Logger, + err: JmapError, + attempt: &mut u32, + policy: &RetryPolicy, + ) -> Result<(), JmapError> { + match transport_disposition(&err) { + retry::Disposition::Fatal => Err(err), + retry::Disposition::Retryable => { + *attempt += 1; + self.inner.retries_total.fetch_add(1, Ordering::Relaxed); + if *attempt > policy.max_retries { + return Err(JmapError::RetriesExhausted(format!("{err}"))); + } + let delay = retry::backoff_delay(policy, *attempt); + if delay >= LONG_RETRY_THRESHOLD { + logger.warn(&format!( + "transient transport failure ({err}); waiting {}s before retry {}/{}", + delay.as_secs(), + attempt, + policy.max_retries, + )); + } + std::thread::sleep(delay); + Ok(()) + } + } + } + + fn one_attempt(&self, req: WireRequest<'_>) -> Attempt { + match self.build_and_send(req) { + Ok(mut resp) => { + let status = resp.status().as_u16(); + let etag = header_str(&resp, "etag"); + let content_type_header = header_str(&resp, "content-type"); + let last_modified = header_str(&resp, "last-modified"); + let location = header_str(&resp, "location"); + let retry_after = resp + .headers() + .get("retry-after") + .and_then(|v| v.to_str().ok()) + .and_then(retry_after_header); + if is_redirect_status(status) && location.is_some() { + return Attempt::Ok { + status, + body: Vec::new(), + retry_after, + etag, + content_type: content_type_header, + last_modified, + location, + }; + } + let body_limit = if (200..300).contains(&status) { + MAX_BODY + } else { + 64 * 1024 + }; + match resp + .body_mut() + .with_config() + .limit(body_limit) + .read_to_vec() + { + Ok(bytes) => Attempt::Ok { + status, + body: bytes, + retry_after, + etag, + content_type: content_type_header, + last_modified, + location, + }, + Err(e) => Attempt::Transport(JmapError::Transport(format!( + "reading response body: {e}" + ))), + } + } + Err(e) => Attempt::Transport(map_ureq_error(e)), + } + } + + fn one_attempt_stream(&self, url: &str, include_auth: bool) -> AttemptStream { + let req = WireRequest { + method: "GET", + url, + body: None, + content_type: None, + depth: None, + accept: ACCEPT_BINARY, + include_auth, + }; + match self.build_and_send(req) { + Ok(resp) => { + let status = resp.status().as_u16(); + let etag = header_str(&resp, "etag"); + let content_type = header_str(&resp, "content-type"); + let last_modified = header_str(&resp, "last-modified"); + let location = header_str(&resp, "location"); + let retry_after = resp + .headers() + .get("retry-after") + .and_then(|v| v.to_str().ok()) + .and_then(retry_after_header); + let body_reader = resp.into_body().into_reader(); + AttemptStream::Ok { + status, + body_reader: Box::new(body_reader), + etag, + content_type, + last_modified, + location, + retry_after, + } + } + Err(e) => AttemptStream::Transport(map_ureq_error(e)), + } + } + + fn build_and_send(&self, req: WireRequest<'_>) -> Result, ureq::Error> { + let method_parsed = Method::from_bytes(req.method.as_bytes()) + .map_err(|e| ureq::Error::Other(Box::new(std::io::Error::other(e))))?; + let mut builder = Request::builder() + .method(method_parsed) + .uri(req.url) + .header("Accept", req.accept) + .header("User-Agent", self.inner.user_agent.as_str()); + if req.include_auth { + builder = builder.header("Authorization", self.inner.auth.header_value()); + } + if let Some(d) = req.depth { + builder = builder.header("Depth", d.to_string()); + } + if let Some(ct) = req.content_type { + builder = builder.header("Content-Type", ct); + } + let payload: &[u8] = req.body.unwrap_or(&[]); + let request = builder + .body(payload) + .map_err(|e| ureq::Error::Other(Box::new(std::io::Error::other(e))))?; + self.inner.agent.run(request) + } +} + +struct WireRequest<'a> { + method: &'a str, + url: &'a str, + body: Option<&'a [u8]>, + content_type: Option<&'a str>, + depth: Option, + accept: &'a str, + include_auth: bool, +} + +pub struct DavStream { + pub status: u16, + pub body_reader: Box, + pub etag: Option, + pub content_type: Option, + pub last_modified: Option, +} + +impl Read for DavStream { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.body_reader.read(buf) + } +} + +enum Attempt { + Ok { + status: u16, + body: Vec, + retry_after: Option, + etag: Option, + content_type: Option, + last_modified: Option, + location: Option, + }, + Transport(JmapError), +} + +enum AttemptStream { + Ok { + status: u16, + body_reader: Box, + etag: Option, + content_type: Option, + last_modified: Option, + location: Option, + retry_after: Option, + }, + Transport(JmapError), +} + +fn header_str(resp: &Response, name: &str) -> Option { + resp.headers() + .get(name) + .and_then(|v| v.to_str().ok()) + .map(str::to_owned) +} + +fn is_redirect_status(status: u16) -> bool { + matches!(status, 301 | 302 | 303 | 307 | 308) +} + +fn resolve_redirect(current: &str, location: &str) -> Result { + crate::dav::href::join_absolute(current, location) + .map_err(|e| JmapError::Connect(format!("redirect: {e}"))) +} + +fn url_host_and_port(url: &str) -> Option<(String, Option)> { + let parsed = url::Url::parse(url).ok()?; + let host = parsed.host_str()?.to_owned(); + Some((host, parsed.port_or_known_default())) +} + +fn same_host(a: &str, b: &str) -> bool { + match (url_host_and_port(a), url_host_and_port(b)) { + (Some(x), Some(y)) => x == y, + _ => false, + } +} + +fn transport_disposition(err: &JmapError) -> retry::Disposition { + match err { + JmapError::Connect(_) => retry::Disposition::Fatal, + _ => retry::Disposition::Retryable, + } +} + +fn map_ureq_error(err: ureq::Error) -> JmapError { + match err { + ureq::Error::Io(e) => JmapError::Transport(format!("io: {e}")), + ureq::Error::Timeout(t) => JmapError::Transport(format!("timeout: {t}")), + ureq::Error::HostNotFound => JmapError::Transport("host not found".to_owned()), + ureq::Error::ConnectionFailed => JmapError::Transport("connection failed".to_owned()), + ureq::Error::BodyStalled => JmapError::Transport("body stalled".to_owned()), + ureq::Error::Tls(m) => JmapError::Transport(format!("tls: {m}")), + ureq::Error::TooManyRedirects => JmapError::Connect("too many redirects".to_owned()), + ureq::Error::RedirectFailed => JmapError::Connect("redirect failed".to_owned()), + ureq::Error::TlsRequired => { + JmapError::Connect("server requires TLS but transport is unsecured".to_owned()) + } + other => JmapError::Connect(other.to_string()), + } +} + +fn truncate(body: &[u8]) -> String { + let text = String::from_utf8_lossy(body); + if text.len() <= 512 { + return text.into_owned(); + } + let end = text.floor_char_boundary(512); + format!("{}...", &text[..end]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn client_constructs_cleanly() { + let c = DavClient::new( + Auth::Basic { + user: "u".into(), + password: "p".into(), + }, + RetryPolicy::new(3), + false, + ); + assert_eq!(c.retries_observed(), 0); + assert_eq!(c.retry_after_sleeps(), 0); + } + + #[test] + fn auth_header_appears_via_builder() { + let c = DavClient::new( + Auth::Bearer { + token: "abc".into(), + }, + RetryPolicy::new(0), + false, + ); + let logger = c.logger(); + assert_eq!(logger.level(), LEVEL_DEFAULT); + } + + #[test] + fn truncate_preserves_char_boundaries() { + let mut body = vec![b'a'; 510]; + body.extend_from_slice("\u{1F4A9}".as_bytes()); + body.extend_from_slice(&[b'b'; 600]); + let s = truncate(&body); + assert!(s.ends_with("...")); + assert!(s.is_char_boundary(s.len())); + } +} diff --git a/src/dav/discover.rs b/src/dav/discover.rs new file mode 100644 index 0000000..f6620fb --- /dev/null +++ b/src/dav/discover.rs @@ -0,0 +1,327 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::dav::client::DavClient; +use crate::dav::href::{Href, absolute_url, normalise}; +use crate::dav::parse::DavResponse; +use crate::dav::xml; +use crate::jmap::error::JmapError; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DavKind { + Caldav, + Carddav, + Webdav, +} + +impl DavKind { + pub fn well_known_uri(self) -> Option<&'static str> { + match self { + DavKind::Caldav => Some("/.well-known/caldav"), + DavKind::Carddav => Some("/.well-known/carddav"), + DavKind::Webdav => None, + } + } + + pub fn home_set_body(self) -> Option { + match self { + DavKind::Caldav => Some(xml::propfind_calendar_home_set()), + DavKind::Carddav => Some(xml::propfind_addressbook_home_set()), + DavKind::Webdav => None, + } + } + + pub fn principal_and_home_set_body(self) -> Option { + match self { + DavKind::Caldav => Some(xml::propfind_principal_and_calendar_home_set()), + DavKind::Carddav => Some(xml::propfind_principal_and_addressbook_home_set()), + DavKind::Webdav => None, + } + } + + pub fn collection_listing_body(self) -> String { + match self { + DavKind::Caldav => xml::propfind_calendar_collections(), + DavKind::Carddav => xml::propfind_addressbook_collections(), + DavKind::Webdav => xml::propfind_webdav_listing(), + } + } + + pub fn extract_home_set(self, props: &crate::dav::parse::ResourceProps) -> Option<&str> { + match self { + DavKind::Caldav => props.calendar_home_set.as_deref(), + DavKind::Carddav => props.addressbook_home_set.as_deref(), + DavKind::Webdav => None, + } + } + + pub fn item_resourcetype_matches(self, props: &crate::dav::parse::ResourceProps) -> bool { + match self { + DavKind::Caldav => props.is_calendar, + DavKind::Carddav => props.is_addressbook, + DavKind::Webdav => props.is_collection, + } + } +} + +#[derive(Debug, Clone)] +pub struct DiscoveredCollection { + pub url: String, + pub href: Href, + pub props: crate::dav::parse::ResourceProps, +} + +#[derive(Debug, thiserror::Error)] +pub enum DiscoveryError { + #[error("transport: {0}")] + Transport(#[from] JmapError), + #[error("xml: {0}")] + Parse(#[from] crate::dav::parse::ParseError), + #[error("href: {0}")] + Href(#[from] crate::dav::href::HrefError), + #[error("no DAV collections found under {url}")] + NotFound { url: String }, +} + +#[derive(Debug, Clone)] +pub struct Discovery { + pub principal_url: Option, + pub home_set_url: String, + pub collections: Vec, +} + +pub fn discover( + client: &DavClient, + kind: DavKind, + user_url: &str, +) -> Result { + if let Some(disc) = try_treat_url_as_home_or_collection(client, kind, user_url)? { + return Ok(disc); + } + if let Some(disc) = try_via_principal(client, kind, user_url)? { + return Ok(disc); + } + if let Some(well_known) = kind.well_known_uri() { + let absolute = absolute_url(user_url, &Href::from_normalised(well_known.to_owned()))?; + if let Some(disc) = try_via_principal(client, kind, &absolute)? { + return Ok(disc); + } + if let Some(disc) = try_treat_url_as_home_or_collection(client, kind, &absolute)? { + return Ok(disc); + } + } + Err(DiscoveryError::NotFound { + url: user_url.to_owned(), + }) +} + +fn try_treat_url_as_home_or_collection( + client: &DavClient, + kind: DavKind, + url: &str, +) -> Result, DiscoveryError> { + let body = kind.collection_listing_body(); + let collections = match propfind_collections(client, kind, url, &body) { + Ok(c) => c, + Err(DiscoveryError::Transport(JmapError::HttpStatus { status, .. })) + if (400..500).contains(&status) => + { + return Ok(None); + } + Err(e) => return Err(e), + }; + if collections.is_empty() { + return Ok(None); + } + Ok(Some(Discovery { + principal_url: None, + home_set_url: url.to_owned(), + collections, + })) +} + +fn try_via_principal( + client: &DavClient, + kind: DavKind, + url: &str, +) -> Result, DiscoveryError> { + let Some(combined_body) = kind.principal_and_home_set_body() else { + return Ok(None); + }; + let first_ms = match client.propfind_responses(url, 0, &combined_body, url) { + Ok(ms) => ms, + Err(JmapError::HttpStatus { status, .. }) if (400..600).contains(&status) => { + return Ok(None); + } + Err(JmapError::RetriesExhausted(_)) => return Ok(None), + Err(e) => return Err(DiscoveryError::Transport(e)), + }; + let first_final_url = first_ms.final_url.clone(); + let direct_home_set = first_ms + .responses + .iter() + .find_map(|r| kind.extract_home_set(&r.props)) + .map(|h| h.to_owned()); + let (principal_url, home_set_url) = if let Some(home) = direct_home_set { + let resolved = absolute_url(&first_final_url, &normalise(&first_final_url, &home)?)?; + (first_final_url.clone(), resolved) + } else { + let principal_href = first_ms + .responses + .iter() + .find_map(|r| r.props.current_user_principal.as_deref()) + .map(|h| h.to_owned()); + let principal_url = match principal_href { + Some(rel) => absolute_url(&first_final_url, &normalise(&first_final_url, &rel)?)?, + None => first_final_url.clone(), + }; + let Some(home_set_body) = kind.home_set_body() else { + return Ok(None); + }; + let home_ms = + match client.propfind_responses(&principal_url, 0, &home_set_body, &principal_url) { + Ok(ms) => ms, + Err(JmapError::HttpStatus { status, .. }) if (400..600).contains(&status) => { + return Ok(None); + } + Err(JmapError::RetriesExhausted(_)) => return Ok(None), + Err(e) => return Err(DiscoveryError::Transport(e)), + }; + let principal_after_redirect = home_ms.final_url.clone(); + let home_set_href = home_ms + .responses + .iter() + .find_map(|r| kind.extract_home_set(&r.props)) + .map(|h| h.to_owned()); + let Some(home_set_rel) = home_set_href else { + return Ok(None); + }; + let resolved = absolute_url( + &principal_after_redirect, + &normalise(&principal_after_redirect, &home_set_rel)?, + )?; + (principal_url, resolved) + }; + let listing_body = kind.collection_listing_body(); + let collections = propfind_collections(client, kind, &home_set_url, &listing_body)?; + if collections.is_empty() { + return Ok(None); + } + Ok(Some(Discovery { + principal_url: Some(principal_url), + home_set_url, + collections, + })) +} + +fn propfind_collections( + client: &DavClient, + kind: DavKind, + url: &str, + body: &str, +) -> Result, DiscoveryError> { + let responses = propfind_depth(client, url, 1, body)?; + let self_href = normalise(url, "").map(|h| h.into_string()).ok(); + let mut out: Vec = Vec::new(); + for r in responses { + let is_self = self_href.as_deref() == Some(r.href.as_str()); + if matches!(kind, DavKind::Webdav) { + if !r.props.is_collection { + continue; + } + if !is_self { + continue; + } + } else { + if !kind.item_resourcetype_matches(&r.props) { + continue; + } + if calendar_supports_only_vfreebusy(&r.props) { + client.logger().warn(&format!( + "calendar {} supports only {:?}; skipping", + r.href.as_str(), + r.props.supported_components, + )); + continue; + } + } + let abs = absolute_url(url, &r.href)?; + out.push(DiscoveredCollection { + url: abs, + href: r.href.clone(), + props: r.props, + }); + } + Ok(out) +} + +fn calendar_supports_only_vfreebusy(props: &crate::dav::parse::ResourceProps) -> bool { + if !props.is_calendar { + return false; + } + if props.supported_components.is_empty() { + return false; + } + let has_useful = props + .supported_components + .iter() + .any(|c| matches!(c.as_str(), "VEVENT" | "VTODO" | "VJOURNAL")); + !has_useful +} + +fn propfind_depth( + client: &DavClient, + url: &str, + depth: u8, + body: &str, +) -> Result, DiscoveryError> { + let ms = client.propfind_responses(url, depth, body, url)?; + Ok(ms.responses) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn well_known_uris_match_rfc_6764() { + assert_eq!( + DavKind::Caldav.well_known_uri(), + Some("/.well-known/caldav") + ); + assert_eq!( + DavKind::Carddav.well_known_uri(), + Some("/.well-known/carddav") + ); + assert_eq!(DavKind::Webdav.well_known_uri(), None); + } + + #[test] + fn home_set_body_only_for_calendar_addressbook() { + assert!(DavKind::Caldav.home_set_body().is_some()); + assert!(DavKind::Carddav.home_set_body().is_some()); + assert!(DavKind::Webdav.home_set_body().is_none()); + } + + #[test] + fn item_resourcetype_matches_calendar_for_caldav() { + let p = crate::dav::parse::ResourceProps { + is_calendar: true, + ..Default::default() + }; + assert!(DavKind::Caldav.item_resourcetype_matches(&p)); + assert!(!DavKind::Carddav.item_resourcetype_matches(&p)); + } + + #[test] + fn item_resourcetype_matches_collection_for_webdav() { + let p = crate::dav::parse::ResourceProps { + is_collection: true, + ..Default::default() + }; + assert!(DavKind::Webdav.item_resourcetype_matches(&p)); + } +} diff --git a/src/dav/href.rs b/src/dav/href.rs new file mode 100644 index 0000000..8ad1b23 --- /dev/null +++ b/src/dav/href.rs @@ -0,0 +1,260 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use url::Url; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Href(String); + +impl Href { + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn into_string(self) -> String { + self.0 + } + + pub fn from_normalised(s: String) -> Href { + Href(s) + } +} + +impl AsRef for Href { + fn as_ref(&self) -> &str { + &self.0 + } +} + +#[derive(Debug, thiserror::Error)] +pub enum HrefError { + #[error("invalid base URL {url:?}: {source}")] + InvalidBase { + url: String, + #[source] + source: url::ParseError, + }, + #[error("invalid href {href:?} relative to {base:?}: {source}")] + InvalidHref { + href: String, + base: String, + #[source] + source: url::ParseError, + }, +} + +pub fn normalise(base: &str, raw: &str) -> Result { + let base_url = Url::parse(base).map_err(|e| HrefError::InvalidBase { + url: base.to_owned(), + source: e, + })?; + let resolved = base_url + .join(raw.trim()) + .map_err(|e| HrefError::InvalidHref { + href: raw.to_owned(), + base: base.to_owned(), + source: e, + })?; + Ok(normalise_path(&resolved)) +} + +pub fn normalise_path(url: &Url) -> Href { + let mut path = url.path().to_owned(); + if path.is_empty() { + path.push('/'); + } + let decoded = percent_decode(&path); + let reencoded = percent_encode_safe(&decoded); + Href(reencoded) +} + +pub fn absolute_url(base: &str, href: &Href) -> Result { + join_absolute(base, href.as_str()) +} + +pub fn join_absolute(base: &str, href: &str) -> Result { + let base_url = Url::parse(base).map_err(|e| HrefError::InvalidBase { + url: base.to_owned(), + source: e, + })?; + let resolved = base_url.join(href).map_err(|e| HrefError::InvalidHref { + href: href.to_owned(), + base: base.to_owned(), + source: e, + })?; + Ok(resolved.to_string()) +} + +pub fn last_path_component(href: &Href) -> String { + let trimmed = href.as_str().trim_end_matches('/'); + let tail = trimmed.rsplit('/').next().unwrap_or(""); + percent_decode(tail) +} + +pub fn parent_collection(href: &Href) -> Href { + let trimmed = href.as_str().trim_end_matches('/'); + if let Some(slash) = trimmed.rfind('/') { + let parent = &trimmed[..=slash]; + Href(parent.to_owned()) + } else { + Href("/".to_owned()) + } +} + +fn percent_decode(s: &str) -> String { + let bytes = s.as_bytes(); + let mut buf: Vec = Vec::with_capacity(s.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' + && i + 2 < bytes.len() + && let (Some(h), Some(l)) = (hex_val(bytes[i + 1]), hex_val(bytes[i + 2])) + { + buf.push((h << 4) | l); + i += 3; + continue; + } + buf.push(bytes[i]); + i += 1; + } + match String::from_utf8(buf) { + Ok(decoded) => decoded, + Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(), + } +} + +fn hex_val(b: u8) -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} + +fn percent_encode_safe(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for byte in s.bytes() { + if is_safe(byte) { + out.push(byte as char); + } else { + out.push('%'); + out.push(upper_hex(byte >> 4)); + out.push(upper_hex(byte & 0x0f)); + } + } + out +} + +fn is_safe(b: u8) -> bool { + matches!(b, + b'A'..=b'Z' + | b'a'..=b'z' + | b'0'..=b'9' + | b'-' | b'_' | b'.' | b'~' + | b'/' | b'@' + ) +} + +fn upper_hex(nibble: u8) -> char { + match nibble { + 0..=9 => (b'0' + nibble) as char, + 10..=15 => (b'A' + nibble - 10) as char, + _ => '0', + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn relative_href_resolves_against_base() { + let h = normalise("https://dav.example.com/", "/dav/cal/user/default/").unwrap(); + assert_eq!(h.as_str(), "/dav/cal/user/default/"); + } + + #[test] + fn absolute_href_strips_scheme_and_host() { + let h = normalise( + "https://dav.example.com/cal/", + "https://dav.example.com/dav/cal/user/default/event-1.ics", + ) + .unwrap(); + assert_eq!(h.as_str(), "/dav/cal/user/default/event-1.ics"); + } + + #[test] + fn percent_encoding_canonicalised_to_uppercase_hex() { + let h = normalise("https://x/", "/dav/cal/user%2bwork/event%201.ics").unwrap(); + assert_eq!(h.as_str(), "/dav/cal/user%2Bwork/event%201.ics"); + } + + #[test] + fn at_sign_is_preserved_in_collection_paths() { + let h = normalise("https://x/", "/dav/cal/user@host/default/").unwrap(); + assert_eq!(h.as_str(), "/dav/cal/user@host/default/"); + } + + #[test] + fn trailing_slash_preserved() { + let coll = normalise("https://x/", "/a/b/").unwrap(); + let item = normalise("https://x/", "/a/b").unwrap(); + assert_eq!(coll.as_str(), "/a/b/"); + assert_eq!(item.as_str(), "/a/b"); + assert_ne!(coll, item); + } + + #[test] + fn href_equality_is_byte_exact_after_normalisation() { + let a = normalise("https://x/", "/Dav/Cal/User/Default/").unwrap(); + let b = normalise("https://x/", "/Dav/Cal/User/Default/").unwrap(); + assert_eq!(a, b); + } + + #[test] + fn last_path_component_decodes() { + let h = Href("/dav/cal/user/event%201.ics".to_owned()); + assert_eq!(last_path_component(&h), "event 1.ics"); + } + + #[test] + fn parent_collection_strips_trailing_slash() { + let h = Href("/dav/cal/user/sub/".to_owned()); + assert_eq!(parent_collection(&h).as_str(), "/dav/cal/user/"); + } + + #[test] + fn parent_collection_of_file_is_parent_dir() { + let h = Href("/dav/cal/user/file.ics".to_owned()); + assert_eq!(parent_collection(&h).as_str(), "/dav/cal/user/"); + } + + #[test] + fn parent_collection_of_root_is_root() { + let h = Href("/".to_owned()); + assert_eq!(parent_collection(&h).as_str(), "/"); + } + + #[test] + fn empty_path_treated_as_root() { + let url = Url::parse("https://dav.example.com").unwrap(); + let h = normalise_path(&url); + assert_eq!(h.as_str(), "/"); + } + + #[test] + fn query_and_fragment_stripped() { + let h = normalise("https://x/", "/dav/cal/?q=1#frag").unwrap(); + assert_eq!(h.as_str(), "/dav/cal/"); + } + + #[test] + fn unicode_path_round_trips_through_utf8_percent_encoding() { + let h = normalise("https://x/", "/dav/cal/caf%C3%A9/").unwrap(); + assert_eq!(h.as_str(), "/dav/cal/caf%C3%A9/"); + } +} diff --git a/src/dav/mod.rs b/src/dav/mod.rs new file mode 100644 index 0000000..0981aef --- /dev/null +++ b/src/dav/mod.rs @@ -0,0 +1,12 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod client; +pub mod discover; +pub mod href; +pub mod parse; +pub mod retry; +pub mod xml; diff --git a/src/dav/parse.rs b/src/dav/parse.rs new file mode 100644 index 0000000..a490574 --- /dev/null +++ b/src/dav/parse.rs @@ -0,0 +1,904 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::BufRead; + +use quick_xml::NsReader; +use quick_xml::events::Event; +use quick_xml::name::ResolveResult; + +use crate::dav::href::{Href, normalise}; +use crate::dav::xml::{NS_APPLE_ICAL, NS_CALDAV, NS_CARDDAV, NS_DAV}; + +#[derive(Debug, thiserror::Error)] +pub enum ParseError { + #[error("xml error: {0}")] + Xml(String), + #[error("invalid href {href:?}: {source}")] + Href { + href: String, + #[source] + source: crate::dav::href::HrefError, + }, + #[error("missing href in response")] + MissingHref, +} + +impl From for ParseError { + fn from(value: quick_xml::Error) -> Self { + ParseError::Xml(value.to_string()) + } +} + +#[derive(Debug, Clone, Default)] +pub struct ResourceProps { + pub is_collection: bool, + pub is_calendar: bool, + pub is_addressbook: bool, + pub displayname: Option, + pub current_user_principal: Option, + pub calendar_home_set: Option, + pub addressbook_home_set: Option, + pub etag: Option, + pub content_type: Option, + pub last_modified: Option, + pub creation_date: Option, + pub content_length: Option, + pub calendar_description: Option, + pub addressbook_description: Option, + pub calendar_color: Option, + pub calendar_order: Option, + pub calendar_timezone: Option, + pub supported_components: Vec, + pub calendar_data: Option, + pub address_data: Option, +} + +#[derive(Debug, Clone)] +pub struct DavResponse { + pub href: Href, + pub status: Option, + pub props: ResourceProps, + pub propstat_errors: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum NsKind { + Dav, + Caldav, + Carddav, + Apple, + Other, +} + +fn classify_ns(ns: &ResolveResult<'_>) -> NsKind { + match ns { + ResolveResult::Bound(prefix) => { + let p = prefix.as_ref(); + if p == NS_DAV.as_bytes() { + NsKind::Dav + } else if p == NS_CALDAV.as_bytes() { + NsKind::Caldav + } else if p == NS_CARDDAV.as_bytes() { + NsKind::Carddav + } else if p == NS_APPLE_ICAL.as_bytes() { + NsKind::Apple + } else { + NsKind::Other + } + } + _ => NsKind::Other, + } +} + +fn is(ns: NsKind, local: &[u8], expected_ns: NsKind, target: &[u8]) -> bool { + ns == expected_ns && local.eq_ignore_ascii_case(target) +} + +#[derive(Debug)] +enum Step { + StartElement { + ns: NsKind, + local: Vec, + attrs: Vec<(Vec, Vec)>, + }, + EmptyElement { + ns: NsKind, + local: Vec, + attrs: Vec<(Vec, Vec)>, + }, + EndElement { + ns: NsKind, + local: Vec, + }, + Text(String), + CData(String), + Other, + Eof, +} + +fn next_step(xml: &mut NsReader, buf: &mut Vec) -> Result { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(buf)?; + let ns_kind = classify_ns(&ns); + Ok(match ev { + Event::Start(e) => { + let local = e.local_name().as_ref().to_vec(); + let mut attrs: Vec<(Vec, Vec)> = Vec::new(); + for a in e.attributes().flatten() { + attrs.push((a.key.as_ref().to_vec(), a.value.as_ref().to_vec())); + } + Step::StartElement { + ns: ns_kind, + local, + attrs, + } + } + Event::Empty(e) => { + let local = e.local_name().as_ref().to_vec(); + let mut attrs: Vec<(Vec, Vec)> = Vec::new(); + for a in e.attributes().flatten() { + attrs.push((a.key.as_ref().to_vec(), a.value.as_ref().to_vec())); + } + Step::EmptyElement { + ns: ns_kind, + local, + attrs, + } + } + Event::End(e) => Step::EndElement { + ns: ns_kind, + local: e.local_name().as_ref().to_vec(), + }, + Event::Text(t) => { + let s = t.decode().map_err(|e| ParseError::Xml(e.to_string()))?; + Step::Text(s.into_owned()) + } + Event::CData(cd) => Step::CData(String::from_utf8_lossy(cd.as_ref()).into_owned()), + Event::Eof => Step::Eof, + _ => Step::Other, + }) +} + +pub fn parse_multistatus( + reader: R, + base_url: &str, +) -> Result, ParseError> { + let mut xml = NsReader::from_reader(reader); + let mut buf = Vec::new(); + let mut out: Vec = Vec::new(); + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + + loop { + let step = next_step(&mut xml, &mut buf)?; + match step { + Step::StartElement { ns, local, .. } if is(ns, &local, NsKind::Dav, b"response") => { + if let Some(r) = parse_response(&mut xml, base_url)? + && seen.insert(r.href.as_str().to_owned()) + { + out.push(r); + } + } + Step::Eof => break, + _ => {} + } + } + Ok(out) +} + +fn parse_response( + xml: &mut NsReader, + base_url: &str, +) -> Result, ParseError> { + let mut buf = Vec::new(); + let mut href: Option = None; + let mut status: Option = None; + let mut props = ResourceProps::default(); + let mut propstat_errors: Vec = Vec::new(); + + loop { + let step = next_step(xml, &mut buf)?; + match step { + Step::StartElement { ns, local, .. } => { + if is(ns, &local, NsKind::Dav, b"href") { + let text = read_token(xml)?; + if href.is_none() { + href = Some(text); + } + } else if is(ns, &local, NsKind::Dav, b"status") { + let text = read_token(xml)?; + status = parse_http_status(&text); + } else if is(ns, &local, NsKind::Dav, b"propstat") { + let (block_status, block_props) = parse_propstat(xml)?; + match block_status { + Some(s) if (200..300).contains(&s) => merge_props(&mut props, block_props), + Some(s) => propstat_errors.push(s), + None => {} + } + } else { + skip_element(xml, &local)?; + } + } + Step::EndElement { ns, local } if is(ns, &local, NsKind::Dav, b"response") => break, + Step::Eof => return Err(ParseError::Xml("unexpected EOF in ".into())), + _ => {} + } + } + + let raw_href = href.ok_or(ParseError::MissingHref)?; + let normalised = normalise(base_url, &raw_href).map_err(|e| ParseError::Href { + href: raw_href, + source: e, + })?; + Ok(Some(DavResponse { + href: normalised, + status, + props, + propstat_errors, + })) +} + +fn parse_propstat( + xml: &mut NsReader, +) -> Result<(Option, ResourceProps), ParseError> { + let mut buf = Vec::new(); + let mut status: Option = None; + let mut props = ResourceProps::default(); + loop { + let step = next_step(xml, &mut buf)?; + match step { + Step::StartElement { ns, local, .. } => { + if is(ns, &local, NsKind::Dav, b"status") { + let text = read_token(xml)?; + status = parse_http_status(&text); + } else if is(ns, &local, NsKind::Dav, b"prop") { + parse_prop_block(xml, &mut props)?; + } else { + skip_element(xml, &local)?; + } + } + Step::EndElement { ns, local } if is(ns, &local, NsKind::Dav, b"propstat") => break, + Step::Eof => return Err(ParseError::Xml("unexpected EOF in ".into())), + _ => {} + } + } + Ok((status, props)) +} + +fn parse_prop_block( + xml: &mut NsReader, + props: &mut ResourceProps, +) -> Result<(), ParseError> { + let mut buf = Vec::new(); + loop { + let step = next_step(xml, &mut buf)?; + match step { + Step::StartElement { ns, local, .. } => { + handle_prop_element(xml, props, ns, &local)?; + } + Step::EmptyElement { .. } => {} + Step::EndElement { ns, local } if is(ns, &local, NsKind::Dav, b"prop") => break, + Step::Eof => return Err(ParseError::Xml("unexpected EOF in ".into())), + _ => {} + } + } + Ok(()) +} + +fn handle_prop_element( + xml: &mut NsReader, + props: &mut ResourceProps, + ns: NsKind, + local: &[u8], +) -> Result<(), ParseError> { + if is(ns, local, NsKind::Dav, b"resourcetype") { + consume_resourcetype(xml, props)?; + } else if is(ns, local, NsKind::Dav, b"displayname") { + props.displayname = Some(read_token(xml)?); + } else if is(ns, local, NsKind::Dav, b"current-user-principal") { + props.current_user_principal = read_first_href(xml, b"current-user-principal")?; + } else if is(ns, local, NsKind::Caldav, b"calendar-home-set") { + props.calendar_home_set = read_first_href(xml, b"calendar-home-set")?; + } else if is(ns, local, NsKind::Carddav, b"addressbook-home-set") { + props.addressbook_home_set = read_first_href(xml, b"addressbook-home-set")?; + } else if is(ns, local, NsKind::Dav, b"getetag") { + let raw = read_token(xml)?; + if !raw.is_empty() { + props.etag = Some(raw); + } + } else if is(ns, local, NsKind::Dav, b"getcontenttype") { + props.content_type = Some(read_token(xml)?); + } else if is(ns, local, NsKind::Dav, b"getlastmodified") { + props.last_modified = Some(read_token(xml)?); + } else if is(ns, local, NsKind::Dav, b"creationdate") { + props.creation_date = Some(read_token(xml)?); + } else if is(ns, local, NsKind::Dav, b"getcontentlength") { + let raw = read_token(xml)?; + props.content_length = raw.parse::().ok(); + } else if is(ns, local, NsKind::Caldav, b"calendar-description") { + props.calendar_description = Some(read_token(xml)?); + } else if is(ns, local, NsKind::Carddav, b"addressbook-description") { + props.addressbook_description = Some(read_token(xml)?); + } else if is(ns, local, NsKind::Apple, b"calendar-color") { + props.calendar_color = Some(read_token(xml)?); + } else if is(ns, local, NsKind::Apple, b"calendar-order") { + let raw = read_token(xml)?; + props.calendar_order = raw.parse::().ok(); + } else if is(ns, local, NsKind::Caldav, b"calendar-timezone") { + props.calendar_timezone = Some(read_token(xml)?); + } else if is( + ns, + local, + NsKind::Caldav, + b"supported-calendar-component-set", + ) { + consume_supported_components(xml, props)?; + } else if is(ns, local, NsKind::Caldav, b"calendar-data") { + props.calendar_data = Some(read_text(xml)?); + } else if is(ns, local, NsKind::Carddav, b"address-data") { + props.address_data = Some(read_text(xml)?); + } else { + skip_element(xml, local)?; + } + Ok(()) +} + +fn consume_resourcetype( + xml: &mut NsReader, + props: &mut ResourceProps, +) -> Result<(), ParseError> { + let mut buf = Vec::new(); + loop { + let step = next_step(xml, &mut buf)?; + match step { + Step::StartElement { ns, local, .. } => { + mark_resourcetype(ns, &local, props); + skip_element(xml, &local)?; + } + Step::EmptyElement { ns, local, .. } => { + mark_resourcetype(ns, &local, props); + } + Step::EndElement { ns, local } if is(ns, &local, NsKind::Dav, b"resourcetype") => break, + Step::Eof => { + return Err(ParseError::Xml("unexpected EOF in ".into())); + } + _ => {} + } + } + Ok(()) +} + +fn mark_resourcetype(ns: NsKind, local: &[u8], props: &mut ResourceProps) { + if is(ns, local, NsKind::Dav, b"collection") { + props.is_collection = true; + } else if is(ns, local, NsKind::Caldav, b"calendar") { + props.is_calendar = true; + } else if is(ns, local, NsKind::Carddav, b"addressbook") { + props.is_addressbook = true; + } +} + +fn consume_supported_components( + xml: &mut NsReader, + props: &mut ResourceProps, +) -> Result<(), ParseError> { + let mut buf = Vec::new(); + loop { + let step = next_step(xml, &mut buf)?; + match step { + Step::StartElement { + ns, + local, + ref attrs, + } if is(ns, &local, NsKind::Caldav, b"comp") => { + if let Some(name) = attr_lookup(attrs, b"name") { + props.supported_components.push(name); + } + skip_element(xml, &local)?; + } + Step::EmptyElement { + ns, + local, + ref attrs, + } if is(ns, &local, NsKind::Caldav, b"comp") => { + if let Some(name) = attr_lookup(attrs, b"name") { + props.supported_components.push(name); + } + } + Step::EndElement { ns, local } + if is( + ns, + &local, + NsKind::Caldav, + b"supported-calendar-component-set", + ) => + { + break; + } + Step::Eof => { + return Err(ParseError::Xml( + "unexpected EOF in ".into(), + )); + } + _ => {} + } + } + Ok(()) +} + +fn attr_lookup(attrs: &[(Vec, Vec)], key: &[u8]) -> Option { + for (k, v) in attrs { + if k == key { + return Some(String::from_utf8_lossy(v).into_owned()); + } + } + None +} + +fn read_text(xml: &mut NsReader) -> Result { + let mut buf = Vec::new(); + let mut out = String::new(); + loop { + let step = next_step(xml, &mut buf)?; + match step { + Step::Text(s) | Step::CData(s) => out.push_str(&s), + Step::StartElement { local, .. } => skip_element(xml, &local)?, + Step::EndElement { .. } | Step::EmptyElement { .. } => break, + Step::Eof => return Err(ParseError::Xml("unexpected EOF reading text".into())), + _ => {} + } + } + Ok(out) +} + +fn read_token(xml: &mut NsReader) -> Result { + Ok(read_text(xml)?.trim().to_owned()) +} + +fn read_first_href( + xml: &mut NsReader, + closing_local: &[u8], +) -> Result, ParseError> { + let mut buf = Vec::new(); + let mut out: Option = None; + loop { + let step = next_step(xml, &mut buf)?; + match step { + Step::StartElement { ns, local, .. } => { + if out.is_none() && is(ns, &local, NsKind::Dav, b"href") { + out = Some(read_token(xml)?); + } else { + skip_element(xml, &local)?; + } + } + Step::EmptyElement { .. } => {} + Step::EndElement { local, .. } if local.eq_ignore_ascii_case(closing_local) => { + break; + } + Step::Eof => return Err(ParseError::Xml("unexpected EOF reading href".into())), + _ => {} + } + } + Ok(out) +} + +fn skip_element(xml: &mut NsReader, target: &[u8]) -> Result<(), ParseError> { + let mut buf = Vec::new(); + let mut depth: i32 = 1; + while depth > 0 { + let step = next_step(xml, &mut buf)?; + match step { + Step::StartElement { .. } => depth += 1, + Step::EndElement { local, .. } => { + depth -= 1; + if depth == 0 && !local.eq_ignore_ascii_case(target) { + return Ok(()); + } + } + Step::Eof => return Err(ParseError::Xml("unexpected EOF in skip".into())), + _ => {} + } + } + Ok(()) +} + +fn merge_props(target: &mut ResourceProps, src: ResourceProps) { + if src.is_collection { + target.is_collection = true; + } + if src.is_calendar { + target.is_calendar = true; + } + if src.is_addressbook { + target.is_addressbook = true; + } + overwrite_some(&mut target.displayname, src.displayname); + overwrite_some( + &mut target.current_user_principal, + src.current_user_principal, + ); + overwrite_some(&mut target.calendar_home_set, src.calendar_home_set); + overwrite_some(&mut target.addressbook_home_set, src.addressbook_home_set); + overwrite_some(&mut target.etag, src.etag); + overwrite_some(&mut target.content_type, src.content_type); + overwrite_some(&mut target.last_modified, src.last_modified); + overwrite_some(&mut target.creation_date, src.creation_date); + overwrite_some(&mut target.content_length, src.content_length); + overwrite_some(&mut target.calendar_description, src.calendar_description); + overwrite_some( + &mut target.addressbook_description, + src.addressbook_description, + ); + overwrite_some(&mut target.calendar_color, src.calendar_color); + overwrite_some(&mut target.calendar_order, src.calendar_order); + overwrite_some(&mut target.calendar_timezone, src.calendar_timezone); + overwrite_some(&mut target.calendar_data, src.calendar_data); + overwrite_some(&mut target.address_data, src.address_data); + if !src.supported_components.is_empty() { + target.supported_components = src.supported_components; + } +} + +fn overwrite_some(target: &mut Option, src: Option) { + if src.is_some() { + *target = src; + } +} + +fn parse_http_status(text: &str) -> Option { + let mut parts = text.split_ascii_whitespace(); + let _version = parts.next()?; + let code = parts.next()?; + code.parse::().ok() +} + +pub fn strip_ascii_control_chars(body: &[u8]) -> std::borrow::Cow<'_, [u8]> { + if body.iter().any(|&b| is_xml_illegal_control(b)) { + let cleaned: Vec = body + .iter() + .copied() + .filter(|b| !is_xml_illegal_control(*b)) + .collect(); + std::borrow::Cow::Owned(cleaned) + } else { + std::borrow::Cow::Borrowed(body) + } +} + +pub fn is_xml_illegal_control(b: u8) -> bool { + matches!(b, 0x00..=0x08 | 0x0b | 0x0c | 0x0e..=0x1f) +} + +pub struct ControlStrippingReader { + inner: R, +} + +impl ControlStrippingReader { + pub fn new(inner: R) -> Self { + Self { inner } + } +} + +impl std::io::Read for ControlStrippingReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + loop { + let n = self.inner.read(buf)?; + if n == 0 { + return Ok(0); + } + let mut write = 0; + for read in 0..n { + let b = buf[read]; + if !is_xml_illegal_control(b) { + buf[write] = b; + write += 1; + } + } + if write > 0 { + return Ok(write); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + fn parse(body: &str) -> Vec { + parse_multistatus(Cursor::new(body), "https://x/").expect("parse") + } + + #[test] + fn parses_simple_principal_response() { + let body = r#" + + + /dav/ + + + + /dav/principals/alice/ + + + HTTP/1.1 200 OK + + + "#; + let r = parse(body); + assert_eq!(r.len(), 1); + assert_eq!( + r[0].props.current_user_principal.as_deref(), + Some("/dav/principals/alice/") + ); + } + + #[test] + fn parses_calendar_collection_with_apple_color_and_order() { + let body = r#" + + + /dav/cal/u/default/ + + + + Default + #3366cc + 5 + Work + + HTTP/1.1 200 OK + + + "#; + let r = parse(body); + assert_eq!(r.len(), 1); + assert!(r[0].props.is_collection); + assert!(r[0].props.is_calendar); + assert_eq!(r[0].props.displayname.as_deref(), Some("Default")); + assert_eq!(r[0].props.calendar_color.as_deref(), Some("#3366cc")); + assert_eq!(r[0].props.calendar_order, Some(5)); + assert_eq!(r[0].props.calendar_description.as_deref(), Some("Work")); + } + + #[test] + fn parses_item_etag_and_content_type() { + let body = r#" + + + /dav/cal/u/d/event1.ics + + + + "abc123" + text/calendar; charset=utf-8 + + HTTP/1.1 200 OK + + + "#; + let r = parse(body); + assert_eq!(r[0].props.etag.as_deref(), Some("\"abc123\"")); + assert!(!r[0].props.is_collection); + assert!( + r[0].props + .content_type + .as_deref() + .unwrap() + .starts_with("text/calendar") + ); + } + + #[test] + fn parses_calendar_multiget_response_with_data() { + let body = r#" + + + /dav/cal/u/d/event1.ics + + + "v2" + BEGIN:VCALENDAR +END:VCALENDAR + + HTTP/1.1 200 OK + + + "#; + let r = parse(body); + assert_eq!(r[0].props.etag.as_deref(), Some("\"v2\"")); + let data = r[0].props.calendar_data.as_deref().unwrap(); + assert!(data.contains("VCALENDAR")); + } + + #[test] + fn drops_duplicate_response_hrefs() { + let body = r#" + + + /dav/cal/u/d/event.ics + + "a" + HTTP/1.1 200 OK + + + + /dav/cal/u/d/event.ics + + "a" + HTTP/1.1 200 OK + + + "#; + let r = parse(body); + assert_eq!(r.len(), 1); + } + + #[test] + fn captures_response_level_status_for_vanished_items() { + let body = r#" + + + /dav/cal/u/d/gone.ics + HTTP/1.1 404 Not Found + + "#; + let r = parse(body); + assert_eq!(r[0].status, Some(404)); + } + + #[test] + fn collects_propstat_errors_separately() { + let body = r#" + + + /dav/cal/u/d/event.ics + + "a" + HTTP/1.1 200 OK + + + + HTTP/1.1 404 Not Found + + + "#; + let r = parse(body); + assert_eq!(r[0].propstat_errors, vec![404]); + assert_eq!(r[0].props.etag.as_deref(), Some("\"a\"")); + } + + #[test] + fn unknown_namespace_elements_ignored() { + let body = r#" + + + /dav/cal/u/d/ + + + + abc + + HTTP/1.1 200 OK + + + "#; + let r = parse(body); + assert!(r[0].props.is_collection); + } + + #[test] + fn strip_control_chars_removes_low_bytes() { + let raw = b"\x01"; + let cleaned = strip_ascii_control_chars(raw); + assert!(!cleaned.contains(&0x01)); + } + + #[test] + fn strip_control_chars_leaves_tab_newline_cr() { + let raw = b"\t\n\r"; + let cleaned = strip_ascii_control_chars(raw); + match cleaned { + std::borrow::Cow::Borrowed(_) => {} + std::borrow::Cow::Owned(_) => panic!("should not have allocated"), + } + } + + #[test] + fn parses_addressbook_collection() { + let body = r#" + + + /dav/card/u/default/ + + + + Contacts + Personal + + HTTP/1.1 200 OK + + + "#; + let r = parse(body); + assert!(r[0].props.is_addressbook); + assert_eq!(r[0].props.displayname.as_deref(), Some("Contacts")); + assert_eq!( + r[0].props.addressbook_description.as_deref(), + Some("Personal") + ); + } + + #[test] + fn empty_multistatus_yields_no_responses() { + let body = r#" + "#; + let r = parse(body); + assert!(r.is_empty()); + } + + #[test] + fn supported_calendar_component_set_captures_empty_element_name_attr() { + let body = r#" + + + /dav/cal/u/d/ + + + + + + + + + HTTP/1.1 200 OK + + + "#; + let r = parse(body); + assert_eq!(r[0].props.supported_components, vec!["VEVENT", "VTODO"]); + } + + #[test] + fn merge_props_last_propstat_wins_for_duplicate_keys() { + let body = r#" + + + /dav/cal/u/d/ + + First + HTTP/1.1 200 OK + + + Second + HTTP/1.1 200 OK + + + "#; + let r = parse(body); + assert_eq!(r[0].props.displayname.as_deref(), Some("Second")); + } + + #[test] + fn content_length_parses_to_u64() { + let body = r#" + + + /file.bin + + + 123456 + + + HTTP/1.1 200 OK + + + "#; + let r = parse(body); + assert_eq!(r[0].props.content_length, Some(123456)); + } +} diff --git a/src/dav/retry.rs b/src/dav/retry.rs new file mode 100644 index 0000000..b33cc86 --- /dev/null +++ b/src/dav/retry.rs @@ -0,0 +1,116 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::Value; + +use crate::jmap::retry::Disposition; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DavOutcome { + Success, + Retryable, + Vanished, + Auth, + Fatal, +} + +pub fn classify(status: u16, body: &[u8]) -> DavOutcome { + if (200..300).contains(&status) { + return DavOutcome::Success; + } + if status == 401 { + return DavOutcome::Auth; + } + if status == 403 { + if is_google_usage_limit(body) { + return DavOutcome::Retryable; + } + return DavOutcome::Auth; + } + if status == 404 || status == 410 { + return DavOutcome::Vanished; + } + if matches!(status, 423 | 429 | 502 | 503 | 504 | 507) { + return DavOutcome::Retryable; + } + match crate::jmap::retry::classify_http_status(status) { + Disposition::Retryable => DavOutcome::Retryable, + Disposition::Fatal => DavOutcome::Fatal, + } +} + +fn is_google_usage_limit(body: &[u8]) -> bool { + let value: Value = match serde_json::from_slice(body) { + Ok(v) => v, + Err(_) => return false, + }; + let errors = value + .get("error") + .and_then(|e| e.get("errors")) + .and_then(Value::as_array); + let Some(errors) = errors else { + return false; + }; + errors.iter().any(|entry| { + entry + .get("domain") + .and_then(Value::as_str) + .map(|d| d == "usageLimits") + .unwrap_or(false) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn success_range() { + assert_eq!(classify(200, b""), DavOutcome::Success); + assert_eq!(classify(207, b""), DavOutcome::Success); + assert_eq!(classify(299, b""), DavOutcome::Success); + } + + #[test] + fn auth_status() { + assert_eq!(classify(401, b""), DavOutcome::Auth); + assert_eq!(classify(403, b""), DavOutcome::Auth); + } + + #[test] + fn vanished_status() { + assert_eq!(classify(404, b""), DavOutcome::Vanished); + assert_eq!(classify(410, b""), DavOutcome::Vanished); + } + + #[test] + fn locked_and_quota_are_retryable() { + assert_eq!(classify(423, b""), DavOutcome::Retryable); + assert_eq!(classify(429, b""), DavOutcome::Retryable); + assert_eq!(classify(503, b""), DavOutcome::Retryable); + assert_eq!(classify(507, b""), DavOutcome::Retryable); + } + + #[test] + fn google_usage_limit_403_is_retryable() { + let body = + br#"{"error":{"errors":[{"domain":"usageLimits","reason":"rateLimitExceeded"}]}}"#; + assert_eq!(classify(403, body), DavOutcome::Retryable); + } + + #[test] + fn non_google_403_is_auth() { + let body = br#"{"error":{"errors":[{"domain":"global","reason":"forbidden"}]}}"#; + assert_eq!(classify(403, body), DavOutcome::Auth); + } + + #[test] + fn fatal_4xx_other() { + assert_eq!(classify(400, b""), DavOutcome::Fatal); + assert_eq!(classify(405, b""), DavOutcome::Fatal); + assert_eq!(classify(412, b""), DavOutcome::Fatal); + } +} diff --git a/src/dav/xml.rs b/src/dav/xml.rs new file mode 100644 index 0000000..791a523 --- /dev/null +++ b/src/dav/xml.rs @@ -0,0 +1,251 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::dav::href::Href; + +pub const NS_DAV: &str = "DAV:"; +pub const NS_CALDAV: &str = "urn:ietf:params:xml:ns:caldav"; +pub const NS_CARDDAV: &str = "urn:ietf:params:xml:ns:carddav"; +pub const NS_APPLE_ICAL: &str = "http://apple.com/ns/ical/"; + +pub fn propfind_current_user_principal() -> String { + String::from( + "\n\ + \ + \ + ", + ) +} + +pub fn propfind_calendar_home_set() -> String { + String::from( + "\n\ + \ + \ + ", + ) +} + +pub fn propfind_addressbook_home_set() -> String { + String::from( + "\n\ + \ + \ + ", + ) +} + +pub fn propfind_principal_and_calendar_home_set() -> String { + String::from( + "\n\ + \ + \ + \ + \ + \ + ", + ) +} + +pub fn propfind_principal_and_addressbook_home_set() -> String { + String::from( + "\n\ + \ + \ + \ + \ + \ + ", + ) +} + +pub fn propfind_calendar_collections() -> String { + String::from( + "\n\ + \ + \ + \ + \ + \ + \ + \ + \ + \ + \ + ", + ) +} + +pub fn propfind_addressbook_collections() -> String { + String::from( + "\n\ + \ + \ + \ + \ + \ + \ + ", + ) +} + +pub fn propfind_webdav_listing() -> String { + String::from( + "\n\ + \ + \ + \ + \ + \ + \ + \ + \ + \ + \ + ", + ) +} + +pub fn propfind_dav_items() -> String { + String::from( + "\n\ + \ + \ + \ + \ + \ + \ + \ + ", + ) +} + +pub fn calendar_multiget(hrefs: &[Href]) -> String { + let mut out = String::with_capacity(256 + hrefs.len() * 64); + out.push_str( + "\n\ + \ + \ + \ + \ + ", + ); + for h in hrefs { + write_href_element(&mut out, h); + } + out.push_str(""); + out +} + +pub fn addressbook_multiget(hrefs: &[Href]) -> String { + let mut out = String::with_capacity(256 + hrefs.len() * 64); + out.push_str( + "\n\ + \ + \ + \ + \ + ", + ); + for h in hrefs { + write_href_element(&mut out, h); + } + out.push_str(""); + out +} + +fn write_href_element(out: &mut String, h: &Href) { + out.push_str(""); + write_escaped(out, h.as_str()); + out.push_str(""); +} + +fn write_escaped(out: &mut String, s: &str) { + for ch in s.chars() { + match ch { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\'' => out.push_str("'"), + _ => out.push(ch), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dav::href::Href; + + #[test] + fn principal_body_matches_expected_shape() { + let body = propfind_current_user_principal(); + assert!(body.starts_with("")); + assert!(body.contains("xmlns:d=\"DAV:\"")); + } + + #[test] + fn calendar_home_set_includes_caldav_namespace() { + let body = propfind_calendar_home_set(); + assert!(body.contains("xmlns:c=\"urn:ietf:params:xml:ns:caldav\"")); + assert!(body.contains("")); + } + + #[test] + fn addressbook_home_set_includes_carddav_namespace() { + let body = propfind_addressbook_home_set(); + assert!(body.contains("xmlns:c=\"urn:ietf:params:xml:ns:carddav\"")); + assert!(body.contains("")); + } + + #[test] + fn calendar_multiget_lists_hrefs_in_order() { + let hrefs = vec![ + Href::from_normalised("/dav/cal/u/d/a.ics".to_owned()), + Href::from_normalised("/dav/cal/u/d/b.ics".to_owned()), + ]; + let body = calendar_multiget(&hrefs); + let a_pos = body.find("/a.ics").unwrap(); + let b_pos = body.find("/b.ics").unwrap(); + assert!(a_pos < b_pos); + assert!(body.contains("")); + } + + #[test] + fn addressbook_multiget_emits_address_data_element() { + let hrefs = vec![Href::from_normalised("/dav/card/u/d/a.vcf".to_owned())]; + let body = addressbook_multiget(&hrefs); + assert!(body.contains("")); + assert!(body.contains("/dav/card/u/d/a.vcf")); + } + + #[test] + fn escape_handles_xml_metacharacters() { + let h = Href::from_normalised("/dav/cal/u/d/<weird>.ics".to_owned()); + let body = calendar_multiget(&[h]); + assert!(body.contains("&lt;weird&gt;.ics")); + } + + #[test] + fn webdav_listing_requests_filesystem_props() { + let body = propfind_webdav_listing(); + assert!(body.contains("")); + assert!(body.contains("")); + assert!(body.contains("")); + assert!(body.contains("")); + } +} diff --git a/src/db/blobs.rs b/src/db/blobs.rs new file mode 100644 index 0000000..1c91668 --- /dev/null +++ b/src/db/blobs.rs @@ -0,0 +1,111 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::{Connection, OptionalExtension, params}; + +pub fn intern_blob(conn: &Connection, bytes: &[u8]) -> Result { + let hash = blake3::hash(bytes); + let hash_bytes = hash.as_bytes().as_slice(); + conn.execute( + "INSERT OR IGNORE INTO blobs (hash, data) VALUES (?1, ?2)", + params![hash_bytes, bytes], + )?; + conn.query_row( + "SELECT id FROM blobs WHERE hash = ?1", + params![hash_bytes], + |row| row.get(0), + ) +} + +pub fn blob_bytes(conn: &Connection, id: i64) -> Result>, rusqlite::Error> { + conn.query_row("SELECT data FROM blobs WHERE id = ?1", params![id], |row| { + row.get(0) + }) + .optional() +} + +pub fn gc_orphan_blobs(conn: &Connection) -> Result { + conn.execute( + "DELETE FROM blobs WHERE id NOT IN ( + SELECT blob_id FROM emails + UNION SELECT blob_id FROM sieve_scripts + UNION SELECT blob_id FROM file_nodes WHERE blob_id IS NOT NULL + UNION SELECT json_tree.atom FROM contact_cards, json_tree(contact_cards.data) + WHERE json_tree.key = '@blob' + UNION SELECT json_tree.atom FROM calendar_events, json_tree(calendar_events.data) + WHERE json_tree.key = '@blob')", + [], + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::init; + + fn mem() -> Connection { + let c = Connection::open_in_memory().unwrap(); + init::apply_schema(&c).unwrap(); + c + } + + #[test] + fn intern_dedups_identical_bytes() { + let c = mem(); + let a = intern_blob(&c, b"hello world").unwrap(); + let b = intern_blob(&c, b"hello world").unwrap(); + let other = intern_blob(&c, b"different").unwrap(); + assert_eq!(a, b); + assert_ne!(a, other); + let n: i64 = c + .query_row("SELECT count(*) FROM blobs", [], |r| r.get(0)) + .unwrap(); + assert_eq!(n, 2); + } + + #[test] + fn blob_bytes_roundtrip() { + let c = mem(); + let id = intern_blob(&c, b"payload").unwrap(); + assert_eq!( + blob_bytes(&c, id).unwrap().as_deref(), + Some(&b"payload"[..]) + ); + assert_eq!(blob_bytes(&c, 9999).unwrap(), None); + } + + #[test] + fn gc_keeps_referenced_and_reaps_orphans() { + let c = mem(); + let referenced = intern_blob(&c, b"keep me").unwrap(); + let _orphan = intern_blob(&c, b"reap me").unwrap(); + c.execute( + "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords) + VALUES (?1, '2020-01-01T00:00:00Z', '[1]', '[]')", + params![referenced], + ) + .unwrap(); + let removed = gc_orphan_blobs(&c).unwrap(); + assert_eq!(removed, 1); + assert!(blob_bytes(&c, referenced).unwrap().is_some()); + } + + #[test] + fn gc_follows_blob_sentinel_in_json_data() { + let c = mem(); + let photo = intern_blob(&c, b"photo bytes").unwrap(); + let _orphan = intern_blob(&c, b"orphan").unwrap(); + c.execute( + "INSERT INTO contact_cards (uid, address_book_ids, data) + VALUES ('u1', '[1]', ?1)", + params![format!(r#"{{"photos":{{"p":{{"@blob":{photo}}}}}}}"#)], + ) + .unwrap(); + let removed = gc_orphan_blobs(&c).unwrap(); + assert_eq!(removed, 1); + assert!(blob_bytes(&c, photo).unwrap().is_some()); + } +} diff --git a/src/db/dav_ids.rs b/src/db/dav_ids.rs new file mode 100644 index 0000000..a7cf537 --- /dev/null +++ b/src/db/dav_ids.rs @@ -0,0 +1,336 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use rusqlite::{Connection, OptionalExtension, params}; + +pub const CALENDAR: &str = "calendar"; +pub const CALENDAR_EVENT: &str = "calendarevent"; +pub const ADDRESS_BOOK: &str = "addressbook"; +pub const CONTACT_CARD: &str = "contactcard"; +pub const FILE_NODE: &str = "filenode"; + +#[derive(Debug, Clone)] +pub struct ItemRow { + pub item_href: String, + pub etag: String, + pub local_id: i64, +} + +pub fn insert( + conn: &Connection, + source_id: i64, + type_name: &str, + collection_href: &str, + item_href: &str, + etag: &str, + local_id: i64, +) -> Result<(), rusqlite::Error> { + conn.execute( + "INSERT INTO sync_id_dav + (source_id, type_name, collection_href, item_href, etag, local_id) + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + source_id, + type_name, + collection_href, + item_href, + etag, + local_id + ], + )?; + Ok(()) +} + +pub fn update_etag( + conn: &Connection, + source_id: i64, + type_name: &str, + item_href: &str, + etag: &str, +) -> Result<(), rusqlite::Error> { + conn.execute( + "UPDATE sync_id_dav + SET etag = ?4 + WHERE source_id = ?1 AND type_name = ?2 AND item_href = ?3", + params![source_id, type_name, item_href, etag], + )?; + Ok(()) +} + +pub fn local_for_item( + conn: &Connection, + source_id: i64, + type_name: &str, + item_href: &str, +) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT local_id FROM sync_id_dav + WHERE source_id = ?1 AND type_name = ?2 AND item_href = ?3", + params![source_id, type_name, item_href], + |row| row.get(0), + ) + .optional() +} + +pub fn items_in_collection( + conn: &Connection, + source_id: i64, + type_name: &str, + collection_href: &str, +) -> Result, rusqlite::Error> { + let mut stmt = conn.prepare( + "SELECT item_href, etag, local_id FROM sync_id_dav + WHERE source_id = ?1 AND type_name = ?2 AND collection_href = ?3", + )?; + let rows = stmt.query_map(params![source_id, type_name, collection_href], |row| { + Ok(ItemRow { + item_href: row.get(0)?, + etag: row.get(1)?, + local_id: row.get(2)?, + }) + })?; + let mut out = Vec::new(); + for r in rows { + out.push(r?); + } + Ok(out) +} + +pub fn collections_of_type( + conn: &Connection, + source_id: i64, + type_name: &str, +) -> Result, rusqlite::Error> { + let mut stmt = conn.prepare( + "SELECT item_href, local_id FROM sync_id_dav + WHERE source_id = ?1 AND type_name = ?2", + )?; + let rows = stmt.query_map(params![source_id, type_name], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + })?; + let mut map = HashMap::new(); + for r in rows { + let (href, id) = r?; + map.insert(href, id); + } + Ok(map) +} + +pub fn delete_item( + conn: &Connection, + source_id: i64, + type_name: &str, + item_href: &str, +) -> Result<(), rusqlite::Error> { + conn.execute( + "DELETE FROM sync_id_dav + WHERE source_id = ?1 AND type_name = ?2 AND item_href = ?3", + params![source_id, type_name, item_href], + )?; + Ok(()) +} + +pub fn delete_collection( + conn: &Connection, + source_id: i64, + type_name: &str, + collection_href: &str, +) -> Result { + let removed = conn.execute( + "DELETE FROM sync_id_dav + WHERE source_id = ?1 AND type_name = ?2 + AND (item_href = ?3 OR collection_href = ?3)", + params![source_id, type_name, collection_href], + )?; + Ok(removed) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::init; + use crate::db::sources::{SourceKey, upsert_source}; + + fn setup() -> (rusqlite::Connection, i64) { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + init::apply_schema(&conn).unwrap(); + let key = SourceKey { + kind: "caldav".to_owned(), + session_url: "https://x/".to_owned(), + account_id: "u".to_owned(), + }; + let sid = upsert_source(&conn, &key, Some("u"), "u").unwrap(); + (conn, sid) + } + + #[test] + fn insert_and_lookup_round_trip() { + let (conn, sid) = setup(); + insert( + &conn, + sid, + CALENDAR_EVENT, + "/cal/a/", + "/cal/a/e.ics", + "\"v1\"", + 7, + ) + .unwrap(); + let id = local_for_item(&conn, sid, CALENDAR_EVENT, "/cal/a/e.ics") + .unwrap() + .unwrap(); + assert_eq!(id, 7); + } + + #[test] + fn replace_keeps_local_id_on_etag_change() { + let (conn, sid) = setup(); + insert( + &conn, + sid, + CALENDAR_EVENT, + "/cal/a/", + "/cal/a/e.ics", + "\"v1\"", + 7, + ) + .unwrap(); + update_etag(&conn, sid, CALENDAR_EVENT, "/cal/a/e.ics", "\"v2\"").unwrap(); + let rows = items_in_collection(&conn, sid, CALENDAR_EVENT, "/cal/a/").unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].etag, "\"v2\""); + assert_eq!(rows[0].local_id, 7); + } + + #[test] + fn items_in_collection_filters_by_collection_href() { + let (conn, sid) = setup(); + insert( + &conn, + sid, + CALENDAR_EVENT, + "/cal/a/", + "/cal/a/e1.ics", + "v1", + 1, + ) + .unwrap(); + insert( + &conn, + sid, + CALENDAR_EVENT, + "/cal/a/", + "/cal/a/e2.ics", + "v2", + 2, + ) + .unwrap(); + insert( + &conn, + sid, + CALENDAR_EVENT, + "/cal/b/", + "/cal/b/e3.ics", + "v3", + 3, + ) + .unwrap(); + let rows = items_in_collection(&conn, sid, CALENDAR_EVENT, "/cal/a/").unwrap(); + assert_eq!(rows.len(), 2); + } + + #[test] + fn delete_item_removes_only_one_row() { + let (conn, sid) = setup(); + insert( + &conn, + sid, + CALENDAR_EVENT, + "/cal/a/", + "/cal/a/e1.ics", + "v", + 1, + ) + .unwrap(); + insert( + &conn, + sid, + CALENDAR_EVENT, + "/cal/a/", + "/cal/a/e2.ics", + "v", + 2, + ) + .unwrap(); + delete_item(&conn, sid, CALENDAR_EVENT, "/cal/a/e1.ics").unwrap(); + let rows = items_in_collection(&conn, sid, CALENDAR_EVENT, "/cal/a/").unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].item_href, "/cal/a/e2.ics"); + } + + #[test] + fn delete_collection_removes_collection_row_and_items() { + let (conn, sid) = setup(); + insert(&conn, sid, CALENDAR, "/cal/a/", "/cal/a/", "", 10).unwrap(); + insert( + &conn, + sid, + CALENDAR_EVENT, + "/cal/a/", + "/cal/a/e.ics", + "v", + 1, + ) + .unwrap(); + let removed = delete_collection(&conn, sid, CALENDAR_EVENT, "/cal/a/").unwrap(); + assert_eq!(removed, 1); + let rows = items_in_collection(&conn, sid, CALENDAR_EVENT, "/cal/a/").unwrap(); + assert!(rows.is_empty()); + } + + #[test] + fn collections_of_type_lists_all_known_collections() { + let (conn, sid) = setup(); + insert(&conn, sid, CALENDAR, "/cal/a/", "/cal/a/", "", 1).unwrap(); + insert(&conn, sid, CALENDAR, "/cal/b/", "/cal/b/", "", 2).unwrap(); + let map = collections_of_type(&conn, sid, CALENDAR).unwrap(); + assert_eq!(map.len(), 2); + assert_eq!(map.get("/cal/a/"), Some(&1)); + assert_eq!(map.get("/cal/b/"), Some(&2)); + } + + #[test] + fn unique_local_id_per_type_enforced() { + let (conn, sid) = setup(); + insert( + &conn, + sid, + CALENDAR_EVENT, + "/cal/a/", + "/cal/a/e1.ics", + "v", + 1, + ) + .unwrap(); + let err = insert( + &conn, + sid, + CALENDAR_EVENT, + "/cal/a/", + "/cal/a/e2.ics", + "v", + 1, + ) + .unwrap_err(); + let msg = format!("{err}"); + assert!( + msg.contains("UNIQUE"), + "expected UNIQUE violation, got {msg}" + ); + } +} diff --git a/src/db/exchange_ews_ids.rs b/src/db/exchange_ews_ids.rs new file mode 100644 index 0000000..05d68b0 --- /dev/null +++ b/src/db/exchange_ews_ids.rs @@ -0,0 +1,265 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use rusqlite::{Connection, OptionalExtension, params}; + +pub const MAILBOX: &str = "mailbox"; +pub const CALENDAR: &str = "calendar"; +pub const ADDRESS_BOOK: &str = "addressbook"; +pub const EMAIL: &str = "email"; +pub const CALENDAR_EVENT: &str = "calendarevent"; +pub const CONTACT_CARD: &str = "contactcard"; + +#[derive(Debug, Clone)] +pub struct ItemRow { + pub item_id: String, + pub change_key: String, + pub local_id: i64, +} + +#[derive(Debug, Clone)] +pub struct FolderRow { + pub item_id: String, + pub folder_id: String, + pub change_key: String, + pub local_id: i64, +} + +pub fn insert( + conn: &Connection, + source_id: i64, + type_name: &str, + folder_id: &str, + item_id: &str, + change_key: &str, + local_id: i64, +) -> Result<(), rusqlite::Error> { + conn.execute( + "INSERT INTO sync_id_exchange_ews + (source_id, type_name, folder_id, item_id, change_key, sync_state, local_id) + VALUES (?1, ?2, ?3, ?4, ?5, '', ?6)", + params![ + source_id, type_name, folder_id, item_id, change_key, local_id + ], + )?; + Ok(()) +} + +pub fn update_change_key( + conn: &Connection, + source_id: i64, + type_name: &str, + item_id: &str, + change_key: &str, +) -> Result<(), rusqlite::Error> { + conn.execute( + "UPDATE sync_id_exchange_ews + SET change_key = ?4 + WHERE source_id = ?1 AND type_name = ?2 AND item_id = ?3", + params![source_id, type_name, item_id, change_key], + )?; + Ok(()) +} + +pub fn delete_item( + conn: &Connection, + source_id: i64, + type_name: &str, + item_id: &str, +) -> Result<(), rusqlite::Error> { + conn.execute( + "DELETE FROM sync_id_exchange_ews + WHERE source_id = ?1 AND type_name = ?2 AND item_id = ?3", + params![source_id, type_name, item_id], + )?; + Ok(()) +} + +pub fn local_for_item( + conn: &Connection, + source_id: i64, + type_name: &str, + item_id: &str, +) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT local_id FROM sync_id_exchange_ews + WHERE source_id = ?1 AND type_name = ?2 AND item_id = ?3", + params![source_id, type_name, item_id], + |row| row.get(0), + ) + .optional() +} + +pub fn items_in_folder( + conn: &Connection, + source_id: i64, + type_name: &str, + folder_id: &str, +) -> Result, rusqlite::Error> { + let mut stmt = conn.prepare( + "SELECT item_id, change_key, local_id FROM sync_id_exchange_ews + WHERE source_id = ?1 AND type_name = ?2 AND folder_id = ?3", + )?; + let rows = stmt.query_map(params![source_id, type_name, folder_id], |row| { + Ok(ItemRow { + item_id: row.get(0)?, + change_key: row.get(1)?, + local_id: row.get(2)?, + }) + })?; + let mut out = Vec::new(); + for r in rows { + out.push(r?); + } + Ok(out) +} + +pub fn folders_of_type( + conn: &Connection, + source_id: i64, + type_name: &str, +) -> Result, rusqlite::Error> { + let mut stmt = conn.prepare( + "SELECT item_id, folder_id, change_key, local_id FROM sync_id_exchange_ews + WHERE source_id = ?1 AND type_name = ?2", + )?; + let rows = stmt.query_map(params![source_id, type_name], |row| { + Ok(FolderRow { + item_id: row.get(0)?, + folder_id: row.get(1)?, + change_key: row.get(2)?, + local_id: row.get(3)?, + }) + })?; + let mut map = HashMap::new(); + for r in rows { + let f = r?; + map.insert(f.item_id.clone(), f); + } + Ok(map) +} + +pub fn get_sync_state( + conn: &Connection, + source_id: i64, + type_name: &str, + item_id: &str, +) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT sync_state FROM sync_id_exchange_ews + WHERE source_id = ?1 AND type_name = ?2 AND item_id = ?3", + params![source_id, type_name, item_id], + |row| row.get::<_, String>(0), + ) + .optional() +} + +pub fn set_sync_state( + conn: &Connection, + source_id: i64, + type_name: &str, + item_id: &str, + sync_state: &str, +) -> Result<(), rusqlite::Error> { + conn.execute( + "UPDATE sync_id_exchange_ews + SET sync_state = ?4 + WHERE source_id = ?1 AND type_name = ?2 AND item_id = ?3", + params![source_id, type_name, item_id, sync_state], + )?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::init; + use crate::db::sources::{SourceKey, upsert_source}; + + fn setup() -> (rusqlite::Connection, i64) { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + init::apply_schema(&conn).unwrap(); + let key = SourceKey { + kind: "exchange_ews".to_owned(), + session_url: "https://x/EWS/Exchange.asmx".to_owned(), + account_id: "u@d".to_owned(), + }; + let sid = upsert_source(&conn, &key, Some("u"), "u@d").unwrap(); + (conn, sid) + } + + #[test] + fn insert_and_lookup_round_trip() { + let (conn, sid) = setup(); + insert(&conn, sid, EMAIL, "FOLDER1", "ITEM1", "CK1", 42).unwrap(); + let id = local_for_item(&conn, sid, EMAIL, "ITEM1").unwrap().unwrap(); + assert_eq!(id, 42); + let rows = items_in_folder(&conn, sid, EMAIL, "FOLDER1").unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].change_key, "CK1"); + } + + #[test] + fn update_change_key_preserves_local_id() { + let (conn, sid) = setup(); + insert(&conn, sid, EMAIL, "F1", "I1", "A", 7).unwrap(); + update_change_key(&conn, sid, EMAIL, "I1", "B").unwrap(); + let rows = items_in_folder(&conn, sid, EMAIL, "F1").unwrap(); + assert_eq!(rows[0].change_key, "B"); + assert_eq!(rows[0].local_id, 7); + } + + #[test] + fn folders_of_type_lists_by_item_id() { + let (conn, sid) = setup(); + insert(&conn, sid, MAILBOX, "", "F1", "CK1", 1).unwrap(); + insert(&conn, sid, MAILBOX, "F1", "F2", "CK2", 2).unwrap(); + let map = folders_of_type(&conn, sid, MAILBOX).unwrap(); + assert_eq!(map.len(), 2); + assert_eq!(map.get("F1").map(|r| r.local_id), Some(1)); + assert_eq!(map.get("F2").map(|r| r.local_id), Some(2)); + } + + #[test] + fn sync_state_round_trips() { + let (conn, sid) = setup(); + insert(&conn, sid, MAILBOX, "", "F1", "", 1).unwrap(); + assert_eq!( + get_sync_state(&conn, sid, MAILBOX, "F1") + .unwrap() + .as_deref(), + Some("") + ); + set_sync_state(&conn, sid, MAILBOX, "F1", "opaque-state").unwrap(); + assert_eq!( + get_sync_state(&conn, sid, MAILBOX, "F1") + .unwrap() + .as_deref(), + Some("opaque-state") + ); + } + + #[test] + fn delete_item_removes_only_one() { + let (conn, sid) = setup(); + insert(&conn, sid, EMAIL, "F1", "I1", "A", 1).unwrap(); + insert(&conn, sid, EMAIL, "F1", "I2", "A", 2).unwrap(); + delete_item(&conn, sid, EMAIL, "I1").unwrap(); + let rows = items_in_folder(&conn, sid, EMAIL, "F1").unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].item_id, "I2"); + } + + #[test] + fn unique_local_id_per_type_is_enforced() { + let (conn, sid) = setup(); + insert(&conn, sid, EMAIL, "F1", "I1", "A", 1).unwrap(); + let err = insert(&conn, sid, EMAIL, "F1", "I2", "A", 1).unwrap_err(); + assert!(err.to_string().contains("UNIQUE")); + } +} diff --git a/src/db/exchange_graph_ids.rs b/src/db/exchange_graph_ids.rs new file mode 100644 index 0000000..815bc1e --- /dev/null +++ b/src/db/exchange_graph_ids.rs @@ -0,0 +1,167 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use rusqlite::{Connection, OptionalExtension, params}; + +pub const MAILBOX: &str = "mailbox"; +pub const EMAIL: &str = "email"; +pub const CALENDAR: &str = "calendar"; +pub const CALENDAR_EVENT: &str = "calendarevent"; +pub const ADDRESS_BOOK: &str = "addressbook"; +pub const CONTACT_CARD: &str = "contactcard"; + +#[derive(Debug, Clone)] +pub struct IdRow { + pub graph_id: String, + pub local_id: i64, +} + +pub fn insert( + conn: &Connection, + source_id: i64, + type_name: &str, + graph_id: &str, + local_id: i64, +) -> Result<(), rusqlite::Error> { + conn.execute( + "INSERT INTO sync_id_exchange_graph (source_id, type_name, graph_id, local_id) + VALUES (?1, ?2, ?3, ?4)", + params![source_id, type_name, graph_id, local_id], + )?; + Ok(()) +} + +pub fn delete( + conn: &Connection, + source_id: i64, + type_name: &str, + graph_id: &str, +) -> Result<(), rusqlite::Error> { + conn.execute( + "DELETE FROM sync_id_exchange_graph + WHERE source_id = ?1 AND type_name = ?2 AND graph_id = ?3", + params![source_id, type_name, graph_id], + )?; + Ok(()) +} + +pub fn local_for_graph_id( + conn: &Connection, + source_id: i64, + type_name: &str, + graph_id: &str, +) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT local_id FROM sync_id_exchange_graph + WHERE source_id = ?1 AND type_name = ?2 AND graph_id = ?3", + params![source_id, type_name, graph_id], + |row| row.get(0), + ) + .optional() +} + +pub fn ids_of_type( + conn: &Connection, + source_id: i64, + type_name: &str, +) -> Result, rusqlite::Error> { + let mut stmt = conn.prepare( + "SELECT graph_id, local_id FROM sync_id_exchange_graph + WHERE source_id = ?1 AND type_name = ?2", + )?; + let rows = stmt.query_map(params![source_id, type_name], |row| { + Ok(IdRow { + graph_id: row.get(0)?, + local_id: row.get(1)?, + }) + })?; + let mut map = HashMap::new(); + for r in rows { + let row = r?; + map.insert(row.graph_id, row.local_id); + } + Ok(map) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::init; + use crate::db::sources::{SourceKey, upsert_source}; + + fn setup() -> (Connection, i64) { + let conn = Connection::open_in_memory().unwrap(); + init::apply_schema(&conn).unwrap(); + let key = SourceKey { + kind: "exchange_graph".to_owned(), + session_url: "https://x|https://graph.microsoft.com/v1.0".to_owned(), + account_id: "u-uuid".to_owned(), + }; + let sid = upsert_source(&conn, &key, Some("u@d"), "u@d").unwrap(); + (conn, sid) + } + + #[test] + fn insert_and_lookup_round_trip() { + let (conn, sid) = setup(); + insert(&conn, sid, EMAIL, "AAkA-base64==", 17).unwrap(); + let id = local_for_graph_id(&conn, sid, EMAIL, "AAkA-base64==") + .unwrap() + .unwrap(); + assert_eq!(id, 17); + } + + #[test] + fn graph_id_is_case_sensitive() { + let (conn, sid) = setup(); + insert(&conn, sid, EMAIL, "AaBb", 1).unwrap(); + assert!( + local_for_graph_id(&conn, sid, EMAIL, "aabb") + .unwrap() + .is_none() + ); + assert!( + local_for_graph_id(&conn, sid, EMAIL, "AaBb") + .unwrap() + .is_some() + ); + } + + #[test] + fn unique_local_id_per_type_is_enforced() { + let (conn, sid) = setup(); + insert(&conn, sid, EMAIL, "ID-A", 1).unwrap(); + let err = insert(&conn, sid, EMAIL, "ID-B", 1).unwrap_err(); + assert!(err.to_string().contains("UNIQUE")); + } + + #[test] + fn ids_of_type_filters_by_type() { + let (conn, sid) = setup(); + insert(&conn, sid, EMAIL, "M1", 1).unwrap(); + insert(&conn, sid, EMAIL, "M2", 2).unwrap(); + insert(&conn, sid, MAILBOX, "F1", 3).unwrap(); + let emails = ids_of_type(&conn, sid, EMAIL).unwrap(); + let folders = ids_of_type(&conn, sid, MAILBOX).unwrap(); + assert_eq!(emails.len(), 2); + assert_eq!(folders.len(), 1); + assert_eq!(emails.get("M1"), Some(&1)); + assert_eq!(folders.get("F1"), Some(&3)); + } + + #[test] + fn delete_removes_only_one() { + let (conn, sid) = setup(); + insert(&conn, sid, EMAIL, "M1", 1).unwrap(); + insert(&conn, sid, EMAIL, "M2", 2).unwrap(); + delete(&conn, sid, EMAIL, "M1").unwrap(); + let remaining = ids_of_type(&conn, sid, EMAIL).unwrap(); + assert_eq!(remaining.len(), 1); + assert!(remaining.contains_key("M2")); + } +} diff --git a/src/db/ids.rs b/src/db/ids.rs new file mode 100644 index 0000000..aa7b34d --- /dev/null +++ b/src/db/ids.rs @@ -0,0 +1,114 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use rusqlite::{Connection, OptionalExtension, params}; + +use crate::types::ObjectType; + +pub fn insert( + conn: &Connection, + source_id: i64, + ty: ObjectType, + jmap_id: &str, + local_id: i64, +) -> Result<(), rusqlite::Error> { + conn.execute( + "INSERT OR REPLACE INTO sync_id_jmap (source_id, type_name, jmap_id, local_id) + VALUES (?1, ?2, ?3, ?4)", + params![source_id, ty.jmap_name(), jmap_id, local_id], + )?; + Ok(()) +} + +pub fn local_for( + conn: &Connection, + source_id: i64, + ty: ObjectType, + jmap_id: &str, +) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT local_id FROM sync_id_jmap + WHERE source_id = ?1 AND type_name = ?2 AND jmap_id = ?3", + params![source_id, ty.jmap_name(), jmap_id], + |row| row.get(0), + ) + .optional() +} + +pub fn jmap_to_local( + conn: &Connection, + source_id: i64, + ty: ObjectType, +) -> Result, rusqlite::Error> { + let mut stmt = conn.prepare( + "SELECT jmap_id, local_id FROM sync_id_jmap WHERE source_id = ?1 AND type_name = ?2", + )?; + let rows = stmt.query_map(params![source_id, ty.jmap_name()], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + })?; + let mut map = HashMap::new(); + for r in rows { + let (j, l) = r?; + map.insert(j, l); + } + Ok(map) +} + +pub fn delete( + conn: &Connection, + source_id: i64, + ty: ObjectType, + jmap_id: &str, +) -> Result<(), rusqlite::Error> { + conn.execute( + "DELETE FROM sync_id_jmap + WHERE source_id = ?1 AND type_name = ?2 AND jmap_id = ?3", + params![source_id, ty.jmap_name(), jmap_id], + )?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::init; + use crate::db::sources::{SourceKey, upsert_source}; + + fn setup() -> (Connection, i64) { + let c = Connection::open_in_memory().unwrap(); + init::apply_schema(&c).unwrap(); + let sid = upsert_source( + &c, + &SourceKey { + kind: "jmap".to_owned(), + session_url: "u".to_owned(), + account_id: "w".to_owned(), + }, + None, + "alice", + ) + .unwrap(); + (c, sid) + } + + #[test] + fn crud_roundtrip() { + let (c, sid) = setup(); + insert(&c, sid, ObjectType::Mailbox, "M1", 1).unwrap(); + insert(&c, sid, ObjectType::Mailbox, "M2", 2).unwrap(); + assert_eq!( + local_for(&c, sid, ObjectType::Mailbox, "M1").unwrap(), + Some(1) + ); + let map = jmap_to_local(&c, sid, ObjectType::Mailbox).unwrap(); + assert_eq!(map.len(), 2); + assert_eq!(map.get("M2"), Some(&2)); + delete(&c, sid, ObjectType::Mailbox, "M1").unwrap(); + assert_eq!(local_for(&c, sid, ObjectType::Mailbox, "M1").unwrap(), None); + } +} diff --git a/src/db/imap_ids.rs b/src/db/imap_ids.rs new file mode 100644 index 0000000..cb784f5 --- /dev/null +++ b/src/db/imap_ids.rs @@ -0,0 +1,292 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; + +use rusqlite::{Connection, OptionalExtension, params}; + +const MAILBOX: &str = "mailbox"; +const EMAIL: &str = "email"; + +pub fn insert_mailbox( + conn: &Connection, + source_id: i64, + folder: &str, + local_id: i64, +) -> Result<(), rusqlite::Error> { + conn.execute( + "INSERT OR REPLACE INTO sync_id_imap + (source_id, type_name, folder, uidvalidity, uid, local_id) + VALUES (?1, ?2, ?3, 0, 0, ?4)", + params![source_id, MAILBOX, folder, local_id], + )?; + Ok(()) +} + +pub fn insert_email( + conn: &Connection, + source_id: i64, + folder: &str, + uidvalidity: u32, + uid: u32, + local_id: i64, +) -> Result<(), rusqlite::Error> { + conn.execute( + "INSERT OR REPLACE INTO sync_id_imap + (source_id, type_name, folder, uidvalidity, uid, local_id) + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![source_id, EMAIL, folder, uidvalidity, uid, local_id], + )?; + Ok(()) +} + +pub fn local_for_mailbox( + conn: &Connection, + source_id: i64, + folder: &str, +) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT local_id FROM sync_id_imap + WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3 + AND uidvalidity = 0 AND uid = 0", + params![source_id, MAILBOX, folder], + |row| row.get(0), + ) + .optional() +} + +pub fn local_for_email( + conn: &Connection, + source_id: i64, + folder: &str, + uidvalidity: u32, + uid: u32, +) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT local_id FROM sync_id_imap + WHERE source_id = ?1 AND type_name = ?2 + AND folder = ?3 AND uidvalidity = ?4 AND uid = ?5", + params![source_id, EMAIL, folder, uidvalidity, uid], + |row| row.get(0), + ) + .optional() +} + +pub fn mailbox_folders( + conn: &Connection, + source_id: i64, +) -> Result, rusqlite::Error> { + let mut stmt = conn.prepare( + "SELECT folder, local_id FROM sync_id_imap + WHERE source_id = ?1 AND type_name = ?2", + )?; + let rows = stmt.query_map(params![source_id, MAILBOX], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + })?; + let mut map = HashMap::new(); + for r in rows { + let (f, l) = r?; + map.insert(f, l); + } + Ok(map) +} + +pub fn email_uids_in_folder( + conn: &Connection, + source_id: i64, + folder: &str, +) -> Result, rusqlite::Error> { + let mut stmt = conn.prepare( + "SELECT uidvalidity, uid, local_id FROM sync_id_imap + WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3", + )?; + let rows = stmt.query_map(params![source_id, EMAIL, folder], |row| { + Ok(( + row.get::<_, u32>(0)?, + row.get::<_, u32>(1)?, + row.get::<_, i64>(2)?, + )) + })?; + let mut map = HashMap::new(); + for r in rows { + let (v, u, l) = r?; + map.insert((v, u), l); + } + Ok(map) +} + +pub fn delete_mailbox( + conn: &Connection, + source_id: i64, + folder: &str, +) -> Result<(), rusqlite::Error> { + conn.execute( + "DELETE FROM sync_id_imap + WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3 + AND uidvalidity = 0 AND uid = 0", + params![source_id, MAILBOX, folder], + )?; + Ok(()) +} + +pub fn delete_email( + conn: &Connection, + source_id: i64, + folder: &str, + uidvalidity: u32, + uid: u32, +) -> Result<(), rusqlite::Error> { + conn.execute( + "DELETE FROM sync_id_imap + WHERE source_id = ?1 AND type_name = ?2 + AND folder = ?3 AND uidvalidity = ?4 AND uid = ?5", + params![source_id, EMAIL, folder, uidvalidity, uid], + )?; + Ok(()) +} + +pub fn delete_all_emails_in_folder( + conn: &Connection, + source_id: i64, + folder: &str, +) -> Result { + conn.execute( + "DELETE FROM sync_id_imap + WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3", + params![source_id, EMAIL, folder], + ) +} + +pub fn email_uids_at_validity( + conn: &Connection, + source_id: i64, + folder: &str, + uidvalidity: u32, +) -> Result, rusqlite::Error> { + let mut stmt = conn.prepare( + "SELECT uid FROM sync_id_imap + WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3 + AND uidvalidity = ?4 + ORDER BY uid", + )?; + let rows = stmt.query_map(params![source_id, EMAIL, folder, uidvalidity], |row| { + row.get::<_, u32>(0) + })?; + let mut out = Vec::new(); + for r in rows { + out.push(r?); + } + Ok(out) +} + +pub fn folders_with_emails( + conn: &Connection, + source_id: i64, +) -> Result, rusqlite::Error> { + let mut stmt = conn.prepare( + "SELECT DISTINCT folder FROM sync_id_imap + WHERE source_id = ?1 AND type_name = ?2", + )?; + let rows = stmt.query_map(params![source_id, EMAIL], |row| row.get::<_, String>(0))?; + let mut out = HashSet::new(); + for r in rows { + out.insert(r?); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::init; + use crate::db::sources::{SourceKey, upsert_source}; + + fn setup() -> (Connection, i64) { + let c = Connection::open_in_memory().unwrap(); + init::apply_schema(&c).unwrap(); + let sid = upsert_source( + &c, + &SourceKey { + kind: "imap".to_owned(), + session_url: "imaps://host:993".to_owned(), + account_id: "alice".to_owned(), + }, + None, + "alice", + ) + .unwrap(); + (c, sid) + } + + #[test] + fn mailbox_roundtrip() { + let (c, sid) = setup(); + insert_mailbox(&c, sid, "INBOX", 7).unwrap(); + insert_mailbox(&c, sid, "INBOX/Sub", 8).unwrap(); + assert_eq!(local_for_mailbox(&c, sid, "INBOX").unwrap(), Some(7)); + let all = mailbox_folders(&c, sid).unwrap(); + assert_eq!(all.len(), 2); + assert_eq!(all.get("INBOX/Sub"), Some(&8)); + delete_mailbox(&c, sid, "INBOX").unwrap(); + assert_eq!(local_for_mailbox(&c, sid, "INBOX").unwrap(), None); + } + + #[test] + fn email_roundtrip() { + let (c, sid) = setup(); + insert_email(&c, sid, "INBOX", 12345, 100, 1).unwrap(); + insert_email(&c, sid, "INBOX", 12345, 101, 2).unwrap(); + insert_email(&c, sid, "Sent", 7777, 1, 3).unwrap(); + assert_eq!( + local_for_email(&c, sid, "INBOX", 12345, 100).unwrap(), + Some(1) + ); + let inbox = email_uids_in_folder(&c, sid, "INBOX").unwrap(); + assert_eq!(inbox.len(), 2); + assert_eq!(inbox.get(&(12345, 100)), Some(&1)); + let folders = folders_with_emails(&c, sid).unwrap(); + assert!(folders.contains("INBOX")); + assert!(folders.contains("Sent")); + } + + #[test] + fn delete_all_emails_in_folder_wipes_only_that_folder() { + let (c, sid) = setup(); + insert_email(&c, sid, "INBOX", 1, 10, 1).unwrap(); + insert_email(&c, sid, "INBOX", 1, 11, 2).unwrap(); + insert_email(&c, sid, "Sent", 1, 10, 3).unwrap(); + let removed = delete_all_emails_in_folder(&c, sid, "INBOX").unwrap(); + assert_eq!(removed, 2); + assert!(email_uids_in_folder(&c, sid, "INBOX").unwrap().is_empty()); + assert_eq!(email_uids_in_folder(&c, sid, "Sent").unwrap().len(), 1); + } + + #[test] + fn re_insert_with_same_local_id_replaces() { + let (c, sid) = setup(); + insert_mailbox(&c, sid, "A", 1).unwrap(); + insert_mailbox(&c, sid, "B", 1).unwrap(); + assert_eq!(local_for_mailbox(&c, sid, "A").unwrap(), None); + assert_eq!(local_for_mailbox(&c, sid, "B").unwrap(), Some(1)); + let all = mailbox_folders(&c, sid).unwrap(); + assert_eq!(all.len(), 1); + } + + #[test] + fn pk_composite_is_per_type_per_folder_per_uid() { + let (c, sid) = setup(); + insert_email(&c, sid, "INBOX", 1, 10, 1).unwrap(); + insert_email(&c, sid, "INBOX", 1, 10, 1).unwrap(); + assert_eq!(email_uids_in_folder(&c, sid, "INBOX").unwrap().len(), 1); + } + + #[test] + fn mailbox_and_email_namespaces_do_not_collide_on_local_id() { + let (c, sid) = setup(); + insert_mailbox(&c, sid, "INBOX", 42).unwrap(); + insert_email(&c, sid, "INBOX", 1, 1, 42).unwrap(); + } +} diff --git a/src/db/imap_state.rs b/src/db/imap_state.rs new file mode 100644 index 0000000..81a8481 --- /dev/null +++ b/src/db/imap_state.rs @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::{Connection, OptionalExtension, params}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FolderState { + pub uidvalidity: u32, + pub uidnext: u32, + pub last_seen: String, +} + +pub fn get( + conn: &Connection, + source_id: i64, + folder: &str, +) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT uidvalidity, uidnext, last_seen FROM imap_folder_state + WHERE source_id = ?1 AND folder = ?2", + params![source_id, folder], + |row| { + Ok(FolderState { + uidvalidity: row.get(0)?, + uidnext: row.get(1)?, + last_seen: row.get(2)?, + }) + }, + ) + .optional() +} + +pub fn upsert( + conn: &Connection, + source_id: i64, + folder: &str, + uidvalidity: u32, + uidnext: u32, + last_seen_rfc3339: &str, +) -> Result<(), rusqlite::Error> { + conn.execute( + "INSERT INTO imap_folder_state (source_id, folder, uidvalidity, uidnext, last_seen) + VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT (source_id, folder) + DO UPDATE SET uidvalidity = excluded.uidvalidity, + uidnext = excluded.uidnext, + last_seen = excluded.last_seen", + params![source_id, folder, uidvalidity, uidnext, last_seen_rfc3339], + )?; + Ok(()) +} + +pub fn delete(conn: &Connection, source_id: i64, folder: &str) -> Result<(), rusqlite::Error> { + conn.execute( + "DELETE FROM imap_folder_state WHERE source_id = ?1 AND folder = ?2", + params![source_id, folder], + )?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::init; + use crate::db::sources::{SourceKey, upsert_source}; + + fn setup() -> (Connection, i64) { + let c = Connection::open_in_memory().unwrap(); + init::apply_schema(&c).unwrap(); + let sid = upsert_source( + &c, + &SourceKey { + kind: "imap".to_owned(), + session_url: "imaps://host:993".to_owned(), + account_id: "alice".to_owned(), + }, + None, + "alice", + ) + .unwrap(); + (c, sid) + } + + #[test] + fn upsert_then_get_returns_latest() { + let (c, sid) = setup(); + assert!(get(&c, sid, "INBOX").unwrap().is_none()); + upsert(&c, sid, "INBOX", 100, 50, "2026-05-24T00:00:00Z").unwrap(); + let st = get(&c, sid, "INBOX").unwrap().unwrap(); + assert_eq!(st.uidvalidity, 100); + assert_eq!(st.uidnext, 50); + upsert(&c, sid, "INBOX", 100, 75, "2026-05-24T01:00:00Z").unwrap(); + let st = get(&c, sid, "INBOX").unwrap().unwrap(); + assert_eq!(st.uidnext, 75); + assert_eq!(st.last_seen, "2026-05-24T01:00:00Z"); + } + + #[test] + fn delete_removes_only_named_folder() { + let (c, sid) = setup(); + upsert(&c, sid, "INBOX", 1, 1, "2026-05-24T00:00:00Z").unwrap(); + upsert(&c, sid, "Sent", 2, 1, "2026-05-24T00:00:00Z").unwrap(); + delete(&c, sid, "INBOX").unwrap(); + assert!(get(&c, sid, "INBOX").unwrap().is_none()); + assert!(get(&c, sid, "Sent").unwrap().is_some()); + } +} diff --git a/src/db/init.rs b/src/db/init.rs new file mode 100644 index 0000000..bc765f4 --- /dev/null +++ b/src/db/init.rs @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::Connection; + +pub const SCHEMA_SQL: &str = include_str!("schema.sql"); + +pub fn open(path: &std::path::Path) -> Result { + let conn = Connection::open(path)?; + apply_pragmas(&conn)?; + apply_schema(&conn)?; + Ok(conn) +} + +pub fn apply_schema(conn: &Connection) -> Result<(), OpenError> { + let tx = conn.unchecked_transaction()?; + tx.execute_batch(SCHEMA_SQL)?; + ensure_calendar_events_data_type(&tx)?; + tx.commit()?; + Ok(()) +} + +fn ensure_calendar_events_data_type(conn: &Connection) -> Result<(), OpenError> { + let mut stmt = conn.prepare("PRAGMA table_info(calendar_events)")?; + let rows = stmt.query_map([], |row| row.get::<_, String>(1))?; + let has_column = rows.filter_map(|r| r.ok()).any(|name| name == "data_type"); + if !has_column { + conn.execute( + "ALTER TABLE calendar_events ADD COLUMN data_type TEXT NOT NULL DEFAULT 'Event'", + [], + )?; + } + Ok(()) +} + +fn apply_pragmas(conn: &Connection) -> Result<(), OpenError> { + conn.pragma_update(None, "journal_mode", "WAL")?; + conn.pragma_update(None, "foreign_keys", "ON")?; + conn.pragma_update(None, "locking_mode", "EXCLUSIVE")?; + Ok(()) +} + +#[derive(Debug, thiserror::Error)] +pub enum OpenError { + #[error("sqlite error: {0}")] + Sqlite(#[from] rusqlite::Error), +} diff --git a/src/db/maildir_ids.rs b/src/db/maildir_ids.rs new file mode 100644 index 0000000..a3fd905 --- /dev/null +++ b/src/db/maildir_ids.rs @@ -0,0 +1,269 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; + +use rusqlite::{Connection, OptionalExtension, params}; + +const MAILBOX: &str = "mailbox"; +const EMAIL: &str = "email"; + +pub fn insert_mailbox( + conn: &Connection, + source_id: i64, + folder: &str, + local_id: i64, +) -> Result<(), rusqlite::Error> { + conn.execute( + "INSERT INTO sync_id_maildir + (source_id, type_name, folder, unique_id, local_id) + VALUES (?1, ?2, ?3, '', ?4)", + params![source_id, MAILBOX, folder, local_id], + )?; + Ok(()) +} + +pub fn insert_email( + conn: &Connection, + source_id: i64, + folder: &str, + unique_id: &str, + local_id: i64, +) -> Result<(), rusqlite::Error> { + conn.execute( + "INSERT INTO sync_id_maildir + (source_id, type_name, folder, unique_id, local_id) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![source_id, EMAIL, folder, unique_id, local_id], + )?; + Ok(()) +} + +pub fn local_for_mailbox( + conn: &Connection, + source_id: i64, + folder: &str, +) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT local_id FROM sync_id_maildir + WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3 AND unique_id = ''", + params![source_id, MAILBOX, folder], + |row| row.get(0), + ) + .optional() +} + +pub fn local_for_email( + conn: &Connection, + source_id: i64, + folder: &str, + unique_id: &str, +) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT local_id FROM sync_id_maildir + WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3 AND unique_id = ?4", + params![source_id, EMAIL, folder, unique_id], + |row| row.get(0), + ) + .optional() +} + +pub fn mailbox_folders( + conn: &Connection, + source_id: i64, +) -> Result, rusqlite::Error> { + let mut stmt = conn.prepare( + "SELECT folder, local_id FROM sync_id_maildir + WHERE source_id = ?1 AND type_name = ?2 AND unique_id = ''", + )?; + let rows = stmt.query_map(params![source_id, MAILBOX], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + })?; + let mut map = HashMap::new(); + for r in rows { + let (f, l) = r?; + map.insert(f, l); + } + Ok(map) +} + +pub fn email_ids_in_folder( + conn: &Connection, + source_id: i64, + folder: &str, +) -> Result, rusqlite::Error> { + let mut stmt = conn.prepare( + "SELECT unique_id, local_id FROM sync_id_maildir + WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3", + )?; + let rows = stmt.query_map(params![source_id, EMAIL, folder], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + })?; + let mut map = HashMap::new(); + for r in rows { + let (u, l) = r?; + map.insert(u, l); + } + Ok(map) +} + +pub fn delete_mailbox( + conn: &Connection, + source_id: i64, + folder: &str, +) -> Result<(), rusqlite::Error> { + conn.execute( + "DELETE FROM sync_id_maildir + WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3 AND unique_id = ''", + params![source_id, MAILBOX, folder], + )?; + Ok(()) +} + +pub fn delete_email( + conn: &Connection, + source_id: i64, + folder: &str, + unique_id: &str, +) -> Result<(), rusqlite::Error> { + conn.execute( + "DELETE FROM sync_id_maildir + WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3 AND unique_id = ?4", + params![source_id, EMAIL, folder, unique_id], + )?; + Ok(()) +} + +pub fn delete_all_emails_in_folder( + conn: &Connection, + source_id: i64, + folder: &str, +) -> Result { + conn.execute( + "DELETE FROM sync_id_maildir + WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3", + params![source_id, EMAIL, folder], + ) +} + +pub fn folders_with_emails( + conn: &Connection, + source_id: i64, +) -> Result, rusqlite::Error> { + let mut stmt = conn.prepare( + "SELECT DISTINCT folder FROM sync_id_maildir + WHERE source_id = ?1 AND type_name = ?2", + )?; + let rows = stmt.query_map(params![source_id, EMAIL], |row| row.get::<_, String>(0))?; + let mut out = HashSet::new(); + for r in rows { + out.insert(r?); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::init; + use crate::db::sources::{SourceKey, upsert_source}; + + fn setup() -> (Connection, i64) { + let c = Connection::open_in_memory().unwrap(); + init::apply_schema(&c).unwrap(); + let sid = upsert_source( + &c, + &SourceKey { + kind: "maildir".to_owned(), + session_url: "file:///home/alice/Maildir".to_owned(), + account_id: "/home/alice/Maildir".to_owned(), + }, + Some("Maildir"), + "", + ) + .unwrap(); + (c, sid) + } + + #[test] + fn mailbox_roundtrip() { + let (c, sid) = setup(); + insert_mailbox(&c, sid, "INBOX", 7).unwrap(); + insert_mailbox(&c, sid, "Archive.2025", 8).unwrap(); + assert_eq!(local_for_mailbox(&c, sid, "INBOX").unwrap(), Some(7)); + let all = mailbox_folders(&c, sid).unwrap(); + assert_eq!(all.len(), 2); + assert_eq!(all.get("Archive.2025"), Some(&8)); + delete_mailbox(&c, sid, "INBOX").unwrap(); + assert_eq!(local_for_mailbox(&c, sid, "INBOX").unwrap(), None); + } + + #[test] + fn email_roundtrip() { + let (c, sid) = setup(); + insert_email(&c, sid, "INBOX", "1739471123.M001.host", 1).unwrap(); + insert_email(&c, sid, "INBOX", "1739471200.M002.host", 2).unwrap(); + insert_email(&c, sid, "Sent", "1739471300.M003.host", 3).unwrap(); + assert_eq!( + local_for_email(&c, sid, "INBOX", "1739471123.M001.host").unwrap(), + Some(1) + ); + let inbox = email_ids_in_folder(&c, sid, "INBOX").unwrap(); + assert_eq!(inbox.len(), 2); + assert_eq!(inbox.get("1739471123.M001.host"), Some(&1)); + let folders = folders_with_emails(&c, sid).unwrap(); + assert!(folders.contains("INBOX")); + assert!(folders.contains("Sent")); + } + + #[test] + fn delete_email_removes_only_that_row() { + let (c, sid) = setup(); + insert_email(&c, sid, "INBOX", "u1", 1).unwrap(); + insert_email(&c, sid, "INBOX", "u2", 2).unwrap(); + delete_email(&c, sid, "INBOX", "u1").unwrap(); + let inbox = email_ids_in_folder(&c, sid, "INBOX").unwrap(); + assert_eq!(inbox.len(), 1); + assert!(inbox.contains_key("u2")); + } + + #[test] + fn delete_all_emails_in_folder_wipes_only_that_folder() { + let (c, sid) = setup(); + insert_email(&c, sid, "INBOX", "u1", 1).unwrap(); + insert_email(&c, sid, "INBOX", "u2", 2).unwrap(); + insert_email(&c, sid, "Sent", "u3", 3).unwrap(); + let removed = delete_all_emails_in_folder(&c, sid, "INBOX").unwrap(); + assert_eq!(removed, 2); + assert!(email_ids_in_folder(&c, sid, "INBOX").unwrap().is_empty()); + assert_eq!(email_ids_in_folder(&c, sid, "Sent").unwrap().len(), 1); + } + + #[test] + fn mailbox_and_email_namespaces_do_not_collide_on_local_id() { + let (c, sid) = setup(); + insert_mailbox(&c, sid, "INBOX", 42).unwrap(); + insert_email(&c, sid, "INBOX", "u1", 42).unwrap(); + } + + #[test] + fn re_insert_with_same_unique_id_is_rejected_by_pk() { + let (c, sid) = setup(); + insert_email(&c, sid, "INBOX", "u1", 1).unwrap(); + let err = insert_email(&c, sid, "INBOX", "u1", 2).expect_err("PK conflict"); + assert!(matches!(err, rusqlite::Error::SqliteFailure(_, _))); + assert_eq!(local_for_email(&c, sid, "INBOX", "u1").unwrap(), Some(1)); + assert_eq!(email_ids_in_folder(&c, sid, "INBOX").unwrap().len(), 1); + } + + #[test] + fn re_insert_with_same_local_id_is_rejected_by_unique_constraint() { + let (c, sid) = setup(); + insert_email(&c, sid, "INBOX", "u1", 42).unwrap(); + let err = insert_email(&c, sid, "INBOX", "u2", 42).expect_err("UNIQUE conflict"); + assert!(matches!(err, rusqlite::Error::SqliteFailure(_, _))); + } +} diff --git a/src/db/managesieve_ids.rs b/src/db/managesieve_ids.rs new file mode 100644 index 0000000..d55d100 --- /dev/null +++ b/src/db/managesieve_ids.rs @@ -0,0 +1,129 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use rusqlite::{Connection, OptionalExtension, params}; + +pub fn insert( + conn: &Connection, + source_id: i64, + name: &str, + local_id: i64, +) -> Result<(), rusqlite::Error> { + conn.execute( + "INSERT OR REPLACE INTO sync_id_managesieve (source_id, name, local_id) + VALUES (?1, ?2, ?3)", + params![source_id, name, local_id], + )?; + Ok(()) +} + +pub fn local_for( + conn: &Connection, + source_id: i64, + name: &str, +) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT local_id FROM sync_id_managesieve WHERE source_id = ?1 AND name = ?2", + params![source_id, name], + |row| row.get(0), + ) + .optional() +} + +pub fn all_names( + conn: &Connection, + source_id: i64, +) -> Result, rusqlite::Error> { + let mut stmt = + conn.prepare("SELECT name, local_id FROM sync_id_managesieve WHERE source_id = ?1")?; + let rows = stmt.query_map(params![source_id], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + })?; + let mut out = HashMap::new(); + for r in rows { + let (n, l) = r?; + out.insert(n, l); + } + Ok(out) +} + +pub fn delete(conn: &Connection, source_id: i64, name: &str) -> Result<(), rusqlite::Error> { + conn.execute( + "DELETE FROM sync_id_managesieve WHERE source_id = ?1 AND name = ?2", + params![source_id, name], + )?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::init; + use crate::db::sources::{SourceKey, upsert_source}; + + fn setup() -> (Connection, i64) { + let c = Connection::open_in_memory().unwrap(); + init::apply_schema(&c).unwrap(); + let sid = upsert_source( + &c, + &SourceKey { + kind: "managesieve".to_owned(), + session_url: "sieve://host:4190".to_owned(), + account_id: "alice".to_owned(), + }, + None, + "alice", + ) + .unwrap(); + (c, sid) + } + + #[test] + fn insert_and_lookup_by_name() { + let (c, sid) = setup(); + insert(&c, sid, "vacation", 7).unwrap(); + assert_eq!(local_for(&c, sid, "vacation").unwrap(), Some(7)); + assert_eq!(local_for(&c, sid, "missing").unwrap(), None); + } + + #[test] + fn all_names_returns_full_map() { + let (c, sid) = setup(); + insert(&c, sid, "a", 1).unwrap(); + insert(&c, sid, "b", 2).unwrap(); + let map = all_names(&c, sid).unwrap(); + assert_eq!(map.len(), 2); + assert_eq!(map.get("a"), Some(&1)); + } + + #[test] + fn delete_removes_row() { + let (c, sid) = setup(); + insert(&c, sid, "a", 1).unwrap(); + delete(&c, sid, "a").unwrap(); + assert_eq!(local_for(&c, sid, "a").unwrap(), None); + } + + #[test] + fn re_insert_with_same_name_replaces_local_id() { + let (c, sid) = setup(); + insert(&c, sid, "a", 1).unwrap(); + insert(&c, sid, "a", 2).unwrap(); + assert_eq!(local_for(&c, sid, "a").unwrap(), Some(2)); + assert_eq!(all_names(&c, sid).unwrap().len(), 1); + } + + #[test] + fn insert_or_replace_reassigns_when_local_id_clashes() { + let (c, sid) = setup(); + insert(&c, sid, "a", 1).unwrap(); + insert(&c, sid, "b", 1).unwrap(); + assert!(local_for(&c, sid, "a").unwrap().is_none()); + assert_eq!(local_for(&c, sid, "b").unwrap(), Some(1)); + } +} diff --git a/src/db/mod.rs b/src/db/mod.rs new file mode 100644 index 0000000..1d3fdc7 --- /dev/null +++ b/src/db/mod.rs @@ -0,0 +1,18 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod blobs; +pub mod dav_ids; +pub mod exchange_ews_ids; +pub mod exchange_graph_ids; +pub mod ids; +pub mod imap_ids; +pub mod imap_state; +pub mod init; +pub mod maildir_ids; +pub mod managesieve_ids; +pub mod sources; +pub mod takeout_ids; diff --git a/src/db/schema.sql b/src/db/schema.sql new file mode 100644 index 0000000..3ca3a57 --- /dev/null +++ b/src/db/schema.sql @@ -0,0 +1,249 @@ +CREATE TABLE IF NOT EXISTS sources ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL CHECK (kind IN ('jmap','imap','caldav','carddav','webdav','managesieve','maildir','exchange_ews','exchange_graph','takeout')), + session_url TEXT NOT NULL, + account_id TEXT NOT NULL, + account_name TEXT, + username TEXT NOT NULL, + UNIQUE (kind, session_url, account_id) +); + +CREATE TABLE IF NOT EXISTS blobs ( + id INTEGER PRIMARY KEY, + hash BLOB NOT NULL UNIQUE, + data BLOB NOT NULL +); + +CREATE INDEX IF NOT EXISTS blobs_hash_idx ON blobs (hash); + +CREATE TABLE IF NOT EXISTS sync_id_jmap ( + source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + type_name TEXT NOT NULL, + jmap_id TEXT NOT NULL, + local_id INTEGER NOT NULL, + PRIMARY KEY (source_id, type_name, jmap_id), + UNIQUE (source_id, type_name, local_id) +); + +CREATE INDEX IF NOT EXISTS sync_id_jmap_local_idx + ON sync_id_jmap (type_name, local_id); + +CREATE TABLE IF NOT EXISTS sync_id_imap ( + source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + type_name TEXT NOT NULL CHECK (type_name IN ('mailbox','email')), + folder TEXT NOT NULL, + uidvalidity INTEGER NOT NULL, + uid INTEGER NOT NULL, + local_id INTEGER NOT NULL, + PRIMARY KEY (source_id, type_name, folder, uidvalidity, uid), + UNIQUE (source_id, type_name, local_id) +); + +CREATE INDEX IF NOT EXISTS sync_id_imap_folder_idx + ON sync_id_imap (source_id, type_name, folder); + +CREATE TABLE IF NOT EXISTS imap_folder_state ( + source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + folder TEXT NOT NULL, + uidvalidity INTEGER NOT NULL, + uidnext INTEGER NOT NULL, + last_seen TEXT NOT NULL, + PRIMARY KEY (source_id, folder) +); + +CREATE TABLE IF NOT EXISTS sync_id_dav ( + source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + type_name TEXT NOT NULL CHECK (type_name IN ( + 'calendar','calendarevent', + 'addressbook','contactcard', + 'filenode')), + collection_href TEXT NOT NULL, + item_href TEXT NOT NULL, + etag TEXT NOT NULL DEFAULT '', + local_id INTEGER NOT NULL, + PRIMARY KEY (source_id, type_name, item_href), + UNIQUE (source_id, type_name, local_id) +); + +CREATE INDEX IF NOT EXISTS sync_id_dav_collection_idx + ON sync_id_dav (source_id, type_name, collection_href); + +CREATE TABLE IF NOT EXISTS sync_id_managesieve ( + source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + name TEXT NOT NULL, + local_id INTEGER NOT NULL, + PRIMARY KEY (source_id, name), + UNIQUE (source_id, local_id) +); + +CREATE TABLE IF NOT EXISTS sync_id_maildir ( + source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + type_name TEXT NOT NULL CHECK (type_name IN ('mailbox','email')), + folder TEXT NOT NULL, + unique_id TEXT NOT NULL, + local_id INTEGER NOT NULL, + PRIMARY KEY (source_id, type_name, folder, unique_id), + UNIQUE (source_id, type_name, local_id) +); + +CREATE INDEX IF NOT EXISTS sync_id_maildir_folder_idx + ON sync_id_maildir (source_id, type_name, folder); + +CREATE TABLE IF NOT EXISTS sync_id_exchange_ews ( + source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + type_name TEXT NOT NULL CHECK (type_name IN ( + 'mailbox','calendar','addressbook', + 'email','calendarevent','contactcard')), + folder_id TEXT NOT NULL DEFAULT '', + item_id TEXT NOT NULL, + change_key TEXT NOT NULL DEFAULT '', + sync_state TEXT NOT NULL DEFAULT '', + local_id INTEGER NOT NULL, + PRIMARY KEY (source_id, type_name, item_id), + UNIQUE (source_id, type_name, local_id) +); + +CREATE INDEX IF NOT EXISTS sync_id_exchange_ews_folder_idx + ON sync_id_exchange_ews (source_id, type_name, folder_id); + +CREATE TABLE IF NOT EXISTS sync_id_exchange_graph ( + source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + type_name TEXT NOT NULL CHECK (type_name IN ( + 'mailbox','email', + 'calendar','calendarevent', + 'addressbook','contactcard')), + graph_id TEXT NOT NULL, + local_id INTEGER NOT NULL, + PRIMARY KEY (source_id, type_name, graph_id), + UNIQUE (source_id, type_name, local_id) +); + +CREATE INDEX IF NOT EXISTS sync_id_exchange_graph_type_idx + ON sync_id_exchange_graph (source_id, type_name); + +CREATE TABLE IF NOT EXISTS mailboxes ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + parent_id INTEGER REFERENCES mailboxes(id) ON DELETE RESTRICT, + role TEXT, + sort_order INTEGER NOT NULL DEFAULT 0, + is_subscribed INTEGER NOT NULL DEFAULT 1, + UNIQUE (parent_id, name) +); + +CREATE TABLE IF NOT EXISTS emails ( + id INTEGER PRIMARY KEY, + blob_id INTEGER NOT NULL REFERENCES blobs(id), + received_at TEXT NOT NULL, + mailbox_ids TEXT NOT NULL CHECK (json_valid(mailbox_ids)), + keywords TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(keywords)), + message_match TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(message_match)) +); + +CREATE INDEX IF NOT EXISTS emails_blob_idx ON emails (blob_id); + +CREATE TABLE IF NOT EXISTS identities ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL DEFAULT '', + email TEXT NOT NULL, + reply_to TEXT CHECK (reply_to IS NULL OR json_valid(reply_to)), + bcc TEXT CHECK (bcc IS NULL OR json_valid(bcc)), + text_signature TEXT NOT NULL DEFAULT '', + html_signature TEXT NOT NULL DEFAULT '' +); + +CREATE TABLE IF NOT EXISTS sieve_scripts ( + id INTEGER PRIMARY KEY, + name TEXT, + is_active INTEGER NOT NULL DEFAULT 0, + blob_id INTEGER NOT NULL REFERENCES blobs(id), + UNIQUE (name) +); + +CREATE UNIQUE INDEX IF NOT EXISTS sieve_scripts_one_active_idx + ON sieve_scripts (is_active) WHERE is_active = 1; + +CREATE TABLE IF NOT EXISTS address_books ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + description TEXT, + sort_order INTEGER NOT NULL DEFAULT 0, + is_default INTEGER NOT NULL DEFAULT 0, + is_subscribed INTEGER NOT NULL DEFAULT 1 +); + +CREATE TABLE IF NOT EXISTS contact_cards ( + id INTEGER PRIMARY KEY, + uid TEXT NOT NULL UNIQUE, + address_book_ids TEXT NOT NULL CHECK (json_valid(address_book_ids)), + data TEXT NOT NULL CHECK (json_valid(data)) +); + +CREATE TABLE IF NOT EXISTS calendars ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + description TEXT, + color TEXT, + sort_order INTEGER NOT NULL DEFAULT 0, + is_subscribed INTEGER NOT NULL DEFAULT 1, + is_visible INTEGER NOT NULL DEFAULT 1, + is_default INTEGER NOT NULL DEFAULT 0, + include_in_availability TEXT NOT NULL DEFAULT 'all' + CHECK (include_in_availability IN ('all','attending','none')), + default_alerts_with_time TEXT CHECK (default_alerts_with_time IS NULL + OR json_valid(default_alerts_with_time)), + default_alerts_without_time TEXT CHECK (default_alerts_without_time IS NULL + OR json_valid(default_alerts_without_time)), + time_zone TEXT +); + +CREATE TABLE IF NOT EXISTS calendar_events ( + id INTEGER PRIMARY KEY, + calendar_ids TEXT NOT NULL CHECK (json_valid(calendar_ids)), + is_draft INTEGER NOT NULL DEFAULT 0, + use_default_alerts INTEGER NOT NULL DEFAULT 0, + data TEXT NOT NULL CHECK (json_valid(data)), + data_type TEXT NOT NULL DEFAULT 'Event' + CHECK (data_type IN ('Event','Task','Note','Group')) +); + +CREATE TABLE IF NOT EXISTS participant_identities ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL DEFAULT '', + calendar_address TEXT NOT NULL, + is_default INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE IF NOT EXISTS file_nodes ( + id INTEGER PRIMARY KEY, + parent_id INTEGER REFERENCES file_nodes(id) ON DELETE RESTRICT, + node_type TEXT NOT NULL + CHECK (node_type IN ('file','directory','symlink')), + blob_id INTEGER REFERENCES blobs(id), + target TEXT CHECK (target IS NULL OR json_valid(target)), + name TEXT NOT NULL, + media_type TEXT, + created TEXT NOT NULL, + modified TEXT, + is_subscribed INTEGER NOT NULL DEFAULT 1, + role TEXT, + UNIQUE (parent_id, name) +); + +CREATE INDEX IF NOT EXISTS file_nodes_parent_idx ON file_nodes (parent_id); +CREATE INDEX IF NOT EXISTS file_nodes_blob_idx ON file_nodes (blob_id); + +CREATE TABLE IF NOT EXISTS sync_id_takeout ( + source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + type_name TEXT NOT NULL CHECK (type_name IN ( + 'mailbox','email', + 'addressbook','contactcard', + 'calendar','calendarevent')), + source_obj_id TEXT NOT NULL, + local_id INTEGER NOT NULL, + PRIMARY KEY (source_id, type_name, source_obj_id), + UNIQUE (source_id, type_name, local_id) +); + +CREATE INDEX IF NOT EXISTS sync_id_takeout_type_idx + ON sync_id_takeout (source_id, type_name); diff --git a/src/db/sources.rs b/src/db/sources.rs new file mode 100644 index 0000000..8a2600a --- /dev/null +++ b/src/db/sources.rs @@ -0,0 +1,105 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::{Connection, OptionalExtension, params}; + +#[derive(Debug, Clone)] +pub struct SourceKey { + pub kind: String, + pub session_url: String, + pub account_id: String, +} + +pub fn find_source(conn: &Connection, key: &SourceKey) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT id FROM sources WHERE kind = ?1 AND session_url = ?2 AND account_id = ?3", + params![key.kind, key.session_url, key.account_id], + |row| row.get(0), + ) + .optional() +} + +pub fn conflicting_source( + conn: &Connection, + kind: &str, + session_url: &str, + account_id: &str, +) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT session_url, account_id FROM sources + WHERE kind = ?1 AND NOT (session_url = ?2 AND account_id = ?3) LIMIT 1", + params![kind, session_url, account_id], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() +} + +pub fn upsert_source( + conn: &Connection, + key: &SourceKey, + account_name: Option<&str>, + username: &str, +) -> Result { + conn.execute( + "INSERT INTO sources (kind, session_url, account_id, account_name, username) + VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT (kind, session_url, account_id) + DO UPDATE SET account_name = excluded.account_name, username = excluded.username", + params![ + key.kind, + key.session_url, + key.account_id, + account_name, + username + ], + )?; + find_source(conn, key).map(|opt| opt.unwrap_or_else(|| conn.last_insert_rowid())) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::init; + + fn mem() -> Connection { + let c = Connection::open_in_memory().unwrap(); + init::apply_schema(&c).unwrap(); + c + } + + fn key(url: &str, acc: &str) -> SourceKey { + SourceKey { + kind: "jmap".to_owned(), + session_url: url.to_owned(), + account_id: acc.to_owned(), + } + } + + #[test] + fn upsert_is_idempotent_and_returns_stable_id() { + let c = mem(); + let k = key("https://a/jmap", "w"); + let id1 = upsert_source(&c, &k, Some("alice"), "alice").unwrap(); + let id2 = upsert_source(&c, &k, Some("alice2"), "alice").unwrap(); + assert_eq!(id1, id2); + assert!(find_source(&c, &k).unwrap().is_some()); + } + + #[test] + fn conflicting_source_detects_a_different_account() { + let c = mem(); + upsert_source(&c, &key("https://a/jmap", "w"), None, "alice").unwrap(); + assert!( + conflicting_source(&c, "jmap", "https://a/jmap", "w") + .unwrap() + .is_none() + ); + let conflict = conflicting_source(&c, "jmap", "https://b/jmap", "v") + .unwrap() + .unwrap(); + assert_eq!(conflict, ("https://a/jmap".to_owned(), "w".to_owned())); + } +} diff --git a/src/db/takeout_ids.rs b/src/db/takeout_ids.rs new file mode 100644 index 0000000..0fda03f --- /dev/null +++ b/src/db/takeout_ids.rs @@ -0,0 +1,177 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use rusqlite::{Connection, OptionalExtension, params}; + +pub const MAILBOX: &str = "mailbox"; +pub const EMAIL: &str = "email"; +pub const ADDRESS_BOOK: &str = "addressbook"; +pub const CONTACT_CARD: &str = "contactcard"; +pub const CALENDAR: &str = "calendar"; +pub const CALENDAR_EVENT: &str = "calendarevent"; + +pub fn insert( + conn: &Connection, + source_id: i64, + type_name: &str, + source_obj_id: &str, + local_id: i64, +) -> Result<(), rusqlite::Error> { + conn.execute( + "INSERT INTO sync_id_takeout + (source_id, type_name, source_obj_id, local_id) + VALUES (?1, ?2, ?3, ?4)", + params![source_id, type_name, source_obj_id, local_id], + )?; + Ok(()) +} + +pub fn local_for( + conn: &Connection, + source_id: i64, + type_name: &str, + source_obj_id: &str, +) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT local_id FROM sync_id_takeout + WHERE source_id = ?1 AND type_name = ?2 AND source_obj_id = ?3", + params![source_id, type_name, source_obj_id], + |row| row.get(0), + ) + .optional() +} + +pub fn delete( + conn: &Connection, + source_id: i64, + type_name: &str, + source_obj_id: &str, +) -> Result<(), rusqlite::Error> { + conn.execute( + "DELETE FROM sync_id_takeout + WHERE source_id = ?1 AND type_name = ?2 AND source_obj_id = ?3", + params![source_id, type_name, source_obj_id], + )?; + Ok(()) +} + +pub fn all_for_type( + conn: &Connection, + source_id: i64, + type_name: &str, +) -> Result, rusqlite::Error> { + let mut stmt = conn.prepare( + "SELECT source_obj_id, local_id FROM sync_id_takeout + WHERE source_id = ?1 AND type_name = ?2", + )?; + let rows = stmt.query_map(params![source_id, type_name], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + })?; + let mut map = HashMap::new(); + for r in rows { + let (k, v) = r?; + map.insert(k, v); + } + Ok(map) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::init; + use crate::db::sources::{SourceKey, upsert_source}; + + fn setup() -> (Connection, i64) { + let c = Connection::open_in_memory().unwrap(); + init::apply_schema(&c).unwrap(); + let sid = upsert_source( + &c, + &SourceKey { + kind: "takeout".to_owned(), + session_url: "file:///tmp/Takeout".to_owned(), + account_id: "/tmp/Takeout".to_owned(), + }, + Some("Takeout"), + "", + ) + .unwrap(); + (c, sid) + } + + #[test] + fn insert_and_lookup_roundtrip() { + let (c, sid) = setup(); + insert(&c, sid, MAILBOX, "Inbox", 7).unwrap(); + insert(&c, sid, MAILBOX, "Label/Sub", 8).unwrap(); + assert_eq!(local_for(&c, sid, MAILBOX, "Inbox").unwrap(), Some(7)); + assert_eq!(local_for(&c, sid, MAILBOX, "Label/Sub").unwrap(), Some(8)); + assert_eq!(local_for(&c, sid, MAILBOX, "Missing").unwrap(), None); + } + + #[test] + fn types_are_isolated_namespaces() { + let (c, sid) = setup(); + insert(&c, sid, MAILBOX, "Inbox", 1).unwrap(); + insert(&c, sid, EMAIL, "ab12", 2).unwrap(); + insert(&c, sid, ADDRESS_BOOK, "Imported", 3).unwrap(); + insert(&c, sid, CONTACT_CARD, "uid-1", 4).unwrap(); + insert(&c, sid, CALENDAR, "Imported", 5).unwrap(); + insert(&c, sid, CALENDAR_EVENT, "uid-e1", 6).unwrap(); + assert_eq!(all_for_type(&c, sid, MAILBOX).unwrap().len(), 1); + assert_eq!(all_for_type(&c, sid, EMAIL).unwrap().len(), 1); + assert_eq!(all_for_type(&c, sid, ADDRESS_BOOK).unwrap().len(), 1); + assert_eq!(all_for_type(&c, sid, CONTACT_CARD).unwrap().len(), 1); + assert_eq!(all_for_type(&c, sid, CALENDAR).unwrap().len(), 1); + assert_eq!(all_for_type(&c, sid, CALENDAR_EVENT).unwrap().len(), 1); + } + + #[test] + fn delete_removes_only_target_row() { + let (c, sid) = setup(); + insert(&c, sid, EMAIL, "hash-a", 1).unwrap(); + insert(&c, sid, EMAIL, "hash-b", 2).unwrap(); + delete(&c, sid, EMAIL, "hash-a").unwrap(); + assert_eq!(local_for(&c, sid, EMAIL, "hash-a").unwrap(), None); + assert_eq!(local_for(&c, sid, EMAIL, "hash-b").unwrap(), Some(2)); + } + + #[test] + fn pk_rejects_duplicate_source_obj_id() { + let (c, sid) = setup(); + insert(&c, sid, EMAIL, "h", 1).unwrap(); + let err = insert(&c, sid, EMAIL, "h", 2).expect_err("PK conflict"); + assert!(matches!(err, rusqlite::Error::SqliteFailure(_, _))); + } + + #[test] + fn unique_constraint_rejects_duplicate_local_id_within_type() { + let (c, sid) = setup(); + insert(&c, sid, EMAIL, "h1", 42).unwrap(); + let err = insert(&c, sid, EMAIL, "h2", 42).expect_err("UNIQUE conflict"); + assert!(matches!(err, rusqlite::Error::SqliteFailure(_, _))); + } + + #[test] + fn same_local_id_allowed_across_different_types() { + let (c, sid) = setup(); + insert(&c, sid, MAILBOX, "Inbox", 42).unwrap(); + insert(&c, sid, EMAIL, "h1", 42).unwrap(); + } + + #[test] + fn all_for_type_returns_full_map() { + let (c, sid) = setup(); + insert(&c, sid, EMAIL, "h1", 1).unwrap(); + insert(&c, sid, EMAIL, "h2", 2).unwrap(); + insert(&c, sid, EMAIL, "h3", 3).unwrap(); + let all = all_for_type(&c, sid, EMAIL).unwrap(); + assert_eq!(all.len(), 3); + assert_eq!(all.get("h1"), Some(&1)); + assert_eq!(all.get("h3"), Some(&3)); + } +} diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..f87af6f --- /dev/null +++ b/src/error.rs @@ -0,0 +1,77 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::db::init::OpenError; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("usage error: {0}")] + Usage(String), + + #[error("connection error: {0}")] + Connection(String), + + #[error("account resolution failed: {0}")] + Account(String), + + #[error("source-change protection: {0}")] + SourceChange(String), + + #[error("partial failure: {0}")] + Partial(String), + + #[error("aborted at prune confirmation")] + PruneAborted, + + #[error("not yet implemented: {0}")] + Unimplemented(&'static str), + + #[error("archive error: {0}")] + Db(#[from] OpenError), + + #[error("i/o error: {0}")] + Io(#[from] std::io::Error), +} + +impl From for Error { + fn from(e: rusqlite::Error) -> Self { + Error::Db(OpenError::Sqlite(e)) + } +} + +impl Error { + pub fn exit_code(&self) -> i32 { + match self { + Error::Usage(_) => 1, + Error::Connection(_) => 2, + Error::Account(_) => 3, + Error::SourceChange(_) => 4, + Error::Partial(_) => 5, + Error::PruneAborted => 6, + Error::Unimplemented(_) => 1, + Error::Db(_) => 7, + Error::Io(_) => 7, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn exit_codes_match_cli_md_table() { + assert_eq!(Error::Usage("x".into()).exit_code(), 1); + assert_eq!(Error::Unimplemented("x").exit_code(), 1); + assert_eq!(Error::Connection("x".into()).exit_code(), 2); + assert_eq!(Error::Account("x".into()).exit_code(), 3); + assert_eq!(Error::SourceChange("x".into()).exit_code(), 4); + assert_eq!(Error::Partial("x".into()).exit_code(), 5); + assert_eq!(Error::PruneAborted.exit_code(), 6); + let io = Error::Io(std::io::Error::other("disk")); + assert_eq!(io.exit_code(), 7); + } +} diff --git a/src/exchange/mod.rs b/src/exchange/mod.rs new file mode 100644 index 0000000..0a3f097 --- /dev/null +++ b/src/exchange/mod.rs @@ -0,0 +1,7 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod tz; diff --git a/src/exchange/tz.rs b/src/exchange/tz.rs new file mode 100644 index 0000000..35b14b6 --- /dev/null +++ b/src/exchange/tz.rs @@ -0,0 +1,258 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub fn windows_to_iana(windows_name: &str) -> Option<&'static str> { + let key = windows_name.trim(); + for (w, i) in TABLE { + if w.eq_ignore_ascii_case(key) { + return Some(i); + } + } + None +} + +const TABLE: &[(&str, &str)] = &[ + ("UTC", "Etc/UTC"), + ("Coordinated Universal Time", "Etc/UTC"), + ("UTC-11", "Etc/GMT+11"), + ("UTC-09", "Etc/GMT+9"), + ("UTC-08", "Etc/GMT+8"), + ("UTC-02", "Etc/GMT+2"), + ("UTC+12", "Etc/GMT-12"), + ("UTC+13", "Etc/GMT-13"), + ("Dateline Standard Time", "Etc/GMT+12"), + ("Aleutian Standard Time", "America/Adak"), + ("Hawaiian Standard Time", "Pacific/Honolulu"), + ("Marquesas Standard Time", "Pacific/Marquesas"), + ("Alaskan Standard Time", "America/Anchorage"), + ("Pacific Standard Time (Mexico)", "America/Tijuana"), + ("Pacific Standard Time", "America/Los_Angeles"), + ("US Mountain Standard Time", "America/Phoenix"), + ("Mountain Standard Time (Mexico)", "America/Mazatlan"), + ("Mountain Standard Time", "America/Denver"), + ("Yukon Standard Time", "America/Whitehorse"), + ("Central America Standard Time", "America/Guatemala"), + ("Central Standard Time", "America/Chicago"), + ("Easter Island Standard Time", "Pacific/Easter"), + ("Central Standard Time (Mexico)", "America/Mexico_City"), + ("Canada Central Standard Time", "America/Regina"), + ("SA Pacific Standard Time", "America/Bogota"), + ("Eastern Standard Time (Mexico)", "America/Cancun"), + ("Eastern Standard Time", "America/New_York"), + ("Haiti Standard Time", "America/Port-au-Prince"), + ("Cuba Standard Time", "America/Havana"), + ("US Eastern Standard Time", "America/Indianapolis"), + ("Turks And Caicos Standard Time", "America/Grand_Turk"), + ("Paraguay Standard Time", "America/Asuncion"), + ("Atlantic Standard Time", "America/Halifax"), + ("Venezuela Standard Time", "America/Caracas"), + ("Central Brazilian Standard Time", "America/Cuiaba"), + ("SA Western Standard Time", "America/La_Paz"), + ("Pacific SA Standard Time", "America/Santiago"), + ("Newfoundland Standard Time", "America/St_Johns"), + ("Tocantins Standard Time", "America/Araguaina"), + ("E. South America Standard Time", "America/Sao_Paulo"), + ("SA Eastern Standard Time", "America/Cayenne"), + ("Argentina Standard Time", "America/Argentina/Buenos_Aires"), + ("Greenland Standard Time", "America/Godthab"), + ("Montevideo Standard Time", "America/Montevideo"), + ("Magallanes Standard Time", "America/Punta_Arenas"), + ("Saint Pierre Standard Time", "America/Miquelon"), + ("Bahia Standard Time", "America/Bahia"), + ("Mid-Atlantic Standard Time", "Atlantic/South_Georgia"), + ("Azores Standard Time", "Atlantic/Azores"), + ("Cape Verde Standard Time", "Atlantic/Cape_Verde"), + ("GMT Standard Time", "Europe/London"), + ("Greenwich Standard Time", "Atlantic/Reykjavik"), + ("Sao Tome Standard Time", "Africa/Sao_Tome"), + ("Morocco Standard Time", "Africa/Casablanca"), + ("W. Europe Standard Time", "Europe/Berlin"), + ("Central Europe Standard Time", "Europe/Budapest"), + ("Romance Standard Time", "Europe/Paris"), + ("Central European Standard Time", "Europe/Warsaw"), + ("W. Central Africa Standard Time", "Africa/Lagos"), + ("Jordan Standard Time", "Asia/Amman"), + ("GTB Standard Time", "Europe/Bucharest"), + ("Middle East Standard Time", "Asia/Beirut"), + ("Egypt Standard Time", "Africa/Cairo"), + ("E. Europe Standard Time", "Europe/Chisinau"), + ("Syria Standard Time", "Asia/Damascus"), + ("West Bank Standard Time", "Asia/Hebron"), + ("South Africa Standard Time", "Africa/Johannesburg"), + ("FLE Standard Time", "Europe/Kiev"), + ("Israel Standard Time", "Asia/Jerusalem"), + ("South Sudan Standard Time", "Africa/Juba"), + ("Kaliningrad Standard Time", "Europe/Kaliningrad"), + ("Sudan Standard Time", "Africa/Khartoum"), + ("Libya Standard Time", "Africa/Tripoli"), + ("Namibia Standard Time", "Africa/Windhoek"), + ("Arabic Standard Time", "Asia/Baghdad"), + ("Turkey Standard Time", "Europe/Istanbul"), + ("Arab Standard Time", "Asia/Riyadh"), + ("Belarus Standard Time", "Europe/Minsk"), + ("Russian Standard Time", "Europe/Moscow"), + ("E. Africa Standard Time", "Africa/Nairobi"), + ("Iran Standard Time", "Asia/Tehran"), + ("Arabian Standard Time", "Asia/Dubai"), + ("Astrakhan Standard Time", "Europe/Astrakhan"), + ("Azerbaijan Standard Time", "Asia/Baku"), + ("Russia Time Zone 3", "Europe/Samara"), + ("Mauritius Standard Time", "Indian/Mauritius"), + ("Saratov Standard Time", "Europe/Saratov"), + ("Georgian Standard Time", "Asia/Tbilisi"), + ("Volgograd Standard Time", "Europe/Volgograd"), + ("Caucasus Standard Time", "Asia/Yerevan"), + ("Afghanistan Standard Time", "Asia/Kabul"), + ("West Asia Standard Time", "Asia/Tashkent"), + ("Ekaterinburg Standard Time", "Asia/Yekaterinburg"), + ("Pakistan Standard Time", "Asia/Karachi"), + ("Qyzylorda Standard Time", "Asia/Qyzylorda"), + ("India Standard Time", "Asia/Kolkata"), + ("Sri Lanka Standard Time", "Asia/Colombo"), + ("Nepal Standard Time", "Asia/Katmandu"), + ("Central Asia Standard Time", "Asia/Almaty"), + ("Bangladesh Standard Time", "Asia/Dhaka"), + ("Omsk Standard Time", "Asia/Omsk"), + ("Myanmar Standard Time", "Asia/Rangoon"), + ("SE Asia Standard Time", "Asia/Bangkok"), + ("Altai Standard Time", "Asia/Barnaul"), + ("W. Mongolia Standard Time", "Asia/Hovd"), + ("North Asia Standard Time", "Asia/Krasnoyarsk"), + ("N. Central Asia Standard Time", "Asia/Novosibirsk"), + ("Tomsk Standard Time", "Asia/Tomsk"), + ("China Standard Time", "Asia/Shanghai"), + ("North Asia East Standard Time", "Asia/Irkutsk"), + ("Singapore Standard Time", "Asia/Singapore"), + ("W. Australia Standard Time", "Australia/Perth"), + ("Taipei Standard Time", "Asia/Taipei"), + ("Ulaanbaatar Standard Time", "Asia/Ulaanbaatar"), + ("Aus Central W. Standard Time", "Australia/Eucla"), + ("Transbaikal Standard Time", "Asia/Chita"), + ("Tokyo Standard Time", "Asia/Tokyo"), + ("North Korea Standard Time", "Asia/Pyongyang"), + ("Korea Standard Time", "Asia/Seoul"), + ("Yakutsk Standard Time", "Asia/Yakutsk"), + ("Cen. Australia Standard Time", "Australia/Adelaide"), + ("AUS Central Standard Time", "Australia/Darwin"), + ("E. Australia Standard Time", "Australia/Brisbane"), + ("AUS Eastern Standard Time", "Australia/Sydney"), + ("West Pacific Standard Time", "Pacific/Port_Moresby"), + ("Tasmania Standard Time", "Australia/Hobart"), + ("Vladivostok Standard Time", "Asia/Vladivostok"), + ("Lord Howe Standard Time", "Australia/Lord_Howe"), + ("Bougainville Standard Time", "Pacific/Bougainville"), + ("Russia Time Zone 10", "Asia/Srednekolymsk"), + ("Magadan Standard Time", "Asia/Magadan"), + ("Norfolk Standard Time", "Pacific/Norfolk"), + ("Sakhalin Standard Time", "Asia/Sakhalin"), + ("Central Pacific Standard Time", "Pacific/Guadalcanal"), + ("Russia Time Zone 11", "Asia/Kamchatka"), + ("New Zealand Standard Time", "Pacific/Auckland"), + ("Fiji Standard Time", "Pacific/Fiji"), + ("Kamchatka Standard Time", "Asia/Kamchatka"), + ("Chatham Islands Standard Time", "Pacific/Chatham"), + ("Tonga Standard Time", "Pacific/Tongatapu"), + ("Samoa Standard Time", "Pacific/Apia"), + ("Line Islands Standard Time", "Pacific/Kiritimati"), +]; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pacific_standard_time_maps_to_los_angeles() { + assert_eq!( + windows_to_iana("Pacific Standard Time"), + Some("America/Los_Angeles") + ); + } + + #[test] + fn romance_standard_time_maps_to_paris() { + assert_eq!( + windows_to_iana("Romance Standard Time"), + Some("Europe/Paris") + ); + } + + #[test] + fn unknown_zone_returns_none() { + assert_eq!(windows_to_iana("Made Up Zone"), None); + } + + #[test] + fn case_insensitive_lookup() { + assert_eq!( + windows_to_iana("eastern standard time"), + Some("America/New_York") + ); + } + + #[test] + fn whitespace_is_trimmed() { + assert_eq!( + windows_to_iana(" GMT Standard Time "), + Some("Europe/London") + ); + } + + #[test] + fn full_cldr_table_has_at_least_140_rows() { + assert!(TABLE.len() >= 140, "got {} rows", TABLE.len()); + } + + #[test] + fn spot_check_coverage_across_continents() { + assert_eq!(windows_to_iana("Tokyo Standard Time"), Some("Asia/Tokyo")); + assert_eq!( + windows_to_iana("AUS Eastern Standard Time"), + Some("Australia/Sydney") + ); + assert_eq!( + windows_to_iana("Cen. Australia Standard Time"), + Some("Australia/Adelaide") + ); + assert_eq!( + windows_to_iana("Greenland Standard Time"), + Some("America/Godthab") + ); + assert_eq!( + windows_to_iana("China Standard Time"), + Some("Asia/Shanghai") + ); + assert_eq!( + windows_to_iana("Israel Standard Time"), + Some("Asia/Jerusalem") + ); + assert_eq!( + windows_to_iana("Chatham Islands Standard Time"), + Some("Pacific/Chatham") + ); + assert_eq!( + windows_to_iana("Newfoundland Standard Time"), + Some("America/St_Johns") + ); + assert_eq!(windows_to_iana("UTC-08"), Some("Etc/GMT+8")); + assert_eq!(windows_to_iana("UTC+13"), Some("Etc/GMT-13")); + assert_eq!( + windows_to_iana("Yukon Standard Time"), + Some("America/Whitehorse") + ); + assert_eq!( + windows_to_iana("Aus Central W. Standard Time"), + Some("Australia/Eucla") + ); + } + + #[test] + fn coordinated_universal_time_alias_resolves_to_utc() { + assert_eq!( + windows_to_iana("Coordinated Universal Time"), + Some("Etc/UTC") + ); + } +} diff --git a/src/exchange_ews/autodiscover.rs b/src/exchange_ews/autodiscover.rs new file mode 100644 index 0000000..e3d6e72 --- /dev/null +++ b/src/exchange_ews/autodiscover.rs @@ -0,0 +1,561 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use quick_xml::NsReader; +use quick_xml::events::Event; +use serde_json::Value; +use ureq::Agent; +use ureq::config::Config; +use ureq::tls::{RootCerts, TlsConfig}; + +use crate::exchange_ews::error::EwsError; + +const V2_HOST: &str = "https://outlook.office365.com"; +const POX_REQ_NS: &str = + "http://schemas.microsoft.com/exchange/autodiscover/outlook/requestschema/2006"; +const POX_RESP_NS: &str = + "http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a"; + +pub fn is_fully_qualified_ews_url(url: &str) -> bool { + let parsed = match url::Url::parse(url) { + Ok(u) => u, + Err(_) => return false, + }; + if parsed.scheme() != "https" { + return false; + } + parsed.path().eq_ignore_ascii_case("/EWS/Exchange.asmx") +} + +#[derive(Debug, Clone)] +pub struct DiscoveryResult { + pub ews_url: String, + pub source: DiscoverySource, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DiscoverySource { + V2, + V1, + SuppliedUrl, +} + +pub fn discover( + supplied_url: Option<&str>, + email: Option<&str>, + auth_header: Option<&str>, + allow_invalid_certs: bool, +) -> Result { + if let Some(url) = supplied_url + && is_fully_qualified_ews_url(url) + { + return Ok(DiscoveryResult { + ews_url: url.to_owned(), + source: DiscoverySource::SuppliedUrl, + }); + } + let Some(email) = email else { + return Err(EwsError::AutodiscoverFailed( + "either a fully-qualified --url or --mailbox is required".to_owned(), + )); + }; + let agent = build_agent(allow_invalid_certs); + if let Ok(url) = autodiscover_v2(&agent, email) { + return Ok(DiscoveryResult { + ews_url: url, + source: DiscoverySource::V2, + }); + } + let mut tried: Vec = Vec::new(); + let mut redirects: u8 = 0; + let mut url_redirects: u8 = 0; + let mut current_email = email.to_owned(); + 'outer: loop { + let domain = current_email + .split('@') + .nth(1) + .ok_or_else(|| EwsError::AutodiscoverFailed("--mailbox is not an email".to_owned()))?; + let candidates = pox_candidates(domain); + for candidate in &candidates { + tried.push(candidate.clone()); + match autodiscover_v1(&agent, candidate, ¤t_email, auth_header) { + Ok(PoxOutcome::EwsUrl(url)) => { + return Ok(DiscoveryResult { + ews_url: url, + source: DiscoverySource::V1, + }); + } + Ok(PoxOutcome::RedirectAddr(addr)) => { + if redirects >= 4 { + return Err(EwsError::AutodiscoverLoop); + } + redirects += 1; + current_email = addr; + continue 'outer; + } + Ok(PoxOutcome::RedirectUrl(url)) => { + if url_redirects >= 1 { + return Err(EwsError::AutodiscoverLoop); + } + url_redirects += 1; + tried.push(url.clone()); + if let Ok(PoxOutcome::EwsUrl(u)) = + autodiscover_v1(&agent, &url, ¤t_email, auth_header) + { + return Ok(DiscoveryResult { + ews_url: u, + source: DiscoverySource::V1, + }); + } + continue; + } + Err(_) | Ok(PoxOutcome::Failed) => continue, + } + } + break; + } + if let Some(url) = supplied_url { + return Ok(DiscoveryResult { + ews_url: url.to_owned(), + source: DiscoverySource::SuppliedUrl, + }); + } + Err(EwsError::AutodiscoverFailed(format!( + "tried {} candidate(s), none returned a v1 EwsUrl", + tried.len() + ))) +} + +fn build_agent(allow_invalid_certs: bool) -> Agent { + let config: Config = Config::builder() + .http_status_as_error(false) + .tls_config( + TlsConfig::builder() + .root_certs(RootCerts::PlatformVerifier) + .disable_verification(allow_invalid_certs) + .build(), + ) + .build(); + config.new_agent() +} + +fn autodiscover_v2(agent: &Agent, email: &str) -> Result { + let url = format!( + "{V2_HOST}/autodiscover/autodiscover.json?Email={}&Protocol=Ews", + urlencode(email) + ); + let mut resp = agent + .get(&url) + .header("Accept", "application/json") + .call() + .map_err(|e| EwsError::Transport(format!("autodiscover v2: {e}")))?; + let status = resp.status().as_u16(); + if !(200..300).contains(&status) { + return Err(EwsError::AutodiscoverFailed(format!( + "v2 returned http {status}" + ))); + } + let body: Value = resp + .body_mut() + .read_json() + .map_err(|e| EwsError::Malformed(format!("v2 body: {e}")))?; + let protocol = body.get("Protocol").and_then(Value::as_str).unwrap_or(""); + if !protocol.eq_ignore_ascii_case("Ews") { + return Err(EwsError::AutodiscoverFailed(format!( + "v2 returned protocol {protocol:?}" + ))); + } + let url = body + .get("Url") + .and_then(Value::as_str) + .ok_or_else(|| EwsError::AutodiscoverFailed("v2 missing Url".to_owned()))?; + Ok(url.to_owned()) +} + +#[derive(Debug, Clone)] +enum PoxOutcome { + EwsUrl(String), + RedirectAddr(String), + RedirectUrl(String), + Failed, +} + +fn pox_candidates(domain: &str) -> Vec { + let mut out = vec![ + format!("https://autodiscover.{domain}/autodiscover/autodiscover.xml"), + format!("https://{domain}/autodiscover/autodiscover.xml"), + ]; + for SrvRecord { target, port, .. } in srv_lookup(&format!("_autodiscover._tcp.{domain}")) { + let host = target.trim_end_matches('.'); + let suffix = if port == 443 { + String::new() + } else { + format!(":{port}") + }; + out.push(format!( + "https://{host}{suffix}/autodiscover/autodiscover.xml" + )); + } + out +} + +#[derive(Debug, Clone)] +struct SrvRecord { + priority: u16, + weight: u16, + port: u16, + target: String, +} + +fn srv_lookup(qname: &str) -> Vec { + use std::net::UdpSocket; + use std::time::Duration; + + let nameservers = resolv_conf_nameservers(); + if nameservers.is_empty() { + return Vec::new(); + } + let mut query = Vec::with_capacity(64); + let txn_id: u16 = 0x1234; + query.extend_from_slice(&txn_id.to_be_bytes()); + query.extend_from_slice(&0x0100u16.to_be_bytes()); + query.extend_from_slice(&1u16.to_be_bytes()); + query.extend_from_slice(&[0u8; 6]); + encode_qname(&mut query, qname); + query.extend_from_slice(&33u16.to_be_bytes()); + query.extend_from_slice(&1u16.to_be_bytes()); + + let mut out: Vec = Vec::new(); + for ns in nameservers { + let Ok(socket) = UdpSocket::bind("0.0.0.0:0") else { + continue; + }; + let _ = socket.set_read_timeout(Some(Duration::from_secs(3))); + if socket.send_to(&query, format!("{ns}:53")).is_err() { + continue; + } + let mut buf = [0u8; 1500]; + let Ok((n, _)) = socket.recv_from(&mut buf) else { + continue; + }; + if let Some(records) = parse_srv_response(&buf[..n]) { + out = records; + break; + } + } + out.sort_by(|a, b| a.priority.cmp(&b.priority).then(b.weight.cmp(&a.weight))); + out +} + +fn resolv_conf_nameservers() -> Vec { + let path = std::path::Path::new("/etc/resolv.conf"); + let Ok(text) = std::fs::read_to_string(path) else { + return Vec::new(); + }; + let mut out: Vec = Vec::new(); + for line in text.lines() { + let line = line.split('#').next().unwrap_or("").trim(); + if let Some(rest) = line.strip_prefix("nameserver") { + let ns = rest.trim(); + if !ns.is_empty() { + out.push(ns.to_owned()); + } + } + } + out +} + +fn encode_qname(out: &mut Vec, qname: &str) { + for label in qname.trim_end_matches('.').split('.') { + if label.is_empty() { + continue; + } + out.push(label.len() as u8); + out.extend_from_slice(label.as_bytes()); + } + out.push(0); +} + +fn parse_srv_response(buf: &[u8]) -> Option> { + if buf.len() < 12 { + return None; + } + let qd = u16::from_be_bytes([buf[4], buf[5]]) as usize; + let an = u16::from_be_bytes([buf[6], buf[7]]) as usize; + let mut cursor = 12; + for _ in 0..qd { + cursor = skip_name(buf, cursor)?; + cursor = cursor.checked_add(4)?; + if cursor > buf.len() { + return None; + } + } + let mut out: Vec = Vec::new(); + for _ in 0..an { + cursor = skip_name(buf, cursor)?; + if cursor + 10 > buf.len() { + return None; + } + let rtype = u16::from_be_bytes([buf[cursor], buf[cursor + 1]]); + let rdlength = u16::from_be_bytes([buf[cursor + 8], buf[cursor + 9]]) as usize; + cursor += 10; + let rdata_end = cursor.checked_add(rdlength)?; + if rdata_end > buf.len() { + return None; + } + if rtype == 33 && rdlength >= 7 { + let priority = u16::from_be_bytes([buf[cursor], buf[cursor + 1]]); + let weight = u16::from_be_bytes([buf[cursor + 2], buf[cursor + 3]]); + let port = u16::from_be_bytes([buf[cursor + 4], buf[cursor + 5]]); + let target = read_name(buf, cursor + 6)?; + out.push(SrvRecord { + priority, + weight, + port, + target, + }); + } + cursor = rdata_end; + } + Some(out) +} + +fn skip_name(buf: &[u8], mut cursor: usize) -> Option { + loop { + if cursor >= buf.len() { + return None; + } + let len = buf[cursor]; + if len == 0 { + return Some(cursor + 1); + } + if (len & 0xC0) == 0xC0 { + return Some(cursor + 2); + } + cursor = cursor.checked_add(1 + len as usize)?; + } +} + +fn read_name(buf: &[u8], start: usize) -> Option { + let mut out = String::new(); + let mut cursor = start; + let mut hops = 0; + loop { + if cursor >= buf.len() { + return None; + } + let len = buf[cursor]; + if len == 0 { + break; + } + if (len & 0xC0) == 0xC0 { + if cursor + 1 >= buf.len() { + return None; + } + let pointer = ((len & 0x3F) as usize) << 8 | (buf[cursor + 1] as usize); + hops += 1; + if hops > 8 { + return None; + } + cursor = pointer; + continue; + } + cursor += 1; + let end = cursor.checked_add(len as usize)?; + if end > buf.len() { + return None; + } + if !out.is_empty() { + out.push('.'); + } + out.push_str(std::str::from_utf8(&buf[cursor..end]).ok()?); + cursor = end; + } + Some(out) +} + +fn autodiscover_v1( + agent: &Agent, + url: &str, + email: &str, + auth_header: Option<&str>, +) -> Result { + let body = format!( + "\ + \ + {}\ + {POX_RESP_NS}\ + ", + xml_escape(email) + ); + let mut req = agent + .post(url) + .header("Content-Type", "text/xml; charset=utf-8"); + if let Some(h) = auth_header { + req = req.header("Authorization", h); + } + let mut resp = req + .send(body.as_bytes()) + .map_err(|e| EwsError::Transport(format!("v1 POST: {e}")))?; + let status = resp.status().as_u16(); + if !(200..300).contains(&status) { + return Err(EwsError::AutodiscoverFailed(format!( + "{url} returned http {status}" + ))); + } + let bytes = resp + .body_mut() + .with_config() + .limit(2 * 1024 * 1024) + .read_to_vec() + .map_err(|e| EwsError::Malformed(format!("v1 body: {e}")))?; + parse_pox_response(&bytes) +} + +fn parse_pox_response(body: &[u8]) -> Result { + let mut xml = NsReader::from_reader(body); + xml.config_mut().trim_text(true); + let mut buf = Vec::new(); + let mut current: Option<&'static str> = None; + let mut current_type: Option = None; + let mut ews_url: Option = None; + let mut redirect_addr: Option = None; + let mut redirect_url: Option = None; + let mut action: Option = None; + loop { + buf.clear(); + let (_, ev) = xml.read_resolved_event_into(&mut buf)?; + match ev { + Event::Start(e) => { + let local = e.local_name().as_ref().to_vec(); + if local.eq_ignore_ascii_case(b"Protocol") { + current_type = None; + } + if local.eq_ignore_ascii_case(b"Type") { + current = Some("type"); + } else if local.eq_ignore_ascii_case(b"EwsUrl") { + if matches!(current_type.as_deref(), Some("EXPR") | Some("EXCH")) { + current = Some("ewsUrl"); + } else { + current = None; + } + } else if local.eq_ignore_ascii_case(b"Action") { + current = Some("action"); + } else if local.eq_ignore_ascii_case(b"RedirectAddr") { + current = Some("redirectAddr"); + } else if local.eq_ignore_ascii_case(b"RedirectUrl") { + current = Some("redirectUrl"); + } else { + current = None; + } + } + Event::End(e) => { + let local = e.local_name().as_ref().to_vec(); + if local.eq_ignore_ascii_case(b"Protocol") { + current_type = None; + } + current = None; + } + Event::Text(t) => { + let text = t.decode().map(|c| c.into_owned()).unwrap_or_default(); + match current { + Some("type") => current_type = Some(text), + Some("ewsUrl") => ews_url = Some(text), + Some("action") => action = Some(text), + Some("redirectAddr") => redirect_addr = Some(text), + Some("redirectUrl") => redirect_url = Some(text), + _ => {} + } + } + Event::Eof => break, + _ => {} + } + } + if let Some(addr) = redirect_addr.filter(|_| action.as_deref() == Some("redirectAddr")) { + return Ok(PoxOutcome::RedirectAddr(addr)); + } + if let Some(url) = redirect_url.filter(|_| action.as_deref() == Some("redirectUrl")) { + return Ok(PoxOutcome::RedirectUrl(url)); + } + if let Some(url) = ews_url { + return Ok(PoxOutcome::EwsUrl(url)); + } + Ok(PoxOutcome::Failed) +} + +fn xml_escape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for ch in s.chars() { + match ch { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + _ => out.push(ch), + } + } + out +} + +fn urlencode(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for b in s.bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(b as char); + } + _ => out.push_str(&format!("%{b:02X}")), + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detects_fully_qualified_ews_url() { + assert!(is_fully_qualified_ews_url( + "https://outlook.office365.com/EWS/Exchange.asmx" + )); + assert!(is_fully_qualified_ews_url( + "https://exchange.example.com/EWS/Exchange.asmx" + )); + assert!(!is_fully_qualified_ews_url( + "http://outlook.office365.com/EWS/Exchange.asmx" + )); + assert!(!is_fully_qualified_ews_url( + "https://outlook.office365.com/" + )); + } + + #[test] + fn pox_response_extracts_ews_url() { + let body = format!( + "\ + \ + EXPRhttps://srv/EWS/Exchange.asmx" + ); + match parse_pox_response(body.as_bytes()).unwrap() { + PoxOutcome::EwsUrl(u) => assert_eq!(u, "https://srv/EWS/Exchange.asmx"), + _ => panic!("expected EwsUrl"), + } + } + + #[test] + fn pox_response_handles_redirect_addr() { + let body = format!( + "\ + \ + redirectAddrbob@other.com" + ); + match parse_pox_response(body.as_bytes()).unwrap() { + PoxOutcome::RedirectAddr(a) => assert_eq!(a, "bob@other.com"), + _ => panic!("expected RedirectAddr"), + } + } +} diff --git a/src/exchange_ews/calendar_map.rs b/src/exchange_ews/calendar_map.rs new file mode 100644 index 0000000..72afffb --- /dev/null +++ b/src/exchange_ews/calendar_map.rs @@ -0,0 +1,580 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::{Map, Value, json}; + +use crate::exchange_ews::parse::{CalendarItemRaw, RawAttendee, RawOccurrence}; +use crate::exchange_ews::recurrence::to_jscalendar_rule; +use crate::exchange_ews::tz::windows_to_iana; + +pub struct EventValue { + pub data: Value, + pub is_draft: bool, + pub use_default_alerts: bool, +} + +pub fn to_jscalendar(raw: &CalendarItemRaw) -> EventValue { + let mut event = Map::new(); + event.insert("@type".to_owned(), Value::String("Event".to_owned())); + if let Some(uid) = raw.uid.as_ref() { + event.insert("uid".to_owned(), Value::String(uid.clone())); + } else { + event.insert( + "uid".to_owned(), + Value::String(format!( + "vandelay-ews-event-{}", + blake3::hash(raw.id.id.as_bytes()).to_hex() + )), + ); + } + if let Some(subject) = raw.subject.as_ref() { + event.insert("title".to_owned(), Value::String(subject.clone())); + } + if let Some(loc) = raw.location.as_ref() { + event.insert( + "locations".to_owned(), + Value::Object(map_singleton( + "1", + json!({"@type": "Location", "name": loc}), + )), + ); + } + if let Some(body) = raw.body_text.as_ref() { + event.insert("description".to_owned(), Value::String(body.clone())); + } else if let Some(body) = raw.body_html.as_ref() { + event.insert("description".to_owned(), Value::String(body.clone())); + event.insert( + "descriptionContentType".to_owned(), + Value::String("text/html".to_owned()), + ); + } + if let Some(created) = raw.created.as_ref() { + event.insert( + "created".to_owned(), + Value::String(normalise_utc_datetime(created)), + ); + } + if let Some(updated) = raw.last_modified.as_ref() { + event.insert( + "updated".to_owned(), + Value::String(normalise_utc_datetime(updated)), + ); + } + if !raw.categories.is_empty() { + let mut kw = Map::new(); + for c in &raw.categories { + kw.insert(c.to_ascii_lowercase(), Value::Bool(true)); + } + event.insert("keywords".to_owned(), Value::Object(kw)); + } + let iana = raw + .start_tz + .as_deref() + .map(|tz| windows_to_iana(tz).unwrap_or(tz).to_owned()); + if let Some(true) = raw.is_all_day_event { + if let Some(start) = raw.start.as_ref() { + let date_only = start.split('T').next().unwrap_or(start.as_str()); + event.insert( + "start".to_owned(), + Value::String(format!("{date_only}T00:00:00")), + ); + } + event.insert("showWithoutTime".to_owned(), Value::Bool(true)); + let days = match (raw.start.as_ref(), raw.end.as_ref()) { + (Some(s), Some(e)) => all_day_span_days(s, e).max(1), + _ => 1, + }; + event.insert("duration".to_owned(), Value::String(format!("P{days}D"))); + } else { + if let Some(start) = raw.start.as_ref() { + event.insert( + "start".to_owned(), + Value::String(to_local_datetime_in(start, iana.as_deref())), + ); + } + if let (Some(start), Some(end)) = (raw.start.as_ref(), raw.end.as_ref()) + && let Some(dur) = duration_iso8601(start, end) + { + event.insert("duration".to_owned(), Value::String(dur)); + } + if let Some(tz) = iana.as_deref() { + event.insert("timeZone".to_owned(), Value::String(tz.to_owned())); + } + } + if let Some(status) = raw.legacy_free_busy_status.as_ref() { + let mapped = match status.as_str() { + "Free" => "free", + _ => "busy", + }; + event.insert( + "freeBusyStatus".to_owned(), + Value::String(mapped.to_owned()), + ); + } + let mut participants = Map::new(); + let mut next_id = 1; + if let Some(email) = raw.organizer_smtp.as_ref() { + let key = next_id.to_string(); + next_id += 1; + let cal_addr = format!("mailto:{email}"); + event.insert( + "organizerCalendarAddress".to_owned(), + Value::String(cal_addr.clone()), + ); + let mut p = Map::new(); + p.insert("@type".to_owned(), Value::String("Participant".to_owned())); + p.insert("calendarAddress".to_owned(), Value::String(cal_addr)); + p.insert("email".to_owned(), Value::String(email.clone())); + let mut roles = Map::new(); + roles.insert("owner".to_owned(), Value::Bool(true)); + roles.insert("chair".to_owned(), Value::Bool(true)); + p.insert("roles".to_owned(), Value::Object(roles)); + if let Some(name) = raw.organizer_name.as_ref() { + p.insert("name".to_owned(), Value::String(name.clone())); + } + participants.insert(key, Value::Object(p)); + } + add_attendees( + &mut participants, + &mut next_id, + &raw.required_attendees, + true, + ); + add_attendees( + &mut participants, + &mut next_id, + &raw.optional_attendees, + false, + ); + if !participants.is_empty() { + event.insert("participants".to_owned(), Value::Object(participants)); + } + if let Some(rec) = raw.recurrence.as_ref() + && let Some(rule) = to_jscalendar_rule(rec) + { + event.insert("recurrenceRules".to_owned(), Value::Array(vec![rule])); + } + let overrides = build_recurrence_overrides( + &raw.modified_occurrences, + &raw.deleted_occurrences, + iana.as_deref(), + ); + if let Some(overrides) = overrides { + event.insert("recurrenceOverrides".to_owned(), overrides); + } + EventValue { + data: Value::Object(event), + is_draft: false, + use_default_alerts: false, + } +} + +fn add_attendees( + out: &mut Map, + next_id: &mut u32, + attendees: &[RawAttendee], + required: bool, +) { + for att in attendees { + let key = next_id.to_string(); + *next_id += 1; + let mut p = Map::new(); + p.insert("@type".to_owned(), Value::String("Participant".to_owned())); + if let Some(email) = att.email.as_ref() { + p.insert( + "calendarAddress".to_owned(), + Value::String(format!("mailto:{email}")), + ); + p.insert("email".to_owned(), Value::String(email.clone())); + } + if let Some(name) = att.name.as_ref() { + p.insert("name".to_owned(), Value::String(name.clone())); + } + let mut roles = Map::new(); + if required { + roles.insert("required".to_owned(), Value::Bool(true)); + } else { + roles.insert("optional".to_owned(), Value::Bool(true)); + } + p.insert("roles".to_owned(), Value::Object(roles)); + if let Some(rt) = att.response_type.as_ref() { + let mapped = match rt.as_str() { + "Accept" => "accepted", + "Tentative" => "tentative", + "Decline" => "declined", + "Organizer" => "accepted", + _ => "needs-action", + }; + p.insert( + "participationStatus".to_owned(), + Value::String(mapped.to_owned()), + ); + } + out.insert(key, Value::Object(p)); + } +} + +fn build_recurrence_overrides( + modified: &[RawOccurrence], + deleted: &[RawOccurrence], + iana: Option<&str>, +) -> Option { + if modified.is_empty() && deleted.is_empty() { + return None; + } + let mut map = Map::new(); + for occ in modified { + let Some(key) = occ + .original_start + .as_deref() + .or(occ.start.as_deref()) + .map(|s| to_local_datetime_in(s, iana)) + else { + continue; + }; + let mut o = Map::new(); + if let Some(s) = occ.start.as_ref() { + o.insert( + "start".to_owned(), + Value::String(to_local_datetime_in(s, iana)), + ); + } + if let (Some(s), Some(e)) = (occ.start.as_ref(), occ.end.as_ref()) + && let Some(dur) = duration_iso8601(s, e) + { + o.insert("duration".to_owned(), Value::String(dur)); + } + map.insert(key, Value::Object(o)); + } + for occ in deleted { + let Some(key) = occ.start.as_deref().map(|s| to_local_datetime_in(s, iana)) else { + continue; + }; + map.insert(key, json!({"excluded": true})); + } + if map.is_empty() { + None + } else { + Some(Value::Object(map)) + } +} + +fn normalise_utc_datetime(s: &str) -> String { + let trimmed = s.trim(); + let stripped = trimmed + .strip_suffix('Z') + .or_else(|| { + trimmed + .rfind(['+', '-']) + .filter(|i| *i >= 10) + .map(|i| &trimmed[..i]) + }) + .unwrap_or(trimmed); + let (date, time) = match stripped.split_once('T') { + Some((d, t)) => (d, t), + None => return format!("{stripped}T00:00:00Z"), + }; + let time_clean = if let Some(dot) = time.find('.') { + &time[..dot] + } else { + time + }; + format!("{date}T{time_clean}Z") +} + +fn to_local_datetime_in(s: &str, iana: Option<&str>) -> String { + let trimmed = s.trim(); + let utc_anchored = + trimmed.ends_with('Z') || trimmed.rfind(['+', '-']).filter(|i| *i >= 10).is_some(); + if utc_anchored + && let Some(tz) = iana + && let Some(out) = convert_utc_to_local(trimmed, tz) + { + return out; + } + let stripped = trimmed + .strip_suffix('Z') + .or_else(|| { + trimmed + .rfind(['+', '-']) + .filter(|i| *i >= 10) + .map(|i| &trimmed[..i]) + }) + .unwrap_or(trimmed); + let (date, time) = match stripped.split_once('T') { + Some((d, t)) => (d, t), + None => return format!("{stripped}T00:00:00"), + }; + let time_clean = if let Some(dot) = time.find('.') { + &time[..dot] + } else { + time + }; + format!("{date}T{time_clean}") +} + +fn convert_utc_to_local(s: &str, iana: &str) -> Option { + use chrono::DateTime; + let utc: DateTime = DateTime::parse_from_rfc3339(s) + .ok()? + .with_timezone(&chrono::Utc); + let tz: chrono_tz::Tz = iana.parse().ok()?; + let local = utc.with_timezone(&tz); + Some(local.format("%Y-%m-%dT%H:%M:%S").to_string()) +} + +fn all_day_span_days(start: &str, end: &str) -> u32 { + fn date_only(s: &str) -> &str { + s.split('T').next().unwrap_or(s) + } + fn ymd(s: &str) -> Option<(i64, u32, u32)> { + let mut p = s.split('-'); + let y: i64 = p.next()?.parse().ok()?; + let m: u32 = p.next()?.parse().ok()?; + let d: u32 = p.next()?.parse().ok()?; + Some((y, m, d)) + } + fn to_days(y: i64, m: u32, d: u32) -> i64 { + let (mut y, m) = if m <= 2 { + (y - 1, m as i64 + 12) + } else { + (y, m as i64) + }; + y += 4800; + let mm = m + 1; + 365 * y + y / 4 - y / 100 + y / 400 + 30 * mm + 3 * (mm + 1) / 5 + d as i64 - 32045 + } + let (Some(s), Some(e)) = (ymd(date_only(start)), ymd(date_only(end))) else { + return 1; + }; + let diff = to_days(e.0, e.1, e.2) - to_days(s.0, s.1, s.2); + if diff <= 0 { 1 } else { diff as u32 } +} + +fn duration_iso8601(start: &str, end: &str) -> Option { + use time::OffsetDateTime; + use time::format_description::well_known::Rfc3339; + let s = OffsetDateTime::parse(start, &Rfc3339).ok()?; + let e = OffsetDateTime::parse(end, &Rfc3339).ok()?; + let dur = e - s; + if dur.is_zero() { + return Some("PT0S".to_owned()); + } + let total_seconds = dur.whole_seconds(); + if total_seconds < 0 { + return None; + } + let hours = total_seconds / 3600; + let minutes = (total_seconds % 3600) / 60; + let seconds = total_seconds % 60; + let mut out = String::from("PT"); + if hours > 0 { + out.push_str(&format!("{hours}H")); + } + if minutes > 0 { + out.push_str(&format!("{minutes}M")); + } + if seconds > 0 || (hours == 0 && minutes == 0) { + out.push_str(&format!("{seconds}S")); + } + Some(out) +} + +fn map_singleton(key: &str, value: Value) -> Map { + let mut m = Map::new(); + m.insert(key.to_owned(), value); + m +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::exchange_ews::parse::{RecurrencePattern, RecurrenceRange}; + + #[test] + fn simple_event_round_trip_converts_utc_to_local_wall_clock() { + let raw = CalendarItemRaw { + uid: Some("uid-1".to_owned()), + subject: Some("Meeting".to_owned()), + start: Some("2025-06-15T14:00:00Z".to_owned()), + end: Some("2025-06-15T15:00:00Z".to_owned()), + location: Some("HQ".to_owned()), + start_tz: Some("Pacific Standard Time".to_owned()), + ..CalendarItemRaw::default() + }; + let v = to_jscalendar(&raw).data; + assert_eq!(v["@type"], "Event"); + assert_eq!(v["uid"], "uid-1"); + assert_eq!(v["title"], "Meeting"); + assert_eq!(v["start"], "2025-06-15T07:00:00"); + assert_eq!(v["duration"], "PT1H"); + assert_eq!(v["timeZone"], "America/Los_Angeles"); + assert_eq!(v["locations"]["1"]["name"], "HQ"); + } + + #[test] + fn naive_start_without_timezone_stays_floating() { + let raw = CalendarItemRaw { + uid: Some("uid-naive".to_owned()), + start: Some("2025-06-15T14:00:00".to_owned()), + end: Some("2025-06-15T15:00:00".to_owned()), + ..CalendarItemRaw::default() + }; + let v = to_jscalendar(&raw).data; + assert_eq!(v["start"], "2025-06-15T14:00:00"); + assert!(v.get("timeZone").is_none()); + } + + #[test] + fn unknown_windows_timezone_passes_through_verbatim() { + let raw = CalendarItemRaw { + uid: Some("uid-x".to_owned()), + start: Some("2025-06-15T14:00:00Z".to_owned()), + end: Some("2025-06-15T15:00:00Z".to_owned()), + start_tz: Some("Made Up Time".to_owned()), + ..CalendarItemRaw::default() + }; + let v = to_jscalendar(&raw).data; + assert_eq!(v["start"], "2025-06-15T14:00:00"); + assert_eq!(v["timeZone"], "Made Up Time"); + } + + #[test] + fn recurrence_override_key_uses_local_time_in_event_timezone() { + let raw = CalendarItemRaw { + uid: Some("uid-tz".to_owned()), + start: Some("2025-06-15T21:00:00Z".to_owned()), + end: Some("2025-06-15T22:00:00Z".to_owned()), + start_tz: Some("Pacific Standard Time".to_owned()), + recurrence: Some(crate::exchange_ews::parse::RawRecurrence { + pattern: Some(RecurrencePattern::Daily { interval: 1 }), + range: Some(RecurrenceRange::Numbered { + start_date: "2025-06-15".to_owned(), + number_of_occurrences: 3, + }), + }), + deleted_occurrences: vec![RawOccurrence { + item_id: Default::default(), + start: Some("2025-06-16T21:00:00Z".to_owned()), + end: None, + original_start: None, + }], + ..CalendarItemRaw::default() + }; + let v = to_jscalendar(&raw).data; + let over = v["recurrenceOverrides"].as_object().unwrap(); + assert!(over.contains_key("2025-06-16T14:00:00")); + } + + #[test] + fn all_day_event_uses_date_only_start() { + let raw = CalendarItemRaw { + uid: Some("uid-2".to_owned()), + subject: Some("Holiday".to_owned()), + is_all_day_event: Some(true), + start: Some("2025-12-25T00:00:00".to_owned()), + end: Some("2025-12-26T00:00:00".to_owned()), + ..CalendarItemRaw::default() + }; + let v = to_jscalendar(&raw).data; + assert_eq!(v["start"], "2025-12-25T00:00:00"); + assert_eq!(v["showWithoutTime"], true); + assert_eq!(v["duration"], "P1D"); + assert!(v.get("timeZone").is_none()); + } + + #[test] + fn recurring_master_with_overrides() { + let raw = CalendarItemRaw { + uid: Some("uid-3".to_owned()), + start: Some("2025-06-15T14:00:00Z".to_owned()), + end: Some("2025-06-15T15:00:00Z".to_owned()), + recurrence: Some(crate::exchange_ews::parse::RawRecurrence { + pattern: Some(RecurrencePattern::Daily { interval: 1 }), + range: Some(RecurrenceRange::Numbered { + start_date: "2025-06-15".to_owned(), + number_of_occurrences: 3, + }), + }), + modified_occurrences: vec![RawOccurrence { + item_id: Default::default(), + start: Some("2025-06-16T15:00:00Z".to_owned()), + end: Some("2025-06-16T16:30:00Z".to_owned()), + original_start: Some("2025-06-16T14:00:00Z".to_owned()), + }], + deleted_occurrences: vec![RawOccurrence { + item_id: Default::default(), + start: Some("2025-06-17T14:00:00Z".to_owned()), + end: None, + original_start: None, + }], + ..CalendarItemRaw::default() + }; + let v = to_jscalendar(&raw).data; + assert_eq!(v["recurrenceRules"][0]["frequency"], "daily"); + assert_eq!(v["recurrenceRules"][0]["count"], 3); + let over = v["recurrenceOverrides"].as_object().unwrap(); + assert!(over.contains_key("2025-06-16T14:00:00")); + assert_eq!(over["2025-06-17T14:00:00"]["excluded"], true); + } + + #[test] + fn multi_day_all_day_spans_correct_number_of_days() { + let raw = CalendarItemRaw { + uid: Some("uid-multi".to_owned()), + is_all_day_event: Some(true), + start: Some("2025-07-04T00:00:00".to_owned()), + end: Some("2025-07-07T00:00:00".to_owned()), + ..CalendarItemRaw::default() + }; + let v = to_jscalendar(&raw).data; + assert_eq!(v["duration"], "P3D"); + assert_eq!(v["showWithoutTime"], true); + } + + #[test] + fn attendees_get_required_or_optional_role() { + let raw = CalendarItemRaw { + uid: Some("uid-att".to_owned()), + start: Some("2025-06-15T14:00:00Z".to_owned()), + end: Some("2025-06-15T15:00:00Z".to_owned()), + organizer_smtp: Some("alice@x".to_owned()), + required_attendees: vec![crate::exchange_ews::parse::RawAttendee { + email: Some("bob@x".to_owned()), + name: None, + response_type: Some("Accept".to_owned()), + }], + optional_attendees: vec![crate::exchange_ews::parse::RawAttendee { + email: Some("eve@x".to_owned()), + name: None, + response_type: None, + }], + ..CalendarItemRaw::default() + }; + let v = to_jscalendar(&raw).data; + assert_eq!(v["organizerCalendarAddress"], "mailto:alice@x"); + let participants = v["participants"].as_object().unwrap(); + let required = participants + .values() + .find(|p| p["email"] == "bob@x") + .unwrap(); + assert_eq!(required["roles"]["required"], true); + assert!(required["roles"].get("optional").is_none()); + let optional = participants + .values() + .find(|p| p["email"] == "eve@x") + .unwrap(); + assert_eq!(optional["roles"]["optional"], true); + } + + #[test] + fn all_day_span_helper_is_inclusive_of_start() { + assert_eq!(all_day_span_days("2025-07-04", "2025-07-04"), 1); + assert_eq!(all_day_span_days("2025-07-04", "2025-07-05"), 1); + assert_eq!(all_day_span_days("2025-07-04", "2025-07-07"), 3); + assert_eq!(all_day_span_days("bad", "input"), 1); + } +} diff --git a/src/exchange_ews/client.rs b/src/exchange_ews/client.rs new file mode 100644 index 0000000..6d24010 --- /dev/null +++ b/src/exchange_ews/client.rs @@ -0,0 +1,445 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::time::Duration; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use ureq::Agent; +use ureq::config::{Config, RedirectAuthHeaders}; +use ureq::tls::{RootCerts, TlsConfig}; + +use crate::exchange_ews::error::EwsError; +use crate::exchange_ews::parse::{EnvelopeKind, read_envelope_summary}; +use crate::exchange_ews::retry::{FaultDisposition, classify_fault, classify_http_status}; +use crate::exchange_ews::soap::{EnvelopeOptions, soap_action, wrap_envelope}; +use crate::exchange_ews::types::ServerVersion; +use crate::jmap::http::{Auth, RetryPolicy, retry_after_header}; +use crate::jmap::retry::{self, Disposition, RateLimitState}; +use crate::logging::{LEVEL_BODIES, LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; + +const MAX_BODY: u64 = 2 * 1024 * 1024 * 1024; +const LONG_RETRY_THRESHOLD: Duration = Duration::from_secs(10); + +struct Inner { + agent: Agent, + auth: Mutex, + impersonated_smtp: Mutex>, + anchor_mailbox: Mutex>, + retry: RetryPolicy, + rate_limit: RateLimitState, + version: Mutex, + log_level: AtomicU8, + retries_total: AtomicU64, + retry_after_sleeps: AtomicU64, + soap_calls: AtomicU64, + user_agent: String, +} + +#[derive(Clone)] +pub struct EwsClient { + inner: Arc, +} + +#[derive(Debug, Clone)] +pub struct SoapResponse { + pub body: Vec, + pub server_version: Option, +} + +impl EwsClient { + pub fn new(auth: Auth, retry: RetryPolicy, allow_invalid_certs: bool) -> EwsClient { + let config: Config = Config::builder() + .http_status_as_error(false) + .redirect_auth_headers(RedirectAuthHeaders::SameHost) + .tls_config( + TlsConfig::builder() + .root_certs(RootCerts::PlatformVerifier) + .disable_verification(allow_invalid_certs) + .build(), + ) + .build(); + EwsClient { + inner: Arc::new(Inner { + agent: config.new_agent(), + auth: Mutex::new(auth), + impersonated_smtp: Mutex::new(None), + anchor_mailbox: Mutex::new(None), + retry, + rate_limit: RateLimitState::new(), + version: Mutex::new(ServerVersion::Exchange2013Sp1), + log_level: AtomicU8::new(LEVEL_DEFAULT), + retries_total: AtomicU64::new(0), + retry_after_sleeps: AtomicU64::new(0), + soap_calls: AtomicU64::new(0), + user_agent: format!("vandelay/{}", env!("CARGO_PKG_VERSION")), + }), + } + } + + pub fn set_auth(&self, auth: Auth) { + if let Ok(mut g) = self.inner.auth.lock() { + *g = auth; + } + } + + pub fn set_anchor_mailbox(&self, smtp: Option) { + if let Ok(mut g) = self.inner.anchor_mailbox.lock() { + *g = smtp; + } + } + + pub fn set_impersonation(&self, smtp: Option) { + if let Ok(mut g) = self.inner.impersonated_smtp.lock() { + *g = smtp; + } + } + + pub fn set_logger(&self, logger: Logger) { + self.inner + .log_level + .store(logger.level(), Ordering::Relaxed); + } + + pub fn logger(&self) -> Logger { + Logger::new(self.inner.log_level.load(Ordering::Relaxed)) + } + + pub fn set_server_version(&self, version: ServerVersion) { + if let Ok(mut g) = self.inner.version.lock() { + *g = version; + } + } + + pub fn server_version(&self) -> ServerVersion { + self.inner + .version + .lock() + .map(|g| *g) + .unwrap_or(ServerVersion::Exchange2013Sp1) + } + + pub fn retries_observed(&self) -> u64 { + self.inner.retries_total.load(Ordering::Relaxed) + } + + pub fn retry_after_sleeps(&self) -> u64 { + self.inner.retry_after_sleeps.load(Ordering::Relaxed) + } + + pub fn soap_calls(&self) -> u64 { + self.inner.soap_calls.load(Ordering::Relaxed) + } + + pub fn rate_limit(&self) -> &RateLimitState { + &self.inner.rate_limit + } + + pub fn call(&self, url: &str, operation: &str, body: &str) -> Result { + let envelope = self.wrap(body); + let action = soap_action(operation); + self.inner.soap_calls.fetch_add(1, Ordering::Relaxed); + self.execute(url, &envelope, &action) + } + + fn wrap(&self, body: &str) -> String { + let version = self.server_version(); + let smtp = self + .inner + .impersonated_smtp + .lock() + .ok() + .and_then(|g| g.clone()); + let opts = EnvelopeOptions { + version, + impersonated_smtp: smtp.as_deref(), + }; + wrap_envelope(opts, body) + } + + fn auth_header(&self) -> String { + let auth = self.inner.auth.lock().ok().map(|g| g.clone()); + match auth { + Some(Auth::Basic { user, password }) => { + format!("Basic {}", STANDARD.encode(format!("{user}:{password}"))) + } + Some(Auth::Bearer { token }) => format!("Bearer {token}"), + None => String::new(), + } + } + + fn anchor_header(&self) -> Option { + self.inner + .anchor_mailbox + .lock() + .ok() + .and_then(|g| g.clone()) + } + + fn execute(&self, url: &str, body: &str, action: &str) -> Result { + let logger = self.logger(); + let policy = self.inner.retry; + let mut attempt: u32 = 0; + loop { + self.inner.rate_limit.cooldown().wait(); + let outcome = self.one_attempt(url, body, action); + match outcome { + AttemptOutcome::Ok { + status, + body: bytes, + retry_after, + } => { + if (200..300).contains(&status) { + self.inner.rate_limit.on_success(); + match read_envelope_summary(&bytes) { + Ok(EnvelopeKind::Body { version }) => { + if let Some(v) = version.to_server_version() { + self.set_server_version(v); + } + return Ok(SoapResponse { + body: bytes, + server_version: version.to_server_version(), + }); + } + Ok(EnvelopeKind::Fault { fault, version }) => { + if let Some(v) = version.to_server_version() { + self.set_server_version(v); + } + match classify_fault(&fault.response_code) { + FaultDisposition::Fatal => { + return Err(EwsError::SoapFault { + code: fault.response_code, + reason: fault.fault_string, + }); + } + FaultDisposition::Auth => { + return Err(EwsError::Auth(fault.fault_string)); + } + FaultDisposition::Retryable { delay } => { + attempt += 1; + self.inner.retries_total.fetch_add(1, Ordering::Relaxed); + if attempt > policy.max_retries { + return Err(EwsError::RetriesExhausted(format!( + "{operation} kept returning {code}", + operation = action, + code = fault.response_code + ))); + } + let chosen = + self.inner.rate_limit.on_throttle(&policy, delay); + if chosen >= LONG_RETRY_THRESHOLD { + logger.warn(&format!( + "EWS soap fault {} ({}); waiting {}s before retry {}/{}", + fault.response_code, + fault.fault_string, + chosen.as_secs(), + attempt, + policy.max_retries + )); + } + if logger.enabled(LEVEL_BODIES) { + eprintln!( + "retry {}/{} {} after {:?} ({})", + attempt, + policy.max_retries, + action, + chosen, + fault.response_code, + ); + } + std::thread::sleep(chosen); + continue; + } + } + } + Err(e) => return Err(e), + } + } + if status == 401 { + return Err(EwsError::Auth(format!( + "server returned 401: {}", + truncate(&bytes) + ))); + } + if status == 403 { + return Err(EwsError::HttpStatus { + status: 403, + body: truncate(&bytes), + }); + } + match classify_http_status(status) { + Disposition::Fatal => { + return Err(EwsError::HttpStatus { + status, + body: truncate(&bytes), + }); + } + Disposition::Retryable => { + attempt += 1; + self.inner.retries_total.fetch_add(1, Ordering::Relaxed); + if attempt > policy.max_retries { + return Err(EwsError::RetriesExhausted(format!( + "{action} kept returning http {status}" + ))); + } + if retry_after.is_some() { + self.inner + .retry_after_sleeps + .fetch_add(1, Ordering::Relaxed); + } + let chosen = self.inner.rate_limit.on_throttle(&policy, retry_after); + if chosen >= LONG_RETRY_THRESHOLD { + logger.warn(&format!( + "EWS {action} rate-limited (http {status}); waiting {}s", + chosen.as_secs() + )); + } + if logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "retry {}/{} (http {status})", + attempt, policy.max_retries + ); + } + std::thread::sleep(chosen); + } + } + } + AttemptOutcome::Transport(err) => { + attempt += 1; + self.inner.retries_total.fetch_add(1, Ordering::Relaxed); + if attempt > policy.max_retries { + return Err(err); + } + let delay = retry::backoff_delay(&policy, attempt); + if delay >= LONG_RETRY_THRESHOLD { + logger.warn(&format!( + "transient transport failure ({err}); waiting {}s before retry {}/{}", + delay.as_secs(), + attempt, + policy.max_retries + )); + } + std::thread::sleep(delay); + } + } + } + } + + fn one_attempt(&self, url: &str, body: &str, action: &str) -> AttemptOutcome { + let mut req = self + .inner + .agent + .post(url) + .header("Authorization", self.auth_header()) + .header("Content-Type", "text/xml; charset=utf-8") + .header("Accept", "text/xml, application/soap+xml, application/xml") + .header("Accept-Encoding", "gzip") + .header("SOAPAction", action.to_owned()) + .header("User-Agent", self.inner.user_agent.as_str()); + if let Some(anchor) = self.anchor_header() { + req = req.header("X-AnchorMailbox", anchor); + } + let result = req.send(body.as_bytes()); + match result { + Ok(mut resp) => { + let status = resp.status().as_u16(); + let retry_after = resp + .headers() + .get("retry-after") + .and_then(|v| v.to_str().ok()) + .and_then(retry_after_header); + match resp.body_mut().with_config().limit(MAX_BODY).read_to_vec() { + Ok(bytes) => AttemptOutcome::Ok { + status, + body: bytes, + retry_after, + }, + Err(e) => AttemptOutcome::Transport(EwsError::Transport(format!( + "reading response body: {e}" + ))), + } + } + Err(e) => AttemptOutcome::Transport(map_ureq_error(e)), + } + } +} + +enum AttemptOutcome { + Ok { + status: u16, + body: Vec, + retry_after: Option, + }, + Transport(EwsError), +} + +fn map_ureq_error(err: ureq::Error) -> EwsError { + match err { + ureq::Error::Io(e) => EwsError::Transport(format!("io: {e}")), + ureq::Error::Timeout(t) => EwsError::Transport(format!("timeout: {t}")), + ureq::Error::HostNotFound => EwsError::Transport("host not found".to_owned()), + ureq::Error::ConnectionFailed => EwsError::Transport("connection failed".to_owned()), + ureq::Error::BodyStalled => EwsError::Transport("body stalled".to_owned()), + ureq::Error::Tls(m) => EwsError::Transport(format!("tls: {m}")), + ureq::Error::TooManyRedirects => EwsError::Connect("too many redirects".to_owned()), + ureq::Error::RedirectFailed => EwsError::Connect("redirect failed".to_owned()), + ureq::Error::TlsRequired => { + EwsError::Connect("server requires TLS but transport is unsecured".to_owned()) + } + other => EwsError::Connect(other.to_string()), + } +} + +fn truncate(body: &[u8]) -> String { + let text = String::from_utf8_lossy(body); + if text.len() <= 512 { + return text.into_owned(); + } + let end = text.floor_char_boundary(512); + format!("{}...", &text[..end]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn client_constructs_with_defaults() { + let c = EwsClient::new( + Auth::Bearer { token: "t".into() }, + RetryPolicy::new(3), + false, + ); + assert_eq!(c.server_version(), ServerVersion::Exchange2013Sp1); + assert_eq!(c.retries_observed(), 0); + assert_eq!(c.retry_after_sleeps(), 0); + } + + #[test] + fn server_version_can_be_pinned() { + let c = EwsClient::new( + Auth::Bearer { token: "t".into() }, + RetryPolicy::new(0), + false, + ); + c.set_server_version(ServerVersion::Exchange2019); + assert_eq!(c.server_version(), ServerVersion::Exchange2019); + } + + #[test] + fn anchor_mailbox_round_trips() { + let c = EwsClient::new( + Auth::Bearer { token: "t".into() }, + RetryPolicy::new(0), + false, + ); + c.set_anchor_mailbox(Some("alice@x".to_owned())); + assert_eq!(c.anchor_header().as_deref(), Some("alice@x")); + } +} diff --git a/src/exchange_ews/contact_map.rs b/src/exchange_ews/contact_map.rs new file mode 100644 index 0000000..1ffe4c8 --- /dev/null +++ b/src/exchange_ews/contact_map.rs @@ -0,0 +1,454 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::{Map, Value, json}; + +use crate::exchange_ews::parse::{ContactItemRaw, RawContactAddress}; + +pub fn synthetic_uid(item_id: &str) -> String { + let hash = blake3::hash(item_id.as_bytes()).to_hex(); + format!("vandelay-ews-syn-{hash}") +} + +pub fn to_jscontact(raw: &ContactItemRaw) -> Value { + let mut card = Map::new(); + card.insert("@type".to_owned(), Value::String("Card".to_owned())); + card.insert("version".to_owned(), Value::String("1.0".to_owned())); + if let Some(full) = raw.display_name.as_ref() { + let mut name = Map::new(); + name.insert("@type".to_owned(), Value::String("Name".to_owned())); + name.insert("full".to_owned(), Value::String(full.clone())); + let components = name_components(raw); + if !components.is_empty() { + name.insert("components".to_owned(), Value::Array(components)); + } + card.insert("name".to_owned(), Value::Object(name)); + } + if let Some(n) = raw.nickname.as_ref() { + card.insert( + "nicknames".to_owned(), + Value::Object(map_singleton("1", json!({"@type": "Nickname", "name": n}))), + ); + } + let mut organizations: Map = Map::new(); + let mut have_org = false; + let mut org_inner: Map = Map::new(); + org_inner.insert("@type".to_owned(), Value::String("Organization".to_owned())); + if let Some(co) = raw.company_name.as_ref() { + org_inner.insert("name".to_owned(), Value::String(co.clone())); + have_org = true; + } + let mut org_units: Vec = Vec::new(); + if let Some(dep) = raw.department.as_ref() { + org_units.push(json!({"@type": "OrgUnit", "name": dep})); + } + if let Some(office) = raw.office_location.as_ref() { + org_units.push(json!({"@type": "OrgUnit", "name": office})); + } + if !org_units.is_empty() { + org_inner.insert("units".to_owned(), Value::Array(org_units)); + have_org = true; + } + if have_org { + organizations.insert("1".to_owned(), Value::Object(org_inner)); + } + let mut next_org_idx = 2u32; + for extra in &raw.companies { + if matches!(raw.company_name.as_deref(), Some(c) if c == extra) { + continue; + } + organizations.insert( + next_org_idx.to_string(), + json!({"@type": "Organization", "name": extra}), + ); + next_org_idx += 1; + } + if !organizations.is_empty() { + card.insert("organizations".to_owned(), Value::Object(organizations)); + } + if let Some(t) = raw.job_title.as_ref() { + card.insert( + "titles".to_owned(), + Value::Object(map_singleton("1", json!({"@type": "Title", "name": t}))), + ); + } + if !raw.emails.is_empty() { + let mut map = Map::new(); + for (i, (key, addr)) in raw.emails.iter().enumerate() { + let entry_id = format!("{}", i + 1); + let mut entry = Map::new(); + entry.insert("@type".to_owned(), Value::String("EmailAddress".to_owned())); + entry.insert("address".to_owned(), Value::String(addr.clone())); + entry.insert("contexts".to_owned(), email_contexts(key)); + map.insert(entry_id, Value::Object(entry)); + } + card.insert("emails".to_owned(), Value::Object(map)); + } + if !raw.phones.is_empty() { + let mut map = Map::new(); + for (i, (key, num)) in raw.phones.iter().enumerate() { + let entry_id = format!("{}", i + 1); + let mut entry = Map::new(); + entry.insert("@type".to_owned(), Value::String("Phone".to_owned())); + entry.insert("number".to_owned(), Value::String(num.clone())); + entry.insert("contexts".to_owned(), phone_contexts(key)); + entry.insert("features".to_owned(), phone_features(key)); + map.insert(entry_id, Value::Object(entry)); + } + card.insert("phones".to_owned(), Value::Object(map)); + } + if !raw.ims.is_empty() { + let mut map = Map::new(); + for (i, (key, addr)) in raw.ims.iter().enumerate() { + let entry_id = format!("{}", i + 1); + let mut entry = Map::new(); + entry.insert( + "@type".to_owned(), + Value::String("OnlineService".to_owned()), + ); + entry.insert("user".to_owned(), Value::String(addr.clone())); + if let Some(service) = im_service_from_key(key) { + entry.insert("service".to_owned(), Value::String(service)); + } + map.insert(entry_id, Value::Object(entry)); + } + card.insert("onlineServices".to_owned(), Value::Object(map)); + } + if !raw.addresses.is_empty() { + let pref_key = raw + .postal_address_index + .as_deref() + .map(str::to_ascii_lowercase); + let mut map = Map::new(); + for (i, addr) in raw.addresses.iter().enumerate() { + let entry_id = format!("{}", i + 1); + let is_pref = matches!(&pref_key, Some(p) if !p.is_empty() && p == &addr.key.to_ascii_lowercase()); + map.insert(entry_id, address_to_jscontact(addr, is_pref)); + } + card.insert("addresses".to_owned(), Value::Object(map)); + } + if let Some(url) = raw.url.as_ref() { + card.insert( + "links".to_owned(), + Value::Object(map_singleton("1", json!({"@type": "Link", "uri": url}))), + ); + } + if let Some(bday) = raw.birthday.as_ref() + && let Some(partial) = partial_date(bday) + { + card.insert( + "anniversaries".to_owned(), + Value::Object(map_singleton( + "birth", + json!({"@type": "Anniversary", "kind": "birth", "date": partial}), + )), + ); + } + if let Some(anniv) = raw.wedding_anniversary.as_ref() + && let Some(partial) = partial_date(anniv) + { + let key = "wedding"; + let entry = json!({"@type": "Anniversary", "kind": "wedding", "date": partial}); + if let Some(map) = card.get_mut("anniversaries").and_then(Value::as_object_mut) { + map.insert(key.to_owned(), entry); + } else { + card.insert( + "anniversaries".to_owned(), + Value::Object(map_singleton(key, entry)), + ); + } + } + if !raw.categories.is_empty() { + let mut keywords = Map::new(); + for cat in &raw.categories { + keywords.insert(cat.to_ascii_lowercase(), Value::Bool(true)); + } + card.insert("keywords".to_owned(), Value::Object(keywords)); + } + if let Some(notes) = raw.notes.as_ref() { + card.insert( + "notes".to_owned(), + Value::Object(map_singleton("1", json!({"@type": "Note", "note": notes}))), + ); + } + let mut related: Map = Map::new(); + if let Some(v) = raw.spouse.as_ref() { + related.insert( + v.clone(), + json!({"@type": "Relation", "relation": {"spouse": true}}), + ); + } + for child in &raw.children { + related.insert( + child.clone(), + json!({"@type": "Relation", "relation": {"child": true}}), + ); + } + if let Some(v) = raw.manager.as_ref() { + related.insert( + v.clone(), + json!({"@type": "Relation", "relation": {"x-manager": true}}), + ); + } + if let Some(v) = raw.assistant.as_ref() { + related.insert( + v.clone(), + json!({"@type": "Relation", "relation": {"x-assistant": true}}), + ); + } + if !related.is_empty() { + card.insert("relatedTo".to_owned(), Value::Object(related)); + } + Value::Object(card) +} + +fn name_components(raw: &ContactItemRaw) -> Vec { + let mut out: Vec = Vec::new(); + if let Some(v) = raw.given_name.as_ref() { + out.push(json!({"@type": "NameComponent", "kind": "given", "value": v})); + } + if let Some(v) = raw.middle_name.as_ref() { + out.push(json!({"@type": "NameComponent", "kind": "given2", "value": v})); + } + if let Some(v) = raw.surname.as_ref() { + out.push(json!({"@type": "NameComponent", "kind": "surname", "value": v})); + } + if let Some(v) = raw.generation.as_ref() { + out.push(json!({"@type": "NameComponent", "kind": "generation", "value": v})); + } + out +} + +fn email_contexts(key: &str) -> Value { + let mut o = Map::new(); + let lower = key.to_ascii_lowercase(); + if lower.contains("business") || lower.contains("work") { + o.insert("work".to_owned(), Value::Bool(true)); + } else if lower.contains("home") { + o.insert("private".to_owned(), Value::Bool(true)); + } + Value::Object(o) +} + +fn phone_contexts(key: &str) -> Value { + let mut o = Map::new(); + let lower = key.to_ascii_lowercase(); + if lower.starts_with("business") || lower.starts_with("company") { + o.insert("work".to_owned(), Value::Bool(true)); + } else if lower.starts_with("home") || lower.starts_with("mobile") { + o.insert("private".to_owned(), Value::Bool(true)); + } + Value::Object(o) +} + +fn phone_features(key: &str) -> Value { + let mut o = Map::new(); + let lower = key.to_ascii_lowercase(); + if lower.contains("mobile") { + o.insert("mobile".to_owned(), Value::Bool(true)); + } else if lower.contains("fax") { + o.insert("fax".to_owned(), Value::Bool(true)); + } else if lower.contains("pager") { + o.insert("pager".to_owned(), Value::Bool(true)); + } else { + o.insert("voice".to_owned(), Value::Bool(true)); + } + Value::Object(o) +} + +fn address_to_jscontact(addr: &RawContactAddress, is_pref: bool) -> Value { + let mut o = Map::new(); + o.insert("@type".to_owned(), Value::String("Address".to_owned())); + let mut components: Vec = Vec::new(); + if let Some(s) = addr.street.as_ref() { + components.push(json!({"@type": "AddressComponent", "kind": "name", "value": s})); + } + if let Some(s) = addr.city.as_ref() { + components.push(json!({"@type": "AddressComponent", "kind": "locality", "value": s})); + } + if let Some(s) = addr.state.as_ref() { + components.push(json!({"@type": "AddressComponent", "kind": "region", "value": s})); + } + if let Some(s) = addr.postal_code.as_ref() { + components.push(json!({"@type": "AddressComponent", "kind": "postcode", "value": s})); + } + if let Some(s) = addr.country.as_ref() { + components.push(json!({"@type": "AddressComponent", "kind": "country", "value": s})); + } + if !components.is_empty() { + o.insert("components".to_owned(), Value::Array(components)); + } + let mut contexts = Map::new(); + let lower = addr.key.to_ascii_lowercase(); + if lower == "business" || lower == "work" { + contexts.insert("work".to_owned(), Value::Bool(true)); + } else if lower == "home" { + contexts.insert("private".to_owned(), Value::Bool(true)); + } + if !contexts.is_empty() { + o.insert("contexts".to_owned(), Value::Object(contexts)); + } + if is_pref { + o.insert("pref".to_owned(), Value::from(1u32)); + } + Value::Object(o) +} + +fn im_service_from_key(key: &str) -> Option { + let trimmed: String = key.chars().filter(|c| !c.is_ascii_digit()).collect(); + let canonical = trimmed.trim_start_matches("ImAddress"); + if canonical.is_empty() { + return None; + } + Some(canonical.to_owned()) +} + +fn partial_date(s: &str) -> Option { + let head = s.split('T').next().unwrap_or(s); + let mut parts = head.splitn(3, '-'); + let year: u32 = parts.next()?.parse().ok()?; + let month: u32 = parts.next()?.parse().ok()?; + let day: u32 = parts.next()?.parse().ok()?; + let mut o = Map::new(); + o.insert("@type".to_owned(), Value::String("PartialDate".to_owned())); + if year > 0 { + o.insert("year".to_owned(), Value::from(year)); + } + o.insert("month".to_owned(), Value::from(month)); + o.insert("day".to_owned(), Value::from(day)); + Some(Value::Object(o)) +} + +fn map_singleton(key: &str, value: Value) -> Map { + let mut m = Map::new(); + m.insert(key.to_owned(), value); + m +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn basic_contact_round_trip() { + let raw = ContactItemRaw { + display_name: Some("Alice Doe".to_owned()), + given_name: Some("Alice".to_owned()), + surname: Some("Doe".to_owned()), + emails: vec![("EmailAddress1".to_owned(), "alice@x".to_owned())], + phones: vec![("BusinessPhone1".to_owned(), "+1".to_owned())], + ..ContactItemRaw::default() + }; + let v = to_jscontact(&raw); + assert_eq!(v["@type"], "Card"); + assert_eq!(v["name"]["full"], "Alice Doe"); + let comps = v["name"]["components"].as_array().unwrap(); + assert_eq!(comps[0]["kind"], "given"); + assert_eq!(comps[0]["value"], "Alice"); + assert_eq!(v["emails"]["1"]["address"], "alice@x"); + assert_eq!(v["phones"]["1"]["contexts"]["work"], true); + } + + #[test] + fn synthetic_uid_is_stable() { + let u1 = synthetic_uid("ITEM-1"); + let u2 = synthetic_uid("ITEM-1"); + assert_eq!(u1, u2); + assert_ne!(u1, synthetic_uid("ITEM-2")); + assert!(u1.starts_with("vandelay-ews-syn-")); + } + + #[test] + fn anniversaries_combine_birth_and_wedding() { + let raw = ContactItemRaw { + display_name: Some("X".to_owned()), + birthday: Some("1990-04-15T00:00:00Z".to_owned()), + wedding_anniversary: Some("2015-08-22T00:00:00".to_owned()), + ..ContactItemRaw::default() + }; + let v = to_jscontact(&raw); + assert_eq!(v["anniversaries"]["birth"]["kind"], "birth"); + assert_eq!(v["anniversaries"]["birth"]["date"]["month"], 4); + assert_eq!(v["anniversaries"]["wedding"]["kind"], "wedding"); + } + + #[test] + fn office_location_is_added_as_org_unit() { + let raw = ContactItemRaw { + display_name: Some("X".to_owned()), + company_name: Some("Initech".to_owned()), + department: Some("R&D".to_owned()), + office_location: Some("Bldg 4".to_owned()), + ..ContactItemRaw::default() + }; + let v = to_jscontact(&raw); + let units = v["organizations"]["1"]["units"].as_array().unwrap(); + assert_eq!(units[0]["name"], "R&D"); + assert_eq!(units[1]["name"], "Bldg 4"); + } + + #[test] + fn extra_companies_become_additional_organizations() { + let raw = ContactItemRaw { + display_name: Some("X".to_owned()), + company_name: Some("Initech".to_owned()), + companies: vec!["Initech".to_owned(), "Initrode".to_owned()], + ..ContactItemRaw::default() + }; + let v = to_jscontact(&raw); + assert_eq!(v["organizations"]["1"]["name"], "Initech"); + assert_eq!(v["organizations"]["2"]["name"], "Initrode"); + } + + #[test] + fn postal_address_index_sets_pref_on_matching_entry() { + let raw = ContactItemRaw { + display_name: Some("X".to_owned()), + addresses: vec![ + RawContactAddress { + key: "Home".to_owned(), + street: Some("1 St".to_owned()), + ..Default::default() + }, + RawContactAddress { + key: "Business".to_owned(), + street: Some("2 Av".to_owned()), + ..Default::default() + }, + ], + postal_address_index: Some("Business".to_owned()), + ..ContactItemRaw::default() + }; + let v = to_jscontact(&raw); + assert!(v["addresses"]["1"].get("pref").is_none()); + assert_eq!(v["addresses"]["2"]["pref"], 1); + } + + #[test] + fn im_service_extracted_from_key() { + let raw = ContactItemRaw { + display_name: Some("X".to_owned()), + ims: vec![("ImAddress1".to_owned(), "alice@msn".to_owned())], + ..ContactItemRaw::default() + }; + let v = to_jscontact(&raw); + assert_eq!(v["onlineServices"]["1"]["user"], "alice@msn"); + assert!(v["onlineServices"]["1"].get("service").is_none()); + } + + #[test] + fn relations_use_x_prefix_for_manager_and_assistant() { + let raw = ContactItemRaw { + display_name: Some("X".to_owned()), + manager: Some("Boss".to_owned()), + assistant: Some("Asst".to_owned()), + ..ContactItemRaw::default() + }; + let v = to_jscontact(&raw); + assert_eq!(v["relatedTo"]["Boss"]["relation"]["x-manager"], true); + assert_eq!(v["relatedTo"]["Asst"]["relation"]["x-assistant"], true); + } +} diff --git a/src/exchange_ews/error.rs b/src/exchange_ews/error.rs new file mode 100644 index 0000000..334617e --- /dev/null +++ b/src/exchange_ews/error.rs @@ -0,0 +1,86 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::error::Error; +use crate::exchange_ews::types::ResponseCode; +use crate::jmap::error::JmapError; + +#[derive(Debug, thiserror::Error)] +pub enum EwsError { + #[error("transport: {0}")] + Transport(String), + + #[error("connection: {0}")] + Connect(String), + + #[error("authentication rejected: {0}")] + Auth(String), + + #[error("http {status}: {body}")] + HttpStatus { status: u16, body: String }, + + #[error("retries exhausted: {0}")] + RetriesExhausted(String), + + #[error("soap fault {code}: {reason}")] + SoapFault { code: ResponseCode, reason: String }, + + #[error("malformed EWS response: {0}")] + Malformed(String), + + #[error("EWS schema unsupported: {0}")] + Unsupported(String), + + #[error("autodiscover failed: {0}")] + AutodiscoverFailed(String), + + #[error("autodiscover redirect loop")] + AutodiscoverLoop, + + #[error("oauth: {0}")] + OAuth(String), + + #[error("xml: {0}")] + Xml(String), +} + +impl From for EwsError { + fn from(value: quick_xml::Error) -> Self { + EwsError::Xml(value.to_string()) + } +} + +impl From for EwsError { + fn from(value: JmapError) -> Self { + match value { + JmapError::Transport(m) => EwsError::Transport(m), + JmapError::Connect(m) => EwsError::Connect(m), + JmapError::Auth(m) => EwsError::Auth(m), + JmapError::HttpStatus { status, body } => EwsError::HttpStatus { status, body }, + JmapError::RetriesExhausted(m) => EwsError::RetriesExhausted(m), + JmapError::Malformed(m) => EwsError::Malformed(m), + other => EwsError::Connect(other.to_string()), + } + } +} + +impl From for Error { + fn from(value: EwsError) -> Self { + match value { + EwsError::Auth(m) => Error::Connection(format!("authentication rejected: {m}")), + EwsError::Connect(m) | EwsError::Transport(m) => Error::Connection(m), + EwsError::AutodiscoverFailed(m) => { + Error::Connection(format!("autodiscover failed: {m}")) + } + EwsError::AutodiscoverLoop => { + Error::Connection("autodiscover redirect loop".to_owned()) + } + EwsError::OAuth(m) => Error::Connection(format!("oauth: {m}")), + EwsError::Unsupported(m) => Error::Usage(m), + other => Error::Connection(other.to_string()), + } + } +} diff --git a/src/exchange_ews/mod.rs b/src/exchange_ews/mod.rs new file mode 100644 index 0000000..df972b6 --- /dev/null +++ b/src/exchange_ews/mod.rs @@ -0,0 +1,26 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod autodiscover; +pub mod calendar_map; +pub mod client; +pub mod contact_map; +pub mod error; +pub mod oauth; +pub mod parse; +pub mod recurrence; +pub mod retry; +pub mod soap; +pub mod types; +pub mod xml; + +pub use crate::exchange::tz; + +pub use client::EwsClient; +pub use error::EwsError; +pub use types::{ + DistinguishedFolderId, FolderId, ItemId, MailboxKind, ResponseCode, ServerVersion, +}; diff --git a/src/exchange_ews/oauth.rs b/src/exchange_ews/oauth.rs new file mode 100644 index 0000000..2fb2795 --- /dev/null +++ b/src/exchange_ews/oauth.rs @@ -0,0 +1,373 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::{self, Write}; +use std::time::Duration; + +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use serde_json::Value; +use ureq::config::Config; +use ureq::tls::{RootCerts, TlsConfig}; + +use crate::exchange_ews::error::EwsError; + +pub const SCOPE_APP_ONLY: &str = "https://outlook.office365.com/.default"; +pub const SCOPE_DELEGATED: &str = + "https://outlook.office365.com/EWS.AccessAsUser.All offline_access"; + +#[derive(Debug, Clone)] +pub struct AcquiredToken { + pub access_token: String, + pub refresh_token: Option, + pub expires_in: Option, + pub tenant_id: Option, + pub upn: Option, + pub name: Option, +} + +#[derive(Debug, Clone)] +pub enum OAuthFlow { + PreAcquired { + token: String, + }, + ClientCredentials { + tenant: String, + client_id: String, + client_secret: String, + }, + DeviceCode { + tenant: String, + client_id: String, + }, +} + +pub fn acquire(flow: &OAuthFlow, allow_invalid_certs: bool) -> Result { + match flow { + OAuthFlow::PreAcquired { token } => { + let claims = decode_jwt_claims(token).unwrap_or_default(); + Ok(AcquiredToken { + access_token: token.clone(), + refresh_token: None, + expires_in: None, + tenant_id: claims.tenant_id, + upn: claims.upn, + name: claims.name, + }) + } + OAuthFlow::ClientCredentials { + tenant, + client_id, + client_secret, + } => client_credentials(tenant, client_id, client_secret, allow_invalid_certs), + OAuthFlow::DeviceCode { tenant, client_id } => { + device_code_flow(tenant, client_id, allow_invalid_certs) + } + } +} + +#[derive(Debug, Clone, Default)] +pub struct JwtClaims { + pub tenant_id: Option, + pub upn: Option, + pub name: Option, + pub exp: Option, +} + +pub fn decode_jwt_claims(token: &str) -> Option { + let mut parts = token.split('.'); + let _header = parts.next()?; + let payload = parts.next()?; + let bytes = URL_SAFE_NO_PAD + .decode(payload) + .or_else(|_| base64::engine::general_purpose::STANDARD_NO_PAD.decode(payload)) + .or_else(|_| base64::engine::general_purpose::STANDARD.decode(payload)) + .ok()?; + let value: Value = serde_json::from_slice(&bytes).ok()?; + Some(JwtClaims { + tenant_id: value.get("tid").and_then(Value::as_str).map(str::to_owned), + upn: value + .get("upn") + .and_then(Value::as_str) + .or_else(|| value.get("preferred_username").and_then(Value::as_str)) + .map(str::to_owned), + name: value.get("name").and_then(Value::as_str).map(str::to_owned), + exp: value.get("exp").and_then(Value::as_u64), + }) +} + +fn token_endpoint(tenant: &str) -> String { + format!("https://login.microsoftonline.com/{tenant}/oauth2/v2.0/token") +} + +fn device_code_endpoint(tenant: &str) -> String { + format!("https://login.microsoftonline.com/{tenant}/oauth2/v2.0/devicecode") +} + +fn build_agent(allow_invalid_certs: bool) -> ureq::Agent { + let config: Config = Config::builder() + .http_status_as_error(false) + .tls_config( + TlsConfig::builder() + .root_certs(RootCerts::PlatformVerifier) + .disable_verification(allow_invalid_certs) + .build(), + ) + .build(); + config.new_agent() +} + +fn client_credentials( + tenant: &str, + client_id: &str, + client_secret: &str, + allow_invalid_certs: bool, +) -> Result { + let agent = build_agent(allow_invalid_certs); + let body = form_encode(&[ + ("client_id", client_id), + ("client_secret", client_secret), + ("scope", SCOPE_APP_ONLY), + ("grant_type", "client_credentials"), + ]); + let endpoint = token_endpoint(tenant); + let resp = agent + .post(&endpoint) + .header("Content-Type", "application/x-www-form-urlencoded") + .send(body.as_bytes()) + .map_err(|e| EwsError::OAuth(format!("token endpoint: {e}")))?; + parse_token_response(resp) +} + +fn device_code_flow( + tenant: &str, + client_id: &str, + allow_invalid_certs: bool, +) -> Result { + let agent = build_agent(allow_invalid_certs); + let body = form_encode(&[("client_id", client_id), ("scope", SCOPE_DELEGATED)]); + let endpoint = device_code_endpoint(tenant); + let mut resp = agent + .post(&endpoint) + .header("Content-Type", "application/x-www-form-urlencoded") + .send(body.as_bytes()) + .map_err(|e| EwsError::OAuth(format!("device code endpoint: {e}")))?; + let json: Value = resp + .body_mut() + .read_json() + .map_err(|e| EwsError::OAuth(format!("device code body: {e}")))?; + let device_code = json + .get("device_code") + .and_then(Value::as_str) + .ok_or_else(|| EwsError::OAuth("device_code missing".to_owned()))? + .to_owned(); + let user_code = json.get("user_code").and_then(Value::as_str).unwrap_or("?"); + let verify_uri = json + .get("verification_uri") + .and_then(Value::as_str) + .unwrap_or("https://microsoft.com/devicelogin"); + let interval = json.get("interval").and_then(Value::as_u64).unwrap_or(5); + let message = json.get("message").and_then(Value::as_str); + if let Some(m) = message { + eprintln!("{m}"); + } else { + eprintln!("To sign in, open {verify_uri} and enter the code {user_code}"); + } + io::stdout() + .flush() + .map_err(|e| EwsError::OAuth(format!("flushing stdout: {e}")))?; + let endpoint = token_endpoint(tenant); + let mut delay = Duration::from_secs(interval); + let deadline = std::time::Instant::now() + + Duration::from_secs( + json.get("expires_in") + .and_then(Value::as_u64) + .unwrap_or(900), + ); + loop { + std::thread::sleep(delay); + if std::time::Instant::now() >= deadline { + return Err(EwsError::OAuth( + "device code expired before the user completed sign-in".to_owned(), + )); + } + let body = form_encode(&[ + ("client_id", client_id), + ("grant_type", "urn:ietf:params:oauth:grant-type:device_code"), + ("device_code", &device_code), + ]); + let mut resp = agent + .post(&endpoint) + .header("Content-Type", "application/x-www-form-urlencoded") + .send(body.as_bytes()) + .map_err(|e| EwsError::OAuth(format!("device code polling: {e}")))?; + let status = resp.status().as_u16(); + if (200..300).contains(&status) { + return parse_token_response(resp); + } + let payload: Value = resp.body_mut().read_json().unwrap_or(Value::Null); + let error = payload.get("error").and_then(Value::as_str).unwrap_or(""); + match error { + "authorization_pending" => continue, + "slow_down" => { + delay = delay.saturating_add(Duration::from_secs(5)); + continue; + } + "expired_token" => { + return Err(EwsError::OAuth( + "device code expired; restart the import".to_owned(), + )); + } + "" => { + return Err(EwsError::OAuth(format!( + "device code polling: http {status}" + ))); + } + other => { + let desc = payload + .get("error_description") + .and_then(Value::as_str) + .unwrap_or(""); + return Err(EwsError::OAuth(format!( + "device code polling: {other}: {desc}" + ))); + } + } + } +} + +fn parse_token_response( + mut resp: ureq::http::Response, +) -> Result { + let status = resp.status().as_u16(); + let json: Value = resp + .body_mut() + .read_json() + .map_err(|e| EwsError::OAuth(format!("token body: {e}")))?; + if !(200..300).contains(&status) { + let desc = json + .get("error_description") + .and_then(Value::as_str) + .unwrap_or("(no error_description)"); + return Err(EwsError::OAuth(format!("http {status}: {desc}"))); + } + let access = json + .get("access_token") + .and_then(Value::as_str) + .ok_or_else(|| EwsError::OAuth("access_token missing".to_owned()))? + .to_owned(); + let refresh_token = json + .get("refresh_token") + .and_then(Value::as_str) + .map(str::to_owned); + let expires_in = json.get("expires_in").and_then(Value::as_u64); + let claims = decode_jwt_claims(&access).unwrap_or_default(); + Ok(AcquiredToken { + access_token: access, + refresh_token, + expires_in, + tenant_id: claims.tenant_id, + upn: claims.upn, + name: claims.name, + }) +} + +pub fn refresh_with_token( + tenant: &str, + client_id: &str, + refresh_token: &str, + allow_invalid_certs: bool, +) -> Result { + let agent = build_agent(allow_invalid_certs); + let body = form_encode(&[ + ("client_id", client_id), + ("grant_type", "refresh_token"), + ("refresh_token", refresh_token), + ("scope", SCOPE_DELEGATED), + ]); + let endpoint = token_endpoint(tenant); + let resp = agent + .post(&endpoint) + .header("Content-Type", "application/x-www-form-urlencoded") + .send(body.as_bytes()) + .map_err(|e| EwsError::OAuth(format!("refresh token endpoint: {e}")))?; + parse_token_response(resp) +} + +fn form_encode(pairs: &[(&str, &str)]) -> String { + let mut out = String::new(); + for (i, (k, v)) in pairs.iter().enumerate() { + if i > 0 { + out.push('&'); + } + out.push_str(&urlencode(k)); + out.push('='); + out.push_str(&urlencode(v)); + } + out +} + +fn urlencode(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for b in s.bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(b as char); + } + _ => out.push_str(&format!("%{b:02X}")), + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use base64::engine::general_purpose::URL_SAFE_NO_PAD; + + fn make_jwt(tid: &str, upn: &str, exp: u64) -> String { + let header = URL_SAFE_NO_PAD.encode(b"{\"alg\":\"none\"}"); + let claims = format!(r#"{{"tid":"{tid}","upn":"{upn}","exp":{exp}}}"#); + let payload = URL_SAFE_NO_PAD.encode(claims.as_bytes()); + format!("{header}.{payload}.") + } + + #[test] + fn jwt_decoder_extracts_tid_and_upn() { + let token = make_jwt("tenant-1", "alice@x.com", 9999999999); + let claims = decode_jwt_claims(&token).unwrap(); + assert_eq!(claims.tenant_id.as_deref(), Some("tenant-1")); + assert_eq!(claims.upn.as_deref(), Some("alice@x.com")); + assert_eq!(claims.exp, Some(9999999999)); + } + + #[test] + fn malformed_token_returns_none() { + assert!(decode_jwt_claims("garbage").is_none()); + assert!(decode_jwt_claims("only.two").is_none()); + } + + #[test] + fn pre_acquired_flow_decodes_claims() { + let token = make_jwt("t-2", "bob@x", 9999999999); + let acq = acquire( + &OAuthFlow::PreAcquired { + token: token.clone(), + }, + false, + ) + .unwrap(); + assert_eq!(acq.access_token, token); + assert_eq!(acq.tenant_id.as_deref(), Some("t-2")); + assert_eq!(acq.upn.as_deref(), Some("bob@x")); + } + + #[test] + fn urlencode_handles_special_chars() { + assert_eq!(urlencode("a b/c"), "a%20b%2Fc"); + assert_eq!(urlencode("a.b-c_d"), "a.b-c_d"); + } +} diff --git a/src/exchange_ews/parse.rs b/src/exchange_ews/parse.rs new file mode 100644 index 0000000..0f60bec --- /dev/null +++ b/src/exchange_ews/parse.rs @@ -0,0 +1,2332 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::BufRead; + +use quick_xml::NsReader; +use quick_xml::XmlVersion; +use quick_xml::events::{BytesStart, Event}; +use quick_xml::name::ResolveResult; + +use crate::exchange_ews::error::EwsError; +use crate::exchange_ews::soap::{NS_MESSAGES, NS_TYPES}; +use crate::exchange_ews::types::{CalendarItemType, FolderId, ItemId, ResponseCode, ServerVersion}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Ns { + Messages, + Types, + Other, +} + +fn classify(ns: &ResolveResult<'_>) -> Ns { + match ns { + ResolveResult::Bound(prefix) => { + let p = prefix.as_ref(); + if p == NS_MESSAGES.as_bytes() { + Ns::Messages + } else if p == NS_TYPES.as_bytes() { + Ns::Types + } else { + Ns::Other + } + } + _ => Ns::Other, + } +} + +fn is(ns: Ns, local: &[u8], expected_ns: Ns, target: &[u8]) -> bool { + ns == expected_ns && local.eq_ignore_ascii_case(target) +} + +fn attr_value(e: &BytesStart<'_>, name: &[u8]) -> Option { + for a in e.attributes().flatten() { + let key = a.key.local_name(); + if key.as_ref().eq_ignore_ascii_case(name) { + return Some( + a.normalized_value(XmlVersion::Implicit1_0) + .map(|c| c.into_owned()) + .unwrap_or_else(|_| String::from_utf8_lossy(a.value.as_ref()).into_owned()), + ); + } + } + None +} + +fn capture_id_attrs(e: &BytesStart<'_>, id_out: &mut String, ck_out: &mut String) { + for a in e.attributes().flatten() { + let key = a.key.local_name(); + let kb = key.as_ref(); + let v = a + .normalized_value(XmlVersion::Implicit1_0) + .map(|c| c.into_owned()) + .unwrap_or_else(|_| String::from_utf8_lossy(a.value.as_ref()).into_owned()); + if kb.eq_ignore_ascii_case(b"Id") { + *id_out = v; + } else if kb.eq_ignore_ascii_case(b"ChangeKey") { + *ck_out = v; + } + } +} + +#[derive(Debug, Clone, Default)] +pub struct ServerVersionInfo { + pub major_version: Option, + pub minor_version: Option, + pub major_build: Option, + pub minor_build: Option, + pub version: Option, +} + +impl ServerVersionInfo { + pub fn to_server_version(&self) -> Option { + match (self.major_version, self.minor_version) { + (Some(maj), Some(min)) => Some(ServerVersion::from_build(maj, min)), + _ => None, + } + } +} + +fn read_version_attrs(e: &BytesStart<'_>, out: &mut ServerVersionInfo) { + for a in e.attributes().flatten() { + let key = a.key.local_name(); + let kb = key.as_ref(); + let v = String::from_utf8_lossy(a.value.as_ref()).into_owned(); + if kb.eq_ignore_ascii_case(b"MajorVersion") { + out.major_version = v.parse().ok(); + } else if kb.eq_ignore_ascii_case(b"MinorVersion") { + out.minor_version = v.parse().ok(); + } else if kb.eq_ignore_ascii_case(b"MajorBuildNumber") { + out.major_build = v.parse().ok(); + } else if kb.eq_ignore_ascii_case(b"MinorBuildNumber") { + out.minor_build = v.parse().ok(); + } else if kb.eq_ignore_ascii_case(b"Version") { + out.version = Some(v); + } + } +} + +#[derive(Debug, Clone)] +pub struct SoapFault { + pub fault_code: String, + pub fault_string: String, + pub response_code: ResponseCode, + pub back_off_ms: Option, +} + +#[derive(Debug, Clone)] +pub enum EnvelopeKind { + Fault { + version: ServerVersionInfo, + fault: SoapFault, + }, + Body { + version: ServerVersionInfo, + }, +} + +pub fn read_envelope_summary(bytes: &[u8]) -> Result { + let mut xml = NsReader::from_reader(bytes); + xml.config_mut().trim_text(true); + let mut buf = Vec::new(); + let mut version = ServerVersionInfo::default(); + let mut seen_body = false; + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + let ns_kind = classify(&ns); + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + let local = e.local_name().as_ref().to_vec(); + if is(ns_kind, &local, Ns::Types, b"ServerVersionInfo") { + read_version_attrs(e, &mut version); + } else if local.eq_ignore_ascii_case(b"Fault") { + let fault = parse_fault(&mut xml)?; + return Ok(EnvelopeKind::Fault { version, fault }); + } else if local.eq_ignore_ascii_case(b"Body") { + seen_body = true; + } + } + Event::Eof => break, + _ => {} + } + } + if seen_body { + Ok(EnvelopeKind::Body { version }) + } else { + Err(EwsError::Malformed("envelope had no soap:Body".to_owned())) + } +} + +fn parse_fault(xml: &mut NsReader) -> Result { + let mut buf = Vec::new(); + let mut fault_code = String::new(); + let mut fault_string = String::new(); + let mut code = ResponseCode::Other(String::new()); + let mut back_off: Option = None; + let mut text_target: Option<&'static str> = None; + let mut last_value_name: Option = None; + let mut depth: u32 = 1; + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + let ns_kind = classify(&ns); + match ev { + Event::Start(e) => { + depth += 1; + let local = e.local_name().as_ref().to_vec(); + if local.eq_ignore_ascii_case(b"faultcode") { + text_target = Some("faultcode"); + } else if local.eq_ignore_ascii_case(b"faultstring") { + text_target = Some("faultstring"); + } else if local.eq_ignore_ascii_case(b"ResponseCode") { + text_target = Some("responseCode"); + } else if local.eq_ignore_ascii_case(b"Value") && ns_kind == Ns::Types { + last_value_name = attr_value(&e, b"Name"); + text_target = Some("messageXmlValue"); + } else { + text_target = None; + } + } + Event::Empty(e) => { + let local = e.local_name().as_ref().to_vec(); + if is(ns_kind, &local, Ns::Types, b"Value") { + last_value_name = attr_value(&e, b"Name"); + } + } + Event::End(_) => { + if depth == 0 { + break; + } + depth -= 1; + text_target = None; + if depth == 0 { + break; + } + } + Event::Text(t) => { + let text = t.decode().map(|c| c.into_owned()).unwrap_or_default(); + match text_target { + Some("faultcode") => fault_code = text, + Some("faultstring") => fault_string = text, + Some("responseCode") => code = ResponseCode::parse(&text), + Some("messageXmlValue") + if last_value_name.as_deref() == Some("BackOffMilliseconds") => + { + back_off = text.trim().parse().ok(); + } + _ => {} + } + } + Event::CData(c) if text_target == Some("faultstring") => { + fault_string = String::from_utf8_lossy(c.as_ref()).into_owned(); + } + Event::Eof => break, + _ => {} + } + } + if matches!(code, ResponseCode::ServerBusy { .. }) { + code = ResponseCode::ServerBusy { + back_off_ms: back_off, + }; + } + Ok(SoapFault { + fault_code, + fault_string, + response_code: code, + back_off_ms: back_off, + }) +} + +#[derive(Debug, Clone, Default)] +pub struct FindFolderResponse { + pub folders: Vec, + pub more: bool, + pub total_in_view: Option, +} + +#[derive(Debug, Clone)] +pub struct FolderEntry { + pub element: FolderElement, + pub folder_id: FolderId, + pub parent_id: Option, + pub display_name: String, + pub folder_class: String, + pub total_count: Option, + pub child_count: Option, +} + +impl Default for FolderEntry { + fn default() -> Self { + FolderEntry { + element: FolderElement::Folder, + folder_id: FolderId { + id: String::new(), + change_key: String::new(), + }, + parent_id: None, + display_name: String::new(), + folder_class: String::new(), + total_count: None, + child_count: None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FolderElement { + Folder, + CalendarFolder, + ContactsFolder, + TasksFolder, + SearchFolder, +} + +impl FolderElement { + fn from_local(local: &[u8]) -> Option { + if local.eq_ignore_ascii_case(b"Folder") { + Some(FolderElement::Folder) + } else if local.eq_ignore_ascii_case(b"CalendarFolder") { + Some(FolderElement::CalendarFolder) + } else if local.eq_ignore_ascii_case(b"ContactsFolder") { + Some(FolderElement::ContactsFolder) + } else if local.eq_ignore_ascii_case(b"TasksFolder") { + Some(FolderElement::TasksFolder) + } else if local.eq_ignore_ascii_case(b"SearchFolder") { + Some(FolderElement::SearchFolder) + } else { + None + } + } +} + +pub fn parse_find_folder_response(body: &[u8]) -> Result { + let mut xml = NsReader::from_reader(body); + xml.config_mut().trim_text(true); + let mut buf = Vec::new(); + let mut out = FindFolderResponse::default(); + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + let ns_kind = classify(&ns); + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + let local = e.local_name().as_ref().to_vec(); + if is(ns_kind, &local, Ns::Messages, b"RootFolder") { + if let Some(v) = attr_value(e, b"TotalItemsInView") { + out.total_in_view = v.trim().parse().ok(); + } + if let Some(v) = attr_value(e, b"IncludesLastItemInRange") { + out.more = !matches!(v.trim(), "true" | "1"); + } + } else if ns_kind == Ns::Types + && let Some(elem) = FolderElement::from_local(&local) + { + let entry = parse_folder_element(&mut xml, elem)?; + out.folders.push(entry); + } + } + Event::Eof => break, + _ => {} + } + } + Ok(out) +} + +pub fn parse_folder_inner(inner_xml: &str) -> Result, EwsError> { + let mut xml = NsReader::from_reader(inner_xml.as_bytes()); + xml.config_mut().trim_text(true); + let mut buf = Vec::new(); + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + let ns_kind = classify(&ns); + match ev { + Event::Start(_) | Event::Empty(_) => { + let local = match &ev { + Event::Start(e) | Event::Empty(e) => e.local_name().as_ref().to_vec(), + _ => continue, + }; + if ns_kind == Ns::Types + && let Some(elem) = FolderElement::from_local(&local) + { + return Ok(Some(parse_folder_element(&mut xml, elem)?)); + } + } + Event::Eof => return Ok(None), + _ => {} + } + } +} + +pub fn parse_get_folder_response(body: &[u8]) -> Result, EwsError> { + let mut xml = NsReader::from_reader(body); + xml.config_mut().trim_text(true); + let mut buf = Vec::new(); + let mut out: Vec = Vec::new(); + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + let ns_kind = classify(&ns); + match ev { + Event::Start(_) | Event::Empty(_) => { + let local = match &ev { + Event::Start(e) | Event::Empty(e) => e.local_name().as_ref().to_vec(), + _ => continue, + }; + if ns_kind == Ns::Types + && let Some(elem) = FolderElement::from_local(&local) + { + let entry = parse_folder_element(&mut xml, elem)?; + out.push(entry); + } + } + Event::Eof => break, + _ => {} + } + } + Ok(out) +} + +fn parse_folder_element( + xml: &mut NsReader, + element: FolderElement, +) -> Result { + let mut entry = FolderEntry { + element, + ..FolderEntry::default() + }; + let mut buf = Vec::new(); + let mut depth: u32 = 1; + let mut current: Option<&'static str> = None; + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + let ns_kind = classify(&ns); + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + let is_empty = matches!(ev, Event::Empty(_)); + if !is_empty { + depth += 1; + } + let local = e.local_name().as_ref().to_vec(); + if ns_kind == Ns::Types { + if local.eq_ignore_ascii_case(b"FolderId") { + capture_id_attrs( + e, + &mut entry.folder_id.id, + &mut entry.folder_id.change_key, + ); + } else if local.eq_ignore_ascii_case(b"ParentFolderId") { + let mut pid = String::new(); + let mut ck = String::new(); + capture_id_attrs(e, &mut pid, &mut ck); + if !pid.is_empty() { + entry.parent_id = Some(pid); + } + } else if local.eq_ignore_ascii_case(b"DisplayName") { + current = Some("displayName"); + } else if local.eq_ignore_ascii_case(b"FolderClass") { + current = Some("folderClass"); + } else if local.eq_ignore_ascii_case(b"TotalCount") { + current = Some("totalCount"); + } else if local.eq_ignore_ascii_case(b"ChildFolderCount") { + current = Some("childCount"); + } else { + current = None; + } + } + if is_empty { + current = None; + } + } + Event::End(_) => { + if depth == 0 { + break; + } + depth -= 1; + current = None; + if depth == 0 { + break; + } + } + Event::Text(t) => { + let text = t.decode().map(|c| c.into_owned()).unwrap_or_default(); + match current { + Some("displayName") => entry.display_name = text, + Some("folderClass") => entry.folder_class = text, + Some("totalCount") => entry.total_count = text.trim().parse().ok(), + Some("childCount") => entry.child_count = text.trim().parse().ok(), + _ => {} + } + } + Event::Eof => break, + _ => {} + } + } + Ok(entry) +} + +#[derive(Debug, Clone, Default)] +pub struct FindItemResponse { + pub items: Vec, + pub total_in_view: Option, + pub more: bool, +} + +#[derive(Debug, Clone)] +pub struct ItemEntry { + pub element: String, + pub id: ItemId, +} + +pub fn parse_find_item_response(body: &[u8]) -> Result { + let mut xml = NsReader::from_reader(body); + xml.config_mut().trim_text(true); + let mut buf = Vec::new(); + let mut out = FindItemResponse::default(); + let mut in_root = false; + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + let ns_kind = classify(&ns); + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + let local = e.local_name().as_ref().to_vec(); + if is(ns_kind, &local, Ns::Messages, b"RootFolder") { + in_root = true; + if let Some(v) = attr_value(e, b"TotalItemsInView") { + out.total_in_view = v.trim().parse().ok(); + } + if let Some(v) = attr_value(e, b"IncludesLastItemInRange") { + out.more = !matches!(v.trim(), "true" | "1"); + } + } else if in_root && ns_kind == Ns::Types { + if is_item_element(&local) { + out.items.push(ItemEntry { + element: String::from_utf8_lossy(&local).into_owned(), + id: ItemId::default(), + }); + } else if local.eq_ignore_ascii_case(b"ItemId") + && let Some(last) = out.items.last_mut() + { + capture_id_attrs(e, &mut last.id.id, &mut last.id.change_key); + } + } + } + Event::End(e) => { + let local = e.local_name().as_ref().to_vec(); + if local.eq_ignore_ascii_case(b"RootFolder") { + in_root = false; + } + } + Event::Eof => break, + _ => {} + } + } + Ok(out) +} + +fn is_item_element(local: &[u8]) -> bool { + matches!( + local.to_ascii_lowercase().as_slice(), + b"message" + | b"calendaritem" + | b"contact" + | b"meetingrequest" + | b"meetingresponse" + | b"meetingmessage" + | b"meetingcancellation" + | b"item" + ) +} + +#[derive(Debug, Clone)] +pub struct ResponseMessage { + pub success: bool, + pub response_code: ResponseCode, + pub message_text: String, + pub inner_xml: String, +} + +pub fn parse_response_messages( + body: &[u8], + response_message_local: &[u8], +) -> Result, EwsError> { + let mut xml = NsReader::from_reader(body); + xml.config_mut().trim_text(false); + let mut buf = Vec::new(); + let mut out: Vec = Vec::new(); + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + let ns_kind = classify(&ns); + match ev { + Event::Start(e) => { + let local = e.local_name().as_ref().to_vec(); + if is(ns_kind, &local, Ns::Messages, response_message_local) { + let response_class = + attr_value(&e, b"ResponseClass").unwrap_or_else(|| "Success".to_owned()); + let msg = parse_one_response_message(&mut xml, response_class)?; + out.push(msg); + } + } + Event::Eof => break, + _ => {} + } + } + Ok(out) +} + +fn parse_one_response_message( + xml: &mut NsReader, + response_class: String, +) -> Result { + let mut buf = Vec::new(); + let mut response_code = ResponseCode::NoError; + let mut message_text = String::new(); + let mut inner = String::new(); + let mut depth: u32 = 1; + let mut current: Option<&'static str> = None; + let mut capture_depth: u32 = 0; + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + let ns_kind = classify(&ns); + let in_capture = capture_depth > 0; + match ev { + Event::Start(e) => { + depth += 1; + let local = e.local_name().as_ref().to_vec(); + if !in_capture && is(ns_kind, &local, Ns::Messages, b"ResponseCode") { + current = Some("responseCode"); + } else if !in_capture && is(ns_kind, &local, Ns::Messages, b"MessageText") { + current = Some("messageText"); + } else if !in_capture + && (is(ns_kind, &local, Ns::Messages, b"Items") + || is(ns_kind, &local, Ns::Messages, b"Attachments") + || is(ns_kind, &local, Ns::Messages, b"Folders")) + { + capture_depth = 1; + current = None; + } else if in_capture { + write_start_xml(&mut inner, &e); + capture_depth += 1; + } else { + current = None; + } + } + Event::Empty(e) => { + let local = e.local_name().as_ref().to_vec(); + if in_capture { + write_empty_xml(&mut inner, &e); + } else if is(ns_kind, &local, Ns::Messages, b"ResponseCode") + || is(ns_kind, &local, Ns::Messages, b"MessageText") + { + } + } + Event::End(e) => { + if in_capture { + if capture_depth == 1 { + capture_depth = 0; + } else { + write_end_xml_event(&mut inner, &e); + capture_depth -= 1; + } + } + if depth == 0 { + break; + } + depth -= 1; + current = None; + if depth == 0 { + break; + } + } + Event::Text(t) => { + let text = t.decode().map(|c| c.into_owned()).unwrap_or_default(); + if in_capture { + write_text_xml(&mut inner, &text); + } else { + match current { + Some("responseCode") => response_code = ResponseCode::parse(text.trim()), + Some("messageText") => message_text = text, + _ => {} + } + } + } + Event::CData(c) => { + let text = String::from_utf8_lossy(c.as_ref()).into_owned(); + if in_capture { + write_text_xml(&mut inner, &text); + } + } + Event::Eof => break, + _ => {} + } + } + Ok(ResponseMessage { + success: matches!(response_class.as_str(), "Success" | "Warning"), + response_code, + message_text, + inner_xml: wrap_inner_with_namespaces(&inner), + }) +} + +fn wrap_inner_with_namespaces(inner: &str) -> String { + if inner.is_empty() { + return String::new(); + } + format!( + "{inner}", + ) +} + +fn write_start_xml(out: &mut String, e: &BytesStart<'_>) { + out.push('<'); + out.push_str(&String::from_utf8_lossy(e.name().as_ref())); + for a in e.attributes().flatten() { + out.push(' '); + out.push_str(&String::from_utf8_lossy(a.key.as_ref())); + out.push_str("=\""); + let val = String::from_utf8_lossy(a.value.as_ref()); + for ch in val.chars() { + match ch { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + _ => out.push(ch), + } + } + out.push('"'); + } + out.push('>'); +} + +fn write_empty_xml(out: &mut String, e: &BytesStart<'_>) { + out.push('<'); + out.push_str(&String::from_utf8_lossy(e.name().as_ref())); + for a in e.attributes().flatten() { + out.push(' '); + out.push_str(&String::from_utf8_lossy(a.key.as_ref())); + out.push_str("=\""); + let val = String::from_utf8_lossy(a.value.as_ref()); + for ch in val.chars() { + match ch { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + _ => out.push(ch), + } + } + out.push('"'); + } + out.push_str("/>"); +} + +fn write_end_xml_event(out: &mut String, e: &quick_xml::events::BytesEnd<'_>) { + out.push_str("'); +} + +fn write_text_xml(out: &mut String, text: &str) { + for ch in text.chars() { + match ch { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + _ => out.push(ch), + } + } +} + +#[derive(Debug, Clone)] +pub struct SyncFolderItemsResponse { + pub sync_state: String, + pub more: bool, + pub changes: Vec, +} + +#[derive(Debug, Clone)] +pub enum SyncChange { + Create { id: ItemId, element: String }, + Update { id: ItemId, element: String }, + Delete { id: ItemId }, + ReadFlagChange { id: ItemId, is_read: bool }, +} + +pub fn parse_sync_folder_items_response(body: &[u8]) -> Result { + let mut xml = NsReader::from_reader(body); + xml.config_mut().trim_text(true); + let mut buf = Vec::new(); + let mut sync_state = String::new(); + let mut more = false; + let mut changes: Vec = Vec::new(); + let mut current_text: Option<&'static str> = None; + let mut in_changes = false; + let mut current_change: Option = None; + let mut reading_is_read = false; + let mut pending_is_read: Option = None; + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + let ns_kind = classify(&ns); + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + let local = e.local_name().as_ref().to_vec(); + let is_empty = matches!(ev, Event::Empty(_)); + if is(ns_kind, &local, Ns::Messages, b"SyncState") { + current_text = Some("syncState"); + } else if is(ns_kind, &local, Ns::Messages, b"IncludesLastItemInRange") { + current_text = Some("includesLast"); + } else if is(ns_kind, &local, Ns::Messages, b"Changes") { + in_changes = true; + } else if in_changes && ns_kind == Ns::Types { + if local.eq_ignore_ascii_case(b"Create") { + current_change = Some(SyncChange::Create { + id: ItemId::default(), + element: String::new(), + }); + } else if local.eq_ignore_ascii_case(b"Update") { + current_change = Some(SyncChange::Update { + id: ItemId::default(), + element: String::new(), + }); + } else if local.eq_ignore_ascii_case(b"Delete") { + current_change = Some(SyncChange::Delete { + id: ItemId::default(), + }); + } else if local.eq_ignore_ascii_case(b"ReadFlagChange") { + current_change = Some(SyncChange::ReadFlagChange { + id: ItemId::default(), + is_read: false, + }); + pending_is_read = None; + } else if local.eq_ignore_ascii_case(b"ItemId") { + if let Some(change) = current_change.as_mut() { + let (id, ck) = item_id_mut(change); + capture_id_attrs(e, id, ck); + } + } else if local.eq_ignore_ascii_case(b"IsRead") { + reading_is_read = true; + } else if is_item_element(&local) { + match current_change.as_mut() { + Some(SyncChange::Create { element, .. }) + | Some(SyncChange::Update { element, .. }) => { + *element = String::from_utf8_lossy(&local).into_owned(); + } + _ => {} + } + } + } + if is_empty { + current_text = None; + } + } + Event::End(e) => { + let local = e.local_name().as_ref().to_vec(); + let lower = local.to_ascii_lowercase(); + match lower.as_slice() { + b"create" | b"update" | b"delete" | b"readflagchange" => { + if let Some(mut change) = current_change.take() { + if let SyncChange::ReadFlagChange { is_read, .. } = &mut change + && let Some(v) = pending_is_read.take() + { + *is_read = v; + } + changes.push(change); + } + } + b"isread" => reading_is_read = false, + b"changes" => in_changes = false, + _ => {} + } + current_text = None; + } + Event::Text(t) => { + let text = t.decode().map(|c| c.into_owned()).unwrap_or_default(); + if reading_is_read { + pending_is_read = Some(matches!(text.trim(), "true" | "1")); + } + match current_text { + Some("syncState") => sync_state = text, + Some("includesLast") => more = !matches!(text.trim(), "true" | "1"), + _ => {} + } + } + Event::CData(c) => { + let text = String::from_utf8_lossy(c.as_ref()).into_owned(); + if current_text == Some("syncState") { + sync_state = text; + } + } + Event::Eof => break, + _ => {} + } + } + Ok(SyncFolderItemsResponse { + sync_state, + more, + changes, + }) +} + +fn item_id_mut(change: &mut SyncChange) -> (&mut String, &mut String) { + match change { + SyncChange::Create { id, .. } + | SyncChange::Update { id, .. } + | SyncChange::Delete { id } + | SyncChange::ReadFlagChange { id, .. } => (&mut id.id, &mut id.change_key), + } +} + +#[derive(Debug, Clone, Default)] +pub struct MessageItem { + pub element: String, + pub id: ItemId, + pub parent_folder_id: Option, + pub mime_content: Option, + pub mime_charset: Option, + pub subject: Option, + pub date_time_received: Option, + pub is_read: Option, + pub is_draft: Option, + pub is_read_receipt_requested: Option, + pub categories: Vec, + pub flag_status: Option, +} + +pub fn parse_message_item(inner_xml: &str) -> Result { + let mut xml = NsReader::from_reader(inner_xml.as_bytes()); + xml.config_mut().trim_text(false); + let mut buf = Vec::new(); + let mut item = MessageItem::default(); + let mut current: Option<&'static str> = None; + let mut mime_charset: Option = None; + let mut category_collecting = false; + let mut in_flag = false; + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + let ns_kind = classify(&ns); + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + let local = e.local_name().as_ref().to_vec(); + let is_empty = matches!(ev, Event::Empty(_)); + if item.element.is_empty() && is_item_element(&local) { + item.element = String::from_utf8_lossy(&local).into_owned(); + } else if ns_kind == Ns::Types { + if local.eq_ignore_ascii_case(b"ItemId") { + capture_id_attrs(e, &mut item.id.id, &mut item.id.change_key); + } else if local.eq_ignore_ascii_case(b"ParentFolderId") { + let mut pid = String::new(); + let mut ck = String::new(); + capture_id_attrs(e, &mut pid, &mut ck); + if !pid.is_empty() { + item.parent_folder_id = Some(pid); + } + } else if local.eq_ignore_ascii_case(b"MimeContent") { + mime_charset = attr_value(e, b"CharacterSet"); + current = Some("mimeContent"); + } else if local.eq_ignore_ascii_case(b"Subject") { + current = Some("subject"); + } else if local.eq_ignore_ascii_case(b"DateTimeReceived") { + current = Some("received"); + } else if local.eq_ignore_ascii_case(b"IsRead") { + current = Some("isRead"); + } else if local.eq_ignore_ascii_case(b"IsDraft") { + current = Some("isDraft"); + } else if local.eq_ignore_ascii_case(b"IsReadReceiptRequested") { + current = Some("readReceipt"); + } else if local.eq_ignore_ascii_case(b"Categories") { + category_collecting = true; + } else if category_collecting && local.eq_ignore_ascii_case(b"String") { + current = Some("category"); + } else if local.eq_ignore_ascii_case(b"Flag") { + in_flag = true; + } else if in_flag && local.eq_ignore_ascii_case(b"FlagStatus") { + current = Some("flagStatus"); + } else { + current = None; + } + } + if is_empty { + current = None; + } + } + Event::End(e) => { + let local = e.local_name().as_ref().to_vec(); + let lower = local.to_ascii_lowercase(); + if lower == b"categories" { + category_collecting = false; + } else if lower == b"flag" { + in_flag = false; + } + current = None; + } + Event::Text(t) => { + let text = t.decode().map(|c| c.into_owned()).unwrap_or_default(); + match current { + Some("mimeContent") => { + item.mime_content + .get_or_insert_with(String::new) + .push_str(&text); + item.mime_charset = mime_charset.clone(); + } + Some("subject") => item.subject = Some(text), + Some("received") => item.date_time_received = Some(text), + Some("isRead") => item.is_read = Some(matches!(text.trim(), "true" | "1")), + Some("isDraft") => item.is_draft = Some(matches!(text.trim(), "true" | "1")), + Some("readReceipt") => { + item.is_read_receipt_requested = Some(matches!(text.trim(), "true" | "1")); + } + Some("category") => item.categories.push(text), + Some("flagStatus") => item.flag_status = Some(text), + _ => {} + } + } + Event::CData(c) => { + let text = String::from_utf8_lossy(c.as_ref()).into_owned(); + if current == Some("mimeContent") { + item.mime_content + .get_or_insert_with(String::new) + .push_str(&text); + item.mime_charset = mime_charset.clone(); + } + } + Event::Eof => break, + _ => {} + } + } + Ok(item) +} + +#[derive(Debug, Clone, Default)] +pub struct CalendarItemRaw { + pub element: String, + pub id: ItemId, + pub parent_folder_id: Option, + pub uid: Option, + pub subject: Option, + pub start: Option, + pub end: Option, + pub original_start: Option, + pub is_all_day_event: Option, + pub legacy_free_busy_status: Option, + pub location: Option, + pub calendar_item_type: Option, + pub recurrence_id: Option, + pub start_tz: Option, + pub end_tz: Option, + pub recurrence: Option, + pub modified_occurrences: Vec, + pub deleted_occurrences: Vec, + pub organizer_smtp: Option, + pub organizer_name: Option, + pub required_attendees: Vec, + pub optional_attendees: Vec, + pub categories: Vec, + pub created: Option, + pub last_modified: Option, + pub body_text: Option, + pub body_html: Option, + pub attachments: Vec, +} + +#[derive(Debug, Clone, Default)] +pub struct RawOccurrence { + pub item_id: ItemId, + pub start: Option, + pub end: Option, + pub original_start: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct RawAttendee { + pub email: Option, + pub name: Option, + pub response_type: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct RawAttachmentRef { + pub attachment_id: String, + pub name: Option, + pub content_type: Option, + pub is_contact_photo: bool, + pub is_item_attachment: bool, +} + +#[derive(Debug, Clone, Default)] +pub struct RawRecurrence { + pub pattern: Option, + pub range: Option, +} + +#[derive(Debug, Clone)] +pub enum RecurrencePattern { + Daily { + interval: u32, + }, + Weekly { + interval: u32, + days_of_week: Vec, + }, + AbsoluteMonthly { + interval: u32, + day_of_month: u32, + }, + RelativeMonthly { + interval: u32, + day_of_week_index: String, + days_of_week: Vec, + }, + AbsoluteYearly { + month: String, + day_of_month: u32, + }, + RelativeYearly { + month: String, + day_of_week_index: String, + days_of_week: Vec, + }, +} + +#[derive(Debug, Clone)] +pub enum RecurrenceRange { + NoEnd { + start_date: String, + }, + EndDate { + start_date: String, + end_date: String, + }, + Numbered { + start_date: String, + number_of_occurrences: u32, + }, +} + +pub fn parse_calendar_item(inner_xml: &str) -> Result { + let mut xml = NsReader::from_reader(inner_xml.as_bytes()); + xml.config_mut().trim_text(false); + let mut buf = Vec::new(); + let mut item = CalendarItemRaw::default(); + let mut text_target: Option<&'static str> = None; + let mut category_collecting = false; + let mut attendee_kind: Option<&'static str> = None; + let mut current_attendee: Option = None; + let mut in_mailbox = false; + let mut in_organizer = false; + let mut occurrence_stack: Vec = Vec::new(); + let mut deleted_stack: Vec = Vec::new(); + let mut in_modified = false; + let mut in_deleted = false; + let mut recurrence_path: Vec> = Vec::new(); + let mut recurrence_text: Option<&'static str> = None; + let mut recurrence = RawRecurrence::default(); + let mut pending = PendingRecurrence::default(); + + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + let ns_kind = classify(&ns); + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + let local = e.local_name().as_ref().to_vec(); + let is_empty = matches!(ev, Event::Empty(_)); + if item.element.is_empty() && is_item_element(&local) { + item.element = String::from_utf8_lossy(&local).into_owned(); + continue; + } + if ns_kind != Ns::Types { + continue; + } + if !recurrence_path.is_empty() { + recurrence_path.push(local.clone()); + if local.eq_ignore_ascii_case(b"Interval") { + recurrence_text = Some("interval"); + } else if local.eq_ignore_ascii_case(b"DaysOfWeek") { + recurrence_text = Some("daysOfWeek"); + } else if local.eq_ignore_ascii_case(b"DayOfMonth") { + recurrence_text = Some("dayOfMonth"); + } else if local.eq_ignore_ascii_case(b"DayOfWeekIndex") { + recurrence_text = Some("dayOfWeekIndex"); + } else if local.eq_ignore_ascii_case(b"Month") { + recurrence_text = Some("month"); + } else if local.eq_ignore_ascii_case(b"StartDate") { + recurrence_text = Some("startDate"); + } else if local.eq_ignore_ascii_case(b"EndDate") { + recurrence_text = Some("endDate"); + } else if local.eq_ignore_ascii_case(b"NumberOfOccurrences") { + recurrence_text = Some("numberOfOccurrences"); + } else if local.eq_ignore_ascii_case(b"DailyRecurrence") { + pending.pattern_choice = Some("Daily"); + } else if local.eq_ignore_ascii_case(b"WeeklyRecurrence") { + pending.pattern_choice = Some("Weekly"); + } else if local.eq_ignore_ascii_case(b"AbsoluteMonthlyRecurrence") { + pending.pattern_choice = Some("AbsoluteMonthly"); + } else if local.eq_ignore_ascii_case(b"RelativeMonthlyRecurrence") { + pending.pattern_choice = Some("RelativeMonthly"); + } else if local.eq_ignore_ascii_case(b"AbsoluteYearlyRecurrence") { + pending.pattern_choice = Some("AbsoluteYearly"); + } else if local.eq_ignore_ascii_case(b"RelativeYearlyRecurrence") { + pending.pattern_choice = Some("RelativeYearly"); + } else if local.eq_ignore_ascii_case(b"NoEndRecurrence") { + pending.range_choice = Some("NoEnd"); + } else if local.eq_ignore_ascii_case(b"EndDateRecurrence") { + pending.range_choice = Some("EndDate"); + } else if local.eq_ignore_ascii_case(b"NumberedRecurrence") { + pending.range_choice = Some("Numbered"); + } else { + recurrence_text = None; + } + if is_empty { + recurrence_path.pop(); + recurrence_text = None; + } + continue; + } + if local.eq_ignore_ascii_case(b"ItemId") { + if in_modified { + if let Some(occ) = occurrence_stack.last_mut() { + capture_id_attrs(e, &mut occ.item_id.id, &mut occ.item_id.change_key); + } + } else if in_deleted { + if let Some(occ) = deleted_stack.last_mut() { + capture_id_attrs(e, &mut occ.item_id.id, &mut occ.item_id.change_key); + } + } else { + capture_id_attrs(e, &mut item.id.id, &mut item.id.change_key); + } + } else if local.eq_ignore_ascii_case(b"ParentFolderId") { + let mut pid = String::new(); + let mut ck = String::new(); + capture_id_attrs(e, &mut pid, &mut ck); + if !pid.is_empty() { + item.parent_folder_id = Some(pid); + } + } else if local.eq_ignore_ascii_case(b"UID") { + text_target = Some("uid"); + } else if local.eq_ignore_ascii_case(b"Subject") { + text_target = Some("subject"); + } else if local.eq_ignore_ascii_case(b"Start") { + if in_modified || in_deleted { + text_target = Some("occStart"); + } else { + text_target = Some("start"); + } + } else if local.eq_ignore_ascii_case(b"End") { + if in_modified || in_deleted { + text_target = Some("occEnd"); + } else { + text_target = Some("end"); + } + } else if local.eq_ignore_ascii_case(b"OriginalStart") { + if in_modified || in_deleted { + text_target = Some("occOrig"); + } else { + text_target = Some("originalStart"); + } + } else if local.eq_ignore_ascii_case(b"IsAllDayEvent") { + text_target = Some("isAllDay"); + } else if local.eq_ignore_ascii_case(b"LegacyFreeBusyStatus") { + text_target = Some("freeBusy"); + } else if local.eq_ignore_ascii_case(b"Location") { + text_target = Some("location"); + } else if local.eq_ignore_ascii_case(b"CalendarItemType") { + text_target = Some("calendarItemType"); + } else if local.eq_ignore_ascii_case(b"RecurrenceId") { + text_target = Some("recurrenceId"); + } else if local.eq_ignore_ascii_case(b"StartTimeZone") { + if let Some(v) = attr_value(e, b"Id") { + item.start_tz = Some(v); + } + } else if local.eq_ignore_ascii_case(b"EndTimeZone") { + if let Some(v) = attr_value(e, b"Id") { + item.end_tz = Some(v); + } + } else if local.eq_ignore_ascii_case(b"Recurrence") { + recurrence_path.push(local.clone()); + } else if local.eq_ignore_ascii_case(b"ModifiedOccurrences") { + in_modified = true; + } else if local.eq_ignore_ascii_case(b"DeletedOccurrences") { + in_deleted = true; + } else if local.eq_ignore_ascii_case(b"Occurrence") { + let mut occ = RawOccurrence::default(); + capture_id_attrs(e, &mut occ.item_id.id, &mut occ.item_id.change_key); + occurrence_stack.push(occ); + } else if local.eq_ignore_ascii_case(b"DeletedOccurrence") { + deleted_stack.push(RawOccurrence::default()); + } else if local.eq_ignore_ascii_case(b"Organizer") { + in_organizer = true; + } else if local.eq_ignore_ascii_case(b"RequiredAttendees") { + attendee_kind = Some("required"); + } else if local.eq_ignore_ascii_case(b"OptionalAttendees") { + attendee_kind = Some("optional"); + } else if local.eq_ignore_ascii_case(b"Attendee") && attendee_kind.is_some() { + current_attendee = Some(RawAttendee::default()); + } else if local.eq_ignore_ascii_case(b"Mailbox") { + in_mailbox = true; + } else if local.eq_ignore_ascii_case(b"Name") && in_mailbox { + text_target = Some(if in_organizer { + "organizerName" + } else { + "attendeeName" + }); + } else if local.eq_ignore_ascii_case(b"EmailAddress") && in_mailbox { + text_target = Some(if in_organizer { + "organizerEmail" + } else { + "attendeeEmail" + }); + } else if local.eq_ignore_ascii_case(b"ResponseType") && current_attendee.is_some() + { + text_target = Some("attendeeResponse"); + } else if local.eq_ignore_ascii_case(b"Categories") { + category_collecting = true; + } else if category_collecting && local.eq_ignore_ascii_case(b"String") { + text_target = Some("category"); + } else if local.eq_ignore_ascii_case(b"DateTimeCreated") { + text_target = Some("created"); + } else if local.eq_ignore_ascii_case(b"LastModifiedTime") { + text_target = Some("lastModified"); + } else if local.eq_ignore_ascii_case(b"Body") { + let body_type = attr_value(e, b"BodyType").unwrap_or_default(); + text_target = Some(if body_type.eq_ignore_ascii_case("HTML") { + "bodyHtml" + } else { + "bodyText" + }); + } else if local.eq_ignore_ascii_case(b"FileAttachment") + || local.eq_ignore_ascii_case(b"ItemAttachment") + { + let att = parse_attachment_ref(&mut xml, &local)?; + item.attachments.push(att); + } else { + text_target = None; + } + if is_empty { + text_target = None; + } + } + Event::End(e) => { + let local = e.local_name().as_ref().to_vec(); + let lower = local.to_ascii_lowercase(); + if !recurrence_path.is_empty() { + recurrence_path.pop(); + if recurrence_path.is_empty() { + pending.finalize(&mut recurrence); + item.recurrence = Some(recurrence.clone()); + pending = PendingRecurrence::default(); + } + recurrence_text = None; + continue; + } + match lower.as_slice() { + b"modifiedoccurrences" => in_modified = false, + b"deletedoccurrences" => in_deleted = false, + b"occurrence" => { + if let Some(occ) = occurrence_stack.pop() { + item.modified_occurrences.push(occ); + } + } + b"deletedoccurrence" => { + if let Some(occ) = deleted_stack.pop() { + item.deleted_occurrences.push(occ); + } + } + b"attendee" => { + if let Some(att) = current_attendee.take() { + match attendee_kind { + Some("required") => item.required_attendees.push(att), + Some("optional") => item.optional_attendees.push(att), + _ => {} + } + } + } + b"requiredattendees" | b"optionalattendees" => attendee_kind = None, + b"organizer" => in_organizer = false, + b"mailbox" => in_mailbox = false, + b"categories" => category_collecting = false, + _ => {} + } + text_target = None; + } + Event::Text(_) | Event::CData(_) => { + let text = match ev { + Event::Text(ref t) => t.decode().map(|c| c.into_owned()).unwrap_or_default(), + Event::CData(ref c) => String::from_utf8_lossy(c.as_ref()).into_owned(), + _ => unreachable!(), + }; + if !recurrence_path.is_empty() { + match recurrence_text { + Some("interval") => pending.interval = text.trim().parse().unwrap_or(1), + Some("daysOfWeek") => { + for word in text.split_whitespace() { + pending.days_of_week.push(word.to_owned()); + } + } + Some("dayOfMonth") => { + pending.day_of_month = text.trim().parse().unwrap_or(0); + } + Some("dayOfWeekIndex") => { + pending.day_of_week_index = text.trim().to_owned() + } + Some("month") => pending.month = text.trim().to_owned(), + Some("startDate") => pending.start_date = text.trim().to_owned(), + Some("endDate") => pending.end_date = text.trim().to_owned(), + Some("numberOfOccurrences") => { + pending.number_of_occurrences = text.trim().parse().unwrap_or(0); + } + _ => {} + } + + continue; + } + + match text_target { + Some("uid") => item.uid = Some(text), + Some("subject") => item.subject = Some(text), + Some("start") => item.start = Some(text), + Some("end") => item.end = Some(text), + Some("originalStart") => item.original_start = Some(text), + Some("isAllDay") => { + item.is_all_day_event = Some(matches!(text.trim(), "true" | "1")); + } + Some("freeBusy") => item.legacy_free_busy_status = Some(text), + Some("location") => item.location = Some(text), + Some("calendarItemType") => { + item.calendar_item_type = CalendarItemType::parse(text.trim()); + } + Some("recurrenceId") => item.recurrence_id = Some(text), + Some("organizerName") => item.organizer_name = Some(text), + Some("organizerEmail") => item.organizer_smtp = Some(text), + Some("attendeeName") => { + if let Some(att) = current_attendee.as_mut() { + att.name = Some(text); + } + } + Some("attendeeEmail") => { + if let Some(att) = current_attendee.as_mut() { + att.email = Some(text); + } + } + Some("attendeeResponse") => { + if let Some(att) = current_attendee.as_mut() { + att.response_type = Some(text); + } + } + Some("category") => item.categories.push(text), + Some("created") => item.created = Some(text), + Some("lastModified") => item.last_modified = Some(text), + Some("bodyText") => { + item.body_text + .get_or_insert_with(String::new) + .push_str(&text); + } + Some("bodyHtml") => { + item.body_html + .get_or_insert_with(String::new) + .push_str(&text); + } + Some("occStart") => { + if let Some(occ) = occurrence_stack.last_mut() { + occ.start = Some(text.clone()); + } + if let Some(occ) = deleted_stack.last_mut() { + occ.start = Some(text); + } + } + Some("occEnd") => { + if let Some(occ) = occurrence_stack.last_mut() { + occ.end = Some(text); + } + } + Some("occOrig") => { + if let Some(occ) = occurrence_stack.last_mut() { + occ.original_start = Some(text); + } + } + _ => {} + } + } + Event::Eof => break, + _ => {} + } + } + Ok(item) +} + +#[derive(Default)] +struct PendingRecurrence { + pattern_choice: Option<&'static str>, + range_choice: Option<&'static str>, + interval: u32, + days_of_week: Vec, + day_of_month: u32, + day_of_week_index: String, + month: String, + start_date: String, + end_date: String, + number_of_occurrences: u32, +} + +impl PendingRecurrence { + fn finalize(self, out: &mut RawRecurrence) { + let interval = if self.interval == 0 { 1 } else { self.interval }; + out.pattern = match self.pattern_choice { + Some("Daily") => Some(RecurrencePattern::Daily { interval }), + Some("Weekly") => Some(RecurrencePattern::Weekly { + interval, + days_of_week: self.days_of_week.clone(), + }), + Some("AbsoluteMonthly") => Some(RecurrencePattern::AbsoluteMonthly { + interval, + day_of_month: self.day_of_month, + }), + Some("RelativeMonthly") => Some(RecurrencePattern::RelativeMonthly { + interval, + day_of_week_index: self.day_of_week_index.clone(), + days_of_week: self.days_of_week.clone(), + }), + Some("AbsoluteYearly") => Some(RecurrencePattern::AbsoluteYearly { + month: self.month.clone(), + day_of_month: self.day_of_month, + }), + Some("RelativeYearly") => Some(RecurrencePattern::RelativeYearly { + month: self.month.clone(), + day_of_week_index: self.day_of_week_index.clone(), + days_of_week: self.days_of_week.clone(), + }), + _ => None, + }; + out.range = match self.range_choice { + Some("NoEnd") => Some(RecurrenceRange::NoEnd { + start_date: self.start_date, + }), + Some("EndDate") => Some(RecurrenceRange::EndDate { + start_date: self.start_date, + end_date: self.end_date, + }), + Some("Numbered") => Some(RecurrenceRange::Numbered { + start_date: self.start_date, + number_of_occurrences: self.number_of_occurrences, + }), + _ => None, + }; + } +} + +#[derive(Debug, Clone, Default)] +pub struct ContactItemRaw { + pub id: ItemId, + pub parent_folder_id: Option, + pub display_name: Option, + pub given_name: Option, + pub middle_name: Option, + pub surname: Option, + pub initials: Option, + pub nickname: Option, + pub company_name: Option, + pub department: Option, + pub job_title: Option, + pub generation: Option, + pub office_location: Option, + pub url: Option, + pub birthday: Option, + pub wedding_anniversary: Option, + pub notes: Option, + pub categories: Vec, + pub emails: Vec<(String, String)>, + pub phones: Vec<(String, String)>, + pub addresses: Vec, + pub ims: Vec<(String, String)>, + pub manager: Option, + pub spouse: Option, + pub assistant: Option, + pub children: Vec, + pub companies: Vec, + pub profession: Option, + pub postal_address_index: Option, + pub created: Option, + pub last_modified: Option, + pub attachments: Vec, +} + +#[derive(Debug, Clone, Default)] +pub struct RawContactAddress { + pub key: String, + pub street: Option, + pub city: Option, + pub state: Option, + pub country: Option, + pub postal_code: Option, +} + +pub fn parse_contact_item(inner_xml: &str) -> Result { + let mut xml = NsReader::from_reader(inner_xml.as_bytes()); + xml.config_mut().trim_text(false); + let mut buf = Vec::new(); + let mut item = ContactItemRaw::default(); + let mut text_target: Option<&'static str> = None; + let mut category_collecting = false; + let mut children_collecting = false; + let mut companies_collecting = false; + let mut entry_container: Option<&'static str> = None; + let mut entry_key: Option = None; + let mut current_address: Option = None; + let mut address_text: Option<&'static str> = None; + let mut seen_root_contact = false; + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + let ns_kind = classify(&ns); + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + let local = e.local_name().as_ref().to_vec(); + let is_empty = matches!(ev, Event::Empty(_)); + if !seen_root_contact && local.eq_ignore_ascii_case(b"Contact") { + seen_root_contact = true; + continue; + } + if ns_kind != Ns::Types { + continue; + } + if local.eq_ignore_ascii_case(b"ItemId") { + capture_id_attrs(e, &mut item.id.id, &mut item.id.change_key); + } else if local.eq_ignore_ascii_case(b"ParentFolderId") { + let mut pid = String::new(); + let mut ck = String::new(); + capture_id_attrs(e, &mut pid, &mut ck); + if !pid.is_empty() { + item.parent_folder_id = Some(pid); + } + } else if local.eq_ignore_ascii_case(b"DisplayName") { + text_target = Some("displayName"); + } else if local.eq_ignore_ascii_case(b"GivenName") { + text_target = Some("givenName"); + } else if local.eq_ignore_ascii_case(b"MiddleName") { + text_target = Some("middleName"); + } else if local.eq_ignore_ascii_case(b"Surname") { + text_target = Some("surname"); + } else if local.eq_ignore_ascii_case(b"Initials") { + text_target = Some("initials"); + } else if local.eq_ignore_ascii_case(b"Nickname") { + text_target = Some("nickname"); + } else if local.eq_ignore_ascii_case(b"CompanyName") { + text_target = Some("companyName"); + } else if local.eq_ignore_ascii_case(b"Department") { + text_target = Some("department"); + } else if local.eq_ignore_ascii_case(b"JobTitle") { + text_target = Some("jobTitle"); + } else if local.eq_ignore_ascii_case(b"Generation") { + text_target = Some("generation"); + } else if local.eq_ignore_ascii_case(b"OfficeLocation") { + text_target = Some("officeLocation"); + } else if local.eq_ignore_ascii_case(b"BusinessHomePage") { + text_target = Some("url"); + } else if local.eq_ignore_ascii_case(b"Birthday") { + text_target = Some("birthday"); + } else if local.eq_ignore_ascii_case(b"WeddingAnniversary") { + text_target = Some("weddingAnniversary"); + } else if local.eq_ignore_ascii_case(b"Manager") { + text_target = Some("manager"); + } else if local.eq_ignore_ascii_case(b"SpouseName") { + text_target = Some("spouse"); + } else if local.eq_ignore_ascii_case(b"AssistantName") { + text_target = Some("assistant"); + } else if local.eq_ignore_ascii_case(b"Profession") { + text_target = Some("profession"); + } else if local.eq_ignore_ascii_case(b"PostalAddressIndex") { + text_target = Some("postalAddressIndex"); + } else if local.eq_ignore_ascii_case(b"Body") { + text_target = Some("notes"); + } else if local.eq_ignore_ascii_case(b"DateTimeCreated") { + text_target = Some("created"); + } else if local.eq_ignore_ascii_case(b"LastModifiedTime") { + text_target = Some("lastModified"); + } else if local.eq_ignore_ascii_case(b"Categories") { + category_collecting = true; + } else if category_collecting && local.eq_ignore_ascii_case(b"String") { + text_target = Some("category"); + } else if local.eq_ignore_ascii_case(b"Children") { + children_collecting = true; + } else if children_collecting && local.eq_ignore_ascii_case(b"String") { + text_target = Some("child"); + } else if local.eq_ignore_ascii_case(b"Companies") { + companies_collecting = true; + } else if companies_collecting && local.eq_ignore_ascii_case(b"String") { + text_target = Some("company"); + } else if local.eq_ignore_ascii_case(b"EmailAddresses") { + entry_container = Some("email"); + } else if local.eq_ignore_ascii_case(b"PhoneNumbers") { + entry_container = Some("phone"); + } else if local.eq_ignore_ascii_case(b"ImAddresses") { + entry_container = Some("im"); + } else if local.eq_ignore_ascii_case(b"PhysicalAddresses") { + entry_container = Some("address"); + } else if local.eq_ignore_ascii_case(b"Entry") { + entry_key = attr_value(e, b"Key"); + if entry_container == Some("address") { + current_address = Some(RawContactAddress { + key: entry_key.clone().unwrap_or_default(), + ..RawContactAddress::default() + }); + } else { + text_target = Some("entryValue"); + } + } else if current_address.is_some() { + if local.eq_ignore_ascii_case(b"Street") { + address_text = Some("street"); + } else if local.eq_ignore_ascii_case(b"City") { + address_text = Some("city"); + } else if local.eq_ignore_ascii_case(b"State") { + address_text = Some("state"); + } else if local.eq_ignore_ascii_case(b"CountryOrRegion") { + address_text = Some("country"); + } else if local.eq_ignore_ascii_case(b"PostalCode") { + address_text = Some("postal"); + } else { + address_text = None; + } + } else if local.eq_ignore_ascii_case(b"FileAttachment") + || local.eq_ignore_ascii_case(b"ItemAttachment") + { + let att = parse_attachment_ref(&mut xml, &local)?; + item.attachments.push(att); + } else { + text_target = None; + } + if is_empty { + text_target = None; + address_text = None; + } + } + Event::End(e) => { + let local = e.local_name().as_ref().to_vec(); + let lower = local.to_ascii_lowercase(); + match lower.as_slice() { + b"categories" => category_collecting = false, + b"children" => children_collecting = false, + b"companies" => companies_collecting = false, + b"emailaddresses" | b"phonenumbers" | b"imaddresses" | b"physicaladdresses" => { + entry_container = None + } + b"entry" => { + if let Some(addr) = current_address.take() { + item.addresses.push(addr); + } + entry_key = None; + } + b"street" | b"city" | b"state" | b"countryorregion" | b"postalcode" => { + address_text = None; + } + _ => {} + } + text_target = None; + } + Event::Text(_) | Event::CData(_) => { + let text = match ev { + Event::Text(ref t) => t.decode().map(|c| c.into_owned()).unwrap_or_default(), + Event::CData(ref c) => String::from_utf8_lossy(c.as_ref()).into_owned(), + _ => unreachable!(), + }; + + if let Some(at) = address_text + && let Some(addr) = current_address.as_mut() + { + match at { + "street" => addr.street = Some(text.clone()), + "city" => addr.city = Some(text.clone()), + "state" => addr.state = Some(text.clone()), + "country" => addr.country = Some(text.clone()), + "postal" => addr.postal_code = Some(text.clone()), + _ => {} + } + continue; + } + match text_target { + Some("displayName") => item.display_name = Some(text), + Some("givenName") => item.given_name = Some(text), + Some("middleName") => item.middle_name = Some(text), + Some("surname") => item.surname = Some(text), + Some("initials") => item.initials = Some(text), + Some("nickname") => item.nickname = Some(text), + Some("companyName") => item.company_name = Some(text), + Some("department") => item.department = Some(text), + Some("jobTitle") => item.job_title = Some(text), + Some("generation") => item.generation = Some(text), + Some("officeLocation") => item.office_location = Some(text), + Some("url") => item.url = Some(text), + Some("birthday") => item.birthday = Some(text), + Some("weddingAnniversary") => item.wedding_anniversary = Some(text), + Some("manager") => item.manager = Some(text), + Some("spouse") => item.spouse = Some(text), + Some("assistant") => item.assistant = Some(text), + Some("profession") => item.profession = Some(text), + Some("postalAddressIndex") => item.postal_address_index = Some(text), + Some("notes") => item.notes = Some(text), + Some("created") => item.created = Some(text), + Some("lastModified") => item.last_modified = Some(text), + Some("category") => item.categories.push(text), + Some("child") => item.children.push(text), + Some("company") => item.companies.push(text), + Some("entryValue") => { + let key = entry_key.clone().unwrap_or_default(); + match entry_container { + Some("email") => item.emails.push((key, text)), + Some("phone") => item.phones.push((key, text)), + Some("im") => item.ims.push((key, text)), + _ => {} + } + } + _ => {} + } + } + Event::Eof => break, + _ => {} + } + } + Ok(item) +} + +fn parse_attachment_ref( + xml: &mut NsReader, + element_local: &[u8], +) -> Result { + let mut att = RawAttachmentRef { + is_item_attachment: element_local.eq_ignore_ascii_case(b"itemattachment"), + ..RawAttachmentRef::default() + }; + let mut buf = Vec::new(); + let mut depth: u32 = 1; + let mut current: Option<&'static str> = None; + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + let ns_kind = classify(&ns); + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + let is_empty = matches!(ev, Event::Empty(_)); + if !is_empty { + depth += 1; + } + let local = e.local_name().as_ref().to_vec(); + if ns_kind == Ns::Types { + if local.eq_ignore_ascii_case(b"AttachmentId") { + if let Some(v) = attr_value(e, b"Id") { + att.attachment_id = v; + } + } else if local.eq_ignore_ascii_case(b"Name") { + current = Some("name"); + } else if local.eq_ignore_ascii_case(b"ContentType") { + current = Some("contentType"); + } else if local.eq_ignore_ascii_case(b"IsContactPhoto") { + current = Some("isContactPhoto"); + } else { + current = None; + } + } + if is_empty { + current = None; + } + } + Event::End(_) => { + if depth == 0 { + break; + } + depth -= 1; + current = None; + if depth == 0 { + break; + } + } + Event::Text(t) => { + let text = t.decode().map(|c| c.into_owned()).unwrap_or_default(); + match current { + Some("name") => att.name = Some(text), + Some("contentType") => att.content_type = Some(text), + Some("isContactPhoto") => { + att.is_contact_photo = matches!(text.trim(), "true" | "1"); + } + _ => {} + } + } + Event::Eof => break, + _ => {} + } + } + Ok(att) +} + +#[derive(Debug, Clone, Default)] +pub struct GetAttachmentInline { + pub attachment_id: String, + pub name: Option, + pub content_type: Option, + pub is_contact_photo: bool, + pub content_base64: String, +} + +pub fn parse_get_attachment_inline(body: &[u8]) -> Result, EwsError> { + let mut xml = NsReader::from_reader(body); + xml.config_mut().trim_text(true); + let mut buf = Vec::new(); + let mut out: Vec = Vec::new(); + let mut current: Option = None; + let mut text_target: Option<&'static str> = None; + loop { + buf.clear(); + let (ns, ev) = xml.read_resolved_event_into(&mut buf)?; + let ns_kind = classify(&ns); + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + let local = e.local_name().as_ref().to_vec(); + let is_empty = matches!(ev, Event::Empty(_)); + if ns_kind == Ns::Types + && (local.eq_ignore_ascii_case(b"FileAttachment") + || local.eq_ignore_ascii_case(b"ItemAttachment")) + { + current = Some(GetAttachmentInline::default()); + } else if let Some(cur) = current.as_mut() + && ns_kind == Ns::Types + { + if local.eq_ignore_ascii_case(b"AttachmentId") { + if let Some(v) = attr_value(e, b"Id") { + cur.attachment_id = v; + } + } else if local.eq_ignore_ascii_case(b"Name") { + text_target = Some("name"); + } else if local.eq_ignore_ascii_case(b"ContentType") { + text_target = Some("contentType"); + } else if local.eq_ignore_ascii_case(b"IsContactPhoto") { + text_target = Some("isContactPhoto"); + } else if local.eq_ignore_ascii_case(b"Content") { + text_target = Some("content"); + } else { + text_target = None; + } + } + if is_empty { + text_target = None; + } + } + Event::End(e) => { + let local = e.local_name().as_ref().to_vec(); + if (local.eq_ignore_ascii_case(b"FileAttachment") + || local.eq_ignore_ascii_case(b"ItemAttachment")) + && let Some(att) = current.take() + { + out.push(att); + } + text_target = None; + } + Event::Text(_) | Event::CData(_) => { + let text = match ev { + Event::Text(ref t) => t.decode().map(|c| c.into_owned()).unwrap_or_default(), + Event::CData(ref c) => String::from_utf8_lossy(c.as_ref()).into_owned(), + _ => unreachable!(), + }; + + if let Some(cur) = current.as_mut() { + match text_target { + Some("name") => cur.name = Some(text), + Some("contentType") => cur.content_type = Some(text), + Some("isContactPhoto") => { + cur.is_contact_photo = matches!(text.trim(), "true" | "1"); + } + Some("content") => cur.content_base64.push_str(&text), + _ => {} + } + } + } + Event::Eof => break, + _ => {} + } + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + const NS: &str = " xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\" \ + xmlns:t=\"http://schemas.microsoft.com/exchange/services/2006/types\" \ + xmlns:m=\"http://schemas.microsoft.com/exchange/services/2006/messages\""; + + #[test] + fn server_busy_fault_extracts_backoff_milliseconds() { + let body = format!( + "\ + soap:Serverbusy\ + ErrorServerBusy\ + 297749\ + " + ); + let env = read_envelope_summary(body.as_bytes()).unwrap(); + match env { + EnvelopeKind::Fault { fault, version } => { + assert!(matches!( + fault.response_code, + ResponseCode::ServerBusy { + back_off_ms: Some(297749) + } + )); + assert_eq!(fault.back_off_ms, Some(297749)); + assert_eq!( + version.to_server_version(), + Some(ServerVersion::Exchange2016) + ); + } + _ => panic!("expected fault"), + } + } + + #[test] + fn body_envelope_returns_version_only() { + let body = format!( + "\ + " + ); + let env = read_envelope_summary(body.as_bytes()).unwrap(); + assert!(matches!(env, EnvelopeKind::Body { .. })); + } + + #[test] + fn find_folder_parses_two_folders() { + let body = format!( + "\ + NoError\ + \ + \ + \ + IPF.NoteInbox\ + 70\ + \ + IPF.AppointmentCalendar\ + \ + " + ); + let r = parse_find_folder_response(body.as_bytes()).unwrap(); + assert_eq!(r.folders.len(), 2); + assert_eq!(r.folders[0].folder_id.id, "F1"); + assert_eq!(r.folders[0].folder_id.change_key, "C1"); + assert_eq!(r.folders[0].display_name, "Inbox"); + assert_eq!(r.folders[0].folder_class, "IPF.Note"); + assert_eq!(r.folders[0].parent_id.as_deref(), Some("ROOT")); + assert_eq!(r.folders[1].element, FolderElement::CalendarFolder); + assert!(!r.more); + } + + #[test] + fn find_item_parses_two_messages_with_pagination_flag() { + let body = format!( + "\ + NoError\ + \ + \ + \ + \ + " + ); + let r = parse_find_item_response(body.as_bytes()).unwrap(); + assert_eq!(r.items.len(), 2); + assert_eq!(r.items[0].id.id, "I1"); + assert_eq!(r.items[0].id.change_key, "CK1"); + assert_eq!(r.items[1].id.id, "I2"); + assert!(r.more); + assert_eq!(r.total_in_view, Some(3)); + } + + #[test] + fn parse_response_messages_carries_inner_xml_and_codes() { + let body = format!( + "\ + \ + NoError\ + \ + Hitrue\ + SGVsbG8=\ + \ + \ + ErrorItemNotFound\ + not found\ + \ + " + ); + let r = parse_response_messages(body.as_bytes(), b"GetItemResponseMessage").unwrap(); + assert_eq!(r.len(), 2); + assert!(r[0].success); + assert!(r[0].inner_xml.contains("\ + NoError\ + AAAA\ + true\ + \ + \ + \ + true\ + " + ); + let r = parse_sync_folder_items_response(body.as_bytes()).unwrap(); + assert_eq!(r.sync_state, "AAAA"); + assert!(!r.more); + assert_eq!(r.changes.len(), 3); + match &r.changes[0] { + SyncChange::Create { id, element } => { + assert_eq!(id.id, "I1"); + assert_eq!(element, "Message"); + } + _ => panic!("expected Create"), + } + match &r.changes[2] { + SyncChange::ReadFlagChange { id, is_read } => { + assert_eq!(id.id, "I3"); + assert!(*is_read); + } + _ => panic!("expected ReadFlagChange"), + } + } + + #[test] + fn calendar_master_recurrence_and_occurrences_round_trip() { + let body = format!( + "\ + \ + Daily\ + uid-1\ + 2025-06-15T14:00:00Z\ + 2025-06-15T15:00:00Z\ + RecurringMaster\ + \ + 1\ + 2025-06-153\ + \ + \ + 2025-06-16T15:00:00Z2025-06-16T16:30:00Z2025-06-16T14:00:00Z\ + \ + \ + 2025-06-17T14:00:00Z\ + \ + " + ); + let parsed = parse_calendar_item(&body).unwrap(); + assert_eq!(parsed.id.id, "M1"); + assert_eq!(parsed.uid.as_deref(), Some("uid-1")); + assert_eq!( + parsed.calendar_item_type, + Some(CalendarItemType::RecurringMaster) + ); + assert_eq!(parsed.modified_occurrences.len(), 1); + assert_eq!( + parsed.modified_occurrences[0].original_start.as_deref(), + Some("2025-06-16T14:00:00Z") + ); + assert_eq!(parsed.deleted_occurrences.len(), 1); + assert_eq!( + parsed.deleted_occurrences[0].start.as_deref(), + Some("2025-06-17T14:00:00Z") + ); + let rec = parsed.recurrence.unwrap(); + assert!(matches!( + rec.pattern, + Some(RecurrencePattern::Daily { interval: 1 }) + )); + assert!(matches!( + rec.range, + Some(RecurrenceRange::Numbered { + number_of_occurrences: 3, + .. + }) + )); + } + + #[test] + fn contact_parses_emails_phones_addresses() { + let body = format!( + "\ + \ + Alice Doe\ + Alice\ + Doe\ + \ + alice@example.com\ + \ + \ + +15551234\ + \ + \ + 1 MainTownUS\ + \ + " + ); + let parsed = parse_contact_item(&body).unwrap(); + assert_eq!(parsed.given_name.as_deref(), Some("Alice")); + assert_eq!(parsed.surname.as_deref(), Some("Doe")); + assert_eq!(parsed.emails.len(), 1); + assert_eq!(parsed.emails[0].0, "EmailAddress1"); + assert_eq!(parsed.emails[0].1, "alice@example.com"); + assert_eq!(parsed.phones.len(), 1); + assert_eq!(parsed.phones[0].0, "BusinessPhone1"); + assert_eq!(parsed.addresses.len(), 1); + assert_eq!(parsed.addresses[0].key, "Home"); + assert_eq!(parsed.addresses[0].street.as_deref(), Some("1 Main")); + assert_eq!(parsed.addresses[0].country.as_deref(), Some("US")); + } + + #[test] + fn get_item_with_unprefixed_types_elements_parses() { + let body = format!( + "\ + \ + NoError\ + \ + \ + Hello\ + SGk=\ + \ + " + ); + let msgs = parse_response_messages(body.as_bytes(), b"GetItemResponseMessage").unwrap(); + assert_eq!(msgs.len(), 1); + let item = parse_message_item(&msgs[0].inner_xml).unwrap(); + assert_eq!(item.id.id, "NP1"); + assert_eq!(item.subject.as_deref(), Some("Hello")); + assert_eq!(item.mime_content.as_deref(), Some("SGk=")); + } + + #[test] + fn default_namespace_response_parses_via_resolved_names() { + let body = "\ + \ + \ + \ + NoError\ + \ + \ + IPF.NoteInbox\ + \ + \ + "; + let parsed = parse_find_folder_response(body.as_bytes()).unwrap(); + assert_eq!(parsed.folders.len(), 1); + assert_eq!(parsed.folders[0].folder_id.id, "D1"); + assert_eq!(parsed.folders[0].display_name, "Inbox"); + } + + #[test] + fn fault_response_code_in_messages_namespace_is_recognised() { + let body = format!( + "\ + soap:Serverbusy\ + ErrorServerBusy\ + 42\ + " + ); + let env = read_envelope_summary(body.as_bytes()).unwrap(); + match env { + EnvelopeKind::Fault { fault, .. } => { + assert!(matches!( + fault.response_code, + ResponseCode::ServerBusy { .. } + )); + assert_eq!(fault.back_off_ms, Some(42)); + } + _ => panic!("expected fault"), + } + } + + #[test] + fn change_key_attribute_with_entities_is_xml_decoded() { + let body = format!( + "\ + NoError\ + \ + \ + X\ + " + ); + let parsed = parse_find_folder_response(body.as_bytes()).unwrap(); + assert_eq!(parsed.folders[0].folder_id.change_key, "A&B"); + } + + #[test] + fn warning_response_class_is_treated_as_success() { + let body = format!( + "\ + \ + NoError\ + \ + " + ); + let r = parse_response_messages(body.as_bytes(), b"GetItemResponseMessage").unwrap(); + assert_eq!(r.len(), 1); + assert!(r[0].success, "Warning should be success-equivalent"); + } + + #[test] + fn get_folder_response_message_captures_folder_inner() { + let body = format!( + "\ + \ + NoError\ + \ + Inbox\ + \ + \ + ErrorAccessDenied\ + " + ); + let msgs = parse_response_messages(body.as_bytes(), b"GetFolderResponseMessage").unwrap(); + assert_eq!(msgs.len(), 2); + assert!(msgs[0].success); + let f = parse_folder_inner(&msgs[0].inner_xml).unwrap().unwrap(); + assert_eq!(f.folder_id.id, "F1"); + assert!(!msgs[1].success); + } + + #[test] + fn get_attachment_inline_decodes_content_field() { + let body = format!( + "\ + \ + NoError\ + \ + p.png\ + image/png\ + true\ + iVBORw0KGgo=\ + " + ); + let items = parse_get_attachment_inline(body.as_bytes()).unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0].attachment_id, "A1"); + assert_eq!(items[0].name.as_deref(), Some("p.png")); + assert_eq!(items[0].content_type.as_deref(), Some("image/png")); + assert!(items[0].is_contact_photo); + assert_eq!(items[0].content_base64, "iVBORw0KGgo="); + } +} diff --git a/src/exchange_ews/recurrence.rs b/src/exchange_ews/recurrence.rs new file mode 100644 index 0000000..d7ead56 --- /dev/null +++ b/src/exchange_ews/recurrence.rs @@ -0,0 +1,309 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::{Map, Value, json}; + +use crate::exchange_ews::parse::{RawRecurrence, RecurrencePattern, RecurrenceRange}; + +pub fn to_jscalendar_rule(raw: &RawRecurrence) -> Option { + let mut rule: Map = Map::new(); + match raw.pattern.as_ref()? { + RecurrencePattern::Daily { interval } => { + rule.insert("frequency".to_owned(), Value::String("daily".to_owned())); + if *interval > 1 { + rule.insert("interval".to_owned(), Value::from(*interval)); + } + } + RecurrencePattern::Weekly { + interval, + days_of_week, + } => { + rule.insert("frequency".to_owned(), Value::String("weekly".to_owned())); + if *interval > 1 { + rule.insert("interval".to_owned(), Value::from(*interval)); + } + let days: Vec = days_of_week + .iter() + .filter_map(|d| day_token(d).map(|t| json!({"day": t}))) + .collect(); + if !days.is_empty() { + rule.insert("byDay".to_owned(), Value::Array(days)); + } + } + RecurrencePattern::AbsoluteMonthly { + interval, + day_of_month, + } => { + rule.insert("frequency".to_owned(), Value::String("monthly".to_owned())); + if *interval > 1 { + rule.insert("interval".to_owned(), Value::from(*interval)); + } + rule.insert( + "byMonthDay".to_owned(), + Value::Array(vec![Value::from(*day_of_month)]), + ); + } + RecurrencePattern::RelativeMonthly { + interval, + day_of_week_index, + days_of_week, + } => { + rule.insert("frequency".to_owned(), Value::String("monthly".to_owned())); + if *interval > 1 { + rule.insert("interval".to_owned(), Value::from(*interval)); + } + let nth = nth_of_period(day_of_week_index); + let days: Vec = days_of_week + .iter() + .filter_map(|d| { + day_token(d).map(|t| { + let mut o = Map::new(); + o.insert("day".to_owned(), Value::String(t.to_owned())); + if let Some(n) = nth { + o.insert("nthOfPeriod".to_owned(), Value::from(n)); + } + Value::Object(o) + }) + }) + .collect(); + if !days.is_empty() { + rule.insert("byDay".to_owned(), Value::Array(days)); + } + } + RecurrencePattern::AbsoluteYearly { + month, + day_of_month, + } => { + rule.insert("frequency".to_owned(), Value::String("yearly".to_owned())); + if let Some(n) = month_number(month) { + rule.insert( + "byMonth".to_owned(), + Value::Array(vec![Value::String(n.to_string())]), + ); + } + rule.insert( + "byMonthDay".to_owned(), + Value::Array(vec![Value::from(*day_of_month)]), + ); + } + RecurrencePattern::RelativeYearly { + month, + day_of_week_index, + days_of_week, + } => { + rule.insert("frequency".to_owned(), Value::String("yearly".to_owned())); + if let Some(n) = month_number(month) { + rule.insert( + "byMonth".to_owned(), + Value::Array(vec![Value::String(n.to_string())]), + ); + } + let nth = nth_of_period(day_of_week_index); + let days: Vec = days_of_week + .iter() + .filter_map(|d| { + day_token(d).map(|t| { + let mut o = Map::new(); + o.insert("day".to_owned(), Value::String(t.to_owned())); + if let Some(n) = nth { + o.insert("nthOfPeriod".to_owned(), Value::from(n)); + } + Value::Object(o) + }) + }) + .collect(); + if !days.is_empty() { + rule.insert("byDay".to_owned(), Value::Array(days)); + } + } + } + match raw.range.as_ref() { + Some(RecurrenceRange::NoEnd { .. }) | None => {} + Some(RecurrenceRange::EndDate { end_date, .. }) => { + let local = if end_date.contains('T') { + end_date.clone() + } else { + format!("{end_date}T23:59:59") + }; + rule.insert("until".to_owned(), Value::String(local)); + } + Some(RecurrenceRange::Numbered { + number_of_occurrences, + .. + }) => { + rule.insert("count".to_owned(), Value::from(*number_of_occurrences)); + } + } + Some(Value::Object(rule)) +} + +fn day_token(d: &str) -> Option<&'static str> { + match d.to_ascii_lowercase().as_str() { + "monday" | "mo" => Some("mo"), + "tuesday" | "tu" => Some("tu"), + "wednesday" | "we" => Some("we"), + "thursday" | "th" => Some("th"), + "friday" | "fr" => Some("fr"), + "saturday" | "sa" => Some("sa"), + "sunday" | "su" => Some("su"), + _ => None, + } +} + +fn nth_of_period(index: &str) -> Option { + match index { + "First" => Some(1), + "Second" => Some(2), + "Third" => Some(3), + "Fourth" => Some(4), + "Last" => Some(-1), + _ => None, + } +} + +fn month_number(m: &str) -> Option { + Some(match m { + "January" => 1, + "February" => 2, + "March" => 3, + "April" => 4, + "May" => 5, + "June" => 6, + "July" => 7, + "August" => 8, + "September" => 9, + "October" => 10, + "November" => 11, + "December" => 12, + _ => return None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn daily_interval_round_trips() { + let raw = RawRecurrence { + pattern: Some(RecurrencePattern::Daily { interval: 3 }), + range: Some(RecurrenceRange::Numbered { + start_date: "2025-01-01".to_owned(), + number_of_occurrences: 5, + }), + }; + let rule = to_jscalendar_rule(&raw).unwrap(); + assert_eq!(rule["frequency"], "daily"); + assert_eq!(rule["interval"], 3); + assert_eq!(rule["count"], 5); + } + + #[test] + fn relative_monthly_nth_of_period() { + let raw = RawRecurrence { + pattern: Some(RecurrencePattern::RelativeMonthly { + interval: 1, + day_of_week_index: "First".to_owned(), + days_of_week: vec!["Monday".to_owned()], + }), + range: Some(RecurrenceRange::NoEnd { + start_date: "2025-01-01".to_owned(), + }), + }; + let rule = to_jscalendar_rule(&raw).unwrap(); + assert_eq!(rule["frequency"], "monthly"); + let by_day = rule["byDay"].as_array().unwrap(); + assert_eq!(by_day[0]["day"], "mo"); + assert_eq!(by_day[0]["nthOfPeriod"], 1); + } + + #[test] + fn relative_yearly_last_friday_in_june() { + let raw = RawRecurrence { + pattern: Some(RecurrencePattern::RelativeYearly { + month: "June".to_owned(), + day_of_week_index: "Last".to_owned(), + days_of_week: vec!["Friday".to_owned()], + }), + range: Some(RecurrenceRange::NoEnd { + start_date: "2020-01-01".to_owned(), + }), + }; + let rule = to_jscalendar_rule(&raw).unwrap(); + assert_eq!(rule["frequency"], "yearly"); + assert_eq!(rule["byMonth"][0], "6"); + assert_eq!(rule["byDay"][0]["day"], "fr"); + assert_eq!(rule["byDay"][0]["nthOfPeriod"], -1); + } + + #[test] + fn absolute_monthly_translates_to_by_month_day() { + let raw = RawRecurrence { + pattern: Some(RecurrencePattern::AbsoluteMonthly { + interval: 2, + day_of_month: 15, + }), + range: Some(RecurrenceRange::NoEnd { + start_date: "2025-01-15".to_owned(), + }), + }; + let rule = to_jscalendar_rule(&raw).unwrap(); + assert_eq!(rule["frequency"], "monthly"); + assert_eq!(rule["interval"], 2); + assert_eq!(rule["byMonthDay"][0], 15); + } + + #[test] + fn absolute_yearly_translates_to_by_month_and_day() { + let raw = RawRecurrence { + pattern: Some(RecurrencePattern::AbsoluteYearly { + month: "January".to_owned(), + day_of_month: 1, + }), + range: Some(RecurrenceRange::NoEnd { + start_date: "2025-01-01".to_owned(), + }), + }; + let rule = to_jscalendar_rule(&raw).unwrap(); + assert_eq!(rule["frequency"], "yearly"); + assert_eq!(rule["byMonth"][0], "1"); + assert_eq!(rule["byMonthDay"][0], 1); + } + + #[test] + fn no_end_range_emits_no_until_or_count() { + let raw = RawRecurrence { + pattern: Some(RecurrencePattern::Daily { interval: 1 }), + range: Some(RecurrenceRange::NoEnd { + start_date: "2025-01-01".to_owned(), + }), + }; + let rule = to_jscalendar_rule(&raw).unwrap(); + assert!(rule.get("until").is_none()); + assert!(rule.get("count").is_none()); + } + + #[test] + fn weekly_two_days() { + let raw = RawRecurrence { + pattern: Some(RecurrencePattern::Weekly { + interval: 1, + days_of_week: vec!["Monday".to_owned(), "Wednesday".to_owned()], + }), + range: Some(RecurrenceRange::EndDate { + start_date: "2025-01-06".to_owned(), + end_date: "2025-06-30".to_owned(), + }), + }; + let rule = to_jscalendar_rule(&raw).unwrap(); + assert_eq!(rule["frequency"], "weekly"); + assert_eq!(rule["until"], "2025-06-30T23:59:59"); + let by_day = rule["byDay"].as_array().unwrap(); + assert_eq!(by_day.len(), 2); + assert_eq!(by_day[0]["day"], "mo"); + assert_eq!(by_day[1]["day"], "we"); + } +} diff --git a/src/exchange_ews/retry.rs b/src/exchange_ews/retry.rs new file mode 100644 index 0000000..33a7450 --- /dev/null +++ b/src/exchange_ews/retry.rs @@ -0,0 +1,157 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::time::Duration; + +use crate::exchange_ews::types::ResponseCode; +use crate::jmap::retry::Disposition; + +pub fn classify_http_status(status: u16) -> Disposition { + match status { + 429 | 502 | 503 | 504 => Disposition::Retryable, + _ => Disposition::Fatal, + } +} + +#[derive(Debug, Clone)] +pub enum FaultDisposition { + Fatal, + Auth, + Retryable { delay: Option }, +} + +pub fn classify_fault(code: &ResponseCode) -> FaultDisposition { + match code { + ResponseCode::ServerBusy { back_off_ms } => FaultDisposition::Retryable { + delay: back_off_ms.map(Duration::from_millis), + }, + ResponseCode::TimeoutExpired + | ResponseCode::InternalServerTransientError + | ResponseCode::MailboxStoreUnavailable + | ResponseCode::ConnectionFailed + | ResponseCode::AdUnavailable + | ResponseCode::BatchProcessingStopped => FaultDisposition::Retryable { delay: None }, + ResponseCode::AuthenticationRequired => FaultDisposition::Auth, + _ => FaultDisposition::Fatal, + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ItemAction { + Success, + Vanished, + Skip, + Retryable, + RetryDiscardSyncState, +} + +pub fn classify_item_response(code: &ResponseCode) -> ItemAction { + match code { + ResponseCode::NoError => ItemAction::Success, + ResponseCode::ItemNotFound | ResponseCode::AttachmentNotFound => ItemAction::Vanished, + ResponseCode::ServerBusy { .. } + | ResponseCode::TimeoutExpired + | ResponseCode::InternalServerTransientError + | ResponseCode::MailboxStoreUnavailable + | ResponseCode::ConnectionFailed + | ResponseCode::AdUnavailable + | ResponseCode::BatchProcessingStopped => ItemAction::Retryable, + ResponseCode::InvalidSyncStateData => ItemAction::RetryDiscardSyncState, + ResponseCode::AccessDenied + | ResponseCode::InvalidIdMalformed + | ResponseCode::MimeContentConversionFailed + | ResponseCode::ImpersonationFailed => ItemAction::Skip, + ResponseCode::AuthenticationRequired => ItemAction::Skip, + ResponseCode::Other(_) => ItemAction::Skip, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn server_busy_with_backoff_is_retryable_with_delay() { + let code = ResponseCode::ServerBusy { + back_off_ms: Some(297749), + }; + match classify_fault(&code) { + FaultDisposition::Retryable { delay: Some(d) } => { + assert_eq!(d, Duration::from_millis(297749)); + } + _ => panic!("expected retryable with delay"), + } + } + + #[test] + fn item_responses_are_classified() { + assert_eq!( + classify_item_response(&ResponseCode::NoError), + ItemAction::Success + ); + assert_eq!( + classify_item_response(&ResponseCode::ItemNotFound), + ItemAction::Vanished + ); + assert_eq!( + classify_item_response(&ResponseCode::AttachmentNotFound), + ItemAction::Vanished + ); + assert_eq!( + classify_item_response(&ResponseCode::AccessDenied), + ItemAction::Skip + ); + assert_eq!( + classify_item_response(&ResponseCode::InvalidSyncStateData), + ItemAction::RetryDiscardSyncState + ); + assert_eq!( + classify_item_response(&ResponseCode::ServerBusy { back_off_ms: None }), + ItemAction::Retryable + ); + } + + #[test] + fn http_status_table() { + assert_eq!(classify_http_status(429), Disposition::Retryable); + assert_eq!(classify_http_status(503), Disposition::Retryable); + assert_eq!(classify_http_status(500), Disposition::Fatal); + assert_eq!(classify_http_status(401), Disposition::Fatal); + } + + #[test] + fn transient_codes_are_retryable_at_item_and_fault_level() { + for code in [ + ResponseCode::InternalServerTransientError, + ResponseCode::MailboxStoreUnavailable, + ResponseCode::ConnectionFailed, + ResponseCode::AdUnavailable, + ResponseCode::BatchProcessingStopped, + ] { + assert_eq!(classify_item_response(&code), ItemAction::Retryable); + assert!(matches!( + classify_fault(&code), + FaultDisposition::Retryable { .. } + )); + } + } + + #[test] + fn mime_content_conversion_failure_skips_item() { + assert_eq!( + classify_item_response(&ResponseCode::MimeContentConversionFailed), + ItemAction::Skip + ); + } + + #[test] + fn other_unknown_codes_skip_at_item_level() { + assert_eq!( + classify_item_response(&ResponseCode::Other("ErrorWhoKnows".to_owned())), + ItemAction::Skip + ); + } +} diff --git a/src/exchange_ews/soap.rs b/src/exchange_ews/soap.rs new file mode 100644 index 0000000..765e239 --- /dev/null +++ b/src/exchange_ews/soap.rs @@ -0,0 +1,117 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::exchange_ews::types::ServerVersion; + +pub const NS_SOAP: &str = "http://schemas.xmlsoap.org/soap/envelope/"; +pub const NS_TYPES: &str = "http://schemas.microsoft.com/exchange/services/2006/types"; +pub const NS_MESSAGES: &str = "http://schemas.microsoft.com/exchange/services/2006/messages"; + +pub const SOAP_ACTION_BASE: &str = "http://schemas.microsoft.com/exchange/services/2006/messages"; + +#[derive(Debug, Clone, Copy)] +pub struct EnvelopeOptions<'a> { + pub version: ServerVersion, + pub impersonated_smtp: Option<&'a str>, +} + +impl Default for EnvelopeOptions<'_> { + fn default() -> Self { + EnvelopeOptions { + version: ServerVersion::Exchange2013Sp1, + impersonated_smtp: None, + } + } +} + +pub fn wrap_envelope(opts: EnvelopeOptions<'_>, body: &str) -> String { + let mut out = String::with_capacity(body.len() + 512); + out.push_str(""); + out.push_str(""); + out.push_str(""); + if let Some(smtp) = opts.impersonated_smtp { + out.push_str(""); + write_escaped(&mut out, smtp); + out.push_str(""); + } + out.push_str(""); + out.push_str(body); + out.push_str(""); + out +} + +pub fn soap_action(operation: &str) -> String { + format!("\"{SOAP_ACTION_BASE}/{operation}\"") +} + +pub fn write_escaped(out: &mut String, s: &str) { + for ch in s.chars() { + match ch { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\'' => out.push_str("'"), + _ => out.push(ch), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn envelope_has_canonical_namespaces() { + let env = wrap_envelope(EnvelopeOptions::default(), ""); + assert!(env.contains("xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\"")); + assert!( + env.contains("xmlns:t=\"http://schemas.microsoft.com/exchange/services/2006/types\"") + ); + assert!( + env.contains( + "xmlns:m=\"http://schemas.microsoft.com/exchange/services/2006/messages\"" + ) + ); + assert!(env.contains("")); + assert!(env.contains("")); + } + + #[test] + fn envelope_impersonation_includes_smtp() { + let opts = EnvelopeOptions { + version: ServerVersion::Exchange2016, + impersonated_smtp: Some("alice@contoso.com"), + }; + let env = wrap_envelope(opts, ""); + assert!(env.contains("")); + assert!(env.contains("alice@contoso.com")); + assert!(env.contains("Version=\"Exchange2016\"")); + } + + #[test] + fn soap_action_quotes_operation() { + assert_eq!( + soap_action("FindItem"), + "\"http://schemas.microsoft.com/exchange/services/2006/messages/FindItem\"" + ); + } + + #[test] + fn write_escaped_handles_metachars() { + let mut s = String::new(); + write_escaped(&mut s, ""); + assert_eq!(s, "<a&b>"); + } +} diff --git a/src/exchange_ews/types.rs b/src/exchange_ews/types.rs new file mode 100644 index 0000000..f2fb225 --- /dev/null +++ b/src/exchange_ews/types.rs @@ -0,0 +1,314 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fmt; + +#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] +pub struct ItemId { + pub id: String, + pub change_key: String, +} + +impl ItemId { + pub fn new(id: impl Into, change_key: impl Into) -> ItemId { + ItemId { + id: id.into(), + change_key: change_key.into(), + } + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] +pub struct FolderId { + pub id: String, + pub change_key: String, +} + +impl FolderId { + pub fn new(id: impl Into, change_key: impl Into) -> FolderId { + FolderId { + id: id.into(), + change_key: change_key.into(), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MailboxKind { + Primary, + Archive, + PublicFolders, +} + +impl MailboxKind { + pub fn as_str(self) -> &'static str { + match self { + MailboxKind::Primary => "primary", + MailboxKind::Archive => "archive", + MailboxKind::PublicFolders => "public-folders", + } + } + + pub fn distinguished_root(self) -> DistinguishedFolderId { + match self { + MailboxKind::Primary => DistinguishedFolderId::MsgFolderRoot, + MailboxKind::Archive => DistinguishedFolderId::ArchiveRoot, + MailboxKind::PublicFolders => DistinguishedFolderId::PublicFoldersRoot, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DistinguishedFolderId { + MsgFolderRoot, + ArchiveRoot, + PublicFoldersRoot, + Inbox, + SentItems, + Drafts, + DeletedItems, + JunkEmail, + Outbox, + ConversationHistory, + Archive, +} + +impl DistinguishedFolderId { + pub fn as_str(self) -> &'static str { + match self { + DistinguishedFolderId::MsgFolderRoot => "msgfolderroot", + DistinguishedFolderId::ArchiveRoot => "archiveroot", + DistinguishedFolderId::PublicFoldersRoot => "publicfoldersroot", + DistinguishedFolderId::Inbox => "inbox", + DistinguishedFolderId::SentItems => "sentitems", + DistinguishedFolderId::Drafts => "drafts", + DistinguishedFolderId::DeletedItems => "deleteditems", + DistinguishedFolderId::JunkEmail => "junkemail", + DistinguishedFolderId::Outbox => "outbox", + DistinguishedFolderId::ConversationHistory => "conversationhistory", + DistinguishedFolderId::Archive => "archive", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ServerVersion { + Exchange2007, + Exchange2010, + Exchange2010Sp1, + Exchange2010Sp2, + Exchange2013, + Exchange2013Sp1, + Exchange2016, + Exchange2019, +} + +impl ServerVersion { + pub fn as_str(self) -> &'static str { + match self { + ServerVersion::Exchange2007 => "Exchange2007", + ServerVersion::Exchange2010 => "Exchange2010", + ServerVersion::Exchange2010Sp1 => "Exchange2010_SP1", + ServerVersion::Exchange2010Sp2 => "Exchange2010_SP2", + ServerVersion::Exchange2013 => "Exchange2013", + ServerVersion::Exchange2013Sp1 => "Exchange2013_SP1", + ServerVersion::Exchange2016 => "Exchange2016", + ServerVersion::Exchange2019 => "Exchange2019", + } + } + + pub fn from_build(major: u32, minor: u32) -> ServerVersion { + match (major, minor) { + (8, _) => ServerVersion::Exchange2007, + (14, 0) => ServerVersion::Exchange2010, + (14, 1) => ServerVersion::Exchange2010Sp1, + (14, 2 | 3) => ServerVersion::Exchange2010Sp2, + (15, 0) => ServerVersion::Exchange2013, + (15, 1) => ServerVersion::Exchange2016, + (15, 2) => ServerVersion::Exchange2019, + _ if major >= 15 => ServerVersion::Exchange2019, + _ => ServerVersion::Exchange2016, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FolderClass { + Mail, + Calendar, + Contacts, + Skipped, +} + +impl FolderClass { + pub fn from_ipf(class: &str) -> FolderClass { + if class.eq_ignore_ascii_case("IPF.Note") { + FolderClass::Mail + } else if class.eq_ignore_ascii_case("IPF.Appointment") + || class.starts_with("IPF.Appointment.") + { + FolderClass::Calendar + } else if class.eq_ignore_ascii_case("IPF.Contact") || class.starts_with("IPF.Contact.") { + FolderClass::Contacts + } else { + FolderClass::Skipped + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CalendarItemType { + Single, + RecurringMaster, + Occurrence, + Exception, +} + +impl CalendarItemType { + pub fn parse(value: &str) -> Option { + match value { + "Single" => Some(CalendarItemType::Single), + "RecurringMaster" => Some(CalendarItemType::RecurringMaster), + "Occurrence" => Some(CalendarItemType::Occurrence), + "Exception" => Some(CalendarItemType::Exception), + _ => None, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ResponseCode { + NoError, + ItemNotFound, + AccessDenied, + InvalidIdMalformed, + ServerBusy { back_off_ms: Option }, + TimeoutExpired, + InvalidSyncStateData, + MimeContentConversionFailed, + AttachmentNotFound, + AuthenticationRequired, + ImpersonationFailed, + BatchProcessingStopped, + InternalServerTransientError, + MailboxStoreUnavailable, + ConnectionFailed, + AdUnavailable, + Other(String), +} + +impl ResponseCode { + pub fn parse(code: &str) -> ResponseCode { + match code { + "NoError" => ResponseCode::NoError, + "ErrorItemNotFound" => ResponseCode::ItemNotFound, + "ErrorAccessDenied" => ResponseCode::AccessDenied, + "ErrorInvalidIdMalformed" => ResponseCode::InvalidIdMalformed, + "ErrorServerBusy" => ResponseCode::ServerBusy { back_off_ms: None }, + "ErrorTimeoutExpired" => ResponseCode::TimeoutExpired, + "ErrorInvalidSyncStateData" => ResponseCode::InvalidSyncStateData, + "ErrorMimeContentConversionFailed" => ResponseCode::MimeContentConversionFailed, + "ErrorAttachmentNotFound" => ResponseCode::AttachmentNotFound, + "ErrorAuthenticationRequired" => ResponseCode::AuthenticationRequired, + "ErrorImpersonationFailed" => ResponseCode::ImpersonationFailed, + "ErrorBatchProcessingStopped" => ResponseCode::BatchProcessingStopped, + "ErrorInternalServerTransientError" => ResponseCode::InternalServerTransientError, + "ErrorMailboxStoreUnavailable" => ResponseCode::MailboxStoreUnavailable, + "ErrorConnectionFailed" => ResponseCode::ConnectionFailed, + "ErrorADUnavailable" => ResponseCode::AdUnavailable, + other => ResponseCode::Other(other.to_owned()), + } + } + + pub fn as_wire(&self) -> &str { + match self { + ResponseCode::NoError => "NoError", + ResponseCode::ItemNotFound => "ErrorItemNotFound", + ResponseCode::AccessDenied => "ErrorAccessDenied", + ResponseCode::InvalidIdMalformed => "ErrorInvalidIdMalformed", + ResponseCode::ServerBusy { .. } => "ErrorServerBusy", + ResponseCode::TimeoutExpired => "ErrorTimeoutExpired", + ResponseCode::InvalidSyncStateData => "ErrorInvalidSyncStateData", + ResponseCode::MimeContentConversionFailed => "ErrorMimeContentConversionFailed", + ResponseCode::AttachmentNotFound => "ErrorAttachmentNotFound", + ResponseCode::AuthenticationRequired => "ErrorAuthenticationRequired", + ResponseCode::ImpersonationFailed => "ErrorImpersonationFailed", + ResponseCode::BatchProcessingStopped => "ErrorBatchProcessingStopped", + ResponseCode::InternalServerTransientError => "ErrorInternalServerTransientError", + ResponseCode::MailboxStoreUnavailable => "ErrorMailboxStoreUnavailable", + ResponseCode::ConnectionFailed => "ErrorConnectionFailed", + ResponseCode::AdUnavailable => "ErrorADUnavailable", + ResponseCode::Other(s) => s.as_str(), + } + } +} + +impl fmt::Display for ResponseCode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_wire()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn folder_class_mapping() { + assert_eq!(FolderClass::from_ipf("IPF.Note"), FolderClass::Mail); + assert_eq!( + FolderClass::from_ipf("IPF.Appointment"), + FolderClass::Calendar + ); + assert_eq!( + FolderClass::from_ipf("IPF.Appointment.Birthday"), + FolderClass::Calendar + ); + assert_eq!(FolderClass::from_ipf("IPF.Contact"), FolderClass::Contacts); + assert_eq!(FolderClass::from_ipf("IPF.Task"), FolderClass::Skipped); + } + + #[test] + fn server_version_from_build() { + assert_eq!( + ServerVersion::from_build(15, 0), + ServerVersion::Exchange2013 + ); + assert_eq!( + ServerVersion::from_build(15, 1), + ServerVersion::Exchange2016 + ); + assert_eq!( + ServerVersion::from_build(15, 2), + ServerVersion::Exchange2019 + ); + assert_eq!( + ServerVersion::from_build(99, 99), + ServerVersion::Exchange2019 + ); + } + + #[test] + fn response_code_round_trip() { + let codes = [ + "NoError", + "ErrorItemNotFound", + "ErrorAccessDenied", + "ErrorInvalidIdMalformed", + "ErrorServerBusy", + "ErrorTimeoutExpired", + "ErrorInvalidSyncStateData", + "ErrorMimeContentConversionFailed", + "ErrorAttachmentNotFound", + "ErrorAuthenticationRequired", + "ErrorImpersonationFailed", + "ErrorWhatever", + ]; + for c in codes { + assert_eq!(ResponseCode::parse(c).as_wire(), c); + } + } +} diff --git a/src/exchange_ews/xml.rs b/src/exchange_ews/xml.rs new file mode 100644 index 0000000..b7ac5b7 --- /dev/null +++ b/src/exchange_ews/xml.rs @@ -0,0 +1,340 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::exchange_ews::soap::write_escaped; +use crate::exchange_ews::types::{DistinguishedFolderId, FolderId, ItemId}; + +#[derive(Debug, Clone, Copy)] +pub enum FolderRef<'a> { + Distinguished(DistinguishedFolderId), + Concrete(&'a FolderId), +} + +#[derive(Debug, Clone, Copy)] +pub enum FolderShape { + IdOnly, +} + +impl FolderShape { + fn as_str(self) -> &'static str { + match self { + FolderShape::IdOnly => "IdOnly", + } + } +} + +#[derive(Debug, Clone, Copy)] +pub enum Traversal { + Shallow, + Deep, +} + +impl Traversal { + fn as_str(self) -> &'static str { + match self { + Traversal::Shallow => "Shallow", + Traversal::Deep => "Deep", + } + } +} + +pub fn find_folder_body(parent: FolderRef<'_>, traversal: Traversal) -> String { + let mut out = String::with_capacity(512); + out.push_str(""); + out.push_str("Default"); + out.push_str(""); + out.push_str(""); + out.push_str(""); + out.push_str(""); + out.push_str(""); + out.push_str(""); + out.push_str(""); + out.push_str(""); + write_folder_ref(&mut out, parent); + out.push_str(""); + out +} + +pub fn get_folder_body(folders: &[FolderRef<'_>], shape: FolderShape) -> String { + let mut out = String::with_capacity(256 + folders.len() * 64); + out.push_str(""); + out.push_str(""); + out.push_str(shape.as_str()); + out.push_str(""); + out.push_str(""); + for f in folders { + write_folder_ref(&mut out, *f); + } + out.push_str(""); + out +} + +pub fn find_item_body( + parent: FolderRef<'_>, + traversal: Traversal, + offset: u32, + page_size: u32, +) -> String { + let mut out = String::with_capacity(512); + out.push_str("IdOnly"); + out.push_str(""); + out.push_str(""); + write_folder_ref(&mut out, parent); + out.push_str(""); + out +} + +pub fn sync_folder_items_body(folder: &FolderId, sync_state: &str, max_changes: u32) -> String { + let mut out = String::with_capacity(512); + out.push_str(""); + out.push_str("IdOnly"); + out.push_str(""); + write_folder_id_element(&mut out, folder); + out.push_str(""); + out.push_str(""); + write_escaped(&mut out, sync_state); + out.push_str(""); + out.push_str(""); + out.push_str(&max_changes.to_string()); + out.push_str(""); + out.push_str("NormalItems"); + out.push_str(""); + out +} + +#[derive(Debug, Clone, Copy)] +pub enum ItemShape { + Message, + CalendarItem, + Contact, +} + +pub fn get_item_body(shape: ItemShape, ids: &[ItemId]) -> String { + let mut out = String::with_capacity(256 + ids.len() * 128); + out.push_str(""); + match shape { + ItemShape::Message => { + out.push_str("IdOnly"); + out.push_str("true"); + out.push_str("Best"); + out.push_str(""); + out.push_str(""); + out.push_str(""); + out.push_str(""); + out.push_str(""); + out.push_str(""); + out.push_str(""); + out.push_str(""); + out.push_str(""); + } + ItemShape::CalendarItem => { + out.push_str("IdOnly"); + out.push_str("Best"); + out.push_str(""); + for f in CALENDAR_FIELDS { + out.push_str(""); + } + out.push_str(""); + } + ItemShape::Contact => { + out.push_str("AllProperties"); + } + } + out.push_str(""); + for id in ids { + write_item_id_element(&mut out, id); + } + out.push_str(""); + out +} + +const CALENDAR_FIELDS: &[&str] = &[ + "calendar:UID", + "calendar:RecurrenceId", + "calendar:Start", + "calendar:End", + "calendar:OriginalStart", + "calendar:IsAllDayEvent", + "calendar:LegacyFreeBusyStatus", + "calendar:Location", + "calendar:IsRecurring", + "calendar:CalendarItemType", + "calendar:Organizer", + "calendar:RequiredAttendees", + "calendar:OptionalAttendees", + "calendar:Recurrence", + "calendar:ModifiedOccurrences", + "calendar:DeletedOccurrences", + "calendar:StartTimeZone", + "calendar:EndTimeZone", + "calendar:Duration", + "item:Subject", + "item:Body", + "item:Categories", + "item:DateTimeCreated", + "item:LastModifiedTime", + "item:ParentFolderId", +]; + +pub fn get_attachment_body(ids: &[&str]) -> String { + let mut out = String::with_capacity(256 + ids.len() * 64); + out.push_str(""); + out.push_str("true"); + out.push_str("Best"); + out.push_str(""); + for id in ids { + out.push_str(""); + } + out.push_str(""); + out +} + +pub fn write_folder_id_element(out: &mut String, folder: &FolderId) { + out.push_str(""); +} + +pub fn write_item_id_element(out: &mut String, id: &ItemId) { + out.push_str(""); +} + +fn write_folder_ref(out: &mut String, parent: FolderRef<'_>) { + match parent { + FolderRef::Distinguished(d) => { + out.push_str(""); + } + FolderRef::Concrete(f) => write_folder_id_element(out, f), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn find_folder_uses_deep_traversal_for_msgfolderroot() { + let body = find_folder_body( + FolderRef::Distinguished(DistinguishedFolderId::MsgFolderRoot), + Traversal::Deep, + ); + assert!(body.contains("Traversal=\"Deep\"")); + assert!(body.contains("")); + assert!(body.contains("folder:FolderClass")); + assert!(body.contains("folder:DisplayName")); + assert!(body.contains("folder:ParentFolderId")); + } + + #[test] + fn find_folder_archive_uses_archiveroot() { + let body = find_folder_body( + FolderRef::Distinguished(DistinguishedFolderId::ArchiveRoot), + Traversal::Deep, + ); + assert!(body.contains("")); + } + + #[test] + fn find_item_paginates_with_offset_and_page_size() { + let folder = FolderId::new("FID", "FCK"); + let body = find_item_body(FolderRef::Concrete(&folder), Traversal::Shallow, 200, 50); + assert!(body.contains("Offset=\"200\"")); + assert!(body.contains("MaxEntriesReturned=\"50\"")); + assert!(body.contains("")); + } + + #[test] + fn get_item_message_shape_requests_mime() { + let ids = vec![ItemId::new("I1", "CK1"), ItemId::new("I2", "")]; + let body = get_item_body(ItemShape::Message, &ids); + assert!(body.contains("true")); + assert!(body.contains("")); + assert!(body.contains("")); + } + + #[test] + fn get_item_calendar_shape_lists_calendar_fields() { + let body = get_item_body(ItemShape::CalendarItem, &[ItemId::new("X", "")]); + assert!(body.contains("calendar:Recurrence")); + assert!(body.contains("calendar:ModifiedOccurrences")); + assert!(body.contains("calendar:DeletedOccurrences")); + assert!(body.contains("calendar:UID")); + assert!(!body.contains("")); + } + + #[test] + fn get_item_contact_shape_requests_all_properties() { + let body = get_item_body(ItemShape::Contact, &[ItemId::new("X", "")]); + assert!(body.contains("AllProperties")); + } + + #[test] + fn sync_folder_items_body_carries_state_and_max() { + let folder = FolderId::new("FID", "FCK"); + let body = sync_folder_items_body(&folder, "STATE", 512); + assert!(body.contains("")); + assert!(body.contains("STATE")); + assert!(body.contains("512")); + assert!(body.contains("NormalItems")); + } + + #[test] + fn sync_folder_items_empty_state_round_trips() { + let folder = FolderId::new("FID", ""); + let body = sync_folder_items_body(&folder, "", 100); + assert!(body.contains("")); + } + + #[test] + fn get_attachment_lists_ids() { + let body = get_attachment_body(&["AID1", "AID2"]); + assert!(body.contains("")); + assert!(body.contains("")); + assert!(body.contains("true")); + } + + #[test] + fn get_folder_emits_distinguished_ids() { + let body = get_folder_body( + &[ + FolderRef::Distinguished(DistinguishedFolderId::Inbox), + FolderRef::Distinguished(DistinguishedFolderId::SentItems), + ], + FolderShape::IdOnly, + ); + assert!(body.contains("")); + assert!(body.contains("")); + assert!(body.contains("IdOnly")); + } +} diff --git a/src/exchange_graph/api.rs b/src/exchange_graph/api.rs new file mode 100644 index 0000000..a7d8d2f --- /dev/null +++ b/src/exchange_graph/api.rs @@ -0,0 +1,286 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::Value; + +use crate::exchange_graph::client::GraphClient; +use crate::exchange_graph::error::GraphError; +use crate::exchange_graph::types::MailboxKind; + +pub const DEFAULT_API_BASE: &str = "https://graph.microsoft.com/v1.0"; + +pub const PREFER_TIMEZONE_UTC: &str = "outlook.timezone=\"UTC\""; +pub const PREFER_BODY_TEXT: &str = "outlook.body-content-type=\"text\""; +pub const PREFER_BODY_HTML: &str = "outlook.body-content-type=\"html\""; + +#[derive(Debug, Clone)] +pub struct Endpoints { + pub api_base: String, + pub user_path: String, +} + +impl Endpoints { + pub fn for_me(api_base: &str) -> Endpoints { + Endpoints { + api_base: api_base.trim_end_matches('/').to_owned(), + user_path: "/me".to_owned(), + } + } + + pub fn for_user(api_base: &str, user_id: &str) -> Endpoints { + Endpoints { + api_base: api_base.trim_end_matches('/').to_owned(), + user_path: format!("/users/{user_id}"), + } + } + + pub fn me_or_user(&self) -> String { + format!("{}{}", self.api_base, self.user_path) + } + + pub fn mail_folders_root(&self, kind: MailboxKind, top: usize) -> String { + match kind { + MailboxKind::Primary => { + format!( + "{}/mailFolders?$top={top}&includeHiddenFolders=true", + self.me_or_user() + ) + } + MailboxKind::Archive => { + format!( + "{}/mailFolders/archive/childFolders?$top={top}&includeHiddenFolders=true", + self.me_or_user() + ) + } + } + } + + pub fn mail_folder_child_folders(&self, folder_id: &str, top: usize) -> String { + format!( + "{}/mailFolders/{}/childFolders?$top={top}&includeHiddenFolders=true", + self.me_or_user(), + url_escape(folder_id) + ) + } + + pub fn folder_messages_ids(&self, folder_id: &str, top: usize) -> String { + format!( + "{}/mailFolders/{}/messages?$top={top}&$select=id", + self.me_or_user(), + url_escape(folder_id) + ) + } + + pub fn message_mime(&self, message_id: &str) -> String { + format!( + "{}/messages/{}/$value", + self.me_or_user(), + url_escape(message_id) + ) + } + + pub fn well_known_folder(&self, well_known: &str) -> String { + format!( + "{}/mailFolders/{}?$select=id", + self.me_or_user(), + well_known + ) + } + + pub fn calendars(&self, top: usize) -> String { + format!("{}/calendars?$top={top}", self.me_or_user()) + } + + pub fn calendar_events_ids(&self, calendar_id: &str, top: usize) -> String { + format!( + "{}/calendars/{}/events?$top={top}&$select=id,type,seriesMasterId", + self.me_or_user(), + url_escape(calendar_id) + ) + } + + pub fn event(&self, event_id: &str) -> String { + format!("{}/events/{}", self.me_or_user(), url_escape(event_id)) + } + + pub fn contact_folders(&self, top: usize) -> String { + format!("{}/contactFolders?$top={top}", self.me_or_user()) + } + + pub fn contact_folder_children(&self, folder_id: &str, top: usize) -> String { + format!( + "{}/contactFolders/{}/childFolders?$top={top}", + self.me_or_user(), + url_escape(folder_id) + ) + } + + pub fn contact_folder_contacts_ids(&self, folder_id: &str, top: usize) -> String { + format!( + "{}/contactFolders/{}/contacts?$top={top}&$select=id", + self.me_or_user(), + url_escape(folder_id) + ) + } + + pub fn contact(&self, contact_id: &str) -> String { + format!("{}/contacts/{}", self.me_or_user(), url_escape(contact_id)) + } + + pub fn mailbox_settings_timezone(&self) -> String { + format!("{}/mailboxSettings?$select=timeZone", self.me_or_user()) + } + + pub fn me_select_id_upn(&self) -> String { + format!( + "{}?$select=id,userPrincipalName,displayName,mail", + self.me_or_user() + ) + } +} + +fn url_escape(segment: &str) -> String { + let mut out = String::with_capacity(segment.len()); + for b in segment.bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(b as char); + } + _ => { + use std::fmt::Write; + let _ = write!(out, "%{b:02X}"); + } + } + } + out +} + +pub fn paged_collect( + client: &GraphClient, + initial_url: &str, + prefer: &[&str], + mut on_page: F, +) -> Result<(), GraphError> +where + F: FnMut(&Value) -> Result<(), GraphError>, +{ + let mut url = initial_url.to_owned(); + loop { + let body = client.get_json_with_prefer(&url, prefer)?; + on_page(&body)?; + match body.get("@odata.nextLink").and_then(Value::as_str) { + Some(next) => url = next.to_owned(), + None => return Ok(()), + } + } +} + +pub fn collect_all_ids( + client: &GraphClient, + initial_url: &str, + prefer: &[&str], +) -> Result, GraphError> { + let mut out = Vec::new(); + paged_collect(client, initial_url, prefer, |page| { + if let Some(arr) = page.get("value").and_then(Value::as_array) { + for item in arr { + if let Some(id) = item.get("id").and_then(Value::as_str) { + out.push(id.to_owned()); + } + } + } + Ok(()) + })?; + Ok(out) +} + +pub fn collect_all_values( + client: &GraphClient, + initial_url: &str, + prefer: &[&str], +) -> Result, GraphError> { + let mut out = Vec::new(); + paged_collect(client, initial_url, prefer, |page| { + if let Some(arr) = page.get("value").and_then(Value::as_array) { + for item in arr { + out.push(item.clone()); + } + } + Ok(()) + })?; + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn primary_mail_folders_root_uses_me_with_hidden() { + let e = Endpoints::for_me(DEFAULT_API_BASE); + let url = e.mail_folders_root(MailboxKind::Primary, 100); + assert!(url.starts_with("https://graph.microsoft.com/v1.0/me/mailFolders?")); + assert!(url.contains("includeHiddenFolders=true")); + assert!(url.contains("$top=100")); + } + + #[test] + fn archive_mail_folders_root_uses_archive_child_folders() { + let e = Endpoints::for_me(DEFAULT_API_BASE); + let url = e.mail_folders_root(MailboxKind::Archive, 50); + assert!( + url.starts_with( + "https://graph.microsoft.com/v1.0/me/mailFolders/archive/childFolders?" + ) + ); + assert!(url.contains("$top=50")); + } + + #[test] + fn for_user_replaces_me_segment() { + let e = Endpoints::for_user(DEFAULT_API_BASE, "uid"); + let url = e.mail_folders_root(MailboxKind::Primary, 100); + assert!(url.contains("/users/uid/mailFolders")); + assert!(!url.contains("/me/")); + } + + #[test] + fn top_threads_through_calendar_and_contact_endpoints() { + let e = Endpoints::for_me(DEFAULT_API_BASE); + assert!(e.calendars(75).contains("$top=75")); + assert!(e.contact_folders(75).contains("$top=75")); + assert!(e.contact_folder_children("F", 75).contains("$top=75")); + assert!(e.mail_folder_child_folders("F", 75).contains("$top=75")); + } + + #[test] + fn url_escape_encodes_unsafe_bytes_only() { + assert_eq!(url_escape("AAkAAA="), "AAkAAA%3D"); + assert_eq!(url_escape("plain.id_123-x"), "plain.id_123-x"); + } + + #[test] + fn message_mime_endpoint_uses_value() { + let e = Endpoints::for_me(DEFAULT_API_BASE); + let url = e.message_mime("AAkA=="); + assert!(url.ends_with("/messages/AAkA%3D%3D/$value")); + } + + #[test] + fn well_known_folder_uses_short_name() { + let e = Endpoints::for_me(DEFAULT_API_BASE); + let url = e.well_known_folder("inbox"); + assert!(url.ends_with("/mailFolders/inbox?$select=id")); + } + + #[test] + fn event_ids_endpoint_requests_routing_fields() { + let e = Endpoints::for_me(DEFAULT_API_BASE); + let url = e.calendar_events_ids("CAL", 100); + assert!(url.contains("$select=id,type,seriesMasterId")); + assert!(url.contains("$top=100")); + } +} diff --git a/src/exchange_graph/calendar_map.rs b/src/exchange_graph/calendar_map.rs new file mode 100644 index 0000000..9cfdd0e --- /dev/null +++ b/src/exchange_graph/calendar_map.rs @@ -0,0 +1,735 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::{Map, Value, json}; + +use crate::exchange::tz::windows_to_iana; +use crate::exchange_graph::error::GraphError; +use crate::exchange_graph::recurrence::convert_patterned_recurrence_rule; + +pub fn graph_calendar_color_to_hex(value: &str) -> Option<&'static str> { + match value { + "lightBlue" => Some("#A6CEE3"), + "lightGreen" => Some("#B2DF8A"), + "lightOrange" => Some("#FDBF6F"), + "lightGray" => Some("#CCCCCC"), + "lightYellow" => Some("#FFFF99"), + "lightTeal" => Some("#A0E7E5"), + "lightPink" => Some("#FB9A99"), + "lightBrown" => Some("#B15928"), + "lightRed" => Some("#FB9A99"), + _ => None, + } +} + +pub fn windows_or_iana_to_iana(value: &str) -> Option { + if value.is_empty() { + return None; + } + if let Some(iana) = windows_to_iana(value) { + return Some(iana.to_owned()); + } + if value.contains('/') { + return Some(value.to_owned()); + } + None +} + +#[derive(Debug, Clone)] +pub struct ConvertedEvent { + pub uid: String, + pub data: Value, + pub is_draft: bool, + pub use_default_alerts: bool, + pub series_master_id: Option, + pub event_type: EventType, + pub original_start: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EventType { + SingleInstance, + SeriesMaster, + Occurrence, + Exception, +} + +pub fn classify_event_type(value: &Value) -> EventType { + match value.get("type").and_then(Value::as_str).unwrap_or("") { + "seriesMaster" => EventType::SeriesMaster, + "occurrence" => EventType::Occurrence, + "exception" => EventType::Exception, + _ => EventType::SingleInstance, + } +} + +pub fn convert_event( + graph_event: &Value, + fallback_calendar_tz: Option<&str>, +) -> Result { + let uid = graph_event + .get("iCalUId") + .and_then(Value::as_str) + .or_else(|| graph_event.get("id").and_then(Value::as_str)) + .ok_or_else(|| GraphError::Malformed("event has neither iCalUId nor id".to_owned()))? + .to_owned(); + + let event_type = classify_event_type(graph_event); + let series_master_id = graph_event + .get("seriesMasterId") + .and_then(Value::as_str) + .map(str::to_owned); + let original_start = graph_event + .get("originalStart") + .and_then(Value::as_str) + .map(str::to_owned); + let is_draft = graph_event + .get("isDraft") + .and_then(Value::as_bool) + .unwrap_or(false); + + let mut card = Map::new(); + card.insert("@type".to_owned(), Value::from("Event")); + card.insert("version".to_owned(), Value::from("2.0")); + card.insert("uid".to_owned(), Value::from(uid.clone())); + + if let Some(subject) = graph_event.get("subject").and_then(Value::as_str) + && !subject.is_empty() + { + card.insert("title".to_owned(), Value::from(subject.to_owned())); + } + + if let Some(body) = graph_event.get("body").and_then(Value::as_object) + && let Some(content) = body.get("content").and_then(Value::as_str) + && !content.is_empty() + { + card.insert("description".to_owned(), Value::from(content.to_owned())); + } + + let start_dt = extract_local_datetime(graph_event.get("start")).map(strip_fractional); + let end_dt = extract_local_datetime(graph_event.get("end")).map(strip_fractional); + + if let Some(start) = start_dt.as_deref() { + card.insert("start".to_owned(), Value::from(start.to_owned())); + } + + if let (Some(start), Some(end)) = (start_dt.as_deref(), end_dt.as_deref()) + && let Some(dur) = duration_iso8601(start, end) + { + card.insert("duration".to_owned(), Value::from(dur)); + } + + let event_tz = graph_event + .get("originalStartTimeZone") + .and_then(Value::as_str) + .and_then(windows_or_iana_to_iana) + .or_else(|| { + graph_event + .get("start") + .and_then(|s| s.get("timeZone")) + .and_then(Value::as_str) + .and_then(windows_or_iana_to_iana) + }) + .or_else(|| fallback_calendar_tz.map(str::to_owned)); + if let Some(tz) = event_tz { + card.insert("timeZone".to_owned(), Value::from(tz)); + } + + if let Some(all_day) = graph_event.get("isAllDay").and_then(Value::as_bool) + && all_day + { + card.insert("showWithoutTime".to_owned(), Value::Bool(true)); + } + + if let Some(true) = graph_event.get("isCancelled").and_then(Value::as_bool) { + card.insert("status".to_owned(), Value::from("cancelled")); + } else { + card.insert("status".to_owned(), Value::from("confirmed")); + } + + if let Some(sens) = graph_event.get("sensitivity").and_then(Value::as_str) { + card.insert("privacy".to_owned(), Value::from(privacy_for(sens))); + } + + if let Some(imp) = graph_event.get("importance").and_then(Value::as_str) { + card.insert("priority".to_owned(), Value::from(priority_for(imp))); + } + + if let Some(show) = graph_event.get("showAs").and_then(Value::as_str) { + card.insert( + "freeBusyStatus".to_owned(), + Value::from(free_busy_for(show)), + ); + } + + if let Some(cats) = graph_event.get("categories").and_then(Value::as_array) + && !cats.is_empty() + { + let mut map = Map::new(); + for cat in cats.iter().filter_map(Value::as_str) { + map.insert(cat.to_owned(), Value::Bool(true)); + } + if !map.is_empty() { + card.insert("keywords".to_owned(), Value::Object(map)); + } + } + + if let Some(locs) = graph_event.get("locations").and_then(Value::as_array) + && !locs.is_empty() + { + let mut locations = Map::new(); + let width = pad_width(locs.len()); + for (i, loc) in locs.iter().enumerate() { + let key = format!("loc-{:0width$}", i + 1, width = width); + let mut object = Map::new(); + object.insert("@type".to_owned(), Value::from("Location")); + let display_name = loc + .get("displayName") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()); + let address_str = loc + .get("address") + .and_then(Value::as_object) + .map(|addr| { + ["street", "city", "state", "postalCode", "countryOrRegion"] + .iter() + .filter_map(|f| addr.get(*f).and_then(Value::as_str)) + .filter(|s| !s.is_empty()) + .collect::>() + .join(", ") + }) + .filter(|s| !s.is_empty()); + let name = match (display_name, address_str.as_deref()) { + (Some(dn), Some(addr)) if dn != addr => Some(format!("{dn}, {addr}")), + (Some(dn), _) => Some(dn.to_owned()), + (None, Some(addr)) => Some(addr.to_owned()), + (None, None) => None, + }; + if let Some(n) = name { + object.insert("name".to_owned(), Value::from(n)); + } + locations.insert(key, Value::Object(object)); + } + if !locations.is_empty() { + card.insert("locations".to_owned(), Value::Object(locations)); + } + } + + if let Some(recurrence) = graph_event.get("recurrence") + && !recurrence.is_null() + { + let rule = convert_patterned_recurrence_rule(recurrence)?; + card.insert("recurrenceRule".to_owned(), Value::Object(rule)); + } + + let mut participants: Vec<(String, Value)> = Vec::new(); + if let Some(org) = graph_event + .get("organizer") + .and_then(|o| o.get("emailAddress")) + { + if let Some(addr) = org.get("address").and_then(Value::as_str) + && !addr.is_empty() + { + card.insert( + "organizerCalendarAddress".to_owned(), + Value::from(format!("mailto:{addr}")), + ); + } + let p = build_participant(org, &["owner", "chair"], None, None); + participants.push(("organizer".to_owned(), p)); + } + if let Some(attendees) = graph_event.get("attendees").and_then(Value::as_array) { + let width = pad_width(attendees.len()); + for (i, att) in attendees.iter().enumerate() { + let key = format!("att-{:0width$}", i + 1, width = width); + let Some(email) = att.get("emailAddress") else { + continue; + }; + let role = att + .get("type") + .and_then(Value::as_str) + .map(attendee_type_to_role) + .unwrap_or("required"); + let status = att + .get("status") + .and_then(|s| s.get("response")) + .and_then(Value::as_str) + .and_then(attendee_response_to_status); + let p = build_participant(email, &[role], status, Some(true)); + participants.push((key, p)); + } + } + if !participants.is_empty() { + let map: Map = participants.into_iter().collect(); + card.insert("participants".to_owned(), Value::Object(map)); + } + + if let Some(created) = graph_event.get("createdDateTime").and_then(Value::as_str) { + card.insert( + "created".to_owned(), + Value::from(strip_fractional_utc(created)), + ); + } + if let Some(updated) = graph_event + .get("lastModifiedDateTime") + .and_then(Value::as_str) + { + card.insert( + "updated".to_owned(), + Value::from(strip_fractional_utc(updated)), + ); + } + + let use_default_alerts = false; + if let Some(true) = graph_event.get("isReminderOn").and_then(Value::as_bool) + && let Some(mins) = graph_event + .get("reminderMinutesBeforeStart") + .and_then(Value::as_i64) + { + let trigger = format!("-PT{}M", mins.unsigned_abs()); + let alert = json!({ + "@type": "Alert", + "trigger": { + "@type": "OffsetTrigger", + "relativeTo": "start", + "offset": trigger + } + }); + let mut alerts = Map::new(); + alerts.insert("alert-1".to_owned(), alert); + card.insert("alerts".to_owned(), Value::Object(alerts)); + } + + if let Some(cancels) = graph_event + .get("cancelledOccurrences") + .and_then(Value::as_array) + && !cancels.is_empty() + { + let mut overrides = Map::new(); + for entry in cancels.iter().filter_map(Value::as_str) { + overrides.insert(entry.to_owned(), json!({"excluded": true})); + } + if !overrides.is_empty() { + card.insert("recurrenceOverrides".to_owned(), Value::Object(overrides)); + } + } + + Ok(ConvertedEvent { + uid, + data: Value::Object(card), + is_draft, + use_default_alerts, + series_master_id, + event_type, + original_start, + }) +} + +fn extract_local_datetime(slot: Option<&Value>) -> Option { + let slot = slot?; + let dt = slot.get("dateTime").and_then(Value::as_str)?; + Some(dt.to_owned()) +} + +fn build_participant( + email: &Value, + roles: &[&str], + participation_status: Option<&'static str>, + expect_reply: Option, +) -> Value { + let mut map = Map::new(); + map.insert("@type".to_owned(), Value::from("Participant")); + if let Some(addr) = email.get("address").and_then(Value::as_str) { + map.insert( + "calendarAddress".to_owned(), + Value::from(format!("mailto:{addr}")), + ); + map.insert("email".to_owned(), Value::from(addr.to_owned())); + } + if let Some(name) = email.get("name").and_then(Value::as_str) + && !name.is_empty() + { + map.insert("name".to_owned(), Value::from(name.to_owned())); + } + let role_map: Map = roles + .iter() + .map(|r| ((*r).to_owned(), Value::Bool(true))) + .collect(); + map.insert("roles".to_owned(), Value::Object(role_map)); + if let Some(status) = participation_status { + map.insert( + "participationStatus".to_owned(), + Value::from(status.to_owned()), + ); + } + if let Some(b) = expect_reply { + map.insert("expectReply".to_owned(), Value::Bool(b)); + } + Value::Object(map) +} + +fn attendee_type_to_role(t: &str) -> &'static str { + match t { + "optional" => "optional", + "resource" => "informational", + _ => "required", + } +} + +fn attendee_response_to_status(r: &str) -> Option<&'static str> { + match r { + "accepted" | "organizer" => Some("accepted"), + "declined" => Some("declined"), + "tentativelyAccepted" => Some("tentative"), + "notResponded" | "none" => Some("needs-action"), + _ => None, + } +} + +fn pad_width(n: usize) -> usize { + let mut digits = 1; + let mut k = n; + while k >= 10 { + digits += 1; + k /= 10; + } + digits +} + +fn privacy_for(value: &str) -> &'static str { + match value { + "private" => "private", + "confidential" => "secret", + _ => "public", + } +} + +fn priority_for(value: &str) -> i64 { + match value { + "high" => 1, + "low" => 9, + _ => 5, + } +} + +fn free_busy_for(value: &str) -> &'static str { + match value { + "free" => "free", + _ => "busy", + } +} + +fn duration_iso8601(start: &str, end: &str) -> Option { + let s = parse_naive_seconds(start)?; + let e = parse_naive_seconds(end)?; + if e <= s { + return None; + } + let total = e - s; + let hours = total / 3600; + let minutes = (total % 3600) / 60; + let seconds = total % 60; + let mut out = "PT".to_owned(); + use std::fmt::Write; + if hours > 0 { + let _ = write!(out, "{hours}H"); + } + if minutes > 0 || (hours > 0 && seconds > 0) { + let _ = write!(out, "{minutes}M"); + } + if seconds > 0 || (hours == 0 && minutes == 0) { + let _ = write!(out, "{seconds}S"); + } + Some(out) +} + +fn strip_fractional(s: String) -> String { + if let Some(dot) = s.find('.') { + s[..dot].to_owned() + } else { + s + } +} + +fn strip_fractional_utc(raw: &str) -> String { + let trailing_z = raw.ends_with('Z'); + let trimmed = raw.trim_end_matches('Z'); + let base = match trimmed.find('.') { + Some(dot) => &trimmed[..dot], + None => trimmed, + }; + if trailing_z { + format!("{base}Z") + } else { + base.to_owned() + } +} + +fn parse_naive_seconds(s: &str) -> Option { + let (date, time) = s.split_once('T')?; + let mut date_parts = date.split('-'); + let year: i64 = date_parts.next()?.parse().ok()?; + let month: i64 = date_parts.next()?.parse().ok()?; + let day: i64 = date_parts.next()?.parse().ok()?; + let time = time.split('.').next()?; + let time = time.trim_end_matches('Z'); + let mut time_parts = time.split(':'); + let hh: i64 = time_parts.next()?.parse().ok()?; + let mm: i64 = time_parts.next()?.parse().ok()?; + let ss: i64 = time_parts.next().unwrap_or("0").parse().ok()?; + Some(days_from_civil(year, month, day) * 86400 + hh * 3600 + mm * 60 + ss) +} + +fn days_from_civil(y: i64, m: i64, d: i64) -> i64 { + let y = if m <= 2 { y - 1 } else { y }; + let era = (if y >= 0 { y } else { y - 399 }) / 400; + let yoe = (y - era * 400) as u64; + let m_adj: i64 = if m > 2 { m - 3 } else { m + 9 }; + let doy = ((153 * m_adj + 2) / 5 + d - 1) as u64; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + era * 146097 + (doe as i64) - 719468 +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn sample() -> Value { + json!({ + "id": "AAA", + "iCalUId": "uid-1", + "type": "singleInstance", + "subject": "Sync up", + "body": {"contentType": "text", "content": "Agenda"}, + "start": {"dateTime": "2026-05-27T10:00:00.0000000", "timeZone": "UTC"}, + "end": {"dateTime": "2026-05-27T11:00:00.0000000", "timeZone": "UTC"}, + "isAllDay": false, + "isCancelled": false, + "sensitivity": "private", + "importance": "high", + "showAs": "busy", + "categories": ["Red", "Internal"], + "createdDateTime": "2026-05-01T08:00:00Z", + "lastModifiedDateTime": "2026-05-02T08:00:00Z", + "isReminderOn": true, + "reminderMinutesBeforeStart": 15, + "isDraft": false, + "organizer": {"emailAddress": {"name": "Alice", "address": "alice@x.com"}}, + "attendees": [ + {"emailAddress": {"name": "Bob", "address": "bob@x.com"}, "type": "required"} + ] + }) + } + + #[test] + fn classify_event_type_dispatch() { + assert_eq!( + classify_event_type(&json!({"type": "seriesMaster"})), + EventType::SeriesMaster + ); + assert_eq!( + classify_event_type(&json!({"type": "occurrence"})), + EventType::Occurrence + ); + assert_eq!( + classify_event_type(&json!({"type": "exception"})), + EventType::Exception + ); + assert_eq!(classify_event_type(&json!({})), EventType::SingleInstance); + } + + #[test] + fn single_instance_converts_to_event_value() { + let conv = convert_event(&sample(), Some("UTC")).unwrap(); + assert_eq!(conv.uid, "uid-1"); + assert_eq!(conv.data["@type"], "Event"); + assert_eq!(conv.data["title"], "Sync up"); + assert_eq!(conv.data["description"], "Agenda"); + assert_eq!(conv.data["start"], "2026-05-27T10:00:00"); + assert_eq!(conv.data["duration"], "PT1H"); + assert_eq!(conv.data["status"], "confirmed"); + assert_eq!(conv.data["privacy"], "private"); + assert_eq!(conv.data["priority"], 1); + assert_eq!(conv.data["freeBusyStatus"], "busy"); + let kws = conv.data["keywords"].as_object().unwrap(); + assert!(kws.contains_key("Red")); + assert!(kws.contains_key("Internal")); + assert!(conv.data.get("categories").is_none()); + assert_eq!(conv.data["version"], "2.0"); + let alerts = conv.data["alerts"].as_object().unwrap(); + assert_eq!(alerts["alert-1"]["trigger"]["offset"], "-PT15M"); + } + + #[test] + fn cancelled_event_status_is_cancelled() { + let mut v = sample(); + v["isCancelled"] = Value::Bool(true); + let conv = convert_event(&v, None).unwrap(); + assert_eq!(conv.data["status"], "cancelled"); + } + + #[test] + fn all_day_sets_show_without_time() { + let mut v = sample(); + v["isAllDay"] = Value::Bool(true); + let conv = convert_event(&v, None).unwrap(); + assert_eq!(conv.data["showWithoutTime"], true); + } + + #[test] + fn priority_mapping_covers_three_levels() { + let conv = convert_event(&sample(), None).unwrap(); + assert_eq!(conv.data["priority"], 1); + let mut v = sample(); + v["importance"] = Value::from("normal"); + assert_eq!(convert_event(&v, None).unwrap().data["priority"], 5); + v["importance"] = Value::from("low"); + assert_eq!(convert_event(&v, None).unwrap().data["priority"], 9); + } + + #[test] + fn duration_handles_subhour() { + assert_eq!( + duration_iso8601("2026-05-27T10:00:00", "2026-05-27T10:30:00").as_deref(), + Some("PT30M") + ); + } + + #[test] + fn duration_handles_multi_day() { + assert_eq!( + duration_iso8601("2026-05-27T10:00:00", "2026-05-28T11:30:45").as_deref(), + Some("PT25H30M45S") + ); + } + + #[test] + fn windows_tz_fallback_returns_iana() { + assert_eq!( + windows_or_iana_to_iana("Pacific Standard Time").as_deref(), + Some("America/Los_Angeles") + ); + assert_eq!( + windows_or_iana_to_iana("America/New_York").as_deref(), + Some("America/New_York") + ); + assert!(windows_or_iana_to_iana("Made Up Zone").is_none()); + } + + #[test] + fn organizer_and_attendees_become_participants() { + let conv = convert_event(&sample(), None).unwrap(); + let participants = conv.data["participants"].as_object().unwrap(); + assert!(participants.contains_key("organizer")); + assert_eq!(participants["organizer"]["email"], "alice@x.com"); + assert_eq!( + participants["organizer"]["calendarAddress"], + "mailto:alice@x.com" + ); + assert_eq!(participants["organizer"]["roles"]["owner"], true); + assert_eq!(participants["organizer"]["roles"]["chair"], true); + assert!(participants.contains_key("att-1")); + assert_eq!(participants["att-1"]["email"], "bob@x.com"); + assert_eq!(participants["att-1"]["roles"]["required"], true); + } + + #[test] + fn optional_attendee_keeps_role_and_uses_status_when_present() { + let mut v = sample(); + v["attendees"] = json!([ + { + "emailAddress": {"name": "Carol", "address": "carol@x.com"}, + "type": "optional", + "status": {"response": "tentativelyAccepted"} + } + ]); + let conv = convert_event(&v, None).unwrap(); + let p = &conv.data["participants"]["att-1"]; + assert_eq!(p["roles"]["optional"], true); + assert!(p["roles"].get("required").is_none()); + assert_eq!(p["participationStatus"], "tentative"); + } + + #[test] + fn many_attendees_zero_pad_keys_for_stable_sort() { + let mut v = sample(); + let mut atts = Vec::new(); + for i in 1..=11 { + atts.push(json!({ + "emailAddress": {"address": format!("u{i}@x.com")}, + "type": "required" + })); + } + v["attendees"] = Value::Array(atts); + let conv = convert_event(&v, None).unwrap(); + let map = conv.data["participants"].as_object().unwrap(); + assert!(map.contains_key("att-01")); + assert!(map.contains_key("att-11")); + } + + #[test] + fn original_start_time_zone_is_preserved() { + let mut v = sample(); + v["originalStartTimeZone"] = Value::from("Pacific Standard Time"); + v["start"]["timeZone"] = Value::from("UTC"); + let conv = convert_event(&v, None).unwrap(); + assert_eq!(conv.data["timeZone"], "America/Los_Angeles"); + } + + #[test] + fn duration_h_and_s_no_minutes_includes_zero_m() { + assert_eq!( + duration_iso8601("2026-05-27T10:00:00", "2026-05-27T11:00:05").as_deref(), + Some("PT1H0M5S") + ); + } + + #[test] + fn fractional_seconds_are_stripped_from_timestamps() { + let mut v = sample(); + v["createdDateTime"] = Value::from("2026-05-01T08:00:00.0000000Z"); + v["lastModifiedDateTime"] = Value::from("2026-05-02T08:00:00.123Z"); + let conv = convert_event(&v, None).unwrap(); + assert_eq!(conv.data["created"], "2026-05-01T08:00:00Z"); + assert_eq!(conv.data["updated"], "2026-05-02T08:00:00Z"); + } + + #[test] + fn organizer_calendar_address_is_set_alongside_participants() { + let conv = convert_event(&sample(), None).unwrap(); + assert_eq!(conv.data["organizerCalendarAddress"], "mailto:alice@x.com"); + } + + #[test] + fn recurrence_emits_singular_recurrence_rule_object() { + let mut v = sample(); + v["recurrence"] = json!({ + "pattern": {"type": "daily", "interval": 1}, + "range": {"type": "noEnd"} + }); + let conv = convert_event(&v, None).unwrap(); + assert!(conv.data.get("recurrenceRules").is_none()); + let rule = &conv.data["recurrenceRule"]; + assert!(rule.is_object()); + assert_eq!(rule["@type"], "RecurrenceRule"); + assert_eq!(rule["frequency"], "daily"); + } + + #[test] + fn location_with_address_only_uses_name_property() { + let mut v = sample(); + v["locations"] = json!([{ + "address": {"street": "1 Infinite Loop", "city": "Cupertino"} + }]); + let conv = convert_event(&v, None).unwrap(); + let locs = conv.data["locations"].as_object().unwrap(); + let only = locs.values().next().unwrap(); + assert_eq!(only["@type"], "Location"); + assert_eq!(only["name"], "1 Infinite Loop, Cupertino"); + assert!(only.get("description").is_none()); + } +} diff --git a/src/exchange_graph/client.rs b/src/exchange_graph/client.rs new file mode 100644 index 0000000..0c54104 --- /dev/null +++ b/src/exchange_graph/client.rs @@ -0,0 +1,454 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::time::Duration; + +use serde_json::Value; +use ureq::Agent; +use ureq::config::{Config, RedirectAuthHeaders}; +use ureq::tls::{RootCerts, TlsConfig}; + +use crate::exchange_graph::error::GraphError; +use crate::exchange_graph::retry::{HttpClass, classify_http_status, is_throttled}; +use crate::jmap::http::{RetryPolicy, retry_after_header}; +use crate::jmap::retry::{self, RateLimitState}; +use crate::logging::{LEVEL_BODIES, LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; + +const MAX_BODY: u64 = 256 * 1024 * 1024; +const LONG_RETRY_THRESHOLD: Duration = Duration::from_secs(10); + +#[derive(Debug, Clone, Copy)] +pub enum Accept { + Json, + Text, +} + +impl Accept { + fn header_value(self) -> &'static str { + match self { + Accept::Json => "application/json", + Accept::Text => "text/plain", + } + } +} + +#[derive(Debug, Clone)] +pub struct GraphResponse { + pub status: u16, + pub body: Vec, + pub content_type: Option, +} + +impl GraphResponse { + pub fn as_str(&self) -> Result<&str, GraphError> { + std::str::from_utf8(&self.body) + .map_err(|e| GraphError::Malformed(format!("response is not utf-8: {e}"))) + } + + pub fn json(&self) -> Result { + serde_json::from_slice(&self.body) + .map_err(|e| GraphError::Malformed(format!("response is not valid json: {e}"))) + } +} + +struct Inner { + agent: Agent, + bearer: Mutex, + retry: RetryPolicy, + rate_limit: RateLimitState, + log_level: AtomicU8, + retries_total: AtomicU64, + retry_after_sleeps: AtomicU64, + requests_total: AtomicU64, + user_agent: String, +} + +#[derive(Clone)] +pub struct GraphClient { + inner: Arc, +} + +enum Attempt { + Ok { + status: u16, + body: Vec, + retry_after: Option, + content_type: Option, + }, + Transport(GraphError), +} + +impl GraphClient { + pub fn new(bearer: String, retry: RetryPolicy, allow_invalid_certs: bool) -> GraphClient { + let config: Config = Config::builder() + .http_status_as_error(false) + .redirect_auth_headers(RedirectAuthHeaders::SameHost) + .tls_config( + TlsConfig::builder() + .root_certs(RootCerts::PlatformVerifier) + .disable_verification(allow_invalid_certs) + .build(), + ) + .build(); + GraphClient { + inner: Arc::new(Inner { + agent: config.new_agent(), + bearer: Mutex::new(bearer), + retry, + rate_limit: RateLimitState::new(), + log_level: AtomicU8::new(LEVEL_DEFAULT), + retries_total: AtomicU64::new(0), + retry_after_sleeps: AtomicU64::new(0), + requests_total: AtomicU64::new(0), + user_agent: format!("vandelay/{}", env!("CARGO_PKG_VERSION")), + }), + } + } + + pub fn set_bearer(&self, bearer: String) { + let mut g = match self.inner.bearer.lock() { + Ok(g) => g, + Err(p) => p.into_inner(), + }; + *g = bearer; + } + + pub fn set_logger(&self, logger: Logger) { + self.inner + .log_level + .store(logger.level(), Ordering::Relaxed); + } + + fn logger(&self) -> Logger { + Logger::new(self.inner.log_level.load(Ordering::Relaxed)) + } + + fn bearer(&self) -> String { + match self.inner.bearer.lock() { + Ok(g) => g.clone(), + Err(p) => p.into_inner().clone(), + } + } + + fn auth_header(&self) -> String { + format!("Bearer {}", self.bearer()) + } + + pub fn retries_observed(&self) -> u64 { + self.inner.retries_total.load(Ordering::Relaxed) + } + + pub fn retry_after_sleeps(&self) -> u64 { + self.inner.retry_after_sleeps.load(Ordering::Relaxed) + } + + pub fn requests_observed(&self) -> u64 { + self.inner.requests_total.load(Ordering::Relaxed) + } + + pub fn rate_limit(&self) -> &RateLimitState { + &self.inner.rate_limit + } + + pub fn get(&self, url: &str, accept: Accept) -> Result { + self.execute("GET", url, accept, &[]) + } + + pub fn get_with_prefer( + &self, + url: &str, + accept: Accept, + prefer: &[&str], + ) -> Result { + self.execute("GET", url, accept, prefer) + } + + pub fn get_json(&self, url: &str) -> Result { + self.get(url, Accept::Json).and_then(|r| r.json()) + } + + pub fn get_json_with_prefer(&self, url: &str, prefer: &[&str]) -> Result { + self.get_with_prefer(url, Accept::Json, prefer) + .and_then(|r| r.json()) + } + + fn execute( + &self, + method: &str, + url: &str, + accept: Accept, + extra_prefer: &[&str], + ) -> Result { + let policy = self.inner.retry; + let logger = self.logger(); + let mut attempt: u32 = 0; + loop { + self.inner.rate_limit.cooldown().wait(); + self.inner.requests_total.fetch_add(1, Ordering::Relaxed); + let outcome = self.one_attempt(method, url, accept, extra_prefer); + match outcome { + Attempt::Ok { + status, + body, + retry_after, + content_type, + } => { + let class = classify_http_status(status); + match class { + HttpClass::Success => { + self.inner.rate_limit.on_success(); + return Ok(GraphResponse { + status, + body, + content_type, + }); + } + HttpClass::Vanished => { + return Err(GraphError::Vanished); + } + HttpClass::Auth => { + return Err(GraphError::Auth(format!( + "server returned {status}: {}", + truncate(&body) + ))); + } + HttpClass::Fatal => { + return Err(GraphError::HttpStatus { + status, + body: truncate(&body), + }); + } + HttpClass::Retryable => { + attempt += 1; + self.inner.retries_total.fetch_add(1, Ordering::Relaxed); + if attempt > policy.max_retries { + return Err(GraphError::RetriesExhausted(format!( + "{method} {url} kept returning http {status}" + ))); + } + if retry_after.is_some() { + self.inner + .retry_after_sleeps + .fetch_add(1, Ordering::Relaxed); + } + let delay = if is_throttled(status) { + self.inner.rate_limit.on_throttle(&policy, retry_after) + } else { + match retry_after { + Some(d) => d, + None => retry::backoff_delay(&policy, attempt), + } + }; + if delay >= LONG_RETRY_THRESHOLD { + logger.warn(&format!( + "graph rate-limited (http {status}); waiting {} before retry {}/{} (shared level {})", + format_retry_wait(delay), + attempt, + policy.max_retries, + self.inner.rate_limit.level(), + )); + } + self.log_retry( + &logger, + RetryLog { + method, + url, + attempt, + delay, + status, + body: &body, + }, + ); + std::thread::sleep(delay); + } + } + } + Attempt::Transport(err) => { + if matches!(err, GraphError::Connect(_)) { + return Err(err); + } + attempt += 1; + self.inner.retries_total.fetch_add(1, Ordering::Relaxed); + if attempt > policy.max_retries { + return Err(GraphError::RetriesExhausted(format!( + "{method} {url}: {err}" + ))); + } + let delay = retry::backoff_delay(&policy, attempt); + if delay >= LONG_RETRY_THRESHOLD { + logger.warn(&format!( + "transient transport failure ({err}); waiting {} before retry {}/{}", + format_retry_wait(delay), + attempt, + policy.max_retries + )); + } + std::thread::sleep(delay); + } + } + } + } + + fn one_attempt( + &self, + method: &str, + url: &str, + accept: Accept, + extra_prefer: &[&str], + ) -> Attempt { + let mut req = match method { + "GET" => self.inner.agent.get(url), + other => { + return Attempt::Transport(GraphError::Connect(format!( + "unsupported method {other} (graph importer is read-only)" + ))); + } + }; + req = req + .header("Authorization", self.auth_header()) + .header("Accept", accept.header_value()) + .header("User-Agent", self.inner.user_agent.as_str()); + req = req.header("Prefer", "IdType=\"ImmutableId\""); + for value in extra_prefer { + req = req.header("Prefer", *value); + } + match req.call() { + Ok(mut resp) => { + let status = resp.status().as_u16(); + let retry_after = resp + .headers() + .get("retry-after") + .and_then(|v| v.to_str().ok()) + .and_then(retry_after_header); + let content_type = resp + .headers() + .get("content-type") + .and_then(|v| v.to_str().ok()) + .map(str::to_owned); + match resp.body_mut().with_config().limit(MAX_BODY).read_to_vec() { + Ok(bytes) => Attempt::Ok { + status, + body: bytes, + retry_after, + content_type, + }, + Err(e) => Attempt::Transport(GraphError::Transport(format!( + "reading response body: {e}" + ))), + } + } + Err(e) => Attempt::Transport(map_ureq_error(e)), + } + } + + fn log_retry(&self, logger: &Logger, info: RetryLog<'_>) { + let RetryLog { + method, + url, + attempt, + delay, + status, + body, + } = info; + if logger.enabled(LEVEL_BODIES) { + eprintln!( + "retry {}/{} {} {} after {:?} (http {status}) body={}", + attempt, + self.inner.retry.max_retries, + method, + url, + delay, + truncate(body) + ); + } else if logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "retry {}/{} (http {status})", + attempt, self.inner.retry.max_retries + ); + } + } +} + +struct RetryLog<'a> { + method: &'a str, + url: &'a str, + attempt: u32, + delay: Duration, + status: u16, + body: &'a [u8], +} + +fn map_ureq_error(err: ureq::Error) -> GraphError { + match err { + ureq::Error::Io(e) => GraphError::Transport(format!("io: {e}")), + ureq::Error::Timeout(t) => GraphError::Transport(format!("timeout: {t}")), + ureq::Error::HostNotFound => GraphError::Transport("host not found".to_owned()), + ureq::Error::ConnectionFailed => GraphError::Transport("connection failed".to_owned()), + ureq::Error::BodyStalled => GraphError::Transport("body stalled".to_owned()), + ureq::Error::Tls(m) => GraphError::Transport(format!("tls: {m}")), + ureq::Error::TooManyRedirects => GraphError::Connect("too many redirects".to_owned()), + ureq::Error::RedirectFailed => GraphError::Connect("redirect failed".to_owned()), + ureq::Error::TlsRequired => { + GraphError::Connect("server requires TLS but transport is unsecured".to_owned()) + } + other => GraphError::Connect(other.to_string()), + } +} + +fn truncate(body: &[u8]) -> String { + let text = String::from_utf8_lossy(body); + if text.len() <= 512 { + return text.into_owned(); + } + let end = text.floor_char_boundary(512); + format!("{}...", &text[..end]) +} + +fn format_retry_wait(d: Duration) -> String { + let total = d.as_secs(); + if total < 60 { + format!("{total}s") + } else if total < 3600 { + format!("{}m{:02}s", total / 60, total % 60) + } else { + format!( + "{}h{:02}m{:02}s", + total / 3600, + (total % 3600) / 60, + total % 60 + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults_construct() { + let c = GraphClient::new("token".to_owned(), RetryPolicy::new(3), false); + assert_eq!(c.retries_observed(), 0); + assert_eq!(c.retry_after_sleeps(), 0); + assert_eq!(c.requests_observed(), 0); + assert_eq!(c.auth_header(), "Bearer token"); + } + + #[test] + fn bearer_can_be_swapped_at_runtime() { + let c = GraphClient::new("old".to_owned(), RetryPolicy::new(0), false); + c.set_bearer("new".to_owned()); + assert_eq!(c.auth_header(), "Bearer new"); + } + + #[test] + fn accept_header_values_are_what_the_spec_requires() { + assert_eq!(Accept::Json.header_value(), "application/json"); + assert_eq!(Accept::Text.header_value(), "text/plain"); + } +} diff --git a/src/exchange_graph/contact_map.rs b/src/exchange_graph/contact_map.rs new file mode 100644 index 0000000..947e2ce --- /dev/null +++ b/src/exchange_graph/contact_map.rs @@ -0,0 +1,586 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::{Map, Value, json}; + +use crate::exchange_graph::error::GraphError; + +#[derive(Debug, Clone)] +pub struct ConvertedContact { + pub uid: String, + pub data: Value, +} + +pub fn convert_contact(graph_contact: &Value) -> Result { + let graph_id = graph_contact + .get("id") + .and_then(Value::as_str) + .ok_or_else(|| GraphError::Malformed("contact has no id".to_owned()))?; + let uid = synthetic_uid(graph_id); + + let mut card = Map::new(); + card.insert("@type".to_owned(), Value::from("Card")); + card.insert("version".to_owned(), Value::from("1.0")); + card.insert("uid".to_owned(), Value::from(uid.clone())); + card.insert("kind".to_owned(), Value::from("individual")); + + let mut name_object = Map::new(); + name_object.insert("@type".to_owned(), Value::from("Name")); + if let Some(display) = graph_contact.get("displayName").and_then(Value::as_str) + && !display.is_empty() + { + name_object.insert("full".to_owned(), Value::from(display.to_owned())); + } + let name_slots = [ + ("title", graph_contact.get("title").and_then(Value::as_str)), + ( + "given", + graph_contact.get("givenName").and_then(Value::as_str), + ), + ( + "given2", + graph_contact.get("middleName").and_then(Value::as_str), + ), + ( + "surname", + graph_contact.get("surname").and_then(Value::as_str), + ), + ( + "generation", + graph_contact.get("generation").and_then(Value::as_str), + ), + ]; + let mut components: Vec = Vec::new(); + let mut slot_index: std::collections::HashMap<&'static str, usize> = + std::collections::HashMap::new(); + for (kind, value) in name_slots { + let Some(v) = value else { continue }; + if v.is_empty() { + continue; + } + slot_index.insert(kind, components.len()); + components.push(json!({"@type": "NameComponent", "kind": kind, "value": v})); + } + if !components.is_empty() { + name_object.insert("components".to_owned(), Value::Array(components)); + } + if !name_object.is_empty() { + card.insert("name".to_owned(), Value::Object(name_object)); + } + + if let Some(nick) = graph_contact.get("nickName").and_then(Value::as_str) + && !nick.is_empty() + { + let mut nicks = Map::new(); + nicks.insert( + "nick-1".to_owned(), + json!({"@type": "Nickname", "name": nick}), + ); + card.insert("nicknames".to_owned(), Value::Object(nicks)); + } + + let mut emails = Map::new(); + if let Some(addrs) = graph_contact + .get("emailAddresses") + .and_then(Value::as_array) + { + for (i, e) in addrs.iter().enumerate() { + let Some(addr) = e.get("address").and_then(Value::as_str) else { + continue; + }; + if addr.is_empty() { + continue; + } + let key = format!("email-{}", i + 1); + let mut entry = Map::new(); + entry.insert("@type".to_owned(), Value::from("EmailAddress")); + entry.insert("address".to_owned(), Value::from(addr.to_owned())); + if i == 0 { + entry.insert("pref".to_owned(), Value::from(1)); + } + emails.insert(key, Value::Object(entry)); + } + } + if !emails.is_empty() { + card.insert("emails".to_owned(), Value::Object(emails)); + } + + let mut phones = Map::new(); + push_phones( + &mut phones, + graph_contact + .get("businessPhones") + .and_then(Value::as_array), + "work", + ); + push_phones( + &mut phones, + graph_contact.get("homePhones").and_then(Value::as_array), + "private", + ); + if let Some(mobile) = graph_contact.get("mobilePhone").and_then(Value::as_str) + && !mobile.is_empty() + { + let key = format!("phone-{}", phones.len() + 1); + phones.insert( + key, + json!({ + "@type": "Phone", + "number": mobile, + "features": {"mobile": true, "voice": true} + }), + ); + } + if !phones.is_empty() { + card.insert("phones".to_owned(), Value::Object(phones)); + } + + let mut addresses = Map::new(); + push_address(&mut addresses, graph_contact.get("homeAddress"), "private"); + push_address(&mut addresses, graph_contact.get("businessAddress"), "work"); + push_address(&mut addresses, graph_contact.get("otherAddress"), "other"); + if !addresses.is_empty() { + card.insert("addresses".to_owned(), Value::Object(addresses)); + } + + let mut organizations = Map::new(); + let has_company = graph_contact + .get("companyName") + .and_then(Value::as_str) + .is_some_and(|s| !s.is_empty()); + let has_dept = graph_contact + .get("department") + .and_then(Value::as_str) + .is_some_and(|s| !s.is_empty()); + let has_office = graph_contact + .get("officeLocation") + .and_then(Value::as_str) + .is_some_and(|s| !s.is_empty()); + let has_title = graph_contact + .get("jobTitle") + .and_then(Value::as_str) + .is_some_and(|s| !s.is_empty()); + if has_company || has_dept || has_office { + let mut org = Map::new(); + org.insert("@type".to_owned(), Value::from("Organization")); + if let Some(name) = graph_contact.get("companyName").and_then(Value::as_str) + && !name.is_empty() + { + org.insert("name".to_owned(), Value::from(name.to_owned())); + } + let mut units: Vec = Vec::new(); + if let Some(dept) = graph_contact.get("department").and_then(Value::as_str) + && !dept.is_empty() + { + units.push(json!({"@type": "OrgUnit", "name": dept})); + } + if let Some(office) = graph_contact.get("officeLocation").and_then(Value::as_str) + && !office.is_empty() + { + units.push(json!({"@type": "OrgUnit", "name": office})); + } + if !units.is_empty() { + org.insert("units".to_owned(), Value::Array(units)); + } + organizations.insert("org-1".to_owned(), Value::Object(org)); + } + if has_title + && let Some(title) = graph_contact.get("jobTitle").and_then(Value::as_str) + && !title.is_empty() + { + let mut titles = Map::new(); + let mut title_obj = Map::new(); + title_obj.insert("@type".to_owned(), Value::from("Title")); + title_obj.insert("name".to_owned(), Value::from(title.to_owned())); + if organizations.contains_key("org-1") { + title_obj.insert("organizationId".to_owned(), Value::from("org-1")); + } + titles.insert("title-1".to_owned(), Value::Object(title_obj)); + card.insert("titles".to_owned(), Value::Object(titles)); + } + let has_org_one = organizations.contains_key("org-1"); + if !organizations.is_empty() { + card.insert("organizations".to_owned(), Value::Object(organizations)); + } + + if let Some(notes) = graph_contact.get("personalNotes").and_then(Value::as_str) + && !notes.is_empty() + { + let mut map = Map::new(); + map.insert("note-1".to_owned(), json!({"@type": "Note", "note": notes})); + card.insert("notes".to_owned(), Value::Object(map)); + } + + let mut localizations = Map::new(); + let mut ja = Map::new(); + if let Some(name) = graph_contact.get("yomiCompanyName").and_then(Value::as_str) + && !name.is_empty() + && has_org_one + { + ja.insert( + "organizations/org-1/name".to_owned(), + Value::from(name.to_owned()), + ); + } + if let Some(name) = graph_contact.get("yomiGivenName").and_then(Value::as_str) + && !name.is_empty() + && let Some(idx) = slot_index.get("given") + { + ja.insert( + format!("name/components/{idx}/value"), + Value::from(name.to_owned()), + ); + } + if let Some(name) = graph_contact.get("yomiSurname").and_then(Value::as_str) + && !name.is_empty() + && let Some(idx) = slot_index.get("surname") + { + ja.insert( + format!("name/components/{idx}/value"), + Value::from(name.to_owned()), + ); + } + if !ja.is_empty() { + localizations.insert("ja".to_owned(), Value::Object(ja)); + } + if !localizations.is_empty() { + card.insert("localizations".to_owned(), Value::Object(localizations)); + } + + if let Some(birthday) = graph_contact.get("birthday").and_then(Value::as_str) + && let Some(date) = parse_partial_date(birthday) + { + let mut anniversaries = Map::new(); + anniversaries.insert( + "anniversary-1".to_owned(), + json!({ + "@type": "Anniversary", + "kind": "birth", + "date": date, + }), + ); + card.insert("anniversaries".to_owned(), Value::Object(anniversaries)); + } + + let mut related = Map::new(); + if let Some(m) = graph_contact.get("manager").and_then(Value::as_str) + && !m.is_empty() + { + related.insert( + m.to_owned(), + json!({"@type": "Relation", "relation": {"co-worker": true}}), + ); + } + if let Some(s) = graph_contact.get("spouseName").and_then(Value::as_str) + && !s.is_empty() + { + related.insert( + s.to_owned(), + json!({"@type": "Relation", "relation": {"spouse": true}}), + ); + } + if let Some(kids) = graph_contact.get("children").and_then(Value::as_array) { + for kid in kids.iter().filter_map(Value::as_str) { + related.insert( + kid.to_owned(), + json!({"@type": "Relation", "relation": {"child": true}}), + ); + } + } + if !related.is_empty() { + card.insert("relatedTo".to_owned(), Value::Object(related)); + } + + if let Some(link) = graph_contact + .get("businessHomePage") + .and_then(Value::as_str) + && !link.is_empty() + { + let mut links = Map::new(); + links.insert( + "link-1".to_owned(), + json!({"@type": "Link", "uri": link, "contexts": {"work": true}}), + ); + card.insert("links".to_owned(), Value::Object(links)); + } + + if let Some(created) = graph_contact.get("createdDateTime").and_then(Value::as_str) { + card.insert("created".to_owned(), Value::from(created.to_owned())); + } + if let Some(updated) = graph_contact + .get("lastModifiedDateTime") + .and_then(Value::as_str) + { + card.insert("updated".to_owned(), Value::from(updated.to_owned())); + } + + Ok(ConvertedContact { + uid, + data: Value::Object(card), + }) +} + +pub fn synthetic_uid(graph_id: &str) -> String { + let hash = blake3::hash(graph_id.as_bytes()); + format!("vandelay-graph-{}", hash.to_hex()) +} + +fn parse_partial_date(raw: &str) -> Option { + let date_part = raw.split('T').next().unwrap_or(raw); + let mut iter = date_part.split('-'); + let year: u64 = iter.next()?.parse().ok()?; + let mut obj = Map::new(); + obj.insert("@type".to_owned(), Value::from("PartialDate")); + obj.insert("year".to_owned(), Value::from(year)); + if let Some(month_str) = iter.next() + && let Ok(month) = month_str.parse::() + && (1..=12).contains(&month) + { + obj.insert("month".to_owned(), Value::from(month)); + if let Some(day_str) = iter.next() + && let Ok(day) = day_str.parse::() + && (1..=31).contains(&day) + { + obj.insert("day".to_owned(), Value::from(day)); + } + } + Some(Value::Object(obj)) +} + +fn push_phones(target: &mut Map, list: Option<&Vec>, context: &str) { + let Some(list) = list else { return }; + for n in list.iter().filter_map(Value::as_str) { + if n.is_empty() { + continue; + } + let key = format!("phone-{}", target.len() + 1); + target.insert( + key, + json!({ + "@type": "Phone", + "number": n, + "contexts": {context: true} + }), + ); + } +} + +fn push_address(target: &mut Map, source: Option<&Value>, context: &str) { + let Some(source) = source else { return }; + let Some(map) = source.as_object() else { + return; + }; + let street = map + .get("street") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()); + let component_fields: Vec<(&str, &str)> = [ + ("city", "locality"), + ("state", "region"), + ("postalCode", "postcode"), + ("countryOrRegion", "country"), + ] + .into_iter() + .filter_map(|(graph_key, jsc_kind)| { + map.get(graph_key) + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .map(|s| (jsc_kind, s)) + }) + .collect(); + if street.is_none() && component_fields.is_empty() { + return; + } + let mut address = Map::new(); + address.insert("@type".to_owned(), Value::from("Address")); + address.insert("contexts".to_owned(), json!({context: true})); + if let Some(s) = street { + address.insert("full".to_owned(), Value::from(s.to_owned())); + } + if !component_fields.is_empty() { + let components: Vec = component_fields + .iter() + .map(|(kind, value)| json!({"@type": "AddressComponent", "kind": kind, "value": value})) + .collect(); + address.insert("components".to_owned(), Value::Array(components)); + } + let key = format!("addr-{}", target.len() + 1); + target.insert(key, Value::Object(address)); +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn minimal_contact_yields_card_with_synthetic_uid() { + let c = json!({"id": "GRAPH-1", "displayName": "Alice"}); + let conv = convert_contact(&c).unwrap(); + assert!(conv.uid.starts_with("vandelay-graph-")); + assert_eq!(conv.data["@type"], "Card"); + assert_eq!(conv.data["kind"], "individual"); + assert_eq!(conv.data["name"]["full"], "Alice"); + } + + #[test] + fn synthetic_uid_is_stable_per_graph_id() { + let a = synthetic_uid("AAA"); + let b = synthetic_uid("AAA"); + let c = synthetic_uid("BBB"); + assert_eq!(a, b); + assert_ne!(a, c); + } + + #[test] + fn emails_map_with_first_as_pref() { + let c = json!({ + "id": "X", + "emailAddresses": [ + {"address": "a@x.com", "name": "Alice"}, + {"address": "b@x.com", "name": "Alice B"} + ] + }); + let conv = convert_contact(&c).unwrap(); + let emails = conv.data["emails"].as_object().unwrap(); + assert_eq!(emails["email-1"]["address"], "a@x.com"); + assert_eq!(emails["email-1"]["pref"], 1); + assert_eq!(emails["email-2"]["address"], "b@x.com"); + assert!(emails["email-2"].get("pref").is_none()); + } + + #[test] + fn phones_carry_context_per_source_field() { + let c = json!({ + "id": "X", + "businessPhones": ["+1-555-2000"], + "homePhones": ["+1-555-1000"], + "mobilePhone": "+1-555-3000" + }); + let conv = convert_contact(&c).unwrap(); + let phones = conv.data["phones"].as_object().unwrap(); + assert_eq!(phones.len(), 3); + let has_work = phones + .values() + .any(|v| v["contexts"]["work"] == Value::Bool(true)); + let has_private = phones + .values() + .any(|v| v["contexts"]["private"] == Value::Bool(true)); + let has_mobile = phones + .values() + .any(|v| v["features"]["mobile"] == Value::Bool(true)); + assert!(has_work && has_private && has_mobile); + } + + #[test] + fn addresses_map_three_contexts() { + let c = json!({ + "id": "X", + "homeAddress": {"city": "Home City"}, + "businessAddress": {"city": "Work City"}, + "otherAddress": {"city": "Other City"} + }); + let conv = convert_contact(&c).unwrap(); + let addresses = conv.data["addresses"].as_object().unwrap(); + assert_eq!(addresses.len(), 3); + } + + #[test] + fn org_and_title_pair_into_separate_jscontact_objects() { + let c = json!({ + "id": "X", + "companyName": "Acme", + "department": "Engineering", + "officeLocation": "Building 9", + "jobTitle": "Architect" + }); + let conv = convert_contact(&c).unwrap(); + let orgs = conv.data["organizations"].as_object().unwrap(); + assert!(orgs.contains_key("org-1")); + assert_eq!(orgs["org-1"]["name"], "Acme"); + let titles = conv.data["titles"].as_object().unwrap(); + assert_eq!(titles["title-1"]["name"], "Architect"); + assert_eq!(titles["title-1"]["organizationId"], "org-1"); + } + + #[test] + fn yomi_fields_populate_japanese_localization() { + let c = json!({ + "id": "X", + "displayName": "Yamada Taro", + "givenName": "Taro", + "surname": "Yamada", + "companyName": "Acme", + "yomiCompanyName": "アクメ", + "yomiGivenName": "タロウ", + "yomiSurname": "ヤマダ" + }); + let conv = convert_contact(&c).unwrap(); + let ja = conv.data["localizations"]["ja"].as_object().unwrap(); + assert_eq!(ja["organizations/org-1/name"], "アクメ"); + } + + #[test] + fn birthday_emits_partialdate_object() { + let c = json!({"id": "X", "birthday": "1980-04-12T00:00:00Z"}); + let conv = convert_contact(&c).unwrap(); + let ann = conv.data["anniversaries"].as_object().unwrap(); + let date = &ann["anniversary-1"]["date"]; + assert_eq!(date["@type"], "PartialDate"); + assert_eq!(date["year"], 1980); + assert_eq!(date["month"], 4); + assert_eq!(date["day"], 12); + assert_eq!(ann["anniversary-1"]["kind"], "birth"); + } + + #[test] + fn mobile_phone_uses_features_not_contexts() { + let c = json!({"id": "X", "mobilePhone": "+1-555-3000"}); + let conv = convert_contact(&c).unwrap(); + let phones = conv.data["phones"].as_object().unwrap(); + let phone = phones.values().next().unwrap(); + assert_eq!(phone["features"]["mobile"], true); + assert!(phone.get("contexts").is_none()); + } + + #[test] + fn generation_maps_to_generation_kind() { + let c = json!({"id": "X", "generation": "Jr."}); + let conv = convert_contact(&c).unwrap(); + let comps = conv.data["name"]["components"].as_array().unwrap(); + assert!( + comps + .iter() + .any(|c| c["kind"] == "generation" && c["value"] == "Jr.") + ); + } + + #[test] + fn only_jobtitle_does_not_emit_empty_organization() { + let c = json!({"id": "X", "jobTitle": "Architect"}); + let conv = convert_contact(&c).unwrap(); + assert!(conv.data.get("organizations").is_none()); + let titles = conv.data["titles"].as_object().unwrap(); + assert_eq!(titles["title-1"]["name"], "Architect"); + assert!(titles["title-1"].get("organizationId").is_none()); + } + + #[test] + fn manager_and_children_become_related_to() { + let c = json!({ + "id": "X", + "manager": "alice@x.com", + "spouseName": "Bob", + "children": ["Kid A", "Kid B"] + }); + let conv = convert_contact(&c).unwrap(); + let related = conv.data["relatedTo"].as_object().unwrap(); + assert!(related.contains_key("alice@x.com")); + assert!(related.contains_key("Bob")); + assert!(related.contains_key("Kid A")); + assert!(related.contains_key("Kid B")); + } +} diff --git a/src/exchange_graph/error.rs b/src/exchange_graph/error.rs new file mode 100644 index 0000000..84a6de2 --- /dev/null +++ b/src/exchange_graph/error.rs @@ -0,0 +1,51 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::error::Error; + +#[derive(Debug, thiserror::Error)] +pub enum GraphError { + #[error("oauth error: {0}")] + OAuth(String), + + #[error("graph http {status}: {body}")] + HttpStatus { status: u16, body: String }, + + #[error("graph auth failure: {0}")] + Auth(String), + + #[error("graph transport error: {0}")] + Transport(String), + + #[error("graph connect error: {0}")] + Connect(String), + + #[error("retries exhausted: {0}")] + RetriesExhausted(String), + + #[error("malformed graph response: {0}")] + Malformed(String), + + #[error("graph item vanished")] + Vanished, +} + +impl From for Error { + fn from(e: GraphError) -> Self { + match e { + GraphError::OAuth(m) => Error::Connection(format!("oauth: {m}")), + GraphError::Auth(m) => Error::Connection(format!("auth: {m}")), + GraphError::Connect(m) => Error::Connection(m), + GraphError::Transport(m) => Error::Connection(m), + GraphError::RetriesExhausted(m) => Error::Connection(m), + GraphError::HttpStatus { status, body } => { + Error::Connection(format!("http {status}: {body}")) + } + GraphError::Malformed(m) => Error::Partial(format!("malformed: {m}")), + GraphError::Vanished => Error::Partial("graph item vanished".to_owned()), + } + } +} diff --git a/src/exchange_graph/mod.rs b/src/exchange_graph/mod.rs new file mode 100644 index 0000000..cebeef7 --- /dev/null +++ b/src/exchange_graph/mod.rs @@ -0,0 +1,19 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod api; +pub mod calendar_map; +pub mod client; +pub mod contact_map; +pub mod error; +pub mod oauth; +pub mod recurrence; +pub mod retry; +pub mod types; + +pub use client::{GraphClient, GraphResponse}; +pub use error::GraphError; +pub use types::{EventBodyFormat, MailboxKind, ResolvedPrincipal}; diff --git a/src/exchange_graph/oauth.rs b/src/exchange_graph/oauth.rs new file mode 100644 index 0000000..d32f68b --- /dev/null +++ b/src/exchange_graph/oauth.rs @@ -0,0 +1,534 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::{self, Write}; +use std::time::{Duration, Instant}; + +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use serde_json::Value; +use ureq::config::Config; +use ureq::tls::{RootCerts, TlsConfig}; + +use crate::exchange_graph::error::GraphError; + +pub const SCOPES: &str = + "offline_access User.Read Mail.Read MailboxSettings.Read Calendars.Read Contacts.Read"; + +pub fn default_authority(tenant: &str) -> String { + format!("https://login.microsoftonline.com/{tenant}") +} + +pub fn device_code_endpoint(authority: &str) -> String { + format!("{authority}/oauth2/v2.0/devicecode") +} + +pub fn token_endpoint(authority: &str) -> String { + format!("{authority}/oauth2/v2.0/token") +} + +#[derive(Debug, Clone)] +pub enum OAuthFlow { + PreAcquired { + token: String, + }, + DeviceCode { + authority: String, + client_id: String, + }, +} + +#[derive(Debug, Clone, Default)] +pub struct JwtClaims { + pub tenant_id: Option, + pub upn: Option, + pub name: Option, + pub exp: Option, +} + +pub fn decode_jwt_claims(token: &str) -> Option { + let mut parts = token.split('.'); + let _header = parts.next()?; + let payload = parts.next()?; + let bytes = URL_SAFE_NO_PAD + .decode(payload) + .or_else(|_| base64::engine::general_purpose::STANDARD_NO_PAD.decode(payload)) + .or_else(|_| base64::engine::general_purpose::STANDARD.decode(payload)) + .ok()?; + let value: Value = serde_json::from_slice(&bytes).ok()?; + Some(JwtClaims { + tenant_id: value.get("tid").and_then(Value::as_str).map(str::to_owned), + upn: value + .get("upn") + .and_then(Value::as_str) + .or_else(|| value.get("preferred_username").and_then(Value::as_str)) + .map(str::to_owned), + name: value.get("name").and_then(Value::as_str).map(str::to_owned), + exp: value.get("exp").and_then(Value::as_u64), + }) +} + +#[derive(Debug, Clone)] +pub struct AcquiredToken { + pub access_token: String, + pub refresh_token: Option, + pub expires_in: Option, + pub tenant_id: Option, + pub upn: Option, + pub name: Option, +} + +fn build_agent(allow_invalid_certs: bool) -> ureq::Agent { + let config: Config = Config::builder() + .http_status_as_error(false) + .tls_config( + TlsConfig::builder() + .root_certs(RootCerts::PlatformVerifier) + .disable_verification(allow_invalid_certs) + .build(), + ) + .build(); + config.new_agent() +} + +pub fn acquire(flow: &OAuthFlow, allow_invalid_certs: bool) -> Result { + match flow { + OAuthFlow::PreAcquired { token } => Ok(token_from_string(token.clone())), + OAuthFlow::DeviceCode { + authority, + client_id, + } => device_code_flow(authority, client_id, allow_invalid_certs), + } +} + +fn token_from_string(token: String) -> AcquiredToken { + let claims = decode_jwt_claims(&token).unwrap_or_default(); + AcquiredToken { + access_token: token, + refresh_token: None, + expires_in: None, + tenant_id: claims.tenant_id, + upn: claims.upn, + name: claims.name, + } +} + +#[derive(Debug, Clone)] +pub struct DeviceCodeResponse { + pub device_code: String, + pub user_code: String, + pub verification_uri: String, + pub interval: u64, + pub expires_in: u64, + pub message: Option, +} + +pub fn parse_device_code_response(json: &Value) -> Result { + let device_code = json + .get("device_code") + .and_then(Value::as_str) + .ok_or_else(|| GraphError::OAuth("device_code missing".to_owned()))? + .to_owned(); + let user_code = json + .get("user_code") + .and_then(Value::as_str) + .unwrap_or("?") + .to_owned(); + let verification_uri = json + .get("verification_uri") + .and_then(Value::as_str) + .unwrap_or("https://microsoft.com/devicelogin") + .to_owned(); + let interval = json.get("interval").and_then(Value::as_u64).unwrap_or(5); + let expires_in = json + .get("expires_in") + .and_then(Value::as_u64) + .unwrap_or(900); + let message = json + .get("message") + .and_then(Value::as_str) + .map(str::to_owned); + Ok(DeviceCodeResponse { + device_code, + user_code, + verification_uri, + interval, + expires_in, + message, + }) +} + +#[derive(Debug, Clone)] +pub enum TokenResponse { + Ok(AcquiredToken), + Pending, + SlowDown, + Expired, + Declined, + BadCode, + Other { error: String, description: String }, +} + +pub fn parse_token_response(status: u16, json: &Value) -> TokenResponse { + if (200..300).contains(&status) { + let Some(access) = json.get("access_token").and_then(Value::as_str) else { + return TokenResponse::Other { + error: "missing_access_token".to_owned(), + description: String::new(), + }; + }; + let claims = decode_jwt_claims(access).unwrap_or_default(); + return TokenResponse::Ok(AcquiredToken { + access_token: access.to_owned(), + refresh_token: json + .get("refresh_token") + .and_then(Value::as_str) + .map(str::to_owned), + expires_in: json.get("expires_in").and_then(Value::as_u64), + tenant_id: claims.tenant_id, + upn: claims.upn, + name: claims.name, + }); + } + let error = json + .get("error") + .and_then(Value::as_str) + .unwrap_or("") + .to_owned(); + let description = json + .get("error_description") + .and_then(Value::as_str) + .unwrap_or("") + .to_owned(); + match error.as_str() { + "authorization_pending" => TokenResponse::Pending, + "slow_down" => TokenResponse::SlowDown, + "expired_token" => TokenResponse::Expired, + "authorization_declined" => TokenResponse::Declined, + "bad_verification_code" => TokenResponse::BadCode, + _ => TokenResponse::Other { error, description }, + } +} + +fn device_code_flow( + authority: &str, + client_id: &str, + allow_invalid_certs: bool, +) -> Result { + let agent = build_agent(allow_invalid_certs); + let body = form_encode(&[("client_id", client_id), ("scope", SCOPES)]); + let endpoint = device_code_endpoint(authority); + let mut resp = agent + .post(&endpoint) + .header("Content-Type", "application/x-www-form-urlencoded") + .send(body.as_bytes()) + .map_err(|e| GraphError::OAuth(format!("device code endpoint: {e}")))?; + let json: Value = resp + .body_mut() + .read_json() + .map_err(|e| GraphError::OAuth(format!("device code body: {e}")))?; + let response = parse_device_code_response(&json)?; + if let Some(m) = response.message.as_deref() { + eprintln!("{m}"); + } else { + eprintln!( + "To sign in, open {} and enter the code {}", + response.verification_uri, response.user_code + ); + } + let _ = io::stdout().flush(); + let token_endpoint = token_endpoint(authority); + let deadline = Instant::now() + Duration::from_secs(response.expires_in); + let mut delay = Duration::from_secs(response.interval); + loop { + std::thread::sleep(delay); + if Instant::now() >= deadline { + return Err(GraphError::OAuth( + "device code expired before user completed sign-in".to_owned(), + )); + } + let body = form_encode(&[ + ("client_id", client_id), + ("grant_type", "urn:ietf:params:oauth:grant-type:device_code"), + ("device_code", response.device_code.as_str()), + ]); + let mut resp = agent + .post(&token_endpoint) + .header("Content-Type", "application/x-www-form-urlencoded") + .send(body.as_bytes()) + .map_err(|e| GraphError::OAuth(format!("device code polling: {e}")))?; + let status = resp.status().as_u16(); + let payload: Value = resp.body_mut().read_json().unwrap_or(Value::Null); + match parse_token_response(status, &payload) { + TokenResponse::Ok(t) => return Ok(t), + TokenResponse::Pending => continue, + TokenResponse::SlowDown => { + delay = delay.saturating_add(Duration::from_secs(5)); + } + TokenResponse::Expired => { + return Err(GraphError::OAuth( + "device code expired; restart the import".to_owned(), + )); + } + TokenResponse::Declined => { + return Err(GraphError::OAuth( + "the user declined the device-code sign-in request".to_owned(), + )); + } + TokenResponse::BadCode => { + return Err(GraphError::OAuth("device code rejected".to_owned())); + } + TokenResponse::Other { error, description } => { + return Err(GraphError::OAuth(format!( + "device code polling: {error}: {description}" + ))); + } + } + } +} + +pub fn refresh_access_token( + authority: &str, + client_id: &str, + refresh_token: &str, + allow_invalid_certs: bool, +) -> Result { + let agent = build_agent(allow_invalid_certs); + let body = form_encode(&[ + ("client_id", client_id), + ("grant_type", "refresh_token"), + ("refresh_token", refresh_token), + ("scope", SCOPES), + ]); + let endpoint = token_endpoint(authority); + let mut delay = Duration::from_millis(500); + let mut attempts: u32 = 0; + loop { + attempts += 1; + let send = agent + .post(&endpoint) + .header("Content-Type", "application/x-www-form-urlencoded") + .send(body.as_bytes()); + let mut resp = match send { + Ok(r) => r, + Err(e) if attempts < 4 => { + std::thread::sleep(delay); + delay = (delay * 2).min(Duration::from_secs(8)); + eprintln!("graph token refresh transport error: {e}; retrying"); + continue; + } + Err(e) => return Err(GraphError::OAuth(format!("refresh endpoint: {e}"))), + }; + let status = resp.status().as_u16(); + let payload: Value = resp.body_mut().read_json().unwrap_or(Value::Null); + if matches!(status, 500..=599) && attempts < 4 { + std::thread::sleep(delay); + delay = (delay * 2).min(Duration::from_secs(8)); + continue; + } + return match parse_token_response(status, &payload) { + TokenResponse::Ok(t) => Ok(t), + other => Err(GraphError::OAuth(format!( + "refresh failed (http {status}): {other:?}" + ))), + }; + } +} + +fn form_encode(pairs: &[(&str, &str)]) -> String { + let mut out = String::new(); + for (i, (k, v)) in pairs.iter().enumerate() { + if i > 0 { + out.push('&'); + } + out.push_str(&urlencode(k)); + out.push('='); + out.push_str(&urlencode(v)); + } + out +} + +fn urlencode(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for b in s.bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(b as char); + } + _ => { + use std::fmt::Write; + let _ = write!(out, "%{b:02X}"); + } + } + } + out +} + +pub fn run_device_code_polling_against( + authority: &str, + client_id: &str, + response: &DeviceCodeResponse, + mut step: F, +) -> Result +where + F: FnMut(&str, &str) -> Result<(u16, Value), GraphError>, +{ + let token_endpoint = token_endpoint(authority); + let deadline = Instant::now() + Duration::from_secs(response.expires_in.max(1)); + let mut delay = Duration::from_millis(20); + loop { + if Instant::now() >= deadline { + return Err(GraphError::OAuth( + "device code expired before user completed sign-in".to_owned(), + )); + } + std::thread::sleep(delay); + let body = form_encode(&[ + ("client_id", client_id), + ("grant_type", "urn:ietf:params:oauth:grant-type:device_code"), + ("device_code", response.device_code.as_str()), + ]); + let (status, payload) = step(&token_endpoint, &body)?; + match parse_token_response(status, &payload) { + TokenResponse::Ok(t) => return Ok(t), + TokenResponse::Pending => continue, + TokenResponse::SlowDown => { + delay = delay.saturating_add(Duration::from_millis(20)); + } + TokenResponse::Expired => { + return Err(GraphError::OAuth("device code expired".to_owned())); + } + TokenResponse::Declined => { + return Err(GraphError::OAuth("device code declined by user".to_owned())); + } + TokenResponse::BadCode => { + return Err(GraphError::OAuth("device code rejected".to_owned())); + } + TokenResponse::Other { error, description } => { + return Err(GraphError::OAuth(format!("{error}: {description}"))); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_jwt(tid: &str, upn: &str, exp: u64) -> String { + let header = URL_SAFE_NO_PAD.encode(b"{\"alg\":\"none\"}"); + let claims = format!(r#"{{"tid":"{tid}","upn":"{upn}","exp":{exp}}}"#); + let payload = URL_SAFE_NO_PAD.encode(claims.as_bytes()); + format!("{header}.{payload}.") + } + + #[test] + fn jwt_claim_extraction() { + let token = make_jwt("t-1", "alice@x.com", 9999999999); + let claims = decode_jwt_claims(&token).unwrap(); + assert_eq!(claims.tenant_id.as_deref(), Some("t-1")); + assert_eq!(claims.upn.as_deref(), Some("alice@x.com")); + assert_eq!(claims.exp, Some(9999999999)); + } + + #[test] + fn malformed_token_returns_none() { + assert!(decode_jwt_claims("garbage").is_none()); + } + + #[test] + fn device_code_response_parses_required_fields() { + let json = serde_json::json!({ + "device_code": "ABCDEF", + "user_code": "XXX-YYY", + "verification_uri": "https://microsoft.com/devicelogin", + "interval": 5, + "expires_in": 900, + "message": "go sign in" + }); + let r = parse_device_code_response(&json).unwrap(); + assert_eq!(r.device_code, "ABCDEF"); + assert_eq!(r.user_code, "XXX-YYY"); + assert_eq!(r.interval, 5); + assert_eq!(r.expires_in, 900); + } + + #[test] + fn token_response_pending() { + let json = serde_json::json!({ + "error": "authorization_pending", + "error_description": "user has not yet completed" + }); + assert!(matches!( + parse_token_response(400, &json), + TokenResponse::Pending + )); + } + + #[test] + fn token_response_declined() { + let json = serde_json::json!({"error": "authorization_declined"}); + assert!(matches!( + parse_token_response(400, &json), + TokenResponse::Declined + )); + } + + #[test] + fn token_response_bad_code() { + let json = serde_json::json!({"error": "bad_verification_code"}); + assert!(matches!( + parse_token_response(400, &json), + TokenResponse::BadCode + )); + } + + #[test] + fn token_response_expired() { + let json = serde_json::json!({"error": "expired_token"}); + assert!(matches!( + parse_token_response(400, &json), + TokenResponse::Expired + )); + } + + #[test] + fn token_response_success_includes_refresh() { + let token = make_jwt("t-2", "bob@x.com", 9999999999); + let json = serde_json::json!({ + "access_token": token, + "refresh_token": "REFRESH", + "expires_in": 3599 + }); + match parse_token_response(200, &json) { + TokenResponse::Ok(t) => { + assert!(t.access_token.starts_with("eyJhbGciOiJub25lIn0")); + assert_eq!(t.refresh_token.as_deref(), Some("REFRESH")); + assert_eq!(t.expires_in, Some(3599)); + assert_eq!(t.upn.as_deref(), Some("bob@x.com")); + } + other => panic!("expected Ok, got {other:?}"), + } + } + + #[test] + fn urlencode_escapes_special_chars() { + assert_eq!(urlencode("a b/c"), "a%20b%2Fc"); + assert_eq!(urlencode("plain.123_-"), "plain.123_-"); + } + + #[test] + fn default_authority_uses_login_microsoftonline_com() { + assert_eq!( + default_authority("common"), + "https://login.microsoftonline.com/common" + ); + assert_eq!( + default_authority("contoso.onmicrosoft.com"), + "https://login.microsoftonline.com/contoso.onmicrosoft.com" + ); + } +} diff --git a/src/exchange_graph/recurrence.rs b/src/exchange_graph/recurrence.rs new file mode 100644 index 0000000..2c44dd7 --- /dev/null +++ b/src/exchange_graph/recurrence.rs @@ -0,0 +1,277 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::{Value, json}; + +use crate::exchange_graph::error::GraphError; + +pub fn convert_patterned_recurrence(pr: &Value) -> Result { + Ok(Value::Object(convert_patterned_recurrence_rule(pr)?)) +} + +pub fn convert_patterned_recurrence_rule( + pr: &Value, +) -> Result, GraphError> { + let pattern = pr + .get("pattern") + .ok_or_else(|| GraphError::Malformed("recurrence.pattern missing".to_owned()))?; + let range = pr + .get("range") + .ok_or_else(|| GraphError::Malformed("recurrence.range missing".to_owned()))?; + + let mut rule = serde_json::Map::new(); + rule.insert("@type".to_owned(), Value::from("RecurrenceRule")); + rule.insert("frequency".to_owned(), Value::from(frequency_for(pattern))); + + if let Some(interval) = pattern.get("interval").and_then(Value::as_u64) + && interval != 1 + { + rule.insert("interval".to_owned(), Value::from(interval)); + } + + if let Some(first_day) = pattern.get("firstDayOfWeek").and_then(Value::as_str) + && let Some(short) = day_short(first_day) + { + rule.insert("firstDayOfWeek".to_owned(), Value::from(short)); + } + + if let Some(days) = pattern.get("daysOfWeek").and_then(Value::as_array) { + let by_day: Vec = days + .iter() + .filter_map(Value::as_str) + .filter_map(day_short) + .map(|d| json!({"@type": "NDay", "day": d})) + .collect(); + if !by_day.is_empty() { + rule.insert("byDay".to_owned(), Value::Array(by_day)); + } + } + + if let Some(dom) = pattern.get("dayOfMonth").and_then(Value::as_i64) + && dom != 0 + { + rule.insert( + "byMonthDay".to_owned(), + Value::Array(vec![Value::from(dom)]), + ); + } + + if let Some(month) = pattern.get("month").and_then(Value::as_i64) + && month != 0 + { + rule.insert( + "byMonth".to_owned(), + Value::Array(vec![Value::String(month.to_string())]), + ); + } + + if let Some(index) = pattern.get("index").and_then(Value::as_str) + && let Some(setpos) = set_position_for(index) + { + rule.insert( + "bySetPosition".to_owned(), + Value::Array(vec![Value::from(setpos)]), + ); + } + + let range_type = range.get("type").and_then(Value::as_str).unwrap_or("noEnd"); + match range_type { + "endDate" => { + if let Some(end) = range.get("endDate").and_then(Value::as_str) { + let local = if end.contains('T') { + end.to_owned() + } else { + format!("{end}T23:59:59") + }; + rule.insert("until".to_owned(), Value::from(local)); + } + } + "numbered" => { + if let Some(count) = range.get("numberOfOccurrences").and_then(Value::as_u64) { + rule.insert("count".to_owned(), Value::from(count)); + } + } + _ => {} + } + + Ok(rule) +} + +fn frequency_for(pattern: &Value) -> &'static str { + match pattern.get("type").and_then(Value::as_str).unwrap_or("") { + "daily" => "daily", + "weekly" => "weekly", + "absoluteMonthly" | "relativeMonthly" => "monthly", + "absoluteYearly" | "relativeYearly" => "yearly", + _ => "daily", + } +} + +fn day_short(name: &str) -> Option<&'static str> { + match name.to_ascii_lowercase().as_str() { + "monday" => Some("mo"), + "tuesday" => Some("tu"), + "wednesday" => Some("we"), + "thursday" => Some("th"), + "friday" => Some("fr"), + "saturday" => Some("sa"), + "sunday" => Some("su"), + _ => None, + } +} + +fn set_position_for(index: &str) -> Option { + match index.to_ascii_lowercase().as_str() { + "first" => Some(1), + "second" => Some(2), + "third" => Some(3), + "fourth" => Some(4), + "last" => Some(-1), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn daily_with_interval() { + let pr = json!({ + "pattern": {"type": "daily", "interval": 3}, + "range": {"type": "noEnd"} + }); + let out = convert_patterned_recurrence(&pr).unwrap(); + let rule = &out; + assert_eq!(rule["frequency"], "daily"); + assert_eq!(rule["interval"], 3); + assert!(rule.get("until").is_none()); + assert!(rule.get("count").is_none()); + } + + #[test] + fn weekly_with_days_of_week() { + let pr = json!({ + "pattern": { + "type": "weekly", + "interval": 1, + "daysOfWeek": ["monday", "wednesday", "friday"], + "firstDayOfWeek": "monday" + }, + "range": {"type": "noEnd"} + }); + let out = convert_patterned_recurrence(&pr).unwrap(); + let rule = &out; + assert_eq!(rule["frequency"], "weekly"); + assert_eq!(rule["firstDayOfWeek"], "mo"); + let by_day = rule["byDay"].as_array().unwrap(); + assert_eq!(by_day.len(), 3); + assert_eq!(by_day[0]["day"], "mo"); + assert_eq!(by_day[1]["day"], "we"); + assert_eq!(by_day[2]["day"], "fr"); + } + + #[test] + fn absolute_monthly_uses_by_month_day() { + let pr = json!({ + "pattern": {"type": "absoluteMonthly", "interval": 1, "dayOfMonth": 15}, + "range": {"type": "noEnd"} + }); + let out = convert_patterned_recurrence(&pr).unwrap(); + let rule = &out; + assert_eq!(rule["frequency"], "monthly"); + assert_eq!(rule["byMonthDay"][0], 15); + } + + #[test] + fn relative_monthly_emits_by_set_position() { + let pr = json!({ + "pattern": { + "type": "relativeMonthly", + "interval": 1, + "daysOfWeek": ["thursday"], + "index": "third" + }, + "range": {"type": "noEnd"} + }); + let out = convert_patterned_recurrence(&pr).unwrap(); + let rule = &out; + assert_eq!(rule["frequency"], "monthly"); + assert_eq!(rule["bySetPosition"][0], 3); + assert_eq!(rule["byDay"][0]["day"], "th"); + } + + #[test] + fn absolute_yearly_with_month_and_day() { + let pr = json!({ + "pattern": {"type": "absoluteYearly", "interval": 1, "month": 7, "dayOfMonth": 4}, + "range": {"type": "noEnd"} + }); + let out = convert_patterned_recurrence(&pr).unwrap(); + let rule = &out; + assert_eq!(rule["frequency"], "yearly"); + assert_eq!(rule["byMonth"][0], "7"); + assert_eq!(rule["byMonthDay"][0], 4); + } + + #[test] + fn relative_yearly_uses_setpos_and_byday() { + let pr = json!({ + "pattern": { + "type": "relativeYearly", "interval": 1, "daysOfWeek": ["monday"], + "index": "last", "month": 5 + }, + "range": {"type": "noEnd"} + }); + let out = convert_patterned_recurrence(&pr).unwrap(); + let rule = &out; + assert_eq!(rule["frequency"], "yearly"); + assert_eq!(rule["bySetPosition"][0], -1); + assert_eq!(rule["byMonth"][0], "5"); + } + + #[test] + fn range_end_date_maps_to_until_local_datetime() { + let pr = json!({ + "pattern": {"type": "daily", "interval": 1}, + "range": {"type": "endDate", "endDate": "2026-12-31"} + }); + let out = convert_patterned_recurrence(&pr).unwrap(); + assert_eq!(out["until"], "2026-12-31T23:59:59"); + } + + #[test] + fn range_end_datetime_preserved() { + let pr = json!({ + "pattern": {"type": "daily", "interval": 1}, + "range": {"type": "endDate", "endDate": "2026-12-31T23:59:59"} + }); + let out = convert_patterned_recurrence(&pr).unwrap(); + assert_eq!(out["until"], "2026-12-31T23:59:59"); + } + + #[test] + fn range_numbered_maps_to_count() { + let pr = json!({ + "pattern": {"type": "weekly", "interval": 1, "daysOfWeek": ["monday"]}, + "range": {"type": "numbered", "numberOfOccurrences": 10} + }); + let out = convert_patterned_recurrence(&pr).unwrap(); + assert_eq!(out["count"], 10); + } + + #[test] + fn no_end_range_omits_until_and_count() { + let pr = json!({ + "pattern": {"type": "daily", "interval": 1}, + "range": {"type": "noEnd"} + }); + let out = convert_patterned_recurrence(&pr).unwrap(); + assert!(out.get("until").is_none()); + assert!(out.get("count").is_none()); + } +} diff --git a/src/exchange_graph/retry.rs b/src/exchange_graph/retry.rs new file mode 100644 index 0000000..c9b69f9 --- /dev/null +++ b/src/exchange_graph/retry.rs @@ -0,0 +1,78 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HttpClass { + Success, + Retryable, + Auth, + Vanished, + Fatal, +} + +pub fn classify_http_status(status: u16) -> HttpClass { + match status { + 200..=299 => HttpClass::Success, + 401 => HttpClass::Auth, + 403 => HttpClass::Auth, + 404 | 410 => HttpClass::Vanished, + 429 | 500 | 502 | 503 | 504 => HttpClass::Retryable, + _ => HttpClass::Fatal, + } +} + +pub fn is_throttled(status: u16) -> bool { + matches!(status, 429 | 503) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rate_limit_status_is_retryable() { + assert_eq!(classify_http_status(429), HttpClass::Retryable); + assert!(is_throttled(429)); + } + + #[test] + fn service_unavailable_is_retryable_and_throttle_marked() { + assert_eq!(classify_http_status(503), HttpClass::Retryable); + assert!(is_throttled(503)); + } + + #[test] + fn bad_gateway_and_timeout_retry() { + assert_eq!(classify_http_status(502), HttpClass::Retryable); + assert_eq!(classify_http_status(504), HttpClass::Retryable); + } + + #[test] + fn insufficient_storage_is_per_item_fatal() { + assert_eq!(classify_http_status(507), HttpClass::Fatal); + } + + #[test] + fn auth_failure_is_marked_distinctly() { + assert_eq!(classify_http_status(401), HttpClass::Auth); + assert_eq!(classify_http_status(403), HttpClass::Auth); + } + + #[test] + fn not_found_and_gone_are_vanished() { + assert_eq!(classify_http_status(404), HttpClass::Vanished); + assert_eq!(classify_http_status(410), HttpClass::Vanished); + } + + #[test] + fn unknown_4xx_is_fatal() { + assert_eq!(classify_http_status(400), HttpClass::Fatal); + assert_eq!(classify_http_status(405), HttpClass::Fatal); + assert_eq!(classify_http_status(408), HttpClass::Fatal); + assert_eq!(classify_http_status(422), HttpClass::Fatal); + assert_eq!(classify_http_status(423), HttpClass::Fatal); + } +} diff --git a/src/exchange_graph/types.rs b/src/exchange_graph/types.rs new file mode 100644 index 0000000..a0210c2 --- /dev/null +++ b/src/exchange_graph/types.rs @@ -0,0 +1,116 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::error::Error; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MailboxKind { + Primary, + Archive, +} + +impl MailboxKind { + pub fn parse(value: &str) -> Result { + match value { + "primary" => Ok(MailboxKind::Primary), + "archive" => Ok(MailboxKind::Archive), + "public-folders" => Err(Error::Usage( + "Microsoft Graph does not expose public folders. Run `vandelay import exchange-ews --mailbox-kind public-folders` instead.".to_owned(), + )), + other => Err(Error::Usage(format!( + "--mailbox-kind must be primary | archive, got {other:?}" + ))), + } + } + + pub fn as_str(self) -> &'static str { + match self { + MailboxKind::Primary => "primary", + MailboxKind::Archive => "archive", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EventBodyFormat { + Text, + Html, +} + +impl EventBodyFormat { + pub fn parse(value: &str) -> Result { + match value { + "text" => Ok(EventBodyFormat::Text), + "html" => Ok(EventBodyFormat::Html), + other => Err(Error::Usage(format!( + "--event-body-format must be text | html, got {other:?}" + ))), + } + } + + pub fn prefer_value(self) -> &'static str { + match self { + EventBodyFormat::Text => "outlook.body-content-type=\"text\"", + EventBodyFormat::Html => "outlook.body-content-type=\"html\"", + } + } +} + +#[derive(Debug, Clone)] +pub struct ResolvedPrincipal { + pub id: String, + pub user_principal_name: String, +} + +pub fn synthetic_account_id(directory_id: &str, kind: MailboxKind) -> String { + match kind { + MailboxKind::Primary => directory_id.to_owned(), + MailboxKind::Archive => format!("{directory_id}#archive"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_primary_and_archive() { + assert_eq!(MailboxKind::parse("primary").unwrap(), MailboxKind::Primary); + assert_eq!(MailboxKind::parse("archive").unwrap(), MailboxKind::Archive); + } + + #[test] + fn public_folders_is_redirected_to_ews() { + let err = MailboxKind::parse("public-folders").unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("public folders")); + assert!(msg.contains("exchange-ews")); + } + + #[test] + fn synthetic_id_matches_ews_pattern() { + assert_eq!( + synthetic_account_id("u-uuid", MailboxKind::Primary), + "u-uuid" + ); + assert_eq!( + synthetic_account_id("u-uuid", MailboxKind::Archive), + "u-uuid#archive" + ); + } + + #[test] + fn body_format_prefer_value() { + assert_eq!( + EventBodyFormat::Text.prefer_value(), + "outlook.body-content-type=\"text\"" + ); + assert_eq!( + EventBodyFormat::Html.prefer_value(), + "outlook.body-content-type=\"html\"" + ); + } +} diff --git a/src/imap/automap.rs b/src/imap/automap.rs new file mode 100644 index 0000000..52d1a94 --- /dev/null +++ b/src/imap/automap.rs @@ -0,0 +1,236 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub fn role_from_special_use(attributes: &[String]) -> Option<&'static str> { + for attr in attributes { + match attr.to_ascii_lowercase().as_str() { + "\\all" => return Some("all"), + "\\archive" => return Some("archive"), + "\\drafts" => return Some("drafts"), + "\\flagged" => return Some("flagged"), + "\\important" => return Some("important"), + "\\junk" => return Some("junk"), + "\\sent" => return Some("sent"), + "\\trash" => return Some("trash"), + _ => {} + } + } + None +} + +pub fn role_for_folder( + folder_name: &str, + attributes: &[String], + namespace_prefix: &str, + automap_enabled: bool, +) -> Option<&'static str> { + if folder_name.eq_ignore_ascii_case("INBOX") { + return Some("inbox"); + } + if let Some(r) = role_from_special_use(attributes) { + return Some(r); + } + if !automap_enabled { + return None; + } + let stripped = strip_prefix_case_insensitive(folder_name, namespace_prefix); + let leaf = leaf_component(stripped); + role_from_name(stripped, leaf) +} + +fn role_from_name(full: &str, leaf: &str) -> Option<&'static str> { + for (role, patterns) in ROLE_PATTERNS { + for p in *patterns { + if leaf.eq_ignore_ascii_case(p) || full.eq_ignore_ascii_case(p) { + return Some(role); + } + } + } + None +} + +fn strip_prefix_case_insensitive<'a>(name: &'a str, prefix: &str) -> &'a str { + if prefix.is_empty() { + return name; + } + if name.len() < prefix.len() { + return name; + } + if name[..prefix.len()].eq_ignore_ascii_case(prefix) { + &name[prefix.len()..] + } else { + name + } +} + +fn leaf_component(name: &str) -> &str { + if let Some(idx) = name.rfind(['/', '.']) { + &name[idx + 1..] + } else { + name + } +} + +const ROLE_PATTERNS: &[(&str, &[&str])] = &[ + ( + "sent", + &[ + "Sent", + "Sent Mail", + "Sent Items", + "Sent Messages", + "Sent-Mail", + "[Gmail]/Sent Mail", + "[Gmail]/Sent", + ], + ), + ( + "drafts", + &["Drafts", "Draft", "Brouillons", "[Gmail]/Drafts"], + ), + ( + "trash", + &[ + "Trash", + "Deleted", + "Deleted Items", + "Deleted Messages", + "[Gmail]/Trash", + "[Gmail]/Bin", + ], + ), + ( + "junk", + &["Junk", "Spam", "Junk E-mail", "Junk Email", "[Gmail]/Spam"], + ), + ("all", &["[Gmail]/All Mail"]), + ("archive", &["Archive", "Archives"]), +]; + +#[cfg(test)] +mod tests { + use super::*; + + fn attrs(s: &[&str]) -> Vec { + s.iter().map(|x| (*x).to_owned()).collect() + } + + #[test] + fn inbox_always_maps_to_inbox() { + assert_eq!(role_for_folder("INBOX", &[], "", true), Some("inbox")); + assert_eq!(role_for_folder("inbox", &[], "", true), Some("inbox")); + assert_eq!(role_for_folder("Inbox", &[], "", false), Some("inbox")); + } + + #[test] + fn special_use_drives_role() { + let r = role_for_folder("Whatever", &attrs(&["\\Sent"]), "", false); + assert_eq!(r, Some("sent")); + let r = role_for_folder("Anywhere", &attrs(&["\\Drafts"]), "", true); + assert_eq!(r, Some("drafts")); + } + + #[test] + fn special_use_table_covers_all_iana_roles() { + for (attr, expected) in [ + ("\\All", "all"), + ("\\Archive", "archive"), + ("\\Drafts", "drafts"), + ("\\Flagged", "flagged"), + ("\\Important", "important"), + ("\\Junk", "junk"), + ("\\Sent", "sent"), + ("\\Trash", "trash"), + ] { + let r = role_from_special_use(&attrs(&[attr])); + assert_eq!(r, Some(expected), "for {attr}"); + } + } + + #[test] + fn special_use_attrs_unrelated_to_roles_yield_none() { + for s in [ + "\\Subscribed", + "\\HasChildren", + "\\HasNoChildren", + "\\Marked", + "\\Unmarked", + "\\Noselect", + "\\Noinferiors", + "\\Remote", + ] { + assert_eq!(role_from_special_use(&attrs(&[s])), None); + } + } + + #[test] + fn name_heuristic_when_no_special_use() { + assert_eq!(role_for_folder("Sent", &[], "", true), Some("sent")); + assert_eq!(role_for_folder("Sent Items", &[], "", true), Some("sent")); + assert_eq!(role_for_folder("Drafts", &[], "", true), Some("drafts")); + assert_eq!(role_for_folder("Trash", &[], "", true), Some("trash")); + assert_eq!( + role_for_folder("Deleted Items", &[], "", true), + Some("trash") + ); + assert_eq!(role_for_folder("Junk", &[], "", true), Some("junk")); + assert_eq!(role_for_folder("Spam", &[], "", true), Some("junk")); + assert_eq!(role_for_folder("Archive", &[], "", true), Some("archive")); + assert_eq!( + role_for_folder("[Gmail]/All Mail", &[], "", true), + Some("all") + ); + assert_eq!(role_for_folder("Archives", &[], "", true), Some("archive")); + } + + #[test] + fn name_heuristic_disabled_when_automap_off() { + assert_eq!(role_for_folder("Sent", &[], "", false), None); + assert_eq!(role_for_folder("Trash", &[], "", false), None); + } + + #[test] + fn namespace_prefix_stripped_before_name_match() { + assert_eq!( + role_for_folder("INBOX.Sent", &[], "INBOX.", true), + Some("sent") + ); + assert_eq!( + role_for_folder("INBOX.Drafts", &[], "INBOX.", true), + Some("drafts") + ); + } + + #[test] + fn unknown_folder_yields_none() { + assert_eq!(role_for_folder("My Custom Folder", &[], "", true), None); + assert_eq!(role_for_folder("Projects/Alpha", &[], "", true), None); + } + + #[test] + fn case_insensitive_name_match() { + assert_eq!(role_for_folder("sent", &[], "", true), Some("sent")); + assert_eq!(role_for_folder("DRAFTS", &[], "", true), Some("drafts")); + } + + #[test] + fn all_special_use_wins_over_heuristic_archive() { + let r = role_for_folder("[Gmail]/All Mail", &attrs(&["\\All"]), "", true); + assert_eq!(r, Some("all")); + } + + #[test] + fn gmail_all_mail_no_attribute_is_all_not_archive() { + let r = role_for_folder("[Gmail]/All Mail", &[], "", true); + assert_eq!(r, Some("all")); + } + + #[test] + fn plain_archive_name_resolves_to_archive() { + assert_eq!(role_for_folder("Archive", &[], "", true), Some("archive")); + assert_eq!(role_for_folder("Archives", &[], "", true), Some("archive")); + } +} diff --git a/src/imap/client.rs b/src/imap/client.rs new file mode 100644 index 0000000..89ae2a6 --- /dev/null +++ b/src/imap/client.rs @@ -0,0 +1,652 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::BTreeSet; +use std::io::{BufReader, Read, Write}; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64; + +use super::command::{self, CommandBuilder}; +use super::error::{ImapError, NoError}; +use super::response::{Response, Status, StatusLine, Untagged, parse_response}; +use super::transport::{Connector, ImapStream}; + +pub struct ImapClient { + reader: BufReader>, + tags: CommandBuilder, + pub capabilities: BTreeSet, + pub host: String, + closed: bool, + utf8_accept: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConnectMode { + ImplicitTls, + Plain, + StartTls, +} + +#[derive(Debug, Clone)] +pub struct Greeting { + pub status: Status, + pub text: String, + pub code: Option, +} + +impl ImapClient { + pub fn connect( + connector: &Connector, + host: &str, + port: u16, + mode: ConnectMode, + ) -> Result { + let stream: Box = match mode { + ConnectMode::ImplicitTls => connector.connect_tls(host, port)?, + ConnectMode::Plain | ConnectMode::StartTls => connector.connect_plain(host, port)?, + }; + let mut client = ImapClient { + reader: BufReader::new(stream), + tags: CommandBuilder::new(), + capabilities: BTreeSet::new(), + host: host.to_owned(), + closed: false, + utf8_accept: false, + }; + client.read_greeting()?; + client.send_capability()?; + if matches!(mode, ConnectMode::StartTls) { + if !client.has_capability("STARTTLS") { + return Err(ImapError::Unsupported( + "server does not advertise STARTTLS but cleartext credentials are forbidden" + .into(), + )); + } + client.starttls(connector)?; + client.send_capability()?; + } + Ok(client) + } + + pub fn has_capability(&self, name: &str) -> bool { + self.capabilities + .iter() + .any(|c| c.eq_ignore_ascii_case(name)) + } + + fn read_greeting(&mut self) -> Result { + let resp = parse_response(&mut self.reader)?; + match resp { + Response::Untagged(Untagged::StatusLine(line)) => { + if let Some(code) = &line.code + && code == "CAPABILITY" + && let Some(args) = &line.code_args + { + self.capabilities = parse_caps(args); + } + Ok(Greeting { + status: line.status, + text: line.text, + code: line.code, + }) + } + Response::Untagged(Untagged::Bye(text)) => Err(ImapError::Bye(text)), + other => Err(ImapError::Protocol(format!( + "expected greeting, got {other:?}" + ))), + } + } + + pub fn refresh_capabilities(&mut self) -> Result<(), ImapError> { + self.send_capability() + } + + fn send_capability(&mut self) -> Result<(), ImapError> { + let resp = self.run_collect(command::capability())?; + for u in resp.untagged { + if let Untagged::Capability(caps) = u { + self.capabilities = caps.into_iter().collect(); + } + } + if self.capabilities.is_empty() { + return Err(ImapError::Protocol( + "server returned no CAPABILITY response".into(), + )); + } + Ok(()) + } + + fn starttls(&mut self, connector: &Connector) -> Result<(), ImapError> { + self.run_collect(command::starttls())?; + let stream = self.reader.get_ref(); + if stream.is_tls() { + return Err(ImapError::Protocol( + "STARTTLS issued but stream already TLS".into(), + )); + } + let buffered = std::mem::replace(&mut self.reader, BufReader::new(noop_stream())); + let inner = buffered.into_inner(); + let upgraded = inner.upgrade_tls(connector, &self.host)?; + self.reader = BufReader::new(upgraded); + Ok(()) + } + + pub fn utf8_accept(&self) -> bool { + self.utf8_accept + } + + pub fn enable(&mut self, extensions: &[&str]) -> Result, ImapError> { + if !self.has_capability("ENABLE") { + return Err(ImapError::Unsupported( + "server does not advertise ENABLE".into(), + )); + } + let resp = self.run_collect(&command::enable(extensions))?; + let mut enabled: Vec = Vec::new(); + for u in resp.untagged { + if let Untagged::Enabled(exts) = u { + enabled.extend(exts); + } + } + if enabled + .iter() + .any(|e| e.eq_ignore_ascii_case("UTF8=ACCEPT")) + { + self.utf8_accept = true; + } + Ok(enabled) + } + + pub fn compress_deflate(&mut self) -> Result<(), ImapError> { + if !self.has_capability("COMPRESS=DEFLATE") { + return Err(ImapError::Unsupported( + "server does not advertise COMPRESS=DEFLATE".into(), + )); + } + self.run_collect(command::compress_deflate())?; + let buffered = std::mem::replace(&mut self.reader, BufReader::new(noop_stream())); + let primer = buffered.buffer().to_vec(); + let inner = buffered.into_inner(); + let deflated = super::transport::DeflateImapStream::wrap(inner, &primer); + self.reader = BufReader::new(deflated); + Ok(()) + } + + pub fn login(&mut self, user: &str, password: &str) -> Result<(), ImapError> { + if self.capabilities.iter().any(|c| c == "LOGINDISABLED") { + return Err(ImapError::Unsupported( + "server advertises LOGINDISABLED".into(), + )); + } + let cmd = command::login(user, password); + match self.run_collect(&cmd) { + Ok(_) => Ok(()), + Err(ImapError::No(no)) if no.is_auth_failed() => Err(ImapError::AuthFailed(no.text)), + Err(e) => Err(e), + } + } + + pub fn authenticate_plain(&mut self, authcid: &str, password: &str) -> Result<(), ImapError> { + let mut payload = Vec::with_capacity(authcid.len() + password.len() + 3); + payload.push(0); + payload.extend_from_slice(authcid.as_bytes()); + payload.push(0); + payload.extend_from_slice(password.as_bytes()); + let encoded = BASE64.encode(&payload); + if self.has_capability("SASL-IR") { + let cmd = command::authenticate_with_ir("PLAIN", &encoded); + return match self.run_collect(&cmd) { + Ok(_) => Ok(()), + Err(ImapError::No(no)) if no.is_auth_failed() => { + Err(ImapError::AuthFailed(no.text)) + } + Err(e) => Err(e), + }; + } + let (tag, bytes) = self.tags.build(&command::authenticate("PLAIN")); + self.write_all(&bytes)?; + loop { + let resp = parse_response(&mut self.reader)?; + match resp { + Response::Continuation(_) => { + let mut line = encoded.clone().into_bytes(); + line.extend_from_slice(b"\r\n"); + self.write_all(&line)?; + } + Response::Tagged { tag: t, line } if t == tag => { + self.update_capabilities_from_code(&line); + return self.finish_tagged(line); + } + Response::Untagged(Untagged::Capability(caps)) => { + self.capabilities = caps.into_iter().collect(); + } + _ => {} + } + } + } + + pub fn authenticate_xoauth2(&mut self, user: &str, token: &str) -> Result<(), ImapError> { + let mut payload = String::with_capacity(user.len() + token.len() + 32); + payload.push_str("user="); + payload.push_str(user); + payload.push('\x01'); + payload.push_str("auth=Bearer "); + payload.push_str(token); + payload.push('\x01'); + payload.push('\x01'); + self.drive_bearer_sasl("XOAUTH2", payload.as_bytes()) + } + + pub fn authenticate_oauthbearer(&mut self, user: &str, token: &str) -> Result<(), ImapError> { + let mut payload = String::with_capacity(user.len() + token.len() + 32); + payload.push_str("n,a="); + payload.push_str(user); + payload.push(','); + payload.push('\x01'); + payload.push_str("auth=Bearer "); + payload.push_str(token); + payload.push('\x01'); + payload.push('\x01'); + self.drive_bearer_sasl("OAUTHBEARER", payload.as_bytes()) + } + + fn drive_bearer_sasl(&mut self, mechanism: &str, payload: &[u8]) -> Result<(), ImapError> { + let encoded = BASE64.encode(payload); + if self.has_capability("SASL-IR") { + let cmd = command::authenticate_with_ir(mechanism, &encoded); + return match self.run_collect(&cmd) { + Ok(_) => Ok(()), + Err(ImapError::No(no)) if no.is_auth_failed() => { + Err(ImapError::AuthFailed(no.text)) + } + Err(e) => Err(e), + }; + } + let (tag, bytes) = self.tags.build(&command::authenticate(mechanism)); + self.write_all(&bytes)?; + let mut sent = false; + loop { + let resp = parse_response(&mut self.reader)?; + match resp { + Response::Continuation(_) => { + if sent { + self.write_all(b"\r\n")?; + } else { + let mut line = encoded.clone().into_bytes(); + line.extend_from_slice(b"\r\n"); + self.write_all(&line)?; + sent = true; + } + } + Response::Tagged { tag: t, line } if t == tag => { + self.update_capabilities_from_code(&line); + return self.finish_tagged(line); + } + Response::Untagged(Untagged::Capability(caps)) => { + self.capabilities = caps.into_iter().collect(); + } + _ => {} + } + } + } + + fn update_capabilities_from_code(&mut self, line: &StatusLine) { + if line.code.as_deref() == Some("CAPABILITY") + && let Some(args) = &line.code_args + { + self.capabilities = parse_caps(args); + } + } + + pub fn noop(&mut self) -> Result { + self.run_collect(command::noop()) + } + + pub fn logout(&mut self) -> Result<(), ImapError> { + if self.closed { + return Ok(()); + } + let _ = self.run_collect(command::logout()); + self.closed = true; + Ok(()) + } + + pub fn run(&mut self, command: &str) -> Result { + self.run_collect(command) + } + + pub fn run_streamed( + &mut self, + command: &str, + mut on_untagged: F, + ) -> Result + where + F: FnMut(Untagged), + { + let (tag, bytes) = self.tags.build(command); + if command::contains_literal(command.as_bytes()) + && !self.has_capability("LITERAL+") + && !self.has_capability("LITERAL-") + { + return Err(ImapError::Unsupported( + "command contains a non-ASCII literal but server does not advertise LITERAL+ or LITERAL-" + .into(), + )); + } + self.write_all(&bytes)?; + loop { + let resp = parse_response(&mut self.reader)?; + match resp { + Response::Tagged { tag: t, line } if t == tag => { + self.update_capabilities_from_code(&line); + return match line.status { + Status::Ok => Ok(line), + Status::No => Err(ImapError::No(line.into_no_error())), + Status::Bad => Err(ImapError::Bad(line.text)), + Status::Bye => Err(ImapError::Bye(line.text)), + Status::PreAuth => Err(ImapError::Protocol( + "unexpected PREAUTH on tagged response".into(), + )), + }; + } + Response::Tagged { tag: other, .. } => { + return Err(ImapError::Protocol(format!( + "expected tag {tag}, got {other}" + ))); + } + Response::Untagged(Untagged::Bye(text)) => { + self.closed = true; + return Err(ImapError::Bye(text)); + } + Response::Untagged(u) => { + if let Untagged::Capability(caps) = &u { + self.capabilities = caps.iter().cloned().collect(); + } + on_untagged(u); + } + Response::Continuation(_) => { + return Err(ImapError::Protocol( + "unexpected continuation outside AUTHENTICATE".into(), + )); + } + } + } + } + + pub fn run_collect(&mut self, command: &str) -> Result { + let (tag, bytes) = self.tags.build(command); + if command::contains_literal(command.as_bytes()) + && !self.has_capability("LITERAL+") + && !self.has_capability("LITERAL-") + { + return Err(ImapError::Unsupported( + "command contains a non-ASCII literal but server does not advertise LITERAL+ or LITERAL-" + .into(), + )); + } + self.write_all(&bytes)?; + let mut untagged = Vec::new(); + loop { + let resp = parse_response(&mut self.reader)?; + match resp { + Response::Tagged { tag: t, line } if t == tag => { + self.update_capabilities_from_code(&line); + return match line.status { + Status::Ok => Ok(CollectedResponse { + tag, + line, + untagged, + }), + Status::No => Err(ImapError::No(line.into_no_error())), + Status::Bad => Err(ImapError::Bad(line.text)), + Status::Bye => Err(ImapError::Bye(line.text)), + Status::PreAuth => Err(ImapError::Protocol( + "unexpected PREAUTH on tagged response".into(), + )), + }; + } + Response::Tagged { tag: other, .. } => { + return Err(ImapError::Protocol(format!( + "expected tag {tag}, got {other}" + ))); + } + Response::Untagged(Untagged::Bye(text)) => { + self.closed = true; + return Err(ImapError::Bye(text)); + } + Response::Untagged(u) => { + if let Untagged::Capability(caps) = &u { + self.capabilities = caps.iter().cloned().collect(); + } + untagged.push(u); + } + Response::Continuation(_) => { + return Err(ImapError::Protocol( + "unexpected continuation outside AUTHENTICATE".into(), + )); + } + } + } + } + + fn finish_tagged(&mut self, line: StatusLine) -> Result<(), ImapError> { + match line.status { + Status::Ok => Ok(()), + Status::No => { + let no = NoError::new(line.text, line.code); + if no.is_auth_failed() { + Err(ImapError::AuthFailed(no.text)) + } else { + Err(ImapError::No(no)) + } + } + Status::Bad => Err(ImapError::Bad(line.text)), + Status::Bye => Err(ImapError::Bye(line.text)), + Status::PreAuth => Err(ImapError::Protocol( + "unexpected PREAUTH on tagged response".into(), + )), + } + } + + fn write_all(&mut self, bytes: &[u8]) -> Result<(), ImapError> { + let stream = self.reader.get_mut(); + stream.write_all(bytes)?; + stream.flush()?; + Ok(()) + } +} + +impl Drop for ImapClient { + fn drop(&mut self) { + if !self.closed { + let _ = self.logout(); + } + } +} + +#[derive(Debug)] +pub struct CollectedResponse { + pub tag: String, + pub line: StatusLine, + pub untagged: Vec, +} + +fn parse_caps(s: &str) -> BTreeSet { + s.split_ascii_whitespace().map(|w| w.to_owned()).collect() +} + +fn noop_stream() -> Box { + Box::new(NoopStream) +} + +struct NoopStream; + +impl Read for NoopStream { + fn read(&mut self, _buf: &mut [u8]) -> std::io::Result { + Ok(0) + } +} + +impl Write for NoopStream { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl ImapStream for NoopStream { + fn upgrade_tls( + self: Box, + _connector: &Connector, + _host: &str, + ) -> Result, ImapError> { + Ok(self) + } + fn is_tls(&self) -> bool { + false + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + struct MockStream { + reader: Cursor>, + written: Vec, + } + + impl MockStream { + fn boxed(server_says: &[u8]) -> Box { + Box::new(MockStream { + reader: Cursor::new(server_says.to_vec()), + written: Vec::new(), + }) + } + } + + impl Read for MockStream { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.reader.read(buf) + } + } + + impl Write for MockStream { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.written.extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl ImapStream for MockStream { + fn upgrade_tls( + self: Box, + _c: &Connector, + _h: &str, + ) -> Result, ImapError> { + Ok(self) + } + fn is_tls(&self) -> bool { + false + } + } + + fn client_with(server: &[u8]) -> ImapClient { + ImapClient { + reader: BufReader::new(MockStream::boxed(server)), + tags: CommandBuilder::new(), + capabilities: BTreeSet::new(), + host: "test.example".to_owned(), + closed: false, + utf8_accept: false, + } + } + + #[test] + fn run_collect_drains_untagged_and_returns_ok() { + let server = b"* CAPABILITY IMAP4rev1 STARTTLS\r\nA0001 OK done\r\n"; + let mut c = client_with(server); + let r = c.run_collect("CAPABILITY").unwrap(); + assert_eq!(r.untagged.len(), 1); + assert!(matches!(r.line.status, Status::Ok)); + } + + #[test] + fn run_collect_propagates_no_as_imaperror_no() { + let server = b"A0001 NO mailbox not found\r\n"; + let mut c = client_with(server); + let err = c.run_collect("SELECT bogus").unwrap_err(); + match err { + ImapError::No(no) => assert_eq!(no.text, "mailbox not found"), + other => panic!("expected No, got {other:?}"), + } + } + + #[test] + fn run_collect_propagates_bad_as_imaperror_bad() { + let server = b"A0001 BAD syntax error\r\n"; + let mut c = client_with(server); + let err = c.run_collect("BOGUS").unwrap_err(); + assert!(matches!(err, ImapError::Bad(_))); + } + + #[test] + fn run_collect_translates_authenticationfailed_via_login_path() { + let server = b"A0001 NO [AUTHENTICATIONFAILED] bad creds\r\n"; + let mut c = client_with(server); + let err = c.login("alice", "wrong").unwrap_err(); + assert!(matches!(err, ImapError::AuthFailed(_))); + } + + #[test] + fn login_refused_when_logindisabled() { + let server = b"A0001 OK any\r\n"; + let mut c = client_with(server); + c.capabilities.insert("LOGINDISABLED".to_owned()); + let err = c.login("a", "b").unwrap_err(); + assert!(matches!(err, ImapError::Unsupported(_))); + } + + #[test] + fn bye_mid_response_closes_client() { + let server = b"* BYE going away\r\n"; + let mut c = client_with(server); + let err = c.run_collect("NOOP").unwrap_err(); + assert!(matches!(err, ImapError::Bye(_))); + assert!(c.closed); + } + + #[test] + fn authenticate_plain_sasl_ir_path() { + let server = b"A0001 OK auth done\r\n"; + let mut c = client_with(server); + c.capabilities.insert("SASL-IR".to_owned()); + c.authenticate_plain("alice", "p@ss").unwrap(); + } + + #[test] + fn authenticate_plain_continuation_path() { + let server = b"+ \r\nA0001 OK auth done\r\n"; + let mut c = client_with(server); + c.authenticate_plain("alice", "p@ss").unwrap(); + } + + #[test] + fn has_capability_is_case_insensitive() { + let mut c = client_with(b""); + c.capabilities.insert("STARTTLS".to_owned()); + assert!(c.has_capability("starttls")); + assert!(c.has_capability("STARTTLS")); + assert!(!c.has_capability("MOVE")); + } +} diff --git a/src/imap/command.rs b/src/imap/command.rs new file mode 100644 index 0000000..0056bff --- /dev/null +++ b/src/imap/command.rs @@ -0,0 +1,384 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fmt::Write as _; + +pub struct CommandBuilder { + next_tag: u32, +} + +impl Default for CommandBuilder { + fn default() -> Self { + Self::new() + } +} + +impl CommandBuilder { + pub fn new() -> Self { + Self { next_tag: 1 } + } + + pub fn next_tag(&mut self) -> String { + let tag = format!("A{:04}", self.next_tag); + self.next_tag = self.next_tag.wrapping_add(1).max(1); + tag + } + + pub fn build(&mut self, command: &str) -> (String, Vec) { + let tag = self.next_tag(); + let mut out = String::with_capacity(tag.len() + command.len() + 4); + out.push_str(&tag); + out.push(' '); + out.push_str(command); + out.push_str("\r\n"); + (tag, out.into_bytes()) + } +} + +pub fn quote_astring(s: &str) -> String { + let needs_literal = s.bytes().any(|b| !(0x20..=0x7e).contains(&b)); + if needs_literal { + format!("{{{}+}}\r\n{}", s.len(), s) + } else { + let mut out = String::with_capacity(s.len() + 2); + out.push('"'); + for c in s.chars() { + if c == '"' || c == '\\' { + out.push('\\'); + } + out.push(c); + } + out.push('"'); + out + } +} + +pub fn contains_literal(bytes: &[u8]) -> bool { + let mut i = 0; + while i + 2 < bytes.len() { + if bytes[i] == b'{' { + let mut j = i + 1; + while j < bytes.len() && bytes[j].is_ascii_digit() { + j += 1; + } + if j < bytes.len() && bytes[j] == b'+' { + j += 1; + } + if j < bytes.len() && bytes[j] == b'}' { + return true; + } + } + i += 1; + } + false +} + +pub fn login(user: &str, password: &str) -> String { + format!("LOGIN {} {}", quote_astring(user), quote_astring(password)) +} + +pub fn authenticate(mechanism: &str) -> String { + format!("AUTHENTICATE {mechanism}") +} + +pub fn authenticate_with_ir(mechanism: &str, initial_response: &str) -> String { + format!("AUTHENTICATE {mechanism} {initial_response}") +} + +pub fn capability() -> &'static str { + "CAPABILITY" +} + +pub fn starttls() -> &'static str { + "STARTTLS" +} + +pub fn compress_deflate() -> &'static str { + "COMPRESS DEFLATE" +} + +pub fn noop() -> &'static str { + "NOOP" +} + +pub fn logout() -> &'static str { + "LOGOUT" +} + +pub fn namespace() -> &'static str { + "NAMESPACE" +} + +pub fn enable(extensions: &[&str]) -> String { + let mut out = String::from("ENABLE"); + for ext in extensions { + out.push(' '); + out.push_str(ext); + } + out +} + +pub fn list(reference: &str, pattern: &str) -> String { + format!( + "LIST {} {}", + quote_astring(reference), + quote_astring(pattern) + ) +} + +pub fn list_extended( + reference: &str, + patterns: &[&str], + selection: &[&str], + return_options: &[&str], +) -> String { + let mut out = String::from("LIST"); + if !selection.is_empty() { + out.push_str(" ("); + for (i, s) in selection.iter().enumerate() { + if i > 0 { + out.push(' '); + } + out.push_str(s); + } + out.push(')'); + } + out.push(' '); + out.push_str("e_astring(reference)); + if patterns.len() == 1 { + out.push(' '); + out.push_str("e_astring(patterns[0])); + } else { + out.push_str(" ("); + for (i, p) in patterns.iter().enumerate() { + if i > 0 { + out.push(' '); + } + out.push_str("e_astring(p)); + } + out.push(')'); + } + if !return_options.is_empty() { + out.push_str(" RETURN ("); + for (i, r) in return_options.iter().enumerate() { + if i > 0 { + out.push(' '); + } + out.push_str(r); + } + out.push(')'); + } + out +} + +pub fn lsub(reference: &str, pattern: &str) -> String { + format!( + "LSUB {} {}", + quote_astring(reference), + quote_astring(pattern) + ) +} + +pub fn select(mailbox: &str) -> String { + format!("SELECT {}", quote_astring(mailbox)) +} + +pub fn examine(mailbox: &str) -> String { + format!("EXAMINE {}", quote_astring(mailbox)) +} + +pub fn status(mailbox: &str, items: &[&str]) -> String { + let mut out = String::from("STATUS "); + out.push_str("e_astring(mailbox)); + out.push_str(" ("); + for (i, s) in items.iter().enumerate() { + if i > 0 { + out.push(' '); + } + out.push_str(s); + } + out.push(')'); + out +} + +pub fn uid_search_esearch_all() -> &'static str { + "UID SEARCH RETURN (ALL) ALL" +} + +pub fn uid_search_all() -> &'static str { + "UID SEARCH ALL" +} + +pub fn uid_fetch_all_uids() -> &'static str { + "UID FETCH 1:* (UID)" +} + +pub fn uid_fetch(set: &str, items: &[&str]) -> String { + let mut out = String::from("UID FETCH "); + out.push_str(set); + out.push_str(" ("); + for (i, s) in items.iter().enumerate() { + if i > 0 { + out.push(' '); + } + out.push_str(s); + } + out.push(')'); + out +} + +pub fn format_uid_set(uids: &[u32], is_sorted: bool) -> String { + if uids.is_empty() { + return String::new(); + } + let owned_sorted: Vec; + let slice: &[u32] = if is_sorted { + uids + } else { + let mut s = uids.to_vec(); + s.sort_unstable(); + s.dedup(); + owned_sorted = s; + &owned_sorted[..] + }; + + let mut out = String::with_capacity(slice.len() * 8); + let mut i = 0; + let mut first = true; + while i < slice.len() { + let start = slice[i]; + let mut end = start; + let mut j = i + 1; + while j < slice.len() && slice[j] == end + 1 { + end = slice[j]; + j += 1; + } + if !first { + out.push(','); + } + if start == end { + let _ = write!(&mut out, "{start}"); + } else { + let _ = write!(&mut out, "{start}:{end}"); + } + first = false; + i = j; + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn next_tag_increments() { + let mut b = CommandBuilder::new(); + assert_eq!(b.next_tag(), "A0001"); + assert_eq!(b.next_tag(), "A0002"); + assert_eq!(b.next_tag(), "A0003"); + } + + #[test] + fn build_appends_crlf() { + let mut b = CommandBuilder::new(); + let (tag, bytes) = b.build("NOOP"); + assert_eq!(tag, "A0001"); + assert_eq!(bytes, b"A0001 NOOP\r\n"); + } + + #[test] + fn quote_astring_quotes_safe_text() { + assert_eq!(quote_astring("hello"), "\"hello\""); + assert_eq!(quote_astring("INBOX"), "\"INBOX\""); + } + + #[test] + fn quote_astring_escapes_quote_and_backslash() { + assert_eq!(quote_astring("a\"b"), "\"a\\\"b\""); + assert_eq!(quote_astring("a\\b"), "\"a\\\\b\""); + } + + #[test] + fn quote_astring_switches_to_literal_for_non_ascii() { + let q = quote_astring("éclair"); + assert!(q.starts_with("{")); + assert!(q.contains("\r\n")); + assert!(q.ends_with("éclair")); + } + + #[test] + fn login_command_format() { + let s = login("alice", "p@ss"); + assert_eq!(s, "LOGIN \"alice\" \"p@ss\""); + } + + #[test] + fn list_extended_with_return_clause() { + let s = list_extended("", &["*"], &[], &["SPECIAL-USE", "SUBSCRIBED"]); + assert_eq!(s, "LIST \"\" \"*\" RETURN (SPECIAL-USE SUBSCRIBED)"); + } + + #[test] + fn list_extended_with_selection_and_multi_pattern() { + let s = list_extended("", &["INBOX", "Sent"], &["SUBSCRIBED"], &[]); + assert_eq!(s, "LIST (SUBSCRIBED) \"\" (\"INBOX\" \"Sent\")"); + } + + #[test] + fn status_items_serialise_in_order() { + let s = status("INBOX", &["UIDVALIDITY", "UIDNEXT", "MESSAGES"]); + assert_eq!(s, "STATUS \"INBOX\" (UIDVALIDITY UIDNEXT MESSAGES)"); + } + + #[test] + fn uid_fetch_with_items() { + let s = uid_fetch("1:100", &["UID", "FLAGS", "INTERNALDATE", "RFC822.SIZE"]); + assert_eq!(s, "UID FETCH 1:100 (UID FLAGS INTERNALDATE RFC822.SIZE)"); + } + + #[test] + fn format_uid_set_collapses_consecutive_sorted_input() { + assert_eq!(format_uid_set(&[], true), ""); + assert_eq!(format_uid_set(&[5], true), "5"); + assert_eq!(format_uid_set(&[1, 2, 3, 5, 7, 8], true), "1:3,5,7:8"); + } + + #[test] + fn format_uid_set_sorts_when_caller_unsure() { + assert_eq!(format_uid_set(&[3, 1, 2], false), "1:3"); + assert_eq!(format_uid_set(&[1, 1, 1, 2, 2, 3], false), "1:3"); + } + + #[test] + fn enable_with_multiple_extensions() { + assert_eq!( + enable(&["UTF8=ACCEPT", "CONDSTORE"]), + "ENABLE UTF8=ACCEPT CONDSTORE" + ); + } + + #[test] + fn quote_astring_uses_literal_plus_for_non_ascii() { + let q = quote_astring("éclair"); + assert!(q.starts_with("{")); + assert!(q.contains("+}\r\n")); + assert!(q.ends_with("éclair")); + } + + #[test] + fn contains_literal_detects_literal_plus_and_sync() { + assert!(contains_literal(b"LOGIN \"u\" {6+}\r\nsecret")); + assert!(contains_literal(b"LOGIN \"u\" {6}\r\nsecret")); + assert!(!contains_literal(b"LOGIN \"u\" \"p\"")); + } + + #[test] + fn format_uid_set_trusts_sorted_flag() { + let s = format_uid_set(&[1, 2, 3, 7, 9], true); + assert_eq!(s, "1:3,7,9"); + } +} diff --git a/src/imap/error.rs b/src/imap/error.rs new file mode 100644 index 0000000..8dec7a1 --- /dev/null +++ b/src/imap/error.rs @@ -0,0 +1,132 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fmt; +use std::io; + +#[derive(Debug, thiserror::Error)] +pub enum ImapError { + #[error("transport error: {0}")] + Io(#[from] io::Error), + + #[error("tls error: {0}")] + Tls(String), + + #[error("protocol error: {0}")] + Protocol(String), + + #[error("response parse error: {0}")] + Parse(String), + + #[error("authentication failed: {0}")] + AuthFailed(String), + + #[error("server said BAD: {0}")] + Bad(String), + + #[error("server said NO: {0}")] + No(NoError), + + #[error("server closed connection: {0}")] + Bye(String), + + #[error("connection is no longer usable")] + Disconnected, + + #[error("unsupported by server: {0}")] + Unsupported(String), +} + +#[derive(Debug, Clone)] +pub struct NoError { + pub text: String, + pub code: Option, +} + +impl NoError { + pub fn new(text: impl Into, code: Option) -> Self { + Self { + text: text.into(), + code, + } + } + + pub fn is_transient(&self) -> bool { + let lower = self.text.to_ascii_lowercase(); + const TRANSIENT_MARKERS: &[&str] = &[ + "temp", + "try again", + "try later", + "unavailable", + "over quota", + "overquota", + "throttle", + "rate limit", + "too many", + ]; + TRANSIENT_MARKERS.iter().any(|m| lower.contains(m)) + } + + pub fn is_auth_failed(&self) -> bool { + matches!( + self.code.as_deref(), + Some("AUTHENTICATIONFAILED") | Some("PRIVACYREQUIRED") | Some("AUTHORIZATIONFAILED") + ) + } +} + +impl fmt::Display for NoError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.code { + Some(code) => write!(f, "[{code}] {}", self.text), + None => f.write_str(&self.text), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn transient_markers_detected_case_insensitively() { + for s in [ + "Temporary failure", + "Try again later", + "Server unavailable", + "OVER QUOTA", + "user is rate limited", + "Too many connections", + ] { + assert!( + NoError::new(s, None).is_transient(), + "should be transient: {s}" + ); + } + } + + #[test] + fn non_transient_markers_not_flagged() { + for s in [ + "Mailbox does not exist", + "Permission denied", + "Invalid argument", + ] { + assert!( + !NoError::new(s, None).is_transient(), + "should NOT be transient: {s}" + ); + } + } + + #[test] + fn auth_failed_codes_detected() { + assert!(NoError::new("x", Some("AUTHENTICATIONFAILED".into())).is_auth_failed()); + assert!(NoError::new("x", Some("PRIVACYREQUIRED".into())).is_auth_failed()); + assert!(!NoError::new("x", Some("ALERT".into())).is_auth_failed()); + assert!(!NoError::new("x", None).is_auth_failed()); + } +} diff --git a/src/imap/mod.rs b/src/imap/mod.rs new file mode 100644 index 0000000..af211de --- /dev/null +++ b/src/imap/mod.rs @@ -0,0 +1,17 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod automap; +pub mod client; +pub mod command; +pub mod error; +pub mod name; +pub mod response; +pub mod retry; +pub mod transport; + +pub use client::{CollectedResponse, ConnectMode, Greeting, ImapClient}; +pub use error::ImapError; diff --git a/src/imap/name.rs b/src/imap/name.rs new file mode 100644 index 0000000..a4a3ad9 --- /dev/null +++ b/src/imap/name.rs @@ -0,0 +1,235 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use super::error::ImapError; + +const MODIFIED_UTF7_ALPHABET: &[u8] = + b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,"; + +pub fn canonicalise_inbox(name: &str) -> String { + if name.eq_ignore_ascii_case("INBOX") { + "INBOX".to_owned() + } else { + name.to_owned() + } +} + +pub fn decode_mailbox_name(input: &str) -> Result { + decode_mailbox_name_with(input, false) +} + +pub fn decode_mailbox_name_with(input: &str, utf8_accept: bool) -> Result { + if utf8_accept { + return Ok(input.to_owned()); + } + if input.is_ascii() && !input.contains('&') { + return Ok(input.to_owned()); + } + let bytes = input.as_bytes(); + let mut out = String::with_capacity(input.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] != b'&' { + out.push(bytes[i] as char); + i += 1; + continue; + } + if i + 1 < bytes.len() && bytes[i + 1] == b'-' { + out.push('&'); + i += 2; + continue; + } + let mut end = i + 1; + while end < bytes.len() && bytes[end] != b'-' { + end += 1; + } + if end == bytes.len() { + return Err(ImapError::Parse( + "unterminated modified UTF-7 sequence".into(), + )); + } + let encoded = &bytes[i + 1..end]; + let decoded = decode_b64_modified(encoded)?; + let u16s: Vec = decoded + .chunks(2) + .filter(|c| c.len() == 2) + .map(|c| u16::from_be_bytes([c[0], c[1]])) + .collect(); + let s = String::from_utf16(&u16s).map_err(|e| ImapError::Parse(format!("utf-16: {e}")))?; + out.push_str(&s); + i = end + 1; + } + Ok(out) +} + +pub fn encode_mailbox_name(input: &str) -> String { + encode_mailbox_name_with(input, false) +} + +pub fn encode_mailbox_name_with(input: &str, utf8_accept: bool) -> String { + if utf8_accept { + return input.to_owned(); + } + let mut out = String::with_capacity(input.len()); + let mut pending: Vec = Vec::new(); + let flush = |pending: &mut Vec, out: &mut String| { + if pending.is_empty() { + return; + } + out.push('&'); + let mut raw = Vec::with_capacity(pending.len() * 2); + for w in pending.iter() { + raw.extend_from_slice(&w.to_be_bytes()); + } + out.push_str(&encode_b64_modified(&raw)); + out.push('-'); + pending.clear(); + }; + for c in input.chars() { + let cp = c as u32; + if c == '&' { + flush(&mut pending, &mut out); + out.push_str("&-"); + } else if (0x20..=0x7E).contains(&cp) { + flush(&mut pending, &mut out); + out.push(c); + } else { + let mut buf = [0u16; 2]; + let units = c.encode_utf16(&mut buf); + pending.extend_from_slice(units); + } + } + flush(&mut pending, &mut out); + out +} + +fn decode_b64_modified(bytes: &[u8]) -> Result, ImapError> { + let mut out = Vec::with_capacity(bytes.len() * 3 / 4); + let mut acc: u32 = 0; + let mut bits: u32 = 0; + for &b in bytes { + let v = match MODIFIED_UTF7_ALPHABET.iter().position(|&x| x == b) { + Some(idx) => idx as u32, + None => { + return Err(ImapError::Parse(format!( + "invalid modified UTF-7 byte {b:#04x}" + ))); + } + }; + acc = (acc << 6) | v; + bits += 6; + if bits >= 8 { + bits -= 8; + let byte = ((acc >> bits) & 0xFF) as u8; + out.push(byte); + } + } + Ok(out) +} + +fn encode_b64_modified(bytes: &[u8]) -> String { + let mut out = String::with_capacity(bytes.len() * 4 / 3 + 1); + let mut acc: u32 = 0; + let mut bits: u32 = 0; + for &b in bytes { + acc = (acc << 8) | b as u32; + bits += 8; + while bits >= 6 { + bits -= 6; + let idx = ((acc >> bits) & 0x3F) as usize; + out.push(MODIFIED_UTF7_ALPHABET[idx] as char); + } + } + if bits > 0 { + let idx = ((acc << (6 - bits)) & 0x3F) as usize; + out.push(MODIFIED_UTF7_ALPHABET[idx] as char); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ascii_passes_through_decode() { + assert_eq!(decode_mailbox_name("INBOX").unwrap(), "INBOX"); + assert_eq!(decode_mailbox_name("Sent/Items").unwrap(), "Sent/Items"); + } + + #[test] + fn ampersand_dash_decodes_to_literal_amp() { + assert_eq!(decode_mailbox_name("R&-D").unwrap(), "R&D"); + assert_eq!(decode_mailbox_name("&-").unwrap(), "&"); + } + + #[test] + fn non_ascii_decode_japanese() { + let decoded = decode_mailbox_name("~peter/mail/&ZeVnLIqe-/&U,BTFw-").unwrap(); + assert_eq!(decoded, "~peter/mail/日本語/台北"); + } + + #[test] + fn unterminated_sequence_errors() { + let err = decode_mailbox_name("&ZeVnLIqe").unwrap_err(); + assert!(matches!(err, ImapError::Parse(_))); + } + + #[test] + fn ascii_passes_through_encode() { + assert_eq!(encode_mailbox_name("INBOX"), "INBOX"); + assert_eq!(encode_mailbox_name("Sent Items"), "Sent Items"); + } + + #[test] + fn literal_amp_encodes_to_amp_dash() { + assert_eq!(encode_mailbox_name("R&D"), "R&-D"); + assert_eq!(encode_mailbox_name("&"), "&-"); + } + + #[test] + fn non_ascii_encode_japanese() { + let encoded = encode_mailbox_name("~peter/mail/日本語/台北"); + assert_eq!(encoded, "~peter/mail/&ZeVnLIqe-/&U,BTFw-"); + } + + #[test] + fn encode_decode_roundtrip_various() { + for s in [ + "INBOX", + "Sent Mail", + "R&D", + "日本語", + "Hé llo", + "INBOX.Projects.Alpha", + "Junk E-mail", + ] { + let enc = encode_mailbox_name(s); + let dec = decode_mailbox_name(&enc).unwrap(); + assert_eq!(dec, s, "roundtrip failed for {s:?}, enc={enc:?}"); + } + } + + #[test] + fn inbox_canonical_uppercase() { + assert_eq!(canonicalise_inbox("inbox"), "INBOX"); + assert_eq!(canonicalise_inbox("Inbox"), "INBOX"); + assert_eq!(canonicalise_inbox("INBOX"), "INBOX"); + assert_eq!(canonicalise_inbox("Inbox/Sub"), "Inbox/Sub"); + } + + #[test] + fn utf8_accept_skips_mutf7_decode_so_ampersand_passes_through() { + assert_eq!(decode_mailbox_name_with("R&D", true).unwrap(), "R&D"); + assert_eq!(decode_mailbox_name_with("日本語", true).unwrap(), "日本語"); + } + + #[test] + fn utf8_accept_skips_mutf7_encode_so_input_passes_through() { + assert_eq!(encode_mailbox_name_with("日本語", true), "日本語"); + assert_eq!(encode_mailbox_name_with("R&D", true), "R&D"); + } +} diff --git a/src/imap/response.rs b/src/imap/response.rs new file mode 100644 index 0000000..d5302a0 --- /dev/null +++ b/src/imap/response.rs @@ -0,0 +1,1093 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::BTreeMap; +use std::io::BufRead; + +use super::error::{ImapError, NoError}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Status { + Ok, + No, + Bad, + Bye, + PreAuth, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StatusLine { + pub status: Status, + pub code: Option, + pub code_args: Option, + pub text: String, +} + +impl StatusLine { + pub fn into_no_error(self) -> NoError { + NoError::new(self.text, self.code) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Value { + Atom(String), + Str(String), + Bytes(Vec), + Number(u64), + Nil, + List(Vec), +} + +impl Value { + pub fn as_str(&self) -> Option<&str> { + match self { + Value::Atom(s) | Value::Str(s) => Some(s), + _ => None, + } + } + + pub fn as_number(&self) -> Option { + match self { + Value::Number(n) => Some(*n), + _ => None, + } + } + + pub fn as_list(&self) -> Option<&[Value]> { + match self { + Value::List(items) => Some(items.as_slice()), + _ => None, + } + } + + pub fn into_bytes(self) -> Option> { + match self { + Value::Bytes(b) => Some(b), + Value::Str(s) => Some(s.into_bytes()), + _ => None, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Untagged { + Capability(Vec), + List { + attributes: Vec, + delimiter: Option, + name: String, + }, + Lsub { + attributes: Vec, + delimiter: Option, + name: String, + }, + Status { + mailbox: String, + items: BTreeMap, + }, + Search(Vec), + Esearch { + tag: Option, + all: Vec, + count: Option, + }, + Exists(u32), + Recent(u32), + Expunge(u32), + Fetch { + seq: u32, + items: Vec<(String, Value)>, + }, + Flags(Vec), + Namespace { + personal: Vec, + others: Vec, + shared: Vec, + }, + Enabled(Vec), + StatusLine(StatusLine), + Bye(String), + Raw(String), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NamespaceEntry { + pub prefix: String, + pub delimiter: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Response { + Tagged { tag: String, line: StatusLine }, + Untagged(Untagged), + Continuation(String), +} + +pub fn parse_response(reader: &mut R) -> Result { + let mut parser = Parser::new(reader); + parser.parse_response() +} + +struct Parser<'r, R: BufRead> { + reader: &'r mut R, + buf: Vec, + pos: usize, +} + +impl<'r, R: BufRead> Parser<'r, R> { + fn new(reader: &'r mut R) -> Self { + Self { + reader, + buf: Vec::new(), + pos: 0, + } + } + + fn parse_response(&mut self) -> Result { + self.read_line()?; + if self.buf.is_empty() { + return Err(ImapError::Disconnected); + } + if self.peek() == Some(b'+') { + self.bump(); + self.skip_ws(); + let text = self.read_until_crlf(); + return Ok(Response::Continuation(text)); + } + if self.peek() == Some(b'*') { + self.bump(); + self.skip_ws(); + return self.parse_untagged().map(Response::Untagged); + } + let tag = self.read_atom_string()?; + self.skip_ws(); + let line = self.parse_status_line()?; + Ok(Response::Tagged { tag, line }) + } + + fn parse_untagged(&mut self) -> Result { + let first = self.read_atom_string()?; + let upper = first.to_ascii_uppercase(); + match upper.as_str() { + "OK" | "NO" | "BAD" | "BYE" | "PREAUTH" => { + self.skip_ws(); + let mut line = self.parse_status_line_body()?; + line.status = match upper.as_str() { + "OK" => Status::Ok, + "NO" => Status::No, + "BAD" => Status::Bad, + "BYE" => Status::Bye, + "PREAUTH" => Status::PreAuth, + _ => unreachable!(), + }; + if matches!(line.status, Status::Bye) { + return Ok(Untagged::Bye(line.text)); + } + Ok(Untagged::StatusLine(line)) + } + "CAPABILITY" => { + let mut caps = Vec::new(); + while !self.at_end() { + self.skip_ws(); + if self.at_end() { + break; + } + caps.push(self.read_atom_string()?); + } + Ok(Untagged::Capability(caps)) + } + "LIST" | "LSUB" => { + self.skip_ws(); + self.expect(b'(')?; + let mut attrs = Vec::new(); + while self.peek() != Some(b')') { + self.skip_ws(); + if self.peek() == Some(b')') { + break; + } + attrs.push(self.read_atom_string()?); + } + self.expect(b')')?; + self.skip_ws(); + let delim_value = self.parse_value()?; + let delimiter = match delim_value { + Value::Nil => None, + Value::Str(s) | Value::Atom(s) => s.chars().next(), + Value::Bytes(b) if !b.is_empty() => Some(b[0] as char), + _ => None, + }; + self.skip_ws(); + let name_val = self.parse_value()?; + let name = match name_val { + Value::Str(s) | Value::Atom(s) => s, + Value::Bytes(b) => String::from_utf8_lossy(&b).into_owned(), + _ => return Err(ImapError::Parse("LIST mailbox name missing".into())), + }; + if upper == "LIST" { + Ok(Untagged::List { + attributes: attrs, + delimiter, + name, + }) + } else { + Ok(Untagged::Lsub { + attributes: attrs, + delimiter, + name, + }) + } + } + "STATUS" => { + self.skip_ws(); + let mailbox = match self.parse_value()? { + Value::Str(s) | Value::Atom(s) => s, + Value::Bytes(b) => String::from_utf8_lossy(&b).into_owned(), + _ => return Err(ImapError::Parse("STATUS mailbox missing".into())), + }; + self.skip_ws(); + self.expect(b'(')?; + let mut items = BTreeMap::new(); + while self.peek() != Some(b')') { + self.skip_ws(); + if self.peek() == Some(b')') { + break; + } + let key = self.read_atom_string()?.to_ascii_uppercase(); + self.skip_ws(); + let val = self.parse_value()?; + if let Some(n) = val.as_number() { + items.insert(key, n); + } + } + self.expect(b')')?; + Ok(Untagged::Status { mailbox, items }) + } + "SEARCH" => { + let mut ids = Vec::new(); + while !self.at_end() { + self.skip_ws(); + if self.at_end() { + break; + } + let v = self.parse_value()?; + if let Some(n) = v.as_number() { + ids.push(n as u32); + } + } + Ok(Untagged::Search(ids)) + } + "ESEARCH" => self.parse_esearch(), + "FLAGS" => { + self.skip_ws(); + let v = self.parse_value()?; + let flags = match v { + Value::List(items) => items + .into_iter() + .filter_map(|x| match x { + Value::Atom(s) | Value::Str(s) => Some(s), + _ => None, + }) + .collect(), + _ => Vec::new(), + }; + Ok(Untagged::Flags(flags)) + } + "NAMESPACE" => self.parse_namespace(), + "ENABLED" => { + let mut exts = Vec::new(); + while !self.at_end() { + self.skip_ws(); + if self.at_end() { + break; + } + exts.push(self.read_atom_string()?); + } + Ok(Untagged::Enabled(exts)) + } + _ => { + if let Ok(num) = upper.parse::() { + self.skip_ws(); + let kw = self.read_atom_string()?.to_ascii_uppercase(); + match kw.as_str() { + "EXISTS" => return Ok(Untagged::Exists(num)), + "RECENT" => return Ok(Untagged::Recent(num)), + "EXPUNGE" => return Ok(Untagged::Expunge(num)), + "FETCH" => return self.parse_fetch_body(num), + _ => { + let rest = self.read_until_crlf(); + return Ok(Untagged::Raw(format!("{num} {kw} {rest}"))); + } + } + } + let rest = self.read_until_crlf(); + Ok(Untagged::Raw(format!("{first} {rest}"))) + } + } + } + + fn parse_esearch(&mut self) -> Result { + let mut tag = None; + let mut all = Vec::new(); + let mut count = None; + if self.peek() == Some(b' ') { + self.skip_ws(); + } + if self.peek() == Some(b'(') { + self.bump(); + self.skip_ws(); + let key = self.read_atom_string()?.to_ascii_uppercase(); + if key == "TAG" { + self.skip_ws(); + if let Value::Str(s) = self.parse_value()? { + tag = Some(s); + } + } + self.skip_ws(); + self.expect(b')')?; + } + while !self.at_end() { + self.skip_ws(); + if self.at_end() { + break; + } + let key = self.read_atom_string()?.to_ascii_uppercase(); + match key.as_str() { + "UID" => {} + "ALL" => { + self.skip_ws(); + let s = self.read_seq_set_token(); + all = parse_seq_set(&s); + } + "COUNT" => { + self.skip_ws(); + if let Value::Number(n) = self.parse_value()? { + count = Some(n as u32); + } + } + _ => { + self.skip_ws(); + let _ = self.parse_value(); + } + } + } + Ok(Untagged::Esearch { tag, all, count }) + } + + fn parse_namespace(&mut self) -> Result { + self.skip_ws(); + let personal = self.parse_namespace_list()?; + self.skip_ws(); + let others = self.parse_namespace_list()?; + self.skip_ws(); + let shared = self.parse_namespace_list()?; + Ok(Untagged::Namespace { + personal, + others, + shared, + }) + } + + fn parse_namespace_list(&mut self) -> Result, ImapError> { + match self.parse_value()? { + Value::Nil => Ok(Vec::new()), + Value::List(entries) => { + let mut out = Vec::new(); + for e in entries { + if let Value::List(inner) = e + && let Some(prefix) = inner.first().and_then(|v| v.as_str()) + { + let delim = inner.get(1).and_then(|v| match v { + Value::Nil => None, + Value::Str(s) | Value::Atom(s) => s.chars().next(), + _ => None, + }); + out.push(NamespaceEntry { + prefix: prefix.to_owned(), + delimiter: delim, + }); + } + } + Ok(out) + } + _ => Ok(Vec::new()), + } + } + + fn parse_fetch_body(&mut self, seq: u32) -> Result { + self.skip_ws(); + self.expect(b'(')?; + let mut items = Vec::new(); + while self.peek() != Some(b')') { + self.skip_ws(); + if self.peek() == Some(b')') { + break; + } + let name = self.read_fetch_item_name()?; + self.skip_ws(); + let val = self.parse_value()?; + items.push((name, val)); + } + self.expect(b')')?; + Ok(Untagged::Fetch { seq, items }) + } + + fn read_fetch_item_name(&mut self) -> Result { + let start = self.pos; + while let Some(b) = self.peek() { + match b { + b' ' | b'\r' | b'\n' | b')' | b'(' => break, + _ => self.bump(), + } + } + let bytes = &self.buf[start..self.pos]; + std::str::from_utf8(bytes) + .map(|s| s.to_ascii_uppercase()) + .map_err(|e| ImapError::Parse(format!("non-utf8 fetch item name: {e}"))) + } + + fn parse_status_line(&mut self) -> Result { + let kw = self.read_atom_string()?; + self.skip_ws(); + let mut line = self.parse_status_line_body()?; + line.status = match kw.to_ascii_uppercase().as_str() { + "OK" => Status::Ok, + "NO" => Status::No, + "BAD" => Status::Bad, + "BYE" => Status::Bye, + "PREAUTH" => Status::PreAuth, + other => return Err(ImapError::Parse(format!("unknown status: {other}"))), + }; + Ok(line) + } + + fn parse_status_line_body(&mut self) -> Result { + let mut code = None; + let mut code_args = None; + if self.peek() == Some(b'[') { + self.bump(); + let start = self.pos; + let mut depth = 1usize; + while let Some(b) = self.peek() { + if b == b'[' { + depth += 1; + self.bump(); + } else if b == b']' { + depth -= 1; + if depth == 0 { + break; + } + self.bump(); + } else { + self.bump(); + } + } + let inside = &self.buf[start..self.pos]; + let inside_str = std::str::from_utf8(inside) + .map_err(|e| ImapError::Parse(format!("non-utf8 status code: {e}")))?; + let (head, tail) = match inside_str.split_once(' ') { + Some((h, t)) => (h.to_owned(), Some(t.to_owned())), + None => (inside_str.to_owned(), None), + }; + code = Some(head.to_ascii_uppercase()); + code_args = tail; + self.expect(b']')?; + self.skip_ws(); + } + let text = self.read_until_crlf(); + Ok(StatusLine { + status: Status::Ok, + code, + code_args, + text, + }) + } + + fn parse_value(&mut self) -> Result { + self.skip_ws(); + match self.peek() { + Some(b'"') => self.parse_quoted(), + Some(b'{') => self.parse_literal(), + Some(b'(') => self.parse_list(), + Some(b) + if b.is_ascii_digit() + || b == b'-' + || b == b'+' + || is_atom_byte(b) + || b == b'\\' => + { + self.parse_atom_or_number() + } + None => Err(ImapError::Parse("expected value, got EOL".into())), + Some(other) => Err(ImapError::Parse(format!( + "unexpected byte {other:#04x} in value" + ))), + } + } + + fn parse_quoted(&mut self) -> Result { + self.expect(b'"')?; + let mut s = String::new(); + while let Some(b) = self.peek() { + match b { + b'"' => { + self.bump(); + return Ok(Value::Str(s)); + } + b'\\' => { + self.bump(); + let next = self + .peek() + .ok_or_else(|| ImapError::Parse("trailing \\ in quoted".into()))?; + s.push(next as char); + self.bump(); + } + _ => { + s.push(b as char); + self.bump(); + } + } + } + Err(ImapError::Parse("unterminated quoted string".into())) + } + + fn parse_literal(&mut self) -> Result { + self.expect(b'{')?; + let mut len_str = String::new(); + while let Some(b) = self.peek() { + if b.is_ascii_digit() { + len_str.push(b as char); + self.bump(); + } else { + break; + } + } + if self.peek() == Some(b'+') { + self.bump(); + } + self.expect(b'}')?; + let len: usize = len_str + .parse() + .map_err(|e| ImapError::Parse(format!("literal length: {e}")))?; + if !self.at_end_of_line() { + return Err(ImapError::Parse("garbage after literal header".into())); + } + let mut data = vec![0u8; len]; + self.reader.read_exact(&mut data).map_err(ImapError::Io)?; + self.buf.clear(); + self.read_line()?; + if let Ok(s) = std::str::from_utf8(&data) { + Ok(Value::Str(s.to_owned())) + } else { + Ok(Value::Bytes(data)) + } + } + + fn parse_list(&mut self) -> Result { + self.expect(b'(')?; + let mut items = Vec::new(); + loop { + self.skip_ws(); + if self.peek() == Some(b')') { + self.bump(); + return Ok(Value::List(items)); + } + if self.peek().is_none() { + return Err(ImapError::Parse("unterminated list".into())); + } + items.push(self.parse_value()?); + } + } + + fn parse_atom_or_number(&mut self) -> Result { + let start = self.pos; + while let Some(b) = self.peek() { + if is_atom_byte(b) || b == b'\\' || b == b'*' || b == b']' || b == b'+' { + self.bump(); + } else { + break; + } + } + let raw = &self.buf[start..self.pos]; + let s = std::str::from_utf8(raw) + .map_err(|e| ImapError::Parse(format!("non-utf8 atom: {e}")))? + .to_owned(); + if s.is_empty() { + return Err(ImapError::Parse("empty atom".into())); + } + if s.eq_ignore_ascii_case("NIL") { + return Ok(Value::Nil); + } + if let Ok(n) = s.parse::() { + return Ok(Value::Number(n)); + } + Ok(Value::Atom(s)) + } + + fn read_atom_string(&mut self) -> Result { + let start = self.pos; + while let Some(b) = self.peek() { + if is_atom_byte(b) { + self.bump(); + } else { + break; + } + } + if start == self.pos { + return Err(ImapError::Parse("expected atom".into())); + } + let bytes = &self.buf[start..self.pos]; + std::str::from_utf8(bytes) + .map(|s| s.to_owned()) + .map_err(|e| ImapError::Parse(format!("non-utf8 atom: {e}"))) + } + + fn read_seq_set_token(&mut self) -> String { + let start = self.pos; + while let Some(b) = self.peek() { + if b == b' ' || b == b'\r' || b == b'\n' { + break; + } + self.bump(); + } + String::from_utf8_lossy(&self.buf[start..self.pos]).into_owned() + } + + fn read_until_crlf(&mut self) -> String { + let start = self.pos; + let end = self.buf.len().saturating_sub(2); + let end = end.max(start); + self.pos = self.buf.len(); + String::from_utf8_lossy(&self.buf[start..end]) + .trim_end() + .to_owned() + } + + fn read_line(&mut self) -> Result<(), ImapError> { + self.buf.clear(); + let n = self.reader.read_until(b'\n', &mut self.buf)?; + self.pos = 0; + if n == 0 { + return Ok(()); + } + Ok(()) + } + + fn peek(&self) -> Option { + let end = self.buf.len().saturating_sub(self.tail_crlf_len()); + if self.pos >= end { + None + } else { + Some(self.buf[self.pos]) + } + } + + fn bump(&mut self) { + self.pos += 1; + } + + fn skip_ws(&mut self) { + while self.peek() == Some(b' ') { + self.bump(); + } + } + + fn at_end(&self) -> bool { + self.peek().is_none() + } + + fn at_end_of_line(&self) -> bool { + let end = self.buf.len().saturating_sub(self.tail_crlf_len()); + self.pos >= end + } + + fn tail_crlf_len(&self) -> usize { + if self.buf.ends_with(b"\r\n") { + 2 + } else if self.buf.ends_with(b"\n") { + 1 + } else { + 0 + } + } + + fn expect(&mut self, b: u8) -> Result<(), ImapError> { + if self.peek() == Some(b) { + self.bump(); + Ok(()) + } else { + Err(ImapError::Parse(format!( + "expected {:?}, got {:?}", + b as char, + self.peek().map(|x| x as char) + ))) + } + } +} + +fn is_atom_byte(b: u8) -> bool { + matches!(b, b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' + | b'-' | b'.' | b'_' | b'!' | b'#' | b'$' | b'&' + | b'\'' | b'+' | b'/' | b';' | b'<' | b'=' | b'>' | b'?' + | b'@' | b'\\' | b'^' | b'`' | b'|' | b'~' | b'*') +} + +fn parse_seq_set(s: &str) -> Vec { + let mut out = Vec::new(); + for part in s.split(',') { + let part = part.trim(); + if part.is_empty() { + continue; + } + if let Some((lo, hi)) = part.split_once(':') { + let lo: u32 = match lo.parse() { + Ok(n) => n, + Err(_) => continue, + }; + let hi: u32 = match hi.parse() { + Ok(n) => n, + Err(_) => continue, + }; + let (a, b) = if lo <= hi { (lo, hi) } else { (hi, lo) }; + for n in a..=b { + out.push(n); + } + } else if let Ok(n) = part.parse::() { + out.push(n); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + fn parse(input: &[u8]) -> Response { + parse_response(&mut Cursor::new(input)).unwrap() + } + + #[test] + fn tagged_ok() { + let r = parse(b"A001 OK LOGIN completed\r\n"); + match r { + Response::Tagged { tag, line } => { + assert_eq!(tag, "A001"); + assert_eq!(line.status, Status::Ok); + assert_eq!(line.text, "LOGIN completed"); + } + _ => panic!("expected Tagged"), + } + } + + #[test] + fn tagged_no_with_code() { + let r = parse(b"A002 NO [AUTHENTICATIONFAILED] bad credentials\r\n"); + match r { + Response::Tagged { tag, line } => { + assert_eq!(tag, "A002"); + assert_eq!(line.status, Status::No); + assert_eq!(line.code.as_deref(), Some("AUTHENTICATIONFAILED")); + assert_eq!(line.text, "bad credentials"); + } + _ => panic!("expected Tagged"), + } + } + + #[test] + fn untagged_capability() { + let r = parse(b"* CAPABILITY IMAP4rev1 STARTTLS AUTH=PLAIN AUTH=LOGIN\r\n"); + match r { + Response::Untagged(Untagged::Capability(caps)) => { + assert_eq!( + caps, + vec!["IMAP4rev1", "STARTTLS", "AUTH=PLAIN", "AUTH=LOGIN"] + ); + } + _ => panic!("expected Capability"), + } + } + + #[test] + fn untagged_capability_with_literal_plus() { + let r = parse( + b"* CAPABILITY IMAP4rev2 IMAP4rev1 ENABLE SASL-IR LITERAL+ ID UTF8=ACCEPT JMAPACCESS AUTH=PLAIN AUTH=OAUTHBEARER AUTH=XOAUTH2\r\n", + ); + match r { + Response::Untagged(Untagged::Capability(caps)) => { + assert!(caps.iter().any(|c| c == "LITERAL+")); + assert!(caps.iter().any(|c| c == "AUTH=OAUTHBEARER")); + assert_eq!(caps.len(), 11); + } + _ => panic!("expected Capability"), + } + } + + #[test] + fn untagged_list_with_special_use() { + let r = parse(b"* LIST (\\HasNoChildren \\Sent) \"/\" \"Sent Mail\"\r\n"); + match r { + Response::Untagged(Untagged::List { + attributes, + delimiter, + name, + }) => { + assert_eq!(attributes, vec!["\\HasNoChildren", "\\Sent"]); + assert_eq!(delimiter, Some('/')); + assert_eq!(name, "Sent Mail"); + } + _ => panic!("expected List"), + } + } + + #[test] + fn untagged_list_with_nil_delimiter() { + let r = parse(b"* LIST () NIL INBOX\r\n"); + match r { + Response::Untagged(Untagged::List { + attributes, + delimiter, + name, + }) => { + assert!(attributes.is_empty()); + assert_eq!(delimiter, None); + assert_eq!(name, "INBOX"); + } + _ => panic!("expected List"), + } + } + + #[test] + fn untagged_status() { + let r = parse(b"* STATUS INBOX (UIDVALIDITY 12345 UIDNEXT 42 MESSAGES 7)\r\n"); + match r { + Response::Untagged(Untagged::Status { mailbox, items }) => { + assert_eq!(mailbox, "INBOX"); + assert_eq!(items.get("UIDVALIDITY"), Some(&12345)); + assert_eq!(items.get("UIDNEXT"), Some(&42)); + assert_eq!(items.get("MESSAGES"), Some(&7)); + } + _ => panic!("expected Status"), + } + } + + #[test] + fn untagged_search_with_uids() { + let r = parse(b"* SEARCH 1 3 5 7\r\n"); + match r { + Response::Untagged(Untagged::Search(ids)) => { + assert_eq!(ids, vec![1, 3, 5, 7]); + } + _ => panic!("expected Search"), + } + } + + #[test] + fn untagged_search_empty() { + let r = parse(b"* SEARCH\r\n"); + match r { + Response::Untagged(Untagged::Search(ids)) => { + assert!(ids.is_empty()); + } + _ => panic!("expected Search"), + } + } + + #[test] + fn untagged_esearch_all_set() { + let r = parse(b"* ESEARCH (TAG \"q1\") UID ALL 1:3,7\r\n"); + match r { + Response::Untagged(Untagged::Esearch { tag, all, .. }) => { + assert_eq!(tag.as_deref(), Some("q1")); + assert_eq!(all, vec![1, 2, 3, 7]); + } + _ => panic!("expected Esearch"), + } + } + + #[test] + fn untagged_exists_recent() { + let r = parse(b"* 42 EXISTS\r\n"); + assert!(matches!(r, Response::Untagged(Untagged::Exists(42)))); + let r = parse(b"* 3 RECENT\r\n"); + assert!(matches!(r, Response::Untagged(Untagged::Recent(3)))); + } + + #[test] + fn untagged_fetch_flags_and_uid() { + let r = parse( + b"* 17 FETCH (UID 100 FLAGS (\\Seen \\Flagged) RFC822.SIZE 4242 INTERNALDATE \"17-Jul-1996 02:44:25 -0700\")\r\n", + ); + match r { + Response::Untagged(Untagged::Fetch { seq, items }) => { + assert_eq!(seq, 17); + let uid = items.iter().find(|(n, _)| n == "UID").unwrap(); + assert_eq!(uid.1.as_number(), Some(100)); + let flags = items.iter().find(|(n, _)| n == "FLAGS").unwrap(); + let list = flags.1.as_list().unwrap(); + assert_eq!(list.len(), 2); + let size = items.iter().find(|(n, _)| n == "RFC822.SIZE").unwrap(); + assert_eq!(size.1.as_number(), Some(4242)); + let internaldate = items.iter().find(|(n, _)| n == "INTERNALDATE").unwrap(); + assert_eq!(internaldate.1.as_str(), Some("17-Jul-1996 02:44:25 -0700")); + } + _ => panic!("expected Fetch"), + } + } + + #[test] + fn untagged_fetch_with_body_literal() { + let input = b"* 1 FETCH (UID 5 BODY[] {11}\r\nHello world)\r\n"; + let r = parse(input); + match r { + Response::Untagged(Untagged::Fetch { seq, items }) => { + assert_eq!(seq, 1); + let body = items.iter().find(|(n, _)| n == "BODY[]").unwrap(); + match &body.1 { + Value::Str(s) => assert_eq!(s, "Hello world"), + Value::Bytes(b) => assert_eq!(b, b"Hello world"), + other => panic!("expected literal, got {other:?}"), + } + } + _ => panic!("expected Fetch"), + } + } + + #[test] + fn untagged_fetch_with_binary_literal() { + let mut input = b"* 1 FETCH (UID 5 BODY[] {4}\r\n".to_vec(); + input.extend_from_slice(&[0xFF, 0xFE, 0x00, 0x01]); + input.extend_from_slice(b")\r\n"); + let r = parse_response(&mut Cursor::new(input)).unwrap(); + match r { + Response::Untagged(Untagged::Fetch { items, .. }) => { + let body = items.iter().find(|(n, _)| n == "BODY[]").unwrap(); + match &body.1 { + Value::Bytes(b) => assert_eq!(b, &[0xFF, 0xFE, 0x00, 0x01]), + other => panic!("expected Bytes, got {other:?}"), + } + } + _ => panic!("expected Fetch"), + } + } + + #[test] + fn untagged_flags() { + let r = parse(b"* FLAGS (\\Answered \\Flagged \\Draft \\Deleted \\Seen)\r\n"); + match r { + Response::Untagged(Untagged::Flags(flags)) => { + assert_eq!(flags.len(), 5); + assert!(flags.contains(&"\\Seen".to_owned())); + } + _ => panic!("expected Flags"), + } + } + + #[test] + fn untagged_bye() { + let r = parse(b"* BYE server logging out\r\n"); + match r { + Response::Untagged(Untagged::Bye(text)) => { + assert_eq!(text, "server logging out"); + } + _ => panic!("expected Bye"), + } + } + + #[test] + fn untagged_namespace() { + let r = parse(b"* NAMESPACE ((\"INBOX.\" \".\")) NIL NIL\r\n"); + match r { + Response::Untagged(Untagged::Namespace { + personal, + others, + shared, + }) => { + assert_eq!(personal.len(), 1); + assert_eq!(personal[0].prefix, "INBOX."); + assert_eq!(personal[0].delimiter, Some('.')); + assert!(others.is_empty()); + assert!(shared.is_empty()); + } + _ => panic!("expected Namespace"), + } + } + + #[test] + fn continuation_request() { + let r = parse(b"+ Ready for additional command text\r\n"); + match r { + Response::Continuation(text) => { + assert_eq!(text, "Ready for additional command text"); + } + _ => panic!("expected Continuation"), + } + } + + #[test] + fn untagged_status_with_response_code() { + let r = parse(b"* OK [PERMANENTFLAGS (\\Seen \\Flagged \\*)] Limited\r\n"); + match r { + Response::Untagged(Untagged::StatusLine(line)) => { + assert_eq!(line.status, Status::Ok); + assert_eq!(line.code.as_deref(), Some("PERMANENTFLAGS")); + assert!(line.code_args.as_deref().is_some()); + assert_eq!(line.text, "Limited"); + } + _ => panic!("expected StatusLine"), + } + } + + #[test] + fn untagged_ok_with_uidvalidity() { + let r = parse(b"* OK [UIDVALIDITY 3857529045] UIDs valid\r\n"); + match r { + Response::Untagged(Untagged::StatusLine(line)) => { + assert_eq!(line.code.as_deref(), Some("UIDVALIDITY")); + assert_eq!(line.code_args.as_deref(), Some("3857529045")); + } + _ => panic!("expected StatusLine"), + } + } + + #[test] + fn quoted_string_with_escape() { + let r = parse(b"* LIST () \"/\" \"He said \\\"hi\\\"\"\r\n"); + match r { + Response::Untagged(Untagged::List { name, .. }) => { + assert_eq!(name, "He said \"hi\""); + } + _ => panic!("expected List"), + } + } + + #[test] + fn seq_set_parses_ranges_and_singletons() { + assert_eq!(parse_seq_set("1:3,5,7:8"), vec![1, 2, 3, 5, 7, 8]); + assert_eq!(parse_seq_set("42"), vec![42]); + assert!(parse_seq_set("").is_empty()); + } + + #[test] + fn empty_read_is_disconnected() { + let mut empty: &[u8] = b""; + let err = parse_response(&mut empty).unwrap_err(); + assert!(matches!(err, ImapError::Disconnected)); + } + + #[test] + fn untagged_enabled_parses() { + let r = parse(b"* ENABLED UTF8=ACCEPT CONDSTORE\r\n"); + match r { + Response::Untagged(Untagged::Enabled(exts)) => { + assert_eq!(exts, vec!["UTF8=ACCEPT", "CONDSTORE"]); + } + other => panic!("expected Enabled, got {other:?}"), + } + } +} diff --git a/src/imap/retry.rs b/src/imap/retry.rs new file mode 100644 index 0000000..eb2e615 --- /dev/null +++ b/src/imap/retry.rs @@ -0,0 +1,352 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::sync::Arc; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::time::Duration; + +use super::error::ImapError; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Disposition { + TransportDrop, + Transient, + Permanent, + PerMessageRecoverable, + PerFolderPermanent, + FeatureNegotiation, + AuthFailed, +} + +#[derive(Debug, Clone, Copy)] +pub struct RetryPolicy { + pub max_retries: u32, +} + +impl RetryPolicy { + pub fn new(max_retries: u32) -> Self { + Self { max_retries } + } +} + +const SHARED_BASE_MS: u64 = 1_000; +const SHARED_CAP_MS: u64 = 30_000; +const TRANSPORT_BASE_MS: u64 = 500; +const TRANSPORT_CAP_MS: u64 = 30_000; +const LEVEL_CAP: u32 = 16; + +#[derive(Default, Clone)] +pub struct BackoffState { + inner: Arc, +} + +#[derive(Default)] +struct BackoffInner { + level: AtomicU32, + transient_retries: AtomicU32, + transport_retries: AtomicU32, +} + +impl BackoffState { + pub fn new() -> Self { + Self::default() + } + + pub fn reset(&self) { + self.inner.level.store(0, Ordering::Relaxed); + } + + pub fn current_level(&self) -> u32 { + self.inner.level.load(Ordering::Relaxed) + } + + pub fn transient_retries(&self) -> u32 { + self.inner.transient_retries.load(Ordering::Relaxed) + } + + pub fn transport_retries(&self) -> u32 { + self.inner.transport_retries.load(Ordering::Relaxed) + } + + pub fn total_retries(&self) -> u64 { + self.transient_retries() as u64 + self.transport_retries() as u64 + } + + pub fn next_shared_delay(&self) -> Duration { + let prev = self.inner.level.fetch_add(1, Ordering::Relaxed); + self.inner.transient_retries.fetch_add(1, Ordering::Relaxed); + let n = prev.saturating_add(1).min(LEVEL_CAP); + let shift = n.saturating_sub(1).min(15); + let scaled = SHARED_BASE_MS.saturating_mul(1u64 << shift); + let scaled = scaled.min(SHARED_CAP_MS); + let half = scaled / 2; + Duration::from_millis(jitter_in_range(half, scaled)) + } + + pub fn transport_delay(&self, attempt: u32) -> Duration { + self.inner.transport_retries.fetch_add(1, Ordering::Relaxed); + let n = attempt.min(LEVEL_CAP); + let shift = n.saturating_sub(1).min(15); + let scaled = TRANSPORT_BASE_MS.saturating_mul(1u64 << shift); + let scaled = scaled.min(TRANSPORT_CAP_MS); + Duration::from_millis(jitter_in_range(0, scaled)) + } +} + +fn jitter_in_range(lo: u64, hi: u64) -> u64 { + if hi <= lo { + return lo; + } + let span = hi - lo; + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.subsec_nanos() as u64) + .unwrap_or(0); + lo + (nanos % span) +} + +pub fn is_negotiation_failure(err: &ImapError) -> bool { + match err { + ImapError::Bad(_) => true, + ImapError::No(no) => !no.is_transient(), + ImapError::Unsupported(_) => true, + _ => false, + } +} + +pub fn classify(err: &ImapError) -> Disposition { + match err { + ImapError::Io(_) => Disposition::TransportDrop, + ImapError::Tls(_) => Disposition::Permanent, + ImapError::Disconnected => Disposition::TransportDrop, + ImapError::Bye(_) => Disposition::TransportDrop, + ImapError::AuthFailed(_) => Disposition::AuthFailed, + ImapError::Bad(_) => Disposition::FeatureNegotiation, + ImapError::No(no) => { + if no.is_auth_failed() { + Disposition::AuthFailed + } else if no.is_transient() { + Disposition::Transient + } else { + Disposition::Permanent + } + } + ImapError::Protocol(_) => Disposition::Permanent, + ImapError::Parse(_) => Disposition::Permanent, + ImapError::Unsupported(_) => Disposition::Permanent, + } +} + +pub fn folder_disposition(err: &ImapError) -> Disposition { + let base = classify(err); + if matches!( + base, + Disposition::Permanent | Disposition::FeatureNegotiation + ) { + Disposition::PerFolderPermanent + } else { + base + } +} + +pub fn message_disposition(err: &ImapError) -> Disposition { + let base = classify(err); + if matches!( + base, + Disposition::Permanent | Disposition::FeatureNegotiation + ) { + Disposition::PerMessageRecoverable + } else { + base + } +} + +#[cfg(test)] +mod tests { + use super::super::error::NoError; + use super::*; + use std::io::{self, ErrorKind}; + + #[test] + fn transport_errors_are_transport_drop() { + for k in [ + ErrorKind::ConnectionReset, + ErrorKind::ConnectionAborted, + ErrorKind::BrokenPipe, + ErrorKind::UnexpectedEof, + ErrorKind::TimedOut, + ] { + let e = ImapError::Io(io::Error::new(k, "x")); + assert_eq!(classify(&e), Disposition::TransportDrop, "{k:?}"); + } + } + + #[test] + fn bye_is_transport_drop() { + assert_eq!( + classify(&ImapError::Bye("going away".into())), + Disposition::TransportDrop + ); + } + + #[test] + fn auth_failed_classified_separately() { + let e = ImapError::AuthFailed("bad creds".into()); + assert_eq!(classify(&e), Disposition::AuthFailed); + let e = ImapError::No(NoError::new("x", Some("AUTHENTICATIONFAILED".into()))); + assert_eq!(classify(&e), Disposition::AuthFailed); + } + + #[test] + fn transient_no_text_marked_transient() { + let e = ImapError::No(NoError::new("Try again later", None)); + assert_eq!(classify(&e), Disposition::Transient); + let e = ImapError::No(NoError::new("user over quota", None)); + assert_eq!(classify(&e), Disposition::Transient); + } + + #[test] + fn non_transient_no_is_permanent() { + let e = ImapError::No(NoError::new("mailbox does not exist", None)); + assert_eq!(classify(&e), Disposition::Permanent); + } + + #[test] + fn bad_is_feature_negotiation() { + let e = ImapError::Bad("unrecognised command".into()); + assert_eq!(classify(&e), Disposition::FeatureNegotiation); + } + + #[test] + fn protocol_and_parse_are_permanent() { + assert_eq!( + classify(&ImapError::Protocol("x".into())), + Disposition::Permanent + ); + assert_eq!( + classify(&ImapError::Parse("x".into())), + Disposition::Permanent + ); + } + + #[test] + fn folder_disposition_lifts_permanent_to_per_folder() { + let e = ImapError::No(NoError::new("no such mailbox", None)); + assert_eq!(folder_disposition(&e), Disposition::PerFolderPermanent); + } + + #[test] + fn folder_disposition_keeps_transient_class() { + let e = ImapError::No(NoError::new("try again", None)); + assert_eq!(folder_disposition(&e), Disposition::Transient); + } + + #[test] + fn message_disposition_demotes_permanent_to_per_message() { + let e = ImapError::Parse("bad fetch".into()); + assert_eq!(message_disposition(&e), Disposition::PerMessageRecoverable); + } + + #[test] + fn shared_delay_grows_with_consecutive_throttles() { + let s = BackoffState::new(); + let d1 = s.next_shared_delay(); + let d3 = { + let _ = s.next_shared_delay(); + s.next_shared_delay() + }; + assert!(d1 <= Duration::from_millis(SHARED_BASE_MS)); + assert!( + d3 >= d1, + "expected non-decreasing delay, got {d1:?} -> {d3:?}" + ); + assert!(d3 <= Duration::from_millis(SHARED_CAP_MS)); + } + + #[test] + fn shared_delay_caps_at_cap_ms() { + let s = BackoffState::new(); + for _ in 0..40 { + let _ = s.next_shared_delay(); + } + let d = s.next_shared_delay(); + assert!(d <= Duration::from_millis(SHARED_CAP_MS)); + } + + #[test] + fn reset_clears_level() { + let s = BackoffState::new(); + let _ = s.next_shared_delay(); + let _ = s.next_shared_delay(); + assert!(s.current_level() > 0); + s.reset(); + assert_eq!(s.current_level(), 0); + } + + #[test] + fn transport_delay_does_not_touch_shared_level() { + let s = BackoffState::new(); + let _ = s.transport_delay(1); + let _ = s.transport_delay(2); + let _ = s.transport_delay(3); + assert_eq!(s.current_level(), 0, "transport delays are local"); + } + + #[test] + fn transport_delay_bounded() { + let s = BackoffState::new(); + let d1 = s.transport_delay(1); + let d6 = s.transport_delay(6); + assert!(d1 <= Duration::from_millis(TRANSPORT_BASE_MS)); + assert!(d6 <= Duration::from_millis(TRANSPORT_CAP_MS)); + } + + #[test] + fn shared_state_is_cloneable_and_shared() { + let a = BackoffState::new(); + let b = a.clone(); + let _ = a.next_shared_delay(); + assert_eq!(a.current_level(), b.current_level()); + } + + #[test] + fn is_negotiation_failure_matches_bad_unsupported_and_nontransient_no() { + assert!(is_negotiation_failure(&ImapError::Bad("x".into()))); + assert!(is_negotiation_failure(&ImapError::Unsupported("x".into()))); + assert!(is_negotiation_failure(&ImapError::No(NoError::new( + "no such mailbox", + None + )))); + } + + #[test] + fn is_negotiation_failure_excludes_transient_no_and_others() { + assert!(!is_negotiation_failure(&ImapError::No(NoError::new( + "try again later", + None + )))); + assert!(!is_negotiation_failure(&ImapError::Bye("x".into()))); + assert!(!is_negotiation_failure(&ImapError::Disconnected)); + } + + #[test] + fn transient_retries_counter_increments_on_shared_delay() { + let s = BackoffState::new(); + assert_eq!(s.transient_retries(), 0); + let _ = s.next_shared_delay(); + let _ = s.next_shared_delay(); + assert_eq!(s.transient_retries(), 2); + } + + #[test] + fn transport_retries_counter_increments_on_transport_delay() { + let s = BackoffState::new(); + assert_eq!(s.transport_retries(), 0); + let _ = s.transport_delay(1); + let _ = s.transport_delay(2); + assert_eq!(s.transport_retries(), 2); + } +} diff --git a/src/imap/transport.rs b/src/imap/transport.rs new file mode 100644 index 0000000..a538f8e --- /dev/null +++ b/src/imap/transport.rs @@ -0,0 +1,435 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::{self, Read, Write}; +use std::net::TcpStream; +use std::sync::Arc; +use std::time::Duration; + +use rustls::pki_types::ServerName; +use rustls::{ClientConfig, ClientConnection, StreamOwned}; +use rustls_platform_verifier::BuilderVerifierExt; + +use super::error::ImapError; + +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); + +pub trait ImapStream: Read + Write + Send { + fn upgrade_tls( + self: Box, + connector: &Connector, + host: &str, + ) -> Result, ImapError>; + + fn is_tls(&self) -> bool; +} + +pub struct Connector { + config: Arc, +} + +impl Connector { + pub fn new(allow_invalid_certs: bool) -> Result { + let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + let builder = ClientConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .map_err(|e| ImapError::Tls(format!("rustls protocol versions: {e}")))?; + let config = if allow_invalid_certs { + builder + .dangerous() + .with_custom_certificate_verifier(Arc::new(NoVerify)) + .with_no_client_auth() + } else { + builder + .with_platform_verifier() + .map_err(|e| ImapError::Tls(format!("rustls platform verifier: {e}")))? + .with_no_client_auth() + }; + Ok(Connector { + config: Arc::new(config), + }) + } + + pub fn connect_plain(&self, host: &str, port: u16) -> Result, ImapError> { + let tcp = tcp_connect(host, port)?; + Ok(Box::new(TcpImapStream { tcp })) + } + + pub fn connect_tls(&self, host: &str, port: u16) -> Result, ImapError> { + let tcp = tcp_connect(host, port)?; + self.wrap_tls(tcp, host) + } + + fn wrap_tls(&self, tcp: TcpStream, host: &str) -> Result, ImapError> { + let server_name = ServerName::try_from(host.to_owned()) + .map_err(|e| ImapError::Tls(format!("invalid server name {host}: {e}")))?; + let conn = ClientConnection::new(self.config.clone(), server_name) + .map_err(|e| ImapError::Tls(format!("rustls client: {e}")))?; + Ok(Box::new(TlsImapStream { + inner: StreamOwned::new(conn, tcp), + })) + } +} + +fn tcp_connect(host: &str, port: u16) -> Result { + let tcp = TcpStream::connect((host, port))?; + tcp.set_read_timeout(Some(DEFAULT_TIMEOUT))?; + tcp.set_write_timeout(Some(DEFAULT_TIMEOUT))?; + tcp.set_nodelay(true)?; + Ok(tcp) +} + +pub struct TcpImapStream { + tcp: TcpStream, +} + +impl Read for TcpImapStream { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.tcp.read(buf) + } +} + +impl Write for TcpImapStream { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.tcp.write(buf) + } + fn flush(&mut self) -> io::Result<()> { + self.tcp.flush() + } +} + +impl ImapStream for TcpImapStream { + fn upgrade_tls( + self: Box, + connector: &Connector, + host: &str, + ) -> Result, ImapError> { + connector.wrap_tls(self.tcp, host) + } + + fn is_tls(&self) -> bool { + false + } +} + +pub struct TlsImapStream { + inner: StreamOwned, +} + +impl Read for TlsImapStream { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.inner.read(buf) + } +} + +impl Write for TlsImapStream { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.inner.write(buf) + } + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +impl ImapStream for TlsImapStream { + fn upgrade_tls( + self: Box, + _connector: &Connector, + _host: &str, + ) -> Result, ImapError> { + Ok(self) + } + + fn is_tls(&self) -> bool { + true + } +} + +pub struct DeflateImapStream { + inner: Box, + decompress: flate2::Decompress, + compress: flate2::Compress, + rx_raw: Vec, + rx_raw_pos: usize, + rx_decoded: Vec, + rx_decoded_pos: usize, +} + +impl DeflateImapStream { + pub fn wrap(inner: Box, primer: &[u8]) -> Box { + Box::new(DeflateImapStream { + inner, + decompress: flate2::Decompress::new(false), + compress: flate2::Compress::new(flate2::Compression::default(), false), + rx_raw: primer.to_vec(), + rx_raw_pos: 0, + rx_decoded: Vec::new(), + rx_decoded_pos: 0, + }) + } +} + +impl Read for DeflateImapStream { + fn read(&mut self, out: &mut [u8]) -> io::Result { + loop { + if self.rx_decoded_pos < self.rx_decoded.len() { + let avail = &self.rx_decoded[self.rx_decoded_pos..]; + let n = avail.len().min(out.len()); + out[..n].copy_from_slice(&avail[..n]); + self.rx_decoded_pos += n; + return Ok(n); + } + if self.rx_raw_pos >= self.rx_raw.len() { + self.rx_raw.clear(); + self.rx_raw_pos = 0; + self.rx_raw.resize(8192, 0); + let n = self.inner.read(&mut self.rx_raw)?; + if n == 0 { + return Ok(0); + } + self.rx_raw.truncate(n); + } + self.rx_decoded.clear(); + self.rx_decoded_pos = 0; + self.rx_decoded.resize(out.len().max(8192) * 8, 0); + let in_before = self.decompress.total_in(); + let out_before = self.decompress.total_out(); + let status = self.decompress.decompress( + &self.rx_raw[self.rx_raw_pos..], + &mut self.rx_decoded, + flate2::FlushDecompress::None, + ); + match status { + Ok(_) => {} + Err(e) => { + return Err(io::Error::new(io::ErrorKind::InvalidData, e)); + } + } + let consumed = (self.decompress.total_in() - in_before) as usize; + let produced = (self.decompress.total_out() - out_before) as usize; + self.rx_raw_pos += consumed; + self.rx_decoded.truncate(produced); + if produced == 0 && consumed == 0 { + self.rx_raw.clear(); + self.rx_raw_pos = 0; + continue; + } + } + } +} + +impl Write for DeflateImapStream { + fn write(&mut self, buf: &[u8]) -> io::Result { + let mut out = vec![0u8; buf.len() + 256]; + let in_before = self.compress.total_in(); + let out_before = self.compress.total_out(); + self.compress + .compress(buf, &mut out, flate2::FlushCompress::Sync) + .map_err(io::Error::other)?; + let consumed = (self.compress.total_in() - in_before) as usize; + let produced = (self.compress.total_out() - out_before) as usize; + self.inner.write_all(&out[..produced])?; + Ok(consumed) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +impl ImapStream for DeflateImapStream { + fn upgrade_tls( + self: Box, + _connector: &Connector, + _host: &str, + ) -> Result, ImapError> { + Err(ImapError::Protocol( + "cannot STARTTLS after COMPRESS DEFLATE".into(), + )) + } + + fn is_tls(&self) -> bool { + self.inner.is_tls() + } +} + +#[derive(Debug)] +struct NoVerify; + +impl rustls::client::danger::ServerCertVerifier for NoVerify { + fn verify_server_cert( + &self, + _end_entity: &rustls::pki_types::CertificateDer<'_>, + _intermediates: &[rustls::pki_types::CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, + _ocsp_response: &[u8], + _now: rustls::pki_types::UnixTime, + ) -> Result { + Ok(rustls::client::danger::ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _message: &[u8], + _cert: &rustls::pki_types::CertificateDer<'_>, + _dss: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _message: &[u8], + _cert: &rustls::pki_types::CertificateDer<'_>, + _dss: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) + } + + fn supported_verify_schemes(&self) -> Vec { + use rustls::SignatureScheme::*; + vec![ + RSA_PKCS1_SHA256, + RSA_PKCS1_SHA384, + RSA_PKCS1_SHA512, + ECDSA_NISTP256_SHA256, + ECDSA_NISTP384_SHA384, + ECDSA_NISTP521_SHA512, + RSA_PSS_SHA256, + RSA_PSS_SHA384, + RSA_PSS_SHA512, + ED25519, + ] + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn connector_constructs_with_invalid_certs_disabled() { + let _c = Connector::new(false).unwrap(); + } + + #[test] + fn connector_constructs_with_invalid_certs_allowed() { + let _c = Connector::new(true).unwrap(); + } + + struct PeerStream { + outbound: Vec, + inbound: Vec, + inbound_pos: usize, + } + + impl PeerStream { + fn new() -> Self { + Self { + outbound: Vec::new(), + inbound: Vec::new(), + inbound_pos: 0, + } + } + } + + impl Read for PeerStream { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if self.inbound_pos >= self.inbound.len() { + return Ok(0); + } + let avail = &self.inbound[self.inbound_pos..]; + let n = avail.len().min(buf.len()); + buf[..n].copy_from_slice(&avail[..n]); + self.inbound_pos += n; + Ok(n) + } + } + + impl Write for PeerStream { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.outbound.extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + impl ImapStream for PeerStream { + fn upgrade_tls( + self: Box, + _c: &Connector, + _h: &str, + ) -> Result, ImapError> { + Ok(self) + } + fn is_tls(&self) -> bool { + false + } + } + + #[test] + fn deflate_writes_round_trip_through_peer_decompressor() { + let peer = Box::new(PeerStream::new()); + let mut stream = DeflateImapStream::wrap(peer, b""); + stream.write_all(b"hello world").unwrap(); + + stream.write_all(b" again").unwrap(); + stream.flush().unwrap(); + } + + #[test] + fn deflate_reads_decompressed_payload_from_inner() { + + let mut peer = PeerStream::new(); + let mut compressor = flate2::Compress::new(flate2::Compression::default(), false); + let plaintext = b"* OK [CAPABILITY IMAP4rev2 COMPRESS=DEFLATE] hi\r\n"; + let mut out = vec![0u8; plaintext.len() + 256]; + compressor + .compress(plaintext, &mut out, flate2::FlushCompress::Sync) + .unwrap(); + let n = compressor.total_out() as usize; + peer.inbound.extend_from_slice(&out[..n]); + + let mut stream = DeflateImapStream::wrap(Box::new(peer), b""); + let mut got = vec![0u8; 256]; + let mut total = 0usize; + while total < plaintext.len() { + let n = stream.read(&mut got[total..]).unwrap(); + if n == 0 { + break; + } + total += n; + } + assert_eq!(&got[..total], plaintext); + } + + #[test] + fn deflate_consumes_primer_before_reading_inner() { + let mut compressor = flate2::Compress::new(flate2::Compression::default(), false); + let plaintext = b"primed payload\r\n"; + let mut out = vec![0u8; plaintext.len() + 256]; + compressor + .compress(plaintext, &mut out, flate2::FlushCompress::Sync) + .unwrap(); + let n = compressor.total_out() as usize; + let primer = out[..n].to_vec(); + + let peer = Box::new(PeerStream::new()); + let mut stream = DeflateImapStream::wrap(peer, &primer); + let mut buf = vec![0u8; 256]; + let mut total = 0usize; + while total < plaintext.len() { + let n = stream.read(&mut buf[total..]).unwrap(); + if n == 0 { + break; + } + total += n; + } + assert_eq!(&buf[..total], plaintext); + } +} diff --git a/src/inspect/list.rs b/src/inspect/list.rs new file mode 100644 index 0000000..2690b06 --- /dev/null +++ b/src/inspect/list.rs @@ -0,0 +1,631 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; +use std::io::Write; + +use rusqlite::Connection; +use serde_json::Value; + +use super::{blob_summary, format_count}; +use crate::error::Error; +use crate::types::ObjectType; + +pub fn write_list( + conn: &Connection, + ty: ObjectType, + limit: Option, + offset: usize, + out: &mut impl Write, +) -> Result<(), Error> { + let table = crate::sync::table_name(ty); + let total: i64 = conn.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r.get(0))?; + let total = total as u64; + write_header(out, ty, total, limit, offset)?; + if total == 0 { + writeln!(out, "(no {} in archive)", ty.token())?; + return Ok(()); + } + let names = Names::load(conn, ty)?; + let mut written = 0u64; + match ty { + ObjectType::Identity => write_identities(conn, limit, offset, out, &mut written)?, + ObjectType::Email => write_emails(conn, &names, limit, offset, out, &mut written)?, + ObjectType::SieveScript => write_sieves(conn, limit, offset, out, &mut written)?, + ObjectType::AddressBook => write_address_books(conn, limit, offset, out, &mut written)?, + ObjectType::ContactCard => { + write_contact_cards(conn, &names, limit, offset, out, &mut written)? + } + ObjectType::Calendar => write_calendars(conn, limit, offset, out, &mut written)?, + ObjectType::CalendarEvent => { + write_calendar_events(conn, &names, limit, offset, out, &mut written)? + } + ObjectType::ParticipantIdentity => { + write_participant_identities(conn, limit, offset, out, &mut written)? + } + ObjectType::Mailbox | ObjectType::FileNode => unreachable!("tree types do not list"), + } + if let Some(lim) = limit { + let next_offset = (offset as u64) + written; + if next_offset < total { + writeln!( + out, + "... ({} more; --offset {} --limit {} to continue)", + format_count(total - next_offset), + next_offset, + lim + )?; + } + } + Ok(()) +} + +fn write_header( + out: &mut impl Write, + ty: ObjectType, + total: u64, + limit: Option, + offset: usize, +) -> Result<(), Error> { + match limit { + None if offset == 0 => writeln!(out, "{} ({})", ty.jmap_name(), format_count(total))?, + None => writeln!( + out, + "{} ({}, skipping first {})", + ty.jmap_name(), + format_count(total), + format_count(offset as u64) + )?, + Some(lim) => writeln!( + out, + "{} ({} total, showing up to {} from offset {})", + ty.jmap_name(), + format_count(total), + format_count(lim as u64), + format_count(offset as u64) + )?, + } + writeln!(out)?; + Ok(()) +} + +pub(crate) struct Names { + mailboxes: HashMap, + address_books: HashMap, + calendars: HashMap, +} + +impl Names { + pub(crate) fn load(conn: &Connection, ty: ObjectType) -> Result { + let mailboxes = if matches!(ty, ObjectType::Email) { + load_name_map(conn, "SELECT id, name FROM mailboxes")? + } else { + HashMap::new() + }; + let address_books = if matches!(ty, ObjectType::ContactCard) { + load_name_map(conn, "SELECT id, name FROM address_books")? + } else { + HashMap::new() + }; + let calendars = if matches!(ty, ObjectType::CalendarEvent) { + load_name_map(conn, "SELECT id, name FROM calendars")? + } else { + HashMap::new() + }; + Ok(Names { + mailboxes, + address_books, + calendars, + }) + } + + fn render(&self, kind: ObjectType, ids: &[i64]) -> String { + let map = match kind { + ObjectType::Mailbox => &self.mailboxes, + ObjectType::AddressBook => &self.address_books, + ObjectType::Calendar => &self.calendars, + _ => return format!("{ids:?}"), + }; + let parts: Vec = ids + .iter() + .map(|id| match map.get(id) { + Some(name) => format!("{name:?}"), + None => format!("#{id} (missing)"), + }) + .collect(); + format!("[{}]", parts.join(", ")) + } +} + +fn load_name_map(conn: &Connection, sql: &str) -> Result, Error> { + let mut stmt = conn.prepare(sql)?; + let rows = stmt.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)))?; + let mut out = HashMap::new(); + for row in rows { + let (id, name) = row?; + out.insert(id, name); + } + Ok(out) +} + +fn limit_clause(limit: Option, offset: usize) -> String { + match limit { + Some(n) => format!(" LIMIT {n} OFFSET {offset}"), + None if offset > 0 => format!(" LIMIT -1 OFFSET {offset}"), + None => String::new(), + } +} + +fn parse_local_ids(text: &str) -> Vec { + serde_json::from_str::(text) + .ok() + .and_then(|v| { + v.as_array() + .map(|arr| arr.iter().filter_map(|x| x.as_i64()).collect::>()) + }) + .unwrap_or_default() +} + +fn json_pretty_or_raw(text: &str) -> String { + match serde_json::from_str::(text) { + Ok(v) => serde_json::to_string_pretty(&v).unwrap_or_else(|_| text.to_owned()), + Err(_) => text.to_owned(), + } +} + +fn keywords_summary(text: &str) -> String { + let ids = serde_json::from_str::>(text).unwrap_or_default(); + if ids.is_empty() { + "(none)".to_owned() + } else { + format!("[{}]", ids.join(", ")) + } +} + +fn opt_or_null(s: Option) -> String { + s.unwrap_or_else(|| "(null)".to_owned()) +} + +fn bool_str(b: bool) -> String { + (if b { "true" } else { "false" }).to_owned() +} + +fn write_record(out: &mut impl Write, title: &str, fields: &[(&str, String)]) -> Result<(), Error> { + writeln!(out, "{title}")?; + let max_label = fields.iter().map(|(l, _)| l.len()).max().unwrap_or(0); + let pad = " ".repeat(max_label + 4); + for (label, value) in fields { + for (i, line) in value.split('\n').enumerate() { + if i == 0 { + writeln!(out, " {label:, + offset: usize, + out: &mut impl Write, + written: &mut u64, +) -> Result<(), Error> { + let sql = format!( + "SELECT id, name, email, reply_to, bcc, text_signature, html_signature + FROM identities ORDER BY id{}", + limit_clause(limit, offset) + ); + let mut stmt = conn.prepare(&sql)?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let id: i64 = row.get(0)?; + let fields = vec![ + ("name", row.get::<_, String>(1)?), + ("email", row.get::<_, String>(2)?), + ("reply_to", opt_or_null(row.get(3)?)), + ("bcc", opt_or_null(row.get(4)?)), + ("text_signature", row.get::<_, String>(5)?), + ("html_signature", row.get::<_, String>(6)?), + ]; + write_record(out, &format!("identity #{id}"), &fields)?; + *written += 1; + } + Ok(()) +} + +fn write_emails( + conn: &Connection, + names: &Names, + limit: Option, + offset: usize, + out: &mut impl Write, + written: &mut u64, +) -> Result<(), Error> { + let sql = format!( + "SELECT id, blob_id, received_at, mailbox_ids, keywords, message_match + FROM emails ORDER BY id{}", + limit_clause(limit, offset) + ); + let mut stmt = conn.prepare(&sql)?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let id: i64 = row.get(0)?; + let blob_id: i64 = row.get(1)?; + let mailbox_ids: String = row.get(3)?; + let keywords: String = row.get(4)?; + let message_match: String = row.get(5)?; + let locals = parse_local_ids(&mailbox_ids); + let fields = vec![ + ("blob", blob_summary(conn, blob_id)?), + ("received_at", row.get::<_, String>(2)?), + ("mailboxes", names.render(ObjectType::Mailbox, &locals)), + ("keywords", keywords_summary(&keywords)), + ("message_match", json_pretty_or_raw(&message_match)), + ]; + write_record(out, &format!("email #{id}"), &fields)?; + *written += 1; + } + Ok(()) +} + +fn write_sieves( + conn: &Connection, + limit: Option, + offset: usize, + out: &mut impl Write, + written: &mut u64, +) -> Result<(), Error> { + let sql = format!( + "SELECT id, name, is_active, blob_id FROM sieve_scripts ORDER BY id{}", + limit_clause(limit, offset) + ); + let mut stmt = conn.prepare(&sql)?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let id: i64 = row.get(0)?; + let name: Option = row.get(1)?; + let active: i64 = row.get(2)?; + let blob_id: i64 = row.get(3)?; + let fields = vec![ + ("name", opt_or_null(name)), + ("is_active", bool_str(active != 0)), + ("blob", blob_summary(conn, blob_id)?), + ]; + write_record(out, &format!("sievescript #{id}"), &fields)?; + *written += 1; + } + Ok(()) +} + +fn write_address_books( + conn: &Connection, + limit: Option, + offset: usize, + out: &mut impl Write, + written: &mut u64, +) -> Result<(), Error> { + let sql = format!( + "SELECT id, name, description, sort_order, is_default, is_subscribed + FROM address_books ORDER BY id{}", + limit_clause(limit, offset) + ); + let mut stmt = conn.prepare(&sql)?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let id: i64 = row.get(0)?; + let fields = vec![ + ("name", row.get::<_, String>(1)?), + ("description", opt_or_null(row.get(2)?)), + ("sort_order", row.get::<_, i64>(3)?.to_string()), + ("is_default", bool_str(row.get::<_, i64>(4)? != 0)), + ("is_subscribed", bool_str(row.get::<_, i64>(5)? != 0)), + ]; + write_record(out, &format!("addressbook #{id}"), &fields)?; + *written += 1; + } + Ok(()) +} + +fn write_contact_cards( + conn: &Connection, + names: &Names, + limit: Option, + offset: usize, + out: &mut impl Write, + written: &mut u64, +) -> Result<(), Error> { + let sql = format!( + "SELECT id, uid, address_book_ids, data FROM contact_cards ORDER BY id{}", + limit_clause(limit, offset) + ); + let mut stmt = conn.prepare(&sql)?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let id: i64 = row.get(0)?; + let uid: String = row.get(1)?; + let ab_ids: String = row.get(2)?; + let data: String = row.get(3)?; + let locals = parse_local_ids(&ab_ids); + let fields = vec![ + ("uid", uid), + ( + "address_books", + names.render(ObjectType::AddressBook, &locals), + ), + ("data", redact_blobs_in_json(&data)), + ]; + write_record(out, &format!("contactcard #{id}"), &fields)?; + *written += 1; + } + Ok(()) +} + +fn write_calendars( + conn: &Connection, + limit: Option, + offset: usize, + out: &mut impl Write, + written: &mut u64, +) -> Result<(), Error> { + let sql = format!( + "SELECT id, name, description, color, sort_order, is_subscribed, is_visible, is_default, + include_in_availability, default_alerts_with_time, default_alerts_without_time, + time_zone + FROM calendars ORDER BY id{}", + limit_clause(limit, offset) + ); + let mut stmt = conn.prepare(&sql)?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let id: i64 = row.get(0)?; + let fields = vec![ + ("name", row.get::<_, String>(1)?), + ("description", opt_or_null(row.get(2)?)), + ("color", opt_or_null(row.get(3)?)), + ("sort_order", row.get::<_, i64>(4)?.to_string()), + ("is_subscribed", bool_str(row.get::<_, i64>(5)? != 0)), + ("is_visible", bool_str(row.get::<_, i64>(6)? != 0)), + ("is_default", bool_str(row.get::<_, i64>(7)? != 0)), + ("include_in_availability", row.get::<_, String>(8)?), + ("default_alerts_with_time", opt_or_null(row.get(9)?)), + ("default_alerts_without_time", opt_or_null(row.get(10)?)), + ("time_zone", opt_or_null(row.get(11)?)), + ]; + write_record(out, &format!("calendar #{id}"), &fields)?; + *written += 1; + } + Ok(()) +} + +fn write_calendar_events( + conn: &Connection, + names: &Names, + limit: Option, + offset: usize, + out: &mut impl Write, + written: &mut u64, +) -> Result<(), Error> { + let sql = format!( + "SELECT id, calendar_ids, is_draft, use_default_alerts, data + FROM calendar_events ORDER BY id{}", + limit_clause(limit, offset) + ); + let mut stmt = conn.prepare(&sql)?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let id: i64 = row.get(0)?; + let cal_ids: String = row.get(1)?; + let data: String = row.get(4)?; + let locals = parse_local_ids(&cal_ids); + let fields = vec![ + ("calendars", names.render(ObjectType::Calendar, &locals)), + ("is_draft", bool_str(row.get::<_, i64>(2)? != 0)), + ("use_default_alerts", bool_str(row.get::<_, i64>(3)? != 0)), + ("data", redact_blobs_in_json(&data)), + ]; + write_record(out, &format!("calendarevent #{id}"), &fields)?; + *written += 1; + } + Ok(()) +} + +fn write_participant_identities( + conn: &Connection, + limit: Option, + offset: usize, + out: &mut impl Write, + written: &mut u64, +) -> Result<(), Error> { + let sql = format!( + "SELECT id, name, calendar_address, is_default + FROM participant_identities ORDER BY id{}", + limit_clause(limit, offset) + ); + let mut stmt = conn.prepare(&sql)?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let id: i64 = row.get(0)?; + let fields = vec![ + ("name", row.get::<_, String>(1)?), + ("calendar_address", row.get::<_, String>(2)?), + ("is_default", bool_str(row.get::<_, i64>(3)? != 0)), + ]; + write_record(out, &format!("participantidentity #{id}"), &fields)?; + *written += 1; + } + Ok(()) +} + +fn redact_blobs_in_json(text: &str) -> String { + let mut value = match serde_json::from_str::(text) { + Ok(v) => v, + Err(_) => return text.to_owned(), + }; + redact_blobs(&mut value); + serde_json::to_string_pretty(&value).unwrap_or_else(|_| text.to_owned()) +} + +fn redact_blobs(value: &mut Value) { + match value { + Value::Object(map) => { + if let Some(bid) = map.get("@blob").and_then(|v| v.as_i64()) { + map.clear(); + map.insert( + "@blob".to_owned(), + Value::String(format!("local #{bid} (omitted)")), + ); + return; + } + for v in map.values_mut() { + redact_blobs(v); + } + } + Value::Array(arr) => { + for v in arr.iter_mut() { + redact_blobs(v); + } + } + _ => {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::{blobs, init}; + use rusqlite::params; + + fn mem() -> Connection { + let c = Connection::open_in_memory().unwrap(); + init::apply_schema(&c).unwrap(); + c + } + + fn add_mailbox(c: &Connection, name: &str) -> i64 { + c.execute("INSERT INTO mailboxes (name) VALUES (?1)", params![name]) + .unwrap(); + c.last_insert_rowid() + } + + #[test] + fn empty_table_prints_none_message() { + let c = mem(); + let mut buf = Vec::new(); + write_list(&c, ObjectType::Identity, None, 0, &mut buf).unwrap(); + let s = String::from_utf8(buf).unwrap(); + assert!(s.contains("Identity (0)")); + assert!(s.contains("(no identity in archive)")); + } + + #[test] + fn identity_record_uses_aligned_fields_and_redacts_nothing() { + let c = mem(); + c.execute( + "INSERT INTO identities (name, email, text_signature) VALUES ('Alice', 'a@x.test', 'best,A')", + [], + ) + .unwrap(); + let mut buf = Vec::new(); + write_list(&c, ObjectType::Identity, None, 0, &mut buf).unwrap(); + let s = String::from_utf8(buf).unwrap(); + assert!(s.contains("identity #1")); + assert!(s.contains(" name Alice"), "got:\n{s}"); + assert!(s.contains(" email a@x.test")); + assert!(s.contains(" bcc (null)")); + } + + #[test] + fn email_resolves_mailbox_names_and_summarises_blob() { + let c = mem(); + let inbox = add_mailbox(&c, "Inbox"); + let other = add_mailbox(&c, "Personal"); + let blob = blobs::intern_blob(&c, b"From: a@x.test\r\nSubject: hi\r\n\r\nhello").unwrap(); + c.execute( + "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords) + VALUES (?1, '2024-01-01T00:00:00Z', ?2, ?3)", + params![ + blob, + format!("[{inbox},{other}]"), + "[\"$seen\",\"Important\"]" + ], + ) + .unwrap(); + let mut buf = Vec::new(); + write_list(&c, ObjectType::Email, None, 0, &mut buf).unwrap(); + let s = String::from_utf8(buf).unwrap(); + assert!(s.contains("email #1")); + assert!(s.contains("blob ")); + assert!(s.contains("blake3=")); + assert!(s.contains("mailboxes ")); + assert!(s.contains("\"Inbox\""), "got:\n{s}"); + assert!(s.contains("\"Personal\"")); + assert!(s.contains("keywords ")); + assert!(s.contains("[$seen, Important]")); + } + + #[test] + fn contact_card_redacts_blob_sentinel_in_data() { + let c = mem(); + c.execute("INSERT INTO address_books (name) VALUES ('Personal')", []) + .unwrap(); + c.execute( + "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)", + params![ + "urn:uuid:1", + "[1]", + r#"{"name":{"full":"Alice"},"photos":{"p":{"@blob":17}}}"# + ], + ) + .unwrap(); + let mut buf = Vec::new(); + write_list(&c, ObjectType::ContactCard, None, 0, &mut buf).unwrap(); + let s = String::from_utf8(buf).unwrap(); + assert!(s.contains("contactcard #1")); + assert!(s.contains("\"Personal\"")); + assert!(s.contains("local #17 (omitted)")); + assert!(!s.contains("\"@blob\": 17")); + } + + #[test] + fn pagination_writes_continuation_hint_when_more_remain() { + let c = mem(); + for i in 0..5 { + c.execute( + "INSERT INTO identities (name, email) VALUES (?1, ?2)", + params![format!("u{i}"), format!("u{i}@x.test")], + ) + .unwrap(); + } + let mut buf = Vec::new(); + write_list(&c, ObjectType::Identity, Some(2), 0, &mut buf).unwrap(); + let s = String::from_utf8(buf).unwrap(); + assert!(s.contains("Identity (5 total, showing up to 2 from offset 0)")); + assert!(s.contains("identity #1")); + assert!(s.contains("identity #2")); + assert!(!s.contains("identity #3")); + assert!(s.contains("... (3 more; --offset 2 --limit 2 to continue)")); + } + + #[test] + fn pagination_at_end_does_not_print_continuation() { + let c = mem(); + for i in 0..3 { + c.execute( + "INSERT INTO identities (name, email) VALUES (?1, ?2)", + params![format!("u{i}"), format!("u{i}@x.test")], + ) + .unwrap(); + } + let mut buf = Vec::new(); + write_list(&c, ObjectType::Identity, Some(5), 0, &mut buf).unwrap(); + let s = String::from_utf8(buf).unwrap(); + assert!(!s.contains("more;"), "got:\n{s}"); + } +} diff --git a/src/inspect/mod.rs b/src/inspect/mod.rs new file mode 100644 index 0000000..f09e273 --- /dev/null +++ b/src/inspect/mod.rs @@ -0,0 +1,285 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; + +use rusqlite::{Connection, OptionalExtension, params}; + +use crate::db::init; +use crate::error::Error; +use crate::types::ObjectType; + +pub mod list; +pub mod tree; + +pub struct InspectConfig { + pub archive: PathBuf, + pub target: Option, + pub limit: Option, + pub offset: usize, +} + +pub fn run(config: InspectConfig) -> Result<(), Error> { + let conn = init::open(&config.archive)?; + let stdout = io::stdout(); + let mut out = stdout.lock(); + dispatch(&conn, &config, &mut out)?; + Ok(()) +} + +fn dispatch(conn: &Connection, cfg: &InspectConfig, out: &mut impl Write) -> Result<(), Error> { + match cfg.target { + None => write_summary(conn, &cfg.archive, out), + Some(ObjectType::Mailbox) => { + warn_if_paged(out, cfg, "mailbox")?; + tree::write_mailboxes(conn, out) + } + Some(ObjectType::FileNode) => { + warn_if_paged(out, cfg, "filenode")?; + tree::write_file_nodes(conn, out) + } + Some(ty) => list::write_list(conn, ty, cfg.limit, cfg.offset, out), + } +} + +fn warn_if_paged(out: &mut impl Write, cfg: &InspectConfig, token: &str) -> Result<(), Error> { + if cfg.limit.is_some() || cfg.offset > 0 { + writeln!( + out, + "note: --limit/--offset are ignored for {token} (tree view always shows the full hierarchy)" + )?; + } + Ok(()) +} + +fn write_summary(conn: &Connection, archive: &Path, out: &mut impl Write) -> Result<(), Error> { + writeln!(out, "Archive: {}", archive.display())?; + match read_source(conn)? { + Some(src) => { + let name = src.account_name.unwrap_or_else(|| "(unnamed)".to_owned()); + writeln!( + out, + "Source: {} {} (account {} / {})", + src.kind, src.session_url, src.account_id, name + )?; + } + None => writeln!(out, "Source: (no source recorded)")?, + } + writeln!(out)?; + let counts = read_type_counts(conn)?; + let label_width = ObjectType::ALL + .iter() + .map(|t| t.token().len()) + .max() + .unwrap_or(0); + let count_width = counts + .values() + .map(|n| format_count(*n).len()) + .max() + .unwrap_or(1) + .max("blobs".len()); + for ty in ObjectType::ALL { + let n = counts.get(&ty).copied().unwrap_or(0); + writeln!( + out, + " {:cw$}", + ty.token(), + format_count(n), + lw = label_width, + cw = count_width + )?; + } + let (blob_count, blob_bytes) = read_blob_totals(conn)?; + writeln!(out)?; + writeln!( + out, + " {:cw$} ({})", + "blobs", + format_count(blob_count), + format_bytes(blob_bytes), + lw = label_width, + cw = count_width + )?; + Ok(()) +} + +struct SourceRow { + kind: String, + session_url: String, + account_id: String, + account_name: Option, +} + +fn read_source(conn: &Connection) -> Result, Error> { + let row = conn + .query_row( + "SELECT kind, session_url, account_id, account_name FROM sources LIMIT 1", + [], + |r| { + Ok(SourceRow { + kind: r.get(0)?, + session_url: r.get(1)?, + account_id: r.get(2)?, + account_name: r.get(3)?, + }) + }, + ) + .optional()?; + Ok(row) +} + +fn read_type_counts(conn: &Connection) -> Result, Error> { + let mut out = HashMap::new(); + for ty in ObjectType::ALL { + let sql = format!("SELECT COUNT(*) FROM {}", crate::sync::table_name(ty)); + let n: i64 = conn.query_row(&sql, [], |r| r.get(0))?; + out.insert(ty, n as u64); + } + Ok(out) +} + +fn read_blob_totals(conn: &Connection) -> Result<(u64, u64), Error> { + let (count, bytes): (i64, i64) = conn.query_row( + "SELECT COUNT(*), COALESCE(SUM(length(data)), 0) FROM blobs", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + )?; + Ok((count as u64, bytes.max(0) as u64)) +} + +pub(crate) fn format_count(n: u64) -> String { + let s = n.to_string(); + let bytes = s.as_bytes(); + let len = bytes.len(); + let mut out = String::with_capacity(len + len / 3); + for (i, ch) in bytes.iter().enumerate() { + if i > 0 && (len - i).is_multiple_of(3) { + out.push(','); + } + out.push(*ch as char); + } + out +} + +pub(crate) fn format_bytes(n: u64) -> String { + const UNITS: &[&str] = &["B", "KB", "MB", "GB", "TB"]; + if n < 1024 { + return format!("{n} B"); + } + let mut value = n as f64; + let mut i = 0; + while value >= 1024.0 && i + 1 < UNITS.len() { + value /= 1024.0; + i += 1; + } + format!("{value:.1} {}", UNITS[i]) +} + +pub(crate) fn blob_summary(conn: &Connection, blob_id: i64) -> Result { + let row = conn + .query_row( + "SELECT length(data), substr(lower(hex(hash)), 1, 12) FROM blobs WHERE id = ?1", + params![blob_id], + |r| { + let len: i64 = r.get(0)?; + let hex: String = r.get(1)?; + Ok((len, hex)) + }, + ) + .optional()?; + match row { + Some((len, hex)) => Ok(format!("{} blake3={hex}", format_bytes(len.max(0) as u64))), + None => Ok(format!("(missing blob #{blob_id})")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::init; + + fn mem() -> Connection { + let c = Connection::open_in_memory().unwrap(); + init::apply_schema(&c).unwrap(); + c + } + + #[test] + fn format_count_inserts_grouping_commas() { + assert_eq!(format_count(0), "0"); + assert_eq!(format_count(42), "42"); + assert_eq!(format_count(999), "999"); + assert_eq!(format_count(1_000), "1,000"); + assert_eq!(format_count(12_345), "12,345"); + assert_eq!(format_count(1_234_567_890), "1,234,567,890"); + } + + #[test] + fn format_bytes_picks_appropriate_unit() { + assert_eq!(format_bytes(0), "0 B"); + assert_eq!(format_bytes(512), "512 B"); + assert_eq!(format_bytes(1024), "1.0 KB"); + assert_eq!(format_bytes(1_536), "1.5 KB"); + assert_eq!(format_bytes(10 * 1024 * 1024), "10.0 MB"); + } + + #[test] + fn blob_summary_reports_size_and_short_hash() { + let c = mem(); + let id = crate::db::blobs::intern_blob(&c, b"hello world").unwrap(); + let s = blob_summary(&c, id).unwrap(); + assert!(s.starts_with("11 B blake3="), "got {s}"); + assert!(s.len() > "11 B blake3=".len()); + } + + #[test] + fn blob_summary_handles_missing_id() { + let c = mem(); + let s = blob_summary(&c, 9_999).unwrap(); + assert_eq!(s, "(missing blob #9999)"); + } + + #[test] + fn summary_lists_every_type_and_blob_totals() { + let c = mem(); + crate::db::sources::upsert_source( + &c, + &crate::db::sources::SourceKey { + kind: "jmap".to_owned(), + session_url: "https://example.test/jmap".to_owned(), + account_id: "u-1".to_owned(), + }, + Some("alice@example.test"), + "alice@example.test", + ) + .unwrap(); + c.execute("INSERT INTO mailboxes (name) VALUES (?1)", params!["Inbox"]) + .unwrap(); + crate::db::blobs::intern_blob(&c, b"some bytes").unwrap(); + let mut buf = Vec::new(); + write_summary(&c, Path::new("test.sqlite"), &mut buf).unwrap(); + let s = String::from_utf8(buf).unwrap(); + assert!(s.contains("Archive: test.sqlite")); + assert!(s.contains("Source: jmap https://example.test/jmap")); + assert!(s.contains("alice@example.test")); + for ty in ObjectType::ALL { + assert!(s.contains(ty.token()), "missing {} in:\n{s}", ty.token()); + } + assert!(s.contains("blobs")); + assert!(s.contains("10 B")); + } + + #[test] + fn summary_without_source_reports_none() { + let c = mem(); + let mut buf = Vec::new(); + write_summary(&c, Path::new("x.sqlite"), &mut buf).unwrap(); + let s = String::from_utf8(buf).unwrap(); + assert!(s.contains("(no source recorded)")); + } +} diff --git a/src/inspect/tree.rs b/src/inspect/tree.rs new file mode 100644 index 0000000..5eed42f --- /dev/null +++ b/src/inspect/tree.rs @@ -0,0 +1,327 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; +use std::io::Write; + +use rusqlite::Connection; + +use super::{blob_summary, format_count}; +use crate::error::Error; + +struct MailboxNode { + id: i64, + parent_id: Option, + name: String, + role: Option, + sort_order: i64, + msg_count: u64, +} + +pub fn write_mailboxes(conn: &Connection, out: &mut impl Write) -> Result<(), Error> { + let nodes = load_mailboxes(conn)?; + writeln!(out, "Mailboxes ({})", format_count(nodes.len() as u64))?; + if nodes.is_empty() { + writeln!(out, "(none)")?; + return Ok(()); + } + let by_id: HashMap = nodes.iter().map(|n| (n.id, n)).collect(); + let mut by_parent: HashMap, Vec> = HashMap::new(); + for n in &nodes { + by_parent.entry(n.parent_id).or_default().push(n.id); + } + for ids in by_parent.values_mut() { + ids.sort_by(|a, b| { + let na = by_id[a]; + let nb = by_id[b]; + na.sort_order + .cmp(&nb.sort_order) + .then_with(|| na.name.cmp(&nb.name)) + }); + } + let roots: Vec = by_parent.get(&None).cloned().unwrap_or_default(); + let last = roots.len().saturating_sub(1); + for (i, id) in roots.iter().enumerate() { + write_mailbox(out, &by_id, &by_parent, *id, "", i == last)?; + } + Ok(()) +} + +fn write_mailbox( + out: &mut impl Write, + by_id: &HashMap, + by_parent: &HashMap, Vec>, + id: i64, + prefix: &str, + is_last: bool, +) -> Result<(), Error> { + let node = by_id[&id]; + let connector = if is_last { "└── " } else { "├── " }; + let mut tags: Vec = Vec::new(); + if let Some(role) = &node.role { + tags.push(format!("role={role}")); + } + tags.push(format!("{} msgs", format_count(node.msg_count))); + writeln!( + out, + "{prefix}{connector}{} [{}]", + node.name, + tags.join(", ") + )?; + let child_prefix = format!("{prefix}{}", if is_last { " " } else { "│ " }); + if let Some(children) = by_parent.get(&Some(id)) { + let last = children.len() - 1; + for (i, child) in children.iter().enumerate() { + write_mailbox(out, by_id, by_parent, *child, &child_prefix, i == last)?; + } + } + Ok(()) +} + +fn load_mailboxes(conn: &Connection) -> Result, Error> { + let mut stmt = conn.prepare( + "SELECT m.id, m.parent_id, m.name, m.role, m.sort_order, + (SELECT COUNT(*) FROM emails e + WHERE EXISTS (SELECT 1 FROM json_each(e.mailbox_ids) j + WHERE CAST(j.value AS INTEGER) = m.id)) + FROM mailboxes m", + )?; + let rows = stmt.query_map([], |r| { + Ok(MailboxNode { + id: r.get(0)?, + parent_id: r.get(1)?, + name: r.get(2)?, + role: r.get(3)?, + sort_order: r.get(4)?, + msg_count: r.get::<_, i64>(5)? as u64, + }) + })?; + let mut out = Vec::new(); + for row in rows { + out.push(row?); + } + Ok(out) +} + +struct FileNodeRow { + id: i64, + parent_id: Option, + node_type: String, + blob_id: Option, + name: String, + media_type: Option, + target: Option, + child_count: u64, +} + +pub fn write_file_nodes(conn: &Connection, out: &mut impl Write) -> Result<(), Error> { + let nodes = load_file_nodes(conn)?; + writeln!(out, "File nodes ({})", format_count(nodes.len() as u64))?; + if nodes.is_empty() { + writeln!(out, "(none)")?; + return Ok(()); + } + let by_id: HashMap = nodes.iter().map(|n| (n.id, n)).collect(); + let mut by_parent: HashMap, Vec> = HashMap::new(); + for n in &nodes { + by_parent.entry(n.parent_id).or_default().push(n.id); + } + for ids in by_parent.values_mut() { + ids.sort_by(|a, b| { + let na = by_id[a]; + let nb = by_id[b]; + (na.node_type != "directory") + .cmp(&(nb.node_type != "directory")) + .then_with(|| na.name.cmp(&nb.name)) + }); + } + let roots: Vec = by_parent.get(&None).cloned().unwrap_or_default(); + let last = roots.len().saturating_sub(1); + for (i, id) in roots.iter().enumerate() { + write_file_node(conn, out, &by_id, &by_parent, *id, "", i == last)?; + } + Ok(()) +} + +fn write_file_node( + conn: &Connection, + out: &mut impl Write, + by_id: &HashMap, + by_parent: &HashMap, Vec>, + id: i64, + prefix: &str, + is_last: bool, +) -> Result<(), Error> { + let node = by_id[&id]; + let connector = if is_last { "└── " } else { "├── " }; + let display = match node.node_type.as_str() { + "directory" => format!("{}/", node.name), + _ => node.name.clone(), + }; + let detail = match node.node_type.as_str() { + "directory" => format!("({} children)", format_count(node.child_count)), + "symlink" => { + let target = node.target.as_deref().unwrap_or("[]"); + format!("symlink -> {target}") + } + _ => { + let media = node + .media_type + .as_deref() + .unwrap_or("application/octet-stream"); + match node.blob_id { + Some(bid) => format!("{media}, {}", blob_summary(conn, bid)?), + None => format!("{media}, (no blob)"), + } + } + }; + writeln!(out, "{prefix}{connector}{display} {detail}")?; + let child_prefix = format!("{prefix}{}", if is_last { " " } else { "│ " }); + if let Some(children) = by_parent.get(&Some(id)) { + let last = children.len() - 1; + for (i, child) in children.iter().enumerate() { + write_file_node( + conn, + out, + by_id, + by_parent, + *child, + &child_prefix, + i == last, + )?; + } + } + Ok(()) +} + +fn load_file_nodes(conn: &Connection) -> Result, Error> { + let mut stmt = conn.prepare( + "SELECT f.id, f.parent_id, f.node_type, f.blob_id, f.name, f.media_type, f.target, + (SELECT COUNT(*) FROM file_nodes c WHERE c.parent_id = f.id) + FROM file_nodes f", + )?; + let rows = stmt.query_map([], |r| { + Ok(FileNodeRow { + id: r.get(0)?, + parent_id: r.get(1)?, + node_type: r.get(2)?, + blob_id: r.get(3)?, + name: r.get(4)?, + media_type: r.get(5)?, + target: r.get(6)?, + child_count: r.get::<_, i64>(7)? as u64, + }) + })?; + let mut out = Vec::new(); + for row in rows { + out.push(row?); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::{blobs, init}; + use rusqlite::params; + + fn mem() -> Connection { + let c = Connection::open_in_memory().unwrap(); + init::apply_schema(&c).unwrap(); + c + } + + fn insert_mailbox( + c: &Connection, + name: &str, + parent: Option, + role: Option<&str>, + sort: i64, + ) -> i64 { + c.execute( + "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) + VALUES (?1, ?2, ?3, ?4, 1)", + params![name, parent, role, sort], + ) + .unwrap(); + c.last_insert_rowid() + } + + fn insert_email_in(c: &Connection, mailbox_id: i64) { + let blob = blobs::intern_blob(c, format!("seed-{mailbox_id}").as_bytes()).unwrap(); + c.execute( + "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords) + VALUES (?1, '2024-01-01T00:00:00Z', ?2, '[]')", + params![blob, format!("[{mailbox_id}]")], + ) + .unwrap(); + } + + #[test] + fn mailbox_tree_draws_box_characters_and_msg_counts() { + let c = mem(); + let inbox = insert_mailbox(&c, "Inbox", None, Some("inbox"), 0); + let _archived = insert_mailbox(&c, "Archived", Some(inbox), None, 0); + let _y2024 = insert_mailbox(&c, "2024", Some(inbox), None, 1); + let _sent = insert_mailbox(&c, "Sent", None, Some("sent"), 1); + insert_email_in(&c, inbox); + insert_email_in(&c, inbox); + let mut buf = Vec::new(); + write_mailboxes(&c, &mut buf).unwrap(); + let s = String::from_utf8(buf).unwrap(); + assert!(s.contains("Mailboxes (4)"), "header missing: {s}"); + assert!(s.contains("├── Inbox [role=inbox, 2 msgs]"), "got:\n{s}"); + assert!(s.contains("│ ├── Archived"), "got:\n{s}"); + assert!(s.contains("│ └── 2024"), "got:\n{s}"); + assert!(s.contains("└── Sent [role=sent, 0 msgs]"), "got:\n{s}"); + } + + #[test] + fn mailbox_tree_empty_says_none() { + let c = mem(); + let mut buf = Vec::new(); + write_mailboxes(&c, &mut buf).unwrap(); + let s = String::from_utf8(buf).unwrap(); + assert!(s.contains("Mailboxes (0)")); + assert!(s.contains("(none)")); + } + + #[test] + fn file_node_tree_distinguishes_dirs_files_symlinks() { + let c = mem(); + let docs: i64 = { + c.execute( + "INSERT INTO file_nodes (parent_id, node_type, name, created) + VALUES (NULL, 'directory', 'Documents', '2024-01-01T00:00:00Z')", + [], + ) + .unwrap(); + c.last_insert_rowid() + }; + let blob = blobs::intern_blob(&c, b"hello world").unwrap(); + c.execute( + "INSERT INTO file_nodes (parent_id, node_type, blob_id, name, media_type, created) + VALUES (?1, 'file', ?2, 'note.txt', 'text/plain', '2024-01-01T00:00:00Z')", + params![docs, blob], + ) + .unwrap(); + c.execute( + "INSERT INTO file_nodes (parent_id, node_type, target, name, created) + VALUES (?1, 'symlink', '[\"..\",\"etc\"]', 'shortcut', '2024-01-01T00:00:00Z')", + params![docs], + ) + .unwrap(); + let mut buf = Vec::new(); + write_file_nodes(&c, &mut buf).unwrap(); + let s = String::from_utf8(buf).unwrap(); + assert!(s.contains("File nodes (3)")); + assert!(s.contains("Documents/")); + assert!(s.contains("(2 children)")); + assert!(s.contains("note.txt")); + assert!(s.contains("text/plain, 11 B blake3=")); + assert!(s.contains("symlink -> [\"..\",\"etc\"]")); + } +} diff --git a/src/jmap/account.rs b/src/jmap/account.rs new file mode 100644 index 0000000..12e2cd8 --- /dev/null +++ b/src/jmap/account.rs @@ -0,0 +1,215 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::{Value, json}; + +use crate::error::Error; +use crate::jmap::http::HttpClient; +use crate::jmap::request::{Request, check_method_error}; +use crate::jmap::session::Session; + +#[derive(Debug, Clone)] +pub enum AccountSelector { + Id(String), + Name(String), +} + +const OWNER_URN: &str = "urn:ietf:params:jmap:principals:owner"; + +pub fn resolve( + selector: &AccountSelector, + session: &Session, + client: &HttpClient, +) -> Result { + match selector { + AccountSelector::Id(id) => Ok(id.clone()), + AccountSelector::Name(name) => { + for (id, account) in &session.accounts { + if account.name == *name { + return Ok(id.clone()); + } + } + resolve_via_principal(name, session, client) + } + } +} + +fn resolve_via_principal( + name: &str, + session: &Session, + client: &HttpClient, +) -> Result { + let mut req = Request::new(); + req.call( + "Principal/query", + json!({ "filter": { "name": name } }), + "q", + ); + req.call( + "Principal/get", + json!({ + "#ids": { "resultOf": "q", "name": "Principal/query", "path": "/ids" }, + "properties": ["id", "name", "accounts"] + }), + "g", + ); + let resp = req + .send(client, &session.api_url) + .map_err(|e| Error::Account(format!("principal resolution request failed: {e}")))?; + + let query = resp + .by_call_id("q") + .map_err(|e| Error::Account(e.to_string()))?; + check_method_error(query).map_err(|e| Error::Account(e.to_string()))?; + let get = resp + .by_call_id("g") + .map_err(|e| Error::Account(e.to_string()))?; + check_method_error(get).map_err(|e| Error::Account(e.to_string()))?; + + let list = get + .args + .get("list") + .and_then(Value::as_array) + .ok_or_else(|| Error::Account("Principal/get returned no list".to_owned()))?; + + let mut matches: Vec<&Value> = list + .iter() + .filter(|p| { + p.get("name") + .and_then(Value::as_str) + .map(|n| n == name) + .unwrap_or(false) + }) + .collect(); + + match matches.len() { + 0 => Err(Error::Account(format!( + "account name '{name}' not found via Principal/query" + ))), + 1 => extract_account_id(matches.remove(0), name), + _ => { + let ids: Vec = matches + .iter() + .filter_map(|p| p.get("id").and_then(Value::as_str).map(str::to_owned)) + .collect(); + Err(Error::Account(format!( + "account name '{name}' is ambiguous; candidate principals: {}", + ids.join(", ") + ))) + } + } +} + +fn extract_account_id(principal: &Value, name: &str) -> Result { + if let Some(accounts) = principal.get("accounts").and_then(Value::as_object) { + for data in accounts.values() { + if let Some(owner) = data.get(OWNER_URN) + && let Some(id) = owner.get("accountIdForPrincipal").and_then(Value::as_str) + { + return Ok(id.to_owned()); + } + } + for data in accounts.values() { + if let Some(cap) = data.as_object() { + for v in cap.values() { + if let Some(id) = v.get("accountId").and_then(Value::as_str) { + return Ok(id.to_owned()); + } + } + } + } + } + principal + .get("id") + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| { + Error::Account(format!( + "principal for '{name}' has no resolvable data account id" + )) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn session_with(name: &str, id: &str) -> Session { + let raw = serde_json::json!({ + "apiUrl": "https://example.org/api", + "uploadUrl": "https://example.org/upload", + "downloadUrl": "https://example.org/download", + "capabilities": {}, + "accounts": { id: { "name": name } } + }); + serde_json::from_value(raw).unwrap() + } + + fn client() -> HttpClient { + HttpClient::new( + crate::jmap::http::Auth::Bearer { token: "t".into() }, + crate::jmap::http::RetryPolicy::new(0), + false, + ) + } + + #[test] + fn id_selector_is_verbatim() { + let s = session_with("alice", "a"); + assert_eq!( + resolve(&AccountSelector::Id("zzz".into()), &s, &client()).unwrap(), + "zzz" + ); + } + + #[test] + fn name_selector_exact_case_sensitive_match() { + let s = session_with("alice@example.org", "w"); + assert_eq!( + resolve( + &AccountSelector::Name("alice@example.org".into()), + &s, + &client() + ) + .unwrap(), + "w" + ); + } + + #[test] + fn substring_near_match_is_rejected_exactly() { + let list = json!([ + { "id": "p1", "name": "alice2", "accounts": {} }, + { "id": "p2", "name": "alice", "accounts": { + "w": { "urn:ietf:params:jmap:principals:owner": + { "accountIdForPrincipal": "w" } } } } + ]); + let matches: Vec<&Value> = list + .as_array() + .unwrap() + .iter() + .filter(|p| p.get("name").and_then(Value::as_str) == Some("alice")) + .collect(); + assert_eq!(matches.len(), 1); + assert_eq!(extract_account_id(matches[0], "alice").unwrap(), "w"); + } + + #[test] + fn extract_falls_back_to_principal_id() { + let p = json!({ "id": "acc-7", "name": "bob", "accounts": {} }); + assert_eq!(extract_account_id(&p, "bob").unwrap(), "acc-7"); + } + + #[test] + fn extract_uses_per_capability_account_id() { + let p = json!({ + "id": "px", + "name": "bob", + "accounts": { "z": { "urn:ietf:params:jmap:mail": { "accountId": "data-9" } } } + }); + assert_eq!(extract_account_id(&p, "bob").unwrap(), "data-9"); + } +} diff --git a/src/jmap/blob.rs b/src/jmap/blob.rs new file mode 100644 index 0000000..19714cd --- /dev/null +++ b/src/jmap/blob.rs @@ -0,0 +1,93 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::{Map, Value}; + +pub const SENTINEL_KEY: &str = "@blob"; + +pub fn import_blob_ids(value: &mut Value, mut resolve: F) -> Result<(), BlobWalkError> +where + F: FnMut(&str) -> Result, +{ + walk_import(value, &mut resolve) +} + +fn walk_import(value: &mut Value, resolve: &mut F) -> Result<(), BlobWalkError> +where + F: FnMut(&str) -> Result, +{ + match value { + Value::Object(map) => { + + if let Some(Value::String(blob_id)) = map.remove("blobId") { + let local = resolve(&blob_id)?; + map.insert(SENTINEL_KEY.to_owned(), Value::from(local)); + } + for child in map.values_mut() { + walk_import(child, resolve)?; + } + } + Value::Array(items) => { + for item in items { + walk_import(item, resolve)?; + } + } + _ => {} + } + Ok(()) +} + +pub fn export_blob_ids(value: &mut Value, mut resolve: F) -> Result<(), BlobWalkError> +where + F: FnMut(i64) -> Result, +{ + walk_export(value, &mut resolve) +} + +fn walk_export(value: &mut Value, resolve: &mut F) -> Result<(), BlobWalkError> +where + F: FnMut(i64) -> Result, +{ + match value { + Value::Object(map) => { + if let Some(sentinel) = map.remove(SENTINEL_KEY) { + let local_id = sentinel.as_i64().ok_or(BlobWalkError::MalformedSentinel)?; + let target_jmap_id = resolve(local_id)?; + map.insert("blobId".to_owned(), Value::String(target_jmap_id)); + } + for child in map.values_mut() { + walk_export(child, resolve)?; + } + } + Value::Array(items) => { + for item in items { + walk_export(item, resolve)?; + } + } + _ => {} + } + Ok(()) +} + +pub fn prepend_property(target: &mut Map, key: &str, value: Value) { + let existing: Vec<(String, Value)> = + target.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + target.clear(); + target.insert(key.to_owned(), value); + for (k, v) in existing { + if k != key { + target.insert(k, v); + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum BlobWalkError { + #[error("malformed @blob sentinel: expected integer local id")] + MalformedSentinel, + #[error("blob resolver failed: {0}")] + Resolver(String), +} diff --git a/src/jmap/blobxfer.rs b/src/jmap/blobxfer.rs new file mode 100644 index 0000000..353c532 --- /dev/null +++ b/src/jmap/blobxfer.rs @@ -0,0 +1,40 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::Value; + +use crate::jmap::error::JmapError; +use crate::jmap::http::HttpClient; +use crate::jmap::session::Session; +use crate::jmap::wire::JmapId; + +pub fn upload_bytes( + client: &HttpClient, + session: &Session, + account_id: &str, + content_type: &str, + bytes: &[u8], +) -> Result { + let url = session.upload_url_for(account_id); + let response = client.upload(&url, content_type, bytes)?; + response + .get("blobId") + .and_then(Value::as_str) + .map(|s| JmapId(s.to_owned())) + .ok_or_else(|| JmapError::malformed("upload response has no blobId")) +} + +pub fn download_bytes( + client: &HttpClient, + session: &Session, + account_id: &str, + blob_id: &str, + type_hint: &str, + name: &str, +) -> Result, JmapError> { + let url = session.download_url_for(account_id, blob_id, type_hint, name); + client.download(&url) +} diff --git a/src/jmap/connect.rs b/src/jmap/connect.rs new file mode 100644 index 0000000..ab10fb9 --- /dev/null +++ b/src/jmap/connect.rs @@ -0,0 +1,66 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::error::Error; +use crate::jmap::account; +use crate::jmap::session::{Limits, Session}; +use crate::sync::{ConnectConfig, Context}; +use crate::types::ObjectType; + +pub struct Connected { + pub session: Session, + pub limits: Limits, + pub account_id: String, + supported: Vec, +} + +impl Connected { + pub fn supports(&self, ty: ObjectType) -> bool { + self.supported.contains(&ty) + } + + pub fn supported_types(&self) -> &[ObjectType] { + &self.supported + } +} + +pub fn prepare(ctx: &Context, connect: &ConnectConfig) -> Result { + ctx.client.set_logger(ctx.common.logger); + + let session = Session::discover(&ctx.client, &connect.url)?; + let limits = session.core_limits()?; + ctx.client.set_limits(&limits); + let account_id = account::resolve(&connect.account, &session, &ctx.client)?; + + if session.account(&account_id).is_none() + && !matches!(connect.account, account::AccountSelector::Name(_)) + { + ctx.common.logger.warn(&format!( + "account id {account_id} is not enumerated in this session; capabilities cannot be gated and every type will be attempted" + )); + } + + let mut supported = Vec::new(); + for ty in ObjectType::ALL { + let urn = ty.capability_urn(); + let known = session.account(&account_id).is_some(); + if !known || session.supports(&account_id, urn) { + supported.push(ty); + } else { + ctx.common.logger.warn(&format!( + "target account does not advertise {urn}; skipping type {}", + ty.jmap_name() + )); + } + } + + Ok(Connected { + session, + limits, + account_id, + supported, + }) +} diff --git a/src/jmap/error.rs b/src/jmap/error.rs new file mode 100644 index 0000000..f014853 --- /dev/null +++ b/src/jmap/error.rs @@ -0,0 +1,70 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::error::Error; +use crate::jmap::blob::BlobWalkError; + +#[derive(Debug, thiserror::Error)] +pub enum JmapError { + #[error("transport failure: {0}")] + Transport(String), + + #[error("connection failure: {0}")] + Connect(String), + + #[error("authentication rejected: {0}")] + Auth(String), + + #[error("http status {status}: {body}")] + HttpStatus { status: u16, body: String }, + + #[error("retries exhausted: {0}")] + RetriesExhausted(String), + + #[error("request too large")] + RequestTooLarge, + + #[error("single object exceeds the server size limit and cannot be split: {0}")] + SingleObjectTooLarge(String), + + #[error("query anchor not found")] + AnchorNotFound, + + #[error("jmap method error in call {call_id}: {error_type}{}", .description.as_deref().map(|d| format!(" ({d})")).unwrap_or_default())] + Method { + call_id: String, + error_type: String, + description: Option, + }, + + #[error("malformed jmap response: {0}")] + Malformed(String), + + #[error("json error: {0}")] + Json(#[from] serde_json::Error), + + #[error("sqlite error: {0}")] + Sqlite(#[from] rusqlite::Error), + + #[error("blob walk error: {0}")] + Blob(#[from] BlobWalkError), +} + +impl JmapError { + pub fn malformed(context: impl Into) -> JmapError { + JmapError::Malformed(context.into()) + } +} + +impl From for Error { + fn from(value: JmapError) -> Self { + match value { + JmapError::Connect(m) | JmapError::Transport(m) => Error::Connection(m), + JmapError::Auth(m) => Error::Connection(format!("authentication rejected: {m}")), + other => Error::Connection(other.to_string()), + } + } +} diff --git a/src/jmap/http.rs b/src/jmap/http.rs new file mode 100644 index 0000000..1f9555d --- /dev/null +++ b/src/jmap/http.rs @@ -0,0 +1,766 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::sync::Arc; +use std::sync::OnceLock; +use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +use std::time::{Duration, SystemTime}; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use serde_json::Value; +use ureq::Agent; +use ureq::config::{Config, RedirectAuthHeaders}; +use ureq::tls::{RootCerts, TlsConfig}; + +use crate::jmap::error::JmapError; +use crate::jmap::inflight::{Permit, Semaphore}; +use crate::jmap::retry::{self, Disposition, RateLimitState}; +use crate::jmap::session::Limits; +use crate::logging::{LEVEL_BODIES, LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; + +const MAX_BODY: u64 = 512 * 1024 * 1024; + +const LONG_RETRY_THRESHOLD: Duration = Duration::from_secs(10); + +#[derive(Debug, Clone)] +pub enum Auth { + Basic { user: String, password: String }, + Bearer { token: String }, +} + +impl Auth { + pub fn header_value(&self) -> String { + match self { + Auth::Basic { user, password } => { + format!("Basic {}", STANDARD.encode(format!("{user}:{password}"))) + } + Auth::Bearer { token } => format!("Bearer {token}"), + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct RetryPolicy { + pub max_retries: u32, + pub base: Duration, + pub cap: Duration, +} + +impl RetryPolicy { + pub fn new(max_retries: u32) -> Self { + RetryPolicy { + max_retries, + base: Duration::from_millis(500), + cap: Duration::from_secs(30), + } + } +} + +struct Inner { + agent: Agent, + auth: Auth, + retry: RetryPolicy, + allow_invalid_certs: bool, + rate_limit: RateLimitState, + log_level: AtomicU8, + requests_gate: OnceLock, + uploads_gate: OnceLock, + max_upload_bytes: AtomicU64, + retries_total: AtomicU64, + retry_after_sleeps: AtomicU64, +} + +#[derive(Debug, Clone, Copy)] +enum Kind { + Api, + Upload, +} + +#[derive(Clone)] +pub struct HttpClient { + inner: Arc, +} + +enum Attempt { + Ok { + status: u16, + body: Vec, + retry_after: Option, + rate_limit_headers: Vec<(String, String)>, + }, + Transport(JmapError), +} + +impl HttpClient { + pub fn new(auth: Auth, retry: RetryPolicy, allow_invalid_certs: bool) -> Self { + let config: Config = Config::builder() + .http_status_as_error(false) + .redirect_auth_headers(RedirectAuthHeaders::SameHost) + .tls_config( + TlsConfig::builder() + .root_certs(RootCerts::PlatformVerifier) + .disable_verification(allow_invalid_certs) + .build(), + ) + .build(); + HttpClient { + inner: Arc::new(Inner { + agent: config.new_agent(), + auth, + retry, + allow_invalid_certs, + rate_limit: RateLimitState::new(), + log_level: AtomicU8::new(LEVEL_DEFAULT), + requests_gate: OnceLock::new(), + uploads_gate: OnceLock::new(), + max_upload_bytes: AtomicU64::new(0), + retries_total: AtomicU64::new(0), + retry_after_sleeps: AtomicU64::new(0), + }), + } + } + + pub fn set_limits(&self, limits: &Limits) { + let _ = self + .inner + .requests_gate + .set(Semaphore::new(limits.max_concurrent_requests)); + let _ = self + .inner + .uploads_gate + .set(Semaphore::new(limits.max_concurrent_upload)); + self.inner + .max_upload_bytes + .store(limits.max_size_upload, Ordering::Relaxed); + } + + pub fn retries_observed(&self) -> u64 { + self.inner.retries_total.load(Ordering::Relaxed) + } + + pub fn retry_after_sleeps(&self) -> u64 { + self.inner.retry_after_sleeps.load(Ordering::Relaxed) + } + + fn acquire_requests(&self) -> Option> { + self.inner.requests_gate.get().map(Semaphore::acquire) + } + + fn acquire_uploads(&self) -> Option> { + self.inner.uploads_gate.get().map(Semaphore::acquire) + } + + pub fn auth(&self) -> &Auth { + &self.inner.auth + } + + pub fn retry(&self) -> &RetryPolicy { + &self.inner.retry + } + + pub fn allow_invalid_certs(&self) -> bool { + self.inner.allow_invalid_certs + } + + pub fn rate_limit(&self) -> &RateLimitState { + &self.inner.rate_limit + } + + pub fn throttle_level(&self) -> u32 { + self.inner.rate_limit.level() + } + + pub fn set_logger(&self, logger: Logger) { + self.inner + .log_level + .store(logger.level(), Ordering::Relaxed); + } + + fn logger(&self) -> Logger { + Logger::new(self.inner.log_level.load(Ordering::Relaxed)) + } + + pub fn get(&self, url: &str) -> Result { + let body = self.execute(Kind::Api, "GET", url, None, None)?; + String::from_utf8(body).map_err(|e| JmapError::Malformed(format!("non-utf8 body: {e}"))) + } + + pub fn post_json(&self, url: &str, body: &Value) -> Result { + let payload = serde_json::to_vec(body)?; + let raw = self.execute( + Kind::Api, + "POST", + url, + Some(&payload), + Some("application/json"), + )?; + serde_json::from_slice(&raw) + .map_err(|e| JmapError::Malformed(format!("response is not valid json: {e}"))) + } + + pub fn upload( + &self, + upload_url: &str, + content_type: &str, + bytes: &[u8], + ) -> Result { + let cap = self.inner.max_upload_bytes.load(Ordering::Relaxed); + if cap > 0 && (bytes.len() as u64) > cap { + return Err(JmapError::SingleObjectTooLarge(format!( + "blob upload of {} bytes exceeds maxSizeUpload ({cap})", + bytes.len() + ))); + } + let raw = self.execute( + Kind::Upload, + "POST", + upload_url, + Some(bytes), + Some(content_type), + )?; + serde_json::from_slice(&raw) + .map_err(|e| JmapError::Malformed(format!("upload response is not valid json: {e}"))) + } + + pub fn download(&self, url: &str) -> Result, JmapError> { + self.execute(Kind::Api, "GET", url, None, None) + } + + fn execute( + &self, + kind: Kind, + method: &str, + url: &str, + body: Option<&[u8]>, + content_type: Option<&str>, + ) -> Result, JmapError> { + let policy = self.inner.retry; + let logger = self.logger(); + let mut attempt: u32 = 0; + loop { + self.inner.rate_limit.cooldown().wait(); + let attempt_outcome = { + let _r = self.acquire_requests(); + let _u = if matches!(kind, Kind::Upload) { + self.acquire_uploads() + } else { + None + }; + self.one_attempt(url, body, content_type) + }; + match attempt_outcome { + Attempt::Ok { status, body, .. } if (200..300).contains(&status) => { + self.inner.rate_limit.on_success(); + return Ok(body); + } + Attempt::Ok { + status, + body, + retry_after, + rate_limit_headers, + } => { + let disposition = self.status_disposition(status, &body); + match disposition { + StatusOutcome::Auth => { + return Err(JmapError::Auth(format!( + "server returned {status}: {}", + truncate(&body) + ))); + } + StatusOutcome::RequestTooLarge => { + return Err(JmapError::RequestTooLarge); + } + StatusOutcome::Fatal => { + return Err(JmapError::HttpStatus { + status, + body: truncate(&body), + }); + } + StatusOutcome::Retryable => { + attempt += 1; + self.inner.retries_total.fetch_add(1, Ordering::Relaxed); + let detail = problem_detail(&body); + let headers_blurb = format_rate_headers(&rate_limit_headers); + let reason = match (&detail, headers_blurb.as_deref()) { + (Some(d), Some(h)) => format!("http {status}: {d} [{h}]"), + (Some(d), None) => format!("http {status}: {d}"), + (None, Some(h)) => format!("http {status} [{h}]"), + (None, None) => format!("http {status}"), + }; + if attempt > policy.max_retries { + return Err(JmapError::RetriesExhausted(format!( + "{method} {url} kept returning {reason}" + ))); + } + if retry_after.is_some() { + self.inner + .retry_after_sleeps + .fetch_add(1, Ordering::Relaxed); + } + let delay = self.inner.rate_limit.on_throttle(&policy, retry_after); + if delay >= LONG_RETRY_THRESHOLD { + logger.warn(&format!( + "server rate-limited ({reason}); waiting {} before retry {}/{} (shared level {})", + format_retry_wait(delay), + attempt, + policy.max_retries, + self.inner.rate_limit.level(), + )); + } + self.log_retry( + &logger, + RetryLog { + method, + url, + attempt, + delay, + reason: &reason, + body: &body, + }, + ); + std::thread::sleep(delay); + } + } + } + Attempt::Transport(err) => match transport_disposition(&err) { + Disposition::Fatal => return Err(err), + Disposition::Retryable => { + attempt += 1; + self.inner.retries_total.fetch_add(1, Ordering::Relaxed); + if attempt > policy.max_retries { + return Err(JmapError::RetriesExhausted(format!( + "{method} {url}: {err}" + ))); + } + let delay = retry::backoff_delay(&policy, attempt); + if delay >= LONG_RETRY_THRESHOLD { + logger.warn(&format!( + "transient transport failure ({err}); waiting {} before retry {}/{}", + format_retry_wait(delay), + attempt, + policy.max_retries, + )); + } + self.log_retry( + &logger, + RetryLog { + method, + url, + attempt, + delay, + reason: &err.to_string(), + body: &[], + }, + ); + std::thread::sleep(delay); + } + }, + } + } + } + + fn one_attempt(&self, url: &str, body: Option<&[u8]>, content_type: Option<&str>) -> Attempt { + let auth = self.inner.auth.header_value(); + let result = if let Some(payload) = body { + let mut req = self + .inner + .agent + .post(url) + .header("Authorization", auth) + .header("Accept", "application/json"); + if let Some(ct) = content_type { + req = req.header("Content-Type", ct); + } + req.send(payload) + } else { + self.inner + .agent + .get(url) + .header("Authorization", auth) + .header("Accept", "application/json") + .call() + }; + match result { + Ok(mut resp) => { + let status = resp.status().as_u16(); + let retry_after = resp + .headers() + .get("retry-after") + .and_then(|v| v.to_str().ok()) + .and_then(retry_after_header); + let rate_limit_headers: Vec<(String, String)> = resp + .headers() + .iter() + .filter_map(|(name, value)| { + let lower = name.as_str().to_ascii_lowercase(); + if matches!( + lower.as_str(), + "retry-after" | "ratelimit" | "ratelimit-policy" | "content-type" + ) { + value.to_str().ok().map(|v| (lower, v.to_owned())) + } else { + None + } + }) + .collect(); + match resp.body_mut().with_config().limit(MAX_BODY).read_to_vec() { + Ok(bytes) => Attempt::Ok { + status, + body: bytes, + retry_after, + rate_limit_headers, + }, + Err(e) => Attempt::Transport(JmapError::Transport(format!( + "reading response body: {e}" + ))), + } + } + Err(e) => Attempt::Transport(map_ureq_error(e)), + } + } + + fn status_disposition(&self, status: u16, body: &[u8]) -> StatusOutcome { + if status == 401 || status == 403 { + return StatusOutcome::Auth; + } + if status == 413 { + return StatusOutcome::RequestTooLarge; + } + if let Ok(problem) = serde_json::from_slice::(body) + && let Some(kind) = problem.get("type").and_then(Value::as_str) + { + if kind.ends_with(":requestTooLarge") { + return StatusOutcome::RequestTooLarge; + } + if kind.ends_with(":limit") { + let which = problem.get("limit").and_then(Value::as_str).unwrap_or(""); + let subtype = problem.get("subType").and_then(Value::as_str).unwrap_or(""); + match which { + "maxSizeRequest" | "maxSizeUpload" => { + return StatusOutcome::RequestTooLarge; + } + "maxConcurrentRequests" | "maxConcurrentUpload" => { + return StatusOutcome::Retryable; + } + _ if subtype == "rateLimit" => { + return StatusOutcome::Retryable; + } + _ => {} + } + } + } + match retry::classify_http_status(status) { + Disposition::Retryable => StatusOutcome::Retryable, + Disposition::Fatal => StatusOutcome::Fatal, + } + } + + fn log_retry(&self, logger: &Logger, r: RetryLog<'_>) { + if logger.enabled(LEVEL_BODIES) { + let snippet = if r.body.is_empty() { + String::new() + } else { + format!(" body={}", truncate(r.body)) + }; + eprintln!( + "retry {}/{} {} {} after {:?} ({}){snippet}", + r.attempt, self.inner.retry.max_retries, r.method, r.url, r.delay, r.reason, + ); + } else if logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "retry {}/{} ({})", + r.attempt, self.inner.retry.max_retries, r.reason + ); + } + } +} + +enum StatusOutcome { + Retryable, + Fatal, + Auth, + RequestTooLarge, +} + +struct RetryLog<'a> { + method: &'a str, + url: &'a str, + attempt: u32, + delay: Duration, + reason: &'a str, + body: &'a [u8], +} + +fn transport_disposition(err: &JmapError) -> Disposition { + match err { + JmapError::Connect(_) => Disposition::Fatal, + _ => Disposition::Retryable, + } +} + +fn map_ureq_error(err: ureq::Error) -> JmapError { + match err { + ureq::Error::Io(e) => JmapError::Transport(format!("io: {e}")), + ureq::Error::Timeout(t) => JmapError::Transport(format!("timeout: {t}")), + ureq::Error::HostNotFound => JmapError::Transport("host not found".to_owned()), + ureq::Error::ConnectionFailed => JmapError::Transport("connection failed".to_owned()), + ureq::Error::BodyStalled => JmapError::Transport("body stalled".to_owned()), + ureq::Error::Tls(m) => JmapError::Transport(format!("tls: {m}")), + ureq::Error::TooManyRedirects => JmapError::Connect("too many redirects".to_owned()), + ureq::Error::RedirectFailed => JmapError::Connect("redirect failed".to_owned()), + ureq::Error::TlsRequired => { + JmapError::Connect("server requires TLS but transport is unsecured".to_owned()) + } + other => JmapError::Connect(other.to_string()), + } +} + +fn truncate(body: &[u8]) -> String { + let text = String::from_utf8_lossy(body); + if text.len() <= 512 { + return text.into_owned(); + } + let end = text.floor_char_boundary(512); + format!("{}...", &text[..end]) +} + +pub fn retry_after_header(value: &str) -> Option { + retry::parse_retry_after(value, SystemTime::now()) +} + +fn problem_detail(body: &[u8]) -> Option { + let value: Value = serde_json::from_slice(body).ok()?; + let title = value.get("title").and_then(Value::as_str); + let detail = value.get("detail").and_then(Value::as_str); + match (title, detail) { + (Some(t), Some(d)) => Some(format!("{t}: {d}")), + (Some(t), None) => Some(t.to_owned()), + (None, Some(d)) => Some(d.to_owned()), + (None, None) => None, + } +} + +fn format_rate_headers(headers: &[(String, String)]) -> Option { + if headers.is_empty() { + return None; + } + let mut parts: Vec = headers + .iter() + .filter(|(name, _)| name != "content-type") + .map(|(name, value)| format!("{name}={value}")) + .collect(); + if parts.is_empty() { + return None; + } + parts.sort(); + Some(parts.join("; ")) +} + +fn format_retry_wait(d: Duration) -> String { + let total = d.as_secs(); + if total < 60 { + format!("{total}s") + } else if total < 3600 { + format!("{}m{:02}s", total / 60, total % 60) + } else { + format!( + "{}h{:02}m{:02}s", + total / 3600, + (total % 3600) / 60, + total % 60 + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn basic_header_matches_rfc7617_example() { + let auth = Auth::Basic { + user: "Aladdin".to_owned(), + password: "open sesame".to_owned(), + }; + assert_eq!(auth.header_value(), "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="); + } + + #[test] + fn bearer_header() { + let auth = Auth::Bearer { + token: "abc.def".to_owned(), + }; + assert_eq!(auth.header_value(), "Bearer abc.def"); + } + + #[test] + fn truncate_does_not_panic_on_multibyte_boundary() { + let mut body = vec![b'a'; 510]; + body.extend_from_slice("\u{1F4A9}".as_bytes()); + body.extend_from_slice(&[b'b'; 600]); + let s = truncate(&body); + assert!(s.ends_with("...")); + assert!(s.is_char_boundary(s.len())); + let inner = s.trim_end_matches('.'); + assert!(inner.is_char_boundary(inner.len())); + } + + #[test] + fn truncate_short_body_returns_verbatim() { + let s = truncate("héllo".as_bytes()); + assert_eq!(s, "héllo"); + } + + #[test] + fn limit_problem_unknown_kind_on_429_is_retried() { + let client = HttpClient::new( + Auth::Bearer { + token: "t".to_owned(), + }, + RetryPolicy::new(0), + false, + ); + let body = br#"{"type":"urn:ietf:params:jmap:error:limit","limit":"someServerLimit"}"#; + assert!(matches!( + client.status_disposition(429, body), + StatusOutcome::Retryable + )); + } + + #[test] + fn limit_problem_max_size_request_is_resplit() { + let client = HttpClient::new( + Auth::Bearer { + token: "t".to_owned(), + }, + RetryPolicy::new(0), + false, + ); + let body = br#"{"type":"urn:ietf:params:jmap:error:limit","limit":"maxSizeRequest"}"#; + assert!(matches!( + client.status_disposition(400, body), + StatusOutcome::RequestTooLarge + )); + } + + #[test] + fn limit_problem_max_concurrent_is_retried() { + let client = HttpClient::new( + Auth::Bearer { + token: "t".to_owned(), + }, + RetryPolicy::new(0), + false, + ); + let body = + br#"{"type":"urn:ietf:params:jmap:error:limit","limit":"maxConcurrentRequests"}"#; + assert!(matches!( + client.status_disposition(429, body), + StatusOutcome::Retryable + )); + } + + fn limits_with(max_upload: u64, req: u64, up: u64) -> Limits { + Limits { + max_objects_in_get: 500, + max_objects_in_set: 500, + max_calls_in_request: 16, + max_concurrent_requests: req, + max_concurrent_upload: up, + max_size_request: 10_000_000, + max_size_upload: max_upload, + } + } + + #[test] + fn upload_rejects_oversize_before_sending() { + let client = HttpClient::new( + Auth::Bearer { + token: "t".to_owned(), + }, + RetryPolicy::new(0), + false, + ); + client.set_limits(&limits_with(10, 4, 4)); + let err = client + .upload( + "http://127.0.0.1:1/upload", + "application/octet-stream", + &[0u8; 11], + ) + .unwrap_err(); + assert!( + matches!(err, JmapError::SingleObjectTooLarge(_)), + "got {err:?}" + ); + assert_eq!(client.retries_observed(), 0); + } + + #[test] + fn upload_within_max_size_is_not_short_circuited() { + let client = HttpClient::new( + Auth::Bearer { + token: "t".to_owned(), + }, + RetryPolicy::new(0), + false, + ); + client.set_limits(&limits_with(1024, 4, 4)); + let err = client + .upload( + "http://127.0.0.1:1/upload", + "application/octet-stream", + b"abc", + ) + .unwrap_err(); + assert!( + !matches!(err, JmapError::SingleObjectTooLarge(_)), + "expected the call to proceed to transport (got SingleObjectTooLarge): {err:?}" + ); + } + + #[test] + fn problem_detail_extracts_title_and_detail() { + let body = br#"{"type":"about:blank","status":429,"title":"Quota exceeded","detail":"You have exceeded the blob upload quota of 1000 files or 50000000 bytes."}"#; + let got = problem_detail(body).expect("title+detail"); + assert!(got.contains("Quota exceeded")); + assert!(got.contains("50000000 bytes")); + } + + #[test] + fn problem_detail_title_only() { + let body = br#"{"title":"Too Many Requests"}"#; + assert_eq!(problem_detail(body).as_deref(), Some("Too Many Requests")); + } + + #[test] + fn problem_detail_returns_none_for_empty_body() { + assert_eq!(problem_detail(b"{}"), None); + assert_eq!(problem_detail(b"not json"), None); + } + + #[test] + fn format_retry_wait_short_medium_long() { + assert_eq!(format_retry_wait(Duration::from_secs(7)), "7s"); + assert_eq!(format_retry_wait(Duration::from_secs(67)), "1m07s"); + assert_eq!(format_retry_wait(Duration::from_secs(1427)), "23m47s"); + assert_eq!(format_retry_wait(Duration::from_secs(3661)), "1h01m01s"); + } + + #[test] + fn counters_start_at_zero() { + let client = HttpClient::new( + Auth::Bearer { + token: "t".to_owned(), + }, + RetryPolicy::new(0), + false, + ); + assert_eq!(client.retries_observed(), 0); + assert_eq!(client.retry_after_sleeps(), 0); + } +} diff --git a/src/jmap/inflight.rs b/src/jmap/inflight.rs new file mode 100644 index 0000000..67fb001 --- /dev/null +++ b/src/jmap/inflight.rs @@ -0,0 +1,122 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::sync::{Condvar, Mutex}; + +pub struct Semaphore { + state: Mutex, + cond: Condvar, +} + +impl Semaphore { + pub fn new(capacity: u64) -> Semaphore { + Semaphore { + state: Mutex::new(capacity.max(1)), + cond: Condvar::new(), + } + } + + pub fn acquire(&self) -> Permit<'_> { + let mut guard = lock(&self.state); + loop { + if *guard > 0 { + *guard -= 1; + return Permit { sem: self }; + } + guard = wait(&self.cond, guard); + } + } + + fn release(&self) { + let mut guard = lock(&self.state); + *guard = guard.saturating_add(1); + self.cond.notify_one(); + } +} + +pub struct Permit<'a> { + sem: &'a Semaphore, +} + +impl Drop for Permit<'_> { + fn drop(&mut self) { + self.sem.release(); + } +} + +fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, u64> { + match m.lock() { + Ok(g) => g, + Err(p) => p.into_inner(), + } +} + +fn wait<'a>(cv: &Condvar, g: std::sync::MutexGuard<'a, u64>) -> std::sync::MutexGuard<'a, u64> { + match cv.wait(g) { + Ok(g) => g, + Err(p) => p.into_inner(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::thread; + use std::time::{Duration, Instant}; + + #[test] + fn permit_decrements_and_drop_releases() { + let s = Semaphore::new(1); + { + let _p = s.acquire(); + let s2 = &s; + let started = Instant::now(); + thread::scope(|sc| { + let h = sc.spawn(move || { + let _q = s2.acquire(); + started.elapsed() + }); + thread::sleep(Duration::from_millis(80)); + drop(_p); + let waited = h.join().unwrap(); + assert!( + waited >= Duration::from_millis(60), + "second acquirer should have waited until first released, waited {waited:?}" + ); + }); + } + } + + #[test] + fn capacity_bounds_concurrent_acquirers() { + let s = Arc::new(Semaphore::new(2)); + let in_flight = Arc::new(AtomicU64::new(0)); + let max = Arc::new(AtomicU64::new(0)); + let mut handles = Vec::new(); + for _ in 0..8 { + let s = s.clone(); + let inf = in_flight.clone(); + let mx = max.clone(); + handles.push(thread::spawn(move || { + let _p = s.acquire(); + let cur = inf.fetch_add(1, Ordering::SeqCst) + 1; + mx.fetch_max(cur, Ordering::SeqCst); + thread::sleep(Duration::from_millis(30)); + inf.fetch_sub(1, Ordering::SeqCst); + })); + } + for h in handles { + h.join().unwrap(); + } + assert_eq!( + max.load(Ordering::SeqCst), + 2, + "max in-flight must equal capacity" + ); + } +} diff --git a/src/jmap/mod.rs b/src/jmap/mod.rs new file mode 100644 index 0000000..c99cd74 --- /dev/null +++ b/src/jmap/mod.rs @@ -0,0 +1,17 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod account; +pub mod blob; +pub mod blobxfer; +pub mod connect; +pub mod error; +pub mod http; +pub mod inflight; +pub mod request; +pub mod retry; +pub mod session; +pub mod wire; diff --git a/src/jmap/request.rs b/src/jmap/request.rs new file mode 100644 index 0000000..4a63b99 --- /dev/null +++ b/src/jmap/request.rs @@ -0,0 +1,685 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use indexmap::IndexSet; +use serde::de::DeserializeOwned; +use serde_json::{Map, Value, json}; + +use crate::jmap::error::JmapError; +use crate::jmap::http::HttpClient; +use crate::jmap::session::Limits; +use crate::jmap::wire::JmapId; + +pub const URN_CORE: &str = "urn:ietf:params:jmap:core"; + +pub fn using_urn(method: &str) -> &'static str { + let prefix = method.split('/').next().unwrap_or(method); + match prefix { + "Mailbox" | "Email" | "Identity" => "urn:ietf:params:jmap:mail", + "SieveScript" => "urn:ietf:params:jmap:sieve", + "AddressBook" | "ContactCard" => "urn:ietf:params:jmap:contacts", + "Calendar" | "CalendarEvent" | "ParticipantIdentity" => "urn:ietf:params:jmap:calendars", + "FileNode" => "urn:ietf:params:jmap:filenode", + "Principal" => "urn:ietf:params:jmap:principals", + "x:Account" | "x:Domain" => "urn:stalwart:jmap", + _ => URN_CORE, + } +} + +#[derive(Debug, Clone)] +pub struct MethodCall { + pub name: String, + pub args: Value, + pub call_id: String, +} + +#[derive(Debug, Clone, Default)] +pub struct Request { + calls: Vec, +} + +impl Request { + pub fn new() -> Request { + Request { calls: Vec::new() } + } + + pub fn call( + &mut self, + name: impl Into, + args: Value, + call_id: impl Into, + ) -> &mut Request { + self.calls.push(MethodCall { + name: name.into(), + args, + call_id: call_id.into(), + }); + self + } + + pub fn len(&self) -> usize { + self.calls.len() + } + + pub fn is_empty(&self) -> bool { + self.calls.is_empty() + } + + pub fn using(&self) -> Vec { + let mut set: IndexSet = IndexSet::new(); + set.insert(URN_CORE.to_owned()); + for c in &self.calls { + set.insert(using_urn(&c.name).to_owned()); + } + set.into_iter().collect() + } + + fn envelope(&self) -> Value { + let method_calls: Vec = self + .calls + .iter() + .map(|c| json!([c.name, c.args, c.call_id])) + .collect(); + json!({ "using": self.using(), "methodCalls": method_calls }) + } + + pub fn fits(&self, limits: &Limits) -> Result<(), JmapError> { + if self.calls.len() as u64 > limits.max_calls_in_request { + return Err(JmapError::RequestTooLarge); + } + let size = serde_json::to_vec(&self.envelope())?.len() as u64; + if size > limits.max_size_request { + return Err(JmapError::RequestTooLarge); + } + Ok(()) + } + + pub fn send(&self, client: &HttpClient, api_url: &str) -> Result { + let value = client.post_json(api_url, &self.envelope())?; + Response::parse(value) + } +} + +#[derive(Debug)] +pub struct Response { + pub method_responses: Vec, +} + +impl Response { + fn parse(value: Value) -> Result { + let arr = value + .get("methodResponses") + .and_then(Value::as_array) + .ok_or_else(|| JmapError::malformed("response has no methodResponses array"))?; + let mut out = Vec::with_capacity(arr.len()); + for entry in arr { + let triple = entry.as_array().filter(|t| t.len() == 3).ok_or_else(|| { + JmapError::malformed("methodResponse is not a [name,args,id] triple") + })?; + let name = triple[0] + .as_str() + .ok_or_else(|| JmapError::malformed("methodResponse name is not a string"))? + .to_owned(); + let call_id = triple[2] + .as_str() + .ok_or_else(|| JmapError::malformed("methodResponse callId is not a string"))? + .to_owned(); + out.push(MethodCall { + name, + args: triple[1].clone(), + call_id, + }); + } + Ok(Response { + method_responses: out, + }) + } + + pub fn first(&self) -> Result<&MethodCall, JmapError> { + self.method_responses + .first() + .ok_or_else(|| JmapError::malformed("empty methodResponses")) + } + + pub fn by_call_id(&self, call_id: &str) -> Result<&MethodCall, JmapError> { + self.method_responses + .iter() + .find(|m| m.call_id == call_id) + .ok_or_else(|| JmapError::malformed(format!("no response for callId {call_id}"))) + } +} + +pub fn check_method_error(mr: &MethodCall) -> Result<(), JmapError> { + if mr.name != "error" { + return Ok(()); + } + let error_type = mr + .args + .get("type") + .and_then(Value::as_str) + .unwrap_or("unknown") + .to_owned(); + if error_type == "anchorNotFound" { + return Err(JmapError::AnchorNotFound); + } + let description = mr + .args + .get("description") + .and_then(Value::as_str) + .map(str::to_owned); + Err(JmapError::Method { + call_id: mr.call_id.clone(), + error_type, + description, + }) +} + +fn ids_array(ids: &[JmapId]) -> Value { + Value::Array(ids.iter().map(|i| Value::String(i.0.clone())).collect()) +} + +pub fn query_all_ids( + client: &HttpClient, + api_url: &str, + account_id: &str, + type_name: &str, + limits: &Limits, +) -> Result, JmapError> { + let mut restarts = 0u32; + loop { + match query_pages(client, api_url, account_id, type_name, limits) { + Ok(ids) => return Ok(ids), + Err(JmapError::AnchorNotFound) if restarts < 2 => { + restarts += 1; + } + Err(e) => return Err(e), + } + } +} + +fn query_pages( + client: &HttpClient, + api_url: &str, + account_id: &str, + type_name: &str, + limits: &Limits, +) -> Result, JmapError> { + let limit = limits.max_objects_in_get.max(1); + let mut collected: Vec = Vec::new(); + let mut page_size: Option = None; + let mut anchor: Option = None; + + loop { + let mut args = Map::new(); + args.insert("accountId".to_owned(), Value::String(account_id.to_owned())); + args.insert("limit".to_owned(), Value::from(limit)); + if let Some(a) = &anchor { + args.insert("anchor".to_owned(), Value::String(a.clone())); + args.insert("anchorOffset".to_owned(), Value::from(1)); + } + let mut req = Request::new(); + req.call(format!("{type_name}/query"), Value::Object(args), "q"); + let resp = req.send(client, api_url)?; + let mr = resp.first()?; + check_method_error(mr)?; + let ids = mr + .args + .get("ids") + .and_then(Value::as_array) + .ok_or_else(|| JmapError::malformed("query response has no ids array"))?; + let this_len = ids.len(); + for v in ids { + let s = v + .as_str() + .ok_or_else(|| JmapError::malformed("query id is not a string"))?; + collected.push(JmapId(s.to_owned())); + } + match page_size { + None => { + if this_len == 0 { + break; + } + page_size = Some(this_len); + } + Some(full) => { + if this_len == 0 || this_len < full { + break; + } + } + } + let last = collected + .last() + .map(|i| i.0.clone()) + .ok_or_else(|| JmapError::malformed("non-empty page yielded no anchor"))?; + anchor = Some(last); + } + Ok(collected) +} + +#[derive(Debug)] +pub struct GetResult { + pub list: Vec, + pub not_found: Vec, +} + +impl Default for GetResult { + fn default() -> Self { + GetResult { + list: Vec::new(), + not_found: Vec::new(), + } + } +} + +struct GetCtx<'a> { + client: &'a HttpClient, + api_url: &'a str, + account_id: &'a str, + type_name: &'a str, + properties: Option<&'a [&'a str]>, + limits: &'a Limits, +} + +pub fn get_objects( + client: &HttpClient, + api_url: &str, + account_id: &str, + type_name: &str, + ids: &[JmapId], + properties: Option<&[&str]>, + limits: &Limits, +) -> Result, JmapError> { + let ctx = GetCtx { + client, + api_url, + account_id, + type_name, + properties, + limits, + }; + let mut out = GetResult::default(); + let chunk = limits.max_objects_in_get.max(1) as usize; + let mut start = 0; + while start < ids.len() { + let end = (start + chunk).min(ids.len()); + get_chunk(&ctx, &ids[start..end], &mut out)?; + start = end; + } + Ok(out) +} + +fn get_chunk( + ctx: &GetCtx<'_>, + ids: &[JmapId], + out: &mut GetResult, +) -> Result<(), JmapError> { + let mut args = Map::new(); + args.insert( + "accountId".to_owned(), + Value::String(ctx.account_id.to_owned()), + ); + args.insert("ids".to_owned(), ids_array(ids)); + if let Some(props) = ctx.properties { + let mut p: Vec = props + .iter() + .map(|s| Value::String((*s).to_owned())) + .collect(); + p.push(Value::String("id".to_owned())); + args.insert("properties".to_owned(), Value::Array(p)); + } + let mut req = Request::new(); + req.call(format!("{}/get", ctx.type_name), Value::Object(args), "g"); + let send_result: Result<(), JmapError> = if req.fits(ctx.limits).is_err() { + Err(JmapError::RequestTooLarge) + } else { + match req.send(ctx.client, ctx.api_url) { + Ok(resp) => { + let mr = resp.first()?; + check_method_error(mr)?; + decode_get(mr, out) + } + Err(e) => Err(e), + } + }; + match send_result { + Ok(()) => Ok(()), + Err(JmapError::RequestTooLarge) => { + if ids.len() <= 1 { + return Err(JmapError::SingleObjectTooLarge(format!( + "{}/get of a single id exceeds maxSizeRequest", + ctx.type_name + ))); + } + let mid = ids.len() / 2; + get_chunk(ctx, &ids[..mid], out)?; + get_chunk(ctx, &ids[mid..], out) + } + Err(e) => Err(e), + } +} + +pub fn get_all( + client: &HttpClient, + api_url: &str, + account_id: &str, + type_name: &str, +) -> Result, JmapError> { + let mut args = Map::new(); + args.insert("accountId".to_owned(), Value::String(account_id.to_owned())); + args.insert("ids".to_owned(), Value::Null); + let mut req = Request::new(); + req.call(format!("{type_name}/get"), Value::Object(args), "g"); + let resp = req.send(client, api_url)?; + let mr = resp.first()?; + check_method_error(mr)?; + let mut out = GetResult::default(); + decode_get(mr, &mut out)?; + Ok(out) +} + +fn decode_get( + mr: &MethodCall, + out: &mut GetResult, +) -> Result<(), JmapError> { + let list = mr + .args + .get("list") + .and_then(Value::as_array) + .ok_or_else(|| JmapError::malformed("get response has no list array"))?; + for item in list { + out.list.push(serde_json::from_value(item.clone())?); + } + if let Some(nf) = mr.args.get("notFound").and_then(Value::as_array) { + for v in nf { + if let Some(s) = v.as_str() { + out.not_found.push(JmapId(s.to_owned())); + } + } + } + Ok(()) +} + +#[derive(Debug, Default)] +pub struct SetOutcome { + pub created: Vec<(String, Value)>, + pub updated: Vec, + pub destroyed: Vec, + pub not_created: Vec<(String, Value)>, + pub not_updated: Vec<(String, Value)>, + pub not_destroyed: Vec<(String, Value)>, +} + +impl SetOutcome { + fn absorb(&mut self, mut other: SetOutcome) { + self.created.append(&mut other.created); + self.updated.append(&mut other.updated); + self.destroyed.append(&mut other.destroyed); + self.not_created.append(&mut other.not_created); + self.not_updated.append(&mut other.not_updated); + self.not_destroyed.append(&mut other.not_destroyed); + } +} + +#[derive(Debug, Default)] +pub struct SetRequest<'a> { + pub create: Option, + pub update: Option, + pub destroy: Option, + pub extra_args: &'a [(&'a str, Value)], +} + +#[derive(Debug, Clone, Copy)] +enum Section { + Create, + Update, + Destroy, +} + +fn collect_items(value: Option<&Value>, section: Section) -> Vec<(Section, String, Value)> { + match (section, value) { + (Section::Destroy, Some(Value::Array(ids))) => ids + .iter() + .filter_map(|v| v.as_str().map(|s| (section, s.to_owned(), Value::Null))) + .collect(), + (_, Some(Value::Object(m))) => m + .iter() + .map(|(k, v)| (section, k.clone(), v.clone())) + .collect(), + _ => Vec::new(), + } +} + +pub fn set_call( + client: &HttpClient, + api_url: &str, + account_id: &str, + type_name: &str, + body: SetRequest<'_>, + limits: &Limits, +) -> Result { + let mut items = collect_items(body.create.as_ref(), Section::Create); + items.extend(collect_items(body.update.as_ref(), Section::Update)); + items.extend(collect_items(body.destroy.as_ref(), Section::Destroy)); + + if items.is_empty() { + return set_send( + client, + api_url, + account_id, + type_name, + &[], + body.extra_args, + limits, + ); + } + + let chunk = limits.max_objects_in_set.max(1) as usize; + let mut outcome = SetOutcome::default(); + let mut start = 0; + while start < items.len() { + let end = (start + chunk).min(items.len()); + let part = set_send( + client, + api_url, + account_id, + type_name, + &items[start..end], + body.extra_args, + limits, + )?; + outcome.absorb(part); + start = end; + } + Ok(outcome) +} + +fn set_send( + client: &HttpClient, + api_url: &str, + account_id: &str, + type_name: &str, + items: &[(Section, String, Value)], + extra_args: &[(&str, Value)], + limits: &Limits, +) -> Result { + let mut args = Map::new(); + args.insert("accountId".to_owned(), Value::String(account_id.to_owned())); + let mut create = Map::new(); + let mut update = Map::new(); + let mut destroy: Vec = Vec::new(); + for (section, key, val) in items { + match section { + Section::Create => { + create.insert(key.clone(), val.clone()); + } + Section::Update => { + update.insert(key.clone(), val.clone()); + } + Section::Destroy => destroy.push(Value::String(key.clone())), + } + } + if !create.is_empty() { + args.insert("create".to_owned(), Value::Object(create)); + } + if !update.is_empty() { + args.insert("update".to_owned(), Value::Object(update)); + } + if !destroy.is_empty() { + args.insert("destroy".to_owned(), Value::Array(destroy)); + } + for (k, v) in extra_args { + args.insert((*k).to_owned(), v.clone()); + } + let mut req = Request::new(); + req.call(format!("{type_name}/set"), Value::Object(args), "s"); + + let oversize = req.fits(limits).is_err(); + let send_result = if oversize { + Err(JmapError::RequestTooLarge) + } else { + req.send(client, api_url).and_then(|resp| { + let mr = resp.first()?; + check_method_error(mr)?; + Ok(decode_set(mr)) + }) + }; + + match send_result { + Ok(outcome) => Ok(outcome), + Err(JmapError::RequestTooLarge) => { + if items.len() <= 1 { + return Err(JmapError::SingleObjectTooLarge(format!( + "{type_name}/set of a single object exceeds maxSizeRequest" + ))); + } + let mid = items.len() / 2; + let mut left = set_send( + client, + api_url, + account_id, + type_name, + &items[..mid], + extra_args, + limits, + )?; + let right = set_send( + client, + api_url, + account_id, + type_name, + &items[mid..], + extra_args, + limits, + )?; + left.absorb(right); + Ok(left) + } + Err(e) => Err(e), + } +} + +fn pairs(args: &Value, key: &str) -> Vec<(String, Value)> { + args.get(key) + .and_then(Value::as_object) + .map(|m| m.iter().map(|(k, v)| (k.clone(), v.clone())).collect()) + .unwrap_or_default() +} + +fn keys(args: &Value, key: &str) -> Vec { + args.get(key) + .and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(|v| v.as_str().map(str::to_owned)) + .collect() + }) + .unwrap_or_default() +} + +fn decode_set(mr: &MethodCall) -> SetOutcome { + SetOutcome { + created: pairs(&mr.args, "created"), + updated: mr + .args + .get("updated") + .and_then(Value::as_object) + .map(|m| m.keys().cloned().collect()) + .unwrap_or_default(), + destroyed: keys(&mr.args, "destroyed"), + not_created: pairs(&mr.args, "notCreated"), + not_updated: pairs(&mr.args, "notUpdated"), + not_destroyed: pairs(&mr.args, "notDestroyed"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn using_union_includes_core_and_dedups() { + let mut r = Request::new(); + r.call("Mailbox/get", json!({}), "a"); + r.call("Email/query", json!({}), "b"); + let u = r.using(); + assert!(u.contains(&URN_CORE.to_owned())); + assert!(u.contains(&"urn:ietf:params:jmap:mail".to_owned())); + assert_eq!(u.iter().filter(|x| x.as_str() == URN_CORE).count(), 1); + } + + #[test] + fn using_maps_principal_and_stalwart() { + assert_eq!( + using_urn("Principal/query"), + "urn:ietf:params:jmap:principals" + ); + assert_eq!(using_urn("x:Account/set"), "urn:stalwart:jmap"); + assert_eq!( + using_urn("CalendarEvent/set"), + "urn:ietf:params:jmap:calendars" + ); + } + + #[test] + fn fits_rejects_too_many_calls() { + let limits = Limits { + max_objects_in_get: 10, + max_objects_in_set: 10, + max_calls_in_request: 2, + max_concurrent_requests: 4, + max_concurrent_upload: 4, + max_size_request: 10_000_000, + max_size_upload: 10_000_000, + }; + let mut r = Request::new(); + r.call("Mailbox/get", json!({}), "a"); + r.call("Mailbox/get", json!({}), "b"); + r.call("Mailbox/get", json!({}), "c"); + assert!(matches!(r.fits(&limits), Err(JmapError::RequestTooLarge))); + } + + #[test] + fn parse_response_and_method_error() { + let v = json!({ + "methodResponses": [ + ["Mailbox/get", {"list": [], "notFound": []}, "g"], + ["error", {"type": "invalidArguments"}, "x"] + ] + }); + let r = Response::parse(v).unwrap(); + assert!(check_method_error(r.by_call_id("g").unwrap()).is_ok()); + let err = check_method_error(r.by_call_id("x").unwrap()).unwrap_err(); + assert!(matches!(err, JmapError::Method { .. })); + } + + #[test] + fn anchor_not_found_is_classified() { + let v = json!({ "methodResponses": [["error", {"type": "anchorNotFound"}, "q"]] }); + let r = Response::parse(v).unwrap(); + let err = check_method_error(r.first().unwrap()).unwrap_err(); + assert!(matches!(err, JmapError::AnchorNotFound)); + } +} diff --git a/src/jmap/retry.rs b/src/jmap/retry.rs new file mode 100644 index 0000000..0ab5487 --- /dev/null +++ b/src/jmap/retry.rs @@ -0,0 +1,312 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::cell::Cell; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use crate::jmap::http::RetryPolicy; + +pub const THROTTLE_BASE: Duration = Duration::from_secs(1); +const THROTTLE_LEVEL_CAP: u32 = 16; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Disposition { + Retryable, + Fatal, +} + +pub fn classify_http_status(status: u16) -> Disposition { + match status { + 429 | 502 | 503 | 504 => Disposition::Retryable, + _ => Disposition::Fatal, + } +} + +pub fn jmap_method_disposition(error_type: &str) -> Disposition { + match error_type { + "serverUnavailable" | "serverPartialFail" => Disposition::Retryable, + _ => Disposition::Fatal, + } +} + +pub fn parse_retry_after(value: &str, now: SystemTime) -> Option { + let trimmed = value.trim(); + if let Ok(secs) = trimmed.parse::() { + return Some(Duration::from_secs(secs)); + } + let imf_fixdate = time::macros::format_description!( + "[weekday repr:short], [day] [month repr:short] [year] \ + [hour]:[minute]:[second] GMT" + ); + let parsed = time::PrimitiveDateTime::parse(trimmed, imf_fixdate).ok()?; + let target: SystemTime = parsed.assume_utc().into(); + target.duration_since(now).ok() +} + +thread_local! { + static RNG_STATE: Cell = Cell::new(seed()); +} + +fn seed() -> u64 { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos() as u64) + .unwrap_or(0x9E37_79B9_7F4A_7C15); + let tid = { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + std::hash::Hash::hash(&std::thread::current().id(), &mut hasher); + std::hash::Hasher::finish(&hasher) + }; + (nanos ^ tid).max(1) +} + +fn next_u64() -> u64 { + RNG_STATE.with(|cell| { + let mut x = cell.get(); + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + cell.set(x); + x + }) +} + +fn jitter_uniform(span: Duration) -> Duration { + let nanos = span.as_nanos(); + if nanos == 0 { + return Duration::ZERO; + } + let pick = (next_u64() as u128) % (nanos + 1); + Duration::from_nanos(pick.min(u64::MAX as u128) as u64) +} + +pub fn backoff_delay(policy: &RetryPolicy, attempt: u32) -> Duration { + let shift = attempt.saturating_sub(1).min(32); + let scaled = policy + .base + .saturating_mul(1u32.checked_shl(shift).unwrap_or(u32::MAX)); + let capped = scaled.min(policy.cap); + jitter_uniform(capped) +} + +#[derive(Debug, Default)] +pub struct Cooldown { + until: Mutex>, +} + +impl Cooldown { + pub fn new() -> Cooldown { + Cooldown { + until: Mutex::new(None), + } + } + + pub fn arm(&self, delay: Duration) { + let target = Instant::now() + delay; + if let Ok(mut guard) = self.until.lock() { + let extend = guard.map(|cur| target > cur).unwrap_or(true); + if extend { + *guard = Some(target); + } + } + } + + pub fn remaining(&self) -> Option { + let guard = self.until.lock().ok()?; + let until = (*guard)?; + let now = Instant::now(); + if until > now { Some(until - now) } else { None } + } + + pub fn wait(&self) { + while let Some(remaining) = self.remaining() { + std::thread::sleep(remaining); + } + } +} + +#[derive(Debug, Default)] +pub struct RateLimitState { + cooldown: Cooldown, + level: AtomicU32, +} + +impl RateLimitState { + pub fn new() -> RateLimitState { + RateLimitState { + cooldown: Cooldown::new(), + level: AtomicU32::new(0), + } + } + + pub fn cooldown(&self) -> &Cooldown { + &self.cooldown + } + + pub fn level(&self) -> u32 { + self.level.load(Ordering::Relaxed) + } + + pub fn on_success(&self) { + self.level.store(0, Ordering::Relaxed); + } + + pub fn on_throttle(&self, policy: &RetryPolicy, retry_after: Option) -> Duration { + let n = self + .level + .fetch_add(1, Ordering::Relaxed) + .saturating_add(1) + .min(THROTTLE_LEVEL_CAP); + let delay = retry_after.unwrap_or_else(|| throttle_backoff(policy, n)); + self.cooldown.arm(delay); + delay + } +} + +pub fn throttle_backoff(policy: &RetryPolicy, level: u32) -> Duration { + let shift = level.saturating_sub(1).min(20); + let scaled = THROTTLE_BASE + .saturating_mul(1u32.checked_shl(shift).unwrap_or(u32::MAX)) + .min(policy.cap); + let half = scaled / 2; + half + jitter_uniform(scaled - half) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn policy() -> RetryPolicy { + RetryPolicy::new(5) + } + + #[test] + fn backoff_is_bounded_by_cap_with_full_jitter() { + let p = policy(); + for attempt in 1..=12 { + let d = backoff_delay(&p, attempt); + assert!(d <= p.cap, "attempt {attempt} delay {d:?} exceeds cap"); + } + } + + #[test] + fn backoff_grows_until_cap() { + let p = policy(); + let big = backoff_delay(&p, 30); + assert!(big <= p.cap); + } + + #[test] + fn retry_after_seconds() { + assert_eq!( + parse_retry_after("120", SystemTime::now()), + Some(Duration::from_secs(120)) + ); + } + + #[test] + fn retry_after_http_date_in_future() { + let now = SystemTime::UNIX_EPOCH + Duration::from_secs(784_111_777); + let d = parse_retry_after("Sun, 06 Nov 1994 08:49:47 GMT", now); + assert_eq!(d, Some(Duration::from_secs(10))); + } + + #[test] + fn retry_after_garbage_is_none() { + assert_eq!(parse_retry_after("soon", SystemTime::now()), None); + } + + #[test] + fn status_classification() { + assert_eq!(classify_http_status(429), Disposition::Retryable); + assert_eq!(classify_http_status(503), Disposition::Retryable); + assert_eq!(classify_http_status(400), Disposition::Fatal); + assert_eq!(classify_http_status(401), Disposition::Fatal); + } + + #[test] + fn method_classification() { + assert_eq!( + jmap_method_disposition("serverUnavailable"), + Disposition::Retryable + ); + assert_eq!( + jmap_method_disposition("invalidArguments"), + Disposition::Fatal + ); + } + + #[test] + fn cooldown_arms_and_clears() { + let c = Cooldown::new(); + assert!(c.remaining().is_none()); + c.arm(Duration::from_millis(40)); + assert!(c.remaining().is_some()); + std::thread::sleep(Duration::from_millis(60)); + assert!(c.remaining().is_none()); + } + + #[test] + fn throttle_backoff_doubles_then_clamps_to_cap() { + let p = RetryPolicy::new(8); + let mut prev = throttle_backoff(&p, 1); + for level in 2..=4 { + let d = throttle_backoff(&p, level); + assert!( + d >= prev / 2, + "level {level} ({d:?}) collapsed below half of prev ({prev:?})" + ); + prev = d; + } + assert!(throttle_backoff(&p, 20) <= p.cap); + } + + #[test] + fn rate_limit_state_escalates_across_concurrent_callers() { + let p = RetryPolicy::new(8); + let s = RateLimitState::new(); + let first = s.on_throttle(&p, None); + let second = s.on_throttle(&p, None); + let third = s.on_throttle(&p, None); + assert!( + third >= first, + "level 3 delay ({third:?}) should not be smaller than level 1 ({first:?})" + ); + assert!( + second >= first, + "level 2 delay ({second:?}) should not be smaller than level 1 ({first:?})" + ); + assert!(s.level() >= 3); + } + + #[test] + fn rate_limit_state_resets_on_success() { + let p = RetryPolicy::new(8); + let s = RateLimitState::new(); + let _ = s.on_throttle(&p, None); + let _ = s.on_throttle(&p, None); + let _ = s.on_throttle(&p, None); + assert!(s.level() >= 3); + s.on_success(); + assert_eq!(s.level(), 0); + let next = s.on_throttle(&p, None); + assert!( + next <= THROTTLE_BASE, + "first throttle after reset should be small ({next:?})" + ); + } + + #[test] + fn rate_limit_state_honours_retry_after_over_schedule() { + let p = RetryPolicy::new(8); + let s = RateLimitState::new(); + let d = s.on_throttle(&p, Some(Duration::from_millis(750))); + assert_eq!(d, Duration::from_millis(750)); + assert_eq!(s.level(), 1); + } +} diff --git a/src/jmap/session.rs b/src/jmap/session.rs new file mode 100644 index 0000000..063ab5a --- /dev/null +++ b/src/jmap/session.rs @@ -0,0 +1,250 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use indexmap::IndexMap; +use serde::Deserialize; +use serde_json::Value; + +use crate::error::Error; +use crate::jmap::error::JmapError; +use crate::jmap::http::HttpClient; + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Session { + pub api_url: String, + pub upload_url: String, + pub download_url: String, + pub capabilities: IndexMap, + pub accounts: IndexMap, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Account { + pub name: String, + #[serde(default)] + pub account_capabilities: IndexMap, +} + +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Limits { + pub max_objects_in_get: u64, + pub max_objects_in_set: u64, + pub max_calls_in_request: u64, + pub max_concurrent_requests: u64, + pub max_concurrent_upload: u64, + pub max_size_request: u64, + pub max_size_upload: u64, +} + +fn parse_session(body: &str) -> Option { + let value: Value = serde_json::from_str(body).ok()?; + let has_shape = value.get("apiUrl").is_some() + && value.get("accounts").is_some() + && value.get("capabilities").is_some(); + if !has_shape { + return None; + } + serde_json::from_value(value).ok() +} + +fn well_known_url(base: &str) -> String { + let trimmed = base.trim_end_matches('/'); + format!("{trimmed}/.well-known/jmap") +} + +impl Session { + pub fn discover(client: &HttpClient, url: &str) -> Result { + match client.get(url) { + Ok(direct) => { + if let Some(session) = parse_session(&direct) { + return session.ensure_authenticated(); + } + } + Err(JmapError::Auth(m)) => return Err(JmapError::Auth(m)), + Err(_) => {} + } + let well_known = well_known_url(url); + match client.get(&well_known) { + Ok(body) => { + if let Some(session) = parse_session(&body) { + return session.ensure_authenticated(); + } + Err(JmapError::Connect(format!( + "no JMAP Session object at {url} or {well_known}" + ))) + } + Err(JmapError::Auth(m)) => Err(JmapError::Auth(m)), + Err(e) => Err(JmapError::Connect(format!( + "session discovery failed: no Session at {url}; {well_known}: {e}" + ))), + } + } + + fn ensure_authenticated(self) -> Result { + if self.accounts.is_empty() { + return Err(JmapError::Auth( + "session enumerates no accounts (anonymous session: authentication failed)" + .to_owned(), + )); + } + Ok(self) + } + + pub fn core_limits(&self) -> Result { + let core = self + .capabilities + .get("urn:ietf:params:jmap:core") + .ok_or_else(|| { + Error::Connection("session has no urn:ietf:params:jmap:core capability".to_owned()) + })?; + serde_json::from_value(core.clone()) + .map_err(|e| Error::Connection(format!("session core capability is malformed: {e}"))) + } + + pub fn account(&self, account_id: &str) -> Option<&Account> { + self.accounts.get(account_id) + } + + pub fn account_capabilities(&self, account_id: &str) -> Option<&IndexMap> { + self.accounts + .get(account_id) + .map(|a| &a.account_capabilities) + } + + pub fn supports(&self, account_id: &str, capability_urn: &str) -> bool { + self.account_capabilities(account_id) + .map(|caps| caps.contains_key(capability_urn)) + .unwrap_or(false) + } + + pub fn upload_url_for(&self, account_id: &str) -> String { + self.upload_url + .replace("{accountId}", &encode_segment(account_id)) + } + + pub fn download_url_for( + &self, + account_id: &str, + blob_id: &str, + type_hint: &str, + name: &str, + ) -> String { + self.download_url + .replace("{accountId}", &encode_segment(account_id)) + .replace("{blobId}", &encode_segment(blob_id)) + .replace("{type}", &encode_segment(type_hint)) + .replace("{name}", &encode_segment(name)) + } +} + +fn encode_segment(input: &str) -> String { + const HEX: &[u8; 16] = b"0123456789ABCDEF"; + let mut out = String::with_capacity(input.len()); + for byte in input.as_bytes() { + let b = *byte; + let unreserved = b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.' | b'~'); + if unreserved { + out.push(b as char); + } else { + out.push('%'); + out.push(HEX[(b >> 4) as usize] as char); + out.push(HEX[(b & 0x0F) as usize] as char); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn raw_session() -> &'static str { + r#"{ + "apiUrl": "https://example.org/jmap/api", + "uploadUrl": "https://example.org/jmap/upload/{accountId}/", + "downloadUrl": "https://example.org/jmap/download/{accountId}/{blobId}/{type}/{name}", + "capabilities": { + "urn:ietf:params:jmap:core": { + "maxObjectsInGet": 500, + "maxObjectsInSet": 500, + "maxCallsInRequest": 16, + "maxConcurrentRequests": 4, + "maxConcurrentUpload": 4, + "maxSizeRequest": 10000000, + "maxSizeUpload": 50000000 + } + }, + "accounts": { + "w": { + "name": "vspec-user@example.org", + "accountCapabilities": { "urn:ietf:params:jmap:mail": {} } + } + } + }"# + } + + #[test] + fn parses_a_session_and_reads_limits() { + let session = parse_session(raw_session()).unwrap(); + let limits = session.core_limits().unwrap(); + assert_eq!(limits.max_objects_in_get, 500); + assert_eq!(limits.max_size_upload, 50000000); + assert_eq!(session.accounts["w"].name, "vspec-user@example.org"); + } + + #[test] + fn capability_gate_reads_account_capabilities() { + let session = parse_session(raw_session()).unwrap(); + assert!(session.supports("w", "urn:ietf:params:jmap:mail")); + assert!(!session.supports("w", "urn:ietf:params:jmap:filenode")); + assert!(!session.supports("missing", "urn:ietf:params:jmap:mail")); + } + + #[test] + fn templates_upload_and_download_urls_with_encoding() { + let session = parse_session(raw_session()).unwrap(); + assert_eq!( + session.upload_url_for("a c"), + "https://example.org/jmap/upload/a%20c/" + ); + assert_eq!( + session.download_url_for("w", "G1/2", "application/sieve", "my script"), + "https://example.org/jmap/download/w/G1%2F2/application%2Fsieve/my%20script" + ); + } + + #[test] + fn rejects_body_that_is_not_a_session() { + assert!(parse_session("{\"hello\":true}").is_none()); + assert!(parse_session("not json").is_none()); + } + + #[test] + fn anonymous_session_is_auth_failure() { + let raw = + r#"{"apiUrl":"x","uploadUrl":"u","downloadUrl":"d","capabilities":{},"accounts":{}}"#; + let session = parse_session(raw).unwrap(); + assert!(matches!( + session.ensure_authenticated(), + Err(JmapError::Auth(_)) + )); + } + + #[test] + fn well_known_appends_correctly() { + assert_eq!( + well_known_url("https://h.example/"), + "https://h.example/.well-known/jmap" + ); + assert_eq!( + well_known_url("https://h.example"), + "https://h.example/.well-known/jmap" + ); + } +} diff --git a/src/jmap/wire/address_book.rs b/src/jmap/wire/address_book.rs new file mode 100644 index 0000000..2e4e9cb --- /dev/null +++ b/src/jmap/wire/address_book.rs @@ -0,0 +1,34 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde::{Deserialize, Serialize}; + +use super::JmapId; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AddressBook { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + + pub name: String, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + + #[serde(default)] + pub sort_order: u32, + + #[serde(default)] + pub is_default: bool, + + #[serde(default = "default_true")] + pub is_subscribed: bool, +} + +fn default_true() -> bool { + true +} diff --git a/src/jmap/wire/calendar.rs b/src/jmap/wire/calendar.rs new file mode 100644 index 0000000..326ff44 --- /dev/null +++ b/src/jmap/wire/calendar.rs @@ -0,0 +1,59 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use super::JmapId; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Calendar { + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + + pub name: String, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub color: Option, + + #[serde(default)] + pub sort_order: u32, + + #[serde(default = "default_true")] + pub is_subscribed: bool, + + #[serde(default = "default_true")] + pub is_visible: bool, + + #[serde(default)] + pub is_default: bool, + + #[serde(default = "default_include_in_availability")] + pub include_in_availability: String, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default_alerts_with_time: Option>, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default_alerts_without_time: Option>, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub time_zone: Option, +} + +fn default_true() -> bool { + true +} + +fn default_include_in_availability() -> String { + "all".to_owned() +} diff --git a/src/jmap/wire/calendar_event.rs b/src/jmap/wire/calendar_event.rs new file mode 100644 index 0000000..44aa062 --- /dev/null +++ b/src/jmap/wire/calendar_event.rs @@ -0,0 +1,30 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use super::JmapId; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CalendarEvent { + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + + pub calendar_ids: IndexMap, + + #[serde(default)] + pub is_draft: bool, + + #[serde(default)] + pub use_default_alerts: bool, + + #[serde(flatten)] + pub rest: IndexMap, +} diff --git a/src/jmap/wire/common.rs b/src/jmap/wire/common.rs new file mode 100644 index 0000000..c8ab03a --- /dev/null +++ b/src/jmap/wire/common.rs @@ -0,0 +1,47 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct JmapId(pub String); + +impl From for JmapId { + fn from(value: String) -> Self { + JmapId(value) + } +} + +impl From<&str> for JmapId { + fn from(value: &str) -> Self { + JmapId(value.to_owned()) + } +} + +impl std::fmt::Display for JmapId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +pub type UtcDate = time::OffsetDateTime; + +pub type Date = time::OffsetDateTime; + +pub fn bool_or_true<'de, D>(d: D) -> Result +where + D: serde::Deserializer<'de>, +{ + Ok(Option::::deserialize(d)?.unwrap_or(true)) +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EmailAddress { + pub name: Option, + pub email: String, +} diff --git a/src/jmap/wire/contact_card.rs b/src/jmap/wire/contact_card.rs new file mode 100644 index 0000000..4eb72cc --- /dev/null +++ b/src/jmap/wire/contact_card.rs @@ -0,0 +1,24 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use super::JmapId; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ContactCard { + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + + pub address_book_ids: IndexMap, + + #[serde(flatten)] + pub rest: IndexMap, +} diff --git a/src/jmap/wire/email.rs b/src/jmap/wire/email.rs new file mode 100644 index 0000000..bf3df50 --- /dev/null +++ b/src/jmap/wire/email.rs @@ -0,0 +1,46 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; + +use super::{JmapId, UtcDate}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Email { + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + + pub blob_id: JmapId, + + #[serde(with = "time::serde::rfc3339")] + pub received_at: UtcDate, + + pub mailbox_ids: IndexMap, + + pub keywords: IndexMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EmailImport { + + pub blob_id: JmapId, + + pub mailbox_ids: IndexMap, + + #[serde(default, skip_serializing_if = "IndexMap::is_empty")] + pub keywords: IndexMap, + + #[serde( + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub received_at: Option, +} diff --git a/src/jmap/wire/file_node.rs b/src/jmap/wire/file_node.rs new file mode 100644 index 0000000..d5cd229 --- /dev/null +++ b/src/jmap/wire/file_node.rs @@ -0,0 +1,62 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde::{Deserialize, Serialize}; + +use super::common::bool_or_true; +use super::{JmapId, UtcDate}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FileNode { + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parent_id: Option, + + pub node_type: NodeType, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub blob_id: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target: Option>, + + pub name: String, + + #[serde(rename = "type", default, skip_serializing_if = "Option::is_none")] + pub media_type: Option, + + #[serde(with = "time::serde::rfc3339")] + pub created: UtcDate, + + #[serde( + default, + with = "time::serde::rfc3339::option", + skip_serializing_if = "Option::is_none" + )] + pub modified: Option, + + #[serde(default = "default_true", deserialize_with = "bool_or_true")] + pub is_subscribed: bool, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum NodeType { + File, + Directory, + Symlink, +} + +fn default_true() -> bool { + true +} diff --git a/src/jmap/wire/identity.rs b/src/jmap/wire/identity.rs new file mode 100644 index 0000000..b88144e --- /dev/null +++ b/src/jmap/wire/identity.rs @@ -0,0 +1,34 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde::{Deserialize, Serialize}; + +use super::{EmailAddress, JmapId}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Identity { + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + + #[serde(default)] + pub name: String, + + pub email: String, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reply_to: Option>, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bcc: Option>, + + #[serde(default)] + pub text_signature: String, + + #[serde(default)] + pub html_signature: String, +} diff --git a/src/jmap/wire/mailbox.rs b/src/jmap/wire/mailbox.rs new file mode 100644 index 0000000..0b4a46f --- /dev/null +++ b/src/jmap/wire/mailbox.rs @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde::{Deserialize, Serialize}; + +use super::JmapId; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Mailbox { + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + + pub name: String, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parent_id: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + + #[serde(default)] + pub sort_order: u32, + + #[serde(default = "default_true")] + pub is_subscribed: bool, +} + +fn default_true() -> bool { + true +} diff --git a/src/jmap/wire/mod.rs b/src/jmap/wire/mod.rs new file mode 100644 index 0000000..2250540 --- /dev/null +++ b/src/jmap/wire/mod.rs @@ -0,0 +1,20 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod common; + +pub mod address_book; +pub mod calendar; +pub mod calendar_event; +pub mod contact_card; +pub mod email; +pub mod file_node; +pub mod identity; +pub mod mailbox; +pub mod participant_identity; +pub mod sieve_script; + +pub use common::{Date, EmailAddress, JmapId, UtcDate}; diff --git a/src/jmap/wire/participant_identity.rs b/src/jmap/wire/participant_identity.rs new file mode 100644 index 0000000..ed69d04 --- /dev/null +++ b/src/jmap/wire/participant_identity.rs @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde::{Deserialize, Serialize}; + +use super::JmapId; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ParticipantIdentity { + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + + #[serde(default)] + pub name: String, + + pub calendar_address: String, + + #[serde(default)] + pub is_default: bool, +} diff --git a/src/jmap/wire/sieve_script.rs b/src/jmap/wire/sieve_script.rs new file mode 100644 index 0000000..95c5d9c --- /dev/null +++ b/src/jmap/wire/sieve_script.rs @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde::{Deserialize, Serialize}; + +use super::JmapId; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SieveScript { + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + + #[serde(default)] + pub name: Option, + + pub blob_id: JmapId, + + #[serde(default)] + pub is_active: bool, +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..10b7642 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod cli; +pub mod dav; +pub mod db; +pub mod error; +pub mod exchange; +pub mod exchange_ews; +pub mod exchange_graph; +pub mod imap; +pub mod inspect; +pub mod jmap; +pub mod logging; +pub mod managesieve; +pub mod secret; +pub mod sync; +pub mod types; + +use std::sync::Once; + +static CRYPTO_INIT: Once = Once::new(); + +pub fn install_default_crypto_provider() { + CRYPTO_INIT.call_once(|| { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + }); +} diff --git a/src/logging.rs b/src/logging.rs new file mode 100644 index 0000000..5903f5c --- /dev/null +++ b/src/logging.rs @@ -0,0 +1,77 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub const LEVEL_QUIET: u8 = 0; +pub const LEVEL_DEFAULT: u8 = 1; +pub const LEVEL_PROGRESS: u8 = 2; +pub const LEVEL_METHOD: u8 = 3; +pub const LEVEL_BODIES: u8 = 4; + +#[derive(Debug, Clone, Copy)] +pub struct Logger { + level: u8, +} + +impl Logger { + pub fn new(level: u8) -> Self { + Logger { + level: level.min(LEVEL_BODIES), + } + } + + pub fn from_flags(quiet: bool, verbose: u8) -> Self { + let level = if quiet { + LEVEL_QUIET + } else { + LEVEL_DEFAULT.saturating_add(verbose) + }; + Logger::new(level) + } + + pub fn level(&self) -> u8 { + self.level + } + + pub fn enabled(&self, level: u8) -> bool { + self.level >= level + } + + pub fn warn(&self, message: &str) { + eprintln!("warning: {message}"); + } + + pub fn error(&self, message: &str) { + eprintln!("error: {message}"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn quiet_is_level_zero() { + assert_eq!(Logger::from_flags(true, 3).level(), LEVEL_QUIET); + } + + #[test] + fn default_is_level_one() { + assert_eq!(Logger::from_flags(false, 0).level(), LEVEL_DEFAULT); + } + + #[test] + fn verbose_caps_at_four() { + assert_eq!(Logger::from_flags(false, 9).level(), LEVEL_BODIES); + } + + #[test] + fn enabled_is_a_superset_scale() { + let l = Logger::from_flags(false, 1); + assert!(l.enabled(LEVEL_DEFAULT)); + assert!(l.enabled(LEVEL_PROGRESS)); + assert!(!l.enabled(LEVEL_METHOD)); + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..26eea8e --- /dev/null +++ b/src/main.rs @@ -0,0 +1,105 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use clap::Parser; + +use vandelay::cli::{Action, Cli}; +use vandelay::error::Error; +use vandelay::inspect; +use vandelay::sync::{self, Summary}; + +fn main() { + vandelay::install_default_crypto_provider(); + let code = run(); + std::process::exit(code); +} + +fn run() -> i32 { + let cli = match Cli::try_parse() { + Ok(cli) => cli, + Err(err) => { + let _ = err.print(); + return if err.use_stderr() { 1 } else { 0 }; + } + }; + + let action = match cli.resolve() { + Ok(action) => action, + Err(err) => return fail(&err), + }; + + let result = match action { + Action::Import(common, config) => { + let logger = common.logger; + sync::import_jmap::run(common, config).map(|s| (s, logger)) + } + Action::ImportImap(common, config) => { + let logger = common.logger; + sync::import_imap::run(common, config).map(|s| (s, logger)) + } + Action::ImportDav(common, config) => { + let logger = common.logger; + sync::import_dav::run(common, config).map(|s| (s, logger)) + } + Action::ImportManageSieve(common, config) => { + let logger = common.logger; + sync::import_managesieve::run(common, config).map(|s| (s, logger)) + } + Action::ImportMaildir(common, config) => { + let logger = common.logger; + sync::import_maildir::run(common, config).map(|s| (s, logger)) + } + Action::ImportTakeout(common, config) => { + let logger = common.logger; + sync::import_takeout::run(common, config).map(|s| (s, logger)) + } + Action::ImportExchangeEws(common, config) => { + let logger = common.logger; + sync::import_exchange_ews::run(common, config).map(|s| (s, logger)) + } + Action::ImportExchangeGraph(common, config) => { + let logger = common.logger; + sync::import_exchange_graph::run(common, config).map(|s| (s, logger)) + } + Action::Export(common, config) => { + let logger = common.logger; + sync::export::run(common, config).map(|s| (s, logger)) + } + Action::Inspect(config) => { + return match inspect::run(config) { + Ok(()) => 0, + Err(err) => fail(&err), + }; + } + }; + + match result { + Ok((summary, logger)) => { + report(&summary); + if summary.any_failed() { + logger.error("some objects failed; the archive is consistent and resumable"); + 5 + } else { + 0 + } + } + Err(err) => fail(&err), + } +} + +fn fail(err: &Error) -> i32 { + eprintln!("error: {err}"); + err.exit_code() +} + +fn report(summary: &Summary) { + for (type_name, counts) in &summary.per_type { + println!( + "{type_name}: created={} fetched={} deleted={} skipped={} failed={}", + counts.created, counts.fetched, counts.deleted, counts.skipped, counts.failed + ); + } +} diff --git a/src/managesieve/client.rs b/src/managesieve/client.rs new file mode 100644 index 0000000..619487c --- /dev/null +++ b/src/managesieve/client.rs @@ -0,0 +1,504 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::{BufRead, BufReader, Read, Write}; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64; + +use crate::imap::transport::{Connector, ImapStream}; + +use super::command; +use super::error::SieveError; +use super::response::{ + Capabilities, ResponseBlock, Status, StatusLine, Token, parse_capabilities, read_response, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConnectMode { + ImplicitTls, + Plain, + StartTls, +} + +pub struct SieveClient { + reader: BufReader>, + pub host: String, + pub capabilities: Capabilities, + closed: bool, + fresh_post_auth_caps: bool, +} + +impl SieveClient { + pub fn connect( + connector: &Connector, + host: &str, + port: u16, + mode: ConnectMode, + allow_cleartext: bool, + ) -> Result { + let stream: Box = match mode { + ConnectMode::ImplicitTls => connector + .connect_tls(host, port) + .map_err(|e| SieveError::Tls(e.to_string()))?, + ConnectMode::Plain | ConnectMode::StartTls => connector + .connect_plain(host, port) + .map_err(|e| SieveError::Io(std::io::Error::other(e.to_string())))?, + }; + let mut client = SieveClient { + reader: BufReader::new(stream), + host: host.to_owned(), + capabilities: Capabilities::default(), + closed: false, + fresh_post_auth_caps: false, + }; + client.read_initial_capabilities()?; + if matches!(mode, ConnectMode::StartTls) { + if client.capabilities.starttls { + client.start_tls(connector)?; + client.read_initial_capabilities()?; + } else if !allow_cleartext { + return Err(SieveError::Unsupported( + "server does not advertise STARTTLS; pass --allow-cleartext to permit \ + credentials over an unencrypted socket" + .into(), + )); + } + } + Ok(client) + } + + pub fn capabilities(&self) -> &Capabilities { + &self.capabilities + } + + pub fn is_closed(&self) -> bool { + self.closed + } + + fn read_initial_capabilities(&mut self) -> Result<(), SieveError> { + let block = read_response(&mut self.reader)?; + match block.status.status { + Status::Ok => { + self.capabilities = parse_capabilities(&block.data); + Ok(()) + } + Status::Bye => { + self.closed = true; + Err(SieveError::Bye(block.status.text)) + } + Status::No => Err(SieveError::No(block.status.into_no_error())), + } + } + + pub fn refresh_capabilities(&mut self) -> Result<(), SieveError> { + let block = self.run(&command::capability())?; + let caps = parse_capabilities(&block.data); + self.capabilities = caps; + self.fresh_post_auth_caps = true; + Ok(()) + } + + pub fn had_fresh_post_auth_caps(&self) -> bool { + self.fresh_post_auth_caps + } + + fn consume_post_auth_caps(&mut self, data: &[Vec]) { + if data.is_empty() { + return; + } + let parsed = parse_capabilities(data); + let mentions_any = + !parsed.sasl.is_empty() || parsed.implementation.is_some() || parsed.version.is_some(); + if mentions_any { + self.capabilities = parsed; + self.fresh_post_auth_caps = true; + } + } + + fn start_tls(&mut self, connector: &Connector) -> Result<(), SieveError> { + let block = self.run(&command::starttls())?; + let _ = block; + let stream = self.reader.get_ref(); + if stream.is_tls() { + return Err(SieveError::Protocol( + "STARTTLS issued but stream already TLS".into(), + )); + } + let buffered = std::mem::replace(&mut self.reader, BufReader::new(noop_stream())); + if !buffered.buffer().is_empty() { + return Err(SieveError::Protocol( + "buffered data found after STARTTLS OK; possible response injection".into(), + )); + } + let inner = buffered.into_inner(); + let upgraded = inner + .upgrade_tls(connector, &self.host) + .map_err(|e| SieveError::Tls(e.to_string()))?; + self.reader = BufReader::new(upgraded); + Ok(()) + } + + pub fn run(&mut self, command: &str) -> Result { + self.write_all(command.as_bytes())?; + let block = read_response(&mut self.reader)?; + finish_block(block, &mut self.closed) + } + + pub fn run_raw(&mut self, bytes: &[u8]) -> Result { + self.write_all(bytes)?; + let block = read_response(&mut self.reader)?; + finish_block(block, &mut self.closed) + } + + pub fn listscripts(&mut self) -> Result { + self.run(&command::listscripts()) + } + + pub fn getscript(&mut self, name: &str) -> Result { + self.run(&command::getscript(name)) + } + + pub fn noop(&mut self) -> Result<(), SieveError> { + let _ = self.run(&command::noop())?; + Ok(()) + } + + pub fn logout(&mut self) -> Result<(), SieveError> { + if self.closed { + return Ok(()); + } + let _ = self.write_all(command::logout().as_bytes()); + self.closed = true; + Ok(()) + } + + pub fn authenticate_plain(&mut self, authcid: &str, password: &str) -> Result<(), SieveError> { + let mut payload: Vec = Vec::with_capacity(authcid.len() + password.len() + 3); + payload.push(0); + payload.extend_from_slice(authcid.as_bytes()); + payload.push(0); + payload.extend_from_slice(password.as_bytes()); + let encoded = BASE64.encode(&payload); + self.send_authenticate("PLAIN", &encoded) + } + + pub fn authenticate_login(&mut self, authcid: &str, password: &str) -> Result<(), SieveError> { + self.fresh_post_auth_caps = false; + self.write_all(command::authenticate("LOGIN").as_bytes())?; + let user_payload = BASE64.encode(authcid.as_bytes()); + self.expect_continuation()?; + self.write_all(command::continuation_payload(&user_payload).as_bytes())?; + let pass_payload = BASE64.encode(password.as_bytes()); + self.expect_continuation()?; + self.write_all(command::continuation_payload(&pass_payload).as_bytes())?; + let block = read_response(&mut self.reader)?; + match finish_block(block, &mut self.closed) { + Ok(block) => { + self.consume_post_auth_caps(&block.data); + Ok(()) + } + Err(SieveError::No(no)) if no.is_referral() => { + Err(SieveError::Referral(no.to_string())) + } + Err(other) => Err(other), + } + } + + pub fn authenticate_oauthbearer(&mut self, user: &str, token: &str) -> Result<(), SieveError> { + let mut payload = String::with_capacity(user.len() + token.len() + 32); + payload.push_str("n,a="); + payload.push_str(user); + payload.push(','); + payload.push('\x01'); + payload.push_str("auth=Bearer "); + payload.push_str(token); + payload.push('\x01'); + payload.push('\x01'); + let encoded = BASE64.encode(payload.as_bytes()); + self.send_authenticate("OAUTHBEARER", &encoded) + } + + fn send_authenticate(&mut self, mechanism: &str, b64: &str) -> Result<(), SieveError> { + self.fresh_post_auth_caps = false; + self.write_all(command::authenticate_with_initial(mechanism, b64).as_bytes())?; + let block = read_response(&mut self.reader)?; + match finish_block(block, &mut self.closed) { + Ok(block) => { + self.consume_post_auth_caps(&block.data); + Ok(()) + } + Err(SieveError::No(no)) if no.is_referral() => { + Err(SieveError::Referral(no.to_string())) + } + Err(other) => Err(other), + } + } + + fn expect_continuation(&mut self) -> Result<(), SieveError> { + let mut line = Vec::new(); + loop { + line.clear(); + let n = self.reader.read_until(b'\n', &mut line)?; + if n == 0 { + return Err(SieveError::Io(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "expected continuation", + ))); + } + while matches!(line.last(), Some(b'\n') | Some(b'\r')) { + line.pop(); + } + if line.is_empty() { + continue; + } + let s = String::from_utf8_lossy(&line); + let trimmed = s.trim_start(); + if trimmed.starts_with('"') || trimmed.starts_with('{') { + return Ok(()); + } + if trimmed.starts_with("NO") || trimmed.starts_with("BYE") { + let line_tokens = tokens_from_text(&line)?; + let status = super::response::try_parse_status(&line_tokens) + .ok_or_else(|| SieveError::Parse(format!("bad status: {s:?}")))?; + let block = ResponseBlock { + data: Vec::new(), + status, + }; + return finish_block(block, &mut self.closed).map(|_| ()); + } + return Err(SieveError::Parse(format!( + "unexpected line during AUTHENTICATE LOGIN: {s:?}" + ))); + } + } + + fn write_all(&mut self, bytes: &[u8]) -> Result<(), SieveError> { + let stream = self.reader.get_mut(); + stream.write_all(bytes)?; + stream.flush()?; + Ok(()) + } +} + +impl Drop for SieveClient { + fn drop(&mut self) { + if !self.closed { + let _ = self.logout(); + } + } +} + +impl SieveClient { + pub fn new_from_stream(stream: Box, host: String) -> Self { + SieveClient { + reader: BufReader::new(stream), + host, + capabilities: Capabilities::default(), + closed: false, + fresh_post_auth_caps: false, + } + } + + pub fn read_response_block(&mut self) -> Result { + let block = read_response(&mut self.reader)?; + finish_block(block, &mut self.closed) + } +} + +fn tokens_from_text(line: &[u8]) -> Result, SieveError> { + let mut buf: Vec = line.to_vec(); + buf.extend_from_slice(b"\r\n"); + let mut cursor = std::io::Cursor::new(buf); + let mut reader = BufReader::new(&mut cursor); + let block = read_response_with_eof_ok(&mut reader)?; + if let Some(line) = block.data.into_iter().next() { + return Ok(line); + } + Ok(Vec::new()) +} + +fn read_response_with_eof_ok( + reader: &mut R, +) -> Result { + match read_response(reader) { + Ok(b) => Ok(b), + Err(SieveError::Io(_)) => Ok(ResponseBlock::default()), + Err(e) => Err(e), + } +} + +fn finish_block(block: ResponseBlock, closed: &mut bool) -> Result { + match block.status.status { + Status::Ok => Ok(block), + Status::No => { + let no = StatusLine::into_no_error(block.status); + Err(SieveError::No(no)) + } + Status::Bye => { + *closed = true; + Err(SieveError::Bye(block.status.text)) + } + } +} + +fn noop_stream() -> Box { + Box::new(NoopStream) +} + +struct NoopStream; + +impl Read for NoopStream { + fn read(&mut self, _buf: &mut [u8]) -> std::io::Result { + Ok(0) + } +} + +impl Write for NoopStream { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl ImapStream for NoopStream { + fn upgrade_tls( + self: Box, + _connector: &Connector, + _host: &str, + ) -> Result, crate::imap::ImapError> { + Ok(self) + } + fn is_tls(&self) -> bool { + false + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::imap::ImapError; + use std::io::Cursor; + + struct MockStream { + reader: Cursor>, + written: Vec, + } + + impl MockStream { + fn new(server_says: &[u8]) -> Self { + MockStream { + reader: Cursor::new(server_says.to_vec()), + written: Vec::new(), + } + } + } + + impl Read for MockStream { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.reader.read(buf) + } + } + + impl Write for MockStream { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.written.extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl ImapStream for MockStream { + fn upgrade_tls( + self: Box, + _c: &Connector, + _h: &str, + ) -> Result, ImapError> { + Ok(self) + } + fn is_tls(&self) -> bool { + false + } + } + + fn client_with(server: &[u8]) -> SieveClient { + SieveClient::new_from_stream(Box::new(MockStream::new(server)), "test.example".to_owned()) + } + + #[test] + fn run_listscripts_returns_data_and_ok() { + let server = b"\"a\"\r\n\"b\" ACTIVE\r\nOK\r\n"; + let mut c = client_with(server); + let r = c.listscripts().unwrap(); + let scripts = super::super::response::parse_listscripts(&r.data).unwrap(); + assert_eq!(scripts.len(), 2); + assert!(scripts[1].active); + } + + #[test] + fn run_returns_no_as_sieve_error_no() { + let server = b"NO (NONEXISTENT) \"not here\"\r\n"; + let mut c = client_with(server); + let err = c.getscript("missing").unwrap_err(); + match err { + SieveError::No(no) => { + assert_eq!(no.code.as_deref(), Some("NONEXISTENT")); + assert!(no.is_nonexistent()); + } + other => panic!("expected No, got {other:?}"), + } + } + + #[test] + fn bye_closes_client() { + let server = b"BYE \"going away\"\r\n"; + let mut c = client_with(server); + let err = c.noop().unwrap_err(); + assert!(matches!(err, SieveError::Bye(_))); + assert!(c.is_closed()); + } + + #[test] + fn authenticate_plain_sends_initial_response() { + let server = b"OK\r\n"; + let mut c = client_with(server); + c.authenticate_plain("alice", "p@ss").unwrap(); + } + + #[test] + fn authenticate_oauthbearer_sends_initial_response() { + let server = b"OK\r\n"; + let mut c = client_with(server); + c.authenticate_oauthbearer("alice@x", "tok").unwrap(); + } + + #[test] + fn authenticate_login_walks_two_continuations() { + let server = b"\"VXNlcm5hbWU6\"\r\n\"UGFzc3dvcmQ6\"\r\nOK\r\n"; + let mut c = client_with(server); + c.authenticate_login("alice", "p@ss").unwrap(); + } + + #[test] + fn authenticate_no_response_propagates_as_no() { + let server = b"NO \"bad creds\"\r\n"; + let mut c = client_with(server); + let err = c.authenticate_plain("alice", "wrong").unwrap_err(); + assert!(matches!(err, SieveError::No(_)), "got {err:?}"); + } + + #[test] + fn authenticate_referral_no_translates_to_referral() { + let server = b"NO (REFERRAL \"sieve://other\") \"go\"\r\n"; + let mut c = client_with(server); + let err = c.authenticate_plain("alice", "x").unwrap_err(); + assert!(matches!(err, SieveError::Referral(_)), "got {err:?}"); + } +} diff --git a/src/managesieve/command.rs b/src/managesieve/command.rs new file mode 100644 index 0000000..94ac480 --- /dev/null +++ b/src/managesieve/command.rs @@ -0,0 +1,136 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use super::name::encode_argument; + +pub fn capability() -> String { + "CAPABILITY\r\n".to_owned() +} + +pub fn starttls() -> String { + "STARTTLS\r\n".to_owned() +} + +pub fn noop() -> String { + "NOOP\r\n".to_owned() +} + +pub fn logout() -> String { + "LOGOUT\r\n".to_owned() +} + +pub fn listscripts() -> String { + "LISTSCRIPTS\r\n".to_owned() +} + +pub fn getscript(name: &str) -> String { + let mut out = String::with_capacity(name.len() + 16); + out.push_str("GETSCRIPT "); + encode_argument(name, &mut out); + out.push_str("\r\n"); + out +} + +pub fn authenticate(mechanism: &str) -> String { + let mut out = String::with_capacity(mechanism.len() + 20); + out.push_str("AUTHENTICATE "); + encode_argument(mechanism, &mut out); + out.push_str("\r\n"); + out +} + +pub fn authenticate_with_initial(mechanism: &str, initial_b64: &str) -> String { + let mut out = String::with_capacity(mechanism.len() + initial_b64.len() + 32); + out.push_str("AUTHENTICATE "); + encode_argument(mechanism, &mut out); + out.push(' '); + encode_argument(initial_b64, &mut out); + out.push_str("\r\n"); + out +} + +pub fn continuation_payload(initial_b64: &str) -> String { + let mut out = String::with_capacity(initial_b64.len() + 8); + encode_argument(initial_b64, &mut out); + out.push_str("\r\n"); + out +} + +pub use super::name::contains_literal; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn capability_is_uppercase_with_crlf() { + assert_eq!(capability(), "CAPABILITY\r\n"); + } + + #[test] + fn getscript_quoted_for_ascii_names() { + assert_eq!(getscript("vacation"), "GETSCRIPT \"vacation\"\r\n"); + } + + #[test] + fn getscript_escapes_quotes_and_backslashes() { + let out = getscript("a\"b\\c"); + assert!(out.contains("\\\"")); + assert!(out.contains("\\\\")); + } + + #[test] + fn getscript_uses_literal_for_non_ascii() { + let name = "fancy\u{00e7}y"; + let out = getscript(name); + let nbytes = name.len(); + let expected = { + let mut v: Vec = Vec::new(); + v.extend_from_slice(format!("GETSCRIPT {{{nbytes}+}}\r\n").as_bytes()); + v.extend_from_slice(name.as_bytes()); + v.extend_from_slice(b"\r\n"); + v + }; + assert_eq!( + out.as_bytes(), + expected.as_slice(), + "GETSCRIPT wire bytes for non-ascii name must be byte-exact" + ); + } + + #[test] + fn getscript_uses_literal_for_names_containing_crlf() { + let name = "with\nlinebreak"; + let out = getscript(name); + assert!( + out.contains("{") && out.contains("+}\r\n"), + "expected literal for newline-bearing name: {out:?}" + ); + } + + #[test] + fn authenticate_with_initial_emits_two_arguments() { + let out = authenticate_with_initial("PLAIN", "AGZvbwBiYXI="); + assert_eq!(out, "AUTHENTICATE \"PLAIN\" \"AGZvbwBiYXI=\"\r\n"); + } + + #[test] + fn authenticate_without_initial_emits_one_argument() { + let out = authenticate("LOGIN"); + assert_eq!(out, "AUTHENTICATE \"LOGIN\"\r\n"); + } + + #[test] + fn continuation_payload_quotes_simple_base64() { + assert_eq!(continuation_payload("Zm9v"), "\"Zm9v\"\r\n"); + } + + #[test] + fn contains_literal_detects_open_brace() { + assert!(contains_literal(b"GETSCRIPT {3+}\r\nabc\r\n")); + assert!(!contains_literal(b"GETSCRIPT \"abc\"\r\n")); + } +} diff --git a/src/managesieve/error.rs b/src/managesieve/error.rs new file mode 100644 index 0000000..52570e8 --- /dev/null +++ b/src/managesieve/error.rs @@ -0,0 +1,166 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::fmt; +use std::io; + +#[derive(Debug, thiserror::Error)] +pub enum SieveError { + #[error("transport error: {0}")] + Io(#[from] io::Error), + + #[error("tls error: {0}")] + Tls(String), + + #[error("protocol error: {0}")] + Protocol(String), + + #[error("response parse error: {0}")] + Parse(String), + + #[error("server said NO: {0}")] + No(NoError), + + #[error("server closed connection: {0}")] + Bye(String), + + #[error("unsupported by server: {0}")] + Unsupported(String), + + #[error("referral not supported: {0}")] + Referral(String), +} + +#[derive(Debug, Clone)] +pub struct NoError { + pub text: String, + pub code: Option, +} + +impl NoError { + pub fn new(text: impl Into, code: Option) -> Self { + Self { + text: text.into(), + code, + } + } + + pub fn is_transient(&self) -> bool { + if let Some(c) = &self.code { + let cu = c.to_ascii_uppercase(); + if cu == "TRYLATER" { + return true; + } + if matches!( + cu.as_str(), + "QUOTA" + | "QUOTA/MAXSCRIPTS" + | "AUTH-TOO-WEAK" + | "ENCRYPT-NEEDED" + | "TRANSITION-NEEDED" + | "NONEXISTENT" + | "ALREADYEXISTS" + | "ACTIVE" + | "SASL" + | "TAG" + | "WARNINGS" + ) { + return false; + } + } + let lower = self.text.to_ascii_lowercase(); + const TRANSIENT_MARKERS: &[&str] = &[ + "temp", + "try again", + "try later", + "unavailable", + "throttle", + "rate limit", + "too many", + ]; + TRANSIENT_MARKERS.iter().any(|m| lower.contains(m)) + } + + pub fn is_referral(&self) -> bool { + matches!( + self.code.as_deref().map(str::to_ascii_uppercase).as_deref(), + Some("REFERRAL") + ) + } + + pub fn is_nonexistent(&self) -> bool { + matches!( + self.code.as_deref().map(str::to_ascii_uppercase).as_deref(), + Some("NONEXISTENT") + ) + } +} + +impl fmt::Display for NoError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.code { + Some(code) => write!(f, "({code}) {}", self.text), + None => f.write_str(&self.text), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn transient_markers_detected_case_insensitively() { + for s in [ + "Temporary failure", + "Try again later", + "Server unavailable", + "user is rate limited", + "Too many connections", + ] { + assert!( + NoError::new(s, None).is_transient(), + "should be transient: {s}" + ); + } + } + + #[test] + fn trylater_code_is_transient() { + assert!(NoError::new("x", Some("TRYLATER".into())).is_transient()); + } + + #[test] + fn quota_code_is_permanent_per_spec() { + assert!(!NoError::new("x", Some("QUOTA".into())).is_transient()); + assert!(!NoError::new("x", Some("QUOTA/MAXSCRIPTS".into())).is_transient()); + } + + #[test] + fn permanent_codes_override_text_match() { + + assert!(!NoError::new("try again later", Some("QUOTA".into())).is_transient()); + assert!(!NoError::new("temporarily", Some("AUTH-TOO-WEAK".into())).is_transient()); + assert!(!NoError::new("rate limit", Some("TRANSITION-NEEDED".into())).is_transient()); + } + + #[test] + fn referral_code_detected() { + assert!(NoError::new("x", Some("REFERRAL".into())).is_referral()); + assert!(!NoError::new("x", None).is_referral()); + } + + #[test] + fn nonexistent_code_detected() { + assert!(NoError::new("x", Some("NONEXISTENT".into())).is_nonexistent()); + assert!(NoError::new("x", Some("nonexistent".into())).is_nonexistent()); + } + + #[test] + fn non_transient_no_not_flagged() { + assert!(!NoError::new("Permission denied", None).is_transient()); + } +} diff --git a/src/managesieve/mod.rs b/src/managesieve/mod.rs new file mode 100644 index 0000000..dfbebad --- /dev/null +++ b/src/managesieve/mod.rs @@ -0,0 +1,19 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod client; +pub mod command; +pub mod error; +pub mod name; +pub mod response; +pub mod retry; + +pub use client::{ConnectMode, SieveClient}; +pub use error::{NoError, SieveError}; +pub use response::{ + Capabilities, ListedScript, ResponseBlock, Status, StatusLine, Token, parse_capabilities, + parse_getscript, parse_listscripts, read_response, try_parse_status, +}; diff --git a/src/managesieve/name.rs b/src/managesieve/name.rs new file mode 100644 index 0000000..c774459 --- /dev/null +++ b/src/managesieve/name.rs @@ -0,0 +1,123 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub fn encode_argument(s: &str, out: &mut String) { + let bytes = s.as_bytes(); + if can_be_quoted(bytes) { + out.push('"'); + for ch in s.chars() { + if ch == '"' || ch == '\\' { + out.push('\\'); + } + out.push(ch); + } + out.push('"'); + } else { + out.push('{'); + out.push_str(&bytes.len().to_string()); + out.push_str("+}\r\n"); + out.push_str(s); + } +} + +pub fn can_be_quoted(bytes: &[u8]) -> bool { + if bytes.len() > 1024 { + return false; + } + bytes + .iter() + .all(|&b| b != b'\r' && b != b'\n' && b != 0 && (b == b'\t' || (0x20..=0x7e).contains(&b))) +} + +pub fn contains_literal(bytes: &[u8]) -> bool { + bytes.contains(&b'{') +} + +#[cfg(test)] +mod tests { + use super::*; + + fn encoded(s: &str) -> String { + let mut out = String::new(); + encode_argument(s, &mut out); + out + } + + #[test] + fn ascii_round_trips_as_quoted_string() { + assert_eq!(encoded("vacation"), "\"vacation\""); + assert_eq!(encoded(""), "\"\""); + } + + #[test] + fn quotes_and_backslashes_are_escaped() { + assert_eq!(encoded("a\"b"), "\"a\\\"b\""); + assert_eq!(encoded("a\\b"), "\"a\\\\b\""); + } + + #[test] + fn non_ascii_promotes_to_literal_with_byte_exact_body() { + let name = "fancy\u{00e7}y"; + let out = encoded(name); + let in_bytes = name.as_bytes(); + let header = format!("{{{}+}}\r\n", in_bytes.len()); + assert!(out.starts_with(&header), "got {out:?}"); + let body = &out.as_bytes()[header.len()..]; + assert_eq!(body, in_bytes, "literal body must be byte-exact UTF-8"); + assert_eq!(body.len(), in_bytes.len()); + } + + #[test] + fn literal_length_matches_payload_byte_count_for_multibyte_runs() { + for s in [ + "\u{00e7}", + "na\u{00ef}ve", + "\u{4e2d}\u{6587}", + "mix-\u{00e9}\u{00e0}\u{00f6}", + ] { + let out = encoded(s); + let wire = out.as_bytes(); + let header = format!("{{{}+}}\r\n", s.len()); + assert!( + wire.starts_with(header.as_bytes()), + "expected header {header:?} for {s:?}, got {out:?}" + ); + let body = &wire[header.len()..]; + assert_eq!(body, s.as_bytes(), "literal body must equal input bytes"); + } + } + + #[test] + fn crlf_in_name_promotes_to_literal() { + let out = encoded("with\nline"); + assert!(out.starts_with('{')); + assert!(out.contains("+}\r\n")); + let header = format!("{{{}+}}\r\n", "with\nline".len()); + assert!(out.starts_with(&header)); + } + + #[test] + fn tab_is_allowed_in_quoted_string() { + assert!(can_be_quoted(b"abc\tdef")); + } + + #[test] + fn nul_byte_forces_literal() { + assert!(!can_be_quoted(b"a\0b")); + } + + #[test] + fn names_over_quoted_threshold_use_literal() { + let long: Vec = (0..2048).map(|_| b'x').collect(); + assert!(!can_be_quoted(&long)); + } + + #[test] + fn contains_literal_detects_open_brace() { + assert!(contains_literal(b"GETSCRIPT {3+}\r\nabc\r\n")); + assert!(!contains_literal(b"GETSCRIPT \"abc\"\r\n")); + } +} diff --git a/src/managesieve/response.rs b/src/managesieve/response.rs new file mode 100644 index 0000000..f67032d --- /dev/null +++ b/src/managesieve/response.rs @@ -0,0 +1,623 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::{self, BufRead}; + +use super::error::{NoError, SieveError}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Token { + Atom(String), + QStr(String), + Literal(Vec), + LParen, + RParen, +} + +impl Token { + pub fn as_string_bytes(&self) -> Option> { + match self { + Token::QStr(s) => Some(s.as_bytes().to_vec()), + Token::Literal(b) => Some(b.clone()), + _ => None, + } + } + + pub fn as_string(&self) -> Option { + match self { + Token::QStr(s) => Some(s.clone()), + Token::Literal(b) => Some(String::from_utf8_lossy(b).into_owned()), + Token::Atom(s) => Some(s.clone()), + _ => None, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Status { + Ok, + No, + Bye, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StatusLine { + pub status: Status, + pub code: Option, + pub code_args: Option, + pub text: String, +} + +impl Default for StatusLine { + fn default() -> Self { + StatusLine { + status: Status::Ok, + code: None, + code_args: None, + text: String::new(), + } + } +} + +impl StatusLine { + pub fn into_no_error(self) -> NoError { + NoError::new(self.text, self.code) + } +} + +#[derive(Debug, Default)] +pub struct ResponseBlock { + pub data: Vec>, + pub status: StatusLine, +} + +pub fn read_response(reader: &mut R) -> Result { + let mut data: Vec> = Vec::new(); + loop { + let line = read_logical_line(reader)?; + if line.is_empty() { + continue; + } + if let Some(status) = try_parse_status(&line) { + return Ok(ResponseBlock { data, status }); + } + data.push(line); + } +} + +pub fn try_parse_status(tokens: &[Token]) -> Option { + let first = tokens.first()?; + let status = match first { + Token::Atom(s) => match s.as_str() { + "OK" | "ok" | "Ok" => Status::Ok, + "NO" | "no" | "No" => Status::No, + "BYE" | "bye" | "Bye" => Status::Bye, + _ => return None, + }, + _ => return None, + }; + let mut idx = 1usize; + let mut code: Option = None; + let mut code_args: Option = None; + if matches!(tokens.get(idx), Some(Token::LParen)) { + idx += 1; + let mut depth: i32 = 1; + let mut inside: Vec = Vec::new(); + while idx < tokens.len() && depth > 0 { + match &tokens[idx] { + Token::LParen => { + depth += 1; + inside.push(tokens[idx].clone()); + } + Token::RParen => { + depth -= 1; + if depth > 0 { + inside.push(tokens[idx].clone()); + } + } + t => inside.push(t.clone()), + } + idx += 1; + } + if !inside.is_empty() { + if let Some(name) = inside[0].as_string() { + code = Some(name); + } + if inside.len() > 1 { + let mut joined = String::new(); + for (i, t) in inside.iter().enumerate().skip(1) { + if let Some(s) = t.as_string() { + if i > 1 { + joined.push(' '); + } + joined.push_str(&s); + } + } + if !joined.is_empty() { + code_args = Some(joined); + } + } + } + } + let text = match tokens.get(idx) { + Some(Token::QStr(s)) => s.clone(), + Some(Token::Literal(b)) => String::from_utf8_lossy(b).into_owned(), + Some(Token::Atom(s)) => s.clone(), + _ => String::new(), + }; + Some(StatusLine { + status, + code, + code_args, + text, + }) +} + +fn read_logical_line(reader: &mut R) -> Result, SieveError> { + let mut tokens: Vec = Vec::new(); + let mut physical = Vec::with_capacity(256); + read_physical_line(reader, &mut physical)?; + let mut pos = 0usize; + loop { + while pos < physical.len() && matches!(physical[pos], b' ' | b'\t') { + pos += 1; + } + if pos >= physical.len() { + return Ok(tokens); + } + match physical[pos] { + b'"' => { + let (s, new_pos) = parse_quoted_string(&physical, pos + 1)?; + tokens.push(Token::QStr(s)); + pos = new_pos; + } + b'{' => { + let (n, plus, new_pos) = parse_literal_marker(&physical, pos + 1)?; + let _ = plus; + let trailing = &physical[new_pos..]; + if trailing.iter().any(|b| !matches!(b, b' ' | b'\t')) { + return Err(SieveError::Parse(format!( + "literal marker {{{n}}} not at end of line: trailing {:?}", + String::from_utf8_lossy(trailing) + ))); + } + let mut data = vec![0u8; n]; + reader.read_exact(&mut data)?; + tokens.push(Token::Literal(data)); + physical.clear(); + read_physical_line(reader, &mut physical)?; + pos = 0; + } + b'(' => { + tokens.push(Token::LParen); + pos += 1; + } + b')' => { + tokens.push(Token::RParen); + pos += 1; + } + _ => { + let start = pos; + while pos < physical.len() + && !matches!(physical[pos], b' ' | b'\t' | b'"' | b'{' | b'(' | b')') + { + pos += 1; + } + let atom = String::from_utf8_lossy(&physical[start..pos]).into_owned(); + tokens.push(Token::Atom(atom)); + } + } + } +} + +fn read_physical_line(reader: &mut R, buf: &mut Vec) -> Result<(), SieveError> { + buf.clear(); + let n = reader.read_until(b'\n', buf)?; + if n == 0 { + return Err(SieveError::Io(io::Error::new( + io::ErrorKind::UnexpectedEof, + "managesieve closed stream", + ))); + } + while matches!(buf.last(), Some(b'\n') | Some(b'\r')) { + buf.pop(); + } + Ok(()) +} + +fn parse_quoted_string(buf: &[u8], start: usize) -> Result<(String, usize), SieveError> { + let mut bytes: Vec = Vec::new(); + let mut i = start; + while i < buf.len() { + match buf[i] { + b'"' => { + let s = String::from_utf8_lossy(&bytes).into_owned(); + return Ok((s, i + 1)); + } + b'\\' if i + 1 < buf.len() => { + bytes.push(buf[i + 1]); + i += 2; + } + _ => { + bytes.push(buf[i]); + i += 1; + } + } + } + Err(SieveError::Parse("unterminated quoted-string".into())) +} + +fn parse_literal_marker(buf: &[u8], start: usize) -> Result<(usize, bool, usize), SieveError> { + let mut i = start; + let mut n: usize = 0; + let digits_start = i; + while i < buf.len() && buf[i].is_ascii_digit() { + n = n + .checked_mul(10) + .and_then(|v| v.checked_add((buf[i] - b'0') as usize)) + .ok_or_else(|| SieveError::Parse("literal size overflow".into()))?; + i += 1; + } + if i == digits_start { + return Err(SieveError::Parse("literal marker missing digits".into())); + } + let plus = i < buf.len() && buf[i] == b'+'; + if plus { + i += 1; + } + if i >= buf.len() || buf[i] != b'}' { + return Err(SieveError::Parse( + "literal marker missing closing brace".into(), + )); + } + Ok((n, plus, i + 1)) +} + +#[derive(Debug, Default, Clone)] +pub struct ListedScript { + pub name: String, + pub active: bool, +} + +pub fn parse_listscripts(data: &[Vec]) -> Result, SieveError> { + let mut out = Vec::with_capacity(data.len()); + for line in data { + let mut iter = line.iter(); + let first = match iter.next() { + Some(t) => t, + None => continue, + }; + let name = first + .as_string() + .ok_or_else(|| SieveError::Parse(format!("LISTSCRIPTS line missing name: {line:?}")))?; + let active = iter.any(|t| matches!(t, Token::Atom(a) if a.eq_ignore_ascii_case("ACTIVE"))); + out.push(ListedScript { name, active }); + } + Ok(out) +} + +pub fn parse_getscript(data: &[Vec]) -> Result, SieveError> { + let mut buf: Vec = Vec::new(); + let mut found = false; + for line in data { + for token in line { + match token { + Token::Literal(b) => { + if found { + return Err(SieveError::Parse( + "GETSCRIPT returned more than one body token".into(), + )); + } + buf = b.clone(); + found = true; + } + Token::QStr(s) => { + if found { + return Err(SieveError::Parse( + "GETSCRIPT returned more than one body token".into(), + )); + } + buf = s.as_bytes().to_vec(); + found = true; + } + _ => {} + } + } + } + if !found { + return Ok(Vec::new()); + } + Ok(buf) +} + +pub fn parse_capabilities(data: &[Vec]) -> Capabilities { + let mut caps = Capabilities::default(); + for line in data { + if line.is_empty() { + continue; + } + let name = match line[0].as_string() { + Some(s) => s, + None => continue, + }; + let value = if line.len() > 1 { + line[1..] + .iter() + .filter_map(|t| t.as_string()) + .collect::>() + .join(" ") + } else { + String::new() + }; + match name.to_ascii_uppercase().as_str() { + "IMPLEMENTATION" => caps.implementation = Some(value), + "VERSION" => caps.version = Some(value), + "SASL" => { + caps.sasl = value + .split_ascii_whitespace() + .map(|s| s.to_ascii_uppercase()) + .collect() + } + "SIEVE" => caps.sieve = Some(value), + "STARTTLS" => caps.starttls = true, + "MAXREDIRECTS" => caps.maxredirects = value.parse().ok(), + "NOTIFY" => caps.notify = Some(value), + "LANGUAGE" => caps.language = Some(value), + "OWNER" => caps.owner = Some(value), + other => { + caps.raw.push((other.to_ascii_uppercase(), value)); + } + } + } + caps +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Capabilities { + pub implementation: Option, + pub version: Option, + pub sasl: Vec, + pub sieve: Option, + pub starttls: bool, + pub maxredirects: Option, + pub notify: Option, + pub language: Option, + pub owner: Option, + pub raw: Vec<(String, String)>, +} + +impl Capabilities { + pub fn has_sasl(&self, mech: &str) -> bool { + self.sasl.iter().any(|m| m.eq_ignore_ascii_case(mech)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + fn cursor(s: &[u8]) -> Cursor> { + Cursor::new(s.to_vec()) + } + + #[test] + fn parses_greeting_with_quoted_pairs_then_ok() { + let input = + b"\"IMPLEMENTATION\" \"Test Server\"\r\n\"VERSION\" \"1.0\"\r\n\"STARTTLS\"\r\n\"SASL\" \"PLAIN LOGIN\"\r\nOK \"hello\"\r\n"; + let mut c = cursor(input); + let r = read_response(&mut c).unwrap(); + assert_eq!(r.status.status, Status::Ok); + assert_eq!(r.status.text, "hello"); + let caps = parse_capabilities(&r.data); + assert_eq!(caps.implementation, Some("Test Server".to_owned())); + assert_eq!(caps.version, Some("1.0".to_owned())); + assert!(caps.starttls); + assert!(caps.has_sasl("PLAIN")); + assert!(caps.has_sasl("login")); + assert!(!caps.has_sasl("XOAUTH2")); + } + + #[test] + fn parses_listscripts_with_active_marker() { + let input = b"\"vacation\"\r\n\"out-of-office\" ACTIVE\r\n\"forward\"\r\nOK\r\n"; + let mut c = cursor(input); + let r = read_response(&mut c).unwrap(); + assert_eq!(r.status.status, Status::Ok); + let list = parse_listscripts(&r.data).unwrap(); + assert_eq!(list.len(), 3); + assert!(!list[0].active); + assert!(list[1].active); + assert_eq!(list[1].name, "out-of-office"); + assert!(!list[2].active); + } + + #[test] + fn parses_listscripts_with_literal_name_then_active_on_same_logical_line() { + let input = b"{12+}\r\nfun\xc3\xa7y-name SP ACTIVE\r\nOK\r\n"; + let _ = input; + let body = b"\x67\xc3\xa9\x6e\xc3\xa9\x72\x69\x71\x75\x65"; + let mut data: Vec = Vec::new(); + data.extend_from_slice(format!("{{{}}}\r\n", body.len()).as_bytes()); + data.extend_from_slice(body); + data.extend_from_slice(b" ACTIVE\r\n"); + data.extend_from_slice(b"OK\r\n"); + let mut c = cursor(&data); + let r = read_response(&mut c).unwrap(); + assert_eq!(r.status.status, Status::Ok); + let list = parse_listscripts(&r.data).unwrap(); + assert_eq!(list.len(), 1); + assert_eq!(list[0].name.as_bytes(), body); + assert!(list[0].active); + } + + #[test] + fn parses_getscript_literal_body() { + let script = b"require [\"fileinto\"];\nif true { fileinto \"INBOX\"; }\n"; + let mut data: Vec = Vec::new(); + data.extend_from_slice(format!("{{{}}}\r\n", script.len()).as_bytes()); + data.extend_from_slice(script); + data.extend_from_slice(b"\r\nOK\r\n"); + let mut c = cursor(&data); + let r = read_response(&mut c).unwrap(); + let body = parse_getscript(&r.data).unwrap(); + assert_eq!(body, script); + } + + #[test] + fn parses_getscript_quoted_one_line_body() { + let data = b"\"stop;\"\r\nOK\r\n"; + let mut c = cursor(data); + let r = read_response(&mut c).unwrap(); + let body = parse_getscript(&r.data).unwrap(); + assert_eq!(body, b"stop;"); + } + + #[test] + fn parses_no_with_code_and_text() { + let data = b"NO (NONEXISTENT) \"no such script\"\r\n"; + let mut c = cursor(data); + let r = read_response(&mut c).unwrap(); + assert_eq!(r.status.status, Status::No); + assert_eq!(r.status.code.as_deref(), Some("NONEXISTENT")); + assert_eq!(r.status.text, "no such script"); + } + + #[test] + fn parses_no_without_code_and_quoted_text() { + let data = b"NO \"failed\"\r\n"; + let mut c = cursor(data); + let r = read_response(&mut c).unwrap(); + assert_eq!(r.status.status, Status::No); + assert!(r.status.code.is_none()); + assert_eq!(r.status.text, "failed"); + } + + #[test] + fn parses_bye_referral_with_code_args() { + let data = b"BYE (REFERRAL \"sieve://other.example/\") \"go away\"\r\n"; + let mut c = cursor(data); + let r = read_response(&mut c).unwrap(); + assert_eq!(r.status.status, Status::Bye); + assert_eq!(r.status.code.as_deref(), Some("REFERRAL")); + assert_eq!(r.status.text, "go away"); + assert!(r.status.code_args.unwrap().contains("other.example")); + } + + #[test] + fn parses_bare_ok_response() { + let data = b"OK\r\n"; + let mut c = cursor(data); + let r = read_response(&mut c).unwrap(); + assert_eq!(r.status.status, Status::Ok); + assert!(r.status.text.is_empty()); + assert!(r.data.is_empty()); + } + + #[test] + fn quoted_string_escapes_backslash_and_quote() { + let data = b"\"a \\\"b\\\\\" \r\nOK\r\n"; + let _ = data; + let mut buf: Vec = Vec::new(); + buf.extend_from_slice(b"\"a \\\"b\\\\\"\r\nOK\r\n"); + let mut c = cursor(&buf); + let r = read_response(&mut c).unwrap(); + assert_eq!(r.data.len(), 1); + match &r.data[0][0] { + Token::QStr(s) => assert_eq!(s, "a \"b\\"), + other => panic!("expected QStr, got {other:?}"), + } + } + + #[test] + fn literal_size_overflow_is_parse_error() { + let data = b"{99999999999999999999}\r\nbody\r\nOK\r\n"; + let mut c = cursor(data); + let err = read_response(&mut c).unwrap_err(); + assert!(matches!(err, SieveError::Parse(_)), "got {err:?}"); + } + + #[test] + fn unterminated_quoted_string_is_parse_error() { + let data = b"\"unterminated\r\n"; + let mut c = cursor(data); + let err = read_response(&mut c).unwrap_err(); + assert!(matches!(err, SieveError::Parse(_)), "got {err:?}"); + } + + #[test] + fn capability_atoms_recognised_without_quotes() { + let data = b"IMPLEMENTATION \"X\"\r\nSTARTTLS\r\nSASL PLAIN\r\nOK\r\n"; + let mut c = cursor(data); + let r = read_response(&mut c).unwrap(); + let caps = parse_capabilities(&r.data); + assert_eq!(caps.implementation.as_deref(), Some("X")); + assert!(caps.starttls); + assert!(caps.has_sasl("PLAIN")); + } + + #[test] + fn capability_sasl_as_multiple_unquoted_atoms_captures_every_mechanism() { + let data = b"IMPLEMENTATION \"X\"\r\nSASL PLAIN LOGIN OAUTHBEARER\r\nOK\r\n"; + let mut c = cursor(data); + let r = read_response(&mut c).unwrap(); + let caps = parse_capabilities(&r.data); + assert!(caps.has_sasl("PLAIN"), "PLAIN missing: {:?}", caps.sasl); + assert!(caps.has_sasl("LOGIN"), "LOGIN missing: {:?}", caps.sasl); + assert!( + caps.has_sasl("OAUTHBEARER"), + "OAUTHBEARER missing: {:?}", + caps.sasl + ); + } + + #[test] + fn capability_sasl_as_single_quoted_string_captures_every_mechanism() { + let data = b"\"SASL\" \"PLAIN LOGIN OAUTHBEARER\"\r\nOK\r\n"; + let mut c = cursor(data); + let r = read_response(&mut c).unwrap(); + let caps = parse_capabilities(&r.data); + assert!(caps.has_sasl("PLAIN")); + assert!(caps.has_sasl("LOGIN")); + assert!(caps.has_sasl("OAUTHBEARER")); + } + + #[test] + fn quoted_string_preserves_non_ascii_utf8_bytes() { + let name_bytes: &[u8] = b"fancy\xc3\xa7y"; + let mut data: Vec = Vec::new(); + data.push(b'"'); + data.extend_from_slice(name_bytes); + data.push(b'"'); + data.extend_from_slice(b"\r\nOK\r\n"); + let mut c = cursor(&data); + let r = read_response(&mut c).unwrap(); + assert_eq!(r.data.len(), 1); + match &r.data[0][0] { + Token::QStr(s) => assert_eq!(s.as_bytes(), name_bytes), + other => panic!("expected QStr, got {other:?}"), + } + } + + #[test] + fn listscripts_quoted_name_with_non_ascii_is_byte_exact() { + let mut data: Vec = Vec::new(); + data.extend_from_slice(b"\"fun\xc3\xa7y\"\r\nOK\r\n"); + let mut c = cursor(&data); + let r = read_response(&mut c).unwrap(); + let list = parse_listscripts(&r.data).unwrap(); + assert_eq!(list.len(), 1); + assert_eq!(list[0].name.as_bytes(), b"fun\xc3\xa7y"); + } + + #[test] + fn parses_two_logical_lines_with_intervening_blank_skipped() { + let data = b"\"a\"\r\n\r\n\"b\"\r\nOK\r\n"; + let mut c = cursor(data); + let r = read_response(&mut c).unwrap(); + assert_eq!(r.data.len(), 2); + } +} diff --git a/src/managesieve/retry.rs b/src/managesieve/retry.rs new file mode 100644 index 0000000..521e52c --- /dev/null +++ b/src/managesieve/retry.rs @@ -0,0 +1,127 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use super::error::SieveError; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Disposition { + TransportDrop, + Transient, + Permanent, + PerScriptRecoverable, + FeatureNegotiation, + Referral, +} + +pub fn classify(err: &SieveError) -> Disposition { + match err { + SieveError::Io(_) => Disposition::TransportDrop, + SieveError::Tls(_) => Disposition::Permanent, + SieveError::Bye(_) => Disposition::TransportDrop, + SieveError::No(no) => { + if no.is_referral() { + Disposition::Referral + } else if no.is_transient() { + Disposition::Transient + } else { + Disposition::Permanent + } + } + SieveError::Protocol(_) | SieveError::Parse(_) | SieveError::Unsupported(_) => { + Disposition::Permanent + } + SieveError::Referral(_) => Disposition::Referral, + } +} + +pub fn is_negotiation_failure(err: &SieveError) -> bool { + match err { + SieveError::No(no) => !no.is_transient() && !no.is_referral(), + SieveError::Unsupported(_) => true, + _ => false, + } +} + +pub fn script_disposition(err: &SieveError) -> Disposition { + let base = classify(err); + match base { + Disposition::Permanent | Disposition::FeatureNegotiation => { + Disposition::PerScriptRecoverable + } + other => other, + } +} + +#[cfg(test)] +mod tests { + use super::super::error::NoError; + use super::*; + use std::io::{self, ErrorKind}; + + #[test] + fn io_errors_are_transport_drop() { + for k in [ + ErrorKind::ConnectionReset, + ErrorKind::ConnectionAborted, + ErrorKind::BrokenPipe, + ErrorKind::UnexpectedEof, + ErrorKind::TimedOut, + ] { + let e = SieveError::Io(io::Error::new(k, "x")); + assert_eq!(classify(&e), Disposition::TransportDrop, "{k:?}"); + } + } + + #[test] + fn bye_is_transport_drop() { + assert_eq!( + classify(&SieveError::Bye("going away".into())), + Disposition::TransportDrop + ); + } + + #[test] + fn transient_no_text_marked_transient() { + let e = SieveError::No(NoError::new("Try again later", None)); + assert_eq!(classify(&e), Disposition::Transient); + } + + #[test] + fn trylater_code_marked_transient() { + let e = SieveError::No(NoError::new("x", Some("TRYLATER".into()))); + assert_eq!(classify(&e), Disposition::Transient); + } + + #[test] + fn referral_code_classified_as_referral() { + let e = SieveError::No(NoError::new("x", Some("REFERRAL".into()))); + assert_eq!(classify(&e), Disposition::Referral); + } + + #[test] + fn nonexistent_no_is_permanent_but_per_script_demotes_it() { + let e = SieveError::No(NoError::new("no such script", Some("NONEXISTENT".into()))); + assert_eq!(classify(&e), Disposition::Permanent); + assert_eq!(script_disposition(&e), Disposition::PerScriptRecoverable); + } + + #[test] + fn negotiation_failure_matches_non_transient_no_and_unsupported() { + assert!(is_negotiation_failure(&SieveError::Unsupported("x".into()))); + assert!(is_negotiation_failure(&SieveError::No(NoError::new( + "no such mech", + None + )))); + assert!(!is_negotiation_failure(&SieveError::No(NoError::new( + "try again", + None + )))); + assert!(!is_negotiation_failure(&SieveError::No(NoError::new( + "x", + Some("REFERRAL".into()) + )))); + } +} diff --git a/src/secret.rs b/src/secret.rs new file mode 100644 index 0000000..153427e --- /dev/null +++ b/src/secret.rs @@ -0,0 +1,63 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::{BufRead, IsTerminal, Write}; +use std::process::Command; + +use crate::error::Error; + +pub fn resolve( + flag_value: Option<&str>, + env_var: &str, + prompt_label: &str, +) -> Result { + if let Some(value) = flag_value { + return Ok(value.to_owned()); + } + if let Ok(value) = std::env::var(env_var) + && !value.is_empty() + { + return Ok(value); + } + if std::io::stdin().is_terminal() { + return prompt_no_echo(prompt_label); + } + Err(Error::Usage(format!( + "no {prompt_label} provided: pass the flag, set ${env_var}, or run on a TTY for an interactive prompt" + ))) +} + +fn prompt_no_echo(label: &str) -> Result { + let mut stderr = std::io::stderr(); + write!(stderr, "{label}: ")?; + stderr.flush()?; + + let echo_off = set_echo(false); + let mut line = String::new(); + let read = std::io::stdin().lock().read_line(&mut line); + if echo_off { + let _ = set_echo(true); + } + writeln!(stderr)?; + stderr.flush()?; + read?; + + let trimmed = line.trim_end_matches(['\r', '\n']); + if trimmed.is_empty() { + return Err(Error::Usage(format!("empty {label}"))); + } + Ok(trimmed.to_owned()) +} + +fn set_echo(on: bool) -> bool { + let arg = if on { "echo" } else { "-echo" }; + Command::new("stty") + .arg(arg) + .stdin(std::process::Stdio::inherit()) + .status() + .map(|s| s.success()) + .unwrap_or(false) +} diff --git a/src/sync/emailmeta.rs b/src/sync/emailmeta.rs new file mode 100644 index 0000000..104e1d4 --- /dev/null +++ b/src/sync/emailmeta.rs @@ -0,0 +1,113 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use mail_parser::{Address, HeaderName, HeaderValue, MessageParser}; + +use crate::sync::keys::{EmailIndex, email_index}; + +fn addrs(a: Option<&Address>) -> Vec { + match a { + Some(addr) => addr + .iter() + .filter_map(|x| x.address().map(|s| s.to_owned())) + .collect(), + None => Vec::new(), + } +} + +fn message_ids(value: &HeaderValue) -> Vec { + match value { + HeaderValue::Text(s) => vec![s.to_string()], + HeaderValue::TextList(l) => l.iter().map(|s| s.to_string()).collect(), + _ => Vec::new(), + } +} + +pub fn email_index_from_blob(bytes: &[u8]) -> EmailIndex { + email_meta_from_blob(bytes).0 +} + +pub fn email_meta_from_blob(bytes: &[u8]) -> (EmailIndex, Option) { + let Some(m) = MessageParser::default().parse(bytes) else { + return (email_index(&[], &[], "", "", &[]), None); + }; + let mut mids: Vec = Vec::new(); + for v in m.header_values(HeaderName::MessageId) { + mids.extend(message_ids(v)); + } + if mids.is_empty() + && let Some(single) = m.message_id() + { + mids.push(single.to_owned()); + } + let date = m.date().map(|d| d.to_rfc3339()); + let idx = email_index( + &mids, + &addrs(m.from()), + m.subject().unwrap_or(""), + date.as_deref().unwrap_or(""), + &addrs(m.to()), + ); + (idx, date) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sync::keys::{EmailKey, email_key}; + + #[test] + fn parses_core_fields_and_keys_by_message_id() { + let raw = b"From: Alice \r\nTo: Bob \r\nSubject: Hi there\r\nMessage-ID: \r\nDate: Mon, 12 May 2025 10:00:00 +0200\r\n\r\nbody\r\n"; + let idx = email_index_from_blob(raw); + assert_eq!(idx.mids, vec!["abc-1@host".to_owned()]); + assert_eq!( + email_key(&idx), + EmailKey::MessageId("abc-1@host".to_owned()) + ); + } + + #[test] + fn sent_at_offset_is_preserved_then_z_for_utc() { + let off = email_index_from_blob( + b"Message-ID: \r\nDate: Mon, 12 May 2025 10:00:00 +0200\r\n\r\nx", + ); + let utc = email_index_from_blob( + b"Message-ID: \r\nDate: Mon, 12 May 2025 08:00:00 +0000\r\n\r\nx", + ); + + assert_ne!(off.fb, utc.fb); + } + + #[test] + fn missing_message_id_falls_back_to_hash() { + let idx = email_index_from_blob(b"From: a@x\r\nSubject: only\r\n\r\nbody"); + assert!(idx.mids.is_empty()); + assert!(matches!(email_key(&idx), EmailKey::Fallback(_))); + } + + #[test] + fn multiple_message_id_values_are_all_captured() { + let idx = + email_index_from_blob(b"Message-ID: \r\nMessage-ID: \r\nSubject: s\r\n\r\nx"); + assert_eq!(idx.mids, vec!["a@h".to_owned(), "b@h".to_owned()]); + assert_eq!( + email_key(&idx), + EmailKey::MessageId("a@h\u{1f}b@h".to_owned()) + ); + } + + #[test] + fn same_message_different_received_metadata_same_key() { + let a = email_index_from_blob( + b"From: a@x\r\nSubject: S\r\nMessage-ID: \r\nDate: Mon, 12 May 2025 10:00:00 +0200\r\n\r\nbody", + ); + let b = email_index_from_blob( + b"From: a@x\r\nSubject: S\r\nMessage-ID: \r\nDate: Tue, 13 May 2025 11:00:00 +0000\r\n\r\nbody", + ); + assert_eq!(email_key(&a), email_key(&b)); + } +} diff --git a/src/sync/export.rs b/src/sync/export.rs new file mode 100644 index 0000000..51e3cc5 --- /dev/null +++ b/src/sync/export.rs @@ -0,0 +1,501 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; +use std::io::{IsTerminal, Write}; + +use rusqlite::Connection; +use serde_json::{Map, Value, json}; + +use crate::db; +use crate::error::Error; +use crate::jmap::blobxfer; +use crate::jmap::connect::{self, Connected}; +use crate::jmap::error::JmapError; +use crate::jmap::http::HttpClient; +use crate::jmap::request::{Request, SetRequest, get_all, get_objects, query_all_ids, set_call}; +use crate::jmap::session::{Limits, Session}; +use crate::jmap::wire::JmapId; +use crate::logging::{LEVEL_DEFAULT, Logger}; +use crate::sync::import_jmap::mapping::{BlobUpload, TargetResolver}; +use crate::sync::{CommonConfig, Context, ExportConfig, Summary, TypeCounts}; +use crate::types::ObjectType; + +const EXPORT_ORDER: [ObjectType; 10] = [ + ObjectType::Mailbox, + ObjectType::AddressBook, + ObjectType::Calendar, + ObjectType::FileNode, + ObjectType::Identity, + ObjectType::SieveScript, + ObjectType::ParticipantIdentity, + ObjectType::Email, + ObjectType::ContactCard, + ObjectType::CalendarEvent, +]; + +type IdMap = HashMap; + +#[derive(Default)] +struct Maps { + m: HashMap, +} + +impl Maps { + fn insert(&mut self, ty: ObjectType, local: i64, target: JmapId) { + self.m.entry(ty).or_default().insert(local, target); + } +} + +impl TargetResolver for Maps { + fn target(&self, ty: ObjectType, local_id: i64) -> Option { + self.m.get(&ty)?.get(&local_id).cloned() + } +} + +struct Uploader<'a> { + net: &'a Net, + conn: &'a Connection, + cache: HashMap, +} + +impl<'a> Uploader<'a> { + fn new(net: &'a Net, conn: &'a Connection) -> Uploader<'a> { + Uploader { + net, + conn, + cache: HashMap::new(), + } + } + + fn upload_with(&mut self, local_id: i64, content_type: &str) -> Result { + if let Some(id) = self.cache.get(&local_id) { + return Ok(id.clone()); + } + let id = if self.net.dry_run { + let _exists = db::blobs::blob_bytes(self.conn, local_id)? + .ok_or_else(|| JmapError::malformed(format!("blob local id {local_id} missing")))?; + JmapId(format!("dryrun-blob-{local_id}")) + } else { + let bytes = db::blobs::blob_bytes(self.conn, local_id)? + .ok_or_else(|| JmapError::malformed(format!("blob local id {local_id} missing")))?; + blobxfer::upload_bytes( + &self.net.client, + &self.net.session, + &self.net.account, + content_type, + &bytes, + )? + }; + self.cache.insert(local_id, id.clone()); + Ok(id) + } +} + +impl BlobUpload for Uploader<'_> { + fn upload(&mut self, local_id: i64) -> Result { + self.upload_with(local_id, "application/octet-stream") + } +} + +#[derive(Clone)] +struct Net { + client: HttpClient, + api: String, + account: String, + limits: Limits, + session: Session, + dry_run: bool, +} + +fn has_rows(conn: &Connection, ty: ObjectType) -> bool { + let table = crate::sync::table_name(ty); + conn.query_row(&format!("SELECT EXISTS(SELECT 1 FROM {table})"), [], |r| { + r.get::<_, i64>(0) + }) + .map(|n| n != 0) + .unwrap_or(false) +} + +pub fn run(common: CommonConfig, config: ExportConfig) -> Result { + let logger = common.logger; + let ctx = Context::open(common, &config.connect)?; + let connected = connect::prepare(&ctx, &config.connect)?; + + let net = Net { + client: ctx.client.clone(), + api: connected.session.api_url.clone(), + account: connected.account_id.clone(), + limits: connected.limits, + session: connected.session.clone(), + dry_run: ctx.dry_run(), + }; + + let work = work_list(&ctx.conn, &config, &connected, &logger); + let mut maps = Maps::default(); + let mut summary = Summary::default(); + let mut dry_rows: Vec<(&'static str, u64, u64, u64)> = Vec::new(); + let mut plans: HashMap = HashMap::new(); + let mut counts_per_type: HashMap = HashMap::new(); + + for ty in &work { + if logger.enabled(LEVEL_DEFAULT) { + eprintln!("export: {} ...", ty.jmap_name()); + } + let mut counts = TypeCounts::default(); + let res = reconcile_type( + &ctx, + &net, + *ty, + &mut maps, + &logger, + &mut counts, + &mut dry_rows, + ); + let plan = match res { + Ok(p) => p, + Err(e) => { + logger.warn(&format!("type {} aborted: {e}", ty.jmap_name())); + counts.failed += 1; + Plan::default() + } + }; + plans.insert(*ty, plan); + counts_per_type.insert(*ty, counts); + } + + if config.prune { + prune_phase( + &ctx, + &net, + &work, + &plans, + &config, + &logger, + &mut counts_per_type, + )?; + } + + for ty in &work { + if let Some(counts) = counts_per_type.remove(ty) { + summary.per_type.push((ty.jmap_name(), counts)); + } + } + + if ctx.dry_run() { + print_dry_run(&dry_rows, config.prune); + return Ok(Summary::default()); + } + summary.retries_observed = ctx.client.retries_observed(); + summary.retry_after_sleeps = ctx.client.retry_after_sleeps(); + Ok(summary) +} + +fn prune_phase( + ctx: &Context, + net: &Net, + work: &[ObjectType], + plans: &HashMap, + config: &ExportConfig, + logger: &Logger, + counts_per_type: &mut HashMap, +) -> Result<(), Error> { + let totals: Vec<(ObjectType, &Plan)> = work + .iter() + .filter_map(|ty| plans.get(ty).map(|p| (*ty, p))) + .filter(|(_, p)| !p.prune_candidates.is_empty()) + .collect(); + if totals.is_empty() { + return Ok(()); + } + eprintln!("prune plan:"); + let total: usize = totals.iter().map(|(_, p)| p.prune_candidates.len()).sum(); + for (ty, p) in &totals { + eprintln!( + " {:<22} {:>6} candidate(s); sample: {}", + ty.jmap_name(), + p.prune_candidates.len(), + sample(&p.prune_candidates), + ); + } + eprintln!(" {:<22} {:>6} total", "(all types)", total); + if ctx.dry_run() { + return Ok(()); + } + if !config.yes && std::io::stdin().is_terminal() { + eprint!("destroy all {total} objects across all types? [y/N] "); + let _ = std::io::stderr().flush(); + let mut line = String::new(); + std::io::stdin() + .read_line(&mut line) + .map_err(|e| Error::Partial(e.to_string()))?; + if !matches!(line.trim(), "y" | "Y" | "yes") { + return Err(Error::PruneAborted); + } + } + for ty in work.iter().rev() { + if let Some(plan) = plans.get(ty) + && !plan.prune_candidates.is_empty() + && let Some(counts) = counts_per_type.get_mut(ty) + { + do_destroy(net, *ty, plan, logger, counts); + } + } + Ok(()) +} + +fn work_list( + conn: &Connection, + config: &ExportConfig, + connected: &Connected, + logger: &Logger, +) -> Vec { + let selected = config.objects.as_ref(); + EXPORT_ORDER + .into_iter() + .filter(|ty| selected.map(|s| s.contains(ty)).unwrap_or(true)) + .filter(|ty| has_rows(conn, *ty)) + .filter(|ty| { + if connected.supports(*ty) { + true + } else { + logger.warn(&format!( + "target does not support {}; skipping", + ty.jmap_name() + )); + false + } + }) + .collect() +} + +#[allow(clippy::too_many_arguments)] +fn reconcile_type( + ctx: &Context, + net: &Net, + ty: ObjectType, + maps: &mut Maps, + logger: &Logger, + counts: &mut TypeCounts, + dry_rows: &mut Vec<(&'static str, u64, u64, u64)>, +) -> Result { + let plan = match ty { + ObjectType::Mailbox | ObjectType::FileNode => { + tree::reconcile(ctx, net, ty, maps, counts, logger) + } + ObjectType::AddressBook | ObjectType::Calendar => { + flat::reconcile(ctx, net, ty, maps, counts, logger) + } + ObjectType::Identity => keyed::reconcile_identity(ctx, net, maps, counts, logger), + ObjectType::ParticipantIdentity => { + keyed::reconcile_participant(ctx, net, maps, counts, logger) + } + ObjectType::SieveScript => sieve::reconcile(ctx, net, maps, counts, logger), + ObjectType::ContactCard | ObjectType::CalendarEvent => { + uidtype::reconcile(ctx, net, ty, maps, counts, logger) + } + ObjectType::Email => email::reconcile(ctx, net, maps, counts, logger), + }?; + + if ctx.dry_run() { + dry_rows.push(( + ty.jmap_name(), + counts.created, + counts.skipped, + plan.prune_candidates.len() as u64, + )); + } + + Ok(plan) +} + +#[derive(Default)] +pub struct Plan { + pub prune_candidates: Vec, + pub active_sieve_target: Option, +} + +fn do_destroy(net: &Net, ty: ObjectType, plan: &Plan, logger: &Logger, counts: &mut TypeCounts) { + if ty == ObjectType::SieveScript { + deactivate_active_sieve_script(net, logger); + } + let destroy = Value::Array( + plan.prune_candidates + .iter() + .map(|s| Value::String(s.clone())) + .collect(), + ); + let extra = destroy_contents_arg(ty); + match set_call( + &net.client, + &net.api, + &net.account, + ty.jmap_name(), + SetRequest { + destroy: Some(destroy), + extra_args: &extra, + ..Default::default() + }, + &net.limits, + ) { + Ok(outcome) => { + counts.deleted += outcome.destroyed.len() as u64; + for (id, err) in &outcome.not_destroyed { + logger.warn(&format!( + "prune: {} {id} not destroyed: {err}", + ty.jmap_name() + )); + counts.skipped += 1; + } + } + Err(e) => { + logger.warn(&format!( + "prune {}: destroy request failed: {e}", + ty.jmap_name() + )); + counts.failed += plan.prune_candidates.len() as u64; + } + } +} + +fn deactivate_active_sieve_script(net: &Net, logger: &Logger) { + let mut req = Request::new(); + req.call( + "SieveScript/set", + json!({ "accountId": net.account, "onSuccessDeactivateScript": true }), + "d", + ); + let outcome = req.send(&net.client, &net.api).and_then(|resp| { + let mr = resp.first()?; + crate::jmap::request::check_method_error(mr) + }); + if let Err(e) = outcome { + logger.warn(&format!( + "prune: SieveScript deactivation failed before destroy: {e}" + )); + } +} + +fn destroy_contents_arg(ty: ObjectType) -> Vec<(&'static str, Value)> { + match ty { + ObjectType::AddressBook => vec![("onDestroyRemoveContents", Value::Bool(false))], + ObjectType::Calendar => vec![("onDestroyRemoveEvents", Value::Bool(false))], + ObjectType::FileNode => vec![("onDestroyRemoveChildren", Value::Bool(false))], + _ => Vec::new(), + } +} + +fn sample(ids: &[String]) -> String { + let n = ids.len().min(5); + ids[..n].join(", ") +} + +fn print_dry_run(rows: &[(&'static str, u64, u64, u64)], prune: bool) { + if prune { + println!( + "{:<22} {:>10} {:>10} {:>12}", + "TYPE", "CREATE", "MATCHED", "WOULD-DESTROY" + ); + for (ty, c, m, d) in rows { + println!("{ty:<22} {c:>10} {m:>10} {d:>12}"); + } + } else { + println!("{:<22} {:>10} {:>10}", "TYPE", "CREATE", "MATCHED"); + for (ty, c, m, _) in rows { + println!("{ty:<22} {c:>10} {m:>10}"); + } + } +} + +mod tree; + +mod flat; + +mod keyed; + +mod sieve; + +mod uidtype; + +mod email; + +mod common { + use super::*; + + pub fn target_query_get( + net: &Net, + ty: ObjectType, + props: Option<&[&str]>, + ) -> Result, JmapError> { + let ids = query_all_ids( + &net.client, + &net.api, + &net.account, + ty.jmap_name(), + &net.limits, + )?; + if ids.is_empty() { + return Ok(Vec::new()); + } + let got = get_objects::( + &net.client, + &net.api, + &net.account, + ty.jmap_name(), + &ids, + props, + &net.limits, + )?; + Ok(got.list) + } + + pub fn target_get_all(net: &Net, ty: ObjectType) -> Result, JmapError> { + Ok(get_all::(&net.client, &net.api, &net.account, ty.jmap_name())?.list) + } + + pub fn jid(v: &Value) -> Option { + v.get("id").and_then(Value::as_str).map(str::to_owned) + } + + pub fn create_batch( + net: &Net, + ty: ObjectType, + creates: Vec<(String, Value)>, + ) -> Result { + if net.dry_run { + return Ok(synthesize_dry_run_outcome(ty, &creates)); + } + let mut map = Map::new(); + for (cid, obj) in creates { + map.insert(cid, obj); + } + set_call( + &net.client, + &net.api, + &net.account, + ty.jmap_name(), + SetRequest { + create: Some(Value::Object(map)), + ..Default::default() + }, + &net.limits, + ) + } + + fn synthesize_dry_run_outcome( + ty: ObjectType, + creates: &[(String, Value)], + ) -> crate::jmap::request::SetOutcome { + let mut outcome = crate::jmap::request::SetOutcome::default(); + for (cid, _) in creates { + let synthetic = serde_json::json!({ + "id": format!("dryrun-{}-{cid}", ty.jmap_name()) + }); + outcome.created.push((cid.clone(), synthetic)); + } + outcome + } +} diff --git a/src/sync/export/email.rs b/src/sync/export/email.rs new file mode 100644 index 0000000..f1d75ec --- /dev/null +++ b/src/sync/export/email.rs @@ -0,0 +1,253 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; + +use serde_json::{Map, Value, json}; + +use super::common::{jid, target_query_get}; +use super::{Maps, Net, Plan, Uploader}; +use crate::error::Error; +use crate::jmap::error::JmapError; +use crate::jmap::request::{Request, check_method_error, get_objects}; +use crate::jmap::wire::JmapId; +use crate::logging::Logger; +use crate::sync::import_jmap::mapping::{EMAIL_SELECT, TargetResolver, row_to_email}; +use crate::sync::keys::{EmailIndex, EmailKey, email_index, email_keys, index_from_json}; +use crate::sync::{Context, TypeCounts}; +use crate::types::ObjectType; + +fn server_index(v: &Value) -> EmailIndex { + let arr = |k: &str| { + v.get(k) + .and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(|x| x.get("email").and_then(Value::as_str).map(str::to_owned)) + .collect::>() + }) + .unwrap_or_default() + }; + let mids: Vec = v + .get("messageId") + .and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(|x| x.as_str().map(str::to_owned)) + .collect() + }) + .unwrap_or_default(); + email_index( + &mids, + &arr("from"), + v.get("subject").and_then(Value::as_str).unwrap_or(""), + v.get("sentAt").and_then(Value::as_str).unwrap_or(""), + &arr("to"), + ) +} + +pub fn reconcile( + ctx: &Context, + net: &Net, + maps: &mut Maps, + counts: &mut TypeCounts, + logger: &Logger, +) -> Result { + let ty = ObjectType::Email; + + let target_min = target_query_get(net, ty, Some(&["messageId"])).map_err(Error::from)?; + let mut indices: Vec = target_min.iter().map(server_index).collect(); + + let fallback_ids: Vec = target_min + .iter() + .zip(indices.iter()) + .filter(|(_, i)| i.mids.is_empty()) + .filter_map(|(v, _)| jid(v).map(JmapId)) + .collect(); + if !fallback_ids.is_empty() { + let got = get_objects::( + &net.client, + &net.api, + &net.account, + ty.jmap_name(), + &fallback_ids, + Some(&["messageId", "from", "subject", "sentAt", "to"]), + &net.limits, + ) + .map_err(Error::from)?; + let by_id: HashMap = got + .list + .iter() + .filter_map(|v| jid(v).map(|i| (i, v))) + .collect(); + for (v, slot) in target_min.iter().zip(indices.iter_mut()) { + if let Some(full) = jid(v).and_then(|i| by_id.get(&i)) { + *slot = server_index(full); + } + } + } + let target_keys: HashSet = email_keys(&indices).into_iter().collect(); + + let local: Vec<(i64, crate::sync::import_jmap::mapping::EmailRow)> = { + let mut stmt = ctx + .conn + .prepare(EMAIL_SELECT) + .map_err(|e| Error::Partial(e.to_string()))?; + stmt.query_map([], |row| { + let id: i64 = row.get(0)?; + Ok((id, row_to_email(row))) + }) + .and_then(|m| m.collect::, _>>()) + .map_err(|e| Error::Partial(e.to_string()))? + .into_iter() + .map(|(id, r)| Ok((id, r.map_err(Error::from)?))) + .collect::>()? + }; + + let local_indices: Vec = local + .iter() + .map(|(_, r)| index_from_json(&r.message_match)) + .collect(); + let local_keys = email_keys(&local_indices); + + let mut uploader = Uploader::new(net, &ctx.conn); + for (i, key) in local_keys.iter().enumerate() { + if target_keys.contains(key) { + counts.skipped += 1; + continue; + } + let (local_id, row) = &local[i]; + let mut mids = Map::new(); + let mut all_resolved = true; + for ml in &row.mailbox_locals { + match maps.target(ObjectType::Mailbox, *ml) { + Some(t) => { + mids.insert(t.0, Value::Bool(true)); + } + None => { + all_resolved = false; + break; + } + } + } + if !all_resolved { + logger.warn(&format!( + "email local {local_id} skipped: mailbox not on target" + )); + counts.failed += 1; + continue; + } + let blob = match uploader.upload_with(row.blob_local_id, "message/rfc822") { + Ok(b) => b.0, + Err(e) => { + logger.warn(&format!("email blob upload failed: {e}")); + counts.failed += 1; + continue; + } + }; + let mut kw = Map::new(); + for k in &row.keywords { + kw.insert(k.clone(), Value::Bool(true)); + } + let item = json!({ + "blobId": blob, + "mailboxIds": Value::Object(mids), + "keywords": Value::Object(kw), + "receivedAt": row.received_at, + }); + send_import_chunk(net, &[(format!("e{local_id}"), item)], counts, logger); + } + + Ok(Plan::default()) +} + +fn send_import_chunk( + net: &Net, + items: &[(String, Value)], + counts: &mut TypeCounts, + logger: &Logger, +) { + if items.is_empty() { + return; + } + if net.dry_run { + counts.created += items.len() as u64; + return; + } + let mut map = Map::new(); + for (k, v) in items { + map.insert(k.clone(), v.clone()); + } + let mut req = Request::new(); + req.call( + "Email/import", + json!({ "accountId": net.account, "emails": Value::Object(map) }), + "i", + ); + if req.fits(&net.limits).is_err() { + resplit_or_fail(net, items, counts, logger); + return; + } + match req.send(&net.client, &net.api) { + Ok(resp) => match resp.first().and_then(|mr| { + check_method_error(mr)?; + Ok(mr) + }) { + Ok(mr) => absorb_import(mr, counts, logger), + Err(JmapError::RequestTooLarge) => resplit_or_fail(net, items, counts, logger), + Err(JmapError::Method { error_type, .. }) if error_type == "requestTooLarge" => { + resplit_or_fail(net, items, counts, logger); + } + Err(e) => { + logger.warn(&format!( + "Email/import method error ({} items): {e}", + items.len() + )); + counts.failed += items.len() as u64; + } + }, + Err(JmapError::RequestTooLarge) => resplit_or_fail(net, items, counts, logger), + Err(e) => { + logger.warn(&format!( + "Email/import send failed ({} items): {e}", + items.len() + )); + counts.failed += items.len() as u64; + } + } +} + +fn resplit_or_fail(net: &Net, items: &[(String, Value)], counts: &mut TypeCounts, logger: &Logger) { + if items.len() <= 1 { + for (cid, _) in items { + logger.warn(&format!( + "Email/import {cid} exceeds maxSizeRequest alone; skipped" + )); + } + counts.failed += items.len() as u64; + return; + } + let mid = items.len() / 2; + send_import_chunk(net, &items[..mid], counts, logger); + send_import_chunk(net, &items[mid..], counts, logger); +} + +fn absorb_import(mr: &crate::jmap::request::MethodCall, counts: &mut TypeCounts, logger: &Logger) { + if let Some(created) = mr.args.get("created").and_then(Value::as_object) { + counts.created += created.len() as u64; + } + if let Some(nc) = mr.args.get("notCreated").and_then(Value::as_object) { + for (cid, err) in nc { + let etype = err.get("type").and_then(Value::as_str).unwrap_or(""); + if etype == "alreadyExists" { + counts.skipped += 1; + } else { + logger.warn(&format!("Email/import {cid} failed: {err}")); + counts.failed += 1; + } + } + } +} diff --git a/src/sync/export/flat.rs b/src/sync/export/flat.rs new file mode 100644 index 0000000..0723fc9 --- /dev/null +++ b/src/sync/export/flat.rs @@ -0,0 +1,141 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::params; +use serde_json::{Value, json}; + +use super::common::{create_batch, jid, target_get_all}; +use super::{Maps, Net, Plan}; +use crate::error::Error; +use crate::jmap::wire::JmapId; +use crate::logging::Logger; +use crate::sync::import_jmap::mapping::{ + ADDRESS_BOOK_SELECT, CALENDAR_SELECT, row_to_address_book, row_to_calendar, +}; +use crate::sync::keys::fold_name; +use crate::sync::prune::{TargetObj, candidates}; +use crate::sync::{Context, TypeCounts}; +use crate::types::ObjectType; + +pub fn reconcile( + ctx: &Context, + net: &Net, + ty: ObjectType, + maps: &mut Maps, + counts: &mut TypeCounts, + logger: &Logger, +) -> Result { + let table = crate::sync::table_name(ty); + let select = if ty == ObjectType::AddressBook { + ADDRESS_BOOK_SELECT + } else { + CALENDAR_SELECT + }; + + let locals: Vec<(i64, String, bool)> = { + let mut stmt = ctx + .conn + .prepare(&format!("SELECT id, name, is_default FROM {table}")) + .map_err(|e| Error::Partial(e.to_string()))?; + stmt.query_map([], |r| { + Ok((r.get(0)?, r.get::<_, String>(1)?, r.get::<_, i64>(2)? != 0)) + }) + .and_then(|m| m.collect::, _>>()) + .map_err(|e| Error::Partial(e.to_string()))? + }; + + let targets = target_get_all(net, ty).map_err(Error::from)?; + let mut tmatched = std::collections::HashSet::new(); + + let mut to_create: Vec<(i64, bool)> = Vec::new(); + for (local, name, is_default) in &locals { + let hit = targets.iter().find(|t| { + let tid = jid(t); + t.get("name") + .and_then(Value::as_str) + .map(|n| fold_name(n) == fold_name(name)) + .unwrap_or(false) + && tid.as_ref().map(|i| !tmatched.contains(i)).unwrap_or(false) + }); + match hit.and_then(jid) { + Some(tid) => { + tmatched.insert(tid.clone()); + maps.insert(ty, *local, JmapId(tid)); + counts.skipped += 1; + } + None => to_create.push((*local, *is_default)), + } + } + + if !to_create.is_empty() { + let mut batch = Vec::new(); + for (local, _) in &to_create { + let wire = ctx + .conn + .query_row(&format!("{select} WHERE id = ?1"), params![local], |row| { + Ok(if ty == ObjectType::AddressBook { + row_to_address_book(row).map(|w| serde_json::to_value(&w)) + } else { + row_to_calendar(row).map(|w| serde_json::to_value(&w)) + }) + }) + .map_err(|e| Error::Partial(e.to_string()))? + .map_err(Error::from)? + .map_err(|e| Error::Partial(e.to_string()))?; + let mut wire = wire; + if let Value::Object(m) = &mut wire { + m.remove("isDefault"); + } + batch.push((format!("c{local}"), wire)); + } + let outcome = create_batch(net, ty, batch).map_err(Error::from)?; + for (cid, v) in &outcome.created { + if let Some(local) = cid.strip_prefix('c').and_then(|s| s.parse::().ok()) + && let Some(id) = jid(v) + { + maps.insert(ty, local, JmapId(id.clone())); + counts.created += 1; + if to_create.iter().any(|(l, d)| *l == local && *d) { + let mut req = crate::jmap::request::Request::new(); + req.call( + format!("{}/set", ty.jmap_name()), + json!({ "accountId": net.account, "onSuccessSetIsDefault": id }), + "d", + ); + if let Err(e) = req.send(&net.client, &net.api) { + logger.warn(&format!("{} isDefault not set: {e}", ty.jmap_name())); + } + } + } + } + for (cid, err) in &outcome.not_created { + logger.warn(&format!("{} {cid} not created: {err}", ty.jmap_name())); + counts.failed += 1; + } + } + + let objs: Vec = targets + .iter() + .filter_map(|t| { + let id = jid(t)?; + Some(TargetObj { + id: id.clone(), + matched: tmatched.contains(&id), + protected: t.get("isDefault").and_then(Value::as_bool).unwrap_or(false), + may_delete: t + .get("myRights") + .and_then(|r| r.get("mayDelete")) + .and_then(Value::as_bool) + .unwrap_or(true), + parent: None, + }) + }) + .collect(); + Ok(Plan { + prune_candidates: candidates(&objs, false), + active_sieve_target: None, + }) +} diff --git a/src/sync/export/keyed.rs b/src/sync/export/keyed.rs new file mode 100644 index 0000000..c1e565f --- /dev/null +++ b/src/sync/export/keyed.rs @@ -0,0 +1,145 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashSet; + +use serde_json::Value; + +use super::common::{create_batch, target_get_all}; +use super::{Maps, Net, Plan}; +use crate::error::Error; +use crate::logging::Logger; +use crate::sync::import_jmap::mapping::{ + IDENTITY_SELECT, PARTICIPANT_IDENTITY_SELECT, row_to_identity, row_to_participant_identity, +}; +use crate::sync::keys::{identity_key, participant_identity_key}; +use crate::sync::{Context, TypeCounts}; +use crate::types::ObjectType; + +fn run( + net: &Net, + ty: ObjectType, + counts: &mut TypeCounts, + logger: &Logger, + target_key: impl Fn(&Value) -> Option<[u8; 32]>, + local_rows: Vec<(i64, [u8; 32], Value)>, +) -> Result { + let targets = target_get_all(net, ty).map_err(Error::from)?; + let mut target_keys: HashSet<[u8; 32]> = HashSet::new(); + for t in &targets { + if let Some(k) = target_key(t) { + target_keys.insert(k); + } + } + + let mut seen: HashSet<[u8; 32]> = HashSet::new(); + let mut batch = Vec::new(); + for (local, key, wire) in local_rows { + if target_keys.contains(&key) || !seen.insert(key) { + counts.skipped += 1; + continue; + } + batch.push((format!("c{local}"), wire)); + } + if !batch.is_empty() { + let outcome = create_batch(net, ty, batch).map_err(Error::from)?; + counts.created += outcome.created.len() as u64; + for (cid, err) in &outcome.not_created { + logger.warn(&format!( + "{} {cid} not created (expected for non-owned addresses): {err}", + ty.jmap_name() + )); + counts.skipped += 1; + } + } + Ok(Plan::default()) +} + +pub fn reconcile_identity( + ctx: &Context, + net: &Net, + _maps: &mut Maps, + counts: &mut TypeCounts, + logger: &Logger, +) -> Result { + let mut stmt = ctx + .conn + .prepare(IDENTITY_SELECT) + .map_err(|e| Error::Partial(e.to_string()))?; + let rows: Vec<(i64, [u8; 32], Value)> = stmt + .query_map([], |row| { + let id: i64 = row.get(0)?; + Ok((id, row_to_identity(row))) + }) + .and_then(|m| m.collect::, _>>()) + .map_err(|e| Error::Partial(e.to_string()))? + .into_iter() + .map(|(id, w)| { + let w = w.map_err(Error::from)?; + let key = identity_key(&w.name, &w.email); + let v = serde_json::to_value(&w).map_err(|e| Error::Partial(e.to_string()))?; + Ok((id, key, v)) + }) + .collect::>()?; + drop(stmt); + run( + net, + ObjectType::Identity, + counts, + logger, + |t| { + Some(identity_key( + t.get("name").and_then(Value::as_str).unwrap_or(""), + t.get("email").and_then(Value::as_str).unwrap_or(""), + )) + }, + rows, + ) +} + +pub fn reconcile_participant( + ctx: &Context, + net: &Net, + _maps: &mut Maps, + counts: &mut TypeCounts, + logger: &Logger, +) -> Result { + let mut stmt = ctx + .conn + .prepare(PARTICIPANT_IDENTITY_SELECT) + .map_err(|e| Error::Partial(e.to_string()))?; + let rows: Vec<(i64, [u8; 32], Value)> = stmt + .query_map([], |row| { + let id: i64 = row.get(0)?; + Ok((id, row_to_participant_identity(row))) + }) + .and_then(|m| m.collect::, _>>()) + .map_err(|e| Error::Partial(e.to_string()))? + .into_iter() + .map(|(id, w)| { + let w = w.map_err(Error::from)?; + let key = participant_identity_key(&w.calendar_address, &w.name); + let v = serde_json::to_value(&w).map_err(|e| Error::Partial(e.to_string()))?; + Ok((id, key, v)) + }) + .collect::>()?; + drop(stmt); + run( + net, + ObjectType::ParticipantIdentity, + counts, + logger, + |t| { + Some(participant_identity_key( + t.get("calendarAddress") + .and_then(Value::as_str) + .unwrap_or(""), + t.get("name").and_then(Value::as_str).unwrap_or(""), + )) + }, + rows, + ) +} diff --git a/src/sync/export/sieve.rs b/src/sync/export/sieve.rs new file mode 100644 index 0000000..1cdb59b --- /dev/null +++ b/src/sync/export/sieve.rs @@ -0,0 +1,151 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use serde_json::{Value, json}; + +use super::common::{create_batch, jid, target_get_all}; +use super::{Maps, Net, Plan, Uploader}; +use crate::db; +use crate::error::Error; +use crate::jmap::blobxfer; +use crate::jmap::request::Request; +use crate::logging::Logger; +use crate::sync::import_jmap::mapping::{SIEVE_SELECT, row_to_sieve_script}; +use crate::sync::keys::blake3_bytes; +use crate::sync::{Context, TypeCounts}; +use crate::types::ObjectType; + +pub fn reconcile( + ctx: &Context, + net: &Net, + _maps: &mut Maps, + counts: &mut TypeCounts, + logger: &Logger, +) -> Result { + let ty = ObjectType::SieveScript; + let targets = target_get_all(net, ty).map_err(Error::from)?; + + let mut target_by_key: HashMap<[u8; 32], String> = HashMap::new(); + for t in &targets { + let (Some(id), Some(blob)) = (jid(t), t.get("blobId").and_then(Value::as_str)) else { + continue; + }; + let bytes = blobxfer::download_bytes( + &net.client, + &net.session, + &net.account, + blob, + "application/sieve", + "script", + ) + .map_err(Error::from)?; + target_by_key.insert(blake3_bytes(&bytes), id); + } + + let locals: Vec<(i64, Option, bool, i64)> = { + let mut stmt = ctx + .conn + .prepare(SIEVE_SELECT) + .map_err(|e| Error::Partial(e.to_string()))?; + stmt.query_map([], |row| { + let sr = row_to_sieve_script(row); + Ok((row.get::<_, i64>(0)?, sr)) + }) + .and_then(|m| m.collect::, _>>()) + .map_err(|e| Error::Partial(e.to_string()))? + .into_iter() + .map(|(id, sr)| { + let sr = sr.map_err(Error::from)?; + Ok((id, sr.name, sr.is_active, sr.blob_local_id)) + }) + .collect::>()? + }; + + let mut active_target: Option = None; + let mut deactivate = false; + let mut uploader = Uploader::new(net, &ctx.conn); + + for (local, name, is_active, blob_local) in &locals { + let bytes = db::blobs::blob_bytes(&ctx.conn, *blob_local) + .map_err(|e| Error::Partial(e.to_string()))? + .ok_or_else(|| Error::Partial("sieve blob missing".to_owned()))?; + let key = blake3_bytes(&bytes); + let target_id = if let Some(id) = target_by_key.get(&key) { + counts.skipped += 1; + id.clone() + } else { + let blob_id = uploader + .upload_with(*blob_local, "application/sieve") + .map_err(Error::from)?; + let mut obj = serde_json::Map::new(); + if let Some(n) = name { + obj.insert("name".to_owned(), Value::String(n.clone())); + } + obj.insert("blobId".to_owned(), Value::String(blob_id.0)); + let outcome = create_batch(net, ty, vec![(format!("c{local}"), Value::Object(obj))]) + .map_err(Error::from)?; + match outcome.created.first().and_then(|(_, v)| jid(v)) { + Some(id) => { + counts.created += 1; + target_by_key.insert(key, id.clone()); + id + } + None => { + for (cid, err) in &outcome.not_created { + logger.warn(&format!("SieveScript {cid} not created: {err}")); + } + counts.failed += 1; + continue; + } + } + }; + if *is_active { + active_target = Some(target_id); + } + } + + if active_target.is_none() && locals.iter().all(|(_, _, a, _)| !*a) { + deactivate = true; + } + + if !net.dry_run { + let mut req = Request::new(); + let args = if let Some(id) = &active_target { + json!({ "accountId": net.account, "onSuccessActivateScript": id }) + } else if deactivate { + json!({ "accountId": net.account, "onSuccessDeactivateScript": true }) + } else { + json!({ "accountId": net.account }) + }; + req.call("SieveScript/set", args, "a"); + if let Err(e) = req.send(&net.client, &net.api) { + logger.warn(&format!("SieveScript activation failed: {e}")); + } + } + + let local_keys: std::collections::HashSet<[u8; 32]> = locals + .iter() + .filter_map(|(_, _, _, b)| { + db::blobs::blob_bytes(&ctx.conn, *b) + .ok() + .flatten() + .map(|by| blake3_bytes(&by)) + }) + .collect(); + let mut prune_candidates: Vec = target_by_key + .iter() + .filter(|(k, _)| !local_keys.contains(*k)) + .map(|(_, id)| id.clone()) + .collect(); + prune_candidates.sort(); + + Ok(Plan { + prune_candidates, + active_sieve_target: active_target, + }) +} diff --git a/src/sync/export/tree.rs b/src/sync/export/tree.rs new file mode 100644 index 0000000..6b5390b --- /dev/null +++ b/src/sync/export/tree.rs @@ -0,0 +1,321 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; + +use rusqlite::params; +use serde_json::Value; + +use super::common::{create_batch, jid, target_query_get}; +use super::{Net, Plan, Uploader}; +use crate::error::Error; +use crate::jmap::wire::JmapId; +use crate::logging::Logger; +use crate::sync::import_jmap::mapping::{ + FILE_NODE_SELECT, MAILBOX_SELECT, TargetResolver, row_to_file_node, row_to_mailbox, +}; +use crate::sync::keys::fold_name; +use crate::sync::{Context, TypeCounts}; +use crate::types::ObjectType; + +use super::Maps; + +struct LocalNode { + local: i64, + parent: Option, + name: String, + role: Option, +} + +struct TargetNode { + id: String, + parent: Option, + name: String, + role: Option, + may_delete: bool, +} + +fn load_local(ctx: &Context, ty: ObjectType) -> Result, Error> { + let table = crate::sync::table_name(ty); + let mut stmt = ctx + .conn + .prepare(&format!("SELECT id, parent_id, name, role FROM {table}")) + .map_err(|e| Error::Partial(e.to_string()))?; + let rows = stmt + .query_map([], |r| { + Ok(LocalNode { + local: r.get(0)?, + parent: r.get(1)?, + name: r.get(2)?, + role: r.get(3)?, + }) + }) + .and_then(|m| m.collect::, _>>()) + .map_err(|e| Error::Partial(e.to_string()))?; + Ok(rows) +} + +fn load_target(net: &Net, ty: ObjectType) -> Result, Error> { + let props: &[&str] = &["role", "name", "parentId", "myRights"]; + let list = target_query_get(net, ty, Some(props)).map_err(Error::from)?; + Ok(list + .iter() + .filter_map(|v| { + Some(TargetNode { + id: jid(v)?, + parent: v.get("parentId").and_then(Value::as_str).map(str::to_owned), + name: v.get("name").and_then(Value::as_str)?.to_owned(), + role: v.get("role").and_then(Value::as_str).map(str::to_owned), + may_delete: v + .get("myRights") + .and_then(|r| r.get("mayDelete")) + .and_then(Value::as_bool) + .unwrap_or(true), + }) + }) + .collect()) +} + +fn depth(local: i64, by: &HashMap>) -> usize { + let mut d = 0; + let mut cur = by.get(&local).copied().flatten(); + let mut seen = HashSet::new(); + while let Some(p) = cur { + if !seen.insert(p) { + break; + } + d += 1; + cur = by.get(&p).copied().flatten(); + } + d +} + +pub fn reconcile( + ctx: &Context, + net: &Net, + ty: ObjectType, + maps: &mut Maps, + counts: &mut TypeCounts, + logger: &Logger, +) -> Result { + let locals = load_local(ctx, ty)?; + let targets = load_target(net, ty)?; + + let mut matched: HashMap = HashMap::new(); + let mut tmatched: HashSet = HashSet::new(); + + if ty == ObjectType::Mailbox { + for t in &targets { + if let Some(r) = &t.role + && let Some(l) = locals.iter().find(|l| { + l.role.as_deref() == Some(r.as_str()) && !matched.contains_key(&l.local) + }) + { + matched.insert(l.local, t.id.clone()); + tmatched.insert(t.id.clone()); + } + } + } + + let mut stack: Vec<(Option, Option)> = vec![(None, None)]; + for (l, tid) in matched.clone() { + stack.push((Some(l), Some(tid))); + } + while let Some((lp, tp)) = stack.pop() { + let lchildren: Vec<&LocalNode> = locals + .iter() + .filter(|n| n.parent == lp && !matched.contains_key(&n.local)) + .collect(); + for ln in lchildren { + if let Some(tn) = targets.iter().find(|t| { + t.parent.as_deref() == tp.as_deref() + && !tmatched.contains(&t.id) + && fold_name(&t.name) == fold_name(&ln.name) + }) { + matched.insert(ln.local, tn.id.clone()); + tmatched.insert(tn.id.clone()); + stack.push((Some(ln.local), Some(tn.id.clone()))); + } + } + } + + for (l, tid) in &matched { + maps.insert(ty, *l, JmapId(tid.clone())); + } + counts.skipped += matched.len() as u64; + + let by: HashMap> = locals.iter().map(|n| (n.local, n.parent)).collect(); + let mut to_create: Vec<&LocalNode> = locals + .iter() + .filter(|n| !matched.contains_key(&n.local)) + .collect(); + to_create.sort_by_key(|n| depth(n.local, &by)); + let max_depth = to_create + .iter() + .map(|n| depth(n.local, &by)) + .max() + .unwrap_or(0); + + let mut uploader = Uploader::new(net, &ctx.conn); + let interleave = ty == ObjectType::FileNode; + for d in 0..=max_depth { + let level: Vec<&LocalNode> = to_create + .iter() + .copied() + .filter(|n| depth(n.local, &by) == d) + .collect(); + if interleave { + for n in &level { + if let Some(p) = n.parent + && maps.target(ty, p).is_none() + { + logger.warn(&format!( + "{} local {} skipped: parent {} not created", + ty.jmap_name(), + n.local, + p + )); + counts.failed += 1; + continue; + } + let obj = match build_create(ctx, ty, n.local, maps, &mut uploader) { + Ok(o) => o, + Err(e) => { + logger.warn(&format!( + "{} local {} skipped: {e}", + ty.jmap_name(), + n.local + )); + counts.failed += 1; + continue; + } + }; + let outcome = create_batch(net, ty, vec![(format!("c{}", n.local), obj)]) + .map_err(Error::from)?; + for (cid, v) in &outcome.created { + if let Some(local) = cid.strip_prefix('c').and_then(|s| s.parse::().ok()) + && let Some(id) = jid(v) + { + maps.insert(ty, local, JmapId(id)); + counts.created += 1; + } + } + for (cid, err) in &outcome.not_created { + logger.warn(&format!("{} {cid} not created: {err}", ty.jmap_name())); + counts.failed += 1; + } + } + continue; + } + let mut batch: Vec<(String, Value)> = Vec::new(); + for n in &level { + if let Some(p) = n.parent + && maps.target(ty, p).is_none() + { + logger.warn(&format!( + "{} local {} skipped: parent {} not created", + ty.jmap_name(), + n.local, + p + )); + counts.failed += 1; + continue; + } + match build_create(ctx, ty, n.local, maps, &mut uploader) { + Ok(obj) => { + batch.push((format!("c{}", n.local), obj)); + } + Err(e) => { + logger.warn(&format!( + "{} local {} skipped: {e}", + ty.jmap_name(), + n.local + )); + counts.failed += 1; + } + } + } + if batch.is_empty() { + continue; + } + let outcome = create_batch(net, ty, batch).map_err(Error::from)?; + for (cid, v) in &outcome.created { + if let Some(local) = cid.strip_prefix('c').and_then(|s| s.parse::().ok()) + && let Some(id) = jid(v) + { + maps.insert(ty, local, JmapId(id)); + counts.created += 1; + } + } + for (cid, err) in &outcome.not_created { + logger.warn(&format!("{} {cid} not created: {err}", ty.jmap_name())); + counts.failed += 1; + } + } + + let objs: Vec = targets + .iter() + .map(|t| crate::sync::prune::TargetObj { + id: t.id.clone(), + matched: tmatched.contains(&t.id), + protected: t.role.is_some(), + may_delete: t.may_delete, + parent: t.parent.clone(), + }) + .collect(); + Ok(Plan { + prune_candidates: crate::sync::prune::candidates(&objs, true), + active_sieve_target: None, + }) +} + +fn build_create( + ctx: &Context, + ty: ObjectType, + local: i64, + maps: &Maps, + uploader: &mut Uploader<'_>, +) -> Result { + if ty == ObjectType::Mailbox { + let wire = ctx + .conn + .query_row( + &format!("{MAILBOX_SELECT} WHERE id = ?1"), + params![local], + |row| Ok(row_to_mailbox(row, maps)), + ) + .map_err(|e| Error::Partial(e.to_string()))? + .map_err(Error::from)?; + return serde_json::to_value(&wire).map_err(|e| Error::Partial(e.to_string())); + } + let fnrow = ctx + .conn + .query_row( + &format!("{FILE_NODE_SELECT} WHERE id = ?1"), + params![local], + |row| Ok(row_to_file_node(row, maps)), + ) + .map_err(|e| Error::Partial(e.to_string()))? + .map_err(Error::from)?; + let mut wire = fnrow.wire; + if let (crate::jmap::wire::file_node::NodeType::File, Some(blob_local)) = + (&wire.node_type, fnrow.blob_local_id) + { + let ct = wire + .media_type + .clone() + .unwrap_or_else(|| "application/octet-stream".to_owned()); + let id = uploader.upload_with(blob_local, &ct).map_err(Error::from)?; + wire.blob_id = Some(id); + } + let mut obj = serde_json::to_value(&wire).map_err(|e| Error::Partial(e.to_string()))?; + if let Value::Object(m) = &mut obj { + m.remove("created"); + m.remove("modified"); + m.remove("isSubscribed"); + } + Ok(obj) +} diff --git a/src/sync/export/uidtype.rs b/src/sync/export/uidtype.rs new file mode 100644 index 0000000..61734c2 --- /dev/null +++ b/src/sync/export/uidtype.rs @@ -0,0 +1,149 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashSet; + +use serde_json::Value; + +use super::common::{create_batch, jid, target_query_get}; +use super::{Maps, Net, Plan, Uploader}; +use crate::error::Error; +use crate::logging::Logger; +use crate::sync::import_jmap::mapping::{ + CALENDAR_EVENT_SELECT, CONTACT_CARD_SELECT, calendar_event_to_wire, contact_card_to_wire, +}; +use crate::sync::prune::{TargetObj, candidates}; +use crate::sync::{Context, TypeCounts}; +use crate::types::ObjectType; + +fn target_uid(v: &Value) -> Option { + v.get("uid").and_then(Value::as_str).map(str::to_owned) +} + +pub fn reconcile( + ctx: &Context, + net: &Net, + ty: ObjectType, + maps: &mut Maps, + counts: &mut TypeCounts, + logger: &Logger, +) -> Result { + let targets = target_query_get(net, ty, None).map_err(Error::from)?; + let mut by_uid: std::collections::HashMap = std::collections::HashMap::new(); + for t in &targets { + if let (Some(uid), Some(id)) = (target_uid(t), jid(t)) { + by_uid.entry(uid).or_insert(id); + } + } + + let select = if ty == ObjectType::ContactCard { + CONTACT_CARD_SELECT + } else { + CALENDAR_EVENT_SELECT + }; + let rows: Vec<(i64, String)> = { + let mut stmt = ctx + .conn + .prepare(select) + .map_err(|e| Error::Partial(e.to_string()))?; + stmt.query_map([], |r| { + if ty == ObjectType::ContactCard { + Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)) + } else { + let data: String = r.get(4)?; + let uid = serde_json::from_str::(&data) + .ok() + .and_then(|v| v.get("uid").and_then(Value::as_str).map(str::to_owned)) + .unwrap_or_default(); + Ok((r.get::<_, i64>(0)?, uid)) + } + }) + .and_then(|m| m.collect::, _>>()) + .map_err(|e| Error::Partial(e.to_string()))? + }; + + let mut matched_uids: HashSet = HashSet::new(); + let mut uploader = Uploader::new(net, &ctx.conn); + for (local, uid) in &rows { + if let Some(tid) = by_uid.get(uid) { + maps.insert(ty, *local, crate::jmap::wire::JmapId(tid.clone())); + matched_uids.insert(uid.clone()); + counts.skipped += 1; + continue; + } + let wire = match build_wire(ctx, ty, *local, maps, &mut uploader) { + Ok(w) => w, + Err(e) => { + logger.warn(&format!("{} local {local} skipped: {e}", ty.jmap_name())); + counts.failed += 1; + continue; + } + }; + let outcome = + create_batch(net, ty, vec![(format!("c{local}"), wire)]).map_err(Error::from)?; + for (cid, v) in &outcome.created { + if let Some(parsed) = cid.strip_prefix('c').and_then(|s| s.parse::().ok()) + && let Some(id) = jid(v) + { + maps.insert(ty, parsed, crate::jmap::wire::JmapId(id)); + counts.created += 1; + } + } + for (cid, err) in &outcome.not_created { + logger.warn(&format!("{} {cid} not created: {err}", ty.jmap_name())); + counts.failed += 1; + } + } + + let objs: Vec = targets + .iter() + .filter_map(|t| { + let id = jid(t)?; + let uid = target_uid(t); + Some(TargetObj { + id: id.clone(), + matched: uid.map(|u| matched_uids.contains(&u)).unwrap_or(false), + protected: false, + may_delete: true, + parent: None, + }) + }) + .collect(); + Ok(Plan { + prune_candidates: candidates(&objs, false), + active_sieve_target: None, + }) +} + +fn build_wire( + ctx: &Context, + ty: ObjectType, + local: i64, + maps: &Maps, + up: &mut Uploader<'_>, +) -> Result { + if ty == ObjectType::ContactCard { + let (uid, abids, data): (String, String, String) = ctx + .conn + .query_row( + &format!("{CONTACT_CARD_SELECT} WHERE id = ?1"), + rusqlite::params![local], + |r| Ok((r.get(1)?, r.get(2)?, r.get(3)?)), + ) + .map_err(|e| Error::Partial(e.to_string()))?; + contact_card_to_wire(&uid, &abids, &data, maps, up).map_err(Error::from) + } else { + let (cal, dr, ud, data): (String, i64, i64, String) = ctx + .conn + .query_row( + &format!("{CALENDAR_EVENT_SELECT} AND id = ?1"), + rusqlite::params![local], + |r| Ok((r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)), + ) + .map_err(|e| Error::Partial(e.to_string()))?; + calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, maps, up).map_err(Error::from) + } +} diff --git a/src/sync/import_dav.rs b/src/sync/import_dav.rs new file mode 100644 index 0000000..2b434fb --- /dev/null +++ b/src/sync/import_dav.rs @@ -0,0 +1,13 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod calcard; +pub mod collections; +pub mod coordinator; +pub mod items; +pub mod tree; + +pub use coordinator::{DavAuth, DavImportConfig, DavKindArg, run}; diff --git a/src/sync/import_dav/calcard.rs b/src/sync/import_dav/calcard.rs new file mode 100644 index 0000000..fbf9c3e --- /dev/null +++ b/src/sync/import_dav/calcard.rs @@ -0,0 +1,281 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use calcard::icalendar::ICalendar; +use calcard::vcard::VCard; +use serde_json::{Map, Value}; + +#[derive(Debug, thiserror::Error)] +pub enum CalcardError { + #[error("vcard parse failed")] + VCardParse, + #[error("icalendar parse failed")] + ICalParse, + #[error("jscontact serialisation: {0}")] + Json(#[from] serde_json::Error), + #[error("no parseable entries in iCalendar")] + NoEntries, + #[error("calcard output is not a JSON object")] + NotAnObject, +} + +#[derive(Debug)] +pub struct JsContact { + pub uid: String, + pub data: Value, +} + +pub fn vcard_to_jscontact(text: &str, item_href: &str) -> Result { + let vcard = VCard::parse(text).map_err(|_| CalcardError::VCardParse)?; + let js = vcard.into_jscontact::(); + let value: Value = serde_json::from_str(&js.to_string_pretty())?; + let uid = value + .get("uid") + .and_then(Value::as_str) + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_owned) + .unwrap_or_else(|| synthesise_uid(item_href)); + let mut obj = match value { + Value::Object(map) => map, + _ => return Err(CalcardError::NotAnObject), + }; + obj.remove("uid"); + Ok(JsContact { + uid, + data: Value::Object(obj), + }) +} + +pub fn synthesise_uid(item_href: &str) -> String { + let hash = blake3::hash(item_href.as_bytes()); + format!("vandelay-syn-{}", hash.to_hex()) +} + +#[derive(Debug)] +pub struct JsCalendarEntry { + pub data: Value, + pub data_type: EntryType, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EntryType { + Event, + Task, + Note, + Group, +} + +impl EntryType { + pub fn as_column(self) -> &'static str { + match self { + EntryType::Event => "Event", + EntryType::Task => "Task", + EntryType::Note => "Note", + EntryType::Group => "Group", + } + } +} + +pub fn ical_to_jscalendar_entries(text: &str) -> Result, CalcardError> { + let ical = ICalendar::parse(text).map_err(|_| CalcardError::ICalParse)?; + let js = ical.into_jscalendar::(); + let value: Value = serde_json::from_str(&js.to_string_pretty())?; + let mut out = Vec::new(); + collect_entries(value, &mut out); + if out.is_empty() { + return Err(CalcardError::NoEntries); + } + Ok(out) +} + +fn collect_entries(value: Value, out: &mut Vec) { + match value { + Value::Object(map) => { + let at_type = map.get("@type").and_then(Value::as_str).map(str::to_owned); + match at_type.as_deref() { + Some("Event") if has_nonempty_uid(&map) => out.push(JsCalendarEntry { + data: Value::Object(map), + data_type: EntryType::Event, + }), + Some("Task") if has_nonempty_uid(&map) => out.push(JsCalendarEntry { + data: Value::Object(map), + data_type: EntryType::Task, + }), + Some("Note") if has_nonempty_uid(&map) => out.push(JsCalendarEntry { + data: Value::Object(map), + data_type: EntryType::Note, + }), + Some("Group") => { + if let Some(entries) = map.get("entries").and_then(Value::as_array) { + for entry in entries { + collect_entries(entry.clone(), out); + } + } + } + _ => {} + } + } + Value::Array(items) => { + for item in items { + collect_entries(item, out); + } + } + _ => {} + } +} + +fn has_nonempty_uid(map: &Map) -> bool { + map.get("uid") + .and_then(Value::as_str) + .map(|u| !u.trim().is_empty()) + .unwrap_or(false) +} + +pub fn strip_extracted_fields_from_event(value: &mut Value) -> (bool, bool, Option) { + let Value::Object(map) = value else { + return (false, false, None); + }; + let is_draft = map + .remove("isDraft") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + let use_default_alerts = map + .remove("useDefaultAlerts") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + map.remove("calendarIds"); + map.remove("method"); + map.remove("utcStart"); + map.remove("utcEnd"); + map.remove("isOrigin"); + map.remove("baseEventId"); + let uid = map.get("uid").and_then(Value::as_str).map(str::to_owned); + (is_draft, use_default_alerts, uid) +} + +#[cfg(test)] +mod tests { + use super::*; + + const SAMPLE_VCARD: &str = "BEGIN:VCARD\r\n\ +VERSION:4.0\r\n\ +UID:abc-123\r\n\ +FN:Alice Smith\r\n\ +EMAIL:alice@example.com\r\n\ +END:VCARD\r\n"; + + const SAMPLE_ICAL: &str = "BEGIN:VCALENDAR\r\n\ +VERSION:2.0\r\n\ +PRODID:-//Test//EN\r\n\ +BEGIN:VEVENT\r\n\ +UID:event-1@example.com\r\n\ +DTSTAMP:20260101T000000Z\r\n\ +DTSTART:20260101T090000Z\r\n\ +DTEND:20260101T100000Z\r\n\ +SUMMARY:Test Event\r\n\ +END:VEVENT\r\n\ +END:VCALENDAR\r\n"; + + #[test] + fn vcard_extracts_uid_into_separate_field() { + let jc = vcard_to_jscontact(SAMPLE_VCARD, "/dav/card/u/d/a.vcf").expect("parse"); + assert_eq!(jc.uid, "abc-123"); + let obj = jc.data.as_object().unwrap(); + assert!(!obj.contains_key("uid"), "uid stripped from data"); + } + + #[test] + fn ical_extracts_at_least_one_event() { + let events = ical_to_jscalendar_entries(SAMPLE_ICAL).expect("parse"); + assert!(!events.is_empty()); + assert_eq!(events[0].data_type, EntryType::Event); + let uid = events[0] + .data + .as_object() + .and_then(|m| m.get("uid")) + .and_then(Value::as_str) + .unwrap(); + assert!(uid.starts_with("event-1")); + } + + #[test] + fn vcard_without_uid_gets_synthetic_uid_from_href() { + let no_uid = "BEGIN:VCARD\r\nVERSION:4.0\r\nFN:Anon\r\nEND:VCARD\r\n"; + let href = "/dav/card/u/d/no-uid.vcf"; + let jc = vcard_to_jscontact(no_uid, href).expect("synthetic uid"); + assert_eq!(jc.uid, synthesise_uid(href)); + assert!(jc.uid.starts_with("vandelay-syn-")); + } + + #[test] + fn synthetic_uid_is_stable_per_href() { + assert_eq!(synthesise_uid("/a"), synthesise_uid("/a")); + assert_ne!(synthesise_uid("/a"), synthesise_uid("/b")); + } + + #[test] + fn ical_with_empty_calendar_fails_no_entries() { + let empty = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nEND:VCALENDAR\r\n"; + let err = ical_to_jscalendar_entries(empty).unwrap_err(); + assert!(matches!(err, CalcardError::NoEntries)); + } + + #[test] + fn vcard_garbage_fails_parse() { + let err = vcard_to_jscontact("not a vcard", "/x").unwrap_err(); + assert!(matches!(err, CalcardError::VCardParse)); + } + + #[test] + fn ical_vtodo_classified_as_task() { + let vtodo = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\n\ + BEGIN:VTODO\r\nUID:t1@example\r\nDTSTAMP:20260101T000000Z\r\n\ + SUMMARY:Buy milk\r\nEND:VTODO\r\nEND:VCALENDAR\r\n"; + let entries = ical_to_jscalendar_entries(vtodo).expect("parse vtodo"); + assert_eq!(entries[0].data_type, EntryType::Task); + } + + #[test] + fn ical_recurrence_overrides_collapse_to_single_event() { + let ical = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\n\ + BEGIN:VEVENT\r\nUID:r1@example\r\nDTSTAMP:20260101T000000Z\r\n\ + DTSTART:20260101T090000Z\r\nDTEND:20260101T100000Z\r\n\ + RRULE:FREQ=DAILY;COUNT=3\r\nSUMMARY:Standup\r\nEND:VEVENT\r\n\ + BEGIN:VEVENT\r\nUID:r1@example\r\nDTSTAMP:20260101T000000Z\r\n\ + RECURRENCE-ID:20260102T090000Z\r\nDTSTART:20260102T093000Z\r\nDTEND:20260102T103000Z\r\n\ + SUMMARY:Standup (moved)\r\nEND:VEVENT\r\n\ + END:VCALENDAR\r\n"; + let entries = ical_to_jscalendar_entries(ical).expect("parse"); + assert_eq!(entries.len(), 1, "calcard merges overrides into master"); + let master = entries[0].data.as_object().unwrap(); + assert!(master.contains_key("recurrenceOverrides")); + } + + #[test] + fn strip_extracted_fields_pulls_isdraft_use_default_alerts() { + let mut v = serde_json::json!({ + "@type": "Event", + "uid": "x", + "title": "T", + "isDraft": true, + "useDefaultAlerts": true, + "calendarIds": {"a": true}, + "utcStart": "2026-01-01T09:00:00Z", + "method": "PUBLISH" + }); + let (d, u, uid) = strip_extracted_fields_from_event(&mut v); + assert!(d); + assert!(u); + assert_eq!(uid.as_deref(), Some("x")); + let m = v.as_object().unwrap(); + assert!(!m.contains_key("calendarIds")); + assert!(!m.contains_key("isDraft")); + assert!(!m.contains_key("useDefaultAlerts")); + assert!(!m.contains_key("method")); + assert!(!m.contains_key("utcStart")); + } +} diff --git a/src/sync/import_dav/collections.rs b/src/sync/import_dav/collections.rs new file mode 100644 index 0000000..4261308 --- /dev/null +++ b/src/sync/import_dav/collections.rs @@ -0,0 +1,416 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use rusqlite::{Connection, params}; + +use crate::dav::discover::DiscoveredCollection; +use crate::dav::href::{Href, last_path_component}; +use crate::db::dav_ids; +use crate::error::Error; +use crate::logging::{LEVEL_PROGRESS, Logger}; +use crate::sync::TypeCounts; + +pub fn reconcile_calendars( + conn: &mut Connection, + source_id: i64, + discovered: &[DiscoveredCollection], + counts: &mut TypeCounts, + logger: Logger, +) -> Result, Error> { + reconcile_calendar_like( + conn, + source_id, + discovered, + &RowOps { + type_name: dav_ids::CALENDAR, + upsert: upsert_calendar_row, + delete: delete_calendar_row, + }, + counts, + logger, + ) +} + +pub fn reconcile_address_books( + conn: &mut Connection, + source_id: i64, + discovered: &[DiscoveredCollection], + counts: &mut TypeCounts, + logger: Logger, +) -> Result, Error> { + reconcile_calendar_like( + conn, + source_id, + discovered, + &RowOps { + type_name: dav_ids::ADDRESS_BOOK, + upsert: upsert_address_book_row, + delete: delete_address_book_row, + }, + counts, + logger, + ) +} + +type RowUpsert = + fn(&Connection, &DiscoveredCollection, Option) -> Result; +type RowDelete = fn(&Connection, i64) -> Result<(), rusqlite::Error>; + +struct RowOps { + type_name: &'static str, + upsert: RowUpsert, + delete: RowDelete, +} + +fn reconcile_calendar_like( + conn: &mut Connection, + source_id: i64, + discovered: &[DiscoveredCollection], + ops: &RowOps, + counts: &mut TypeCounts, + logger: Logger, +) -> Result, Error> { + let type_name = ops.type_name; + let upsert_row = ops.upsert; + let delete_row = ops.delete; + let existing: HashMap = dav_ids::collections_of_type(conn, source_id, type_name) + .map_err(|e| Error::Partial(e.to_string()))?; + + let mut upserted: Vec<(String, i64)> = Vec::new(); + let server_hrefs: std::collections::HashSet = discovered + .iter() + .map(|c| c.href.as_str().to_owned()) + .collect(); + + for coll in discovered { + let collection_href = coll.href.as_str().to_owned(); + let existing_local = existing.get(&collection_href).copied(); + let tx = conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + match upsert_row(&tx, coll, existing_local) { + Ok(local_id) => { + if existing_local.is_none() { + dav_ids::insert( + &tx, + source_id, + type_name, + &collection_href, + &collection_href, + "", + local_id, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + counts.created += 1; + } else { + counts.fetched += 1; + } + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + upserted.push((collection_href.clone(), local_id)); + if logger.enabled(LEVEL_PROGRESS) { + eprintln!("collection upserted: {collection_href} -> {local_id}"); + } + } + Err(e) => { + let _ = tx.rollback(); + logger.warn(&format!( + "collection upsert {collection_href:?} failed: {e}" + )); + counts.failed += 1; + } + } + } + + for (collection_href, local_id) in &existing { + if server_hrefs.contains(collection_href) { + continue; + } + let tx = conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + let item_type = item_type_for(type_name); + if let Some(item_type) = item_type { + let _ = dav_ids::delete_collection(&tx, source_id, item_type, collection_href); + } + let _ = dav_ids::delete_collection(&tx, source_id, type_name, collection_href); + match delete_row(&tx, *local_id) { + Ok(()) => { + counts.deleted += 1; + } + Err(e) => { + logger.warn(&format!( + "vanished collection {collection_href:?} delete failed: {e}" + )); + counts.failed += 1; + } + } + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + } + + Ok(upserted) +} + +fn item_type_for(collection_type: &str) -> Option<&'static str> { + match collection_type { + dav_ids::CALENDAR => Some(dav_ids::CALENDAR_EVENT), + dav_ids::ADDRESS_BOOK => Some(dav_ids::CONTACT_CARD), + _ => None, + } +} + +fn upsert_calendar_row( + conn: &Connection, + coll: &DiscoveredCollection, + existing_local: Option, +) -> Result { + let name = display_or_fallback(&coll.props.displayname, &coll.href); + let description = coll.props.calendar_description.as_deref(); + let color = coll.props.calendar_color.as_deref(); + let sort_order = coll.props.calendar_order.unwrap_or(0).max(0); + let time_zone = parse_tzid_from_vtimezone(coll.props.calendar_timezone.as_deref()); + + if let Some(local) = existing_local { + conn.execute( + "UPDATE calendars SET name = ?1, description = ?2, color = ?3, sort_order = ?4, + time_zone = ?5 + WHERE id = ?6", + params![name, description, color, sort_order, time_zone, local], + )?; + Ok(local) + } else { + conn.execute( + "INSERT INTO calendars (name, description, color, sort_order, is_subscribed, + is_visible, is_default, include_in_availability, + default_alerts_with_time, default_alerts_without_time, time_zone) + VALUES (?1, ?2, ?3, ?4, 1, 1, 0, 'all', NULL, NULL, ?5)", + params![name, description, color, sort_order, time_zone], + )?; + Ok(conn.last_insert_rowid()) + } +} + +fn delete_calendar_row(conn: &Connection, local_id: i64) -> Result<(), rusqlite::Error> { + conn.execute( + "DELETE FROM calendar_events + WHERE EXISTS ( + SELECT 1 FROM json_each(calendar_ids) WHERE value = ?1 + )", + params![local_id], + )?; + conn.execute("DELETE FROM calendars WHERE id = ?1", params![local_id])?; + Ok(()) +} + +fn upsert_address_book_row( + conn: &Connection, + coll: &DiscoveredCollection, + existing_local: Option, +) -> Result { + let name = display_or_fallback(&coll.props.displayname, &coll.href); + let description = coll.props.addressbook_description.as_deref(); + + if let Some(local) = existing_local { + conn.execute( + "UPDATE address_books SET name = ?1, description = ?2 WHERE id = ?3", + params![name, description, local], + )?; + Ok(local) + } else { + conn.execute( + "INSERT INTO address_books (name, description, sort_order, is_default, is_subscribed) + VALUES (?1, ?2, 0, 0, 1)", + params![name, description], + )?; + Ok(conn.last_insert_rowid()) + } +} + +fn delete_address_book_row(conn: &Connection, local_id: i64) -> Result<(), rusqlite::Error> { + conn.execute( + "DELETE FROM contact_cards + WHERE EXISTS ( + SELECT 1 FROM json_each(address_book_ids) WHERE value = ?1 + )", + params![local_id], + )?; + conn.execute("DELETE FROM address_books WHERE id = ?1", params![local_id])?; + Ok(()) +} + +fn display_or_fallback(name: &Option, href: &Href) -> String { + if let Some(n) = name + && !n.trim().is_empty() + { + return n.clone(); + } + let comp = last_path_component(href); + if comp.is_empty() { + "Untitled".to_owned() + } else { + comp + } +} + +fn parse_tzid_from_vtimezone(vt: Option<&str>) -> Option { + let body = vt?; + for line in body.lines() { + let line = line.trim(); + if let Some(rest) = line.strip_prefix("TZID:") { + let v = rest.trim(); + if !v.is_empty() { + return Some(v.to_owned()); + } + } + } + None +} + +pub fn upsert_root_directory( + conn: &Connection, + source_id: i64, + root: &DiscoveredCollection, +) -> Result { + let collection_href = root.href.as_str().to_owned(); + let name = display_or_fallback(&root.props.displayname, &root.href); + if let Some(local) = + dav_ids::local_for_item(conn, source_id, dav_ids::FILE_NODE, &collection_href) + .map_err(|e| Error::Partial(e.to_string()))? + { + conn.execute( + "UPDATE file_nodes SET name = ?1 WHERE id = ?2", + params![name, local], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + return Ok(local); + } + let now = time::OffsetDateTime::now_utc() + .format(&time::format_description::well_known::Rfc3339) + .map_err(|e| Error::Partial(format!("clock: {e}")))?; + conn.execute( + "INSERT INTO file_nodes (parent_id, node_type, blob_id, target, name, media_type, + created, modified, is_subscribed, role) + VALUES (NULL, 'directory', NULL, NULL, ?1, NULL, ?2, NULL, 1, NULL)", + params![name, now], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + let local_id = conn.last_insert_rowid(); + dav_ids::insert( + conn, + source_id, + dav_ids::FILE_NODE, + &collection_href, + &collection_href, + "", + local_id, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + Ok(local_id) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_tzid_from_vtimezone_extracts_first() { + let v = "BEGIN:VTIMEZONE\r\nTZID:Europe/Berlin\r\nEND:VTIMEZONE\r\n"; + assert_eq!( + parse_tzid_from_vtimezone(Some(v)).as_deref(), + Some("Europe/Berlin") + ); + } + + #[test] + fn parse_tzid_returns_none_when_absent() { + assert!(parse_tzid_from_vtimezone(Some("nothing here")).is_none()); + assert!(parse_tzid_from_vtimezone(None).is_none()); + } + + #[test] + fn display_or_fallback_uses_href_when_displayname_empty() { + let href = Href::from_normalised("/dav/cal/u/work/".to_owned()); + let name: Option = None; + assert_eq!(display_or_fallback(&name, &href), "work"); + } + + #[test] + fn display_or_fallback_uses_displayname_when_present() { + let href = Href::from_normalised("/dav/cal/u/default/".to_owned()); + let name = Some("Default".to_owned()); + assert_eq!(display_or_fallback(&name, &href), "Default"); + } + + #[test] + fn delete_calendar_row_cascades_events_and_sync_id_rows() { + use crate::db::init; + use crate::db::sources::{SourceKey, upsert_source}; + let conn = rusqlite::Connection::open_in_memory().unwrap(); + init::apply_schema(&conn).unwrap(); + let sid = upsert_source( + &conn, + &SourceKey { + kind: "caldav".to_owned(), + session_url: "https://x".to_owned(), + account_id: "u".to_owned(), + }, + Some("u"), + "u", + ) + .unwrap(); + conn.execute( + "INSERT INTO calendars (name, sort_order, is_subscribed, is_visible, is_default, + include_in_availability) + VALUES ('Work', 0, 1, 1, 0, 'all')", + [], + ) + .unwrap(); + let cal_local = conn.last_insert_rowid(); + conn.execute( + "INSERT INTO calendar_events (calendar_ids, data, data_type) + VALUES (?1, '{}', 'Event')", + rusqlite::params![format!("[{cal_local}]")], + ) + .unwrap(); + let evt_local = conn.last_insert_rowid(); + dav_ids::insert( + &conn, + sid, + dav_ids::CALENDAR, + "/cal/work/", + "/cal/work/", + "", + cal_local, + ) + .unwrap(); + dav_ids::insert( + &conn, + sid, + dav_ids::CALENDAR_EVENT, + "/cal/work/", + "/cal/work/e.ics", + "\"v\"", + evt_local, + ) + .unwrap(); + let _ = dav_ids::delete_collection(&conn, sid, dav_ids::CALENDAR_EVENT, "/cal/work/"); + let _ = dav_ids::delete_collection(&conn, sid, dav_ids::CALENDAR, "/cal/work/"); + delete_calendar_row(&conn, cal_local).unwrap(); + let cal_count: i64 = conn + .query_row("SELECT count(*) FROM calendars", [], |r| r.get(0)) + .unwrap(); + let evt_count: i64 = conn + .query_row("SELECT count(*) FROM calendar_events", [], |r| r.get(0)) + .unwrap(); + let dav_count: i64 = conn + .query_row("SELECT count(*) FROM sync_id_dav", [], |r| r.get(0)) + .unwrap(); + assert_eq!(cal_count, 0, "calendar row gone"); + assert_eq!(evt_count, 0, "event rows cascaded"); + assert_eq!(dav_count, 0, "sync_id_dav rows for both types cleared"); + } +} diff --git a/src/sync/import_dav/coordinator.rs b/src/sync/import_dav/coordinator.rs new file mode 100644 index 0000000..bd37f20 --- /dev/null +++ b/src/sync/import_dav/coordinator.rs @@ -0,0 +1,520 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::Connection; + +use crate::dav::client::DavClient; +use crate::dav::discover::{DavKind, Discovery, DiscoveryError, discover}; +use crate::db; +use crate::db::sources::SourceKey; +use crate::error::Error; +use crate::jmap::http::{Auth, RetryPolicy}; +use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; +use crate::sync::{CommonConfig, Summary, TypeCounts}; + +use super::collections; +use super::items; +use super::tree; + +#[derive(Debug, Clone, Copy)] +pub enum DavKindArg { + Caldav, + Carddav, + Webdav, +} + +impl DavKindArg { + pub fn kind(self) -> DavKind { + match self { + DavKindArg::Caldav => DavKind::Caldav, + DavKindArg::Carddav => DavKind::Carddav, + DavKindArg::Webdav => DavKind::Webdav, + } + } + + pub fn source_kind(self) -> &'static str { + match self { + DavKindArg::Caldav => "caldav", + DavKindArg::Carddav => "carddav", + DavKindArg::Webdav => "webdav", + } + } +} + +#[derive(Debug, Clone)] +pub enum DavAuth { + Basic { user: String, password: String }, + Bearer { token: String }, +} + +impl DavAuth { + pub fn to_jmap_auth(&self) -> Auth { + match self { + DavAuth::Basic { user, password } => Auth::Basic { + user: user.clone(), + password: password.clone(), + }, + DavAuth::Bearer { token } => Auth::Bearer { + token: token.clone(), + }, + } + } + + pub fn username(&self) -> String { + match self { + DavAuth::Basic { user, .. } => user.clone(), + DavAuth::Bearer { .. } => "(bearer)".to_owned(), + } + } +} + +#[derive(Debug)] +pub struct DavImportConfig { + pub kind: DavKindArg, + pub url: String, + pub auth: DavAuth, + pub allow_cleartext: bool, + pub dav_connections: usize, + pub multiget_batch: usize, + pub allow_source_change: bool, +} + +pub fn run(common: CommonConfig, config: DavImportConfig) -> Result { + let logger = common.logger; + enforce_tls_policy(&config.url, config.allow_cleartext)?; + + let mut conn = db::init::open(&common.archive)?; + + let client = DavClient::new( + config.auth.to_jmap_auth(), + RetryPolicy::new(common.max_retries), + common.allow_invalid_certs, + ); + client.set_logger(logger); + + let discovery = + discover(&client, config.kind.kind(), &config.url).map_err(map_discovery_error)?; + + let session_url = normalise_base_url(&config.url); + let account_id_raw = discovery + .principal_url + .clone() + .unwrap_or_else(|| discovery.home_set_url.clone()); + let account_id = normalise_account_url(&account_id_raw); + + let key = SourceKey { + kind: config.kind.source_kind().to_owned(), + session_url: session_url.clone(), + account_id: account_id.clone(), + }; + + if !common.dry_run + && let Some((url, acc)) = db::sources::conflicting_source( + &conn, + config.kind.source_kind(), + &session_url, + &account_id, + ) + .map_err(|e| Error::Partial(e.to_string()))? + && !config.allow_source_change + { + return Err(Error::SourceChange(format!( + "archive already records {} source ({url}, account {acc}); \ + pass --allow-source-change to import a different account", + config.kind.source_kind() + ))); + } + + if logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "DAV discovery: principal={:?} home_set={} collections={}", + discovery.principal_url, + discovery.home_set_url, + discovery.collections.len() + ); + } + + if common.dry_run { + let mut summary = run_dry_diff(&conn, &client, &discovery, &config, logger)?; + summary.retries_observed = client.retries_observed(); + summary.retry_after_sleeps = client.retry_after_sleeps(); + return Ok(summary); + } + + let username = config.auth.username(); + let source_id = db::sources::upsert_source(&conn, &key, Some(&account_id), &username) + .map_err(|e| Error::Partial(e.to_string()))?; + + let summary = match config.kind { + DavKindArg::Caldav => { + run_caldav(&mut conn, &client, source_id, &discovery, &config, logger)? + } + DavKindArg::Carddav => { + run_carddav(&mut conn, &client, source_id, &discovery, &config, logger)? + } + DavKindArg::Webdav => { + run_webdav(&mut conn, &client, source_id, &discovery, &config, logger)? + } + }; + + if !summary.any_failed() + && let Err(e) = run_gc(&conn) + { + logger.warn(&format!("blob GC skipped: {e}")); + } + + let mut summary = summary; + summary.retries_observed = client.retries_observed(); + summary.retry_after_sleeps = client.retry_after_sleeps(); + Ok(summary) +} + +type ReconcileCollections = fn( + &mut Connection, + i64, + &[crate::dav::discover::DiscoveredCollection], + &mut TypeCounts, + Logger, +) -> Result, Error>; + +type ReconcileItems = + fn(&mut Connection, &items::ItemRunCtx<'_>, &str, i64, &mut TypeCounts) -> Result<(), Error>; + +struct ItemPhase { + container_label: &'static str, + item_label: &'static str, + reconcile_collections: ReconcileCollections, + reconcile_items: ReconcileItems, +} + +fn run_collection_phase( + conn: &mut Connection, + client: &DavClient, + source_id: i64, + discovery: &Discovery, + config: &DavImportConfig, + logger: Logger, + phase: ItemPhase, +) -> Result { + let mut summary = Summary::default(); + let mut container_counts = TypeCounts::default(); + let mut item_counts = TypeCounts::default(); + + let upserted = (phase.reconcile_collections)( + conn, + source_id, + &discovery.collections, + &mut container_counts, + logger, + )?; + if logger.enabled(LEVEL_DEFAULT) { + eprintln!( + "import: {} done (upserted={} deleted={} failed={})", + phase.container_label, + container_counts.created + container_counts.fetched, + container_counts.deleted, + container_counts.failed + ); + } + + let ctx = items::ItemRunCtx { + client, + source_id, + base_url: &discovery.home_set_url, + multiget_batch: config.multiget_batch, + dav_connections: config.dav_connections, + logger, + }; + for (collection_href, local_id) in &upserted { + match (phase.reconcile_items)(conn, &ctx, collection_href, *local_id, &mut item_counts) { + Ok(()) => {} + Err(e) => { + logger.warn(&format!( + "{} {collection_href:?}: items failed: {e}", + phase.container_label + )); + item_counts.failed += 1; + } + } + } + + summary + .per_type + .push((phase.container_label, container_counts)); + summary.per_type.push((phase.item_label, item_counts)); + Ok(summary) +} + +fn run_dry_diff( + conn: &rusqlite::Connection, + client: &DavClient, + discovery: &Discovery, + config: &DavImportConfig, + logger: Logger, +) -> Result { + use crate::dav::href::join_absolute; + use crate::dav::xml; + use crate::db::dav_ids; + let mut summary = Summary::default(); + let (container_label, item_label, container_type, item_type) = match config.kind { + DavKindArg::Caldav => ( + "calendar", + "calendarevent", + dav_ids::CALENDAR, + dav_ids::CALENDAR_EVENT, + ), + DavKindArg::Carddav => ( + "addressbook", + "contactcard", + dav_ids::ADDRESS_BOOK, + dav_ids::CONTACT_CARD, + ), + DavKindArg::Webdav => ( + "filenode", + "filenode", + dav_ids::FILE_NODE, + dav_ids::FILE_NODE, + ), + }; + + let mut container_counts = TypeCounts::default(); + let mut item_counts = TypeCounts::default(); + container_counts.created = discovery.collections.len() as u64; + if logger.enabled(LEVEL_DEFAULT) { + eprintln!( + "dry-run: {} new={} (no archive to diff against)", + container_label, + discovery.collections.len(), + ); + } + + if !matches!(config.kind, DavKindArg::Webdav) { + for coll in &discovery.collections { + let url = join_absolute(&discovery.home_set_url, coll.href.as_str()) + .map_err(|e| Error::Partial(e.to_string()))?; + let ms = client + .propfind_responses(&url, 1, &xml::propfind_dav_items(), &url) + .map_err(Error::from)?; + let new_count = ms + .responses + .iter() + .filter(|r| !r.props.is_collection) + .count(); + item_counts.created += new_count as u64; + if logger.enabled(LEVEL_DEFAULT) { + eprintln!(" {} items: {new_count}", coll.href.as_str()); + } + } + } else if let Some(root) = discovery.collections.first() { + let url = join_absolute(&discovery.home_set_url, root.href.as_str()) + .map_err(|e| Error::Partial(e.to_string()))?; + let ms = client + .propfind_responses(&url, 1, &xml::propfind_webdav_listing(), &url) + .map_err(Error::from)?; + let new_count = ms + .responses + .iter() + .filter(|r| !r.props.is_collection) + .count(); + item_counts.created += new_count as u64; + if logger.enabled(LEVEL_DEFAULT) { + eprintln!(" root {} files: {new_count}", root.href.as_str()); + } + } + + let _ = container_type; + let _ = item_type; + let _ = conn; + summary.per_type.push((container_label, container_counts)); + if !matches!(config.kind, DavKindArg::Webdav) { + summary.per_type.push((item_label, item_counts)); + } + Ok(summary) +} + +fn run_caldav( + conn: &mut Connection, + client: &DavClient, + source_id: i64, + discovery: &Discovery, + config: &DavImportConfig, + logger: Logger, +) -> Result { + run_collection_phase( + conn, + client, + source_id, + discovery, + config, + logger, + ItemPhase { + container_label: "calendar", + item_label: "calendarevent", + reconcile_collections: collections::reconcile_calendars, + reconcile_items: items::reconcile_calendar_events, + }, + ) +} + +fn run_carddav( + conn: &mut Connection, + client: &DavClient, + source_id: i64, + discovery: &Discovery, + config: &DavImportConfig, + logger: Logger, +) -> Result { + run_collection_phase( + conn, + client, + source_id, + discovery, + config, + logger, + ItemPhase { + container_label: "addressbook", + item_label: "contactcard", + reconcile_collections: collections::reconcile_address_books, + reconcile_items: items::reconcile_contact_cards, + }, + ) +} + +fn run_webdav( + conn: &mut Connection, + client: &DavClient, + source_id: i64, + discovery: &Discovery, + config: &DavImportConfig, + logger: Logger, +) -> Result { + let mut summary = Summary::default(); + let mut file_counts = TypeCounts::default(); + + if discovery.collections.is_empty() { + summary.per_type.push(("filenode", file_counts)); + return Ok(summary); + } + let root = &discovery.collections[0]; + let ctx = tree::WebDavCtx { + client, + source_id, + base_url: &discovery.home_set_url, + dav_connections: config.dav_connections, + logger, + }; + tree::reconcile_filenodes(conn, &ctx, root, &mut file_counts)?; + + summary.per_type.push(("filenode", file_counts)); + Ok(summary) +} + +fn map_discovery_error(err: DiscoveryError) -> Error { + match err { + DiscoveryError::NotFound { url } => Error::Usage(format!( + "no DAV collections found under --url {url:?}; \ + check the URL and that the account has access" + )), + DiscoveryError::Transport(e) => Error::from(e), + DiscoveryError::Parse(e) => { + Error::Connection(format!("DAV discovery: malformed multistatus: {e}")) + } + DiscoveryError::Href(e) => Error::Connection(format!("DAV discovery: bad href: {e}")), + } +} + +fn run_gc(conn: &Connection) -> Result<(), Error> { + let tx = conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + db::blobs::gc_orphan_blobs(&tx).map_err(|e| Error::Partial(e.to_string()))?; + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + Ok(()) +} + +fn enforce_tls_policy(url: &str, allow_cleartext: bool) -> Result<(), Error> { + let parsed = + url::Url::parse(url).map_err(|e| Error::Usage(format!("invalid --url {url:?}: {e}")))?; + match parsed.scheme() { + "https" => Ok(()), + "http" => { + if allow_cleartext { + Ok(()) + } else { + Err(Error::Connection( + "http:// URL requires --allow-cleartext".to_owned(), + )) + } + } + other => Err(Error::Usage(format!( + "--url scheme must be http or https, got {other}" + ))), + } +} + +fn normalise_base_url(url: &str) -> String { + let parsed = match url::Url::parse(url) { + Ok(u) => u, + Err(_) => return url.to_owned(), + }; + let scheme = parsed.scheme(); + let host = parsed.host_str().unwrap_or(""); + let port = parsed.port().map(|p| format!(":{p}")).unwrap_or_default(); + format!("{scheme}://{host}{port}") +} + +fn normalise_account_url(url: &str) -> String { + let mut parsed = match url::Url::parse(url) { + Ok(u) => u, + Err(_) => return url.to_owned(), + }; + parsed.set_query(None); + parsed.set_fragment(None); + let path = parsed.path().to_owned(); + if !path.ends_with('/') { + parsed.set_path(&format!("{path}/")); + } + parsed.to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn https_url_is_accepted_without_allow_cleartext() { + assert!(enforce_tls_policy("https://x/", false).is_ok()); + } + + #[test] + fn http_url_requires_allow_cleartext() { + let err = enforce_tls_policy("http://x/", false).unwrap_err(); + assert!(matches!(err, Error::Connection(_))); + assert!(enforce_tls_policy("http://x/", true).is_ok()); + } + + #[test] + fn ftp_url_rejected_as_usage() { + let err = enforce_tls_policy("ftp://x/", false).unwrap_err(); + assert!(matches!(err, Error::Usage(_))); + } + + #[test] + fn normalise_base_url_strips_path() { + assert_eq!( + normalise_base_url("https://dav.example.com/cal/"), + "https://dav.example.com" + ); + } + + #[test] + fn normalise_base_url_keeps_explicit_port() { + assert_eq!( + normalise_base_url("https://dav.example.com:8443/cal/"), + "https://dav.example.com:8443" + ); + } +} diff --git a/src/sync/import_dav/items.rs b/src/sync/import_dav/items.rs new file mode 100644 index 0000000..fed37df --- /dev/null +++ b/src/sync/import_dav/items.rs @@ -0,0 +1,653 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; + +use rusqlite::{Connection, params}; + +use crate::dav::client::{DavClient, MultiStatus}; +use crate::dav::href::{Href, join_absolute, normalise}; +use crate::dav::parse::DavResponse; +use crate::dav::xml; +use crate::db::dav_ids; +use crate::error::Error; +use crate::jmap::error::JmapError; +use crate::logging::{LEVEL_PROGRESS, Logger}; +use crate::sync::TypeCounts; +use crate::sync::import_jmap::pool::Pool; + +use super::calcard; + +pub struct ItemRunCtx<'a> { + pub client: &'a DavClient, + pub source_id: i64, + pub base_url: &'a str, + pub multiget_batch: usize, + pub dav_connections: usize, + pub logger: Logger, +} + +struct PerCollectionCtx<'a> { + run: &'a ItemRunCtx<'a>, + kind: ItemKind, + collection_href: &'a str, + container_local_id: i64, + absolute_url: &'a str, +} + +pub fn reconcile_calendar_events( + conn: &mut Connection, + ctx: &ItemRunCtx<'_>, + collection_href: &str, + calendar_local_id: i64, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let absolute = absolute_collection_url(ctx.base_url, collection_href)?; + let server_items = enumerate_items(ctx.client, &absolute)?; + let pcc = PerCollectionCtx { + run: ctx, + kind: ItemKind::CalendarEvent, + collection_href, + container_local_id: calendar_local_id, + absolute_url: &absolute, + }; + reconcile_items_generic(conn, &pcc, &server_items, counts) +} + +pub fn reconcile_contact_cards( + conn: &mut Connection, + ctx: &ItemRunCtx<'_>, + collection_href: &str, + address_book_local_id: i64, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let absolute = absolute_collection_url(ctx.base_url, collection_href)?; + let server_items = enumerate_items(ctx.client, &absolute)?; + let pcc = PerCollectionCtx { + run: ctx, + kind: ItemKind::ContactCard, + collection_href, + container_local_id: address_book_local_id, + absolute_url: &absolute, + }; + reconcile_items_generic(conn, &pcc, &server_items, counts) +} + +#[derive(Debug, Clone, Copy)] +enum ItemKind { + CalendarEvent, + ContactCard, +} + +impl ItemKind { + fn type_name(self) -> &'static str { + match self { + ItemKind::CalendarEvent => dav_ids::CALENDAR_EVENT, + ItemKind::ContactCard => dav_ids::CONTACT_CARD, + } + } + fn data_query(self) -> fn(&[Href]) -> String { + match self { + ItemKind::CalendarEvent => xml::calendar_multiget, + ItemKind::ContactCard => xml::addressbook_multiget, + } + } + fn data_field(self) -> &'static str { + match self { + ItemKind::CalendarEvent => "calendar_data", + ItemKind::ContactCard => "address_data", + } + } +} + +#[derive(Debug, Clone)] +struct ServerItem { + href: String, + etag: String, +} + +fn enumerate_items(client: &DavClient, url: &str) -> Result, Error> { + let body = xml::propfind_dav_items(); + let ms = client + .propfind_responses(url, 1, &body, url) + .map_err(Error::from)?; + if ms.status >= 400 { + return Err(Error::Partial(format!( + "enumerate {url}: http {}", + ms.status + ))); + } + let self_href = normalise(url, "") + .map(|h| h.into_string()) + .unwrap_or_default(); + let mut out = Vec::new(); + for r in ms.responses { + if r.href.as_str() == self_href { + continue; + } + if r.props.is_collection { + continue; + } + out.push(ServerItem { + href: r.href.into_string(), + etag: r.props.etag.unwrap_or_default(), + }); + } + Ok(out) +} + +struct MultigetJob { + body: String, + hrefs: Vec, +} + +struct MultigetReply { + hrefs: Vec, + result: Result, +} + +struct GetJob { + url: String, + item: ServerItem, +} + +struct GetReply { + item: ServerItem, + result: Result, +} + +struct GetBody { + status: u16, + bytes: Vec, + etag: Option, +} + +fn reconcile_items_generic( + conn: &mut Connection, + pcc: &PerCollectionCtx<'_>, + server_items: &[ServerItem], + counts: &mut TypeCounts, +) -> Result<(), Error> { + let type_name = pcc.kind.type_name(); + let local_rows = + dav_ids::items_in_collection(conn, pcc.run.source_id, type_name, pcc.collection_href) + .map_err(|e| Error::Partial(e.to_string()))?; + let local_map: HashMap = local_rows + .iter() + .map(|r| (r.item_href.clone(), (r.etag.clone(), r.local_id))) + .collect(); + let server_set: HashSet<&str> = server_items.iter().map(|s| s.href.as_str()).collect(); + + let mut to_fetch: Vec = Vec::new(); + for s in server_items { + match local_map.get(&s.href) { + None => to_fetch.push(s.clone()), + Some((local_etag, _)) => { + if s.etag.is_empty() || s.etag != *local_etag { + to_fetch.push(s.clone()); + } + } + } + } + + let vanished: Vec = local_map + .keys() + .filter(|h| !server_set.contains(h.as_str())) + .cloned() + .collect(); + + if !to_fetch.is_empty() { + let fallback = run_multiget_pool(conn, pcc, &to_fetch, &local_map, counts)?; + if !fallback.is_empty() { + run_get_pool(conn, pcc, &fallback, &local_map, counts)?; + } + } + + for href in &vanished { + let local_id = match local_map.get(href) { + Some((_, id)) => *id, + None => continue, + }; + let tx = conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + let table = match pcc.kind { + ItemKind::CalendarEvent => "calendar_events", + ItemKind::ContactCard => "contact_cards", + }; + tx.execute( + &format!("DELETE FROM {table} WHERE id = ?1"), + params![local_id], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + dav_ids::delete_item(&tx, pcc.run.source_id, type_name, href) + .map_err(|e| Error::Partial(e.to_string()))?; + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + counts.deleted += 1; + } + + if pcc.run.logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "items in {}: fetched={} deleted={} failed={}", + pcc.collection_href, counts.fetched, counts.deleted, counts.failed + ); + } + Ok(()) +} + +fn run_multiget_pool( + conn: &mut Connection, + pcc: &PerCollectionCtx<'_>, + to_fetch: &[ServerItem], + local_map: &HashMap, + counts: &mut TypeCounts, +) -> Result, Error> { + let logger = pcc.run.logger; + let absolute_url = pcc.absolute_url; + let batch_size = pcc.run.multiget_batch.max(1); + let workers = pcc.run.dav_connections.clamp(1, 8); + let by_href: HashMap = to_fetch + .iter() + .map(|s| (s.href.clone(), s.clone())) + .collect(); + let client_for_pool = pcc.run.client.clone(); + let url_for_pool = absolute_url.to_owned(); + let pool: Pool = Pool::new(workers, move |job: MultigetJob| { + let result = client_for_pool.report_responses(&url_for_pool, 1, &job.body, &url_for_pool); + MultigetReply { + hrefs: job.hrefs, + result, + } + }); + let mut submitted: usize = 0; + for chunk in to_fetch.chunks(batch_size) { + let hrefs: Vec = chunk + .iter() + .map(|s| Href::from_normalised(s.href.clone())) + .collect(); + let chunk_hrefs: Vec = chunk.iter().map(|s| s.href.clone()).collect(); + pool.submit(MultigetJob { + body: (pcc.kind.data_query())(&hrefs), + hrefs: chunk_hrefs, + }); + submitted += 1; + } + + let mut fallback: Vec = Vec::new(); + for _ in 0..submitted { + let reply = match pool.results().recv() { + Ok(r) => r, + Err(_) => break, + }; + match reply.result { + Err(JmapError::HttpStatus { status, .. }) if status == 405 || status == 501 => { + logger.warn(&format!( + "multiget {absolute_url} returned {status}; server may not implement multiget, falling back to per-item GET" + )); + fallback.extend(reply.hrefs.iter().filter_map(|h| by_href.get(h).cloned())); + } + Err(e) => { + logger.warn(&format!( + "multiget {absolute_url}: {e}; falling back to per-item GET" + )); + fallback.extend(reply.hrefs.iter().filter_map(|h| by_href.get(h).cloned())); + } + Ok(ms) if ms.status >= 400 => { + logger.warn(&format!( + "multiget {absolute_url} returned {}; chunk failed", + ms.status + )); + counts.failed += reply.hrefs.len() as u64; + } + Ok(ms) => { + let requested: HashSet<&str> = reply.hrefs.iter().map(String::as_str).collect(); + let mut tx = conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + let mut pending: usize = 0; + for r in ms.responses { + if !requested.contains(r.href.as_str()) { + logger.warn(&format!( + "multiget {absolute_url}: server returned unsolicited href {}; skipping", + r.href.as_str() + )); + continue; + } + handle_multiget_response(&tx, pcc, &r, local_map, counts); + pending += 1; + if pending >= COMMIT_BATCH { + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + tx = conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + pending = 0; + } + } + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + } + } + } + drop(pool); + Ok(fallback) +} + +fn run_get_pool( + conn: &mut Connection, + pcc: &PerCollectionCtx<'_>, + items: &[ServerItem], + local_map: &HashMap, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let logger = pcc.run.logger; + let absolute_url = pcc.absolute_url; + let workers = pcc.run.dav_connections.clamp(1, 8); + let client_for_pool = pcc.run.client.clone(); + let pool: Pool = Pool::new(workers, move |job: GetJob| { + let r = client_for_pool.get(&job.url).map(|resp| GetBody { + status: resp.status, + bytes: resp.body, + etag: resp.etag, + }); + GetReply { + item: job.item, + result: r, + } + }); + + let mut submitted: usize = 0; + for item in items { + let url = match absolute_item_url(absolute_url, &item.href) { + Ok(u) => u, + Err(e) => { + logger.warn(&format!("item url {}: {e}", item.href)); + counts.failed += 1; + continue; + } + }; + pool.submit(GetJob { + url, + item: item.clone(), + }); + submitted += 1; + } + + let mut tx = conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + let mut pending: usize = 0; + for _ in 0..submitted { + let reply = match pool.results().recv() { + Ok(r) => r, + Err(_) => break, + }; + match reply.result { + Err(e) => { + logger.warn(&format!("GET {}: {e}", reply.item.href)); + counts.failed += 1; + } + Ok(body) if body.status == 404 || body.status == 410 => {} + Ok(body) if body.status >= 400 => { + logger.warn(&format!("GET {}: http {}", reply.item.href, body.status)); + counts.failed += 1; + } + Ok(body) => { + let etag = body.etag.unwrap_or_else(|| reply.item.etag.clone()); + let raw = match std::str::from_utf8(&body.bytes) { + Ok(s) => s.to_owned(), + Err(e) => { + logger.warn(&format!("GET {}: non-utf8 body: {e}", reply.item.href)); + counts.failed += 1; + continue; + } + }; + if let Err(e) = + insert_or_update_tx(&tx, pcc, &reply.item.href, &etag, &raw, local_map) + { + logger.warn(&format!("item {}: {e}", reply.item.href)); + counts.failed += 1; + } else { + counts.fetched += 1; + pending += 1; + if pending >= COMMIT_BATCH { + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + tx = conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + pending = 0; + } + } + } + } + } + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + drop(pool); + Ok(()) +} + +const COMMIT_BATCH: usize = 64; + +fn handle_multiget_response( + tx: &rusqlite::Transaction<'_>, + pcc: &PerCollectionCtx<'_>, + response: &DavResponse, + local_map: &HashMap, + counts: &mut TypeCounts, +) { + let logger = pcc.run.logger; + if response_indicates_vanished(response) { + return; + } + let item_href = response.href.as_str().to_owned(); + let etag = response.props.etag.clone().unwrap_or_default(); + let data = match pcc.kind { + ItemKind::CalendarEvent => response.props.calendar_data.as_deref(), + ItemKind::ContactCard => response.props.address_data.as_deref(), + }; + let Some(raw) = data else { + logger.warn(&format!( + "multiget {item_href}: missing {} payload", + pcc.kind.data_field() + )); + counts.failed += 1; + return; + }; + if let Err(e) = insert_or_update_tx(tx, pcc, &item_href, &etag, raw, local_map) { + logger.warn(&format!("item {item_href}: {e}")); + counts.failed += 1; + } else { + counts.fetched += 1; + } +} + +fn insert_or_update_tx( + tx: &rusqlite::Transaction<'_>, + pcc: &PerCollectionCtx<'_>, + item_href: &str, + etag: &str, + raw: &str, + local_map: &HashMap, +) -> Result<(), Error> { + let existing = local_map.get(item_href).map(|(_, id)| *id); + match pcc.kind { + ItemKind::CalendarEvent => insert_or_update_event( + tx, + pcc.container_local_id, + existing, + raw, + item_href, + pcc.run.logger, + )?, + ItemKind::ContactCard => { + insert_or_update_card(tx, pcc.container_local_id, existing, raw, item_href)? + } + }; + if existing.is_none() { + let new_local = tx.last_insert_rowid(); + dav_ids::insert( + tx, + pcc.run.source_id, + pcc.kind.type_name(), + pcc.collection_href, + item_href, + etag, + new_local, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + } else { + dav_ids::update_etag(tx, pcc.run.source_id, pcc.kind.type_name(), item_href, etag) + .map_err(|e| Error::Partial(e.to_string()))?; + } + Ok(()) +} + +fn insert_or_update_event( + tx: &rusqlite::Transaction<'_>, + calendar_local: i64, + existing: Option, + raw: &str, + item_href: &str, + logger: Logger, +) -> Result<(), Error> { + let entries = calcard::ical_to_jscalendar_entries(raw) + .map_err(|e| Error::Partial(format!("iCalendar parse: {e}")))?; + if entries.len() > 1 && logger.enabled(crate::logging::LEVEL_DEFAULT) { + logger.warn(&format!( + "{item_href}: iCalendar resource has {} entries; storing the first ({}), \ + others dropped (recurrence overrides on the master are preserved)", + entries.len(), + entries[0].data_type.as_column(), + )); + } + let mut first = entries + .into_iter() + .next() + .ok_or_else(|| Error::Partial("iCalendar contained no parseable entries".to_owned()))?; + let (is_draft, use_default_alerts, _uid) = + calcard::strip_extracted_fields_from_event(&mut first.data); + let data_type = first.data_type.as_column(); + let calendar_ids = format!("[{calendar_local}]"); + if let Some(local) = existing { + tx.execute( + "UPDATE calendar_events SET calendar_ids = ?1, is_draft = ?2, + use_default_alerts = ?3, data = ?4, data_type = ?5 + WHERE id = ?6", + params![ + calendar_ids, + is_draft as i64, + use_default_alerts as i64, + first.data.to_string(), + data_type, + local, + ], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + } else { + tx.execute( + "INSERT INTO calendar_events (calendar_ids, is_draft, use_default_alerts, data, data_type) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + calendar_ids, + is_draft as i64, + use_default_alerts as i64, + first.data.to_string(), + data_type, + ], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + } + Ok(()) +} + +fn insert_or_update_card( + tx: &rusqlite::Transaction<'_>, + address_book_local: i64, + existing: Option, + raw: &str, + item_href: &str, +) -> Result<(), Error> { + let card = calcard::vcard_to_jscontact(raw, item_href) + .map_err(|e| Error::Partial(format!("vCard parse: {e}")))?; + let address_book_ids = format!("[{address_book_local}]"); + if let Some(local) = existing { + tx.execute( + "UPDATE contact_cards SET uid = ?1, address_book_ids = ?2, data = ?3 + WHERE id = ?4", + params![card.uid, address_book_ids, card.data.to_string(), local], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + } else { + tx.execute( + "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)", + params![card.uid, address_book_ids, card.data.to_string()], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + } + Ok(()) +} + +fn response_indicates_vanished(response: &DavResponse) -> bool { + if let Some(status) = response.status + && (status == 404 || status == 410) + { + return true; + } + response + .propstat_errors + .iter() + .any(|s| *s == 404 || *s == 410) +} + +fn absolute_collection_url(base_url: &str, href: &str) -> Result { + join_absolute(base_url, href).map_err(|e| Error::Partial(e.to_string())) +} + +fn absolute_item_url(collection_url: &str, item_href: &str) -> Result { + join_absolute(collection_url, item_href).map_err(|e| Error::Partial(e.to_string())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn absolute_collection_url_joins_path() { + let u = absolute_collection_url("https://x/dav/", "/dav/cal/u/d/").unwrap(); + assert_eq!(u, "https://x/dav/cal/u/d/"); + } + + #[test] + fn absolute_item_url_joins_relative() { + let u = absolute_item_url("https://x/dav/cal/u/d/", "/dav/cal/u/d/event.ics").unwrap(); + assert_eq!(u, "https://x/dav/cal/u/d/event.ics"); + } + + fn response(status: Option, propstat_errors: Vec) -> DavResponse { + DavResponse { + href: Href::from_normalised("/dav/cal/u/d/e.ics".to_owned()), + status, + props: crate::dav::parse::ResourceProps::default(), + propstat_errors, + } + } + + #[test] + fn response_level_404_is_vanished() { + assert!(response_indicates_vanished(&response(Some(404), vec![]))); + assert!(response_indicates_vanished(&response(Some(410), vec![]))); + } + + #[test] + fn per_propstat_404_or_410_is_vanished() { + assert!(response_indicates_vanished(&response(None, vec![404]))); + assert!(response_indicates_vanished(&response(None, vec![200, 410]))); + } + + #[test] + fn other_propstat_errors_are_not_vanished() { + assert!(!response_indicates_vanished(&response(None, vec![403]))); + assert!(!response_indicates_vanished(&response(None, vec![]))); + } +} diff --git a/src/sync/import_dav/tree.rs b/src/sync/import_dav/tree.rs new file mode 100644 index 0000000..4aaa35f --- /dev/null +++ b/src/sync/import_dav/tree.rs @@ -0,0 +1,681 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashSet, VecDeque}; +use std::io::Read; + +use rusqlite::{Connection, OptionalExtension, params}; +use time::format_description::well_known::Rfc3339; + +use crate::dav::client::DavClient; +use crate::dav::discover::DiscoveredCollection; +use crate::dav::href::{Href, join_absolute, last_path_component}; +use crate::dav::parse::DavResponse; +use crate::dav::xml; +use crate::db; +use crate::db::dav_ids; +use crate::error::Error; +use crate::jmap::error::JmapError; +use crate::logging::{LEVEL_PROGRESS, Logger}; +use crate::sync::TypeCounts; +use crate::sync::import_jmap::pool::Pool; + +struct FilePlan { + item_href: String, + parent_href: String, + parent_local: i64, + existing_local: Option, + propfind_etag: String, + propfind_content_type: Option, + propfind_last_modified: Option, + propfind_creation_date: Option, + propfind_displayname: Option, + url: String, +} + +struct FileFetch { + plan: FilePlan, +} + +struct FileFetched { + plan: FilePlan, + result: Result, +} + +struct FetchedBody { + status: u16, + bytes: Vec, + etag: Option, + content_type: Option, + last_modified: Option, +} + +pub struct WebDavCtx<'a> { + pub client: &'a DavClient, + pub source_id: i64, + pub base_url: &'a str, + pub dav_connections: usize, + pub logger: Logger, +} + +pub fn reconcile_filenodes( + conn: &mut Connection, + ctx: &WebDavCtx<'_>, + root: &DiscoveredCollection, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let client = ctx.client; + let source_id = ctx.source_id; + let base_url = ctx.base_url; + let dav_connections = ctx.dav_connections; + let logger = ctx.logger; + let absolute_root = absolute(base_url, root.href.as_str())?; + let root_local = super::collections::upsert_root_directory(conn, source_id, root)?; + + let known: Vec<(String, i64)> = + dav_ids::collections_of_type(conn, source_id, dav_ids::FILE_NODE) + .map_err(|e| Error::Partial(e.to_string()))? + .into_iter() + .collect(); + let mut seen: HashSet = HashSet::new(); + seen.insert(root.href.as_str().to_owned()); + + let mut visited: HashSet = HashSet::new(); + visited.insert(root.href.as_str().to_owned()); + let mut queue: VecDeque<(String, String, i64)> = VecDeque::new(); + queue.push_back((absolute_root, root.href.as_str().to_owned(), root_local)); + + let mut file_plans: Vec = Vec::new(); + + while let Some((url, parent_href, parent_local)) = queue.pop_front() { + match walk_one( + conn, + client, + source_id, + WalkPos { + url: &url, + parent_href: &parent_href, + parent_local, + }, + WalkState { + counts, + seen: &mut seen, + file_plans: &mut file_plans, + }, + logger, + ) { + Ok(children) => { + for (child_url, child_href, child_local) in children { + if visited.insert(child_href.clone()) { + queue.push_back((child_url, child_href, child_local)); + } else { + logger.warn(&format!("cycle detected at {child_href}; not recursing")); + } + } + } + Err(e) => { + logger.warn(&format!("PROPFIND {url}: {e}")); + counts.failed += 1; + } + } + } + + fetch_files_parallel( + conn, + client, + source_id, + dav_connections, + file_plans, + counts, + logger, + )?; + + delete_vanished(conn, source_id, &known, &seen, counts, logger)?; + + Ok(()) +} + +fn fetch_files_parallel( + conn: &mut Connection, + client: &DavClient, + source_id: i64, + dav_connections: usize, + plans: Vec, + counts: &mut TypeCounts, + logger: Logger, +) -> Result<(), Error> { + if plans.is_empty() { + return Ok(()); + } + let workers = dav_connections.clamp(1, 8); + let client_for_pool = client.clone(); + let pool: Pool = Pool::new(workers, move |job: FileFetch| { + let result = match client_for_pool.get_stream(&job.plan.url) { + Ok(mut s) => { + let mut bytes = Vec::new(); + match s.read_to_end(&mut bytes) { + Ok(_) => Ok(FetchedBody { + status: s.status, + bytes, + etag: s.etag, + content_type: s.content_type, + last_modified: s.last_modified, + }), + Err(e) => Err(JmapError::Transport(format!("read body: {e}"))), + } + } + Err(e) => Err(e), + }; + FileFetched { + plan: job.plan, + result, + } + }); + + let mut submitted: usize = 0; + for plan in plans { + pool.submit(FileFetch { plan }); + submitted += 1; + } + + const FILE_COMMIT_BATCH: usize = 16; + let mut tx = conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + let mut pending: usize = 0; + for _ in 0..submitted { + let fetched = match pool.results().recv() { + Ok(r) => r, + Err(_) => break, + }; + match fetched.result { + Err(e) => { + logger.warn(&format!("file {}: {e}", fetched.plan.item_href)); + counts.failed += 1; + } + Ok(body) if body.status == 404 || body.status == 410 => {} + Ok(body) if body.status >= 400 => { + logger.warn(&format!( + "file {}: http {}", + fetched.plan.item_href, body.status + )); + counts.failed += 1; + } + Ok(body) => match commit_file(&tx, source_id, &fetched.plan, body, logger) { + Ok(()) => { + counts.fetched += 1; + pending += 1; + if pending >= FILE_COMMIT_BATCH { + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + tx = conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + pending = 0; + } + } + Err(e) => { + logger.warn(&format!("file {}: {e}", fetched.plan.item_href)); + counts.failed += 1; + } + }, + } + } + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + drop(pool); + Ok(()) +} + +fn commit_file( + tx: &rusqlite::Transaction<'_>, + source_id: i64, + plan: &FilePlan, + body: FetchedBody, + logger: Logger, +) -> Result<(), Error> { + let etag = body + .etag + .clone() + .unwrap_or_else(|| plan.propfind_etag.clone()); + let content_type = body + .content_type + .or_else(|| plan.propfind_content_type.clone()); + let last_modified = normalise_dav_date( + &body + .last_modified + .or_else(|| plan.propfind_last_modified.clone()), + ); + let created = format_or_now(&plan.propfind_creation_date)?; + let name = display_or_basename(plan, logger); + + let blob_local = + db::blobs::intern_blob(tx, &body.bytes).map_err(|e| Error::Partial(e.to_string()))?; + if let Some(local) = plan.existing_local { + tx.execute( + "UPDATE file_nodes SET parent_id = ?1, node_type = 'file', blob_id = ?2, + target = NULL, name = ?3, media_type = ?4, + created = ?5, modified = ?6 + WHERE id = ?7", + params![ + plan.parent_local, + blob_local, + name, + content_type, + created, + last_modified, + local, + ], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + dav_ids::update_etag(tx, source_id, dav_ids::FILE_NODE, &plan.item_href, &etag) + .map_err(|e| Error::Partial(e.to_string()))?; + } else { + tx.execute( + "INSERT INTO file_nodes (parent_id, node_type, blob_id, target, name, media_type, + created, modified, is_subscribed, role) + VALUES (?1, 'file', ?2, NULL, ?3, ?4, ?5, ?6, 1, NULL)", + params![ + plan.parent_local, + blob_local, + name, + content_type, + created, + last_modified, + ], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + let local = tx.last_insert_rowid(); + dav_ids::insert( + tx, + source_id, + dav_ids::FILE_NODE, + &plan.parent_href, + &plan.item_href, + &etag, + local, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + } + Ok(()) +} + +fn display_or_basename(plan: &FilePlan, logger: Logger) -> String { + let href = Href::from_normalised(plan.item_href.clone()); + let basename = decoded_basename(&href); + if let Some(n) = &plan.propfind_displayname + && !n.trim().is_empty() + && n.trim() != basename + && logger.enabled(crate::logging::LEVEL_BODIES) + { + eprintln!( + "webdav: {} displayname={:?} differs from basename={basename:?}; using basename", + plan.item_href, n + ); + } + basename +} + +fn delete_vanished( + conn: &mut Connection, + source_id: i64, + known: &[(String, i64)], + seen: &HashSet, + counts: &mut TypeCounts, + logger: Logger, +) -> Result<(), Error> { + let mut vanished: Vec<(String, i64)> = known + .iter() + .filter(|(href, _)| !seen.contains(href)) + .cloned() + .collect(); + if vanished.is_empty() { + return Ok(()); + } + vanished.sort_by(|a, b| { + let depth_a = a.0.matches('/').count(); + let depth_b = b.0.matches('/').count(); + depth_b.cmp(&depth_a).then_with(|| a.0.cmp(&b.0)) + }); + + let tx = conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + for (href, local_id) in &vanished { + if let Err(e) = tx.execute("DELETE FROM file_nodes WHERE id = ?1", params![local_id]) { + logger.warn(&format!( + "vanished file_node {href:?} delete failed: {e}; skipping" + )); + counts.failed += 1; + continue; + } + if let Err(e) = dav_ids::delete_item(&tx, source_id, dav_ids::FILE_NODE, href) { + logger.warn(&format!( + "vanished file_node {href:?} sync_id delete failed: {e}" + )); + counts.failed += 1; + continue; + } + counts.deleted += 1; + } + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + if logger.enabled(LEVEL_PROGRESS) { + eprintln!("webdav: deleted {} vanished file_nodes", vanished.len()); + } + Ok(()) +} + +struct WalkPos<'a> { + url: &'a str, + parent_href: &'a str, + parent_local: i64, +} + +struct WalkState<'a> { + counts: &'a mut TypeCounts, + seen: &'a mut HashSet, + file_plans: &'a mut Vec, +} + +fn walk_one( + conn: &mut Connection, + client: &DavClient, + source_id: i64, + pos: WalkPos<'_>, + state: WalkState<'_>, + logger: Logger, +) -> Result, Error> { + let url = pos.url; + let parent_href = pos.parent_href; + let parent_local = pos.parent_local; + let counts = state.counts; + let seen = state.seen; + let file_plans = state.file_plans; + let body = xml::propfind_webdav_listing(); + let ms = client + .propfind_responses(url, 1, &body, url) + .map_err(Error::from)?; + if ms.status >= 400 { + return Err(Error::Partial(format!("http {}", ms.status))); + } + let parent_norm = crate::dav::href::normalise(url, "") + .map(|h| h.into_string()) + .unwrap_or_default(); + let mut children = Vec::new(); + for r in ms.responses { + if r.href.as_str() == parent_norm || r.href.as_str() == parent_href { + continue; + } + seen.insert(r.href.as_str().to_owned()); + if r.props.is_collection { + let local = upsert_directory(conn, source_id, &r, parent_local, counts)?; + let abs = absolute(url, r.href.as_str())?; + children.push((abs, r.href.as_str().to_owned(), local)); + } else { + match plan_file(conn, source_id, &r, url, parent_local, parent_href) { + Ok(Some(plan)) => file_plans.push(plan), + Ok(None) => {} + Err(e) => { + logger.warn(&format!("file {}: {e}", r.href.as_str())); + counts.failed += 1; + } + } + } + } + if logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "dir {url}: queued {} files (fetched-so-far={} failed-so-far={})", + file_plans.len(), + counts.fetched, + counts.failed + ); + } + Ok(children) +} + +fn plan_file( + conn: &Connection, + source_id: i64, + response: &DavResponse, + collection_url: &str, + parent_local: i64, + parent_href: &str, +) -> Result, Error> { + let item_href = response.href.as_str().to_owned(); + let propfind_etag = response.props.etag.clone().unwrap_or_default(); + let existing = dav_ids::local_for_item(conn, source_id, dav_ids::FILE_NODE, &item_href) + .map_err(|e| Error::Partial(e.to_string()))?; + if let Some(local) = existing { + let stored_etag: Option = conn + .query_row( + "SELECT etag FROM sync_id_dav + WHERE source_id = ?1 AND type_name = ?2 AND item_href = ?3", + params![source_id, dav_ids::FILE_NODE, item_href], + |r| r.get(0), + ) + .optional() + .map_err(|e| Error::Partial(e.to_string()))?; + if let Some(stored) = stored_etag.as_deref() + && !propfind_etag.is_empty() + && stored == propfind_etag + { + return Ok(None); + } + let url = absolute(collection_url, &item_href)?; + Ok(Some(FilePlan { + item_href, + parent_href: parent_href.to_owned(), + parent_local, + existing_local: Some(local), + propfind_etag, + propfind_content_type: response.props.content_type.clone(), + propfind_last_modified: response.props.last_modified.clone(), + propfind_creation_date: response.props.creation_date.clone(), + propfind_displayname: response.props.displayname.clone(), + url, + })) + } else { + let url = absolute(collection_url, &item_href)?; + Ok(Some(FilePlan { + item_href, + parent_href: parent_href.to_owned(), + parent_local, + existing_local: None, + propfind_etag, + propfind_content_type: response.props.content_type.clone(), + propfind_last_modified: response.props.last_modified.clone(), + propfind_creation_date: response.props.creation_date.clone(), + propfind_displayname: response.props.displayname.clone(), + url, + })) + } +} + +fn upsert_directory( + conn: &mut Connection, + source_id: i64, + response: &DavResponse, + parent_local: i64, + counts: &mut TypeCounts, +) -> Result { + let item_href = response.href.as_str().to_owned(); + let name = display_or_path(&response.props.displayname, &response.href); + let modified = normalise_dav_date(&response.props.last_modified); + let existing = dav_ids::local_for_item(conn, source_id, dav_ids::FILE_NODE, &item_href) + .map_err(|e| Error::Partial(e.to_string()))?; + let tx = conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + if let Some(local) = existing { + tx.execute( + "UPDATE file_nodes SET parent_id = ?1, name = ?2, modified = ?3 + WHERE id = ?4", + params![parent_local, name, modified, local], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + counts.fetched += 1; + Ok(local) + } else { + let created = format_or_now(&response.props.creation_date)?; + tx.execute( + "INSERT INTO file_nodes (parent_id, node_type, blob_id, target, name, media_type, + created, modified, is_subscribed, role) + VALUES (?1, 'directory', NULL, NULL, ?2, NULL, ?3, ?4, 1, NULL)", + params![parent_local, name, created, modified], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + let local = tx.last_insert_rowid(); + let parent_href = parent_href_string(&response.href); + dav_ids::insert( + &tx, + source_id, + dav_ids::FILE_NODE, + &parent_href, + &item_href, + "", + local, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + counts.created += 1; + Ok(local) + } +} + +pub(super) fn parent_href_string(href: &Href) -> String { + let s = href.as_str(); + let trimmed = s.trim_end_matches('/'); + if let Some(slash) = trimmed.rfind('/') { + s[..=slash].to_owned() + } else { + "/".to_owned() + } +} + +fn absolute(base: &str, href: &str) -> Result { + join_absolute(base, href).map_err(|e| Error::Partial(e.to_string())) +} + +fn display_or_path(name: &Option, href: &Href) -> String { + if let Some(n) = name + && !n.trim().is_empty() + { + return n.clone(); + } + decoded_basename(href) +} + +fn decoded_basename(href: &Href) -> String { + let c = last_path_component(href); + if c.is_empty() { "_".to_owned() } else { c } +} + +fn normalise_dav_date(input: &Option) -> Option { + let raw = input.as_deref()?.trim(); + if raw.is_empty() { + return None; + } + if let Ok(t) = time::OffsetDateTime::parse(raw, &Rfc3339) { + return t.format(&Rfc3339).ok(); + } + let imf_fixdate = time::macros::format_description!( + "[weekday repr:short], [day] [month repr:short] [year] \ + [hour]:[minute]:[second] GMT" + ); + if let Ok(p) = time::PrimitiveDateTime::parse(raw, imf_fixdate) { + return p.assume_utc().format(&Rfc3339).ok(); + } + None +} + +fn format_or_now(input: &Option) -> Result { + if let Some(s) = normalise_dav_date(input) { + return Ok(s); + } + time::OffsetDateTime::now_utc() + .format(&Rfc3339) + .map_err(|e| Error::Partial(format!("clock: {e}"))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parent_href_string_of_root_is_root() { + let h = Href::from_normalised("/".to_owned()); + assert_eq!(parent_href_string(&h), "/"); + } + + #[test] + fn parent_href_string_of_dir_strips_trailing_segment() { + let h = Href::from_normalised("/dav/file/u/work/".to_owned()); + assert_eq!(parent_href_string(&h), "/dav/file/u/"); + } + + #[test] + fn decoded_basename_handles_percent_encoding() { + let h = Href::from_normalised("/dav/file/u/work/note%201.txt".to_owned()); + assert_eq!(decoded_basename(&h), "note 1.txt"); + } + + fn plan_with(displayname: Option<&str>, href: &str) -> FilePlan { + FilePlan { + item_href: href.to_owned(), + parent_href: "/dav/file/u/".to_owned(), + parent_local: 1, + existing_local: None, + propfind_etag: String::new(), + propfind_content_type: None, + propfind_last_modified: None, + propfind_creation_date: None, + propfind_displayname: displayname.map(str::to_owned), + url: format!("https://x{href}"), + } + } + + #[test] + fn display_or_basename_prefers_url_basename_over_displayname() { + let plan = plan_with(Some("Pretty Name.txt"), "/dav/file/u/photo.jpg"); + let logger = Logger::from_flags(false, 0); + assert_eq!(display_or_basename(&plan, logger), "photo.jpg"); + } + + #[test] + fn display_or_basename_returns_basename_when_no_displayname() { + let plan = plan_with(None, "/dav/file/u/photo.jpg"); + let logger = Logger::from_flags(false, 0); + assert_eq!(display_or_basename(&plan, logger), "photo.jpg"); + } + + #[test] + fn normalise_dav_date_parses_imf_fixdate() { + let n = normalise_dav_date(&Some("Wed, 27 May 2026 07:59:07 GMT".to_owned())) + .expect("imf parse"); + assert!(n.starts_with("2026-05-27T07:59:07")); + } + + #[test] + fn normalise_dav_date_returns_none_on_garbage() { + assert!(normalise_dav_date(&Some("not a date".to_owned())).is_none()); + assert!(normalise_dav_date(&None).is_none()); + } + + #[test] + fn visited_hashset_breaks_recursion_on_cyclic_listing() { + let mut visited: HashSet = HashSet::new(); + visited.insert("/files/u/".to_owned()); + assert!(visited.insert("/files/u/a/".to_owned())); + assert!(visited.insert("/files/u/a/b/".to_owned())); + assert!( + !visited.insert("/files/u/".to_owned()), + "cycle back to root not re-recursed" + ); + assert!( + !visited.insert("/files/u/a/".to_owned()), + "cycle back to a/ not re-recursed" + ); + } +} diff --git a/src/sync/import_exchange_ews.rs b/src/sync/import_exchange_ews.rs new file mode 100644 index 0000000..877a6b7 --- /dev/null +++ b/src/sync/import_exchange_ews.rs @@ -0,0 +1,15 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod attachments; +pub mod calendar; +pub mod contacts; +pub mod coordinator; +pub mod folders; +pub mod items; +pub mod messages; + +pub use coordinator::{EwsAuth, EwsImportConfig, run}; diff --git a/src/sync/import_exchange_ews/attachments.rs b/src/sync/import_exchange_ews/attachments.rs new file mode 100644 index 0000000..7f4fd3e --- /dev/null +++ b/src/sync/import_exchange_ews/attachments.rs @@ -0,0 +1,103 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use base64::Engine; +use base64::engine::general_purpose::STANDARD; + +use crate::db::blobs; +use crate::error::Error; +use crate::exchange_ews::error::EwsError; +use crate::exchange_ews::parse::parse_get_attachment_inline; +use crate::exchange_ews::xml::get_attachment_body; + +use super::items::ItemRunCtx; + +#[derive(Debug, Clone)] +pub struct FetchedAttachment { + pub attachment_id: String, + pub bytes: Vec, + pub media_type: String, +} + +pub fn fetch_contact_photo( + conn: &rusqlite::Connection, + ctx: &ItemRunCtx<'_>, + attachment_id: &str, +) -> Result, Error> { + let Some(att) = fetch_attachment_bytes(ctx, attachment_id).map_err(Error::from)? else { + return Ok(None); + }; + let blob_id = intern_attachment(conn, &att.bytes)?; + Ok(Some((blob_id, att.media_type))) +} + +pub fn fetch_attachment_bytes( + ctx: &ItemRunCtx<'_>, + attachment_id: &str, +) -> Result, EwsError> { + let mut out = fetch_attachments(ctx, &[attachment_id])?; + Ok(out.pop()) +} + +pub fn fetch_attachments( + ctx: &ItemRunCtx<'_>, + attachment_ids: &[&str], +) -> Result, EwsError> { + if attachment_ids.is_empty() { + return Ok(Vec::new()); + } + let batch = ctx.attachment_batch.max(1); + let mut out: Vec = Vec::with_capacity(attachment_ids.len()); + for chunk in attachment_ids.chunks(batch) { + let body = get_attachment_body(chunk); + let resp = ctx.client.call(ctx.url, "GetAttachment", &body)?; + let inline = parse_get_attachment_inline(&resp.body)?; + for att in inline { + let cleaned = strip_ascii_whitespace(att.content_base64.as_bytes()); + if cleaned.is_empty() { + continue; + } + let bytes = STANDARD.decode(&cleaned).map_err(|e| { + EwsError::Malformed(format!("attachment {}: base64: {e}", att.attachment_id)) + })?; + let media_type = att + .content_type + .unwrap_or_else(|| "application/octet-stream".to_owned()); + out.push(FetchedAttachment { + attachment_id: att.attachment_id, + bytes, + media_type, + }); + } + } + Ok(out) +} + +pub fn intern_attachment(conn: &rusqlite::Connection, bytes: &[u8]) -> Result { + blobs::intern_blob(conn, bytes).map_err(|e| Error::Partial(e.to_string())) +} + +fn strip_ascii_whitespace(input: &[u8]) -> Vec { + let mut out = Vec::with_capacity(input.len()); + for b in input { + if !matches!(b, b' ' | b'\t' | b'\n' | b'\r') { + out.push(*b); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::strip_ascii_whitespace; + + #[test] + fn strips_whitespace_efficiently() { + let input = b"AB CD\nEF\tGH\r\nIJ"; + let cleaned = strip_ascii_whitespace(input); + assert_eq!(cleaned, b"ABCDEFGHIJ"); + } +} diff --git a/src/sync/import_exchange_ews/calendar.rs b/src/sync/import_exchange_ews/calendar.rs new file mode 100644 index 0000000..819abd0 --- /dev/null +++ b/src/sync/import_exchange_ews/calendar.rs @@ -0,0 +1,292 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::{Connection, params}; +use serde_json::Value; + +use crate::db::exchange_ews_ids; +use crate::error::Error; +use crate::exchange_ews::calendar_map::to_jscalendar; +use crate::exchange_ews::parse::{CalendarItemRaw, parse_calendar_item}; +use crate::exchange_ews::types::{CalendarItemType, ItemId}; +use crate::exchange_ews::xml::ItemShape; +use crate::logging::LEVEL_PROGRESS; +use crate::sync::TypeCounts; + +use super::attachments::{fetch_attachments, intern_attachment}; +use super::folders::FolderPlan; +use super::items::{ + EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, get_items, plan_for, +}; + +pub fn reconcile_all( + conn: &mut Connection, + ctx: &ItemRunCtx<'_>, + plan: &FolderPlan, + counts: &mut TypeCounts, +) -> Result<(), Error> { + for folder in &plan.calendar { + let folder_id = &folder.folder.folder_id; + let local_folder_id = match exchange_ews_ids::local_for_item( + conn, + ctx.source_id, + exchange_ews_ids::CALENDAR, + &folder_id.id, + ) + .map_err(|e| Error::Partial(e.to_string()))? + { + Some(id) => id, + None => continue, + }; + if let Err(e) = reconcile_one(conn, ctx, folder_id, local_folder_id, counts) { + ctx.logger + .warn(&format!("calendar folder {} failed: {}", folder_id.id, e)); + counts.failed += 1; + } + } + Ok(()) +} + +fn reconcile_one( + conn: &mut Connection, + ctx: &ItemRunCtx<'_>, + folder: &crate::exchange_ews::types::FolderId, + local_folder_id: i64, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let prior = exchange_ews_ids::get_sync_state( + conn, + ctx.source_id, + exchange_ews_ids::CALENDAR, + &folder.id, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + let mut outcome = enumerate_folder(ctx, folder, prior.as_deref()).map_err(Error::from)?; + outcome.items.retain(|s| { + matches!( + s.element.to_ascii_lowercase().as_str(), + "calendaritem" | "item" + ) + }); + let local = exchange_ews_ids::items_in_folder( + conn, + ctx.source_id, + exchange_ews_ids::CALENDAR_EVENT, + &folder.id, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + let plan = plan_for(&outcome, &local); + if ctx.logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "EWS calendar folder {}: new={} changed={} vanished={}", + folder.id, + plan.new.len(), + plan.present_changed.len(), + plan.vanished.len() + ); + } + let mut to_fetch: Vec = plan.new.clone(); + for (id, _) in &plan.present_changed { + to_fetch.push(id.clone()); + } + if !to_fetch.is_empty() { + let outcome = get_items(ctx, ItemShape::CalendarItem, &to_fetch).map_err(Error::from)?; + counts.failed += outcome.failed_items; + for msg in outcome.messages { + if !msg.success { + if matches!( + msg.response_code, + crate::exchange_ews::types::ResponseCode::ItemNotFound + ) { + counts.skipped += 1; + } else { + counts.failed += 1; + ctx.logger + .warn(&format!("GetItem (calendar) error: {}", msg.response_code)); + } + continue; + } + let parsed = parse_calendar_item(&msg.inner_xml).map_err(Error::from)?; + if parsed.id.id.is_empty() { + counts.failed += 1; + continue; + } + if matches!( + parsed.calendar_item_type, + Some(CalendarItemType::Occurrence) | Some(CalendarItemType::Exception) + ) { + counts.skipped += 1; + continue; + } + let existing = plan + .present_changed + .iter() + .find(|(id, _)| id.id == parsed.id.id) + .map(|(_, local)| *local); + apply_event( + conn, + ctx, + &parsed, + local_folder_id, + &folder.id, + existing, + counts, + )?; + } + } + delete_vanished( + conn, + ctx.source_id, + exchange_ews_ids::CALENDAR_EVENT, + "calendar_events", + &plan.vanished, + counts, + )?; + if let EnumerationMode::Delta { new_sync_state, .. } = &outcome.mode { + exchange_ews_ids::set_sync_state( + conn, + ctx.source_id, + exchange_ews_ids::CALENDAR, + &folder.id, + new_sync_state, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + } + Ok(()) +} + +fn apply_event( + conn: &mut Connection, + ctx: &ItemRunCtx<'_>, + raw: &CalendarItemRaw, + local_folder_id: i64, + folder_id: &str, + existing_local_id: Option, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let mut built = to_jscalendar(raw); + attach_calendar_links(conn, ctx, raw, &mut built.data)?; + let calendar_ids = serde_json::json!([local_folder_id]).to_string(); + let data = built.data.to_string(); + let is_draft = if built.is_draft { 1 } else { 0 }; + let use_default_alerts = if built.use_default_alerts { 1 } else { 0 }; + + let tx = conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + match existing_local_id { + Some(id) => { + tx.execute( + "UPDATE calendar_events SET calendar_ids = ?1, is_draft = ?2, \ + use_default_alerts = ?3, data = ?4, data_type = 'Event' WHERE id = ?5", + params![calendar_ids, is_draft, use_default_alerts, data, id], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + exchange_ews_ids::update_change_key( + &tx, + ctx.source_id, + exchange_ews_ids::CALENDAR_EVENT, + &raw.id.id, + &raw.id.change_key, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + counts.fetched += 1; + } + None => { + tx.execute( + "INSERT INTO calendar_events (calendar_ids, is_draft, use_default_alerts, data, data_type) \ + VALUES (?1, ?2, ?3, ?4, 'Event')", + params![calendar_ids, is_draft, use_default_alerts, data], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + let new_id = tx.last_insert_rowid(); + exchange_ews_ids::insert( + &tx, + ctx.source_id, + exchange_ews_ids::CALENDAR_EVENT, + folder_id, + &raw.id.id, + &raw.id.change_key, + new_id, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + counts.created += 1; + } + } + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + Ok(()) +} + +fn attach_calendar_links( + conn: &mut Connection, + ctx: &ItemRunCtx<'_>, + raw: &CalendarItemRaw, + data: &mut Value, +) -> Result<(), Error> { + let refs: Vec<_> = raw + .attachments + .iter() + .filter(|a| !a.is_item_attachment && !a.attachment_id.is_empty()) + .collect(); + if refs.is_empty() { + return Ok(()); + } + let ids: Vec<&str> = refs.iter().map(|a| a.attachment_id.as_str()).collect(); + let fetched = match fetch_attachments(ctx, &ids) { + Ok(f) => f, + Err(e) => { + ctx.logger.warn(&format!( + "calendar attachments fetch failed: {e}; continuing without enclosures" + )); + return Ok(()); + } + }; + let mut links = serde_json::Map::new(); + for (idx, att) in (1u32..).zip(fetched) { + let blob_id = intern_attachment(conn, &att.bytes)?; + let declared = refs + .iter() + .find(|r| r.attachment_id == att.attachment_id) + .and_then(|r| r.content_type.as_deref()); + let name = refs + .iter() + .find(|r| r.attachment_id == att.attachment_id) + .and_then(|r| r.name.clone()); + let key = idx.to_string(); + let mut entry = serde_json::Map::new(); + entry.insert("@type".to_owned(), Value::String("Link".to_owned())); + entry.insert("@blob".to_owned(), Value::from(blob_id)); + entry.insert( + "contentType".to_owned(), + Value::String(content_type_or_from(declared, &att.media_type)), + ); + if let Some(n) = name { + entry.insert("title".to_owned(), Value::String(n)); + } + entry.insert("rel".to_owned(), Value::String("enclosure".to_owned())); + links.insert(key, Value::Object(entry)); + } + if links.is_empty() { + return Ok(()); + } + if let Some(obj) = data.as_object_mut() { + if let Some(existing) = obj.get_mut("links").and_then(Value::as_object_mut) { + for (k, v) in links { + existing.insert(k, v); + } + } else { + obj.insert("links".to_owned(), Value::Object(links)); + } + } + Ok(()) +} + +fn content_type_or_from(declared: Option<&str>, fallback: &str) -> String { + declared + .filter(|s| !s.is_empty()) + .map(str::to_owned) + .unwrap_or_else(|| fallback.to_owned()) +} diff --git a/src/sync/import_exchange_ews/contacts.rs b/src/sync/import_exchange_ews/contacts.rs new file mode 100644 index 0000000..2415948 --- /dev/null +++ b/src/sync/import_exchange_ews/contacts.rs @@ -0,0 +1,227 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::{Connection, params}; + +use crate::db::exchange_ews_ids; +use crate::error::Error; +use crate::exchange_ews::contact_map::{synthetic_uid, to_jscontact}; +use crate::exchange_ews::parse::{ContactItemRaw, parse_contact_item}; +use crate::exchange_ews::types::ItemId; +use crate::exchange_ews::xml::ItemShape; +use crate::logging::LEVEL_PROGRESS; +use crate::sync::TypeCounts; + +use super::attachments::fetch_contact_photo; +use super::folders::FolderPlan; +use super::items::{ + EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, get_items, plan_for, +}; + +pub fn reconcile_all( + conn: &mut Connection, + ctx: &ItemRunCtx<'_>, + plan: &FolderPlan, + counts: &mut TypeCounts, +) -> Result<(), Error> { + for folder in &plan.contacts { + let folder_id = &folder.folder.folder_id; + let local_folder_id = match exchange_ews_ids::local_for_item( + conn, + ctx.source_id, + exchange_ews_ids::ADDRESS_BOOK, + &folder_id.id, + ) + .map_err(|e| Error::Partial(e.to_string()))? + { + Some(id) => id, + None => continue, + }; + if let Err(e) = reconcile_one(conn, ctx, folder_id, local_folder_id, counts) { + ctx.logger + .warn(&format!("contact folder {} failed: {}", folder_id.id, e)); + counts.failed += 1; + } + } + Ok(()) +} + +fn reconcile_one( + conn: &mut Connection, + ctx: &ItemRunCtx<'_>, + folder: &crate::exchange_ews::types::FolderId, + local_folder_id: i64, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let prior = exchange_ews_ids::get_sync_state( + conn, + ctx.source_id, + exchange_ews_ids::ADDRESS_BOOK, + &folder.id, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + let outcome = enumerate_folder(ctx, folder, prior.as_deref()).map_err(Error::from)?; + let local = exchange_ews_ids::items_in_folder( + conn, + ctx.source_id, + exchange_ews_ids::CONTACT_CARD, + &folder.id, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + let plan = plan_for(&outcome, &local); + if ctx.logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "EWS contacts folder {}: new={} changed={} vanished={}", + folder.id, + plan.new.len(), + plan.present_changed.len(), + plan.vanished.len() + ); + } + let mut to_fetch: Vec = plan.new.clone(); + for (id, _) in &plan.present_changed { + to_fetch.push(id.clone()); + } + if !to_fetch.is_empty() { + let outcome = get_items(ctx, ItemShape::Contact, &to_fetch).map_err(Error::from)?; + counts.failed += outcome.failed_items; + for msg in outcome.messages { + if !msg.success { + if matches!( + msg.response_code, + crate::exchange_ews::types::ResponseCode::ItemNotFound + ) { + counts.skipped += 1; + } else { + counts.failed += 1; + ctx.logger + .warn(&format!("GetItem (contact) error: {}", msg.response_code)); + } + continue; + } + let parsed = parse_contact_item(&msg.inner_xml).map_err(Error::from)?; + if parsed.id.id.is_empty() { + counts.failed += 1; + continue; + } + let existing = plan + .present_changed + .iter() + .find(|(id, _)| id.id == parsed.id.id) + .map(|(_, local)| *local); + apply_contact( + conn, + ctx, + &parsed, + local_folder_id, + &folder.id, + existing, + counts, + )?; + } + } + delete_vanished( + conn, + ctx.source_id, + exchange_ews_ids::CONTACT_CARD, + "contact_cards", + &plan.vanished, + counts, + )?; + if let EnumerationMode::Delta { new_sync_state, .. } = &outcome.mode { + exchange_ews_ids::set_sync_state( + conn, + ctx.source_id, + exchange_ews_ids::ADDRESS_BOOK, + &folder.id, + new_sync_state, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + } + Ok(()) +} + +fn apply_contact( + conn: &mut Connection, + ctx: &ItemRunCtx<'_>, + item: &ContactItemRaw, + local_folder_id: i64, + folder_id: &str, + existing_local_id: Option, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let mut card = to_jscontact(item); + if let Some(photo_att) = item.attachments.iter().find(|a| a.is_contact_photo) { + match fetch_contact_photo(conn, ctx, &photo_att.attachment_id) { + Ok(Some((blob_id, media_type))) => { + let mut media = serde_json::Map::new(); + media.insert( + "photo".to_owned(), + serde_json::json!({ + "@type": "Media", + "kind": "photo", + "@blob": blob_id, + "mediaType": media_type, + }), + ); + if let Some(m) = card.as_object_mut() { + m.insert("media".to_owned(), serde_json::Value::Object(media)); + } + } + Ok(None) => {} + Err(e) => { + ctx.logger + .warn(&format!("contact photo fetch failed: {e}; continuing")); + } + } + } + let uid = synthetic_uid(&item.id.id); + let address_book_ids = serde_json::json!([local_folder_id]).to_string(); + let data = card.to_string(); + + let tx = conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + match existing_local_id { + Some(id) => { + tx.execute( + "UPDATE contact_cards SET uid = ?1, address_book_ids = ?2, data = ?3 WHERE id = ?4", + params![uid, address_book_ids, data, id], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + exchange_ews_ids::update_change_key( + &tx, + ctx.source_id, + exchange_ews_ids::CONTACT_CARD, + &item.id.id, + &item.id.change_key, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + counts.fetched += 1; + } + None => { + tx.execute( + "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)", + params![uid, address_book_ids, data], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + let new_id = tx.last_insert_rowid(); + exchange_ews_ids::insert( + &tx, + ctx.source_id, + exchange_ews_ids::CONTACT_CARD, + folder_id, + &item.id.id, + &item.id.change_key, + new_id, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + counts.created += 1; + } + } + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + Ok(()) +} diff --git a/src/sync/import_exchange_ews/coordinator.rs b/src/sync/import_exchange_ews/coordinator.rs new file mode 100644 index 0000000..eaee836 --- /dev/null +++ b/src/sync/import_exchange_ews/coordinator.rs @@ -0,0 +1,521 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::Connection; + +use crate::db; +use crate::db::sources::SourceKey; +use crate::error::Error; +use crate::exchange_ews::EwsClient; +use crate::exchange_ews::autodiscover::{DiscoveryResult, discover}; +use crate::exchange_ews::oauth::{AcquiredToken, OAuthFlow, acquire}; +use crate::exchange_ews::types::MailboxKind; +use crate::jmap::http::{Auth, RetryPolicy}; +use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS}; +use crate::sync::{CommonConfig, Summary, TypeCounts}; + +use super::folders::{self, plan_folders}; +use super::{calendar, contacts, messages}; + +#[derive(Debug, Clone)] +pub enum EwsAuth { + Basic { user: String, password: String }, + Bearer { token: String }, + OAuth(OAuthFlow), +} + +#[derive(Debug, Clone)] +pub struct EwsImportConfig { + pub url: Option, + pub mailbox: Option, + pub mailbox_kind: MailboxKind, + pub auth: EwsAuth, + pub ews_connections: usize, + pub getitem_batch: usize, + pub attachment_batch: usize, + pub use_syncfolderitems: bool, + pub allow_source_change: bool, +} + +pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result { + let logger = common.logger; + let mut conn = db::init::open(&common.archive)?; + + let (auth, acquired) = resolve_auth(&config.auth, common.allow_invalid_certs)?; + let discovery = run_autodiscover(&config, &acquired, common.allow_invalid_certs)?; + if logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "EWS discovery: url={} source={:?}", + discovery.ews_url, discovery.source + ); + } + let mailbox = resolve_mailbox(&config, &acquired)?; + let account_id = synthetic_account_id(&mailbox, config.mailbox_kind); + let session_url = normalise_ews_url(&discovery.ews_url); + enforce_basic_auth_policy(&config.auth, &session_url)?; + let key = SourceKey { + kind: "exchange_ews".to_owned(), + session_url: session_url.clone(), + account_id: account_id.clone(), + }; + + if !common.dry_run + && let Some((url, acc)) = + db::sources::conflicting_source(&conn, "exchange_ews", &session_url, &account_id)? + && !config.allow_source_change + { + return Err(Error::SourceChange(format!( + "archive already records exchange_ews source ({url}, account {acc}); \ + pass --allow-source-change to import a different account" + ))); + } + + let client = EwsClient::new( + auth, + RetryPolicy::new(common.max_retries), + common.allow_invalid_certs, + ); + client.set_logger(logger); + if matches!(config.mailbox_kind, MailboxKind::PublicFolders) { + client.set_anchor_mailbox(None); + } else { + client.set_anchor_mailbox(Some(mailbox.clone())); + } + if let EwsAuth::OAuth(OAuthFlow::ClientCredentials { .. }) = &config.auth { + client.set_impersonation(Some(mailbox.clone())); + } + spawn_token_refresher( + &client, + &config.auth, + &acquired, + common.allow_invalid_certs, + logger, + ); + + let username = match &config.auth { + EwsAuth::Basic { user, .. } => user.clone(), + EwsAuth::Bearer { .. } | EwsAuth::OAuth(_) => acquired + .as_ref() + .and_then(|a| a.upn.clone()) + .unwrap_or_else(|| mailbox.clone()), + }; + + if common.dry_run { + let summary = run_dry(&conn, &client, &session_url, &config, logger)?; + return Ok(summary); + } + + let account_name = acquired.as_ref().and_then(|a| a.name.clone()); + let source_id = db::sources::upsert_source(&conn, &key, account_name.as_deref(), &username)?; + + let mut summary = Summary::default(); + let mut mailbox_counts = TypeCounts::default(); + let mut calendar_counts = TypeCounts::default(); + let mut addressbook_counts = TypeCounts::default(); + let mut email_counts = TypeCounts::default(); + let mut calendar_event_counts = TypeCounts::default(); + let mut contact_counts = TypeCounts::default(); + + let plan = + plan_folders(&client, &session_url, config.mailbox_kind, logger).map_err(Error::from)?; + + folders::reconcile( + &mut conn, + source_id, + &plan, + &mut mailbox_counts, + &mut calendar_counts, + &mut addressbook_counts, + logger, + )?; + + if logger.enabled(LEVEL_DEFAULT) { + eprintln!( + "import: mailbox={} calendar={} addressbook={}", + mailbox_counts.created + mailbox_counts.fetched, + calendar_counts.created + calendar_counts.fetched, + addressbook_counts.created + addressbook_counts.fetched, + ); + } + + let item_ctx = super::items::ItemRunCtx { + client: &client, + url: &session_url, + source_id, + batch_size: config.getitem_batch.max(1), + attachment_batch: config.attachment_batch.max(1), + connections: config.ews_connections.clamp(1, 8), + use_syncfolderitems: config.use_syncfolderitems, + logger, + }; + + messages::reconcile_all(&mut conn, &item_ctx, &plan, &mut email_counts)?; + contacts::reconcile_all(&mut conn, &item_ctx, &plan, &mut contact_counts)?; + calendar::reconcile_all(&mut conn, &item_ctx, &plan, &mut calendar_event_counts)?; + + summary.per_type.push(("mailbox", mailbox_counts)); + summary.per_type.push(("calendar", calendar_counts)); + summary.per_type.push(("addressbook", addressbook_counts)); + summary.per_type.push(("email", email_counts)); + summary + .per_type + .push(("calendarevent", calendar_event_counts)); + summary.per_type.push(("contactcard", contact_counts)); + + if !summary.any_failed() + && let Err(e) = run_gc(&conn) + { + logger.warn(&format!("blob GC skipped: {e}")); + } + + summary.retries_observed = client.retries_observed(); + summary.retry_after_sleeps = client.retry_after_sleeps(); + Ok(summary) +} + +fn run_dry( + _conn: &Connection, + client: &EwsClient, + session_url: &str, + config: &EwsImportConfig, + logger: crate::logging::Logger, +) -> Result { + let plan = + plan_folders(client, session_url, config.mailbox_kind, logger).map_err(Error::from)?; + let mut summary = Summary::default(); + let mut mailbox_counts = TypeCounts::default(); + let mut calendar_counts = TypeCounts::default(); + let mut addressbook_counts = TypeCounts::default(); + mailbox_counts.created = plan.mail.len() as u64; + calendar_counts.created = plan.calendar.len() as u64; + addressbook_counts.created = plan.contacts.len() as u64; + if logger.enabled(LEVEL_DEFAULT) { + eprintln!( + "dry-run: mailbox={} calendar={} addressbook={}", + mailbox_counts.created, calendar_counts.created, addressbook_counts.created + ); + } + summary.per_type.push(("mailbox", mailbox_counts)); + summary.per_type.push(("calendar", calendar_counts)); + summary.per_type.push(("addressbook", addressbook_counts)); + Ok(summary) +} + +fn resolve_auth( + auth: &EwsAuth, + allow_invalid_certs: bool, +) -> Result<(Auth, Option), Error> { + match auth { + EwsAuth::Basic { user, password } => Ok(( + Auth::Basic { + user: user.clone(), + password: password.clone(), + }, + None, + )), + EwsAuth::Bearer { token } => { + let acq = acquire( + &OAuthFlow::PreAcquired { + token: token.clone(), + }, + allow_invalid_certs, + ) + .map_err(Error::from)?; + Ok(( + Auth::Bearer { + token: acq.access_token.clone(), + }, + Some(acq), + )) + } + EwsAuth::OAuth(flow) => { + let acq = acquire(flow, allow_invalid_certs).map_err(Error::from)?; + Ok(( + Auth::Bearer { + token: acq.access_token.clone(), + }, + Some(acq), + )) + } + } +} + +fn run_autodiscover( + config: &EwsImportConfig, + acquired: &Option, + allow_invalid_certs: bool, +) -> Result { + let email = config + .mailbox + .clone() + .or_else(|| acquired.as_ref().and_then(|a| a.upn.clone())); + let result = discover( + config.url.as_deref(), + email.as_deref(), + None, + allow_invalid_certs, + ) + .map_err(Error::from)?; + Ok(result) +} + +fn resolve_mailbox( + config: &EwsImportConfig, + acquired: &Option, +) -> Result { + if let Some(mb) = config.mailbox.as_ref() { + return Ok(mb.clone()); + } + if let Some(acq) = acquired.as_ref() + && let Some(upn) = acq.upn.as_ref() + { + return Ok(upn.clone()); + } + if let EwsAuth::Basic { user, .. } = &config.auth + && user.contains('@') + { + return Ok(user.clone()); + } + Err(Error::Account( + "could not resolve mailbox SMTP; pass --mailbox".to_owned(), + )) +} + +pub fn synthetic_account_id(mailbox: &str, kind: MailboxKind) -> String { + match kind { + MailboxKind::Primary => mailbox.to_owned(), + MailboxKind::Archive => format!("{mailbox}#archive"), + MailboxKind::PublicFolders => { + let domain = mailbox.split('@').nth(1).unwrap_or(""); + format!("__public_folders__@{domain}") + } + } +} + +pub fn normalise_ews_url(raw: &str) -> String { + let Ok(parsed) = url::Url::parse(raw) else { + return raw.to_owned(); + }; + let scheme = parsed.scheme().to_ascii_lowercase(); + let host = parsed.host_str().unwrap_or("").to_ascii_lowercase(); + let default_port = match scheme.as_str() { + "https" => 443, + "http" => 80, + _ => 0, + }; + let port = parsed.port().filter(|p| *p != default_port); + let path = if parsed.path().is_empty() || parsed.path() == "/" { + "/EWS/Exchange.asmx".to_owned() + } else { + let last = parsed.path().rsplit('/').next().unwrap_or(""); + if last.eq_ignore_ascii_case("Exchange.asmx") { + let mut p = String::new(); + let mut parts: Vec<&str> = parsed.path().split('/').collect(); + if let Some(last_part) = parts.last_mut() { + *last_part = "Exchange.asmx"; + } + if let Some(second_last_idx) = parts.len().checked_sub(2) + && parts[second_last_idx].eq_ignore_ascii_case("EWS") + { + parts[second_last_idx] = "EWS"; + } + p.push_str(&parts.join("/")); + p + } else { + parsed.path().to_owned() + } + }; + match port { + Some(p) => format!("{scheme}://{host}:{p}{path}"), + None => format!("{scheme}://{host}{path}"), + } +} + +fn enforce_basic_auth_policy(auth: &EwsAuth, url: &str) -> Result<(), Error> { + if let EwsAuth::Basic { .. } = auth { + let parsed = url::Url::parse(url) + .map_err(|e| Error::Usage(format!("invalid EWS URL {url:?}: {e}")))?; + if parsed.scheme() != "https" { + return Err(Error::Connection( + "EWS requires https:// (basic auth refused on cleartext)".to_owned(), + )); + } + let host = parsed.host_str().unwrap_or("").to_ascii_lowercase(); + let blocked = [ + "outlook.office365.com", + "outlook.office.com", + "outlook.office365.us", + "office365.us", + ]; + if blocked.iter().any(|b| host == *b || host.ends_with(b)) { + return Err(Error::Connection( + "Basic auth is disabled on Exchange Online; use --auth-bearer".to_owned(), + )); + } + } + Ok(()) +} + +fn spawn_token_refresher( + client: &EwsClient, + auth: &EwsAuth, + initial: &Option, + allow_invalid_certs: bool, + logger: crate::logging::Logger, +) { + let flow = match auth { + EwsAuth::OAuth(flow) => flow.clone(), + _ => return, + }; + let exp = initial.as_ref().and_then(|t| { + crate::exchange_ews::oauth::decode_jwt_claims(&t.access_token).and_then(|c| c.exp) + }); + let Some(exp_secs) = exp else { + return; + }; + let client = client.clone(); + let initial_refresh = initial.as_ref().and_then(|t| t.refresh_token.clone()); + std::thread::Builder::new() + .name("vandelay-ews-token-refresh".to_owned()) + .spawn(move || { + let mut next_exp = exp_secs; + let mut refresh_token = initial_refresh; + loop { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + let refresh_at = next_exp.saturating_sub(5 * 60); + if refresh_at > now { + std::thread::sleep(std::time::Duration::from_secs(refresh_at - now)); + } + let result = if let (Some(rt), OAuthFlow::DeviceCode { tenant, client_id }) = + (refresh_token.as_deref(), &flow) + { + crate::exchange_ews::oauth::refresh_with_token( + tenant, + client_id, + rt, + allow_invalid_certs, + ) + } else { + crate::exchange_ews::oauth::acquire(&flow, allow_invalid_certs) + }; + match result { + Ok(tok) => { + client.set_auth(crate::jmap::http::Auth::Bearer { + token: tok.access_token.clone(), + }); + if tok.refresh_token.is_some() { + refresh_token = tok.refresh_token.clone(); + } + if let Some(new_exp) = + crate::exchange_ews::oauth::decode_jwt_claims(&tok.access_token) + .and_then(|c| c.exp) + { + next_exp = new_exp; + } else { + next_exp = next_exp.saturating_add(50 * 60); + } + } + Err(e) => { + logger.warn(&format!( + "EWS token refresh failed: {e}; sleeping 60s before retry" + )); + std::thread::sleep(std::time::Duration::from_secs(60)); + } + } + } + }) + .ok(); +} + +fn run_gc(conn: &Connection) -> Result<(), Error> { + let tx = conn.unchecked_transaction()?; + db::blobs::gc_orphan_blobs(&tx)?; + tx.commit()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn synthetic_account_id_uses_smtp_for_primary() { + assert_eq!( + synthetic_account_id("alice@contoso.com", MailboxKind::Primary), + "alice@contoso.com" + ); + } + + #[test] + fn synthetic_account_id_archive_suffix() { + assert_eq!( + synthetic_account_id("alice@contoso.com", MailboxKind::Archive), + "alice@contoso.com#archive" + ); + } + + #[test] + fn synthetic_account_id_public_folders_uses_tenant() { + assert_eq!( + synthetic_account_id("alice@contoso.com", MailboxKind::PublicFolders), + "__public_folders__@contoso.com" + ); + } + + #[test] + fn basic_against_office365_endpoint_is_refused() { + let auth = EwsAuth::Basic { + user: "alice@x".to_owned(), + password: "p".to_owned(), + }; + let res = + enforce_basic_auth_policy(&auth, "https://outlook.office365.com/EWS/Exchange.asmx"); + assert!(matches!(res, Err(Error::Connection(_)))); + } + + #[test] + fn basic_over_https_on_prem_is_allowed() { + let auth = EwsAuth::Basic { + user: "alice@x".to_owned(), + password: "p".to_owned(), + }; + assert!( + enforce_basic_auth_policy(&auth, "https://exchange.example.com/EWS/Exchange.asmx") + .is_ok() + ); + } + + #[test] + fn basic_over_http_is_refused() { + let auth = EwsAuth::Basic { + user: "alice".to_owned(), + password: "p".to_owned(), + }; + assert!( + enforce_basic_auth_policy(&auth, "http://exchange.example.com/EWS/Exchange.asmx") + .is_err() + ); + } + + #[test] + fn normalise_ews_url_lowercases_host_and_keeps_canonical_path() { + assert_eq!( + normalise_ews_url("https://OUTLOOK.OFFICE365.COM/EWS/Exchange.asmx"), + "https://outlook.office365.com/EWS/Exchange.asmx" + ); + assert_eq!( + normalise_ews_url("https://outlook.office365.com:443/ews/exchange.asmx"), + "https://outlook.office365.com/EWS/Exchange.asmx" + ); + assert_eq!( + normalise_ews_url("https://srv.example.com:8443/EWS/Exchange.asmx"), + "https://srv.example.com:8443/EWS/Exchange.asmx" + ); + } +} diff --git a/src/sync/import_exchange_ews/folders.rs b/src/sync/import_exchange_ews/folders.rs new file mode 100644 index 0000000..99d0029 --- /dev/null +++ b/src/sync/import_exchange_ews/folders.rs @@ -0,0 +1,588 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use rusqlite::{Connection, params}; + +use crate::db::exchange_ews_ids; +use crate::error::Error; +use crate::exchange_ews::EwsClient; +use crate::exchange_ews::error::EwsError; +use crate::exchange_ews::parse::{ + FolderEntry, parse_find_folder_response, parse_folder_inner, parse_get_folder_response, + parse_response_messages, +}; +use crate::exchange_ews::types::{DistinguishedFolderId, FolderClass, MailboxKind}; +use crate::exchange_ews::xml::{ + FolderRef, FolderShape, Traversal, find_folder_body, get_folder_body, +}; +use crate::logging::{LEVEL_PROGRESS, Logger}; +use crate::sync::TypeCounts; + +pub struct FolderPlan { + pub mail: Vec, + pub calendar: Vec, + pub contacts: Vec, + pub well_known_roles: HashMap, + pub root_folder_id: String, +} + +#[derive(Debug, Clone)] +pub struct ClassifiedFolder { + pub folder: FolderEntry, + pub kind: FolderClass, +} + +pub fn plan_folders( + client: &EwsClient, + url: &str, + mailbox_kind: MailboxKind, + logger: Logger, +) -> Result { + let root = mailbox_kind.distinguished_root(); + let request = find_folder_body(FolderRef::Distinguished(root), Traversal::Deep); + let resp = client.call(url, "FindFolder", &request)?; + let parsed = parse_find_folder_response(&resp.body)?; + if logger.enabled(LEVEL_PROGRESS) { + eprintln!("FindFolder returned {} folders", parsed.folders.len()); + } + let mut mail = Vec::new(); + let mut calendar = Vec::new(); + let mut contacts = Vec::new(); + for folder in parsed.folders { + let class = FolderClass::from_ipf(&folder.folder_class); + match (mailbox_kind, class) { + (MailboxKind::Archive, FolderClass::Mail) => { + mail.push(ClassifiedFolder { + folder, + kind: class, + }); + } + (MailboxKind::Archive, _) => {} + (_, FolderClass::Mail) => mail.push(ClassifiedFolder { + folder, + kind: class, + }), + (_, FolderClass::Calendar) => { + calendar.push(ClassifiedFolder { + folder, + kind: class, + }); + } + (_, FolderClass::Contacts) => { + contacts.push(ClassifiedFolder { + folder, + kind: class, + }); + } + (_, FolderClass::Skipped) => {} + } + } + let well_known_roles = resolve_well_known_roles(client, url, mailbox_kind, logger)?; + let root_folder_id = resolve_root_folder_id(client, url, mailbox_kind)?; + Ok(FolderPlan { + mail, + calendar, + contacts, + well_known_roles, + root_folder_id, + }) +} + +fn resolve_well_known_roles( + client: &EwsClient, + url: &str, + mailbox_kind: MailboxKind, + logger: Logger, +) -> Result, EwsError> { + if !matches!(mailbox_kind, MailboxKind::Primary) { + return Ok(HashMap::new()); + } + let folders: Vec> = WELL_KNOWN_ROLES + .iter() + .map(|(d, _)| FolderRef::Distinguished(*d)) + .collect(); + let request = get_folder_body(&folders, FolderShape::IdOnly); + let resp = match client.call(url, "GetFolder", &request) { + Ok(r) => r, + Err(e) => { + if logger.enabled(LEVEL_PROGRESS) { + eprintln!("GetFolder for distinguished ids failed: {e}; roles will be empty"); + } + return Ok(HashMap::new()); + } + }; + let messages = + parse_response_messages(&resp.body, b"GetFolderResponseMessage").unwrap_or_default(); + let mut map: HashMap = HashMap::new(); + for (msg, (_, role)) in messages.into_iter().zip(WELL_KNOWN_ROLES.iter()) { + if !msg.success { + continue; + } + let Ok(Some(entry)) = parse_folder_inner(&msg.inner_xml) else { + continue; + }; + if entry.folder_id.id.is_empty() { + continue; + } + if let Some(r) = role { + map.insert(entry.folder_id.id, r); + } + } + Ok(map) +} + +fn resolve_root_folder_id( + client: &EwsClient, + url: &str, + mailbox_kind: MailboxKind, +) -> Result { + let root = mailbox_kind.distinguished_root(); + let body = get_folder_body(&[FolderRef::Distinguished(root)], FolderShape::IdOnly); + let resp = client.call(url, "GetFolder", &body)?; + let entries = parse_get_folder_response(&resp.body)?; + Ok(entries + .into_iter() + .next() + .map(|e| e.folder_id.id) + .unwrap_or_default()) +} + +const WELL_KNOWN_ROLES: &[(DistinguishedFolderId, Option<&'static str>)] = &[ + (DistinguishedFolderId::Inbox, Some("inbox")), + (DistinguishedFolderId::SentItems, Some("sent")), + (DistinguishedFolderId::Drafts, Some("drafts")), + (DistinguishedFolderId::DeletedItems, Some("trash")), + (DistinguishedFolderId::JunkEmail, Some("junk")), + (DistinguishedFolderId::Archive, Some("archive")), + (DistinguishedFolderId::Outbox, None), + (DistinguishedFolderId::ConversationHistory, None), +]; + +pub fn reconcile( + conn: &mut Connection, + source_id: i64, + plan: &FolderPlan, + mailbox_counts: &mut TypeCounts, + calendar_counts: &mut TypeCounts, + addressbook_counts: &mut TypeCounts, + _logger: Logger, +) -> Result<(), Error> { + let local_mailbox: HashMap = + exchange_ews_ids::folders_of_type(conn, source_id, exchange_ews_ids::MAILBOX) + .map_err(|e| Error::Partial(e.to_string()))?; + let local_calendar: HashMap = + exchange_ews_ids::folders_of_type(conn, source_id, exchange_ews_ids::CALENDAR) + .map_err(|e| Error::Partial(e.to_string()))?; + let local_addressbook: HashMap = + exchange_ews_ids::folders_of_type(conn, source_id, exchange_ews_ids::ADDRESS_BOOK) + .map_err(|e| Error::Partial(e.to_string()))?; + + let mut id_to_local: HashMap = HashMap::new(); + for (id, row) in local_mailbox + .iter() + .chain(local_calendar.iter()) + .chain(local_addressbook.iter()) + { + id_to_local.insert(id.clone(), row.local_id); + } + + let ordered_mail = order_by_parent(&plan.mail); + let ordered_calendar = order_by_parent(&plan.calendar); + let ordered_contacts = order_by_parent(&plan.contacts); + + let mut server_ids_mail: Vec = Vec::new(); + let mut server_ids_calendar: Vec = Vec::new(); + let mut server_ids_addressbook: Vec = Vec::new(); + + for folder in &ordered_mail { + server_ids_mail.push(folder.folder.folder_id.id.clone()); + let role = plan + .well_known_roles + .get(&folder.folder.folder_id.id) + .copied(); + let mut ctx = UpsertCtx { + conn, + source_id, + id_to_local: &mut id_to_local, + local: &local_mailbox, + counts: mailbox_counts, + root_folder_id: &plan.root_folder_id, + }; + upsert_mailbox(&mut ctx, folder, role)?; + } + for folder in &ordered_calendar { + server_ids_calendar.push(folder.folder.folder_id.id.clone()); + let mut ctx = UpsertCtx { + conn, + source_id, + id_to_local: &mut id_to_local, + local: &local_calendar, + counts: calendar_counts, + root_folder_id: &plan.root_folder_id, + }; + upsert_calendar(&mut ctx, folder)?; + } + for folder in &ordered_contacts { + server_ids_addressbook.push(folder.folder.folder_id.id.clone()); + let mut ctx = UpsertCtx { + conn, + source_id, + id_to_local: &mut id_to_local, + local: &local_addressbook, + counts: addressbook_counts, + root_folder_id: &plan.root_folder_id, + }; + upsert_address_book(&mut ctx, folder)?; + } + + delete_vanished( + conn, + source_id, + exchange_ews_ids::MAILBOX, + "mailboxes", + &local_mailbox, + &server_ids_mail, + mailbox_counts, + )?; + delete_vanished( + conn, + source_id, + exchange_ews_ids::CALENDAR, + "calendars", + &local_calendar, + &server_ids_calendar, + calendar_counts, + )?; + delete_vanished( + conn, + source_id, + exchange_ews_ids::ADDRESS_BOOK, + "address_books", + &local_addressbook, + &server_ids_addressbook, + addressbook_counts, + )?; + Ok(()) +} + +fn order_by_parent(folders: &[ClassifiedFolder]) -> Vec<&ClassifiedFolder> { + let by_id: HashMap<&str, &ClassifiedFolder> = folders + .iter() + .map(|f| (f.folder.folder_id.id.as_str(), f)) + .collect(); + let mut depths: HashMap<&str, usize> = HashMap::new(); + fn depth_of<'a>( + id: &'a str, + by_id: &HashMap<&'a str, &'a ClassifiedFolder>, + memo: &mut HashMap<&'a str, usize>, + seen: &mut std::collections::HashSet<&'a str>, + ) -> usize { + if let Some(d) = memo.get(id) { + return *d; + } + if !seen.insert(id) { + return 0; + } + let d = match by_id.get(id).and_then(|f| f.folder.parent_id.as_deref()) { + Some(parent) if by_id.contains_key(parent) => 1 + depth_of(parent, by_id, memo, seen), + _ => 0, + }; + memo.insert(id, d); + d + } + for f in folders { + let id = f.folder.folder_id.id.as_str(); + let mut seen: std::collections::HashSet<&str> = std::collections::HashSet::new(); + let d = depth_of(id, &by_id, &mut depths, &mut seen); + depths.insert(id, d); + } + let mut out: Vec<&ClassifiedFolder> = folders.iter().collect(); + out.sort_by_key(|f| { + depths + .get(f.folder.folder_id.id.as_str()) + .copied() + .unwrap_or(0) + }); + out +} + +struct UpsertCtx<'a> { + conn: &'a mut Connection, + source_id: i64, + id_to_local: &'a mut HashMap, + local: &'a HashMap, + counts: &'a mut TypeCounts, + root_folder_id: &'a str, +} + +fn upsert_mailbox( + ctx: &mut UpsertCtx<'_>, + folder: &ClassifiedFolder, + role: Option<&'static str>, +) -> Result<(), Error> { + let parent_local = parent_local_id(folder, ctx.id_to_local, ctx.root_folder_id); + let tx = ctx + .conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + let existing = ctx.local.get(&folder.folder.folder_id.id); + let local_id = if let Some(row) = existing { + tx.execute( + "UPDATE mailboxes SET name = ?1, parent_id = ?2, role = ?3 WHERE id = ?4", + params![folder.folder.display_name, parent_local, role, row.local_id], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + exchange_ews_ids::update_change_key( + &tx, + ctx.source_id, + exchange_ews_ids::MAILBOX, + &folder.folder.folder_id.id, + &folder.folder.folder_id.change_key, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + ctx.counts.fetched += 1; + row.local_id + } else { + tx.execute( + "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) \ + VALUES (?1, ?2, ?3, 0, 1)", + params![folder.folder.display_name, parent_local, role], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + let new_id = tx.last_insert_rowid(); + exchange_ews_ids::insert( + &tx, + ctx.source_id, + exchange_ews_ids::MAILBOX, + folder.folder.parent_id.as_deref().unwrap_or(""), + &folder.folder.folder_id.id, + &folder.folder.folder_id.change_key, + new_id, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + ctx.counts.created += 1; + new_id + }; + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + ctx.id_to_local + .insert(folder.folder.folder_id.id.clone(), local_id); + Ok(()) +} + +fn upsert_calendar(ctx: &mut UpsertCtx<'_>, folder: &ClassifiedFolder) -> Result<(), Error> { + let tx = ctx + .conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + let existing = ctx.local.get(&folder.folder.folder_id.id); + let local_id = if let Some(row) = existing { + tx.execute( + "UPDATE calendars SET name = ?1 WHERE id = ?2", + params![folder.folder.display_name, row.local_id], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + exchange_ews_ids::update_change_key( + &tx, + ctx.source_id, + exchange_ews_ids::CALENDAR, + &folder.folder.folder_id.id, + &folder.folder.folder_id.change_key, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + ctx.counts.fetched += 1; + row.local_id + } else { + tx.execute( + "INSERT INTO calendars (name, sort_order, is_subscribed, is_visible) \ + VALUES (?1, 0, 1, 1)", + params![folder.folder.display_name], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + let new_id = tx.last_insert_rowid(); + exchange_ews_ids::insert( + &tx, + ctx.source_id, + exchange_ews_ids::CALENDAR, + folder.folder.parent_id.as_deref().unwrap_or(""), + &folder.folder.folder_id.id, + &folder.folder.folder_id.change_key, + new_id, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + ctx.counts.created += 1; + new_id + }; + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + ctx.id_to_local + .insert(folder.folder.folder_id.id.clone(), local_id); + Ok(()) +} + +fn upsert_address_book(ctx: &mut UpsertCtx<'_>, folder: &ClassifiedFolder) -> Result<(), Error> { + let tx = ctx + .conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + let existing = ctx.local.get(&folder.folder.folder_id.id); + let local_id = if let Some(row) = existing { + tx.execute( + "UPDATE address_books SET name = ?1 WHERE id = ?2", + params![folder.folder.display_name, row.local_id], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + exchange_ews_ids::update_change_key( + &tx, + ctx.source_id, + exchange_ews_ids::ADDRESS_BOOK, + &folder.folder.folder_id.id, + &folder.folder.folder_id.change_key, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + ctx.counts.fetched += 1; + row.local_id + } else { + tx.execute( + "INSERT INTO address_books (name, sort_order, is_subscribed) \ + VALUES (?1, 0, 1)", + params![folder.folder.display_name], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + let new_id = tx.last_insert_rowid(); + exchange_ews_ids::insert( + &tx, + ctx.source_id, + exchange_ews_ids::ADDRESS_BOOK, + folder.folder.parent_id.as_deref().unwrap_or(""), + &folder.folder.folder_id.id, + &folder.folder.folder_id.change_key, + new_id, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + ctx.counts.created += 1; + new_id + }; + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + ctx.id_to_local + .insert(folder.folder.folder_id.id.clone(), local_id); + Ok(()) +} + +fn parent_local_id( + folder: &ClassifiedFolder, + id_to_local: &HashMap, + root_folder_id: &str, +) -> Option { + let parent_id = folder.folder.parent_id.as_deref()?; + if parent_id == root_folder_id || parent_id.is_empty() { + return None; + } + id_to_local.get(parent_id).copied() +} + +fn delete_vanished( + conn: &mut Connection, + source_id: i64, + type_name: &str, + table: &str, + local: &HashMap, + server_ids: &[String], + counts: &mut TypeCounts, +) -> Result<(), Error> { + let server_set: std::collections::HashSet<&str> = + server_ids.iter().map(String::as_str).collect(); + let mut vanished: Vec<(&str, &exchange_ews_ids::FolderRow)> = local + .iter() + .filter(|(id, _)| !server_set.contains(id.as_str())) + .map(|(id, row)| (id.as_str(), row)) + .collect(); + if type_name == exchange_ews_ids::MAILBOX { + vanished.sort_by_key(|(_, row)| std::cmp::Reverse(folder_depth(row, local))); + } + for (item_id, row) in vanished { + let tx = conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + let result = tx.execute( + &format!("DELETE FROM {table} WHERE id = ?1"), + params![row.local_id], + ); + match result { + Ok(_) => { + exchange_ews_ids::delete_item(&tx, source_id, type_name, item_id) + .map_err(|e| Error::Partial(e.to_string()))?; + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + counts.deleted += 1; + } + Err(_) => { + let _ = tx.rollback(); + counts.failed += 1; + } + } + } + Ok(()) +} + +fn folder_depth( + row: &exchange_ews_ids::FolderRow, + local: &HashMap, +) -> usize { + let mut depth = 0; + let mut cursor = row.folder_id.as_str(); + let mut visited: std::collections::HashSet<&str> = std::collections::HashSet::new(); + while !cursor.is_empty() && visited.insert(cursor) { + match local.get(cursor) { + Some(parent) => { + depth += 1; + cursor = parent.folder_id.as_str(); + } + None => break, + } + } + depth +} + +#[cfg(test)] +mod tests { + use super::*; + + fn row(id: &str, parent_ews_id: &str, local_id: i64) -> exchange_ews_ids::FolderRow { + exchange_ews_ids::FolderRow { + item_id: id.to_owned(), + folder_id: parent_ews_id.to_owned(), + change_key: String::new(), + local_id, + } + } + + #[test] + fn folder_depth_walks_parent_chain() { + let mut local: HashMap = HashMap::new(); + local.insert("ROOT".to_owned(), row("ROOT", "", 1)); + local.insert("CHILD".to_owned(), row("CHILD", "ROOT", 2)); + local.insert("GRAND".to_owned(), row("GRAND", "CHILD", 3)); + assert_eq!(folder_depth(local.get("ROOT").unwrap(), &local), 0); + assert_eq!(folder_depth(local.get("CHILD").unwrap(), &local), 1); + assert_eq!(folder_depth(local.get("GRAND").unwrap(), &local), 2); + } + + #[test] + fn folder_depth_handles_unknown_parent() { + let mut local: HashMap = HashMap::new(); + local.insert("X".to_owned(), row("X", "ORPHAN", 1)); + assert_eq!(folder_depth(local.get("X").unwrap(), &local), 0); + } + + #[test] + fn folder_depth_stops_on_cycle() { + let mut local: HashMap = HashMap::new(); + local.insert("A".to_owned(), row("A", "B", 1)); + local.insert("B".to_owned(), row("B", "A", 2)); + let d = folder_depth(local.get("A").unwrap(), &local); + assert!(d <= 2); + } +} diff --git a/src/sync/import_exchange_ews/items.rs b/src/sync/import_exchange_ews/items.rs new file mode 100644 index 0000000..722b474 --- /dev/null +++ b/src/sync/import_exchange_ews/items.rs @@ -0,0 +1,469 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use crate::error::Error; +use crate::exchange_ews::EwsClient; +use crate::exchange_ews::error::EwsError; +use crate::exchange_ews::parse::{ + SyncChange, parse_find_item_response, parse_response_messages, parse_sync_folder_items_response, +}; +use crate::exchange_ews::types::{FolderId, ItemId, ResponseCode}; +use crate::exchange_ews::xml::{ + FolderRef, ItemShape, Traversal, find_item_body, get_item_body, sync_folder_items_body, +}; +use crate::logging::Logger; + +pub struct ItemRunCtx<'a> { + pub client: &'a EwsClient, + pub url: &'a str, + pub source_id: i64, + pub batch_size: usize, + pub attachment_batch: usize, + pub connections: usize, + pub use_syncfolderitems: bool, + pub logger: Logger, +} + +#[derive(Debug, Clone)] +pub struct EnumeratedItem { + pub element: String, + pub id: ItemId, +} + +#[derive(Debug, Clone)] +pub struct EnumerationOutcome { + pub items: Vec, + pub mode: EnumerationMode, +} + +#[derive(Debug, Clone)] +pub enum EnumerationMode { + Full, + Delta { + deletions: Vec, + new_sync_state: String, + }, +} + +pub fn enumerate_folder( + ctx: &ItemRunCtx<'_>, + folder: &FolderId, + prior_sync_state: Option<&str>, +) -> Result { + if ctx.use_syncfolderitems + && let Some(state) = prior_sync_state + && !state.is_empty() + && let Some(outcome) = try_sync_folder_items(ctx, folder, state)? + { + return Ok(outcome); + } + enumerate_via_find_item(ctx, folder).map(|items| EnumerationOutcome { + items, + mode: EnumerationMode::Full, + }) +} + +fn sync_fallback_codes(code: &ResponseCode) -> bool { + matches!( + code, + ResponseCode::InvalidIdMalformed | ResponseCode::AccessDenied | ResponseCode::Other(_) + ) +} + +fn try_sync_folder_items( + ctx: &ItemRunCtx<'_>, + folder: &FolderId, + prior_state: &str, +) -> Result, EwsError> { + let mut sync_state = prior_state.to_owned(); + let mut items: Vec = Vec::new(); + let mut deletions: Vec = Vec::new(); + let mut iters = 0; + loop { + let body = sync_folder_items_body(folder, &sync_state, 512); + let resp = match ctx.client.call(ctx.url, "SyncFolderItems", &body) { + Ok(r) => r, + Err(EwsError::SoapFault { + code: ResponseCode::InvalidSyncStateData, + .. + }) => { + return Ok(None); + } + Err(EwsError::SoapFault { code, .. }) if sync_fallback_codes(&code) => { + return Ok(None); + } + Err(e) => return Err(e), + }; + let parsed = parse_sync_folder_items_response(&resp.body)?; + for change in parsed.changes { + match change { + SyncChange::Create { id, element } | SyncChange::Update { id, element } => { + items.push(EnumeratedItem { element, id }); + } + SyncChange::Delete { id } => { + deletions.push(id.id); + } + SyncChange::ReadFlagChange { id, .. } => { + items.push(EnumeratedItem { + element: "Message".to_owned(), + id, + }); + } + } + } + sync_state = parsed.sync_state; + if !parsed.more { + break; + } + iters += 1; + if iters > 200 { + break; + } + } + Ok(Some(EnumerationOutcome { + items, + mode: EnumerationMode::Delta { + deletions, + new_sync_state: sync_state, + }, + })) +} + +fn enumerate_via_find_item( + ctx: &ItemRunCtx<'_>, + folder: &FolderId, +) -> Result, EwsError> { + let mut items: Vec = Vec::new(); + let mut offset: u32 = 0; + let page_size: u32 = 500; + loop { + let body = find_item_body( + FolderRef::Concrete(folder), + Traversal::Shallow, + offset, + page_size, + ); + let resp = ctx.client.call(ctx.url, "FindItem", &body)?; + let parsed = parse_find_item_response(&resp.body)?; + let returned = parsed.items.len() as u32; + for entry in parsed.items { + items.push(EnumeratedItem { + element: entry.element, + id: entry.id, + }); + } + if !parsed.more { + break; + } + if returned == 0 { + break; + } + offset = offset.saturating_add(page_size); + if offset > 1_000_000 { + break; + } + } + Ok(items) +} + +#[derive(Debug, Clone)] +pub struct DiffPlan { + pub new: Vec, + pub vanished: Vec<(String, i64)>, + pub present_changed: Vec<(ItemId, i64)>, + pub present_unchanged: Vec<(String, i64)>, +} + +pub fn plan_for( + outcome: &EnumerationOutcome, + local: &[crate::db::exchange_ews_ids::ItemRow], +) -> DiffPlan { + match &outcome.mode { + EnumerationMode::Full => diff(&outcome.items, local), + EnumerationMode::Delta { deletions, .. } => diff_delta(&outcome.items, deletions, local), + } +} + +fn diff_delta( + server_changes: &[EnumeratedItem], + deletions: &[String], + local: &[crate::db::exchange_ews_ids::ItemRow], +) -> DiffPlan { + let local_map: HashMap<&str, &crate::db::exchange_ews_ids::ItemRow> = + local.iter().map(|r| (r.item_id.as_str(), r)).collect(); + let mut plan = DiffPlan { + new: Vec::new(), + vanished: Vec::new(), + present_changed: Vec::new(), + present_unchanged: Vec::new(), + }; + for s in server_changes { + match local_map.get(s.id.id.as_str()) { + None => plan.new.push(s.id.clone()), + Some(row) => plan.present_changed.push((s.id.clone(), row.local_id)), + } + } + for d in deletions { + if let Some(row) = local_map.get(d.as_str()) { + plan.vanished.push((d.clone(), row.local_id)); + } + } + plan +} + +pub fn diff(server: &[EnumeratedItem], local: &[crate::db::exchange_ews_ids::ItemRow]) -> DiffPlan { + let local_map: HashMap<&str, &crate::db::exchange_ews_ids::ItemRow> = + local.iter().map(|r| (r.item_id.as_str(), r)).collect(); + let server_map: HashMap<&str, &EnumeratedItem> = + server.iter().map(|s| (s.id.id.as_str(), s)).collect(); + let mut plan = DiffPlan { + new: Vec::new(), + vanished: Vec::new(), + present_changed: Vec::new(), + present_unchanged: Vec::new(), + }; + for s in server { + match local_map.get(s.id.id.as_str()) { + None => plan.new.push(s.id.clone()), + Some(row) => { + if s.id.change_key.is_empty() || s.id.change_key == row.change_key { + plan.present_unchanged + .push((row.item_id.clone(), row.local_id)); + } else { + plan.present_changed.push((s.id.clone(), row.local_id)); + } + } + } + } + for row in local { + if !server_map.contains_key(row.item_id.as_str()) { + plan.vanished.push((row.item_id.clone(), row.local_id)); + } + } + plan +} + +pub struct GetItemBatchOutcome { + pub messages: Vec, + pub failed_items: u64, +} + +fn is_per_batch_fault(err: &EwsError) -> bool { + matches!( + err, + EwsError::SoapFault { .. } | EwsError::HttpStatus { .. } | EwsError::Malformed(_) + ) +} + +pub fn get_items( + ctx: &ItemRunCtx<'_>, + shape: ItemShape, + ids: &[ItemId], +) -> Result { + let batch = ctx.batch_size.max(1); + let workers = ctx.connections.clamp(1, 8); + let mut failed_items: u64 = 0; + if workers <= 1 || ids.len() <= batch { + let mut all = Vec::new(); + for chunk in ids.chunks(batch) { + let body = get_item_body(shape, chunk); + match ctx.client.call(ctx.url, "GetItem", &body) { + Ok(resp) => match parse_response_messages(&resp.body, b"GetItemResponseMessage") { + Ok(mut msgs) => all.append(&mut msgs), + Err(e) if is_per_batch_fault(&e) => { + ctx.logger.warn(&format!( + "GetItem batch parse failed ({e}); {} ids left for retry next run", + chunk.len() + )); + failed_items += chunk.len() as u64; + } + Err(e) => return Err(e), + }, + Err(e) if is_per_batch_fault(&e) => { + ctx.logger.warn(&format!( + "GetItem batch failed ({e}); {} ids left for retry next run", + chunk.len() + )); + failed_items += chunk.len() as u64; + } + Err(e) => return Err(e), + } + } + return Ok(GetItemBatchOutcome { + messages: all, + failed_items, + }); + } + let client = ctx.client.clone(); + let url = ctx.url.to_owned(); + type BatchResult = ( + usize, + Result, EwsError>, + ); + let pool: crate::sync::import_jmap::pool::Pool, BatchResult> = + crate::sync::import_jmap::pool::Pool::new(workers, move |chunk: Vec| { + let body = get_item_body(shape, &chunk); + let n = chunk.len(); + let result = match client.call(&url, "GetItem", &body) { + Ok(resp) => parse_response_messages(&resp.body, b"GetItemResponseMessage"), + Err(e) => Err(e), + }; + (n, result) + }); + let mut submitted = 0usize; + for chunk in ids.chunks(batch) { + pool.submit(chunk.to_vec()); + submitted += 1; + } + let mut all = Vec::new(); + let mut abort_err: Option = None; + for _ in 0..submitted { + match pool.results().recv() { + Ok((_, Ok(mut msgs))) => all.append(&mut msgs), + Ok((n, Err(e))) => { + if is_per_batch_fault(&e) { + ctx.logger.warn(&format!( + "GetItem batch failed ({e}); {n} ids left for retry next run" + )); + failed_items += n as u64; + } else if abort_err.is_none() { + abort_err = Some(e); + } + } + Err(_) => break, + } + } + if let Some(e) = abort_err { + return Err(e); + } + Ok(GetItemBatchOutcome { + messages: all, + failed_items, + }) +} + +pub fn delete_vanished( + conn: &mut rusqlite::Connection, + source_id: i64, + type_name: &str, + table: &str, + vanished: &[(String, i64)], + counts: &mut crate::sync::TypeCounts, +) -> Result<(), Error> { + use rusqlite::params; + for (item_id, local_id) in vanished { + let tx = conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + match tx.execute( + &format!("DELETE FROM {table} WHERE id = ?1"), + params![local_id], + ) { + Ok(_) => { + crate::db::exchange_ews_ids::delete_item(&tx, source_id, type_name, item_id) + .map_err(|e| Error::Partial(e.to_string()))?; + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + counts.deleted += 1; + } + Err(_) => { + let _ = tx.rollback(); + counts.failed += 1; + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn diff_delta_only_yields_changes_from_server_response() { + use crate::db::exchange_ews_ids::ItemRow; + let outcome = EnumerationOutcome { + items: vec![EnumeratedItem { + element: "Message".to_owned(), + id: ItemId::new("A", "ck-2"), + }], + mode: EnumerationMode::Delta { + deletions: vec!["Z".to_owned()], + new_sync_state: "STATE2".to_owned(), + }, + }; + let local = vec![ + ItemRow { + item_id: "A".to_owned(), + change_key: "ck-1".to_owned(), + local_id: 1, + }, + ItemRow { + item_id: "B".to_owned(), + change_key: "ck-1".to_owned(), + local_id: 2, + }, + ItemRow { + item_id: "Z".to_owned(), + change_key: "ck-9".to_owned(), + local_id: 99, + }, + ]; + let plan = plan_for(&outcome, &local); + assert!(plan.new.is_empty()); + assert_eq!(plan.present_changed.len(), 1); + assert_eq!(plan.present_changed[0].0.id, "A"); + assert_eq!(plan.vanished.len(), 1); + assert_eq!(plan.vanished[0].0, "Z"); + assert_eq!(plan.present_unchanged.len(), 0); + } + + #[test] + fn diff_splits_into_new_unchanged_changed_vanished() { + use crate::db::exchange_ews_ids::ItemRow; + let server = vec![ + EnumeratedItem { + element: "Message".to_owned(), + id: ItemId::new("A", "ck-1"), + }, + EnumeratedItem { + element: "Message".to_owned(), + id: ItemId::new("B", "ck-1"), + }, + EnumeratedItem { + element: "Message".to_owned(), + id: ItemId::new("C", "ck-2"), + }, + ]; + let local = vec![ + ItemRow { + item_id: "A".to_owned(), + change_key: "ck-1".to_owned(), + local_id: 1, + }, + ItemRow { + item_id: "B".to_owned(), + change_key: "ck-0".to_owned(), + local_id: 2, + }, + ItemRow { + item_id: "Z".to_owned(), + change_key: "ck-9".to_owned(), + local_id: 99, + }, + ]; + let plan = diff(&server, &local); + assert_eq!(plan.new.len(), 1); + assert_eq!(plan.new[0].id, "C"); + assert_eq!(plan.present_unchanged.len(), 1); + assert_eq!(plan.present_unchanged[0].0, "A"); + assert_eq!(plan.present_changed.len(), 1); + assert_eq!(plan.present_changed[0].0.id, "B"); + assert_eq!(plan.vanished.len(), 1); + assert_eq!(plan.vanished[0].0, "Z"); + } +} diff --git a/src/sync/import_exchange_ews/messages.rs b/src/sync/import_exchange_ews/messages.rs new file mode 100644 index 0000000..247c2be --- /dev/null +++ b/src/sync/import_exchange_ews/messages.rs @@ -0,0 +1,262 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use rusqlite::{Connection, params}; +use serde_json::{Value, json}; + +use crate::db::{blobs, exchange_ews_ids}; +use crate::error::Error; +use crate::exchange_ews::parse::{MessageItem, parse_message_item}; +use crate::exchange_ews::types::ItemId; +use crate::exchange_ews::xml::ItemShape; +use crate::logging::LEVEL_PROGRESS; +use crate::sync::TypeCounts; +use crate::sync::emailmeta::email_meta_from_blob; +use crate::sync::keys::index_to_json; + +use super::folders::FolderPlan; +use super::items::{ + EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, get_items, plan_for, +}; + +pub fn reconcile_all( + conn: &mut Connection, + ctx: &ItemRunCtx<'_>, + plan: &FolderPlan, + counts: &mut TypeCounts, +) -> Result<(), Error> { + for folder in &plan.mail { + let folder_id = &folder.folder.folder_id; + let local_folder_id = match exchange_ews_ids::local_for_item( + conn, + ctx.source_id, + exchange_ews_ids::MAILBOX, + &folder_id.id, + ) + .map_err(|e| Error::Partial(e.to_string()))? + { + Some(id) => id, + None => continue, + }; + if let Err(e) = reconcile_one_folder(conn, ctx, folder_id, local_folder_id, counts) { + ctx.logger + .warn(&format!("email folder {} failed: {}", folder_id.id, e)); + counts.failed += 1; + } + } + Ok(()) +} + +fn reconcile_one_folder( + conn: &mut Connection, + ctx: &ItemRunCtx<'_>, + folder: &crate::exchange_ews::types::FolderId, + local_folder_id: i64, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let prior = exchange_ews_ids::get_sync_state( + conn, + ctx.source_id, + exchange_ews_ids::MAILBOX, + &folder.id, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + let outcome = enumerate_folder(ctx, folder, prior.as_deref()).map_err(Error::from)?; + let local = + exchange_ews_ids::items_in_folder(conn, ctx.source_id, exchange_ews_ids::EMAIL, &folder.id) + .map_err(|e| Error::Partial(e.to_string()))?; + let plan = plan_for(&outcome, &local); + if ctx.logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "EWS folder {}: new={} changed={} vanished={} unchanged={}", + folder.id, + plan.new.len(), + plan.present_changed.len(), + plan.vanished.len(), + plan.present_unchanged.len() + ); + } + let mut to_fetch: Vec = plan.new.clone(); + for (id, _local_id) in &plan.present_changed { + to_fetch.push(id.clone()); + } + if !to_fetch.is_empty() { + let outcome = get_items(ctx, ItemShape::Message, &to_fetch).map_err(Error::from)?; + counts.failed += outcome.failed_items; + for msg in outcome.messages { + if !msg.success { + if matches!( + msg.response_code, + crate::exchange_ews::types::ResponseCode::ItemNotFound + ) { + counts.skipped += 1; + } else { + counts.failed += 1; + ctx.logger.warn(&format!( + "GetItem (message) error: {} {}", + msg.response_code, msg.message_text + )); + } + continue; + } + let parsed = parse_message_item(&msg.inner_xml).map_err(Error::from)?; + if parsed.id.id.is_empty() { + counts.failed += 1; + continue; + } + let existing = plan + .present_changed + .iter() + .find(|(id, _)| id.id == parsed.id.id) + .map(|(_, local)| *local); + apply_message( + conn, + ctx, + &parsed, + local_folder_id, + &folder.id, + existing, + counts, + )?; + } + } + delete_vanished( + conn, + ctx.source_id, + exchange_ews_ids::EMAIL, + "emails", + &plan.vanished, + counts, + )?; + if let EnumerationMode::Delta { new_sync_state, .. } = &outcome.mode { + exchange_ews_ids::set_sync_state( + conn, + ctx.source_id, + exchange_ews_ids::MAILBOX, + &folder.id, + new_sync_state, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + } + Ok(()) +} + +fn apply_message( + conn: &mut Connection, + ctx: &ItemRunCtx<'_>, + item: &MessageItem, + local_folder_id: i64, + folder_id: &str, + existing_local_id: Option, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let mime_b64 = match item.mime_content.as_ref() { + Some(s) => s.replace(['\n', '\r', ' ', '\t'], ""), + None => { + counts.skipped += 1; + ctx.logger.warn(&format!( + "message {} has no MimeContent; skipping", + item.id.id + )); + return Ok(()); + } + }; + let bytes = match STANDARD.decode(mime_b64.as_bytes()) { + Ok(b) => b, + Err(e) => { + counts.failed += 1; + ctx.logger.warn(&format!( + "message {} MimeContent base64 decode failed: {e}", + item.id.id + )); + return Ok(()); + } + }; + let (idx, _date_header) = email_meta_from_blob(&bytes); + let message_match = index_to_json(&idx); + let received_at = item + .date_time_received + .clone() + .unwrap_or_else(|| "1970-01-01T00:00:00Z".to_owned()); + let keywords = keyword_array(item); + let mailbox_ids = json!([local_folder_id]).to_string(); + + let tx = conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + let blob_id = blobs::intern_blob(&tx, &bytes).map_err(|e| Error::Partial(e.to_string()))?; + match existing_local_id { + Some(id) => { + tx.execute( + "UPDATE emails SET blob_id = ?1, received_at = ?2, mailbox_ids = ?3, \ + keywords = ?4, message_match = ?5 WHERE id = ?6", + params![ + blob_id, + received_at, + mailbox_ids, + keywords, + message_match, + id + ], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + exchange_ews_ids::update_change_key( + &tx, + ctx.source_id, + exchange_ews_ids::EMAIL, + &item.id.id, + &item.id.change_key, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + counts.fetched += 1; + } + None => { + tx.execute( + "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match) \ + VALUES (?1, ?2, ?3, ?4, ?5)", + params![blob_id, received_at, mailbox_ids, keywords, message_match], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + let new_id = tx.last_insert_rowid(); + exchange_ews_ids::insert( + &tx, + ctx.source_id, + exchange_ews_ids::EMAIL, + folder_id, + &item.id.id, + &item.id.change_key, + new_id, + ) + .map_err(|e| Error::Partial(e.to_string()))?; + counts.created += 1; + } + } + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + Ok(()) +} + +fn keyword_array(item: &MessageItem) -> String { + let mut kws: Vec = Vec::new(); + if matches!(item.is_read, Some(true)) { + kws.push("$seen".to_owned()); + } + if matches!(item.is_draft, Some(true)) { + kws.push("$draft".to_owned()); + } + if matches!(item.is_read_receipt_requested, Some(true)) { + kws.push("$notified".to_owned()); + } + if matches!(item.flag_status.as_deref(), Some("Flagged")) { + kws.push("$flagged".to_owned()); + } + for cat in &item.categories { + kws.push(cat.to_ascii_lowercase()); + } + let value: Value = Value::Array(kws.into_iter().map(Value::String).collect()); + value.to_string() +} diff --git a/src/sync/import_exchange_graph.rs b/src/sync/import_exchange_graph.rs new file mode 100644 index 0000000..17c3c05 --- /dev/null +++ b/src/sync/import_exchange_graph.rs @@ -0,0 +1,13 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod calendar; +pub mod contacts; +pub mod coordinator; +pub mod folders; +pub mod messages; + +pub use coordinator::{GraphAuth, GraphImportConfig, run}; diff --git a/src/sync/import_exchange_graph/calendar.rs b/src/sync/import_exchange_graph/calendar.rs new file mode 100644 index 0000000..02f0923 --- /dev/null +++ b/src/sync/import_exchange_graph/calendar.rs @@ -0,0 +1,443 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use rusqlite::{Connection, Transaction, params}; +use serde_json::{Value, json}; + +use crate::db::exchange_graph_ids; +use crate::error::Error; +use crate::exchange_graph::api::{self, PREFER_BODY_HTML, PREFER_BODY_TEXT, PREFER_TIMEZONE_UTC}; +use crate::exchange_graph::calendar_map::{ + ConvertedEvent, EventType, classify_event_type, convert_event, +}; +use crate::exchange_graph::error::GraphError; +use crate::exchange_graph::types::EventBodyFormat; +use crate::logging::LEVEL_PROGRESS; +use crate::sync::TypeCounts; +use crate::sync::import_jmap::pool::Pool; + +use super::coordinator::{CHUNK_SIZE, GraphCoordinator}; +use super::folders::CalendarFolder; + +pub fn reconcile_all( + conn: &mut Connection, + ctx: &GraphCoordinator<'_>, + calendars: &[CalendarFolder], + counts: &mut TypeCounts, +) -> Result<(), Error> { + let local: HashMap = + exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::CALENDAR_EVENT)?; + let mut server_total: std::collections::HashSet = std::collections::HashSet::new(); + let mut any_failure = false; + + for cal in calendars { + let url = ctx.endpoints.calendar_events_ids(&cal.graph_id, ctx.top); + let stubs = match api::collect_all_values(ctx.client, &url, &[PREFER_TIMEZONE_UTC]) { + Ok(v) => v, + Err(e) => { + ctx.logger + .warn(&format!("calendar {} stub fetch failed: {e}", cal.graph_id)); + counts.failed += 1; + any_failure = true; + continue; + } + }; + let mut want_ids: Vec = Vec::new(); + let mut occurrence_count = 0usize; + for stub in &stubs { + match classify_event_type(stub) { + EventType::Occurrence => occurrence_count += 1, + _ => { + if let Some(id) = stub.get("id").and_then(Value::as_str) { + server_total.insert(id.to_owned()); + if !local.contains_key(id) { + want_ids.push(id.to_owned()); + } + } + } + } + } + if ctx.logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "graph calendar {} events: stubs={} new={} occurrences_skipped={}", + cal.graph_id, + stubs.len(), + want_ids.len(), + occurrence_count + ); + } + if want_ids.is_empty() { + continue; + } + let fetched = fetch_events(ctx, &want_ids); + let mut masters: Vec<(String, ConvertedEvent)> = Vec::new(); + let mut exceptions: Vec<(String, ConvertedEvent)> = Vec::new(); + for (graph_id, result) in fetched { + match result { + Ok(raw) => match convert_event(&raw, None) { + Ok(c) => match c.event_type { + EventType::Exception => exceptions.push((graph_id, c)), + _ => masters.push((graph_id, c)), + }, + Err(e) => { + counts.failed += 1; + ctx.logger + .warn(&format!("graph event {graph_id} convert failed: {e}")); + } + }, + Err(GraphError::Vanished) => counts.skipped += 1, + Err(e) => { + counts.failed += 1; + ctx.logger + .warn(&format!("graph event {graph_id} fetch failed: {e}")); + } + } + } + + let mut master_by_graph_id: HashMap = masters.into_iter().collect(); + for (ex_graph_id, ex) in exceptions { + let Some(master_graph_id) = ex.series_master_id.clone() else { + ctx.logger.warn(&format!( + "graph exception event {ex_graph_id} has no seriesMasterId; dropping" + )); + counts.skipped += 1; + continue; + }; + if let Some(master) = master_by_graph_id.get_mut(&master_graph_id) { + merge_exception_into(master, &ex); + } else if local.contains_key(&master_graph_id) { + merge_exception_into_existing(conn, ctx, &master_graph_id, &ex, counts); + } else { + counts.skipped += 1; + ctx.logger.warn(&format!( + "graph exception {ex_graph_id} references missing master {master_graph_id}; orphaned" + )); + } + } + + let pairs: Vec<(String, ConvertedEvent)> = master_by_graph_id.into_iter().collect(); + insert_events_chunked(conn, ctx, cal.local_id, &pairs, counts)?; + } + + if any_failure { + ctx.logger.warn( + "graph event vanished-cleanup skipped: one or more calendars failed to enumerate; \ + a clean re-run will reconcile deletions", + ); + } else { + delete_vanished(conn, ctx.source_id, &local, &server_total, counts)?; + } + Ok(()) +} + +fn fetch_events( + ctx: &GraphCoordinator<'_>, + ids: &[String], +) -> Vec<(String, Result)> { + type R = (String, Result); + let client = ctx.client.clone(); + let endpoints: crate::exchange_graph::api::Endpoints = (*ctx.endpoints).clone(); + let body_prefer = match ctx.event_body_format { + EventBodyFormat::Text => PREFER_BODY_TEXT, + EventBodyFormat::Html => PREFER_BODY_HTML, + }; + let prefer: Vec = vec![PREFER_TIMEZONE_UTC.to_owned(), body_prefer.to_owned()]; + let pool: Pool = Pool::new(ctx.workers, move |id: String| { + let url = endpoints.event(&id); + let prefer_refs: Vec<&str> = prefer.iter().map(String::as_str).collect(); + let result = client.get_json_with_prefer(&url, &prefer_refs); + (id, result) + }); + for id in ids { + pool.submit(id.clone()); + } + let mut out = Vec::with_capacity(ids.len()); + for _ in 0..ids.len() { + if let Ok(r) = pool.results().recv() { + out.push(r); + } + } + out +} + +fn merge_exception_into(master: &mut ConvertedEvent, ex: &ConvertedEvent) { + let Some(raw_key) = ex.original_start.as_deref() else { + return; + }; + let master_tz = master + .data + .get("timeZone") + .and_then(Value::as_str) + .map(str::to_owned); + let key = normalise_override_key(raw_key, master_tz.as_deref()); + let Value::Object(map) = &mut master.data else { + return; + }; + let overrides = map + .entry("recurrenceOverrides".to_owned()) + .or_insert_with(|| Value::Object(serde_json::Map::new())); + let Value::Object(overrides) = overrides else { + return; + }; + overrides.insert(key, ex.data.clone()); +} + +fn merge_exception_into_existing( + conn: &mut Connection, + ctx: &GraphCoordinator<'_>, + master_graph_id: &str, + ex: &ConvertedEvent, + counts: &mut TypeCounts, +) { + let local_id = match exchange_graph_ids::local_for_graph_id( + conn, + ctx.source_id, + exchange_graph_ids::CALENDAR_EVENT, + master_graph_id, + ) { + Ok(Some(id)) => id, + Ok(None) => { + counts.skipped += 1; + return; + } + Err(e) => { + counts.failed += 1; + ctx.logger.warn(&format!( + "graph exception merge: lookup of master {master_graph_id} failed: {e}" + )); + return; + } + }; + let Some(raw_key) = ex.original_start.as_deref() else { + counts.skipped += 1; + return; + }; + if let Err(e) = merge_persisted_master(conn, local_id, raw_key, &ex.data) { + counts.failed += 1; + ctx.logger.warn(&format!( + "graph exception merge into stored master {master_graph_id} failed: {e}" + )); + } else { + counts.fetched += 1; + } +} + +fn merge_persisted_master( + conn: &Connection, + local_id: i64, + raw_key: &str, + ex_data: &Value, +) -> Result<(), String> { + let tx = conn.unchecked_transaction().map_err(|e| e.to_string())?; + let row: String = tx + .query_row( + "SELECT data FROM calendar_events WHERE id = ?1", + params![local_id], + |row| row.get(0), + ) + .map_err(|e| e.to_string())?; + let mut data: Value = serde_json::from_str(&row).map_err(|e| e.to_string())?; + let master_tz = data + .get("timeZone") + .and_then(Value::as_str) + .map(str::to_owned); + let key = normalise_override_key(raw_key, master_tz.as_deref()); + if let Value::Object(map) = &mut data { + let entry = map + .entry("recurrenceOverrides".to_owned()) + .or_insert_with(|| Value::Object(serde_json::Map::new())); + if let Value::Object(overrides) = entry { + overrides.insert(key, ex_data.clone()); + } + } + tx.execute( + "UPDATE calendar_events SET data = ?1 WHERE id = ?2", + params![data.to_string(), local_id], + ) + .map_err(|e| e.to_string())?; + tx.commit().map_err(|e| e.to_string())?; + Ok(()) +} + +pub fn normalise_override_key(raw: &str, master_tz: Option<&str>) -> String { + if let Some(tz_name) = master_tz + && let Some(local) = utc_offset_to_local(raw, tz_name) + { + return local; + } + strip_offset_and_fractional(raw) +} + +fn utc_offset_to_local(raw: &str, tz_name: &str) -> Option { + use chrono::{DateTime, NaiveDateTime, TimeZone}; + use chrono_tz::Tz; + let tz: Tz = tz_name.parse().ok()?; + let parsed: DateTime = if let Ok(dt) = DateTime::parse_from_rfc3339(raw) { + dt + } else if let Ok(naive) = NaiveDateTime::parse_from_str(raw, "%Y-%m-%dT%H:%M:%S%.f") { + return Some(format_local(naive)); + } else if let Ok(naive) = NaiveDateTime::parse_from_str(raw, "%Y-%m-%dT%H:%M:%S") { + return Some(format_local(naive)); + } else { + return None; + }; + let local = tz.from_utc_datetime(&parsed.naive_utc()).naive_local(); + Some(format_local(local)) +} + +fn format_local(dt: chrono::NaiveDateTime) -> String { + dt.format("%Y-%m-%dT%H:%M:%S").to_string() +} + +fn strip_offset_and_fractional(raw: &str) -> String { + let no_offset = if let Some(idx) = raw.find('Z') { + &raw[..idx] + } else if let Some(idx) = raw.find('+') { + &raw[..idx] + } else if let Some(idx) = raw.rfind('-') + && idx > 10 + { + &raw[..idx] + } else { + raw + }; + let trimmed = no_offset.split('.').next().unwrap_or(no_offset); + if trimmed.contains('T') { + trimmed.to_owned() + } else { + format!("{trimmed}T00:00:00") + } +} + +fn insert_events_chunked( + conn: &mut Connection, + ctx: &GraphCoordinator<'_>, + calendar_local_id: i64, + pairs: &[(String, ConvertedEvent)], + counts: &mut TypeCounts, +) -> Result<(), Error> { + for chunk in pairs.chunks(CHUNK_SIZE) { + let tx = conn.unchecked_transaction()?; + for (graph_id, event) in chunk { + insert_event_in_tx(&tx, ctx, calendar_local_id, graph_id, event, counts)?; + } + tx.commit()?; + } + Ok(()) +} + +fn insert_event_in_tx( + tx: &Transaction<'_>, + ctx: &GraphCoordinator<'_>, + calendar_local_id: i64, + graph_id: &str, + event: &ConvertedEvent, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let calendar_ids = json!([calendar_local_id]).to_string(); + let data = event.data.to_string(); + tx.execute( + "INSERT INTO calendar_events (calendar_ids, is_draft, use_default_alerts, data, data_type) + VALUES (?1, ?2, ?3, ?4, 'Event')", + params![ + calendar_ids, + event.is_draft as i64, + event.use_default_alerts as i64, + data, + ], + )?; + let new_id = tx.last_insert_rowid(); + exchange_graph_ids::insert( + tx, + ctx.source_id, + exchange_graph_ids::CALENDAR_EVENT, + graph_id, + new_id, + )?; + counts.created += 1; + Ok(()) +} + +fn delete_vanished( + conn: &mut Connection, + source_id: i64, + local: &HashMap, + server: &std::collections::HashSet, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let vanished: Vec<(&String, &i64)> = local + .iter() + .filter(|(graph_id, _)| !server.contains(graph_id.as_str())) + .collect(); + for chunk in vanished.chunks(CHUNK_SIZE) { + let tx = conn.unchecked_transaction()?; + for (graph_id, local_id) in chunk { + let result = tx.execute( + "DELETE FROM calendar_events WHERE id = ?1", + params![local_id], + ); + match result { + Ok(_) => { + exchange_graph_ids::delete( + &tx, + source_id, + exchange_graph_ids::CALENDAR_EVENT, + graph_id, + )?; + counts.deleted += 1; + } + Err(_) => { + counts.failed += 1; + } + } + } + tx.commit()?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn override_key_utc_converts_to_master_local_time() { + let local = normalise_override_key("2026-05-11T15:00:00Z", Some("America/New_York")); + assert_eq!(local, "2026-05-11T11:00:00"); + } + + #[test] + fn override_key_no_tz_strips_z_only() { + let local = normalise_override_key("2026-05-11T15:00:00Z", None); + assert_eq!(local, "2026-05-11T15:00:00"); + } + + #[test] + fn override_key_with_explicit_offset_converts() { + let local = normalise_override_key("2026-05-11T15:00:00+00:00", Some("Europe/London")); + assert_eq!(local, "2026-05-11T16:00:00"); + } + + #[test] + fn override_key_naive_localdatetime_passes_through() { + let local = normalise_override_key("2026-05-11T15:00:00", Some("America/New_York")); + assert_eq!(local, "2026-05-11T15:00:00"); + } + + #[test] + fn override_key_unknown_timezone_strips_offset_only() { + let local = normalise_override_key("2026-05-11T15:00:00Z", Some("Not/A_Zone")); + assert_eq!(local, "2026-05-11T15:00:00"); + } + + #[test] + fn override_key_fractional_seconds_stripped() { + let local = normalise_override_key("2026-05-11T15:00:00.1234567Z", None); + assert_eq!(local, "2026-05-11T15:00:00"); + } +} diff --git a/src/sync/import_exchange_graph/contacts.rs b/src/sync/import_exchange_graph/contacts.rs new file mode 100644 index 0000000..1c46b6e --- /dev/null +++ b/src/sync/import_exchange_graph/contacts.rs @@ -0,0 +1,201 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use rusqlite::{Connection, Transaction, params}; +use serde_json::{Value, json}; + +use crate::db::exchange_graph_ids; +use crate::error::Error; +use crate::exchange_graph::api; +use crate::exchange_graph::contact_map::{ConvertedContact, convert_contact}; +use crate::exchange_graph::error::GraphError; +use crate::logging::LEVEL_PROGRESS; +use crate::sync::TypeCounts; +use crate::sync::import_jmap::pool::Pool; + +use super::coordinator::{CHUNK_SIZE, GraphCoordinator}; +use super::folders::ContactFolder; + +pub fn reconcile_all( + conn: &mut Connection, + ctx: &GraphCoordinator<'_>, + books: &[ContactFolder], + counts: &mut TypeCounts, +) -> Result<(), Error> { + let local: HashMap = + exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::CONTACT_CARD)?; + let mut server_total: std::collections::HashSet = std::collections::HashSet::new(); + let mut any_failure = false; + + for book in books { + let url = ctx + .endpoints + .contact_folder_contacts_ids(&book.graph_id, ctx.top); + let ids = match api::collect_all_ids(ctx.client, &url, &[]) { + Ok(v) => v, + Err(e) => { + ctx.logger.warn(&format!( + "contact folder {} enumeration failed: {e}", + book.graph_id + )); + counts.failed += 1; + any_failure = true; + continue; + } + }; + if ctx.logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "graph contact folder {} contacts: {}", + book.graph_id, + ids.len() + ); + } + for id in &ids { + server_total.insert(id.clone()); + } + let new_ids: Vec = ids + .into_iter() + .filter(|id| !local.contains_key(id)) + .collect(); + if new_ids.is_empty() { + continue; + } + let fetched = fetch_contacts(ctx, &new_ids); + let mut converted: Vec<(String, ConvertedContact)> = Vec::new(); + for (graph_id, result) in fetched { + match result { + Ok(raw) => match convert_contact(&raw) { + Ok(c) => converted.push((graph_id, c)), + Err(e) => { + counts.failed += 1; + ctx.logger + .warn(&format!("graph contact {graph_id} convert failed: {e}")); + } + }, + Err(GraphError::Vanished) => counts.skipped += 1, + Err(e) => { + counts.failed += 1; + ctx.logger + .warn(&format!("graph contact {graph_id} fetch failed: {e}")); + } + } + } + insert_contacts_chunked(conn, ctx, book.local_id, &converted, counts)?; + } + + if any_failure { + ctx.logger.warn( + "graph contact vanished-cleanup skipped: one or more contact folders failed to enumerate; \ + a clean re-run will reconcile deletions", + ); + } else { + delete_vanished(conn, ctx.source_id, &local, &server_total, counts)?; + } + Ok(()) +} + +fn fetch_contacts( + ctx: &GraphCoordinator<'_>, + ids: &[String], +) -> Vec<(String, Result)> { + type R = (String, Result); + let client = ctx.client.clone(); + let endpoints: crate::exchange_graph::api::Endpoints = (*ctx.endpoints).clone(); + let pool: Pool = Pool::new(ctx.workers, move |id: String| { + let url = endpoints.contact(&id); + (id, client.get_json_with_prefer(&url, &[])) + }); + for id in ids { + pool.submit(id.clone()); + } + let mut out = Vec::with_capacity(ids.len()); + for _ in 0..ids.len() { + if let Ok(r) = pool.results().recv() { + out.push(r); + } + } + out +} + +fn insert_contacts_chunked( + conn: &mut Connection, + ctx: &GraphCoordinator<'_>, + book_local_id: i64, + pairs: &[(String, ConvertedContact)], + counts: &mut TypeCounts, +) -> Result<(), Error> { + for chunk in pairs.chunks(CHUNK_SIZE) { + let tx = conn.unchecked_transaction()?; + for (graph_id, contact) in chunk { + insert_contact_in_tx(&tx, ctx, book_local_id, graph_id, contact, counts)?; + } + tx.commit()?; + } + Ok(()) +} + +fn insert_contact_in_tx( + tx: &Transaction<'_>, + ctx: &GraphCoordinator<'_>, + book_local_id: i64, + graph_id: &str, + converted: &ConvertedContact, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let book_ids = json!([book_local_id]).to_string(); + let data = converted.data.to_string(); + tx.execute( + "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)", + params![converted.uid, book_ids, data], + )?; + let new_id = tx.last_insert_rowid(); + exchange_graph_ids::insert( + tx, + ctx.source_id, + exchange_graph_ids::CONTACT_CARD, + graph_id, + new_id, + )?; + counts.created += 1; + Ok(()) +} + +fn delete_vanished( + conn: &mut Connection, + source_id: i64, + local: &HashMap, + server: &std::collections::HashSet, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let vanished: Vec<(&String, &i64)> = local + .iter() + .filter(|(graph_id, _)| !server.contains(graph_id.as_str())) + .collect(); + for chunk in vanished.chunks(CHUNK_SIZE) { + let tx = conn.unchecked_transaction()?; + for (graph_id, local_id) in chunk { + let result = tx.execute("DELETE FROM contact_cards WHERE id = ?1", params![local_id]); + match result { + Ok(_) => { + exchange_graph_ids::delete( + &tx, + source_id, + exchange_graph_ids::CONTACT_CARD, + graph_id, + )?; + counts.deleted += 1; + } + Err(_) => { + counts.failed += 1; + } + } + } + tx.commit()?; + } + Ok(()) +} diff --git a/src/sync/import_exchange_graph/coordinator.rs b/src/sync/import_exchange_graph/coordinator.rs new file mode 100644 index 0000000..f242087 --- /dev/null +++ b/src/sync/import_exchange_graph/coordinator.rs @@ -0,0 +1,531 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::Connection; +use serde_json::Value; + +use crate::db; +use crate::db::sources::SourceKey; +use crate::error::Error; +use crate::exchange_graph::api::{DEFAULT_API_BASE, Endpoints}; +use crate::exchange_graph::client::GraphClient; +use crate::exchange_graph::error::GraphError; +use crate::exchange_graph::oauth::{ + AcquiredToken, OAuthFlow, acquire, default_authority, refresh_access_token, +}; +use crate::exchange_graph::types::{EventBodyFormat, MailboxKind, synthetic_account_id}; +use crate::jmap::http::RetryPolicy; +use crate::logging::LEVEL_DEFAULT; +use crate::sync::{CommonConfig, Summary, TypeCounts}; +use crate::types::ObjectType; + +#[derive(Debug, Clone)] +pub enum GraphAuth { + PreAcquired { + token: String, + }, + DeviceCode { + authority: String, + client_id: String, + }, +} + +#[derive(Debug, Clone)] +pub struct GraphImportConfig { + pub auth: GraphAuth, + pub api_base: String, + pub user_target: Option, + pub mailbox_kind: MailboxKind, + pub objects: Option>, + pub event_body_format: EventBodyFormat, + pub graph_connections: usize, + pub top: usize, + pub allow_source_change: bool, +} + +pub const CHUNK_SIZE: usize = 100; + +pub struct GraphCoordinator<'a> { + pub client: &'a GraphClient, + pub endpoints: &'a Endpoints, + pub source_id: i64, + pub top: usize, + pub workers: usize, + pub logger: crate::logging::Logger, + pub event_body_format: EventBodyFormat, +} + +pub fn run(common: CommonConfig, config: GraphImportConfig) -> Result { + let logger = common.logger; + let mut conn = db::init::open(&common.archive)?; + + let acquired = acquire_with_flow(&config.auth, common.allow_invalid_certs)?; + let client = GraphClient::new( + acquired.access_token.clone(), + RetryPolicy::new(common.max_retries), + common.allow_invalid_certs, + ); + client.set_logger(logger); + + let endpoints = resolve_endpoints(&config, &client)?; + let principal = resolve_principal(&client, &endpoints, acquired.upn.as_deref())?; + + let session_url = canonical_session_url(&config); + let account_id = synthetic_account_id(&principal.id, config.mailbox_kind); + + if !common.dry_run + && let Some((url, acc)) = + db::sources::conflicting_source(&conn, "exchange_graph", &session_url, &account_id)? + && !config.allow_source_change + { + return Err(Error::SourceChange(format!( + "archive already records exchange_graph source ({url}, account {acc}); \ + pass --allow-source-change to import a different account" + ))); + } + + if matches!(config.mailbox_kind, MailboxKind::Archive) { + logger.warn( + "--mailbox-kind archive: Microsoft Online Archive holds only mail; \ + calendar and contact surfaces are skipped for this run", + ); + } + + if common.dry_run { + return run_dry( + &client, + &endpoints, + config.mailbox_kind, + &logger, + config.top, + ); + } + + let source_id = db::sources::upsert_source( + &conn, + &SourceKey { + kind: "exchange_graph".to_owned(), + session_url: session_url.clone(), + account_id: account_id.clone(), + }, + Some(&principal.user_principal_name), + &principal.user_principal_name, + )?; + + let _refresher = spawn_token_refresher( + &client, + &config.auth, + &acquired, + common.allow_invalid_certs, + logger, + ); + + let mut summary = Summary::default(); + let mut mailbox_counts = TypeCounts::default(); + let mut email_counts = TypeCounts::default(); + let mut calendar_counts = TypeCounts::default(); + let mut event_counts = TypeCounts::default(); + let mut addressbook_counts = TypeCounts::default(); + let mut contact_counts = TypeCounts::default(); + + let ctx = GraphCoordinator { + client: &client, + endpoints: &endpoints, + source_id, + top: config.top.clamp(1, 1000), + workers: config.graph_connections.clamp(1, 16), + logger, + event_body_format: config.event_body_format, + }; + + let want_mail = config + .objects + .as_ref() + .map(|set| { + set.iter() + .any(|o| matches!(o, ObjectType::Mailbox | ObjectType::Email)) + }) + .unwrap_or(true); + let want_calendar = config + .objects + .as_ref() + .map(|set| { + set.iter() + .any(|o| matches!(o, ObjectType::Calendar | ObjectType::CalendarEvent)) + }) + .unwrap_or(true) + && !matches!(config.mailbox_kind, MailboxKind::Archive); + let want_contacts = config + .objects + .as_ref() + .map(|set| { + set.iter() + .any(|o| matches!(o, ObjectType::AddressBook | ObjectType::ContactCard)) + }) + .unwrap_or(true) + && !matches!(config.mailbox_kind, MailboxKind::Archive); + + if want_mail { + let folders = super::folders::reconcile_mail( + &mut conn, + &ctx, + config.mailbox_kind, + &mut mailbox_counts, + )?; + if logger.enabled(LEVEL_DEFAULT) { + eprintln!( + "graph mail folders: created={} fetched={} deleted={}", + mailbox_counts.created, mailbox_counts.fetched, mailbox_counts.deleted + ); + } + super::messages::reconcile_all(&mut conn, &ctx, &folders, &mut email_counts)?; + } + + if want_calendar { + let calendars = super::folders::reconcile_calendars(&mut conn, &ctx, &mut calendar_counts)?; + super::calendar::reconcile_all(&mut conn, &ctx, &calendars, &mut event_counts)?; + } + + if want_contacts { + let books = + super::folders::reconcile_address_books(&mut conn, &ctx, &mut addressbook_counts)?; + super::contacts::reconcile_all(&mut conn, &ctx, &books, &mut contact_counts)?; + } + + summary.per_type.push(("mailbox", mailbox_counts)); + summary.per_type.push(("email", email_counts)); + summary.per_type.push(("calendar", calendar_counts)); + summary.per_type.push(("calendarevent", event_counts)); + summary.per_type.push(("addressbook", addressbook_counts)); + summary.per_type.push(("contactcard", contact_counts)); + + if !summary.any_failed() + && let Err(e) = run_gc(&conn) + { + logger.warn(&format!("blob GC skipped: {e}")); + } + + summary.retries_observed = client.retries_observed(); + summary.retry_after_sleeps = client.retry_after_sleeps(); + Ok(summary) +} + +fn run_dry( + client: &GraphClient, + endpoints: &Endpoints, + mailbox_kind: MailboxKind, + logger: &crate::logging::Logger, + top: usize, +) -> Result { + let mut mailbox_counts = TypeCounts::default(); + let mut calendar_counts = TypeCounts::default(); + let mut addressbook_counts = TypeCounts::default(); + let folders = + enumerate_mail_folders(client, endpoints, mailbox_kind, top).map_err(Error::from)?; + mailbox_counts.created = folders.len() as u64; + if !matches!(mailbox_kind, MailboxKind::Archive) { + let calendars = + crate::exchange_graph::api::collect_all_values(client, &endpoints.calendars(top), &[]) + .map_err(Error::from)?; + calendar_counts.created = calendars.len() as u64; + let books = crate::exchange_graph::api::collect_all_values( + client, + &endpoints.contact_folders(top), + &[], + ) + .map_err(Error::from)?; + addressbook_counts.created = books.len() as u64; + } + if logger.enabled(LEVEL_DEFAULT) { + eprintln!( + "dry-run: mailbox={} calendar={} addressbook={}", + mailbox_counts.created, calendar_counts.created, addressbook_counts.created + ); + } + let mut summary = Summary::default(); + summary.per_type.push(("mailbox", mailbox_counts)); + summary.per_type.push(("calendar", calendar_counts)); + summary.per_type.push(("addressbook", addressbook_counts)); + Ok(summary) +} + +pub fn enumerate_mail_folders( + client: &GraphClient, + endpoints: &Endpoints, + mailbox_kind: MailboxKind, + top: usize, +) -> Result, GraphError> { + let mut all = Vec::new(); + let mut frontier: Vec = Vec::new(); + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + let initial = endpoints.mail_folders_root(mailbox_kind, top); + let level = crate::exchange_graph::api::collect_all_values(client, &initial, &[])?; + for f in &level { + if let Some(id) = f.get("id").and_then(Value::as_str) + && seen.insert(id.to_owned()) + { + frontier.push(id.to_owned()); + } + } + all.extend(level); + while let Some(parent) = frontier.pop() { + let url = endpoints.mail_folder_child_folders(&parent, top); + let children = crate::exchange_graph::api::collect_all_values(client, &url, &[])?; + for c in &children { + if let Some(id) = c.get("id").and_then(Value::as_str) + && seen.insert(id.to_owned()) + { + frontier.push(id.to_owned()); + } + } + all.extend(children); + } + Ok(all) +} + +fn acquire_with_flow(auth: &GraphAuth, allow_invalid_certs: bool) -> Result { + let flow = match auth { + GraphAuth::PreAcquired { token } => OAuthFlow::PreAcquired { + token: token.clone(), + }, + GraphAuth::DeviceCode { + authority, + client_id, + } => OAuthFlow::DeviceCode { + authority: authority.clone(), + client_id: client_id.clone(), + }, + }; + acquire(&flow, allow_invalid_certs).map_err(Error::from) +} + +fn resolve_endpoints(config: &GraphImportConfig, client: &GraphClient) -> Result { + if let Some(target) = config.user_target.as_deref() { + let resolved = if looks_like_uuid(target) { + target.to_owned() + } else { + resolve_user_id(client, &config.api_base, target) + .map_err(|e| Error::Connection(format!("user resolution: {e}")))? + }; + Ok(Endpoints::for_user(&config.api_base, &resolved)) + } else { + Ok(Endpoints::for_me(&config.api_base)) + } +} + +fn looks_like_uuid(s: &str) -> bool { + s.len() == 36 + && s.chars().enumerate().all(|(i, c)| match i { + 8 | 13 | 18 | 23 => c == '-', + _ => c.is_ascii_hexdigit(), + }) +} + +fn resolve_user_id( + client: &GraphClient, + api_base: &str, + upn_or_id: &str, +) -> Result { + let url = format!( + "{}/users/{}?$select=id,userPrincipalName", + api_base.trim_end_matches('/'), + upn_or_id + ); + let body = client.get_json(&url)?; + let id = body + .get("id") + .and_then(Value::as_str) + .ok_or_else(|| GraphError::Malformed("user resolution missing id".to_owned()))? + .to_owned(); + Ok(id) +} + +fn resolve_principal( + client: &GraphClient, + endpoints: &Endpoints, + fallback_upn: Option<&str>, +) -> Result { + let url = endpoints.me_select_id_upn(); + let value = client.get_json(&url).map_err(Error::from)?; + let id = value + .get("id") + .and_then(Value::as_str) + .ok_or_else(|| Error::Connection("graph principal has no id".to_owned()))? + .to_owned(); + let upn = value + .get("userPrincipalName") + .and_then(Value::as_str) + .map(str::to_owned) + .or_else(|| fallback_upn.map(str::to_owned)) + .unwrap_or_default(); + Ok(crate::exchange_graph::ResolvedPrincipal { + id, + user_principal_name: upn, + }) +} + +fn canonical_session_url(config: &GraphImportConfig) -> String { + let authority = match &config.auth { + GraphAuth::DeviceCode { authority, .. } => authority.clone(), + GraphAuth::PreAcquired { .. } => default_authority("common"), + }; + format!( + "{}|{}", + authority.trim_end_matches('/'), + config.api_base.trim_end_matches('/') + ) +} + +fn spawn_token_refresher( + client: &GraphClient, + auth: &GraphAuth, + initial: &AcquiredToken, + allow_invalid_certs: bool, + logger: crate::logging::Logger, +) -> Option { + let (authority, client_id) = match auth { + GraphAuth::DeviceCode { + authority, + client_id, + } => (authority.clone(), client_id.clone()), + GraphAuth::PreAcquired { .. } => return None, + }; + let refresh = initial.refresh_token.clone()?; + let mut deadline_unix = initial_deadline_unix(initial)?; + let client = client.clone(); + let mut refresh_token = refresh; + let shutdown = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let shutdown_signal = shutdown.clone(); + let handle = std::thread::Builder::new() + .name("vandelay-graph-token-refresh".to_owned()) + .spawn(move || { + while !shutdown_signal.load(std::sync::atomic::Ordering::Relaxed) { + let now = unix_now(); + let refresh_at = deadline_unix.saturating_sub(60); + if refresh_at > now { + let wait = std::time::Duration::from_secs(refresh_at - now); + let chunk = std::time::Duration::from_secs(2); + let mut left = wait; + while left > std::time::Duration::ZERO + && !shutdown_signal.load(std::sync::atomic::Ordering::Relaxed) + { + let step = chunk.min(left); + std::thread::sleep(step); + left = left.saturating_sub(step); + } + if shutdown_signal.load(std::sync::atomic::Ordering::Relaxed) { + break; + } + } + match refresh_access_token( + &authority, + &client_id, + &refresh_token, + allow_invalid_certs, + ) { + Ok(tok) => { + client.set_bearer(tok.access_token.clone()); + if let Some(new_refresh) = tok.refresh_token { + refresh_token = new_refresh; + } + deadline_unix = unix_now() + + tok.expires_in.unwrap_or_else(|| { + crate::exchange_graph::oauth::decode_jwt_claims(&tok.access_token) + .and_then(|c| c.exp) + .and_then(|exp| exp.checked_sub(unix_now())) + .unwrap_or(50 * 60) + }); + } + Err(e) => { + logger.warn(&format!( + "graph token refresh failed: {e}; sleeping 60s before retry" + )); + std::thread::sleep(std::time::Duration::from_secs(60)); + } + } + } + }) + .ok()?; + Some(TokenRefresher { + shutdown, + handle: Some(handle), + }) +} + +pub struct TokenRefresher { + shutdown: std::sync::Arc, + handle: Option>, +} + +impl Drop for TokenRefresher { + fn drop(&mut self) { + self.shutdown + .store(true, std::sync::atomic::Ordering::Relaxed); + if let Some(h) = self.handle.take() { + let _ = h.join(); + } + } +} + +fn unix_now() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +fn initial_deadline_unix(initial: &AcquiredToken) -> Option { + if let Some(expires_in) = initial.expires_in { + return Some(unix_now() + expires_in); + } + crate::exchange_graph::oauth::decode_jwt_claims(&initial.access_token).and_then(|c| c.exp) +} + +fn run_gc(conn: &Connection) -> Result<(), Error> { + let tx = conn.unchecked_transaction()?; + db::blobs::gc_orphan_blobs(&tx)?; + tx.commit()?; + Ok(()) +} + +pub fn default_api_base() -> String { + DEFAULT_API_BASE.to_owned() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn canonical_session_url_concatenates_authority_and_base() { + let config = GraphImportConfig { + auth: GraphAuth::DeviceCode { + authority: "https://login.microsoftonline.com/common".to_owned(), + client_id: "uuid".to_owned(), + }, + api_base: "https://graph.microsoft.com/v1.0".to_owned(), + user_target: None, + mailbox_kind: MailboxKind::Primary, + objects: None, + event_body_format: EventBodyFormat::Text, + graph_connections: 4, + top: 100, + allow_source_change: false, + }; + let url = canonical_session_url(&config); + assert_eq!( + url, + "https://login.microsoftonline.com/common|https://graph.microsoft.com/v1.0" + ); + } + + #[test] + fn looks_like_uuid_only_matches_well_formed() { + assert!(looks_like_uuid("12345678-1234-1234-1234-123456789abc")); + assert!(!looks_like_uuid("alice@x.com")); + assert!(!looks_like_uuid("12345678123412341234123456789abc")); + } +} diff --git a/src/sync/import_exchange_graph/folders.rs b/src/sync/import_exchange_graph/folders.rs new file mode 100644 index 0000000..b9468b7 --- /dev/null +++ b/src/sync/import_exchange_graph/folders.rs @@ -0,0 +1,531 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use rusqlite::{Connection, params}; +use serde_json::Value; + +use crate::db::exchange_graph_ids; +use crate::error::Error; +use crate::exchange_graph::api; +use crate::exchange_graph::calendar_map::{graph_calendar_color_to_hex, windows_or_iana_to_iana}; +use crate::exchange_graph::types::MailboxKind; +use crate::logging::LEVEL_PROGRESS; +use crate::sync::TypeCounts; + +use super::coordinator::{CHUNK_SIZE, GraphCoordinator, enumerate_mail_folders}; + +#[derive(Debug, Clone)] +pub struct MailFolder { + pub graph_id: String, + pub parent_graph_id: Option, + pub display_name: String, + pub is_hidden: bool, + pub local_id: i64, +} + +#[derive(Debug, Clone)] +pub struct CalendarFolder { + pub graph_id: String, + pub local_id: i64, +} + +#[derive(Debug, Clone)] +pub struct ContactFolder { + pub graph_id: String, + pub local_id: i64, +} + +pub fn reconcile_mail( + conn: &mut Connection, + ctx: &GraphCoordinator<'_>, + mailbox_kind: MailboxKind, + counts: &mut TypeCounts, +) -> Result, Error> { + let server = enumerate_mail_folders(ctx.client, ctx.endpoints, mailbox_kind, ctx.top) + .map_err(Error::from)?; + if ctx.logger.enabled(LEVEL_PROGRESS) { + eprintln!("graph mailFolders enumerated: {}", server.len()); + } + let well_known = match mailbox_kind { + MailboxKind::Primary => resolve_well_known_roles(ctx), + MailboxKind::Archive => HashMap::new(), + }; + let local: HashMap = + exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::MAILBOX)?; + + let entries = order_by_parent(server); + let mut by_id: HashMap = HashMap::new(); + let mut out: Vec = Vec::new(); + let mut server_ids: Vec = Vec::new(); + + for chunk in entries.chunks(CHUNK_SIZE) { + let tx = conn.unchecked_transaction()?; + for value in chunk { + let Some(graph_id) = value.get("id").and_then(Value::as_str) else { + continue; + }; + let parent_graph_id = value + .get("parentFolderId") + .and_then(Value::as_str) + .map(str::to_owned); + let display_name = value + .get("displayName") + .and_then(Value::as_str) + .unwrap_or("Unnamed") + .to_owned(); + let is_hidden = value + .get("isHidden") + .and_then(Value::as_bool) + .unwrap_or(false); + let role = well_known.get(graph_id).copied(); + let parent_local_id = parent_graph_id + .as_deref() + .and_then(|p| by_id.get(p)) + .copied(); + let existing = local.get(graph_id).copied(); + server_ids.push(graph_id.to_owned()); + + let local_id = if let Some(id) = existing { + tx.execute( + "UPDATE mailboxes SET name = ?1, parent_id = ?2, role = ?3, is_subscribed = ?4 + WHERE id = ?5", + params![display_name, parent_local_id, role, !is_hidden as i64, id,], + )?; + counts.fetched += 1; + id + } else { + tx.execute( + "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) + VALUES (?1, ?2, ?3, 0, ?4)", + params![display_name, parent_local_id, role, !is_hidden as i64], + )?; + let new_id = tx.last_insert_rowid(); + exchange_graph_ids::insert( + &tx, + ctx.source_id, + exchange_graph_ids::MAILBOX, + graph_id, + new_id, + )?; + counts.created += 1; + new_id + }; + by_id.insert(graph_id.to_owned(), local_id); + out.push(MailFolder { + graph_id: graph_id.to_owned(), + parent_graph_id, + display_name, + is_hidden, + local_id, + }); + } + tx.commit()?; + } + + delete_vanished_mailboxes( + conn, + ctx.source_id, + &local, + &server_ids, + counts, + &ctx.logger, + )?; + + Ok(out) +} + +pub fn reconcile_calendars( + conn: &mut Connection, + ctx: &GraphCoordinator<'_>, + counts: &mut TypeCounts, +) -> Result, Error> { + let server = api::collect_all_values(ctx.client, &ctx.endpoints.calendars(ctx.top), &[]) + .map_err(Error::from)?; + let mailbox_tz = mailbox_timezone(ctx); + let local: HashMap = + exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::CALENDAR)?; + let mut out: Vec = Vec::new(); + let mut server_ids: Vec = Vec::new(); + + for chunk in server.chunks(CHUNK_SIZE) { + let tx = conn.unchecked_transaction()?; + for value in chunk { + let Some(graph_id) = value.get("id").and_then(Value::as_str) else { + continue; + }; + server_ids.push(graph_id.to_owned()); + let name = value + .get("name") + .and_then(Value::as_str) + .unwrap_or("Calendar") + .to_owned(); + let color = value + .get("hexColor") + .and_then(Value::as_str) + .map(str::to_owned) + .or_else(|| { + value + .get("color") + .and_then(Value::as_str) + .and_then(graph_calendar_color_to_hex) + .map(str::to_owned) + }); + let is_default = value + .get("isDefaultCalendar") + .and_then(Value::as_bool) + .unwrap_or(false); + let existing = local.get(graph_id).copied(); + let tz = mailbox_tz.clone(); + let local_id = if let Some(id) = existing { + tx.execute( + "UPDATE calendars SET name = ?1, color = ?2, is_default = ?3, time_zone = ?4 + WHERE id = ?5", + params![name, color, is_default as i64, tz, id], + )?; + counts.fetched += 1; + id + } else { + tx.execute( + "INSERT INTO calendars (name, color, sort_order, is_subscribed, is_visible, + is_default, include_in_availability, time_zone) + VALUES (?1, ?2, 0, 1, 1, ?3, 'all', ?4)", + params![name, color, is_default as i64, tz], + )?; + let new_id = tx.last_insert_rowid(); + exchange_graph_ids::insert( + &tx, + ctx.source_id, + exchange_graph_ids::CALENDAR, + graph_id, + new_id, + )?; + counts.created += 1; + new_id + }; + out.push(CalendarFolder { + graph_id: graph_id.to_owned(), + local_id, + }); + } + tx.commit()?; + } + + delete_vanished_flat( + conn, + ctx.source_id, + exchange_graph_ids::CALENDAR, + "calendars", + &local, + &server_ids, + counts, + )?; + Ok(out) +} + +pub fn reconcile_address_books( + conn: &mut Connection, + ctx: &GraphCoordinator<'_>, + counts: &mut TypeCounts, +) -> Result, Error> { + let mut server = + api::collect_all_values(ctx.client, &ctx.endpoints.contact_folders(ctx.top), &[]) + .map_err(Error::from)?; + let mut seen: std::collections::HashSet = server + .iter() + .filter_map(|f| f.get("id").and_then(Value::as_str)) + .map(str::to_owned) + .collect(); + let mut frontier: Vec = seen.iter().cloned().collect(); + while let Some(parent) = frontier.pop() { + let url = ctx.endpoints.contact_folder_children(&parent, ctx.top); + let children = api::collect_all_values(ctx.client, &url, &[]).map_err(Error::from)?; + for c in &children { + if let Some(id) = c.get("id").and_then(Value::as_str) + && seen.insert(id.to_owned()) + { + frontier.push(id.to_owned()); + } + } + server.extend(children); + } + + let local: HashMap = + exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::ADDRESS_BOOK)?; + let mut out: Vec = Vec::new(); + let mut server_ids: Vec = Vec::new(); + for chunk in server.chunks(CHUNK_SIZE) { + let tx = conn.unchecked_transaction()?; + for value in chunk { + let Some(graph_id) = value.get("id").and_then(Value::as_str) else { + continue; + }; + server_ids.push(graph_id.to_owned()); + let name = value + .get("displayName") + .and_then(Value::as_str) + .unwrap_or("Contacts") + .to_owned(); + let existing = local.get(graph_id).copied(); + let local_id = if let Some(id) = existing { + tx.execute( + "UPDATE address_books SET name = ?1 WHERE id = ?2", + params![name, id], + )?; + counts.fetched += 1; + id + } else { + tx.execute( + "INSERT INTO address_books (name, sort_order, is_subscribed) + VALUES (?1, 0, 1)", + params![name], + )?; + let new_id = tx.last_insert_rowid(); + exchange_graph_ids::insert( + &tx, + ctx.source_id, + exchange_graph_ids::ADDRESS_BOOK, + graph_id, + new_id, + )?; + counts.created += 1; + new_id + }; + out.push(ContactFolder { + graph_id: graph_id.to_owned(), + local_id, + }); + } + tx.commit()?; + } + + delete_vanished_flat( + conn, + ctx.source_id, + exchange_graph_ids::ADDRESS_BOOK, + "address_books", + &local, + &server_ids, + counts, + )?; + Ok(out) +} + +fn mailbox_timezone(ctx: &GraphCoordinator<'_>) -> Option { + let url = ctx.endpoints.mailbox_settings_timezone(); + let value = ctx.client.get_json(&url).ok()?; + let tz = value.get("timeZone").and_then(Value::as_str)?; + windows_or_iana_to_iana(tz) +} + +fn resolve_well_known_roles(ctx: &GraphCoordinator<'_>) -> HashMap { + let mapping: &[(&str, &str)] = &[ + ("inbox", "inbox"), + ("drafts", "drafts"), + ("sentitems", "sent"), + ("deleteditems", "trash"), + ("junkemail", "junk"), + ("archive", "archive"), + ]; + let mut out = HashMap::new(); + for (short_name, role) in mapping { + let url = ctx.endpoints.well_known_folder(short_name); + match ctx.client.get_json(&url) { + Ok(value) => { + if let Some(id) = value.get("id").and_then(Value::as_str) { + out.insert(id.to_owned(), *role); + } + } + Err(_) => continue, + } + } + out +} + +fn order_by_parent(server: Vec) -> Vec { + let mut parents: HashMap> = HashMap::new(); + for v in &server { + if let Some(id) = v.get("id").and_then(Value::as_str) { + let parent = v + .get("parentFolderId") + .and_then(Value::as_str) + .map(str::to_owned); + parents.insert(id.to_owned(), parent); + } + } + fn depth_of( + id: &str, + parents: &HashMap>, + memo: &mut HashMap, + seen: &mut std::collections::HashSet, + ) -> usize { + if let Some(d) = memo.get(id) { + return *d; + } + if !seen.insert(id.to_owned()) { + return 0; + } + let parent = parents.get(id).and_then(|p| p.as_deref()); + let d = match parent { + Some(p) if parents.contains_key(p) => 1 + depth_of(p, parents, memo, seen), + _ => 0, + }; + memo.insert(id.to_owned(), d); + d + } + let mut memo: HashMap = HashMap::new(); + let mut entries: Vec<(usize, Value)> = server + .into_iter() + .map(|v| { + let id = v.get("id").and_then(Value::as_str).unwrap_or("").to_owned(); + let mut seen = std::collections::HashSet::new(); + let d = depth_of(&id, &parents, &mut memo, &mut seen); + (d, v) + }) + .collect(); + entries.sort_by_key(|(d, _)| *d); + entries.into_iter().map(|(_, v)| v).collect() +} + +fn delete_vanished_flat( + conn: &mut Connection, + source_id: i64, + type_name: &str, + table: &str, + local: &HashMap, + server_ids: &[String], + counts: &mut TypeCounts, +) -> Result<(), Error> { + let server_set: std::collections::HashSet<&str> = + server_ids.iter().map(String::as_str).collect(); + let vanished: Vec<(&String, &i64)> = local + .iter() + .filter(|(graph_id, _)| !server_set.contains(graph_id.as_str())) + .collect(); + for chunk in vanished.chunks(CHUNK_SIZE) { + let tx = conn.unchecked_transaction()?; + for (graph_id, local_id) in chunk { + let result = tx.execute( + &format!("DELETE FROM {table} WHERE id = ?1"), + params![local_id], + ); + match result { + Ok(_) => { + exchange_graph_ids::delete(&tx, source_id, type_name, graph_id)?; + counts.deleted += 1; + } + Err(_) => { + counts.failed += 1; + } + } + } + tx.commit()?; + } + Ok(()) +} + +fn delete_vanished_mailboxes( + conn: &mut Connection, + source_id: i64, + local: &HashMap, + server_ids: &[String], + counts: &mut TypeCounts, + logger: &crate::logging::Logger, +) -> Result<(), Error> { + let server_set: std::collections::HashSet<&str> = + server_ids.iter().map(String::as_str).collect(); + let mut vanished: Vec<(String, i64)> = local + .iter() + .filter(|(id, _)| !server_set.contains(id.as_str())) + .map(|(id, lid)| (id.clone(), *lid)) + .collect(); + let depths = mailbox_depths(conn, &vanished)?; + vanished.sort_by_key(|(_, local_id)| std::cmp::Reverse(*depths.get(local_id).unwrap_or(&0))); + + for (graph_id, local_id) in vanished { + let tx = conn.unchecked_transaction()?; + let result = tx.execute("DELETE FROM mailboxes WHERE id = ?1", params![local_id]); + match result { + Ok(_) => { + exchange_graph_ids::delete(&tx, source_id, exchange_graph_ids::MAILBOX, &graph_id)?; + tx.commit()?; + counts.deleted += 1; + } + Err(e) => { + let _ = tx.rollback(); + logger.warn(&format!( + "mailbox {graph_id} (local id {local_id}) could not be deleted (live children?): {e}" + )); + counts.failed += 1; + } + } + } + Ok(()) +} + +fn mailbox_depths(conn: &Connection, rows: &[(String, i64)]) -> Result, Error> { + let mut parents: HashMap> = HashMap::new(); + { + let mut stmt = conn.prepare("SELECT id, parent_id FROM mailboxes")?; + let mut iter = stmt.query([])?; + while let Some(row) = iter.next()? { + let id: i64 = row.get(0)?; + let parent: Option = row.get(1)?; + parents.insert(id, parent); + } + } + let mut depths: HashMap = HashMap::new(); + for (_, local_id) in rows { + depths.insert(*local_id, depth_of(*local_id, &parents)); + } + Ok(depths) +} + +fn depth_of(id: i64, parents: &HashMap>) -> usize { + let mut depth = 0; + let mut cursor = id; + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + while seen.insert(cursor) { + match parents.get(&cursor).copied().flatten() { + Some(p) => { + depth += 1; + cursor = p; + } + None => break, + } + } + depth +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn order_by_parent_returns_parents_first() { + let input = vec![ + json!({"id": "C", "parentFolderId": "B"}), + json!({"id": "B", "parentFolderId": "A"}), + json!({"id": "A"}), + ]; + let out = order_by_parent(input); + assert_eq!(out[0]["id"], "A"); + assert_eq!(out[1]["id"], "B"); + assert_eq!(out[2]["id"], "C"); + } + + #[test] + fn order_by_parent_tolerates_missing_parent() { + let input = vec![ + json!({"id": "Orphan", "parentFolderId": "Missing"}), + json!({"id": "A"}), + ]; + let out = order_by_parent(input); + assert_eq!(out.len(), 2); + } +} diff --git a/src/sync/import_exchange_graph/messages.rs b/src/sync/import_exchange_graph/messages.rs new file mode 100644 index 0000000..9b136a9 --- /dev/null +++ b/src/sync/import_exchange_graph/messages.rs @@ -0,0 +1,202 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::{Connection, Transaction, params}; +use serde_json::json; + +use crate::db::{blobs, exchange_graph_ids}; +use crate::error::Error; +use crate::exchange_graph::api; +use crate::exchange_graph::client::Accept; +use crate::exchange_graph::error::GraphError; +use crate::logging::LEVEL_PROGRESS; +use crate::sync::TypeCounts; +use crate::sync::emailmeta::email_meta_from_blob; +use crate::sync::import_jmap::pool::Pool; +use crate::sync::keys::index_to_json; + +use super::coordinator::{CHUNK_SIZE, GraphCoordinator}; +use super::folders::MailFolder; + +pub fn reconcile_all( + conn: &mut Connection, + ctx: &GraphCoordinator<'_>, + folders: &[MailFolder], + counts: &mut TypeCounts, +) -> Result<(), Error> { + let local: std::collections::HashMap = + exchange_graph_ids::ids_of_type(conn, ctx.source_id, exchange_graph_ids::EMAIL)?; + let mut server_total: std::collections::HashSet = std::collections::HashSet::new(); + let mut any_failure = false; + + for folder in folders { + let url = ctx.endpoints.folder_messages_ids(&folder.graph_id, ctx.top); + let ids = match api::collect_all_ids(ctx.client, &url, &[]) { + Ok(v) => v, + Err(e) => { + ctx.logger.warn(&format!( + "folder {} message enumeration failed: {e}", + folder.graph_id + )); + counts.failed += 1; + any_failure = true; + continue; + } + }; + if ctx.logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "graph folder {} enumerated {} messages", + folder.graph_id, + ids.len() + ); + } + for id in &ids { + server_total.insert(id.clone()); + } + let new_ids: Vec = ids + .into_iter() + .filter(|id| !local.contains_key(id)) + .collect(); + if new_ids.is_empty() { + continue; + } + fetch_and_insert(conn, ctx, folder, &new_ids, counts)?; + } + + if any_failure { + ctx.logger.warn( + "graph message vanished-cleanup skipped: one or more folders failed to enumerate; \ + a clean re-run will reconcile deletions", + ); + } else { + delete_vanished(conn, ctx.source_id, &local, &server_total, counts)?; + } + Ok(()) +} + +fn fetch_and_insert( + conn: &mut Connection, + ctx: &GraphCoordinator<'_>, + folder: &MailFolder, + ids: &[String], + counts: &mut TypeCounts, +) -> Result<(), Error> { + let client = ctx.client.clone(); + let endpoints: crate::exchange_graph::api::Endpoints = (*ctx.endpoints).clone(); + type FetchResult = (String, Result, GraphError>); + let pool: Pool = Pool::new(ctx.workers, move |id: String| { + let url = endpoints.message_mime(&id); + match client.get_with_prefer(&url, Accept::Text, &[]) { + Ok(resp) => (id, Ok(resp.body)), + Err(e) => (id, Err(e)), + } + }); + for id in ids { + pool.submit(id.clone()); + } + let mut tx_opt: Option> = None; + let mut in_batch: usize = 0; + for _ in 0..ids.len() { + let Ok((graph_id, result)) = pool.results().recv() else { + break; + }; + match result { + Ok(bytes) => { + if tx_opt.is_none() { + tx_opt = Some(conn.unchecked_transaction()?); + } + let tx = tx_opt.as_mut().expect("tx is Some"); + apply_message_in_tx(tx, ctx, folder, &graph_id, &bytes, counts)?; + in_batch += 1; + if in_batch >= CHUNK_SIZE { + if let Some(t) = tx_opt.take() { + t.commit()?; + } + in_batch = 0; + } + } + Err(GraphError::Vanished) => { + counts.skipped += 1; + } + Err(e) => { + counts.failed += 1; + ctx.logger + .warn(&format!("graph message {graph_id} fetch failed: {e}")); + } + } + } + if let Some(t) = tx_opt.take() { + t.commit()?; + } + Ok(()) +} + +fn apply_message_in_tx( + tx: &Transaction<'_>, + ctx: &GraphCoordinator<'_>, + folder: &MailFolder, + graph_id: &str, + bytes: &[u8], + counts: &mut TypeCounts, +) -> Result<(), Error> { + let (idx, date_header) = email_meta_from_blob(bytes); + let message_match = index_to_json(&idx); + let received_at = date_header.unwrap_or_else(|| "1970-01-01T00:00:00Z".to_owned()); + let mailbox_ids = json!([folder.local_id]).to_string(); + let keywords = "[]".to_owned(); + + let blob_id = blobs::intern_blob(tx, bytes)?; + tx.execute( + "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![blob_id, received_at, mailbox_ids, keywords, message_match], + )?; + let new_id = tx.last_insert_rowid(); + exchange_graph_ids::insert( + tx, + ctx.source_id, + exchange_graph_ids::EMAIL, + graph_id, + new_id, + )?; + counts.created += 1; + Ok(()) +} + +fn delete_vanished( + conn: &mut Connection, + source_id: i64, + local: &std::collections::HashMap, + server: &std::collections::HashSet, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let vanished: Vec<(&String, &i64)> = local + .iter() + .filter(|(graph_id, _)| !server.contains(graph_id.as_str())) + .collect(); + for chunk in vanished.chunks(CHUNK_SIZE) { + let tx = conn.unchecked_transaction()?; + for (graph_id, local_id) in chunk { + let result = tx.execute("DELETE FROM emails WHERE id = ?1", params![local_id]); + match result { + Ok(_) => { + exchange_graph_ids::delete( + &tx, + source_id, + exchange_graph_ids::EMAIL, + graph_id, + )?; + counts.deleted += 1; + } + Err(_) => { + counts.failed += 1; + } + } + } + tx.commit()?; + } + Ok(()) +} diff --git a/src/sync/import_imap.rs b/src/sync/import_imap.rs new file mode 100644 index 0000000..351d71f --- /dev/null +++ b/src/sync/import_imap.rs @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod fetch; +pub mod folders; +pub mod internaldate; +pub mod keywords; +pub mod messages; +pub mod pool; + +pub mod coordinator; + +pub use coordinator::{ImapAuth, ImapImportConfig, run}; diff --git a/src/sync/import_imap/coordinator.rs b/src/sync/import_imap/coordinator.rs new file mode 100644 index 0000000..d0d96a8 --- /dev/null +++ b/src/sync/import_imap/coordinator.rs @@ -0,0 +1,1141 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; + +use crossbeam_channel::RecvTimeoutError; +use regex::Regex; +use rusqlite::{Connection, OptionalExtension, params}; +use serde_json::Value; +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; +use url::Url; + +use crate::db; +use crate::db::sources::SourceKey; +use crate::error::Error; +use crate::imap::client::{ConnectMode, ImapClient}; +use crate::imap::command; +use crate::imap::error::ImapError; +use crate::imap::name::encode_mailbox_name_with; +use crate::imap::response::{NamespaceEntry, Untagged}; +use crate::imap::retry::{ + BackoffState, Disposition, RetryPolicy, classify, is_negotiation_failure, +}; +use crate::imap::transport::Connector; +use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; +use crate::sync::emailmeta::email_index_from_blob; +use crate::sync::keys::index_to_json; +use crate::sync::{CommonConfig, Summary, TypeCounts}; + +use super::fetch; +use super::folders::{ + FolderFilters, ResolvedFolder, apply_filters, collect_from_list, sort_by_depth, + vanished_depth_sort, vanished_folders, +}; +use super::internaldate::imap_internaldate_to_rfc3339; +use super::keywords::translate_flags; +use super::messages::{chunks, diff_uids, select_uids}; +use super::pool::{FetchEvent, FetchJob, WorkerArgs, WorkerPool}; + +const EMAIL_TYPE: &str = "email"; + +#[derive(Clone, Copy)] +pub(super) struct RunOpts { + source_id: i64, + fetch_batch: usize, + include_deleted: bool, + logger: Logger, +} + +#[derive(Clone)] +pub(super) struct ControlCtx { + pub connector: std::sync::Arc, + pub endpoint: std::sync::Arc, + pub mode: ConnectMode, + pub auth: ImapAuth, + pub compress: bool, + pub policy: RetryPolicy, + pub backoff: BackoffState, + pub logger: Logger, +} + +pub(super) fn control_run_collect( + client: &mut ImapClient, + ctx: &ControlCtx, + command: &str, +) -> Result { + let mut transient_attempts: u32 = 0; + let mut transport_attempts: u32 = 0; + loop { + match client.run_collect(command) { + Ok(r) => { + ctx.backoff.reset(); + return Ok(r); + } + Err(e) => match classify(&e) { + Disposition::TransportDrop => { + if transport_attempts >= ctx.policy.max_retries { + return Err(Error::Connection(format!("control connection lost: {e}"))); + } + transport_attempts += 1; + std::thread::sleep(ctx.backoff.transport_delay(transport_attempts)); + match reconnect_control(client, ctx) { + Ok(()) => { + log_at(ctx.logger, LEVEL_DEFAULT, "control connection reconnected"); + } + Err(e2) => { + return Err(Error::Connection(format!( + "control reconnect failed: {e2}" + ))); + } + } + } + Disposition::Transient => { + if transient_attempts >= ctx.policy.max_retries { + return Err(Error::Partial(format!( + "{command:?}: {e} (max retries exhausted)" + ))); + } + transient_attempts += 1; + std::thread::sleep(ctx.backoff.next_shared_delay()); + } + _ => { + return Err(Error::Partial(format!("{command:?}: {e}"))); + } + }, + } + } +} + +pub(super) fn call_with_retry( + client: &mut ImapClient, + ctx: &ControlCtx, + mut attempt: F, +) -> Result +where + F: FnMut(&mut ImapClient) -> Result, +{ + let mut transient_attempts: u32 = 0; + let mut transport_attempts: u32 = 0; + loop { + match attempt(client) { + Ok(v) => { + ctx.backoff.reset(); + return Ok(v); + } + Err(e) => match classify(&e) { + Disposition::TransportDrop => { + if transport_attempts >= ctx.policy.max_retries { + return Err(e); + } + transport_attempts += 1; + std::thread::sleep(ctx.backoff.transport_delay(transport_attempts)); + reconnect_control(client, ctx)?; + } + Disposition::Transient => { + if transient_attempts >= ctx.policy.max_retries { + return Err(e); + } + transient_attempts += 1; + std::thread::sleep(ctx.backoff.next_shared_delay()); + } + _ => return Err(e), + }, + } + } +} + +fn reconnect_control(client: &mut ImapClient, ctx: &ControlCtx) -> Result<(), ImapError> { + let new_client = ImapClient::connect( + &ctx.connector, + &ctx.endpoint.host, + ctx.endpoint.port, + ctx.mode, + )?; + *client = new_client; + authenticate_client(client, &ctx.auth).map_err(|e| ImapError::AuthFailed(e.to_string()))?; + let _ = client.refresh_capabilities(); + if ctx.compress && client.has_capability("COMPRESS=DEFLATE") { + client.compress_deflate()?; + } + if client.has_capability("ENABLE") && client.has_capability("UTF8=ACCEPT") { + let _ = client.enable(&["UTF8=ACCEPT"]); + } + Ok(()) +} + +fn log_at(logger: Logger, level: u8, msg: &str) { + if logger.enabled(level) { + println!("{msg}"); + } +} + +pub struct ImapImportConfig { + pub url: String, + pub auth: ImapAuth, + pub allow_cleartext: bool, + pub compress: bool, + pub include: Vec, + pub exclude: Vec, + pub exclude_special: Vec, + pub folder: Vec, + pub subscribed_only: bool, + pub automap: bool, + pub include_deleted: bool, + pub fetch_batch: usize, + pub imap_connections: usize, + pub allow_source_change: bool, +} + +#[derive(Debug, Clone)] +pub enum ImapAuth { + Basic { user: String, password: String }, + Bearer { user: String, token: String }, +} + +pub fn run(common: CommonConfig, config: ImapImportConfig) -> Result { + let logger = common.logger; + let mut conn = db::init::open(&common.archive)?; + + let endpoint = parse_endpoint(&config.url)?; + let session_url = format!( + "{}://{}:{}", + if endpoint.implicit_tls { + "imaps" + } else { + "imap" + }, + endpoint.host, + endpoint.port + ); + + let connector = std::sync::Arc::new( + Connector::new(common.allow_invalid_certs).map_err(|e| Error::Connection(e.to_string()))?, + ); + let endpoint = std::sync::Arc::new(endpoint); + let mode = if endpoint.implicit_tls { + ConnectMode::ImplicitTls + } else if config.allow_cleartext { + ConnectMode::Plain + } else { + ConnectMode::StartTls + }; + let mut client = ImapClient::connect(&connector, &endpoint.host, endpoint.port, mode) + .map_err(|e| Error::Connection(e.to_string()))?; + + let account_id = authenticate(&mut client, &config.auth)?; + log_at( + logger, + LEVEL_PROGRESS, + &format!("authenticated as {account_id} on {session_url}"), + ); + client + .refresh_capabilities() + .map_err(|e| Error::Connection(format!("post-auth CAPABILITY: {e}")))?; + + if config.compress { + if client.has_capability("COMPRESS=DEFLATE") { + client + .compress_deflate() + .map_err(|e| Error::Connection(format!("COMPRESS DEFLATE: {e}")))?; + log_at(logger, LEVEL_PROGRESS, "COMPRESS DEFLATE enabled"); + } else { + log_at( + logger, + LEVEL_PROGRESS, + "--compress requested but server does not advertise COMPRESS=DEFLATE; ignored", + ); + } + } + + if client.has_capability("ENABLE") && client.has_capability("UTF8=ACCEPT") { + match client.enable(&["UTF8=ACCEPT"]) { + Ok(_) => log_at(logger, LEVEL_PROGRESS, "ENABLE UTF8=ACCEPT"), + Err(e) => log_at( + logger, + LEVEL_PROGRESS, + &format!("ENABLE UTF8=ACCEPT failed, falling back to modified UTF-7: {e}"), + ), + } + } + + let source_key = SourceKey { + kind: "imap".to_owned(), + session_url: session_url.clone(), + account_id: account_id.clone(), + }; + if let Some((existing_url, existing_account)) = + db::sources::conflicting_source(&conn, "imap", &session_url, &account_id)? + && !config.allow_source_change + { + return Err(Error::SourceChange(format!( + "archive already records imap source {existing_url} / {existing_account}; \ + re-run with --allow-source-change or use a fresh archive" + ))); + } + let source_id = if common.dry_run { + -1 + } else { + db::sources::upsert_source(&conn, &source_key, Some(&account_id), &account_id)? + }; + + let backoff = BackoffState::new(); + let policy = RetryPolicy::new(common.max_retries); + let control_ctx = ControlCtx { + connector: connector.clone(), + endpoint: endpoint.clone(), + mode, + auth: config.auth.clone(), + compress: config.compress, + policy, + backoff: backoff.clone(), + logger, + }; + + let namespace_prefix = discover_namespace(&mut client, logger)?; + + let extended_supported = client.has_capability("LIST-EXTENDED"); + let list_status_supported = extended_supported && client.has_capability("LIST-STATUS"); + let list_cmd = if list_status_supported { + command::list_extended( + "", + &["*"], + &[], + &[ + "SPECIAL-USE", + "SUBSCRIBED", + "CHILDREN", + "STATUS (UIDVALIDITY UIDNEXT MESSAGES)", + ], + ) + } else if extended_supported { + command::list_extended("", &["*"], &[], &["SPECIAL-USE", "SUBSCRIBED"]) + } else { + command::list("", "*") + }; + let list_resp = control_run_collect(&mut client, &control_ctx, &list_cmd)?; + let mut all_responses = list_resp.untagged; + if (!extended_supported || !list_status_supported) + && let Ok(lsub_resp) = + control_run_collect(&mut client, &control_ctx, &command::lsub("", "*")) + { + all_responses.extend(lsub_resp.untagged); + } + + let discovered = collect_from_list(&all_responses, client.utf8_accept()) + .map_err(|e| Error::Partial(format!("LIST parse: {e}")))?; + let filters = FolderFilters { + include: config.include, + exclude: config.exclude, + exclude_special: config.exclude_special, + explicit: config.folder, + subscribed_only: config.subscribed_only, + automap_enabled: config.automap, + namespace_prefix, + }; + let mut resolved = apply_filters(discovered, &filters); + sort_by_depth(&mut resolved); + + if common.dry_run { + let existing_source = db::sources::find_source(&conn, &source_key)?; + return dry_run_summary( + &conn, + existing_source, + &mut client, + &control_ctx, + &resolved, + logger, + ); + } + + let mut mailbox_counts = TypeCounts::default(); + let mut email_counts = TypeCounts::default(); + + let local_mailboxes = db::imap_ids::mailbox_folders(&conn, source_id)?; + let server_folder_set: HashSet = resolved.iter().map(|f| f.name.clone()).collect(); + + upsert_mailboxes(&mut conn, source_id, &resolved, &mut mailbox_counts)?; + + let server_delimiter = resolved.iter().find_map(|f| f.delimiter).unwrap_or('/'); + + let mut vanished = vanished_folders(&local_mailboxes, &server_folder_set); + vanished_depth_sort(&mut vanished, server_delimiter); + if !vanished.is_empty() { + delete_vanished_folders( + &mut conn, + source_id, + &vanished, + &mut mailbox_counts, + &mut email_counts, + logger, + )?; + } + + let opts = RunOpts { + source_id, + fetch_batch: config.fetch_batch.max(1), + include_deleted: config.include_deleted, + logger, + }; + + let pool = WorkerPool::start( + WorkerArgs { + connector: connector.clone(), + endpoint: endpoint.clone(), + mode, + auth: config.auth.clone(), + compress: config.compress, + policy, + backoff: backoff.clone(), + }, + config.imap_connections.max(1), + ) + .map_err(|e| Error::Connection(format!("worker pool: {e}")))?; + + for (i, folder) in resolved.iter().enumerate() { + if i > 0 { + let _ = control_run_collect(&mut client, &control_ctx, "NOOP"); + } + match reconcile_folder( + &mut conn, + &mut client, + &control_ctx, + &pool, + folder, + opts, + &mut email_counts, + ) { + Ok(()) => {} + Err(e) => { + log_at( + logger, + LEVEL_DEFAULT, + &format!("folder {:?}: {e}", folder.name), + ); + email_counts.failed += 1; + } + } + } + + pool.shutdown(); + let _ = client.logout(); + + Ok(Summary { + per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)], + retries_observed: backoff.total_retries(), + retry_after_sleeps: backoff.transient_retries() as u64, + }) +} + +#[derive(Debug, Clone)] +pub struct Endpoint { + pub host: String, + pub port: u16, + pub implicit_tls: bool, +} + +fn parse_endpoint(url: &str) -> Result { + let parsed = + Url::parse(url).map_err(|e| Error::Usage(format!("invalid --url {url:?}: {e}")))?; + let scheme = parsed.scheme(); + let implicit_tls = match scheme { + "imaps" => true, + "imap" => false, + other => { + return Err(Error::Usage(format!( + "--url scheme must be imap or imaps, got {other}" + ))); + } + }; + let host = parsed + .host_str() + .ok_or_else(|| Error::Usage(format!("--url missing host: {url}")))? + .to_owned(); + let port = parsed + .port() + .unwrap_or(if implicit_tls { 993 } else { 143 }); + Ok(Endpoint { + host, + port, + implicit_tls, + }) +} + +pub(super) fn authenticate_client(client: &mut ImapClient, auth: &ImapAuth) -> Result<(), Error> { + authenticate(client, auth).map(|_| ()) +} + +fn authenticate(client: &mut ImapClient, auth: &ImapAuth) -> Result { + match auth { + ImapAuth::Basic { user, password } => { + if client.has_capability("AUTH=PLAIN") { + match client.authenticate_plain(user, password) { + Ok(()) => return Ok(user.clone()), + Err(e) if is_auth_failed(&e) => { + return Err(Error::Connection(format!("auth failed: {e}"))); + } + Err(e) if is_negotiation_failure(&e) => {} + Err(e) => return Err(Error::Connection(e.to_string())), + } + } + match client.login(user, password) { + Ok(()) => Ok(user.clone()), + Err(e) if is_auth_failed(&e) => Err(Error::Connection(format!("auth failed: {e}"))), + Err(e) => Err(Error::Connection(format!("LOGIN failed: {e}"))), + } + } + ImapAuth::Bearer { user, token } => { + let has_xoauth2 = client.has_capability("AUTH=XOAUTH2"); + let has_oauthbearer = client.has_capability("AUTH=OAUTHBEARER"); + if !has_xoauth2 && !has_oauthbearer { + return Err(Error::Connection( + "server does not advertise AUTH=XOAUTH2 or AUTH=OAUTHBEARER".into(), + )); + } + if has_xoauth2 { + match client.authenticate_xoauth2(user, token) { + Ok(()) => return Ok(user.clone()), + Err(e) if is_auth_failed(&e) => { + return Err(Error::Connection(format!("auth failed: {e}"))); + } + Err(e) if is_negotiation_failure(&e) && has_oauthbearer => {} + Err(e) => return Err(Error::Connection(format!("XOAUTH2 failed: {e}"))), + } + } + client + .authenticate_oauthbearer(user, token) + .map_err(|e| Error::Connection(format!("OAUTHBEARER failed: {e}")))?; + Ok(user.clone()) + } + } +} + +fn is_auth_failed(e: &ImapError) -> bool { + match e { + ImapError::AuthFailed(_) => true, + ImapError::No(no) => no.is_auth_failed(), + _ => false, + } +} + +fn discover_namespace(client: &mut ImapClient, logger: Logger) -> Result { + if !client.has_capability("NAMESPACE") { + return Ok(String::new()); + } + let resp = match client.run_collect(command::namespace()) { + Ok(r) => r, + Err(e) => { + log_at( + logger, + LEVEL_PROGRESS, + &format!("NAMESPACE failed, ignoring: {e}"), + ); + return Ok(String::new()); + } + }; + for u in resp.untagged { + if let Untagged::Namespace { personal, .. } = u + && let Some(NamespaceEntry { prefix, .. }) = personal.into_iter().next() + { + return Ok(prefix); + } + } + Ok(String::new()) +} + +fn upsert_mailboxes( + conn: &mut Connection, + source_id: i64, + folders: &[ResolvedFolder], + counts: &mut TypeCounts, +) -> Result<(), Error> { + let tx = conn.transaction()?; + let mut local_ids: HashMap = HashMap::new(); + for folder in folders { + let parent_local = match &folder.parent_path { + Some(p) => local_ids.get(p.as_str()).copied().or_else(|| { + db::imap_ids::local_for_mailbox(&tx, source_id, p) + .ok() + .flatten() + }), + None => None, + }; + let existing = db::imap_ids::local_for_mailbox(&tx, source_id, &folder.name)?; + let id = if let Some(id) = existing { + tx.execute( + "UPDATE mailboxes SET name = ?1, parent_id = ?2, role = ?3, + is_subscribed = ?4 WHERE id = ?5", + params![ + folder.leaf, + parent_local, + folder.role, + folder.subscribed as i64, + id + ], + )?; + id + } else { + tx.execute( + "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) + VALUES (?1, ?2, ?3, 0, ?4)", + params![ + folder.leaf, + parent_local, + folder.role, + folder.subscribed as i64 + ], + )?; + let new_id = tx.last_insert_rowid(); + db::imap_ids::insert_mailbox(&tx, source_id, &folder.name, new_id)?; + counts.created += 1; + counts.fetched += 1; + new_id + }; + local_ids.insert(folder.name.clone(), id); + } + tx.commit()?; + Ok(()) +} + +fn delete_vanished_folders( + conn: &mut Connection, + source_id: i64, + folders: &[String], + mailbox_counts: &mut TypeCounts, + email_counts: &mut TypeCounts, + logger: Logger, +) -> Result<(), Error> { + let tx = conn.transaction()?; + for name in folders { + let local_id = match db::imap_ids::local_for_mailbox(&tx, source_id, name)? { + Some(id) => id, + None => continue, + }; + let surviving_child: Option = tx + .query_row( + "SELECT id FROM mailboxes WHERE parent_id = ?1 LIMIT 1", + params![local_id], + |row| row.get(0), + ) + .optional()?; + if surviving_child.is_some() { + log_at( + logger, + LEVEL_DEFAULT, + &format!( + "folder {name:?}: vanished from server but still has child mailboxes in the archive; skipping delete (parent_id RESTRICT)" + ), + ); + mailbox_counts.failed += 1; + continue; + } + let email_ids: Vec = tx + .prepare( + "SELECT local_id FROM sync_id_imap + WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3", + )? + .query_map(params![source_id, EMAIL_TYPE, name], |row| row.get(0))? + .collect::, _>>()?; + for eid in &email_ids { + tx.execute("DELETE FROM emails WHERE id = ?1", params![eid])?; + email_counts.deleted += 1; + } + db::imap_ids::delete_all_emails_in_folder(&tx, source_id, name)?; + db::imap_state::delete(&tx, source_id, name)?; + tx.execute("DELETE FROM mailboxes WHERE id = ?1", params![local_id])?; + db::imap_ids::delete_mailbox(&tx, source_id, name)?; + mailbox_counts.deleted += 1; + } + tx.commit()?; + Ok(()) +} + +fn reconcile_folder( + conn: &mut Connection, + client: &mut ImapClient, + control_ctx: &ControlCtx, + pool: &WorkerPool, + folder: &ResolvedFolder, + opts: RunOpts, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let RunOpts { + source_id, + fetch_batch, + include_deleted: _, + logger, + } = opts; + + if let Some(status) = folder.status.as_ref() + && status.messages == Some(0) + && let Some(uv) = status.uidvalidity + && let Some(un) = status.uidnext + { + let local = db::imap_ids::email_uids_in_folder(conn, source_id, &folder.name)?; + if local.is_empty() { + let now = OffsetDateTime::now_utc() + .format(&Rfc3339) + .unwrap_or_else(|_| String::from("1970-01-01T00:00:00Z")); + db::imap_state::upsert(conn, source_id, &folder.name, uv as u32, un as u32, &now)?; + log_at( + logger, + LEVEL_PROGRESS, + &format!("folder {:?}: empty (LIST-STATUS), skipped", folder.name), + ); + return Ok(()); + } + } + + let wire_name = encode_mailbox_name_with(&folder.name, client.utf8_accept()); + let resp = control_run_collect(client, control_ctx, &command::select(&wire_name))?; + let mut uidvalidity: u32 = 0; + let mut uidnext: u32 = 0; + for u in &resp.untagged { + if let Untagged::StatusLine(line) = u + && let Some(code) = &line.code + && let Some(args) = &line.code_args + { + match code.as_str() { + "UIDVALIDITY" => { + if let Ok(n) = args.parse() { + uidvalidity = n; + } + } + "UIDNEXT" => { + if let Ok(n) = args.parse() { + uidnext = n; + } + } + _ => {} + } + } + } + if uidvalidity == 0 { + return Err(Error::Partial(format!( + "SELECT {:?}: server gave no UIDVALIDITY", + folder.name + ))); + } + + if let Some(prev) = db::imap_state::get(conn, source_id, &folder.name)? + && prev.uidvalidity != uidvalidity + { + log_at( + logger, + LEVEL_DEFAULT, + &format!( + "folder {:?}: UIDVALIDITY changed {} -> {} (wiping local rows)", + folder.name, prev.uidvalidity, uidvalidity + ), + ); + wipe_folder_emails(conn, source_id, &folder.name, counts)?; + } + + let server_uids = call_with_retry(client, control_ctx, select_uids) + .map_err(|e| Error::Partial(format!("UID enum {:?}: {e}", folder.name)))?; + let local_uids = + db::imap_ids::email_uids_at_validity(conn, source_id, &folder.name, uidvalidity)?; + let diff = diff_uids(&local_uids, &server_uids); + + log_at( + logger, + LEVEL_PROGRESS, + &format!( + "folder {:?}: new={} vanished={} present={}", + folder.name, + diff.new.len(), + diff.vanished.len(), + diff.present.len() + ), + ); + + if !diff.vanished.is_empty() { + delete_vanished_emails( + conn, + source_id, + &folder.name, + uidvalidity, + &diff.vanished, + counts, + )?; + } + + if !diff.new.is_empty() { + let mailbox_local = db::imap_ids::local_for_mailbox(conn, source_id, &folder.name)? + .ok_or_else(|| { + Error::Partial(format!( + "mailbox {:?} missing from sync_id_imap", + folder.name + )) + })?; + let batches: Vec<&[u32]> = chunks(&diff.new, fetch_batch); + let n_batches = batches.len(); + for batch in &batches { + pool.submit(FetchJob { + folder: folder.name.clone(), + uidvalidity, + uids: batch.to_vec(), + }); + } + let keepalive_interval = std::time::Duration::from_secs(45); + let mut chunks_done: usize = 0; + let tx = conn.transaction()?; + let target = FetchTarget { + folder: folder.name.as_str(), + uidvalidity, + mailbox_local, + }; + while chunks_done < n_batches { + let event = loop { + match pool.recv_timeout(keepalive_interval) { + Ok(r) => break r, + Err(RecvTimeoutError::Timeout) => { + let _ = control_run_collect(client, control_ctx, "NOOP"); + } + Err(RecvTimeoutError::Disconnected) => { + return Err(Error::Partial("worker channel disconnected".to_owned())); + } + } + }; + match event { + FetchEvent::Item { attrs, .. } => { + insert_single_message(&tx, &target, &attrs, opts, counts)?; + } + FetchEvent::ChunkDone { + folder: chunk_folder, + outcome, + uids_requested, + .. + } => { + chunks_done += 1; + if let Err(e) = outcome { + log_at( + logger, + LEVEL_DEFAULT, + &format!("UID FETCH chunk failed for folder {chunk_folder:?}: {e}",), + ); + counts.failed += uids_requested.len() as u64; + } + } + } + } + tx.commit()?; + } + + let now = OffsetDateTime::now_utc() + .format(&Rfc3339) + .unwrap_or_else(|_| String::from("1970-01-01T00:00:00Z")); + db::imap_state::upsert(conn, source_id, &folder.name, uidvalidity, uidnext, &now)?; + Ok(()) +} + +fn wipe_folder_emails( + conn: &mut Connection, + source_id: i64, + folder: &str, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let tx = conn.transaction()?; + let ids: Vec = tx + .prepare( + "SELECT local_id FROM sync_id_imap + WHERE source_id = ?1 AND type_name = ?2 AND folder = ?3", + )? + .query_map(params![source_id, EMAIL_TYPE, folder], |row| row.get(0))? + .collect::, _>>()?; + for id in &ids { + tx.execute("DELETE FROM emails WHERE id = ?1", params![id])?; + counts.deleted += 1; + } + db::imap_ids::delete_all_emails_in_folder(&tx, source_id, folder)?; + tx.commit()?; + Ok(()) +} + +fn delete_vanished_emails( + conn: &mut Connection, + source_id: i64, + folder: &str, + uidvalidity: u32, + uids: &[u32], + counts: &mut TypeCounts, +) -> Result<(), Error> { + let tx = conn.transaction()?; + for &uid in uids { + if let Some(local_id) = + db::imap_ids::local_for_email(&tx, source_id, folder, uidvalidity, uid)? + { + tx.execute("DELETE FROM emails WHERE id = ?1", params![local_id])?; + db::imap_ids::delete_email(&tx, source_id, folder, uidvalidity, uid)?; + counts.deleted += 1; + } + } + tx.commit()?; + Ok(()) +} + +pub(super) struct FetchTarget<'a> { + pub folder: &'a str, + pub uidvalidity: u32, + pub mailbox_local: i64, +} + +fn insert_single_message( + tx: &rusqlite::Transaction<'_>, + target: &FetchTarget<'_>, + attrs: &fetch::FetchAttrs, + opts: RunOpts, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let RunOpts { + source_id, + fetch_batch: _, + include_deleted, + logger, + } = opts; + let folder = target.folder; + let uidvalidity = target.uidvalidity; + let mailbox_local = target.mailbox_local; + + let Some(uid) = attrs.uid else { + return Ok(()); + }; + let Some(body) = attrs.body.as_ref() else { + counts.skipped += 1; + return Ok(()); + }; + if let Some(declared_size) = attrs.size + && (declared_size as usize) != body.len() + { + log_at( + logger, + LEVEL_DEFAULT, + &format!( + "folder {folder:?} uid {uid}: BODY[] {} bytes vs RFC822.SIZE {declared_size}, skipping", + body.len() + ), + ); + counts.skipped += 1; + return Ok(()); + } + let translation = translate_flags(&attrs.flags, include_deleted); + if translation.has_deleted_flag && !include_deleted { + counts.skipped += 1; + return Ok(()); + } + let received_at = match attrs.internaldate.as_deref() { + Some(s) => imap_internaldate_to_rfc3339(s)?, + None => OffsetDateTime::now_utc() + .format(&Rfc3339) + .unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_owned()), + }; + let blob_id = db::blobs::intern_blob(tx, body)?; + let message_match = index_to_json(&email_index_from_blob(body)); + let mailbox_ids = Value::Array(vec![Value::from(mailbox_local)]); + let keywords = Value::Array( + translation + .keywords + .iter() + .map(|k| Value::String(k.clone())) + .collect(), + ); + tx.execute( + "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + blob_id, + received_at, + mailbox_ids.to_string(), + keywords.to_string(), + message_match, + ], + )?; + let email_local = tx.last_insert_rowid(); + db::imap_ids::insert_email(tx, source_id, folder, uidvalidity, uid, email_local)?; + counts.created += 1; + counts.fetched += 1; + Ok(()) +} + +fn dry_run_summary( + conn: &Connection, + source_id: Option, + client: &mut ImapClient, + control_ctx: &ControlCtx, + folders: &[ResolvedFolder], + logger: Logger, +) -> Result { + let mut mailbox = TypeCounts::default(); + let mut email = TypeCounts::default(); + + let server_set: HashSet = folders.iter().map(|f| f.name.clone()).collect(); + let local_mailboxes = match source_id { + Some(sid) => db::imap_ids::mailbox_folders(conn, sid)?, + None => HashMap::new(), + }; + let new_folders: Vec<&str> = folders + .iter() + .map(|f| f.name.as_str()) + .filter(|n| !local_mailboxes.contains_key(*n)) + .collect(); + let vanished_folder_names: Vec<&str> = local_mailboxes + .keys() + .filter(|n| !server_set.contains(n.as_str())) + .map(|s| s.as_str()) + .collect(); + mailbox.created += new_folders.len() as u64; + mailbox.deleted += vanished_folder_names.len() as u64; + mailbox.fetched += new_folders.len() as u64; + if !new_folders.is_empty() { + log_at( + logger, + LEVEL_PROGRESS, + &format!( + "dry-run: folders new={} vanished={}", + new_folders.len(), + vanished_folder_names.len() + ), + ); + } + + for folder in folders { + let wire = encode_mailbox_name_with(&folder.name, client.utf8_accept()); + let select_resp = match control_run_collect(client, control_ctx, &command::select(&wire)) { + Ok(r) => r, + Err(e) => { + log_at( + logger, + LEVEL_DEFAULT, + &format!("dry-run SELECT {:?}: {e}", folder.name), + ); + email.failed += 1; + continue; + } + }; + let mut server_uidvalidity: u32 = 0; + for u in &select_resp.untagged { + if let Untagged::StatusLine(line) = u + && line.code.as_deref() == Some("UIDVALIDITY") + && let Some(args) = &line.code_args + && let Ok(n) = args.parse() + { + server_uidvalidity = n; + break; + } + } + let server_uids = match call_with_retry(client, control_ctx, select_uids) { + Ok(v) => v, + Err(e) => { + log_at( + logger, + LEVEL_DEFAULT, + &format!("dry-run UID enum {:?}: {e}", folder.name), + ); + email.failed += 1; + continue; + } + }; + let local_pairs = match source_id { + Some(sid) => db::imap_ids::email_uids_in_folder(conn, sid, &folder.name)?, + None => HashMap::new(), + }; + let prev_state = match source_id { + Some(sid) => db::imap_state::get(conn, sid, &folder.name)?, + None => None, + }; + let uv_changed = prev_state + .as_ref() + .is_some_and(|p| p.uidvalidity != server_uidvalidity && server_uidvalidity != 0); + let local_uids: Vec = if uv_changed { + Vec::new() + } else { + local_pairs + .keys() + .filter(|(v, _)| server_uidvalidity == 0 || *v == server_uidvalidity) + .map(|(_, u)| *u) + .collect() + }; + let diff = diff_uids(&local_uids, &server_uids); + email.created += diff.new.len() as u64; + email.deleted += diff.vanished.len() as u64; + email.fetched += diff.new.len() as u64; + let uv_note = if uv_changed { + format!( + " (UIDVALIDITY {} -> {})", + prev_state.as_ref().map_or(0, |p| p.uidvalidity), + server_uidvalidity, + ) + } else { + String::new() + }; + log_at( + logger, + LEVEL_PROGRESS, + &format!( + "dry-run: {:?} new={} vanished={} present={}{uv_note}", + folder.name, + diff.new.len(), + diff.vanished.len(), + diff.present.len(), + ), + ); + } + for name in &vanished_folder_names { + if let Some(sid) = source_id { + let n = db::imap_ids::email_uids_in_folder(conn, sid, name)?.len() as u64; + email.deleted += n; + log_at( + logger, + LEVEL_PROGRESS, + &format!( + "dry-run: folder {:?} vanished ({n} emails would be removed)", + name + ), + ); + } + } + + let _ = client.logout(); + Ok(Summary { + per_type: vec![("mailbox", mailbox), ("email", email)], + retries_observed: control_ctx.backoff.total_retries(), + retry_after_sleeps: control_ctx.backoff.transient_retries() as u64, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_endpoint_imaps_defaults_to_993() { + let e = parse_endpoint("imaps://mail.example.com").unwrap(); + assert_eq!(e.host, "mail.example.com"); + assert_eq!(e.port, 993); + assert!(e.implicit_tls); + } + + #[test] + fn parse_endpoint_imap_defaults_to_143() { + let e = parse_endpoint("imap://mail.example.com").unwrap(); + assert_eq!(e.host, "mail.example.com"); + assert_eq!(e.port, 143); + assert!(!e.implicit_tls); + } + + #[test] + fn parse_endpoint_explicit_port() { + let e = parse_endpoint("imaps://mail.example.com:1993").unwrap(); + assert_eq!(e.port, 1993); + } + + #[test] + fn parse_endpoint_rejects_other_schemes() { + let err = parse_endpoint("http://example.com").unwrap_err(); + assert!(matches!(err, Error::Usage(_))); + } +} diff --git a/src/sync/import_imap/fetch.rs b/src/sync/import_imap/fetch.rs new file mode 100644 index 0000000..7bc2fc4 --- /dev/null +++ b/src/sync/import_imap/fetch.rs @@ -0,0 +1,113 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::imap::response::{Untagged, Value}; + +#[derive(Debug, Default, Clone)] +pub struct FetchAttrs { + pub uid: Option, + pub flags: Vec, + pub internaldate: Option, + pub size: Option, + pub body: Option>, +} + +pub fn extract(u: &Untagged) -> Option { + let Untagged::Fetch { items, .. } = u else { + return None; + }; + let mut out = FetchAttrs::default(); + for (name, value) in items { + match name.as_str() { + "UID" => { + if let Some(n) = value.as_number() { + out.uid = Some(n as u32); + } + } + "FLAGS" => { + if let Value::List(items) = value { + out.flags = items + .iter() + .filter_map(|v| match v { + Value::Atom(s) | Value::Str(s) => Some(s.clone()), + _ => None, + }) + .collect(); + } + } + "INTERNALDATE" => { + if let Some(s) = value.as_str() { + out.internaldate = Some(s.to_owned()); + } + } + "RFC822.SIZE" => { + if let Some(n) = value.as_number() { + out.size = Some(n); + } + } + n if n == "BODY[]" || n == "RFC822" => { + out.body = match value { + Value::Bytes(b) => Some(b.clone()), + Value::Str(s) => Some(s.clone().into_bytes()), + _ => None, + }; + } + _ => {} + } + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + fn parse_fetch(input: &[u8]) -> FetchAttrs { + let resp = crate::imap::response::parse_response(&mut Cursor::new(input)).unwrap(); + match resp { + crate::imap::response::Response::Untagged(u) => extract(&u).expect("fetch attrs"), + _ => panic!("expected untagged"), + } + } + + #[test] + fn extracts_uid_flags_size_internaldate() { + let f = parse_fetch( + b"* 1 FETCH (UID 42 FLAGS (\\Seen) INTERNALDATE \"01-Jan-2024 12:00:00 +0000\" RFC822.SIZE 4242)\r\n", + ); + assert_eq!(f.uid, Some(42)); + assert_eq!(f.flags, vec!["\\Seen"]); + assert_eq!( + f.internaldate.as_deref(), + Some("01-Jan-2024 12:00:00 +0000") + ); + assert_eq!(f.size, Some(4242)); + assert!(f.body.is_none()); + } + + #[test] + fn extracts_body_literal_as_bytes() { + let input = b"* 1 FETCH (UID 5 BODY[] {11}\r\nHello world)\r\n"; + let f = parse_fetch(input); + assert_eq!(f.body.as_deref(), Some(&b"Hello world"[..])); + } + + #[test] + fn ignores_unknown_attrs() { + let f = parse_fetch(b"* 1 FETCH (UID 1 X-CUSTOM \"ignored\" MODSEQ (12345))\r\n"); + assert_eq!(f.uid, Some(1)); + assert!(f.flags.is_empty()); + assert!(f.size.is_none()); + } + + #[test] + fn empty_flags_list() { + let f = parse_fetch(b"* 1 FETCH (UID 7 FLAGS ())\r\n"); + assert_eq!(f.uid, Some(7)); + assert!(f.flags.is_empty()); + } +} diff --git a/src/sync/import_imap/folders.rs b/src/sync/import_imap/folders.rs new file mode 100644 index 0000000..c3810cb --- /dev/null +++ b/src/sync/import_imap/folders.rs @@ -0,0 +1,504 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{BTreeMap, HashMap, HashSet}; + +use regex::Regex; + +use crate::imap::automap; +use crate::imap::name::{canonicalise_inbox, decode_mailbox_name_with}; +use crate::imap::response::Untagged; + +#[derive(Debug, Clone, Default)] +pub struct FolderStatus { + pub uidvalidity: Option, + pub uidnext: Option, + pub messages: Option, +} + +#[derive(Debug, Clone)] +pub struct DiscoveredFolder { + pub name: String, + pub delimiter: Option, + pub attributes: Vec, + pub subscribed: bool, + pub selectable: bool, + pub status: Option, +} + +#[derive(Debug, Clone)] +pub struct ResolvedFolder { + pub name: String, + pub leaf: String, + pub parent_path: Option, + pub delimiter: Option, + pub role: Option<&'static str>, + pub subscribed: bool, + pub status: Option, +} + +pub struct FolderFilters { + pub include: Vec, + pub exclude: Vec, + pub exclude_special: Vec, + pub explicit: Vec, + pub subscribed_only: bool, + pub automap_enabled: bool, + pub namespace_prefix: String, +} + +pub fn collect_from_list( + untagged: &[Untagged], + utf8_accept: bool, +) -> Result, String> { + let mut subscribed = HashSet::new(); + let mut statuses: BTreeMap = BTreeMap::new(); + let mut found = BTreeMap::new(); + for u in untagged { + match u { + Untagged::Lsub { name, .. } => { + let decoded = + decode_mailbox_name_with(name, utf8_accept).map_err(|e| e.to_string())?; + subscribed.insert(canonicalise_inbox(&decoded)); + } + Untagged::List { + attributes, + delimiter, + name, + } => { + let decoded = + decode_mailbox_name_with(name, utf8_accept).map_err(|e| e.to_string())?; + let canonical = canonicalise_inbox(&decoded); + let attrs_lower: Vec = + attributes.iter().map(|a| a.to_ascii_lowercase()).collect(); + let selectable = !attrs_lower + .iter() + .any(|a| a == "\\noselect" || a == "\\nonexistent"); + let extended_subscribed = attrs_lower.iter().any(|a| a == "\\subscribed"); + if extended_subscribed { + subscribed.insert(canonical.clone()); + } + found.insert( + canonical.clone(), + DiscoveredFolder { + name: canonical, + delimiter: *delimiter, + attributes: attributes.clone(), + subscribed: false, + selectable, + status: None, + }, + ); + } + Untagged::Status { mailbox, items } => { + let decoded = + decode_mailbox_name_with(mailbox, utf8_accept).map_err(|e| e.to_string())?; + let canonical = canonicalise_inbox(&decoded); + let mut st = FolderStatus::default(); + if let Some(v) = items.get("UIDVALIDITY") { + st.uidvalidity = Some(*v); + } + if let Some(v) = items.get("UIDNEXT") { + st.uidnext = Some(*v); + } + if let Some(v) = items.get("MESSAGES") { + st.messages = Some(*v); + } + statuses.insert(canonical, st); + } + _ => {} + } + } + let mut out: Vec = found.into_values().collect(); + for f in &mut out { + if subscribed.contains(&f.name) { + f.subscribed = true; + } + if let Some(s) = statuses.remove(&f.name) { + f.status = Some(s); + } + } + Ok(out) +} + +pub fn apply_filters( + folders: Vec, + filters: &FolderFilters, +) -> Vec { + let mut keep: Vec = folders + .into_iter() + .filter(|f| f.selectable) + .filter(|f| match_filters(&f.name, filters)) + .collect(); + if filters.subscribed_only { + keep.retain(|f| f.subscribed); + } + let mut resolved: Vec = Vec::with_capacity(keep.len()); + for f in keep { + let role = automap::role_for_folder( + &f.name, + &f.attributes, + &filters.namespace_prefix, + filters.automap_enabled, + ); + if let Some(r) = role + && filters + .exclude_special + .iter() + .any(|s| s.eq_ignore_ascii_case(r)) + { + continue; + } + let delim = f.delimiter; + let (leaf, parent_path) = split_parent(&f.name, delim); + resolved.push(ResolvedFolder { + name: f.name, + leaf, + parent_path, + delimiter: delim, + role, + subscribed: f.subscribed, + status: f.status, + }); + } + resolved +} + +fn match_filters(name: &str, filters: &FolderFilters) -> bool { + if !filters.explicit.is_empty() { + return filters.explicit.iter().any(|e| e == name); + } + if !filters.include.is_empty() && !filters.include.iter().any(|r| r.is_match(name)) { + return false; + } + if filters.exclude.iter().any(|r| r.is_match(name)) { + return false; + } + true +} + +fn split_parent(name: &str, delim: Option) -> (String, Option) { + let Some(d) = delim else { + return (name.to_owned(), None); + }; + if let Some(idx) = name.rfind(d) { + let parent = &name[..idx]; + let leaf = &name[idx + d.len_utf8()..]; + (leaf.to_owned(), Some(parent.to_owned())) + } else { + (name.to_owned(), None) + } +} + +pub fn sort_by_depth(resolved: &mut [ResolvedFolder]) { + resolved.sort_by(|a, b| { + let da = a.parent_path.as_deref().map_or(0, |s| { + a.delimiter.map(|d| s.matches(d).count() + 1).unwrap_or(0) + }); + let db = b.parent_path.as_deref().map_or(0, |s| { + b.delimiter.map(|d| s.matches(d).count() + 1).unwrap_or(0) + }); + da.cmp(&db).then_with(|| a.name.cmp(&b.name)) + }); +} + +pub fn vanished_folders(local: &HashMap, server: &HashSet) -> Vec { + local + .keys() + .filter(|k| !server.contains(*k)) + .cloned() + .collect() +} + +pub fn vanished_depth_sort(names: &mut [String], delimiter: char) { + names.sort_by(|a, b| { + let depth_a = a.matches(delimiter).count(); + let depth_b = b.matches(delimiter).count(); + depth_b.cmp(&depth_a).then_with(|| a.cmp(b)) + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn lst(name: &str, delim: &str, attrs: &[&str]) -> Untagged { + Untagged::List { + attributes: attrs.iter().map(|s| (*s).to_owned()).collect(), + delimiter: delim.chars().next(), + name: name.to_owned(), + } + } + + #[test] + fn collect_filters_noselect_and_canonicalises_inbox() { + let resps = vec![ + lst("Inbox", "/", &[]), + lst("Sent", "/", &["\\Sent"]), + lst("Hidden", "/", &["\\Noselect"]), + ]; + let folders = collect_from_list(&resps, false).unwrap(); + assert_eq!(folders.len(), 3); + let inbox = folders.iter().find(|f| f.name == "INBOX").unwrap(); + assert!(inbox.selectable); + let hidden = folders.iter().find(|f| f.name == "Hidden").unwrap(); + assert!(!hidden.selectable); + } + + #[test] + fn collect_marks_subscribed_via_lsub_or_extended() { + let resps = vec![ + lst("Inbox", "/", &[]), + lst("Sent", "/", &["\\Subscribed"]), + Untagged::Lsub { + attributes: vec![], + delimiter: Some('/'), + name: "Inbox".to_owned(), + }, + ]; + let folders = collect_from_list(&resps, false).unwrap(); + let inbox = folders.iter().find(|f| f.name == "INBOX").unwrap(); + assert!(inbox.subscribed); + let sent = folders.iter().find(|f| f.name == "Sent").unwrap(); + assert!(sent.subscribed); + } + + #[test] + fn collect_attaches_list_status_data_per_mailbox() { + let mut inbox_status: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + inbox_status.insert("UIDVALIDITY".into(), 12345); + inbox_status.insert("UIDNEXT".into(), 42); + inbox_status.insert("MESSAGES".into(), 7); + let resps = vec![ + lst("INBOX", "/", &[]), + lst("Sent", "/", &["\\Sent"]), + Untagged::Status { + mailbox: "INBOX".to_owned(), + items: inbox_status, + }, + ]; + let folders = collect_from_list(&resps, false).unwrap(); + let inbox = folders.iter().find(|f| f.name == "INBOX").unwrap(); + let st = inbox.status.as_ref().expect("INBOX has status"); + assert_eq!(st.uidvalidity, Some(12345)); + assert_eq!(st.uidnext, Some(42)); + assert_eq!(st.messages, Some(7)); + + let sent = folders.iter().find(|f| f.name == "Sent").unwrap(); + assert!(sent.status.is_none()); + } + + #[test] + fn collect_decodes_modified_utf7_names() { + let resps = vec![lst("&ZeVnLIqe-", "/", &[])]; + let folders = collect_from_list(&resps, false).unwrap(); + assert_eq!(folders[0].name, "日本語"); + } + + fn filters_default() -> FolderFilters { + FolderFilters { + include: Vec::new(), + exclude: Vec::new(), + exclude_special: Vec::new(), + explicit: Vec::new(), + subscribed_only: false, + automap_enabled: true, + namespace_prefix: String::new(), + } + } + + #[test] + fn filters_default_keep_everything() { + let folders = collect_from_list( + &[ + lst("INBOX", "/", &[]), + lst("Sent", "/", &["\\Sent"]), + lst("Drafts", "/", &[]), + ], + false, + ) + .unwrap(); + let res = apply_filters(folders, &filters_default()); + assert_eq!(res.len(), 3); + let sent = res.iter().find(|f| f.name == "Sent").unwrap(); + assert_eq!(sent.role, Some("sent")); + } + + #[test] + fn include_excludes_non_matching() { + let folders = collect_from_list( + &[ + lst("INBOX", "/", &[]), + lst("Trash", "/", &[]), + lst("Sent", "/", &["\\Sent"]), + ], + false, + ) + .unwrap(); + let mut f = filters_default(); + f.include = vec![Regex::new(r"^(INBOX|Sent)$").unwrap()]; + let res = apply_filters(folders, &f); + assert_eq!(res.len(), 2); + assert!(res.iter().any(|x| x.name == "INBOX")); + assert!(res.iter().any(|x| x.name == "Sent")); + } + + #[test] + fn exclude_drops_matching() { + let folders = + collect_from_list(&[lst("INBOX", "/", &[]), lst("Trash", "/", &[])], false).unwrap(); + let mut f = filters_default(); + f.exclude = vec![Regex::new(r"^Trash$").unwrap()]; + let res = apply_filters(folders, &f); + assert_eq!(res.len(), 1); + assert_eq!(res[0].name, "INBOX"); + } + + #[test] + fn exclude_special_drops_by_role() { + let folders = collect_from_list( + &[lst("INBOX", "/", &[]), lst("Trash", "/", &["\\Trash"])], + false, + ) + .unwrap(); + let mut f = filters_default(); + f.exclude_special = vec!["trash".to_owned()]; + let res = apply_filters(folders, &f); + assert_eq!(res.len(), 1); + assert_eq!(res[0].name, "INBOX"); + } + + #[test] + fn explicit_folder_list_overrides_include_exclude() { + let folders = collect_from_list( + &[ + lst("INBOX", "/", &[]), + lst("Sent", "/", &[]), + lst("Trash", "/", &[]), + ], + false, + ) + .unwrap(); + let mut f = filters_default(); + f.explicit = vec!["Sent".to_owned()]; + let res = apply_filters(folders, &f); + assert_eq!(res.len(), 1); + assert_eq!(res[0].name, "Sent"); + } + + #[test] + fn subscribed_only_filters_unsubscribed() { + let mut folders = collect_from_list( + &[lst("INBOX", "/", &["\\Subscribed"]), lst("Other", "/", &[])], + false, + ) + .unwrap(); + + let mut f = filters_default(); + f.subscribed_only = true; + + for x in &mut folders { + if x.name == "INBOX" { + x.subscribed = true; + } + } + let res = apply_filters(folders, &f); + assert_eq!(res.len(), 1); + assert_eq!(res[0].name, "INBOX"); + } + + #[test] + fn split_parent_handles_root_and_nested() { + assert_eq!(split_parent("INBOX", Some('/')), ("INBOX".to_owned(), None)); + assert_eq!( + split_parent("Projects/Alpha", Some('/')), + ("Alpha".to_owned(), Some("Projects".to_owned())) + ); + assert_eq!( + split_parent("Projects/Alpha/Beta", Some('/')), + ("Beta".to_owned(), Some("Projects/Alpha".to_owned())) + ); + } + + #[test] + fn split_parent_with_no_delimiter_returns_root() { + assert_eq!( + split_parent("Anything", None), + ("Anything".to_owned(), None) + ); + } + + #[test] + fn sort_by_depth_places_roots_first() { + let mut r = vec![ + ResolvedFolder { + name: "Projects/Alpha".into(), + leaf: "Alpha".into(), + parent_path: Some("Projects".into()), + delimiter: Some('/'), + role: None, + subscribed: false, + status: None, + }, + ResolvedFolder { + name: "INBOX".into(), + leaf: "INBOX".into(), + parent_path: None, + delimiter: Some('/'), + role: Some("inbox"), + subscribed: true, + status: None, + }, + ResolvedFolder { + name: "Projects".into(), + leaf: "Projects".into(), + parent_path: None, + delimiter: Some('/'), + role: None, + subscribed: false, + status: None, + }, + ]; + sort_by_depth(&mut r); + assert_eq!(r[0].name, "INBOX"); + assert_eq!(r[1].name, "Projects"); + assert_eq!(r[2].name, "Projects/Alpha"); + } + + #[test] + fn vanished_folders_is_set_minus() { + let mut local = HashMap::new(); + local.insert("INBOX".to_owned(), 1); + local.insert("Old".to_owned(), 2); + local.insert("Sent".to_owned(), 3); + let mut server = HashSet::new(); + server.insert("INBOX".to_owned()); + server.insert("Sent".to_owned()); + let v = vanished_folders(&local, &server); + assert_eq!(v, vec!["Old".to_owned()]); + } + + #[test] + fn vanished_depth_sort_deepest_first() { + let mut names = vec!["A".to_owned(), "A/B/C".to_owned(), "A/B".to_owned()]; + vanished_depth_sort(&mut names, '/'); + assert_eq!(names, vec!["A/B/C", "A/B", "A"]); + } + + #[test] + fn untagged_status_does_not_become_a_folder() { + let resps = vec![ + lst("INBOX", "/", &[]), + Untagged::Status { + mailbox: "INBOX".to_owned(), + items: BTreeMap::new(), + }, + ]; + let folders = collect_from_list(&resps, false).unwrap(); + assert_eq!(folders.len(), 1); + } +} diff --git a/src/sync/import_imap/internaldate.rs b/src/sync/import_imap/internaldate.rs new file mode 100644 index 0000000..039a7dd --- /dev/null +++ b/src/sync/import_imap/internaldate.rs @@ -0,0 +1,152 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::error::Error; + +pub fn imap_internaldate_to_rfc3339(s: &str) -> Result { + let s = s.trim_start_matches(' ').trim_end(); + let (day, rest) = s + .split_once('-') + .ok_or_else(|| Error::Partial(format!("INTERNALDATE: missing day in {s:?}")))?; + let (mon, rest) = rest + .split_once('-') + .ok_or_else(|| Error::Partial(format!("INTERNALDATE: missing month in {s:?}")))?; + let (year, rest) = rest + .split_once(' ') + .ok_or_else(|| Error::Partial(format!("INTERNALDATE: missing year in {s:?}")))?; + let (time, zone) = rest + .split_once(' ') + .ok_or_else(|| Error::Partial(format!("INTERNALDATE: missing zone in {s:?}")))?; + let (h, ms) = time + .split_once(':') + .ok_or_else(|| Error::Partial(format!("INTERNALDATE: bad time {time:?}")))?; + let (m, sec) = ms + .split_once(':') + .ok_or_else(|| Error::Partial(format!("INTERNALDATE: bad time {time:?}")))?; + let day: u32 = day + .trim() + .parse() + .map_err(|e| Error::Partial(format!("INTERNALDATE day {day:?}: {e}")))?; + let mon_idx = month_to_num(mon)?; + let year: i32 = year + .parse() + .map_err(|e| Error::Partial(format!("INTERNALDATE year {year:?}: {e}")))?; + let h: u32 = h + .parse() + .map_err(|e| Error::Partial(format!("INTERNALDATE hour {h:?}: {e}")))?; + let m: u32 = m + .parse() + .map_err(|e| Error::Partial(format!("INTERNALDATE minute {m:?}: {e}")))?; + let sec: u32 = sec + .parse() + .map_err(|e| Error::Partial(format!("INTERNALDATE second {sec:?}: {e}")))?; + let (sign, hh, mm) = parse_zone(zone)?; + let mut out = String::with_capacity(32); + use std::fmt::Write; + let _ = write!( + &mut out, + "{year:04}-{mon_idx:02}-{day:02}T{h:02}:{m:02}:{sec:02}" + ); + if hh == 0 && mm == 0 { + out.push('Z'); + } else { + let _ = write!(&mut out, "{sign}{hh:02}:{mm:02}"); + } + Ok(out) +} + +fn month_to_num(s: &str) -> Result { + let m = match s { + "Jan" => 1, + "Feb" => 2, + "Mar" => 3, + "Apr" => 4, + "May" => 5, + "Jun" => 6, + "Jul" => 7, + "Aug" => 8, + "Sep" => 9, + "Oct" => 10, + "Nov" => 11, + "Dec" => 12, + other => return Err(Error::Partial(format!("INTERNALDATE month {other:?}"))), + }; + Ok(m) +} + +fn parse_zone(s: &str) -> Result<(char, u32, u32), Error> { + let s = s.trim(); + if s.len() != 5 { + return Err(Error::Partial(format!("INTERNALDATE zone {s:?}"))); + } + let sign = s.as_bytes()[0] as char; + if sign != '+' && sign != '-' { + return Err(Error::Partial(format!("INTERNALDATE zone sign {sign}"))); + } + let hh: u32 = s[1..3] + .parse() + .map_err(|e| Error::Partial(format!("INTERNALDATE zone hour: {e}")))?; + let mm: u32 = s[3..5] + .parse() + .map_err(|e| Error::Partial(format!("INTERNALDATE zone min: {e}")))?; + Ok((sign, hh, mm)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn utc_zone_becomes_z() { + assert_eq!( + imap_internaldate_to_rfc3339("17-Jul-1996 02:44:25 +0000").unwrap(), + "1996-07-17T02:44:25Z" + ); + } + + #[test] + fn positive_offset_preserved() { + assert_eq!( + imap_internaldate_to_rfc3339("17-Jul-1996 02:44:25 +0200").unwrap(), + "1996-07-17T02:44:25+02:00" + ); + } + + #[test] + fn negative_offset_preserved() { + assert_eq!( + imap_internaldate_to_rfc3339("17-Jul-1996 02:44:25 -0700").unwrap(), + "1996-07-17T02:44:25-07:00" + ); + } + + #[test] + fn single_digit_day_padded() { + assert_eq!( + imap_internaldate_to_rfc3339("5-Jan-2024 09:05:01 +0000").unwrap(), + "2024-01-05T09:05:01Z" + ); + } + + #[test] + fn leading_space_single_digit_day() { + assert_eq!( + imap_internaldate_to_rfc3339(" 5-Jan-2024 09:05:01 +0000").unwrap(), + "2024-01-05T09:05:01Z" + ); + } + + #[test] + fn invalid_month_errors() { + assert!(imap_internaldate_to_rfc3339("1-XYZ-2024 00:00:00 +0000").is_err()); + } + + #[test] + fn malformed_input_errors() { + assert!(imap_internaldate_to_rfc3339("garbage").is_err()); + assert!(imap_internaldate_to_rfc3339("17-Jul-1996").is_err()); + } +} diff --git a/src/sync/import_imap/keywords.rs b/src/sync/import_imap/keywords.rs new file mode 100644 index 0000000..9dcc74f --- /dev/null +++ b/src/sync/import_imap/keywords.rs @@ -0,0 +1,173 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FlagAction { + Keep(&'static str), + Drop, + Verbatim, + Deleted, +} + +pub fn translate_system_flag(flag: &str) -> FlagAction { + if !flag.starts_with('\\') { + return FlagAction::Verbatim; + } + match &flag[1..].to_ascii_lowercase()[..] { + "seen" => FlagAction::Keep("$seen"), + "flagged" => FlagAction::Keep("$flagged"), + "answered" => FlagAction::Keep("$answered"), + "draft" => FlagAction::Keep("$draft"), + "recent" => FlagAction::Drop, + "deleted" => FlagAction::Deleted, + _ => FlagAction::Verbatim, + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Translation { + pub keywords: Vec, + pub has_deleted_flag: bool, +} + +pub fn translate_flags(imap_flags: &[String], include_deleted: bool) -> Translation { + let mut out: Vec = Vec::with_capacity(imap_flags.len()); + let mut has_deleted = false; + for f in imap_flags { + if f.is_empty() { + continue; + } + match translate_system_flag(f) { + FlagAction::Keep(k) => { + if !out.iter().any(|x| x.eq_ignore_ascii_case(k)) { + out.push(k.to_owned()); + } + } + FlagAction::Drop => {} + FlagAction::Verbatim => { + if !out.iter().any(|x| x.eq_ignore_ascii_case(f)) { + out.push(f.clone()); + } + } + FlagAction::Deleted => { + has_deleted = true; + if include_deleted && !out.iter().any(|x| x.eq_ignore_ascii_case("$deleted")) { + out.push("$deleted".to_owned()); + } + } + } + } + Translation { + keywords: out, + has_deleted_flag: has_deleted, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn flags(s: &[&str]) -> Vec { + s.iter().map(|x| (*x).to_owned()).collect() + } + + #[test] + fn system_flag_mapping() { + assert!(matches!( + translate_system_flag("\\Seen"), + FlagAction::Keep("$seen") + )); + assert!(matches!( + translate_system_flag("\\Flagged"), + FlagAction::Keep("$flagged") + )); + assert!(matches!( + translate_system_flag("\\Answered"), + FlagAction::Keep("$answered") + )); + assert!(matches!( + translate_system_flag("\\Draft"), + FlagAction::Keep("$draft") + )); + assert!(matches!( + translate_system_flag("\\Recent"), + FlagAction::Drop + )); + assert!(matches!( + translate_system_flag("\\Deleted"), + FlagAction::Deleted + )); + } + + #[test] + fn unknown_system_flag_passes_verbatim() { + assert!(matches!( + translate_system_flag("\\Forwarded"), + FlagAction::Verbatim + )); + assert!(matches!( + translate_system_flag("\\MDNSent"), + FlagAction::Verbatim + )); + } + + #[test] + fn custom_keywords_pass_verbatim_case() { + let t = translate_flags(&flags(&["$Junk", "NotJunk", "MyLabel"]), false); + assert_eq!(t.keywords, vec!["$Junk", "NotJunk", "MyLabel"]); + assert!(!t.has_deleted_flag); + } + + #[test] + fn translates_seen_and_flagged() { + let t = translate_flags(&flags(&["\\Seen", "\\Flagged"]), false); + assert_eq!(t.keywords, vec!["$seen", "$flagged"]); + } + + #[test] + fn drops_recent() { + let t = translate_flags(&flags(&["\\Seen", "\\Recent"]), false); + assert_eq!(t.keywords, vec!["$seen"]); + } + + #[test] + fn deleted_flag_without_include_marks_only() { + let t = translate_flags(&flags(&["\\Deleted", "\\Seen"]), false); + assert!(t.has_deleted_flag); + assert_eq!(t.keywords, vec!["$seen"]); + } + + #[test] + fn deleted_flag_with_include_adds_dollar_deleted() { + let t = translate_flags(&flags(&["\\Deleted", "\\Seen"]), true); + assert!(t.has_deleted_flag); + assert_eq!(t.keywords, vec!["$deleted", "$seen"]); + } + + #[test] + fn duplicate_keywords_deduped_case_insensitively() { + let t = translate_flags(&flags(&["mykey", "MYKEY", "MyKey"]), false); + assert_eq!(t.keywords, vec!["mykey"]); + } + + #[test] + fn empty_flag_strings_ignored() { + let t = translate_flags(&flags(&["", "\\Seen", ""]), false); + assert_eq!(t.keywords, vec!["$seen"]); + } + + #[test] + fn order_preserved() { + let t = translate_flags( + &flags(&["MyLabel", "\\Seen", "$Important", "\\Flagged"]), + false, + ); + assert_eq!( + t.keywords, + vec!["MyLabel", "$seen", "$Important", "$flagged"] + ); + } +} diff --git a/src/sync/import_imap/messages.rs b/src/sync/import_imap/messages.rs new file mode 100644 index 0000000..95249ca --- /dev/null +++ b/src/sync/import_imap/messages.rs @@ -0,0 +1,158 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashSet; + +use crate::imap::client::ImapClient; +use crate::imap::command; +use crate::imap::error::ImapError; +use crate::imap::response::Untagged; +use crate::imap::retry::is_negotiation_failure; + +pub fn select_uids(client: &mut ImapClient) -> Result, ImapError> { + if client.has_capability("ESEARCH") { + match client.run_collect(command::uid_search_esearch_all()) { + Ok(resp) => { + for u in &resp.untagged { + if let Untagged::Esearch { all, .. } = u { + return Ok(all.clone()); + } + } + } + Err(e) if is_negotiation_failure(&e) => {} + Err(e) => return Err(e), + } + } + match client.run_collect(command::uid_search_all()) { + Ok(resp) => { + for u in &resp.untagged { + if let Untagged::Search(uids) = u { + return Ok(uids.clone()); + } + } + Ok(Vec::new()) + } + Err(e) if is_negotiation_failure(&e) => uids_via_fetch(client), + Err(e) => Err(e), + } +} + +fn uids_via_fetch(client: &mut ImapClient) -> Result, ImapError> { + let resp = client.run_collect(command::uid_fetch_all_uids())?; + let mut out = Vec::new(); + for u in resp.untagged { + if let Untagged::Fetch { items, .. } = u { + for (n, v) in items { + if n == "UID" + && let Some(num) = v.as_number() + { + out.push(num as u32); + } + } + } + } + Ok(out) +} + +#[derive(Debug, Clone, Default)] +pub struct UidDiff { + pub new: Vec, + pub vanished: Vec, + pub present: Vec, +} + +pub fn diff_uids(local: &[u32], server: &[u32]) -> UidDiff { + let local_set: HashSet = local.iter().copied().collect(); + let server_set: HashSet = server.iter().copied().collect(); + let mut new = Vec::new(); + let mut present = Vec::new(); + for &uid in server { + if local_set.contains(&uid) { + present.push(uid); + } else { + new.push(uid); + } + } + let mut vanished: Vec = local + .iter() + .copied() + .filter(|u| !server_set.contains(u)) + .collect(); + new.sort_unstable(); + new.dedup(); + vanished.sort_unstable(); + vanished.dedup(); + present.sort_unstable(); + present.dedup(); + UidDiff { + new, + vanished, + present, + } +} + +pub fn chunks(uids: &[u32], chunk_size: usize) -> Vec<&[u32]> { + if chunk_size == 0 { + return vec![uids]; + } + uids.chunks(chunk_size).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn diff_basic() { + let local = vec![1, 2, 5]; + let server = vec![2, 3, 5, 7]; + let d = diff_uids(&local, &server); + assert_eq!(d.new, vec![3, 7]); + assert_eq!(d.vanished, vec![1]); + assert_eq!(d.present, vec![2, 5]); + } + + #[test] + fn diff_empty_local_is_all_new() { + let d = diff_uids(&[], &[1, 2, 3]); + assert_eq!(d.new, vec![1, 2, 3]); + assert!(d.vanished.is_empty()); + assert!(d.present.is_empty()); + } + + #[test] + fn diff_empty_server_is_all_vanished() { + let d = diff_uids(&[1, 2, 3], &[]); + assert_eq!(d.vanished, vec![1, 2, 3]); + assert!(d.new.is_empty()); + assert!(d.present.is_empty()); + } + + #[test] + fn diff_dedups_inputs() { + let d = diff_uids(&[1, 1, 2, 2], &[2, 2, 3, 3]); + assert_eq!(d.new, vec![3]); + assert_eq!(d.vanished, vec![1]); + assert_eq!(d.present, vec![2]); + } + + #[test] + fn chunks_splits_evenly() { + let v: Vec = (1..=10).collect(); + let c = chunks(&v, 3); + assert_eq!(c.len(), 4); + assert_eq!(c[0], &[1, 2, 3][..]); + assert_eq!(c[3], &[10][..]); + } + + #[test] + fn chunks_zero_returns_whole() { + let v: Vec = vec![1, 2, 3]; + let c = chunks(&v, 0); + assert_eq!(c.len(), 1); + assert_eq!(c[0], &v[..]); + } +} diff --git a/src/sync/import_imap/pool.rs b/src/sync/import_imap/pool.rs new file mode 100644 index 0000000..bb3afa4 --- /dev/null +++ b/src/sync/import_imap/pool.rs @@ -0,0 +1,256 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::Arc; +use std::thread; + +use crossbeam_channel::{Receiver, Sender, unbounded}; + +use crate::imap::client::{ConnectMode, ImapClient}; +use crate::imap::command; +use crate::imap::error::ImapError; +use crate::imap::name::encode_mailbox_name_with; +use crate::imap::response::Untagged; +use crate::imap::retry::{BackoffState, Disposition, RetryPolicy, classify}; +use crate::imap::transport::Connector; + +use super::coordinator::{Endpoint, ImapAuth, authenticate_client}; +use super::fetch::FetchAttrs; + +pub const HARD_CAP: usize = 8; + +pub struct FetchJob { + pub folder: String, + pub uidvalidity: u32, + pub uids: Vec, +} + +pub enum FetchEvent { + Item { + folder: String, + uidvalidity: u32, + attrs: FetchAttrs, + }, + ChunkDone { + folder: String, + uidvalidity: u32, + uids_requested: Vec, + outcome: Result<(), ImapError>, + }, +} + +pub struct WorkerArgs { + pub connector: Arc, + pub endpoint: Arc, + pub mode: ConnectMode, + pub auth: ImapAuth, + pub compress: bool, + pub policy: RetryPolicy, + pub backoff: BackoffState, +} + +pub struct WorkerPool { + job_tx: Sender, + event_rx: Receiver, + handles: Vec>, +} + +impl WorkerPool { + pub fn start(args: WorkerArgs, pool_size: usize) -> Result { + let size = pool_size.clamp(1, HARD_CAP); + let (job_tx, job_rx) = unbounded::(); + let (event_tx, event_rx) = unbounded::(); + let mut handles = Vec::with_capacity(size); + let args = Arc::new(args); + + for _ in 0..size { + let args = args.clone(); + let job_rx = job_rx.clone(); + let event_tx = event_tx.clone(); + let handle = thread::spawn(move || { + worker_loop(args, job_rx, event_tx); + }); + handles.push(handle); + } + + Ok(WorkerPool { + job_tx, + event_rx, + handles, + }) + } + + pub fn submit(&self, job: FetchJob) { + let _ = self.job_tx.send(job); + } + + pub fn recv(&self) -> Result { + self.event_rx.recv() + } + + pub fn recv_timeout( + &self, + timeout: std::time::Duration, + ) -> Result { + self.event_rx.recv_timeout(timeout) + } + + pub fn shutdown(self) { + drop(self.job_tx); + for h in self.handles { + let _ = h.join(); + } + } +} + +fn worker_loop(args: Arc, job_rx: Receiver, event_tx: Sender) { + let mut client: Option = None; + let mut current_folder: Option = None; + while let Ok(job) = job_rx.recv() { + let job_folder = job.folder.clone(); + let job_uv = job.uidvalidity; + let job_uids = job.uids.clone(); + let event_tx_for_job = event_tx.clone(); + let outcome = match catch_unwind(AssertUnwindSafe(|| { + run_job_with_retry( + &args, + &mut client, + &mut current_folder, + &job, + &event_tx_for_job, + ) + })) { + Ok(r) => r, + Err(_) => { + client = None; + current_folder = None; + Err(ImapError::Protocol("worker thread panicked".into())) + } + }; + let _ = event_tx.send(FetchEvent::ChunkDone { + folder: job_folder, + uidvalidity: job_uv, + uids_requested: job_uids, + outcome, + }); + } +} + +fn run_job_with_retry( + args: &WorkerArgs, + client_slot: &mut Option, + current_folder: &mut Option, + job: &FetchJob, + event_tx: &Sender, +) -> Result<(), ImapError> { + let mut transient_attempts: u32 = 0; + let mut transport_attempts: u32 = 0; + loop { + if client_slot.is_none() { + match connect_and_auth(args) { + Ok(c) => { + *client_slot = Some(c); + *current_folder = None; + } + Err(e) => { + let disp = classify(&e); + if disp == Disposition::TransportDrop + && transport_attempts < args.policy.max_retries + { + transport_attempts += 1; + std::thread::sleep(args.backoff.transport_delay(transport_attempts)); + continue; + } + return Err(e); + } + } + } + let Some(client) = client_slot.as_mut() else { + return Err(ImapError::Protocol( + "worker pool: client slot empty after connect".into(), + )); + }; + match run_one_job(client, current_folder, job, event_tx) { + Ok(()) => { + args.backoff.reset(); + return Ok(()); + } + Err(e) => match classify(&e) { + Disposition::TransportDrop => { + *client_slot = None; + *current_folder = None; + if transport_attempts >= args.policy.max_retries { + return Err(e); + } + transport_attempts += 1; + std::thread::sleep(args.backoff.transport_delay(transport_attempts)); + } + Disposition::Transient => { + if transient_attempts >= args.policy.max_retries { + return Err(e); + } + transient_attempts += 1; + std::thread::sleep(args.backoff.next_shared_delay()); + } + _ => return Err(e), + }, + } + } +} + +fn connect_and_auth(args: &WorkerArgs) -> Result { + let mut client = ImapClient::connect( + &args.connector, + &args.endpoint.host, + args.endpoint.port, + args.mode, + )?; + authenticate_client(&mut client, &args.auth) + .map_err(|e| ImapError::AuthFailed(e.to_string()))?; + let _ = client.refresh_capabilities(); + if args.compress && client.has_capability("COMPRESS=DEFLATE") { + client.compress_deflate()?; + } + if client.has_capability("ENABLE") && client.has_capability("UTF8=ACCEPT") { + let _ = client.enable(&["UTF8=ACCEPT"]); + } + Ok(client) +} + +fn run_one_job( + client: &mut ImapClient, + current_folder: &mut Option, + job: &FetchJob, + event_tx: &Sender, +) -> Result<(), ImapError> { + if current_folder.as_deref() != Some(job.folder.as_str()) { + let wire = encode_mailbox_name_with(&job.folder, client.utf8_accept()); + client.run_collect(&command::select(&wire))?; + *current_folder = Some(job.folder.clone()); + } + let set = command::format_uid_set(&job.uids, true); + let folder = job.folder.clone(); + let uv = job.uidvalidity; + client.run_streamed( + &command::uid_fetch( + &set, + &["UID", "FLAGS", "INTERNALDATE", "RFC822.SIZE", "BODY.PEEK[]"], + ), + |u| { + if let Untagged::Fetch { .. } = &u + && let Some(attrs) = super::fetch::extract(&u) + { + let _ = event_tx.send(FetchEvent::Item { + folder: folder.clone(), + uidvalidity: uv, + attrs, + }); + } + }, + )?; + Ok(()) +} diff --git a/src/sync/import_jmap.rs b/src/sync/import_jmap.rs new file mode 100644 index 0000000..818e322 --- /dev/null +++ b/src/sync/import_jmap.rs @@ -0,0 +1,1012 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod diff; +pub mod mapping; +pub mod pool; +pub mod tree; + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use rusqlite::Connection; +use serde_json::Value; + +use self::diff::diff; +use self::mapping::{BlobIntern, LocalResolver}; +use self::pool::{Pool, effective_workers}; +use self::tree::topo_order; +use crate::db; +use crate::db::sources::SourceKey; +use crate::error::Error; +use crate::jmap::blobxfer; +use crate::jmap::connect::{self, Connected}; +use crate::jmap::error::JmapError; +use crate::jmap::http::{Auth, HttpClient}; +use crate::jmap::request::{get_all, get_objects, query_all_ids}; +use crate::jmap::session::{Limits, Session}; +use crate::jmap::wire::JmapId; +use crate::jmap::wire::email::Email; +use crate::jmap::wire::file_node::{FileNode, NodeType}; +use crate::jmap::wire::sieve_script::SieveScript; +use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; +use crate::sync::{CommonConfig, Context, ImportConfig, Summary, TypeCounts}; +use crate::types::ObjectType; + +const IMPORT_ORDER: [ObjectType; 10] = [ + ObjectType::Mailbox, + ObjectType::AddressBook, + ObjectType::Calendar, + ObjectType::FileNode, + ObjectType::Identity, + ObjectType::SieveScript, + ObjectType::ParticipantIdentity, + ObjectType::Email, + ObjectType::ContactCard, + ObjectType::CalendarEvent, +]; + +fn is_queryless(ty: ObjectType) -> bool { + matches!( + ty, + ObjectType::Identity + | ObjectType::AddressBook + | ObjectType::Calendar + | ObjectType::ParticipantIdentity + ) +} + +fn get_props(ty: ObjectType) -> Option<&'static [&'static str]> { + match ty { + ObjectType::Mailbox => Some(&["name", "parentId", "role", "sortOrder", "isSubscribed"]), + ObjectType::Email => Some(&["blobId", "receivedAt", "mailboxIds", "keywords"]), + ObjectType::SieveScript => Some(&["name", "isActive", "blobId"]), + ObjectType::FileNode => Some(&[ + "parentId", + "nodeType", + "blobId", + "target", + "name", + "type", + "created", + "modified", + "isSubscribed", + "role", + ]), + _ => None, + } +} + +type GetMsg = Result<(Vec, usize), JmapError>; +type BlobRef = (String, String, String); +type DlMsg = (String, Result, JmapError>); + +#[derive(Clone)] +struct Net { + client: HttpClient, + api: String, + account: String, + limits: Limits, + session: Session, +} + +struct DbResolver<'a> { + conn: &'a Connection, + source_id: i64, +} + +impl LocalResolver for DbResolver<'_> { + fn local(&self, ty: ObjectType, jmap_id: &str) -> Option { + db::ids::local_for(self.conn, self.source_id, ty, jmap_id) + .ok() + .flatten() + } +} + +struct PrefetchedBlobs<'a> { + conn: &'a Connection, + bytes: &'a HashMap>, +} + +impl BlobIntern for PrefetchedBlobs<'_> { + fn intern(&mut self, jmap_blob_id: &str) -> Result { + let data = self.bytes.get(jmap_blob_id).ok_or_else(|| { + JmapError::malformed(format!("blob {jmap_blob_id} was not prefetched")) + })?; + Ok(db::blobs::intern_blob(self.conn, data)?) + } +} + +fn collect_blob_ids(value: &Value, out: &mut Vec) { + match value { + Value::Object(map) => { + if let Some(Value::String(b)) = map.get("blobId") { + out.push(b.clone()); + } + for v in map.values() { + collect_blob_ids(v, out); + } + } + Value::Array(items) => { + for v in items { + collect_blob_ids(v, out); + } + } + _ => {} + } +} + +fn username_of(auth: &Auth) -> String { + match auth { + Auth::Basic { user, .. } => user.clone(), + Auth::Bearer { .. } => "(bearer)".to_owned(), + } +} + +pub fn run(common: CommonConfig, config: ImportConfig) -> Result { + let logger = common.logger; + let ctx = Context::open(common, &config.connect)?; + let connected = connect::prepare(&ctx, &config.connect)?; + + let work = work_list(&config, &connected); + let key = SourceKey { + kind: "jmap".to_owned(), + session_url: config.connect.url.clone(), + account_id: connected.account_id.clone(), + }; + + if !ctx.dry_run() + && let Some((url, acc)) = + db::sources::conflicting_source(&ctx.conn, "jmap", &key.session_url, &key.account_id) + .map_err(|e| Error::Partial(e.to_string()))? + && !config.allow_source_change + { + return Err(Error::SourceChange(format!( + "archive already records JMAP source ({url}, account {acc}); \ + pass --allow-source-change to import a different account" + ))); + } + + let source_id = if ctx.dry_run() { + db::sources::find_source(&ctx.conn, &key).map_err(|e| Error::Partial(e.to_string()))? + } else { + let account_name = match &config.connect.account { + crate::jmap::account::AccountSelector::Name(n) => Some(n.clone()), + crate::jmap::account::AccountSelector::Id(_) => None, + }; + Some( + db::sources::upsert_source( + &ctx.conn, + &key, + account_name.as_deref(), + &username_of(ctx.client.auth()), + ) + .map_err(|e| Error::Partial(e.to_string()))?, + ) + }; + + let net = Net { + client: ctx.client.clone(), + api: connected.session.api_url.clone(), + account: connected.account_id.clone(), + limits: connected.limits, + session: connected.session.clone(), + }; + + let mut summary = Summary::default(); + let mut dry_rows: Vec<(&'static str, u64, u64, u64)> = Vec::new(); + let threads = ctx.common.threads; + + for ty in work { + if logger.enabled(LEVEL_DEFAULT) { + eprintln!("import: {} ...", ty.jmap_name()); + } + let mut counts = TypeCounts::default(); + match reconcile_type( + &ctx, + &net, + ty, + source_id, + threads, + &logger, + &mut counts, + &mut dry_rows, + ) { + Ok(()) => {} + Err(e) => { + logger.warn(&format!( + "type {} aborted: {e}; continuing (run will exit 5)", + ty.jmap_name() + )); + counts.failed += 1; + } + } + summary.per_type.push((ty.jmap_name(), counts)); + } + + if ctx.dry_run() { + print_dry_run(&dry_rows); + return Ok(Summary::default()); + } + + if !summary.any_failed() + && let Err(e) = run_gc(&ctx.conn) + { + logger.warn(&format!("blob GC skipped: {e}")); + } + + summary.retries_observed = ctx.client.retries_observed(); + summary.retry_after_sleeps = ctx.client.retry_after_sleeps(); + Ok(summary) +} + +fn work_list(config: &ImportConfig, connected: &Connected) -> Vec { + let selected = config.objects.as_ref(); + IMPORT_ORDER + .into_iter() + .filter(|ty| connected.supports(*ty)) + .filter(|ty| selected.map(|s| s.contains(ty)).unwrap_or(true)) + .collect() +} + +#[allow(clippy::too_many_arguments)] +fn reconcile_type( + ctx: &Context, + net: &Net, + ty: ObjectType, + source_id: Option, + threads: usize, + logger: &Logger, + counts: &mut TypeCounts, + dry_rows: &mut Vec<(&'static str, u64, u64, u64)>, +) -> Result<(), Error> { + let local_map: HashMap = match source_id { + Some(sid) => { + db::ids::jmap_to_local(&ctx.conn, sid, ty).map_err(|e| Error::Partial(e.to_string()))? + } + None => HashMap::new(), + }; + let local_ids: HashSet = local_map.keys().cloned().collect(); + + let (server_ids, preloaded): (Vec, Option>) = if is_queryless(ty) { + let got = get_all::(&net.client, &net.api, &net.account, ty.jmap_name()) + .map_err(Error::from)?; + let ids = got + .list + .iter() + .filter_map(|v| { + v.get("id") + .and_then(Value::as_str) + .map(|s| JmapId(s.to_owned())) + }) + .collect(); + (ids, Some(got.list)) + } else { + let ids = query_all_ids( + &net.client, + &net.api, + &net.account, + ty.jmap_name(), + &net.limits, + ) + .map_err(Error::from)?; + (ids, None) + }; + + let d = diff(&server_ids, &local_ids); + + if ctx.dry_run() { + dry_rows.push(( + ty.jmap_name(), + d.new.len() as u64, + d.vanished.len() as u64, + d.present.len() as u64, + )); + return Ok(()); + } + + let source_id = source_id.expect("source_id present outside dry-run"); + + if !d.new.is_empty() { + let objects = match preloaded { + Some(list) => { + let want: HashSet<&str> = d.new.iter().map(|i| i.0.as_str()).collect(); + list.into_iter() + .filter(|v| { + v.get("id") + .and_then(Value::as_str) + .map(|s| want.contains(s)) + .unwrap_or(false) + }) + .collect() + } + None => fetch_objects(net, ty, &d.new, threads, logger, counts), + }; + insert_objects(ctx, net, ty, source_id, objects, threads, logger, counts)?; + } + + if !d.vanished.is_empty() { + delete_vanished(&ctx.conn, ty, source_id, &d.vanished, &local_map, logger)?; + counts.deleted += d.vanished.len() as u64; + } + + if logger.enabled(LEVEL_DEFAULT) { + eprintln!( + "import: {} done (fetched={} deleted={} skipped={} failed={})", + ty.jmap_name(), + counts.fetched, + counts.deleted, + counts.skipped, + counts.failed + ); + } + Ok(()) +} + +fn fetch_objects( + net: &Net, + ty: ObjectType, + new_ids: &[JmapId], + threads: usize, + logger: &Logger, + counts: &mut TypeCounts, +) -> Vec { + let chunk = net.limits.max_objects_in_get.max(1) as usize; + let props = get_props(ty); + let workers = effective_workers(threads, &net.limits, false); + let net = Arc::new(net.clone()); + let pool: Pool, GetMsg> = Pool::new(workers, { + let net = net.clone(); + move |ids: Vec| { + let n = ids.len(); + get_objects::( + &net.client, + &net.api, + &net.account, + ty.jmap_name(), + &ids, + props, + &net.limits, + ) + .map(|r| (r.list, n)) + } + }); + + let mut batches = 0u64; + for c in new_ids.chunks(chunk) { + pool.submit(c.to_vec()); + batches += 1; + } + let mut out = Vec::with_capacity(new_ids.len()); + let mut done = 0u64; + for res in pool.finish() { + match res { + Ok((list, _)) => out.extend(list), + Err(e) => { + logger.warn(&format!("{} /get chunk failed: {e}", ty.jmap_name())); + counts.failed += 1; + } + } + done += 1; + } + if logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "import: {} fetched {}/{} chunks", + ty.jmap_name(), + done, + batches + ); + } + out +} + +#[allow(clippy::too_many_arguments)] +fn insert_objects( + ctx: &Context, + net: &Net, + ty: ObjectType, + source_id: i64, + objects: Vec, + threads: usize, + logger: &Logger, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let blob_refs = blob_references(ty, &objects); + let blobs = if blob_refs.is_empty() { + HashMap::new() + } else { + download_blobs(net, blob_refs, threads, logger, counts) + }; + + let existing_parents: HashSet = + if matches!(ty, ObjectType::Mailbox | ObjectType::FileNode) { + db::ids::jmap_to_local(&ctx.conn, source_id, ty) + .map(|m| m.into_keys().collect()) + .map_err(|e| Error::Partial(e.to_string()))? + } else { + HashSet::new() + }; + let (ordered, warnings) = order_objects(ty, &objects, &existing_parents); + for w in &warnings { + logger.warn(w); + } + + for batch in ordered.chunks(200) { + let tx = ctx + .conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + for &idx in batch { + let obj = &objects[idx]; + let jmap_id = match obj.get("id").and_then(Value::as_str) { + Some(s) => s.to_owned(), + None => { + counts.failed += 1; + continue; + } + }; + match insert_one(&tx, ty, source_id, obj, &blobs) { + Ok(local_id) => { + db::ids::insert(&tx, source_id, ty, &jmap_id, local_id) + .map_err(|e| Error::Partial(e.to_string()))?; + counts.fetched += 1; + } + Err(e) => { + logger.warn(&format!("{} {jmap_id} skipped: {e}", ty.jmap_name())); + counts.failed += 1; + } + } + } + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + } + Ok(()) +} + +fn blob_references(ty: ObjectType, objects: &[Value]) -> Vec { + let mut seen: HashSet = HashSet::new(); + let mut refs = Vec::new(); + match ty { + ObjectType::Email => { + for o in objects { + if let Some(b) = o.get("blobId").and_then(Value::as_str) + && seen.insert(b.to_owned()) + { + refs.push(( + b.to_owned(), + "message/rfc822".to_owned(), + "email".to_owned(), + )); + } + } + } + ObjectType::SieveScript => { + for o in objects { + if let Some(b) = o.get("blobId").and_then(Value::as_str) + && seen.insert(b.to_owned()) + { + refs.push(( + b.to_owned(), + "application/sieve".to_owned(), + "script".to_owned(), + )); + } + } + } + ObjectType::FileNode => { + for o in objects { + if let Some(b) = o.get("blobId").and_then(Value::as_str) + && seen.insert(b.to_owned()) + { + let mt = o + .get("type") + .and_then(Value::as_str) + .unwrap_or("application/octet-stream") + .to_owned(); + let name = o + .get("name") + .and_then(Value::as_str) + .unwrap_or("file") + .to_owned(); + refs.push((b.to_owned(), mt, name)); + } + } + } + ObjectType::ContactCard | ObjectType::CalendarEvent => { + for o in objects { + let mut ids = Vec::new(); + collect_blob_ids(o, &mut ids); + for b in ids { + if seen.insert(b.clone()) { + refs.push((b, "application/octet-stream".to_owned(), "blob".to_owned())); + } + } + } + } + _ => {} + } + refs +} + +fn download_blobs( + net: &Net, + refs: Vec, + threads: usize, + logger: &Logger, + counts: &mut TypeCounts, +) -> HashMap> { + let workers = effective_workers(threads, &net.limits, true); + let net_arc = Arc::new(net.clone()); + let pool: Pool = Pool::new(workers, { + let net = net_arc.clone(); + move |(blob_id, type_hint, name): (String, String, String)| { + let r = blobxfer::download_bytes( + &net.client, + &net.session, + &net.account, + &blob_id, + &type_hint, + &name, + ); + (blob_id, r) + } + }); + let total = refs.len(); + for r in refs { + pool.submit(r); + } + let mut out = HashMap::new(); + for (blob_id, res) in pool.finish() { + match res { + Ok(bytes) => { + out.insert(blob_id, bytes); + } + Err(e) => { + logger.warn(&format!("blob {blob_id} download failed: {e}")); + counts.failed += 1; + } + } + } + if logger.enabled(LEVEL_PROGRESS) { + eprintln!("import: downloaded {}/{} blobs", out.len(), total); + } + out +} + +fn order_objects( + ty: ObjectType, + objects: &[Value], + existing_parents: &HashSet, +) -> (Vec, Vec) { + if !matches!(ty, ObjectType::Mailbox | ObjectType::FileNode) { + return ((0..objects.len()).collect(), Vec::new()); + } + let items: Vec<(String, Option)> = objects + .iter() + .map(|o| { + ( + o.get("id") + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned(), + o.get("parentId").and_then(Value::as_str).map(str::to_owned), + ) + }) + .collect(); + let result = topo_order(&items, existing_parents); + let mut warnings = Vec::new(); + for i in &result.orphans { + let (id, parent) = &items[*i]; + warnings.push(format!( + "{} {id}: parent {} not present in batch or archive; inserting with NULL parent", + ty.jmap_name(), + parent.as_deref().unwrap_or(""), + )); + } + for i in &result.cycle_roots { + warnings.push(format!( + "{} {}: parent cycle detected; broken at this node", + ty.jmap_name(), + items[*i].0, + )); + } + (result.order, warnings) +} + +fn insert_one( + conn: &Connection, + ty: ObjectType, + source_id: i64, + obj: &Value, + blobs: &HashMap>, +) -> Result { + let resolver = DbResolver { conn, source_id }; + match ty { + ObjectType::Mailbox => { + let w = serde_json::from_value(obj.clone())?; + mapping::insert_mailbox(conn, &w, &resolver) + } + ObjectType::Identity => { + let w = serde_json::from_value(obj.clone())?; + mapping::insert_identity(conn, &w) + } + ObjectType::AddressBook => { + let w = serde_json::from_value(obj.clone())?; + mapping::insert_address_book(conn, &w) + } + ObjectType::Calendar => { + let w = serde_json::from_value(obj.clone())?; + mapping::insert_calendar(conn, &w) + } + ObjectType::ParticipantIdentity => { + let w = serde_json::from_value(obj.clone())?; + mapping::insert_participant_identity(conn, &w) + } + ObjectType::Email => { + let w: Email = serde_json::from_value(obj.clone())?; + let data = blobs + .get(&w.blob_id.0) + .ok_or_else(|| JmapError::malformed("email blob missing"))?; + let blob_local = db::blobs::intern_blob(conn, data)?; + let mm = crate::sync::keys::index_to_json( + &crate::sync::emailmeta::email_index_from_blob(data), + ); + mapping::insert_email(conn, &w, blob_local, &mm, &resolver) + } + ObjectType::SieveScript => { + let w: SieveScript = serde_json::from_value(obj.clone())?; + let data = blobs + .get(&w.blob_id.0) + .ok_or_else(|| JmapError::malformed("sieve blob missing"))?; + let blob_local = db::blobs::intern_blob(conn, data)?; + mapping::insert_sieve_script(conn, &w, blob_local) + } + ObjectType::FileNode => { + let w: FileNode = serde_json::from_value(obj.clone())?; + let blob_local = match (&w.node_type, &w.blob_id) { + (NodeType::File, Some(b)) => { + let data = blobs + .get(&b.0) + .ok_or_else(|| JmapError::malformed("file blob missing"))?; + Some(db::blobs::intern_blob(conn, data)?) + } + _ => None, + }; + mapping::insert_file_node(conn, &w, blob_local, &resolver) + } + ObjectType::ContactCard => { + let w = serde_json::from_value(obj.clone())?; + let mut bi = PrefetchedBlobs { conn, bytes: blobs }; + mapping::insert_contact_card(conn, &w, &resolver, &mut bi) + } + ObjectType::CalendarEvent => { + let w = serde_json::from_value(obj.clone())?; + let mut bi = PrefetchedBlobs { conn, bytes: blobs }; + mapping::insert_calendar_event(conn, &w, &resolver, &mut bi) + } + } +} + +fn delete_vanished( + conn: &Connection, + ty: ObjectType, + source_id: i64, + vanished: &[JmapId], + local_map: &HashMap, + logger: &Logger, +) -> Result<(), Error> { + let locals: Vec = vanished + .iter() + .filter_map(|j| local_map.get(&j.0).copied()) + .collect(); + if locals.is_empty() { + return Ok(()); + } + let tx = conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + let vanished_set: HashSet = locals.iter().copied().collect(); + + match ty { + ObjectType::Mailbox | ObjectType::FileNode => { + let table = if ty == ObjectType::Mailbox { + "mailboxes" + } else { + "file_nodes" + }; + let mut stmt = tx + .prepare(&format!("SELECT id, parent_id FROM {table}")) + .map_err(|e| Error::Partial(e.to_string()))?; + let rows: Vec<(i64, Option)> = stmt + .query_map([], |r| Ok((r.get(0)?, r.get(1)?))) + .and_then(|m| m.collect()) + .map_err(|e| Error::Partial(e.to_string()))?; + drop(stmt); + for (id, parent) in &rows { + if let Some(p) = parent + && vanished_set.contains(p) + && !vanished_set.contains(id) + { + tx.execute( + &format!("UPDATE {table} SET parent_id = NULL WHERE id = ?1"), + rusqlite::params![id], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + } + } + let by: HashMap> = rows.iter().copied().collect(); + let depth = |mut id: i64| -> usize { + let mut d = 0; + let mut seen = HashSet::new(); + while let Some(Some(p)) = by.get(&id) { + if !seen.insert(id) || !vanished_set.contains(p) { + break; + } + d += 1; + id = *p; + } + d + }; + let depths: HashMap = locals.iter().map(|id| (*id, depth(*id))).collect(); + let mut ordered = locals.clone(); + ordered.sort_by(|a, b| depths[b].cmp(&depths[a]).then(a.cmp(b))); + for id in ordered { + tx.execute( + &format!("DELETE FROM {table} WHERE id = ?1"), + rusqlite::params![id], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + } + } + _ => { + let table = crate::sync::table_name(ty); + for id in &locals { + tx.execute( + &format!("DELETE FROM {table} WHERE id = ?1"), + rusqlite::params![id], + ) + .map_err(|e| Error::Partial(e.to_string()))?; + } + } + } + + if let Some((table, column)) = cross_ref_target(ty) { + prune_id_arrays(&tx, table, column, &vanished_set) + .map_err(|e| Error::Partial(e.to_string()))?; + } + + for j in vanished { + db::ids::delete(&tx, source_id, ty, &j.0).map_err(|e| Error::Partial(e.to_string()))?; + } + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + if logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "import: {} deleted {} vanished", + ty.jmap_name(), + locals.len() + ); + } + Ok(()) +} + +fn cross_ref_target(ty: ObjectType) -> Option<(&'static str, &'static str)> { + match ty { + ObjectType::Mailbox => Some(("emails", "mailbox_ids")), + ObjectType::Calendar => Some(("calendar_events", "calendar_ids")), + ObjectType::AddressBook => Some(("contact_cards", "address_book_ids")), + _ => None, + } +} + +fn prune_id_arrays( + conn: &Connection, + table: &str, + column: &str, + removed: &HashSet, +) -> Result<(), rusqlite::Error> { + let mut stmt = conn.prepare(&format!("SELECT id, {column} FROM {table}"))?; + let rows: Vec<(i64, String)> = stmt + .query_map([], |r| Ok((r.get(0)?, r.get(1)?)))? + .collect::>()?; + drop(stmt); + for (id, json) in rows { + let arr: Vec = serde_json::from_str(&json).unwrap_or_default(); + if arr.iter().any(|v| removed.contains(v)) { + let kept: Vec = arr.into_iter().filter(|v| !removed.contains(v)).collect(); + let new_json = serde_json::to_string(&kept).unwrap_or_else(|_| "[]".to_owned()); + conn.execute( + &format!("UPDATE {table} SET {column} = ?1 WHERE id = ?2"), + rusqlite::params![new_json, id], + )?; + } + } + Ok(()) +} + +fn run_gc(conn: &Connection) -> Result<(), Error> { + let tx = conn + .unchecked_transaction() + .map_err(|e| Error::Partial(e.to_string()))?; + db::blobs::gc_orphan_blobs(&tx).map_err(|e| Error::Partial(e.to_string()))?; + tx.commit().map_err(|e| Error::Partial(e.to_string()))?; + Ok(()) +} + +fn print_dry_run(rows: &[(&'static str, u64, u64, u64)]) { + println!( + "{:<22} {:>8} {:>10} {:>9}", + "TYPE", "NEW", "VANISHED", "PRESENT" + ); + for (ty, new, vanished, present) in rows { + println!("{ty:<22} {new:>8} {vanished:>10} {present:>9}"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::init; + use crate::db::sources::{SourceKey, upsert_source}; + use crate::jmap::wire::JmapId; + use crate::logging::Logger; + + fn mem() -> (Connection, i64) { + let c = Connection::open_in_memory().unwrap(); + init::apply_schema(&c).unwrap(); + let sid = upsert_source( + &c, + &SourceKey { + kind: "jmap".to_owned(), + session_url: "u".to_owned(), + account_id: "w".to_owned(), + }, + None, + "u", + ) + .unwrap(); + (c, sid) + } + + #[test] + fn vanished_tree_deletes_leaf_first_and_nulls_true_orphan() { + let (c, sid) = mem(); + c.execute( + "INSERT INTO mailboxes (id,name,parent_id) VALUES (1,'root',NULL)", + [], + ) + .unwrap(); + c.execute( + "INSERT INTO mailboxes (id,name,parent_id) VALUES (2,'mid',1)", + [], + ) + .unwrap(); + c.execute( + "INSERT INTO mailboxes (id,name,parent_id) VALUES (3,'leaf',2)", + [], + ) + .unwrap(); + c.execute( + "INSERT INTO mailboxes (id,name,parent_id) VALUES (4,'survivor',2)", + [], + ) + .unwrap(); + for (jid, local) in [("R", 1), ("M", 2), ("L", 3), ("S", 4)] { + db::ids::insert(&c, sid, ObjectType::Mailbox, jid, local).unwrap(); + } + let local_map: HashMap = [ + ("R".to_owned(), 1), + ("M".to_owned(), 2), + ("L".to_owned(), 3), + ] + .into_iter() + .collect(); + let vanished = vec![ + JmapId("R".to_owned()), + JmapId("M".to_owned()), + JmapId("L".to_owned()), + ]; + delete_vanished( + &c, + ObjectType::Mailbox, + sid, + &vanished, + &local_map, + &Logger::from_flags(true, 0), + ) + .unwrap(); + let remaining: i64 = c + .query_row("SELECT count(*) FROM mailboxes", [], |r| r.get(0)) + .unwrap(); + assert_eq!(remaining, 1, "only the surviving child remains"); + let parent: Option = c + .query_row("SELECT parent_id FROM mailboxes WHERE id=4", [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!(parent, None, "true orphan's parent was nulled"); + assert!( + db::ids::local_for(&c, sid, ObjectType::Mailbox, "L") + .unwrap() + .is_none(), + "vanished sync_id rows removed" + ); + } + + #[test] + fn vanished_mailbox_is_pruned_from_email_mailbox_ids() { + let (c, sid) = mem(); + c.execute( + "INSERT INTO mailboxes (id,name,parent_id) VALUES (1,'a',NULL)", + [], + ) + .unwrap(); + c.execute( + "INSERT INTO mailboxes (id,name,parent_id) VALUES (2,'b',NULL)", + [], + ) + .unwrap(); + let blob = db::blobs::intern_blob(&c, b"msg").unwrap(); + c.execute( + "INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords) + VALUES (?1,'2020-01-01T00:00:00Z','[1,2]','[]')", + rusqlite::params![blob], + ) + .unwrap(); + db::ids::insert(&c, sid, ObjectType::Mailbox, "A", 1).unwrap(); + db::ids::insert(&c, sid, ObjectType::Mailbox, "B", 2).unwrap(); + let local_map: HashMap = [("A".to_owned(), 1)].into_iter().collect(); + delete_vanished( + &c, + ObjectType::Mailbox, + sid, + &[JmapId("A".to_owned())], + &local_map, + &Logger::from_flags(true, 0), + ) + .unwrap(); + let ids: String = c + .query_row("SELECT mailbox_ids FROM emails", [], |r| r.get(0)) + .unwrap(); + assert_eq!( + ids, "[2]", + "deleted mailbox id removed from referring email" + ); + } + + #[test] + fn order_objects_does_not_warn_when_parent_already_in_archive() { + let objects = vec![serde_json::json!({"id":"child","parentId":"existing"})]; + let existing: HashSet = ["existing".to_owned()].into_iter().collect(); + let (ordered, warnings) = order_objects(ObjectType::Mailbox, &objects, &existing); + assert_eq!(ordered, vec![0]); + assert!( + warnings.is_empty(), + "no orphan warning expected: {warnings:?}" + ); + } + + #[test] + fn order_objects_warns_on_orphan_parent() { + let objects = vec![serde_json::json!({"id":"orphan","parentId":"ghost"})]; + let (ordered, warnings) = order_objects(ObjectType::Mailbox, &objects, &HashSet::new()); + assert_eq!(ordered, vec![0]); + assert_eq!(warnings.len(), 1); + assert!(warnings[0].contains("ghost"), "msg was: {}", warnings[0]); + } + + #[test] + fn order_objects_warns_on_cycle() { + let objects = vec![ + serde_json::json!({"id":"a","parentId":"b"}), + serde_json::json!({"id":"b","parentId":"a"}), + ]; + let (ordered, warnings) = order_objects(ObjectType::Mailbox, &objects, &HashSet::new()); + assert_eq!(ordered.len(), 2); + assert!( + warnings.iter().any(|w| w.contains("cycle")), + "expected a cycle warning, got: {warnings:?}" + ); + } +} diff --git a/src/sync/import_jmap/diff.rs b/src/sync/import_jmap/diff.rs new file mode 100644 index 0000000..5b6dd7a --- /dev/null +++ b/src/sync/import_jmap/diff.rs @@ -0,0 +1,88 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashSet; + +use crate::jmap::wire::JmapId; + +pub struct IdSetDiff { + pub new: Vec, + pub vanished: Vec, + pub present: Vec, +} + +pub fn diff(server_ids: &[JmapId], local_ids: &HashSet) -> IdSetDiff { + let server_set: HashSet<&str> = server_ids.iter().map(|i| i.0.as_str()).collect(); + let mut seen: HashSet<&str> = HashSet::new(); + let mut new = Vec::new(); + let mut present = Vec::new(); + for id in server_ids { + if !seen.insert(id.0.as_str()) { + continue; + } + if local_ids.contains(&id.0) { + present.push(id.clone()); + } else { + new.push(id.clone()); + } + } + let mut vanished = Vec::new(); + for local in local_ids { + if !server_set.contains(local.as_str()) { + vanished.push(JmapId(local.clone())); + } + } + vanished.sort_by(|a, b| a.0.cmp(&b.0)); + IdSetDiff { + new, + vanished, + present, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ids(v: &[&str]) -> Vec { + v.iter().map(|s| JmapId((*s).to_owned())).collect() + } + + fn locals(v: &[&str]) -> HashSet { + v.iter().map(|s| (*s).to_owned()).collect() + } + + #[test] + fn partitions_new_vanished_present() { + let d = diff(&ids(&["a", "b", "c"]), &locals(&["b", "c", "d"])); + assert_eq!(d.new, ids(&["a"])); + assert_eq!(d.present, ids(&["b", "c"])); + assert_eq!(d.vanished, ids(&["d"])); + } + + #[test] + fn empty_local_means_all_new() { + let d = diff(&ids(&["x", "y"]), &locals(&[])); + assert_eq!(d.new, ids(&["x", "y"])); + assert!(d.vanished.is_empty()); + assert!(d.present.is_empty()); + } + + #[test] + fn empty_server_means_all_vanished() { + let d = diff(&ids(&[]), &locals(&["p", "q"])); + assert!(d.new.is_empty()); + assert_eq!(d.vanished, ids(&["p", "q"])); + } + + #[test] + fn duplicate_server_ids_are_collapsed() { + let d = diff(&ids(&["a", "a", "b"]), &locals(&["a"])); + assert_eq!(d.new, ids(&["b"])); + assert_eq!(d.present, ids(&["a"])); + assert!(d.vanished.is_empty()); + } +} diff --git a/src/sync/import_jmap/mapping.rs b/src/sync/import_jmap/mapping.rs new file mode 100644 index 0000000..4902d33 --- /dev/null +++ b/src/sync/import_jmap/mapping.rs @@ -0,0 +1,1020 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use indexmap::IndexMap; +use rusqlite::{Connection, Row, params}; +use serde_json::{Map, Value}; + +use crate::jmap::blob::{export_blob_ids, import_blob_ids}; +use crate::jmap::error::JmapError; +use crate::jmap::wire::JmapId; +use crate::jmap::wire::address_book::AddressBook; +use crate::jmap::wire::calendar::Calendar; +use crate::jmap::wire::calendar_event::CalendarEvent; +use crate::jmap::wire::contact_card::ContactCard; +use crate::jmap::wire::email::Email; +use crate::jmap::wire::file_node::FileNode; +use crate::jmap::wire::identity::Identity; +use crate::jmap::wire::mailbox::Mailbox; +use crate::jmap::wire::participant_identity::ParticipantIdentity; +use crate::jmap::wire::sieve_script::SieveScript; +use crate::types::ObjectType; + +pub trait LocalResolver { + fn local(&self, ty: ObjectType, jmap_id: &str) -> Option; +} + +pub trait TargetResolver { + fn target(&self, ty: ObjectType, local_id: i64) -> Option; +} + +pub trait BlobIntern { + fn intern(&mut self, jmap_blob_id: &str) -> Result; +} + +pub trait BlobUpload { + fn upload(&mut self, local_id: i64) -> Result; +} + +fn translate_in( + ids: &IndexMap, + ty: ObjectType, + resolver: &impl LocalResolver, +) -> Result, JmapError> { + let mut out = Vec::with_capacity(ids.len()); + for (id, _) in ids { + match resolver.local(ty, &id.0) { + Some(local) => out.push(local), + None => { + return Err(JmapError::malformed(format!( + "unresolved {} reference {} on insert", + ty.jmap_name(), + id.0 + ))); + } + } + } + Ok(out) +} + +fn translate_out( + locals: &[i64], + ty: ObjectType, + resolver: &impl TargetResolver, +) -> Result, JmapError> { + let mut out = IndexMap::new(); + for local in locals { + match resolver.target(ty, *local) { + Some(target) => { + out.insert(target, true); + } + None => { + return Err(JmapError::malformed(format!( + "unresolved local {} id {local} on export", + ty.jmap_name() + ))); + } + } + } + Ok(out) +} + +fn id_array_json(ids: &[i64]) -> String { + let vals: Vec = ids.iter().map(|i| Value::from(*i)).collect(); + Value::Array(vals).to_string() +} + +fn parse_local_id_array(text: &str) -> Result, JmapError> { + let v: Value = serde_json::from_str(text)?; + v.as_array() + .ok_or_else(|| JmapError::malformed("id column is not a JSON array"))? + .iter() + .map(|x| { + x.as_i64() + .ok_or_else(|| JmapError::malformed("id array element is not an integer")) + }) + .collect() +} + +fn opt_parent( + resolver: &impl LocalResolver, + ty: ObjectType, + parent: &Option, +) -> Option { + parent.as_ref().and_then(|p| resolver.local(ty, &p.0)) +} + +pub fn insert_mailbox( + conn: &Connection, + wire: &Mailbox, + resolver: &impl LocalResolver, +) -> Result { + let parent = opt_parent(resolver, ObjectType::Mailbox, &wire.parent_id); + conn.execute( + "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + wire.name, + parent, + wire.role, + wire.sort_order, + wire.is_subscribed as i64 + ], + )?; + Ok(conn.last_insert_rowid()) +} + +pub const MAILBOX_SELECT: &str = + "SELECT id, name, parent_id, role, sort_order, is_subscribed FROM mailboxes"; + +pub fn row_to_mailbox(row: &Row, resolver: &impl TargetResolver) -> Result { + let parent_local: Option = row.get(2)?; + let parent_id = match parent_local { + Some(p) => Some( + resolver + .target(ObjectType::Mailbox, p) + .ok_or_else(|| JmapError::malformed("unresolved mailbox parent on export"))?, + ), + None => None, + }; + Ok(Mailbox { + id: None, + name: row.get(1)?, + parent_id, + role: row.get(3)?, + sort_order: row.get::<_, i64>(4)? as u32, + is_subscribed: row.get::<_, i64>(5)? != 0, + }) +} + +pub fn insert_email( + conn: &Connection, + wire: &Email, + blob_local_id: i64, + message_match: &str, + resolver: &impl LocalResolver, +) -> Result { + let mailbox_locals = translate_in(&wire.mailbox_ids, ObjectType::Mailbox, resolver)?; + if mailbox_locals.is_empty() { + return Err(JmapError::malformed("email has no resolvable mailbox")); + } + let keywords: Vec = wire.keywords.keys().cloned().collect(); + conn.execute( + "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + blob_local_id, + format_utc(&wire.received_at)?, + id_array_json(&mailbox_locals), + Value::Array(keywords.iter().map(|k| Value::from(k.as_str())).collect()).to_string(), + message_match + ], + )?; + Ok(conn.last_insert_rowid()) +} + +#[derive(Debug, Clone)] +pub struct EmailRow { + pub blob_local_id: i64, + pub received_at: String, + pub mailbox_locals: Vec, + pub keywords: Vec, + pub message_match: String, +} + +pub const EMAIL_SELECT: &str = + "SELECT id, blob_id, received_at, mailbox_ids, keywords, message_match FROM emails"; + +pub fn row_to_email(row: &Row) -> Result { + let mailbox_text: String = row.get(3)?; + let keyword_text: String = row.get(4)?; + let keywords: Vec = serde_json::from_str(&keyword_text)?; + Ok(EmailRow { + blob_local_id: row.get(1)?, + received_at: row.get(2)?, + mailbox_locals: parse_local_id_array(&mailbox_text)?, + keywords, + message_match: row.get(5)?, + }) +} + +pub fn insert_identity(conn: &Connection, wire: &Identity) -> Result { + conn.execute( + "INSERT INTO identities (name, email, reply_to, bcc, text_signature, html_signature) + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + wire.name, + wire.email, + opt_json(&wire.reply_to)?, + opt_json(&wire.bcc)?, + wire.text_signature, + wire.html_signature + ], + )?; + Ok(conn.last_insert_rowid()) +} + +pub const IDENTITY_SELECT: &str = + "SELECT id, name, email, reply_to, bcc, text_signature, html_signature FROM identities"; + +pub fn row_to_identity(row: &Row) -> Result { + Ok(Identity { + id: None, + name: row.get(1)?, + email: row.get(2)?, + reply_to: from_opt_json(row.get::<_, Option>(3)?)?, + bcc: from_opt_json(row.get::<_, Option>(4)?)?, + text_signature: row.get(5)?, + html_signature: row.get(6)?, + }) +} + +pub fn insert_sieve_script( + conn: &Connection, + wire: &SieveScript, + blob_local_id: i64, +) -> Result { + conn.execute( + "INSERT INTO sieve_scripts (name, is_active, blob_id) VALUES (?1, ?2, ?3)", + params![wire.name, wire.is_active as i64, blob_local_id], + )?; + Ok(conn.last_insert_rowid()) +} + +pub const SIEVE_SELECT: &str = "SELECT id, name, is_active, blob_id FROM sieve_scripts"; + +pub fn insert_address_book(conn: &Connection, wire: &AddressBook) -> Result { + conn.execute( + "INSERT INTO address_books (name, description, sort_order, is_default, is_subscribed) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + wire.name, + wire.description, + wire.sort_order, + wire.is_default as i64, + wire.is_subscribed as i64 + ], + )?; + Ok(conn.last_insert_rowid()) +} + +pub const ADDRESS_BOOK_SELECT: &str = + "SELECT id, name, description, sort_order, is_default, is_subscribed FROM address_books"; + +pub fn insert_calendar(conn: &Connection, wire: &Calendar) -> Result { + conn.execute( + "INSERT INTO calendars (name, description, color, sort_order, is_subscribed, is_visible, + is_default, include_in_availability, default_alerts_with_time, + default_alerts_without_time, time_zone) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", + params![ + wire.name, + wire.description, + wire.color, + wire.sort_order, + wire.is_subscribed as i64, + wire.is_visible as i64, + wire.is_default as i64, + wire.include_in_availability, + opt_json(&wire.default_alerts_with_time)?, + opt_json(&wire.default_alerts_without_time)?, + wire.time_zone + ], + )?; + Ok(conn.last_insert_rowid()) +} + +pub const CALENDAR_SELECT: &str = "SELECT id, name, description, color, sort_order, is_subscribed, \ + is_visible, is_default, include_in_availability, default_alerts_with_time, \ + default_alerts_without_time, time_zone FROM calendars"; + +pub fn insert_participant_identity( + conn: &Connection, + wire: &ParticipantIdentity, +) -> Result { + conn.execute( + "INSERT INTO participant_identities (name, calendar_address, is_default) + VALUES (?1, ?2, ?3)", + params![wire.name, wire.calendar_address, wire.is_default as i64], + )?; + Ok(conn.last_insert_rowid()) +} + +pub const PARTICIPANT_IDENTITY_SELECT: &str = + "SELECT id, name, calendar_address, is_default FROM participant_identities"; + +pub fn insert_file_node( + conn: &Connection, + wire: &FileNode, + blob_local_id: Option, + resolver: &impl LocalResolver, +) -> Result { + let parent = opt_parent(resolver, ObjectType::FileNode, &wire.parent_id); + let node_type = serde_json::to_value(wire.node_type)? + .as_str() + .unwrap_or("file") + .to_owned(); + let target = match &wire.target { + Some(t) => Some(serde_json::to_string(t)?), + None => None, + }; + conn.execute( + "INSERT INTO file_nodes (parent_id, node_type, blob_id, target, name, media_type, + created, modified, is_subscribed, role) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", + params![ + parent, + node_type, + blob_local_id, + target, + wire.name, + wire.media_type, + format_utc(&wire.created)?, + opt_format_utc(&wire.modified)?, + wire.is_subscribed as i64, + wire.role + ], + )?; + Ok(conn.last_insert_rowid()) +} + +pub const FILE_NODE_SELECT: &str = "SELECT id, parent_id, node_type, blob_id, target, name, \ + media_type, created, modified, is_subscribed, role FROM file_nodes"; + +pub fn insert_contact_card( + conn: &Connection, + wire: &ContactCard, + resolver: &impl LocalResolver, + blobs: &mut impl BlobIntern, +) -> Result { + let address_books = translate_in(&wire.address_book_ids, ObjectType::AddressBook, resolver)?; + let mut data = Value::Object(value_map(&wire.rest)); + let uid = take_string(&mut data, "uid") + .ok_or_else(|| JmapError::malformed("ContactCard has no uid"))?; + rewrite_blobs_in(&mut data, blobs)?; + conn.execute( + "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)", + params![uid, id_array_json(&address_books), data.to_string()], + )?; + Ok(conn.last_insert_rowid()) +} + +pub const CONTACT_CARD_SELECT: &str = "SELECT id, uid, address_book_ids, data FROM contact_cards"; + +pub fn insert_calendar_event( + conn: &Connection, + wire: &CalendarEvent, + resolver: &impl LocalResolver, + blobs: &mut impl BlobIntern, +) -> Result { + let calendars = translate_in(&wire.calendar_ids, ObjectType::Calendar, resolver)?; + let mut data = Value::Object(value_map(&wire.rest)); + for drop_key in ["method", "utcStart", "utcEnd", "isOrigin", "baseEventId"] { + if let Value::Object(m) = &mut data { + m.remove(drop_key); + } + } + rewrite_blobs_in(&mut data, blobs)?; + conn.execute( + "INSERT INTO calendar_events (calendar_ids, is_draft, use_default_alerts, data) + VALUES (?1, ?2, ?3, ?4)", + params![ + id_array_json(&calendars), + wire.is_draft as i64, + wire.use_default_alerts as i64, + data.to_string() + ], + )?; + Ok(conn.last_insert_rowid()) +} + +pub const CALENDAR_EVENT_SELECT: &str = "SELECT id, calendar_ids, is_draft, use_default_alerts, data FROM calendar_events \ + WHERE data_type = 'Event'"; + +pub fn contact_card_to_wire( + uid: &str, + address_book_ids: &str, + data: &str, + resolver: &impl TargetResolver, + blobs: &mut impl BlobUpload, +) -> Result { + let mut value: Value = serde_json::from_str(data)?; + restore_blobs_in(&mut value, blobs)?; + let locals = parse_local_id_array(address_book_ids)?; + let abids = translate_out(&locals, ObjectType::AddressBook, resolver)?; + if let Value::Object(map) = &mut value { + map.insert("uid".to_owned(), Value::from(uid)); + map.insert("addressBookIds".to_owned(), id_bool_map(&abids)); + } + Ok(value) +} + +pub fn calendar_event_to_wire( + calendar_ids: &str, + is_draft: bool, + use_default_alerts: bool, + data: &str, + resolver: &impl TargetResolver, + blobs: &mut impl BlobUpload, +) -> Result { + let mut value: Value = serde_json::from_str(data)?; + restore_blobs_in(&mut value, blobs)?; + let locals = parse_local_id_array(calendar_ids)?; + let calids = translate_out(&locals, ObjectType::Calendar, resolver)?; + if let Value::Object(map) = &mut value { + map.insert("calendarIds".to_owned(), id_bool_map(&calids)); + map.insert("isDraft".to_owned(), Value::Bool(is_draft)); + map.insert( + "useDefaultAlerts".to_owned(), + Value::Bool(use_default_alerts), + ); + } + Ok(value) +} + +fn id_bool_map(ids: &IndexMap) -> Value { + let mut m = Map::new(); + for (id, _) in ids { + m.insert(id.0.clone(), Value::Bool(true)); + } + Value::Object(m) +} + +fn value_map(rest: &IndexMap) -> Map { + let mut m = Map::new(); + for (k, v) in rest { + m.insert(k.clone(), v.clone()); + } + m +} + +fn take_string(value: &mut Value, key: &str) -> Option { + if let Value::Object(map) = value + && let Some(Value::String(s)) = map.remove(key) + { + return Some(s); + } + None +} + +fn rewrite_blobs_in(data: &mut Value, blobs: &mut impl BlobIntern) -> Result<(), JmapError> { + import_blob_ids(data, |jmap_blob_id| { + blobs + .intern(jmap_blob_id) + .map_err(|e| crate::jmap::blob::BlobWalkError::Resolver(e.to_string())) + }) + .map_err(JmapError::from) +} + +fn restore_blobs_in(data: &mut Value, blobs: &mut impl BlobUpload) -> Result<(), JmapError> { + export_blob_ids(data, |local_id| { + blobs + .upload(local_id) + .map(|id| id.0) + .map_err(|e| crate::jmap::blob::BlobWalkError::Resolver(e.to_string())) + }) + .map_err(JmapError::from) +} + +fn opt_json(value: &Option) -> Result, JmapError> { + match value { + Some(v) => Ok(Some(serde_json::to_string(v)?)), + None => Ok(None), + } +} + +fn from_opt_json( + text: Option, +) -> Result, JmapError> { + match text { + Some(t) => Ok(Some(serde_json::from_str(&t)?)), + None => Ok(None), + } +} + +pub fn row_to_address_book(row: &Row) -> Result { + Ok(AddressBook { + id: None, + name: row.get(1)?, + description: row.get(2)?, + sort_order: row.get::<_, i64>(3)? as u32, + is_default: row.get::<_, i64>(4)? != 0, + is_subscribed: row.get::<_, i64>(5)? != 0, + }) +} + +pub fn row_to_calendar(row: &Row) -> Result { + Ok(Calendar { + id: None, + name: row.get(1)?, + description: row.get(2)?, + color: row.get(3)?, + sort_order: row.get::<_, i64>(4)? as u32, + is_subscribed: row.get::<_, i64>(5)? != 0, + is_visible: row.get::<_, i64>(6)? != 0, + is_default: row.get::<_, i64>(7)? != 0, + include_in_availability: row.get(8)?, + default_alerts_with_time: from_opt_json(row.get::<_, Option>(9)?)?, + default_alerts_without_time: from_opt_json(row.get::<_, Option>(10)?)?, + time_zone: row.get(11)?, + }) +} + +pub fn row_to_participant_identity(row: &Row) -> Result { + Ok(ParticipantIdentity { + id: None, + name: row.get(1)?, + calendar_address: row.get(2)?, + is_default: row.get::<_, i64>(3)? != 0, + }) +} + +#[derive(Debug, Clone)] +pub struct SieveRow { + pub name: Option, + pub is_active: bool, + pub blob_local_id: i64, +} + +pub fn row_to_sieve_script(row: &Row) -> Result { + Ok(SieveRow { + name: row.get(1)?, + is_active: row.get::<_, i64>(2)? != 0, + blob_local_id: row.get(3)?, + }) +} + +#[derive(Debug, Clone)] +pub struct FileNodeRow { + pub wire: FileNode, + pub blob_local_id: Option, +} + +pub fn row_to_file_node( + row: &Row, + resolver: &impl TargetResolver, +) -> Result { + let parent_local: Option = row.get(1)?; + let parent_id = match parent_local { + Some(p) => Some( + resolver + .target(ObjectType::FileNode, p) + .ok_or_else(|| JmapError::malformed("unresolved file_node parent on export"))?, + ), + None => None, + }; + let node_type = serde_json::from_value(Value::from(row.get::<_, String>(2)?))?; + let target: Option> = from_opt_json(row.get::<_, Option>(4)?)?; + let blob_local_id: Option = row.get(3)?; + Ok(FileNodeRow { + wire: FileNode { + id: None, + parent_id, + node_type, + blob_id: None, + target, + name: row.get(5)?, + media_type: row.get(6)?, + created: parse_utc(&row.get::<_, String>(7)?)?, + modified: opt_parse_utc(row.get::<_, Option>(8)?)?, + is_subscribed: row.get::<_, i64>(9)? != 0, + role: row.get(10)?, + }, + blob_local_id, + }) +} + +fn parse_utc(text: &str) -> Result { + time::OffsetDateTime::parse(text, &time::format_description::well_known::Rfc3339) + .map_err(|e| JmapError::malformed(format!("cannot parse date {text}: {e}"))) +} + +fn opt_parse_utc(text: Option) -> Result, JmapError> { + match text { + Some(t) => Ok(Some(parse_utc(&t)?)), + None => Ok(None), + } +} + +fn format_utc(value: &time::OffsetDateTime) -> Result { + value + .format(&time::format_description::well_known::Rfc3339) + .map_err(|e| JmapError::malformed(format!("cannot format date: {e}"))) +} + +fn opt_format_utc(value: &Option) -> Result, JmapError> { + match value { + Some(v) => Ok(Some(format_utc(v)?)), + None => Ok(None), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::init; + use std::collections::HashMap; + + struct MapResolver { + to_local: HashMap<(ObjectType, String), i64>, + to_target: HashMap<(ObjectType, i64), String>, + } + impl LocalResolver for MapResolver { + fn local(&self, ty: ObjectType, jmap_id: &str) -> Option { + self.to_local.get(&(ty, jmap_id.to_owned())).copied() + } + } + impl TargetResolver for MapResolver { + fn target(&self, ty: ObjectType, local_id: i64) -> Option { + self.to_target + .get(&(ty, local_id)) + .map(|s| JmapId(s.clone())) + } + } + + struct FakeBlobs; + impl BlobIntern for FakeBlobs { + fn intern(&mut self, _jmap_blob_id: &str) -> Result { + Ok(42) + } + } + impl BlobUpload for FakeBlobs { + fn upload(&mut self, local_id: i64) -> Result { + Ok(JmapId(format!("T{local_id}"))) + } + } + + fn mem() -> Connection { + let c = Connection::open_in_memory().unwrap(); + init::apply_schema(&c).unwrap(); + c + } + + #[test] + fn mailbox_roundtrips_with_parent_translation() { + let c = mem(); + let root: Mailbox = serde_json::from_value(serde_json::json!({ + "id": "M1", "name": "Root", "role": "archive", "sortOrder": 3 + })) + .unwrap(); + let empty = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + let root_local = insert_mailbox(&c, &root, &empty).unwrap(); + + let child: Mailbox = serde_json::from_value(serde_json::json!({ + "id": "M2", "name": "Child", "parentId": "M1" + })) + .unwrap(); + let mut res = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + res.to_local + .insert((ObjectType::Mailbox, "M1".to_owned()), root_local); + let child_local = insert_mailbox(&c, &child, &res).unwrap(); + + res.to_target + .insert((ObjectType::Mailbox, root_local), "TGT1".to_owned()); + let wire = c + .query_row( + &format!("{MAILBOX_SELECT} WHERE id = ?1"), + params![child_local], + |row| Ok(row_to_mailbox(row, &res)), + ) + .unwrap() + .unwrap(); + assert_eq!(wire.name, "Child"); + assert_eq!(wire.parent_id, Some(JmapId("TGT1".to_owned()))); + } + + #[test] + fn email_keywords_keep_verbatim_case_and_mailbox_translation() { + let c = mem(); + let mut res = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + res.to_local + .insert((ObjectType::Mailbox, "MB".to_owned()), 7); + let email: Email = serde_json::from_value(serde_json::json!({ + "id": "E1", + "blobId": "B1", + "receivedAt": "2021-05-04T10:00:00Z", + "mailboxIds": { "MB": true }, + "keywords": { "$Seen": true, "MyTag": true } + })) + .unwrap(); + let blob_id = crate::db::blobs::intern_blob(&c, b"rfc5322 bytes").unwrap(); + let local = insert_email(&c, &email, blob_id, "{}", &res).unwrap(); + let got = c + .query_row( + &format!("{EMAIL_SELECT} WHERE id = ?1"), + params![local], + |row| Ok(row_to_email(row)), + ) + .unwrap() + .unwrap(); + assert_eq!(got.blob_local_id, blob_id); + assert_eq!(got.mailbox_locals, vec![7]); + assert_eq!(got.keywords, vec!["$Seen".to_owned(), "MyTag".to_owned()]); + } + + #[test] + fn unresolved_foreign_key_on_insert_is_malformed() { + let c = mem(); + let empty = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + let email: Email = serde_json::from_value(serde_json::json!({ + "id": "E1", + "blobId": "B1", + "receivedAt": "2021-05-04T10:00:00Z", + "mailboxIds": { "UNKNOWN": true }, + "keywords": {} + })) + .unwrap(); + let err = insert_email(&c, &email, 1, "{}", &empty).unwrap_err(); + assert!(matches!(err, JmapError::Malformed(_)), "got {err:?}"); + } + + #[test] + fn unresolved_target_on_export_is_malformed() { + let c = mem(); + let root: Mailbox = serde_json::from_value(serde_json::json!({ + "id": "M1", "name": "Root", "parentId": "P9" + })) + .unwrap(); + let mut res = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + res.to_local + .insert((ObjectType::Mailbox, "P9".to_owned()), 1); + let local = insert_mailbox(&c, &root, &res).unwrap(); + let err = c + .query_row( + &format!("{MAILBOX_SELECT} WHERE id = ?1"), + params![local], + |row| Ok(row_to_mailbox(row, &res)), + ) + .unwrap() + .unwrap_err(); + assert!(matches!(err, JmapError::Malformed(_)), "got {err:?}"); + } + + #[test] + fn contact_card_strips_uid_and_rewrites_blob() { + let c = mem(); + let res = MapResolver { + to_local: HashMap::from([((ObjectType::AddressBook, "AB".to_owned()), 1)]), + to_target: HashMap::from([((ObjectType::AddressBook, 1), "TAB".to_owned())]), + }; + let card: ContactCard = serde_json::from_value(serde_json::json!({ + "id": "C1", + "addressBookIds": { "AB": true }, + "uid": "urn:uuid:42", + "name": { "full": "Jane" }, + "photos": { "p1": { "blobId": "PB", "mediaType": "image/png" } } + })) + .unwrap(); + let mut blobs = FakeBlobs; + let local = insert_contact_card(&c, &card, &res, &mut blobs).unwrap(); + let (uid, abids, data): (String, String, String) = c + .query_row( + &format!("{CONTACT_CARD_SELECT} WHERE id = ?1"), + params![local], + |r| Ok((r.get(1)?, r.get(2)?, r.get(3)?)), + ) + .unwrap(); + assert_eq!(uid, "urn:uuid:42"); + let stored: Value = serde_json::from_str(&data).unwrap(); + assert!(stored.get("uid").is_none()); + assert_eq!(stored["photos"]["p1"]["@blob"], Value::from(42)); + + let mut up = FakeBlobs; + let wire = contact_card_to_wire(&uid, &abids, &data, &res, &mut up).unwrap(); + assert_eq!(wire["uid"], Value::from("urn:uuid:42")); + assert_eq!(wire["addressBookIds"]["TAB"], Value::Bool(true)); + assert_eq!(wire["photos"]["p1"]["blobId"], Value::from("T42")); + assert!(wire["photos"]["p1"].get("@blob").is_none()); + } + + #[test] + fn calendar_event_extracts_columns_and_drops_method() { + let c = mem(); + let res = MapResolver { + to_local: HashMap::from([((ObjectType::Calendar, "CAL".to_owned()), 5)]), + to_target: HashMap::from([((ObjectType::Calendar, 5), "TCAL".to_owned())]), + }; + let ev: CalendarEvent = serde_json::from_value(serde_json::json!({ + "id": "EV1", + "calendarIds": { "CAL": true }, + "isDraft": true, + "useDefaultAlerts": false, + "method": "request", + "utcStart": "2020-01-01T00:00:00Z", + "title": "Sprint", + "@type": "Event" + })) + .unwrap(); + let mut blobs = FakeBlobs; + let local = insert_calendar_event(&c, &ev, &res, &mut blobs).unwrap(); + let (cal, dr, ud, data): (String, i64, i64, String) = c + .query_row( + &format!("{CALENDAR_EVENT_SELECT} AND id = ?1"), + params![local], + |r| Ok((r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)), + ) + .unwrap(); + assert_eq!(dr, 1); + assert_eq!(ud, 0); + let stored: Value = serde_json::from_str(&data).unwrap(); + assert!(stored.get("method").is_none()); + assert!(stored.get("utcStart").is_none()); + assert!(stored.get("calendarIds").is_none()); + assert_eq!(stored["title"], Value::from("Sprint")); + + let mut up = FakeBlobs; + let wire = calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, &res, &mut up).unwrap(); + assert_eq!(wire["calendarIds"]["TCAL"], Value::Bool(true)); + assert_eq!(wire["isDraft"], Value::Bool(true)); + assert_eq!(wire["title"], Value::from("Sprint")); + } + + #[test] + fn identity_json_columns_roundtrip() { + let c = mem(); + let id: Identity = serde_json::from_value(serde_json::json!({ + "id": "I1", + "name": "Alice", + "email": "a@x.test", + "replyTo": [ { "name": "A", "email": "a@x.test" } ] + })) + .unwrap(); + let local = insert_identity(&c, &id).unwrap(); + let wire = c + .query_row( + &format!("{IDENTITY_SELECT} WHERE id = ?1"), + params![local], + |row| Ok(row_to_identity(row)), + ) + .unwrap() + .unwrap(); + assert_eq!(wire.email, "a@x.test"); + assert_eq!(wire.reply_to.unwrap()[0].email, "a@x.test"); + assert!(wire.bcc.is_none()); + } + + #[test] + fn file_node_enum_and_target_roundtrip() { + let c = mem(); + let empty = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + let node: FileNode = serde_json::from_value(serde_json::json!({ + "id": "F1", + "nodeType": "symlink", + "target": ["..", "etc", "hosts"], + "name": "link", + "created": "2022-02-02T02:02:02Z" + })) + .unwrap(); + let local = insert_file_node(&c, &node, None, &empty).unwrap(); + let (nt, tgt): (String, Option) = c + .query_row( + &format!("{FILE_NODE_SELECT} WHERE id = ?1"), + params![local], + |r| Ok((r.get(2)?, r.get(4)?)), + ) + .unwrap(); + assert_eq!(nt, "symlink"); + assert_eq!( + serde_json::from_str::>(&tgt.unwrap()).unwrap(), + vec!["..", "etc", "hosts"] + ); + } + + #[test] + fn address_book_and_calendar_row_to_wire_roundtrip() { + let c = mem(); + let ab: AddressBook = serde_json::from_value(serde_json::json!({ + "id": "A1", "name": "Personal", "description": "d", "sortOrder": 2, + "isDefault": true, "isSubscribed": false + })) + .unwrap(); + let abl = insert_address_book(&c, &ab).unwrap(); + let got = c + .query_row( + &format!("{ADDRESS_BOOK_SELECT} WHERE id = ?1"), + params![abl], + |r| Ok(row_to_address_book(r)), + ) + .unwrap() + .unwrap(); + assert_eq!(got.name, "Personal"); + assert!(got.is_default); + assert!(!got.is_subscribed); + + let cal: Calendar = serde_json::from_value(serde_json::json!({ + "id": "C1", "name": "Work", "color": "#ff0000", "timeZone": "Europe/Rome", + "isVisible": false + })) + .unwrap(); + let cl = insert_calendar(&c, &cal).unwrap(); + let gc = c + .query_row( + &format!("{CALENDAR_SELECT} WHERE id = ?1"), + params![cl], + |r| Ok(row_to_calendar(r)), + ) + .unwrap() + .unwrap(); + assert_eq!(gc.name, "Work"); + assert_eq!(gc.color.as_deref(), Some("#ff0000")); + assert_eq!(gc.time_zone.as_deref(), Some("Europe/Rome")); + assert!(!gc.is_visible); + } + + #[test] + fn participant_identity_and_sieve_row_to_wire_roundtrip() { + let c = mem(); + let pi: ParticipantIdentity = serde_json::from_value(serde_json::json!({ + "id": "P1", "name": "Me", "calendarAddress": "mailto:me@x.test", + "isDefault": true + })) + .unwrap(); + let pil = insert_participant_identity(&c, &pi).unwrap(); + let gp = c + .query_row( + &format!("{PARTICIPANT_IDENTITY_SELECT} WHERE id = ?1"), + params![pil], + |r| Ok(row_to_participant_identity(r)), + ) + .unwrap() + .unwrap(); + assert_eq!(gp.calendar_address, "mailto:me@x.test"); + assert!(gp.is_default); + + let ss: SieveScript = serde_json::from_value(serde_json::json!({ + "id": "S1", "name": "main", "isActive": true, "blobId": "B1" + })) + .unwrap(); + let blob = crate::db::blobs::intern_blob(&c, b"keep;").unwrap(); + let ssl = insert_sieve_script(&c, &ss, blob).unwrap(); + let gs = c + .query_row( + &format!("{SIEVE_SELECT} WHERE id = ?1"), + params![ssl], + |r| Ok(row_to_sieve_script(r)), + ) + .unwrap() + .unwrap(); + assert_eq!(gs.name.as_deref(), Some("main")); + assert!(gs.is_active); + assert_eq!(gs.blob_local_id, blob); + } + + #[test] + fn file_node_row_to_wire_translates_parent() { + let c = mem(); + let mut res = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + let dir: FileNode = serde_json::from_value(serde_json::json!({ + "id": "D1", "nodeType": "directory", "name": "Docs", + "created": "2022-02-02T02:02:02Z" + })) + .unwrap(); + let dirl = insert_file_node(&c, &dir, None, &res).unwrap(); + res.to_local + .insert((ObjectType::FileNode, "D1".to_owned()), dirl); + let file: FileNode = serde_json::from_value(serde_json::json!({ + "id": "F1", "nodeType": "file", "parentId": "D1", "name": "a.bin", + "type": "application/octet-stream", "created": "2022-02-02T02:02:02Z" + })) + .unwrap(); + let blob = crate::db::blobs::intern_blob(&c, b"bytes").unwrap(); + let fl = insert_file_node(&c, &file, Some(blob), &res).unwrap(); + res.to_target + .insert((ObjectType::FileNode, dirl), "TGTD".to_owned()); + let got = c + .query_row( + &format!("{FILE_NODE_SELECT} WHERE id = ?1"), + params![fl], + |r| Ok(row_to_file_node(r, &res)), + ) + .unwrap() + .unwrap(); + assert_eq!(got.wire.name, "a.bin"); + assert_eq!(got.wire.parent_id, Some(JmapId("TGTD".to_owned()))); + assert_eq!(got.blob_local_id, Some(blob)); + } +} diff --git a/src/sync/import_jmap/pool.rs b/src/sync/import_jmap/pool.rs new file mode 100644 index 0000000..ca86bbc --- /dev/null +++ b/src/sync/import_jmap/pool.rs @@ -0,0 +1,134 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::thread::JoinHandle; + +use crossbeam_channel::{Receiver, Sender, unbounded}; + +use crate::jmap::session::Limits; + +pub fn effective_workers(requested: usize, limits: &Limits, upload: bool) -> usize { + let server_cap = if upload { + limits.max_concurrent_upload + } else { + limits.max_concurrent_requests + }; + let cap = server_cap.max(1).min(usize::MAX as u64) as usize; + requested.clamp(1, cap) +} + +pub struct Pool +where + J: Send + 'static, + R: Send + 'static, +{ + job_tx: Option>, + result_rx: Receiver, + workers: Vec>, +} + +impl Pool +where + J: Send + 'static, + R: Send + 'static, +{ + pub fn new(threads: usize, worker: F) -> Pool + where + F: Fn(J) -> R + Send + Sync + 'static, + { + let threads = threads.max(1); + let (job_tx, job_rx) = unbounded::(); + let (result_tx, result_rx) = unbounded::(); + let worker = std::sync::Arc::new(worker); + let mut workers = Vec::with_capacity(threads); + for _ in 0..threads { + let job_rx = job_rx.clone(); + let result_tx = result_tx.clone(); + let worker = worker.clone(); + workers.push(std::thread::spawn(move || { + while let Ok(job) = job_rx.recv() { + if result_tx.send(worker(job)).is_err() { + break; + } + } + })); + } + Pool { + job_tx: Some(job_tx), + result_rx, + workers, + } + } + + pub fn submit(&self, job: J) { + if let Some(tx) = &self.job_tx { + let _ = tx.send(job); + } + } + + pub fn results(&self) -> &Receiver { + &self.result_rx + } + + pub fn finish(mut self) -> Vec { + drop(self.job_tx.take()); + for handle in self.workers.drain(..) { + let _ = handle.join(); + } + self.result_rx.try_iter().collect() + } +} + +impl Drop for Pool +where + J: Send + 'static, + R: Send + 'static, +{ + fn drop(&mut self) { + drop(self.job_tx.take()); + for handle in self.workers.drain(..) { + let _ = handle.join(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn limits(req: u64, up: u64) -> Limits { + Limits { + max_objects_in_get: 500, + max_objects_in_set: 500, + max_calls_in_request: 16, + max_concurrent_requests: req, + max_concurrent_upload: up, + max_size_request: 10_000_000, + max_size_upload: 50_000_000, + } + } + + #[test] + fn worker_count_is_clamped_to_server_limits() { + let l = limits(4, 2); + assert_eq!(effective_workers(16, &l, false), 4); + assert_eq!(effective_workers(16, &l, true), 2); + assert_eq!(effective_workers(1, &l, false), 1); + assert_eq!(effective_workers(0, &l, false), 1); + } + + #[test] + fn pool_processes_all_jobs_across_workers() { + let pool: Pool = Pool::new(4, |n| n * n); + for n in 0..100 { + pool.submit(n); + } + let mut got = pool.finish(); + got.sort_unstable(); + let expected: Vec = (0..100).map(|n| n * n).collect(); + assert_eq!(got, expected); + } +} diff --git a/src/sync/import_jmap/tree.rs b/src/sync/import_jmap/tree.rs new file mode 100644 index 0000000..77cb554 --- /dev/null +++ b/src/sync/import_jmap/tree.rs @@ -0,0 +1,131 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; + +pub struct TreeOrder { + pub order: Vec, + pub orphans: Vec, + pub cycle_roots: Vec, +} + +pub fn topo_order( + items: &[(String, Option)], + existing_parents: &HashSet, +) -> TreeOrder { + let index: HashMap<&str, usize> = items + .iter() + .enumerate() + .map(|(i, (id, _))| (id.as_str(), i)) + .collect(); + + let mut order = Vec::with_capacity(items.len()); + let mut orphans = Vec::new(); + let mut cycle_roots = Vec::new(); + let mut placed: HashSet = HashSet::new(); + + loop { + let mut progressed = false; + for (i, (_, parent)) in items.iter().enumerate() { + if placed.contains(&i) { + continue; + } + let ready = match parent { + None => true, + Some(p) => match index.get(p.as_str()) { + Some(&pi) => placed.contains(&pi), + None => { + if !existing_parents.contains(p) { + orphans.push(i); + } + true + } + }, + }; + if ready { + order.push(i); + placed.insert(i); + progressed = true; + } + } + if placed.len() == items.len() { + break; + } + if !progressed { + let mut remaining: Vec = + (0..items.len()).filter(|i| !placed.contains(i)).collect(); + remaining.sort_by(|&a, &b| items[a].0.cmp(&items[b].0)); + let root = remaining[0]; + cycle_roots.push(root); + order.push(root); + placed.insert(root); + } + } + + TreeOrder { + order, + orphans, + cycle_roots, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn item(id: &str, parent: Option<&str>) -> (String, Option) { + (id.to_owned(), parent.map(str::to_owned)) + } + + fn empty() -> HashSet { + HashSet::new() + } + + fn pos(order: &[usize], items: &[(String, Option)], id: &str) -> usize { + order + .iter() + .position(|&i| items[i].0 == id) + .expect("id in order") + } + + #[test] + fn parents_precede_children() { + let items = vec![item("c", Some("b")), item("b", Some("a")), item("a", None)]; + let t = topo_order(&items, &empty()); + assert_eq!(t.order.len(), 3); + assert!(pos(&t.order, &items, "a") < pos(&t.order, &items, "b")); + assert!(pos(&t.order, &items, "b") < pos(&t.order, &items, "c")); + assert!(t.orphans.is_empty()); + assert!(t.cycle_roots.is_empty()); + } + + #[test] + fn parent_not_fetched_and_not_existing_is_orphan() { + let items = vec![item("x", Some("missing"))]; + let t = topo_order(&items, &empty()); + assert_eq!(t.order, vec![0]); + assert_eq!(t.orphans, vec![0]); + } + + #[test] + fn parent_already_in_db_is_not_orphan() { + let items = vec![item("x", Some("p"))]; + let existing: HashSet = ["p".to_owned()].into_iter().collect(); + let t = topo_order(&items, &existing); + assert_eq!(t.order, vec![0]); + assert!(t.orphans.is_empty()); + } + + #[test] + fn cycle_is_broken_by_smallest_id() { + let items = vec![item("b", Some("a")), item("a", Some("b"))]; + let t = topo_order(&items, &empty()); + assert_eq!(t.order.len(), 2); + assert_eq!(t.cycle_roots.len(), 1); + assert_eq!(items[t.cycle_roots[0]].0, "a"); + assert!(pos(&t.order, &items, "a") < pos(&t.order, &items, "b")); + } +} diff --git a/src/sync/import_maildir.rs b/src/sync/import_maildir.rs new file mode 100644 index 0000000..8590d50 --- /dev/null +++ b/src/sync/import_maildir.rs @@ -0,0 +1,12 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod coordinator; +pub mod keywords; +pub mod messages; +pub mod tree; + +pub use coordinator::{MaildirImportConfig, run}; diff --git a/src/sync/import_maildir/coordinator.rs b/src/sync/import_maildir/coordinator.rs new file mode 100644 index 0000000..2ff08f0 --- /dev/null +++ b/src/sync/import_maildir/coordinator.rs @@ -0,0 +1,504 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; + +use regex::Regex; +use rusqlite::{Connection, OptionalExtension, params}; +use url::Url; + +use crate::db; +use crate::db::sources::SourceKey; +use crate::error::Error; +use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; +use crate::sync::{CommonConfig, Summary, TypeCounts}; + +use super::messages; +use super::tree; +use super::tree::{DiscoverError, FolderFilters, ResolvedFolder}; + +pub struct MaildirImportConfig { + pub maildir: PathBuf, + pub include: Vec, + pub exclude: Vec, + pub folder: Vec, + pub automap: bool, + pub include_deleted: bool, + pub allow_source_change: bool, +} + +pub fn run(common: CommonConfig, config: MaildirImportConfig) -> Result { + let logger = common.logger; + if common.threads > 1 { + log_at( + logger, + LEVEL_PROGRESS, + "maildir importer is single-threaded; --threads value will be ignored", + ); + } + + let canonical = std::fs::canonicalize(&config.maildir) + .map_err(|e| Error::Usage(format!("--maildir {:?}: {e}", config.maildir)))?; + + warn_if_looks_like_subfolder(&canonical, logger); + + let discovered = match tree::discover(&canonical, config.automap) { + Ok(v) => v, + Err(DiscoverError::NotFound(p)) => { + return Err(Error::Usage(format!("maildir path does not exist: {p:?}"))); + } + Err(DiscoverError::NotADirectory(p)) => { + return Err(Error::Usage(format!("not a directory: {p:?}"))); + } + Err(DiscoverError::NotAMaildir(p)) => { + return Err(Error::Usage(format!( + "not a Maildir: {p:?} (missing cur/ subdirectory)" + ))); + } + Err(DiscoverError::NotMaildirPlus(name)) => { + return Err(Error::Usage(format!( + "only Maildir++ layout is supported; found non-prefixed subfolder {name:?} \ + (subfolders must be named with a leading '.', e.g. '.{name}')" + ))); + } + Err(DiscoverError::Io(p, e)) => { + return Err(Error::Usage(format!("walking {p:?}: {e}"))); + } + }; + + let include_deleted = config.include_deleted; + let filters = FolderFilters { + include: config.include, + exclude: config.exclude, + explicit: config.folder, + }; + if !filters.explicit.is_empty() && (!filters.include.is_empty() || !filters.exclude.is_empty()) + { + return Err(Error::Usage( + "--folder is mutually exclusive with --include/--exclude".to_owned(), + )); + } + let mut resolved = tree::apply_filters(discovered, &filters); + tree::restore_ephemeral_parents(&mut resolved, &canonical); + let run_flags = RunFlags { include_deleted }; + + let mut conn = db::init::open(&common.archive)?; + let session_url = file_url_for(&canonical)?; + let account_id = canonical.to_string_lossy().into_owned(); + let account_name = canonical + .file_name() + .and_then(|n| n.to_str()) + .map(str::to_owned); + + if let Some((existing_url, existing_account)) = + db::sources::conflicting_source(&conn, "maildir", &session_url, &account_id)? + && !config.allow_source_change + { + return Err(Error::SourceChange(format!( + "archive already records maildir source {existing_url} / {existing_account}; \ + re-run with --allow-source-change or use a fresh archive" + ))); + } + let source_key = SourceKey { + kind: "maildir".to_owned(), + session_url: session_url.clone(), + account_id: account_id.clone(), + }; + let source_id = if common.dry_run { + db::sources::find_source(&conn, &source_key)?.unwrap_or(-1) + } else { + db::sources::upsert_source(&conn, &source_key, account_name.as_deref(), "")? + }; + + log_at( + logger, + LEVEL_PROGRESS, + &format!("maildir source {session_url}"), + ); + + if common.dry_run { + let summary = build_dry_run_summary(&conn, source_id, &resolved, logger)?; + return Ok(summary); + } + + let mut mailbox_counts = TypeCounts::default(); + let mut email_counts = TypeCounts::default(); + + let local_mailboxes = db::maildir_ids::mailbox_folders(&conn, source_id)?; + let mailbox_local_ids = upsert_mailboxes(&mut conn, source_id, &resolved, &mut mailbox_counts)?; + + let server_names: HashSet<&str> = resolved.iter().map(|f| f.name.as_str()).collect(); + let mut vanished: Vec = local_mailboxes + .keys() + .filter(|n| !server_names.contains(n.as_str())) + .cloned() + .collect(); + tree::vanished_depth_sort(&mut vanished); + if !vanished.is_empty() { + delete_vanished_folders( + &mut conn, + source_id, + &vanished, + &mut mailbox_counts, + &mut email_counts, + logger, + )?; + } + + for folder in &resolved { + if folder.ephemeral { + continue; + } + let mailbox_local = match mailbox_local_ids.get(&folder.name).copied() { + Some(id) => id, + None => { + log_at( + logger, + LEVEL_DEFAULT, + &format!("folder {:?}: missing from mailbox id map", folder.name), + ); + email_counts.failed += 1; + continue; + } + }; + if let Err(e) = reconcile_folder( + &mut conn, + source_id, + folder, + mailbox_local, + &run_flags, + &mut email_counts, + logger, + ) { + log_at( + logger, + LEVEL_DEFAULT, + &format!("folder {:?}: {e}", folder.name), + ); + email_counts.failed += 1; + } + } + + if email_counts.failed == 0 && mailbox_counts.failed == 0 { + let tx = conn.unchecked_transaction()?; + db::blobs::gc_orphan_blobs(&tx)?; + tx.commit()?; + } + + log_at( + logger, + LEVEL_DEFAULT, + &format!( + "maildir: mailbox created={} deleted={} skipped={} failed={}; \ + email created={} fetched={} deleted={} skipped={} failed={}", + mailbox_counts.created, + mailbox_counts.deleted, + mailbox_counts.skipped, + mailbox_counts.failed, + email_counts.created, + email_counts.fetched, + email_counts.deleted, + email_counts.skipped, + email_counts.failed + ), + ); + + Ok(Summary { + per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)], + retries_observed: 0, + retry_after_sleeps: 0, + }) +} + +struct RunFlags { + include_deleted: bool, +} + +fn file_url_for(p: &Path) -> Result { + Url::from_file_path(p).map(|u| u.to_string()).map_err(|_| { + Error::Usage(format!( + "cannot encode maildir path as a file:// URL: {p:?}" + )) + }) +} + +fn warn_if_looks_like_subfolder(canonical: &Path, logger: Logger) { + let Some(basename) = canonical.file_name().and_then(|n| n.to_str()) else { + return; + }; + if !basename.starts_with('.') { + return; + } + let Some(parent) = canonical.parent() else { + return; + }; + if !parent.join("cur").is_dir() { + return; + } + log_at( + logger, + LEVEL_DEFAULT, + &format!( + "warning: {canonical:?} looks like a Maildir++ subfolder of {parent:?} \ + (basename starts with '.' and the parent has its own cur/); proceeding, \ + but you may have meant to import the parent" + ), + ); +} + +fn upsert_mailboxes( + conn: &mut Connection, + source_id: i64, + folders: &[ResolvedFolder], + counts: &mut TypeCounts, +) -> Result, Error> { + let tx = conn.transaction()?; + let mut local_ids: HashMap = HashMap::new(); + for folder in folders { + let parent_local = match folder.parent_path.as_deref() { + Some(p) => local_ids.get(p).copied().or_else(|| { + db::maildir_ids::local_for_mailbox(&tx, source_id, p) + .ok() + .flatten() + }), + None => None, + }; + let is_subscribed: i64 = if folder.ephemeral { 0 } else { 1 }; + let existing = db::maildir_ids::local_for_mailbox(&tx, source_id, &folder.name)?; + let id = if let Some(id) = existing { + tx.execute( + "UPDATE mailboxes SET name = ?1, parent_id = ?2, role = ?3, is_subscribed = ?4 + WHERE id = ?5", + params![folder.leaf, parent_local, folder.role, is_subscribed, id], + )?; + id + } else { + tx.execute( + "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) + VALUES (?1, ?2, ?3, 0, ?4)", + params![folder.leaf, parent_local, folder.role, is_subscribed], + )?; + let new_id = tx.last_insert_rowid(); + db::maildir_ids::insert_mailbox(&tx, source_id, &folder.name, new_id)?; + counts.created += 1; + counts.fetched += 1; + new_id + }; + local_ids.insert(folder.name.clone(), id); + } + tx.commit()?; + Ok(local_ids) +} + +fn delete_vanished_folders( + conn: &mut Connection, + source_id: i64, + folders: &[String], + mailbox_counts: &mut TypeCounts, + email_counts: &mut TypeCounts, + logger: Logger, +) -> Result<(), Error> { + let tx = conn.transaction()?; + for name in folders { + let local_id = match db::maildir_ids::local_for_mailbox(&tx, source_id, name)? { + Some(id) => id, + None => continue, + }; + let surviving_child: Option = tx + .query_row( + "SELECT id FROM mailboxes WHERE parent_id = ?1 LIMIT 1", + params![local_id], + |row| row.get(0), + ) + .optional()?; + if surviving_child.is_some() { + log_at( + logger, + LEVEL_DEFAULT, + &format!( + "folder {name:?}: vanished from disk but still has child mailboxes in the \ + archive; skipping delete (parent_id RESTRICT)" + ), + ); + mailbox_counts.failed += 1; + continue; + } + let email_ids: Vec = tx + .prepare( + "SELECT local_id FROM sync_id_maildir + WHERE source_id = ?1 AND type_name = 'email' AND folder = ?2", + )? + .query_map(params![source_id, name], |row| row.get(0))? + .collect::, _>>()?; + for eid in &email_ids { + tx.execute("DELETE FROM emails WHERE id = ?1", params![eid])?; + email_counts.deleted += 1; + } + db::maildir_ids::delete_all_emails_in_folder(&tx, source_id, name)?; + tx.execute("DELETE FROM mailboxes WHERE id = ?1", params![local_id])?; + db::maildir_ids::delete_mailbox(&tx, source_id, name)?; + mailbox_counts.deleted += 1; + } + tx.commit()?; + Ok(()) +} + +fn reconcile_folder( + conn: &mut Connection, + source_id: i64, + folder: &ResolvedFolder, + mailbox_local: i64, + flags: &RunFlags, + counts: &mut TypeCounts, + logger: Logger, +) -> Result<(), Error> { + let listing = match messages::list_folder(&folder.path) { + Ok(l) => l, + Err(e) => { + return Err(Error::Partial(format!( + "list {folder:?}: {e}", + folder = folder.path + ))); + } + }; + if listing.io_failures > 0 { + counts.failed += listing.io_failures; + } + let local = db::maildir_ids::email_ids_in_folder(conn, source_id, &folder.name)?; + let diff = messages::diff(listing.entries, &local); + + log_at( + logger, + LEVEL_PROGRESS, + &format!( + "folder {:?}: new={} present={} vanished={}", + folder.name, + diff.new.len(), + diff.present.len(), + diff.vanished.len() + ), + ); + + let ctx = messages::InsertContext { + source_id, + folder: &folder.name, + mailbox_local, + include_deleted: flags.include_deleted, + }; + messages::apply_folder(conn, ctx, diff, counts, logger)?; + Ok(()) +} + +fn build_dry_run_summary( + conn: &Connection, + source_id: i64, + folders: &[ResolvedFolder], + logger: Logger, +) -> Result { + let mut mailbox = TypeCounts::default(); + let mut email = TypeCounts::default(); + let local_mailboxes = if source_id < 0 { + HashMap::new() + } else { + db::maildir_ids::mailbox_folders(conn, source_id)? + }; + let server_set: HashSet<&str> = folders.iter().map(|f| f.name.as_str()).collect(); + let new_folders: Vec<&str> = folders + .iter() + .map(|f| f.name.as_str()) + .filter(|n| !local_mailboxes.contains_key(*n)) + .collect(); + let vanished_folder_names: Vec<&str> = local_mailboxes + .keys() + .filter(|n| !server_set.contains(n.as_str())) + .map(|s| s.as_str()) + .collect(); + mailbox.created += new_folders.len() as u64; + mailbox.fetched += new_folders.len() as u64; + mailbox.deleted += vanished_folder_names.len() as u64; + + for folder in folders { + if folder.ephemeral { + continue; + } + let listing = match messages::list_folder(&folder.path) { + Ok(l) => l, + Err(e) => { + logger.warn(&format!("dry-run list {:?}: {e}", folder.name)); + email.failed += 1; + continue; + } + }; + email.failed += listing.io_failures; + let disk_ids: HashSet = listing + .entries + .iter() + .map(|e| e.unique_id.clone()) + .collect(); + let local = if source_id < 0 { + HashMap::new() + } else { + db::maildir_ids::email_ids_in_folder(conn, source_id, &folder.name)? + }; + let local_keys: HashSet<&String> = local.keys().collect(); + let new_count = disk_ids.iter().filter(|u| !local.contains_key(*u)).count() as u64; + let vanished_count = local_keys + .iter() + .filter(|k| !disk_ids.contains(k.as_str())) + .count() as u64; + let present_count = disk_ids.iter().filter(|u| local.contains_key(*u)).count() as u64; + email.created += new_count; + email.fetched += new_count; + email.deleted += vanished_count; + email.skipped += present_count; + } + for name in &vanished_folder_names { + if source_id >= 0 { + let n = db::maildir_ids::email_ids_in_folder(conn, source_id, name)?.len() as u64; + email.deleted += n; + } + } + + Ok(Summary { + per_type: vec![("mailbox", mailbox), ("email", email)], + retries_observed: 0, + retry_after_sleeps: 0, + }) +} + +fn log_at(logger: Logger, level: u8, msg: &str) { + if logger.enabled(level) { + eprintln!("{msg}"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn file_url_for_ascii_path_is_passthrough() { + let p = std::path::Path::new("/home/alice/Maildir"); + let url = file_url_for(p).unwrap(); + assert_eq!(url, "file:///home/alice/Maildir"); + } + + #[test] + fn file_url_for_percent_encodes_spaces_and_unicode() { + let p = std::path::Path::new("/home/alice/My Maildir/Sübfolder"); + let url = file_url_for(p).unwrap(); + assert!(url.starts_with("file:///home/alice/My%20Maildir/")); + assert!(url.contains("S%C3%BCbfolder")); + } + + #[test] + fn file_url_for_rejects_relative_paths() { + let p = std::path::Path::new("./relative"); + let err = file_url_for(p).unwrap_err(); + assert!(matches!(err, Error::Usage(_))); + } +} diff --git a/src/sync/import_maildir/keywords.rs b/src/sync/import_maildir/keywords.rs new file mode 100644 index 0000000..c0e74be --- /dev/null +++ b/src/sync/import_maildir/keywords.rs @@ -0,0 +1,202 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use mail_parser::mailbox::maildir::Flag; + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct Translation { + pub keywords: Vec, + pub has_trashed_flag: bool, +} + +pub fn translate_flags(flags: &[Flag], include_deleted: bool) -> Translation { + let mut out: Vec = Vec::new(); + let mut has_trashed = false; + for f in flags { + match f { + Flag::Passed => push_unique(&mut out, "$forwarded"), + Flag::Replied => push_unique(&mut out, "$answered"), + Flag::Seen => push_unique(&mut out, "$seen"), + Flag::Draft => push_unique(&mut out, "$draft"), + Flag::Flagged => push_unique(&mut out, "$flagged"), + Flag::Trashed => { + has_trashed = true; + if include_deleted { + push_unique(&mut out, "$deleted"); + } + } + } + } + out.sort(); + Translation { + keywords: out, + has_trashed_flag: has_trashed, + } +} + +fn push_unique(out: &mut Vec, k: &str) { + if !out.iter().any(|x| x == k) { + out.push(k.to_owned()); + } +} + +pub fn flags_from_filename(filename: &str) -> Vec { + let Some(part) = filename.rsplit_once("2,").map(|(_, s)| s) else { + return Vec::new(); + }; + let mut out = Vec::new(); + for &ch in part.as_bytes() { + match ch { + b'P' => out.push(Flag::Passed), + b'R' => out.push(Flag::Replied), + b'S' => out.push(Flag::Seen), + b'T' => out.push(Flag::Trashed), + b'D' => out.push(Flag::Draft), + b'F' => out.push(Flag::Flagged), + other => { + if !other.is_ascii_alphanumeric() { + break; + } + } + } + } + out +} + +pub fn unique_id_from_filename(filename: &str) -> &str { + match filename.split_once(':') { + Some((unique, _)) => unique, + None => filename, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_flags_yields_empty_keywords() { + let t = translate_flags(&[], false); + assert!(t.keywords.is_empty()); + assert!(!t.has_trashed_flag); + } + + #[test] + fn each_flag_maps_to_expected_jmap_keyword() { + let t = translate_flags( + &[ + Flag::Passed, + Flag::Replied, + Flag::Seen, + Flag::Draft, + Flag::Flagged, + ], + false, + ); + let mut expected = vec!["$forwarded", "$answered", "$seen", "$draft", "$flagged"]; + expected.sort(); + let expected: Vec = expected.into_iter().map(str::to_owned).collect(); + assert_eq!(t.keywords, expected); + assert!(!t.has_trashed_flag); + } + + #[test] + fn trashed_dropped_by_default_marks_flag() { + let t = translate_flags(&[Flag::Trashed, Flag::Seen], false); + assert!(t.has_trashed_flag); + assert_eq!(t.keywords, vec!["$seen".to_owned()]); + } + + #[test] + fn trashed_with_include_deleted_adds_dollar_deleted() { + let t = translate_flags(&[Flag::Trashed, Flag::Seen], true); + assert!(t.has_trashed_flag); + let mut expected = vec!["$seen", "$deleted"]; + expected.sort(); + let expected: Vec = expected.into_iter().map(str::to_owned).collect(); + assert_eq!(t.keywords, expected); + } + + #[test] + fn order_independent_keywords_are_stable() { + let a = translate_flags(&[Flag::Seen, Flag::Flagged], false); + let b = translate_flags(&[Flag::Flagged, Flag::Seen], false); + assert_eq!(a.keywords, b.keywords); + } + + #[test] + fn duplicate_flags_dedup() { + let t = translate_flags(&[Flag::Seen, Flag::Seen, Flag::Seen], false); + assert_eq!(t.keywords, vec!["$seen".to_owned()]); + } + + #[test] + fn unique_id_strips_at_first_colon() { + assert_eq!( + unique_id_from_filename("1739471123.M001P01234V0.host:2,RS"), + "1739471123.M001P01234V0.host" + ); + } + + #[test] + fn unique_id_no_colon_is_whole_basename() { + assert_eq!( + unique_id_from_filename("1739471123.M001P0.host"), + "1739471123.M001P0.host" + ); + } + + #[test] + fn unique_id_strips_legacy_one_form() { + assert_eq!( + unique_id_from_filename("1234.M5.host:1,experimental"), + "1234.M5.host" + ); + } + + #[test] + fn flags_from_filename_recognises_full_set() { + let mut flags = flags_from_filename("name:2,DFPRST"); + flags.sort(); + let mut want = vec![ + Flag::Draft, + Flag::Flagged, + Flag::Passed, + Flag::Replied, + Flag::Seen, + Flag::Trashed, + ]; + want.sort(); + assert_eq!(flags, want); + } + + #[test] + fn flags_from_filename_no_info_section_yields_empty() { + assert!(flags_from_filename("plain").is_empty()); + assert!(flags_from_filename("name:").is_empty()); + assert!(flags_from_filename("name:1,X").is_empty()); + } + + #[test] + fn flags_from_filename_dovecot_extension_metadata_passes_through() { + + let mut flags = flags_from_filename("uid,S=1234,W=1300:2,RS"); + flags.sort(); + let mut want = vec![Flag::Replied, Flag::Seen]; + want.sort(); + assert_eq!(flags, want); + } + + #[test] + fn flags_from_filename_unknown_alpha_chars_skipped() { + + let mut flags = flags_from_filename("uid:2,SaT"); + flags.sort(); + let mut want = vec![Flag::Seen, Flag::Trashed]; + want.sort(); + assert_eq!(flags, want); + } +} diff --git a/src/sync/import_maildir/messages.rs b/src/sync/import_maildir/messages.rs new file mode 100644 index 0000000..e381169 --- /dev/null +++ b/src/sync/import_maildir/messages.rs @@ -0,0 +1,726 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::time::{Duration, UNIX_EPOCH}; + +use mail_parser::mailbox::maildir::Flag; +use rusqlite::{Connection, params}; +use serde_json::Value; +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; + +use crate::db; +use crate::sync::TypeCounts; +use crate::sync::emailmeta::email_index_from_blob; +use crate::sync::keys::index_to_json; + +use super::keywords::{Translation, flags_from_filename, translate_flags, unique_id_from_filename}; + +#[derive(Debug, Clone)] +pub struct DiskEntry { + pub unique_id: String, + pub filename: String, + pub path: PathBuf, + pub flags: Vec, + pub mtime_unix: u64, +} + +#[derive(Debug, Clone, Default)] +pub struct DiskListing { + pub entries: Vec, + pub io_failures: u64, +} + +pub fn list_folder(folder_path: &Path) -> std::io::Result { + let mut out = DiskListing::default(); + let mut seen: std::collections::HashMap = std::collections::HashMap::new(); + for sub in ["cur", "new"] { + let dir = folder_path.join(sub); + if !dir.is_dir() { + continue; + } + let entries = std::fs::read_dir(&dir)?; + for entry in entries { + let entry = match entry { + Ok(e) => e, + Err(_) => { + out.io_failures += 1; + continue; + } + }; + let path = entry.path(); + let file_type = match entry.file_type() { + Ok(t) => t, + Err(_) => { + out.io_failures += 1; + continue; + } + }; + if !file_type.is_file() { + continue; + } + let Some(raw_name) = path.file_name() else { + continue; + }; + let filename: String = raw_name.to_string_lossy().into_owned(); + if filename.starts_with('.') { + continue; + } + let unique_id = unique_id_from_filename(&filename).to_owned(); + if unique_id.is_empty() { + continue; + } + if seen.contains_key(&unique_id) { + + continue; + } + let flags = flags_from_filename(&filename); + let mtime_unix = match entry.metadata().and_then(|m| m.modified()).and_then(|t| { + t.duration_since(UNIX_EPOCH).map_err(|e| { + std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()) + }) + }) { + Ok(d) => d.as_secs(), + Err(_) => 0, + }; + seen.insert(unique_id.clone(), out.entries.len()); + out.entries.push(DiskEntry { + unique_id, + filename, + path, + flags, + mtime_unix, + }); + } + } + Ok(out) +} + +#[derive(Debug, Clone, Default)] +pub struct FolderDiff { + pub new: Vec, + pub present: Vec<(DiskEntry, i64)>, + pub vanished: Vec<(String, i64)>, +} + +pub fn diff(disk: Vec, local: &HashMap) -> FolderDiff { + let mut diff = FolderDiff::default(); + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + for entry in disk { + match local.get(&entry.unique_id) { + None => { + seen.insert(entry.unique_id.clone()); + diff.new.push(entry); + } + Some(&local_id) => { + seen.insert(entry.unique_id.clone()); + diff.present.push((entry, local_id)); + } + } + } + for (uid, local_id) in local { + if !seen.contains(uid) { + diff.vanished.push((uid.clone(), *local_id)); + } + } + diff.vanished.sort_by(|a, b| a.0.cmp(&b.0)); + diff +} + +#[derive(Clone, Copy)] +pub struct InsertContext<'a> { + pub source_id: i64, + pub folder: &'a str, + pub mailbox_local: i64, + pub include_deleted: bool, +} + +pub fn insert_new( + tx: &rusqlite::Transaction<'_>, + ctx: InsertContext<'_>, + entry: &DiskEntry, +) -> Result, InsertError> { + let bytes = std::fs::read(&entry.path).map_err(InsertError::Io)?; + let translation = translate_flags(&entry.flags, ctx.include_deleted); + if translation.has_trashed_flag && !ctx.include_deleted { + return Ok(None); + } + let received_at = format_unix_rfc3339(entry.mtime_unix); + let blob_id = db::blobs::intern_blob(tx, &bytes)?; + let message_match = index_to_json(&email_index_from_blob(&bytes)); + let mailbox_ids = Value::Array(vec![Value::from(ctx.mailbox_local)]); + let keywords = keywords_json(&translation); + tx.execute( + "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + blob_id, + received_at, + mailbox_ids.to_string(), + keywords, + message_match + ], + )?; + let local_id = tx.last_insert_rowid(); + db::maildir_ids::insert_email(tx, ctx.source_id, ctx.folder, &entry.unique_id, local_id)?; + Ok(Some(local_id)) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PresentOutcome { + + Unchanged, + + KeywordsUpdated, + + DeleteRequested, +} + +pub fn refresh_flags( + tx: &rusqlite::Transaction<'_>, + local_id: i64, + entry: &DiskEntry, + stored_keywords: &str, + include_deleted: bool, +) -> Result { + let translation = translate_flags(&entry.flags, include_deleted); + if translation.has_trashed_flag && !include_deleted { + return Ok(PresentOutcome::DeleteRequested); + } + let expected = keywords_json(&translation); + if stored_keywords == expected { + return Ok(PresentOutcome::Unchanged); + } + tx.execute( + "UPDATE emails SET keywords = ?1 WHERE id = ?2", + params![expected, local_id], + )?; + Ok(PresentOutcome::KeywordsUpdated) +} + +pub fn load_present_keywords( + conn: &Connection, + source_id: i64, + folder: &str, +) -> Result, rusqlite::Error> { + let mut stmt = conn.prepare( + "SELECT e.id, e.keywords FROM emails e + JOIN sync_id_maildir m ON m.local_id = e.id AND m.type_name = 'email' + WHERE m.source_id = ?1 AND m.folder = ?2", + )?; + let rows = stmt.query_map(params![source_id, folder], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) + })?; + let mut out = HashMap::new(); + for r in rows { + let (id, kw) = r?; + out.insert(id, kw); + } + Ok(out) +} + +pub fn delete_vanished( + tx: &rusqlite::Transaction<'_>, + source_id: i64, + folder: &str, + unique_id: &str, + local_id: i64, +) -> Result<(), rusqlite::Error> { + tx.execute("DELETE FROM emails WHERE id = ?1", params![local_id])?; + db::maildir_ids::delete_email(tx, source_id, folder, unique_id)?; + Ok(()) +} + +const PROGRESS_TICK: u64 = 1000; + +pub fn apply_folder( + conn: &mut Connection, + ctx: InsertContext<'_>, + diff: FolderDiff, + counts: &mut TypeCounts, + logger: crate::logging::Logger, +) -> Result<(), crate::error::Error> { + let stored_keywords = + load_present_keywords(conn, ctx.source_id, ctx.folder).unwrap_or_default(); + let tx = conn.transaction()?; + let total_new = diff.new.len() as u64; + let mut inserted: u64 = 0; + for entry in &diff.new { + match insert_new(&tx, ctx, entry) { + Ok(Some(_)) => { + counts.created += 1; + counts.fetched += 1; + inserted += 1; + if inserted.is_multiple_of(PROGRESS_TICK) + && logger.enabled(crate::logging::LEVEL_PROGRESS) + { + eprintln!( + "folder {folder:?}: inserted {inserted}/{total_new}", + folder = ctx.folder + ); + } + } + Ok(None) => { + counts.skipped += 1; + } + Err(e) => { + logger.warn(&format!( + "maildir {folder:?}/{name}: {e}", + folder = ctx.folder, + name = entry.filename + )); + counts.failed += 1; + } + } + } + for (entry, local_id) in &diff.present { + let stored = stored_keywords + .get(local_id) + .map(String::as_str) + .unwrap_or("[]"); + match refresh_flags(&tx, *local_id, entry, stored, ctx.include_deleted) { + Ok(PresentOutcome::Unchanged) => counts.skipped += 1, + Ok(PresentOutcome::KeywordsUpdated) => counts.fetched += 1, + Ok(PresentOutcome::DeleteRequested) => { + match delete_vanished(&tx, ctx.source_id, ctx.folder, &entry.unique_id, *local_id) { + Ok(()) => counts.deleted += 1, + Err(e) => { + logger.warn(&format!( + "maildir {folder:?}/{name}: T-flag delete failed: {e}", + folder = ctx.folder, + name = entry.filename + )); + counts.failed += 1; + } + } + } + Err(e) => { + logger.warn(&format!( + "maildir {folder:?}/{name}: flag update failed: {e}", + folder = ctx.folder, + name = entry.filename + )); + counts.failed += 1; + } + } + } + for (unique_id, local_id) in &diff.vanished { + if let Err(e) = delete_vanished(&tx, ctx.source_id, ctx.folder, unique_id, *local_id) { + logger.warn(&format!( + "maildir {folder:?}/{unique_id}: delete failed: {e}", + folder = ctx.folder + )); + counts.failed += 1; + } else { + counts.deleted += 1; + } + } + tx.commit()?; + Ok(()) +} + +fn keywords_json(translation: &Translation) -> String { + Value::Array( + translation + .keywords + .iter() + .map(|k| Value::String(k.clone())) + .collect(), + ) + .to_string() +} + +fn format_unix_rfc3339(secs: u64) -> String { + let when = UNIX_EPOCH + Duration::from_secs(secs); + OffsetDateTime::from(when) + .format(&Rfc3339) + .unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_owned()) +} + +#[derive(Debug, thiserror::Error)] +pub enum InsertError { + #[error("io: {0}")] + Io(#[from] std::io::Error), + #[error("db: {0}")] + Db(#[from] rusqlite::Error), +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::init; + use crate::db::sources::{SourceKey, upsert_source}; + use std::fs; + + fn fresh_archive() -> (Connection, i64) { + let c = Connection::open_in_memory().unwrap(); + init::apply_schema(&c).unwrap(); + let sid = upsert_source( + &c, + &SourceKey { + kind: "maildir".to_owned(), + session_url: "file:///tmp/Maildir".to_owned(), + account_id: "/tmp/Maildir".to_owned(), + }, + Some("Maildir"), + "", + ) + .unwrap(); + (c, sid) + } + + fn write_maildir_message( + folder_path: &Path, + sub: &str, + filename: &str, + body: &[u8], + ) -> PathBuf { + let dir = folder_path.join(sub); + fs::create_dir_all(&dir).unwrap(); + let p = dir.join(filename); + fs::write(&p, body).unwrap(); + p + } + + fn ensure_folder_skel(folder_path: &Path) { + for sub in ["cur", "new", "tmp"] { + fs::create_dir_all(folder_path.join(sub)).unwrap(); + } + } + + #[test] + fn list_folder_returns_cur_and_new_skipping_tmp() { + let td = tempfile::tempdir().unwrap(); + ensure_folder_skel(td.path()); + write_maildir_message(td.path(), "cur", "1.M0.host:2,S", b"a"); + write_maildir_message(td.path(), "new", "2.M0.host", b"b"); + write_maildir_message(td.path(), "tmp", "3.M0.host", b"c"); + let listing = list_folder(td.path()).unwrap(); + assert_eq!(listing.entries.len(), 2); + assert!(listing.entries.iter().any(|e| e.unique_id == "1.M0.host")); + assert!(listing.entries.iter().any(|e| e.unique_id == "2.M0.host")); + } + + #[test] + fn list_folder_skips_dotfiles() { + let td = tempfile::tempdir().unwrap(); + ensure_folder_skel(td.path()); + write_maildir_message(td.path(), "cur", ".uidvalidity", b"meta"); + write_maildir_message(td.path(), "cur", ".dovecot-uidlist", b"meta"); + write_maildir_message(td.path(), "cur", "1.M0.host:2,S", b"a"); + let listing = list_folder(td.path()).unwrap(); + assert_eq!(listing.entries.len(), 1); + assert_eq!(listing.entries[0].unique_id, "1.M0.host"); + } + + #[test] + fn list_folder_parses_dovecot_extension_filename() { + let td = tempfile::tempdir().unwrap(); + ensure_folder_skel(td.path()); + write_maildir_message( + td.path(), + "cur", + "1739471123.M001P01234V0000I0000abcd.host,S=42,W=44:2,RS", + b"a", + ); + let listing = list_folder(td.path()).unwrap(); + assert_eq!(listing.entries.len(), 1); + let e = &listing.entries[0]; + assert_eq!( + e.unique_id, + "1739471123.M001P01234V0000I0000abcd.host,S=42,W=44" + ); + assert!(e.flags.contains(&Flag::Replied)); + assert!(e.flags.contains(&Flag::Seen)); + } + + #[cfg(target_os = "linux")] + #[test] + fn list_folder_renders_non_utf8_filename_lossy() { + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + let td = tempfile::tempdir().unwrap(); + ensure_folder_skel(td.path()); + let mut bytes = b"1.M0.host".to_vec(); + bytes.extend_from_slice(&[0xff, 0xfe]); + bytes.extend_from_slice(b":2,S"); + let bad_name = OsStr::from_bytes(&bytes); + let p = td.path().join("cur").join(bad_name); + fs::write(&p, b"body").unwrap(); + let listing = list_folder(td.path()).unwrap(); + assert_eq!(listing.entries.len(), 1); + let e = &listing.entries[0]; + assert!( + e.unique_id.contains('\u{FFFD}'), + "lossy substitution: {:?}", + e.unique_id + ); + assert!(e.unique_id.starts_with("1.M0.host")); + assert!(e.flags.contains(&Flag::Seen)); + } + + #[test] + fn diff_partitions_new_present_vanished() { + let entries = vec![ + DiskEntry { + unique_id: "a".into(), + filename: "a".into(), + path: PathBuf::new(), + flags: vec![], + mtime_unix: 0, + }, + DiskEntry { + unique_id: "b".into(), + filename: "b".into(), + path: PathBuf::new(), + flags: vec![], + mtime_unix: 0, + }, + ]; + let mut local = HashMap::new(); + local.insert("b".to_owned(), 10); + local.insert("c".to_owned(), 11); + let d = diff(entries, &local); + assert_eq!(d.new.len(), 1); + assert_eq!(d.new[0].unique_id, "a"); + assert_eq!(d.present.len(), 1); + assert_eq!(d.present[0].0.unique_id, "b"); + assert_eq!(d.present[0].1, 10); + assert_eq!(d.vanished, vec![("c".to_owned(), 11)]); + } + + #[test] + fn insert_new_writes_blob_email_and_sync_row() { + let (mut c, sid) = fresh_archive(); + c.execute( + "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) + VALUES ('INBOX', NULL, 'inbox', 0, 1)", + [], + ) + .unwrap(); + let mailbox_local: i64 = c.last_insert_rowid(); + db::maildir_ids::insert_mailbox(&c, sid, "INBOX", mailbox_local).unwrap(); + + let td = tempfile::tempdir().unwrap(); + ensure_folder_skel(td.path()); + let body = b"From: a@x\r\nSubject: hi\r\nMessage-ID: \r\n\r\nhi"; + write_maildir_message(td.path(), "cur", "uid1.M0.host:2,S", body); + let listing = list_folder(td.path()).unwrap(); + let entry = listing.entries.into_iter().next().unwrap(); + + let tx = c.transaction().unwrap(); + let inserted = insert_new( + &tx, + InsertContext { + source_id: sid, + folder: "INBOX", + mailbox_local, + include_deleted: false, + }, + &entry, + ) + .unwrap(); + tx.commit().unwrap(); + assert!(inserted.is_some()); + + let email_count: i64 = c + .query_row("SELECT count(*) FROM emails", [], |r| r.get(0)) + .unwrap(); + assert_eq!(email_count, 1); + let local = db::maildir_ids::local_for_email(&c, sid, "INBOX", "uid1.M0.host") + .unwrap() + .unwrap(); + assert_eq!(local, inserted.unwrap()); + let kws: String = c + .query_row( + "SELECT keywords FROM emails WHERE id = ?1", + params![local], + |r| r.get(0), + ) + .unwrap(); + assert!(kws.contains("$seen")); + } + + #[test] + fn insert_new_skips_trashed_without_include_deleted() { + let (mut c, sid) = fresh_archive(); + c.execute( + "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) + VALUES ('INBOX', NULL, 'inbox', 0, 1)", + [], + ) + .unwrap(); + let mailbox_local: i64 = c.last_insert_rowid(); + db::maildir_ids::insert_mailbox(&c, sid, "INBOX", mailbox_local).unwrap(); + + let td = tempfile::tempdir().unwrap(); + ensure_folder_skel(td.path()); + write_maildir_message(td.path(), "cur", "uid.host:2,T", b"x"); + let listing = list_folder(td.path()).unwrap(); + let entry = listing.entries.into_iter().next().unwrap(); + let tx = c.transaction().unwrap(); + let inserted = insert_new( + &tx, + InsertContext { + source_id: sid, + folder: "INBOX", + mailbox_local, + include_deleted: false, + }, + &entry, + ) + .unwrap(); + tx.commit().unwrap(); + assert!(inserted.is_none()); + let email_count: i64 = c + .query_row("SELECT count(*) FROM emails", [], |r| r.get(0)) + .unwrap(); + assert_eq!(email_count, 0); + } + + #[test] + fn refresh_flags_updates_only_when_changed() { + let (mut c, sid) = fresh_archive(); + c.execute( + "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) + VALUES ('INBOX', NULL, 'inbox', 0, 1)", + [], + ) + .unwrap(); + let mailbox_local = c.last_insert_rowid(); + db::maildir_ids::insert_mailbox(&c, sid, "INBOX", mailbox_local).unwrap(); + let td = tempfile::tempdir().unwrap(); + ensure_folder_skel(td.path()); + write_maildir_message(td.path(), "cur", "u.host:2,", b"hi"); + let entry = list_folder(td.path()).unwrap().entries.remove(0); + let tx = c.transaction().unwrap(); + let local = insert_new( + &tx, + InsertContext { + source_id: sid, + folder: "INBOX", + mailbox_local, + include_deleted: false, + }, + &entry, + ) + .unwrap() + .unwrap(); + tx.commit().unwrap(); + + let stored = load_present_keywords(&c, sid, "INBOX").unwrap(); + let stored_for_local = stored + .get(&local) + .cloned() + .unwrap_or_else(|| "[]".to_owned()); + + let tx = c.transaction().unwrap(); + assert_eq!( + refresh_flags(&tx, local, &entry, &stored_for_local, false).unwrap(), + PresentOutcome::Unchanged + ); + tx.commit().unwrap(); + + fs::rename(&entry.path, entry.path.with_file_name("u.host:2,S")).unwrap(); + let entry2 = list_folder(td.path()).unwrap().entries.remove(0); + let tx = c.transaction().unwrap(); + assert_eq!( + refresh_flags(&tx, local, &entry2, &stored_for_local, false).unwrap(), + PresentOutcome::KeywordsUpdated + ); + tx.commit().unwrap(); + let kws: String = c + .query_row( + "SELECT keywords FROM emails WHERE id = ?1", + params![local], + |r| r.get(0), + ) + .unwrap(); + assert!(kws.contains("$seen")); + } + + #[test] + fn refresh_flags_returns_delete_when_trashed_added_and_include_deleted_off() { + let (mut c, sid) = fresh_archive(); + c.execute( + "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) + VALUES ('INBOX', NULL, 'inbox', 0, 1)", + [], + ) + .unwrap(); + let mailbox_local = c.last_insert_rowid(); + db::maildir_ids::insert_mailbox(&c, sid, "INBOX", mailbox_local).unwrap(); + let td = tempfile::tempdir().unwrap(); + ensure_folder_skel(td.path()); + write_maildir_message(td.path(), "cur", "u.host:2,", b"hi"); + let entry = list_folder(td.path()).unwrap().entries.remove(0); + let tx = c.transaction().unwrap(); + let local = insert_new( + &tx, + InsertContext { + source_id: sid, + folder: "INBOX", + mailbox_local, + include_deleted: false, + }, + &entry, + ) + .unwrap() + .unwrap(); + tx.commit().unwrap(); + + fs::rename(&entry.path, entry.path.with_file_name("u.host:2,T")).unwrap(); + let entry2 = list_folder(td.path()).unwrap().entries.remove(0); + let tx = c.transaction().unwrap(); + assert_eq!( + refresh_flags(&tx, local, &entry2, "[]", false).unwrap(), + PresentOutcome::DeleteRequested + ); + tx.commit().unwrap(); + } + + #[test] + fn refresh_flags_keeps_trashed_with_include_deleted_on() { + let (mut c, sid) = fresh_archive(); + c.execute( + "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) + VALUES ('INBOX', NULL, 'inbox', 0, 1)", + [], + ) + .unwrap(); + let mailbox_local = c.last_insert_rowid(); + db::maildir_ids::insert_mailbox(&c, sid, "INBOX", mailbox_local).unwrap(); + let td = tempfile::tempdir().unwrap(); + ensure_folder_skel(td.path()); + write_maildir_message(td.path(), "cur", "u.host:2,T", b"hi"); + let entry = list_folder(td.path()).unwrap().entries.remove(0); + let tx = c.transaction().unwrap(); + let local = insert_new( + &tx, + InsertContext { + source_id: sid, + folder: "INBOX", + mailbox_local, + include_deleted: true, + }, + &entry, + ) + .unwrap() + .unwrap(); + tx.commit().unwrap(); + + let stored = load_present_keywords(&c, sid, "INBOX").unwrap(); + let stored_for_local = stored.get(&local).cloned().unwrap(); + let tx = c.transaction().unwrap(); + assert_eq!( + refresh_flags(&tx, local, &entry, &stored_for_local, true).unwrap(), + PresentOutcome::Unchanged + ); + tx.commit().unwrap(); + } +} diff --git a/src/sync/import_maildir/tree.rs b/src/sync/import_maildir/tree.rs new file mode 100644 index 0000000..68c3d51 --- /dev/null +++ b/src/sync/import_maildir/tree.rs @@ -0,0 +1,495 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::path::{Path, PathBuf}; + +use regex::Regex; + +use crate::imap::automap; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedFolder { + pub name: String, + pub leaf: String, + pub parent_path: Option, + pub path: PathBuf, + pub role: Option<&'static str>, + pub ephemeral: bool, +} + +pub struct FolderFilters { + pub include: Vec, + pub exclude: Vec, + pub explicit: Vec, +} + +#[derive(Debug, thiserror::Error)] +pub enum DiscoverError { + #[error("path does not exist: {0}")] + NotFound(PathBuf), + #[error("path is not a directory: {0}")] + NotADirectory(PathBuf), + #[error("not a Maildir: {0} (missing cur/ subdirectory)")] + NotAMaildir(PathBuf), + #[error( + "only Maildir++ layout is supported: found directory {0:?} that looks like a non-prefixed \ + subfolder (subfolders must be named with a leading '.', e.g. '.Sent')" + )] + NotMaildirPlus(String), + #[error("io error walking {0}: {1}")] + Io(PathBuf, std::io::Error), +} + +pub fn discover(root: &Path, automap_enabled: bool) -> Result, DiscoverError> { + if !root.exists() { + return Err(DiscoverError::NotFound(root.to_path_buf())); + } + let meta = std::fs::metadata(root).map_err(|e| DiscoverError::Io(root.to_path_buf(), e))?; + if !meta.is_dir() { + return Err(DiscoverError::NotADirectory(root.to_path_buf())); + } + if !root.join("cur").is_dir() { + return Err(DiscoverError::NotAMaildir(root.to_path_buf())); + } + let mut folders = vec![ResolvedFolder { + name: "INBOX".to_owned(), + leaf: "INBOX".to_owned(), + parent_path: None, + path: root.to_path_buf(), + role: Some("inbox"), + ephemeral: false, + }]; + + let mut subfolders: Vec<(String, PathBuf)> = Vec::new(); + let entries = std::fs::read_dir(root).map_err(|e| DiscoverError::Io(root.to_path_buf(), e))?; + for entry in entries { + let entry = entry.map_err(|e| DiscoverError::Io(root.to_path_buf(), e))?; + let path = entry.path(); + if !path.is_dir() { + continue; + } + let Some(raw_name) = path.file_name() else { + continue; + }; + let name: String = raw_name.to_string_lossy().into_owned(); + if matches!(name.as_str(), "cur" | "new" | "tmp") { + continue; + } + let has_cur = path.join("cur").is_dir(); + if let Some(stripped) = name.strip_prefix('.') { + if has_cur && !stripped.is_empty() { + subfolders.push((stripped.to_owned(), path)); + } + } else if has_cur { + return Err(DiscoverError::NotMaildirPlus(name)); + } + } + + subfolders.sort_by(|a, b| a.0.cmp(&b.0)); + + let mut known: Vec = vec!["INBOX".to_owned()]; + for (canonical, path) in subfolders { + let parent_path = split_parent(&canonical); + if let Some(p) = parent_path.as_deref() { + ensure_ephemeral_parents(p, &mut known, &mut folders, root); + } + let leaf = leaf_of(&canonical); + let role = if automap_enabled { + automap::role_for_folder(&canonical, &[], "", true) + } else { + None + }; + folders.push(ResolvedFolder { + name: canonical.clone(), + leaf: leaf.to_owned(), + parent_path, + path, + role, + ephemeral: false, + }); + known.push(canonical); + } + Ok(folders) +} + +fn ensure_ephemeral_parents( + parent_path: &str, + known: &mut Vec, + folders: &mut Vec, + root: &Path, +) { + for ancestor in ancestor_chain(parent_path) { + if known.iter().any(|n| n == &ancestor) { + continue; + } + let leaf = leaf_of(&ancestor).to_owned(); + let parent_path = split_parent(&ancestor); + folders.push(ResolvedFolder { + name: ancestor.clone(), + leaf, + parent_path, + path: root.to_path_buf(), + role: None, + ephemeral: true, + }); + known.push(ancestor); + } +} + +fn ancestor_chain(name: &str) -> Vec { + let mut out = Vec::new(); + let mut acc = String::new(); + for part in name.split('.') { + if !acc.is_empty() { + acc.push('.'); + } + acc.push_str(part); + out.push(acc.clone()); + } + out +} + +fn split_parent(name: &str) -> Option { + name.rfind('.').map(|i| name[..i].to_owned()) +} + +fn leaf_of(name: &str) -> &str { + match name.rfind('.') { + Some(i) => &name[i + 1..], + None => name, + } +} + +pub fn apply_filters(folders: Vec, filters: &FolderFilters) -> Vec { + folders + .into_iter() + .filter(|f| match_filters(&f.name, filters)) + .collect() +} + +pub fn restore_ephemeral_parents(folders: &mut Vec, root: &Path) { + let known: std::collections::HashSet = folders.iter().map(|f| f.name.clone()).collect(); + let mut to_add: Vec = Vec::new(); + for f in folders.iter() { + if let Some(parent) = f.parent_path.as_deref() { + for ancestor in ancestor_chain(parent) { + if known.contains(&ancestor) || to_add.iter().any(|x| x.name == ancestor) { + continue; + } + let leaf = leaf_of(&ancestor).to_owned(); + let parent_path = split_parent(&ancestor); + to_add.push(ResolvedFolder { + name: ancestor, + leaf, + parent_path, + path: root.to_path_buf(), + role: None, + ephemeral: true, + }); + } + } + } + if to_add.is_empty() { + return; + } + folders.extend(to_add); + folders.sort_by_key(|f| f.name.matches('.').count()); +} + +fn match_filters(name: &str, filters: &FolderFilters) -> bool { + if !filters.explicit.is_empty() { + return filters.explicit.iter().any(|e| e == name); + } + if !filters.include.is_empty() && !filters.include.iter().any(|r| r.is_match(name)) { + return false; + } + if filters.exclude.iter().any(|r| r.is_match(name)) { + return false; + } + true +} + +pub fn vanished_folders<'a, I>(local: I, server: &[ResolvedFolder]) -> Vec +where + I: IntoIterator, +{ + let server_set: std::collections::HashSet<&str> = + server.iter().map(|f| f.name.as_str()).collect(); + local + .into_iter() + .filter(|n| !server_set.contains(*n)) + .map(str::to_owned) + .collect() +} + +pub fn vanished_depth_sort(names: &mut [String]) { + names.sort_by(|a, b| { + let da = a.matches('.').count(); + let db = b.matches('.').count(); + db.cmp(&da).then_with(|| a.cmp(b)) + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + fn make_maildir(root: &Path, folders: &[&str]) { + for sub in ["cur", "new", "tmp"] { + fs::create_dir_all(root.join(sub)).unwrap(); + } + for name in folders { + let p = root.join(name); + for sub in ["cur", "new", "tmp"] { + fs::create_dir_all(p.join(sub)).unwrap(); + } + } + } + + #[test] + fn discover_just_inbox_when_root_alone() { + let td = tempfile::tempdir().unwrap(); + make_maildir(td.path(), &[]); + let folders = discover(td.path(), true).unwrap(); + assert_eq!(folders.len(), 1); + assert_eq!(folders[0].name, "INBOX"); + assert_eq!(folders[0].role, Some("inbox")); + assert!(folders[0].parent_path.is_none()); + } + + #[test] + fn discover_dovecot_style_subfolders() { + let td = tempfile::tempdir().unwrap(); + make_maildir(td.path(), &[".Sent", ".Drafts", ".Trash", ".Junk"]); + let folders = discover(td.path(), true).unwrap(); + let names: Vec<&str> = folders.iter().map(|f| f.name.as_str()).collect(); + assert!(names.contains(&"INBOX")); + assert!(names.contains(&"Sent")); + assert!(names.contains(&"Drafts")); + let sent = folders.iter().find(|f| f.name == "Sent").unwrap(); + assert_eq!(sent.role, Some("sent")); + let drafts = folders.iter().find(|f| f.name == "Drafts").unwrap(); + assert_eq!(drafts.role, Some("drafts")); + let trash = folders.iter().find(|f| f.name == "Trash").unwrap(); + assert_eq!(trash.role, Some("trash")); + let junk = folders.iter().find(|f| f.name == "Junk").unwrap(); + assert_eq!(junk.role, Some("junk")); + } + + #[test] + fn discover_courier_maildir_plus_deep_hierarchy() { + let td = tempfile::tempdir().unwrap(); + make_maildir(td.path(), &[".Archive", ".Archive.2024", ".Archive.2025"]); + let folders = discover(td.path(), true).unwrap(); + let names: Vec<&str> = folders.iter().map(|f| f.name.as_str()).collect(); + assert!(names.contains(&"Archive")); + assert!(names.contains(&"Archive.2024")); + assert!(names.contains(&"Archive.2025")); + let a2025 = folders.iter().find(|f| f.name == "Archive.2025").unwrap(); + assert_eq!(a2025.leaf, "2025"); + assert_eq!(a2025.parent_path.as_deref(), Some("Archive")); + let archive = folders.iter().find(|f| f.name == "Archive").unwrap(); + assert_eq!(archive.role, Some("archive")); + + assert!(a2025.role.is_none()); + } + + #[test] + fn discover_inserts_ephemeral_parent_for_orphan_branch() { + let td = tempfile::tempdir().unwrap(); + make_maildir(td.path(), &[".Lists.maildir-dev"]); + let folders = discover(td.path(), true).unwrap(); + let lists = folders.iter().find(|f| f.name == "Lists").unwrap(); + assert!(lists.ephemeral); + assert!(lists.role.is_none()); + assert!(lists.parent_path.is_none()); + let leaf = folders + .iter() + .find(|f| f.name == "Lists.maildir-dev") + .unwrap(); + assert!(!leaf.ephemeral); + assert_eq!(leaf.parent_path.as_deref(), Some("Lists")); + assert_eq!(leaf.leaf, "maildir-dev"); + } + + #[test] + fn discover_skips_non_dot_subfolders_silently_only_if_no_cur() { + let td = tempfile::tempdir().unwrap(); + make_maildir(td.path(), &[".Sent"]); + + fs::create_dir(td.path().join(".dovecot.imap")).unwrap(); + fs::write(td.path().join("dovecot.index.log"), "").unwrap(); + let folders = discover(td.path(), true).unwrap(); + assert_eq!(folders.len(), 2); + assert!(folders.iter().any(|f| f.name == "INBOX")); + assert!(folders.iter().any(|f| f.name == "Sent")); + } + + #[test] + fn discover_rejects_dovecot_layout_fs_tree() { + + let td = tempfile::tempdir().unwrap(); + make_maildir(td.path(), &["Sent"]); + let err = discover(td.path(), true).unwrap_err(); + assert!(matches!(err, DiscoverError::NotMaildirPlus(_))); + } + + #[test] + fn discover_rejects_path_without_cur_subdir() { + let td = tempfile::tempdir().unwrap(); + fs::create_dir_all(td.path().join("new")).unwrap(); + let err = discover(td.path(), true).unwrap_err(); + assert!(matches!(err, DiscoverError::NotAMaildir(_))); + } + + #[test] + fn discover_rejects_nonexistent_path() { + let err = discover(Path::new("/definitely/not/a/maildir/at/all"), true).unwrap_err(); + assert!(matches!(err, DiscoverError::NotFound(_))); + } + + #[test] + fn discover_returns_parents_before_children() { + let td = tempfile::tempdir().unwrap(); + make_maildir( + td.path(), + &[".Archive.2025", ".Archive", ".Lists.maildir-dev"], + ); + let folders = discover(td.path(), true).unwrap(); + let index = |name: &str| folders.iter().position(|f| f.name == name).unwrap(); + assert!(index("Archive") < index("Archive.2025")); + assert!(index("Lists") < index("Lists.maildir-dev")); + assert_eq!(index("INBOX"), 0); + } + + #[test] + fn ancestor_chain_walks_dot_segments() { + assert_eq!(ancestor_chain("A"), vec!["A".to_owned()]); + assert_eq!( + ancestor_chain("A.B.C"), + vec!["A".to_owned(), "A.B".to_owned(), "A.B.C".to_owned()] + ); + } + + #[test] + fn leaf_and_split_handle_root_and_nested() { + assert_eq!(leaf_of("INBOX"), "INBOX"); + assert_eq!(leaf_of("Archive.2025"), "2025"); + assert_eq!(split_parent("INBOX"), None); + assert_eq!(split_parent("Archive.2025"), Some("Archive".to_owned())); + } + + fn folders_for(names: &[&str]) -> Vec { + names + .iter() + .map(|n| ResolvedFolder { + name: (*n).to_owned(), + leaf: (*n).to_owned(), + parent_path: None, + path: PathBuf::new(), + role: None, + ephemeral: false, + }) + .collect() + } + + #[test] + fn include_filter_keeps_only_matches() { + let folders = folders_for(&["INBOX", "Sent", "Trash"]); + let filters = FolderFilters { + include: vec![Regex::new("^(INBOX|Sent)$").unwrap()], + exclude: Vec::new(), + explicit: Vec::new(), + }; + let kept = apply_filters(folders, &filters); + assert_eq!(kept.len(), 2); + assert!(kept.iter().any(|f| f.name == "INBOX")); + assert!(kept.iter().any(|f| f.name == "Sent")); + } + + #[test] + fn exclude_filter_drops_matches() { + let folders = folders_for(&["INBOX", "Trash"]); + let filters = FolderFilters { + include: Vec::new(), + exclude: vec![Regex::new("^Trash$").unwrap()], + explicit: Vec::new(), + }; + let kept = apply_filters(folders, &filters); + assert_eq!(kept.len(), 1); + assert_eq!(kept[0].name, "INBOX"); + } + + #[test] + fn explicit_folder_overrides_include_exclude() { + let folders = folders_for(&["INBOX", "Sent", "Trash"]); + let filters = FolderFilters { + include: Vec::new(), + exclude: Vec::new(), + explicit: vec!["Sent".to_owned()], + }; + let kept = apply_filters(folders, &filters); + assert_eq!(kept.len(), 1); + assert_eq!(kept[0].name, "Sent"); + } + + #[test] + fn discover_with_automap_off_clears_roles_except_inbox() { + let td = tempfile::tempdir().unwrap(); + make_maildir(td.path(), &[".Sent", ".Drafts"]); + let folders = discover(td.path(), false).unwrap(); + let inbox = folders.iter().find(|f| f.name == "INBOX").unwrap(); + assert_eq!(inbox.role, Some("inbox")); + let sent = folders.iter().find(|f| f.name == "Sent").unwrap(); + assert_eq!(sent.role, None); + let drafts = folders.iter().find(|f| f.name == "Drafts").unwrap(); + assert_eq!(drafts.role, None); + } + + #[cfg(target_os = "linux")] + #[test] + fn discover_renders_non_utf8_folder_name_lossy() { + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + let td = tempfile::tempdir().unwrap(); + make_maildir(td.path(), &[]); + let mut bytes = b".Foo".to_vec(); + bytes.extend_from_slice(&[0xff, 0xfe]); + let dotted = OsStr::from_bytes(&bytes); + let p = td.path().join(dotted); + for sub in ["cur", "new", "tmp"] { + fs::create_dir_all(p.join(sub)).unwrap(); + } + let folders = discover(td.path(), true).unwrap(); + assert_eq!(folders.len(), 2); + let sub = folders.iter().find(|f| f.name != "INBOX").unwrap(); + assert!( + sub.name.contains('\u{FFFD}'), + "lossy substitution: {:?}", + sub.name + ); + assert!(sub.name.starts_with("Foo")); + } + + #[test] + fn vanished_depth_sort_deepest_first() { + let mut names = vec![ + "Archive".to_owned(), + "Archive.2025.January".to_owned(), + "Archive.2025".to_owned(), + ]; + vanished_depth_sort(&mut names); + assert_eq!( + names, + vec![ + "Archive.2025.January".to_owned(), + "Archive.2025".to_owned(), + "Archive".to_owned(), + ] + ); + } +} diff --git a/src/sync/import_managesieve.rs b/src/sync/import_managesieve.rs new file mode 100644 index 0000000..b51e01d --- /dev/null +++ b/src/sync/import_managesieve.rs @@ -0,0 +1,10 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod coordinator; +pub mod reconcile; + +pub use coordinator::{ManageSieveAuth, ManageSieveImportConfig, run}; diff --git a/src/sync/import_managesieve/coordinator.rs b/src/sync/import_managesieve/coordinator.rs new file mode 100644 index 0000000..fa683cb --- /dev/null +++ b/src/sync/import_managesieve/coordinator.rs @@ -0,0 +1,638 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::sync::Arc; + +use url::Url; + +use crate::db; +use crate::db::sources::SourceKey; +use crate::error::Error; +use crate::imap::retry::BackoffState; +use crate::imap::transport::Connector; +use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; +use crate::managesieve::error::SieveError; +use crate::managesieve::response::parse_listscripts; +use crate::managesieve::retry::{Disposition, classify, is_negotiation_failure}; +use crate::managesieve::{ConnectMode, SieveClient, parse_getscript}; +use crate::sync::keys::blake3_bytes; +use crate::sync::{CommonConfig, Summary, TypeCounts}; + +use super::reconcile::{ + Action, PresentOutcome, apply_active_assignment, apply_content_update, apply_delete, apply_new, + load_local_state, plan, +}; + +pub struct ManageSieveImportConfig { + pub url: String, + pub auth: ManageSieveAuth, + pub allow_cleartext: bool, + pub allow_source_change: bool, +} + +#[derive(Debug, Clone)] +pub enum ManageSieveAuth { + Basic { user: String, password: String }, + Bearer { user: String, token: String }, +} + +#[derive(Debug, Clone)] +struct Endpoint { + host: String, + port: u16, + implicit_tls: bool, +} + +struct ControlCtx { + connector: Arc, + host: String, + port: u16, + mode: ConnectMode, + allow_cleartext: bool, + auth: ManageSieveAuth, + backoff: BackoffState, + max_retries: u32, + logger: Logger, +} + +fn reconnect(client: &mut SieveClient, ctx: &ControlCtx) -> Result<(), SieveError> { + let new = SieveClient::connect( + &ctx.connector, + &ctx.host, + ctx.port, + ctx.mode, + ctx.allow_cleartext, + )?; + *client = new; + match do_authenticate(client, &ctx.auth) { + Ok(()) => {} + Err(SieveAuthError::Wire(e)) => return Err(e), + Err(SieveAuthError::TerminallyRefused(text)) => { + return Err(SieveError::Protocol(format!( + "auth refused on reconnect: {text}" + ))); + } + Err(SieveAuthError::NoUsableMechanism(text)) => { + return Err(SieveError::Unsupported(text)); + } + } + if !client.had_fresh_post_auth_caps() + && let Err(e) = client.refresh_capabilities() + { + ctx.logger.warn(&format!( + "post-reconnect CAPABILITY refresh failed: {e}; continuing" + )); + } + Ok(()) +} + +pub fn run(common: CommonConfig, config: ManageSieveImportConfig) -> Result { + let logger = common.logger; + if common.threads > 1 { + log_at( + logger, + LEVEL_PROGRESS, + "managesieve importer is single-threaded; --threads value will be ignored", + ); + } + + let mut conn = db::init::open(&common.archive)?; + + let endpoint = parse_endpoint(&config.url)?; + let session_url = format!( + "{}://{}:{}", + if endpoint.implicit_tls { + "sieves" + } else { + "sieve" + }, + endpoint.host, + endpoint.port + ); + + let connector = Arc::new( + Connector::new(common.allow_invalid_certs).map_err(|e| Error::Connection(e.to_string()))?, + ); + let mode = if endpoint.implicit_tls { + ConnectMode::ImplicitTls + } else { + ConnectMode::StartTls + }; + + let mut client = SieveClient::connect( + &connector, + &endpoint.host, + endpoint.port, + mode, + config.allow_cleartext, + ) + .map_err(|e| Error::Connection(e.to_string()))?; + + let account_id = authenticate(&mut client, &config.auth)?; + log_at( + logger, + LEVEL_PROGRESS, + &format!("authenticated as {account_id} on {session_url}"), + ); + if !client.had_fresh_post_auth_caps() + && let Err(e) = client.refresh_capabilities() + { + logger.warn(&format!( + "post-auth CAPABILITY refresh failed: {e}; continuing" + )); + } + + let source_key = SourceKey { + kind: "managesieve".to_owned(), + session_url: session_url.clone(), + account_id: account_id.clone(), + }; + if let Some((existing_url, existing_account)) = + db::sources::conflicting_source(&conn, "managesieve", &session_url, &account_id)? + && !config.allow_source_change + { + return Err(Error::SourceChange(format!( + "archive already records managesieve source {existing_url} / {existing_account}; \ + re-run with --allow-source-change or use a fresh archive" + ))); + } + let source_id = if common.dry_run { + db::sources::find_source(&conn, &source_key)?.unwrap_or(-1) + } else { + db::sources::upsert_source(&conn, &source_key, Some(&account_id), &account_id)? + }; + + let backoff = BackoffState::new(); + let control_ctx = ControlCtx { + connector: connector.clone(), + host: endpoint.host.clone(), + port: endpoint.port, + mode, + allow_cleartext: config.allow_cleartext, + auth: config.auth.clone(), + backoff: backoff.clone(), + max_retries: common.max_retries, + logger, + }; + let server_listing = listscripts_with_retry(&mut client, &control_ctx)?; + + let mut server_with_bytes: Vec<(String, bool, [u8; 32], Vec)> = + Vec::with_capacity(server_listing.len()); + let mut counts = TypeCounts::default(); + for (name, active) in &server_listing { + match getscript_with_retry(&mut client, &control_ctx, name) { + Ok(bytes) => { + let hash = blake3_bytes(&bytes); + server_with_bytes.push((name.clone(), *active, hash, bytes)); + } + Err(e) => match classify(&e) { + Disposition::Permanent + | Disposition::PerScriptRecoverable + | Disposition::Transient => { + logger.warn(&format!("GETSCRIPT {name:?} failed, skipping: {e}")); + counts.failed += 1; + } + Disposition::Referral => { + return Err(Error::Connection(format!( + "ManageSieve referral on GETSCRIPT {name:?}: {e}; vandelay does not follow referrals" + ))); + } + Disposition::TransportDrop => { + return Err(Error::Connection(format!( + "GETSCRIPT {name:?} transport drop exhausted retries: {e}" + ))); + } + Disposition::FeatureNegotiation => { + return Err(Error::Connection(format!("GETSCRIPT {name:?}: {e}"))); + } + }, + } + } + + if common.dry_run { + let summary = + build_dry_run_summary(&conn, source_id, &server_with_bytes, counts, &backoff)?; + let _ = client.logout(); + return Ok(summary); + } + + let local_map = db::managesieve_ids::all_names(&conn, source_id)?; + let (local_hashes, local_active) = load_local_state(&conn, source_id)?; + let server_keyed: Vec<(String, bool, [u8; 32])> = server_with_bytes + .iter() + .map(|(n, a, h, _)| (n.clone(), *a, *h)) + .collect(); + let plan = plan(&server_keyed, &local_map, &local_hashes, &local_active); + + let bytes_by_name: std::collections::HashMap> = server_with_bytes + .iter() + .map(|(n, _, _, b)| (n.clone(), b)) + .collect(); + + let mut active_local: Option = None; + + for action in &plan.actions { + match action { + Action::New { name, active } => { + let bytes = bytes_by_name.get(name).cloned().unwrap(); + let tx = conn.transaction()?; + match apply_new(&tx, source_id, name, bytes) { + Ok(local_id) => { + tx.commit()?; + if *active { + active_local = Some(local_id); + } + counts.created += 1; + counts.fetched += 1; + } + Err(e) => { + let _ = tx.rollback(); + logger.warn(&format!("INSERT sieve_scripts {name:?} failed: {e}")); + counts.failed += 1; + } + } + } + Action::Present { + name, + local_id, + outcome, + active, + } => { + match outcome { + PresentOutcome::Unchanged => { + counts.skipped += 1; + } + PresentOutcome::ActiveOnly => { + counts.skipped += 1; + } + PresentOutcome::ContentUpdated => { + let bytes = bytes_by_name.get(name).cloned().unwrap(); + let tx = conn.transaction()?; + match apply_content_update(&tx, *local_id, bytes) { + Ok(()) => { + tx.commit()?; + counts.fetched += 1; + } + Err(e) => { + let _ = tx.rollback(); + logger.warn(&format!("UPDATE sieve_scripts {name:?} failed: {e}")); + counts.failed += 1; + } + } + } + } + if *active { + active_local = Some(*local_id); + } + } + Action::Vanished { name, local_id } => { + let tx = conn.transaction()?; + match apply_delete(&tx, source_id, name, *local_id) { + Ok(()) => { + tx.commit()?; + counts.deleted += 1; + } + Err(e) => { + let _ = tx.rollback(); + logger.warn(&format!("DELETE sieve_scripts {name:?} failed: {e}")); + counts.failed += 1; + } + } + } + } + } + + apply_active_assignment(&mut conn, active_local)?; + + let _ = client.logout(); + + if counts.failed == 0 { + let tx = conn.unchecked_transaction()?; + db::blobs::gc_orphan_blobs(&tx)?; + tx.commit()?; + } + + log_at( + logger, + LEVEL_DEFAULT, + &format!( + "managesieve: created={} fetched={} deleted={} skipped={} failed={}", + counts.created, counts.fetched, counts.deleted, counts.skipped, counts.failed + ), + ); + + Ok(Summary { + per_type: vec![("sievescript", counts)], + retries_observed: backoff.total_retries(), + retry_after_sleeps: backoff.transient_retries() as u64, + }) +} + +fn build_dry_run_summary( + conn: &rusqlite::Connection, + source_id: i64, + server: &[(String, bool, [u8; 32], Vec)], + mut counts: TypeCounts, + backoff: &BackoffState, +) -> Result { + let server_keyed: Vec<(String, bool, [u8; 32])> = server + .iter() + .map(|(n, a, h, _)| (n.clone(), *a, *h)) + .collect(); + let (local_map, local_hashes, local_active) = if source_id < 0 { + ( + std::collections::HashMap::new(), + std::collections::HashMap::new(), + std::collections::HashMap::new(), + ) + } else { + let local_map = db::managesieve_ids::all_names(conn, source_id)?; + let (h, a) = load_local_state(conn, source_id)?; + (local_map, h, a) + }; + let plan = plan(&server_keyed, &local_map, &local_hashes, &local_active); + for action in &plan.actions { + match action { + Action::New { .. } => { + counts.created += 1; + counts.fetched += 1; + } + Action::Present { + outcome: PresentOutcome::ContentUpdated, + .. + } => counts.fetched += 1, + Action::Present { .. } => counts.skipped += 1, + Action::Vanished { .. } => counts.deleted += 1, + } + } + Ok(Summary { + per_type: vec![("sievescript", counts)], + retries_observed: backoff.total_retries(), + retry_after_sleeps: backoff.transient_retries() as u64, + }) +} + +fn parse_endpoint(url: &str) -> Result { + let parsed = + Url::parse(url).map_err(|e| Error::Usage(format!("invalid --url {url:?}: {e}")))?; + let scheme = parsed.scheme(); + let implicit_tls = match scheme { + "sieves" => true, + "sieve" => false, + other => { + return Err(Error::Usage(format!( + "--url scheme must be sieve or sieves, got {other}" + ))); + } + }; + let host = parsed + .host_str() + .ok_or_else(|| Error::Usage(format!("--url missing host: {url}")))? + .to_owned(); + let port = parsed.port().unwrap_or(4190); + Ok(Endpoint { + host, + port, + implicit_tls, + }) +} + +fn authenticate(client: &mut SieveClient, auth: &ManageSieveAuth) -> Result { + do_authenticate(client, auth).map_err(|e| match e { + SieveAuthError::TerminallyRefused(text) => { + Error::Connection(format!("auth failed: {text}")) + } + SieveAuthError::NoUsableMechanism(text) => Error::Connection(text), + SieveAuthError::Wire(e) => Error::Connection(e.to_string()), + })?; + Ok(account_id_for(auth)) +} + +fn account_id_for(auth: &ManageSieveAuth) -> String { + match auth { + ManageSieveAuth::Basic { user, .. } => user.clone(), + ManageSieveAuth::Bearer { user, .. } => user.clone(), + } +} + +#[derive(Debug)] +enum SieveAuthError { + + TerminallyRefused(String), + + NoUsableMechanism(String), + + Wire(SieveError), +} + +fn do_authenticate(client: &mut SieveClient, auth: &ManageSieveAuth) -> Result<(), SieveAuthError> { + match auth { + ManageSieveAuth::Basic { user, password } => { + let caps = client.capabilities().clone(); + let mut last_plain_error: Option = None; + if caps.has_sasl("PLAIN") { + match client.authenticate_plain(user, password) { + Ok(()) => return Ok(()), + Err(e) if is_negotiation_failure(&e) => { + last_plain_error = Some(e.to_string()); + } + Err(e) => return Err(SieveAuthError::Wire(e)), + } + } + if caps.has_sasl("LOGIN") { + match client.authenticate_login(user, password) { + Ok(()) => return Ok(()), + Err(e) if is_negotiation_failure(&e) => { + let combined = match last_plain_error { + Some(p) => format!("LOGIN: {e}; PLAIN: {p}"), + None => format!("LOGIN: {e}"), + }; + return Err(SieveAuthError::TerminallyRefused(combined)); + } + Err(e) => return Err(SieveAuthError::Wire(e)), + } + } + if let Some(p) = last_plain_error { + return Err(SieveAuthError::TerminallyRefused(format!("PLAIN: {p}"))); + } + Err(SieveAuthError::NoUsableMechanism(format!( + "server SASL list lacks PLAIN and LOGIN (advertised: {:?})", + caps.sasl + ))) + } + ManageSieveAuth::Bearer { user, token } => { + if !client.capabilities().has_sasl("OAUTHBEARER") { + return Err(SieveAuthError::NoUsableMechanism(format!( + "server does not advertise OAUTHBEARER (advertised: {:?})", + client.capabilities().sasl + ))); + } + match client.authenticate_oauthbearer(user, token) { + Ok(()) => Ok(()), + Err(e) if is_negotiation_failure(&e) => Err(SieveAuthError::TerminallyRefused( + format!("OAUTHBEARER: {e}"), + )), + Err(e) => Err(SieveAuthError::Wire(e)), + } + } + } +} + +fn listscripts_with_retry( + client: &mut SieveClient, + ctx: &ControlCtx, +) -> Result, Error> { + let mut transient_attempts: u32 = 0; + let mut transport_attempts: u32 = 0; + loop { + match client.listscripts() { + Ok(block) => { + ctx.backoff.reset(); + let listed = parse_listscripts(&block.data) + .map_err(|e| Error::Connection(format!("LISTSCRIPTS parse: {e}")))?; + return Ok(listed.into_iter().map(|s| (s.name, s.active)).collect()); + } + Err(e) => match classify(&e) { + Disposition::Transient => { + if transient_attempts >= ctx.max_retries { + return Err(Error::Connection(format!( + "LISTSCRIPTS retries exhausted: {e}" + ))); + } + transient_attempts += 1; + log_at( + ctx.logger, + LEVEL_PROGRESS, + &format!("LISTSCRIPTS transient: {e}; retrying"), + ); + std::thread::sleep(ctx.backoff.next_shared_delay()); + } + Disposition::TransportDrop => { + if transport_attempts >= ctx.max_retries { + return Err(Error::Connection(format!( + "LISTSCRIPTS transport drop, retries exhausted: {e}" + ))); + } + transport_attempts += 1; + std::thread::sleep(ctx.backoff.transport_delay(transport_attempts)); + match reconnect(client, ctx) { + Ok(()) => { + log_at( + ctx.logger, + LEVEL_DEFAULT, + &format!("managesieve reconnected after transport drop ({e})"), + ); + } + Err(e2) => { + return Err(Error::Connection(format!( + "LISTSCRIPTS reconnect failed: {e2}" + ))); + } + } + } + Disposition::Referral => { + return Err(Error::Connection(format!( + "LISTSCRIPTS referral: {e}; vandelay does not follow referrals" + ))); + } + _ => return Err(Error::Connection(format!("LISTSCRIPTS failed: {e}"))), + }, + } + } +} + +fn getscript_with_retry( + client: &mut SieveClient, + ctx: &ControlCtx, + name: &str, +) -> Result, SieveError> { + let mut transient_attempts: u32 = 0; + let mut transport_attempts: u32 = 0; + loop { + match client.getscript(name) { + Ok(block) => { + ctx.backoff.reset(); + return parse_getscript(&block.data); + } + Err(e) => match classify(&e) { + Disposition::Transient => { + if transient_attempts >= ctx.max_retries { + return Err(e); + } + transient_attempts += 1; + log_at( + ctx.logger, + LEVEL_PROGRESS, + &format!("GETSCRIPT {name:?} transient: {e}; retrying"), + ); + std::thread::sleep(ctx.backoff.next_shared_delay()); + } + Disposition::TransportDrop => { + if transport_attempts >= ctx.max_retries { + return Err(e); + } + transport_attempts += 1; + std::thread::sleep(ctx.backoff.transport_delay(transport_attempts)); + reconnect(client, ctx)?; + log_at( + ctx.logger, + LEVEL_DEFAULT, + &format!( + "managesieve reconnected mid-GETSCRIPT {name:?} after transport drop" + ), + ); + } + _ => return Err(e), + }, + } + } +} + +fn log_at(logger: Logger, level: u8, msg: &str) { + if logger.enabled(level) { + eprintln!("{msg}"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_endpoint_sieves_defaults_to_4190() { + let e = parse_endpoint("sieves://mail.example.com").unwrap(); + assert_eq!(e.host, "mail.example.com"); + assert_eq!(e.port, 4190); + assert!(e.implicit_tls); + } + + #[test] + fn parse_endpoint_sieve_defaults_to_4190() { + let e = parse_endpoint("sieve://mail.example.com").unwrap(); + assert_eq!(e.host, "mail.example.com"); + assert_eq!(e.port, 4190); + assert!(!e.implicit_tls); + } + + #[test] + fn parse_endpoint_explicit_port() { + let e = parse_endpoint("sieve://mail.example.com:14190").unwrap(); + assert_eq!(e.port, 14190); + assert!(!e.implicit_tls); + } + + #[test] + fn parse_endpoint_rejects_other_schemes() { + let err = parse_endpoint("imap://example.com").unwrap_err(); + assert!(matches!(err, Error::Usage(_))); + } + + #[test] + fn parse_endpoint_rejects_missing_host() { + let err = parse_endpoint("sieve:///path").unwrap_err(); + assert!(matches!(err, Error::Usage(_))); + } +} diff --git a/src/sync/import_managesieve/reconcile.rs b/src/sync/import_managesieve/reconcile.rs new file mode 100644 index 0000000..184b603 --- /dev/null +++ b/src/sync/import_managesieve/reconcile.rs @@ -0,0 +1,480 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; + +use rusqlite::{Connection, OptionalExtension, params}; + +use crate::db; +use crate::error::Error; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PresentOutcome { + Unchanged, + ActiveOnly, + ContentUpdated, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Action { + New { + name: String, + active: bool, + }, + Present { + name: String, + local_id: i64, + outcome: PresentOutcome, + active: bool, + }, + Vanished { + name: String, + local_id: i64, + }, +} + +pub struct Plan { + pub actions: Vec, +} + +pub fn plan( + server: &[(String, bool, [u8; 32])], + local: &HashMap, + local_hashes: &HashMap, + local_active: &HashMap, +) -> Plan { + let mut actions = Vec::new(); + let mut server_names: HashSet<&str> = HashSet::new(); + for (name, active, hash) in server { + server_names.insert(name.as_str()); + match local.get(name) { + None => actions.push(Action::New { + name: name.clone(), + active: *active, + }), + Some(local_id) => { + let cur_hash = local_hashes.get(local_id); + let cur_active = local_active.get(local_id).copied().unwrap_or(false); + let outcome = if cur_hash != Some(hash) { + PresentOutcome::ContentUpdated + } else if cur_active != *active { + PresentOutcome::ActiveOnly + } else { + PresentOutcome::Unchanged + }; + actions.push(Action::Present { + name: name.clone(), + local_id: *local_id, + outcome, + active: *active, + }); + } + } + } + for (name, local_id) in local { + if !server_names.contains(name.as_str()) { + actions.push(Action::Vanished { + name: name.clone(), + local_id: *local_id, + }); + } + } + Plan { actions } +} + +pub type LocalState = (HashMap, HashMap); + +pub fn load_local_state(conn: &Connection, source_id: i64) -> Result { + let mut stmt = conn.prepare( + "SELECT s.id, b.hash, s.is_active + FROM sieve_scripts s JOIN blobs b ON b.id = s.blob_id + JOIN sync_id_managesieve m ON m.local_id = s.id + WHERE m.source_id = ?1", + )?; + let rows = stmt.query_map(params![source_id], |row| { + let id: i64 = row.get(0)?; + let hash: Vec = row.get(1)?; + let active: i64 = row.get(2)?; + Ok((id, hash, active != 0)) + })?; + let mut hashes = HashMap::new(); + let mut active = HashMap::new(); + for r in rows { + let (id, hash, a) = r?; + if hash.len() == 32 { + let mut arr = [0u8; 32]; + arr.copy_from_slice(&hash); + hashes.insert(id, arr); + } + active.insert(id, a); + } + Ok((hashes, active)) +} + +pub fn apply_new( + tx: &rusqlite::Transaction<'_>, + source_id: i64, + name: &str, + bytes: &[u8], +) -> Result { + let blob_id = db::blobs::intern_blob(tx, bytes)?; + tx.execute( + "INSERT INTO sieve_scripts (name, is_active, blob_id) VALUES (?1, 0, ?2)", + params![name, blob_id], + )?; + let local_id = tx.last_insert_rowid(); + db::managesieve_ids::insert(tx, source_id, name, local_id)?; + Ok(local_id) +} + +pub fn apply_content_update( + tx: &rusqlite::Transaction<'_>, + local_id: i64, + bytes: &[u8], +) -> Result<(), Error> { + let blob_id = db::blobs::intern_blob(tx, bytes)?; + tx.execute( + "UPDATE sieve_scripts SET blob_id = ?1 WHERE id = ?2", + params![blob_id, local_id], + )?; + Ok(()) +} + +pub fn apply_delete( + tx: &rusqlite::Transaction<'_>, + source_id: i64, + name: &str, + local_id: i64, +) -> Result<(), Error> { + tx.execute("DELETE FROM sieve_scripts WHERE id = ?1", params![local_id])?; + db::managesieve_ids::delete(tx, source_id, name)?; + Ok(()) +} + +pub fn apply_active_assignment( + conn: &mut Connection, + new_active_local: Option, +) -> Result<(), Error> { + let tx = conn.transaction()?; + let existing_active: Option = tx + .query_row( + "SELECT id FROM sieve_scripts WHERE is_active = 1", + [], + |r| r.get(0), + ) + .optional()?; + if existing_active != new_active_local { + if existing_active.is_some() { + tx.execute( + "UPDATE sieve_scripts SET is_active = 0 WHERE is_active = 1", + [], + )?; + } + if let Some(id) = new_active_local { + tx.execute( + "UPDATE sieve_scripts SET is_active = 1 WHERE id = ?1", + params![id], + )?; + } + } + tx.commit()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::init; + use crate::db::sources::{SourceKey, upsert_source}; + use blake3::Hasher; + + fn hash(b: &[u8]) -> [u8; 32] { + let mut h = Hasher::new(); + h.update(b); + h.finalize().into() + } + + fn mem() -> (Connection, i64) { + let c = Connection::open_in_memory().unwrap(); + init::apply_schema(&c).unwrap(); + let sid = upsert_source( + &c, + &SourceKey { + kind: "managesieve".to_owned(), + session_url: "sieve://h:4190".to_owned(), + account_id: "a".to_owned(), + }, + None, + "a", + ) + .unwrap(); + (c, sid) + } + + #[test] + fn plan_classifies_new_present_unchanged_and_vanished() { + let h_v = hash(b"v"); + let h_w = hash(b"w"); + let server: Vec<(String, bool, [u8; 32])> = vec![ + ("a".into(), false, h_v), + ("b".into(), true, h_w), + ("c".into(), false, h_v), + ]; + let mut local: HashMap = HashMap::new(); + local.insert("b".into(), 2); + local.insert("c".into(), 3); + local.insert("d".into(), 4); + let mut lh: HashMap = HashMap::new(); + lh.insert(2, h_w); + lh.insert(3, h_v); + lh.insert(4, h_v); + let mut la: HashMap = HashMap::new(); + la.insert(2, true); + la.insert(3, false); + la.insert(4, false); + let p = plan(&server, &local, &lh, &la); + let mut new_count = 0; + let mut unchanged = 0; + let mut content_updated = 0; + let mut active_only = 0; + let mut vanished = 0; + for a in p.actions { + match a { + Action::New { .. } => new_count += 1, + Action::Present { + outcome: PresentOutcome::Unchanged, + .. + } => unchanged += 1, + Action::Present { + outcome: PresentOutcome::ContentUpdated, + .. + } => content_updated += 1, + Action::Present { + outcome: PresentOutcome::ActiveOnly, + .. + } => active_only += 1, + Action::Vanished { .. } => vanished += 1, + } + } + assert_eq!(new_count, 1); + assert_eq!(unchanged, 2); + assert_eq!(content_updated, 0); + assert_eq!(active_only, 0); + assert_eq!(vanished, 1); + } + + #[test] + fn plan_flags_content_update_when_hash_differs() { + let h_v = hash(b"v"); + let h_w = hash(b"w"); + let server: Vec<(String, bool, [u8; 32])> = vec![("a".into(), false, h_w)]; + let mut local: HashMap = HashMap::new(); + local.insert("a".into(), 1); + let mut lh = HashMap::new(); + lh.insert(1, h_v); + let mut la = HashMap::new(); + la.insert(1, false); + let p = plan(&server, &local, &lh, &la); + assert!(matches!( + p.actions[0], + Action::Present { + outcome: PresentOutcome::ContentUpdated, + .. + } + )); + } + + #[test] + fn plan_flags_active_only_when_only_flag_differs() { + let h = hash(b"same"); + let server: Vec<(String, bool, [u8; 32])> = vec![("a".into(), true, h)]; + let mut local: HashMap = HashMap::new(); + local.insert("a".into(), 1); + let mut lh = HashMap::new(); + lh.insert(1, h); + let mut la = HashMap::new(); + la.insert(1, false); + let p = plan(&server, &local, &lh, &la); + assert!(matches!( + p.actions[0], + Action::Present { + outcome: PresentOutcome::ActiveOnly, + .. + } + )); + } + + #[test] + fn apply_new_inserts_script_and_id_mapping() { + let (mut c, sid) = mem(); + let tx = c.transaction().unwrap(); + let id = apply_new(&tx, sid, "vacation", b"require[\"x\"];").unwrap(); + tx.commit().unwrap(); + assert!(id > 0); + let local = db::managesieve_ids::local_for(&c, sid, "vacation") + .unwrap() + .unwrap(); + assert_eq!(local, id); + let count: i64 = c + .query_row("SELECT count(*) FROM sieve_scripts", [], |r| r.get(0)) + .unwrap(); + assert_eq!(count, 1); + } + + #[test] + fn apply_content_update_swaps_blob_pointer() { + let (mut c, sid) = mem(); + let tx = c.transaction().unwrap(); + let id = apply_new(&tx, sid, "vacation", b"v1").unwrap(); + tx.commit().unwrap(); + let tx = c.transaction().unwrap(); + apply_content_update(&tx, id, b"v2").unwrap(); + tx.commit().unwrap(); + let blob: Vec = c + .query_row( + "SELECT b.data FROM sieve_scripts s JOIN blobs b ON b.id=s.blob_id WHERE s.id=?1", + params![id], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(blob, b"v2"); + } + + #[test] + fn apply_active_assignment_swaps_active_in_one_transaction() { + let (mut c, sid) = mem(); + let tx = c.transaction().unwrap(); + let id_a = apply_new(&tx, sid, "a", b"x").unwrap(); + let id_b = apply_new(&tx, sid, "b", b"y").unwrap(); + tx.commit().unwrap(); + apply_active_assignment(&mut c, Some(id_a)).unwrap(); + let cur: Option = c + .query_row( + "SELECT id FROM sieve_scripts WHERE is_active = 1", + [], + |r| r.get(0), + ) + .optional() + .unwrap(); + assert_eq!(cur, Some(id_a)); + apply_active_assignment(&mut c, Some(id_b)).unwrap(); + let cur: Option = c + .query_row( + "SELECT id FROM sieve_scripts WHERE is_active = 1", + [], + |r| r.get(0), + ) + .optional() + .unwrap(); + assert_eq!(cur, Some(id_b)); + apply_active_assignment(&mut c, None).unwrap(); + let cur: Option = c + .query_row( + "SELECT id FROM sieve_scripts WHERE is_active = 1", + [], + |r| r.get(0), + ) + .optional() + .unwrap(); + assert_eq!(cur, None); + } + + #[test] + fn apply_delete_clears_active_before_destroy() { + let (mut c, sid) = mem(); + let tx = c.transaction().unwrap(); + let id = apply_new(&tx, sid, "a", b"x").unwrap(); + tx.commit().unwrap(); + apply_active_assignment(&mut c, Some(id)).unwrap(); + let tx = c.transaction().unwrap(); + apply_delete(&tx, sid, "a", id).unwrap(); + tx.commit().unwrap(); + let count: i64 = c + .query_row("SELECT count(*) FROM sieve_scripts", [], |r| r.get(0)) + .unwrap(); + assert_eq!(count, 0); + assert!( + db::managesieve_ids::local_for(&c, sid, "a") + .unwrap() + .is_none() + ); + } + + #[test] + fn apply_new_dedups_identical_script_bytes_at_blob_level() { + let (mut c, sid) = mem(); + let tx = c.transaction().unwrap(); + let id_a = apply_new(&tx, sid, "a", b"require [\"fileinto\"];\n").unwrap(); + let id_b = apply_new(&tx, sid, "b", b"require [\"fileinto\"];\n").unwrap(); + tx.commit().unwrap(); + let blob_a: i64 = c + .query_row( + "SELECT blob_id FROM sieve_scripts WHERE id = ?1", + params![id_a], + |r| r.get(0), + ) + .unwrap(); + let blob_b: i64 = c + .query_row( + "SELECT blob_id FROM sieve_scripts WHERE id = ?1", + params![id_b], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(blob_a, blob_b, "identical bytes must share a blob row"); + let blob_count: i64 = c + .query_row("SELECT count(*) FROM blobs", [], |r| r.get(0)) + .unwrap(); + assert_eq!(blob_count, 1, "blob table must contain a single row"); + } + + #[test] + fn apply_content_update_orphans_old_blob_for_gc() { + let (mut c, sid) = mem(); + let tx = c.transaction().unwrap(); + let id = apply_new(&tx, sid, "a", b"v1").unwrap(); + tx.commit().unwrap(); + let tx = c.transaction().unwrap(); + apply_content_update(&tx, id, b"v2").unwrap(); + tx.commit().unwrap(); + let blob_count_before_gc: i64 = c + .query_row("SELECT count(*) FROM blobs", [], |r| r.get(0)) + .unwrap(); + assert_eq!(blob_count_before_gc, 2, "old blob still present pre-GC"); + let tx = c.unchecked_transaction().unwrap(); + crate::db::blobs::gc_orphan_blobs(&tx).unwrap(); + tx.commit().unwrap(); + let blob_count_after_gc: i64 = c + .query_row("SELECT count(*) FROM blobs", [], |r| r.get(0)) + .unwrap(); + assert_eq!(blob_count_after_gc, 1, "orphan blob reclaimed by GC"); + } + + #[test] + fn partial_unique_index_rejects_two_active_rows() { + let (mut c, sid) = mem(); + let tx = c.transaction().unwrap(); + let id_a = apply_new(&tx, sid, "a", b"x").unwrap(); + let id_b = apply_new(&tx, sid, "b", b"y").unwrap(); + tx.commit().unwrap(); + c.execute( + "UPDATE sieve_scripts SET is_active = 1 WHERE id = ?1", + params![id_a], + ) + .unwrap(); + let err = c + .execute( + "UPDATE sieve_scripts SET is_active = 1 WHERE id = ?1", + params![id_b], + ) + .unwrap_err(); + let msg = err.to_string(); + assert!( + msg.contains("UNIQUE") || msg.contains("unique"), + "expected unique-constraint failure, got {msg}" + ); + } +} diff --git a/src/sync/import_takeout.rs b/src/sync/import_takeout.rs new file mode 100644 index 0000000..26655b0 --- /dev/null +++ b/src/sync/import_takeout.rs @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod calendar; +pub mod contacts; +pub mod coordinator; +pub mod labels; +pub mod mail; +pub mod mbox; +pub mod tree; +pub mod walk; + +pub use coordinator::{TakeoutImportConfig, run}; diff --git a/src/sync/import_takeout/calendar.rs b/src/sync/import_takeout/calendar.rs new file mode 100644 index 0000000..bb6dd95 --- /dev/null +++ b/src/sync/import_takeout/calendar.rs @@ -0,0 +1,309 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; +use std::path::Path; + +use rusqlite::{Connection, OptionalExtension, Transaction, params}; +use serde_json::Value; + +use crate::db::takeout_ids; +use crate::error::Error; +use crate::logging::Logger; +use crate::sync::TypeCounts; +use crate::sync::import_dav::calcard; + +const DEFAULT_CALENDAR: &str = "Imported"; + +pub fn process_file( + conn: &mut Connection, + file_path: &Path, + source_id: i64, + calendar_cache: &mut HashMap, + calendar_counts: &mut TypeCounts, + event_counts: &mut TypeCounts, + logger: Logger, +) -> Result<(), Error> { + let text = match std::fs::read_to_string(file_path) { + Ok(s) => s, + Err(e) => { + logger.warn(&format!("{file_path:?}: read: {e}")); + event_counts.failed += 1; + return Ok(()); + } + }; + + let entries = match calcard::ical_to_jscalendar_entries(&text) { + Ok(v) => v, + Err(calcard::CalcardError::NoEntries) => { + logger.warn(&format!("{file_path:?}: iCalendar contains no events")); + return Ok(()); + } + Err(e) => { + logger.warn(&format!("{file_path:?}: iCalendar parse: {e}")); + event_counts.failed += 1; + return Ok(()); + } + }; + + let metadata = parse_calendar_metadata(&text); + let container_name = metadata + .x_wr_calname + .clone() + .unwrap_or_else(|| DEFAULT_CALENDAR.to_owned()); + + let tx = conn.transaction()?; + let calendar_local = ensure_calendar( + &tx, + source_id, + &container_name, + &metadata, + calendar_cache, + calendar_counts, + )?; + + for (ordinal, mut entry) in entries.into_iter().enumerate() { + match insert_event( + &tx, + source_id, + calendar_local, + file_path, + ordinal, + &mut entry, + event_counts, + ) { + Ok(()) => {} + Err(e) => { + logger.warn(&format!("{file_path:?}#{ordinal}: event insert: {e}")); + event_counts.failed += 1; + } + } + } + tx.commit()?; + Ok(()) +} + +#[derive(Debug, Default, Clone)] +struct CalendarMetadata { + x_wr_calname: Option, + x_wr_caldesc: Option, + x_wr_timezone: Option, +} + +fn ensure_calendar( + tx: &Transaction<'_>, + source_id: i64, + name: &str, + metadata: &CalendarMetadata, + cache: &mut HashMap, + counts: &mut TypeCounts, +) -> Result { + if let Some(&id) = cache.get(name) { + return Ok(id); + } + if let Some(id) = takeout_ids::local_for(tx, source_id, takeout_ids::CALENDAR, name)? { + cache.insert(name.to_owned(), id); + return Ok(id); + } + tx.execute( + "INSERT INTO calendars + (name, description, time_zone, sort_order, is_subscribed, is_visible, is_default) + VALUES (?1, ?2, ?3, 0, 1, 1, 0)", + params![name, metadata.x_wr_caldesc, metadata.x_wr_timezone], + )?; + let id = tx.last_insert_rowid(); + takeout_ids::insert(tx, source_id, takeout_ids::CALENDAR, name, id)?; + cache.insert(name.to_owned(), id); + counts.created += 1; + counts.fetched += 1; + Ok(id) +} + +fn insert_event( + tx: &Transaction<'_>, + source_id: i64, + calendar_local: i64, + file_path: &Path, + ordinal: usize, + entry: &mut calcard::JsCalendarEntry, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let (is_draft, use_default_alerts, uid_field) = + calcard::strip_extracted_fields_from_event(&mut entry.data); + let uid = match uid_field + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + { + Some(u) => u.to_owned(), + None => calcard::synthesise_uid(&format!("{}#{ordinal}", file_path.to_string_lossy())), + }; + let data_type = entry.data_type.as_column(); + let data_json = entry.data.to_string(); + + let existing = takeout_ids::local_for(tx, source_id, takeout_ids::CALENDAR_EVENT, &uid)?; + + if let Some(local) = existing { + let calendar_ids = + merge_container_id(tx, "calendar_events", "calendar_ids", local, calendar_local)?; + let current: (String, i64, i64, String, String) = tx.query_row( + "SELECT calendar_ids, is_draft, use_default_alerts, data, data_type + FROM calendar_events WHERE id = ?1", + params![local], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)), + )?; + let want = ( + calendar_ids.clone(), + is_draft as i64, + use_default_alerts as i64, + data_json.clone(), + data_type.to_owned(), + ); + if current == want { + counts.skipped += 1; + return Ok(()); + } + tx.execute( + "UPDATE calendar_events SET calendar_ids = ?1, is_draft = ?2, + use_default_alerts = ?3, data = ?4, data_type = ?5 + WHERE id = ?6", + params![ + calendar_ids, + is_draft as i64, + use_default_alerts as i64, + data_json, + data_type, + local, + ], + )?; + counts.fetched += 1; + return Ok(()); + } + + let calendar_ids = format!("[{calendar_local}]"); + tx.execute( + "INSERT INTO calendar_events + (calendar_ids, is_draft, use_default_alerts, data, data_type) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + calendar_ids, + is_draft as i64, + use_default_alerts as i64, + data_json, + data_type, + ], + )?; + let local_id = tx.last_insert_rowid(); + takeout_ids::insert(tx, source_id, takeout_ids::CALENDAR_EVENT, &uid, local_id)?; + counts.created += 1; + counts.fetched += 1; + Ok(()) +} + +fn merge_container_id( + tx: &Transaction<'_>, + table: &str, + column: &str, + local_id: i64, + new_id: i64, +) -> Result { + let current: String = tx + .query_row( + &format!("SELECT {column} FROM {table} WHERE id = ?1"), + params![local_id], + |r| r.get(0), + ) + .optional()? + .unwrap_or_else(|| "[]".to_owned()); + let mut ids: Vec = match serde_json::from_str::(¤t) { + Ok(Value::Array(arr)) => arr.into_iter().filter_map(|v| v.as_i64()).collect(), + _ => Vec::new(), + }; + if !ids.contains(&new_id) { + ids.push(new_id); + } + ids.sort_unstable(); + ids.dedup(); + Ok(Value::Array(ids.into_iter().map(Value::from).collect()).to_string()) +} + +fn parse_calendar_metadata(text: &str) -> CalendarMetadata { + let mut out = CalendarMetadata::default(); + let mut inside_vevent = false; + for raw_line in text.lines() { + let line = raw_line.trim_end_matches('\r'); + if line.starts_with("BEGIN:VEVENT") + || line.starts_with("BEGIN:VTODO") + || line.starts_with("BEGIN:VJOURNAL") + { + inside_vevent = true; + continue; + } + if line.starts_with("END:VEVENT") + || line.starts_with("END:VTODO") + || line.starts_with("END:VJOURNAL") + { + inside_vevent = false; + continue; + } + if inside_vevent { + continue; + } + if let Some(rest) = line.strip_prefix("X-WR-CALNAME:") { + out.x_wr_calname = Some(rest.trim().to_owned()); + } else if let Some(rest) = line.strip_prefix("X-WR-CALDESC:") { + out.x_wr_caldesc = Some(rest.trim().to_owned()); + } else if let Some(rest) = line.strip_prefix("X-WR-TIMEZONE:") { + out.x_wr_timezone = Some(rest.trim().to_owned()); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_metadata_finds_xwr_fields() { + let ical = "BEGIN:VCALENDAR\r\n\ + PRODID:-//Google Inc//Google Calendar 70.9054//EN\r\n\ + X-WR-CALNAME:My Calendar\r\n\ + X-WR-CALDESC:Some description\r\n\ + X-WR-TIMEZONE:Europe/Madrid\r\n\ + BEGIN:VEVENT\r\nUID:e1@google.com\r\nDTSTAMP:20250101T000000Z\r\n\ + END:VEVENT\r\nEND:VCALENDAR\r\n"; + let m = parse_calendar_metadata(ical); + assert_eq!(m.x_wr_calname.as_deref(), Some("My Calendar")); + assert_eq!(m.x_wr_caldesc.as_deref(), Some("Some description")); + assert_eq!(m.x_wr_timezone.as_deref(), Some("Europe/Madrid")); + } + + #[test] + fn parse_metadata_ignores_xwr_inside_vevent() { + let ical = "BEGIN:VCALENDAR\r\n\ + BEGIN:VEVENT\r\nUID:e1\r\nX-WR-CALNAME:NOT THIS\r\nEND:VEVENT\r\n\ + END:VCALENDAR\r\n"; + let m = parse_calendar_metadata(ical); + assert!(m.x_wr_calname.is_none()); + } + + #[test] + fn parse_metadata_empty_when_missing() { + let ical = "BEGIN:VCALENDAR\r\nEND:VCALENDAR\r\n"; + let m = parse_calendar_metadata(ical); + assert!(m.x_wr_calname.is_none()); + assert!(m.x_wr_caldesc.is_none()); + assert!(m.x_wr_timezone.is_none()); + } + + #[test] + fn parse_metadata_handles_lf_only_line_endings() { + let ical = "BEGIN:VCALENDAR\nX-WR-CALNAME:Foo\nEND:VCALENDAR\n"; + let m = parse_calendar_metadata(ical); + assert_eq!(m.x_wr_calname.as_deref(), Some("Foo")); + } +} diff --git a/src/sync/import_takeout/contacts.rs b/src/sync/import_takeout/contacts.rs new file mode 100644 index 0000000..4d13149 --- /dev/null +++ b/src/sync/import_takeout/contacts.rs @@ -0,0 +1,264 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::path::Path; + +use rusqlite::{Connection, Transaction, params}; + +use crate::db::takeout_ids; +use crate::error::Error; +use crate::logging::Logger; +use crate::sync::TypeCounts; +use crate::sync::import_dav::calcard; + +const DEFAULT_ADDRESS_BOOK: &str = "Imported"; + +pub fn process_file( + conn: &mut Connection, + file_path: &Path, + source_id: i64, + address_book_local: &mut Option, + book_counts: &mut TypeCounts, + card_counts: &mut TypeCounts, + logger: Logger, +) -> Result<(), Error> { + let text = match std::fs::read_to_string(file_path) { + Ok(s) => s, + Err(e) => { + logger.warn(&format!("{file_path:?}: read: {e}")); + card_counts.failed += 1; + return Ok(()); + } + }; + + let chunks = split_vcards(&text); + if chunks.is_empty() { + return Ok(()); + } + + let tx = conn.transaction()?; + let book_local = match *address_book_local { + Some(id) => id, + None => { + let id = ensure_default_address_book(&tx, source_id, book_counts)?; + *address_book_local = Some(id); + id + } + }; + + for (ordinal, chunk) in chunks.into_iter().enumerate() { + match insert_card( + &tx, + source_id, + book_local, + file_path, + ordinal, + &chunk, + card_counts, + ) { + Ok(()) => {} + Err(e) => { + logger.warn(&format!("{file_path:?}#{ordinal}: vCard: {e}")); + card_counts.failed += 1; + } + } + } + tx.commit()?; + Ok(()) +} + +fn ensure_default_address_book( + tx: &Transaction<'_>, + source_id: i64, + counts: &mut TypeCounts, +) -> Result { + if let Some(id) = takeout_ids::local_for( + tx, + source_id, + takeout_ids::ADDRESS_BOOK, + DEFAULT_ADDRESS_BOOK, + )? { + return Ok(id); + } + tx.execute( + "INSERT INTO address_books + (name, description, sort_order, is_default, is_subscribed) + VALUES (?1, NULL, 0, 0, 1)", + params![DEFAULT_ADDRESS_BOOK], + )?; + let id = tx.last_insert_rowid(); + takeout_ids::insert( + tx, + source_id, + takeout_ids::ADDRESS_BOOK, + DEFAULT_ADDRESS_BOOK, + id, + )?; + counts.created += 1; + counts.fetched += 1; + Ok(id) +} + +fn insert_card( + tx: &Transaction<'_>, + source_id: i64, + book_local: i64, + file_path: &Path, + ordinal: usize, + chunk: &str, + counts: &mut TypeCounts, +) -> Result<(), Error> { + let synthetic_href = format!("{}#{ordinal}", file_path.to_string_lossy()); + let card = calcard::vcard_to_jscontact(chunk, &synthetic_href) + .map_err(|e| Error::Partial(format!("vCard parse: {e}")))?; + let data_json = card.data.to_string(); + let address_book_ids = format!("[{book_local}]"); + + let existing = takeout_ids::local_for(tx, source_id, takeout_ids::CONTACT_CARD, &card.uid)?; + + if let Some(local) = existing { + let current: (String, String, String) = tx.query_row( + "SELECT uid, address_book_ids, data FROM contact_cards WHERE id = ?1", + params![local], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + )?; + let want = ( + card.uid.clone(), + address_book_ids.clone(), + data_json.clone(), + ); + if current == want { + counts.skipped += 1; + return Ok(()); + } + tx.execute( + "UPDATE contact_cards SET uid = ?1, address_book_ids = ?2, data = ?3 + WHERE id = ?4", + params![card.uid, address_book_ids, data_json, local], + )?; + counts.fetched += 1; + return Ok(()); + } + + tx.execute( + "INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)", + params![card.uid, address_book_ids, data_json], + )?; + let local_id = tx.last_insert_rowid(); + takeout_ids::insert( + tx, + source_id, + takeout_ids::CONTACT_CARD, + &card.uid, + local_id, + )?; + counts.created += 1; + counts.fetched += 1; + Ok(()) +} + +fn split_vcards(text: &str) -> Vec { + let mut chunks: Vec = Vec::new(); + let mut current: Option = None; + for raw_line in text.lines() { + let line = raw_line.trim_end_matches('\r'); + let bytes = line.as_bytes(); + if starts_with_ignore_ascii_case(bytes, b"BEGIN:VCARD") { + current = Some(String::new()); + } + if let Some(buf) = current.as_mut() { + buf.push_str(line); + buf.push('\n'); + } + if starts_with_ignore_ascii_case(bytes, b"END:VCARD") + && let Some(buf) = current.take() + { + chunks.push(buf); + } + } + chunks +} + +fn starts_with_ignore_ascii_case(haystack: &[u8], needle: &[u8]) -> bool { + haystack.len() >= needle.len() && haystack[..needle.len()].eq_ignore_ascii_case(needle) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn splits_single_card() { + let text = "BEGIN:VCARD\r\nVERSION:3.0\r\nFN:A\r\nEND:VCARD\r\n"; + let chunks = split_vcards(text); + assert_eq!(chunks.len(), 1); + assert!(chunks[0].contains("FN:A")); + } + + #[test] + fn splits_multiple_cards() { + let text = "BEGIN:VCARD\r\nVERSION:3.0\r\nFN:A\r\nEND:VCARD\r\n\ + BEGIN:VCARD\r\nVERSION:3.0\r\nFN:B\r\nEND:VCARD\r\n\ + BEGIN:VCARD\r\nVERSION:3.0\r\nFN:C\r\nEND:VCARD\r\n"; + let chunks = split_vcards(text); + assert_eq!(chunks.len(), 3); + assert!(chunks[0].contains("FN:A")); + assert!(chunks[1].contains("FN:B")); + assert!(chunks[2].contains("FN:C")); + } + + #[test] + fn skips_lines_outside_any_card() { + let text = "Some preamble garbage\r\n\ + BEGIN:VCARD\r\nVERSION:3.0\r\nFN:A\r\nEND:VCARD\r\n\ + garbage between cards\r\n\ + BEGIN:VCARD\r\nVERSION:3.0\r\nFN:B\r\nEND:VCARD\r\n\ + trailing garbage\r\n"; + let chunks = split_vcards(text); + assert_eq!(chunks.len(), 2); + assert!(!chunks[0].contains("garbage")); + } + + #[test] + fn preserves_folded_continuation_lines_inside_a_card() { + let text = "BEGIN:VCARD\r\nVERSION:3.0\r\n\ + ADR:;;1 Infinite Loop\\nCupertino\\, CA 95014\\nUnited States\r\n \ + ;;;;;\r\nEND:VCARD\r\n"; + let chunks = split_vcards(text); + assert_eq!(chunks.len(), 1); + assert!(chunks[0].contains("ADR:")); + assert!(chunks[0].contains(";;;;;")); + } + + #[test] + fn handles_lf_only_line_endings() { + let text = "BEGIN:VCARD\nVERSION:3.0\nFN:LF\nEND:VCARD\n"; + let chunks = split_vcards(text); + assert_eq!(chunks.len(), 1); + assert!(chunks[0].contains("FN:LF")); + } + + #[test] + fn empty_input_yields_no_chunks() { + assert!(split_vcards("").is_empty()); + assert!(split_vcards("not a vcard\n").is_empty()); + } + + #[test] + fn unterminated_card_is_dropped() { + let text = "BEGIN:VCARD\r\nVERSION:3.0\r\nFN:A\r\n"; + let chunks = split_vcards(text); + assert!(chunks.is_empty()); + } + + #[test] + fn case_insensitive_boundary_match() { + let text = "begin:vcard\r\nFN:CaseTest\r\nend:vcard\r\n"; + let chunks = split_vcards(text); + assert_eq!(chunks.len(), 1); + assert!(chunks[0].contains("FN:CaseTest")); + } +} diff --git a/src/sync/import_takeout/coordinator.rs b/src/sync/import_takeout/coordinator.rs new file mode 100644 index 0000000..53b2b77 --- /dev/null +++ b/src/sync/import_takeout/coordinator.rs @@ -0,0 +1,382 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use rusqlite::{Connection, params}; +use url::Url; + +use crate::db; +use crate::db::sources::SourceKey; +use crate::db::takeout_ids; +use crate::error::Error; +use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; +use crate::sync::{CommonConfig, Summary, TypeCounts}; + +use super::calendar; +use super::contacts; +use super::labels::MappingOptions; +use super::mail; +use super::walk::{self, FileKind}; + +#[derive(Debug)] +pub struct TakeoutImportConfig { + pub takeout_root: PathBuf, + pub allow_source_change: bool, + pub automap: bool, +} + +pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result { + let logger = common.logger; + if common.threads > 1 && logger.enabled(LEVEL_PROGRESS) { + eprintln!("takeout importer is single-threaded; --threads value will be ignored"); + } + + let canonical = std::fs::canonicalize(&config.takeout_root) + .map_err(|e| Error::Usage(format!("--path {:?}: {e}", config.takeout_root)))?; + if !canonical.is_dir() { + return Err(Error::Usage(format!("not a directory: {canonical:?}"))); + } + + let walk_result = walk::walk_with_logger(&canonical, logger) + .map_err(|e| Error::Usage(format!("walking {canonical:?}: {e}")))?; + if walk_result.is_empty() { + return Err(Error::Usage(format!( + "{canonical:?}: no .mbox / .ics / .vcf files found under this path" + ))); + } + + if logger.enabled(LEVEL_PROGRESS) { + let mbox_n = walk_result.by_kind(FileKind::Mbox).count(); + let ics_n = walk_result.by_kind(FileKind::Ics).count(); + let vcf_n = walk_result.by_kind(FileKind::Vcf).count(); + eprintln!( + "takeout discovery: {} .mbox, {} .ics, {} .vcf (io failures: {}, symlink cycles: {})", + mbox_n, ics_n, vcf_n, walk_result.io_failures, walk_result.symlink_cycles + ); + } + + let mut conn = db::init::open(&common.archive)?; + let session_url = file_url_for(&canonical)?; + let account_id = canonical.to_string_lossy().into_owned(); + let account_name = canonical + .file_name() + .and_then(|n| n.to_str()) + .map(str::to_owned); + + if let Some((existing_url, existing_account)) = + db::sources::conflicting_source(&conn, "takeout", &session_url, &account_id)? + && !config.allow_source_change + { + return Err(Error::SourceChange(format!( + "archive already records takeout source {existing_url} / {existing_account}; \ + re-run with --allow-source-change or use a fresh archive" + ))); + } + + let source_key = SourceKey { + kind: "takeout".to_owned(), + session_url: session_url.clone(), + account_id: account_id.clone(), + }; + let source_id = if common.dry_run { + db::sources::find_source(&conn, &source_key)?.unwrap_or(-1) + } else { + db::sources::upsert_source(&conn, &source_key, account_name.as_deref(), "")? + }; + + if common.dry_run { + return Ok(dry_run_summary(&conn, source_id, &walk_result, logger)); + } + + let options = MappingOptions { + automap: config.automap, + }; + + let mut mailbox_counts = TypeCounts::default(); + let mut email_counts = TypeCounts::default(); + let mut calendar_counts = TypeCounts::default(); + let mut calendar_event_counts = TypeCounts::default(); + let mut book_counts = TypeCounts::default(); + let mut card_counts = TypeCounts::default(); + + let mut mailbox_cache: HashMap = + takeout_ids::all_for_type(&conn, source_id, takeout_ids::MAILBOX)?; + + process_mbox_files( + &mut conn, + source_id, + &walk_result, + options, + &mut mailbox_cache, + &mut mailbox_counts, + &mut email_counts, + logger, + )?; + + process_ics_files( + &mut conn, + source_id, + &walk_result, + &mut calendar_counts, + &mut calendar_event_counts, + logger, + )?; + + process_vcf_files( + &mut conn, + source_id, + &walk_result, + &mut book_counts, + &mut card_counts, + logger, + )?; + + let no_failures = mailbox_counts.failed == 0 + && email_counts.failed == 0 + && calendar_counts.failed == 0 + && calendar_event_counts.failed == 0 + && book_counts.failed == 0 + && card_counts.failed == 0; + if no_failures { + let tx = conn.unchecked_transaction()?; + db::blobs::gc_orphan_blobs(&tx)?; + tx.commit()?; + } + + if logger.enabled(LEVEL_DEFAULT) { + eprintln!( + "takeout: mailbox c={} f={} | email c={} u={} f={} | calendar c={} f={} | \ + event c={} u={} f={} | addressbook c={} | card c={} u={} f={}", + mailbox_counts.created, + mailbox_counts.failed, + email_counts.created, + email_counts.fetched.saturating_sub(email_counts.created), + email_counts.failed, + calendar_counts.created, + calendar_counts.failed, + calendar_event_counts.created, + calendar_event_counts + .fetched + .saturating_sub(calendar_event_counts.created), + calendar_event_counts.failed, + book_counts.created, + card_counts.created, + card_counts.fetched.saturating_sub(card_counts.created), + card_counts.failed, + ); + } + + Ok(Summary { + per_type: vec![ + ("mailbox", mailbox_counts), + ("email", email_counts), + ("calendar", calendar_counts), + ("calendarevent", calendar_event_counts), + ("addressbook", book_counts), + ("contactcard", card_counts), + ], + retries_observed: 0, + retry_after_sleeps: 0, + }) +} + +#[allow(clippy::too_many_arguments)] +fn process_mbox_files( + conn: &mut Connection, + source_id: i64, + walk_result: &walk::WalkResult, + options: MappingOptions, + mailbox_cache: &mut HashMap, + mailbox_counts: &mut TypeCounts, + email_counts: &mut TypeCounts, + logger: Logger, +) -> Result<(), Error> { + for file in walk_result.by_kind(FileKind::Mbox) { + let fallback = mailbox_name_from_filename(&file.path); + if logger.enabled(LEVEL_PROGRESS) { + eprintln!( + "mbox: processing {:?} (fallback mailbox: {:?})", + file.path, fallback + ); + } + let ctx = mail::InsertContext { + source_id, + fallback_mailbox: &fallback, + options, + mailbox_cache, + }; + match mail::process_file(conn, &file.path, ctx, mailbox_counts, email_counts, logger) { + Ok(()) => {} + Err(e) => { + logger.warn(&format!("mbox {:?}: {e}", file.path)); + email_counts.failed += 1; + } + } + } + Ok(()) +} + +fn process_ics_files( + conn: &mut Connection, + source_id: i64, + walk_result: &walk::WalkResult, + calendar_counts: &mut TypeCounts, + event_counts: &mut TypeCounts, + logger: Logger, +) -> Result<(), Error> { + let mut cache: HashMap = + takeout_ids::all_for_type(conn, source_id, takeout_ids::CALENDAR)?; + for file in walk_result.by_kind(FileKind::Ics) { + if logger.enabled(LEVEL_PROGRESS) { + eprintln!("ics: processing {:?}", file.path); + } + if let Err(e) = calendar::process_file( + conn, + &file.path, + source_id, + &mut cache, + calendar_counts, + event_counts, + logger, + ) { + logger.warn(&format!("ics {:?}: {e}", file.path)); + event_counts.failed += 1; + } + } + Ok(()) +} + +fn process_vcf_files( + conn: &mut Connection, + source_id: i64, + walk_result: &walk::WalkResult, + book_counts: &mut TypeCounts, + card_counts: &mut TypeCounts, + logger: Logger, +) -> Result<(), Error> { + let mut book_local: Option = + takeout_ids::local_for(conn, source_id, takeout_ids::ADDRESS_BOOK, "Imported")?; + for file in walk_result.by_kind(FileKind::Vcf) { + if logger.enabled(LEVEL_PROGRESS) { + eprintln!("vcf: processing {:?}", file.path); + } + if let Err(e) = contacts::process_file( + conn, + &file.path, + source_id, + &mut book_local, + book_counts, + card_counts, + logger, + ) { + logger.warn(&format!("vcf {:?}: {e}", file.path)); + card_counts.failed += 1; + } + } + Ok(()) +} + +fn mailbox_name_from_filename(path: &Path) -> String { + path.file_stem() + .and_then(|s| s.to_str()) + .map(str::to_owned) + .unwrap_or_else(|| "Imported".to_owned()) +} + +fn file_url_for(p: &Path) -> Result { + Url::from_file_path(p) + .map(|u| u.to_string()) + .map_err(|_| Error::Usage(format!("cannot encode takeout path as file:// URL: {p:?}"))) +} + +fn dry_run_summary( + conn: &Connection, + source_id: i64, + walk_result: &walk::WalkResult, + logger: Logger, +) -> Summary { + let mbox_n = walk_result.by_kind(FileKind::Mbox).count(); + let ics_n = walk_result.by_kind(FileKind::Ics).count(); + let vcf_n = walk_result.by_kind(FileKind::Vcf).count(); + let local_emails = if source_id < 0 { + 0 + } else { + count_rows(conn, source_id, takeout_ids::EMAIL) + }; + let local_events = if source_id < 0 { + 0 + } else { + count_rows(conn, source_id, takeout_ids::CALENDAR_EVENT) + }; + let local_cards = if source_id < 0 { + 0 + } else { + count_rows(conn, source_id, takeout_ids::CONTACT_CARD) + }; + if logger.enabled(LEVEL_DEFAULT) { + eprintln!("dry-run: mbox files={mbox_n} ics files={ics_n} vcf files={vcf_n}"); + eprintln!( + " existing in archive: emails={local_emails} events={local_events} cards={local_cards}" + ); + } + Summary { + per_type: vec![ + ("mailbox", TypeCounts::default()), + ("email", TypeCounts::default()), + ("calendar", TypeCounts::default()), + ("calendarevent", TypeCounts::default()), + ("addressbook", TypeCounts::default()), + ("contactcard", TypeCounts::default()), + ], + retries_observed: 0, + retry_after_sleeps: 0, + } +} + +fn count_rows(conn: &Connection, source_id: i64, type_name: &str) -> u64 { + conn.query_row( + "SELECT COUNT(*) FROM sync_id_takeout WHERE source_id = ?1 AND type_name = ?2", + params![source_id, type_name], + |row| row.get::<_, i64>(0), + ) + .unwrap_or(0) as u64 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn file_url_encodes_unicode_path() { + let p = Path::new("/tmp/Año"); + let u = file_url_for(p).unwrap(); + assert!(u.starts_with("file://")); + assert!(u.contains("A%C3%B1o")); + } + + #[test] + fn file_url_rejects_relative_path() { + let err = file_url_for(Path::new("./relative")).unwrap_err(); + assert!(matches!(err, Error::Usage(_))); + } + + #[test] + fn mailbox_name_from_filename_strips_extension() { + let p = Path::new("/some/dir/Github.mbox"); + assert_eq!(mailbox_name_from_filename(p), "Github"); + } + + #[test] + fn mailbox_name_from_filename_handles_dotted_basename() { + let p = Path::new("/x/All mail Including Spam and Trash.mbox"); + assert_eq!( + mailbox_name_from_filename(p), + "All mail Including Spam and Trash" + ); + } +} diff --git a/src/sync/import_takeout/labels.rs b/src/sync/import_takeout/labels.rs new file mode 100644 index 0000000..1a51446 --- /dev/null +++ b/src/sync/import_takeout/labels.rs @@ -0,0 +1,302 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::BTreeSet; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LabelClassification { + pub mailboxes: Vec, + pub keywords: BTreeSet, + pub had_explicit_seen_signal: bool, + pub had_unread_signal: bool, + pub opened_won_over_unread: bool, +} + +impl LabelClassification { + pub fn keywords_sorted(&self) -> Vec { + self.keywords.iter().cloned().collect() + } + + pub fn is_empty(&self) -> bool { + self.mailboxes.is_empty() && self.keywords.is_empty() + } +} + +#[derive(Debug, Clone, Copy)] +pub struct MappingOptions { + pub automap: bool, +} + +impl Default for MappingOptions { + fn default() -> Self { + Self { automap: true } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MailboxAssignment { + pub canonical_path: String, + pub role: Option<&'static str>, +} + +pub fn parse_header(value: &str) -> Vec { + value + .split(',') + .map(str::trim) + .filter(|t| !t.is_empty()) + .map(str::to_owned) + .collect() +} + +pub fn classify(tokens: &[String]) -> LabelClassification { + let mut mailboxes: Vec = Vec::new(); + let mut keywords: BTreeSet = BTreeSet::new(); + let mut had_opened = false; + let mut had_unread = false; + + for tok in tokens { + if is_category_token(tok) { + continue; + } + match tok.as_str() { + "Starred" => { + keywords.insert("$flagged".to_owned()); + } + "Important" => { + keywords.insert("$important".to_owned()); + } + "Opened" => { + had_opened = true; + keywords.insert("$seen".to_owned()); + } + "Unread" => { + had_unread = true; + } + other => { + let canonical = canonical_label(other); + if !mailboxes.iter().any(|m| m == &canonical) { + mailboxes.push(canonical); + } + } + } + } + + let opened_won_over_unread = had_opened && had_unread; + LabelClassification { + mailboxes, + keywords, + had_explicit_seen_signal: had_opened, + had_unread_signal: had_unread, + opened_won_over_unread, + } +} + +pub fn role_for_mailbox(canonical_path: &str, opts: MappingOptions) -> Option<&'static str> { + if !opts.automap { + return None; + } + if canonical_path.contains('/') { + return None; + } + match canonical_path { + "Inbox" => Some("inbox"), + "Sent" => Some("sent"), + "Drafts" => Some("drafts"), + "Trash" => Some("trash"), + "Spam" => Some("junk"), + "Archive" => Some("archive"), + _ => None, + } +} + +pub fn assignments_for( + classification: &LabelClassification, + opts: MappingOptions, +) -> Vec { + classification + .mailboxes + .iter() + .map(|p| MailboxAssignment { + canonical_path: p.clone(), + role: role_for_mailbox(p, opts), + }) + .collect() +} + +fn is_category_token(t: &str) -> bool { + matches!( + t, + "Category Personal" + | "Category Promotions" + | "Category Social" + | "Category Updates" + | "Category Forums" + ) +} + +fn canonical_label(raw: &str) -> String { + match raw { + "Archived" => "Archive".to_owned(), + other => other.to_owned(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn classify_str(s: &str) -> LabelClassification { + classify(&parse_header(s)) + } + + #[test] + fn parse_header_splits_on_comma_and_trims() { + let v = parse_header("Inbox, Opened ,Starred,,Important"); + assert_eq!(v, vec!["Inbox", "Opened", "Starred", "Important"]); + } + + #[test] + fn parse_header_empty_yields_empty() { + assert!(parse_header("").is_empty()); + assert!(parse_header(",, ,, ").is_empty()); + } + + #[test] + fn inbox_token_becomes_inbox_mailbox_with_role() { + let c = classify_str("Inbox"); + assert_eq!(c.mailboxes, vec!["Inbox".to_owned()]); + let a = assignments_for(&c, MappingOptions::default()); + assert_eq!(a[0].role, Some("inbox")); + } + + #[test] + fn sent_drafts_trash_spam_archive_map_to_roles() { + for (token, role) in [ + ("Sent", "sent"), + ("Drafts", "drafts"), + ("Trash", "trash"), + ("Spam", "junk"), + ("Archived", "archive"), + ] { + let c = classify_str(token); + let a = assignments_for(&c, MappingOptions::default()); + assert_eq!(a[0].role, Some(role), "token {token}"); + } + let c = classify_str("Archived"); + assert_eq!(c.mailboxes, vec!["Archive".to_owned()]); + } + + #[test] + fn starred_important_become_keywords_not_mailboxes() { + let c = classify_str("Starred,Important"); + assert!(c.mailboxes.is_empty()); + assert!(c.keywords.contains("$flagged")); + assert!(c.keywords.contains("$important")); + } + + #[test] + fn opened_yields_seen_keyword() { + let c = classify_str("Inbox,Opened"); + assert!(c.keywords.contains("$seen")); + } + + #[test] + fn unread_alone_means_no_seen() { + let c = classify_str("Inbox,Unread"); + assert!(!c.keywords.contains("$seen")); + assert!(c.had_unread_signal); + } + + #[test] + fn opened_plus_unread_marks_collision_and_keeps_seen() { + let c = classify_str("Inbox,Opened,Unread"); + assert!(c.keywords.contains("$seen")); + assert!(c.opened_won_over_unread); + } + + #[test] + fn category_tokens_are_dropped_silently() { + let c = classify_str("Inbox,Category Promotions,Category Updates,Opened"); + assert_eq!(c.mailboxes, vec!["Inbox".to_owned()]); + assert!(c.keywords.contains("$seen")); + } + + #[test] + fn nested_labels_preserve_full_path() { + let c = classify_str("Inbox,Label_001/Label_002_under_Label_001"); + assert!( + c.mailboxes + .contains(&"Label_001/Label_002_under_Label_001".to_owned()) + ); + let a = assignments_for(&c, MappingOptions::default()); + let nested = a + .iter() + .find(|m| m.canonical_path == "Label_001/Label_002_under_Label_001") + .unwrap(); + assert_eq!(nested.role, None); + } + + #[test] + fn custom_labels_get_no_role() { + let c = classify_str("Inbox,Github,Newsletter"); + let a = assignments_for(&c, MappingOptions::default()); + let github = a.iter().find(|m| m.canonical_path == "Github").unwrap(); + assert_eq!(github.role, None); + } + + #[test] + fn realistic_fixture_label_sets_classify_as_expected() { + let c = classify_str("Archived,Important,Opened,Category Social,Github"); + assert!(c.mailboxes.contains(&"Archive".to_owned())); + assert!(c.mailboxes.contains(&"Github".to_owned())); + assert!(c.keywords.contains("$important")); + assert!(c.keywords.contains("$seen")); + assert!(!c.mailboxes.iter().any(|m| m.contains("Category"))); + + let c = classify_str("Important,Trash,Category Social,Unread,Github"); + assert!(c.mailboxes.contains(&"Trash".to_owned())); + assert!(c.mailboxes.contains(&"Github".to_owned())); + assert!(c.keywords.contains("$important")); + assert!(!c.keywords.contains("$seen")); + assert!(c.had_unread_signal); + } + + #[test] + fn noautomap_suppresses_roles_but_keeps_names() { + let c = classify_str("Inbox,Sent,Github"); + let a = assignments_for(&c, MappingOptions { automap: false }); + assert!(a.iter().all(|m| m.role.is_none())); + let names: Vec<&str> = a.iter().map(|m| m.canonical_path.as_str()).collect(); + assert!(names.contains(&"Inbox")); + assert!(names.contains(&"Sent")); + assert!(names.contains(&"Github")); + } + + #[test] + fn keywords_sorted_returns_deterministic_order() { + let c = classify_str("Starred,Important,Opened"); + let k = c.keywords_sorted(); + assert_eq!( + k, + vec![ + "$flagged".to_owned(), + "$important".to_owned(), + "$seen".to_owned() + ] + ); + } + + #[test] + fn duplicate_mailbox_token_dedups() { + let c = classify_str("Github,Github,Inbox"); + let count = c + .mailboxes + .iter() + .filter(|m| m.as_str() == "Github") + .count(); + assert_eq!(count, 1); + } +} diff --git a/src/sync/import_takeout/mail.rs b/src/sync/import_takeout/mail.rs new file mode 100644 index 0000000..e34321b --- /dev/null +++ b/src/sync/import_takeout/mail.rs @@ -0,0 +1,416 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; +use std::fs::File; +use std::io::BufReader; +use std::path::Path; +use std::time::{Duration, UNIX_EPOCH}; + +use rusqlite::{Connection, OptionalExtension, Transaction, params}; +use serde_json::Value; +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; + +use crate::db; +use crate::db::takeout_ids; +use crate::error::Error; +use crate::logging::{LEVEL_PROGRESS, Logger}; +use crate::sync::TypeCounts; +use crate::sync::emailmeta::email_meta_from_blob; +use crate::sync::keys::index_to_json; + +use super::labels::{self, MappingOptions}; +use super::mbox::{Message, MessageIterator}; + +pub struct InsertContext<'a> { + pub source_id: i64, + pub fallback_mailbox: &'a str, + pub options: MappingOptions, + pub mailbox_cache: &'a mut HashMap, +} + +pub fn process_file( + conn: &mut Connection, + file_path: &Path, + ctx: InsertContext<'_>, + mailbox_counts: &mut TypeCounts, + email_counts: &mut TypeCounts, + logger: Logger, +) -> Result<(), Error> { + let file = + File::open(file_path).map_err(|e| Error::Partial(format!("open {file_path:?}: {e}")))?; + let reader = BufReader::new(file); + let mut iter = MessageIterator::new(reader); + + let InsertContext { + source_id, + fallback_mailbox, + options, + mailbox_cache, + } = ctx; + let tx = conn.transaction()?; + let mut commit_pending: u64 = 0; + + loop { + match iter.next() { + None => break, + Some(Err(e)) => { + logger.warn(&format!("{file_path:?}: mbox io: {e}")); + email_counts.failed += 1; + break; + } + Some(Ok(msg)) => { + match process_message( + &tx, + source_id, + fallback_mailbox, + options, + &msg, + mailbox_cache, + mailbox_counts, + email_counts, + logger, + ) { + Ok(()) => { + commit_pending += 1; + if commit_pending.is_multiple_of(PROGRESS_TICK) + && logger.enabled(LEVEL_PROGRESS) + { + eprintln!("{file_path:?}: processed {commit_pending} messages"); + } + } + Err(e) => { + logger.warn(&format!("{file_path:?}: message: {e}")); + email_counts.failed += 1; + } + } + } + } + } + tx.commit()?; + Ok(()) +} + +const PROGRESS_TICK: u64 = 256; + +#[allow(clippy::too_many_arguments)] +fn process_message( + tx: &Transaction<'_>, + source_id: i64, + fallback_mailbox: &str, + options: MappingOptions, + msg: &Message, + cache: &mut HashMap, + mailbox_counts: &mut TypeCounts, + email_counts: &mut TypeCounts, + logger: Logger, +) -> Result<(), Error> { + let bytes = msg.contents(); + let msg_hash = blake3::hash(bytes).to_hex().to_string(); + + let raw_header = extract_x_gmail_labels(bytes); + let had_labels_header = raw_header.is_some(); + let tokens = match &raw_header { + Some(s) => labels::parse_header(s), + None => Vec::new(), + }; + let mut classification = labels::classify(&tokens); + if classification.mailboxes.is_empty() { + let fallback = fallback_mailbox.trim(); + if fallback.is_empty() { + classification.mailboxes.push("Imported".to_owned()); + } else { + classification.mailboxes.push(fallback.to_owned()); + } + if had_labels_header { + logger.warn(&format!( + "message {msg_hash}: X-Gmail-Labels present but empty after parse; \ + placing in fallback mailbox {:?}", + classification.mailboxes[0] + )); + } else { + logger.warn(&format!( + "message {msg_hash}: no X-Gmail-Labels header; placing in fallback mailbox {:?}", + classification.mailboxes[0] + )); + } + } + if classification.opened_won_over_unread { + logger.warn(&format!( + "message {msg_hash}: X-Gmail-Labels carried both 'Opened' and 'Unread'; \ + treating as $seen (Opened wins)" + )); + } + + let mut mailbox_ids: Vec = Vec::with_capacity(classification.mailboxes.len()); + for path in &classification.mailboxes { + let id = ensure_mailbox(tx, source_id, path, options, cache, mailbox_counts)?; + mailbox_ids.push(id); + } + mailbox_ids.sort_unstable(); + mailbox_ids.dedup(); + + let keywords_json = json_string_array(&classification.keywords_sorted()); + let mailbox_ids_json = json_int_array(&mailbox_ids); + + if let Some(local_id) = takeout_ids::local_for(tx, source_id, takeout_ids::EMAIL, &msg_hash)? { + let current: (String, String) = tx + .query_row( + "SELECT mailbox_ids, keywords FROM emails WHERE id = ?1", + params![local_id], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .optional()? + .unwrap_or_default(); + if current.0 != mailbox_ids_json || current.1 != keywords_json { + tx.execute( + "UPDATE emails SET mailbox_ids = ?1, keywords = ?2 WHERE id = ?3", + params![mailbox_ids_json, keywords_json, local_id], + )?; + email_counts.fetched += 1; + } else { + email_counts.skipped += 1; + } + return Ok(()); + } + + let blob_id = db::blobs::intern_blob(tx, bytes)?; + let (index, date_rfc3339) = email_meta_from_blob(bytes); + let message_match = index_to_json(&index); + let received_at = pick_received_at(msg.internal_date(), date_rfc3339.as_deref()); + + tx.execute( + "INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords, message_match) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + blob_id, + received_at, + mailbox_ids_json, + keywords_json, + message_match + ], + )?; + let local_id = tx.last_insert_rowid(); + takeout_ids::insert(tx, source_id, takeout_ids::EMAIL, &msg_hash, local_id)?; + email_counts.created += 1; + email_counts.fetched += 1; + Ok(()) +} + +fn ensure_mailbox( + tx: &Transaction<'_>, + source_id: i64, + path: &str, + options: MappingOptions, + cache: &mut HashMap, + counts: &mut TypeCounts, +) -> Result { + if let Some(&id) = cache.get(path) { + return Ok(id); + } + if let Some(id) = takeout_ids::local_for(tx, source_id, takeout_ids::MAILBOX, path)? { + cache.insert(path.to_owned(), id); + return Ok(id); + } + let (parent_id, leaf) = match path.rsplit_once('/') { + Some((parent, leaf)) => ( + Some(ensure_mailbox( + tx, source_id, parent, options, cache, counts, + )?), + leaf, + ), + None => (None, path), + }; + let role = labels::role_for_mailbox(path, options); + tx.execute( + "INSERT INTO mailboxes (name, parent_id, role, sort_order, is_subscribed) + VALUES (?1, ?2, ?3, 0, 1)", + params![leaf, parent_id, role], + )?; + let id = tx.last_insert_rowid(); + takeout_ids::insert(tx, source_id, takeout_ids::MAILBOX, path, id)?; + cache.insert(path.to_owned(), id); + counts.created += 1; + counts.fetched += 1; + Ok(id) +} + +fn json_string_array(values: &[String]) -> String { + Value::Array(values.iter().map(|s| Value::String(s.clone())).collect()).to_string() +} + +fn json_int_array(values: &[i64]) -> String { + Value::Array(values.iter().map(|&i| Value::from(i)).collect()).to_string() +} + +fn pick_received_at(internal_date: u64, date_rfc3339: Option<&str>) -> String { + if internal_date > 0 { + return format_unix_rfc3339(internal_date); + } + if let Some(d) = date_rfc3339 { + return d.to_owned(); + } + OffsetDateTime::now_utc() + .format(&Rfc3339) + .unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_owned()) +} + +fn format_unix_rfc3339(secs: u64) -> String { + let when = UNIX_EPOCH + Duration::from_secs(secs); + OffsetDateTime::from(when) + .format(&Rfc3339) + .unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_owned()) +} + +fn extract_x_gmail_labels(bytes: &[u8]) -> Option { + let mut values: Vec = Vec::new(); + let mut current: Option = None; + let mut i = 0; + while i < bytes.len() { + let nl = bytes[i..].iter().position(|&b| b == b'\n'); + let (line, advance) = match nl { + Some(p) => (&bytes[i..i + p], p + 1), + None => (&bytes[i..], bytes.len() - i), + }; + let line = line.strip_suffix(b"\r").unwrap_or(line); + if line.is_empty() { + break; + } + let is_continuation = line + .first() + .map(|b| *b == b' ' || *b == b'\t') + .unwrap_or(false); + if is_continuation { + if let Some(buf) = current.as_mut() { + let extra = std::str::from_utf8(line).unwrap_or("").trim_start(); + if !extra.is_empty() { + buf.push(' '); + buf.push_str(extra); + } + } + } else { + if let Some(buf) = current.take() { + values.push(buf); + } + if let Some(colon) = line.iter().position(|&b| b == b':') { + let name = &line[..colon]; + if name.eq_ignore_ascii_case(b"X-Gmail-Labels") { + let value = &line[colon + 1..]; + let value = std::str::from_utf8(value).unwrap_or("").trim(); + current = Some(value.to_owned()); + } + } + } + i += advance; + } + if let Some(buf) = current.take() { + values.push(buf); + } + if values.is_empty() { + None + } else { + Some(values.join(",")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extract_labels_from_simple_message() { + let bytes = b"From: a@b\r\n\ + X-Gmail-Labels: Inbox,Opened\r\n\ + Subject: hi\r\n\ + \r\n\ + body\r\n"; + assert_eq!( + extract_x_gmail_labels(bytes), + Some("Inbox,Opened".to_owned()) + ); + } + + #[test] + fn extract_labels_handles_lf_only_line_endings() { + let bytes = b"From: a@b\nX-Gmail-Labels: Inbox\n\nbody\n"; + assert_eq!(extract_x_gmail_labels(bytes), Some("Inbox".to_owned())); + } + + #[test] + fn extract_labels_handles_folded_continuation() { + let bytes = b"From: a@b\r\n\ + X-Gmail-Labels: Inbox,\r\n\ + \tImportant,Github\r\n\ + Subject: hi\r\n\ + \r\n\ + body\r\n"; + assert_eq!( + extract_x_gmail_labels(bytes), + Some("Inbox, Important,Github".to_owned()) + ); + } + + #[test] + fn extract_labels_returns_none_when_header_absent() { + let bytes = b"From: a@b\r\nSubject: hi\r\n\r\nbody\r\n"; + assert_eq!(extract_x_gmail_labels(bytes), None); + } + + #[test] + fn extract_labels_case_insensitive_name_match() { + let bytes = b"x-gmail-labels: Sent\r\n\r\n"; + assert_eq!(extract_x_gmail_labels(bytes), Some("Sent".to_owned())); + } + + #[test] + fn extract_labels_concatenates_multiple_header_occurrences() { + let bytes = b"X-Gmail-Labels: Inbox\r\nX-Gmail-Labels: Github\r\n\r\nbody"; + assert_eq!( + extract_x_gmail_labels(bytes), + Some("Inbox,Github".to_owned()) + ); + } + + #[test] + fn extract_labels_stops_at_blank_line() { + let bytes = b"X-Gmail-Labels: Inbox\r\n\r\nX-Gmail-Labels: NOT-A-HEADER\r\n"; + assert_eq!(extract_x_gmail_labels(bytes), Some("Inbox".to_owned())); + } + + #[test] + fn json_int_array_emits_compact_form() { + assert_eq!(json_int_array(&[1, 2, 3]), "[1,2,3]"); + assert_eq!(json_int_array(&[]), "[]"); + } + + #[test] + fn json_string_array_quotes_each_value() { + assert_eq!( + json_string_array(&["$seen".to_owned(), "$flagged".to_owned()]), + "[\"$seen\",\"$flagged\"]" + ); + } + + #[test] + fn pick_received_at_prefers_envelope_internal_date() { + let got = pick_received_at(1763065233, Some("2025-05-12T10:00:00+02:00")); + assert_eq!(got, "2025-11-13T20:20:33Z"); + } + + #[test] + fn pick_received_at_falls_back_to_date_header() { + let got = pick_received_at(0, Some("2025-05-12T10:00:00+02:00")); + assert_eq!(got, "2025-05-12T10:00:00+02:00"); + } + + #[test] + fn pick_received_at_last_resort_is_now_in_rfc3339() { + let got = pick_received_at(0, None); + assert!(got.ends_with("Z") || got.contains("+") || got.contains("-")); + assert!(got.contains("T")); + } +} diff --git a/src/sync/import_takeout/mbox.rs b/src/sync/import_takeout/mbox.rs new file mode 100644 index 0000000..33548e7 --- /dev/null +++ b/src/sync/import_takeout/mbox.rs @@ -0,0 +1,278 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::BufRead; + +use time::{Date, Month, OffsetDateTime, Time, UtcOffset}; + +pub struct MessageIterator { + reader: T, + message: Option, +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct Message { + internal_date: u64, + envelope_from: String, + contents: Vec, +} + +impl MessageIterator +where + T: BufRead, +{ + pub fn new(reader: T) -> MessageIterator { + MessageIterator { + reader, + message: None, + } + } +} + +impl Iterator for MessageIterator +where + T: BufRead, +{ + type Item = std::io::Result; + + fn next(&mut self) -> Option { + let mut line = Vec::with_capacity(80); + + loop { + match self.reader.read_until(b'\n', &mut line) { + Ok(0) => return self.message.take().map(Ok), + Ok(_) => {} + Err(e) => return Some(Err(e)), + } + + if line.starts_with(b"From ") { + let finished = self.message.take().map(Ok); + let header = std::str::from_utf8(&line).unwrap_or(""); + self.message = Some(Message::from_envelope(header)); + if finished.is_some() { + return finished; + } + line.clear(); + continue; + } + + if let Some(message) = &mut self.message { + if line.first() != Some(&b'>') { + message.contents.extend_from_slice(&line); + line.clear(); + continue; + } + let non_gt = line.iter().position(|&c| c != b'>').unwrap_or(line.len()); + if line[non_gt..].starts_with(b"From ") { + message.contents.extend_from_slice(&line[1..]); + } else { + message.contents.extend_from_slice(&line); + } + } + line.clear(); + } + } +} + +impl Message { + fn from_envelope(header: &str) -> Self { + let trimmed = header.strip_prefix("From ").unwrap_or(header); + let (envelope_from, date_str) = match trimmed.split_once(' ') { + Some((from, rest)) => (from.trim().to_owned(), rest), + None => (String::new(), ""), + }; + let internal_date = parse_envelope_date(date_str).unwrap_or(0); + Self { + internal_date, + envelope_from, + contents: Vec::with_capacity(1024), + } + } + + pub fn internal_date(&self) -> u64 { + self.internal_date + } + + pub fn envelope_from(&self) -> &str { + &self.envelope_from + } + + pub fn contents(&self) -> &[u8] { + &self.contents + } + + pub fn unwrap_contents(self) -> Vec { + self.contents + } +} + +fn parse_envelope_date(s: &str) -> Option { + let parts: Vec<&str> = s.split_whitespace().collect(); + let (month_pos, day_pos, time_pos, offset_pos, year_pos) = match parts.len() { + 5 => (1, 2, 3, None, 4), + 6 => (1, 2, 3, Some(4), 5), + _ => return None, + }; + + let month = parse_month(parts[month_pos])?; + let day: u8 = parts[day_pos].parse().ok()?; + let (hour, minute, second) = parse_hms(parts[time_pos])?; + let year: i32 = parts[year_pos].parse().ok()?; + let offset = match offset_pos { + Some(p) => parse_offset(parts[p])?, + None => UtcOffset::UTC, + }; + + let date = Date::from_calendar_date(year, month, day).ok()?; + let time = Time::from_hms(hour, minute, second).ok()?; + let ts = OffsetDateTime::new_in_offset(date, time, offset).unix_timestamp(); + u64::try_from(ts).ok() +} + +fn parse_month(s: &str) -> Option { + Some(match s { + x if x.eq_ignore_ascii_case("jan") => Month::January, + x if x.eq_ignore_ascii_case("feb") => Month::February, + x if x.eq_ignore_ascii_case("mar") => Month::March, + x if x.eq_ignore_ascii_case("apr") => Month::April, + x if x.eq_ignore_ascii_case("may") => Month::May, + x if x.eq_ignore_ascii_case("jun") => Month::June, + x if x.eq_ignore_ascii_case("jul") => Month::July, + x if x.eq_ignore_ascii_case("aug") => Month::August, + x if x.eq_ignore_ascii_case("sep") => Month::September, + x if x.eq_ignore_ascii_case("oct") => Month::October, + x if x.eq_ignore_ascii_case("nov") => Month::November, + x if x.eq_ignore_ascii_case("dec") => Month::December, + _ => return None, + }) +} + +fn parse_hms(s: &str) -> Option<(u8, u8, u8)> { + let mut it = s.split(':'); + let h: u8 = it.next()?.parse().ok()?; + let m: u8 = it.next()?.parse().ok()?; + let sec: u8 = it.next()?.parse().ok()?; + if it.next().is_some() { + return None; + } + Some((h, m, sec)) +} + +fn parse_offset(s: &str) -> Option { + let bytes = s.as_bytes(); + if bytes.len() != 5 { + return None; + } + let sign: i8 = match bytes[0] { + b'+' => 1, + b'-' => -1, + _ => return None, + }; + let hh: i8 = std::str::from_utf8(&bytes[1..3]).ok()?.parse().ok()?; + let mm: i8 = std::str::from_utf8(&bytes[3..5]).ok()?.parse().ok()?; + UtcOffset::from_hms(sign * hh, sign * mm, 0).ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classic_ctime_envelope_parses() { + let raw = b"From god@heaven.af.mil Sat Jan 3 01:05:34 1996\n\ + Message 1\n\ + \n\ + From cras@irccrew.org Tue Jul 23 19:39:23 2002\n\ + Message 2\n\ + \n"; + let mut it = MessageIterator::new(&raw[..]); + let m1 = it.next().unwrap().unwrap(); + assert_eq!(m1.envelope_from(), "god@heaven.af.mil"); + assert_eq!(m1.internal_date(), 820631134); + assert_eq!(m1.contents(), b"Message 1\n\n"); + let m2 = it.next().unwrap().unwrap(); + assert_eq!(m2.envelope_from(), "cras@irccrew.org"); + assert_eq!(m2.internal_date(), 1027453163); + assert!(it.next().is_none()); + } + + #[test] + fn mboxrd_unescape_recovers_lines() { + let raw = b"From a@b Tue Aug 6 13:34:34 2002\n\ + Message 3\n\ + >From hello\n\ + >>From world\n\ + >>>From test\n\ + \n"; + let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap(); + assert_eq!( + m.contents(), + b"Message 3\nFrom hello\n>From world\n>>From test\n\n" + ); + } + + #[test] + fn mboxrd_leaves_non_from_lines_starting_with_gt_alone() { + let raw = b"From a@b Mon Jan 15 15:30:00 2018\n\ + Message 4\n\ + > From\n\ + >F\n"; + let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap(); + assert_eq!(m.contents(), b"Message 4\n> From\n>F\n"); + } + + #[test] + fn takeout_envelope_with_utc_offset_parses() { + let raw = b"From 1848707889910060830@xxx Thu Nov 13 20:20:33 +0000 2025\n\ + X-GM-THRID: 1848707889910060830\n\ + X-Gmail-Labels: Inbox,Opened\n\ + Subject: hi\n\ + \n\ + body\n"; + let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap(); + assert_eq!(m.envelope_from(), "1848707889910060830@xxx"); + assert_eq!(m.internal_date(), 1763065233); + assert!(m.contents().starts_with(b"X-GM-THRID:")); + } + + #[test] + fn takeout_envelope_with_negative_offset_applies_offset() { + let raw = b"From x@y Thu Nov 13 12:20:33 -0800 2025\n\ + body\n"; + let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap(); + assert_eq!(m.internal_date(), 1763065233); + } + + #[test] + fn malformed_envelope_date_yields_zero_not_error() { + let raw = b"From who knows what\nbody\n"; + let m = MessageIterator::new(&raw[..]).next().unwrap().unwrap(); + assert_eq!(m.internal_date(), 0); + } + + #[test] + fn streams_multiple_takeout_format_messages() { + let raw = b"From 1@xxx Thu Nov 13 20:20:33 +0000 2025\n\ + X-GM-THRID: 1\n\ + X-Gmail-Labels: Inbox\n\ + Subject: A\n\ + \n\ + body a\n\ + From 2@xxx Thu Nov 13 21:00:00 +0000 2025\n\ + X-GM-THRID: 2\n\ + X-Gmail-Labels: Sent,Archived\n\ + Subject: B\n\ + \n\ + body b\n"; + let parsed: Vec<_> = MessageIterator::new(&raw[..]) + .collect::, _>>() + .unwrap(); + assert_eq!(parsed.len(), 2); + assert_eq!(parsed[0].envelope_from(), "1@xxx"); + assert_eq!(parsed[1].envelope_from(), "2@xxx"); + assert!(parsed[0].contents().starts_with(b"X-GM-THRID: 1")); + } +} diff --git a/src/sync/import_takeout/tree.rs b/src/sync/import_takeout/tree.rs new file mode 100644 index 0000000..fdba6b8 --- /dev/null +++ b/src/sync/import_takeout/tree.rs @@ -0,0 +1,210 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::BTreeSet; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PlannedMailbox { + pub canonical_path: String, + pub leaf: String, + pub parent_path: Option, + pub role: Option<&'static str>, + pub ephemeral: bool, +} + +pub fn assemble(input: &[String], role_for: F) -> Vec +where + F: Fn(&str) -> Option<&'static str>, +{ + let explicit: BTreeSet = input.iter().cloned().collect(); + let mut all_paths: BTreeSet = BTreeSet::new(); + for p in &explicit { + for prefix in prefixes(p) { + all_paths.insert(prefix); + } + } + + let mut planned: Vec = Vec::with_capacity(all_paths.len()); + for path in all_paths { + let (parent_path, leaf) = split_path(&path); + let ephemeral = !explicit.contains(&path); + let role = role_for(&path); + planned.push(PlannedMailbox { + canonical_path: path, + leaf, + parent_path, + role, + ephemeral, + }); + } + planned +} + +pub fn vanished_depth_sort(paths: &mut [String]) { + paths.sort_by(|a, b| depth_of(b).cmp(&depth_of(a)).then_with(|| a.cmp(b))); +} + +fn prefixes(path: &str) -> Vec { + let parts: Vec<&str> = path.split('/').collect(); + let mut out = Vec::with_capacity(parts.len()); + for i in 1..=parts.len() { + out.push(parts[..i].join("/")); + } + out +} + +fn split_path(path: &str) -> (Option, String) { + match path.rsplit_once('/') { + Some((parent, leaf)) => (Some(parent.to_owned()), leaf.to_owned()), + None => (None, path.to_owned()), + } +} + +fn depth_of(path: &str) -> usize { + path.bytes().filter(|b| *b == b'/').count() + 1 +} + +#[cfg(test)] +mod tests { + use super::*; + + fn no_roles(_: &str) -> Option<&'static str> { + None + } + + fn inbox_role(p: &str) -> Option<&'static str> { + if p == "Inbox" { Some("inbox") } else { None } + } + + #[test] + fn flat_paths_pass_through_unchanged() { + let r = assemble( + &["Inbox".to_owned(), "Sent".to_owned(), "Github".to_owned()], + no_roles, + ); + let names: Vec<&str> = r.iter().map(|m| m.canonical_path.as_str()).collect(); + assert_eq!(names, vec!["Github", "Inbox", "Sent"]); + assert!(r.iter().all(|m| !m.ephemeral)); + assert!(r.iter().all(|m| m.parent_path.is_none())); + } + + #[test] + fn nested_path_creates_ephemeral_parent() { + let r = assemble( + &["Label_001/Label_002_under_Label_001".to_owned()], + no_roles, + ); + assert_eq!(r.len(), 2); + assert_eq!(r[0].canonical_path, "Label_001"); + assert!(r[0].ephemeral); + assert_eq!(r[0].parent_path, None); + assert_eq!(r[1].canonical_path, "Label_001/Label_002_under_Label_001"); + assert!(!r[1].ephemeral); + assert_eq!(r[1].parent_path, Some("Label_001".to_owned())); + assert_eq!(r[1].leaf, "Label_002_under_Label_001"); + } + + #[test] + fn explicit_parent_is_not_marked_ephemeral() { + let r = assemble( + &[ + "Label_001".to_owned(), + "Label_001/Sub".to_owned(), + "Label_001/Other".to_owned(), + ], + no_roles, + ); + let parent = r.iter().find(|m| m.canonical_path == "Label_001").unwrap(); + assert!(!parent.ephemeral); + } + + #[test] + fn three_level_deep_path_creates_all_missing_ancestors() { + let r = assemble(&["A/B/C".to_owned()], no_roles); + let names: Vec<&str> = r.iter().map(|m| m.canonical_path.as_str()).collect(); + assert_eq!(names, vec!["A", "A/B", "A/B/C"]); + assert!(r[0].ephemeral); + assert!(r[1].ephemeral); + assert!(!r[2].ephemeral); + assert_eq!(r[2].parent_path, Some("A/B".to_owned())); + } + + #[test] + fn role_closure_consulted_for_ephemeral_parents_too() { + let r = assemble(&["Inbox/Sub".to_owned()], inbox_role); + let inbox = r.iter().find(|m| m.canonical_path == "Inbox").unwrap(); + assert!(inbox.ephemeral, "auto-created from the nested label"); + assert_eq!( + inbox.role, + Some("inbox"), + "an auto-created Inbox is still the inbox" + ); + } + + #[test] + fn role_closure_applies_to_explicit_top_level() { + let r = assemble(&["Inbox".to_owned()], inbox_role); + assert_eq!(r[0].role, Some("inbox")); + } + + #[test] + fn order_is_parents_before_children() { + let r = assemble( + &[ + "Zeta/Alpha".to_owned(), + "Alpha".to_owned(), + "Beta/Gamma/Delta".to_owned(), + ], + no_roles, + ); + let names: Vec<&str> = r.iter().map(|m| m.canonical_path.as_str()).collect(); + assert_eq!( + names, + vec![ + "Alpha", + "Beta", + "Beta/Gamma", + "Beta/Gamma/Delta", + "Zeta", + "Zeta/Alpha", + ] + ); + } + + #[test] + fn duplicate_inputs_dedup() { + let r = assemble( + &["Inbox".to_owned(), "Inbox".to_owned(), "Inbox".to_owned()], + no_roles, + ); + assert_eq!(r.len(), 1); + } + + #[test] + fn vanished_depth_sort_orders_leaves_first() { + let mut v = vec![ + "Foo".to_owned(), + "Foo/Bar/Baz".to_owned(), + "Other".to_owned(), + "Foo/Bar".to_owned(), + ]; + vanished_depth_sort(&mut v); + assert_eq!(v, vec!["Foo/Bar/Baz", "Foo/Bar", "Foo", "Other",]); + } + + #[test] + fn vanished_depth_sort_handles_ties_lexicographically() { + let mut v = vec!["B".to_owned(), "A".to_owned(), "C".to_owned()]; + vanished_depth_sort(&mut v); + assert_eq!(v, vec!["A", "B", "C"]); + } + + #[test] + fn empty_input_yields_empty_output() { + let r = assemble(&[], no_roles); + assert!(r.is_empty()); + } +} diff --git a/src/sync/import_takeout/walk.rs b/src/sync/import_takeout/walk.rs new file mode 100644 index 0000000..44fb4cb --- /dev/null +++ b/src/sync/import_takeout/walk.rs @@ -0,0 +1,267 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashSet; +use std::fs; +use std::io; +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; + +use crate::logging::Logger; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum FileKind { + Mbox, + Ics, + Vcf, +} + +impl FileKind { + fn from_extension(ext: &str) -> Option { + let lower = ext.to_ascii_lowercase(); + match lower.as_str() { + "mbox" => Some(FileKind::Mbox), + "ics" => Some(FileKind::Ics), + "vcf" => Some(FileKind::Vcf), + _ => None, + } + } +} + +#[derive(Debug, Clone)] +pub struct DiscoveredFile { + pub path: PathBuf, + pub kind: FileKind, +} + +#[derive(Debug, Default)] +pub struct WalkResult { + pub files: Vec, + pub symlink_cycles: u64, + pub io_failures: u64, +} + +impl WalkResult { + pub fn by_kind(&self, kind: FileKind) -> impl Iterator { + self.files.iter().filter(move |f| f.kind == kind) + } + + pub fn is_empty(&self) -> bool { + self.files.is_empty() + } +} + +pub fn walk(root: &Path) -> io::Result { + walk_with_logger(root, Logger::from_flags(true, 0)) +} + +pub fn walk_with_logger(root: &Path, logger: Logger) -> io::Result { + let mut result = WalkResult::default(); + let mut visited: HashSet<(u64, u64)> = HashSet::new(); + walk_dir(root, &mut result, &mut visited, logger)?; + result.files.sort_by(|a, b| a.path.cmp(&b.path)); + Ok(result) +} + +fn walk_dir( + dir: &Path, + result: &mut WalkResult, + visited: &mut HashSet<(u64, u64)>, + logger: Logger, +) -> io::Result<()> { + let metadata = fs::metadata(dir)?; + let key = (metadata.dev(), metadata.ino()); + if !visited.insert(key) { + result.symlink_cycles += 1; + logger.warn(&format!( + "takeout walk: symlink cycle broken at {dir:?} (target already visited)" + )); + return Ok(()); + } + + let entries = match fs::read_dir(dir) { + Ok(it) => it, + Err(e) => { + result.io_failures += 1; + logger.warn(&format!("takeout walk: read_dir {dir:?}: {e}")); + return Ok(()); + } + }; + + for entry in entries { + let entry = match entry { + Ok(e) => e, + Err(e) => { + result.io_failures += 1; + logger.warn(&format!("takeout walk: entry in {dir:?}: {e}")); + continue; + } + }; + let path = entry.path(); + let file_type = match entry.file_type() { + Ok(t) => t, + Err(e) => { + result.io_failures += 1; + logger.warn(&format!("takeout walk: file_type {path:?}: {e}")); + continue; + } + }; + if file_type.is_dir() { + walk_dir(&path, result, visited, logger)?; + continue; + } + if file_type.is_symlink() { + let target_meta = match fs::metadata(&path) { + Ok(m) => m, + Err(e) => { + result.io_failures += 1; + logger.warn(&format!("takeout walk: symlink target {path:?}: {e}")); + continue; + } + }; + if target_meta.is_dir() { + walk_dir(&path, result, visited, logger)?; + continue; + } + if !target_meta.is_file() { + continue; + } + } else if !file_type.is_file() { + continue; + } + if let Some(kind) = classify(&path) { + result.files.push(DiscoveredFile { path, kind }); + } + } + Ok(()) +} + +fn classify(path: &Path) -> Option { + let ext = path.extension()?.to_str()?; + FileKind::from_extension(ext) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use std::os::unix::fs::symlink; + use tempfile::TempDir; + + fn touch(path: &Path) { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).unwrap(); + } + fs::write(path, b"").unwrap(); + } + + #[test] + fn collects_only_matching_extensions() { + let td = TempDir::new().unwrap(); + let root = td.path(); + touch(&root.join("Mail/All.mbox")); + touch(&root.join("Calendar/cal.ics")); + touch(&root.join("Contacts/c.vcf")); + touch(&root.join("Calendar/meet_settings.json")); + touch(&root.join("archive_browser.html")); + touch(&root.join("Contacts/photo.jpg")); + let r = walk(root).unwrap(); + assert_eq!(r.files.len(), 3); + let mut kinds: Vec = r.files.iter().map(|f| f.kind).collect(); + kinds.sort_by_key(|k| format!("{k:?}")); + assert_eq!(kinds, vec![FileKind::Ics, FileKind::Mbox, FileKind::Vcf]); + } + + #[test] + fn extension_match_is_case_insensitive() { + let td = TempDir::new().unwrap(); + let root = td.path(); + touch(&root.join("Inbox.MBOX")); + touch(&root.join("Cal.Ics")); + touch(&root.join("All.Vcf")); + let r = walk(root).unwrap(); + assert_eq!(r.files.len(), 3); + } + + #[test] + fn recurses_into_nested_directories() { + let td = TempDir::new().unwrap(); + let root = td.path(); + touch(&root.join("Takeout/Mail/a.mbox")); + touch(&root.join("Takeout/Calendar/deep/b.ics")); + touch(&root.join("Takeout/Contacts/All Contacts/c.vcf")); + let r = walk(root).unwrap(); + assert_eq!(r.files.len(), 3); + } + + #[test] + fn empty_or_non_matching_tree_returns_empty() { + let td = TempDir::new().unwrap(); + let root = td.path(); + touch(&root.join("readme.txt")); + touch(&root.join("nested/photo.jpg")); + let r = walk(root).unwrap(); + assert!(r.is_empty()); + } + + #[test] + fn results_are_sorted_for_determinism() { + let td = TempDir::new().unwrap(); + let root = td.path(); + touch(&root.join("z.mbox")); + touch(&root.join("a.mbox")); + touch(&root.join("m.ics")); + let r = walk(root).unwrap(); + let names: Vec<&str> = r + .files + .iter() + .map(|f| f.path.file_name().unwrap().to_str().unwrap()) + .collect(); + assert_eq!(names, vec!["a.mbox", "m.ics", "z.mbox"]); + } + + #[test] + fn by_kind_filters_returned_iterator() { + let td = TempDir::new().unwrap(); + let root = td.path(); + touch(&root.join("a.mbox")); + touch(&root.join("b.mbox")); + touch(&root.join("c.ics")); + let r = walk(root).unwrap(); + assert_eq!(r.by_kind(FileKind::Mbox).count(), 2); + assert_eq!(r.by_kind(FileKind::Ics).count(), 1); + assert_eq!(r.by_kind(FileKind::Vcf).count(), 0); + } + + #[test] + fn symlink_to_directory_is_followed_once_not_infinitely() { + let td = TempDir::new().unwrap(); + let root = td.path(); + touch(&root.join("real/a.mbox")); + symlink(root.join("real"), root.join("link-to-real")).unwrap(); + let r = walk(root).unwrap(); + assert_eq!(r.files.len(), 1); + } + + #[test] + fn symlink_cycle_is_broken() { + let td = TempDir::new().unwrap(); + let root = td.path(); + fs::create_dir_all(root.join("a/b")).unwrap(); + symlink(root.join("a"), root.join("a/b/loop")).unwrap(); + let r = walk(root).unwrap(); + assert!(r.symlink_cycles >= 1, "cycle counter incremented"); + } + + #[test] + fn dotted_directory_basenames_traverse_fine() { + let td = TempDir::new().unwrap(); + let root = td.path(); + touch(&root.join(".hidden/m.mbox")); + let r = walk(root).unwrap(); + assert_eq!(r.files.len(), 1); + } +} diff --git a/src/sync/keys.rs b/src/sync/keys.rs new file mode 100644 index 0000000..046f667 --- /dev/null +++ b/src/sync/keys.rs @@ -0,0 +1,269 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::{Value, json}; + +pub fn normalize_message_id(raw: &str) -> String { + let t = raw.trim(); + let t = t.strip_prefix('<').unwrap_or(t); + let t = t.strip_suffix('>').unwrap_or(t); + t.trim().to_lowercase() +} + +pub fn fold_name(name: &str) -> String { + name.trim().to_lowercase() +} + +pub fn blake3_fields(fields: &[&str]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + for (i, f) in fields.iter().enumerate() { + if i > 0 { + hasher.update(b"\x1f"); + } + hasher.update(f.as_bytes()); + } + *hasher.finalize().as_bytes() +} + +pub fn blake3_bytes(bytes: &[u8]) -> [u8; 32] { + *blake3::hash(bytes).as_bytes() +} + +pub fn identity_key(name: &str, email: &str) -> [u8; 32] { + blake3_fields(&[name, &email.to_lowercase()]) +} + +pub fn participant_identity_key(calendar_address: &str, name: &str) -> [u8; 32] { + blake3_fields(&[&calendar_address.to_lowercase(), name]) +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum EmailKey { + MessageId(String), + Fallback([u8; 32]), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EmailIndex { + + pub mids: Vec, + + pub fb: [u8; 32], +} + +fn join_addrs(addrs: &[String]) -> String { + addrs + .iter() + .map(|a| a.to_lowercase()) + .collect::>() + .join(",") +} + +pub fn email_index( + message_ids: &[String], + from: &[String], + subject: &str, + sent_at: &str, + to: &[String], +) -> EmailIndex { + let mut mids: Vec = message_ids + .iter() + .map(|m| normalize_message_id(m)) + .filter(|s| !s.is_empty()) + .collect(); + mids.sort(); + mids.dedup(); + let fb = blake3_fields(&[&join_addrs(from), subject, sent_at, &join_addrs(to)]); + EmailIndex { mids, fb } +} + +pub fn index_to_json(idx: &EmailIndex) -> String { + json!({ "m": idx.mids, "f": hex(&idx.fb) }).to_string() +} + +pub fn index_from_json(text: &str) -> EmailIndex { + let v: Value = serde_json::from_str(text).unwrap_or(Value::Null); + let mids = v + .get("m") + .and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(|x| x.as_str().map(str::to_owned)) + .collect() + }) + .unwrap_or_default(); + let fb = v + .get("f") + .and_then(Value::as_str) + .and_then(unhex) + .unwrap_or([0u8; 32]); + EmailIndex { mids, fb } +} + +fn hex(bytes: &[u8; 32]) -> String { + let mut s = String::with_capacity(64); + for b in bytes { + use std::fmt::Write; + let _ = write!(s, "{b:02x}"); + } + s +} + +fn unhex(s: &str) -> Option<[u8; 32]> { + if s.len() != 64 { + return None; + } + let mut out = [0u8; 32]; + for (i, slot) in out.iter_mut().enumerate() { + *slot = u8::from_str_radix(&s[i * 2..i * 2 + 2], 16).ok()?; + } + Some(out) +} + +pub fn email_key(idx: &EmailIndex) -> EmailKey { + if idx.mids.is_empty() { + EmailKey::Fallback(idx.fb) + } else { + EmailKey::MessageId(idx.mids.join("\x1f")) + } +} + +pub fn email_keys(indices: &[EmailIndex]) -> Vec { + indices.iter().map(email_key).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn message_id_normalization_strips_brackets_and_lowercases() { + assert_eq!(normalize_message_id(" "), "abc@host.com"); + assert_eq!(normalize_message_id("plain@x"), "plain@x"); + } + + #[test] + fn name_fold_is_case_insensitive() { + assert_eq!(fold_name(" Sales "), fold_name("sales")); + assert_ne!(fold_name("Sales"), fold_name("sale")); + } + + #[test] + fn identity_collapse_ignores_signature() { + assert_eq!( + identity_key("Alice", "A@x.test"), + identity_key("Alice", "a@x.test") + ); + } + + #[test] + fn participant_identity_key_lowercases_address_only() { + let a = participant_identity_key("MailTo:User@Host", "Bob"); + let b = participant_identity_key("mailto:user@host", "Bob"); + assert_eq!(a, b); + assert_ne!(a, participant_identity_key("mailto:user@host", "bob")); + } + + fn idx(mids: &[&str], from: &[&str], subj: &str, sent: &str, to: &[&str]) -> EmailIndex { + let m: Vec = mids.iter().map(|s| (*s).to_owned()).collect(); + let f: Vec = from.iter().map(|s| (*s).to_owned()).collect(); + let t: Vec = to.iter().map(|s| (*s).to_owned()).collect(); + email_index(&m, &f, subj, sent, &t) + } + + #[test] + fn message_id_array_is_normalized_sorted_and_order_independent() { + let a = idx(&["", ""], &[], "", "", &[]); + let b = idx(&["", ""], &[], "", "", &[]); + assert_eq!(a.mids, vec!["a@h".to_owned(), "b@h".to_owned()]); + assert_eq!(a.mids, b.mids); + } + + #[test] + fn unique_single_message_id_uses_message_id_key() { + let keys = email_keys(&[ + idx( + &[""], + &["a@x"], + "Hi", + "2020-01-01T00:00:00Z", + &["b@y"], + ), + idx( + &[""], + &["a@x"], + "Yo", + "2020-01-02T00:00:00Z", + &["b@y"], + ), + ]); + assert_eq!(keys[0], EmailKey::MessageId("m1@h".to_owned())); + assert_eq!(keys[1], EmailKey::MessageId("m2@h".to_owned())); + } + + #[test] + fn shared_message_id_matches_regardless_of_other_fields() { + let keys = email_keys(&[ + idx( + &[""], + &["a@x"], + "S", + "2020-01-01T00:00:00Z", + &["b@y"], + ), + idx( + &[""], + &["c@x"], + "T", + "2020-01-02T00:00:00Z", + &["d@y"], + ), + ]); + assert_eq!(keys[0], EmailKey::MessageId("dup@h".to_owned())); + assert_eq!(keys[0], keys[1], "same Message-ID is the same message"); + } + + #[test] + fn only_absent_message_id_falls_back_multi_is_still_message_id() { + let keys = email_keys(&[ + idx(&[], &["a@x"], "S", "2020-01-01T00:00:00Z", &["b@y"]), + idx( + &["", ""], + &["a@x"], + "S", + "2020-01-01T00:00:00Z", + &["b@y"], + ), + ]); + assert!(matches!(keys[0], EmailKey::Fallback(_))); + assert_eq!(keys[1], EmailKey::MessageId("a@h\u{1f}b@h".to_owned())); + } + + #[test] + fn fallback_is_case_insensitive_on_addresses_and_offset_preserving_on_sent_at() { + let a = idx(&[], &["A@X"], "Subj", "2020-01-01T10:00:00+02:00", &["B@Y"]); + let b = idx(&[], &["a@x"], "Subj", "2020-01-01T10:00:00+02:00", &["b@y"]); + let c = idx(&[], &["a@x"], "Subj", "2020-01-01T08:00:00Z", &["b@y"]); + assert_eq!(a.fb, b.fb); + assert_ne!(a.fb, c.fb); + } + + #[test] + fn index_json_roundtrips() { + let i = idx(&[""], &["a@x"], "S", "2020-01-01T00:00:00Z", &["b@y"]); + let j = index_to_json(&i); + let back = index_from_json(&j); + assert_eq!(i, back); + assert_eq!(email_keys(&[i]), email_keys(&[back])); + } + + #[test] + fn malformed_stored_json_degrades_to_empty_fallback() { + let back = index_from_json("{}"); + assert!(back.mids.is_empty()); + assert_eq!(back.fb, [0u8; 32]); + } +} diff --git a/src/sync/mod.rs b/src/sync/mod.rs new file mode 100644 index 0000000..1efbb14 --- /dev/null +++ b/src/sync/mod.rs @@ -0,0 +1,120 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +pub mod emailmeta; +pub mod export; +pub mod import_dav; +pub mod import_exchange_ews; +pub mod import_exchange_graph; +pub mod import_imap; +pub mod import_jmap; +pub mod import_maildir; +pub mod import_managesieve; +pub mod import_takeout; +pub mod keys; +pub mod prune; + +use std::path::PathBuf; + +use rusqlite::Connection; + +use crate::db; +use crate::error::Error; + +pub(crate) fn table_name(ty: ObjectType) -> &'static str { + match ty { + ObjectType::Mailbox => "mailboxes", + ObjectType::Email => "emails", + ObjectType::Identity => "identities", + ObjectType::SieveScript => "sieve_scripts", + ObjectType::AddressBook => "address_books", + ObjectType::ContactCard => "contact_cards", + ObjectType::Calendar => "calendars", + ObjectType::CalendarEvent => "calendar_events", + ObjectType::ParticipantIdentity => "participant_identities", + ObjectType::FileNode => "file_nodes", + } +} +use crate::jmap::account::AccountSelector; +use crate::jmap::http::{Auth, HttpClient, RetryPolicy}; +use crate::logging::Logger; +use crate::types::ObjectType; + +pub struct CommonConfig { + pub archive: PathBuf, + pub threads: usize, + pub dry_run: bool, + pub max_retries: u32, + pub allow_invalid_certs: bool, + pub logger: Logger, +} + +pub struct ConnectConfig { + pub url: String, + pub auth: Auth, + pub account: AccountSelector, +} + +pub struct ImportConfig { + pub connect: ConnectConfig, + pub objects: Option>, + pub allow_source_change: bool, +} + +pub struct ExportConfig { + pub connect: ConnectConfig, + pub objects: Option>, + pub prune: bool, + pub yes: bool, +} + +#[derive(Debug, Default, Clone)] +pub struct TypeCounts { + pub created: u64, + pub fetched: u64, + pub deleted: u64, + pub skipped: u64, + pub failed: u64, +} + +#[derive(Debug, Default, Clone)] +pub struct Summary { + pub per_type: Vec<(&'static str, TypeCounts)>, + pub retries_observed: u64, + pub retry_after_sleeps: u64, +} + +impl Summary { + pub fn any_failed(&self) -> bool { + self.per_type.iter().any(|(_, c)| c.failed > 0) + } +} + +pub struct Context { + pub conn: Connection, + pub client: HttpClient, + pub common: CommonConfig, +} + +impl Context { + pub fn open(common: CommonConfig, connect: &ConnectConfig) -> Result { + let conn = db::init::open(&common.archive)?; + let client = HttpClient::new( + connect.auth.clone(), + RetryPolicy::new(common.max_retries), + common.allow_invalid_certs, + ); + Ok(Context { + conn, + client, + common, + }) + } + + pub fn dry_run(&self) -> bool { + self.common.dry_run + } +} diff --git a/src/sync/prune.rs b/src/sync/prune.rs new file mode 100644 index 0000000..61eedaf --- /dev/null +++ b/src/sync/prune.rs @@ -0,0 +1,124 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::{HashMap, HashSet}; + +#[derive(Debug, Clone)] +pub struct TargetObj { + pub id: String, + pub matched: bool, + pub protected: bool, + pub may_delete: bool, + pub parent: Option, +} + +fn protected_with_ancestors(objs: &[TargetObj]) -> HashSet { + let by_id: HashMap<&str, &TargetObj> = objs.iter().map(|o| (o.id.as_str(), o)).collect(); + let mut protected: HashSet = HashSet::new(); + for o in objs { + if o.protected { + let mut cur = Some(o.id.clone()); + while let Some(id) = cur { + if !protected.insert(id.clone()) { + break; + } + cur = by_id + .get(id.as_str()) + .and_then(|x| x.parent.clone()) + .filter(|p| by_id.contains_key(p.as_str())); + } + } + } + protected +} + +fn depth(id: &str, by_id: &HashMap<&str, &TargetObj>) -> usize { + let mut d = 0; + let mut cur = by_id.get(id).and_then(|o| o.parent.clone()); + let mut seen: HashSet = HashSet::new(); + while let Some(p) = cur { + if !seen.insert(p.clone()) || !by_id.contains_key(p.as_str()) { + break; + } + d += 1; + cur = by_id.get(p.as_str()).and_then(|o| o.parent.clone()); + } + d +} + +pub fn candidates(objs: &[TargetObj], tree: bool) -> Vec { + let protected = protected_with_ancestors(objs); + let by_id: HashMap<&str, &TargetObj> = objs.iter().map(|o| (o.id.as_str(), o)).collect(); + let mut out: Vec<&TargetObj> = objs + .iter() + .filter(|o| !o.matched && o.may_delete && !protected.contains(&o.id)) + .collect(); + if tree { + out.sort_by(|a, b| { + depth(&b.id, &by_id) + .cmp(&depth(&a.id, &by_id)) + .then_with(|| a.id.cmp(&b.id)) + }); + } else { + out.sort_by(|a, b| a.id.cmp(&b.id)); + } + out.into_iter().map(|o| o.id.clone()).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn o(id: &str, matched: bool, protected: bool, parent: Option<&str>) -> TargetObj { + TargetObj { + id: id.to_owned(), + matched, + protected, + may_delete: true, + parent: parent.map(str::to_owned), + } + } + + #[test] + fn unmatched_unprotected_are_candidates() { + let objs = vec![ + o("a", true, false, None), + o("b", false, false, None), + o("c", false, true, None), + ]; + assert_eq!(candidates(&objs, false), vec!["b".to_owned()]); + } + + #[test] + fn protected_excludes_ancestors() { + let objs = vec![ + o("root", false, false, None), + o("mid", false, false, Some("root")), + o("leaf", false, true, Some("mid")), + ]; + assert!(candidates(&objs, true).is_empty()); + } + + #[test] + fn may_delete_false_is_skipped() { + let mut x = o("x", false, false, None); + x.may_delete = false; + assert!(candidates(&[x], false).is_empty()); + } + + #[test] + fn tree_destroy_is_leaf_first() { + let objs = vec![ + o("root", false, false, None), + o("child", false, false, Some("root")), + o("grand", false, false, Some("child")), + ]; + assert_eq!( + candidates(&objs, true), + vec!["grand".to_owned(), "child".to_owned(), "root".to_owned()] + ); + } +} diff --git a/src/types.rs b/src/types.rs new file mode 100644 index 0000000..66649e1 --- /dev/null +++ b/src/types.rs @@ -0,0 +1,142 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use crate::error::Error; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ObjectType { + Mailbox, + Email, + Identity, + SieveScript, + AddressBook, + ContactCard, + Calendar, + CalendarEvent, + ParticipantIdentity, + FileNode, +} + +impl ObjectType { + pub const ALL: [ObjectType; 10] = [ + ObjectType::Mailbox, + ObjectType::Email, + ObjectType::Identity, + ObjectType::SieveScript, + ObjectType::AddressBook, + ObjectType::ContactCard, + ObjectType::Calendar, + ObjectType::CalendarEvent, + ObjectType::ParticipantIdentity, + ObjectType::FileNode, + ]; + + pub fn token(self) -> &'static str { + match self { + ObjectType::Mailbox => "mailbox", + ObjectType::Email => "email", + ObjectType::Identity => "identity", + ObjectType::SieveScript => "sievescript", + ObjectType::AddressBook => "addressbook", + ObjectType::ContactCard => "contactcard", + ObjectType::Calendar => "calendar", + ObjectType::CalendarEvent => "calendarevent", + ObjectType::ParticipantIdentity => "participantidentity", + ObjectType::FileNode => "filenode", + } + } + + pub fn jmap_name(self) -> &'static str { + match self { + ObjectType::Mailbox => "Mailbox", + ObjectType::Email => "Email", + ObjectType::Identity => "Identity", + ObjectType::SieveScript => "SieveScript", + ObjectType::AddressBook => "AddressBook", + ObjectType::ContactCard => "ContactCard", + ObjectType::Calendar => "Calendar", + ObjectType::CalendarEvent => "CalendarEvent", + ObjectType::ParticipantIdentity => "ParticipantIdentity", + ObjectType::FileNode => "FileNode", + } + } + + pub fn capability_urn(self) -> &'static str { + match self { + ObjectType::Mailbox | ObjectType::Email | ObjectType::Identity => { + "urn:ietf:params:jmap:mail" + } + ObjectType::SieveScript => "urn:ietf:params:jmap:sieve", + ObjectType::AddressBook | ObjectType::ContactCard => "urn:ietf:params:jmap:contacts", + ObjectType::Calendar | ObjectType::CalendarEvent | ObjectType::ParticipantIdentity => { + "urn:ietf:params:jmap:calendars" + } + ObjectType::FileNode => "urn:ietf:params:jmap:filenode", + } + } + + pub fn parse(token: &str) -> Result { + let normalized = token.trim().to_ascii_lowercase(); + ObjectType::ALL + .into_iter() + .find(|t| t.token() == normalized) + .ok_or_else(|| Error::Usage(format!("unknown object type: {token}"))) + } +} + +pub fn parse_object_list(list: &str) -> Result, Error> { + let mut out = Vec::new(); + for token in list.split(',') { + let token = token.trim(); + if token.is_empty() { + continue; + } + let parsed = ObjectType::parse(token)?; + if !out.contains(&parsed) { + out.push(parsed); + } + } + if out.is_empty() { + return Err(Error::Usage( + "--objects given but resolved to an empty type list".to_owned(), + )); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn token_roundtrip() { + for t in ObjectType::ALL { + assert_eq!(ObjectType::parse(t.token()).unwrap(), t); + } + } + + #[test] + fn parse_list_dedups_and_trims() { + let parsed = parse_object_list(" mailbox, email ,mailbox").unwrap(); + assert_eq!(parsed, vec![ObjectType::Mailbox, ObjectType::Email]); + } + + #[test] + fn parse_list_case_insensitive() { + let parsed = parse_object_list("Mailbox,EMAIL").unwrap(); + assert_eq!(parsed, vec![ObjectType::Mailbox, ObjectType::Email]); + } + + #[test] + fn parse_list_rejects_unknown() { + assert!(parse_object_list("mailbox,bogus").is_err()); + } + + #[test] + fn parse_list_rejects_empty() { + assert!(parse_object_list(" , ").is_err()); + } +} diff --git a/tests/integration/baikal.rs b/tests/integration/baikal.rs new file mode 100644 index 0000000..e200421 --- /dev/null +++ b/tests/integration/baikal.rs @@ -0,0 +1,463 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashSet; +use std::time::Duration; + +use rusqlite::params; +use testcontainers::core::{IntoContainerPort, WaitFor}; +use testcontainers::runners::{SyncBuilder, SyncRunner}; +use testcontainers::{Container, GenericBuildableImage, GenericImage, ImageExt}; + +use super::data::{RawFixture, load_icals, load_vcards, malformed_ical, rewrite_uid}; +use super::dav_client::DavSeed; +use super::error::{ContainerError, ContainerResult}; +use super::layouts::{self, Layout}; +use super::{Account, Endpoint}; + +const BASE_IMAGE: &str = "ckulka/baikal:0.10.1-nginx"; +const HTTP_PORT: u16 = 80; +const REALM: &str = "BaikalDAV"; + +const ADMIN_PASSWORD_HASH: &str = "004b53ae84286ba6003b34e8ef404826"; +const DIGEST_USER1: &str = "b7a47e657a4a966d0d844b1211416024"; +const DIGEST_USER2: &str = "9fc67834c5a099f0a2eccea3f6918169"; +const DIGEST_USER3: &str = "e1b30b108dad01b323a999b0bb1252ab"; + +const SCHEMA: &str = include_str!("baikal_schema.sql"); + +const BAIKAL_YAML: &str = "system: + configured_version: '0.10.1' + timezone: 'UTC' + card_enabled: true + cal_enabled: true + invite_from: 'noreply@vandelay.test' + dav_auth_type: 'Basic' + admin_passwordhash: '004b53ae84286ba6003b34e8ef404826' + failed_access_message: 'user %u authentication failure for Baikal' + auth_realm: 'BaikalDAV' + base_uri: '' +database: + backend: 'sqlite' + sqlite_file: '/var/www/baikal/Specific/db/db.sqlite' +"; + +const DOCKERFILE: &str = r#"FROM ckulka/baikal:0.10.1-nginx + +COPY baikal.yaml /var/www/baikal/config/baikal.yaml +COPY db.sqlite /var/www/baikal/Specific/db/db.sqlite +RUN touch /var/www/baikal/Specific/INSTALL_DISABLED && \ + chown -R www-data:www-data /var/www/baikal/config /var/www/baikal/Specific && \ + chmod 0640 /var/www/baikal/config/baikal.yaml /var/www/baikal/Specific/db/db.sqlite +"#; + +pub struct Baikal { + container: Container, + pub http: Endpoint, + pub accounts: Vec, +} + +impl Baikal { + pub fn start() -> ContainerResult { + let db_bytes = build_sqlite_db()?; + + let image: GenericImage = GenericBuildableImage::new("vandelay-baikal", "test") + .with_dockerfile_string(DOCKERFILE.to_owned()) + .with_data(BAIKAL_YAML.as_bytes().to_vec(), "baikal.yaml") + .with_data(db_bytes, "db.sqlite") + .build_image() + .map_err(|e| ContainerError::Seed(format!("baikal build: {e}")))?; + + let _ = BASE_IMAGE; + + let request = image + .with_exposed_port(HTTP_PORT.tcp()) + .with_wait_for(WaitFor::message_on_stderr("start worker process")) + .with_startup_timeout(Duration::from_secs(180)); + + let container = request.start()?; + let host = container.get_host()?.to_string(); + let http = Endpoint::new(host, container.get_host_port_ipv4(HTTP_PORT.tcp())?); + + let accounts: Vec = layouts::accounts() + .iter() + .map(|name| Account { + username: (*name).to_owned(), + password: layouts::PASSWORD.to_owned(), + layout: layouts::layout_for(name), + }) + .collect(); + + Ok(Self { + container, + http, + accounts, + }) + } + + pub fn base_url(&self) -> String { + self.http.http_base() + } + + pub fn dav_root(&self) -> String { + format!("{}/dav.php", self.base_url()) + } + + pub fn calendar_home(&self, account: &Account) -> String { + format!("{}/calendars/{}/", self.dav_root(), account.username) + } + + pub fn addressbook_home(&self, account: &Account) -> String { + format!("{}/addressbooks/{}/", self.dav_root(), account.username) + } + + pub fn seed_all(&self) -> ContainerResult> { + let icals = load_icals()?; + let vcards = load_vcards()?; + let mut out = Vec::new(); + for acct in &self.accounts { + let seed = self.seed_account(acct, &icals, &vcards)?; + out.push(seed); + } + Ok(out) + } + + pub fn seed_account( + &self, + account: &Account, + icals: &[RawFixture], + vcards: &[RawFixture], + ) -> ContainerResult { + let client = DavSeed::new(self.dav_root(), &account.username, &account.password); + let layout = &account.layout; + let mut seed = AccountSeed::new(account.username.clone()); + + for (idx, cal) in layout.calendars.iter().enumerate() { + let segment = collection_segment("cal", idx, cal); + let path = format!("/calendars/{}/{}/", account.username, segment); + let mkcal = format!( + r#" + + + {cal} + + + + + +"# + ); + client.mkcalendar(&path, Some(&mkcal))?; + let mut plan = CollectionPlan::new((*cal).to_owned(), path.clone()); + + let count = events_per_calendar(layout, idx); + let offset = events_offset(layout, idx); + for (i, fx) in icals.iter().cycle().skip(offset).take(count).enumerate() { + let item = format!("{path}{}-{}.ics", fx.name, i); + let suffix = format!("{segment}-{i}"); + let (bytes, uid) = rewrite_uid(&fx.bytes, &suffix).ok_or_else(|| { + ContainerError::Seed(format!("ical fixture {} has no UID", fx.name)) + })?; + client.put(&item, "text/calendar; charset=utf-8", &bytes)?; + plan.items.push(SeededItem { + uid, + href: item, + source: bytes, + }); + } + seed.calendars.push(plan); + } + + for (idx, ab) in layout.address_books.iter().enumerate() { + let segment = collection_segment("ab", idx, ab); + let path = format!("/addressbooks/{}/{}/", account.username, segment); + let mkbook = format!( + r#" + + + + {ab} + +"# + ); + client.mkcol(&path, Some(&mkbook))?; + let mut plan = CollectionPlan::new((*ab).to_owned(), path.clone()); + + let count = contacts_per_book(layout, idx); + let offset = contacts_offset(layout, idx); + for (i, fx) in vcards.iter().cycle().skip(offset).take(count).enumerate() { + let item = format!("{path}{}-{}.vcf", fx.name, i); + let suffix = format!("{segment}-{i}"); + let (bytes, uid) = rewrite_uid(&fx.bytes, &suffix).ok_or_else(|| { + ContainerError::Seed(format!("vcard fixture {} has no UID", fx.name)) + })?; + client.put(&item, "text/vcard; charset=utf-8", &bytes)?; + plan.items.push(SeededItem { + uid, + href: item, + source: bytes, + }); + } + seed.address_books.push(plan); + } + + Ok(seed) + } + + pub fn seed_malformed_event( + &self, + account: &Account, + calendar_segment_idx: usize, + tag: &str, + ) -> ContainerResult { + let client = DavSeed::new(self.dav_root(), &account.username, &account.password); + let segment = collection_segment( + "cal", + calendar_segment_idx, + account.layout.calendars[calendar_segment_idx], + ); + let path = format!( + "/calendars/{}/{}/broken-{tag}.ics", + account.username, segment + ); + let bad = malformed_ical(tag); + client.put(&path, "text/calendar; charset=utf-8", &bad.bytes)?; + Ok(path) + } + + pub fn delete_item(&self, account: &Account, path: &str) -> ContainerResult<()> { + let client = DavSeed::new(self.dav_root(), &account.username, &account.password); + client.delete(path)?; + Ok(()) + } + + pub fn add_event( + &self, + account: &Account, + calendar_segment_idx: usize, + tag: &str, + icals: &[RawFixture], + ) -> ContainerResult<(String, String)> { + let client = DavSeed::new(self.dav_root(), &account.username, &account.password); + let display = account.layout.calendars[calendar_segment_idx]; + let segment = collection_segment("cal", calendar_segment_idx, display); + let fixture = icals.first().ok_or_else(|| { + ContainerError::Seed("no ical fixture available for add_event".to_owned()) + })?; + let suffix = format!("added-{tag}"); + let (bytes, uid) = rewrite_uid(&fixture.bytes, &suffix) + .ok_or_else(|| ContainerError::Seed("ical fixture has no UID".to_owned()))?; + let path = format!("/calendars/{}/{segment}/added-{tag}.ics", account.username); + client.put(&path, "text/calendar; charset=utf-8", &bytes)?; + Ok((path, uid)) + } + + pub fn add_contact( + &self, + account: &Account, + book_segment_idx: usize, + tag: &str, + vcards: &[RawFixture], + ) -> ContainerResult<(String, String)> { + let client = DavSeed::new(self.dav_root(), &account.username, &account.password); + let display = account.layout.address_books[book_segment_idx]; + let segment = collection_segment("ab", book_segment_idx, display); + let fixture = vcards.first().ok_or_else(|| { + ContainerError::Seed("no vcard fixture available for add_contact".to_owned()) + })?; + let suffix = format!("added-{tag}"); + let (bytes, uid) = rewrite_uid(&fixture.bytes, &suffix) + .ok_or_else(|| ContainerError::Seed("vcard fixture has no UID".to_owned()))?; + let path = format!( + "/addressbooks/{}/{segment}/added-{tag}.vcf", + account.username + ); + client.put(&path, "text/vcard; charset=utf-8", &bytes)?; + Ok((path, uid)) + } + + pub fn verify_seed(&self, seeds: &[AccountSeed]) -> ContainerResult<()> { + for (acct, seed) in self.accounts.iter().zip(seeds) { + let client = DavSeed::new(self.dav_root(), &acct.username, &acct.password); + let body = client.propfind(&format!("/calendars/{}/", acct.username), 1)?; + if !body.contains("multistatus") { + return Err(ContainerError::Protocol(format!( + "baikal propfind for {} returned no multistatus", + acct.username + ))); + } + let _ = seed; + } + Ok(()) + } + + pub fn stop(self) -> ContainerResult<()> { + self.container.stop()?; + Ok(()) + } +} + +fn build_sqlite_db() -> ContainerResult> { + let tmp = tempfile::NamedTempFile::new() + .map_err(|e| ContainerError::Seed(format!("baikal tempfile: {e}")))?; + let path = tmp.path().to_path_buf(); + drop(tmp); + + { + let conn = rusqlite::Connection::open(&path) + .map_err(|e| ContainerError::Seed(format!("baikal sqlite open: {e}")))?; + conn.execute_batch(SCHEMA) + .map_err(|e| ContainerError::Seed(format!("baikal schema: {e}")))?; + + let users: [(&str, &str); 3] = [ + (layouts::ACCOUNT1, DIGEST_USER1), + (layouts::ACCOUNT2, DIGEST_USER2), + (layouts::ACCOUNT3, DIGEST_USER3), + ]; + for (name, digest) in users { + conn.execute( + "INSERT INTO users (username, digesta1) VALUES (?1, ?2)", + params![name, digest], + ) + .map_err(|e| ContainerError::Seed(format!("baikal user {name}: {e}")))?; + let uri = format!("principals/{name}"); + let email = format!("{name}@vandelay.test"); + let display = format!("Vandelay {name}"); + conn.execute( + "INSERT INTO principals (uri, email, displayname) VALUES (?1, ?2, ?3)", + params![uri, email, display], + ) + .map_err(|e| ContainerError::Seed(format!("baikal principal {name}: {e}")))?; + conn.execute( + "INSERT INTO principals (uri, email, displayname) VALUES (?1, NULL, NULL)", + params![format!("{uri}/calendar-proxy-read")], + ) + .map_err(|e| ContainerError::Seed(format!("baikal cal-proxy-read {name}: {e}")))?; + conn.execute( + "INSERT INTO principals (uri, email, displayname) VALUES (?1, NULL, NULL)", + params![format!("{uri}/calendar-proxy-write")], + ) + .map_err(|e| ContainerError::Seed(format!("baikal cal-proxy-write {name}: {e}")))?; + } + + let _ = REALM; + let _ = ADMIN_PASSWORD_HASH; + } + + let bytes = + std::fs::read(&path).map_err(|e| ContainerError::Seed(format!("baikal read db: {e}")))?; + let _ = std::fs::remove_file(&path); + Ok(bytes) +} + +fn collection_segment(prefix: &str, idx: usize, name: &str) -> String { + let mut out = format!("{prefix}-{:02}-", idx); + for c in name.chars() { + if c.is_ascii_alphanumeric() { + out.push(c.to_ascii_lowercase()); + } else if !out.ends_with('-') { + out.push('-'); + } + } + out.trim_end_matches('-').to_owned() +} + +fn events_per_calendar(layout: &Layout, idx: usize) -> usize { + if layout.calendars.is_empty() { + return 0; + } + let base = layout.event_count / layout.calendars.len().max(1); + let extra = if idx == 0 { + layout.event_count % layout.calendars.len() + } else { + 0 + }; + base + extra +} + +fn events_offset(layout: &Layout, idx: usize) -> usize { + (0..idx).map(|i| events_per_calendar(layout, i)).sum() +} + +fn contacts_per_book(layout: &Layout, idx: usize) -> usize { + if layout.address_books.is_empty() { + return 0; + } + let base = layout.contact_count / layout.address_books.len().max(1); + let extra = if idx == 0 { + layout.contact_count % layout.address_books.len() + } else { + 0 + }; + base + extra +} + +fn contacts_offset(layout: &Layout, idx: usize) -> usize { + (0..idx).map(|i| contacts_per_book(layout, i)).sum() +} + +#[derive(Debug, Clone)] +pub struct SeededItem { + pub uid: String, + pub href: String, + pub source: Vec, +} + +#[derive(Debug, Clone)] +pub struct CollectionPlan { + pub display_name: String, + pub base_href: String, + pub items: Vec, +} + +impl CollectionPlan { + fn new(display_name: String, base_href: String) -> Self { + Self { + display_name, + base_href, + items: Vec::new(), + } + } +} + +#[derive(Debug, Clone)] +pub struct AccountSeed { + pub username: String, + pub calendars: Vec, + pub address_books: Vec, +} + +impl AccountSeed { + fn new(username: String) -> Self { + Self { + username, + calendars: Vec::new(), + address_books: Vec::new(), + } + } + + pub fn total_events(&self) -> usize { + self.calendars.iter().map(|c| c.items.len()).sum() + } + + pub fn total_contacts(&self) -> usize { + self.address_books.iter().map(|c| c.items.len()).sum() + } + + pub fn event_uids(&self) -> HashSet { + self.calendars + .iter() + .flat_map(|c| c.items.iter().map(|i| i.uid.clone())) + .collect() + } + + pub fn contact_uids(&self) -> HashSet { + self.address_books + .iter() + .flat_map(|c| c.items.iter().map(|i| i.uid.clone())) + .collect() + } +} diff --git a/tests/integration/baikal_schema.sql b/tests/integration/baikal_schema.sql new file mode 100644 index 0000000..6d3bf7c --- /dev/null +++ b/tests/integration/baikal_schema.sql @@ -0,0 +1,147 @@ +CREATE TABLE addressbooks ( + id integer primary key asc NOT NULL, + principaluri text NOT NULL, + displayname text, + uri text NOT NULL, + description text, + synctoken integer DEFAULT 1 NOT NULL +); + +CREATE TABLE cards ( + id integer primary key asc NOT NULL, + addressbookid integer NOT NULL, + carddata blob, + uri text NOT NULL, + lastmodified integer, + etag text, + size integer +); + +CREATE TABLE addressbookchanges ( + id integer primary key asc NOT NULL, + uri text, + synctoken integer NOT NULL, + addressbookid integer NOT NULL, + operation integer NOT NULL +); + +CREATE INDEX addressbookid_synctoken ON addressbookchanges (addressbookid, synctoken); +CREATE TABLE calendarobjects ( + id integer primary key asc NOT NULL, + calendardata blob NOT NULL, + uri text NOT NULL, + calendarid integer NOT NULL, + lastmodified integer NOT NULL, + etag text NOT NULL, + size integer NOT NULL, + componenttype text, + firstoccurence integer, + lastoccurence integer, + uid text +); + +CREATE TABLE calendars ( + id integer primary key asc NOT NULL, + synctoken integer DEFAULT 1 NOT NULL, + components text NOT NULL +); + +CREATE TABLE calendarinstances ( + id integer primary key asc NOT NULL, + calendarid integer, + principaluri text, + access integer, + displayname text, + uri text NOT NULL, + description text, + calendarorder integer, + calendarcolor text, + timezone text, + transparent bool, + share_href text, + share_displayname text, + share_invitestatus integer DEFAULT '2', + UNIQUE (principaluri, uri), + UNIQUE (calendarid, principaluri), + UNIQUE (calendarid, share_href) +); + +CREATE TABLE calendarchanges ( + id integer primary key asc NOT NULL, + uri text, + synctoken integer NOT NULL, + calendarid integer NOT NULL, + operation integer NOT NULL +); + +CREATE INDEX calendarid_synctoken ON calendarchanges (calendarid, synctoken); + +CREATE TABLE calendarsubscriptions ( + id integer primary key asc NOT NULL, + uri text NOT NULL, + principaluri text NOT NULL, + source text NOT NULL, + displayname text, + refreshrate text, + calendarorder integer, + calendarcolor text, + striptodos bool, + stripalarms bool, + stripattachments bool, + lastmodified int +); + +CREATE TABLE schedulingobjects ( + id integer primary key asc NOT NULL, + principaluri text NOT NULL, + calendardata blob, + uri text NOT NULL, + lastmodified integer, + etag text NOT NULL, + size integer NOT NULL +); + +CREATE INDEX principaluri_uri ON calendarsubscriptions (principaluri, uri); +BEGIN TRANSACTION; +CREATE TABLE locks ( + id integer primary key asc NOT NULL, + owner text, + timeout integer, + created integer, + token text, + scope integer, + depth integer, + uri text +); +COMMIT; +CREATE TABLE principals ( + id INTEGER PRIMARY KEY ASC NOT NULL, + uri TEXT NOT NULL, + email TEXT, + displayname TEXT, + UNIQUE(uri) +); + +CREATE TABLE groupmembers ( + id INTEGER PRIMARY KEY ASC NOT NULL, + principal_id INTEGER NOT NULL, + member_id INTEGER NOT NULL, + UNIQUE(principal_id, member_id) +); + +CREATE TABLE propertystorage ( + id integer primary key asc NOT NULL, + path text NOT NULL, + name text NOT NULL, + valuetype integer NOT NULL, + value string +); + + +CREATE UNIQUE INDEX path_property ON propertystorage (path, name); +CREATE TABLE users ( + id integer primary key asc NOT NULL, + username TEXT NOT NULL, + digesta1 TEXT NOT NULL, + UNIQUE(username) +); diff --git a/tests/integration/cyrus.rs b/tests/integration/cyrus.rs new file mode 100644 index 0000000..5103423 --- /dev/null +++ b/tests/integration/cyrus.rs @@ -0,0 +1,393 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; +use std::time::Duration; + +use testcontainers::core::{IntoContainerPort, WaitFor}; +use testcontainers::runners::{SyncBuilder, SyncRunner}; +use testcontainers::{Container, GenericBuildableImage, GenericImage, ImageExt}; + +use super::data::{MboxMessage, load_mbox}; +use super::error::{ContainerError, ContainerResult}; +use super::imap_client::{ImapSeed, full_path}; +use super::layouts::{self}; +use super::{Account, Endpoint}; + +const IMAP_PORT: u16 = 143; + +const DOCKERFILE: &str = r#"FROM debian:bookworm-20260518-slim + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + cyrus-imapd cyrus-admin cyrus-clients sasl2-bin \ + libsasl2-modules libsasl2-modules-db \ + ca-certificates && \ + rm -rf /var/lib/apt/lists/* && \ + mkdir -p /var/lib/cyrus /var/spool/cyrus /run/cyrus && \ + chown -R cyrus:mail /var/lib/cyrus /var/spool/cyrus /run/cyrus + +COPY imapd.conf /etc/imapd.conf +COPY cyrus.conf /etc/cyrus.conf +COPY sasldb.txt /tmp/sasldb.txt +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +EXPOSE 143 +CMD ["/entrypoint.sh"] +"#; + +const IMAPD_CONF: &str = r#"configdirectory: /var/lib/cyrus +partition-default: /var/spool/cyrus +admins: cyrus +allowanonymouslogin: no +allowplaintext: yes +sasl_mech_list: PLAIN LOGIN +sasl_pwcheck_method: auxprop +sasl_auxprop_plugin: sasldb +sasl_sasldb_path: /etc/sasldb2 +unixhierarchysep: yes +altnamespace: yes +sasl_minimum_layer: 0 +"#; + +const CYRUS_CONF: &str = r#"START { + recover cmd="ctl_cyrusdb -r" +} + +SERVICES { + imap cmd="imapd" listen="143" prefork=0 +} + +EVENTS { + checkpoint cmd="ctl_cyrusdb -c" period=30 + delprune cmd="cyr_expire -E 3" at=0400 +} +"#; + +fn sasldb_entries() -> String { + let mut out = String::new(); + for u in layouts::accounts() { + out.push_str(u); + out.push(' '); + out.push_str(layouts::PASSWORD); + out.push('\n'); + } + out.push_str("cyrus admin-secret\n"); + out +} + +const ENTRYPOINT: &str = r#"#!/bin/bash +set -e + +while IFS=' ' read -r user pw; do + [ -z "$user" ] && continue + echo "$pw" | saslpasswd2 -p -c -f /etc/sasldb2 "$user" +done < /tmp/sasldb.txt +chown cyrus:mail /etc/sasldb2 +chmod 0640 /etc/sasldb2 + +mkdir -p /var/lib/cyrus /var/spool/cyrus /run/cyrus +chown -R cyrus:mail /var/lib/cyrus /var/spool/cyrus /run/cyrus +su cyrus -s /bin/bash -c '/usr/sbin/cyrus makedirs' || true + +/usr/sbin/cyrmaster -C /etc/imapd.conf -M /etc/cyrus.conf & +master_pid=$! + +for _ in $(seq 1 120); do + if (exec 3<>/dev/tcp/127.0.0.1/143) 2>/dev/null; then + exec 3<&- 3>&- + break + fi + sleep 0.5 +done + +provision() { + exec 3<>/dev/tcp/127.0.0.1/143 + read -r _greeting <&3 + printf 'a1 LOGIN cyrus admin-secret\r\n' >&3 + read -r _ <&3 + local tag=1 + for u in "$@"; do + tag=$((tag + 1)) + printf 'a%d CREATE user/%s\r\n' "$tag" "$u" >&3 + read -r _ <&3 + done + tag=$((tag + 1)) + printf 'a%d LOGOUT\r\n' "$tag" >&3 + read -r _ <&3 + exec 3<&- 3>&- +} + +users=() +while IFS=' ' read -r user _pw; do + [ -z "$user" ] && continue + [ "$user" = "cyrus" ] && continue + users+=("$user") +done < /tmp/sasldb.txt +provision "${users[@]}" + +echo "cyrus provisioned" +wait "$master_pid" +"#; + +pub struct Cyrus { + container: Container, + pub imap: Endpoint, + pub accounts: Vec, +} + +impl Cyrus { + pub fn start() -> ContainerResult { + let image: GenericImage = GenericBuildableImage::new("vandelay-cyrus", "test") + .with_dockerfile_string(DOCKERFILE.to_owned()) + .with_data(IMAPD_CONF.as_bytes().to_vec(), "imapd.conf") + .with_data(CYRUS_CONF.as_bytes().to_vec(), "cyrus.conf") + .with_data(sasldb_entries().into_bytes(), "sasldb.txt") + .with_data(ENTRYPOINT.as_bytes().to_vec(), "entrypoint.sh") + .build_image() + .map_err(|e| ContainerError::Seed(format!("cyrus build: {e}")))?; + + let request = image + .with_exposed_port(IMAP_PORT.tcp()) + .with_wait_for(WaitFor::message_on_stdout("cyrus provisioned")) + .with_startup_timeout(Duration::from_secs(180)); + + let container = request.start()?; + let host = container.get_host()?.to_string(); + let imap = Endpoint::new(host, container.get_host_port_ipv4(IMAP_PORT.tcp())?); + + let accounts: Vec = layouts::accounts() + .iter() + .map(|name| Account { + username: (*name).to_owned(), + password: layouts::PASSWORD.to_owned(), + layout: layouts::layout_for(name), + }) + .collect(); + + Ok(Self { + container, + imap, + accounts, + }) + } + + pub fn seed_all(&self) -> ContainerResult> { + let limit = self + .accounts + .iter() + .map(|a| a.layout.email_count) + .max() + .unwrap_or(0) + .max(1); + let messages = load_mbox(limit)?; + let mut out = Vec::new(); + for acct in &self.accounts { + let m = self.seed_account(acct, &messages)?; + out.push(m); + } + Ok(out) + } + + fn seed_account( + &self, + account: &Account, + messages: &[MboxMessage], + ) -> ContainerResult { + let mut client = ImapSeed::connect(&self.imap.host, self.imap.port)?; + client.login(&account.username, &account.password)?; + let sep = client.discover_separator().unwrap_or('/'); + + let mut paths = Vec::new(); + for spec in account.layout.mailboxes { + let path = full_path(account.layout.mailboxes, spec.key, sep).ok_or_else(|| { + ContainerError::Seed(format!("missing mailbox key: {}", spec.key)) + })?; + client.create(&path)?; + client.subscribe(&path)?; + paths.push(path); + } + let mut targets: Vec = vec!["INBOX".to_owned()]; + targets.extend(paths.iter().cloned()); + let mut histogram: HashMap = + targets.iter().map(|t| (t.clone(), 0)).collect(); + let mut appends: Vec = Vec::new(); + + let total = account.layout.email_count.min(messages.len()); + if total < account.layout.email_count { + return Err(ContainerError::Seed(format!( + "{} only has {} messages but layout requires {}", + account.username, + messages.len(), + account.layout.email_count + ))); + } + for (i, msg) in messages.iter().take(total).enumerate() { + let target = targets[i % targets.len()].clone(); + client.append_with_flags(&target, &[], &msg.raw)?; + *histogram.entry(target.clone()).or_insert(0) += 1; + appends.push(SeededAppend { + raw: msg.raw.clone(), + target, + flags: Vec::new(), + tag: SeedTag::Bulk(i), + }); + } + + let mut dedup_target: Option = None; + if targets.len() > 1 && !messages.is_empty() { + let target = targets[1].clone(); + client.append_with_flags(&target, &[], &messages[0].raw)?; + *histogram.entry(target.clone()).or_insert(0) += 1; + appends.push(SeededAppend { + raw: messages[0].raw.clone(), + target: target.clone(), + flags: Vec::new(), + tag: SeedTag::Dedup, + }); + dedup_target = Some(target); + } + + let mut flagged_target: Option = None; + if !messages.is_empty() { + let target = "INBOX".to_owned(); + let probe = flag_probe_message(); + client.append_with_flags(&target, &["\\Seen", "\\Flagged"], &probe)?; + *histogram.entry(target.clone()).or_insert(0) += 1; + appends.push(SeededAppend { + raw: probe, + target: target.clone(), + flags: vec!["$seen".to_owned(), "$flagged".to_owned()], + tag: SeedTag::FlagProbe, + }); + flagged_target = Some(target); + } + + client.logout()?; + + let total_appends = histogram.values().sum(); + Ok(AccountSeed { + username: account.username.clone(), + paths, + histogram, + total_appends, + dedup_target, + flagged_target, + appends, + }) + } + + pub fn delete_first_inbox_message(&self, account: &Account) -> ContainerResult<()> { + let mut client = ImapSeed::connect(&self.imap.host, self.imap.port)?; + client.login(&account.username, &account.password)?; + client.delete_and_expunge_first("INBOX")?; + client.logout()?; + Ok(()) + } + + pub fn append_new_message( + &self, + account: &Account, + mailbox: &str, + tag: &str, + ) -> ContainerResult<(Vec, String)> { + let mut client = ImapSeed::connect(&self.imap.host, self.imap.port)?; + client.login(&account.username, &account.password)?; + let message_id = format!("", account.username); + let body = format!( + "From: cyrus-added-{tag}@vandelay.test\r\n\ + To: {}@vandelay.test\r\n\ + Subject: Cyrus added probe {tag}\r\n\ + Message-ID: {message_id}\r\n\ + Date: Wed, 01 Jan 2025 12:00:00 +0000\r\n\ + \r\n\ + Cyrus added probe body {tag}.\r\n", + account.username + ); + let raw = body.into_bytes(); + client.append_with_flags(mailbox, &[], &raw)?; + client.logout()?; + Ok((raw, message_id)) + } + + pub fn verify_seed(&self, seeds: &[AccountSeed]) -> ContainerResult<()> { + for (acct, seed) in self.accounts.iter().zip(seeds) { + let mut client = ImapSeed::connect(&self.imap.host, self.imap.port)?; + client.login(&acct.username, &acct.password)?; + let names = client.list_all()?; + let expected = acct.layout.mailboxes.len() + 1; + if names.len() < expected { + return Err(ContainerError::Seed(format!( + "cyrus {}: expected >= {expected} mailboxes, got {}", + acct.username, + names.len() + ))); + } + let inbox_n = client.select("INBOX")?; + if acct.layout.email_count > 0 && inbox_n == 0 { + return Err(ContainerError::Seed(format!( + "cyrus {}: INBOX EXISTS = 0 after seed", + acct.username + ))); + } + for path in &seed.paths { + let n = client.select(path)?; + let want = seed.histogram.get(path).copied().unwrap_or(0); + if n < want { + return Err(ContainerError::Seed(format!( + "cyrus {}: mailbox {path} EXISTS={n} but {want} were appended", + acct.username + ))); + } + } + client.logout()?; + } + Ok(()) + } + + pub fn stop(self) -> ContainerResult<()> { + self.container.stop()?; + Ok(()) + } +} + +pub fn flag_probe_message() -> Vec { + let body = "From: probe-flags@vandelay.test\r\n\ + To: user@vandelay.test\r\n\ + Subject: Cyrus flag probe\r\n\ + Message-ID: \r\n\ + Date: Wed, 01 Jan 2025 12:00:00 +0000\r\n\ + \r\n\ + Cyrus flag probe body.\r\n"; + body.as_bytes().to_vec() +} + +#[derive(Debug, Clone)] +pub struct AccountSeed { + pub username: String, + pub paths: Vec, + pub histogram: HashMap, + pub total_appends: usize, + pub dedup_target: Option, + pub flagged_target: Option, + pub appends: Vec, +} + +#[derive(Debug, Clone)] +pub struct SeededAppend { + pub raw: Vec, + pub target: String, + pub flags: Vec, + pub tag: SeedTag, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SeedTag { + Bulk(usize), + Dedup, + FlagProbe, +} diff --git a/tests/integration/data.rs b/tests/integration/data.rs new file mode 100644 index 0000000..7c6b441 --- /dev/null +++ b/tests/integration/data.rs @@ -0,0 +1,475 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::{BufReader, Read}; +use std::path::{Path, PathBuf}; + +use flate2::read::GzDecoder; +use mail_parser::mailbox::mbox::MessageIterator; + +use super::error::{ContainerError, ContainerResult}; + +fn resources_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("resources") +} + +#[derive(Debug, Clone)] +pub struct MboxMessage { + pub raw: Vec, + pub received_at: i64, +} + +pub fn load_mbox(limit: usize) -> ContainerResult> { + let path = resources_dir().join("mailbox.gz"); + if path.exists() { + let file = std::fs::File::open(&path)?; + let mut decoder = GzDecoder::new(file); + let mut bytes = Vec::new(); + decoder.read_to_end(&mut bytes)?; + let reader = BufReader::new(std::io::Cursor::new(bytes)); + let mut out = Vec::new(); + for item in MessageIterator::new(reader) { + let message = item.map_err(|e| ContainerError::Resource(format!("mbox parse: {e}")))?; + let received_at = message.internal_date() as i64; + out.push(MboxMessage { + raw: message.unwrap_contents(), + received_at, + }); + if out.len() >= limit { + break; + } + } + if !out.is_empty() { + return Ok(out); + } + } + if let Ok(takeout) = load_takeout_mbox("all_mail_including_spam_and_trash.mbox") + && !takeout.is_empty() + { + return Ok(takeout.into_iter().take(limit).collect()); + } + Ok(synth_messages(limit)) +} + +#[derive(Debug, Clone)] +pub struct RawFixture { + pub name: String, + pub bytes: Vec, +} + +pub fn load_vcards() -> ContainerResult> { + Ok(synth_vcards(48)) +} + +pub fn load_icals() -> ContainerResult> { + Ok(synth_icals(48)) +} + +pub fn load_takeout_mbox(name: &str) -> ContainerResult> { + let path = resources_dir().join("takeout").join(name); + let bytes = std::fs::read(&path) + .map_err(|e| ContainerError::Resource(format!("open {}: {e}", path.display())))?; + let reader = BufReader::new(std::io::Cursor::new(bytes)); + let mut out = Vec::new(); + for item in MessageIterator::new(reader) { + let message = item.map_err(|e| ContainerError::Resource(format!("mbox parse: {e}")))?; + let received_at = message.internal_date() as i64; + out.push(MboxMessage { + raw: message.unwrap_contents(), + received_at, + }); + } + Ok(out) +} + +fn synth_messages(n: usize) -> Vec { + let mut out = Vec::with_capacity(n); + for i in 0..n { + let body = format!( + "From: sender{i}@vandelay.test\r\n\ + To: user@vandelay.test\r\n\ + Subject: Synthetic test message {i}\r\n\ + Message-ID: \r\n\ + Date: Wed, 01 Jan 2025 12:00:00 +0000\r\n\ + MIME-Version: 1.0\r\n\ + Content-Type: text/plain; charset=utf-8\r\n\ + \r\n\ + This is synthetic seed message #{i} for vandelay container tests.\r\n", + ); + out.push(MboxMessage { + raw: body.into_bytes(), + received_at: 0, + }); + } + out +} + +const VCARD_VARIANTS: &[fn(usize) -> String] = &[ + vcard_v3_basic, + vcard_v3_full, + vcard_v3_multi_email, + vcard_v3_org_and_title, + vcard_v4_basic, + vcard_v4_with_address, + vcard_v4_nickname, + vcard_v4_birthday, +]; + +fn synth_vcards(n: usize) -> Vec { + let mut out = Vec::with_capacity(n); + for i in 0..n { + let body = VCARD_VARIANTS[i % VCARD_VARIANTS.len()](i); + out.push(RawFixture { + name: format!("synth-{i:03}"), + bytes: body.into_bytes(), + }); + } + out +} + +fn vcard_v3_basic(i: usize) -> String { + format!( + "BEGIN:VCARD\r\n\ + VERSION:3.0\r\n\ + UID:vandelay-card-basic-{i}\r\n\ + FN:Basic Contact {i}\r\n\ + N:Contact{i};Basic;;;\r\n\ + EMAIL;TYPE=INTERNET:basic-{i}@vandelay.test\r\n\ + END:VCARD\r\n" + ) +} + +fn vcard_v3_full(i: usize) -> String { + format!( + "BEGIN:VCARD\r\n\ + VERSION:3.0\r\n\ + UID:vandelay-card-full-{i}\r\n\ + FN:Full Contact {i}\r\n\ + N:Contact{i};Full;Middle;;\r\n\ + EMAIL;TYPE=INTERNET;TYPE=WORK:full-{i}@vandelay.test\r\n\ + TEL;TYPE=CELL:+15550010{i:03}\r\n\ + ORG:Vandelay Test\r\n\ + TITLE:Senior Test Engineer\r\n\ + URL:https://vandelay.test/{i}\r\n\ + NOTE:Synthetic full v3 contact #{i}\r\n\ + END:VCARD\r\n" + ) +} + +fn vcard_v3_multi_email(i: usize) -> String { + format!( + "BEGIN:VCARD\r\n\ + VERSION:3.0\r\n\ + UID:vandelay-card-multi-{i}\r\n\ + FN:Multi Email {i}\r\n\ + N:Contact{i};MultiEmail;;;\r\n\ + EMAIL;TYPE=INTERNET;TYPE=HOME:multi-{i}-home@vandelay.test\r\n\ + EMAIL;TYPE=INTERNET;TYPE=WORK:multi-{i}-work@vandelay.test\r\n\ + END:VCARD\r\n" + ) +} + +fn vcard_v3_org_and_title(i: usize) -> String { + format!( + "BEGIN:VCARD\r\n\ + VERSION:3.0\r\n\ + UID:vandelay-card-org-{i}\r\n\ + FN:Org Contact {i}\r\n\ + N:Contact{i};Org;;Dr.;PhD\r\n\ + ORG:Vandelay Industries;Engineering;Test Group\r\n\ + TITLE:Principal Researcher\r\n\ + EMAIL;TYPE=INTERNET:org-{i}@vandelay.test\r\n\ + END:VCARD\r\n" + ) +} + +fn vcard_v4_basic(i: usize) -> String { + format!( + "BEGIN:VCARD\r\n\ + VERSION:4.0\r\n\ + UID:urn:uuid:00000000-0000-4000-a000-0000{i:08x}\r\n\ + FN:V4 Basic {i}\r\n\ + N:Contact{i};V4Basic;;;\r\n\ + EMAIL:v4-basic-{i}@vandelay.test\r\n\ + END:VCARD\r\n" + ) +} + +fn vcard_v4_with_address(i: usize) -> String { + format!( + "BEGIN:VCARD\r\n\ + VERSION:4.0\r\n\ + UID:urn:uuid:00000000-0000-4000-b000-0000{i:08x}\r\n\ + FN:V4 Address {i}\r\n\ + N:Contact{i};V4Address;;;\r\n\ + EMAIL:v4-address-{i}@vandelay.test\r\n\ + ADR;TYPE=home:;;{i} Test Street;Springfield;OR;97477;US\r\n\ + END:VCARD\r\n" + ) +} + +fn vcard_v4_nickname(i: usize) -> String { + format!( + "BEGIN:VCARD\r\n\ + VERSION:4.0\r\n\ + UID:urn:uuid:00000000-0000-4000-c000-0000{i:08x}\r\n\ + FN:V4 Nick {i}\r\n\ + N:Contact{i};V4Nick;;;\r\n\ + NICKNAME:Nicky{i}\r\n\ + EMAIL:v4-nick-{i}@vandelay.test\r\n\ + END:VCARD\r\n" + ) +} + +fn vcard_v4_birthday(i: usize) -> String { + let year = 1970 + (i % 30); + let month = (i % 12) + 1; + let day = (i % 28) + 1; + format!( + "BEGIN:VCARD\r\n\ + VERSION:4.0\r\n\ + UID:urn:uuid:00000000-0000-4000-d000-0000{i:08x}\r\n\ + FN:V4 Birthday {i}\r\n\ + N:Contact{i};V4Birthday;;;\r\n\ + BDAY:{year:04}{month:02}{day:02}\r\n\ + EMAIL:v4-bday-{i}@vandelay.test\r\n\ + END:VCARD\r\n" + ) +} + +const ICAL_VARIANTS: &[fn(usize) -> String] = &[ + ical_simple_dt, + ical_all_day, + ical_multi_day_dt, + ical_with_organizer_attendees, + ical_with_categories_location, + ical_recurring_daily, + ical_recurring_weekly_until, + ical_with_alarm, +]; + +fn synth_icals(n: usize) -> Vec { + let mut out = Vec::with_capacity(n); + for i in 0..n { + let body = ICAL_VARIANTS[i % ICAL_VARIANTS.len()](i); + out.push(RawFixture { + name: format!("synth-{i:03}"), + bytes: body.into_bytes(), + }); + } + out +} + +fn ical_simple_dt(i: usize) -> String { + let month = ((i % 12) + 1) as u32; + let day = ((i % 28) + 1) as u32; + let hour = (i % 24) as u32; + format!( + "BEGIN:VCALENDAR\r\n\ + VERSION:2.0\r\n\ + PRODID:-//vandelay//container-test//EN\r\n\ + BEGIN:VEVENT\r\n\ + UID:vandelay-simple-{i}@vandelay.test\r\n\ + DTSTAMP:20250101T120000Z\r\n\ + DTSTART:2025{month:02}{day:02}T{hour:02}0000Z\r\n\ + DTEND:2025{month:02}{day:02}T{hour:02}3000Z\r\n\ + SUMMARY:Simple datetime event {i}\r\n\ + DESCRIPTION:Simple synthetic event #{i}\r\n\ + END:VEVENT\r\n\ + END:VCALENDAR\r\n" + ) +} + +fn ical_all_day(i: usize) -> String { + let month = ((i % 12) + 1) as u32; + let day = ((i % 28) + 1) as u32; + let next_day = day + 1; + format!( + "BEGIN:VCALENDAR\r\n\ + VERSION:2.0\r\n\ + PRODID:-//vandelay//container-test//EN\r\n\ + BEGIN:VEVENT\r\n\ + UID:vandelay-allday-{i}@vandelay.test\r\n\ + DTSTAMP:20250101T120000Z\r\n\ + DTSTART;VALUE=DATE:2025{month:02}{day:02}\r\n\ + DTEND;VALUE=DATE:2025{month:02}{next_day:02}\r\n\ + SUMMARY:All-day event {i}\r\n\ + END:VEVENT\r\n\ + END:VCALENDAR\r\n" + ) +} + +fn ical_multi_day_dt(i: usize) -> String { + let month = ((i % 12) + 1) as u32; + let day = ((i % 27) + 1) as u32; + format!( + "BEGIN:VCALENDAR\r\n\ + VERSION:2.0\r\n\ + PRODID:-//vandelay//container-test//EN\r\n\ + BEGIN:VEVENT\r\n\ + UID:vandelay-multiday-{i}@vandelay.test\r\n\ + DTSTAMP:20250101T120000Z\r\n\ + DTSTART:2025{month:02}{day:02}T090000Z\r\n\ + DTEND:2025{month:02}{day:02}T170000Z\r\n\ + SUMMARY:Workshop session {i}\r\n\ + DESCRIPTION:Multi-hour workshop with description.\r\n\ + END:VEVENT\r\n\ + END:VCALENDAR\r\n" + ) +} + +fn ical_with_organizer_attendees(i: usize) -> String { + let month = ((i % 12) + 1) as u32; + let day = ((i % 28) + 1) as u32; + format!( + "BEGIN:VCALENDAR\r\n\ + VERSION:2.0\r\n\ + PRODID:-//vandelay//container-test//EN\r\n\ + BEGIN:VEVENT\r\n\ + UID:vandelay-meeting-{i}@vandelay.test\r\n\ + DTSTAMP:20250101T120000Z\r\n\ + DTSTART:2025{month:02}{day:02}T140000Z\r\n\ + DTEND:2025{month:02}{day:02}T150000Z\r\n\ + SUMMARY:Sync meeting {i}\r\n\ + ORGANIZER;CN=Organiser:mailto:organiser-{i}@vandelay.test\r\n\ + ATTENDEE;CN=Alice;PARTSTAT=ACCEPTED:mailto:alice-{i}@vandelay.test\r\n\ + ATTENDEE;CN=Bob;PARTSTAT=NEEDS-ACTION:mailto:bob-{i}@vandelay.test\r\n\ + END:VEVENT\r\n\ + END:VCALENDAR\r\n" + ) +} + +fn ical_with_categories_location(i: usize) -> String { + let month = ((i % 12) + 1) as u32; + let day = ((i % 28) + 1) as u32; + format!( + "BEGIN:VCALENDAR\r\n\ + VERSION:2.0\r\n\ + PRODID:-//vandelay//container-test//EN\r\n\ + BEGIN:VEVENT\r\n\ + UID:vandelay-cat-{i}@vandelay.test\r\n\ + DTSTAMP:20250101T120000Z\r\n\ + DTSTART:2025{month:02}{day:02}T100000Z\r\n\ + DTEND:2025{month:02}{day:02}T110000Z\r\n\ + SUMMARY:Categorised event {i}\r\n\ + LOCATION:Test Lab\\, Room {i}\r\n\ + CATEGORIES:VANDELAY,TEST,FIXTURE\r\n\ + END:VEVENT\r\n\ + END:VCALENDAR\r\n" + ) +} + +fn ical_recurring_daily(i: usize) -> String { + let month = ((i % 12) + 1) as u32; + let day = ((i % 28) + 1) as u32; + format!( + "BEGIN:VCALENDAR\r\n\ + VERSION:2.0\r\n\ + PRODID:-//vandelay//container-test//EN\r\n\ + BEGIN:VEVENT\r\n\ + UID:vandelay-daily-{i}@vandelay.test\r\n\ + DTSTAMP:20250101T120000Z\r\n\ + DTSTART:2025{month:02}{day:02}T070000Z\r\n\ + DTEND:2025{month:02}{day:02}T073000Z\r\n\ + RRULE:FREQ=DAILY;COUNT=10\r\n\ + SUMMARY:Daily standup {i}\r\n\ + END:VEVENT\r\n\ + END:VCALENDAR\r\n" + ) +} + +fn ical_recurring_weekly_until(i: usize) -> String { + let month = ((i % 12) + 1) as u32; + let day = ((i % 28) + 1) as u32; + let until_month = (month % 12) + 1; + format!( + "BEGIN:VCALENDAR\r\n\ + VERSION:2.0\r\n\ + PRODID:-//vandelay//container-test//EN\r\n\ + BEGIN:VEVENT\r\n\ + UID:vandelay-weekly-{i}@vandelay.test\r\n\ + DTSTAMP:20250101T120000Z\r\n\ + DTSTART:2025{month:02}{day:02}T160000Z\r\n\ + DTEND:2025{month:02}{day:02}T170000Z\r\n\ + RRULE:FREQ=WEEKLY;UNTIL=2025{until_month:02}{day:02}T160000Z;BYDAY=MO\r\n\ + SUMMARY:Weekly retro {i}\r\n\ + END:VEVENT\r\n\ + END:VCALENDAR\r\n" + ) +} + +fn ical_with_alarm(i: usize) -> String { + let month = ((i % 12) + 1) as u32; + let day = ((i % 28) + 1) as u32; + format!( + "BEGIN:VCALENDAR\r\n\ + VERSION:2.0\r\n\ + PRODID:-//vandelay//container-test//EN\r\n\ + BEGIN:VEVENT\r\n\ + UID:vandelay-alarm-{i}@vandelay.test\r\n\ + DTSTAMP:20250101T120000Z\r\n\ + DTSTART:2025{month:02}{day:02}T080000Z\r\n\ + DTEND:2025{month:02}{day:02}T083000Z\r\n\ + SUMMARY:Event with alarm {i}\r\n\ + BEGIN:VALARM\r\n\ + ACTION:DISPLAY\r\n\ + DESCRIPTION:Reminder {i}\r\n\ + TRIGGER:-PT15M\r\n\ + END:VALARM\r\n\ + END:VEVENT\r\n\ + END:VCALENDAR\r\n" + ) +} + +pub fn malformed_ical(name: &str) -> RawFixture { + let body = format!( + "BEGIN:VCALENDAR\r\n\ + VERSION:2.0\r\n\ + PRODID:-//vandelay//broken//EN\r\n\ + BEGIN:VEVENT\r\n\ + UID:broken-{name}@vandelay.test\r\n\ + DTSTAMP:20250101T120000Z\r\n\ + DTSTART:NOT-A-DATE\r\n\ + RRULE:FREQ=BOGUS;INTERVAL=oops\r\n\ + END:NOT-A-VEVENT\r\n", + ); + RawFixture { + name: format!("broken-{name}"), + bytes: body.into_bytes(), + } +} + +pub fn rewrite_uid(bytes: &[u8], suffix: &str) -> Option<(Vec, String)> { + let text = std::str::from_utf8(bytes).ok()?; + let mut out = String::with_capacity(text.len() + 32); + let mut new_uid: Option = None; + for line in text.lines() { + if let Some(rest) = line.strip_prefix("UID:") { + let combined = format!("{rest}-{suffix}"); + if new_uid.is_none() { + new_uid = Some(combined.clone()); + } + out.push_str("UID:"); + out.push_str(&combined); + } else { + out.push_str(line); + } + out.push_str("\r\n"); + } + let uid = new_uid?; + Some((out.into_bytes(), uid)) +} + +pub fn extract_uid(bytes: &[u8]) -> Option { + let text = std::str::from_utf8(bytes).ok()?; + for line in text.lines() { + let l = line.trim_end_matches('\r'); + if let Some(rest) = l.strip_prefix("UID:") { + return Some(rest.to_owned()); + } + } + None +} diff --git a/tests/integration/dav_client.rs b/tests/integration/dav_client.rs new file mode 100644 index 0000000..9ff394b --- /dev/null +++ b/tests/integration/dav_client.rs @@ -0,0 +1,167 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::time::Duration; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD as B64; +use ureq::Agent; +use ureq::http::{Method, Request}; + +use super::error::{ContainerError, ContainerResult}; + +pub struct DavSeed { + agent: Agent, + base: String, + auth: String, +} + +impl DavSeed { + pub fn new(base: impl Into, user: &str, password: &str) -> Self { + let agent = ureq::Agent::config_builder() + .http_status_as_error(false) + .allow_non_standard_methods(true) + .timeout_global(Some(Duration::from_secs(30))) + .build() + .new_agent(); + let credentials = B64.encode(format!("{user}:{password}")); + Self { + agent, + base: base.into(), + auth: format!("Basic {credentials}"), + } + } + + fn url(&self, path: &str) -> String { + if path.starts_with("http://") || path.starts_with("https://") { + path.to_owned() + } else if path.starts_with('/') { + format!("{}{path}", self.base.trim_end_matches('/')) + } else { + format!("{}/{}", self.base.trim_end_matches('/'), path) + } + } + + fn run( + &self, + method: &str, + url: &str, + depth: Option, + content_type: Option<&str>, + body: Option<&[u8]>, + ) -> ContainerResult<(u16, Vec)> { + let parsed = Method::from_bytes(method.as_bytes()) + .map_err(|e| ContainerError::Protocol(format!("bad method {method}: {e}")))?; + let mut builder = Request::builder() + .method(parsed) + .uri(url) + .header("Authorization", &self.auth); + if let Some(d) = depth { + builder = builder.header("Depth", d.to_string()); + } + if let Some(ct) = content_type { + builder = builder.header("Content-Type", ct); + } + let payload: Vec = body.map(|b| b.to_vec()).unwrap_or_default(); + let request = builder + .body(payload) + .map_err(|e| ContainerError::Protocol(format!("build request: {e}")))?; + let mut response = self.agent.run(request)?; + let status = response.status().as_u16(); + let bytes = response.body_mut().read_to_vec()?; + Ok((status, bytes)) + } + + pub fn mkcol(&self, path: &str, body: Option<&str>) -> ContainerResult { + self.mkcol_with_method("MKCOL", path, body) + } + + pub fn mkcalendar(&self, path: &str, body: Option<&str>) -> ContainerResult { + self.mkcol_with_method("MKCALENDAR", path, body) + } + + fn mkcol_with_method( + &self, + method: &str, + path: &str, + body: Option<&str>, + ) -> ContainerResult { + let url = self.url(path); + let bytes = body.map(|b| b.as_bytes()); + let (status, response_body) = self.run( + method, + &url, + None, + Some("application/xml; charset=utf-8"), + bytes, + )?; + if !is_collection_created(status) { + return Err(ContainerError::Protocol(format!( + "{method} {url} -> {status}: {}", + truncate(&response_body) + ))); + } + Ok(status) + } + + pub fn put(&self, path: &str, content_type: &str, body: &[u8]) -> ContainerResult { + let url = self.url(path); + let (status, response_body) = + self.run("PUT", &url, None, Some(content_type), Some(body))?; + if !(200..300).contains(&status) { + return Err(ContainerError::Protocol(format!( + "PUT {url} -> {status}: {}", + truncate(&response_body) + ))); + } + Ok(status) + } + + pub fn delete(&self, path: &str) -> ContainerResult { + let url = self.url(path); + let (status, response_body) = self.run("DELETE", &url, None, None, None)?; + if !(200..300).contains(&status) && status != 404 { + return Err(ContainerError::Protocol(format!( + "DELETE {url} -> {status}: {}", + truncate(&response_body) + ))); + } + Ok(status) + } + + pub fn propfind(&self, path: &str, depth: u8) -> ContainerResult { + let url = self.url(path); + let body = r#" +"#; + let (status, bytes) = self.run( + "PROPFIND", + &url, + Some(depth), + Some("application/xml; charset=utf-8"), + Some(body.as_bytes()), + )?; + if !(200..400).contains(&status) { + return Err(ContainerError::Protocol(format!( + "PROPFIND {url} -> {status}: {}", + truncate(&bytes) + ))); + } + Ok(String::from_utf8_lossy(&bytes).into_owned()) + } +} + +fn is_collection_created(status: u16) -> bool { + matches!(status, 200 | 201 | 204) +} + +fn truncate(body: &[u8]) -> String { + let s = String::from_utf8_lossy(body); + if s.len() > 2000 { + format!("{}…", &s[..2000]) + } else { + s.into_owned() + } +} diff --git a/tests/integration/dovecot.rs b/tests/integration/dovecot.rs new file mode 100644 index 0000000..170fb71 --- /dev/null +++ b/tests/integration/dovecot.rs @@ -0,0 +1,503 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; +use std::time::Duration; + +use testcontainers::core::{IntoContainerPort, WaitFor}; +use testcontainers::runners::{SyncBuilder, SyncRunner}; +use testcontainers::{Container, GenericBuildableImage, GenericImage, ImageExt}; + +use super::data::{MboxMessage, load_mbox}; +use super::error::{ContainerError, ContainerResult}; +use super::imap_client::{ImapSeed, full_path}; +use super::layouts::{self}; +use super::sieve_client::SieveSeed as SieveClient; +use super::{Account, Endpoint}; + +const IMAP_PORT: u16 = 143; +const SIEVE_PORT: u16 = 4190; + +const DOCKERFILE: &str = r#"FROM debian:bookworm-20260518-slim + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + dovecot-imapd dovecot-managesieved dovecot-pop3d dovecot-sieve \ + openssl ca-certificates && \ + rm -rf /var/lib/apt/lists/* && \ + addgroup --gid 5000 vmail && \ + adduser --system --no-create-home --uid 5000 --gid 5000 vmail && \ + mkdir -p /var/vmail && chown -R vmail:vmail /var/vmail && \ + openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ + -keyout /etc/dovecot/dovecot.key \ + -out /etc/dovecot/dovecot.crt \ + -subj /CN=localhost && \ + chmod 600 /etc/dovecot/dovecot.key + +COPY dovecot.conf /etc/dovecot/dovecot.conf +COPY users /etc/dovecot/users + +EXPOSE 143 4190 +CMD ["dovecot", "-F"] +"#; + +const DOVECOT_CONF: &str = r#"protocols = imap sieve +listen = * + +mail_location = maildir:/var/vmail/%u/Maildir +mail_uid = 5000 +mail_gid = 5000 +mail_privileged_group = vmail + +namespace inbox { + inbox = yes + separator = / + prefix = +} + +auth_mechanisms = plain login +disable_plaintext_auth = no + +passdb { + driver = passwd-file + args = /etc/dovecot/users +} +userdb { + driver = static + args = uid=5000 gid=5000 home=/var/vmail/%u +} + +service imap-login { + inet_listener imap { + port = 143 + } +} + +service managesieve-login { + inet_listener sieve { + port = 4190 + } +} + +protocol sieve { + managesieve_max_line_length = 1M + managesieve_max_compile_errors = 5 +} + +ssl = yes +ssl_cert = String { + let mut out = String::new(); + for u in layouts::accounts() { + out.push_str(u); + out.push_str(":{plain}"); + out.push_str(layouts::PASSWORD); + out.push_str(":5000:5000::/var/vmail/"); + out.push_str(u); + out.push_str("::\n"); + } + out +} + +pub struct Dovecot { + container: Container, + pub imap: Endpoint, + pub sieve: Endpoint, + pub accounts: Vec, +} + +impl Dovecot { + pub fn start() -> ContainerResult { + let image: GenericImage = GenericBuildableImage::new("vandelay-dovecot", "test") + .with_dockerfile_string(DOCKERFILE.to_owned()) + .with_data(DOVECOT_CONF.as_bytes().to_vec(), "dovecot.conf") + .with_data(users_file().into_bytes(), "users") + .build_image() + .map_err(|e| ContainerError::Seed(format!("dovecot build: {e}")))?; + + let request = image + .with_exposed_port(IMAP_PORT.tcp()) + .with_exposed_port(SIEVE_PORT.tcp()) + .with_wait_for(WaitFor::message_on_stderr("starting up")) + .with_startup_timeout(Duration::from_secs(120)); + + let container = request.start()?; + let host = container.get_host()?.to_string(); + let imap = Endpoint::new(host.clone(), container.get_host_port_ipv4(IMAP_PORT.tcp())?); + let sieve = Endpoint::new(host, container.get_host_port_ipv4(SIEVE_PORT.tcp())?); + + let accounts: Vec = layouts::accounts() + .iter() + .map(|name| Account { + username: (*name).to_owned(), + password: layouts::PASSWORD.to_owned(), + layout: layouts::layout_for(name), + }) + .collect(); + + Ok(Self { + container, + imap, + sieve, + accounts, + }) + } + + pub fn seed_all(&self) -> ContainerResult> { + let limit = self + .accounts + .iter() + .map(|a| a.layout.email_count) + .max() + .unwrap_or(0) + .max(1); + let messages = load_mbox(limit)?; + let mut out = Vec::new(); + for acct in &self.accounts { + let mailbox = self.seed_imap(acct, &messages)?; + let sieve = self.seed_sieve(acct)?; + out.push(AccountSeed { + username: acct.username.clone(), + mailbox, + sieve, + }); + } + Ok(out) + } + + fn seed_imap( + &self, + account: &Account, + messages: &[MboxMessage], + ) -> ContainerResult { + let mut client = ImapSeed::connect(&self.imap.host, self.imap.port)?; + client.login(&account.username, &account.password)?; + let sep = client.discover_separator().unwrap_or('/'); + + let mut paths = Vec::new(); + for spec in account.layout.mailboxes { + let path = full_path(account.layout.mailboxes, spec.key, sep).ok_or_else(|| { + ContainerError::Seed(format!("missing mailbox key: {}", spec.key)) + })?; + client.create(&path)?; + client.subscribe(&path)?; + paths.push(path); + } + + let mut targets: Vec = vec!["INBOX".to_owned()]; + targets.extend(paths.iter().cloned()); + let mut histogram: HashMap = + targets.iter().map(|t| (t.clone(), 0)).collect(); + let mut appends: Vec = Vec::new(); + + let total = account.layout.email_count.min(messages.len()); + if total < account.layout.email_count { + return Err(ContainerError::Seed(format!( + "{} only has {} messages but layout requires {}", + account.username, + messages.len(), + account.layout.email_count + ))); + } + for (i, msg) in messages.iter().take(total).enumerate() { + let target = targets[i % targets.len()].clone(); + client.append_with_flags(&target, &[], &msg.raw)?; + *histogram.entry(target.clone()).or_insert(0) += 1; + appends.push(SeededAppend { + raw: msg.raw.clone(), + target, + flags: Vec::new(), + tag: SeedTag::Bulk(i), + }); + } + + let mut dedup_target: Option = None; + if targets.len() > 1 && !messages.is_empty() { + let target = targets[1].clone(); + client.append_with_flags(&target, &[], &messages[0].raw)?; + *histogram.entry(target.clone()).or_insert(0) += 1; + appends.push(SeededAppend { + raw: messages[0].raw.clone(), + target: target.clone(), + flags: Vec::new(), + tag: SeedTag::Dedup, + }); + dedup_target = Some(target); + } + + let mut flagged_target: Option = None; + if !messages.is_empty() { + let target = "INBOX".to_owned(); + let probe = flag_probe_message(); + client.append_with_flags(&target, &["\\Seen", "\\Flagged"], &probe)?; + *histogram.entry(target.clone()).or_insert(0) += 1; + appends.push(SeededAppend { + raw: probe, + target: target.clone(), + flags: vec!["$seen".to_owned(), "$flagged".to_owned()], + tag: SeedTag::FlagProbe, + }); + flagged_target = Some(target); + } + + let mut nomid_target: Option = None; + if !messages.is_empty() { + let target = "INBOX".to_owned(); + let probe = no_message_id_probe(); + client.append_with_flags(&target, &[], &probe)?; + *histogram.entry(target.clone()).or_insert(0) += 1; + appends.push(SeededAppend { + raw: probe, + target: target.clone(), + flags: Vec::new(), + tag: SeedTag::NoMessageId, + }); + nomid_target = Some(target); + } + + let mut mid_dedup_targets: Option<(String, String)> = None; + if targets.len() >= 3 { + let body = shared_message_id_probe(); + let a = targets[1].clone(); + let b = targets[2].clone(); + client.append_with_flags(&a, &[], &body)?; + client.append_with_flags(&b, &[], &body)?; + *histogram.entry(a.clone()).or_insert(0) += 1; + *histogram.entry(b.clone()).or_insert(0) += 1; + appends.push(SeededAppend { + raw: body.clone(), + target: a.clone(), + flags: Vec::new(), + tag: SeedTag::SharedMid, + }); + appends.push(SeededAppend { + raw: body, + target: b.clone(), + flags: Vec::new(), + tag: SeedTag::SharedMid, + }); + mid_dedup_targets = Some((a, b)); + } + + client.logout()?; + + let extras = ExtraAppends { + dedup_target, + flagged_target, + nomid_target, + mid_dedup_targets, + }; + + let total_appends = histogram_total(&histogram); + Ok(MailboxSeed { + paths, + histogram, + total_appends, + extras, + appends, + }) + } + + fn seed_sieve(&self, account: &Account) -> ContainerResult { + if account.layout.sieve_scripts.is_empty() { + return Ok(SieveSeed::default()); + } + let mut client = SieveClient::connect_seed(&self.sieve.host, self.sieve.port)?; + client.authenticate(&account.username, &account.password)?; + let mut active_name: Option<&'static str> = None; + let mut names = Vec::new(); + for script in account.layout.sieve_scripts { + client.putscript(script.name, script.body)?; + names.push(script.name.to_owned()); + if script.active { + active_name = Some(script.name); + } + } + if let Some(name) = active_name { + client.setactive(name)?; + } + client.logout()?; + Ok(SieveSeed { + names, + active: active_name.map(str::to_owned), + }) + } + + pub fn delete_first_inbox_message(&self, account: &Account) -> ContainerResult<()> { + let mut client = ImapSeed::connect(&self.imap.host, self.imap.port)?; + client.login(&account.username, &account.password)?; + client.delete_and_expunge_first("INBOX")?; + client.logout()?; + Ok(()) + } + + pub fn append_new_message( + &self, + account: &Account, + mailbox: &str, + tag: &str, + ) -> ContainerResult<(Vec, String)> { + let mut client = ImapSeed::connect(&self.imap.host, self.imap.port)?; + client.login(&account.username, &account.password)?; + let message_id = format!("", account.username); + let body = format!( + "From: added-{tag}@vandelay.test\r\n\ + To: {}@vandelay.test\r\n\ + Subject: Added probe {tag}\r\n\ + Message-ID: {message_id}\r\n\ + Date: Wed, 01 Jan 2025 12:00:00 +0000\r\n\ + \r\n\ + Added probe body {tag}.\r\n", + account.username + ); + let raw = body.into_bytes(); + client.append_with_flags(mailbox, &[], &raw)?; + client.logout()?; + Ok((raw, message_id)) + } + + pub fn install_broken_sieve(&self, account: &Account) -> ContainerResult { + let mut client = SieveClient::connect_seed(&self.sieve.host, self.sieve.port)?; + client.authenticate(&account.username, &account.password)?; + let name = "broken-script"; + client.putscript_raw(name, "INVALID:::not_a_sieve_program;;;")?; + client.logout()?; + Ok(name.to_owned()) + } + + pub fn verify_seed(&self, seeds: &[AccountSeed]) -> ContainerResult<()> { + for (acct, seed) in self.accounts.iter().zip(seeds) { + let mut client = ImapSeed::connect(&self.imap.host, self.imap.port)?; + client.login(&acct.username, &acct.password)?; + let names = client.list_all()?; + let expected = acct.layout.mailboxes.len() + 1; + if names.len() < expected { + return Err(ContainerError::Seed(format!( + "{}: expected >= {expected} mailboxes, got {}", + acct.username, + names.len() + ))); + } + let inbox_n = client.select("INBOX")?; + if acct.layout.email_count > 0 && inbox_n == 0 { + return Err(ContainerError::Seed(format!( + "{}: INBOX EXISTS = 0 after seed", + acct.username + ))); + } + for path in &seed.mailbox.paths { + let n = client.select(path)?; + let want = seed.mailbox.histogram.get(path).copied().unwrap_or(0); + if n < want { + return Err(ContainerError::Seed(format!( + "{}: mailbox {path} EXISTS={n} but {want} were appended", + acct.username + ))); + } + } + client.logout()?; + } + Ok(()) + } + + pub fn stop(self) -> ContainerResult<()> { + self.container.stop()?; + Ok(()) + } +} + +fn histogram_total(h: &HashMap) -> usize { + h.values().sum() +} + +pub fn flag_probe_message() -> Vec { + let body = "From: probe-flags@vandelay.test\r\n\ + To: user@vandelay.test\r\n\ + Subject: Flag probe\r\n\ + Message-ID: \r\n\ + Date: Wed, 01 Jan 2025 12:00:00 +0000\r\n\ + \r\n\ + Flag probe body.\r\n"; + body.as_bytes().to_vec() +} + +pub fn no_message_id_probe() -> Vec { + let body = "From: nomid@vandelay.test\r\n\ + To: user@vandelay.test\r\n\ + Subject: No Message-ID probe\r\n\ + Date: Wed, 01 Jan 2025 12:00:00 +0000\r\n\ + \r\n\ + No Message-ID body.\r\n"; + body.as_bytes().to_vec() +} + +pub fn shared_message_id_probe() -> Vec { + let body = "From: shared@vandelay.test\r\n\ + To: user@vandelay.test\r\n\ + Subject: Shared MID probe\r\n\ + Message-ID: \r\n\ + Date: Wed, 01 Jan 2025 12:00:00 +0000\r\n\ + \r\n\ + Shared MID body.\r\n"; + body.as_bytes().to_vec() +} + +#[derive(Debug, Clone)] +pub struct MailboxSeed { + pub paths: Vec, + pub histogram: HashMap, + pub total_appends: usize, + pub extras: ExtraAppends, + pub appends: Vec, +} + +#[derive(Debug, Clone)] +pub struct SeededAppend { + pub raw: Vec, + pub target: String, + pub flags: Vec, + pub tag: SeedTag, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SeedTag { + Bulk(usize), + Dedup, + FlagProbe, + NoMessageId, + SharedMid, +} + +#[derive(Debug, Clone)] +pub struct ExtraAppends { + pub dedup_target: Option, + pub flagged_target: Option, + pub nomid_target: Option, + pub mid_dedup_targets: Option<(String, String)>, +} + +#[derive(Debug, Default, Clone)] +pub struct SieveSeed { + pub names: Vec, + pub active: Option, +} + +#[derive(Debug, Clone)] +pub struct AccountSeed { + pub username: String, + pub mailbox: MailboxSeed, + pub sieve: SieveSeed, +} diff --git a/tests/integration/error.rs b/tests/integration/error.rs new file mode 100644 index 0000000..1317541 --- /dev/null +++ b/tests/integration/error.rs @@ -0,0 +1,37 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io; + +use thiserror::Error; + +pub type ContainerResult = Result; + +#[derive(Debug, Error)] +pub enum ContainerError { + #[error("io: {0}")] + Io(#[from] io::Error), + #[error("testcontainers: {0}")] + Testcontainers(#[from] testcontainers::TestcontainersError), + #[error("ureq: {0}")] + Ureq(#[from] Box), + #[error("utf8: {0}")] + Utf8(#[from] std::string::FromUtf8Error), + #[error("env: {0}")] + Env(String), + #[error("protocol: {0}")] + Protocol(String), + #[error("seed: {0}")] + Seed(String), + #[error("resource: {0}")] + Resource(String), +} + +impl From for ContainerError { + fn from(e: ureq::Error) -> Self { + ContainerError::Ureq(Box::new(e)) + } +} diff --git a/tests/integration/imap_client.rs b/tests/integration/imap_client.rs new file mode 100644 index 0000000..a2a770a --- /dev/null +++ b/tests/integration/imap_client.rs @@ -0,0 +1,283 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::TcpStream; +use std::time::Duration; + +use super::error::{ContainerError, ContainerResult}; +use super::layouts::MailboxSpec; + +pub struct ImapSeed { + reader: BufReader, + writer: TcpStream, + tag_seq: u32, + pub separator: char, +} + +impl ImapSeed { + pub fn connect(host: &str, port: u16) -> ContainerResult { + let deadline = std::time::Instant::now() + Duration::from_secs(30); + let stream = loop { + match TcpStream::connect((host, port)) { + Ok(s) => break s, + Err(e) => { + if std::time::Instant::now() >= deadline { + return Err(e.into()); + } + std::thread::sleep(Duration::from_millis(250)); + } + } + }; + stream.set_read_timeout(Some(Duration::from_secs(30)))?; + stream.set_write_timeout(Some(Duration::from_secs(30)))?; + let writer = stream.try_clone()?; + let mut me = Self { + reader: BufReader::new(stream), + writer, + tag_seq: 0, + separator: '/', + }; + let greeting = me.read_line()?; + if !greeting.starts_with("* OK") && !greeting.starts_with("* PREAUTH") { + return Err(ContainerError::Protocol(format!( + "imap greeting unexpected: {greeting}" + ))); + } + Ok(me) + } + + fn next_tag(&mut self) -> String { + self.tag_seq += 1; + format!("a{:04}", self.tag_seq) + } + + fn read_line(&mut self) -> ContainerResult { + let mut line = String::new(); + let n = self.reader.read_line(&mut line)?; + if n == 0 { + return Err(ContainerError::Protocol("imap eof".to_owned())); + } + if line.ends_with('\n') { + line.pop(); + if line.ends_with('\r') { + line.pop(); + } + } + Ok(line) + } + + fn read_tagged(&mut self, tag: &str) -> ContainerResult> { + let mut untagged = Vec::new(); + loop { + let line = self.read_line()?; + if let Some(rest) = line.strip_prefix(&format!("{tag} ")) { + if rest.starts_with("OK") { + return Ok(untagged); + } + return Err(ContainerError::Protocol(format!( + "imap tagged failure: {line}" + ))); + } + untagged.push(line); + } + } + + fn send(&mut self, line: &str) -> ContainerResult<()> { + self.writer.write_all(line.as_bytes())?; + self.writer.write_all(b"\r\n")?; + self.writer.flush()?; + Ok(()) + } + + pub fn login(&mut self, user: &str, password: &str) -> ContainerResult<()> { + let tag = self.next_tag(); + let escaped_user = quote(user); + let escaped_pw = quote(password); + self.send(&format!("{tag} LOGIN {escaped_user} {escaped_pw}"))?; + self.read_tagged(&tag)?; + Ok(()) + } + + pub fn discover_separator(&mut self) -> ContainerResult { + let tag = self.next_tag(); + self.send(&format!(r#"{tag} LIST "" """#))?; + let untagged = self.read_tagged(&tag)?; + for line in &untagged { + if let Some(rest) = line.strip_prefix("* LIST ") + && let Some(open) = rest.find('"') + && let Some(close) = rest[open + 1..].find('"') + { + let sep = &rest[open + 1..open + 1 + close]; + if let Some(c) = sep.chars().next() { + self.separator = c; + return Ok(c); + } + } + } + Ok('/') + } + + pub fn create(&mut self, full_path: &str) -> ContainerResult<()> { + let tag = self.next_tag(); + self.send(&format!("{tag} CREATE {}", quote(full_path)))?; + match self.read_tagged(&tag) { + Ok(_) => Ok(()), + Err(ContainerError::Protocol(msg)) if msg.contains("already exists") => Ok(()), + Err(e) => Err(e), + } + } + + pub fn subscribe(&mut self, full_path: &str) -> ContainerResult<()> { + let tag = self.next_tag(); + self.send(&format!("{tag} SUBSCRIBE {}", quote(full_path)))?; + let _ = self.read_tagged(&tag); + Ok(()) + } + + pub fn append(&mut self, mailbox: &str, message: &[u8]) -> ContainerResult<()> { + self.append_with_flags(mailbox, &[], message) + } + + pub fn append_with_flags( + &mut self, + mailbox: &str, + flags: &[&str], + message: &[u8], + ) -> ContainerResult<()> { + let tag = self.next_tag(); + let flag_clause = if flags.is_empty() { + String::new() + } else { + format!(" ({})", flags.join(" ")) + }; + let cmd = format!( + "{tag} APPEND {}{flag_clause} {{{}}}\r\n", + quote(mailbox), + message.len(), + ); + self.writer.write_all(cmd.as_bytes())?; + self.writer.flush()?; + let cont = self.read_line()?; + if !cont.starts_with('+') { + return Err(ContainerError::Protocol(format!( + "append expected continuation, got: {cont}" + ))); + } + self.writer.write_all(message)?; + self.writer.write_all(b"\r\n")?; + self.writer.flush()?; + self.read_tagged(&tag)?; + Ok(()) + } + + pub fn list_all(&mut self) -> ContainerResult> { + let tag = self.next_tag(); + self.send(&format!(r#"{tag} LIST "" "*""#))?; + let untagged = self.read_tagged(&tag)?; + let mut names = Vec::new(); + for line in untagged { + if let Some(rest) = line.strip_prefix("* LIST ") + && let Some(name) = parse_list_name(rest) + { + names.push(name); + } + } + Ok(names) + } + + pub fn select(&mut self, mailbox: &str) -> ContainerResult { + let tag = self.next_tag(); + self.send(&format!("{tag} SELECT {}", quote(mailbox)))?; + let untagged = self.read_tagged(&tag)?; + for line in untagged { + if let Some(rest) = line.strip_prefix("* ") + && let Some(num) = rest.strip_suffix(" EXISTS") + && let Ok(n) = num.parse::() + { + return Ok(n); + } + } + Ok(0) + } + + pub fn delete_and_expunge_first(&mut self, mailbox: &str) -> ContainerResult<()> { + self.select(mailbox)?; + let store_tag = self.next_tag(); + self.send(&format!("{store_tag} STORE 1 +FLAGS (\\Deleted)"))?; + self.read_tagged(&store_tag)?; + let expunge_tag = self.next_tag(); + self.send(&format!("{expunge_tag} EXPUNGE"))?; + self.read_tagged(&expunge_tag)?; + Ok(()) + } + + pub fn logout(&mut self) -> ContainerResult<()> { + let tag = self.next_tag(); + self.send(&format!("{tag} LOGOUT"))?; + let mut buf = String::new(); + let _ = self.reader.read_to_string(&mut buf); + Ok(()) + } +} + +pub fn full_path(specs: &[MailboxSpec], key: &str, separator: char) -> Option { + let mut chain: Vec<&str> = Vec::new(); + let mut cur = key; + loop { + let spec = specs.iter().find(|s| s.key == cur)?; + chain.push(spec.name); + match spec.parent { + Some(p) => cur = p, + None => break, + } + } + chain.reverse(); + Some(chain.join(&separator.to_string())) +} + +fn quote(s: &str) -> String { + let mut out = String::with_capacity(s.len() + 2); + out.push('"'); + for c in s.chars() { + if c == '\\' || c == '"' { + out.push('\\'); + } + out.push(c); + } + out.push('"'); + out +} + +fn parse_list_name(line: &str) -> Option { + let mut chars = line.chars(); + let _ = chars.next()?; + let mut depth = 1; + let mut idx = 1; + for (i, c) in line[1..].char_indices() { + match c { + '(' => depth += 1, + ')' => { + depth -= 1; + if depth == 0 { + idx = i + 2; + break; + } + } + _ => {} + } + } + let rest = line[idx..].trim_start(); + let mut parts = rest.splitn(2, ' '); + let _sep_token = parts.next()?; + let name_token = parts.next()?.trim(); + if let Some(stripped) = name_token.strip_prefix('"') { + let end = stripped.find('"')?; + Some(stripped[..end].to_owned()) + } else { + Some(name_token.to_owned()) + } +} diff --git a/tests/integration/layouts.rs b/tests/integration/layouts.rs new file mode 100644 index 0000000..688d41b --- /dev/null +++ b/tests/integration/layouts.rs @@ -0,0 +1,223 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +#[derive(Debug, Clone, Copy)] +pub struct MailboxSpec { + pub key: &'static str, + pub name: &'static str, + pub parent: Option<&'static str>, +} + +#[derive(Debug, Clone, Copy)] +pub struct FileSpec { + pub key: &'static str, + pub name: &'static str, + pub parent: Option<&'static str>, + pub directory: bool, +} + +#[derive(Debug, Clone, Copy)] +pub struct SieveScript { + pub name: &'static str, + pub body: &'static str, + pub active: bool, +} + +#[derive(Debug, Clone, Copy)] +pub struct Layout { + pub mailboxes: &'static [MailboxSpec], + pub files: &'static [FileSpec], + pub calendars: &'static [&'static str], + pub address_books: &'static [&'static str], + pub sieve_scripts: &'static [SieveScript], + pub email_count: usize, + pub contact_count: usize, + pub event_count: usize, +} + +macro_rules! mailbox { + ($k:expr, $n:expr, $p:expr) => { + MailboxSpec { + key: $k, + name: $n, + parent: $p, + } + }; +} + +macro_rules! node { + ($k:expr, $n:expr, $p:expr, $d:expr) => { + FileSpec { + key: $k, + name: $n, + parent: $p, + directory: $d, + } + }; +} + +pub const ACCOUNT1: &str = "user1"; +pub const ACCOUNT2: &str = "user2"; +pub const ACCOUNT3: &str = "user3"; + +pub const PASSWORD: &str = "VandelayUser#2026"; + +pub static MAILBOXES_DEEP: &[MailboxSpec] = &[ + mailbox!("proj", "Projects", None), + mailbox!("y2024", "2024", Some("proj")), + mailbox!("q1", "Q1", Some("y2024")), + mailbox!("acme", "ClientAcme", Some("q1")), + mailbox!("acme_in", "Internal", Some("acme")), + mailbox!("q2", "Q2", Some("y2024")), + mailbox!("globex", "ClientGlobex", Some("q2")), + mailbox!("y2025", "2025", Some("proj")), + mailbox!("alpha", "Alpha", Some("y2025")), + mailbox!("beta", "Beta", Some("y2025")), + mailbox!("travel", "Travel", None), + mailbox!("asia", "Asia", Some("travel")), + mailbox!("jp", "Japan", Some("asia")), + mailbox!("tokyo", "Tokyo", Some("jp")), + mailbox!("eu", "Europe", Some("travel")), + mailbox!("it", "Italy", Some("eu")), + mailbox!("rome", "Rome", Some("it")), + mailbox!("arch", "Archive", None), + mailbox!("arch23", "2023", Some("arch")), + mailbox!("arch24", "2024", Some("arch")), +]; + +pub static MAILBOXES_MEDIUM: &[MailboxSpec] = &[ + mailbox!("work", "Work", None), + mailbox!("clients", "Clients", Some("work")), + mailbox!("acme", "AcmeCorp", Some("clients")), + mailbox!("inv", "Invoices", Some("acme")), + mailbox!("globex", "Globex", Some("clients")), + mailbox!("internal", "Internal", Some("work")), + mailbox!("pers", "Personal", None), + mailbox!("rcpt", "Receipts", Some("pers")), +]; + +pub static MAILBOXES_FLAT: &[MailboxSpec] = &[ + mailbox!("lists", "Lists", None), + mailbox!("news", "Newsletters", Some("lists")), + mailbox!("misc", "Misc", None), +]; + +pub static FILES_DEEP: &[FileSpec] = &[ + node!("docs", "Documents", None, true), + node!("rep", "Reports", Some("docs"), true), + node!("d24", "2024", Some("rep"), true), + node!("q1f", "q1-results.bin", Some("d24"), false), + node!("q2f", "q2-results.bin", Some("d24"), false), + node!("d25", "2025", Some("rep"), true), + node!("q1f25", "q1-results.bin", Some("d25"), false), + node!("spec", "Specs", Some("docs"), true), + node!("ovr", "overview.bin", Some("spec"), false), + node!("notes", "notes.bin", Some("docs"), false), + node!("photos", "Photos", None, true), + node!("vac", "Vacations", Some("photos"), true), + node!("vit", "Italy", Some("vac"), true), + node!("vrome", "rome.bin", Some("vit"), false), + node!("vjp", "Japan", Some("vac"), true), + node!("vtokyo", "tokyo.bin", Some("vjp"), false), + node!("readme", "README.bin", None, false), +]; + +pub static FILES_MEDIUM: &[FileSpec] = &[ + node!("media", "Media", None, true), + node!("audio", "Audio", Some("media"), true), + node!("song", "song.bin", Some("audio"), false), + node!("video", "Video", Some("media"), true), + node!("clip", "clip.bin", Some("video"), false), + node!("proj", "Projects", None, true), + node!("src", "src", Some("proj"), true), + node!("main", "main.bin", Some("src"), false), +]; + +pub static FILES_FLAT: &[FileSpec] = &[ + node!("bak", "Backup", None, true), + node!("snap", "snapshot.bin", Some("bak"), false), + node!("scratch", "scratch.bin", None, false), +]; + +pub static CALENDARS_DEEP: &[&str] = &["Personal", "Work", "Birthdays", "Travel"]; +pub static CALENDARS_MEDIUM: &[&str] = &["Personal", "Work"]; +pub static CALENDARS_FLAT: &[&str] = &["Personal", "Holidays"]; + +pub static ADDRESS_BOOKS_DEEP: &[&str] = &["Personal", "Work", "Friends"]; +pub static ADDRESS_BOOKS_MEDIUM: &[&str] = &["Personal", "Work"]; +pub static ADDRESS_BOOKS_FLAT: &[&str] = &["Personal", "Family"]; + +pub static SIEVE_DEEP: &[SieveScript] = &[ + SieveScript { + name: "spam", + body: "require [\"fileinto\"];\nif header :contains \"X-Spam\" \"YES\" {\n fileinto \"Junk\";\n stop;\n}\n", + active: true, + }, + SieveScript { + name: "vacation", + body: "require [\"vacation\"];\nvacation :days 7 :subject \"Out of office\" \"I am away, will reply later.\";\n", + active: false, + }, + SieveScript { + name: "organize", + body: "require [\"fileinto\",\"mailbox\"];\nif address :is \"from\" \"newsletter@example.com\" {\n fileinto :create \"Projects/2025/Beta\";\n}\n", + active: false, + }, +]; + +pub static SIEVE_MEDIUM: &[SieveScript] = &[ + SieveScript { + name: "filter", + body: "require [\"fileinto\"];\nif address :is \"to\" \"work@example.com\" {\n fileinto \"Work\";\n}\n", + active: true, + }, + SieveScript { + name: "draft", + body: "require [\"reject\"];\nif address :is \"from\" \"blocked@example.com\" {\n reject \"go away\";\n}\n", + active: false, + }, +]; + +pub static SIEVE_FLAT: &[SieveScript] = &[]; + +pub fn layout_for(username: &str) -> Layout { + match username { + s if s == ACCOUNT1 => Layout { + mailboxes: MAILBOXES_DEEP, + files: FILES_DEEP, + calendars: CALENDARS_DEEP, + address_books: ADDRESS_BOOKS_DEEP, + sieve_scripts: SIEVE_DEEP, + email_count: 200, + contact_count: 40, + event_count: 40, + }, + s if s == ACCOUNT2 => Layout { + mailboxes: MAILBOXES_MEDIUM, + files: FILES_MEDIUM, + calendars: CALENDARS_MEDIUM, + address_books: ADDRESS_BOOKS_MEDIUM, + sieve_scripts: SIEVE_MEDIUM, + email_count: 80, + contact_count: 15, + event_count: 15, + }, + _ => Layout { + mailboxes: MAILBOXES_FLAT, + files: FILES_FLAT, + calendars: CALENDARS_FLAT, + address_books: ADDRESS_BOOKS_FLAT, + sieve_scripts: SIEVE_FLAT, + email_count: 20, + contact_count: 10, + event_count: 10, + }, + } +} + +pub fn accounts() -> [&'static str; 3] { + [ACCOUNT1, ACCOUNT2, ACCOUNT3] +} diff --git a/tests/integration/mod.rs b/tests/integration/mod.rs new file mode 100644 index 0000000..e0b04dc --- /dev/null +++ b/tests/integration/mod.rs @@ -0,0 +1,49 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +#![allow(dead_code, unused_imports)] + +pub mod baikal; +pub mod cyrus; +pub mod data; +pub mod dav_client; +pub mod dovecot; +pub mod error; +pub mod imap_client; +pub mod layouts; +pub mod radicale; +pub mod sieve_client; +pub mod stalwart; +pub mod validate; +pub mod webdav; + +pub use error::{ContainerError, ContainerResult}; + +#[derive(Debug, Clone)] +pub struct Endpoint { + pub host: String, + pub port: u16, +} + +impl Endpoint { + pub fn new(host: impl Into, port: u16) -> Self { + Self { + host: host.into(), + port, + } + } + + pub fn http_base(&self) -> String { + format!("http://{}:{}", self.host, self.port) + } +} + +#[derive(Debug, Clone)] +pub struct Account { + pub username: String, + pub password: String, + pub layout: layouts::Layout, +} diff --git a/tests/integration/radicale.rs b/tests/integration/radicale.rs new file mode 100644 index 0000000..f35e630 --- /dev/null +++ b/tests/integration/radicale.rs @@ -0,0 +1,385 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashSet; +use std::time::Duration; + +use testcontainers::core::{IntoContainerPort, WaitFor}; +use testcontainers::runners::SyncRunner; +use testcontainers::{Container, GenericImage, ImageExt}; + +use super::data::{RawFixture, load_icals, load_vcards, malformed_ical, rewrite_uid}; +use super::dav_client::DavSeed; +use super::error::{ContainerError, ContainerResult}; +use super::layouts::{self, Layout}; +use super::{Account, Endpoint}; + +const IMAGE_NAME: &str = "tomsquest/docker-radicale"; +const IMAGE_TAG: &str = "3.7.3.0"; +const RADICALE_PORT: u16 = 5232; + +const CONFIG: &str = "[server] +hosts = 0.0.0.0:5232 + +[auth] +type = htpasswd +htpasswd_filename = /config/users +htpasswd_encryption = plain + +[storage] +filesystem_folder = /data/collections +"; + +pub struct Radicale { + container: Container, + pub endpoint: Endpoint, + pub accounts: Vec, +} + +impl Radicale { + pub fn start() -> ContainerResult { + let mut users = String::new(); + for u in layouts::accounts() { + users.push_str(u); + users.push(':'); + users.push_str(layouts::PASSWORD); + users.push('\n'); + } + + let image = GenericImage::new(IMAGE_NAME, IMAGE_TAG) + .with_exposed_port(RADICALE_PORT.tcp()) + .with_wait_for(WaitFor::message_on_stderr("Radicale server ready")); + + let request = image + .with_copy_to("/config/config", CONFIG.as_bytes().to_vec()) + .with_copy_to("/config/users", users.into_bytes()) + .with_startup_timeout(Duration::from_secs(90)); + + let container = request.start()?; + let host = container.get_host()?.to_string(); + let port = container.get_host_port_ipv4(RADICALE_PORT.tcp())?; + + let accounts: Vec = layouts::accounts() + .iter() + .map(|name| Account { + username: (*name).to_owned(), + password: layouts::PASSWORD.to_owned(), + layout: layouts::layout_for(name), + }) + .collect(); + + Ok(Self { + container, + endpoint: Endpoint::new(host, port), + accounts, + }) + } + + pub fn base_url(&self) -> String { + self.endpoint.http_base() + } + + pub fn account_url(&self, account: &Account) -> String { + format!("{}/{}/", self.base_url(), account.username) + } + + pub fn seed_all(&self) -> ContainerResult> { + let icals = load_icals()?; + let vcards = load_vcards()?; + let mut out = Vec::new(); + for acct in &self.accounts { + let seed = self.seed_account(acct, &icals, &vcards)?; + out.push(seed); + } + Ok(out) + } + + pub fn seed_account( + &self, + account: &Account, + icals: &[RawFixture], + vcards: &[RawFixture], + ) -> ContainerResult { + let client = DavSeed::new(self.base_url(), &account.username, &account.password); + let layout = &account.layout; + let mut seed = AccountSeed::new(account.username.clone()); + + for (idx, cal) in layout.calendars.iter().enumerate() { + let collection = collection_segment("cal", idx, cal); + let base = format!("/{}/{}/", account.username, collection); + let body = format!( + r#" + + + + {cal} + +"# + ); + client.mkcol(&base, Some(&body))?; + let mut plan = CollectionPlan::new((*cal).to_owned(), base.clone()); + + let count = events_per_calendar(layout, idx); + let offset = events_offset(layout, idx); + for (i, fixture) in icals.iter().cycle().skip(offset).take(count).enumerate() { + let path = format!("{base}{}-{}.ics", fixture.name, i); + let suffix = format!("{collection}-{i}"); + let (bytes, uid) = rewrite_uid(&fixture.bytes, &suffix).ok_or_else(|| { + ContainerError::Seed(format!("ical fixture {} has no UID", fixture.name)) + })?; + client.put(&path, "text/calendar; charset=utf-8", &bytes)?; + plan.items.push(SeededItem { + uid, + href: path, + source: bytes, + }); + } + seed.calendars.push(plan); + } + + for (idx, ab) in layout.address_books.iter().enumerate() { + let collection = collection_segment("ab", idx, ab); + let base = format!("/{}/{}/", account.username, collection); + let body = format!( + r#" + + + + {ab} + +"# + ); + client.mkcol(&base, Some(&body))?; + let mut plan = CollectionPlan::new((*ab).to_owned(), base.clone()); + + let count = contacts_per_book(layout, idx); + let offset = contacts_offset(layout, idx); + for (i, fixture) in vcards.iter().cycle().skip(offset).take(count).enumerate() { + let path = format!("{base}{}-{}.vcf", fixture.name, i); + let suffix = format!("{collection}-{i}"); + let (bytes, uid) = rewrite_uid(&fixture.bytes, &suffix).ok_or_else(|| { + ContainerError::Seed(format!("vcard fixture {} has no UID", fixture.name)) + })?; + client.put(&path, "text/vcard; charset=utf-8", &bytes)?; + plan.items.push(SeededItem { + uid, + href: path, + source: bytes, + }); + } + seed.address_books.push(plan); + } + + Ok(seed) + } + + pub fn seed_malformed_event( + &self, + account: &Account, + calendar_segment_idx: usize, + tag: &str, + ) -> ContainerResult { + let client = DavSeed::new(self.base_url(), &account.username, &account.password); + let collection = collection_segment( + "cal", + calendar_segment_idx, + account.layout.calendars[calendar_segment_idx], + ); + let path = format!("/{}/{}/broken-{tag}.ics", account.username, collection); + let bad = malformed_ical(tag); + client.put(&path, "text/calendar; charset=utf-8", &bad.bytes)?; + Ok(path) + } + + pub fn delete_item(&self, account: &Account, path: &str) -> ContainerResult<()> { + let client = DavSeed::new(self.base_url(), &account.username, &account.password); + client.delete(path)?; + Ok(()) + } + + pub fn add_event( + &self, + account: &Account, + calendar_segment_idx: usize, + tag: &str, + icals: &[RawFixture], + ) -> ContainerResult<(String, String)> { + let client = DavSeed::new(self.base_url(), &account.username, &account.password); + let display = account.layout.calendars[calendar_segment_idx]; + let collection = collection_segment("cal", calendar_segment_idx, display); + let fixture = icals.first().ok_or_else(|| { + ContainerError::Seed("no ical fixture available for add_event".to_owned()) + })?; + let suffix = format!("added-{tag}"); + let (bytes, uid) = rewrite_uid(&fixture.bytes, &suffix) + .ok_or_else(|| ContainerError::Seed("ical fixture has no UID".to_owned()))?; + let path = format!("/{}/{collection}/added-{tag}.ics", account.username); + client.put(&path, "text/calendar; charset=utf-8", &bytes)?; + Ok((path, uid)) + } + + pub fn add_contact( + &self, + account: &Account, + book_segment_idx: usize, + tag: &str, + vcards: &[RawFixture], + ) -> ContainerResult<(String, String)> { + let client = DavSeed::new(self.base_url(), &account.username, &account.password); + let display = account.layout.address_books[book_segment_idx]; + let collection = collection_segment("ab", book_segment_idx, display); + let fixture = vcards.first().ok_or_else(|| { + ContainerError::Seed("no vcard fixture available for add_contact".to_owned()) + })?; + let suffix = format!("added-{tag}"); + let (bytes, uid) = rewrite_uid(&fixture.bytes, &suffix) + .ok_or_else(|| ContainerError::Seed("vcard fixture has no UID".to_owned()))?; + let path = format!("/{}/{collection}/added-{tag}.vcf", account.username); + client.put(&path, "text/vcard; charset=utf-8", &bytes)?; + Ok((path, uid)) + } + + pub fn verify_seed(&self, seeds: &[AccountSeed]) -> ContainerResult<()> { + for (acct, seed) in self.accounts.iter().zip(seeds) { + let client = DavSeed::new(self.base_url(), &acct.username, &acct.password); + let body = client.propfind(&format!("/{}/", acct.username), 1)?; + if !body.contains("multistatus") { + return Err(ContainerError::Protocol(format!( + "radicale propfind for {} returned no multistatus", + acct.username + ))); + } + let expected_cal_names: HashSet<&str> = seed + .calendars + .iter() + .map(|c| c.display_name.as_str()) + .collect(); + let expected_ab_names: HashSet<&str> = seed + .address_books + .iter() + .map(|c| c.display_name.as_str()) + .collect(); + if expected_cal_names.is_empty() && expected_ab_names.is_empty() { + continue; + } + } + Ok(()) + } + + pub fn stop(self) -> ContainerResult<()> { + self.container.stop()?; + Ok(()) + } +} + +#[derive(Debug, Clone)] +pub struct SeededItem { + pub uid: String, + pub href: String, + pub source: Vec, +} + +#[derive(Debug, Clone)] +pub struct CollectionPlan { + pub display_name: String, + pub base_href: String, + pub items: Vec, +} + +impl CollectionPlan { + fn new(display_name: String, base_href: String) -> Self { + Self { + display_name, + base_href, + items: Vec::new(), + } + } +} + +#[derive(Debug, Clone)] +pub struct AccountSeed { + pub username: String, + pub calendars: Vec, + pub address_books: Vec, +} + +impl AccountSeed { + fn new(username: String) -> Self { + Self { + username, + calendars: Vec::new(), + address_books: Vec::new(), + } + } + + pub fn total_events(&self) -> usize { + self.calendars.iter().map(|c| c.items.len()).sum() + } + + pub fn total_contacts(&self) -> usize { + self.address_books.iter().map(|c| c.items.len()).sum() + } + + pub fn event_uids(&self) -> HashSet { + self.calendars + .iter() + .flat_map(|c| c.items.iter().map(|i| i.uid.clone())) + .collect() + } + + pub fn contact_uids(&self) -> HashSet { + self.address_books + .iter() + .flat_map(|c| c.items.iter().map(|i| i.uid.clone())) + .collect() + } +} + +fn collection_segment(prefix: &str, idx: usize, name: &str) -> String { + let mut out = format!("{prefix}-{:02}-", idx); + for c in name.chars() { + if c.is_ascii_alphanumeric() { + out.push(c.to_ascii_lowercase()); + } else if !out.ends_with('-') { + out.push('-'); + } + } + out.trim_end_matches('-').to_owned() +} + +fn events_per_calendar(layout: &Layout, idx: usize) -> usize { + if layout.calendars.is_empty() { + return 0; + } + let base = layout.event_count / layout.calendars.len().max(1); + let extra = if idx == 0 { + layout.event_count % layout.calendars.len() + } else { + 0 + }; + base + extra +} + +fn events_offset(layout: &Layout, idx: usize) -> usize { + (0..idx).map(|i| events_per_calendar(layout, i)).sum() +} + +fn contacts_per_book(layout: &Layout, idx: usize) -> usize { + if layout.address_books.is_empty() { + return 0; + } + let base = layout.contact_count / layout.address_books.len().max(1); + let extra = if idx == 0 { + layout.contact_count % layout.address_books.len() + } else { + 0 + }; + base + extra +} + +fn contacts_offset(layout: &Layout, idx: usize) -> usize { + (0..idx).map(|i| contacts_per_book(layout, i)).sum() +} diff --git a/tests/integration/sieve_client.rs b/tests/integration/sieve_client.rs new file mode 100644 index 0000000..c98a721 --- /dev/null +++ b/tests/integration/sieve_client.rs @@ -0,0 +1,150 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::TcpStream; +use std::time::Duration; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD as B64; + +use super::error::{ContainerError, ContainerResult}; + +pub struct SieveSeed { + reader: BufReader, + writer: TcpStream, +} + +impl SieveSeed { + pub fn connect_seed(host: &str, port: u16) -> ContainerResult { + Self::connect(host, port) + } + + pub fn connect(host: &str, port: u16) -> ContainerResult { + let stream = TcpStream::connect((host, port))?; + stream.set_read_timeout(Some(Duration::from_secs(30)))?; + stream.set_write_timeout(Some(Duration::from_secs(30)))?; + let writer = stream.try_clone()?; + let mut me = Self { + reader: BufReader::new(stream), + writer, + }; + me.read_until_ok()?; + Ok(me) + } + + fn read_line(&mut self) -> ContainerResult { + let mut line = String::new(); + let n = self.reader.read_line(&mut line)?; + if n == 0 { + return Err(ContainerError::Protocol("sieve eof".to_owned())); + } + if line.ends_with('\n') { + line.pop(); + if line.ends_with('\r') { + line.pop(); + } + } + Ok(line) + } + + fn read_until_ok(&mut self) -> ContainerResult> { + let mut lines = Vec::new(); + loop { + let line = self.read_line()?; + if line.starts_with("OK") { + return Ok(lines); + } + if line.starts_with("NO") || line.starts_with("BYE") { + return Err(ContainerError::Protocol(format!("sieve: {line}"))); + } + lines.push(line); + } + } + + fn write_all(&mut self, bytes: &[u8]) -> ContainerResult<()> { + self.writer.write_all(bytes)?; + self.writer.flush()?; + Ok(()) + } + + pub fn authenticate(&mut self, user: &str, password: &str) -> ContainerResult<()> { + let mut blob = Vec::new(); + blob.push(0u8); + blob.extend_from_slice(user.as_bytes()); + blob.push(0u8); + blob.extend_from_slice(password.as_bytes()); + let encoded = B64.encode(&blob); + let cmd = format!( + "AUTHENTICATE \"PLAIN\" {{{}+}}\r\n{encoded}\r\n", + encoded.len() + ); + self.write_all(cmd.as_bytes())?; + self.read_until_ok()?; + Ok(()) + } + + pub fn putscript(&mut self, name: &str, body: &str) -> ContainerResult<()> { + let cmd = format!( + "PUTSCRIPT \"{}\" {{{}+}}\r\n{}\r\n", + escape(name), + body.len(), + body + ); + self.write_all(cmd.as_bytes())?; + self.read_until_ok()?; + Ok(()) + } + + pub fn putscript_raw(&mut self, name: &str, body: &str) -> ContainerResult { + let cmd = format!( + "PUTSCRIPT \"{}\" {{{}+}}\r\n{}\r\n", + escape(name), + body.len(), + body + ); + self.write_all(cmd.as_bytes())?; + match self.read_until_ok() { + Ok(_) => Ok(true), + Err(ContainerError::Protocol(_)) => Ok(false), + Err(e) => Err(e), + } + } + + pub fn setactive(&mut self, name: &str) -> ContainerResult<()> { + self.write_all(format!("SETACTIVE \"{}\"\r\n", escape(name)).as_bytes())?; + self.read_until_ok()?; + Ok(()) + } + + pub fn listscripts(&mut self) -> ContainerResult> { + self.write_all(b"LISTSCRIPTS\r\n")?; + let lines = self.read_until_ok()?; + let mut scripts = Vec::new(); + for line in lines { + let trimmed = line.trim(); + if let Some(stripped) = trimmed.strip_prefix('"') { + let close = stripped.find('"').unwrap_or(stripped.len()); + let name = stripped[..close].to_owned(); + let rest = &stripped[close..]; + let active = rest.contains("ACTIVE"); + scripts.push((name, active)); + } + } + Ok(scripts) + } + + pub fn logout(&mut self) -> ContainerResult<()> { + let _ = self.write_all(b"LOGOUT\r\n"); + let mut buf = String::new(); + let _ = self.reader.read_to_string(&mut buf); + Ok(()) + } +} + +fn escape(s: &str) -> String { + s.replace('\\', "\\\\").replace('"', "\\\"") +} diff --git a/tests/integration/stalwart.rs b/tests/integration/stalwart.rs new file mode 100644 index 0000000..6b0a54a --- /dev/null +++ b/tests/integration/stalwart.rs @@ -0,0 +1,165 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::net::TcpListener; +use std::sync::{Arc, OnceLock}; +use std::time::{Duration, Instant}; + +use base64::Engine; +use serde_json::Value; +use testcontainers::core::{IntoContainerPort, WaitFor}; +use testcontainers::runners::SyncRunner; +use testcontainers::{Container, GenericImage, ImageExt}; +use ureq::Agent; +use ureq::config::RedirectAuthHeaders; +use ureq::tls::{TlsConfig, TlsProvider}; + +use super::error::{ContainerError, ContainerResult}; + +const IMAGE_NAME: &str = "stalwartlabs/stalwart"; +const IMAGE_TAG: &str = "latest"; + +const HTTPS_PORT: u16 = 443; +const IMAP_PORT: u16 = 143; +const IMAPS_PORT: u16 = 993; +const SIEVE_PORT: u16 = 4190; + +const CONFIG_PATH: &str = "/etc/stalwart/config.json"; +const CONFIG_JSON: &str = r#"{"@type":"RocksDb","blobSize":16834,"bufferSize":134217728,"path":"/var/lib/stalwart","poolWorkers":null}"#; + +pub const ADMIN_USER: &str = "admin"; +pub const ADMIN_PASSWORD: &str = "admin"; + +pub struct Stalwart { + _container: Container, + pub host: String, + pub https_port: u16, + pub imap_port: u16, + pub imaps_port: u16, + pub sieve_port: u16, + pub public_url: String, +} + +impl Stalwart { + pub fn start() -> ContainerResult { + let https_port = pick_free_port()?; + let imap_port = pick_free_port()?; + let imaps_port = pick_free_port()?; + let sieve_port = pick_free_port()?; + + let host = "127.0.0.1".to_owned(); + let public_url = format!("https://{host}:{https_port}"); + + let image = GenericImage::new(IMAGE_NAME, IMAGE_TAG) + .with_exposed_port(HTTPS_PORT.tcp()) + .with_exposed_port(IMAP_PORT.tcp()) + .with_exposed_port(IMAPS_PORT.tcp()) + .with_exposed_port(SIEVE_PORT.tcp()) + .with_wait_for(WaitFor::seconds(2)); + + let request = image + .with_env_var("STALWART_PUBLIC_URL", &public_url) + .with_env_var("STALWART_RECOVERY_ADMIN", "admin:admin") + .with_copy_to(CONFIG_PATH, CONFIG_JSON.as_bytes().to_vec()) + .with_mapped_port(https_port, HTTPS_PORT.tcp()) + .with_mapped_port(imap_port, IMAP_PORT.tcp()) + .with_mapped_port(imaps_port, IMAPS_PORT.tcp()) + .with_mapped_port(sieve_port, SIEVE_PORT.tcp()) + .with_startup_timeout(Duration::from_secs(180)); + + let container = request.start()?; + + let me = Self { + _container: container, + host, + https_port, + imap_port, + imaps_port, + sieve_port, + public_url, + }; + + me.wait_ready(Duration::from_secs(120))?; + Ok(me) + } + + pub fn base_url(&self) -> &str { + &self.public_url + } + + pub fn fetch_jmap_session(&self) -> ContainerResult { + let agent = build_agent(); + let auth = basic(ADMIN_USER, ADMIN_PASSWORD); + let url = format!("{}/.well-known/jmap", self.public_url); + let mut resp = agent.get(&url).header("Authorization", &auth).call()?; + let status = resp.status().as_u16(); + let text = resp + .body_mut() + .read_to_string() + .map_err(|e| ContainerError::Protocol(format!("body read: {e}")))?; + if status != 200 { + return Err(ContainerError::Protocol(format!( + "jmap session status {status}: {text}" + ))); + } + serde_json::from_str(&text) + .map_err(|e| ContainerError::Protocol(format!("jmap session parse: {e}"))) + } + + fn wait_ready(&self, total: Duration) -> ContainerResult<()> { + let deadline = Instant::now() + total; + let mut last_err = String::from("no probe attempted"); + while Instant::now() < deadline { + match self.fetch_jmap_session() { + Ok(v) if v.get("apiUrl").is_some() => return Ok(()), + Ok(v) => last_err = format!("session missing apiUrl: {v}"), + Err(e) => last_err = e.to_string(), + } + std::thread::sleep(Duration::from_millis(500)); + } + Err(ContainerError::Protocol(format!( + "stalwart did not become ready in {total:?}: {last_err}" + ))) + } +} + +static SHARED: OnceLock = OnceLock::new(); + +pub fn shared() -> &'static Stalwart { + SHARED.get_or_init(|| Stalwart::start().expect("start shared stalwart container")) +} + +fn pick_free_port() -> ContainerResult { + let listener = TcpListener::bind("127.0.0.1:0")?; + Ok(listener.local_addr()?.port()) +} + +fn build_agent() -> Agent { + let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + let tls = TlsConfig::builder() + .provider(TlsProvider::Rustls) + .unversioned_rustls_crypto_provider(provider) + .disable_verification(true) + .build(); + Agent::config_builder() + .tls_config(tls) + .http_status_as_error(false) + .max_redirects(10) + .redirect_auth_headers(RedirectAuthHeaders::SameHost) + .build() + .new_agent() +} + +fn basic(user: &str, password: &str) -> String { + let mut raw = String::with_capacity(user.len() + password.len() + 1); + raw.push_str(user); + raw.push(':'); + raw.push_str(password); + format!( + "Basic {}", + base64::engine::general_purpose::STANDARD.encode(raw.as_bytes()) + ) +} diff --git a/tests/integration/validate.rs b/tests/integration/validate.rs new file mode 100644 index 0000000..f487470 --- /dev/null +++ b/tests/integration/validate.rs @@ -0,0 +1,610 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +use rusqlite::Connection; + +use vandelay::logging::Logger; +use vandelay::sync::CommonConfig; + +pub fn tmp_archive(tag: &str) -> PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!( + "vandelay-container-{tag}-{}-{}.sqlite", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(), + )); + let _ = std::fs::remove_file(&p); + p +} + +pub fn common(archive: &Path) -> CommonConfig { + CommonConfig { + archive: archive.to_path_buf(), + threads: 2, + dry_run: false, + max_retries: 3, + allow_invalid_certs: true, + logger: Logger::from_flags(false, 0), + } +} + +pub fn open_archive(archive: &Path) -> Connection { + Connection::open(archive).expect("open archive") +} + +pub fn count(conn: &Connection, table: &str) -> i64 { + conn.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0)) + .unwrap_or(0) +} + +pub fn blob_bytes(conn: &Connection, blob_id: i64) -> Vec { + conn.query_row("SELECT data FROM blobs WHERE id = ?1", [blob_id], |r| { + r.get::<_, Vec>(0) + }) + .expect("blob fetch") +} + +pub fn mailbox_path(conn: &Connection, mailbox_id: i64, sep: char) -> String { + let mut parts: Vec = Vec::new(); + let mut cur = Some(mailbox_id); + while let Some(id) = cur { + let (name, parent): (String, Option) = conn + .query_row( + "SELECT name, parent_id FROM mailboxes WHERE id = ?1", + [id], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .expect("mailbox path"); + parts.push(name); + cur = parent; + } + parts.reverse(); + parts.join(&sep.to_string()) +} + +pub fn file_path(conn: &Connection, node_id: i64) -> String { + let mut parts: Vec = Vec::new(); + let mut cur = Some(node_id); + while let Some(id) = cur { + let (name, parent): (String, Option) = conn + .query_row( + "SELECT name, parent_id FROM file_nodes WHERE id = ?1", + [id], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .expect("file path"); + parts.push(name); + cur = parent; + } + parts.reverse(); + parts.join("/") +} + +pub fn emails_in_mailbox(conn: &Connection, mailbox_id: i64) -> i64 { + conn.query_row( + "SELECT count(*) FROM emails + WHERE EXISTS (SELECT 1 FROM json_each(emails.mailbox_ids) j WHERE j.value = ?1)", + [mailbox_id], + |r| r.get(0), + ) + .unwrap_or(0) +} + +pub fn email_rows_for_blob(conn: &Connection, blob_id: i64) -> i64 { + conn.query_row( + "SELECT count(*) FROM emails WHERE blob_id = ?1", + [blob_id], + |r| r.get(0), + ) + .unwrap_or(0) +} + +pub fn blob_id_for_hash(conn: &Connection, hash: &[u8]) -> Option { + conn.query_row("SELECT id FROM blobs WHERE hash = ?1", [hash], |r| r.get(0)) + .ok() +} + +pub fn mailbox_id_by_name(conn: &Connection, name: &str) -> Option { + conn.query_row("SELECT id FROM mailboxes WHERE name = ?1", [name], |r| { + r.get(0) + }) + .ok() +} + +pub fn mailbox_id_by_path(conn: &Connection, path: &str, sep: char) -> Option { + let mut parent: Option = None; + for segment in path.split(sep) { + let id: i64 = match parent { + Some(p) => conn + .query_row( + "SELECT id FROM mailboxes WHERE parent_id = ?1 AND name = ?2", + rusqlite::params![p, segment], + |r| r.get(0), + ) + .ok()?, + None => conn + .query_row( + "SELECT id FROM mailboxes WHERE parent_id IS NULL AND name = ?1", + [segment], + |r| r.get(0), + ) + .ok()?, + }; + parent = Some(id); + } + parent +} + +pub fn file_node_id_by_path(conn: &Connection, segments: &[&str]) -> Option { + let mut parent: Option = None; + for segment in segments { + let id: i64 = match parent { + Some(p) => conn + .query_row( + "SELECT id FROM file_nodes WHERE parent_id = ?1 AND name = ?2", + rusqlite::params![p, segment], + |r| r.get(0), + ) + .ok()?, + None => conn + .query_row( + "SELECT id FROM file_nodes WHERE parent_id IS NULL AND name = ?1", + [segment], + |r| r.get(0), + ) + .ok()?, + }; + parent = Some(id); + } + parent +} + +pub fn collection_names(conn: &Connection, table: &str) -> HashSet { + let mut stmt = conn + .prepare(&format!("SELECT name FROM {table}")) + .expect("prepare"); + stmt.query_map([], |r| r.get::<_, String>(0)) + .expect("query") + .filter_map(|r| r.ok()) + .collect() +} + +pub fn collection_id_by_name(conn: &Connection, table: &str, name: &str) -> Option { + conn.query_row( + &format!("SELECT id FROM {table} WHERE name = ?1"), + [name], + |r| r.get(0), + ) + .ok() +} + +pub fn calendar_event_count(conn: &Connection, calendar_id: i64) -> i64 { + conn.query_row( + "SELECT count(*) FROM calendar_events + WHERE EXISTS (SELECT 1 FROM json_each(calendar_events.calendar_ids) j WHERE j.value = ?1)", + [calendar_id], + |r| r.get(0), + ) + .unwrap_or(0) +} + +pub fn contact_card_count(conn: &Connection, address_book_id: i64) -> i64 { + conn.query_row( + "SELECT count(*) FROM contact_cards + WHERE EXISTS (SELECT 1 FROM json_each(contact_cards.address_book_ids) j WHERE j.value = ?1)", + [address_book_id], + |r| r.get(0), + ) + .unwrap_or(0) +} + +pub fn all_event_uids(conn: &Connection) -> HashSet { + let mut stmt = conn + .prepare( + "SELECT json_extract(data, '$.uid') FROM calendar_events + WHERE json_extract(data, '$.uid') IS NOT NULL", + ) + .expect("prepare"); + stmt.query_map([], |r| r.get::<_, String>(0)) + .expect("query") + .filter_map(|r| r.ok()) + .collect() +} + +pub fn all_contact_uids(conn: &Connection) -> HashSet { + let mut stmt = conn + .prepare("SELECT uid FROM contact_cards") + .expect("prepare"); + stmt.query_map([], |r| r.get::<_, String>(0)) + .expect("query") + .filter_map(|r| r.ok()) + .collect() +} + +pub fn keywords_for_blob(conn: &Connection, blob_id: i64) -> Vec { + let json: String = conn + .query_row( + "SELECT keywords FROM emails WHERE blob_id = ?1 LIMIT 1", + [blob_id], + |r| r.get(0), + ) + .expect("keywords fetch"); + serde_json::from_str::>(&json).expect("keywords json") +} + +pub fn event_data_by_uid(conn: &Connection, uid: &str) -> Option { + let raw: String = conn + .query_row( + "SELECT data FROM calendar_events WHERE json_extract(data,'$.uid') = ?1", + [uid], + |r| r.get(0), + ) + .ok()?; + serde_json::from_str(&raw).ok() +} + +pub fn contact_data_by_uid(conn: &Connection, uid: &str) -> Option { + let raw: String = conn + .query_row( + "SELECT data FROM contact_cards WHERE uid = ?1", + [uid], + |r| r.get(0), + ) + .ok()?; + serde_json::from_str(&raw).ok() +} + +pub fn unfold_text(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for line in text.lines() { + let l = line.trim_end_matches('\r'); + if l.starts_with(' ') || l.starts_with('\t') { + out.push_str(&l[1..]); + } else { + if !out.is_empty() { + out.push('\n'); + } + out.push_str(l); + } + } + out +} + +pub fn extract_property(text: &str, name: &str) -> Option { + component_top_level_properties(text, name) + .into_iter() + .next() +} + +pub fn extract_property_all(text: &str, name: &str) -> Vec { + component_top_level_properties(text, name) +} + +fn component_top_level_properties(text: &str, name: &str) -> Vec { + let value_colon = format!("{name}:"); + let value_semi = format!("{name};"); + let mut out = Vec::new(); + let mut depth: i32 = 0; + let mut item_depth: Option = None; + for line in unfold_text(text).lines() { + let l = line.trim_end_matches('\r'); + if l.starts_with("BEGIN:") { + depth += 1; + if matches!(l, "BEGIN:VEVENT" | "BEGIN:VTODO" | "BEGIN:VCARD") { + item_depth = Some(depth); + } + continue; + } + if l.starts_with("END:") { + if Some(depth) == item_depth { + item_depth = None; + } + depth -= 1; + continue; + } + if item_depth.is_some_and(|d| d == depth) { + if let Some(rest) = l.strip_prefix(&value_colon) { + out.push(rest.to_owned()); + } else if l.starts_with(&value_semi) + && let Some(colon) = l.find(':') + { + out.push(l[colon + 1..].to_owned()); + } + } + } + out +} + +pub fn has_line_prefix(text: &str, prefix: &str) -> bool { + let unfolded = unfold_text(text); + unfolded.lines().any(|l| l.starts_with(prefix)) +} + +pub fn json_contains_string(value: &serde_json::Value, needle: &str) -> bool { + let serialised = value.to_string(); + serialised.contains(needle) +} + +pub fn assert_message_round_trip(conn: &Connection, raw: &[u8], target_mailbox: &str, label: &str) { + let hash = blake3::hash(raw); + let blob_id = blob_id_for_hash(conn, hash.as_bytes()) + .unwrap_or_else(|| panic!("{label}: blob for seeded message missing in archive")); + let stored = blob_bytes(conn, blob_id); + assert_eq!( + stored, raw, + "{label}: stored blob bytes differ from seeded raw" + ); + let mailbox_id = mailbox_id_by_path(conn, target_mailbox, '/') + .unwrap_or_else(|| panic!("{label}: target mailbox {target_mailbox} missing in archive")); + let count: i64 = conn + .query_row( + "SELECT count(*) FROM emails + WHERE blob_id = ?1 + AND EXISTS (SELECT 1 FROM json_each(mailbox_ids) j WHERE j.value = ?2)", + rusqlite::params![blob_id, mailbox_id], + |r| r.get(0), + ) + .unwrap(); + assert!( + count >= 1, + "{label}: blob {blob_id} not linked to mailbox {target_mailbox} (id {mailbox_id})" + ); +} + +pub fn assert_event_round_trip(conn: &Connection, source: &[u8], uid: &str, label: &str) { + let text = std::str::from_utf8(source).expect("seeded ical bytes utf-8"); + let data = event_data_by_uid(conn, uid) + .unwrap_or_else(|| panic!("{label}: event uid {uid} missing in archive")); + let obj = data.as_object().expect("event data is object"); + + let stored_uid = obj.get("uid").and_then(|v| v.as_str()).unwrap_or(""); + assert_eq!( + stored_uid, uid, + "{label}: event uid mismatch in stored data" + ); + + let summary = extract_property(text, "SUMMARY").unwrap_or_default(); + if !summary.is_empty() { + let title = obj.get("title").and_then(|v| v.as_str()).unwrap_or(""); + let expected = unescape_text(&summary); + assert!( + title == expected || title.contains(&expected), + "{label}: event {uid} title mismatch: stored={title:?} expected to contain={expected:?}" + ); + } + + if extract_property(text, "DESCRIPTION").is_some() { + let descr = obj + .get("description") + .and_then(|v| v.as_str()) + .unwrap_or(""); + assert!( + !descr.is_empty(), + "{label}: event {uid} description was seeded but stored data has no description" + ); + } + + let start = obj.get("start").and_then(|v| v.as_str()).unwrap_or(""); + assert!( + !start.is_empty(), + "{label}: event {uid} stored data.start is empty" + ); + + if has_line_prefix(text, "RRULE:") || has_line_prefix(text, "RRULE;") { + let has_rule = obj + .get("recurrenceRules") + .map(|v| !v.as_array().is_some_and(|a| a.is_empty())) + .unwrap_or(false) + || obj + .get("recurrenceRule") + .map(|v| { + !v.as_object().is_some_and(|o| o.is_empty()) + && !v.as_array().is_some_and(|a| a.is_empty()) + }) + .unwrap_or(false); + assert!( + has_rule, + "{label}: event {uid} seeded RRULE but no recurrenceRule/recurrenceRules in stored data: {obj:?}" + ); + } + + if has_line_prefix(text, "BEGIN:VALARM") { + let alerts = obj + .get("alerts") + .and_then(|v| v.as_object()) + .cloned() + .unwrap_or_default(); + assert!( + !alerts.is_empty(), + "{label}: event {uid} seeded VALARM but alerts empty" + ); + } + + if has_line_prefix(text, "ORGANIZER:") + || has_line_prefix(text, "ORGANIZER;") + || has_line_prefix(text, "ATTENDEE:") + || has_line_prefix(text, "ATTENDEE;") + { + let parts = obj + .get("participants") + .and_then(|v| v.as_object()) + .cloned() + .unwrap_or_default(); + assert!( + !parts.is_empty(), + "{label}: event {uid} seeded ORGANIZER/ATTENDEE but participants empty" + ); + for addr in extract_property_all(text, "ATTENDEE") { + if let Some(idx) = addr.find("mailto:") + && let Some(end) = addr[idx + 7..].find([',', ';']) + { + let email = &addr[idx + 7..idx + 7 + end]; + assert!( + json_contains_string(&data, email), + "{label}: event {uid} attendee {email} not present in stored JSON" + ); + } else if let Some(idx) = addr.find("mailto:") { + let email = &addr[idx + 7..]; + assert!( + json_contains_string(&data, email), + "{label}: event {uid} attendee {email} not present in stored JSON" + ); + } + } + } + + if let Some(location) = extract_property(text, "LOCATION") { + let cleaned = unescape_text(&location); + let probe = cleaned.split(',').next().unwrap_or(&cleaned).trim(); + if !probe.is_empty() { + assert!( + json_contains_string(&data, probe), + "{label}: event {uid} location fragment {probe:?} not present in stored JSON" + ); + } + } + + if let Some(cats) = extract_property(text, "CATEGORIES") { + let keywords = obj + .get("keywords") + .and_then(|v| v.as_object()) + .cloned() + .unwrap_or_default(); + assert!( + !keywords.is_empty(), + "{label}: event {uid} seeded CATEGORIES={cats} but keywords empty" + ); + for cat in cats.split(',') { + let cat = cat.trim(); + if cat.is_empty() { + continue; + } + assert!( + keywords.contains_key(cat), + "{label}: event {uid} category {cat} not in keywords {keywords:?}" + ); + } + } +} + +pub fn assert_contact_round_trip(conn: &Connection, source: &[u8], uid: &str, label: &str) { + let text = std::str::from_utf8(source).expect("seeded vcard bytes utf-8"); + let data = contact_data_by_uid(conn, uid) + .unwrap_or_else(|| panic!("{label}: contact uid {uid} missing in archive")); + + if let Some(fn_value) = extract_property(text, "FN") { + let cleaned = unescape_text(&fn_value); + assert!( + json_contains_string(&data, &cleaned), + "{label}: contact {uid} FN {cleaned:?} not present in stored JSContact" + ); + } + + for email in extract_property_all(text, "EMAIL") { + assert!( + json_contains_string(&data, &email), + "{label}: contact {uid} EMAIL {email:?} not present in stored JSContact" + ); + } + + for tel in extract_property_all(text, "TEL") { + assert!( + json_contains_string(&data, &tel), + "{label}: contact {uid} TEL {tel:?} not present in stored JSContact" + ); + } + + if let Some(org) = extract_property(text, "ORG") { + let primary = org.split(';').next().unwrap_or(&org).trim(); + if !primary.is_empty() { + assert!( + json_contains_string(&data, primary), + "{label}: contact {uid} ORG {primary:?} not present in stored JSContact" + ); + } + } + + if let Some(nick) = extract_property(text, "NICKNAME") { + assert!( + json_contains_string(&data, &nick), + "{label}: contact {uid} NICKNAME {nick:?} not present" + ); + } + + if let Some(bday) = extract_property(text, "BDAY") { + let iso = format_vcard_date_as_iso(&bday); + let present_raw = json_contains_string(&data, &bday); + let present_iso = iso + .as_deref() + .map(|s| json_contains_string(&data, s)) + .unwrap_or(false); + let obj = data.as_object().expect("contact data is object"); + let has_anniversaries = obj + .get("anniversaries") + .map(|v| !v.as_object().is_some_and(|o| o.is_empty())) + .unwrap_or(false); + assert!( + present_raw || present_iso || has_anniversaries, + "{label}: contact {uid} BDAY {bday:?} not present in stored JSContact (no anniversaries either)" + ); + } +} + +fn format_vcard_date_as_iso(bday: &str) -> Option { + let trimmed = bday.trim(); + if trimmed.len() == 8 && trimmed.bytes().all(|b| b.is_ascii_digit()) { + Some(format!( + "{}-{}-{}", + &trimmed[..4], + &trimmed[4..6], + &trimmed[6..8] + )) + } else { + None + } +} + +fn unescape_text(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + if c == '\\' + && let Some(&next) = chars.peek() + { + match next { + ',' | ';' | '\\' => { + out.push(next); + chars.next(); + continue; + } + 'n' | 'N' => { + out.push('\n'); + chars.next(); + continue; + } + _ => {} + } + } + out.push(c); + } + out +} + +pub fn cleanup(archive: &Path) { + let _ = std::fs::remove_file(archive); + let mut wal = archive.as_os_str().to_owned(); + wal.push("-wal"); + let _ = std::fs::remove_file(Path::new(&wal)); + let mut shm = archive.as_os_str().to_owned(); + shm.push("-shm"); + let _ = std::fs::remove_file(Path::new(&shm)); +} diff --git a/tests/integration/webdav.rs b/tests/integration/webdav.rs new file mode 100644 index 0000000..9e52954 --- /dev/null +++ b/tests/integration/webdav.rs @@ -0,0 +1,321 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::time::Duration; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD as B64; +use testcontainers::core::{IntoContainerPort, WaitFor}; +use testcontainers::runners::{SyncBuilder, SyncRunner}; +use testcontainers::{Container, GenericBuildableImage, GenericImage, ImageExt}; + +use super::dav_client::DavSeed; +use super::error::{ContainerError, ContainerResult}; +use super::layouts::{self, FileSpec}; +use super::{Account, Endpoint}; + +const HTTP_PORT: u16 = 80; + +const DOCKERFILE: &str = r#"FROM debian:bookworm-20260518-slim + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + apache2 apache2-utils ca-certificates && \ + a2enmod dav dav_fs auth_basic authn_file && \ + rm -rf /var/lib/apt/lists/* + +COPY apache.conf /etc/apache2/sites-available/000-default.conf +COPY htpasswd /etc/apache2/htpasswd + +RUN mkdir -p /var/dav && \ + chown -R www-data:www-data /var/dav + +EXPOSE 80 +CMD ["apachectl", "-D", "FOREGROUND"] +"#; + +const APACHE_CONF: &str = r#"DavLockDB /tmp/DavLock + + + DocumentRoot /var/dav + ErrorLog /dev/stderr + CustomLog /dev/stdout combined + + + Dav On + AuthType Basic + AuthName "WebDAV" + AuthUserFile /etc/apache2/htpasswd + Require valid-user + Options Indexes FollowSymLinks + + +"#; + +fn htpasswd_for_test() -> String { + let mut out = String::new(); + for u in layouts::accounts() { + out.push_str(u); + out.push(':'); + out.push_str("{SHA}"); + out.push_str(&sha1_b64(layouts::PASSWORD.as_bytes())); + out.push('\n'); + } + out +} + +fn sha1_b64(bytes: &[u8]) -> String { + let digest = simple_sha1(bytes); + B64.encode(digest) +} + +fn simple_sha1(bytes: &[u8]) -> [u8; 20] { + let mut h0: u32 = 0x6745_2301; + let mut h1: u32 = 0xEFCD_AB89; + let mut h2: u32 = 0x98BA_DCFE; + let mut h3: u32 = 0x1032_5476; + let mut h4: u32 = 0xC3D2_E1F0; + + let bit_len = (bytes.len() as u64).wrapping_mul(8); + let mut msg = bytes.to_vec(); + msg.push(0x80); + while msg.len() % 64 != 56 { + msg.push(0); + } + msg.extend_from_slice(&bit_len.to_be_bytes()); + + for chunk in msg.chunks(64) { + let mut w = [0u32; 80]; + for (i, word) in w.iter_mut().take(16).enumerate() { + *word = u32::from_be_bytes(chunk[i * 4..i * 4 + 4].try_into().unwrap()); + } + for i in 16..80 { + w[i] = (w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16]).rotate_left(1); + } + let (mut a, mut b, mut c, mut d, mut e) = (h0, h1, h2, h3, h4); + for (i, &wi) in w.iter().enumerate() { + let (f, k): (u32, u32) = match i { + 0..=19 => ((b & c) | ((!b) & d), 0x5A82_7999), + 20..=39 => (b ^ c ^ d, 0x6ED9_EBA1), + 40..=59 => ((b & c) | (b & d) | (c & d), 0x8F1B_BCDC), + _ => (b ^ c ^ d, 0xCA62_C1D6), + }; + let temp = a + .rotate_left(5) + .wrapping_add(f) + .wrapping_add(e) + .wrapping_add(k) + .wrapping_add(wi); + e = d; + d = c; + c = b.rotate_left(30); + b = a; + a = temp; + } + h0 = h0.wrapping_add(a); + h1 = h1.wrapping_add(b); + h2 = h2.wrapping_add(c); + h3 = h3.wrapping_add(d); + h4 = h4.wrapping_add(e); + } + let mut out = [0u8; 20]; + out[..4].copy_from_slice(&h0.to_be_bytes()); + out[4..8].copy_from_slice(&h1.to_be_bytes()); + out[8..12].copy_from_slice(&h2.to_be_bytes()); + out[12..16].copy_from_slice(&h3.to_be_bytes()); + out[16..20].copy_from_slice(&h4.to_be_bytes()); + out +} + +pub struct WebDav { + container: Container, + pub http: Endpoint, + pub accounts: Vec, +} + +impl WebDav { + pub fn start() -> ContainerResult { + let image: GenericImage = GenericBuildableImage::new("vandelay-webdav", "test") + .with_dockerfile_string(DOCKERFILE.to_owned()) + .with_data(APACHE_CONF.as_bytes().to_vec(), "apache.conf") + .with_data(htpasswd_for_test().into_bytes(), "htpasswd") + .build_image() + .map_err(|e| ContainerError::Seed(format!("webdav build: {e}")))?; + + let request = image + .with_exposed_port(HTTP_PORT.tcp()) + .with_wait_for(WaitFor::message_on_stderr("AH00558")) + .with_startup_timeout(Duration::from_secs(120)); + + let container = request.start()?; + let host = container.get_host()?.to_string(); + let http = Endpoint::new(host, container.get_host_port_ipv4(HTTP_PORT.tcp())?); + + let accounts: Vec = layouts::accounts() + .iter() + .map(|name| Account { + username: (*name).to_owned(), + password: layouts::PASSWORD.to_owned(), + layout: layouts::layout_for(name), + }) + .collect(); + + Ok(Self { + container, + http, + accounts, + }) + } + + pub fn base_url(&self) -> String { + self.http.http_base() + } + + pub fn account_url(&self, account: &Account) -> String { + format!("{}/{}/", self.base_url(), account.username) + } + + pub fn seed_all(&self) -> ContainerResult> { + let mut out = Vec::new(); + for acct in &self.accounts { + let m = self.seed_account(acct)?; + out.push(m); + } + Ok(out) + } + + pub fn seed_account(&self, account: &Account) -> ContainerResult { + let client = DavSeed::new(self.base_url(), &account.username, &account.password); + let root = format!("/{}/", account.username); + client.mkcol(&root, None)?; + let mut seed = AccountSeed::new(account.username.clone()); + + let specs = account.layout.files; + for spec in specs { + let path = build_path(&root, specs, spec.key) + .ok_or_else(|| ContainerError::Seed(format!("missing file key: {}", spec.key)))?; + if spec.directory { + client.mkcol(&path, None)?; + seed.directories += 1; + } else { + let payload = synth_payload(spec.name); + client.put(&path, "application/octet-stream", payload.as_bytes())?; + seed.files.push(SeededFile { + key: spec.key.to_owned(), + name: spec.name.to_owned(), + href: path, + payload: payload.into_bytes(), + }); + } + } + Ok(seed) + } + + pub fn delete_resource(&self, account: &Account, href: &str) -> ContainerResult<()> { + let client = DavSeed::new(self.base_url(), &account.username, &account.password); + client.delete(href)?; + Ok(()) + } + + pub fn add_file( + &self, + account: &Account, + parent_segments: &[&str], + name: &str, + payload: &[u8], + ) -> ContainerResult { + let client = DavSeed::new(self.base_url(), &account.username, &account.password); + let mut href = format!("/{}/", account.username); + for seg in parent_segments { + href.push_str(seg); + href.push('/'); + } + href.push_str(name); + client.put(&href, "application/octet-stream", payload)?; + Ok(href) + } + + pub fn verify_seed(&self) -> ContainerResult<()> { + for acct in &self.accounts { + let client = DavSeed::new(self.base_url(), &acct.username, &acct.password); + let body = client.propfind(&format!("/{}/", acct.username), 1)?; + if !body.contains("multistatus") { + return Err(ContainerError::Protocol(format!( + "webdav propfind for {} returned no multistatus", + acct.username + ))); + } + } + Ok(()) + } + + pub fn stop(self) -> ContainerResult<()> { + self.container.stop()?; + Ok(()) + } +} + +fn build_path(root: &str, specs: &[FileSpec], key: &str) -> Option { + let mut chain: Vec<&str> = Vec::new(); + let mut cur = key; + let mut leaf_dir = false; + loop { + let spec = specs.iter().find(|s| s.key == cur)?; + chain.push(spec.name); + if chain.len() == 1 { + leaf_dir = spec.directory; + } + match spec.parent { + Some(p) => cur = p, + None => break, + } + } + chain.reverse(); + let mut path = String::from(root); + for (i, name) in chain.iter().enumerate() { + path.push_str(name); + let is_last = i + 1 == chain.len(); + if !is_last || leaf_dir { + path.push('/'); + } + } + Some(path) +} + +pub fn synth_payload(name: &str) -> String { + let mut out = String::new(); + out.push_str(name); + out.push('\n'); + for i in 0..16 { + out.push_str(&format!("line {i} for {name}\n")); + } + out +} + +#[derive(Debug, Clone)] +pub struct SeededFile { + pub key: String, + pub name: String, + pub href: String, + pub payload: Vec, +} + +#[derive(Debug, Clone)] +pub struct AccountSeed { + pub username: String, + pub directories: usize, + pub files: Vec, +} + +impl AccountSeed { + fn new(username: String) -> Self { + Self { + username, + directories: 0, + files: Vec::new(), + } + } +} diff --git a/tests/integration_baikal.rs b/tests/integration_baikal.rs new file mode 100644 index 0000000..042a4eb --- /dev/null +++ b/tests/integration_baikal.rs @@ -0,0 +1,489 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +mod integration; + +use std::collections::HashSet; + +use integration::Account; +use integration::baikal::{AccountSeed, Baikal}; +use integration::validate::{ + all_contact_uids, all_event_uids, assert_contact_round_trip, assert_event_round_trip, + calendar_event_count, cleanup, collection_id_by_name, collection_names, common, + contact_card_count, count, open_archive, tmp_archive, +}; + +use rusqlite::Connection; +use serde_json::Value; +use vandelay::error::Error; +use vandelay::sync::import_dav; +use vandelay::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; + +fn dav_config(kind: DavKindArg, account: &Account, dav_root: &str) -> DavImportConfig { + let path = match kind { + DavKindArg::Caldav => format!("{dav_root}/calendars/{}/", account.username), + DavKindArg::Carddav => format!("{dav_root}/addressbooks/{}/", account.username), + DavKindArg::Webdav => format!("{dav_root}/"), + }; + DavImportConfig { + kind, + url: path, + auth: DavAuth::Basic { + user: account.username.clone(), + password: account.password.clone(), + }, + allow_cleartext: true, + dav_connections: 2, + multiget_batch: 25, + allow_source_change: false, + } +} + +fn dav_config_via_discovery( + kind: DavKindArg, + account: &Account, + dav_root: &str, +) -> DavImportConfig { + DavImportConfig { + kind, + url: format!("{dav_root}/"), + auth: DavAuth::Basic { + user: account.username.clone(), + password: account.password.clone(), + }, + allow_cleartext: true, + dav_connections: 2, + multiget_batch: 25, + allow_source_change: false, + } +} + +#[test] +#[ignore = "requires Docker"] +fn baikal_starts_seeds_and_imports() { + let b = Baikal::start().expect("baikal start"); + let per_account = b.seed_all().expect("baikal seed"); + assert_eq!( + per_account.len(), + b.accounts.len(), + "seed should return stats for every account" + ); + b.verify_seed(&per_account).expect("baikal verify"); + + let dav_root = b.dav_root(); + + for seed in &per_account { + let account = b + .accounts + .iter() + .find(|a| a.username == seed.username) + .expect("account lookup"); + assert!( + !seed.calendars.is_empty(), + "{}: expected calendars seeded", + seed.username + ); + assert!( + !seed.address_books.is_empty(), + "{}: expected address books seeded", + seed.username + ); + assert!( + seed.total_events() > 0, + "{}: expected events seeded", + seed.username + ); + assert!( + seed.total_contacts() > 0, + "{}: expected contacts seeded", + seed.username + ); + + let cal_archive = tmp_archive(&format!("baikal-cal-{}", seed.username)); + let summary = import_dav::run( + common(&cal_archive), + dav_config(DavKindArg::Caldav, account, &dav_root), + ) + .expect("caldav import"); + assert!( + !summary.any_failed(), + "{}: first caldav import had failures: {summary:?}", + seed.username + ); + let conn = open_archive(&cal_archive); + assert_event_collection_exact(&conn, seed); + assert_event_uids_round_trip(&conn, seed); + assert_event_data_well_formed(&conn); + for plan in &seed.calendars { + for item in &plan.items { + assert_event_round_trip( + &conn, + &item.source, + &item.uid, + &format!("{}/{}", seed.username, plan.display_name), + ); + } + } + let cal_count = count(&conn, "calendars") as usize; + let ev_count = count(&conn, "calendar_events") as usize; + let blob_count = count(&conn, "blobs") as usize; + drop(conn); + + let summary2 = import_dav::run( + common(&cal_archive), + dav_config(DavKindArg::Caldav, account, &dav_root), + ) + .expect("idempotent caldav re-import"); + assert!( + !summary2.any_failed(), + "{}: idempotent caldav re-import had failures", + seed.username + ); + let conn = open_archive(&cal_archive); + assert_eq!( + count(&conn, "calendars") as usize, + cal_count, + "{}: idempotent re-import changed calendar count", + seed.username + ); + assert_eq!( + count(&conn, "calendar_events") as usize, + ev_count, + "{}: idempotent re-import changed event count", + seed.username + ); + assert_eq!( + count(&conn, "blobs") as usize, + blob_count, + "{}: idempotent re-import changed blob count", + seed.username + ); + drop(conn); + + let card_archive = tmp_archive(&format!("baikal-card-{}", seed.username)); + let summary = import_dav::run( + common(&card_archive), + dav_config(DavKindArg::Carddav, account, &dav_root), + ) + .expect("carddav import"); + assert!( + !summary.any_failed(), + "{}: first carddav import had failures: {summary:?}", + seed.username + ); + let conn = open_archive(&card_archive); + assert_contact_collection_exact(&conn, seed); + assert_contact_uids_round_trip(&conn, seed); + assert_contact_data_well_formed(&conn); + for plan in &seed.address_books { + for item in &plan.items { + assert_contact_round_trip( + &conn, + &item.source, + &item.uid, + &format!("{}/{}", seed.username, plan.display_name), + ); + } + } + let book_count = count(&conn, "address_books") as usize; + let card_count = count(&conn, "contact_cards") as usize; + drop(conn); + + let summary2 = import_dav::run( + common(&card_archive), + dav_config(DavKindArg::Carddav, account, &dav_root), + ) + .expect("idempotent carddav re-import"); + assert!( + !summary2.any_failed(), + "{}: idempotent carddav re-import had failures", + seed.username + ); + let conn = open_archive(&card_archive); + assert_eq!( + count(&conn, "address_books") as usize, + book_count, + "{}: idempotent re-import changed address book count", + seed.username + ); + assert_eq!( + count(&conn, "contact_cards") as usize, + card_count, + "{}: idempotent re-import changed contact count", + seed.username + ); + drop(conn); + + let vanish_href = seed.calendars[0] + .items + .first() + .map(|i| i.href.clone()) + .expect("at least one event seeded"); + let vanish_uid = seed.calendars[0].items.first().unwrap().uid.clone(); + b.delete_item(account, &vanish_href) + .expect("baikal delete vanished item"); + let summary_after_vanish = import_dav::run( + common(&cal_archive), + dav_config(DavKindArg::Caldav, account, &dav_root), + ) + .expect("caldav re-import after vanish"); + assert!( + !summary_after_vanish.any_failed(), + "{}: re-import after vanish had failures", + seed.username + ); + let conn = open_archive(&cal_archive); + assert_eq!( + count(&conn, "calendar_events") as usize, + ev_count - 1, + "{}: vanished event was not pruned", + seed.username + ); + let uids_now = all_event_uids(&conn); + assert!( + !uids_now.contains(&vanish_uid), + "{}: vanished event uid {vanish_uid} still in archive", + seed.username + ); + drop(conn); + + let icals = integration::data::load_icals().expect("load icals"); + let (_added_event_href, added_event_uid) = b + .add_event(account, 0, &format!("bk-{}", seed.username), &icals) + .expect("baikal add event"); + let summary_after_add_event = import_dav::run( + common(&cal_archive), + dav_config(DavKindArg::Caldav, account, &dav_root), + ) + .expect("caldav re-import after add"); + assert!( + !summary_after_add_event.any_failed(), + "{}: re-import after add-event had failures", + seed.username + ); + let conn = open_archive(&cal_archive); + let after_add_events = count(&conn, "calendar_events") as usize; + assert_eq!( + after_add_events, ev_count, + "{}: add-then-import should restore count to baseline", + seed.username + ); + let uids_now = all_event_uids(&conn); + assert!( + uids_now.contains(&added_event_uid), + "{}: added event uid {added_event_uid} not found in archive", + seed.username + ); + drop(conn); + + let vcards = integration::data::load_vcards().expect("load vcards"); + let (_added_card_href, added_card_uid) = b + .add_contact(account, 0, &format!("bk-{}", seed.username), &vcards) + .expect("baikal add contact"); + let summary_after_add_contact = import_dav::run( + common(&card_archive), + dav_config(DavKindArg::Carddav, account, &dav_root), + ) + .expect("carddav re-import after add"); + assert!( + !summary_after_add_contact.any_failed(), + "{}: re-import after add-contact had failures", + seed.username + ); + let conn = open_archive(&card_archive); + let after_add_contacts = count(&conn, "contact_cards") as usize; + assert_eq!( + after_add_contacts, + card_count + 1, + "{}: add-then-import should grow contact count by 1", + seed.username + ); + let imported_contact_uids = all_contact_uids(&conn); + assert!( + imported_contact_uids.contains(&added_card_uid), + "{}: added contact uid {added_card_uid} not found", + seed.username + ); + drop(conn); + + let discovery_archive = tmp_archive(&format!("baikal-cal-discovery-{}", seed.username)); + let discovery_summary = import_dav::run( + common(&discovery_archive), + dav_config_via_discovery(DavKindArg::Caldav, account, &dav_root), + ) + .expect("caldav import via discovery from dav root"); + assert!( + !discovery_summary.any_failed(), + "{}: caldav import via discovery had failures: {discovery_summary:?}", + seed.username + ); + let conn = open_archive(&discovery_archive); + let discovered_events = count(&conn, "calendar_events") as usize; + assert_eq!( + discovered_events, ev_count, + "{}: discovery-based import event count mismatch (server holds {ev_count} after vanish+add)", + seed.username + ); + drop(conn); + cleanup(&discovery_archive); + + cleanup(&cal_archive); + cleanup(&card_archive); + } + + let primary = &b.accounts[0]; + let other = &b.accounts[1]; + let shared_archive = tmp_archive("baikal-source-change"); + import_dav::run( + common(&shared_archive), + dav_config(DavKindArg::Caldav, primary, &dav_root), + ) + .expect("seed archive with primary user"); + let err = import_dav::run( + common(&shared_archive), + dav_config(DavKindArg::Caldav, other, &dav_root), + ) + .expect_err("expected source-change abort"); + assert!( + matches!(err, Error::SourceChange(_)), + "expected SourceChange, got {err:?}" + ); + cleanup(&shared_archive); + + b.stop().expect("baikal stop"); +} + +fn assert_event_collection_exact(conn: &Connection, seed: &AccountSeed) { + let names = collection_names(conn, "calendars"); + let expected: HashSet = seed + .calendars + .iter() + .map(|c| c.display_name.clone()) + .collect(); + assert_eq!( + names, expected, + "{}: calendar displayname set mismatch", + seed.username + ); + for plan in &seed.calendars { + let cal_id = collection_id_by_name(conn, "calendars", &plan.display_name) + .unwrap_or_else(|| panic!("calendar {} missing", plan.display_name)); + let got = calendar_event_count(conn, cal_id); + assert_eq!( + got as usize, + plan.items.len(), + "{}: calendar {} event count {got} != seeded {}", + seed.username, + plan.display_name, + plan.items.len() + ); + } +} + +fn assert_contact_collection_exact(conn: &Connection, seed: &AccountSeed) { + let names = collection_names(conn, "address_books"); + let expected: HashSet = seed + .address_books + .iter() + .map(|c| c.display_name.clone()) + .collect(); + assert_eq!( + names, expected, + "{}: address book displayname set mismatch", + seed.username + ); + for plan in &seed.address_books { + let book_id = collection_id_by_name(conn, "address_books", &plan.display_name) + .unwrap_or_else(|| panic!("address book {} missing", plan.display_name)); + let got = contact_card_count(conn, book_id); + assert_eq!( + got as usize, + plan.items.len(), + "{}: book {} contact count {got} != seeded {}", + seed.username, + plan.display_name, + plan.items.len() + ); + } +} + +fn assert_event_uids_round_trip(conn: &Connection, seed: &AccountSeed) { + let imported = all_event_uids(conn); + let expected = seed.event_uids(); + let missing: Vec<_> = expected.difference(&imported).collect(); + assert!( + missing.is_empty(), + "{}: event uids missing from archive: {missing:?}", + seed.username + ); +} + +fn assert_contact_uids_round_trip(conn: &Connection, seed: &AccountSeed) { + let imported = all_contact_uids(conn); + let expected = seed.contact_uids(); + let missing: Vec<_> = expected.difference(&imported).collect(); + assert!( + missing.is_empty(), + "{}: contact uids missing from archive: {missing:?}", + seed.username + ); +} + +fn assert_event_data_well_formed(conn: &Connection) { + let mut stmt = conn + .prepare("SELECT data, data_type FROM calendar_events") + .expect("prepare"); + let rows: Vec<(String, String)> = stmt + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))) + .expect("query") + .filter_map(|r| r.ok()) + .collect(); + assert!(!rows.is_empty(), "no calendar_events rows"); + for (raw, data_type) in rows { + assert_eq!( + data_type, "Event", + "all VEVENT seeds must land as Event, got data_type={data_type}, raw={raw}" + ); + let v: Value = serde_json::from_str(&raw) + .unwrap_or_else(|e| panic!("calendar_events.data invalid JSON: {e}; raw={raw}")); + let obj = v.as_object().expect("event data is object"); + let kind = obj + .get("@type") + .or_else(|| obj.get("type")) + .and_then(Value::as_str) + .unwrap_or(""); + assert_eq!(kind, "Event", "JSCalendar @type must be Event, got {kind}"); + assert!(obj.contains_key("uid"), "event JSON missing uid: {obj:?}"); + assert!( + obj.contains_key("start"), + "event JSON missing start: {obj:?}" + ); + } +} + +fn assert_contact_data_well_formed(conn: &Connection) { + let mut stmt = conn + .prepare("SELECT uid, data FROM contact_cards") + .expect("prepare"); + let rows: Vec<(String, String)> = stmt + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))) + .expect("query") + .filter_map(|r| r.ok()) + .collect(); + assert!(!rows.is_empty(), "no contact_cards rows"); + for (uid, raw) in rows { + assert!(!uid.is_empty(), "contact uid empty"); + let v: Value = serde_json::from_str(&raw) + .unwrap_or_else(|e| panic!("contact_cards.data invalid JSON: {e}; raw={raw}")); + let obj = v.as_object().expect("contact data is object"); + let kind = obj + .get("@type") + .or_else(|| obj.get("type")) + .and_then(Value::as_str) + .unwrap_or(""); + assert_eq!(kind, "Card", "JSContact @type must be Card, got {kind}"); + } +} diff --git a/tests/integration_cyrus.rs b/tests/integration_cyrus.rs new file mode 100644 index 0000000..4c68542 --- /dev/null +++ b/tests/integration_cyrus.rs @@ -0,0 +1,320 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +mod integration; + +use std::collections::HashSet; + +use integration::cyrus::{AccountSeed, Cyrus, flag_probe_message}; +use integration::validate::{ + assert_message_round_trip, blob_bytes, blob_id_for_hash, cleanup, common, count, + email_rows_for_blob, emails_in_mailbox, keywords_for_blob, mailbox_id_by_name, + mailbox_id_by_path, mailbox_path, open_archive, tmp_archive, +}; +use integration::{Account, Endpoint}; + +use vandelay::error::Error; +use vandelay::sync::import_imap; +use vandelay::sync::import_imap::{ImapAuth, ImapImportConfig}; + +fn imap_config(account: &Account, imap: &Endpoint) -> ImapImportConfig { + ImapImportConfig { + url: format!("imap://{}:{}", imap.host, imap.port), + auth: ImapAuth::Basic { + user: account.username.clone(), + password: account.password.clone(), + }, + allow_cleartext: true, + compress: false, + include: Vec::new(), + exclude: Vec::new(), + exclude_special: Vec::new(), + folder: Vec::new(), + subscribed_only: false, + automap: true, + include_deleted: false, + fetch_batch: 64, + imap_connections: 2, + allow_source_change: false, + } +} + +#[test] +#[ignore = "requires Docker"] +fn cyrus_starts_seeds_and_imports() { + let c = Cyrus::start().expect("cyrus start"); + let seeds = c.seed_all().expect("cyrus seed"); + c.verify_seed(&seeds).expect("cyrus verify"); + + for seed in &seeds { + let account = c + .accounts + .iter() + .find(|a| a.username == seed.username) + .expect("account"); + assert!( + !seed.paths.is_empty(), + "{}: expected mailboxes seeded", + seed.username + ); + assert!( + seed.total_appends > 0, + "{}: expected emails appended", + seed.username + ); + + let archive = tmp_archive(&format!("cyrus-{}", seed.username)); + let summary = + import_imap::run(common(&archive), imap_config(account, &c.imap)).expect("imap import"); + assert!( + !summary.any_failed(), + "cyrus import failed for {}: {summary:?}", + seed.username + ); + + let conn = open_archive(&archive); + let mailbox_count = count(&conn, "mailboxes") as usize; + let email_count = count(&conn, "emails") as usize; + let blob_count = count(&conn, "blobs") as usize; + let expected_mailboxes = account.layout.mailboxes.len() + 1; + assert_eq!( + mailbox_count, expected_mailboxes, + "{}: imported {mailbox_count} mailboxes, expected exactly {expected_mailboxes}", + seed.username + ); + assert_eq!( + email_count, seed.total_appends, + "{}: every seeded email must land in archive (appended {}, imported {email_count})", + seed.username, seed.total_appends + ); + assert!(blob_count > 0, "{}: blobs interned", seed.username); + + let bad_match: i64 = conn + .query_row( + "SELECT count(*) FROM emails WHERE message_match IS NULL OR message_match = ''", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + bad_match, 0, + "{}: every imported email must have a non-empty message_match", + seed.username + ); + + if let Some(target) = &seed.flagged_target { + let probe = flag_probe_message(); + let probe_hash = blake3::hash(&probe); + let blob_id = blob_id_for_hash(&conn, probe_hash.as_bytes()).expect("flag probe blob"); + let stored = blob_bytes(&conn, blob_id); + assert_eq!( + stored, probe, + "{}: flag probe blob bytes mismatch", + seed.username + ); + let keywords = keywords_for_blob(&conn, blob_id); + let kw_set: HashSet<_> = keywords.iter().map(String::as_str).collect(); + assert!( + kw_set.contains("$seen"), + "{}: flag probe missing $seen in keywords {keywords:?}", + seed.username + ); + assert!( + kw_set.contains("$flagged"), + "{}: flag probe missing $flagged in keywords {keywords:?}", + seed.username + ); + let _ = target; + } + + if let Some(_target) = &seed.dedup_target + && let Some(first) = seeds + .iter() + .find(|s| s.username == seed.username) + .and_then(|_| account.layout.mailboxes.first()) + { + let _ = first; + let total_dup: i64 = conn + .query_row( + "SELECT count(*) FROM emails e + WHERE EXISTS (SELECT 1 FROM emails e2 + WHERE e2.blob_id = e.blob_id AND e2.id <> e.id)", + [], + |r| r.get(0), + ) + .unwrap(); + assert!( + total_dup >= 2, + "{}: dedup append should produce at least two rows sharing a blob", + seed.username + ); + } + + for path in &seed.paths { + let id = mailbox_id_by_path(&conn, path, '/').unwrap_or_else(|| { + panic!("{}: mailbox path {path} missing in archive", seed.username) + }); + assert_eq!(mailbox_path(&conn, id, '/'), *path); + let expected = seed.histogram.get(path).copied().unwrap_or(0); + let got = emails_in_mailbox(&conn, id) as usize; + assert_eq!( + got, expected, + "{}: mailbox {path} email count {got} != appended {expected}", + seed.username + ); + } + for append in &seed.appends { + assert_message_round_trip( + &conn, + &append.raw, + &append.target, + &format!("{}/append/{:?}", seed.username, append.tag), + ); + } + let inbox_role: Option = conn + .query_row("SELECT role FROM mailboxes WHERE name = 'INBOX'", [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!( + inbox_role, + Some("inbox".to_owned()), + "{}: INBOX role tagged", + seed.username + ); + let inbox_id = mailbox_id_by_name(&conn, "INBOX").expect("INBOX"); + let inbox_seeded = seed.histogram.get("INBOX").copied().unwrap_or(0); + assert_eq!( + emails_in_mailbox(&conn, inbox_id) as usize, + inbox_seeded, + "{}: INBOX email count mismatch", + seed.username + ); + let probe_blob = blob_id_for_hash(&conn, blake3::hash(&flag_probe_message()).as_bytes()); + if let Some(blob_id) = probe_blob { + let rows = email_rows_for_blob(&conn, blob_id); + assert!(rows >= 1, "{}: flag probe missing", seed.username); + } + + drop(conn); + + let summary2 = import_imap::run(common(&archive), imap_config(account, &c.imap)) + .expect("idempotent imap re-import"); + assert!( + !summary2.any_failed(), + "{}: idempotent re-import had failures: {summary2:?}", + seed.username + ); + let conn = open_archive(&archive); + let email_count2 = count(&conn, "emails") as usize; + let blob_count2 = count(&conn, "blobs") as usize; + assert_eq!( + email_count2, email_count, + "{}: idempotent re-import added emails", + seed.username + ); + assert_eq!( + blob_count2, blob_count, + "{}: idempotent re-import added blobs", + seed.username + ); + drop(conn); + + c.delete_first_inbox_message(account) + .expect("expunge first INBOX message"); + let summary_after = import_imap::run(common(&archive), imap_config(account, &c.imap)) + .expect("re-import after expunge"); + assert!( + !summary_after.any_failed(), + "{}: re-import after expunge had failures", + seed.username + ); + let conn = open_archive(&archive); + let email_count3 = count(&conn, "emails") as usize; + assert_eq!( + email_count3, + email_count - 1, + "{}: expunged message was not pruned", + seed.username + ); + drop(conn); + + let (added_raw, added_mid) = c + .append_new_message(account, "INBOX", "post-import") + .expect("append new INBOX message"); + let summary_after_append = + import_imap::run(common(&archive), imap_config(account, &c.imap)) + .expect("re-import after append"); + assert!( + !summary_after_append.any_failed(), + "{}: re-import after append had failures", + seed.username + ); + let conn = open_archive(&archive); + let email_count_after_append = count(&conn, "emails") as usize; + assert_eq!( + email_count_after_append, email_count, + "{}: append-after-expunge should restore email count to baseline", + seed.username + ); + let added_hash = blake3::hash(&added_raw); + let added_blob = + blob_id_for_hash(&conn, added_hash.as_bytes()).expect("added message blob present"); + let mid_normalised = added_mid + .trim_matches(|c| c == '<' || c == '>') + .to_ascii_lowercase(); + let mid_found: i64 = conn + .query_row( + "SELECT count(*) FROM emails + WHERE EXISTS (SELECT 1 FROM json_each(json_extract(message_match,'$.m')) j + WHERE j.value = ?1)", + [mid_normalised.as_str()], + |r| r.get(0), + ) + .unwrap(); + assert!( + mid_found >= 1, + "{}: added Message-ID {added_mid} not found in message_match", + seed.username + ); + let added_inbox_id = mailbox_id_by_name(&conn, "INBOX").expect("INBOX id"); + let added_in_inbox: bool = conn + .query_row( + "SELECT EXISTS (SELECT 1 FROM emails + WHERE blob_id = ?1 + AND EXISTS (SELECT 1 FROM json_each(mailbox_ids) j + WHERE j.value = ?2))", + rusqlite::params![added_blob, added_inbox_id], + |r| r.get::<_, i64>(0).map(|n| n != 0), + ) + .unwrap(); + assert!( + added_in_inbox, + "{}: added message blob not linked to INBOX", + seed.username + ); + drop(conn); + + cleanup(&archive); + } + + let primary = &c.accounts[0]; + let other = &c.accounts[1]; + let shared_archive = tmp_archive("cyrus-source-change"); + import_imap::run(common(&shared_archive), imap_config(primary, &c.imap)) + .expect("seed archive with primary user"); + let err = import_imap::run(common(&shared_archive), imap_config(other, &c.imap)) + .expect_err("expected source-change abort"); + assert!( + matches!(err, Error::SourceChange(_)), + "expected SourceChange, got {err:?}" + ); + cleanup(&shared_archive); + + c.stop().expect("cyrus stop"); +} + +fn _unused(_: &AccountSeed) {} diff --git a/tests/integration_dovecot.rs b/tests/integration_dovecot.rs new file mode 100644 index 0000000..d0ae69a --- /dev/null +++ b/tests/integration_dovecot.rs @@ -0,0 +1,516 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +mod integration; + +use std::collections::HashSet; + +use integration::Account; +use integration::dovecot::{AccountSeed, Dovecot, flag_probe_message, shared_message_id_probe}; +use integration::validate::{ + assert_message_round_trip, blob_bytes, blob_id_for_hash, cleanup, common, count, + email_rows_for_blob, emails_in_mailbox, keywords_for_blob, mailbox_id_by_name, + mailbox_id_by_path, mailbox_path, open_archive, tmp_archive, +}; + +use rusqlite::Connection; +use vandelay::error::Error; +use vandelay::sync::import_imap::{ImapAuth, ImapImportConfig}; +use vandelay::sync::import_managesieve::{ManageSieveAuth, ManageSieveImportConfig}; +use vandelay::sync::{import_imap, import_managesieve}; + +fn imap_config(account: &Account, imap: &integration::Endpoint) -> ImapImportConfig { + ImapImportConfig { + url: format!("imap://{}:{}", imap.host, imap.port), + auth: ImapAuth::Basic { + user: account.username.clone(), + password: account.password.clone(), + }, + allow_cleartext: false, + compress: false, + include: Vec::new(), + exclude: Vec::new(), + exclude_special: Vec::new(), + folder: Vec::new(), + subscribed_only: false, + automap: true, + include_deleted: false, + fetch_batch: 64, + imap_connections: 2, + allow_source_change: false, + } +} + +fn sieve_config(account: &Account, sieve: &integration::Endpoint) -> ManageSieveImportConfig { + ManageSieveImportConfig { + url: format!("sieve://{}:{}", sieve.host, sieve.port), + auth: ManageSieveAuth::Basic { + user: account.username.clone(), + password: account.password.clone(), + }, + allow_cleartext: true, + allow_source_change: false, + } +} + +#[test] +#[ignore = "requires Docker"] +fn dovecot_starts_seeds_and_imports() { + let d = Dovecot::start().expect("dovecot start"); + let seeds = d.seed_all().expect("dovecot seed"); + assert_eq!( + seeds.len(), + d.accounts.len(), + "seed should return stats for every account" + ); + d.verify_seed(&seeds).expect("dovecot verify"); + + for seed in &seeds { + let account = d + .accounts + .iter() + .find(|a| a.username == seed.username) + .expect("account"); + assert!( + !seed.mailbox.paths.is_empty(), + "{}: expected mailboxes seeded", + seed.username + ); + assert!( + seed.mailbox.total_appends > 0, + "{}: expected emails appended", + seed.username + ); + + let archive = tmp_archive(&format!("dovecot-{}", seed.username)); + let summary = + import_imap::run(common(&archive), imap_config(account, &d.imap)).expect("imap import"); + assert!( + !summary.any_failed(), + "imap import had failures for {}: {summary:?}", + seed.username + ); + + let conn = open_archive(&archive); + let mailbox_count = count(&conn, "mailboxes") as usize; + let email_count = count(&conn, "emails") as usize; + let blob_count = count(&conn, "blobs") as usize; + let expected_mailboxes = account.layout.mailboxes.len() + 1; + assert_eq!( + mailbox_count, expected_mailboxes, + "{}: imported {mailbox_count} mailboxes, expected exactly {expected_mailboxes}", + seed.username + ); + assert_eq!( + email_count, seed.mailbox.total_appends, + "{}: every seeded email must land in archive (appended {}, imported {email_count})", + seed.username, seed.mailbox.total_appends + ); + assert!(blob_count > 0, "{}: blobs interned", seed.username); + + let bad_match: i64 = conn + .query_row( + "SELECT count(*) FROM emails WHERE message_match IS NULL OR message_match = ''", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + bad_match, 0, + "{}: every imported email must have a non-empty message_match", + seed.username + ); + + let nomid_target = seed + .mailbox + .extras + .nomid_target + .clone() + .expect("nomid target"); + let nomid_hash = blake3::hash(&integration::dovecot::no_message_id_probe()) + .as_bytes() + .to_vec(); + let nomid_matches: Vec = { + let mut nomid_stmt = conn + .prepare( + "SELECT message_match FROM emails + WHERE blob_id = (SELECT id FROM blobs WHERE hash = ?1)", + ) + .expect("prepare nomid"); + nomid_stmt + .query_map([nomid_hash.as_slice()], |r| r.get::<_, String>(0)) + .expect("query nomid") + .filter_map(|r| r.ok()) + .collect() + }; + assert!( + !nomid_matches.is_empty(), + "{}: no-Message-ID probe missing in archive (target {nomid_target})", + seed.username + ); + for m in &nomid_matches { + let parsed: serde_json::Value = serde_json::from_str(m).expect("message_match json"); + let mids = parsed + .get("m") + .and_then(|v| v.as_array()) + .cloned() + .unwrap_or_default(); + assert!( + mids.is_empty(), + "{}: no-Message-ID probe should have empty Message-ID set, got {m}", + seed.username + ); + let fb = parsed.get("f").and_then(|v| v.as_str()).unwrap_or(""); + assert_eq!( + fb.len(), + 64, + "{}: no-Message-ID probe must carry a BLAKE3 fallback hex, got {m}", + seed.username + ); + } + + if let Some(target) = &seed.mailbox.extras.flagged_target { + let probe = flag_probe_message(); + let probe_hash = blake3::hash(&probe); + let blob_id = blob_id_for_hash(&conn, probe_hash.as_bytes()).expect("flag probe blob"); + let stored = blob_bytes(&conn, blob_id); + assert_eq!( + stored, probe, + "{}: flag probe blob bytes mismatch", + seed.username + ); + let keywords = keywords_for_blob(&conn, blob_id); + let kw_set: HashSet<_> = keywords.iter().map(String::as_str).collect(); + assert!( + kw_set.contains("$seen"), + "{}: flag probe missing $seen in keywords {keywords:?}", + seed.username + ); + assert!( + kw_set.contains("$flagged"), + "{}: flag probe missing $flagged in keywords {keywords:?}", + seed.username + ); + let _ = target; + } + + if let Some((a, b)) = &seed.mailbox.extras.mid_dedup_targets { + let body = shared_message_id_probe(); + let hash = blake3::hash(&body); + let blob_id = blob_id_for_hash(&conn, hash.as_bytes()).expect("shared-mid probe blob"); + let rows_for_blob = email_rows_for_blob(&conn, blob_id); + assert_eq!( + rows_for_blob, 2, + "{}: shared-MID probe should produce 2 email rows for the dedup blob, got {rows_for_blob}", + seed.username + ); + let matches: Vec = { + let mut stmt = conn + .prepare("SELECT message_match FROM emails WHERE blob_id = ?1") + .expect("prepare matches"); + stmt.query_map([blob_id], |r| r.get::<_, String>(0)) + .expect("query matches") + .filter_map(|r| r.ok()) + .collect() + }; + assert_eq!(matches.len(), 2, "{}: expected 2 matches", seed.username); + assert_eq!( + matches[0], matches[1], + "{}: shared-MID emails must have identical message_match, got {matches:?}", + seed.username + ); + let _ = (a, b); + } + + if let Some(target) = &seed.mailbox.extras.dedup_target { + let _ = target; + } + + for path in &seed.mailbox.paths { + let id = mailbox_id_by_path(&conn, path, '/').unwrap_or_else(|| { + panic!("{}: mailbox path {path} missing in archive", seed.username) + }); + assert_eq!(mailbox_path(&conn, id, '/'), *path); + let expected = seed.mailbox.histogram.get(path).copied().unwrap_or(0); + let got = emails_in_mailbox(&conn, id) as usize; + assert_eq!( + got, expected, + "{}: mailbox {path} email count {got} != appended {expected}", + seed.username + ); + } + for append in &seed.mailbox.appends { + assert_message_round_trip( + &conn, + &append.raw, + &append.target, + &format!("{}/append/{:?}", seed.username, append.tag), + ); + } + let inbox_role: Option = conn + .query_row("SELECT role FROM mailboxes WHERE name = 'INBOX'", [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!( + inbox_role, + Some("inbox".to_owned()), + "{}: INBOX role tagged", + seed.username + ); + let inbox_id = mailbox_id_by_name(&conn, "INBOX").expect("INBOX"); + let inbox_seeded = seed.mailbox.histogram.get("INBOX").copied().unwrap_or(0); + assert_eq!( + emails_in_mailbox(&conn, inbox_id) as usize, + inbox_seeded, + "{}: INBOX email count mismatch", + seed.username + ); + assert_received_at_rfc3339_ish(&conn); + + drop(conn); + + let summary2 = import_imap::run(common(&archive), imap_config(account, &d.imap)) + .expect("idempotent imap re-import"); + assert!( + !summary2.any_failed(), + "{}: idempotent re-import had failures: {summary2:?}", + seed.username + ); + let conn = open_archive(&archive); + let email_count2 = count(&conn, "emails") as usize; + let mailbox_count2 = count(&conn, "mailboxes") as usize; + let blob_count2 = count(&conn, "blobs") as usize; + assert_eq!( + email_count2, email_count, + "{}: idempotent re-import added emails", + seed.username + ); + assert_eq!( + mailbox_count2, mailbox_count, + "{}: idempotent re-import added mailboxes", + seed.username + ); + assert_eq!( + blob_count2, blob_count, + "{}: idempotent re-import added blobs", + seed.username + ); + drop(conn); + + d.delete_first_inbox_message(account) + .expect("expunge first INBOX message"); + let summary_after = import_imap::run(common(&archive), imap_config(account, &d.imap)) + .expect("re-import after expunge"); + assert!( + !summary_after.any_failed(), + "{}: re-import after expunge had failures", + seed.username + ); + let conn = open_archive(&archive); + let email_count3 = count(&conn, "emails") as usize; + assert_eq!( + email_count3, + email_count - 1, + "{}: expunged message was not pruned", + seed.username + ); + drop(conn); + + let (added_raw, added_mid) = d + .append_new_message(account, "INBOX", "post-import") + .expect("append new INBOX message"); + let summary_after_append = + import_imap::run(common(&archive), imap_config(account, &d.imap)) + .expect("re-import after append"); + assert!( + !summary_after_append.any_failed(), + "{}: re-import after append had failures", + seed.username + ); + let conn = open_archive(&archive); + let email_count_after_append = count(&conn, "emails") as usize; + assert_eq!( + email_count_after_append, email_count, + "{}: append-after-expunge should restore email count to baseline", + seed.username + ); + let added_hash = blake3::hash(&added_raw); + let added_blob = integration::validate::blob_id_for_hash(&conn, added_hash.as_bytes()) + .expect("added message blob present"); + let stored = integration::validate::blob_bytes(&conn, added_blob); + assert_eq!( + stored, added_raw, + "{}: added message blob bytes mismatch", + seed.username + ); + let mid_normalised = added_mid + .trim_matches(|c| c == '<' || c == '>') + .to_ascii_lowercase(); + let mid_found: i64 = conn + .query_row( + "SELECT count(*) FROM emails + WHERE EXISTS (SELECT 1 FROM json_each(json_extract(message_match,'$.m')) j + WHERE j.value = ?1)", + [mid_normalised.as_str()], + |r| r.get(0), + ) + .unwrap(); + assert!( + mid_found >= 1, + "{}: added Message-ID {added_mid} not found in message_match", + seed.username + ); + let added_inbox_id = mailbox_id_by_name(&conn, "INBOX").expect("INBOX id"); + let added_in_inbox: bool = conn + .query_row( + "SELECT EXISTS (SELECT 1 FROM emails + WHERE blob_id = ?1 + AND EXISTS (SELECT 1 FROM json_each(mailbox_ids) j + WHERE j.value = ?2))", + rusqlite::params![added_blob, added_inbox_id], + |r| r.get::<_, i64>(0).map(|n| n != 0), + ) + .unwrap(); + assert!( + added_in_inbox, + "{}: added message blob not linked to INBOX", + seed.username + ); + drop(conn); + + if !account.layout.sieve_scripts.is_empty() { + let sieve_archive = tmp_archive(&format!("dovecot-sieve-{}", seed.username)); + let summary = + import_managesieve::run(common(&sieve_archive), sieve_config(account, &d.sieve)) + .expect("sieve import"); + assert!( + !summary.any_failed(), + "sieve import had failures for {}: {summary:?}", + seed.username + ); + let conn = open_archive(&sieve_archive); + let script_count = count(&conn, "sieve_scripts") as usize; + assert_eq!( + script_count, + account.layout.sieve_scripts.len(), + "{}: sieve scripts count mismatch", + seed.username + ); + let active_seed = account + .layout + .sieve_scripts + .iter() + .filter(|s| s.active) + .count(); + let active_imported: i64 = conn + .query_row( + "SELECT count(*) FROM sieve_scripts WHERE is_active = 1", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + active_imported as usize, active_seed, + "{}: active sieve script count mismatch", + seed.username + ); + + for seeded in account.layout.sieve_scripts { + let blob_id: i64 = conn + .query_row( + "SELECT blob_id FROM sieve_scripts WHERE name = ?1", + [seeded.name], + |r| r.get(0), + ) + .unwrap(); + let body = blob_bytes(&conn, blob_id); + assert_eq!( + body, + seeded.body.as_bytes(), + "{}: sieve script {} body round-trip", + seed.username, + seeded.name + ); + let is_active: i64 = conn + .query_row( + "SELECT is_active FROM sieve_scripts WHERE name = ?1", + [seeded.name], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + is_active == 1, + seeded.active, + "{}: sieve script {} active flag mismatch", + seed.username, + seeded.name + ); + } + let actives: i64 = conn + .query_row( + "SELECT count(*) FROM sieve_scripts WHERE is_active = 1", + [], + |r| r.get(0), + ) + .unwrap(); + assert!( + actives <= 1, + "{}: at most one sieve script may be active", + seed.username + ); + drop(conn); + + let summary2 = + import_managesieve::run(common(&sieve_archive), sieve_config(account, &d.sieve)) + .expect("idempotent sieve re-import"); + assert!( + !summary2.any_failed(), + "{}: idempotent sieve re-import had failures", + seed.username + ); + + cleanup(&sieve_archive); + } + + cleanup(&archive); + } + + let primary = &d.accounts[0]; + let other = &d.accounts[1]; + let shared_archive = tmp_archive("dovecot-source-change"); + import_imap::run(common(&shared_archive), imap_config(primary, &d.imap)) + .expect("seed archive with primary user"); + let err = import_imap::run(common(&shared_archive), imap_config(other, &d.imap)) + .expect_err("expected source-change abort"); + assert!( + matches!(err, Error::SourceChange(_)), + "expected SourceChange, got {err:?}" + ); + cleanup(&shared_archive); + + d.stop().expect("dovecot stop"); +} + +fn assert_received_at_rfc3339_ish(conn: &Connection) { + let mut stmt = conn + .prepare("SELECT received_at FROM emails LIMIT 50") + .expect("prepare"); + let rows: Vec = stmt + .query_map([], |r| r.get::<_, String>(0)) + .expect("query") + .filter_map(|r| r.ok()) + .collect(); + assert!(!rows.is_empty(), "no emails to check received_at"); + for r in rows { + assert!( + r.contains('T') && (r.ends_with('Z') || r.contains('+') || r.contains('-')), + "received_at does not look like RFC 3339: {r}" + ); + } +} + +fn _unused(_: &AccountSeed) {} diff --git a/tests/integration_radicale.rs b/tests/integration_radicale.rs new file mode 100644 index 0000000..97bc139 --- /dev/null +++ b/tests/integration_radicale.rs @@ -0,0 +1,459 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +mod integration; + +use std::collections::HashSet; + +use integration::radicale::{AccountSeed, Radicale}; +use integration::validate::{ + all_contact_uids, all_event_uids, assert_contact_round_trip, assert_event_round_trip, + calendar_event_count, cleanup, collection_id_by_name, collection_names, common, + contact_card_count, count, open_archive, tmp_archive, +}; +use integration::{Account, Endpoint}; + +use rusqlite::Connection; +use serde_json::Value; +use vandelay::error::Error; +use vandelay::sync::import_dav; +use vandelay::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; + +fn dav_config(kind: DavKindArg, account: &Account, http: &Endpoint) -> DavImportConfig { + DavImportConfig { + kind, + url: format!("http://{}:{}/{}/", http.host, http.port, account.username), + auth: DavAuth::Basic { + user: account.username.clone(), + password: account.password.clone(), + }, + allow_cleartext: true, + dav_connections: 2, + multiget_batch: 25, + allow_source_change: false, + } +} + +#[test] +#[ignore = "requires Docker"] +fn radicale_starts_seeds_and_imports() { + let r = Radicale::start().expect("radicale start"); + let per_account = r.seed_all().expect("radicale seed"); + assert_eq!( + per_account.len(), + r.accounts.len(), + "seed should return stats for every account" + ); + r.verify_seed(&per_account).expect("radicale verify"); + + for seed in &per_account { + let account = r + .accounts + .iter() + .find(|a| a.username == seed.username) + .expect("account lookup"); + assert!( + !seed.calendars.is_empty(), + "{}: expected calendars seeded", + seed.username + ); + assert!( + !seed.address_books.is_empty(), + "{}: expected address books seeded", + seed.username + ); + assert!( + seed.total_events() > 0, + "{}: expected events seeded", + seed.username + ); + assert!( + seed.total_contacts() > 0, + "{}: expected contacts seeded", + seed.username + ); + + let cal_archive = tmp_archive(&format!("radicale-cal-{}", seed.username)); + let summary = import_dav::run( + common(&cal_archive), + dav_config(DavKindArg::Caldav, account, &r.endpoint), + ) + .expect("caldav import"); + assert!( + !summary.any_failed(), + "{}: first caldav import had failures: {summary:?}", + seed.username + ); + let conn = open_archive(&cal_archive); + assert_event_collection_exact(&conn, seed); + assert_event_uids_round_trip(&conn, seed); + assert_event_data_well_formed(&conn); + for plan in &seed.calendars { + for item in &plan.items { + assert_event_round_trip( + &conn, + &item.source, + &item.uid, + &format!("{}/{}", seed.username, plan.display_name), + ); + } + } + let cal_count = count(&conn, "calendars") as usize; + let ev_count = count(&conn, "calendar_events") as usize; + let blob_count = count(&conn, "blobs") as usize; + drop(conn); + + let summary2 = import_dav::run( + common(&cal_archive), + dav_config(DavKindArg::Caldav, account, &r.endpoint), + ) + .expect("idempotent caldav re-import"); + assert!( + !summary2.any_failed(), + "{}: idempotent caldav re-import had failures", + seed.username + ); + let conn = open_archive(&cal_archive); + assert_eq!( + count(&conn, "calendars") as usize, + cal_count, + "{}: idempotent re-import changed calendar count", + seed.username + ); + assert_eq!( + count(&conn, "calendar_events") as usize, + ev_count, + "{}: idempotent re-import changed event count", + seed.username + ); + assert_eq!( + count(&conn, "blobs") as usize, + blob_count, + "{}: idempotent re-import changed blob count", + seed.username + ); + drop(conn); + + let card_archive = tmp_archive(&format!("radicale-card-{}", seed.username)); + let summary = import_dav::run( + common(&card_archive), + dav_config(DavKindArg::Carddav, account, &r.endpoint), + ) + .expect("carddav import"); + assert!( + !summary.any_failed(), + "{}: first carddav import had failures: {summary:?}", + seed.username + ); + let conn = open_archive(&card_archive); + assert_contact_collection_exact(&conn, seed); + assert_contact_uids_round_trip(&conn, seed); + assert_contact_data_well_formed(&conn); + for plan in &seed.address_books { + for item in &plan.items { + assert_contact_round_trip( + &conn, + &item.source, + &item.uid, + &format!("{}/{}", seed.username, plan.display_name), + ); + } + } + let book_count = count(&conn, "address_books") as usize; + let card_count = count(&conn, "contact_cards") as usize; + drop(conn); + + let summary2 = import_dav::run( + common(&card_archive), + dav_config(DavKindArg::Carddav, account, &r.endpoint), + ) + .expect("idempotent carddav re-import"); + assert!( + !summary2.any_failed(), + "{}: idempotent carddav re-import had failures", + seed.username + ); + let conn = open_archive(&card_archive); + assert_eq!( + count(&conn, "address_books") as usize, + book_count, + "{}: idempotent re-import changed address book count", + seed.username + ); + assert_eq!( + count(&conn, "contact_cards") as usize, + card_count, + "{}: idempotent re-import changed contact count", + seed.username + ); + drop(conn); + + let vanish_target_href = seed.calendars[0] + .items + .first() + .map(|i| i.href.clone()) + .expect("at least one event seeded"); + let vanish_uid = seed.calendars[0] + .items + .first() + .map(|i| i.uid.clone()) + .expect("uid"); + r.delete_item(account, &vanish_target_href) + .expect("radicale delete vanished item"); + let summary_after_vanish = import_dav::run( + common(&cal_archive), + dav_config(DavKindArg::Caldav, account, &r.endpoint), + ) + .expect("caldav re-import after vanish"); + assert!( + !summary_after_vanish.any_failed(), + "{}: re-import after vanish had failures", + seed.username + ); + let conn = open_archive(&cal_archive); + let after_vanish_events = count(&conn, "calendar_events") as usize; + assert_eq!( + after_vanish_events, + ev_count - 1, + "{}: vanished event was not pruned from archive", + seed.username + ); + let uids_now = all_event_uids(&conn); + assert!( + !uids_now.contains(&vanish_uid), + "{}: vanished event uid {vanish_uid} still in archive", + seed.username + ); + drop(conn); + + let icals = integration::data::load_icals().expect("load icals"); + let (_added_event_href, added_event_uid) = r + .add_event(account, 0, &format!("rad-{}", seed.username), &icals) + .expect("radicale add event"); + let summary_after_add_event = import_dav::run( + common(&cal_archive), + dav_config(DavKindArg::Caldav, account, &r.endpoint), + ) + .expect("caldav re-import after add"); + assert!( + !summary_after_add_event.any_failed(), + "{}: re-import after add-event had failures", + seed.username + ); + let conn = open_archive(&cal_archive); + let after_add_events = count(&conn, "calendar_events") as usize; + assert_eq!( + after_add_events, + ev_count, + "{}: add-then-import should restore count to baseline (was {} after vanish, expected {ev_count})", + seed.username, + ev_count - 1 + ); + let uids_now = all_event_uids(&conn); + assert!( + uids_now.contains(&added_event_uid), + "{}: added event uid {added_event_uid} not found in archive after re-import", + seed.username + ); + let added_cal_id = + collection_id_by_name(&conn, "calendars", &seed.calendars[0].display_name) + .expect("calendar id"); + let new_calendar_count = calendar_event_count(&conn, added_cal_id); + assert_eq!( + new_calendar_count as usize, + seed.calendars[0].items.len(), + "{}: target calendar event count after add+vanish should match original", + seed.username + ); + drop(conn); + + let vcards = integration::data::load_vcards().expect("load vcards"); + let (_added_card_href, added_card_uid) = r + .add_contact(account, 0, &format!("rad-{}", seed.username), &vcards) + .expect("radicale add contact"); + let summary_after_add_contact = import_dav::run( + common(&card_archive), + dav_config(DavKindArg::Carddav, account, &r.endpoint), + ) + .expect("carddav re-import after add"); + assert!( + !summary_after_add_contact.any_failed(), + "{}: re-import after add-contact had failures", + seed.username + ); + let conn = open_archive(&card_archive); + let after_add_contacts = count(&conn, "contact_cards") as usize; + assert_eq!( + after_add_contacts, + card_count + 1, + "{}: add-then-import should grow contact count by 1", + seed.username + ); + let imported_contact_uids = all_contact_uids(&conn); + assert!( + imported_contact_uids.contains(&added_card_uid), + "{}: added contact uid {added_card_uid} not found", + seed.username + ); + drop(conn); + + cleanup(&cal_archive); + cleanup(&card_archive); + } + + let primary = &r.accounts[0]; + let other = &r.accounts[1]; + let shared_archive = tmp_archive("radicale-source-change"); + import_dav::run( + common(&shared_archive), + dav_config(DavKindArg::Caldav, primary, &r.endpoint), + ) + .expect("seed archive with primary user"); + let err = import_dav::run( + common(&shared_archive), + dav_config(DavKindArg::Caldav, other, &r.endpoint), + ) + .expect_err("expected source-change abort when pointing at second user"); + assert!( + matches!(err, Error::SourceChange(_)), + "expected SourceChange, got {err:?}" + ); + cleanup(&shared_archive); + + r.stop().expect("radicale stop"); +} + +fn assert_event_collection_exact(conn: &Connection, seed: &AccountSeed) { + let names = collection_names(conn, "calendars"); + let expected: HashSet = seed + .calendars + .iter() + .map(|c| c.display_name.clone()) + .collect(); + assert_eq!( + names, expected, + "{}: calendar displayname set mismatch", + seed.username + ); + for plan in &seed.calendars { + let cal_id = collection_id_by_name(conn, "calendars", &plan.display_name) + .unwrap_or_else(|| panic!("calendar {} missing", plan.display_name)); + let got = calendar_event_count(conn, cal_id); + assert_eq!( + got as usize, + plan.items.len(), + "{}: calendar {} event count {got} != seeded {}", + seed.username, + plan.display_name, + plan.items.len() + ); + } +} + +fn assert_contact_collection_exact(conn: &Connection, seed: &AccountSeed) { + let names = collection_names(conn, "address_books"); + let expected: HashSet = seed + .address_books + .iter() + .map(|c| c.display_name.clone()) + .collect(); + assert_eq!( + names, expected, + "{}: address book displayname set mismatch", + seed.username + ); + for plan in &seed.address_books { + let book_id = collection_id_by_name(conn, "address_books", &plan.display_name) + .unwrap_or_else(|| panic!("address book {} missing", plan.display_name)); + let got = contact_card_count(conn, book_id); + assert_eq!( + got as usize, + plan.items.len(), + "{}: book {} contact count {got} != seeded {}", + seed.username, + plan.display_name, + plan.items.len() + ); + } +} + +fn assert_event_uids_round_trip(conn: &Connection, seed: &AccountSeed) { + let imported = all_event_uids(conn); + let expected = seed.event_uids(); + let missing: Vec<_> = expected.difference(&imported).collect(); + assert!( + missing.is_empty(), + "{}: event uids missing from archive: {missing:?}", + seed.username + ); +} + +fn assert_contact_uids_round_trip(conn: &Connection, seed: &AccountSeed) { + let imported = all_contact_uids(conn); + let expected = seed.contact_uids(); + let missing: Vec<_> = expected.difference(&imported).collect(); + assert!( + missing.is_empty(), + "{}: contact uids missing from archive: {missing:?}", + seed.username + ); +} + +fn assert_event_data_well_formed(conn: &Connection) { + let mut stmt = conn + .prepare("SELECT data, data_type FROM calendar_events") + .expect("prepare"); + let rows: Vec<(String, String)> = stmt + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))) + .expect("query") + .filter_map(|r| r.ok()) + .collect(); + assert!(!rows.is_empty(), "no calendar_events rows"); + for (raw, data_type) in rows { + assert_eq!( + data_type, "Event", + "all VEVENT seeds must land as Event, got data_type={data_type}, raw={raw}" + ); + let v: Value = serde_json::from_str(&raw) + .unwrap_or_else(|e| panic!("calendar_events.data invalid JSON: {e}; raw={raw}")); + let obj = v.as_object().expect("event data is object"); + let kind = obj + .get("@type") + .or_else(|| obj.get("type")) + .and_then(Value::as_str) + .unwrap_or(""); + assert_eq!(kind, "Event", "JSCalendar @type must be Event, got {kind}"); + assert!(obj.contains_key("uid"), "event JSON missing uid: {obj:?}"); + assert!( + obj.contains_key("start"), + "event JSON missing start: {obj:?}" + ); + } +} + +fn assert_contact_data_well_formed(conn: &Connection) { + let mut stmt = conn + .prepare("SELECT uid, data FROM contact_cards") + .expect("prepare"); + let rows: Vec<(String, String)> = stmt + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))) + .expect("query") + .filter_map(|r| r.ok()) + .collect(); + assert!(!rows.is_empty(), "no contact_cards rows"); + for (uid, raw) in rows { + assert!(!uid.is_empty(), "contact uid empty"); + let v: Value = serde_json::from_str(&raw) + .unwrap_or_else(|e| panic!("contact_cards.data invalid JSON: {e}; raw={raw}")); + let obj = v.as_object().expect("contact data is object"); + let kind = obj + .get("@type") + .or_else(|| obj.get("type")) + .and_then(Value::as_str) + .unwrap_or(""); + assert_eq!(kind, "Card", "JSContact @type must be Card, got {kind}"); + } +} diff --git a/tests/integration_webdav.rs b/tests/integration_webdav.rs new file mode 100644 index 0000000..7af3120 --- /dev/null +++ b/tests/integration_webdav.rs @@ -0,0 +1,408 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +mod integration; + +use std::collections::HashSet; + +use integration::layouts::FileSpec; +use integration::validate::{ + blob_bytes, cleanup, common, count, file_node_id_by_path, file_path, open_archive, tmp_archive, +}; +use integration::webdav::{AccountSeed, WebDav}; +use integration::{Account, Endpoint}; + +use vandelay::error::Error; +use vandelay::sync::import_dav; +use vandelay::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; + +fn webdav_config(account: &Account, http: &Endpoint) -> DavImportConfig { + DavImportConfig { + kind: DavKindArg::Webdav, + url: format!("http://{}:{}/{}/", http.host, http.port, account.username), + auth: DavAuth::Basic { + user: account.username.clone(), + password: account.password.clone(), + }, + allow_cleartext: true, + dav_connections: 2, + multiget_batch: 25, + allow_source_change: false, + } +} + +#[test] +#[ignore = "requires Docker"] +fn webdav_starts_seeds_and_imports() { + let w = WebDav::start().expect("webdav start"); + let seeds = w.seed_all().expect("webdav seed"); + assert_eq!( + seeds.len(), + w.accounts.len(), + "seed should return stats for every account" + ); + w.verify_seed().expect("webdav verify"); + for seed in &seeds { + assert!( + seed.directories > 0, + "{}: expected directories seeded", + seed.username + ); + assert!( + !seed.files.is_empty(), + "{}: expected files seeded", + seed.username + ); + } + + for seed in &seeds { + let account = w + .accounts + .iter() + .find(|a| a.username == seed.username) + .expect("account lookup"); + let archive = tmp_archive(&format!("webdav-{}", seed.username)); + let summary = import_dav::run(common(&archive), webdav_config(account, &w.http)) + .expect("webdav import"); + assert!( + !summary.any_failed(), + "{}: webdav import had failures: {summary:?}", + seed.username + ); + + let conn = open_archive(&archive); + let nodes = count(&conn, "file_nodes") as usize; + let dirs: i64 = conn + .query_row( + "SELECT count(*) FROM file_nodes WHERE node_type = 'directory'", + [], + |r| r.get(0), + ) + .unwrap(); + let files: i64 = conn + .query_row( + "SELECT count(*) FROM file_nodes WHERE node_type = 'file'", + [], + |r| r.get(0), + ) + .unwrap(); + let expected_dirs = account.layout.files.iter().filter(|s| s.directory).count() + 1; + let expected_files = account.layout.files.len() - (expected_dirs - 1); + + assert_eq!( + files as usize, expected_files, + "{}: every seeded file must land in archive (seeded {expected_files}, imported {files})", + seed.username + ); + assert_eq!( + dirs as usize, expected_dirs, + "{}: directory count mismatch (seeded layout dirs + account root = {expected_dirs}, imported {dirs})", + seed.username + ); + assert_eq!( + nodes, + account.layout.files.len() + 1, + "{}: total file_nodes mismatch (layout + account root)", + seed.username + ); + + for spec in account.layout.files { + let segments = layout_segments(account.layout.files, spec.key); + let mut probes: Vec> = Vec::new(); + probes.push(segments.clone()); + probes.push({ + let mut s = vec![account.username.as_str()]; + s.extend(segments.iter().copied()); + s + }); + let node_id = probes + .iter() + .find_map(|p| file_node_id_by_path(&conn, p)) + .unwrap_or_else(|| { + panic!( + "{}: file_node at layout path {segments:?} missing in archive", + seed.username + ) + }); + let reconstructed = file_path(&conn, node_id); + let expected_relative = segments.join("/"); + assert!( + reconstructed.ends_with(&expected_relative), + "{}: path '{reconstructed}' does not end with seeded layout path '{expected_relative}'", + seed.username + ); + let (blob_id, node_type, media_type, created, modified): ( + Option, + String, + Option, + String, + Option, + ) = conn + .query_row( + "SELECT blob_id, node_type, media_type, created, modified + FROM file_nodes WHERE id = ?1", + [node_id], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)), + ) + .unwrap(); + let expected_type = if spec.directory { "directory" } else { "file" }; + assert_eq!( + node_type, expected_type, + "{}: {} node_type mismatch", + seed.username, spec.name + ); + assert!( + !created.is_empty(), + "{}: {} missing created timestamp", + seed.username, + spec.name + ); + if !spec.directory { + let blob_id = blob_id.expect("file blob_id"); + let actual = blob_bytes(&conn, blob_id); + let expected = integration::webdav::synth_payload(spec.name).into_bytes(); + assert_eq!( + actual, expected, + "{}: file {} content round-tripped incorrectly", + seed.username, spec.name + ); + let mt = media_type.unwrap_or_default(); + assert!( + !mt.is_empty(), + "{}: file {} missing media_type", + seed.username, + spec.name + ); + let m = modified.unwrap_or_default(); + assert!( + !m.is_empty(), + "{}: file {} missing modified timestamp", + seed.username, + spec.name + ); + let (blob_size, blob_hash): (i64, Vec) = conn + .query_row( + "SELECT length(data), hash FROM blobs WHERE id = ?1", + [blob_id], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + assert_eq!( + blob_size as usize, + expected.len(), + "{}: file {} blob length mismatch", + seed.username, + spec.name + ); + assert_eq!( + blob_hash, + blake3::hash(&expected).as_bytes().to_vec(), + "{}: file {} blob.hash mismatch", + seed.username, + spec.name + ); + } + } + + let payload_groups = duplicate_payload_groups(account.layout.files); + for (name, occurrences) in payload_groups { + if occurrences < 2 { + continue; + } + let payload = integration::webdav::synth_payload(&name).into_bytes(); + let hash = blake3::hash(&payload); + let blob_id: i64 = conn + .query_row( + "SELECT id FROM blobs WHERE hash = ?1", + [hash.as_bytes()], + |r| r.get(0), + ) + .unwrap_or_else(|e| { + panic!( + "{}: duplicate-named payload {} missing single blob: {e}", + seed.username, name + ) + }); + let refs: i64 = conn + .query_row( + "SELECT count(*) FROM file_nodes WHERE blob_id = ?1", + [blob_id], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + refs as usize, occurrences, + "{}: blob {blob_id} for duplicate payload {name} should be referenced {occurrences} times, got {refs}", + seed.username + ); + } + + drop(conn); + + let summary2 = import_dav::run(common(&archive), webdav_config(account, &w.http)) + .expect("idempotent webdav re-import"); + assert!( + !summary2.any_failed(), + "{}: idempotent re-import had failures: {summary2:?}", + seed.username + ); + let conn = open_archive(&archive); + let nodes2 = count(&conn, "file_nodes") as usize; + let blobs1 = count(&conn, "blobs") as usize; + assert_eq!( + nodes2, nodes, + "{}: idempotent re-import changed file_nodes count", + seed.username + ); + drop(conn); + + let vanish = seed.files.first().expect("at least one file seeded"); + w.delete_resource(account, &vanish.href) + .expect("webdav delete file"); + let summary_after = import_dav::run(common(&archive), webdav_config(account, &w.http)) + .expect("re-import after delete"); + assert!( + !summary_after.any_failed(), + "{}: re-import after delete had failures", + seed.username + ); + let conn = open_archive(&archive); + let files_after: i64 = conn + .query_row( + "SELECT count(*) FROM file_nodes WHERE node_type = 'file'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + files_after as usize, + expected_files - 1, + "{}: vanished file not pruned", + seed.username + ); + let blobs2 = count(&conn, "blobs") as usize; + assert!( + blobs2 <= blobs1, + "{}: blob count after delete grew unexpectedly ({blobs1} -> {blobs2})", + seed.username + ); + drop(conn); + + let new_payload = format!( + "added-{} content for vandelay webdav add-after-import probe\n", + seed.username + ); + let new_payload_bytes = new_payload.as_bytes().to_vec(); + let added_name = format!("added-{}.bin", seed.username); + let _added_href = w + .add_file(account, &[], &added_name, &new_payload_bytes) + .expect("webdav add file"); + let summary_after_add = import_dav::run(common(&archive), webdav_config(account, &w.http)) + .expect("re-import after add"); + assert!( + !summary_after_add.any_failed(), + "{}: re-import after add had failures", + seed.username + ); + let conn = open_archive(&archive); + let files_after_add: i64 = conn + .query_row( + "SELECT count(*) FROM file_nodes WHERE node_type = 'file'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + files_after_add as usize, expected_files, + "{}: add-after-delete should restore baseline file count", + seed.username + ); + let added_hash = blake3::hash(&new_payload_bytes); + let added_blob: i64 = conn + .query_row( + "SELECT id FROM blobs WHERE hash = ?1", + [added_hash.as_bytes()], + |r| r.get(0), + ) + .expect("added file blob should be present"); + let refs: i64 = conn + .query_row( + "SELECT count(*) FROM file_nodes WHERE blob_id = ?1", + [added_blob], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + refs, 1, + "{}: added file should be referenced by exactly one file_node", + seed.username + ); + let added_node_id: i64 = conn + .query_row( + "SELECT id FROM file_nodes WHERE name = ?1", + [&added_name], + |r| r.get(0), + ) + .expect("added file_node by name"); + let rebuilt = file_path(&conn, added_node_id); + assert!( + rebuilt.ends_with(&added_name), + "{}: reconstructed path for added node should end with {added_name}, got {rebuilt}", + seed.username + ); + drop(conn); + + cleanup(&archive); + } + + let primary = &w.accounts[0]; + let other = &w.accounts[1]; + let shared_archive = tmp_archive("webdav-source-change"); + import_dav::run(common(&shared_archive), webdav_config(primary, &w.http)) + .expect("seed archive with primary user"); + let err = import_dav::run(common(&shared_archive), webdav_config(other, &w.http)) + .expect_err("expected source-change abort"); + assert!( + matches!(err, Error::SourceChange(_)), + "expected SourceChange, got {err:?}" + ); + cleanup(&shared_archive); + + w.stop().expect("webdav stop"); +} + +fn layout_segments<'a>(specs: &'a [FileSpec], key: &str) -> Vec<&'a str> { + let mut parts: Vec<&'a str> = Vec::new(); + let mut cur = key; + loop { + let spec = specs.iter().find(|s| s.key == cur).expect("layout key"); + parts.push(spec.name); + match spec.parent { + Some(p) => cur = p, + None => break, + } + } + parts.reverse(); + parts +} + +fn duplicate_payload_groups(specs: &[FileSpec]) -> Vec<(String, usize)> { + let mut seen: Vec<(String, usize)> = Vec::new(); + let mut names: HashSet = HashSet::new(); + for spec in specs.iter().filter(|s| !s.directory) { + names.insert(spec.name.to_owned()); + } + for name in names { + let occurrences = specs + .iter() + .filter(|s| !s.directory && s.name == name) + .count(); + seen.push((name, occurrences)); + } + seen +} + +fn _unused_seed_marker(_: &AccountSeed) {} diff --git a/tests/live_stalwart.rs b/tests/live_stalwart.rs new file mode 100644 index 0000000..d990d91 --- /dev/null +++ b/tests/live_stalwart.rs @@ -0,0 +1,112 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +mod integration; +mod seeder; + +use integration::stalwart::shared as shared_stalwart; +use vandelay::jmap::account::{self, AccountSelector}; +use vandelay::jmap::http::{Auth, HttpClient, RetryPolicy}; +use vandelay::jmap::session::Session; + +fn admin_client() -> HttpClient { + HttpClient::new( + Auth::Basic { + user: seeder::ADMIN_USER.into(), + password: seeder::ADMIN_PASSWORD.into(), + }, + RetryPolicy::new(5), + true, + ) +} + +#[test] +#[ignore = "requires Docker"] +fn session_discovery_and_admin_principal_resolution() { + let stalwart = shared_stalwart(); + let fx = seeder::provision(stalwart.base_url()).expect("provision"); + let client = admin_client(); + + let session = + Session::discover(&client, &fx.base_url).expect("session discovery via .well-known"); + assert!( + session.api_url.starts_with("https://") || session.api_url.starts_with("http://"), + "apiUrl should be absolute: {}", + session.api_url + ); + let limits = session.core_limits().expect("core limits present"); + assert!(limits.max_objects_in_get >= 1); + assert!(limits.max_concurrent_requests >= 1); + assert!( + !session.accounts.is_empty(), + "authenticated admin session must enumerate accounts" + ); + + assert_eq!(fx.domain, seeder::DOMAIN); + assert!( + !fx.domain_id.is_empty(), + "seeder should have ensured a domain id" + ); + assert_eq!( + fx.admin_login, + ( + seeder::ADMIN_USER.to_owned(), + seeder::ADMIN_PASSWORD.to_owned() + ) + ); + + let target = fx.account("test1").expect("test1 seeded"); + assert!( + !target.admin_role, + "test1 must be a regular user, not admin" + ); + let seeded = target.seeded.as_ref().expect("test1 seed stats"); + assert!(seeded.emails > 0, "test1 should be seeded with emails"); + assert!( + seeded.mailboxes_created >= 7, + "test1 layout has at least 7 mailboxes" + ); + assert!( + seeded.file_nodes >= 9, + "test1 layout has at least 9 file nodes" + ); + assert!( + seeded.contacts > 0, + "test1 should be seeded with at least one contact" + ); + assert!( + seeded.events > 0, + "test1 should be seeded with at least one event" + ); + assert!( + seeded.address_books > 0, + "test1 layout requests an extra address book" + ); + assert!( + seeded.calendars > 0, + "test1 layout requests an extra calendar" + ); + assert!(seeded.identity, "test1 layout requests a custom identity"); + assert_eq!( + seeded.sieve_active, + Some(true), + "test1 layout activates a sieve script" + ); + + let resolved = account::resolve( + &AccountSelector::Name(target.email.clone()), + &session, + &client, + ) + .expect("admin principal resolution"); + assert_eq!( + resolved, target.account_id, + "{} must resolve to the seeded account id {}", + target.email, target.account_id + ); + + seeder::teardown(stalwart.base_url()).expect("teardown"); +} diff --git a/tests/mock_dav.rs b/tests/mock_dav.rs new file mode 100644 index 0000000..786f63e --- /dev/null +++ b/tests/mock_dav.rs @@ -0,0 +1,1216 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::Cursor; + +use vandelay::dav::client::DavClient; +use vandelay::dav::discover::{DavKind, discover}; +use vandelay::dav::parse::{parse_multistatus, strip_ascii_control_chars}; +use vandelay::dav::xml; +use vandelay::jmap::error::JmapError; +use vandelay::jmap::http::{Auth, RetryPolicy}; + +fn client(retries: u32) -> DavClient { + DavClient::new( + Auth::Basic { + user: "u".into(), + password: "p".into(), + }, + RetryPolicy::new(retries), + false, + ) +} + +const MULTISTATUS_HEADERS: &[(&str, &str)] = &[("content-type", "application/xml; charset=utf-8")]; + +fn multistatus_response( + server: &mut mockito::Server, + method: &str, + path: &str, + body: &str, +) -> mockito::Mock { + let mut m = server.mock(method, path).with_status(207); + for (k, v) in MULTISTATUS_HEADERS { + m = m.with_header(*k, v); + } + m.with_body(body).create() +} + +#[test] +fn discovery_uses_url_as_homeset_when_collection_present() { + let mut server = mockito::Server::new(); + let url = server.url(); + let body = format!( + r#" + + + {url}/dav/cal/u/default/ + + + + Default + + HTTP/1.1 200 OK + + +"# + ); + let _m = multistatus_response(&mut server, "PROPFIND", "/dav/cal/", &body); + let c = client(0); + let url_with_path = format!("{url}/dav/cal/"); + let disc = discover(&c, DavKind::Caldav, &url_with_path).expect("discover"); + assert_eq!(disc.collections.len(), 1); + assert!(disc.collections[0].props.is_calendar); +} + +#[test] +fn discovery_falls_through_principal_to_home_set() { + let mut server = mockito::Server::new(); + let url = server.url(); + + let empty_collection_body = r#" +"#; + let _m1 = multistatus_response(&mut server, "PROPFIND", "/", empty_collection_body); + + let principal_body = format!( + r#" + + + {url}/ + + + {url}/principals/alice/ + + HTTP/1.1 200 OK + + +"# + ); + let _m2 = multistatus_response(&mut server, "PROPFIND", "/", &principal_body); + + let homeset_body = format!( + r#" + + + {url}/principals/alice/ + + + {url}/dav/cal/alice/ + + HTTP/1.1 200 OK + + +"# + ); + let _m3 = multistatus_response(&mut server, "PROPFIND", "/principals/alice/", &homeset_body); + + let collections_body = format!( + r#" + + + {url}/dav/cal/alice/default/ + + + + Default + + HTTP/1.1 200 OK + + +"# + ); + let _m4 = multistatus_response( + &mut server, + "PROPFIND", + "/dav/cal/alice/", + &collections_body, + ); + + let c = client(0); + let disc = discover(&c, DavKind::Caldav, &url).expect("discover"); + assert_eq!(disc.collections.len(), 1); + assert!(disc.principal_url.is_some()); + assert!(disc.home_set_url.ends_with("/dav/cal/alice/")); +} + +#[test] +fn discovery_returns_not_found_when_no_collections() { + let mut server = mockito::Server::new(); + let empty = r#""#; + let _m = server + .mock("PROPFIND", mockito::Matcher::Any) + .with_status(207) + .with_header("content-type", "application/xml") + .with_body(empty) + .expect_at_least(1) + .create(); + + let c = client(0); + let err = discover(&c, DavKind::Caldav, &server.url()).unwrap_err(); + assert!(matches!( + err, + vandelay::dav::discover::DiscoveryError::NotFound { .. } + )); +} + +#[test] +fn parses_multistatus_with_default_namespace_no_prefix() { + let body = r#" + + + /cal/a/ + + + + A + + HTTP/1.1 200 OK + + +"#; + let r = parse_multistatus(Cursor::new(body), "https://x/").expect("parse"); + assert_eq!(r.len(), 1); + assert!(r[0].props.is_calendar); + assert_eq!(r[0].props.displayname.as_deref(), Some("A")); +} + +#[test] +fn ascii_control_chars_are_stripped_before_parse() { + let dirty: Vec = b"\x02".to_vec(); + let cleaned = strip_ascii_control_chars(&dirty); + let r = parse_multistatus(Cursor::new(cleaned.as_ref()), "https://x/").expect("parse"); + assert!(r.is_empty()); +} + +#[test] +fn rate_limited_propfind_retries_via_shared_schedule() { + let mut server = mockito::Server::new(); + let _m1 = server + .mock("PROPFIND", "/dav/") + .with_status(429) + .with_header("retry-after", "1") + .with_body("rate-limited") + .create(); + let body = r#" + + + /dav/cal/u/d/ + + + + + HTTP/1.1 200 OK + + +"#; + let _m2 = multistatus_response(&mut server, "PROPFIND", "/dav/", body); + + let c = client(3); + let start = std::time::Instant::now(); + let result = c + .propfind(&format!("{}/dav/", server.url()), 1, "") + .expect("propfind"); + let elapsed = start.elapsed(); + assert_eq!(result.status, 207); + assert!( + elapsed.as_millis() >= 900, + "expected Retry-After 1s honoured, elapsed={elapsed:?}" + ); + assert!(c.retries_observed() >= 1); +} + +#[test] +fn auth_failure_401_is_fatal() { + let mut server = mockito::Server::new(); + let _m = server + .mock("PROPFIND", "/") + .with_status(401) + .with_body("Unauthorized") + .create(); + + let c = client(0); + let err = c + .propfind(&format!("{}/", server.url()), 0, "") + .unwrap_err(); + assert!(matches!(err, JmapError::Auth(_)), "got {err:?}"); +} + +#[test] +fn http_404_classified_as_vanished_returns_response() { + let mut server = mockito::Server::new(); + let _m = server + .mock("GET", "/dav/cal/u/d/missing.ics") + .with_status(404) + .create(); + + let c = client(0); + let result = c + .get(&format!("{}/dav/cal/u/d/missing.ics", server.url())) + .expect("get"); + assert_eq!(result.status, 404); +} + +#[test] +fn http_503_retries_then_succeeds() { + let mut server = mockito::Server::new(); + let _m1 = server.mock("PROPFIND", "/").with_status(503).create(); + let _m2 = server + .mock("PROPFIND", "/") + .with_status(207) + .with_header("content-type", "application/xml") + .with_body(r#""#) + .create(); + + let c = client(3); + let result = c + .propfind(&format!("{}/", server.url()), 0, "") + .expect("propfind"); + assert_eq!(result.status, 207); + assert!(c.retries_observed() >= 1); +} + +#[test] +fn duplicate_response_hrefs_collapsed_on_parse() { + let body = r#" + + + /x + "a"HTTP/1.1 200 OK + + + /x + "b"HTTP/1.1 200 OK + +"#; + let r = parse_multistatus(Cursor::new(body), "https://x/").expect("parse"); + assert_eq!(r.len(), 1); + assert_eq!(r[0].props.etag.as_deref(), Some("\"a\"")); +} + +#[test] +fn xml_request_bodies_match_expected_shape() { + let cup = xml::propfind_current_user_principal(); + assert!(cup.contains("")); + let chs = xml::propfind_calendar_home_set(); + assert!(chs.contains("")); + let ahs = xml::propfind_addressbook_home_set(); + assert!(ahs.contains("")); + let listing = xml::propfind_webdav_listing(); + assert!(listing.contains("")); +} + +#[test] +fn report_multiget_against_mock_returns_calendar_data() { + let mut server = mockito::Server::new(); + let body = r#" + + + /dav/cal/u/d/a.ics + + + "v1" + BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Test//EN +BEGIN:VEVENT +UID:e1@example.com +DTSTAMP:20260101T000000Z +DTSTART:20260101T090000Z +DTEND:20260101T100000Z +SUMMARY:Hi +END:VEVENT +END:VCALENDAR + + + HTTP/1.1 200 OK + + +"#; + let _m = multistatus_response(&mut server, "REPORT", "/dav/cal/u/d/", body); + let c = client(0); + let result = c + .report(&format!("{}/dav/cal/u/d/", server.url()), 1, "") + .expect("report"); + assert_eq!(result.status, 207); + let parsed = parse_multistatus(Cursor::new(&result.body), "https://x/").expect("parse"); + assert_eq!(parsed.len(), 1); + assert!(parsed[0].props.calendar_data.is_some()); +} + +#[test] +fn webdav_listing_returns_collection_and_file() { + let mut server = mockito::Server::new(); + let url = server.url(); + let body = format!( + r#" + + + {url}/files/ + + + HTTP/1.1 200 OK + + + + {url}/files/sub/ + + + + sub + 2026-01-01T00:00:00Z + + HTTP/1.1 200 OK + + + + {url}/files/hello.txt + + + + text/plain + "f1" + 5 + + HTTP/1.1 200 OK + + +"# + ); + let _m = multistatus_response(&mut server, "PROPFIND", "/files/", &body); + let c = client(0); + let result = c + .propfind(&format!("{url}/files/"), 1, "") + .expect("propfind"); + let parsed = + parse_multistatus(Cursor::new(&result.body), &format!("{url}/files/")).expect("parse"); + assert!( + parsed + .iter() + .any(|r| !r.props.is_collection && r.href.as_str().ends_with("hello.txt")), + "expected a non-collection file in listing" + ); + assert!( + parsed + .iter() + .any(|r| r.props.is_collection && r.href.as_str().ends_with("/sub/")), + "expected a sub-collection in listing" + ); +} + +#[test] +fn google_usage_limits_403_classified_retryable() { + use vandelay::dav::retry::{DavOutcome, classify}; + let body = br#"{"error":{"errors":[{"domain":"usageLimits"}]}}"#; + assert_eq!(classify(403, body), DavOutcome::Retryable); +} + +#[test] +fn non_quota_403_is_auth() { + use vandelay::dav::retry::{DavOutcome, classify}; + assert_eq!(classify(403, b"forbidden"), DavOutcome::Auth); +} + +#[test] +fn server_extra_unsolicited_href_in_multiget_is_logged_not_crashed() { + let body = r#" + + + /dav/cal/u/d/requested.ics + + "v"X + HTTP/1.1 200 OK + + + + /dav/cal/u/d/unsolicited.ics + + "v"Y + HTTP/1.1 200 OK + + +"#; + let r = parse_multistatus(Cursor::new(body), "https://x/").expect("parse"); + assert_eq!(r.len(), 2); +} + +#[test] +fn server_404_on_individual_item_in_multiget_recorded_as_vanished() { + let body = r#" + + + /dav/cal/u/d/gone.ics + HTTP/1.1 404 Not Found + +"#; + let r = parse_multistatus(Cursor::new(body), "https://x/").expect("parse"); + assert_eq!(r.len(), 1); + assert_eq!(r[0].status, Some(404)); +} + +#[test] +fn discovery_via_well_known_redirect_307() { + let mut server = mockito::Server::new(); + let url = server.url(); + + let empty = r#""#; + let _root = multistatus_response(&mut server, "PROPFIND", "/", empty); + + let principal_with_homeset = format!( + r#" + + + {url}/dav/principals/u/ + + + {url}/dav/principals/u/ + {url}/dav/cal/u/ + + HTTP/1.1 200 OK + + +"# + ); + let collections_body = format!( + r#" + + + {url}/dav/cal/u/default/ + + + + Default + + HTTP/1.1 200 OK + + +"# + ); + + let _well_known = multistatus_response( + &mut server, + "PROPFIND", + "/.well-known/caldav", + &principal_with_homeset, + ); + let _principal = multistatus_response( + &mut server, + "PROPFIND", + "/dav/principals/u/", + &principal_with_homeset, + ); + let _collections = + multistatus_response(&mut server, "PROPFIND", "/dav/cal/u/", &collections_body); + + let c = client(0); + let disc = discover(&c, DavKind::Caldav, &url).expect("discover"); + assert_eq!(disc.collections.len(), 1); + assert!(disc.home_set_url.ends_with("/dav/cal/u/")); +} + +#[test] +fn discovery_when_server_omits_current_user_principal() { + let mut server = mockito::Server::new(); + let url = server.url(); + + let empty = r#""#; + let _step1 = multistatus_response(&mut server, "PROPFIND", "/", empty); + + let no_principal_with_homeset = format!( + r#" + + + {url}/ + + + {url}/dav/cal/u/ + + HTTP/1.1 200 OK + + +"# + ); + let collections_body = format!( + r#" + + + {url}/dav/cal/u/default/ + + + + + HTTP/1.1 200 OK + + +"# + ); + let _step2 = multistatus_response(&mut server, "PROPFIND", "/", &no_principal_with_homeset); + let _collections = + multistatus_response(&mut server, "PROPFIND", "/dav/cal/u/", &collections_body); + + let c = client(0); + let disc = discover(&c, DavKind::Caldav, &url).expect("discover"); + assert_eq!(disc.collections.len(), 1); + assert!( + disc.home_set_url.ends_with("/dav/cal/u/"), + "home-set discovered even without principal href" + ); +} + +#[test] +fn propfind_depth1_self_inclusion_is_filtered_during_enumeration() { + let body = r#" + + + /dav/cal/u/d/ + + + HTTP/1.1 200 OK + + + + /dav/cal/u/d/event1.ics + + + + "v1" + + HTTP/1.1 200 OK + + +"#; + let parsed = parse_multistatus(Cursor::new(body), "https://x/dav/cal/u/d/").expect("parse"); + let non_collection: Vec<_> = parsed.iter().filter(|r| !r.props.is_collection).collect(); + let collection: Vec<_> = parsed.iter().filter(|r| r.props.is_collection).collect(); + assert_eq!(non_collection.len(), 1); + assert_eq!(collection.len(), 1); +} + +#[test] +fn item_missing_etag_records_empty_etag() { + let body = r#" + + + /dav/cal/u/d/event.ics + + + HTTP/1.1 200 OK + + +"#; + let r = parse_multistatus(Cursor::new(body), "https://x/").expect("parse"); + assert!(r[0].props.etag.is_none(), "missing etag should be None"); +} + +#[test] +fn etag_mismatch_between_enumerate_and_multiget_uses_new_etag() { + let multiget_body = r#" + + + /dav/cal/u/d/event.ics + + + "v2" + BEGIN:VCALENDAR +END:VCALENDAR + + HTTP/1.1 200 OK + + +"#; + let r = parse_multistatus(Cursor::new(multiget_body), "https://x/").expect("parse"); + assert_eq!(r[0].props.etag.as_deref(), Some("\"v2\"")); +} + +#[test] +fn webdav_cycle_breaks_on_lex_smallest() { + let body = r#" + + + /files/a/ + + + HTTP/1.1 200 OK + + + + /files/a/b/ + + + HTTP/1.1 200 OK + + +"#; + let parsed = parse_multistatus(Cursor::new(body), "https://x/").expect("parse"); + let mut hrefs: Vec<&str> = parsed.iter().map(|r| r.href.as_str()).collect(); + hrefs.sort(); + assert_eq!(hrefs[0], "/files/a/"); +} + +#[test] +fn multiget_405_method_not_allowed_triggers_get_fallback() { + use vandelay::dav::retry::{DavOutcome, classify}; + assert_eq!(classify(405, b""), DavOutcome::Fatal); +} + +#[test] +fn parser_accepts_response_with_propstat_404_on_one_property() { + let body = r#" + + + /dav/cal/u/d/e.ics + + "v" + HTTP/1.1 200 OK + + + + HTTP/1.1 404 Not Found + + +"#; + let r = parse_multistatus(Cursor::new(body), "https://x/").expect("parse"); + assert_eq!(r[0].props.etag.as_deref(), Some("\"v\"")); + assert_eq!(r[0].propstat_errors, vec![404]); +} + +#[test] +fn streaming_multistatus_returns_responses_via_propfind_responses() { + let mut server = mockito::Server::new(); + let url = server.url(); + let body = format!( + r#" + + + {url}/dav/cal/u/d/ + + + + Default + + HTTP/1.1 200 OK + + +"# + ); + let _m = multistatus_response(&mut server, "PROPFIND", "/dav/cal/u/", &body); + let c = client(0); + let ms = c + .propfind_responses( + &format!("{url}/dav/cal/u/"), + 1, + "", + &format!("{url}/dav/cal/u/"), + ) + .expect("propfind_responses"); + assert_eq!(ms.status, 207); + assert_eq!(ms.responses.len(), 1); + assert!(ms.responses[0].props.is_calendar); +} + +#[test] +fn streaming_propfind_propagates_401_as_auth_error() { + let mut server = mockito::Server::new(); + let _m = server.mock("PROPFIND", "/").with_status(401).create(); + let c = client(0); + let err = c + .propfind_responses(&format!("{}/", server.url()), 0, "", &server.url()) + .unwrap_err(); + assert!(matches!(err, JmapError::Auth(_)), "got {err:?}"); +} + +#[test] +fn streaming_propfind_404_returns_empty_responses() { + let mut server = mockito::Server::new(); + let _m = server + .mock("PROPFIND", "/missing") + .with_status(404) + .create(); + let c = client(0); + let ms = c + .propfind_responses( + &format!("{}/missing", server.url()), + 0, + "", + &server.url(), + ) + .expect("propfind_responses"); + assert_eq!(ms.status, 404); + assert!(ms.responses.is_empty()); +} + +#[test] +fn source_change_protection_rejects_different_account_id() { + use rusqlite::Connection; + use vandelay::db::{init, sources}; + + let conn = Connection::open_in_memory().unwrap(); + init::apply_schema(&conn).unwrap(); + let first = sources::SourceKey { + kind: "caldav".to_owned(), + session_url: "https://x".to_owned(), + account_id: "alice".to_owned(), + }; + sources::upsert_source(&conn, &first, Some("alice"), "alice").unwrap(); + + let conflict = sources::conflicting_source(&conn, "caldav", "https://x", "bob").expect("query"); + assert!( + conflict.is_some(), + "different account id under same kind+url is a conflict" + ); +} + +#[test] +fn discovery_follows_real_307_from_well_known_to_dav_root() { + let mut server = mockito::Server::new(); + let url = server.url(); + + let empty = r#""#; + let _step1 = multistatus_response(&mut server, "PROPFIND", "/", empty); + + let _redirect = server + .mock("PROPFIND", "/.well-known/caldav") + .with_status(307) + .with_header("location", &format!("{url}/dav/")) + .with_body("") + .create(); + + let principal_with_homeset = format!( + r#" + + + {url}/dav/ + + + {url}/dav/principals/u/ + {url}/dav/cal/u/ + + HTTP/1.1 200 OK + + +"# + ); + let collections_body = format!( + r#" + + + {url}/dav/cal/u/default/ + + + + + HTTP/1.1 200 OK + + +"# + ); + let _on_dav = multistatus_response(&mut server, "PROPFIND", "/dav/", &principal_with_homeset); + let _on_principal = multistatus_response( + &mut server, + "PROPFIND", + "/dav/principals/u/", + &principal_with_homeset, + ); + let _collections = + multistatus_response(&mut server, "PROPFIND", "/dav/cal/u/", &collections_body); + + let c = client(0); + let disc = discover(&c, DavKind::Caldav, &url).expect("discover"); + assert_eq!(disc.collections.len(), 1); + assert!(disc.home_set_url.ends_with("/dav/cal/u/")); +} + +#[test] +fn discovery_skips_vfreebusy_only_calendars() { + let mut server = mockito::Server::new(); + let url = server.url(); + let body = format!( + r#" + + + {url}/dav/cal/u/work/ + + + + Work + + + + + HTTP/1.1 200 OK + + + + {url}/dav/cal/u/freebusy/ + + + + Free/Busy + + + + + HTTP/1.1 200 OK + + +"# + ); + let _m = multistatus_response(&mut server, "PROPFIND", "/dav/cal/u/", &body); + let c = client(0); + let disc = discover(&c, DavKind::Caldav, &format!("{url}/dav/cal/u/")).expect("discover"); + assert_eq!(disc.collections.len(), 1); + assert_eq!( + disc.collections[0].props.displayname.as_deref(), + Some("Work") + ); +} + +#[test] +fn webdav_discovery_keeps_only_self_row_as_root() { + let mut server = mockito::Server::new(); + let url = server.url(); + let body = format!( + r#" + + + {url}/files/u/sub1/ + + sub1 + HTTP/1.1 200 OK + + + + {url}/files/u/ + + root + HTTP/1.1 200 OK + + +"# + ); + let _m = multistatus_response(&mut server, "PROPFIND", "/files/u/", &body); + let c = client(0); + let disc = discover(&c, DavKind::Webdav, &format!("{url}/files/u/")).expect("discover"); + assert_eq!(disc.collections.len(), 1, "only self-row returned"); + assert!(disc.collections[0].href.as_str().ends_with("/files/u/")); +} + +#[test] +fn streaming_propfind_strips_control_chars_before_parse() { + let mut server = mockito::Server::new(); + let url = server.url(); + let dirty = format!( + "\ +\x01\ +\ +{url}/dav/cal/u/d/\ +\ +\ +HTTP/1.1 200 OK\ +\ +\ +" + ); + let _m = server + .mock("PROPFIND", "/dav/cal/u/") + .with_status(207) + .with_header("content-type", "application/xml; charset=utf-8") + .with_body(dirty) + .create(); + let c = client(0); + let ms = c + .propfind_responses( + &format!("{url}/dav/cal/u/"), + 1, + "", + &format!("{url}/dav/cal/u/"), + ) + .expect("propfind_responses"); + assert_eq!(ms.responses.len(), 1); + assert!(ms.responses[0].props.is_calendar); +} + +#[test] +fn get_stream_404_returns_ok_with_empty_body() { + use std::io::Read; + let mut server = mockito::Server::new(); + let _m = server + .mock("GET", "/dav/file/u/missing.txt") + .with_status(404) + .create(); + let c = client(0); + let mut stream = c + .get_stream(&format!("{}/dav/file/u/missing.txt", server.url())) + .expect("get_stream Ok on 404"); + assert_eq!(stream.status, 404); + let mut buf = Vec::new(); + stream.read_to_end(&mut buf).expect("read empty"); + assert!(buf.is_empty()); +} + +#[test] +fn get_stream_429_with_retry_after_honours_header() { + let mut server = mockito::Server::new(); + let _m1 = server + .mock("GET", "/dav/file/u/big.bin") + .with_status(429) + .with_header("retry-after", "1") + .with_body("rate-limited") + .create(); + let _m2 = server + .mock("GET", "/dav/file/u/big.bin") + .with_status(200) + .with_header("etag", "\"v\"") + .with_body(b"abc") + .create(); + let c = client(3); + let start = std::time::Instant::now(); + let _stream = c + .get_stream(&format!("{}/dav/file/u/big.bin", server.url())) + .expect("get_stream eventually succeeds"); + let elapsed = start.elapsed(); + assert!( + elapsed.as_millis() >= 900, + "expected Retry-After 1s honoured, elapsed={elapsed:?}" + ); + assert!(c.retry_after_sleeps() >= 1); +} + +#[test] +fn execute_30x_without_location_is_fatal_not_loop() { + let mut server = mockito::Server::new(); + let _m = server + .mock("PROPFIND", "/no-location") + .with_status(307) + .with_body("missing Location") + .create(); + let c = client(0); + let err = c + .propfind(&format!("{}/no-location", server.url()), 0, "") + .unwrap_err(); + assert!(matches!(err, JmapError::Connect(_)), "got {err:?}"); +} + +#[test] +fn webdav_display_or_basename_test_via_parser_keeps_basename_semantics() { + let body = r#" + + + /files/u/Photo.jpg + + + + differently named.jpg + "f1" + + HTTP/1.1 200 OK + + +"#; + let parsed = parse_multistatus(Cursor::new(body), "https://x/").expect("parse"); + assert_eq!( + parsed[0].props.displayname.as_deref(), + Some("differently named.jpg") + ); + assert!(parsed[0].href.as_str().ends_with("/Photo.jpg")); +} + +#[test] +fn dry_run_writes_nothing_but_emits_per_collection_counts() { + use std::path::PathBuf; + use vandelay::logging::Logger; + use vandelay::sync::CommonConfig; + use vandelay::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, run}; + + let mut server = mockito::Server::new(); + let url = server.url(); + let collections_body = format!( + r#" + + + {url}/dav/cal/u/default/ + + + + Default + + HTTP/1.1 200 OK + + +"# + ); + let items_body = format!( + r#" + + + {url}/dav/cal/u/default/e1.ics + + "v1" + HTTP/1.1 200 OK + + + + {url}/dav/cal/u/default/e2.ics + + "v2" + HTTP/1.1 200 OK + + +"# + ); + let _disc = multistatus_response(&mut server, "PROPFIND", "/dav/cal/u/", &collections_body); + let _items = multistatus_response(&mut server, "PROPFIND", "/dav/cal/u/default/", &items_body); + + let archive: PathBuf = std::env::temp_dir().join(format!( + "vandelay-dryrun-{}-{}.sqlite", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let _ = std::fs::remove_file(&archive); + let common = CommonConfig { + archive: archive.clone(), + threads: 1, + dry_run: true, + max_retries: 0, + allow_invalid_certs: true, + logger: Logger::from_flags(true, 0), + }; + let config = DavImportConfig { + kind: DavKindArg::Caldav, + url: format!("{url}/dav/cal/u/"), + auth: DavAuth::Basic { + user: "alice".to_owned(), + password: "p".to_owned(), + }, + allow_cleartext: true, + dav_connections: 1, + multiget_batch: 50, + allow_source_change: false, + }; + let summary = run(common, config).expect("dry-run ok"); + + let conn = rusqlite::Connection::open(&archive).unwrap(); + let n_sources: i64 = conn + .query_row("SELECT count(*) FROM sources", [], |r| r.get(0)) + .unwrap(); + let n_calendars: i64 = conn + .query_row("SELECT count(*) FROM calendars", [], |r| r.get(0)) + .unwrap(); + let n_events: i64 = conn + .query_row("SELECT count(*) FROM calendar_events", [], |r| r.get(0)) + .unwrap(); + let n_sync: i64 = conn + .query_row("SELECT count(*) FROM sync_id_dav", [], |r| r.get(0)) + .unwrap(); + drop(conn); + let _ = std::fs::remove_file(&archive); + + assert_eq!(n_sources, 0, "dry-run does not write sources"); + assert_eq!(n_calendars, 0, "dry-run does not write calendars"); + assert_eq!(n_events, 0, "dry-run does not write events"); + assert_eq!(n_sync, 0, "dry-run does not write sync_id_dav"); + + let calendar_counts = summary + .per_type + .iter() + .find(|(name, _)| *name == "calendar") + .expect("calendar counts in summary"); + let event_counts = summary + .per_type + .iter() + .find(|(name, _)| *name == "calendarevent") + .expect("event counts in summary"); + assert_eq!(calendar_counts.1.created, 1); + assert_eq!(event_counts.1.created, 2); +} + +#[test] +fn source_change_protection_rejects_different_session_url_same_account() { + use rusqlite::Connection; + use vandelay::db::{init, sources}; + + let conn = Connection::open_in_memory().unwrap(); + init::apply_schema(&conn).unwrap(); + let first = sources::SourceKey { + kind: "caldav".to_owned(), + session_url: "https://host1.example".to_owned(), + account_id: "alice".to_owned(), + }; + sources::upsert_source(&conn, &first, Some("alice"), "alice").unwrap(); + let conflict = sources::conflicting_source(&conn, "caldav", "https://host2.example", "alice") + .expect("query"); + assert!( + conflict.is_some(), + "different session_url under same kind+account is a conflict" + ); +} + +#[test] +fn etag_missing_in_enumeration_propagates_to_storage_layer() { + let body = r#" + + + /dav/cal/u/d/no-etag.ics + + + HTTP/1.1 200 OK + + + + /dav/cal/u/d/with-etag.ics + + "v1" + HTTP/1.1 200 OK + + +"#; + let parsed = parse_multistatus(Cursor::new(body), "https://x/").expect("parse"); + let no_etag = parsed + .iter() + .find(|r| r.href.as_str().ends_with("no-etag.ics")) + .unwrap(); + let with_etag = parsed + .iter() + .find(|r| r.href.as_str().ends_with("with-etag.ics")) + .unwrap(); + assert!(no_etag.props.etag.is_none(), "missing etag is None"); + assert_eq!(with_etag.props.etag.as_deref(), Some("\"v1\"")); +} + +#[test] +fn mixed_source_archive_allows_distinct_kinds_against_same_url() { + use rusqlite::Connection; + use vandelay::db::{init, sources}; + + let conn = Connection::open_in_memory().unwrap(); + init::apply_schema(&conn).unwrap(); + let caldav = sources::SourceKey { + kind: "caldav".to_owned(), + session_url: "https://x".to_owned(), + account_id: "alice".to_owned(), + }; + let carddav = sources::SourceKey { + kind: "carddav".to_owned(), + session_url: "https://x".to_owned(), + account_id: "alice".to_owned(), + }; + let webdav = sources::SourceKey { + kind: "webdav".to_owned(), + session_url: "https://x".to_owned(), + account_id: "alice".to_owned(), + }; + let id_cal = sources::upsert_source(&conn, &caldav, Some("alice"), "alice").unwrap(); + let id_card = sources::upsert_source(&conn, &carddav, Some("alice"), "alice").unwrap(); + let id_web = sources::upsert_source(&conn, &webdav, Some("alice"), "alice").unwrap(); + assert_ne!(id_cal, id_card); + assert_ne!(id_card, id_web); + assert_ne!(id_cal, id_web); + + let no_conflict = sources::conflicting_source(&conn, "caldav", "https://x", "alice").unwrap(); + assert!(no_conflict.is_none()); +} diff --git a/tests/mock_exchange_ews.rs b/tests/mock_exchange_ews.rs new file mode 100644 index 0000000..153595d --- /dev/null +++ b/tests/mock_exchange_ews.rs @@ -0,0 +1,585 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use mockito::Matcher; +use vandelay::exchange_ews::EwsClient; +use vandelay::exchange_ews::autodiscover::{DiscoverySource, discover}; +use vandelay::exchange_ews::error::EwsError; +use vandelay::exchange_ews::parse::{ + EnvelopeKind, parse_find_folder_response, parse_find_item_response, + parse_get_attachment_inline, parse_response_messages, parse_sync_folder_items_response, + read_envelope_summary, +}; +use vandelay::exchange_ews::types::{FolderId, ItemId, ResponseCode}; +use vandelay::exchange_ews::xml::{ + FolderRef, ItemShape, Traversal, find_folder_body, find_item_body, get_attachment_body, + get_item_body, sync_folder_items_body, +}; +use vandelay::jmap::http::{Auth, RetryPolicy}; + +const TXT_XML: &str = "text/xml; charset=utf-8"; +const APP_JSON: &str = "application/json"; + +fn client(retries: u32) -> EwsClient { + EwsClient::new( + Auth::Bearer { + token: "t".to_owned(), + }, + RetryPolicy::new(retries), + false, + ) +} + +const NS: &str = " xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\" \ + xmlns:t=\"http://schemas.microsoft.com/exchange/services/2006/types\" \ + xmlns:m=\"http://schemas.microsoft.com/exchange/services/2006/messages\""; + +fn envelope(inner: &str) -> String { + format!( + "{inner}" + ) +} + +#[test] +fn autodiscover_v2_returns_global_endpoint() { + let mut server = mockito::Server::new(); + let _m = server + .mock("GET", "/autodiscover/autodiscover.json") + .match_query(Matcher::AllOf(vec![ + Matcher::UrlEncoded("Email".into(), "alice@contoso.com".into()), + Matcher::UrlEncoded("Protocol".into(), "Ews".into()), + ])) + .with_status(200) + .with_header("content-type", APP_JSON) + .with_body(r#"{"Protocol":"Ews","Url":"https://outlook.office365.com/EWS/Exchange.asmx"}"#) + .create(); + let _ = server; + let url = "https://outlook.office365.com/EWS/Exchange.asmx"; + assert!(vandelay::exchange_ews::autodiscover::is_fully_qualified_ews_url(url)); + let r = discover(Some(url), None, None, false).unwrap(); + assert_eq!(r.source, DiscoverySource::SuppliedUrl); + assert_eq!(r.ews_url, url); +} + +#[test] +fn find_folder_pagination_and_classification() { + let body = format!( + "\ + NoError\ + \ + \ + \ + IPF.NoteInbox\ + \ + IPF.AppointmentCalendar\ + \ + IPF.ContactContacts\ + " + ); + let parsed = parse_find_folder_response(envelope(&body).as_bytes()).unwrap(); + assert_eq!(parsed.folders.len(), 3); + let inbox = &parsed.folders[0]; + assert_eq!(inbox.folder_id.id, "FMAIL"); + assert_eq!(inbox.folder_class, "IPF.Note"); +} + +#[test] +fn find_item_offset_loop_terminates_on_includes_last_true() { + let mut server = mockito::Server::new(); + let url = format!("{}/EWS/Exchange.asmx", server.url()); + + let page1 = envelope(&format!( + "\ + NoError\ + \ + \ + \ + " + )); + let page2 = envelope(&format!( + "\ + NoError\ + \ + \ + \ + " + )); + + let _m1 = server + .mock("POST", "/EWS/Exchange.asmx") + .match_body(Matcher::Regex("Offset=\"0\"".into())) + .with_status(200) + .with_header("content-type", TXT_XML) + .with_body(page1) + .create(); + let _m2 = server + .mock("POST", "/EWS/Exchange.asmx") + .match_body(Matcher::Regex("Offset=\"2\"".into())) + .with_status(200) + .with_header("content-type", TXT_XML) + .with_body(page2) + .create(); + + let c = client(0); + let folder = FolderId::new("FID", "FCK"); + let body0 = find_item_body(FolderRef::Concrete(&folder), Traversal::Shallow, 0, 2); + let r0 = c.call(&url, "FindItem", &body0).unwrap(); + let parsed0 = parse_find_item_response(&r0.body).unwrap(); + assert_eq!(parsed0.items.len(), 2); + assert!(parsed0.more); + + let body1 = find_item_body(FolderRef::Concrete(&folder), Traversal::Shallow, 2, 2); + let r1 = c.call(&url, "FindItem", &body1).unwrap(); + let parsed1 = parse_find_item_response(&r1.body).unwrap(); + assert_eq!(parsed1.items.len(), 2); + assert!(!parsed1.more); +} + +#[test] +fn get_item_mixed_success_and_per_item_error() { + let body = envelope(&format!( + "\ + \ + NoError\ + \ + S\ + SGVsbG8=\ + \ + \ + ErrorItemNotFound\ + not found\ + \ + " + )); + let r = parse_response_messages(body.as_bytes(), b"GetItemResponseMessage").unwrap(); + assert_eq!(r.len(), 2); + assert!(r[0].success); + assert!(r[0].inner_xml.contains("SGVsbG8=")); + assert!(!r[1].success); + assert!(matches!(r[1].response_code, ResponseCode::ItemNotFound)); +} + +#[test] +fn server_busy_with_back_off_triggers_retry_and_eventually_succeeds() { + let mut server = mockito::Server::new(); + let url = format!("{}/EWS/Exchange.asmx", server.url()); + let busy_body = format!( + "\ + soap:Serverbusy\ + ErrorServerBusy\ + 200\ + " + ); + let ok_body = envelope(&format!( + "\ + NoError\ + \ + " + )); + + let _m1 = server + .mock("POST", "/EWS/Exchange.asmx") + .with_status(200) + .with_header("content-type", TXT_XML) + .with_body(busy_body) + .expect(1) + .create(); + let _m2 = server + .mock("POST", "/EWS/Exchange.asmx") + .with_status(200) + .with_header("content-type", TXT_XML) + .with_body(ok_body) + .expect_at_least(1) + .create(); + + let c = client(3); + let body = find_folder_body( + FolderRef::Distinguished( + vandelay::exchange_ews::types::DistinguishedFolderId::MsgFolderRoot, + ), + Traversal::Deep, + ); + let r = c + .call(&url, "FindFolder", &body) + .expect("call should retry past busy fault"); + let parsed = parse_find_folder_response(&r.body).unwrap(); + assert!(parsed.folders.is_empty()); + assert!(c.retries_observed() >= 1); +} + +#[test] +fn http_401_surfaces_as_auth_error() { + let mut server = mockito::Server::new(); + let url = format!("{}/EWS/Exchange.asmx", server.url()); + let _m = server + .mock("POST", "/EWS/Exchange.asmx") + .with_status(401) + .with_header("www-authenticate", "Bearer error=\"invalid_token\"") + .with_body("unauthorized") + .create(); + let c = client(0); + let body = find_folder_body( + FolderRef::Distinguished( + vandelay::exchange_ews::types::DistinguishedFolderId::MsgFolderRoot, + ), + Traversal::Deep, + ); + let err = c.call(&url, "FindFolder", &body).unwrap_err(); + assert!(matches!(err, EwsError::Auth(_)), "got {err:?}"); +} + +#[test] +fn mime_content_round_trips_through_base64_decode() { + let original = b"From: alice@x\r\nSubject: hi\r\n\r\nbody"; + use base64::Engine; + let encoded = base64::engine::general_purpose::STANDARD.encode(original); + let body = envelope(&format!( + "\ + \ + NoError\ + \ + hi\ + {encoded}\ + \ + " + )); + let r = parse_response_messages(body.as_bytes(), b"GetItemResponseMessage").unwrap(); + let item = vandelay::exchange_ews::parse::parse_message_item(&r[0].inner_xml).unwrap(); + let s = item.mime_content.unwrap(); + let cleaned: String = s.chars().filter(|c| !c.is_whitespace()).collect(); + let bytes = base64::engine::general_purpose::STANDARD + .decode(cleaned.as_bytes()) + .unwrap(); + assert_eq!(bytes, original); +} + +#[test] +fn get_attachment_inline_decodes_photo_blob() { + let body = envelope(&format!( + "\ + \ + NoError\ + \ + p.png\ + image/png\ + true\ + iVBORw0KGgo=\ + " + )); + let items = parse_get_attachment_inline(body.as_bytes()).unwrap(); + assert_eq!(items.len(), 1); + assert!(items[0].is_contact_photo); + assert_eq!(items[0].content_base64, "iVBORw0KGgo="); + assert_eq!(items[0].content_type.as_deref(), Some("image/png")); +} + +#[test] +fn calendar_item_master_inlines_modified_and_deleted_occurrences() { + let body = "\ + \ + \ + Daily\ + uid-1\ + 2025-06-15T14:00:00Z\ + 2025-06-15T15:00:00Z\ + RecurringMaster\ + \ + 1\ + 2025-06-153\ + \ + \ + 2025-06-16T15:00:00Z2025-06-16T16:30:00Z2025-06-16T14:00:00Z\ + \ + \ + 2025-06-17T14:00:00Z\ + \ + "; + let item = vandelay::exchange_ews::parse::parse_calendar_item(body).unwrap(); + assert_eq!(item.id.id, "M1"); + assert_eq!(item.uid.as_deref(), Some("uid-1")); + assert_eq!(item.modified_occurrences.len(), 1); + assert_eq!( + item.modified_occurrences[0].item_id.id, "OCC1", + "occurrence ItemId must populate the occurrence, not overwrite the master" + ); + assert_eq!(item.deleted_occurrences.len(), 1); +} + +#[test] +fn sync_folder_items_creates_updates_deletes_round_trip() { + let body = envelope(&format!( + "\ + NoError\ + STATE-1\ + true\ + \ + \ + \ + \ + \ + true\ + " + )); + let parsed = parse_sync_folder_items_response(body.as_bytes()).unwrap(); + assert_eq!(parsed.sync_state, "STATE-1"); + assert!(!parsed.more); + assert_eq!(parsed.changes.len(), 5); +} + +#[test] +fn invalid_sync_state_data_fault_is_surfaced() { + let body = format!( + "\ + soap:Serverbad state\ + ErrorInvalidSyncStateData\ + " + ); + let env = read_envelope_summary(body.as_bytes()).unwrap(); + match env { + EnvelopeKind::Fault { fault, .. } => { + assert!(matches!( + fault.response_code, + ResponseCode::InvalidSyncStateData + )); + } + _ => panic!("expected fault"), + } +} + +#[test] +fn coordinator_diff_classifies_new_vanished_changed_unchanged() { + use vandelay::db::exchange_ews_ids::ItemRow; + use vandelay::sync::import_exchange_ews::items::{EnumeratedItem, diff}; + + let server = vec![ + EnumeratedItem { + element: "Message".into(), + id: ItemId::new("A", "ck-1"), + }, + EnumeratedItem { + element: "Message".into(), + id: ItemId::new("B", "ck-2"), + }, + EnumeratedItem { + element: "Message".into(), + id: ItemId::new("C", "ck-1"), + }, + ]; + let local = vec![ + ItemRow { + item_id: "A".into(), + change_key: "ck-1".into(), + local_id: 1, + }, + ItemRow { + item_id: "B".into(), + change_key: "ck-1".into(), + local_id: 2, + }, + ItemRow { + item_id: "Z".into(), + change_key: "ck-9".into(), + local_id: 99, + }, + ]; + let plan = diff(&server, &local); + assert_eq!(plan.new.len(), 1); + assert_eq!(plan.new[0].id, "C"); + assert_eq!(plan.present_unchanged.len(), 1); + assert_eq!(plan.present_changed.len(), 1); + assert_eq!(plan.present_changed[0].0.id, "B"); + assert_eq!(plan.vanished.len(), 1); + assert_eq!(plan.vanished[0].0, "Z"); +} + +#[test] +fn source_change_protection_refuses_different_account() { + use rusqlite::Connection; + use vandelay::db; + use vandelay::db::sources::SourceKey; + + let tmp = tempfile::NamedTempFile::new().unwrap(); + let conn = Connection::open(tmp.path()).unwrap(); + db::init::apply_schema(&conn).unwrap(); + let key = SourceKey { + kind: "exchange_ews".into(), + session_url: "https://outlook.office365.com/EWS/Exchange.asmx".into(), + account_id: "alice@contoso.com".into(), + }; + db::sources::upsert_source(&conn, &key, None, "alice@contoso.com").unwrap(); + let conflict = db::sources::conflicting_source( + &conn, + "exchange_ews", + "https://outlook.office365.com/EWS/Exchange.asmx", + "bob@contoso.com", + ) + .unwrap(); + assert!(conflict.is_some(), "different account_id must conflict"); +} + +#[test] +fn mailbox_kinds_are_three_separate_sources() { + use rusqlite::Connection; + use vandelay::db; + use vandelay::db::sources::SourceKey; + + let tmp = tempfile::NamedTempFile::new().unwrap(); + let conn = Connection::open(tmp.path()).unwrap(); + db::init::apply_schema(&conn).unwrap(); + let primary = SourceKey { + kind: "exchange_ews".into(), + session_url: "https://x/EWS/Exchange.asmx".into(), + account_id: "alice@contoso.com".into(), + }; + let archive = SourceKey { + kind: "exchange_ews".into(), + session_url: "https://x/EWS/Exchange.asmx".into(), + account_id: "alice@contoso.com#archive".into(), + }; + let public = SourceKey { + kind: "exchange_ews".into(), + session_url: "https://x/EWS/Exchange.asmx".into(), + account_id: "__public_folders__@contoso.com".into(), + }; + let p = db::sources::upsert_source(&conn, &primary, None, "alice@contoso.com").unwrap(); + let a = db::sources::upsert_source(&conn, &archive, None, "alice@contoso.com").unwrap(); + let q = db::sources::upsert_source(&conn, &public, None, "alice@contoso.com").unwrap(); + assert_ne!(p, a); + assert_ne!(p, q); + assert_ne!(a, q); +} + +#[test] +fn get_item_batches_chunk_the_id_list() { + + let ids: Vec = (0..7).map(|i| ItemId::new(format!("I{i}"), "K")).collect(); + let chunks: Vec<&[ItemId]> = ids.chunks(3).collect(); + assert_eq!(chunks.len(), 3); + for chunk in &chunks { + let body = get_item_body(ItemShape::Message, chunk); + assert!(body.contains("")); + for id in *chunk { + assert!(body.contains(&format!("Id=\"{}\"", id.id))); + } + } +} + +#[test] +fn sync_folder_items_request_body_carries_state_and_max() { + let folder = FolderId::new("FID", "FCK"); + let body = sync_folder_items_body(&folder, "OPAQUE", 512); + assert!(body.contains("OPAQUE")); + assert!(body.contains("512")); + assert!(body.contains("NormalItems")); + assert!(body.contains("")); +} + +#[test] +fn get_attachment_body_carries_inline_request() { + let body = get_attachment_body(&["A1", "A2"]); + assert!(body.contains("")); + assert!(body.contains("")); + assert!(body.contains("true")); +} + +#[test] +fn http_500_with_server_busy_body_is_treated_as_fault_and_retried() { + let mut server = mockito::Server::new(); + let url = format!("{}/EWS/Exchange.asmx", server.url()); + let busy_body = format!( + "\ + soap:Serverbusy\ + ErrorServerBusy\ + 100\ + " + ); + let ok_body = envelope(&format!( + "\ + NoError\ + \ + " + )); + let _m1 = server + .mock("POST", "/EWS/Exchange.asmx") + .with_status(200) + .with_header("content-type", TXT_XML) + .with_body(busy_body) + .expect(1) + .create(); + let _m2 = server + .mock("POST", "/EWS/Exchange.asmx") + .with_status(200) + .with_header("content-type", TXT_XML) + .with_body(ok_body) + .create(); + + let c = client(3); + let body = find_folder_body( + FolderRef::Distinguished( + vandelay::exchange_ews::types::DistinguishedFolderId::MsgFolderRoot, + ), + Traversal::Deep, + ); + let r = c.call(&url, "FindFolder", &body).expect("should retry"); + let parsed = parse_find_folder_response(&r.body).unwrap(); + assert!(parsed.folders.is_empty()); +} + +#[test] +fn warning_response_class_is_treated_as_success_in_mock() { + let body = envelope(&format!( + "\ + \ + NoError\ + \ + " + )); + let r = parse_response_messages(body.as_bytes(), b"GetItemResponseMessage").unwrap(); + assert!(r[0].success); +} + +#[test] +fn default_namespace_envelope_parses_via_resolved_names() { + let body = "\ + \ + \ + \ + NoError\ + \ + Inbox\ + \ + \ + "; + let parsed = parse_find_folder_response(body.as_bytes()).unwrap(); + assert_eq!(parsed.folders.len(), 1); + assert_eq!(parsed.folders[0].folder_id.id, "D1"); +} + +#[test] +fn get_folder_messages_preserve_position_when_one_errors() { + let body = envelope(&format!( + "\ + \ + NoError\ + Inbox\ + \ + \ + ErrorAccessDenied\ + \ + \ + NoError\ + Drafts\ + \ + " + )); + let msgs = parse_response_messages(body.as_bytes(), b"GetFolderResponseMessage").unwrap(); + assert_eq!(msgs.len(), 3, "all three messages must be present"); + assert!(msgs[0].success); + assert!(!msgs[1].success); + assert!(msgs[2].success); +} diff --git a/tests/mock_exchange_graph.rs b/tests/mock_exchange_graph.rs new file mode 100644 index 0000000..7e03442 --- /dev/null +++ b/tests/mock_exchange_graph.rs @@ -0,0 +1,2066 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::sync::Once; + +use base64::Engine; +use mockito::{Matcher, Server}; +use serde_json::json; +use vandelay::exchange_graph::api::{Endpoints, collect_all_ids, paged_collect}; +use vandelay::exchange_graph::calendar_map::{EventType, classify_event_type, convert_event}; +use vandelay::exchange_graph::client::{Accept, GraphClient}; +use vandelay::exchange_graph::contact_map::convert_contact; +use vandelay::exchange_graph::error::GraphError; +use vandelay::exchange_graph::oauth::{ + DeviceCodeResponse, TokenResponse, parse_device_code_response, parse_token_response, + run_device_code_polling_against, +}; +use vandelay::exchange_graph::recurrence::convert_patterned_recurrence; +use vandelay::exchange_graph::retry::{HttpClass, classify_http_status}; +use vandelay::jmap::http::RetryPolicy; + +static INIT: Once = Once::new(); + +fn client_with_retries(retries: u32) -> GraphClient { + INIT.call_once(|| {}); + GraphClient::new("BEARER".to_owned(), RetryPolicy::new(retries), false) +} + +fn url_message_collection(server_url: &str, folder: &str, top: usize) -> String { + let e = Endpoints::for_me(server_url); + e.folder_messages_ids(folder, top) +} + +fn make_jwt(exp: u64, upn: &str) -> String { + use base64::engine::general_purpose::URL_SAFE_NO_PAD; + let header = URL_SAFE_NO_PAD.encode(b"{\"alg\":\"none\"}"); + let claims = format!(r#"{{"tid":"tenant-1","upn":"{upn}","exp":{exp}}}"#); + let payload = URL_SAFE_NO_PAD.encode(claims.as_bytes()); + format!("{header}.{payload}.") +} + +#[test] +fn http_status_classifier_matches_spec_table() { + assert_eq!(classify_http_status(200), HttpClass::Success); + assert_eq!(classify_http_status(401), HttpClass::Auth); + assert_eq!(classify_http_status(403), HttpClass::Auth); + assert_eq!(classify_http_status(404), HttpClass::Vanished); + assert_eq!(classify_http_status(410), HttpClass::Vanished); + assert_eq!(classify_http_status(429), HttpClass::Retryable); + assert_eq!(classify_http_status(503), HttpClass::Retryable); + assert_eq!(classify_http_status(504), HttpClass::Retryable); + assert_eq!(classify_http_status(507), HttpClass::Fatal); + assert_eq!(classify_http_status(400), HttpClass::Fatal); + assert_eq!(classify_http_status(405), HttpClass::Fatal); + assert_eq!(classify_http_status(422), HttpClass::Fatal); +} + +#[test] +fn device_code_response_parses_required_fields() { + let body = json!({ + "device_code": "ABCDEF", + "user_code": "BR4S-7XYZ", + "verification_uri": "https://microsoft.com/devicelogin", + "interval": 5, + "expires_in": 900, + "message": "Open the URL and enter the code." + }); + let parsed = parse_device_code_response(&body).unwrap(); + assert_eq!(parsed.device_code, "ABCDEF"); + assert_eq!(parsed.user_code, "BR4S-7XYZ"); + assert_eq!(parsed.interval, 5); + assert_eq!(parsed.expires_in, 900); +} + +#[test] +fn token_response_handles_pending_and_success() { + let pending = parse_token_response(400, &json!({"error": "authorization_pending"})); + assert!(matches!(pending, TokenResponse::Pending)); + let success = parse_token_response( + 200, + &json!({"access_token": make_jwt(9999999999, "alice@x.com"), "expires_in": 3599}), + ); + match success { + TokenResponse::Ok(t) => { + assert!(t.access_token.contains('.')); + assert_eq!(t.expires_in, Some(3599)); + assert_eq!(t.upn.as_deref(), Some("alice@x.com")); + } + other => panic!("expected Ok, got {other:?}"), + } +} + +#[test] +fn token_response_declined_and_bad_code() { + let declined = parse_token_response(400, &json!({"error": "authorization_declined"})); + assert!(matches!(declined, TokenResponse::Declined)); + let bad = parse_token_response(400, &json!({"error": "bad_verification_code"})); + assert!(matches!(bad, TokenResponse::BadCode)); +} + +#[test] +fn device_code_polling_loop_converges_on_success() { + let response = DeviceCodeResponse { + device_code: "DC".to_owned(), + user_code: "U".to_owned(), + verification_uri: "x".to_owned(), + interval: 0, + expires_in: 5, + message: None, + }; + let mut attempt = 0; + let token = run_device_code_polling_against( + "https://login.x/common", + "client-id", + &response, + |_endpoint, _body| { + attempt += 1; + if attempt < 3 { + Ok((400, json!({"error": "authorization_pending"}))) + } else { + Ok(( + 200, + json!({ + "access_token": make_jwt(9999999999, "alice@x"), + "expires_in": 3599 + }), + )) + } + }, + ) + .unwrap(); + assert_eq!(attempt, 3); + assert_eq!(token.upn.as_deref(), Some("alice@x")); +} + +#[test] +fn device_code_polling_loop_aborts_on_decline() { + let response = DeviceCodeResponse { + device_code: "DC".to_owned(), + user_code: "U".to_owned(), + verification_uri: "x".to_owned(), + interval: 0, + expires_in: 5, + message: None, + }; + let err = run_device_code_polling_against( + "https://login.x/common", + "client-id", + &response, + |_, _| Ok((400, json!({"error": "authorization_declined"}))), + ) + .unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("declined")); +} + +#[test] +fn pagination_collects_ids_across_three_pages_with_one_empty() { + let mut server = Server::new(); + let base = server.url(); + let next2 = format!("{base}/page2"); + let next3 = format!("{base}/page3"); + + let page1 = json!({ + "value": [ + {"id": "M1"}, + {"id": "M2"} + ], + "@odata.nextLink": next2 + }) + .to_string(); + let page2 = json!({ + "value": [], + "@odata.nextLink": next3 + }) + .to_string(); + let page3 = json!({ + "value": [ + {"id": "M3"} + ] + }) + .to_string(); + + let _m1 = server + .mock("GET", "/me/mailFolders/F1/messages?$top=100&$select=id") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(page1) + .create(); + let _m2 = server + .mock("GET", "/page2") + .match_header("authorization", "Bearer BEARER") + .match_header("prefer", "IdType=\"ImmutableId\"") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(page2) + .create(); + let _m3 = server + .mock("GET", "/page3") + .match_header("authorization", "Bearer BEARER") + .match_header("prefer", "IdType=\"ImmutableId\"") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(page3) + .create(); + + let client = client_with_retries(0); + let url = url_message_collection(&base, "F1", 100); + let ids = collect_all_ids(&client, &url, &[]).unwrap(); + assert_eq!(ids, vec!["M1".to_owned(), "M2".to_owned(), "M3".to_owned()]); +} + +#[test] +fn pagination_terminates_when_no_next_link_present() { + let mut server = Server::new(); + let base = server.url(); + let body = json!({"value": [{"id": "A"}, {"id": "B"}]}).to_string(); + let _m = server + .mock("GET", "/me/mailFolders/F/messages?$top=100&$select=id") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(body) + .create(); + let client = client_with_retries(0); + let url = url_message_collection(&base, "F", 100); + let mut pages = 0; + paged_collect(&client, &url, &[], |_| { + pages += 1; + Ok(()) + }) + .unwrap(); + assert_eq!(pages, 1); +} + +#[test] +fn header_redelivery_is_asserted_via_mockito_match() { + let mut server = Server::new(); + let base = server.url(); + let next = format!("{base}/page-two-different-segment"); + let page1 = json!({ + "value": [{"id": "OnlyOnPageOne"}], + "@odata.nextLink": next + }) + .to_string(); + let page2 = json!({"value": [{"id": "OnlyOnPageTwo"}]}).to_string(); + let _m1 = server + .mock("GET", "/me/mailFolders/F/messages?$top=100&$select=id") + .match_header("prefer", "IdType=\"ImmutableId\"") + .match_header("authorization", "Bearer BEARER") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(page1) + .create(); + let _m2 = server + .mock("GET", "/page-two-different-segment") + .match_header("prefer", "IdType=\"ImmutableId\"") + .match_header("authorization", "Bearer BEARER") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(page2) + .create(); + let c = client_with_retries(0); + let url = url_message_collection(&base, "F", 100); + let ids = collect_all_ids(&c, &url, &[]).unwrap(); + assert_eq!(ids.len(), 2); +} + +#[test] +fn case_sensitive_immutable_id_round_trip() { + let mut server = Server::new(); + let id = "AAkAAGFsaWNlAAA="; + let body = json!({"value": [{"id": id}]}).to_string(); + let _m = server + .mock("GET", "/me/mailFolders/F/messages?$top=100&$select=id") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(body) + .create(); + let base = server.url(); + let client = client_with_retries(0); + let url = url_message_collection(&base, "F", 100); + let ids = collect_all_ids(&client, &url, &[]).unwrap(); + assert_eq!(ids[0], id); + assert!(ids[0].contains('=')); +} + +#[test] +fn rate_limit_with_retry_after_is_honoured_then_succeeds() { + let mut server = Server::new(); + let path = "/me?$select=id,userPrincipalName,displayName,mail"; + let _m1 = server + .mock("GET", path) + .with_status(429) + .with_header("retry-after", "1") + .with_header("content-type", "application/json") + .with_body(r#"{"error":{"code":"TooManyRequests"}}"#) + .expect(1) + .create(); + let _m2 = server + .mock("GET", path) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"uid","userPrincipalName":"alice@x.com"}"#) + .expect(1) + .create(); + let base = server.url(); + let client = client_with_retries(3); + let started = std::time::Instant::now(); + let resp = client + .get( + &format!("{base}/me?$select=id,userPrincipalName,displayName,mail"), + Accept::Json, + ) + .unwrap(); + let elapsed = started.elapsed(); + assert_eq!(resp.status, 200); + assert!(elapsed >= std::time::Duration::from_millis(800)); + assert!(client.retries_observed() >= 1); + assert!(client.retry_after_sleeps() >= 1); +} + +#[test] +fn rate_limit_without_retry_after_uses_shared_escalation() { + let mut server = Server::new(); + let path = "/me?$select=id,userPrincipalName,displayName,mail"; + let _m1 = server + .mock("GET", path) + .with_status(429) + .with_header("content-type", "application/json") + .with_body(r#"{"error":{"code":"TooManyRequests"}}"#) + .expect(1) + .create(); + let _m2 = server + .mock("GET", path) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"uid","userPrincipalName":"alice@x.com"}"#) + .expect(1) + .create(); + let base = server.url(); + let client = client_with_retries(3); + let r = client + .get( + &format!("{base}/me?$select=id,userPrincipalName,displayName,mail"), + Accept::Json, + ) + .unwrap(); + assert_eq!(r.status, 200); + assert!(client.retries_observed() >= 1); +} + +#[test] +fn service_unavailable_503_is_retried() { + let mut server = Server::new(); + let _m1 = server + .mock("GET", "/me") + .with_status(503) + .expect(1) + .create(); + let _m2 = server + .mock("GET", "/me") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"u","userPrincipalName":"a@b"}"#) + .expect(1) + .create(); + let base = server.url(); + let c = client_with_retries(3); + let r = c.get(&format!("{base}/me"), Accept::Json).unwrap(); + assert_eq!(r.status, 200); +} + +#[test] +fn http_404_on_per_item_get_is_treated_as_vanished() { + let mut server = Server::new(); + let _m = server + .mock("GET", "/me/messages/GONE/$value") + .with_status(404) + .with_body("gone") + .create(); + let base = server.url(); + let c = client_with_retries(0); + let url = format!("{base}/me/messages/GONE/$value"); + let err = c.get(&url, Accept::Text).unwrap_err(); + assert!(matches!(err, GraphError::Vanished)); +} + +#[test] +fn http_401_is_auth_error() { + let mut server = Server::new(); + let _m = server + .mock("GET", "/me") + .with_status(401) + .with_body("unauthenticated") + .create(); + let base = server.url(); + let c = client_with_retries(0); + let url = format!("{base}/me"); + let err = c.get(&url, Accept::Json).unwrap_err(); + assert!(matches!(err, GraphError::Auth(_))); +} + +#[test] +fn malformed_4xx_is_fatal() { + let mut server = Server::new(); + let _m = server + .mock("GET", "/me") + .with_status(422) + .with_body("malformed payload") + .create(); + let base = server.url(); + let c = client_with_retries(0); + let err = c.get(&format!("{base}/me"), Accept::Json).unwrap_err(); + assert!(matches!(err, GraphError::HttpStatus { status: 422, .. })); +} + +#[test] +fn mime_value_endpoint_returns_raw_rfc5322_bytes() { + let mut server = Server::new(); + let mime = "From: a@x\r\nTo: b@x\r\nSubject: hi\r\nDate: Tue, 27 May 2026 10:00:00 +0000\r\nMessage-ID: \r\n\r\nbody"; + let _m = server + .mock("GET", "/me/messages/MID/$value") + .match_header("accept", "text/plain") + .match_header("authorization", "Bearer BEARER") + .match_header("prefer", "IdType=\"ImmutableId\"") + .with_status(200) + .with_header("content-type", "text/plain") + .with_body(mime) + .create(); + let base = server.url(); + let c = client_with_retries(0); + let url = format!("{base}/me/messages/MID/$value"); + let resp = c.get(&url, Accept::Text).unwrap(); + assert_eq!(resp.body, mime.as_bytes()); +} + +#[test] +fn recurrence_daily_with_interval_round_trips() { + let pr = json!({ + "pattern": {"type": "daily", "interval": 2}, + "range": {"type": "noEnd"} + }); + let out = convert_patterned_recurrence(&pr).unwrap(); + assert_eq!(out["frequency"], "daily"); + assert_eq!(out["interval"], 2); +} + +#[test] +fn recurrence_weekly_with_days_of_week_and_count() { + let pr = json!({ + "pattern": { + "type": "weekly", + "interval": 1, + "daysOfWeek": ["monday", "wednesday", "friday"] + }, + "range": {"type": "numbered", "numberOfOccurrences": 12} + }); + let out = convert_patterned_recurrence(&pr).unwrap(); + assert_eq!(out["frequency"], "weekly"); + assert_eq!(out["count"], 12); + assert_eq!(out["byDay"][0]["day"], "mo"); + assert_eq!(out["byDay"][2]["day"], "fr"); +} + +#[test] +fn recurrence_relative_monthly_emits_setpos() { + let pr = json!({ + "pattern": { + "type": "relativeMonthly", "interval": 1, + "daysOfWeek": ["thursday"], "index": "third" + }, + "range": {"type": "noEnd"} + }); + let out = convert_patterned_recurrence(&pr).unwrap(); + assert_eq!(out["frequency"], "monthly"); + assert_eq!(out["bySetPosition"][0], 3); + assert_eq!(out["byDay"][0]["day"], "th"); +} + +#[test] +fn recurrence_yearly_with_end_date() { + let pr = json!({ + "pattern": {"type": "absoluteYearly", "interval": 1, "month": 12, "dayOfMonth": 25}, + "range": {"type": "endDate", "endDate": "2030-12-25"} + }); + let out = convert_patterned_recurrence(&pr).unwrap(); + assert_eq!(out["frequency"], "yearly"); + assert_eq!(out["until"], "2030-12-25T23:59:59"); +} + +#[test] +fn occurrence_event_is_classified_for_skip() { + let occ = json!({"id": "occ-1", "type": "occurrence", "seriesMasterId": "master-1"}); + assert_eq!(classify_event_type(&occ), EventType::Occurrence); +} + +#[test] +fn exception_event_carries_series_master_id() { + let ex = json!({ + "id": "ex-1", + "iCalUId": "uid-1", + "type": "exception", + "seriesMasterId": "master-1", + "originalStart": "2026-06-15T10:00:00Z", + "subject": "Moved", + "start": {"dateTime": "2026-06-15T11:00:00.0000000", "timeZone": "UTC"}, + "end": {"dateTime": "2026-06-15T12:00:00.0000000", "timeZone": "UTC"} + }); + let conv = convert_event(&ex, None).unwrap(); + assert_eq!(conv.event_type, EventType::Exception); + assert_eq!(conv.series_master_id.as_deref(), Some("master-1")); + assert_eq!(conv.original_start.as_deref(), Some("2026-06-15T10:00:00Z")); +} + +#[test] +fn contact_minimal_yields_jscontact_card() { + let c = json!({ + "id": "graph-1", + "displayName": "Alice Liddell", + "givenName": "Alice", + "surname": "Liddell", + "emailAddresses": [{"name": "Alice", "address": "alice@x.com"}] + }); + let out = convert_contact(&c).unwrap(); + assert!(out.uid.starts_with("vandelay-graph-")); + assert_eq!(out.data["@type"], "Card"); + assert_eq!(out.data["name"]["full"], "Alice Liddell"); + let emails = out.data["emails"].as_object().unwrap(); + assert_eq!(emails["email-1"]["address"], "alice@x.com"); +} + +#[test] +fn integration_dry_run_against_mock_server_lists_three_surfaces() { + let mut server = Server::new(); + let _principal = server + .mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"uid","userPrincipalName":"alice@x.com"}"#) + .create(); + let _folders = server + .mock("GET", "/me/mailFolders?$top=100&includeHiddenFolders=true") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[{"id":"FMAIL","displayName":"Inbox","isHidden":false}]}"#) + .create(); + let _children = server + .mock( + "GET", + "/me/mailFolders/FMAIL/childFolders?$top=100&includeHiddenFolders=true", + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + let _calendars = server + .mock("GET", "/me/calendars?$top=100") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[{"id":"CAL1","name":"Calendar","isDefaultCalendar":true}]}"#) + .create(); + let _contact_folders = server + .mock("GET", "/me/contactFolders?$top=100") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[{"id":"CON1","displayName":"Contacts"}]}"#) + .create(); + let base = server.url(); + let common = vandelay::sync::CommonConfig { + archive: tempfile::NamedTempFile::new().unwrap().path().to_owned(), + threads: 2, + dry_run: true, + max_retries: 0, + allow_invalid_certs: false, + logger: vandelay::logging::Logger::new(0), + }; + let config = vandelay::sync::import_exchange_graph::GraphImportConfig { + auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + token: make_jwt(9999999999, "alice@x.com"), + }, + api_base: base.clone(), + user_target: None, + mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + objects: None, + event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + graph_connections: 2, + top: 100, + allow_source_change: false, + }; + let summary = vandelay::sync::import_exchange_graph::run(common, config).unwrap(); + let mailbox = summary + .per_type + .iter() + .find(|(t, _)| *t == "mailbox") + .map(|(_, c)| c.created) + .unwrap_or(0); + let calendar = summary + .per_type + .iter() + .find(|(t, _)| *t == "calendar") + .map(|(_, c)| c.created) + .unwrap_or(0); + let addressbook = summary + .per_type + .iter() + .find(|(t, _)| *t == "addressbook") + .map(|(_, c)| c.created) + .unwrap_or(0); + assert_eq!(mailbox, 1); + assert_eq!(calendar, 1); + assert_eq!(addressbook, 1); +} + +#[test] +fn integration_full_run_mail_only_imports_mime_via_value() { + let mut server = Server::new(); + let _principal = server + .mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"uid-1","userPrincipalName":"alice@x.com"}"#) + .create(); + let _folders = server + .mock("GET", "/me/mailFolders?$top=100&includeHiddenFolders=true") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[{"id":"FMAIL","displayName":"Inbox","isHidden":false}]}"#) + .create(); + let _children = server + .mock( + "GET", + "/me/mailFolders/FMAIL/childFolders?$top=100&includeHiddenFolders=true", + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + let _well_known: Vec = [ + "inbox", + "drafts", + "sentitems", + "deleteditems", + "junkemail", + "archive", + ] + .iter() + .map(|name| { + let path = format!("/me/mailFolders/{name}?$select=id"); + server + .mock("GET", path.as_str()) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(if *name == "inbox" { + r#"{"id":"FMAIL"}"# + } else { + r#"{"id":"OTHER"}"# + }) + .create() + }) + .collect(); + let _ids = server + .mock("GET", "/me/mailFolders/FMAIL/messages?$top=100&$select=id") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[{"id":"MSG-1"}]}"#) + .create(); + let mime = "From: a@x\r\nTo: b@x\r\nSubject: hi\r\nDate: Tue, 27 May 2026 10:00:00 +0000\r\nMessage-ID: \r\n\r\nhello"; + let _value = server + .mock("GET", "/me/messages/MSG-1/$value") + .match_header("accept", "text/plain") + .with_status(200) + .with_header("content-type", "text/plain") + .with_body(mime) + .create(); + + let tmp = tempfile::NamedTempFile::new().unwrap(); + let archive_path = tmp.path().to_owned(); + let common = vandelay::sync::CommonConfig { + archive: archive_path.clone(), + threads: 2, + dry_run: false, + max_retries: 0, + allow_invalid_certs: false, + logger: vandelay::logging::Logger::new(0), + }; + let config = vandelay::sync::import_exchange_graph::GraphImportConfig { + auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + token: make_jwt(9999999999, "alice@x.com"), + }, + api_base: server.url(), + user_target: None, + mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + objects: Some(vec![ + vandelay::types::ObjectType::Mailbox, + vandelay::types::ObjectType::Email, + ]), + event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + graph_connections: 2, + top: 100, + allow_source_change: false, + }; + drop(tmp); + let summary = vandelay::sync::import_exchange_graph::run(common, config).unwrap(); + let emails = summary + .per_type + .iter() + .find(|(t, _)| *t == "email") + .map(|(_, c)| c.created) + .unwrap_or(0); + assert_eq!(emails, 1, "expected one email created, got {emails}"); +} + +#[test] +fn integration_full_run_is_convergent_on_second_invocation() { + let mut server = Server::new(); + let _principal = server + .mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"uid-2","userPrincipalName":"alice@x.com"}"#) + .expect_at_least(2) + .create(); + let _folders = server + .mock("GET", "/me/mailFolders?$top=100&includeHiddenFolders=true") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[{"id":"FMAIL","displayName":"Inbox","isHidden":false}]}"#) + .expect_at_least(2) + .create(); + let _children = server + .mock( + "GET", + "/me/mailFolders/FMAIL/childFolders?$top=100&includeHiddenFolders=true", + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .expect_at_least(2) + .create(); + for name in [ + "inbox", + "drafts", + "sentitems", + "deleteditems", + "junkemail", + "archive", + ] { + server + .mock("GET", format!("/me/mailFolders/{name}?$select=id").as_str()) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(if name == "inbox" { + r#"{"id":"FMAIL"}"# + } else { + r#"{"id":"OTHER"}"# + }) + .expect_at_least(2) + .create(); + } + let _ids = server + .mock("GET", "/me/mailFolders/FMAIL/messages?$top=100&$select=id") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[{"id":"MSG-A"}]}"#) + .expect_at_least(2) + .create(); + let mime = "From: a@x\r\nTo: b@x\r\nSubject: hi\r\nDate: Tue, 27 May 2026 10:00:00 +0000\r\nMessage-ID: \r\n\r\nbody"; + let _value = server + .mock("GET", "/me/messages/MSG-A/$value") + .with_status(200) + .with_header("content-type", "text/plain") + .with_body(mime) + .expect(1) + .create(); + + let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); + let api_base = server.url(); + let make_config = || vandelay::sync::import_exchange_graph::GraphImportConfig { + auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + token: make_jwt(9999999999, "alice@x.com"), + }, + api_base: api_base.clone(), + user_target: None, + mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + objects: Some(vec![ + vandelay::types::ObjectType::Mailbox, + vandelay::types::ObjectType::Email, + ]), + event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + graph_connections: 2, + top: 100, + allow_source_change: false, + }; + let make_common = |path: std::path::PathBuf| vandelay::sync::CommonConfig { + archive: path, + threads: 2, + dry_run: false, + max_retries: 0, + allow_invalid_certs: false, + logger: vandelay::logging::Logger::new(0), + }; + + let first = + vandelay::sync::import_exchange_graph::run(make_common(archive.clone()), make_config()) + .unwrap(); + let created_first = first + .per_type + .iter() + .find(|(t, _)| *t == "email") + .map(|(_, c)| c.created) + .unwrap_or(0); + assert_eq!(created_first, 1); + + let second = + vandelay::sync::import_exchange_graph::run(make_common(archive), make_config()).unwrap(); + let created_second = second + .per_type + .iter() + .find(|(t, _)| *t == "email") + .map(|(_, c)| c.created) + .unwrap_or(0); + assert_eq!(created_second, 0, "second run should not re-import"); +} + +#[test] +fn source_change_protection_refuses_a_different_account() { + let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); + let conn = vandelay::db::init::open(&archive).unwrap(); + vandelay::db::sources::upsert_source( + &conn, + &vandelay::db::sources::SourceKey { + kind: "exchange_graph".to_owned(), + session_url: + "https://login.microsoftonline.com/common|https://graph.microsoft.com/v1.0" + .to_owned(), + account_id: "old-user-id".to_owned(), + }, + Some("old@x.com"), + "old@x.com", + ) + .unwrap(); + drop(conn); + + let mut server = Server::new(); + let _principal = server + .mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"new-user-id","userPrincipalName":"new@x.com"}"#) + .create(); + + let common = vandelay::sync::CommonConfig { + archive, + threads: 2, + dry_run: false, + max_retries: 0, + allow_invalid_certs: false, + logger: vandelay::logging::Logger::new(0), + }; + let config = vandelay::sync::import_exchange_graph::GraphImportConfig { + auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + token: make_jwt(9999999999, "new@x.com"), + }, + api_base: server.url(), + user_target: None, + mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + objects: None, + event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + graph_connections: 2, + top: 100, + allow_source_change: false, + }; + let err = vandelay::sync::import_exchange_graph::run(common, config).unwrap_err(); + assert!( + matches!(err, vandelay::error::Error::SourceChange(_)), + "expected SourceChange error, got {err:?}" + ); +} + +fn stub_well_known_folders(server: &mut Server, inbox_id: &str) { + for name in [ + "drafts", + "sentitems", + "deleteditems", + "junkemail", + "archive", + ] { + let body = format!(r#"{{"id":"OTHER-{name}"}}"#); + server + .mock("GET", format!("/me/mailFolders/{name}?$select=id").as_str()) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(body) + .expect_at_least(0) + .create(); + } + server + .mock("GET", "/me/mailFolders/inbox?$select=id") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(format!(r#"{{"id":"{inbox_id}"}}"#)) + .expect_at_least(0) + .create(); +} + +fn make_common(archive: std::path::PathBuf) -> vandelay::sync::CommonConfig { + vandelay::sync::CommonConfig { + archive, + threads: 2, + dry_run: false, + max_retries: 0, + allow_invalid_certs: false, + logger: vandelay::logging::Logger::new(0), + } +} + +fn make_config( + api_base: String, + user_target: Option, + objects: Option>, +) -> vandelay::sync::import_exchange_graph::GraphImportConfig { + vandelay::sync::import_exchange_graph::GraphImportConfig { + auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + token: make_jwt(9999999999, "alice@x.com"), + }, + api_base, + user_target, + mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + objects, + event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + graph_connections: 2, + top: 100, + allow_source_change: false, + } +} + +#[test] +fn users_upn_routes_through_users_segment_not_me() { + let mut server = Server::new(); + let _resolve = server + .mock( + "GET", + Matcher::Regex(r"^/users/alice[@%][^/]*\?\$select=id".to_owned()), + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"user-uuid-7","userPrincipalName":"alice@x.com"}"#) + .create(); + let _principal = server + .mock( + "GET", + Matcher::Regex(r"^/users/user-uuid-7\?\$select=id".to_owned()), + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + r#"{"id":"user-uuid-7","userPrincipalName":"alice@x.com","displayName":"Alice"}"#, + ) + .create(); + let _folders = server + .mock( + "GET", + "/users/user-uuid-7/mailFolders?$top=100&includeHiddenFolders=true", + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .expect_at_least(1) + .create(); + let _calendars = server + .mock("GET", "/users/user-uuid-7/calendars?$top=100") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .expect_at_least(0) + .create(); + let _contact_folders = server + .mock("GET", "/users/user-uuid-7/contactFolders?$top=100") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .expect_at_least(0) + .create(); + let base = server.url(); + let common = vandelay::sync::CommonConfig { + archive: tempfile::NamedTempFile::new().unwrap().path().to_owned(), + threads: 2, + dry_run: true, + max_retries: 0, + allow_invalid_certs: false, + logger: vandelay::logging::Logger::new(0), + }; + let summary = vandelay::sync::import_exchange_graph::run( + common, + make_config(base, Some("alice@x.com".to_owned()), None), + ) + .unwrap(); + let mailbox = summary + .per_type + .iter() + .find(|(t, _)| *t == "mailbox") + .map(|(_, c)| c.created) + .unwrap_or(99); + assert_eq!(mailbox, 0); +} + +#[test] +fn series_master_with_exception_merges_into_recurrence_overrides() { + let mut server = Server::new(); + server + .mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"uid","userPrincipalName":"alice@x.com"}"#) + .create(); + server + .mock("GET", "/me/mailFolders?$top=100&includeHiddenFolders=true") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + server + .mock("GET", "/me/mailboxSettings?$select=timeZone") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"timeZone":"UTC"}"#) + .create(); + server + .mock("GET", "/me/calendars?$top=100") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[{"id":"CAL1","name":"Calendar","isDefaultCalendar":true}]}"#) + .create(); + server + .mock("GET", "/me/contactFolders?$top=100") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + server + .mock( + "GET", + "/me/calendars/CAL1/events?$top=100&$select=id,type,seriesMasterId", + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + r#"{"value":[ + {"id":"MASTER","type":"seriesMaster","iCalUId":"uid-master"}, + {"id":"EX","type":"exception","seriesMasterId":"MASTER","iCalUId":"uid-master"} + ]}"#, + ) + .create(); + server + .mock("GET", "/me/events/MASTER") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + r#"{ + "id":"MASTER", + "iCalUId":"uid-master", + "type":"seriesMaster", + "subject":"Weekly sync", + "start":{"dateTime":"2026-05-04T15:00:00.0000000","timeZone":"UTC"}, + "end":{"dateTime":"2026-05-04T16:00:00.0000000","timeZone":"UTC"}, + "recurrence":{ + "pattern":{"type":"weekly","interval":1,"daysOfWeek":["monday"]}, + "range":{"type":"noEnd"} + } + }"#, + ) + .create(); + server + .mock("GET", "/me/events/EX") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + r#"{ + "id":"EX", + "iCalUId":"uid-master", + "type":"exception", + "seriesMasterId":"MASTER", + "originalStart":"2026-05-11T15:00:00Z", + "subject":"Moved sync", + "start":{"dateTime":"2026-05-11T16:00:00.0000000","timeZone":"UTC"}, + "end":{"dateTime":"2026-05-11T17:00:00.0000000","timeZone":"UTC"} + }"#, + ) + .create(); + let base = server.url(); + let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); + let summary = vandelay::sync::import_exchange_graph::run( + make_common(archive.clone()), + make_config( + base, + None, + Some(vec![ + vandelay::types::ObjectType::Calendar, + vandelay::types::ObjectType::CalendarEvent, + ]), + ), + ) + .unwrap(); + let events = summary + .per_type + .iter() + .find(|(t, _)| *t == "calendarevent") + .map(|(_, c)| c.created) + .unwrap_or(0); + assert_eq!(events, 1, "exactly one master row, exception merged in"); + + let conn = vandelay::db::init::open(&archive).unwrap(); + let row: String = conn + .query_row("SELECT data FROM calendar_events", [], |row| row.get(0)) + .unwrap(); + let v: serde_json::Value = serde_json::from_str(&row).unwrap(); + let overrides = v + .get("recurrenceOverrides") + .and_then(|o| o.as_object()) + .expect("recurrenceOverrides present"); + assert!( + overrides.contains_key("2026-05-11T15:00:00"), + "override keyed by originalStart as LocalDateTime; got keys: {:?}", + overrides.keys().collect::>() + ); + let override_data = &overrides["2026-05-11T15:00:00"]; + assert_eq!(override_data["title"], "Moved sync"); +} + +#[test] +fn occurrence_event_is_skipped_no_row_no_id_mapping() { + let mut server = Server::new(); + server + .mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"uid","userPrincipalName":"alice@x.com"}"#) + .create(); + server + .mock("GET", "/me/mailFolders?$top=100&includeHiddenFolders=true") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + server + .mock("GET", "/me/mailboxSettings?$select=timeZone") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"timeZone":"UTC"}"#) + .create(); + server + .mock("GET", "/me/calendars?$top=100") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[{"id":"CAL1","name":"Calendar","isDefaultCalendar":true}]}"#) + .create(); + server + .mock("GET", "/me/contactFolders?$top=100") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + server + .mock( + "GET", + "/me/calendars/CAL1/events?$top=100&$select=id,type,seriesMasterId", + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + r#"{"value":[ + {"id":"OCC","type":"occurrence","seriesMasterId":"MASTER"} + ]}"#, + ) + .create(); + let occ_get = server + .mock("GET", "/me/events/OCC") + .with_status(500) + .with_body("MUST NOT BE CALLED") + .expect(0) + .create(); + let base = server.url(); + let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); + let summary = vandelay::sync::import_exchange_graph::run( + make_common(archive.clone()), + make_config( + base, + None, + Some(vec![ + vandelay::types::ObjectType::Calendar, + vandelay::types::ObjectType::CalendarEvent, + ]), + ), + ) + .unwrap(); + let events = summary + .per_type + .iter() + .find(|(t, _)| *t == "calendarevent") + .map(|(_, c)| c.created) + .unwrap_or(99); + assert_eq!(events, 0, "occurrence must not be fetched or stored"); + occ_get.assert(); + + let conn = vandelay::db::init::open(&archive).unwrap(); + let count: i64 = conn + .query_row("SELECT COUNT(*) FROM calendar_events", [], |row| row.get(0)) + .unwrap(); + assert_eq!(count, 0); +} + +#[test] +fn hidden_mail_folder_has_is_subscribed_zero() { + let mut server = Server::new(); + server + .mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"uid","userPrincipalName":"alice@x.com"}"#) + .create(); + server + .mock("GET", "/me/mailFolders?$top=100&includeHiddenFolders=true") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + r#"{"value":[ + {"id":"FVISIBLE","displayName":"Inbox","isHidden":false}, + {"id":"FHIDDEN","displayName":"SearchFolders","isHidden":true} + ]}"#, + ) + .create(); + server + .mock( + "GET", + "/me/mailFolders/FVISIBLE/childFolders?$top=100&includeHiddenFolders=true", + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + server + .mock( + "GET", + "/me/mailFolders/FHIDDEN/childFolders?$top=100&includeHiddenFolders=true", + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + server + .mock( + "GET", + "/me/mailFolders/FVISIBLE/messages?$top=100&$select=id", + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + server + .mock( + "GET", + "/me/mailFolders/FHIDDEN/messages?$top=100&$select=id", + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + stub_well_known_folders(&mut server, "FVISIBLE"); + let base = server.url(); + let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); + let _ = vandelay::sync::import_exchange_graph::run( + make_common(archive.clone()), + make_config( + base, + None, + Some(vec![ + vandelay::types::ObjectType::Mailbox, + vandelay::types::ObjectType::Email, + ]), + ), + ) + .unwrap(); + let conn = vandelay::db::init::open(&archive).unwrap(); + let visible: i64 = conn + .query_row( + "SELECT is_subscribed FROM mailboxes WHERE name = 'Inbox'", + [], + |row| row.get(0), + ) + .unwrap(); + let hidden: i64 = conn + .query_row( + "SELECT is_subscribed FROM mailboxes WHERE name = 'SearchFolders'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(visible, 1); + assert_eq!(hidden, 0); +} + +#[test] +fn well_known_folder_probes_assign_jmap_roles() { + let mut server = Server::new(); + server + .mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"uid","userPrincipalName":"alice@x.com"}"#) + .create(); + server + .mock("GET", "/me/mailFolders?$top=100&includeHiddenFolders=true") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + r#"{"value":[ + {"id":"INBOX-ID","displayName":"Inbox","isHidden":false}, + {"id":"DRAFTS-ID","displayName":"Drafts","isHidden":false}, + {"id":"SENT-ID","displayName":"Sent Items","isHidden":false}, + {"id":"TRASH-ID","displayName":"Deleted Items","isHidden":false}, + {"id":"JUNK-ID","displayName":"Junk Email","isHidden":false}, + {"id":"ARCHIVE-ID","displayName":"Archive","isHidden":false} + ]}"#, + ) + .create(); + for fid in [ + "INBOX-ID", + "DRAFTS-ID", + "SENT-ID", + "TRASH-ID", + "JUNK-ID", + "ARCHIVE-ID", + ] { + server + .mock( + "GET", + format!("/me/mailFolders/{fid}/childFolders?$top=100&includeHiddenFolders=true") + .as_str(), + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + server + .mock( + "GET", + format!("/me/mailFolders/{fid}/messages?$top=100&$select=id").as_str(), + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + } + let pairs = [ + ("inbox", "INBOX-ID"), + ("drafts", "DRAFTS-ID"), + ("sentitems", "SENT-ID"), + ("deleteditems", "TRASH-ID"), + ("junkemail", "JUNK-ID"), + ("archive", "ARCHIVE-ID"), + ]; + for (name, id) in pairs { + server + .mock("GET", format!("/me/mailFolders/{name}?$select=id").as_str()) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(format!(r#"{{"id":"{id}"}}"#)) + .create(); + } + let base = server.url(); + let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); + let _ = vandelay::sync::import_exchange_graph::run( + make_common(archive.clone()), + make_config( + base, + None, + Some(vec![ + vandelay::types::ObjectType::Mailbox, + vandelay::types::ObjectType::Email, + ]), + ), + ) + .unwrap(); + let conn = vandelay::db::init::open(&archive).unwrap(); + let mut stmt = conn + .prepare("SELECT name, role FROM mailboxes ORDER BY name") + .unwrap(); + let rows: Vec<(String, Option)> = stmt + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .unwrap() + .map(Result::unwrap) + .collect(); + let by_name: std::collections::HashMap> = rows.into_iter().collect(); + assert_eq!(by_name["Inbox"].as_deref(), Some("inbox")); + assert_eq!(by_name["Drafts"].as_deref(), Some("drafts")); + assert_eq!(by_name["Sent Items"].as_deref(), Some("sent")); + assert_eq!(by_name["Deleted Items"].as_deref(), Some("trash")); + assert_eq!(by_name["Junk Email"].as_deref(), Some("junk")); + assert_eq!(by_name["Archive"].as_deref(), Some("archive")); +} + +#[test] +fn attachment_message_does_not_trigger_attachments_endpoint() { + let mut server = Server::new(); + server + .mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"uid","userPrincipalName":"alice@x.com"}"#) + .create(); + server + .mock("GET", "/me/mailFolders?$top=100&includeHiddenFolders=true") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[{"id":"FMAIL","displayName":"Inbox","isHidden":false}]}"#) + .create(); + server + .mock( + "GET", + "/me/mailFolders/FMAIL/childFolders?$top=100&includeHiddenFolders=true", + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + stub_well_known_folders(&mut server, "FMAIL"); + server + .mock("GET", "/me/mailFolders/FMAIL/messages?$top=100&$select=id") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[{"id":"MSG-ATT"}]}"#) + .create(); + let mime_with_attachment = "From: a@x\r\nTo: b@x\r\nSubject: with attachment\r\n\ +Date: Tue, 27 May 2026 10:00:00 +0000\r\nMessage-ID: \r\n\ +Content-Type: multipart/mixed; boundary=\"X\"\r\n\r\n\ +--X\r\nContent-Type: text/plain\r\n\r\nbody\r\n\ +--X\r\nContent-Type: application/octet-stream\r\nContent-Disposition: attachment; filename=\"a.bin\"\r\nContent-Transfer-Encoding: base64\r\n\r\n\ +SGVsbG8=\r\n--X--\r\n"; + server + .mock("GET", "/me/messages/MSG-ATT/$value") + .with_status(200) + .with_header("content-type", "text/plain") + .with_body(mime_with_attachment) + .create(); + let no_attachments = server + .mock( + "GET", + Matcher::Regex(r"^/me/messages/[^/]+/attachments".to_owned()), + ) + .with_status(500) + .with_body("MUST NOT BE CALLED") + .expect(0) + .create(); + let base = server.url(); + let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); + let summary = vandelay::sync::import_exchange_graph::run( + make_common(archive), + make_config( + base, + None, + Some(vec![ + vandelay::types::ObjectType::Mailbox, + vandelay::types::ObjectType::Email, + ]), + ), + ) + .unwrap(); + let emails = summary + .per_type + .iter() + .find(|(t, _)| *t == "email") + .map(|(_, c)| c.created) + .unwrap_or(0); + assert_eq!(emails, 1); + no_attachments.assert(); +} + +#[test] +fn event_get_carries_outlook_timezone_and_body_content_type_prefer() { + let mut server = Server::new(); + server + .mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"uid","userPrincipalName":"alice@x.com"}"#) + .create(); + server + .mock("GET", "/me/mailFolders?$top=100&includeHiddenFolders=true") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + server + .mock("GET", "/me/mailboxSettings?$select=timeZone") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"timeZone":"Pacific Standard Time"}"#) + .create(); + server + .mock("GET", "/me/calendars?$top=100") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[{"id":"CAL1","name":"Calendar"}]}"#) + .create(); + server + .mock("GET", "/me/contactFolders?$top=100") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + let _events_list = server + .mock( + "GET", + "/me/calendars/CAL1/events?$top=100&$select=id,type,seriesMasterId", + ) + .match_header( + "prefer", + Matcher::AllOf(vec![ + Matcher::Regex(r#"outlook\.timezone="UTC""#.to_owned()), + Matcher::Regex(r#"IdType="ImmutableId""#.to_owned()), + ]), + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[{"id":"EVT","type":"singleInstance"}]}"#) + .expect(1) + .create(); + let _event_get = server + .mock("GET", "/me/events/EVT") + .match_header( + "prefer", + Matcher::AllOf(vec![ + Matcher::Regex(r#"outlook\.body-content-type="text""#.to_owned()), + Matcher::Regex(r#"outlook\.timezone="UTC""#.to_owned()), + Matcher::Regex(r#"IdType="ImmutableId""#.to_owned()), + ]), + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + r#"{ + "id":"EVT","iCalUId":"uid-evt","type":"singleInstance", + "subject":"Test","start":{"dateTime":"2026-05-27T10:00:00.0000000","timeZone":"UTC"}, + "end":{"dateTime":"2026-05-27T11:00:00.0000000","timeZone":"UTC"} + }"#, + ) + .expect(1) + .create(); + let base = server.url(); + let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); + let _ = vandelay::sync::import_exchange_graph::run( + make_common(archive), + make_config( + base, + None, + Some(vec![ + vandelay::types::ObjectType::Calendar, + vandelay::types::ObjectType::CalendarEvent, + ]), + ), + ) + .unwrap(); +} + +#[test] +fn graph_client_retries_after_401_when_bearer_is_swapped() { + let mut server = Server::new(); + let _m1 = server + .mock("GET", "/me") + .match_header("authorization", "Bearer EXPIRED") + .with_status(401) + .with_body("expired") + .expect(1) + .create(); + let _m2 = server + .mock("GET", "/me") + .match_header("authorization", "Bearer FRESH") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"u","userPrincipalName":"a@b"}"#) + .expect(1) + .create(); + let base = server.url(); + let client = GraphClient::new("EXPIRED".to_owned(), RetryPolicy::new(0), false); + let url = format!("{base}/me"); + let err = client.get(&url, Accept::Json).unwrap_err(); + assert!(matches!(err, GraphError::Auth(_))); + client.set_bearer("FRESH".to_owned()); + let resp = client.get(&url, Accept::Json).unwrap(); + assert_eq!(resp.status, 200); +} + +#[test] +fn full_run_records_graph_id_in_sync_id_exchange_graph_with_padding() { + let mut server = Server::new(); + let _principal = server + .mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"uid-pad","userPrincipalName":"pad@x.com"}"#) + .create(); + let _folders = server + .mock("GET", "/me/mailFolders?$top=100&includeHiddenFolders=true") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[{"id":"AAkA-Padded==","displayName":"Inbox","isHidden":false}]}"#) + .create(); + let _children = server + .mock( + "GET", + "/me/mailFolders/AAkA-Padded%3D%3D/childFolders?$top=100&includeHiddenFolders=true", + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + for name in [ + "inbox", + "drafts", + "sentitems", + "deleteditems", + "junkemail", + "archive", + ] { + server + .mock("GET", format!("/me/mailFolders/{name}?$select=id").as_str()) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"OTHER"}"#) + .create(); + } + let _ids = server + .mock( + "GET", + "/me/mailFolders/AAkA-Padded%3D%3D/messages?$top=100&$select=id", + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + + let tmp = tempfile::NamedTempFile::new().unwrap(); + let archive = tmp.path().to_owned(); + let common = vandelay::sync::CommonConfig { + archive: archive.clone(), + threads: 2, + dry_run: false, + max_retries: 0, + allow_invalid_certs: false, + logger: vandelay::logging::Logger::new(0), + }; + let config = vandelay::sync::import_exchange_graph::GraphImportConfig { + auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + token: make_jwt(9999999999, "pad@x.com"), + }, + api_base: server.url(), + user_target: None, + mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + objects: Some(vec![vandelay::types::ObjectType::Mailbox]), + event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + graph_connections: 2, + top: 100, + allow_source_change: false, + }; + drop(tmp); + let _ = vandelay::sync::import_exchange_graph::run(common, config).unwrap(); + let conn = vandelay::db::init::open(&archive).unwrap(); + let (graph_id, type_name): (String, String) = conn + .query_row( + "SELECT graph_id, type_name FROM sync_id_exchange_graph", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!( + graph_id, "AAkA-Padded==", + "case-sensitive padded id round-trips verbatim" + ); + assert_eq!(type_name, "mailbox"); + let account_id: String = conn + .query_row( + "SELECT account_id FROM sources WHERE kind = 'exchange_graph'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + account_id, "uid-pad", + "primary kind uses directory id verbatim" + ); +} + +#[test] +fn archive_mailbox_kind_encodes_synthetic_suffix_in_account_id() { + let mut server = Server::new(); + let _principal = server + .mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"uid-arch","userPrincipalName":"a@x.com"}"#) + .create(); + let _root = server + .mock( + "GET", + "/me/mailFolders/archive/childFolders?$top=100&includeHiddenFolders=true", + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + + let tmp = tempfile::NamedTempFile::new().unwrap(); + let archive = tmp.path().to_owned(); + let common = vandelay::sync::CommonConfig { + archive: archive.clone(), + threads: 2, + dry_run: false, + max_retries: 0, + allow_invalid_certs: false, + logger: vandelay::logging::Logger::new(0), + }; + let config = vandelay::sync::import_exchange_graph::GraphImportConfig { + auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + token: make_jwt(9999999999, "a@x.com"), + }, + api_base: server.url(), + user_target: None, + mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Archive, + objects: Some(vec![vandelay::types::ObjectType::Mailbox]), + event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + graph_connections: 2, + top: 100, + allow_source_change: false, + }; + drop(tmp); + let _ = vandelay::sync::import_exchange_graph::run(common, config).unwrap(); + let conn = vandelay::db::init::open(&archive).unwrap(); + let account_id: String = conn + .query_row( + "SELECT account_id FROM sources WHERE kind = 'exchange_graph'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(account_id, "uid-arch#archive"); +} + +#[test] +fn allow_source_change_permits_overwriting_a_different_account() { + let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); + let conn = vandelay::db::init::open(&archive).unwrap(); + vandelay::db::sources::upsert_source( + &conn, + &vandelay::db::sources::SourceKey { + kind: "exchange_graph".to_owned(), + session_url: "https://login.microsoftonline.com/common|https://x.example".to_owned(), + account_id: "different-uid".to_owned(), + }, + Some("old@x.com"), + "old@x.com", + ) + .unwrap(); + drop(conn); + + let mut server = Server::new(); + let _principal = server + .mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"new-uid","userPrincipalName":"new@x.com"}"#) + .create(); + let _folders = server + .mock("GET", "/me/mailFolders?$top=100&includeHiddenFolders=true") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + + let common = vandelay::sync::CommonConfig { + archive: archive.clone(), + threads: 2, + dry_run: false, + max_retries: 0, + allow_invalid_certs: false, + logger: vandelay::logging::Logger::new(0), + }; + let config = vandelay::sync::import_exchange_graph::GraphImportConfig { + auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + token: make_jwt(9999999999, "new@x.com"), + }, + api_base: server.url(), + user_target: None, + mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + objects: Some(vec![vandelay::types::ObjectType::Mailbox]), + event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + graph_connections: 2, + top: 100, + allow_source_change: true, + }; + let result = vandelay::sync::import_exchange_graph::run(common, config); + assert!( + result.is_ok(), + "--allow-source-change must permit a different account, got {result:?}" + ); +} + +#[test] +fn dry_run_makes_no_per_item_get_and_no_sqlite_writes() { + let mut server = Server::new(); + let _principal = server + .mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"u","userPrincipalName":"a@x.com"}"#) + .create(); + let _folders = server + .mock("GET", "/me/mailFolders?$top=100&includeHiddenFolders=true") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[{"id":"FMAIL","displayName":"Inbox","isHidden":false}]}"#) + .create(); + let _children = server + .mock( + "GET", + "/me/mailFolders/FMAIL/childFolders?$top=100&includeHiddenFolders=true", + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + let _calendars = server + .mock("GET", "/me/calendars?$top=100") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + let _contacts = server + .mock("GET", "/me/contactFolders?$top=100") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + let no_messages = server + .mock( + "GET", + Matcher::Regex(r"^/me/mailFolders/.*/messages".to_owned()), + ) + .expect(0) + .create(); + let no_well_known = server + .mock( + "GET", + Matcher::Regex(r"^/me/mailFolders/[a-z]+\?".to_owned()), + ) + .expect(0) + .create(); + + let tmp = tempfile::NamedTempFile::new().unwrap(); + let archive = tmp.path().to_owned(); + let common = vandelay::sync::CommonConfig { + archive: archive.clone(), + threads: 2, + dry_run: true, + max_retries: 0, + allow_invalid_certs: false, + logger: vandelay::logging::Logger::new(0), + }; + let config = vandelay::sync::import_exchange_graph::GraphImportConfig { + auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + token: make_jwt(9999999999, "a@x.com"), + }, + api_base: server.url(), + user_target: None, + mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + objects: None, + event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + graph_connections: 2, + top: 100, + allow_source_change: false, + }; + drop(tmp); + let _ = vandelay::sync::import_exchange_graph::run(common, config).unwrap(); + no_messages.assert(); + no_well_known.assert(); + let conn = vandelay::db::init::open(&archive).unwrap(); + let sources: i64 = conn + .query_row("SELECT count(*) FROM sources", [], |row| row.get(0)) + .unwrap(); + assert_eq!(sources, 0, "dry-run must not write the sources row"); + let mailboxes: i64 = conn + .query_row("SELECT count(*) FROM mailboxes", [], |row| row.get(0)) + .unwrap(); + assert_eq!(mailboxes, 0, "dry-run must not insert mailbox rows"); +} + +#[test] +fn refresh_access_token_failure_message_carries_status() { + let other = parse_token_response(401, &json!({"error": "invalid_grant"})); + match other { + TokenResponse::Other { error, .. } => assert_eq!(error, "invalid_grant"), + v => panic!("expected Other for invalid_grant, got {v:?}"), + } +} + +#[test] +fn token_response_expired_recognised() { + let r = parse_token_response(400, &json!({"error": "expired_token"})); + assert!(matches!(r, TokenResponse::Expired)); +} + +#[test] +fn token_response_bad_code_recognised() { + let r = parse_token_response(400, &json!({"error": "bad_verification_code"})); + assert!(matches!(r, TokenResponse::BadCode)); +} + +#[test] +fn folder_enumeration_failure_skips_vanished_deletion() { + let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); + let setup_conn = vandelay::db::init::open(&archive).unwrap(); + let sid = vandelay::db::sources::upsert_source( + &setup_conn, + &vandelay::db::sources::SourceKey { + kind: "exchange_graph".to_owned(), + session_url: "stale".to_owned(), + account_id: "stale-uid".to_owned(), + }, + Some("stale@x.com"), + "stale@x.com", + ) + .unwrap(); + setup_conn + .execute( + "INSERT INTO mailboxes (id, name, sort_order, is_subscribed) VALUES (42, 'Inbox', 0, 1)", + [], + ) + .unwrap(); + vandelay::db::exchange_graph_ids::insert( + &setup_conn, + sid, + vandelay::db::exchange_graph_ids::MAILBOX, + "FMAIL", + 42, + ) + .unwrap(); + setup_conn + .execute( + "INSERT INTO blobs (hash, data) VALUES (x'aa', x'68656c6c6f')", + [], + ) + .unwrap(); + let blob_id: i64 = setup_conn + .query_row("SELECT id FROM blobs WHERE hash = x'aa'", [], |r| r.get(0)) + .unwrap(); + setup_conn + .execute( + "INSERT INTO emails (id, blob_id, received_at, mailbox_ids, keywords, message_match) + VALUES (1, ?1, '2024-01-01T00:00:00Z', '[42]', '[]', '[]')", + rusqlite::params![blob_id], + ) + .unwrap(); + vandelay::db::exchange_graph_ids::insert( + &setup_conn, + sid, + vandelay::db::exchange_graph_ids::EMAIL, + "MSG-LOCAL", + 1, + ) + .unwrap(); + setup_conn + .execute( + "UPDATE sources SET kind = 'exchange_graph', + session_url = ?1, + account_id = 'uid-flake' + WHERE id = ?2", + rusqlite::params![ + "https://login.microsoftonline.com/common|".to_owned() + + "https://graph.microsoft.com/v1.0", + sid + ], + ) + .unwrap(); + drop(setup_conn); + + let mut server = Server::new(); + let _principal = server + .mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"uid-flake","userPrincipalName":"flake@x.com"}"#) + .create(); + let _folders = server + .mock("GET", "/me/mailFolders?$top=100&includeHiddenFolders=true") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[{"id":"FMAIL","displayName":"Inbox","isHidden":false}]}"#) + .create(); + let _children = server + .mock( + "GET", + "/me/mailFolders/FMAIL/childFolders?$top=100&includeHiddenFolders=true", + ) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"value":[]}"#) + .create(); + for name in [ + "inbox", + "drafts", + "sentitems", + "deleteditems", + "junkemail", + "archive", + ] { + server + .mock("GET", format!("/me/mailFolders/{name}?$select=id").as_str()) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"id":"FMAIL"}"#) + .create(); + } + let _enum_fail = server + .mock("GET", "/me/mailFolders/FMAIL/messages?$top=100&$select=id") + .with_status(503) + .with_body("transient outage") + .create(); + + let common = vandelay::sync::CommonConfig { + archive: archive.clone(), + threads: 2, + dry_run: false, + max_retries: 0, + allow_invalid_certs: false, + logger: vandelay::logging::Logger::new(0), + }; + let config = vandelay::sync::import_exchange_graph::GraphImportConfig { + auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + token: make_jwt(9999999999, "flake@x.com"), + }, + api_base: server.url(), + user_target: None, + mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + objects: Some(vec![ + vandelay::types::ObjectType::Mailbox, + vandelay::types::ObjectType::Email, + ]), + event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + graph_connections: 2, + top: 100, + allow_source_change: true, + }; + let _ = vandelay::sync::import_exchange_graph::run(common, config).unwrap(); + let conn = vandelay::db::init::open(&archive).unwrap(); + let remaining: i64 = conn + .query_row("SELECT count(*) FROM emails WHERE id = 1", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!( + remaining, 1, + "transient folder-enumeration failure must NOT delete the locally-known message" + ); +} diff --git a/tests/mock_imap.rs b/tests/mock_imap.rs new file mode 100644 index 0000000..12b21ae --- /dev/null +++ b/tests/mock_imap.rs @@ -0,0 +1,1596 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::io::{BufRead, BufReader, Write}; +use std::net::{TcpListener, TcpStream}; +use std::path::PathBuf; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::thread; +use std::time::Duration; + +use rusqlite::Connection; +use vandelay::db; +use vandelay::imap::client::{ConnectMode, ImapClient}; +use vandelay::imap::error::ImapError; +use vandelay::imap::transport::Connector; +use vandelay::logging::Logger; +use vandelay::sync::CommonConfig; +use vandelay::sync::import_imap::{ImapAuth, ImapImportConfig, run}; + +type Script = Box std::io::Result<()> + Send + 'static>; + +struct MockImap { + addr: String, + port: u16, + _thread: thread::JoinHandle<()>, +} + +struct MockConn { + reader: BufReader, + writer: TcpStream, +} + +impl MockConn { + fn write_line(&mut self, s: &str) -> std::io::Result<()> { + self.writer.write_all(s.as_bytes())?; + self.writer.write_all(b"\r\n")?; + self.writer.flush() + } + + fn write_raw(&mut self, bytes: &[u8]) -> std::io::Result<()> { + self.writer.write_all(bytes)?; + self.writer.flush() + } + + fn read_command(&mut self) -> std::io::Result<(String, String)> { + let mut line = String::new(); + let n = self.reader.read_line(&mut line)?; + if n == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "client closed", + )); + } + let trimmed = line.trim_end_matches(['\r', '\n']); + let (tag, rest) = trimmed + .split_once(' ') + .map(|(t, r)| (t.to_owned(), r.to_owned())) + .unwrap_or_else(|| (trimmed.to_owned(), String::new())); + Ok((tag, rest)) + } +} + +impl MockImap { + fn start(handler: H) -> MockImap + where + H: FnOnce(&mut MockConn) -> std::io::Result<()> + Send + 'static, + { + Self::start_scripts(vec![Box::new(handler)]) + } + + fn start_scripts(scripts: Vec