From 97708c111c03f3ff843517b376ea8821af2a2301 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 30 Oct 2026 15:56:57 -0700 Subject: [PATCH 1/2] Import upstream 486a384 (v1.0.11 and later), renamed Upstream commit 486a3847121732fc46c2433de0e0ed50bbcb5825, renamed by scripts/upstream-import.py. --- .gitea/workflows/announce.yml | 20 + .gitea/workflows/ci.yml | 99 ++++ .gitea/workflows/upstream-watch.yml | 90 ++++ .github/workflows/ci.yml | 281 +++++++++++ .github/workflows/release.yml | 342 ------------- .gitignore | 2 + CHANGELOG.md | 203 +++++++- Cargo.lock | 2 +- Cargo.toml | 24 +- README.md | 471 +++++------------- assets/importer-exporter.jpg | Bin 7754 -> 0 bytes dist-workspace.toml | 16 - scripts/export_archive.py | 4 +- src/cli.rs | 80 +-- src/dav/client.rs | 2 +- src/exchange/jscalendar.rs | 8 +- src/exchange_ews/calendar_map.rs | 10 +- src/exchange_ews/client.rs | 2 +- src/exchange_ews/contact_map.rs | 4 +- src/exchange_ews/parse.rs | 2 +- src/exchange_graph/calendar_map.rs | 6 +- src/exchange_graph/client.rs | 2 +- src/exchange_graph/contact_map.rs | 4 +- src/exchange_graph/types.rs | 2 +- src/jmap/session.rs | 2 +- src/main.rs | 8 +- src/sync/import_dav/calcard.rs | 4 +- src/sync/import_exchange_ews/coordinator.rs | 2 +- src/sync/import_exchange_graph/coordinator.rs | 2 +- src/sync/import_managesieve/coordinator.rs | 4 +- tests/integration/baikal.rs | 8 +- tests/integration/cyrus.rs | 17 +- tests/integration/data.rs | 88 ++-- tests/integration/dovecot.rs | 24 +- tests/integration/layouts.rs | 2 +- tests/integration/mod.rs | 2 +- tests/integration/validate.rs | 6 +- tests/integration/webdav.rs | 2 +- tests/integration_baikal.rs | 6 +- tests/integration_cyrus.rs | 12 +- tests/integration_dovecot.rs | 20 +- tests/integration_radicale.rs | 6 +- tests/integration_webdav.rs | 8 +- tests/live_stalwart.rs | 6 +- tests/mock_dav.rs | 116 +++-- tests/mock_exchange_ews.rs | 76 +-- tests/mock_exchange_graph.rs | 297 +++++------ tests/mock_imap.rs | 30 +- tests/mock_jmap.rs | 42 +- tests/mock_maildir.rs | 20 +- tests/mock_managesieve.rs | 20 +- tests/mock_sync.rs | 28 +- tests/seed_smoke.rs | 4 +- tests/seeder/mod.rs | 8 +- tests/seeder/seed.rs | 12 +- tests/sync_caldav.rs | 12 +- tests/sync_carddav.rs | 12 +- tests/sync_exchange_graph.rs | 24 +- tests/sync_imap.rs | 12 +- tests/sync_jmap.rs | 78 +-- tests/sync_maildir.rs | 20 +- tests/sync_managesieve.rs | 26 +- tests/sync_takeout.rs | 14 +- tests/sync_webdav.rs | 12 +- wix/main.wxs | 228 --------- 65 files changed, 1450 insertions(+), 1546 deletions(-) create mode 100644 .gitea/workflows/announce.yml create mode 100644 .gitea/workflows/ci.yml create mode 100644 .gitea/workflows/upstream-watch.yml create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/release.yml delete mode 100644 assets/importer-exporter.jpg delete mode 100644 dist-workspace.toml delete mode 100644 wix/main.wxs diff --git a/.gitea/workflows/announce.yml b/.gitea/workflows/announce.yml new file mode 100644 index 0000000..0708bab --- /dev/null +++ b/.gitea/workflows/announce.yml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# Announce each published release on the community forum, in this project's +# Announcements category (coffey-labs/actions discourse-release; the repo -> +# category map is its release-map.json). Safe to re-run: one topic per tag. +name: announce + +on: + release: + types: [published] + +jobs: + announce: + runs-on: light + steps: + - uses: coffey-labs/actions/discourse-release@7de90df00cedcb421f3d318fb62447cd5afe7586 + with: + api-key: ${{ secrets.DISCOURSE_RELEASE_KEY }} + discord-webhook: ${{ secrets.DISCORD_RELEASE_WEBHOOK }} diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..8b9690d --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# CI on the self-hosted Gitea. Gitea reads .gitea/workflows and ignores +# .github/ once this directory exists; .github/workflows is GitHub's side of +# the switch below. +# +# Every job runs in an image pinned by digest (tag in the trailing comment), +# and the only actions used are coffey-labs/actions ones pinned by SHA. The +# instance resolves short `uses:` against itself, never GitHub, so nothing +# unreviewed can be pulled in. +# +# BUILD ON GITHUB. The org variable BUILD_ON decides which forge builds. +# Set to 'github', the test job below skips and .github/workflows/ci.yml does +# the work on GitHub's runners -- GitHub holds a push mirror of this +# repository, updated on every commit -- and reports back as the commit status +# "github/ci (branch)" or "github/ci (tag)". The `github` job waits for that +# status and passes or fails with it, so a run here still says whether the +# commit is good. Unset (or anything but 'github'), the tests run here. +# +# Releases are built only on GitHub: each architecture is compiled on a +# native runner there, and these runners are amd64 only. With BUILD_ON unset +# a v* tag is tested here but not released. +name: ci + +on: + push: + branches: [main] + tags: ["v*"] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # The unit and mock tests; the #[ignore]d ones need live servers or + # containers and run by hand (README, "Testing"). + test: + if: ${{ vars.BUILD_ON != 'github' }} + runs-on: light + container: + image: rust:1-bookworm@sha256:93ce27a88655056a51dbdd8f5f2d7ddc071c7b0070fb288a37b5a285fc83971e # 1-bookworm + steps: + - uses: coffey-labs/actions/checkout@fab0c4d45e0162963965f1555df27b7bed5e20ec + - run: cargo test --locked + + # Stands in for test and release while GitHub builds: waits for the status + # GitHub's ci.yml posts on this commit and passes or fails with it. A pull + # request is checked at its head commit, which is what GitHub built when + # the branch reached the mirror. Two and a half hours covers a slow queue; + # a timeout here with BUILD_ON=github usually means GitHub never got the + # push -- check the mirror's last error in the repository settings. + github: + if: ${{ vars.BUILD_ON == 'github' }} + # Its own runner label with plenty of slots: this job only polls, but holds a slot + # for as long as the GitHub build takes, and must not starve the build runners. + runs-on: wait + timeout-minutes: 150 + container: + image: rust:1-bookworm@sha256:93ce27a88655056a51dbdd8f5f2d7ddc071c7b0070fb288a37b5a285fc83971e # 1-bookworm + steps: + - shell: bash + env: + TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + SHA: ${{ github.event.pull_request.head.sha || github.sha }} + IS_TAG: ${{ startsWith(github.ref, 'refs/tags/') }} + run: | + set -euo pipefail + apt-get update -qq && apt-get install -y -qq --no-install-recommends jq >/dev/null + ctx="github/ci (branch)"; [ "$IS_TAG" = true ] && ctx="github/ci (tag)" + echo "waiting for '$ctx' on $SHA" + while :; do + s="$(curl -fsS -H "Authorization: token $TOKEN" \ + "$CI_SERVER_INTERNAL/api/v1/repos/$REPO/commits/$SHA/statuses?limit=50" \ + | jq -c --arg c "$ctx" '[.[] | select(.context == $c)] | sort_by(.id) | last // empty')" || s="" + state="$(printf '%s' "$s" | jq -r '.status // .state // empty')" + case "$state" in + success) echo "GitHub: success"; exit 0 ;; + failure|error) echo "GitHub: $state -- $(jq -r '.target_url' <<<"$s")"; exit 1 ;; + esac + sleep 20 + done + + # GitHub makes the Release and publishes it only once its files are + # attached; this waits for GitHub's success and announces then. The action + # makes one topic per tag, so announce.yml firing for the same Release is a + # no-op. + announce: + needs: [github] + if: ${{ startsWith(github.ref, 'refs/tags/v') && needs.github.result == 'success' }} + runs-on: light + steps: + - uses: coffey-labs/actions/discourse-release@7de90df00cedcb421f3d318fb62447cd5afe7586 + with: + api-key: ${{ secrets.DISCOURSE_RELEASE_KEY }} + discord-webhook: ${{ secrets.DISCORD_RELEASE_WEBHOOK }} + tag: ${{ github.ref_name }} diff --git a/.gitea/workflows/upstream-watch.yml b/.gitea/workflows/upstream-watch.yml new file mode 100644 index 0000000..a7aca19 --- /dev/null +++ b/.gitea/workflows/upstream-watch.yml @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# Watch upstream for releases not imported yet, and open an issue for each one +# so it waits in the tracker until someone takes it in. +# +# Reads metadata only: the releases list from GitHub's API and the head of +# this repository's `upstream` branch from Gitea's. Importing is by hand, with +# scripts/upstream-import.py, which commits the release, renamed, onto +# `upstream` for main to merge. +# +# The imported release is the version in the `upstream` branch's head commit +# subject ("Import upstream v1.0.13, renamed"). Drafts and pre-releases are +# ignored. An issue is opened once per release: an existing one with the same +# title, open or closed, stops a second. +# +# Daily 06:23 UTC; run it by hand with workflow_dispatch. +name: upstream-watch + +on: + schedule: + - cron: '23 6 * * *' + workflow_dispatch: + +concurrency: + group: upstream-watch + cancel-in-progress: false + +jobs: + upstream-watch: + runs-on: light + container: + image: python:3.13-slim@sha256:8d9d0b8bcf6506481eae4907c18f5e3e7902e629f5f6d684f9e7c32e85e3ddf0 # 3.13-slim + env: + TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + steps: + - shell: bash + run: | + python3 - <<'PY' + import json, os, re, sys, urllib.request + + api = f"{os.environ['CI_SERVER_INTERNAL']}/api/v1/repos/{os.environ['REPO']}" + def call(method, url, body=None, token=os.environ["TOKEN"]): + headers = {"Content-Type": "application/json", "User-Agent": "inbuxa-migrate-upstream-watch"} + if token: + headers["Authorization"] = f"token {token}" + req = urllib.request.Request(url, method=method, headers=headers, + data=json.dumps(body).encode() if body is not None else None) + with urllib.request.urlopen(req, timeout=30) as r: + return json.load(r) + SEMVER = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$") + def key(tag): + return tuple(int(x) for x in SEMVER.match(tag).groups()) + + subject = call("GET", f"{api}/branches/upstream")["commit"]["message"].splitlines()[0] + m = re.search(r"\b(v\d+\.\d+\.\d+)\b", subject) + if not m: + print(f"Can't read the imported release from the upstream branch: {subject!r}", file=sys.stderr); sys.exit(1) + base = m.group(1) + + # Unauthenticated: a public repository, once a day, well inside the limit. + rels = call("GET", "https://api.github.com/repos/stalwartlabs/vandelay/releases?per_page=30", token=None) + newer = sorted((r for r in rels + if not r["draft"] and not r["prerelease"] and SEMVER.match(r["tag_name"]) + and key(r["tag_name"]) > key(base)), + key=lambda r: key(r["tag_name"])) + if not newer: + print(f"Up to date: {base} is the newest upstream release."); sys.exit(0) + + existing = {i["title"] for i in call("GET", f"{api}/issues?state=all&type=issues&q=Import+upstream&limit=50")} + for r in newer: + tag = r["tag_name"] + title = f"Import upstream {tag}" + if title in existing: + print(f"{tag}: issue already exists."); continue + body = (f"Upstream published {tag} on {r['published_at'][:10]}. " + f"The last import is {base}.\n\n" + "Take it in as scripts/upstream-import.py describes:\n\n" + "```bash\n" + "git fetch origin upstream:upstream\n" + f"scripts/upstream-import.py {tag}\n" + f"git switch -c import/upstream-{tag} origin/main\n" + "git merge upstream\n" + "```\n\n" + "Before reading the changes, scan them for hidden characters and text aimed at " + "automated tools. Resolve, run the checks, open a PR, and push `upstream` once it lands.") + issue = call("POST", f"{api}/issues", {"title": title, "body": body}) + print(f"{tag}: opened #{issue['number']}.") + PY diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9b184c5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,281 @@ +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# CI on GitHub Actions, for the GitHub copy of this repository. +# +# The repository lives on the self-hosted Gitea; GitHub holds a push mirror +# that Gitea updates on every commit. Nothing is merged here -- pull requests +# happen on Gitea, and their branches reach GitHub as ordinary pushes, which is +# why this workflow runs on `push` and not on `pull_request`. +# +# Which forge does the building is one switch, the variable BUILD_ON, set on +# both forges at the organization level: +# +# BUILD_ON=github every job here runs; .gitea/workflows/ci.yml skips its +# test job and waits for the status this workflow reports +# back instead. +# unset / other every job here skips; Gitea runs the tests. Releases need +# GitHub: they are built on native amd64 and arm64 runners. +# +# The result goes back to Gitea as one commit status, "github/ci (branch)" or +# "github/ci (tag)", which is what Gitea's `github` job waits on. +# +# v* tags build a release: the tag must be on main and name the version in +# Cargo.toml; each architecture is built on its own native runner by +# scripts/build-release.sh, and the archives plus SHA256SUMS are attached to +# the Gitea Release -- Gitea is where the install guide points. The Release is +# created as a draft, filled, then published, so it is never visible with +# assets missing; if an upload fails, a Release this run created is deleted. +# Gitea announces the release once this run has reported success. +# +# The binaries link glibc dynamically, so they are built on Ubuntu 22.04: the +# oldest glibc GitHub offers, and so the widest range of servers they run on. +# +# Configuration, all at the organization level on GitHub: variables BUILD_ON, +# GITEA_URL; secret GITEA_TOKEN (write:repository on Gitea). +# +# Every `uses:` is pinned to a full commit SHA, with the release it was in the +# trailing comment. Nothing schedules here: schedules belong to Gitea. +name: ci + +on: + push: + # `upstream` holds upstream's releases, renamed, for main to merge + # (scripts/upstream-import.py); main tests them once merged. + branches: ['**', '!upstream'] + tags: ['**'] + # For a run GitHub queued and then orphaned. Run it from a tag to redo that + # tag's release; attaching skips any file already on the Release. + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref_type != 'tag' }} + +permissions: + contents: read + +env: + GITEA_URL: ${{ vars.GITEA_URL }} + CONTEXT: github/ci (${{ github.ref_type }}) + CARGO_TERM_COLOR: never + +jobs: + # Tells Gitea a result is on its way, so a status that is still missing + # reads as "running" rather than "never started". + pending: + if: vars.BUILD_ON == 'github' + runs-on: ubuntu-latest + steps: + - env: + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + run: | + curl -fsS -o /dev/null -X POST -H "Authorization: token $GITEA_TOKEN" -H 'Content-Type: application/json' \ + "$GITEA_URL/api/v1/repos/$GITHUB_REPOSITORY/statuses/$GITHUB_SHA" \ + -d "$(jq -n --arg c "$CONTEXT" --arg u "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ + '{state:"pending", context:$c, target_url:$u, description:"GitHub Actions"}')" + + # The unit and mock tests. The #[ignore]d ones need live servers or + # containers and run by hand (README, "Testing"). + test: + if: vars.BUILD_ON == 'github' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: cargo-test-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }} + restore-keys: cargo-test-${{ runner.os }}-${{ runner.arch }}- + # rustup is on the runner; rust-toolchain.toml picks the toolchain. + - run: rustup show active-toolchain || rustup toolchain install + - run: cargo test --locked + + # Guards shared by both architectures, checked once. + version: + if: vars.BUILD_ON == 'github' && github.ref_type == 'tag' && startsWith(github.ref_name, 'v') + runs-on: ubuntu-latest + env: + TAG: ${{ github.ref_name }} + steps: + # Full history: the ancestry check cannot be answered from a shallow + # clone. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + # A release can never describe code that was not reviewed onto main, + # and its tag must be the version the binary reports. + - run: | + git fetch --quiet origin main + git merge-base --is-ancestor "$(git rev-parse "${TAG}^{commit}")" origin/main \ + || { echo "::error::$TAG is not on main"; exit 1; } + # The tag names the version as the binary shows it: Cargo's 27.1.0 is + # v27.1 and 27.1.0-rc.1 is v27.1-rc.1 (src/version.rs). + want="v$(cargo metadata --no-deps --format-version 1 | jq -r '.packages[0].version' \ + | sed -E 's/^([0-9]+[.][0-9]+)[.]0(-|$)/\1\2/')" + [ "$TAG" = "$want" ] || { echo "::error::$TAG does not match Cargo.toml ($want)"; exit 1; } + + build: + if: vars.BUILD_ON == 'github' && github.ref_type == 'tag' && startsWith(github.ref_name, 'v') + needs: [test, version] + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: true + matrix: + include: + - arch: amd64 + runner: ubuntu-22.04 + - arch: arm64 + runner: ubuntu-22.04-arm + env: + TAG: ${{ github.ref_name }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - run: rustup show active-toolchain || rustup toolchain install + - run: | + SOURCE_DATE_EPOCH="$(git log -1 --format=%ct "$TAG")" scripts/build-release.sh "$TAG" dist + sha256sum dist/*.tar.gz + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: archive-${{ matrix.arch }} + path: dist/*.tar.gz + retention-days: 7 + overwrite: true + if-no-files-found: error + + release: + if: vars.BUILD_ON == 'github' && github.ref_type == 'tag' && startsWith(github.ref_name, 'v') + needs: [build] + runs-on: ubuntu-latest + env: + TAG: ${{ github.ref_name }} + steps: + # The release notes are this version's section of CHANGELOG.md, so the + # release, and the announcement made from it, say what changed. Checked + # out first: a checkout cleans the workspace, and would take dist/ with + # it if it ran after the download. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: CHANGELOG.md + sparse-checkout-cone-mode: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: dist + pattern: archive-* + merge-multiple: true + - run: | + (cd dist && sha256sum ./*.tar.gz | sed 's| \./| |' > SHA256SUMS) + cat dist/SHA256SUMS + - env: + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + run: | + set -euo pipefail + API="$GITEA_URL/api/v1/repos/$GITHUB_REPOSITORY" + auth=(-H "Authorization: token $GITEA_TOKEN") + # Everything the release carries has to be here before a release is made. + for f in inbuxa-migrate-linux-amd64.tar.gz inbuxa-migrate-linux-arm64.tar.gz SHA256SUMS; do + [ -s "dist/$f" ] || { echo "::error::dist/$f is missing; not creating a release"; exit 1; } + done + files="Binaries for linux/amd64 and linux/arm64. Verify with SHA256SUMS." + notes="$(awk -v v="${TAG#v}" 'index($0, "## [" v "]") == 1 {f = 1; next} + f && (/^## / || /^---$/) {exit} + f' CHANGELOG.md | sed -e '/./,$!d')" + if [ -n "$notes" ]; then notes="$notes"$'\n\n'"$files"; else notes="$files"; fi + # Reuse the Release if the tag already has one (a re-run), else make a + # draft of our own. + created=0 + id="$(curl -fsS "${auth[@]}" "$API/releases/tags/$TAG" 2>/dev/null | jq -r '.id // empty' || true)" + if [ -z "$id" ]; then + id="$(curl -fsS "${auth[@]}" -H 'Content-Type: application/json' \ + -d "$(jq -n --arg t "$TAG" --arg b "$notes" '{tag_name:$t, name:$t, draft:true, body:$b, prerelease:($t | contains("-rc."))}')" \ + "$API/releases" | jq -r '.id // empty')" + [ -n "$id" ] || { echo "::error::could not create the release"; exit 1; } + created=1 + fi + # The uploads cross Cloudflare, which dropped 50 MB HTTP/2 uploads + # part-way for inbuxa-server v2026.9.30.1 (curl 92). HTTP/1.1, and retry. + retry=(--retry 5 --retry-all-errors --retry-delay 15) + have="$(curl -fsS "${retry[@]}" "${auth[@]}" "$API/releases/$id/assets" | jq -r '.[].name')" + for f in dist/*; do + n="$(basename "$f")" + if grep -qxF "$n" <<<"$have"; then echo "already attached: $n"; continue; fi + echo "uploading $n" + curl -fsS --http1.1 "${retry[@]}" -o /dev/null "${auth[@]}" --form "attachment=@$f" "$API/releases/$id/assets?name=$n" || { + [ "$created" = 1 ] && curl -sS -o /dev/null "${auth[@]}" -X DELETE "$API/releases/$id" + exit 1 + } + done + if [ "$created" = 1 ]; then + curl -fsS -o /dev/null "${auth[@]}" -H 'Content-Type: application/json' -X PATCH \ + -d '{"draft":false}' "$API/releases/$id" + fi + + # ---------------------------------------------------- github release ------ + # Copies this tag's Gitea release -- notes and files -- to a GitHub release, + # so the replica's Releases page, and anyone watching it, keeps up. Gitea's + # release is the real one; this is left out of the report to Gitea, so a + # failure here cannot fail a release. PR and issue numbers in the notes are + # rewritten to Gitea links: on GitHub a bare #16 is some other PR. + github-release: + if: ${{ vars.BUILD_ON == 'github' && github.ref_type == 'tag' }} + needs: [release] + runs-on: ubuntu-latest + permissions: + contents: write + env: + GITEA_URL: ${{ vars.GITEA_URL }} + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.ref_name }} + steps: + - run: | + set -euo pipefail + if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + echo "GitHub already has a release for $TAG"; exit 0 + fi + # The Gitea release exists by now if this run made it; allow a few + # minutes either way. + code=0 + for _ in $(seq 1 15); do + code="$(curl -sS -o rel.json -w '%{http_code}' "$GITEA_URL/api/v1/repos/$GITHUB_REPOSITORY/releases/tags/$TAG")" + [ "$code" = 200 ] && break + sleep 20 + done + if [ "$code" != 200 ]; then echo "No Gitea release for $TAG; nothing to copy"; exit 0; fi + if [ "$(jq -r .draft rel.json)" = true ]; then echo "The Gitea release is a draft; not copying"; exit 0; fi + export BASE="$(jq -r '.html_url | sub("/releases/tag/.*$"; "")' rel.json)" + jq -r '.body // ""' rel.json | perl -pe 's{(? notes.md + printf '\n\n_Mirrored from [the Gitea release](%s); report issues on [Gitea](%s/issues)._\n' \ + "$(jq -r .html_url rel.json)" "$BASE" >> notes.md + files=() + mkdir -p files + while IFS=$'\t' read -r name url; do + curl -fsSL -o "files/$name" "$url"; files+=("files/$name") + done < <(jq -r '.assets[]? | [.name, .browser_download_url] | @tsv' rel.json) + title="$(jq -r '.name // ""' rel.json)"; [ -n "$title" ] || title="$TAG" + if [ "$(jq -r .prerelease rel.json)" = true ]; then kind=--prerelease; else kind=--latest; fi + gh release create "$TAG" --repo "$GITHUB_REPOSITORY" --verify-tag --title "$title" \ + --notes-file notes.md "$kind" "${files[@]}" + echo "created the GitHub release for $TAG with ${#files[@]} file(s)" + + report: + if: always() && vars.BUILD_ON == 'github' + needs: [pending, test, version, build, release] + runs-on: ubuntu-latest + steps: + - env: + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + STATE: ${{ contains(needs.*.result, 'failure') && 'failure' || (contains(needs.*.result, 'cancelled') && 'cancelled' || 'success') }} + run: | + # A cancelled run was superseded by a newer run for the same commit (the + # mirror can push one commit twice); that run reports. Posting "failure" + # here would fail the Gitea check while the real build is still going. + if [ "$STATE" = cancelled ]; then echo "cancelled: leaving the result to the newer run"; exit 0; fi + curl -fsS -o /dev/null -X POST -H "Authorization: token $GITEA_TOKEN" -H 'Content-Type: application/json' \ + "$GITEA_URL/api/v1/repos/$GITHUB_REPOSITORY/statuses/$GITHUB_SHA" \ + -d "$(jq -n --arg s "$STATE" --arg c "$CONTEXT" --arg u "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ + '{state:$s, context:$c, target_url:$u, description:"GitHub Actions"}')" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 55f6aec..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,342 +0,0 @@ -name: Release -permissions: - "contents": "write" - -on: - pull_request: - push: - tags: - - '**[0-9]+.[0-9]+.[0-9]+*' - -jobs: - # Run 'dist plan' (or host) to determine what tasks we need to do - plan: - runs-on: "ubuntu-22.04" - outputs: - val: ${{ steps.plan.outputs.manifest }} - tag: ${{ !github.event.pull_request && github.ref_name || '' }} - tag-flag: ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }} - publishing: ${{ !github.event.pull_request }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - submodules: recursive - - name: Install dist - shell: bash - run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.31.0/cargo-dist-installer.sh | sh" - - name: Cache dist - uses: actions/upload-artifact@v7 - with: - name: cargo-dist-cache - path: ~/.cargo/bin/dist - - id: plan - run: | - dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json - echo "dist ran successfully" - cat plan-dist-manifest.json - echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT" - - name: "Upload dist-manifest.json" - uses: actions/upload-artifact@v7 - with: - name: artifacts-plan-dist-manifest - path: plan-dist-manifest.json - - build-local-artifacts: - name: build-local-artifacts (${{ join(matrix.targets, ', ') }}) - needs: - - plan - if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }} - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }} - runs-on: ${{ matrix.runner }} - container: ${{ matrix.container && matrix.container.image || null }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json - AWS_LC_SYS_PREBUILT_NASM: 1 - permissions: - "attestations": "write" - "contents": "read" - "id-token": "write" - steps: - - name: enable windows longpaths - run: | - git config --global core.longpaths true - - uses: actions/checkout@v6 - with: - persist-credentials: false - submodules: recursive - - name: Install Rust non-interactively if not already installed - if: ${{ matrix.container }} - run: | - if ! command -v cargo > /dev/null 2>&1; then - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - echo "$HOME/.cargo/bin" >> $GITHUB_PATH - fi - - name: Install dist - run: ${{ matrix.install_dist.run }} - - name: Fetch local artifacts - uses: actions/download-artifact@v8 - with: - pattern: artifacts-* - path: target/distrib/ - merge-multiple: true - - name: Install dependencies - run: | - ${{ matrix.packages_install }} - - name: Update Rust toolchain to current stable - shell: bash - run: | - if command -v rustup > /dev/null 2>&1; then - rustup update stable - rustup default stable - fi - - name: Build artifacts - run: | - # Actually do builds and make zips and whatnot - dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json - echo "dist ran successfully" - - name: Attest - uses: actions/attest-build-provenance@v4 - with: - subject-path: "target/distrib/*${{ join(matrix.targets, ', ') }}*" - - id: cargo-dist - name: Post-build - shell: bash - run: | - # Parse out what we just built and upload it to scratch storage - echo "paths<> "$GITHUB_OUTPUT" - dist print-upload-files-from-manifest --manifest dist-manifest.json >> "$GITHUB_OUTPUT" - echo "EOF" >> "$GITHUB_OUTPUT" - - cp dist-manifest.json "$BUILD_MANIFEST_NAME" - - name: "Upload artifacts" - uses: actions/upload-artifact@v7 - with: - name: artifacts-build-local-${{ join(matrix.targets, '_') }} - path: | - ${{ steps.cargo-dist.outputs.paths }} - ${{ env.BUILD_MANIFEST_NAME }} - - build-global-artifacts: - needs: - - plan - - build-local-artifacts - runs-on: "ubuntu-22.04" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - submodules: recursive - - name: Install cached dist - uses: actions/download-artifact@v8 - with: - name: cargo-dist-cache - path: ~/.cargo/bin/ - - run: chmod +x ~/.cargo/bin/dist - - name: Fetch local artifacts - uses: actions/download-artifact@v8 - with: - pattern: artifacts-* - path: target/distrib/ - merge-multiple: true - - id: cargo-dist - shell: bash - run: | - dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json - echo "dist ran successfully" - - # Parse out what we just built and upload it to scratch storage - echo "paths<> "$GITHUB_OUTPUT" - jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT" - echo "EOF" >> "$GITHUB_OUTPUT" - - cp dist-manifest.json "$BUILD_MANIFEST_NAME" - - name: "Upload artifacts" - uses: actions/upload-artifact@v7 - with: - name: artifacts-build-global - path: | - ${{ steps.cargo-dist.outputs.paths }} - ${{ env.BUILD_MANIFEST_NAME }} - - host: - needs: - - plan - - build-local-artifacts - - build-global-artifacts - if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - runs-on: "ubuntu-22.04" - outputs: - val: ${{ steps.host.outputs.manifest }} - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - submodules: recursive - - name: Install cached dist - uses: actions/download-artifact@v8 - with: - name: cargo-dist-cache - path: ~/.cargo/bin/ - - run: chmod +x ~/.cargo/bin/dist - - name: Fetch artifacts - uses: actions/download-artifact@v8 - with: - pattern: artifacts-* - path: target/distrib/ - merge-multiple: true - - id: host - shell: bash - run: | - dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json - echo "artifacts uploaded and released successfully" - cat dist-manifest.json - echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT" - - name: "Upload dist-manifest.json" - uses: actions/upload-artifact@v7 - with: - name: artifacts-dist-manifest - path: dist-manifest.json - - name: "Download GitHub Artifacts" - uses: actions/download-artifact@v8 - with: - pattern: artifacts-* - path: artifacts - merge-multiple: true - - name: Cleanup - run: | - # Remove the granular manifests - rm -f artifacts/*-dist-manifest.json - - name: Create GitHub Release - env: - PRERELEASE_FLAG: "${{ fromJson(steps.host.outputs.manifest).announcement_is_prerelease && '--prerelease' || '' }}" - RELEASE_COMMIT: "${{ github.sha }}" - run: | - awk '/^## \[/{c++} c==1' CHANGELOG.md > $RUNNER_TEMP/notes.txt - - # Tag-push releases are created as drafts; the `announce` job un-drafts - # them only after the build succeeds, so watcher notifications don't - # fire on broken builds. - gh release create "${{ needs.plan.outputs.tag }}" --target "$RELEASE_COMMIT" $PRERELEASE_FLAG --draft --title "${{ needs.plan.outputs.tag }}" --notes-file "$RUNNER_TEMP/notes.txt" artifacts/* - - publish-homebrew-formula: - needs: - - plan - - host - runs-on: "ubuntu-22.04" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PLAN: ${{ needs.plan.outputs.val }} - GITHUB_USER: "axo bot" - GITHUB_EMAIL: "admin+bot@axo.dev" - if: ${{ !fromJson(needs.plan.outputs.val).announcement_is_prerelease || fromJson(needs.plan.outputs.val).publish_prereleases }} - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: true - repository: "stalwartlabs/homebrew-tap" - token: ${{ secrets.HOMEBREW_TAP_TOKEN }} - - name: Fetch homebrew formulae - uses: actions/download-artifact@v8 - with: - pattern: artifacts-* - path: Formula/ - merge-multiple: true - - name: Commit formula files - run: | - git config --global user.name "${GITHUB_USER}" - git config --global user.email "${GITHUB_EMAIL}" - - for release in $(echo "$PLAN" | jq --compact-output '.releases[] | select([.artifacts[] | endswith(".rb")] | any)'); do - filename=$(echo "$release" | jq '.artifacts[] | select(endswith(".rb"))' --raw-output) - name=$(echo "$filename" | sed "s/\.rb$//") - version=$(echo "$release" | jq .app_version --raw-output) - - export PATH="/home/linuxbrew/.linuxbrew/bin:$PATH" - brew update - # We avoid reformatting user-provided data such as the app description and homepage. - brew style --except-cops FormulaAudit/Homepage,FormulaAudit/Desc,FormulaAuditStrict --fix "Formula/${filename}" || true - - git add "Formula/${filename}" - git commit -m "${name} ${version}" - done - git push - - publish-npm: - needs: - - plan - - host - runs-on: "ubuntu-22.04" - if: ${{ !fromJson(needs.plan.outputs.val).announcement_is_prerelease || fromJson(needs.plan.outputs.val).publish_prereleases }} - permissions: - "contents": "read" - "id-token": "write" - steps: - - uses: actions/setup-node@v4 - with: - node-version: '22' - - name: Fetch npm tarball - uses: actions/download-artifact@v8 - with: - pattern: artifacts-* - path: artifacts/ - merge-multiple: true - - name: Upgrade npm for trusted publishing - run: npm install -g npm@latest - - name: Publish to npm - env: - IS_PRERELEASE: ${{ fromJson(needs.plan.outputs.val).announcement_is_prerelease }} - run: | - TAG_FLAG="" - if [ "$IS_PRERELEASE" = "true" ]; then - TAG_FLAG="--tag next" - fi - npm publish ./artifacts/vandelay-npm-package.tar.gz --provenance --access public $TAG_FLAG - - announce: - needs: - - plan - - host - - publish-homebrew-formula - - publish-npm - if: ${{ always() && needs.host.result == 'success' && (needs.publish-homebrew-formula.result == 'skipped' || needs.publish-homebrew-formula.result == 'success') && (needs.publish-npm.result == 'skipped' || needs.publish-npm.result == 'success') }} - runs-on: "ubuntu-22.04" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - submodules: recursive - - name: Un-draft release - run: gh release edit "${{ needs.plan.outputs.tag }}" --draft=false --repo "${{ github.repository }}" - - cleanup: - name: Cleanup failed release - needs: - - plan - - build-local-artifacts - - build-global-artifacts - - host - - publish-homebrew-formula - - publish-npm - - announce - if: ${{ failure() && needs.plan.outputs.publishing == 'true' }} - runs-on: "ubuntu-22.04" - permissions: - contents: write - steps: - - name: Delete draft release and tag - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: gh release delete "${{ needs.plan.outputs.tag }}" --yes --cleanup-tag --repo "${{ github.repository }}" || true diff --git a/.gitignore b/.gitignore index b52a849..d2d71e3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ target !CHANGELOG.md .* !.github +!.gitea +!/docs/*.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d51adc4..c8acae8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,207 @@ # Change Log -All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +All notable changes to this project are recorded here. Versions follow the +inbuxa suite: a series (`27.1`), its patches (`27.1.3`) and release +candidates before a series ships (`27.1-rc.1`). The tag is the version with a +`v`. Releases before 27.1 were dates (`v2026.9.30`). + +## [Unreleased] + +## [27.1-rc.6] + +The sixth release candidate of the 27.1 series: whole domains from Microsoft +365 and Google Workspace, with one admin credential and no one's password. + +### Added +- `import exchange-graph --app-only` signs in as the app itself, with its + client secret, and reads the mailbox `--user` names. With an Entra app + granted application permissions, a whole Microsoft 365 tenant moves in a + plan with no one signing in. The token is renewed before it runs out. A + plan's `clientSecret` now reaches Graph as well as EWS. +- `import imap --google-key` signs in to Gmail as a Google Workspace service + account with domain-wide delegation, acting for the user `--auth-user` + names. A whole Workspace domain moves in a plan with one key and no + one's password. The JWT is signed with the key itself (RS256), and each + connection takes a fresh token when the last is close to running out. +- The guide offers Microsoft 365 (an Entra app, app-only) and Google + Workspace (a service account) as sources. Its check signs the app in and + names any permission it lacks, or asks Google for a token for the first + account. + +## [27.1-rc.5] + +The fifth release candidate of the 27.1 series: the guided mode. A move +into inbuxa can now be walked through in the browser, from the source to the +final pass. + +### Added +- `guide` walks through a move into inbuxa in the browser. It serves a page + on 127.0.0.1 that asks for the source (IMAP with a master user, JMAP with + an administrator, or Maildir folders), the accounts (pasted, from a CSV, + or read from the source) and the inbuxa server. It can create the missing + accounts, previews with a dry run, then moves the mail with live progress + and runs a final pass. It writes an ordinary plan and runs it with + `migrate`, so the same move can be finished from a terminal. Secrets stay + in memory, and only the browser given the address, with its random + token, gets an answer. + +## [27.1-rc.4] + +The fourth release candidate of the 27.1 series: a whole domain moves with +one plan file, set up on inbuxa from an exported directory and exported +account by account with one administrator's key. + +### Added +- `import-directory` sets up on an inbuxa server what `export-directory` + wrote: tenants, roles, domains, groups, users and mailing lists, with + their aliases, quotas, roles and memberships. What the server already has + is left as it is, so running it again creates nothing. Each user created + gets a generated password, written to a new file readable by its owner + only. +- `migrate --plan` moves many accounts with one command. The plan names + the steps, which are the commands in the docs with `{email}` and + `{archive}` filled in per account, and the accounts, listed or taken + from `directory.json`. Several accounts run at once, each with its own + archive and logs. A second run resumes where each stopped, `--again` + runs the final pass, and `report.csv` says what moved and why anything + didn't. Secrets are named by environment variable, never written in the + plan. + +### Fixed +- `export` into inbuxa renames Stalwart's Sieve extensions + (`vnd.stalwart.while` and the rest) to inbuxa's. It looked for inbuxa's + names in the server's list of Sieve extensions, which doesn't name them, + so scripts went in unchanged. It now goes by the session, which says the + server is inbuxa. +- `inspect` opens the archive read-only. It used to create an empty archive + at a mistyped path and bring an older archive's layout up to date; now + the first is an error and the second is refused, and reading a closed + archive leaves no files beside it. +- `import carddav` takes a card that is in two address books into both. + It used to fail on that card on every run. Taking it out of one book, or + deleting that book, now leaves it in the other. +- `import caldav` keeps every occurrence of a series someone was invited to + one occurrence at a time, with no master event. It used to keep the first + and drop the rest. `export` now tells such occurrences apart instead of + writing them all onto one event, and `export-caldav` writes them as one + resource, as CalDAV keeps them. + +## [27.1-rc.3] + +The third release candidate of the 27.1 series: contacts, calendars and +filters go straight into another server, as mail already could. + +### Added +- `export-managesieve` writes an archive's Sieve scripts into any + ManageSieve server and makes the archive's active script the active one. + Like `export-imap`, it writes only what's missing or differs, so running + it again writes nothing. A script the server refuses is counted and the + rest go on; `--dry-run` checks each one with `CHECKSCRIPT`. +- `export-caldav` and `export-carddav` write an archive's calendars and + address books into any CalDAV or CardDAV server. The account's home is + found from the server's root, collections are matched by name or made, + and each event or card is written only when it's missing or differs, under + its etag, so running it again writes nothing. Nothing is deleted. + +### Changed +- `export-files` names its layout in `manifest.json` by the address of its + specification, `https://coffeylabs.org/specs/account-export/`, instead of + `inbuxa-migrate-files`. The layout itself is unchanged (`formatVersion` 1). + +## [27.1-rc.2] + +The second release candidate of the 27.1 series: three ways out of an +archive or a server that need no vendor's store format. + +### Added +- `export-files` writes an archive out as plain files in open formats: mail + as a Maildir++ tree, each address book as a `.vcf`, each calendar as an + `.ics`, Sieve scripts, the file tree, and a `manifest.json` with what those + formats don't hold. It needs no network, writes only into a new or empty + directory, and keeps the directory private to its owner. +- `export-imap` writes an archive's mail into any IMAP server: folders in the + server's own layout (SPECIAL-USE folders for sent, drafts, trash, junk and + archive), flags, keywords where the folder keeps them, and arrival times. + Like `export`, it matches what is already there, so running it again + writes only what changed. +- `export-directory` writes who is on a server: its domains, users, groups, + mailing lists, aliases, quotas, tenants and roles, read from the registry + with an administrator's login, as `directory.json` and CSV files that + another server can be set up from. Credentials, OTP seeds, API keys and + DKIM keys are never asked for. `--domain` keeps it to some domains. + +### Changed +- `import maildir` reads folder names in modified UTF-7, as Dovecot and + Courier write them, and the custom keywords in each folder's + `dovecot-keywords` file (Junk, labels). +- A FETCH section with a header list, such as + `BODY[HEADER.FIELDS (MESSAGE-ID)]`, is read as one item. +- `import takeout` unfolds and unescapes a calendar's `X-WR-CALNAME` and + `X-WR-CALDESC`, so a long name or one with a comma reads back whole. + +## [27.1-rc.1] + +The first release candidate of the 27.1 series, with the rest of the suite. + +### Changed +- `--version` and the User-Agent show the version as tags name it + (`27.1-rc.1`). + +## [2026.10.26] -- 2026-10-26 + +### Changed +- `--auth-bearer` reads `INBUXA_MIGRATE_API_KEY`, the name inbuxa-cli and the + console use. `INBUXA_MIGRATE_TOKEN` still works; when both are set, the API + key wins. + +## [2026.9.30] -- 2026-09-30 + +The first release of inbuxa-migrate. + +### Changed +- Renamed to inbuxa-migrate: the binary, the crate, and the credential + variables, now `INBUXA_MIGRATE_PASSWORD`, `INBUXA_MIGRATE_TOKEN`, + `INBUXA_MIGRATE_EWS_CLIENT_SECRET` and `INBUXA_MIGRATE_GRAPH_TOKEN`. The old + names are not read. Archives written by Vandelay are not read either. +- Registry calls (`x:Account`, `x:Domain` and the other `x:` types) use the + capability the server advertises: `urn:inbuxa:jmap:registry` on inbuxa, + `urn:stalwart:jmap` on a Stalwart source. A server that advertises neither + gets a clear error instead of a request it never offered to accept. +- Released as Linux archives for amd64 and arm64 on the Gitea release page, + with `SHA256SUMS`. The npm, Homebrew, shell and PowerShell installers and + the MSI are gone. +- A second export brings matched items up to date instead of skipping them: + flags and folders on mail, changed contacts and events, and edited Sieve + scripts. +- A message filed in several folders is written once, in every one of them, + and a resumed export adds any folder it was still missing. +- Sieve scripts from a Stalwart server keep working on inbuxa: their + `vnd.stalwart.*` extension names are renamed to inbuxa's, each rename is + logged, and a script that can't be activated fails the export instead of + leaving filtering quietly off. +- `--allow-invalid-certs` covers only the server named with `--url`. Microsoft + and Google sign-in are always verified. +- Export imports mail in batches, uploads in parallel within the target's + limits, and prints progress: count, rate and time left. +- `export --dry-run` predicts what would fail -- messages over the target's + upload limit, objects over its request limit, Sieve scripts it would reject + -- lists what would be created, updated and skipped, and exits as the real + run would. + +### Fixed +- Every connection times out instead of waiting forever on a dropped link, + and a timeout is retried like any other transient error. +- An interrupted IMAP or Maildir import keeps what it wrote, and a message + that can't be read is recorded and skipped rather than ending the folder. +- IMAP and EWS imports hold a bounded amount in memory, in batches capped by + size as well as count. +- The archive is created readable by its owner only, and an existing archive + others can read is reported. +- A dry run no longer writes to the target's address books. + +--- + +# Earlier releases (upstream Vandelay) ## [1.0.11] - 2026-09-28 diff --git a/Cargo.lock b/Cargo.lock index 4c86ec9..f22f4bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2937,7 +2937,7 @@ dependencies = [ ] [[package]] -name = "vandelay" +name = "inbuxa-migrate" version = "1.0.11" dependencies = [ "blake3", diff --git a/Cargo.toml b/Cargo.toml index c3b8088..402db81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,22 +1,16 @@ [package] -name = "vandelay" -description = "JMAP account migration utility" -version = "1.0.11" -authors = ["Stalwart Labs LLC "] +name = "inbuxa-migrate" +description = "Account migration and backup for inbuxa" +version = "27.1.0-rc.6" +authors = ["Coffey Labs LLC "] license = "Apache-2.0 OR MIT" -repository = "https://github.com/stalwartlabs/vandelay" -homepage = "https://github.com/stalwartlabs/vandelay" +repository = "https://git.coffeylabs.org/inbuxa/inbuxa-migrate" +homepage = "https://git.coffeylabs.org/inbuxa/inbuxa-migrate" edition = "2024" readme = "README.md" -[package.metadata.wix] -upgrade-guid = "5C8148E7-34A9-4216-A802-FFB8345AECC4" -path-guid = "FA9E97EB-5E5C-4809-9018-9DC1EDB0E93D" -license = false -eula = false - [[bin]] -name = "vandelay" +name = "inbuxa-migrate" path = "src/main.rs" [features] @@ -52,7 +46,3 @@ psl = "2.1.226" mockito = "1.7.2" tempfile = "3.13" testcontainers = { version = "0.28", features = ["blocking"] } - -[profile.dist] -inherits = "release" -lto = "thin" diff --git a/README.md b/README.md index 2b6c1e7..3d870d6 100644 --- a/README.md +++ b/README.md @@ -1,355 +1,134 @@ -

Vandelay

- -

- The JMAP importer-exporter -
- One-shot account migration and backup across JMAP, IMAP, CalDAV, CardDAV, WebDAV, ManageSieve, Maildir, Google Takeout and Microsoft Exchange. -

- -
+# inbuxa-migrate

- build -   - release -   - npm -   - downloads -   - platforms -   - license + License: Apache-2.0 OR MIT + Latest release + Forum: community.coffeylabs.org + Chat on Discord

-
- - - - - - -
- -- Jerry: Well, what does **he** do? -- George: He's an importer. -- Jerry: Just imports, no exports? -- George: He's an **importer/exporter**, okay? - - -He's an importer-exporter. -
- -  - -## About - -Vandelay is a one-shot account-migration utility for JMAP, the JMAP analogue of `imapsync` generalised to every JMAP data type (mail, contacts, calendars, identities, sieve scripts, file storage). It imports an account from a wide range of source protocols into a local SQLite "archive", then exports that archive into a target JMAP server. Import and export never talk to each other, only to the SQLite archive. One archive holds exactly one account. - -Because the archive is a self-contained SQLite file that fully describes one account, vandelay doubles as a per-account backup tool: run an import on a schedule to capture a fresh snapshot, keep the resulting SQLite file as your backup, and restore it later by running an export against a JMAP target. - -## Features - -- **Many source protocols:** - - JMAP - - IMAP - - CalDAV - - CardDAV - - WebDAV - - ManageSieve - - Maildir++ - - Google Takeout - - Microsoft Exchange Server (on-premises) via EWS - - Microsoft Exchange Online via Graph -- **One target protocol:** JMAP, with type-by-type stateless re-matching on every run. -- **Convergent:** Re-running an interrupted import or export picks up where it left off without bookkeeping flags. -- **Multi-threaded, no async runtime:** Blocking HTTP with per-server concurrency caps respected automatically. -- **Content-addressed blobs:** Emails, sieve scripts and file-storage payloads are stored once by BLAKE3 hash and deduplicated across the archive. -- **Dry-run everywhere:** Every command supports `--dry-run` to compute the full plan without writing. -- **Source-change protection:** An archive remembers which account it was filled from; pointing it at a different one fails unless explicitly permitted. -- **Read-only inspection:** A built-in `inspect` command dumps any object type from an archive for verification. - -## Install - -```sh -# macOS / Linux -curl --proto '=https' --tlsv1.2 -LsSf \ - https://github.com/stalwartlabs/vandelay/releases/latest/download/vandelay-installer.sh | sh - -# Homebrew -brew install stalwartlabs/tap/vandelay - -# Windows -powershell -ExecutionPolicy Bypass -c "irm https://github.com/stalwartlabs/vandelay/releases/latest/download/vandelay-installer.ps1 | iex" - -# npm -npm install -g @stalwartlabs/vandelay - -# From source -cargo install --path . -``` - -A signed `.msi` is also published with each release. - -## Quick start - -A typical run is two commands, one to capture a source account into a local SQLite archive and one to push that archive into a JMAP target. - -```sh -# 1. Import an IMAP mailbox into a fresh archive. -export VANDELAY_PASSWORD='source-app-password' -vandelay import imap \ - --url imaps://imap.example.com \ - --auth-basic alice@example.com \ - alice.sqlite - -# 2. Peek at what landed. -vandelay inspect alice.sqlite # per-type summary -vandelay inspect alice.sqlite mailbox # mailbox tree -vandelay inspect alice.sqlite email --limit 5 - -# 3. Push the archive into a target JMAP server. -export VANDELAY_PASSWORD='target-password' -vandelay export \ - --url https://jmap.example.org \ - --auth-basic alice@example.org \ - --account-name alice@example.org \ - alice.sqlite -``` - -Both commands are convergent: rerun either to resume an interrupted run, or rerun `import` later to pick up new mail since the last snapshot. Use `--dry-run` on either side to compute the full plan without writing. - -## CLI quick reference - -``` -vandelay [args...] -``` - -All actions accept a set of global flags (verbosity, worker pool size, retry policy, TLS handling) in addition to action-specific ones. The most useful are: - -| Flag | Purpose | -| --- | --- | -| `-j, --threads ` | Worker pool size (default: logical CPUs). | -| `--dry-run` | Compute the full plan; perform no writes. | -| `-v`, `-vv`, `-vvv` | Increase log verbosity. | -| `-q, --quiet` | Warnings and errors only. | -| `--max-retries ` | Max retries per request on transient failures (default 5). | -| `--allow-invalid-certs` | Accept self-signed / invalid TLS certs. | - -Credentials should be supplied via the `VANDELAY_PASSWORD` / `VANDELAY_TOKEN` / `VANDELAY_EWS_CLIENT_SECRET` / `VANDELAY_GRAPH_TOKEN` environment variables, or via an interactive prompt; passing them on the command line is supported but not recommended. - -### Import - -``` -vandelay import [source-args...] -``` - -Reads a source account into the local SQLite `ARCHIVE` (created if absent). Every importer accepts `--allow-source-change` to override the archive's source-identity guard. - -#### JMAP - -``` -vandelay import jmap \ - --url \ - (--auth-basic [--auth-password ] | --auth-bearer [TOKEN]) \ - (--account-id | --account-name ) \ - [--objects ] \ - -``` - -Imports a single JMAP account. `--objects` accepts a comma-separated list of object tokens (`mailbox,email,calendar,calendarevent,addressbook,contactcard,identity,sievescript,participantidentity,filenode`); default is everything the server advertises. - -#### IMAP - -``` -vandelay import imap \ - --url imap(s)://host[:port] \ - (--auth-basic [--auth-password ] | --auth-bearer [TOKEN] --auth-user ) \ - [--include ...] [--exclude ...] [--exclude-special ...] \ - [--folder ...] [--subscribed-only] [--noautomap] \ - [--include-deleted] [--allow-cleartext] [--compress] \ - [--fetch-batch ] [--imap-connections <1..8>] \ - -``` - -Imports mail (and only mail) from any IMAP server. Folder selection is via `--include`/`--exclude` regexes (mutually exclusive with the exact-match `--folder`); `--exclude-special` drops by SPECIAL-USE role. - -#### CalDAV - -``` -vandelay import caldav \ - --url \ - (--auth-basic [--auth-password ] | --auth-bearer [TOKEN]) \ - [--allow-cleartext] [--dav-connections <1..8>] [--multiget-batch ] \ - -``` - -Discovers the user's CalDAV principal (or accepts a URL pointing straight at a calendar-home or calendar), then imports calendars and events. - -#### CardDAV - -``` -vandelay import carddav \ - --url \ - (--auth-basic [--auth-password ] | --auth-bearer [TOKEN]) \ - [--allow-cleartext] [--dav-connections <1..8>] [--multiget-batch ] \ - -``` - -Same shape as `caldav`, but for address books and contacts. - -#### WebDAV - -``` -vandelay import webdav \ - --url \ - (--auth-basic [--auth-password ] | --auth-bearer [TOKEN]) \ - [--allow-cleartext] [--dav-connections <1..8>] [--multiget-batch ] \ - -``` - -Imports a plain WebDAV file collection as a JMAP `FileNode` tree. - -#### ManageSieve - -``` -vandelay import managesieve \ - --url sieve(s)://host[:port] \ - (--auth-basic [--auth-password ] | --auth-bearer [TOKEN] --auth-user ) \ - [--allow-cleartext] \ - -``` - -Imports sieve scripts only. Each script is content-addressed in the blob table; the active script is recorded. - -#### Maildir - -``` -vandelay import maildir \ - [--include ...] [--exclude ...] [--folder ...] \ - [--noautomap] [--include-deleted] -``` - -Reads a local Maildir++ tree (a directory with `cur/`, `new/`, `tmp/`). No network. Folder selection mirrors the IMAP importer. - -#### Google Takeout - -``` -vandelay import takeout [--noautomap] -``` - -Scans a directory tree recursively for `.mbox`, `.ics` and `.vcf` files and imports them. Tailored to Google Takeout layouts but works on any such tree; system-label role assignment can be disabled with `--noautomap`. - -#### Microsoft Exchange (EWS) - -``` -vandelay import exchange-ews \ - [--url ] [--mailbox ] \ - [--mailbox-kind primary|archive|public-folders] \ - (--auth-basic [--auth-password ] \ - | --auth-bearer [TOKEN] [--ews-tenant --ews-client-id \ - (--ews-device-code | --ews-client-secret )]) \ - [--ews-connections <1..8>] [--ews-getitem-batch ] [--ews-attachment-batch ] \ - [--ews-no-syncfolderitems] \ - -``` - -Imports a mailbox via EWS from an on-premises Exchange Server. Autodiscover is used when `--url` is omitted (a `--mailbox` SMTP address is then required). Supports Basic, pre-acquired bearer, interactive device-code OAuth, and app-only client-credentials OAuth. - -For Exchange Online, use `exchange-graph` instead. Microsoft is retiring EWS in Exchange Online: from 1 October 2026 it is blocked unless a tenant administrator sets `EwsEnabled` to `True` and adds the client id to `EwsAllowedAppIDs`, and on 1 April 2027 it is switched off for every tenant. On-premises Exchange Server is not affected. - -#### Microsoft Exchange (Graph) - -``` -vandelay import exchange-graph \ - (--client-id [--tenant ] | --access-token [TOKEN]) \ - [--user ] \ - [--mailbox-kind primary|archive] \ - [--objects mail,calendar,contacts] \ - [--event-body-format text|html] \ - [--graph-connections <1..16>] [--top <1..1000>] \ - -``` - -Imports a mailbox from Exchange Online via Microsoft Graph. Without `--access-token`, the interactive device-code flow is used. `public-folders` is rejected here: Graph does not expose public folders, so they can only be imported with `exchange-ews`, which for Exchange Online is subject to the retirement described above. - -### Export - -``` -vandelay export \ - --url \ - (--auth-basic [--auth-password ] | --auth-bearer [TOKEN]) \ - (--account-id | --account-name ) \ - [--objects ] [--prune [--yes]] \ - -``` - -Stateless re-export of `ARCHIVE` into a target JMAP server account. The default behaviour is upsert-only: matched items are updated, unmatched local items are created, but pre-existing target items not covered by the archive are left alone. - -`--prune` enables destructive reconciliation: target objects that do not match anything in the archive are deleted. The confirmation prompt can be skipped with `--yes` for automation. Export speaks JMAP only; no other target protocols are currently supported. - -### Inspect - -``` -vandelay inspect [TYPE] [--limit ] [--offset ] -``` - -Read-only dump of a local archive. This command never opens a network connection and never writes to the archive. - -- Omit `TYPE` for a per-type summary (counts of every object kind plus blob storage stats). -- Pass an object type to dump it: `mailbox`, `email`, `identity`, `sievescript`, `addressbook`, `contactcard`, `calendar`, `calendarevent`, `participantidentity`, `filenode`. -- `mailbox` and `filenode` render as a tree (`--limit`/`--offset` are ignored); all other types use a paginated list and respect `--limit` and `--offset`. - -## Testing - -The default suite is hermetic (unit tests plus `mockito`-scripted JMAP/DAV/EWS/Graph behaviours) and needs no network or Docker: - -```sh -cargo build -cargo clippy --all-targets -cargo test -``` - -### Live and integration tests (Docker required) - -Live integration tests against a Stalwart server and container-based tests against third-party servers (Dovecot, Cyrus, Radicale, Baikal, Apache `mod_dav`) are gated behind `--ignored`. **They require a running Docker daemon**: each test binary boots its own throwaway container via `testcontainers` (images are pulled automatically on first run), so Docker must be installed and `docker info` must succeed before invoking them. - -Run them per binary, and always with `--test-threads=1`: - -```sh -cargo test --test sync_jmap -- --ignored --test-threads=1 # live JMAP import/export/convergence/prune -cargo test --test sync_imap -- --ignored --test-threads=1 -cargo test --test sync_managesieve -- --ignored --test-threads=1 -cargo test --test sync_maildir -- --ignored --test-threads=1 -cargo test --test sync_caldav -- --ignored --test-threads=1 -cargo test --test sync_carddav -- --ignored --test-threads=1 -cargo test --test sync_webdav -- --ignored --test-threads=1 -cargo test --test live_stalwart -- --ignored --test-threads=1 -cargo test --test seed_smoke -- --ignored --test-threads=1 -cargo test --test seed_only -- --ignored --test-threads=1 - -# Third-party-server tests (one container each): -cargo test --test integration_radicale -- --ignored --test-threads=1 -cargo test --test integration_baikal -- --ignored --test-threads=1 -cargo test --test integration_webdav -- --ignored --test-threads=1 -cargo test --test integration_dovecot -- --ignored --test-threads=1 -cargo test --test integration_cyrus -- --ignored --test-threads=1 - -# Slow tests -cargo test --test mock_jmap -- --ignored -``` - -`--test-threads=1` is mandatory, not just advisory: within a binary every test shares a single per-binary container, and each test provisions then tears down the same disposable `vandelay.org` domain (and opens the archive with SQLite `EXCLUSIVE` locking). Separate binaries are isolated (each boots its own container on dynamic host ports), so plain `cargo test --test ` invocations are safe to run one after another. +> [!NOTE] +> Development happens on [git.coffeylabs.org/inbuxa/inbuxa-migrate](https://git.coffeylabs.org/inbuxa/inbuxa-migrate); the copy on GitHub is a read-only mirror. +> Report issues at **[git.coffeylabs.org/inbuxa/inbuxa-migrate/issues](https://git.coffeylabs.org/inbuxa/inbuxa-migrate/issues)**, join discussions at **[community.coffeylabs.org](https://community.coffeylabs.org)**, or chat on **[Discord](https://discord.gg/nqcY4TKfAn)**. + +One program that moves an account into **inbuxa**: its mail, calendars, +contacts, identities, Sieve scripts and files. + + inbuxa-migrate import imap … alice.sqlite an account into an archive + inbuxa-migrate inspect alice.sqlite what landed + inbuxa-migrate export … alice.sqlite the archive into inbuxa + inbuxa-migrate export-files alice.sqlite out the archive as plain files + inbuxa-migrate export-imap … alice.sqlite its mail into any IMAP server + inbuxa-migrate export-managesieve … alice.sqlite its filters, likewise + inbuxa-migrate export-caldav … alice.sqlite its calendars into any CalDAV server + inbuxa-migrate export-carddav … alice.sqlite its contacts into any CardDAV server + inbuxa-migrate export-directory … out a server's accounts, no secrets + inbuxa-migrate import-directory … out those accounts, set up on inbuxa + inbuxa-migrate migrate --plan plan.json work many accounts, by a plan + inbuxa-migrate guide work the same, walked through in the browser + +Between the two sides sits an archive: one SQLite file holding one account. +`import` fills it from the old server; `export` writes it into the new one. +Import and export never talk to each other, only to the archive, so the old +server can be gone before the new one exists. + +Both commands converge. An interrupted run picks up where it stopped, and a +later `import` into the same archive adds what arrived since. Every command +takes `--dry-run` and then only reports the plan. `inspect` reads an archive +and changes nothing. + +The archive is also a backup. Import on a schedule, keep the file, and +restore it later with `export`. An archive remembers which account filled it +and refuses a different one unless told otherwise. + +Export speaks JMAP only. It writes into inbuxa, or into any JMAP server that +advertises the types being written. `export-imap` writes the mail into any +IMAP server, `export-managesieve` the Sieve filters into any ManageSieve +server, `export-caldav` and `export-carddav` the calendars and contacts into +any CalDAV and CardDAV server, and `export-files` is the way out to everything else: it writes +the archive as a Maildir++ tree, vCard, iCalendar and Sieve files that other +servers and mail, calendar and contacts apps open, with no network and no +store format in the way. For a whole server, `export-directory` writes the +list of domains, accounts, groups, lists and aliases to set up again +elsewhere, without passwords, and `import-directory` sets them up on an +inbuxa server, with a new password for each user. `migrate` moves many +accounts with one plan file: the same steps for each account, several at +once, resuming where it stopped, with one report at the end. `guide` +walks through the same move in the browser: it asks where the mail is, +which accounts and which inbuxa server, previews, and runs the plan it +writes. The full +command reference is in [docs/usage.md](docs/usage.md). + +## Sources + +- **JMAP** -- any JMAP account, including an inbuxa or Stalwart server. +- **IMAP** -- mail only. Folders are chosen by name or by pattern. +- **CalDAV, CardDAV** -- calendars and events, address books and contacts. +- **WebDAV** -- a plain file collection, kept as a file tree. +- **ManageSieve** -- Sieve scripts only, with the active one recorded. +- **Maildir** -- a local Maildir++ tree. No network. +- **Google Takeout** -- the `.mbox`, `.ics` and `.vcf` files in a Takeout + export, or in any directory tree laid out that way. +- **Exchange Server** -- an on-premises mailbox, through EWS. +- **Exchange Online** -- through Microsoft Graph. EWS is being retired in + Exchange Online: blocked from October 1, 2026 unless an administrator allows + the client, and switched off on April 1, 2027. Public folders are only + reachable through EWS. + +## Credentials + +Secrets are read from the environment, or asked for at a prompt: + + INBUXA_MIGRATE_PASSWORD a password for --auth-basic + INBUXA_MIGRATE_API_KEY an API key or bearer token for --auth-bearer + (INBUXA_MIGRATE_TOKEN still works) + INBUXA_MIGRATE_EWS_CLIENT_SECRET the OAuth client secret for app-only EWS + INBUXA_MIGRATE_GRAPH_TOKEN a Microsoft Graph access token + +The command line takes them too, but a secret there ends up in the shell's +history and in the process list. Use the environment or the prompt. + +## Installing + +Linux, on amd64 or arm64. Each release carries one archive per architecture +and a `SHA256SUMS` file: + + base=https://git.coffeylabs.org/inbuxa/inbuxa-migrate/releases/latest/download + curl -fLO "$base/inbuxa-migrate-linux-amd64.tar.gz" + curl -fLO "$base/SHA256SUMS" + sha256sum --check --ignore-missing SHA256SUMS + tar -xzf inbuxa-migrate-linux-amd64.tar.gz + ./inbuxa-migrate --version + +On arm64, fetch `inbuxa-migrate-linux-arm64.tar.gz` instead. The binary is +the whole program; put it anywhere on the path. + +## Building and testing + + cargo build --release the binary is target/release/inbuxa-migrate + cargo test the default suite + +The default suite needs no network and no Docker. Unit tests and scripted +mocks stand in for JMAP, DAV, EWS and Graph servers. + +The live tests are marked `#[ignore]`. Each boots a throwaway container -- +Stalwart, Dovecot, Cyrus, Radicale, Baikal or Apache `mod_dav` -- so they +need a working Docker daemon. Run them one binary at a time, and always with +one test thread: + + cargo test --test sync_jmap -- --ignored --test-threads=1 + cargo test --test integration_dovecot -- --ignored --test-threads=1 + +The tests in a binary share one container and one disposable domain, so a +second thread would trip over the first. [docs/usage.md](docs/usage.md) lists +every live test binary. ## License -Licensed under either of +Apache-2.0 OR MIT, at your option. The texts are in [LICENSES](LICENSES). - * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) - * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) +Copyright (C) 2020, Stalwart Labs LLC
+Copyright (C) 2026 Coffey Labs LLC -at your option. - -## Copyright - -Copyright (C) 2020, Stalwart Labs LLC +Forked from Vandelay, originally developed by Stalwart Labs, and distributed +under the same Apache-2.0 OR MIT terms. diff --git a/assets/importer-exporter.jpg b/assets/importer-exporter.jpg deleted file mode 100644 index 1fa96d2a6fffe9312d8f661ae4148a732f788af1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7754 zcmb7pbx;&;(Eq{FNJ=P2AL$X&-Q5C*(ny0K-Q5k+4N^xphj1Vz-Q9=M937&B2(RCF z=KbUM=kK$#GtbV>?z1to&&;z=D^HsMVpT;IMF1Ka0D$(708i_H2mme?HtBOx^8c8e zobm+)B?Top<-Zd-CFP43l#~?Y-3AzWNslyrQ;d>le-oLmqz zJUl!?0zxWcVk%A=8hXzEZ#?w^NN@n((11WRMgTeq8ju9-X#fBM0MLPG|AqR0z`#TY zVgb;w|0yMi0cb#U40Id}Obm21%>T^?9f(1KNy@+@qmA{P(cCpOrHQw=Zhioag>osM~f zXjNEjn@=&4um&Qlo-J|@vDJ(A3eZvd>=)7|3}|4gL=A1}qi-)dlMp0To%Q~ca#V`8Qd{&4#Dws>lswy#IS)>c$agx0 z%6|5SXSNSUC#l=XPk>QyUQqnIaD_XP$C*rgM|%mfAgVF0NInnMHuGw_Il)ry!FQOmFAd2{&1cuZ)QsDM8zn^e zClVal(2=D?)f%z^5WbW0gcqF=$7Xdv{Y)PFG-Hf;;)n-Q8DHN_n!JL{u639e=YuHP zAT^Tx)2xiGW9Pjq1%oZLwSS8f_?Po7km$kZffJ=Vz5N23enzdY8J)7_@dPNT9XDs@Iot&8y(bV?++fhf%FNzVe-bB) zs#Eq{%rJ^FVbrNmpPez@^iDtg2Sx%Ds=$rBc}Oj;hKTADhWj^}b;@qecZ^I__3{)< zg*rEsV(wLXk)?-Nc!r{=q39i8^c~?|Ve=&5_iN2~wb^!#Z|4g^8czVLBGhXG>KQF7 z8Wpcc8GE*>+#!Lm{r+H3Iw_H&dzm^{IsmtY%Rh2*OgjtS9X25XYmxF6iR@`D?({8q zs~U~Zo4?#*gg8Jx*W)MD-k|ZdmbSx)4kPH2k+j@z{n@KLw~X{jd~?13LLZmY`T-dP z(>Y6)6`TmDJIk3NwI;uQu1shP7fMFx>ApXuDf#JIA41UD!4(h$3if`K@~Cg1!mKB- ze;HY?{2Rx_ML>{yu!(Y#!Bq&Tc-ofgJ3I8O?54KTu<%pI#82(dn3q@f3^*vGLGCbU zfG#^2b^%)TSEJ>^8`4v}SLx4dAP#S&4)!Mhj(uAnd`>;zb`Aeca`u zrU0LS)rATgd~*6riQ&@MctZ$3?EGnbsG1&3K0;c@u*P4ut4}%t9Sb$1rw&N6)vC7I z46C=v>Cu=Tg~-T?YZTNqcx9ZC)9ndOZ*WvsrkqV^v;%|*RleZXnp!Jz?dly;QoJ%9 zaLBFn*KLxvvY7M;?JB<|ul1*JE}Lk{*@W&!$8do<-fz4Swp*l8_og|LYmR+cz2_0G zHxkDm)h6cn7xPP3WZl4}c5y{HkPpMMd*`^B6zCAlB^>#=CLE0XuC(AHnW;FDQrs}C zzup?o^Ve)m5`|~%q?l^5-Xp=eWmP@yJsZH1q?~&oe;_P)B{BF-uu$#|Q7XHyD1K$B zlwP7|f;(>}(rKBlFgHT-x!EE-`lIEI=4AqgSY2geT^LqdlZ!J0h@vfWY}#VQVe8)` zn2aB@z+WJz^7rjRB#a^~7m{+8oI|)w&;|*Uri~PYp}N%am$Ju!p`J)-fStgXq9gv+ z#V(?w0q0c3eUh$*gNuZ38}}spttYirDT9~VoshxUT+IfJp*I_bTq@fd^NMUrg4Q*D zbYO7CU4J~6w4GpprIJ|tk7!o|KG)6eR{p4hE^8ANp~c2Q8@Q1TJV29H5dYxC!lcKX zxIi&%u$Mm?EzpC(evp3f3uFRrLYvlPMNJ~89PJtIh}qxT!Hwl4jhfng0?h5D!Kd2$ z1}jcx=E)qCsg~BArYv8AaC=Q^>k@7P7p6PAXH0v3lT`Rw$v-o20>Yma}4jdf3aO zd&j?$rDR;!ZoaEJIzrPSM2`q>A{+8;(xcSNw@&Wc#C6e!X#w^NabK)r>%`8Cg@Z2v ziz#hzDLcIh-qHP%H(xZCxtTWn8evV2#6AO5xPaZ|;9pW&^a;_{E_M=WO*6N2dk-m! ze4HDBy673rGQ?Kala^GtE-h*ylw`{t$6a{_Yd4-(uH?qekF{`+zJLube)iQD6#w)2I(Df@PN zc`orXg$=^DcND9+UPSMOPR(|3k^6Tn2tGj;%kCk+UT`t7-wPexvM+U zcn9Iibf2L%6erCs3;gu@gUutY5W$qi+0X2FUjyS~)BESGxA$c}A04)sjQDMJ=6iqI zZ|c4h%Ho-$K|XZ#x44k;D!9dMrKx5w#kH`%v-l++LdDN?yW)DbnW;qIU6r3m=%Dt| zg})=Un=oFXT|l>aZ$t^){!3m^ynaTX&V!6lum#rRt`KrX^%?3ZAsJeWwHw0iUNvu2 zQ3JJiu&B;08J*hR`T~~aNBTt^5Dq!bF0FvpU$0yss7W<5`30*coquBq%?Z*!8+c9f zuwlsO^3hL(8B-u=x_DLT)sRer>IAGo5Nx?dOLb!)RCY)&1rlHXSMa~fP&XFWF-}oh z;*z~n{*z>rB#eOJ7$mzyby&PZ^?j$A<#yV19RIOJkQsfUg?6;!rhnIx^F`ar^Bzf} zw&wJ+F~yLYWA_wu2cM$+$K7f22z-5`V2|3yt2bH9dT;(I|E!Td^ISt4Q~IlG75!}z`>JH+k>QVQ z661!9X|E?898$CHb<1TM_nVL|@E!d`rg3XjR1*Zfp1b6#Bp_UfyS(Rk(u%EtEcvAs z_?H;6h}(11bCmVY^RzbGeF?J2nU?zd7^5)*jU|o{vXbY)fjQ|aO0dw|9OVR*dD!qF z9fKy)rIjv&D!ORg=?LVV1Y2@9eM!Yw7ptB#xaTT^UR|u!*DW)iijr&%ct@l_B%_^N zS5Q7lSPx3CYfJ0Kyswm+W19hU9GqUlZft%DxTj02(Owm^kYO%u1>r6i-f+@r|4$NU zIp#ia$9(K)Ldv;G*6cg)4;A4aZv~YK6(7y12*yQ>QVg7up`J{Gv<+ToV(>zMi#P3Y z4Vx-Ci>d(_6339B=h>o0zs!eJiX3qs!4MzV#d$k61i-%y<|;dp6*CeVsgz1JxyqvYNsrjsh7)wpp9>jb zCk{(qG?J4VWS!G)y)LWd@UqEs#qh3?)gUQ#h)L?ZI^T}?$i4&q3LwyvOTeY#XYSty z(A?d!D<W|#t>#&yg*OJIFpM|Asotav5%UPfqvArp%zq`S(uRGK-5J+1;_9xW+YQXL7`-OHbupz9{}Yx3&Zv+^&tn(P z@(Y~FAcm8AUkWMjIKCb!sc9KAJirD>l1htuH&~TRl$Iw+?kSR3h_&9A*P&0%-7Gtu zx62wDQW1<9QD3W8AwZzSah1lixwX5L-L`BfJ$yuZ$NVFJx^sEBvO2*H8-Il^V|jJ6 zAPTTgY^cx&(*OL&Bv^Ca2vz7aKf|pIS;x>yzM!z%zX@!+g6wMf`u}|H54uftlqwjP znaej`P?r`;%xVi3;`mE6qy!Gmj5A<8;Pepv6al2c*BU^(`n_A3nNb`<;zC#$fX7y# z$Jf0zNsZlxQ6X|Fu0a)oor*Y6Fp?yf#0jA)^TN4&r=8c@ji&oX3{y2_&DY@lHE-`x zxJ}baJYBJ&p>NO6(j_PuxD5-sbRbSqRhM4F1!^Hs79rONYIZb}T#oVxPAdB%B# z?3zqhFOmXghq9W%fEWS%%JF`sR781y z{sP!nO3uO9`vky)HF=FYKQ#9$s8ef0 z8wBqHgQ`oh@h=-6{rqYwSF12N+98P#$(;PY`TOP1>T^=E!;wogYR<(-Usvc*@2bY_ ztv{y6&(|(%4`K@s1t^qRYWwP{*bPdHxHy`?)c+J!z80@|V3aj0M--T?~o@t?D4rHTaLMsT1;S)y4J*PIox(!Sq$z6J8U^|U5?>^e z>xy`M#j9&pV?ha~|BjAc6lJzW6;uak#%GYfzbbh5$|f6K29k`ngB8cyOVh6PWA;X$ zLYPKSl#Hw$I{975*Wn~QA~_(XurMUis8Y^KJZ_~gbmIJUOlo9(ufv>DA@A?`mEC>% z7%4DUkpS-8dO{Q`udCfvPw1`;jN3>`RGc}*P7oyN1(^Cm^y1`r}7#Qs*x?+rs6R;0XUXTJaRsu>;OaZ%L|*b=(Jxw)j7 zNnq_)bLQ_97w0#$aRlihT6{NBvb9!AXFXDlx_KznAkC_voUivL%t)*IYtUFWYiEl? zLsoM}K(22R>p8NblPhKd5QkLx-JBr|52`fa&U^xB?kWI(zD_dJ)2F^`$APUtYAa!) zjCjQpci6SJCYxQQ5-FR-#{R5*{?h(vmL6X8!JOaeptLXiSqJHK#JFgNSAlY+TuoWn zO%K+5C7DyWAB7h(7B%O;a@u^D48Rf-mK6M&r{CVuU(mb3HCWlA=!O-Bb23EZCODt1 zw?&Rvpk_*|@Ok68JAhDH7mC^ho zrc-_sUk@zRz+711Z~weMo_h6;K~WNcJL7}t!iJOYD6lCEG1NMe;%-JYlfA%8W|RJ3 zA(=HtzAvP9SiRz9sjYkL&v}&Y>~^5)kVm=ELeR|YRnH>F_prFM|9c)QrP2V>D z6ju@yY}Y60S~dW$Wdvmjk5ZaglIJ97tTbkfkuEE(Z$jG*eR7*mF8~QGF(RpH^H}X+~OIYgZquSjMfY zBCI`DI5_B2=Q9iax^H~ZKnZhOvOG>6E?N%R)GOhc-i?l2BLdXh>k%VrssDB#nJ4ZC zsTM&)MH3ACr^NUyOy^W!IcZQ+#xQ%8y6*!u9u23Axi)8~1X&wbD3b+#^{`7A7rcyT^1N1^Yfc6GD(8Y~-%1H6v0N$39JEr2AU zKkvW;+;)~2=gq4gzTr;*J)g!lqnJbUkMuQP;*|tR$mNBtBKQR}3v(8|?1W&RQrmnz zQ-xk;`JuQFIm=Wn+o>ZAKa>6^0DXmyCHii>QSQBsfZ+=V7&&JNS5?Vr%(I(mXm&1J|BIs|8jwPnJGC(y`T=LF>g9=> z^2KjGIsLVqJSeJ%jT zWyOeJTgcg{$c?CrW|A+&hWrczM6#Q?^%JXOAT-Ek{7|3;FAlfmof33^ToEI+?^auN zmHNV(o1-)P4^}dF=CbT{;AClW!BfaKHbuKVHGt+dPs3hUB}e zupa?tQa<`LlzVu2>5c2rXh`QKw0_F0o3s{*oFs76f#b&@jZG-b&ZsUJ%#@5Ebn!W+ z6B092!j_D*2>7MIm7pL`Xj!{RnO9xC1F9)i^1}zf%X+dy!0jm2O+1mn^-CUMwO2J& zF@fsD+5;cd2@;TkH2VnIfCd(k>=L zulr82Mf%00c$(2N*Ppd*7*4pAsB7Vdo)gCDvz4bHz-U*ZLL$>lf;UV+^AZkx^*fIC z_s#M@Kik z5sh(zz5jRVXA%|dh_h4vBNb*x88g8en|Fw>v9ys)!!c0d4K-E)j)K$d{Ym=Pjl69_ zim^>!l8=J2mhFoQy5ZF<9+5?@ZDUJ~H2wNBRO^?Gi*F8!H>#8S4K8z&x>{vzAL%LD z)yf^Bq6jH&r9aj+Gdk)NN3rKYA{tv0W4C*oYBB2^968_PMaO^5Sq0d`3@_UiSR zS5F&Lb8~xYS5gzz;Vz-WE*GIY?WWei?dl@Wy3@q*0?(&j@h{7VkEs;e9`u>x?Zhr(L0N^k3I)B~qI25Lq~T<7jp z*-^sFY3M<8roMhx5nL==M1_iB-?fL<7)qNl<0lFzJxAz_84o+r~JoO#%pj|GiY3ENBG7`PV93f zK{0J{+q#@{B7fseM_~Tap1q45W#f97>Vs4D8;ewLM7^VSJIE^!jfEx6As;q0>}f;& z*hd=_YlJx})JBFL-R{N8DK?O-jQWn|WV5~{*4m2{DesrXnQ_CrPfF2K;bRSy+u(Q~ zCDqJ}jlPT}P$bYsNWX#)Yl8B})7({!5)n57FYdDhk`n6~K8+TXEO|rKa)Q^4&vjBe zsVk35^HsN)I2{Pp#xeNlcl(@w4W#_L#`Z>8duJ&`+?Tm@UUU$}#*BT6xaBf#%F&Rz z-q_T;eh{bRQsG_k=l1F|;rcP9mc`Ed4bjbv&1GDGGvdOA6xx!Ml18|dFi2wA-Upep z)~xu}8|NpenEQWT3yO;!8qcDS79SG;3)$@hn*KF!eM4}&gagHJ_v1> zSG%Cn3-~t`2bHP6${6=W#|k6e9U0>(0YoSqer!7t*stJ4pEKI=#qlIF-?{ESD>)_K eI^AFxT~_lix8{?0)WpcFd(poQCh+sq>i+, @@ -146,7 +146,7 @@ struct JmapImportArgs { long, value_name = "TOKEN", num_args = 0..=1, - help = "Bearer token (value optional: falls back to $VANDELAY_TOKEN or prompt)" + help = "Bearer token (value optional: falls back to $INBUXA_MIGRATE_TOKEN or prompt)" )] auth_bearer: Option>, @@ -199,7 +199,7 @@ struct ImapImportArgs { #[arg( long, value_name = "PASS", - help = "Basic password (prefer $VANDELAY_PASSWORD or prompt)" + help = "Basic password (prefer $INBUXA_MIGRATE_PASSWORD or prompt)" )] auth_password: Option, @@ -207,7 +207,7 @@ struct ImapImportArgs { long, value_name = "TOKEN", num_args = 0..=1, - help = "Bearer token for XOAUTH2 (value optional: falls back to $VANDELAY_TOKEN or prompt)" + help = "Bearer token for XOAUTH2 (value optional: falls back to $INBUXA_MIGRATE_TOKEN or prompt)" )] auth_bearer: Option>, @@ -309,7 +309,7 @@ struct ExportArgs { #[arg( long, value_name = "PASS", - help = "Basic password (prefer $VANDELAY_PASSWORD or prompt)" + help = "Basic password (prefer $INBUXA_MIGRATE_PASSWORD or prompt)" )] auth_password: Option, @@ -317,7 +317,7 @@ struct ExportArgs { long, value_name = "TOKEN", num_args = 0..=1, - help = "Bearer token (value optional: falls back to $VANDELAY_TOKEN or prompt)" + help = "Bearer token (value optional: falls back to $INBUXA_MIGRATE_TOKEN or prompt)" )] auth_bearer: Option>, @@ -407,7 +407,7 @@ pub struct ManageSieveImportArgs { #[arg( long, value_name = "PASS", - help = "Basic password (prefer $VANDELAY_PASSWORD or prompt)" + help = "Basic password (prefer $INBUXA_MIGRATE_PASSWORD or prompt)" )] auth_password: Option, @@ -415,7 +415,7 @@ pub struct ManageSieveImportArgs { long, value_name = "TOKEN", num_args = 0..=1, - help = "Bearer token for OAUTHBEARER (value optional: $VANDELAY_TOKEN or prompt)" + help = "Bearer token for OAUTHBEARER (value optional: $INBUXA_MIGRATE_TOKEN or prompt)" )] auth_bearer: Option>, @@ -457,7 +457,7 @@ pub struct DavImportArgs { #[arg( long, value_name = "PASS", - help = "Basic password (prefer $VANDELAY_PASSWORD or prompt)" + help = "Basic password (prefer $INBUXA_MIGRATE_PASSWORD or prompt)" )] auth_password: Option, @@ -465,7 +465,7 @@ pub struct DavImportArgs { long, value_name = "TOKEN", num_args = 0..=1, - help = "Bearer token (value optional: $VANDELAY_TOKEN or prompt)" + help = "Bearer token (value optional: $INBUXA_MIGRATE_TOKEN or prompt)" )] auth_bearer: Option>, @@ -560,7 +560,7 @@ fn resolve_managesieve_auth( "--auth-user is only valid with --auth-bearer".to_owned(), )); } - let password = secret::resolve(auth_password, "VANDELAY_PASSWORD", "password")?; + let password = secret::resolve(auth_password, "INBUXA_MIGRATE_PASSWORD", "password")?; return Ok(ManageSieveAuth::Basic { user: user.to_owned(), password, @@ -577,7 +577,7 @@ fn resolve_managesieve_auth( let user = auth_user .ok_or_else(|| Error::Usage("--auth-user is required with --auth-bearer".to_owned()))? .to_owned(); - let token = secret::resolve(bearer.as_deref(), "VANDELAY_TOKEN", "bearer token")?; + let token = secret::resolve(bearer.as_deref(), "INBUXA_MIGRATE_TOKEN", "bearer token")?; Ok(ManageSieveAuth::Bearer { user, token }) } @@ -726,7 +726,7 @@ fn resolve_dav_auth( auth_bearer: Option<&Option>, ) -> Result { if let Some(user) = auth_basic { - let password = secret::resolve(auth_password, "VANDELAY_PASSWORD", "password")?; + let password = secret::resolve(auth_password, "INBUXA_MIGRATE_PASSWORD", "password")?; return Ok(DavAuth::Basic { user: user.to_owned(), password, @@ -740,7 +740,7 @@ fn resolve_dav_auth( let bearer = auth_bearer.ok_or_else(|| { Error::Usage("exactly one of --auth-basic / --auth-bearer is required".to_owned()) })?; - let token = secret::resolve(bearer.as_deref(), "VANDELAY_TOKEN", "bearer token")?; + let token = secret::resolve(bearer.as_deref(), "INBUXA_MIGRATE_TOKEN", "bearer token")?; Ok(DavAuth::Bearer { token }) } @@ -828,7 +828,7 @@ fn resolve_imap_auth( "--auth-user is only valid with --auth-bearer".to_owned(), )); } - let password = secret::resolve(auth_password, "VANDELAY_PASSWORD", "password")?; + let password = secret::resolve(auth_password, "INBUXA_MIGRATE_PASSWORD", "password")?; return Ok(ImapAuth::Basic { user: user.to_owned(), password, @@ -845,7 +845,7 @@ fn resolve_imap_auth( let user = auth_user .ok_or_else(|| Error::Usage("--auth-user is required with --auth-bearer".to_owned()))? .to_owned(); - let token = secret::resolve(bearer.as_deref(), "VANDELAY_TOKEN", "bearer token")?; + let token = secret::resolve(bearer.as_deref(), "INBUXA_MIGRATE_TOKEN", "bearer token")?; Ok(ImapAuth::Bearer { user, token }) } @@ -921,7 +921,7 @@ pub struct ExchangeEwsImportArgs { #[arg( long, value_name = "PASS", - help = "Basic password (prefer $VANDELAY_PASSWORD or prompt)" + help = "Basic password (prefer $INBUXA_MIGRATE_PASSWORD or prompt)" )] auth_password: Option, @@ -929,7 +929,7 @@ pub struct ExchangeEwsImportArgs { long, value_name = "TOKEN", num_args = 0..=1, - help = "Bearer token (value optional: $VANDELAY_TOKEN or device-code flow)" + help = "Bearer token (value optional: $INBUXA_MIGRATE_TOKEN or device-code flow)" )] auth_bearer: Option>, @@ -946,7 +946,7 @@ pub struct ExchangeEwsImportArgs { #[arg( long, value_name = "SECRET", - help = "OAuth client secret (app-only flow; prefer $VANDELAY_EWS_CLIENT_SECRET)" + help = "OAuth client secret (app-only flow; prefer $INBUXA_MIGRATE_EWS_CLIENT_SECRET)" )] ews_client_secret: Option, @@ -1029,7 +1029,7 @@ fn resolve_exchange_ews_auth(args: &ExchangeEwsImportArgs) -> Result Result Result Result>, @@ -1168,7 +1168,7 @@ pub struct ExchangeGraphImportArgs { help = "Years either side of today to scan for edited recurrence occurrences (max 5)", long_help = "Years either side of today to scan for edited recurrence occurrences.\n \ Graph reports a series exception only inside an expanded calendar view, and caps\n \ - any single view at five years, so vandelay scans [today - N, today] and\n \ + any single view at five years, so inbuxa-migrate scans [today - N, today] and\n \ [today, today + N]. An occurrence edited outside that span imports with the\n \ series default instead of its edit." )] @@ -1177,7 +1177,7 @@ pub struct ExchangeGraphImportArgs { #[arg( long, help = "Skip contact photos (saves one request per contact on large address books)", - long_help = "Skip contact photos.\n Graph will not say which contacts have a photo: $expand=photo comes back empty\n even when one exists, so the only way to find out is to ask for the bytes.\n vandelay therefore spends one extra request per contact. Pass this to skip\n photos entirely on a large address book." + long_help = "Skip contact photos.\n Graph will not say which contacts have a photo: $expand=photo comes back empty\n even when one exists, so the only way to find out is to ask for the bytes.\n inbuxa-migrate therefore spends one extra request per contact. Pass this to skip\n photos entirely on a large address book." )] skip_contact_photos: bool, @@ -1265,7 +1265,11 @@ fn resolve_graph_auth( tenant: &str, ) -> Result { if let Some(slot) = access_token { - let token = secret::resolve(slot.as_deref(), "VANDELAY_GRAPH_TOKEN", "bearer token")?; + let token = secret::resolve( + slot.as_deref(), + "INBUXA_MIGRATE_GRAPH_TOKEN", + "bearer token", + )?; return Ok(GraphAuth::PreAcquired { token }); } let client_id = client_id @@ -1305,7 +1309,7 @@ fn resolve_auth( auth_bearer: Option<&Option>, ) -> Result { if let Some(user) = auth_basic { - let password = secret::resolve(auth_password, "VANDELAY_PASSWORD", "password")?; + let password = secret::resolve(auth_password, "INBUXA_MIGRATE_PASSWORD", "password")?; return Ok(Auth::Basic { user: user.to_owned(), password, @@ -1319,7 +1323,7 @@ fn resolve_auth( let bearer = auth_bearer.ok_or_else(|| { Error::Usage("exactly one of --auth-basic / --auth-bearer is required".to_owned()) })?; - let token = secret::resolve(bearer.as_deref(), "VANDELAY_TOKEN", "bearer token")?; + let token = secret::resolve(bearer.as_deref(), "INBUXA_MIGRATE_TOKEN", "bearer token")?; Ok(Auth::Bearer { token }) } @@ -1350,8 +1354,8 @@ mod tests { fn auth_password_without_basic_is_usage_error() { let _g = lock(); unsafe { - env::remove_var("VANDELAY_TOKEN"); - env::remove_var("VANDELAY_PASSWORD"); + env::remove_var("INBUXA_MIGRATE_TOKEN"); + env::remove_var("INBUXA_MIGRATE_PASSWORD"); } let bearer: Option = Some("t".to_owned()); let r = resolve_auth(None, Some("hunter2"), Some(&bearer)); @@ -1367,8 +1371,8 @@ mod tests { fn auth_basic_with_password_resolves() { let _g = lock(); unsafe { - env::remove_var("VANDELAY_TOKEN"); - env::remove_var("VANDELAY_PASSWORD"); + env::remove_var("INBUXA_MIGRATE_TOKEN"); + env::remove_var("INBUXA_MIGRATE_PASSWORD"); } let auth = resolve_auth(Some("alice"), Some("pw"), None).unwrap(); match auth { @@ -1384,8 +1388,8 @@ mod tests { fn auth_bearer_inline_token_resolves() { let _g = lock(); unsafe { - env::remove_var("VANDELAY_TOKEN"); - env::remove_var("VANDELAY_PASSWORD"); + env::remove_var("INBUXA_MIGRATE_TOKEN"); + env::remove_var("INBUXA_MIGRATE_PASSWORD"); } let bearer: Option = Some("t".to_owned()); let auth = resolve_auth(None, None, Some(&bearer)).unwrap(); @@ -1394,7 +1398,7 @@ mod tests { fn graph_config(objects: &[&str]) -> Result { let mut argv = vec![ - "vandelay", + "inbuxa-migrate", "import", "exchange-graph", "--access-token", diff --git a/src/dav/client.rs b/src/dav/client.rs index 2766060..0abbbe0 100644 --- a/src/dav/client.rs +++ b/src/dav/client.rs @@ -87,7 +87,7 @@ impl DavClient { log_level: AtomicU8::new(LEVEL_DEFAULT), retries_total: AtomicU64::new(0), retry_after_sleeps: AtomicU64::new(0), - user_agent: format!("vandelay/{}", env!("CARGO_PKG_VERSION")), + user_agent: format!("inbuxa-migrate/{}", env!("CARGO_PKG_VERSION")), }), } } diff --git a/src/exchange/jscalendar.rs b/src/exchange/jscalendar.rs index 0dbf2bf..cec6020 100644 --- a/src/exchange/jscalendar.rs +++ b/src/exchange/jscalendar.rs @@ -36,7 +36,7 @@ const CALENDAR_ADDRESS_DEPENDENTS: &[&str] = &[ pub fn synthetic_attendee_address(identifier: &str) -> String { format!( - "urn:x-vandelay:attendee:{}", + "urn:x-inbuxa-migrate:attendee:{}", blake3::hash(identifier.as_bytes()).to_hex() ) } @@ -146,7 +146,7 @@ mod tests { let event = json!({ "@type": "Event", "uid": "uid-1", - "prodId": "vandelay", + "prodId": "inbuxa-migrate", "privacy": "public", "organizerCalendarAddress": "mailto:alice@example.com", "recurrenceRule": {"frequency": "daily"}, @@ -173,7 +173,7 @@ mod tests { let addr = synthetic_attendee_address("Jane Doe"); assert_eq!(addr, synthetic_attendee_address("Jane Doe")); assert_ne!(addr, synthetic_attendee_address("John Doe")); - assert!(addr.starts_with("urn:x-vandelay:attendee:")); + assert!(addr.starts_with("urn:x-inbuxa-migrate:attendee:")); assert!( !addr.starts_with("mailto:"), "export must never invite a fabricated address" @@ -186,7 +186,7 @@ mod tests { "1": { "@type": "Participant", "name": "Room 4", - "calendarAddress": "urn:x-vandelay:attendee:abc", + "calendarAddress": "urn:x-inbuxa-migrate:attendee:abc", "email": "room4@example.com", "roles": {"required": true}, "kind": "resource", diff --git a/src/exchange_ews/calendar_map.rs b/src/exchange_ews/calendar_map.rs index 0c07345..bc6ad45 100644 --- a/src/exchange_ews/calendar_map.rs +++ b/src/exchange_ews/calendar_map.rs @@ -53,7 +53,7 @@ fn build_event_map(raw: &CalendarItemRaw, iana: Option<&str>) -> Map String { let hash = blake3::hash(item_id.as_bytes()).to_hex(); - format!("vandelay-ews-syn-{hash}") + format!("inbuxa-migrate-ews-syn-{hash}") } pub fn to_jscontact(raw: &ContactItemRaw) -> Value { @@ -389,7 +389,7 @@ mod tests { let u2 = synthetic_uid("ITEM-1"); assert_eq!(u1, u2); assert_ne!(u1, synthetic_uid("ITEM-2")); - assert!(u1.starts_with("vandelay-ews-syn-")); + assert!(u1.starts_with("inbuxa-migrate-ews-syn-")); } #[test] diff --git a/src/exchange_ews/parse.rs b/src/exchange_ews/parse.rs index 0024b54..c14861b 100644 --- a/src/exchange_ews/parse.rs +++ b/src/exchange_ews/parse.rs @@ -724,7 +724,7 @@ fn wrap_inner_with_namespaces(inner: &str) -> String { return String::new(); } format!( - "{inner}", + "{inner}", ) } diff --git a/src/exchange_graph/calendar_map.rs b/src/exchange_graph/calendar_map.rs index c741020..3d7a053 100644 --- a/src/exchange_graph/calendar_map.rs +++ b/src/exchange_graph/calendar_map.rs @@ -701,7 +701,7 @@ mod tests { let p = &conv.data["participants"]["att-1"]; let addr = p["calendarAddress"].as_str().unwrap(); assert!( - addr.starts_with("urn:x-vandelay:attendee:"), + addr.starts_with("urn:x-inbuxa-migrate:attendee:"), "a hybrid X500 reference must not become a malformed mailto, got {addr}" ); assert!( @@ -755,7 +755,7 @@ mod tests { let p = &conv.data["participants"]["att-1"]; let addr = p["calendarAddress"].as_str().unwrap(); assert!( - addr.starts_with("urn:x-vandelay:attendee:"), + addr.starts_with("urn:x-inbuxa-migrate:attendee:"), "a name-only attendee needs a synthetic calendarAddress, got {addr}" ); assert!( @@ -799,7 +799,7 @@ mod tests { v["organizer"] = json!({"emailAddress": {"name": "Alice Example"}}); let conv = convert_event(&v, None).unwrap(); let organizer_address = conv.data["organizerCalendarAddress"].as_str().unwrap(); - assert!(organizer_address.starts_with("urn:x-vandelay:attendee:")); + assert!(organizer_address.starts_with("urn:x-inbuxa-migrate:attendee:")); assert_eq!( conv.data["participants"]["organizer"]["calendarAddress"], organizer_address diff --git a/src/exchange_graph/client.rs b/src/exchange_graph/client.rs index 7840b9e..89b2b47 100644 --- a/src/exchange_graph/client.rs +++ b/src/exchange_graph/client.rs @@ -112,7 +112,7 @@ impl GraphClient { retries_total: AtomicU64::new(0), retry_after_sleeps: AtomicU64::new(0), requests_total: AtomicU64::new(0), - user_agent: format!("vandelay/{}", env!("CARGO_PKG_VERSION")), + user_agent: format!("inbuxa-migrate/{}", env!("CARGO_PKG_VERSION")), }), } } diff --git a/src/exchange_graph/contact_map.rs b/src/exchange_graph/contact_map.rs index 967c7a2..ede4a36 100644 --- a/src/exchange_graph/contact_map.rs +++ b/src/exchange_graph/contact_map.rs @@ -354,7 +354,7 @@ fn has_uri_scheme(value: &str) -> bool { pub fn synthetic_uid(graph_id: &str) -> String { let hash = blake3::hash(graph_id.as_bytes()); - format!("vandelay-graph-{}", hash.to_hex()) + format!("inbuxa-migrate-graph-{}", hash.to_hex()) } fn normalise_utc_datetime(raw: &str) -> String { @@ -465,7 +465,7 @@ mod tests { fn minimal_contact_yields_card_with_synthetic_uid() { let c = json!({"id": "GRAPH-1", "displayName": "Alice"}); let conv = convert_contact(&c).unwrap(); - assert!(conv.uid.starts_with("vandelay-graph-")); + assert!(conv.uid.starts_with("inbuxa-migrate-graph-")); assert_eq!(conv.data["@type"], "Card"); assert_eq!(conv.data["kind"], "individual"); assert_eq!(conv.data["name"]["full"], "Alice"); diff --git a/src/exchange_graph/types.rs b/src/exchange_graph/types.rs index 31bcde0..25cd61d 100644 --- a/src/exchange_graph/types.rs +++ b/src/exchange_graph/types.rs @@ -18,7 +18,7 @@ impl MailboxKind { "primary" => Ok(MailboxKind::Primary), "archive" => Ok(MailboxKind::Archive), "public-folders" => Err(Error::Usage( - "Microsoft Graph does not expose public folders. Run `vandelay import exchange-ews --mailbox-kind public-folders` instead.".to_owned(), + "Microsoft Graph does not expose public folders. Run `inbuxa-migrate import exchange-ews --mailbox-kind public-folders` instead.".to_owned(), )), other => Err(Error::Usage(format!( "--mailbox-kind must be primary | archive, got {other:?}" diff --git a/src/jmap/session.rs b/src/jmap/session.rs index 6a1ccc1..86433bd 100644 --- a/src/jmap/session.rs +++ b/src/jmap/session.rs @@ -139,7 +139,7 @@ impl std::fmt::Display for OriginMismatch { } write!( f, - " on {} (connected to {}); vandelay must use the advertised URL. \ + " on {} (connected to {}); inbuxa-migrate must use the advertised URL. \ If this is wrong, fix the server's advertised HTTP URL setting.", self.session_origin, self.connect_origin ) diff --git a/src/main.rs b/src/main.rs index 9d35884..bcb1baf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,10 +6,10 @@ use clap::Parser; -use vandelay::cli::{Action, Cli}; -use vandelay::error::Error; -use vandelay::inspect; -use vandelay::sync::{self, RunOutcome, Summary}; +use inbuxa_migrate::cli::{Action, Cli}; +use inbuxa_migrate::error::Error; +use inbuxa_migrate::inspect; +use inbuxa_migrate::sync::{self, RunOutcome, Summary}; fn main() { let code = run(); diff --git a/src/sync/import_dav/calcard.rs b/src/sync/import_dav/calcard.rs index fbf9c3e..88b90bb 100644 --- a/src/sync/import_dav/calcard.rs +++ b/src/sync/import_dav/calcard.rs @@ -52,7 +52,7 @@ pub fn vcard_to_jscontact(text: &str, item_href: &str) -> Result String { let hash = blake3::hash(item_href.as_bytes()); - format!("vandelay-syn-{}", hash.to_hex()) + format!("inbuxa-migrate-syn-{}", hash.to_hex()) } #[derive(Debug)] @@ -208,7 +208,7 @@ END:VCALENDAR\r\n"; let href = "/dav/card/u/d/no-uid.vcf"; let jc = vcard_to_jscontact(no_uid, href).expect("synthetic uid"); assert_eq!(jc.uid, synthesise_uid(href)); - assert!(jc.uid.starts_with("vandelay-syn-")); + assert!(jc.uid.starts_with("inbuxa-migrate-syn-")); } #[test] diff --git a/src/sync/import_exchange_ews/coordinator.rs b/src/sync/import_exchange_ews/coordinator.rs index 4ee25cd..1bef8d6 100644 --- a/src/sync/import_exchange_ews/coordinator.rs +++ b/src/sync/import_exchange_ews/coordinator.rs @@ -385,7 +385,7 @@ fn spawn_token_refresher( let client = client.clone(); let initial_refresh = initial.as_ref().and_then(|t| t.refresh_token.clone()); std::thread::Builder::new() - .name("vandelay-ews-token-refresh".to_owned()) + .name("inbuxa-migrate-ews-token-refresh".to_owned()) .spawn(move || { let mut next_exp = exp_secs; let mut refresh_token = initial_refresh; diff --git a/src/sync/import_exchange_graph/coordinator.rs b/src/sync/import_exchange_graph/coordinator.rs index 6e80238..be71758 100644 --- a/src/sync/import_exchange_graph/coordinator.rs +++ b/src/sync/import_exchange_graph/coordinator.rs @@ -395,7 +395,7 @@ fn spawn_token_refresher( let shutdown = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); let shutdown_signal = shutdown.clone(); let handle = std::thread::Builder::new() - .name("vandelay-graph-token-refresh".to_owned()) + .name("inbuxa-migrate-graph-token-refresh".to_owned()) .spawn(move || { while !shutdown_signal.load(std::sync::atomic::Ordering::Relaxed) { let now = unix_now(); diff --git a/src/sync/import_managesieve/coordinator.rs b/src/sync/import_managesieve/coordinator.rs index 71cee22..e92d740 100644 --- a/src/sync/import_managesieve/coordinator.rs +++ b/src/sync/import_managesieve/coordinator.rs @@ -199,7 +199,7 @@ pub fn run(common: CommonConfig, config: ManageSieveImportConfig) -> Result { return Err(Error::Connection(format!( - "ManageSieve referral on GETSCRIPT {name:?}: {e}; vandelay does not follow referrals" + "ManageSieve referral on GETSCRIPT {name:?}: {e}; inbuxa-migrate does not follow referrals" ))); } Disposition::TransportDrop => { @@ -536,7 +536,7 @@ fn listscripts_with_retry( } Disposition::Referral => { return Err(Error::Connection(format!( - "LISTSCRIPTS referral: {e}; vandelay does not follow referrals" + "LISTSCRIPTS referral: {e}; inbuxa-migrate does not follow referrals" ))); } _ => return Err(Error::Connection(format!("LISTSCRIPTS failed: {e}"))), diff --git a/tests/integration/baikal.rs b/tests/integration/baikal.rs index 2ac60dd..4f291d0 100644 --- a/tests/integration/baikal.rs +++ b/tests/integration/baikal.rs @@ -34,7 +34,7 @@ const BAIKAL_YAML: &str = "system: timezone: 'UTC' card_enabled: true cal_enabled: true - invite_from: 'noreply@vandelay.test' + invite_from: 'noreply@inbuxa-migrate.test' dav_auth_type: 'Basic' admin_passwordhash: '004b53ae84286ba6003b34e8ef404826' failed_access_message: 'user %u authentication failure for Baikal' @@ -64,7 +64,7 @@ impl Baikal { pub fn start() -> ContainerResult { let db_bytes = build_sqlite_db()?; - let image: GenericImage = GenericBuildableImage::new("vandelay-baikal", "test") + let image: GenericImage = GenericBuildableImage::new("inbuxa-migrate-baikal", "test") .with_dockerfile_string(DOCKERFILE.to_owned()) .with_data(BAIKAL_YAML.as_bytes().to_vec(), "baikal.yaml") .with_data(db_bytes, "db.sqlite") @@ -325,8 +325,8 @@ fn build_sqlite_db() -> ContainerResult> { ) .map_err(|e| ContainerError::Seed(format!("baikal user {name}: {e}")))?; let uri = format!("principals/{name}"); - let email = format!("{name}@vandelay.test"); - let display = format!("Vandelay {name}"); + let email = format!("{name}@inbuxa-migrate.test"); + let display = format!("Inbuxa {name}"); conn.execute( "INSERT INTO principals (uri, email, displayname) VALUES (?1, ?2, ?3)", params![uri, email, display], diff --git a/tests/integration/cyrus.rs b/tests/integration/cyrus.rs index 29d9afa..8777919 100644 --- a/tests/integration/cyrus.rs +++ b/tests/integration/cyrus.rs @@ -142,7 +142,7 @@ pub struct Cyrus { impl Cyrus { pub fn start() -> ContainerResult { - let image: GenericImage = GenericBuildableImage::new("vandelay-cyrus", "test") + let image: GenericImage = GenericBuildableImage::new("inbuxa-migrate-cyrus", "test") .with_dockerfile_string(DOCKERFILE.to_owned()) .with_data(IMAPD_CONF.as_bytes().to_vec(), "imapd.conf") .with_data(CYRUS_CONF.as_bytes().to_vec(), "cyrus.conf") @@ -313,10 +313,13 @@ impl Cyrus { ) -> ContainerResult<(Vec, String)> { let mut client = ImapSeed::connect(&self.imap.host, self.imap.port)?; client.login(&account.username, &account.password)?; - let message_id = format!("", account.username); + let message_id = format!( + "", + account.username + ); let body = format!( - "From: cyrus-added-{tag}@vandelay.test\r\n\ - To: {}@vandelay.test\r\n\ + "From: cyrus-added-{tag}@inbuxa-migrate.test\r\n\ + To: {}@inbuxa-migrate.test\r\n\ Subject: Cyrus added probe {tag}\r\n\ Message-ID: {message_id}\r\n\ Date: Wed, 01 Jan 2025 12:00:00 +0000\r\n\ @@ -372,10 +375,10 @@ impl Cyrus { } pub fn flag_probe_message() -> Vec { - let body = "From: probe-flags@vandelay.test\r\n\ - To: user@vandelay.test\r\n\ + let body = "From: probe-flags@inbuxa-migrate.test\r\n\ + To: user@inbuxa-migrate.test\r\n\ Subject: Cyrus flag probe\r\n\ - Message-ID: \r\n\ + Message-ID: \r\n\ Date: Wed, 01 Jan 2025 12:00:00 +0000\r\n\ \r\n\ Cyrus flag probe body.\r\n"; diff --git a/tests/integration/data.rs b/tests/integration/data.rs index d34a9f8..6c7e134 100644 --- a/tests/integration/data.rs +++ b/tests/integration/data.rs @@ -89,15 +89,15 @@ fn synth_messages(n: usize) -> Vec { let mut out = Vec::with_capacity(n); for i in 0..n { let body = format!( - "From: sender{i}@vandelay.test\r\n\ - To: user@vandelay.test\r\n\ + "From: sender{i}@inbuxa-migrate.test\r\n\ + To: user@inbuxa-migrate.test\r\n\ Subject: Synthetic test message {i}\r\n\ - Message-ID: \r\n\ + Message-ID: \r\n\ Date: Wed, 01 Jan 2025 12:00:00 +0000\r\n\ MIME-Version: 1.0\r\n\ Content-Type: text/plain; charset=utf-8\r\n\ \r\n\ - This is synthetic seed message #{i} for vandelay container tests.\r\n", + This is synthetic seed message #{i} for inbuxa-migrate container tests.\r\n", ); out.push(MboxMessage { raw: body.into_bytes(), @@ -135,10 +135,10 @@ fn vcard_v3_basic(i: usize) -> String { format!( "BEGIN:VCARD\r\n\ VERSION:3.0\r\n\ - UID:vandelay-card-basic-{i}\r\n\ + UID:inbuxa-migrate-card-basic-{i}\r\n\ FN:Basic Contact {i}\r\n\ N:Contact{i};Basic;;;\r\n\ - EMAIL;TYPE=INTERNET:basic-{i}@vandelay.test\r\n\ + EMAIL;TYPE=INTERNET:basic-{i}@inbuxa-migrate.test\r\n\ END:VCARD\r\n" ) } @@ -147,14 +147,14 @@ fn vcard_v3_full(i: usize) -> String { format!( "BEGIN:VCARD\r\n\ VERSION:3.0\r\n\ - UID:vandelay-card-full-{i}\r\n\ + UID:inbuxa-migrate-card-full-{i}\r\n\ FN:Full Contact {i}\r\n\ N:Contact{i};Full;Middle;;\r\n\ - EMAIL;TYPE=INTERNET;TYPE=WORK:full-{i}@vandelay.test\r\n\ + EMAIL;TYPE=INTERNET;TYPE=WORK:full-{i}@inbuxa-migrate.test\r\n\ TEL;TYPE=CELL:+15550010{i:03}\r\n\ - ORG:Vandelay Test\r\n\ + ORG:Inbuxa Test\r\n\ TITLE:Senior Test Engineer\r\n\ - URL:https://vandelay.test/{i}\r\n\ + URL:https://inbuxa-migrate.test/{i}\r\n\ NOTE:Synthetic full v3 contact #{i}\r\n\ END:VCARD\r\n" ) @@ -164,11 +164,11 @@ fn vcard_v3_multi_email(i: usize) -> String { format!( "BEGIN:VCARD\r\n\ VERSION:3.0\r\n\ - UID:vandelay-card-multi-{i}\r\n\ + UID:inbuxa-migrate-card-multi-{i}\r\n\ FN:Multi Email {i}\r\n\ N:Contact{i};MultiEmail;;;\r\n\ - EMAIL;TYPE=INTERNET;TYPE=HOME:multi-{i}-home@vandelay.test\r\n\ - EMAIL;TYPE=INTERNET;TYPE=WORK:multi-{i}-work@vandelay.test\r\n\ + EMAIL;TYPE=INTERNET;TYPE=HOME:multi-{i}-home@inbuxa-migrate.test\r\n\ + EMAIL;TYPE=INTERNET;TYPE=WORK:multi-{i}-work@inbuxa-migrate.test\r\n\ END:VCARD\r\n" ) } @@ -177,12 +177,12 @@ fn vcard_v3_org_and_title(i: usize) -> String { format!( "BEGIN:VCARD\r\n\ VERSION:3.0\r\n\ - UID:vandelay-card-org-{i}\r\n\ + UID:inbuxa-migrate-card-org-{i}\r\n\ FN:Org Contact {i}\r\n\ N:Contact{i};Org;;Dr.;PhD\r\n\ - ORG:Vandelay Industries;Engineering;Test Group\r\n\ + ORG:Inbuxa Industries;Engineering;Test Group\r\n\ TITLE:Principal Researcher\r\n\ - EMAIL;TYPE=INTERNET:org-{i}@vandelay.test\r\n\ + EMAIL;TYPE=INTERNET:org-{i}@inbuxa-migrate.test\r\n\ END:VCARD\r\n" ) } @@ -194,7 +194,7 @@ fn vcard_v4_basic(i: usize) -> String { UID:urn:uuid:00000000-0000-4000-a000-0000{i:08x}\r\n\ FN:V4 Basic {i}\r\n\ N:Contact{i};V4Basic;;;\r\n\ - EMAIL:v4-basic-{i}@vandelay.test\r\n\ + EMAIL:v4-basic-{i}@inbuxa-migrate.test\r\n\ END:VCARD\r\n" ) } @@ -206,7 +206,7 @@ fn vcard_v4_with_address(i: usize) -> String { UID:urn:uuid:00000000-0000-4000-b000-0000{i:08x}\r\n\ FN:V4 Address {i}\r\n\ N:Contact{i};V4Address;;;\r\n\ - EMAIL:v4-address-{i}@vandelay.test\r\n\ + EMAIL:v4-address-{i}@inbuxa-migrate.test\r\n\ ADR;TYPE=home:;;{i} Test Street;Springfield;OR;97477;US\r\n\ END:VCARD\r\n" ) @@ -220,7 +220,7 @@ fn vcard_v4_nickname(i: usize) -> String { FN:V4 Nick {i}\r\n\ N:Contact{i};V4Nick;;;\r\n\ NICKNAME:Nicky{i}\r\n\ - EMAIL:v4-nick-{i}@vandelay.test\r\n\ + EMAIL:v4-nick-{i}@inbuxa-migrate.test\r\n\ END:VCARD\r\n" ) } @@ -236,7 +236,7 @@ fn vcard_v4_birthday(i: usize) -> String { FN:V4 Birthday {i}\r\n\ N:Contact{i};V4Birthday;;;\r\n\ BDAY:{year:04}{month:02}{day:02}\r\n\ - EMAIL:v4-bday-{i}@vandelay.test\r\n\ + EMAIL:v4-bday-{i}@inbuxa-migrate.test\r\n\ END:VCARD\r\n" ) } @@ -245,10 +245,10 @@ fn vcard_v3_apple_item_labels(i: usize) -> String { format!( "BEGIN:VCARD\r\n\ VERSION:3.0\r\n\ - UID:vandelay-card-ablabels-{i}\r\n\ + UID:inbuxa-migrate-card-ablabels-{i}\r\n\ FN:Apple Labels {i}\r\n\ N:Contact{i};AppleLabels;;;\r\n\ - EMAIL;TYPE=INTERNET:ablabels-{i}@vandelay.test\r\n\ + EMAIL;TYPE=INTERNET:ablabels-{i}@inbuxa-migrate.test\r\n\ ITEM1.X-ABLABEL:Name1\r\n\ ITEM2.X-ABLABEL:Name2\r\n\ ITEM1.X-ABDATE:20171111\r\n\ @@ -287,9 +287,9 @@ fn ical_simple_dt(i: usize) -> String { format!( "BEGIN:VCALENDAR\r\n\ VERSION:2.0\r\n\ - PRODID:-//vandelay//container-test//EN\r\n\ + PRODID:-//inbuxa-migrate//container-test//EN\r\n\ BEGIN:VEVENT\r\n\ - UID:vandelay-simple-{i}@vandelay.test\r\n\ + UID:inbuxa-migrate-simple-{i}@inbuxa-migrate.test\r\n\ DTSTAMP:20250101T120000Z\r\n\ DTSTART:2025{month:02}{day:02}T{hour:02}0000Z\r\n\ DTEND:2025{month:02}{day:02}T{hour:02}3000Z\r\n\ @@ -307,9 +307,9 @@ fn ical_all_day(i: usize) -> String { format!( "BEGIN:VCALENDAR\r\n\ VERSION:2.0\r\n\ - PRODID:-//vandelay//container-test//EN\r\n\ + PRODID:-//inbuxa-migrate//container-test//EN\r\n\ BEGIN:VEVENT\r\n\ - UID:vandelay-allday-{i}@vandelay.test\r\n\ + UID:inbuxa-migrate-allday-{i}@inbuxa-migrate.test\r\n\ DTSTAMP:20250101T120000Z\r\n\ DTSTART;VALUE=DATE:2025{month:02}{day:02}\r\n\ DTEND;VALUE=DATE:2025{month:02}{next_day:02}\r\n\ @@ -325,9 +325,9 @@ fn ical_multi_day_dt(i: usize) -> String { format!( "BEGIN:VCALENDAR\r\n\ VERSION:2.0\r\n\ - PRODID:-//vandelay//container-test//EN\r\n\ + PRODID:-//inbuxa-migrate//container-test//EN\r\n\ BEGIN:VEVENT\r\n\ - UID:vandelay-multiday-{i}@vandelay.test\r\n\ + UID:inbuxa-migrate-multiday-{i}@inbuxa-migrate.test\r\n\ DTSTAMP:20250101T120000Z\r\n\ DTSTART:2025{month:02}{day:02}T090000Z\r\n\ DTEND:2025{month:02}{day:02}T170000Z\r\n\ @@ -344,16 +344,16 @@ fn ical_with_organizer_attendees(i: usize) -> String { format!( "BEGIN:VCALENDAR\r\n\ VERSION:2.0\r\n\ - PRODID:-//vandelay//container-test//EN\r\n\ + PRODID:-//inbuxa-migrate//container-test//EN\r\n\ BEGIN:VEVENT\r\n\ - UID:vandelay-meeting-{i}@vandelay.test\r\n\ + UID:inbuxa-migrate-meeting-{i}@inbuxa-migrate.test\r\n\ DTSTAMP:20250101T120000Z\r\n\ DTSTART:2025{month:02}{day:02}T140000Z\r\n\ DTEND:2025{month:02}{day:02}T150000Z\r\n\ SUMMARY:Sync meeting {i}\r\n\ - ORGANIZER;CN=Organiser:mailto:organiser-{i}@vandelay.test\r\n\ - ATTENDEE;CN=Alice;PARTSTAT=ACCEPTED:mailto:alice-{i}@vandelay.test\r\n\ - ATTENDEE;CN=Bob;PARTSTAT=NEEDS-ACTION:mailto:bob-{i}@vandelay.test\r\n\ + ORGANIZER;CN=Organiser:mailto:organiser-{i}@inbuxa-migrate.test\r\n\ + ATTENDEE;CN=Alice;PARTSTAT=ACCEPTED:mailto:alice-{i}@inbuxa-migrate.test\r\n\ + ATTENDEE;CN=Bob;PARTSTAT=NEEDS-ACTION:mailto:bob-{i}@inbuxa-migrate.test\r\n\ END:VEVENT\r\n\ END:VCALENDAR\r\n" ) @@ -365,15 +365,15 @@ fn ical_with_categories_location(i: usize) -> String { format!( "BEGIN:VCALENDAR\r\n\ VERSION:2.0\r\n\ - PRODID:-//vandelay//container-test//EN\r\n\ + PRODID:-//inbuxa-migrate//container-test//EN\r\n\ BEGIN:VEVENT\r\n\ - UID:vandelay-cat-{i}@vandelay.test\r\n\ + UID:inbuxa-migrate-cat-{i}@inbuxa-migrate.test\r\n\ DTSTAMP:20250101T120000Z\r\n\ DTSTART:2025{month:02}{day:02}T100000Z\r\n\ DTEND:2025{month:02}{day:02}T110000Z\r\n\ SUMMARY:Categorised event {i}\r\n\ LOCATION:Test Lab\\, Room {i}\r\n\ - CATEGORIES:VANDELAY,TEST,FIXTURE\r\n\ + CATEGORIES:INBUXA,TEST,FIXTURE\r\n\ END:VEVENT\r\n\ END:VCALENDAR\r\n" ) @@ -385,9 +385,9 @@ fn ical_recurring_daily(i: usize) -> String { format!( "BEGIN:VCALENDAR\r\n\ VERSION:2.0\r\n\ - PRODID:-//vandelay//container-test//EN\r\n\ + PRODID:-//inbuxa-migrate//container-test//EN\r\n\ BEGIN:VEVENT\r\n\ - UID:vandelay-daily-{i}@vandelay.test\r\n\ + UID:inbuxa-migrate-daily-{i}@inbuxa-migrate.test\r\n\ DTSTAMP:20250101T120000Z\r\n\ DTSTART:2025{month:02}{day:02}T070000Z\r\n\ DTEND:2025{month:02}{day:02}T073000Z\r\n\ @@ -405,9 +405,9 @@ fn ical_recurring_weekly_until(i: usize) -> String { format!( "BEGIN:VCALENDAR\r\n\ VERSION:2.0\r\n\ - PRODID:-//vandelay//container-test//EN\r\n\ + PRODID:-//inbuxa-migrate//container-test//EN\r\n\ BEGIN:VEVENT\r\n\ - UID:vandelay-weekly-{i}@vandelay.test\r\n\ + UID:inbuxa-migrate-weekly-{i}@inbuxa-migrate.test\r\n\ DTSTAMP:20250101T120000Z\r\n\ DTSTART:2025{month:02}{day:02}T160000Z\r\n\ DTEND:2025{month:02}{day:02}T170000Z\r\n\ @@ -424,9 +424,9 @@ fn ical_with_alarm(i: usize) -> String { format!( "BEGIN:VCALENDAR\r\n\ VERSION:2.0\r\n\ - PRODID:-//vandelay//container-test//EN\r\n\ + PRODID:-//inbuxa-migrate//container-test//EN\r\n\ BEGIN:VEVENT\r\n\ - UID:vandelay-alarm-{i}@vandelay.test\r\n\ + UID:inbuxa-migrate-alarm-{i}@inbuxa-migrate.test\r\n\ DTSTAMP:20250101T120000Z\r\n\ DTSTART:2025{month:02}{day:02}T080000Z\r\n\ DTEND:2025{month:02}{day:02}T083000Z\r\n\ @@ -445,9 +445,9 @@ pub fn malformed_ical(name: &str) -> RawFixture { let body = format!( "BEGIN:VCALENDAR\r\n\ VERSION:2.0\r\n\ - PRODID:-//vandelay//broken//EN\r\n\ + PRODID:-//inbuxa-migrate//broken//EN\r\n\ BEGIN:VEVENT\r\n\ - UID:broken-{name}@vandelay.test\r\n\ + UID:broken-{name}@inbuxa-migrate.test\r\n\ DTSTAMP:20250101T120000Z\r\n\ DTSTART:NOT-A-DATE\r\n\ RRULE:FREQ=BOGUS;INTERVAL=oops\r\n\ diff --git a/tests/integration/dovecot.rs b/tests/integration/dovecot.rs index ccb46ee..a1c17cd 100644 --- a/tests/integration/dovecot.rs +++ b/tests/integration/dovecot.rs @@ -126,7 +126,7 @@ pub struct Dovecot { impl Dovecot { pub fn start() -> ContainerResult { - let image: GenericImage = GenericBuildableImage::new("vandelay-dovecot", "test") + let image: GenericImage = GenericBuildableImage::new("inbuxa-migrate-dovecot", "test") .with_dockerfile_string(DOCKERFILE.to_owned()) .with_data(DOVECOT_CONF.as_bytes().to_vec(), "dovecot.conf") .with_data(users_file().into_bytes(), "users") @@ -374,10 +374,10 @@ impl Dovecot { ) -> ContainerResult<(Vec, String)> { let mut client = ImapSeed::connect(&self.imap.host, self.imap.port)?; client.login(&account.username, &account.password)?; - let message_id = format!("", account.username); + let message_id = format!("", account.username); let body = format!( - "From: added-{tag}@vandelay.test\r\n\ - To: {}@vandelay.test\r\n\ + "From: added-{tag}@inbuxa-migrate.test\r\n\ + To: {}@inbuxa-migrate.test\r\n\ Subject: Added probe {tag}\r\n\ Message-ID: {message_id}\r\n\ Date: Wed, 01 Jan 2025 12:00:00 +0000\r\n\ @@ -446,10 +446,10 @@ fn histogram_total(h: &HashMap) -> usize { } pub fn flag_probe_message() -> Vec { - let body = "From: probe-flags@vandelay.test\r\n\ - To: user@vandelay.test\r\n\ + let body = "From: probe-flags@inbuxa-migrate.test\r\n\ + To: user@inbuxa-migrate.test\r\n\ Subject: Flag probe\r\n\ - Message-ID: \r\n\ + Message-ID: \r\n\ Date: Wed, 01 Jan 2025 12:00:00 +0000\r\n\ \r\n\ Flag probe body.\r\n"; @@ -457,8 +457,8 @@ pub fn flag_probe_message() -> Vec { } pub fn no_message_id_probe() -> Vec { - let body = "From: nomid@vandelay.test\r\n\ - To: user@vandelay.test\r\n\ + let body = "From: nomid@inbuxa-migrate.test\r\n\ + To: user@inbuxa-migrate.test\r\n\ Subject: No Message-ID probe\r\n\ Date: Wed, 01 Jan 2025 12:00:00 +0000\r\n\ \r\n\ @@ -467,10 +467,10 @@ pub fn no_message_id_probe() -> Vec { } pub fn shared_message_id_probe() -> Vec { - let body = "From: shared@vandelay.test\r\n\ - To: user@vandelay.test\r\n\ + let body = "From: shared@inbuxa-migrate.test\r\n\ + To: user@inbuxa-migrate.test\r\n\ Subject: Shared MID probe\r\n\ - Message-ID: \r\n\ + Message-ID: \r\n\ Date: Wed, 01 Jan 2025 12:00:00 +0000\r\n\ \r\n\ Shared MID body.\r\n"; diff --git a/tests/integration/layouts.rs b/tests/integration/layouts.rs index 688d41b..686afc4 100644 --- a/tests/integration/layouts.rs +++ b/tests/integration/layouts.rs @@ -63,7 +63,7 @@ pub const ACCOUNT1: &str = "user1"; pub const ACCOUNT2: &str = "user2"; pub const ACCOUNT3: &str = "user3"; -pub const PASSWORD: &str = "VandelayUser#2026"; +pub const PASSWORD: &str = "InbuxaUser#2026"; pub static MAILBOXES_DEEP: &[MailboxSpec] = &[ mailbox!("proj", "Projects", None), diff --git a/tests/integration/mod.rs b/tests/integration/mod.rs index 05c77f1..77f5317 100644 --- a/tests/integration/mod.rs +++ b/tests/integration/mod.rs @@ -22,7 +22,7 @@ pub mod webdav; pub use error::{ContainerError, ContainerResult}; -pub const OWNER_LABEL: &str = "art.stalw.vandelay.itest"; +pub const OWNER_LABEL: &str = "art.stalw.inbuxa-migrate.itest"; use std::sync::Once; diff --git a/tests/integration/validate.rs b/tests/integration/validate.rs index 1de4917..775cf37 100644 --- a/tests/integration/validate.rs +++ b/tests/integration/validate.rs @@ -9,13 +9,13 @@ use std::path::{Path, PathBuf}; use rusqlite::Connection; -use vandelay::logging::Logger; -use vandelay::sync::CommonConfig; +use inbuxa_migrate::logging::Logger; +use inbuxa_migrate::sync::CommonConfig; pub fn tmp_archive(tag: &str) -> PathBuf { let mut p = std::env::temp_dir(); p.push(format!( - "vandelay-container-{tag}-{}-{}.sqlite", + "inbuxa-migrate-container-{tag}-{}-{}.sqlite", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) diff --git a/tests/integration/webdav.rs b/tests/integration/webdav.rs index 15d74c7..eff170d 100644 --- a/tests/integration/webdav.rs +++ b/tests/integration/webdav.rs @@ -137,7 +137,7 @@ pub struct WebDav { impl WebDav { pub fn start() -> ContainerResult { - let image: GenericImage = GenericBuildableImage::new("vandelay-webdav", "test") + let image: GenericImage = GenericBuildableImage::new("inbuxa-migrate-webdav", "test") .with_dockerfile_string(DOCKERFILE.to_owned()) .with_data(APACHE_CONF.as_bytes().to_vec(), "apache.conf") .with_data(htpasswd_for_test().into_bytes(), "htpasswd") diff --git a/tests/integration_baikal.rs b/tests/integration_baikal.rs index 1cc929f..f5f59c1 100644 --- a/tests/integration_baikal.rs +++ b/tests/integration_baikal.rs @@ -16,11 +16,11 @@ use integration::validate::{ contact_card_count, count, open_archive, tmp_archive, }; +use inbuxa_migrate::error::Error; +use inbuxa_migrate::sync::import_dav; +use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; use rusqlite::Connection; use serde_json::Value; -use vandelay::error::Error; -use vandelay::sync::import_dav; -use vandelay::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; fn dav_config(kind: DavKindArg, account: &Account, dav_root: &str) -> DavImportConfig { let path = match kind { diff --git a/tests/integration_cyrus.rs b/tests/integration_cyrus.rs index 6475d0d..b673815 100644 --- a/tests/integration_cyrus.rs +++ b/tests/integration_cyrus.rs @@ -16,9 +16,9 @@ use integration::validate::{ }; use integration::{Account, Endpoint}; -use vandelay::error::Error; -use vandelay::sync::import_imap; -use vandelay::sync::import_imap::{ImapAuth, ImapImportConfig}; +use inbuxa_migrate::error::Error; +use inbuxa_migrate::sync::import_imap; +use inbuxa_migrate::sync::import_imap::{ImapAuth, ImapImportConfig}; fn imap_config(account: &Account, imap: &Endpoint) -> ImapImportConfig { ImapImportConfig { @@ -327,10 +327,10 @@ fn cyrus_non_ascii_mailbox_names_round_trip() { let mutf7_name = "Envoy&AOk-s"; let utf8_name = "Envoyés"; - let message = b"From: accents@vandelay.test\r\n\ - To: user1@vandelay.test\r\n\ + let message = b"From: accents@inbuxa-migrate.test\r\n\ + To: user1@inbuxa-migrate.test\r\n\ Subject: Accents\r\n\ - Message-ID: \r\n\ + Message-ID: \r\n\ Date: Wed, 01 Jan 2025 12:00:00 +0000\r\n\ \r\n\ Accented folder probe.\r\n"; diff --git a/tests/integration_dovecot.rs b/tests/integration_dovecot.rs index f99e8ba..503bdcc 100644 --- a/tests/integration_dovecot.rs +++ b/tests/integration_dovecot.rs @@ -16,14 +16,14 @@ use integration::validate::{ mailbox_id_by_path, mailbox_path, open_archive, tmp_archive, }; +use inbuxa_migrate::error::Error; +use inbuxa_migrate::imap::client::{ConnectMode, ImapClient}; +use inbuxa_migrate::imap::transport::Connector; +use inbuxa_migrate::logging::Logger; +use inbuxa_migrate::sync::import_imap::{ImapAuth, ImapImportConfig}; +use inbuxa_migrate::sync::import_managesieve::{ManageSieveAuth, ManageSieveImportConfig}; +use inbuxa_migrate::sync::{import_imap, import_managesieve}; use rusqlite::Connection; -use vandelay::error::Error; -use vandelay::imap::client::{ConnectMode, ImapClient}; -use vandelay::imap::transport::Connector; -use vandelay::logging::Logger; -use vandelay::sync::import_imap::{ImapAuth, ImapImportConfig}; -use vandelay::sync::import_managesieve::{ManageSieveAuth, ManageSieveImportConfig}; -use vandelay::sync::{import_imap, import_managesieve}; fn imap_config(account: &Account, imap: &integration::Endpoint) -> ImapImportConfig { ImapImportConfig { @@ -526,10 +526,10 @@ fn dovecot_non_ascii_mailbox_names_round_trip() { let mutf7_name = "Envoy&AOk-s"; let utf8_name = "Envoyés"; - let message = b"From: accents@vandelay.test\r\n\ - To: user1@vandelay.test\r\n\ + let message = b"From: accents@inbuxa-migrate.test\r\n\ + To: user1@inbuxa-migrate.test\r\n\ Subject: Accents\r\n\ - Message-ID: \r\n\ + Message-ID: \r\n\ Date: Wed, 01 Jan 2025 12:00:00 +0000\r\n\ \r\n\ Accented folder probe.\r\n"; diff --git a/tests/integration_radicale.rs b/tests/integration_radicale.rs index 97bc139..91a108e 100644 --- a/tests/integration_radicale.rs +++ b/tests/integration_radicale.rs @@ -16,11 +16,11 @@ use integration::validate::{ }; use integration::{Account, Endpoint}; +use inbuxa_migrate::error::Error; +use inbuxa_migrate::sync::import_dav; +use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; use rusqlite::Connection; use serde_json::Value; -use vandelay::error::Error; -use vandelay::sync::import_dav; -use vandelay::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; fn dav_config(kind: DavKindArg, account: &Account, http: &Endpoint) -> DavImportConfig { DavImportConfig { diff --git a/tests/integration_webdav.rs b/tests/integration_webdav.rs index dd4a0f5..ed3ff13 100644 --- a/tests/integration_webdav.rs +++ b/tests/integration_webdav.rs @@ -15,9 +15,9 @@ use integration::validate::{ use integration::webdav::{AccountSeed, WebDav}; use integration::{Account, Endpoint}; -use vandelay::error::Error; -use vandelay::sync::import_dav; -use vandelay::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; +use inbuxa_migrate::error::Error; +use inbuxa_migrate::sync::import_dav; +use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; fn webdav_config(account: &Account, http: &Endpoint) -> DavImportConfig { DavImportConfig { @@ -319,7 +319,7 @@ fn webdav_starts_seeds_and_imports() { drop(conn); let new_payload = format!( - "added-{} content for vandelay webdav add-after-import probe\n", + "added-{} content for inbuxa-migrate webdav add-after-import probe\n", seed.username ); let new_payload_bytes = new_payload.as_bytes().to_vec(); diff --git a/tests/live_stalwart.rs b/tests/live_stalwart.rs index d990d91..e4de292 100644 --- a/tests/live_stalwart.rs +++ b/tests/live_stalwart.rs @@ -7,10 +7,10 @@ mod integration; mod seeder; +use inbuxa_migrate::jmap::account::{self, AccountSelector}; +use inbuxa_migrate::jmap::http::{Auth, HttpClient, RetryPolicy}; +use inbuxa_migrate::jmap::session::Session; use integration::stalwart::shared as shared_stalwart; -use vandelay::jmap::account::{self, AccountSelector}; -use vandelay::jmap::http::{Auth, HttpClient, RetryPolicy}; -use vandelay::jmap::session::Session; fn admin_client() -> HttpClient { HttpClient::new( diff --git a/tests/mock_dav.rs b/tests/mock_dav.rs index 54cf59c..c04f513 100644 --- a/tests/mock_dav.rs +++ b/tests/mock_dav.rs @@ -6,12 +6,12 @@ use std::io::Cursor; -use vandelay::dav::client::DavClient; -use vandelay::dav::discover::{DavKind, discover}; -use vandelay::dav::parse::{parse_multistatus, strip_ascii_control_chars}; -use vandelay::dav::xml; -use vandelay::jmap::error::JmapError; -use vandelay::jmap::http::{Auth, RetryPolicy}; +use inbuxa_migrate::dav::client::DavClient; +use inbuxa_migrate::dav::discover::{DavKind, discover}; +use inbuxa_migrate::dav::parse::{parse_multistatus, strip_ascii_control_chars}; +use inbuxa_migrate::dav::xml; +use inbuxa_migrate::jmap::error::JmapError; +use inbuxa_migrate::jmap::http::{Auth, RetryPolicy}; fn client(retries: u32) -> DavClient { DavClient::new( @@ -75,7 +75,7 @@ fn discovery_resolves_per_user_principal_even_when_url_lists_collections() { r#" - {url}/dav/cal/secondary@vandelay.org/default/ + {url}/dav/cal/secondary@inbuxa-migrate.org/default/ @@ -101,7 +101,7 @@ fn discovery_resolves_per_user_principal_even_when_url_lists_collections() { {url}/dav/cal/ - {url}/dav/principals/secondary@vandelay.org/ + {url}/dav/principals/secondary@inbuxa-migrate.org/ HTTP/1.1 200 OK @@ -121,7 +121,7 @@ fn discovery_resolves_per_user_principal_even_when_url_lists_collections() { assert_eq!(disc.collections.len(), 1); assert_eq!( disc.principal_url.as_deref(), - Some(format!("{url}/dav/principals/secondary@vandelay.org/").as_str()), + Some(format!("{url}/dav/principals/secondary@inbuxa-migrate.org/").as_str()), "account identity must be the per-user principal, not the shared base DAV root" ); } @@ -212,7 +212,7 @@ fn discovery_returns_not_found_when_no_collections() { let err = discover(&c, DavKind::Caldav, &server.url()).unwrap_err(); assert!(matches!( err, - vandelay::dav::discover::DiscoveryError::NotFound { .. } + inbuxa_migrate::dav::discover::DiscoveryError::NotFound { .. } )); } @@ -489,14 +489,14 @@ fn webdav_listing_returns_collection_and_file() { #[test] fn google_usage_limits_403_classified_retryable() { - use vandelay::dav::retry::{DavOutcome, classify}; + use inbuxa_migrate::dav::retry::{DavOutcome, classify}; let body = br#"{"error":{"errors":[{"domain":"usageLimits"}]}}"#; assert_eq!(classify(403, body), DavOutcome::Retryable); } #[test] fn non_quota_403_is_auth() { - use vandelay::dav::retry::{DavOutcome, classify}; + use inbuxa_migrate::dav::retry::{DavOutcome, classify}; assert_eq!(classify(403, b"forbidden"), DavOutcome::Auth); } @@ -801,7 +801,7 @@ fn webdav_cycle_breaks_on_lex_smallest() { #[test] fn multiget_405_method_not_allowed_triggers_get_fallback() { - use vandelay::dav::retry::{DavOutcome, classify}; + use inbuxa_migrate::dav::retry::{DavOutcome, classify}; assert_eq!(classify(405, b""), DavOutcome::Fatal); } @@ -893,8 +893,8 @@ fn streaming_propfind_404_returns_empty_responses() { #[test] fn source_change_protection_rejects_different_account_id() { + use inbuxa_migrate::db::{init, sources}; use rusqlite::Connection; - use vandelay::db::{init, sources}; let conn = Connection::open_in_memory().unwrap(); init::apply_schema(&conn).unwrap(); @@ -1049,15 +1049,15 @@ fn webdav_discovery_keeps_only_self_row_as_root() { #[test] fn webdav_root_collection_is_not_materialised_children_map_to_target_root() { + use inbuxa_migrate::dav::discover::DiscoveredCollection; + use inbuxa_migrate::dav::href::Href; + use inbuxa_migrate::dav::parse::ResourceProps; + use inbuxa_migrate::db; + use inbuxa_migrate::db::sources::SourceKey; + use inbuxa_migrate::logging::Logger; + use inbuxa_migrate::sync::TypeCounts; + use inbuxa_migrate::sync::import_dav::tree::{WebDavCtx, reconcile_filenodes}; use rusqlite::Connection; - use vandelay::dav::discover::DiscoveredCollection; - use vandelay::dav::href::Href; - use vandelay::dav::parse::ResourceProps; - use vandelay::db; - use vandelay::db::sources::SourceKey; - use vandelay::logging::Logger; - use vandelay::sync::TypeCounts; - use vandelay::sync::import_dav::tree::{WebDavCtx, reconcile_filenodes}; let mut server = mockito::Server::new(); let url = server.url(); @@ -1336,10 +1336,10 @@ fn webdav_display_or_basename_test_via_parser_keeps_basename_semantics() { #[test] fn dry_run_writes_nothing_but_emits_per_collection_counts() { + use inbuxa_migrate::logging::Logger; + use inbuxa_migrate::sync::CommonConfig; + use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, run}; use std::path::PathBuf; - use vandelay::logging::Logger; - use vandelay::sync::CommonConfig; - use vandelay::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, run}; let mut server = mockito::Server::new(); let url = server.url(); @@ -1381,7 +1381,7 @@ fn dry_run_writes_nothing_but_emits_per_collection_counts() { let _items = multistatus_response(&mut server, "PROPFIND", "/dav/cal/u/default/", &items_body); let archive: PathBuf = std::env::temp_dir().join(format!( - "vandelay-dryrun-{}-{}.sqlite", + "inbuxa-migrate-dryrun-{}-{}.sqlite", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -1449,10 +1449,10 @@ fn dry_run_writes_nothing_but_emits_per_collection_counts() { #[test] fn dav_source_change_protection_fires_across_users_on_same_root() { use encodify::base64::STANDARD; + use inbuxa_migrate::logging::Logger; + use inbuxa_migrate::sync::CommonConfig; + use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, run}; use std::path::PathBuf; - use vandelay::logging::Logger; - use vandelay::sync::CommonConfig; - use vandelay::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, run}; let mut server = mockito::Server::new(); let url = server.url(); @@ -1499,8 +1499,14 @@ fn dav_source_change_protection_fires_across_users_on_same_root() { .with_body(&empty_items) .create(); - let auth_a = format!("Basic {}", STANDARD.encode("secondary@vandelay.org:passA")); - let auth_b = format!("Basic {}", STANDARD.encode("tertiary@vandelay.org:passB")); + let auth_a = format!( + "Basic {}", + STANDARD.encode("secondary@inbuxa-migrate.org:passA") + ); + let auth_b = format!( + "Basic {}", + STANDARD.encode("tertiary@inbuxa-migrate.org:passB") + ); let principal = |who: &str| { format!( r#" @@ -1521,7 +1527,7 @@ fn dav_source_change_protection_fires_across_users_on_same_root() { .match_header("authorization", auth_a.as_str()) .with_status(207) .with_header("content-type", "application/xml; charset=utf-8") - .with_body(principal("secondary@vandelay.org")) + .with_body(principal("secondary@inbuxa-migrate.org")) .create(); let _pb = server .mock("PROPFIND", "/dav/cal/") @@ -1529,11 +1535,11 @@ fn dav_source_change_protection_fires_across_users_on_same_root() { .match_header("authorization", auth_b.as_str()) .with_status(207) .with_header("content-type", "application/xml; charset=utf-8") - .with_body(principal("tertiary@vandelay.org")) + .with_body(principal("tertiary@inbuxa-migrate.org")) .create(); let archive: PathBuf = std::env::temp_dir().join(format!( - "vandelay-dav-srcchange-{}-{}.sqlite", + "inbuxa-migrate-dav-srcchange-{}-{}.sqlite", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -1563,19 +1569,27 @@ fn dav_source_change_protection_fires_across_users_on_same_root() { allow_source_change: false, }; - run(common(&archive), config("secondary@vandelay.org", "passA")).expect("user A import ok"); - let err = run(common(&archive), config("tertiary@vandelay.org", "passB")).unwrap_err(); + run( + common(&archive), + config("secondary@inbuxa-migrate.org", "passA"), + ) + .expect("user A import ok"); + let err = run( + common(&archive), + config("tertiary@inbuxa-migrate.org", "passB"), + ) + .unwrap_err(); let _ = std::fs::remove_file(&archive); assert!( - matches!(err, vandelay::error::Error::SourceChange(_)), + matches!(err, inbuxa_migrate::error::Error::SourceChange(_)), "importing a different user into the same archive must trigger source-change protection; got {err:?}" ); } #[test] fn source_change_protection_rejects_different_session_url_same_account() { + use inbuxa_migrate::db::{init, sources}; use rusqlite::Connection; - use vandelay::db::{init, sources}; let conn = Connection::open_in_memory().unwrap(); init::apply_schema(&conn).unwrap(); @@ -1627,8 +1641,8 @@ fn etag_missing_in_enumeration_propagates_to_storage_layer() { #[test] fn mixed_source_archive_allows_distinct_kinds_against_same_url() { + use inbuxa_migrate::db::{init, sources}; use rusqlite::Connection; - use vandelay::db::{init, sources}; let conn = Connection::open_in_memory().unwrap(); init::apply_schema(&conn).unwrap(); @@ -1663,7 +1677,7 @@ fn dav_archive() -> std::path::PathBuf { let n = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); let mut p = std::env::temp_dir(); p.push(format!( - "vandelay-mockdav-{}-{:?}-{n}.sqlite", + "inbuxa-migrate-mockdav-{}-{:?}-{n}.sqlite", std::process::id(), std::thread::current().id(), )); @@ -1671,11 +1685,11 @@ fn dav_archive() -> std::path::PathBuf { p } -fn caldav_import_config(url: &str) -> vandelay::sync::import_dav::DavImportConfig { - vandelay::sync::import_dav::DavImportConfig { - kind: vandelay::sync::import_dav::DavKindArg::Caldav, +fn caldav_import_config(url: &str) -> inbuxa_migrate::sync::import_dav::DavImportConfig { + inbuxa_migrate::sync::import_dav::DavImportConfig { + kind: inbuxa_migrate::sync::import_dav::DavKindArg::Caldav, url: url.to_owned(), - auth: vandelay::sync::import_dav::DavAuth::Basic { + auth: inbuxa_migrate::sync::import_dav::DavAuth::Basic { user: "u".to_owned(), password: "p".to_owned(), }, @@ -1686,14 +1700,14 @@ fn caldav_import_config(url: &str) -> vandelay::sync::import_dav::DavImportConfi } } -fn dav_common(archive: &std::path::Path) -> vandelay::sync::CommonConfig { - vandelay::sync::CommonConfig { +fn dav_common(archive: &std::path::Path) -> inbuxa_migrate::sync::CommonConfig { + inbuxa_migrate::sync::CommonConfig { archive: archive.to_path_buf(), threads: 1, dry_run: false, max_retries: 0, allow_invalid_certs: false, - logger: vandelay::logging::Logger::from_flags(true, 0), + logger: inbuxa_migrate::logging::Logger::from_flags(true, 0), } } @@ -1788,7 +1802,7 @@ END:VCALENDAR .create(); let archive = dav_archive(); - let summary = vandelay::sync::import_dav::run( + let summary = inbuxa_migrate::sync::import_dav::run( dav_common(&archive), caldav_import_config(&format!("{url}/dav/cal/u/")), ) @@ -1828,14 +1842,14 @@ fn credentials_rejected_during_discovery_still_aborts_the_run() { .create(); let archive = dav_archive(); - let err = vandelay::sync::import_dav::run( + let err = inbuxa_migrate::sync::import_dav::run( dav_common(&archive), caldav_import_config(&format!("{url}/dav/cal/u/")), ) .expect_err("credentials rejected for the principal aborts the run"); assert!( - matches!(err, vandelay::error::Error::Connection(_)), + matches!(err, inbuxa_migrate::error::Error::Connection(_)), "discovery-level auth rejection is a whole-run failure, got {err:?}" ); assert_eq!(err.exit_code(), 2); @@ -1871,7 +1885,7 @@ fn dav_import_reports_both_phases_when_every_collection_fails() { .create(); let archive = dav_archive(); - let outcome = vandelay::sync::import_dav::run_reporting( + let outcome = inbuxa_migrate::sync::import_dav::run_reporting( dav_common(&archive), caldav_import_config(&format!("{url}/dav/cal/u/")), ); diff --git a/tests/mock_exchange_ews.rs b/tests/mock_exchange_ews.rs index f4dd7ac..11fce23 100644 --- a/tests/mock_exchange_ews.rs +++ b/tests/mock_exchange_ews.rs @@ -5,21 +5,23 @@ */ use encodify::base64::{LENIENT, STANDARD}; -use mockito::Matcher; -use vandelay::exchange_ews::EwsClient; -use vandelay::exchange_ews::autodiscover::{DiscoverySource, discover}; -use vandelay::exchange_ews::error::EwsError; -use vandelay::exchange_ews::parse::{ +use inbuxa_migrate::exchange_ews::EwsClient; +use inbuxa_migrate::exchange_ews::autodiscover::{DiscoverySource, discover}; +use inbuxa_migrate::exchange_ews::error::EwsError; +use inbuxa_migrate::exchange_ews::parse::{ EnvelopeKind, parse_find_folder_response, parse_find_item_response, parse_get_attachment_inline, parse_response_messages, parse_sync_folder_items_response, read_envelope_summary, }; -use vandelay::exchange_ews::types::{FolderClass, FolderId, ItemId, ResponseCode, ServerVersion}; -use vandelay::exchange_ews::xml::{ +use inbuxa_migrate::exchange_ews::types::{ + FolderClass, FolderId, ItemId, ResponseCode, ServerVersion, +}; +use inbuxa_migrate::exchange_ews::xml::{ FolderRef, ItemShape, Traversal, find_folder_body, find_item_body, get_attachment_body, get_item_body, sync_folder_items_body, }; -use vandelay::jmap::http::{Auth, RetryPolicy}; +use inbuxa_migrate::jmap::http::{Auth, RetryPolicy}; +use mockito::Matcher; const TXT_XML: &str = "text/xml; charset=utf-8"; const APP_JSON: &str = "application/json"; @@ -59,7 +61,7 @@ fn autodiscover_v2_returns_global_endpoint() { .create(); let _ = server; let url = "https://outlook.office365.com/EWS/Exchange.asmx"; - assert!(vandelay::exchange_ews::autodiscover::is_fully_qualified_ews_url(url)); + assert!(inbuxa_migrate::exchange_ews::autodiscover::is_fully_qualified_ews_url(url)); let r = discover(Some(url), None, None, false).unwrap(); assert_eq!(r.source, DiscoverySource::SuppliedUrl); assert_eq!(r.ews_url, url); @@ -265,7 +267,7 @@ fn server_busy_with_back_off_triggers_retry_and_eventually_succeeds() { let c = client(3); let body = find_folder_body( FolderRef::Distinguished( - vandelay::exchange_ews::types::DistinguishedFolderId::MsgFolderRoot, + inbuxa_migrate::exchange_ews::types::DistinguishedFolderId::MsgFolderRoot, ), Traversal::Deep, ); @@ -290,7 +292,7 @@ fn http_401_surfaces_as_auth_error() { let c = client(0); let body = find_folder_body( FolderRef::Distinguished( - vandelay::exchange_ews::types::DistinguishedFolderId::MsgFolderRoot, + inbuxa_migrate::exchange_ews::types::DistinguishedFolderId::MsgFolderRoot, ), Traversal::Deep, ); @@ -313,7 +315,7 @@ fn mime_content_round_trips_through_base64_decode() { " )); let r = parse_response_messages(body.as_bytes(), "GetItemResponseMessage").unwrap(); - let item = vandelay::exchange_ews::parse::parse_message_item(&r[0].inner_xml).unwrap(); + let item = inbuxa_migrate::exchange_ews::parse::parse_message_item(&r[0].inner_xml).unwrap(); let s = item.mime_content.unwrap(); let bytes = LENIENT.decode(&s).unwrap(); assert_eq!(bytes, original); @@ -341,7 +343,7 @@ fn get_attachment_inline_decodes_photo_blob() { #[test] fn calendar_item_master_inlines_modified_and_deleted_occurrences() { - let body = "\ + let body = "\ \ \ Daily\ @@ -359,8 +361,8 @@ fn calendar_item_master_inlines_modified_and_deleted_occurrences() { \ 2025-06-17T14:00:00Z\ \ - "; - let item = vandelay::exchange_ews::parse::parse_calendar_item(body).unwrap(); + "; + let item = inbuxa_migrate::exchange_ews::parse::parse_calendar_item(body).unwrap(); assert_eq!(item.id.id, "M1"); assert_eq!(item.uid.as_deref(), Some("uid-1")); assert_eq!(item.modified_occurrences.len(), 1); @@ -373,7 +375,7 @@ fn calendar_item_master_inlines_modified_and_deleted_occurrences() { #[test] fn organizer_and_attendee_addresses_survive_an_ex_routing_type() { - let body = "\ + let body = "\ \ \ Review\ @@ -386,9 +388,9 @@ fn organizer_and_attendee_addresses_survive_an_ex_routing_type() { Kristina Morgental\ /o=ExchangeLabs/ou=Exchange Administrative Group (FYDIBOHF23SPDLT)/cn=Recipients/cn=bdc77b18152647a29d28ce1188376dc9-kristina\ EXUnknown\ - "; - let item = vandelay::exchange_ews::parse::parse_calendar_item(body).unwrap(); - let event = vandelay::exchange_ews::calendar_map::to_jscalendar(&item); + "; + let item = inbuxa_migrate::exchange_ews::parse::parse_calendar_item(body).unwrap(); + let event = inbuxa_migrate::exchange_ews::calendar_map::to_jscalendar(&item); assert_eq!( event.data["organizerCalendarAddress"], "mailto:alice@example.com", @@ -409,7 +411,7 @@ fn organizer_and_attendee_addresses_survive_an_ex_routing_type() { attendee["calendarAddress"] .as_str() .unwrap() - .starts_with("urn:x-vandelay:attendee:"), + .starts_with("urn:x-inbuxa-migrate:attendee:"), "a legacy directory reference cannot be resolved and stays synthetic" ); assert!( @@ -420,7 +422,7 @@ fn organizer_and_attendee_addresses_survive_an_ex_routing_type() { #[test] fn recurrence_end_date_with_a_timezone_offset_yields_a_bounded_series() { - let body = "\ + let body = "\ \ \ Biweekly\ @@ -432,9 +434,9 @@ fn recurrence_end_date_with_a_timezone_offset_yields_a_bounded_series() { 2Wednesday\ 2021-08-04-06:002021-09-30-06:00\ \ - "; - let item = vandelay::exchange_ews::parse::parse_calendar_item(body).unwrap(); - let event = vandelay::exchange_ews::calendar_map::to_jscalendar(&item); + "; + let item = inbuxa_migrate::exchange_ews::parse::parse_calendar_item(body).unwrap(); + let event = inbuxa_migrate::exchange_ews::calendar_map::to_jscalendar(&item); let rule = &event.data["recurrenceRule"]; assert_eq!(rule["frequency"], "weekly"); assert_eq!( @@ -486,8 +488,8 @@ fn invalid_sync_state_data_fault_is_surfaced() { #[test] fn coordinator_diff_classifies_new_vanished_changed_unchanged() { - use vandelay::db::exchange_ews_ids::ItemRow; - use vandelay::sync::import_exchange_ews::items::{EnumeratedItem, diff}; + use inbuxa_migrate::db::exchange_ews_ids::ItemRow; + use inbuxa_migrate::sync::import_exchange_ews::items::{EnumeratedItem, diff}; let server = vec![ EnumeratedItem { @@ -532,9 +534,9 @@ fn coordinator_diff_classifies_new_vanished_changed_unchanged() { #[test] fn source_change_protection_refuses_different_account() { + use inbuxa_migrate::db; + use inbuxa_migrate::db::sources::SourceKey; use rusqlite::Connection; - use vandelay::db; - use vandelay::db::sources::SourceKey; let tmp = tempfile::NamedTempFile::new().unwrap(); let conn = Connection::open(tmp.path()).unwrap(); @@ -557,9 +559,9 @@ fn source_change_protection_refuses_different_account() { #[test] fn mailbox_kinds_are_three_separate_sources() { + use inbuxa_migrate::db; + use inbuxa_migrate::db::sources::SourceKey; use rusqlite::Connection; - use vandelay::db; - use vandelay::db::sources::SourceKey; let tmp = tempfile::NamedTempFile::new().unwrap(); let conn = Connection::open(tmp.path()).unwrap(); @@ -653,7 +655,7 @@ fn http_500_with_server_busy_body_is_treated_as_fault_and_retried() { let c = client(3); let body = find_folder_body( FolderRef::Distinguished( - vandelay::exchange_ews::types::DistinguishedFolderId::MsgFolderRoot, + inbuxa_migrate::exchange_ews::types::DistinguishedFolderId::MsgFolderRoot, ), Traversal::Deep, ); @@ -710,7 +712,7 @@ fn invalid_server_version_fault_downgrades_and_succeeds() { assert_eq!(c.server_version(), ServerVersion::Exchange2013Sp1); let body = find_folder_body( FolderRef::Distinguished( - vandelay::exchange_ews::types::DistinguishedFolderId::MsgFolderRoot, + inbuxa_migrate::exchange_ews::types::DistinguishedFolderId::MsgFolderRoot, ), Traversal::Deep, ); @@ -748,7 +750,7 @@ fn unsupported_version_floor_surfaces_as_soap_fault() { let c = client(0); let body = find_folder_body( FolderRef::Distinguished( - vandelay::exchange_ews::types::DistinguishedFolderId::MsgFolderRoot, + inbuxa_migrate::exchange_ews::types::DistinguishedFolderId::MsgFolderRoot, ), Traversal::Deep, ); @@ -801,7 +803,7 @@ fn server_affinity_cookie_is_captured_and_resent() { let c = client(0); let body = find_folder_body( FolderRef::Distinguished( - vandelay::exchange_ews::types::DistinguishedFolderId::MsgFolderRoot, + inbuxa_migrate::exchange_ews::types::DistinguishedFolderId::MsgFolderRoot, ), Traversal::Deep, ); @@ -823,7 +825,7 @@ fn http_456_surfaces_as_account_locked_auth_error() { let c = client(0); let body = find_folder_body( FolderRef::Distinguished( - vandelay::exchange_ews::types::DistinguishedFolderId::MsgFolderRoot, + inbuxa_migrate::exchange_ews::types::DistinguishedFolderId::MsgFolderRoot, ), Traversal::Deep, ); @@ -836,8 +838,8 @@ fn http_456_surfaces_as_account_locked_auth_error() { #[test] fn for_each_fetched_item_streams_every_id_across_windows() { - use vandelay::logging::Logger; - use vandelay::sync::import_exchange_ews::items::{ItemRunCtx, for_each_fetched_item}; + use inbuxa_migrate::logging::Logger; + use inbuxa_migrate::sync::import_exchange_ews::items::{ItemRunCtx, for_each_fetched_item}; let mut server = mockito::Server::new(); let url = format!("{}/EWS/Exchange.asmx", server.url()); diff --git a/tests/mock_exchange_graph.rs b/tests/mock_exchange_graph.rs index bef2075..6ee20c6 100644 --- a/tests/mock_exchange_graph.rs +++ b/tests/mock_exchange_graph.rs @@ -7,21 +7,21 @@ use std::sync::Once; use encodify::base64::URL_SAFE_NO_PAD; -use mockito::{Matcher, Server}; -use serde_json::json; -use vandelay::exchange_graph::api::{Endpoints, collect_all_ids, paged_collect}; -use vandelay::exchange_graph::calendar_map::{EventType, classify_event_type, convert_event}; -use vandelay::exchange_graph::client::{Accept, GraphClient}; -use vandelay::exchange_graph::contact_map::convert_contact; -use vandelay::exchange_graph::error::GraphError; -use vandelay::exchange_graph::oauth::{ +use inbuxa_migrate::exchange_graph::api::{Endpoints, collect_all_ids, paged_collect}; +use inbuxa_migrate::exchange_graph::calendar_map::{EventType, classify_event_type, convert_event}; +use inbuxa_migrate::exchange_graph::client::{Accept, GraphClient}; +use inbuxa_migrate::exchange_graph::contact_map::convert_contact; +use inbuxa_migrate::exchange_graph::error::GraphError; +use inbuxa_migrate::exchange_graph::oauth::{ DeviceCodeResponse, TokenResponse, parse_device_code_response, parse_token_response, run_device_code_polling_against, }; -use vandelay::exchange_graph::recurrence::convert_patterned_recurrence; -use vandelay::exchange_graph::retry::{HttpClass, classify_http_status}; -use vandelay::exchange_graph::types::Surfaces; -use vandelay::jmap::http::RetryPolicy; +use inbuxa_migrate::exchange_graph::recurrence::convert_patterned_recurrence; +use inbuxa_migrate::exchange_graph::retry::{HttpClass, classify_http_status}; +use inbuxa_migrate::exchange_graph::types::Surfaces; +use inbuxa_migrate::jmap::http::RetryPolicy; +use mockito::{Matcher, Server}; +use serde_json::json; static INIT: Once = Once::new(); @@ -525,7 +525,7 @@ fn contact_minimal_yields_jscontact_card() { "emailAddresses": [{"name": "Alice", "address": "alice@x.com"}] }); let out = convert_contact(&c).unwrap(); - assert!(out.uid.starts_with("vandelay-graph-")); + assert!(out.uid.starts_with("inbuxa-migrate-graph-")); assert_eq!(out.data["@type"], "Card"); assert_eq!(out.data["name"]["full"], "Alice Liddell"); let emails = out.data["emails"].as_object().unwrap(); @@ -569,23 +569,23 @@ fn integration_dry_run_against_mock_server_lists_three_surfaces() { .with_body(r#"{"value":[{"id":"CON1","displayName":"Contacts"}]}"#) .create(); let base = server.url(); - let common = vandelay::sync::CommonConfig { + let common = inbuxa_migrate::sync::CommonConfig { archive: tempfile::NamedTempFile::new().unwrap().path().to_owned(), threads: 2, dry_run: true, max_retries: 0, allow_invalid_certs: false, - logger: vandelay::logging::Logger::new(0), + logger: inbuxa_migrate::logging::Logger::new(0), }; - let config = vandelay::sync::import_exchange_graph::GraphImportConfig { - auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + let config = inbuxa_migrate::sync::import_exchange_graph::GraphImportConfig { + auth: inbuxa_migrate::sync::import_exchange_graph::GraphAuth::PreAcquired { token: make_jwt(9999999999, "alice@x.com"), }, api_base: base.clone(), user_target: None, - mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + mailbox_kind: inbuxa_migrate::exchange_graph::types::MailboxKind::Primary, surfaces: Surfaces::ALL, - event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text, graph_connections: 2, top: 100, exception_window_years: 5, @@ -593,7 +593,7 @@ fn integration_dry_run_against_mock_server_lists_three_surfaces() { event_attachments: true, allow_source_change: false, }; - let summary = vandelay::sync::import_exchange_graph::run(common, config).unwrap(); + let summary = inbuxa_migrate::sync::import_exchange_graph::run(common, config).unwrap(); let mailbox = summary .per_type .iter() @@ -681,23 +681,23 @@ fn integration_full_run_mail_only_imports_mime_via_value() { let tmp = tempfile::NamedTempFile::new().unwrap(); let archive_path = tmp.path().to_owned(); - let common = vandelay::sync::CommonConfig { + let common = inbuxa_migrate::sync::CommonConfig { archive: archive_path.clone(), threads: 2, dry_run: false, max_retries: 0, allow_invalid_certs: false, - logger: vandelay::logging::Logger::new(0), + logger: inbuxa_migrate::logging::Logger::new(0), }; - let config = vandelay::sync::import_exchange_graph::GraphImportConfig { - auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + let config = inbuxa_migrate::sync::import_exchange_graph::GraphImportConfig { + auth: inbuxa_migrate::sync::import_exchange_graph::GraphAuth::PreAcquired { token: make_jwt(9999999999, "alice@x.com"), }, api_base: server.url(), user_target: None, - mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + mailbox_kind: inbuxa_migrate::exchange_graph::types::MailboxKind::Primary, surfaces: surfaces("mail"), - event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text, graph_connections: 2, top: 100, exception_window_years: 5, @@ -706,7 +706,7 @@ fn integration_full_run_mail_only_imports_mime_via_value() { allow_source_change: false, }; drop(tmp); - let summary = vandelay::sync::import_exchange_graph::run(common, config).unwrap(); + let summary = inbuxa_migrate::sync::import_exchange_graph::run(common, config).unwrap(); let emails = summary .per_type .iter() @@ -780,23 +780,23 @@ fn integration_duplicate_message_id_does_not_abort_run() { let tmp = tempfile::NamedTempFile::new().unwrap(); let archive_path = tmp.path().to_owned(); - let common = vandelay::sync::CommonConfig { + let common = inbuxa_migrate::sync::CommonConfig { archive: archive_path.clone(), threads: 2, dry_run: false, max_retries: 0, allow_invalid_certs: false, - logger: vandelay::logging::Logger::new(0), + logger: inbuxa_migrate::logging::Logger::new(0), }; - let config = vandelay::sync::import_exchange_graph::GraphImportConfig { - auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + let config = inbuxa_migrate::sync::import_exchange_graph::GraphImportConfig { + auth: inbuxa_migrate::sync::import_exchange_graph::GraphAuth::PreAcquired { token: make_jwt(9999999999, "alice@x.com"), }, api_base: server.url(), user_target: None, - mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + mailbox_kind: inbuxa_migrate::exchange_graph::types::MailboxKind::Primary, surfaces: surfaces("mail"), - event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text, graph_connections: 2, top: 100, exception_window_years: 5, @@ -805,7 +805,7 @@ fn integration_duplicate_message_id_does_not_abort_run() { allow_source_change: false, }; drop(tmp); - let summary = vandelay::sync::import_exchange_graph::run(common, config).unwrap(); + let summary = inbuxa_migrate::sync::import_exchange_graph::run(common, config).unwrap(); let emails = summary .per_type .iter() @@ -880,15 +880,15 @@ fn integration_full_run_is_convergent_on_second_invocation() { let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); let api_base = server.url(); - let make_config = || vandelay::sync::import_exchange_graph::GraphImportConfig { - auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + let make_config = || inbuxa_migrate::sync::import_exchange_graph::GraphImportConfig { + auth: inbuxa_migrate::sync::import_exchange_graph::GraphAuth::PreAcquired { token: make_jwt(9999999999, "alice@x.com"), }, api_base: api_base.clone(), user_target: None, - mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + mailbox_kind: inbuxa_migrate::exchange_graph::types::MailboxKind::Primary, surfaces: surfaces("mail"), - event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text, graph_connections: 2, top: 100, exception_window_years: 5, @@ -896,18 +896,20 @@ fn integration_full_run_is_convergent_on_second_invocation() { event_attachments: true, allow_source_change: false, }; - let make_common = |path: std::path::PathBuf| vandelay::sync::CommonConfig { + let make_common = |path: std::path::PathBuf| inbuxa_migrate::sync::CommonConfig { archive: path, threads: 2, dry_run: false, max_retries: 0, allow_invalid_certs: false, - logger: vandelay::logging::Logger::new(0), + logger: inbuxa_migrate::logging::Logger::new(0), }; - let first = - vandelay::sync::import_exchange_graph::run(make_common(archive.clone()), make_config()) - .unwrap(); + let first = inbuxa_migrate::sync::import_exchange_graph::run( + make_common(archive.clone()), + make_config(), + ) + .unwrap(); let created_first = first .per_type .iter() @@ -917,7 +919,8 @@ fn integration_full_run_is_convergent_on_second_invocation() { assert_eq!(created_first, 1); let second = - vandelay::sync::import_exchange_graph::run(make_common(archive), make_config()).unwrap(); + inbuxa_migrate::sync::import_exchange_graph::run(make_common(archive), make_config()) + .unwrap(); let created_second = second .per_type .iter() @@ -930,10 +933,10 @@ fn integration_full_run_is_convergent_on_second_invocation() { #[test] fn source_change_protection_refuses_a_different_account() { let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); - let conn = vandelay::db::init::open(&archive).unwrap(); - vandelay::db::sources::upsert_source( + let conn = inbuxa_migrate::db::init::open(&archive).unwrap(); + inbuxa_migrate::db::sources::upsert_source( &conn, - &vandelay::db::sources::SourceKey { + &inbuxa_migrate::db::sources::SourceKey { kind: "exchange_graph".to_owned(), session_url: "https://login.microsoftonline.com/common|https://graph.microsoft.com/v1.0" @@ -954,23 +957,23 @@ fn source_change_protection_refuses_a_different_account() { .with_body(r#"{"id":"new-user-id","userPrincipalName":"new@x.com"}"#) .create(); - let common = vandelay::sync::CommonConfig { + let common = inbuxa_migrate::sync::CommonConfig { archive, threads: 2, dry_run: false, max_retries: 0, allow_invalid_certs: false, - logger: vandelay::logging::Logger::new(0), + logger: inbuxa_migrate::logging::Logger::new(0), }; - let config = vandelay::sync::import_exchange_graph::GraphImportConfig { - auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + let config = inbuxa_migrate::sync::import_exchange_graph::GraphImportConfig { + auth: inbuxa_migrate::sync::import_exchange_graph::GraphAuth::PreAcquired { token: make_jwt(9999999999, "new@x.com"), }, api_base: server.url(), user_target: None, - mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + mailbox_kind: inbuxa_migrate::exchange_graph::types::MailboxKind::Primary, surfaces: Surfaces::ALL, - event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text, graph_connections: 2, top: 100, exception_window_years: 5, @@ -978,9 +981,9 @@ fn source_change_protection_refuses_a_different_account() { event_attachments: true, allow_source_change: false, }; - let err = vandelay::sync::import_exchange_graph::run(common, config).unwrap_err(); + let err = inbuxa_migrate::sync::import_exchange_graph::run(common, config).unwrap_err(); assert!( - matches!(err, vandelay::error::Error::SourceChange(_)), + matches!(err, inbuxa_migrate::error::Error::SourceChange(_)), "expected SourceChange error, got {err:?}" ); } @@ -1011,35 +1014,35 @@ fn stub_well_known_folders(server: &mut Server, inbox_id: &str) { .create(); } -fn make_common(archive: std::path::PathBuf) -> vandelay::sync::CommonConfig { - vandelay::sync::CommonConfig { +fn make_common(archive: std::path::PathBuf) -> inbuxa_migrate::sync::CommonConfig { + inbuxa_migrate::sync::CommonConfig { archive, threads: 2, dry_run: false, max_retries: 0, allow_invalid_certs: false, - logger: vandelay::logging::Logger::new(0), + logger: inbuxa_migrate::logging::Logger::new(0), } } -fn surfaces(list: &str) -> vandelay::exchange_graph::types::Surfaces { - vandelay::exchange_graph::types::Surfaces::parse_list(list).unwrap() +fn surfaces(list: &str) -> inbuxa_migrate::exchange_graph::types::Surfaces { + inbuxa_migrate::exchange_graph::types::Surfaces::parse_list(list).unwrap() } fn make_config( api_base: String, user_target: Option, - surfaces: vandelay::exchange_graph::types::Surfaces, -) -> vandelay::sync::import_exchange_graph::GraphImportConfig { - vandelay::sync::import_exchange_graph::GraphImportConfig { - auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + surfaces: inbuxa_migrate::exchange_graph::types::Surfaces, +) -> inbuxa_migrate::sync::import_exchange_graph::GraphImportConfig { + inbuxa_migrate::sync::import_exchange_graph::GraphImportConfig { + auth: inbuxa_migrate::sync::import_exchange_graph::GraphAuth::PreAcquired { token: make_jwt(9999999999, "alice@x.com"), }, api_base, user_target, - mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + mailbox_kind: inbuxa_migrate::exchange_graph::types::MailboxKind::Primary, surfaces, - event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text, graph_connections: 2, top: 100, exception_window_years: 5, @@ -1097,15 +1100,15 @@ fn users_upn_routes_through_users_segment_not_me() { .expect_at_least(0) .create(); let base = server.url(); - let common = vandelay::sync::CommonConfig { + let common = inbuxa_migrate::sync::CommonConfig { archive: tempfile::NamedTempFile::new().unwrap().path().to_owned(), threads: 2, dry_run: true, max_retries: 0, allow_invalid_certs: false, - logger: vandelay::logging::Logger::new(0), + logger: inbuxa_migrate::logging::Logger::new(0), }; - let summary = vandelay::sync::import_exchange_graph::run( + let summary = inbuxa_migrate::sync::import_exchange_graph::run( common, make_config(base, Some("alice@x.com".to_owned()), Surfaces::ALL), ) @@ -1207,7 +1210,7 @@ fn series_master_with_exception_merges_into_recurrence_overrides() { .create(); let base = server.url(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); - let summary = vandelay::sync::import_exchange_graph::run( + let summary = inbuxa_migrate::sync::import_exchange_graph::run( make_common(archive.clone()), make_config(base, None, surfaces("calendar")), ) @@ -1220,7 +1223,7 @@ fn series_master_with_exception_merges_into_recurrence_overrides() { .unwrap_or(0); assert_eq!(events, 1, "exactly one master row, exception merged in"); - let conn = vandelay::db::init::open(&archive).unwrap(); + let conn = inbuxa_migrate::db::init::open(&archive).unwrap(); let row: String = conn .query_row("SELECT data FROM calendar_events", [], |row| row.get(0)) .unwrap(); @@ -1312,7 +1315,7 @@ fn occurrence_event_is_skipped_no_row_no_id_mapping() { .create(); let base = server.url(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); - let summary = vandelay::sync::import_exchange_graph::run( + let summary = inbuxa_migrate::sync::import_exchange_graph::run( make_common(archive.clone()), make_config(base, None, surfaces("calendar")), ) @@ -1326,7 +1329,7 @@ fn occurrence_event_is_skipped_no_row_no_id_mapping() { assert_eq!(events, 0, "occurrence must not be fetched or stored"); occ_get.assert(); - let conn = vandelay::db::init::open(&archive).unwrap(); + let conn = inbuxa_migrate::db::init::open(&archive).unwrap(); let count: i64 = conn .query_row("SELECT COUNT(*) FROM calendar_events", [], |row| row.get(0)) .unwrap(); @@ -1392,12 +1395,12 @@ fn hidden_mail_folder_has_is_subscribed_zero() { stub_well_known_folders(&mut server, "FVISIBLE"); let base = server.url(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); - let _ = vandelay::sync::import_exchange_graph::run( + let _ = inbuxa_migrate::sync::import_exchange_graph::run( make_common(archive.clone()), make_config(base, None, surfaces("mail")), ) .unwrap(); - let conn = vandelay::db::init::open(&archive).unwrap(); + let conn = inbuxa_migrate::db::init::open(&archive).unwrap(); let visible: i64 = conn .query_row( "SELECT is_subscribed FROM mailboxes WHERE name = 'Inbox'", @@ -1486,12 +1489,12 @@ fn well_known_folder_probes_assign_jmap_roles() { } let base = server.url(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); - let _ = vandelay::sync::import_exchange_graph::run( + let _ = inbuxa_migrate::sync::import_exchange_graph::run( make_common(archive.clone()), make_config(base, None, surfaces("mail")), ) .unwrap(); - let conn = vandelay::db::init::open(&archive).unwrap(); + let conn = inbuxa_migrate::db::init::open(&archive).unwrap(); let mut stmt = conn .prepare("SELECT name, role FROM mailboxes ORDER BY name") .unwrap(); @@ -1563,7 +1566,7 @@ SGVsbG8=\r\n--X--\r\n"; .create(); let base = server.url(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); - let summary = vandelay::sync::import_exchange_graph::run( + let summary = inbuxa_migrate::sync::import_exchange_graph::run( make_common(archive), make_config(base, None, surfaces("mail")), ) @@ -1651,7 +1654,7 @@ fn event_get_carries_outlook_timezone_and_body_content_type_prefer() { .create(); let base = server.url(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); - let _ = vandelay::sync::import_exchange_graph::run( + let _ = inbuxa_migrate::sync::import_exchange_graph::run( make_common(archive), make_config(base, None, surfaces("calendar")), ) @@ -1737,23 +1740,23 @@ fn full_run_records_graph_id_in_sync_id_exchange_graph_with_padding() { let tmp = tempfile::NamedTempFile::new().unwrap(); let archive = tmp.path().to_owned(); - let common = vandelay::sync::CommonConfig { + let common = inbuxa_migrate::sync::CommonConfig { archive: archive.clone(), threads: 2, dry_run: false, max_retries: 0, allow_invalid_certs: false, - logger: vandelay::logging::Logger::new(0), + logger: inbuxa_migrate::logging::Logger::new(0), }; - let config = vandelay::sync::import_exchange_graph::GraphImportConfig { - auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + let config = inbuxa_migrate::sync::import_exchange_graph::GraphImportConfig { + auth: inbuxa_migrate::sync::import_exchange_graph::GraphAuth::PreAcquired { token: make_jwt(9999999999, "pad@x.com"), }, api_base: server.url(), user_target: None, - mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + mailbox_kind: inbuxa_migrate::exchange_graph::types::MailboxKind::Primary, surfaces: surfaces("mail"), - event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text, graph_connections: 2, top: 100, exception_window_years: 5, @@ -1762,8 +1765,8 @@ fn full_run_records_graph_id_in_sync_id_exchange_graph_with_padding() { allow_source_change: false, }; drop(tmp); - let _ = vandelay::sync::import_exchange_graph::run(common, config).unwrap(); - let conn = vandelay::db::init::open(&archive).unwrap(); + let _ = inbuxa_migrate::sync::import_exchange_graph::run(common, config).unwrap(); + let conn = inbuxa_migrate::db::init::open(&archive).unwrap(); let (graph_id, type_name): (String, String) = conn .query_row( "SELECT graph_id, type_name FROM sync_id_exchange_graph", @@ -1810,23 +1813,23 @@ fn archive_mailbox_kind_encodes_synthetic_suffix_in_account_id() { let tmp = tempfile::NamedTempFile::new().unwrap(); let archive = tmp.path().to_owned(); - let common = vandelay::sync::CommonConfig { + let common = inbuxa_migrate::sync::CommonConfig { archive: archive.clone(), threads: 2, dry_run: false, max_retries: 0, allow_invalid_certs: false, - logger: vandelay::logging::Logger::new(0), + logger: inbuxa_migrate::logging::Logger::new(0), }; - let config = vandelay::sync::import_exchange_graph::GraphImportConfig { - auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + let config = inbuxa_migrate::sync::import_exchange_graph::GraphImportConfig { + auth: inbuxa_migrate::sync::import_exchange_graph::GraphAuth::PreAcquired { token: make_jwt(9999999999, "a@x.com"), }, api_base: server.url(), user_target: None, - mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Archive, + mailbox_kind: inbuxa_migrate::exchange_graph::types::MailboxKind::Archive, surfaces: surfaces("mail"), - event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text, graph_connections: 2, top: 100, exception_window_years: 5, @@ -1835,8 +1838,8 @@ fn archive_mailbox_kind_encodes_synthetic_suffix_in_account_id() { allow_source_change: false, }; drop(tmp); - let _ = vandelay::sync::import_exchange_graph::run(common, config).unwrap(); - let conn = vandelay::db::init::open(&archive).unwrap(); + let _ = inbuxa_migrate::sync::import_exchange_graph::run(common, config).unwrap(); + let conn = inbuxa_migrate::db::init::open(&archive).unwrap(); let account_id: String = conn .query_row( "SELECT account_id FROM sources WHERE kind = 'exchange_graph'", @@ -1850,10 +1853,10 @@ fn archive_mailbox_kind_encodes_synthetic_suffix_in_account_id() { #[test] fn allow_source_change_permits_overwriting_a_different_account() { let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); - let conn = vandelay::db::init::open(&archive).unwrap(); - vandelay::db::sources::upsert_source( + let conn = inbuxa_migrate::db::init::open(&archive).unwrap(); + inbuxa_migrate::db::sources::upsert_source( &conn, - &vandelay::db::sources::SourceKey { + &inbuxa_migrate::db::sources::SourceKey { kind: "exchange_graph".to_owned(), session_url: "https://login.microsoftonline.com/common|https://x.example".to_owned(), account_id: "different-uid".to_owned(), @@ -1878,23 +1881,23 @@ fn allow_source_change_permits_overwriting_a_different_account() { .with_body(r#"{"value":[]}"#) .create(); - let common = vandelay::sync::CommonConfig { + let common = inbuxa_migrate::sync::CommonConfig { archive: archive.clone(), threads: 2, dry_run: false, max_retries: 0, allow_invalid_certs: false, - logger: vandelay::logging::Logger::new(0), + logger: inbuxa_migrate::logging::Logger::new(0), }; - let config = vandelay::sync::import_exchange_graph::GraphImportConfig { - auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + let config = inbuxa_migrate::sync::import_exchange_graph::GraphImportConfig { + auth: inbuxa_migrate::sync::import_exchange_graph::GraphAuth::PreAcquired { token: make_jwt(9999999999, "new@x.com"), }, api_base: server.url(), user_target: None, - mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + mailbox_kind: inbuxa_migrate::exchange_graph::types::MailboxKind::Primary, surfaces: surfaces("mail"), - event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text, graph_connections: 2, top: 100, exception_window_years: 5, @@ -1902,7 +1905,7 @@ fn allow_source_change_permits_overwriting_a_different_account() { event_attachments: true, allow_source_change: true, }; - let result = vandelay::sync::import_exchange_graph::run(common, config); + let result = inbuxa_migrate::sync::import_exchange_graph::run(common, config); assert!( result.is_ok(), "--allow-source-change must permit a different account, got {result:?}" @@ -1962,23 +1965,23 @@ fn dry_run_makes_no_per_item_get_and_no_sqlite_writes() { let tmp = tempfile::NamedTempFile::new().unwrap(); let archive = tmp.path().to_owned(); - let common = vandelay::sync::CommonConfig { + let common = inbuxa_migrate::sync::CommonConfig { archive: archive.clone(), threads: 2, dry_run: true, max_retries: 0, allow_invalid_certs: false, - logger: vandelay::logging::Logger::new(0), + logger: inbuxa_migrate::logging::Logger::new(0), }; - let config = vandelay::sync::import_exchange_graph::GraphImportConfig { - auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + let config = inbuxa_migrate::sync::import_exchange_graph::GraphImportConfig { + auth: inbuxa_migrate::sync::import_exchange_graph::GraphAuth::PreAcquired { token: make_jwt(9999999999, "a@x.com"), }, api_base: server.url(), user_target: None, - mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + mailbox_kind: inbuxa_migrate::exchange_graph::types::MailboxKind::Primary, surfaces: Surfaces::ALL, - event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text, graph_connections: 2, top: 100, exception_window_years: 5, @@ -1987,10 +1990,10 @@ fn dry_run_makes_no_per_item_get_and_no_sqlite_writes() { allow_source_change: false, }; drop(tmp); - let _ = vandelay::sync::import_exchange_graph::run(common, config).unwrap(); + let _ = inbuxa_migrate::sync::import_exchange_graph::run(common, config).unwrap(); no_messages.assert(); no_well_known.assert(); - let conn = vandelay::db::init::open(&archive).unwrap(); + let conn = inbuxa_migrate::db::init::open(&archive).unwrap(); let sources: i64 = conn .query_row("SELECT count(*) FROM sources", [], |row| row.get(0)) .unwrap(); @@ -2025,10 +2028,10 @@ fn token_response_bad_code_recognised() { #[test] fn folder_enumeration_failure_skips_vanished_deletion() { let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); - let setup_conn = vandelay::db::init::open(&archive).unwrap(); - let sid = vandelay::db::sources::upsert_source( + let setup_conn = inbuxa_migrate::db::init::open(&archive).unwrap(); + let sid = inbuxa_migrate::db::sources::upsert_source( &setup_conn, - &vandelay::db::sources::SourceKey { + &inbuxa_migrate::db::sources::SourceKey { kind: "exchange_graph".to_owned(), session_url: "stale".to_owned(), account_id: "stale-uid".to_owned(), @@ -2043,10 +2046,10 @@ fn folder_enumeration_failure_skips_vanished_deletion() { [], ) .unwrap(); - vandelay::db::exchange_graph_ids::insert( + inbuxa_migrate::db::exchange_graph_ids::insert( &setup_conn, sid, - vandelay::db::exchange_graph_ids::MAILBOX, + inbuxa_migrate::db::exchange_graph_ids::MAILBOX, "FMAIL", 42, ) @@ -2067,10 +2070,10 @@ fn folder_enumeration_failure_skips_vanished_deletion() { rusqlite::params![blob_id], ) .unwrap(); - vandelay::db::exchange_graph_ids::insert( + inbuxa_migrate::db::exchange_graph_ids::insert( &setup_conn, sid, - vandelay::db::exchange_graph_ids::EMAIL, + inbuxa_migrate::db::exchange_graph_ids::EMAIL, "MSG-LOCAL", 1, ) @@ -2133,23 +2136,23 @@ fn folder_enumeration_failure_skips_vanished_deletion() { .with_body("transient outage") .create(); - let common = vandelay::sync::CommonConfig { + let common = inbuxa_migrate::sync::CommonConfig { archive: archive.clone(), threads: 2, dry_run: false, max_retries: 0, allow_invalid_certs: false, - logger: vandelay::logging::Logger::new(0), + logger: inbuxa_migrate::logging::Logger::new(0), }; - let config = vandelay::sync::import_exchange_graph::GraphImportConfig { - auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { + let config = inbuxa_migrate::sync::import_exchange_graph::GraphImportConfig { + auth: inbuxa_migrate::sync::import_exchange_graph::GraphAuth::PreAcquired { token: make_jwt(9999999999, "flake@x.com"), }, api_base: server.url(), user_target: None, - mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, + mailbox_kind: inbuxa_migrate::exchange_graph::types::MailboxKind::Primary, surfaces: surfaces("mail"), - event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, + event_body_format: inbuxa_migrate::exchange_graph::types::EventBodyFormat::Text, graph_connections: 2, top: 100, exception_window_years: 5, @@ -2157,8 +2160,8 @@ fn folder_enumeration_failure_skips_vanished_deletion() { event_attachments: true, allow_source_change: true, }; - let _ = vandelay::sync::import_exchange_graph::run(common, config).unwrap(); - let conn = vandelay::db::init::open(&archive).unwrap(); + let _ = inbuxa_migrate::sync::import_exchange_graph::run(common, config).unwrap(); + let conn = inbuxa_migrate::db::init::open(&archive).unwrap(); let remaining: i64 = conn .query_row("SELECT count(*) FROM emails WHERE id = 1", [], |row| { row.get(0) @@ -2324,7 +2327,7 @@ fn contacts_surface_imports_only_address_books_and_cards() { let base = server.url(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); - vandelay::sync::import_exchange_graph::run( + inbuxa_migrate::sync::import_exchange_graph::run( make_common(archive.clone()), make_config(base, None, surfaces("contacts")), ) @@ -2332,7 +2335,7 @@ fn contacts_surface_imports_only_address_books_and_cards() { no_mail.assert(); no_calendars.assert(); - let conn = vandelay::db::init::open(&archive).unwrap(); + let conn = inbuxa_migrate::db::init::open(&archive).unwrap(); assert_eq!(row_count(&conn, "address_books"), 1); assert_eq!(row_count(&conn, "contact_cards"), 1); assert_eq!(row_count(&conn, "mailboxes"), 0); @@ -2368,7 +2371,7 @@ fn mail_surface_imports_only_mailboxes_and_emails() { let base = server.url(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); - vandelay::sync::import_exchange_graph::run( + inbuxa_migrate::sync::import_exchange_graph::run( make_common(archive.clone()), make_config(base, None, surfaces("mail")), ) @@ -2376,7 +2379,7 @@ fn mail_surface_imports_only_mailboxes_and_emails() { no_calendars.assert(); no_contact_folders.assert(); - let conn = vandelay::db::init::open(&archive).unwrap(); + let conn = inbuxa_migrate::db::init::open(&archive).unwrap(); assert_eq!(row_count(&conn, "mailboxes"), 1); assert_eq!(row_count(&conn, "emails"), 1); assert_eq!(row_count(&conn, "address_books"), 0); @@ -2442,7 +2445,7 @@ fn default_contact_folder_is_imported_although_contactfolders_omits_it() { let base = server.url(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); - let summary = vandelay::sync::import_exchange_graph::run( + let summary = inbuxa_migrate::sync::import_exchange_graph::run( make_common(archive.clone()), make_config(base, None, surfaces("contacts")), ) @@ -2468,7 +2471,7 @@ fn default_contact_folder_is_imported_although_contactfolders_omits_it() { "both default-folder contacts import" ); - let conn = vandelay::db::init::open(&archive).unwrap(); + let conn = inbuxa_migrate::db::init::open(&archive).unwrap(); let is_default: i64 = conn .query_row("SELECT is_default FROM address_books", [], |row| row.get(0)) .unwrap(); @@ -2518,13 +2521,13 @@ fn default_contact_folder_falls_back_to_parent_of_an_existing_contact() { let base = server.url(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); - vandelay::sync::import_exchange_graph::run( + inbuxa_migrate::sync::import_exchange_graph::run( make_common(archive.clone()), make_config(base, None, surfaces("contacts")), ) .unwrap(); - let conn = vandelay::db::init::open(&archive).unwrap(); + let conn = inbuxa_migrate::db::init::open(&archive).unwrap(); let name: String = conn .query_row("SELECT name FROM address_books", [], |row| row.get(0)) .unwrap(); @@ -2568,7 +2571,7 @@ fn contact_folder_reachable_by_two_paths_is_inserted_once() { let base = server.url(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); - let summary = vandelay::sync::import_exchange_graph::run( + let summary = inbuxa_migrate::sync::import_exchange_graph::run( make_common(archive.clone()), make_config(base, None, surfaces("contacts")), ) @@ -2638,13 +2641,13 @@ fn message_state_becomes_jmap_keywords() { let base = server.url(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); - vandelay::sync::import_exchange_graph::run( + inbuxa_migrate::sync::import_exchange_graph::run( make_common(archive.clone()), make_config(base, None, surfaces("mail")), ) .unwrap(); - let conn = vandelay::db::init::open(&archive).unwrap(); + let conn = inbuxa_migrate::db::init::open(&archive).unwrap(); let mut stmt = conn .prepare("SELECT keywords FROM emails ORDER BY id") .unwrap(); @@ -2708,7 +2711,7 @@ fn drive_items_import_as_file_nodes_and_skip_facetless_items() { let base = server.url(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); - let summary = vandelay::sync::import_exchange_graph::run( + let summary = inbuxa_migrate::sync::import_exchange_graph::run( make_common(archive.clone()), make_config(base, None, surfaces("files")), ) @@ -2725,7 +2728,7 @@ fn drive_items_import_as_file_nodes_and_skip_facetless_items() { "one directory and two files; the remoteItem has no file or folder facet and is skipped" ); - let conn = vandelay::db::init::open(&archive).unwrap(); + let conn = inbuxa_migrate::db::init::open(&archive).unwrap(); let vault: i64 = conn .query_row( "SELECT count(*) FROM file_nodes WHERE name = 'Personal Vault'", @@ -2844,13 +2847,13 @@ fn a_deleted_occurrence_becomes_an_excluded_override() { let base = server.url(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); - vandelay::sync::import_exchange_graph::run( + inbuxa_migrate::sync::import_exchange_graph::run( make_common(archive.clone()), make_config(base, None, surfaces("calendar")), ) .unwrap(); - let conn = vandelay::db::init::open(&archive).unwrap(); + let conn = inbuxa_migrate::db::init::open(&archive).unwrap(); let row: String = conn .query_row("SELECT data FROM calendar_events", [], |row| row.get(0)) .unwrap(); @@ -2910,13 +2913,13 @@ fn event_file_attachments_become_enclosure_links() { let base = server.url(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); - vandelay::sync::import_exchange_graph::run( + inbuxa_migrate::sync::import_exchange_graph::run( make_common(archive.clone()), make_config(base, None, surfaces("calendar")), ) .unwrap(); - let conn = vandelay::db::init::open(&archive).unwrap(); + let conn = inbuxa_migrate::db::init::open(&archive).unwrap(); let row: String = conn .query_row("SELECT data FROM calendar_events", [], |row| row.get(0)) .unwrap(); @@ -2985,9 +2988,9 @@ fn contact_photo_categories_and_im_addresses_are_imported() { let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); let mut config = make_config(base, None, surfaces("contacts")); config.contact_photos = true; - vandelay::sync::import_exchange_graph::run(make_common(archive.clone()), config).unwrap(); + inbuxa_migrate::sync::import_exchange_graph::run(make_common(archive.clone()), config).unwrap(); - let conn = vandelay::db::init::open(&archive).unwrap(); + let conn = inbuxa_migrate::db::init::open(&archive).unwrap(); let row: String = conn .query_row("SELECT data FROM contact_cards", [], |row| row.get(0)) .unwrap(); diff --git a/tests/mock_imap.rs b/tests/mock_imap.rs index 4ec2ab7..d913fb0 100644 --- a/tests/mock_imap.rs +++ b/tests/mock_imap.rs @@ -13,14 +13,14 @@ use std::thread; use std::time::Duration; use encodify::base64::STANDARD; +use inbuxa_migrate::db; +use inbuxa_migrate::imap::client::{ConnectMode, ImapClient}; +use inbuxa_migrate::imap::error::ImapError; +use inbuxa_migrate::imap::transport::Connector; +use inbuxa_migrate::logging::Logger; +use inbuxa_migrate::sync::CommonConfig; +use inbuxa_migrate::sync::import_imap::{ImapAuth, ImapImportConfig, run}; use rusqlite::Connection; -use vandelay::db; -use vandelay::imap::client::{ConnectMode, ImapClient}; -use vandelay::imap::error::ImapError; -use vandelay::imap::transport::Connector; -use vandelay::logging::Logger; -use vandelay::sync::CommonConfig; -use vandelay::sync::import_imap::{ImapAuth, ImapImportConfig, run}; type Script = Box std::io::Result<()> + Send + 'static>; @@ -129,7 +129,7 @@ fn connect_mock(server: &MockImap) -> Result { fn tempfile(label: &str) -> PathBuf { let counter = COUNTER.fetch_add(1, Ordering::SeqCst); let mut p = std::env::temp_dir(); - p.push(format!("vandelay_mock_imap_{label}_{counter}.sqlite")); + p.push(format!("inbuxa_migrate_mock_imap_{label}_{counter}.sqlite")); if p.exists() { let _ = std::fs::remove_file(&p); } @@ -143,7 +143,7 @@ fn run_import( user: &str, archive: PathBuf, config_tweak: impl FnOnce(&mut ImapImportConfig), -) -> Result { +) -> Result { let common = CommonConfig { archive: archive.clone(), threads: 1, @@ -364,14 +364,14 @@ fn run_collect_parses_fetch_with_literal_body() { .run_collect("UID FETCH 5 (UID BODY.PEEK[])") .expect("fetch"); let body = r.untagged.iter().find_map(|u| match u { - vandelay::imap::response::Untagged::Fetch { items, .. } => Some(items.clone()), + inbuxa_migrate::imap::response::Untagged::Fetch { items, .. } => Some(items.clone()), _ => None, }); let items = body.expect("fetch items"); let body_item = items.iter().find(|(n, _)| n == "BODY[]").unwrap(); match &body_item.1 { - vandelay::imap::response::Value::Str(s) => assert_eq!(s, "Hello world"), - vandelay::imap::response::Value::Bytes(b) => assert_eq!(b, b"Hello world"), + inbuxa_migrate::imap::response::Value::Str(s) => assert_eq!(s, "Hello world"), + inbuxa_migrate::imap::response::Value::Bytes(b) => assert_eq!(b, b"Hello world"), other => panic!("unexpected body shape: {other:?}"), } } @@ -1094,7 +1094,7 @@ fn coordinator_source_change_detected_on_different_url() { let server2 = MockImap::start_scripts(vec![control2]); let err = run_import(&server2, "alice", archive.clone(), |_| {}).unwrap_err(); - assert!(matches!(err, vandelay::error::Error::SourceChange(_))); + assert!(matches!(err, inbuxa_migrate::error::Error::SourceChange(_))); } #[test] @@ -1483,7 +1483,7 @@ fn coordinator_auth_plain_and_login_both_refused_aborts_run() { }); let archive = tempfile("dual_refuse"); let err = run_import(&server, "alice", archive, |_| {}).unwrap_err(); - assert!(matches!(err, vandelay::error::Error::Connection(_))); + assert!(matches!(err, inbuxa_migrate::error::Error::Connection(_))); assert_eq!(err.exit_code(), 2); } @@ -1812,7 +1812,7 @@ fn coordinator_authenticationfailed_yields_exit2_connection_error() { }); let archive = tempfile("authfail"); let err = run_import(&server, "alice", archive, |_| {}).unwrap_err(); - assert!(matches!(err, vandelay::error::Error::Connection(_))); + assert!(matches!(err, inbuxa_migrate::error::Error::Connection(_))); assert_eq!(err.exit_code(), 2); } diff --git a/tests/mock_jmap.rs b/tests/mock_jmap.rs index abc4fd7..0d44376 100644 --- a/tests/mock_jmap.rs +++ b/tests/mock_jmap.rs @@ -6,17 +6,17 @@ use std::time::Instant; -use serde_json::json; -use vandelay::jmap::account::{self, AccountSelector}; -use vandelay::jmap::error::JmapError; -use vandelay::jmap::http::{Auth, HttpClient, RetryPolicy}; -use vandelay::jmap::request::{ +use inbuxa_migrate::jmap::account::{self, AccountSelector}; +use inbuxa_migrate::jmap::error::JmapError; +use inbuxa_migrate::jmap::http::{Auth, HttpClient, RetryPolicy}; +use inbuxa_migrate::jmap::request::{ self, SetRequest, get_all, get_changes, get_objects, get_state, set_call, }; -use vandelay::jmap::session::{Limits, Session}; -use vandelay::jmap::wire::JmapId; -use vandelay::jmap::wire::identity::Identity; -use vandelay::jmap::wire::mailbox::Mailbox; +use inbuxa_migrate::jmap::session::{Limits, Session}; +use inbuxa_migrate::jmap::wire::JmapId; +use inbuxa_migrate::jmap::wire::identity::Identity; +use inbuxa_migrate::jmap::wire::mailbox::Mailbox; +use serde_json::json; fn client(retries: u32) -> HttpClient { HttpClient::new( @@ -672,7 +672,7 @@ fn retries_counter_increments_on_429() { #[test] fn upload_size_precheck_blocks_before_contacting_server() { - use vandelay::jmap::blobxfer; + use inbuxa_migrate::jmap::blobxfer; let mut server = mockito::Server::new(); let no_hits = server @@ -739,7 +739,7 @@ fn concurrent_requests_400_problem_is_retried_then_succeeds() { #[test] fn concurrent_uploads_400_problem_is_retried_then_succeeds() { - use vandelay::jmap::blobxfer; + use inbuxa_migrate::jmap::blobxfer; let mut server = mockito::Server::new(); server @@ -772,7 +772,7 @@ fn concurrent_uploads_400_problem_is_retried_then_succeeds() { #[test] fn blob_upload_quota_429_with_retry_after_is_honoured() { - use vandelay::jmap::blobxfer; + use inbuxa_migrate::jmap::blobxfer; let mut server = mockito::Server::new(); server @@ -883,7 +883,7 @@ fn ratelimit_headers_without_retry_after_fall_back_to_backoff() { #[test] fn upload_extracts_blob_id_and_download_streams_via_download_url() { - use vandelay::jmap::blobxfer; + use inbuxa_migrate::jmap::blobxfer; let mut server = mockito::Server::new(); let base = server.url(); @@ -974,7 +974,7 @@ fn persistent_problem_json_429_without_retry_after_eventually_succeeds_via_share #[test] fn persistent_429_on_download_uses_shared_throttle() { - use vandelay::jmap::blobxfer; + use inbuxa_migrate::jmap::blobxfer; let mut server = mockito::Server::new(); let session = upload_session(&server.url()); @@ -1207,7 +1207,7 @@ fn archive_path() -> std::path::PathBuf { let n = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); let mut p = std::env::temp_dir(); p.push(format!( - "vandelay-mockjmap-{}-{n}.sqlite", + "inbuxa-migrate-mockjmap-{}-{n}.sqlite", std::process::id() )); let _ = std::fs::remove_file(&p); @@ -1264,17 +1264,17 @@ fn import_aborts_with_exit_two_when_the_advertised_api_url_is_unreachable() { .create(); let archive = archive_path(); - let err = vandelay::sync::import_jmap::run( - vandelay::sync::CommonConfig { + let err = inbuxa_migrate::sync::import_jmap::run( + inbuxa_migrate::sync::CommonConfig { archive: archive.clone(), threads: 1, dry_run: false, max_retries: 0, allow_invalid_certs: false, - logger: vandelay::logging::Logger::from_flags(true, 0), + logger: inbuxa_migrate::logging::Logger::from_flags(true, 0), }, - vandelay::sync::ImportConfig { - connect: vandelay::sync::ConnectConfig { + inbuxa_migrate::sync::ImportConfig { + connect: inbuxa_migrate::sync::ConnectConfig { url: session_base.clone(), auth: Auth::Basic { user: "u".into(), @@ -1289,7 +1289,7 @@ fn import_aborts_with_exit_two_when_the_advertised_api_url_is_unreachable() { .expect_err("an unreachable apiUrl must abort the run"); assert!( - matches!(err, vandelay::error::Error::Connection(_)), + matches!(err, inbuxa_migrate::error::Error::Connection(_)), "a 404 from the advertised apiUrl is a whole-run connection failure, got {err:?}" ); assert_eq!(err.exit_code(), 2, "must not report a partial failure"); diff --git a/tests/mock_maildir.rs b/tests/mock_maildir.rs index 5697b3d..182ae99 100644 --- a/tests/mock_maildir.rs +++ b/tests/mock_maildir.rs @@ -8,16 +8,16 @@ use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; +use inbuxa_migrate::logging::Logger; +use inbuxa_migrate::sync::CommonConfig; +use inbuxa_migrate::sync::import_maildir::{MaildirImportConfig, run}; use rusqlite::Connection; use tempfile::TempDir; -use vandelay::logging::Logger; -use vandelay::sync::CommonConfig; -use vandelay::sync::import_maildir::{MaildirImportConfig, run}; fn tmp_archive(tag: &str) -> PathBuf { let mut p = std::env::temp_dir(); p.push(format!( - "vandelay-mock-maildir-{tag}-{}-{}.sqlite", + "inbuxa-migrate-mock-maildir-{tag}-{}-{}.sqlite", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -530,7 +530,7 @@ fn rejects_path_without_cur_subdir() { fs::create_dir_all(td.path().join("new")).unwrap(); let archive = tmp_archive("not-a-maildir"); let err = run(common(&archive), base_cfg(td.path())).expect_err("should refuse"); - assert!(matches!(err, vandelay::error::Error::Usage(_))); + assert!(matches!(err, inbuxa_migrate::error::Error::Usage(_))); let _ = fs::remove_file(&archive); } @@ -544,7 +544,7 @@ fn rejects_dovecot_layout_fs_tree() { let archive = tmp_archive("layout-fs"); let err = run(common(&archive), base_cfg(td.path())).expect_err("should refuse"); match err { - vandelay::error::Error::Usage(msg) => { + inbuxa_migrate::error::Error::Usage(msg) => { assert!(msg.contains("Maildir++"), "msg was: {msg}"); } other => panic!("expected Usage error, got {other:?}"), @@ -558,7 +558,7 @@ fn rejects_nonexistent_path() { let mut cfg = base_cfg(Path::new("/definitely/not/a/real/maildir")); cfg.maildir = PathBuf::from("/definitely/not/a/real/maildir"); let err = run(common(&archive), cfg).expect_err("should refuse"); - assert!(matches!(err, vandelay::error::Error::Usage(_))); + assert!(matches!(err, inbuxa_migrate::error::Error::Usage(_))); let _ = fs::remove_file(&archive); } @@ -572,7 +572,7 @@ fn source_change_protection_refuses_second_path() { let archive = tmp_archive("source-change"); run(common(&archive), base_cfg(td_a.path())).expect("first import"); let err = run(common(&archive), base_cfg(td_b.path())).expect_err("second import refused"); - assert!(matches!(err, vandelay::error::Error::SourceChange(_))); + assert!(matches!(err, inbuxa_migrate::error::Error::SourceChange(_))); let _ = fs::remove_file(&archive); } @@ -601,7 +601,7 @@ fn folder_and_include_are_mutually_exclusive() { cfg.include = vec![regex::Regex::new("^Sent$").unwrap()]; let err = run(common(&archive), cfg).expect_err("mutex"); match err { - vandelay::error::Error::Usage(msg) => { + inbuxa_migrate::error::Error::Usage(msg) => { assert!( msg.contains("--folder") && msg.contains("--include"), "msg was: {msg}" @@ -621,7 +621,7 @@ fn folder_and_exclude_are_mutually_exclusive() { cfg.folder = vec!["INBOX".to_owned()]; cfg.exclude = vec![regex::Regex::new("^Trash$").unwrap()]; let err = run(common(&archive), cfg).expect_err("mutex"); - assert!(matches!(err, vandelay::error::Error::Usage(_))); + assert!(matches!(err, inbuxa_migrate::error::Error::Usage(_))); let _ = fs::remove_file(&archive); } diff --git a/tests/mock_managesieve.rs b/tests/mock_managesieve.rs index aa34e71..4d72dbb 100644 --- a/tests/mock_managesieve.rs +++ b/tests/mock_managesieve.rs @@ -12,10 +12,10 @@ use std::sync::atomic::{AtomicU32, Ordering}; use std::thread; use std::time::Duration; +use inbuxa_migrate::logging::Logger; +use inbuxa_migrate::sync::CommonConfig; +use inbuxa_migrate::sync::import_managesieve::{ManageSieveAuth, ManageSieveImportConfig, run}; use rusqlite::Connection; -use vandelay::logging::Logger; -use vandelay::sync::CommonConfig; -use vandelay::sync::import_managesieve::{ManageSieveAuth, ManageSieveImportConfig, run}; type Script = Box std::io::Result<()> + Send + 'static>; @@ -117,7 +117,7 @@ fn tempfile(label: &str) -> PathBuf { let counter = COUNTER.fetch_add(1, Ordering::SeqCst); let mut p = std::env::temp_dir(); p.push(format!( - "vandelay_mock_managesieve_{label}_{counter}.sqlite" + "inbuxa_migrate_mock_managesieve_{label}_{counter}.sqlite" )); if p.exists() { let _ = std::fs::remove_file(&p); @@ -133,7 +133,7 @@ fn count(conn: &Connection, table: &str) -> i64 { fn run_basic( server: &MockSieveServer, archive: &Path, -) -> Result { +) -> Result { let common = CommonConfig { archive: archive.to_path_buf(), threads: 1, @@ -256,7 +256,7 @@ fn auth_no_translates_to_connection_error() { let archive = tempfile("auth_fail"); let err = run_basic(&server, &archive).unwrap_err(); match err { - vandelay::error::Error::Connection(msg) => assert!( + inbuxa_migrate::error::Error::Connection(msg) => assert!( msg.contains("auth failed") || msg.contains("LOGIN"), "got {msg}" ), @@ -274,7 +274,7 @@ fn server_lacking_plain_login_for_basic_is_refused() { }); let archive = tempfile("no_basic_mech"); let err = run_basic(&server, &archive).unwrap_err(); - assert!(matches!(err, vandelay::error::Error::Connection(_))); + assert!(matches!(err, inbuxa_migrate::error::Error::Connection(_))); let _ = std::fs::remove_file(&archive); } @@ -423,7 +423,7 @@ fn referral_aborts_run_with_connection_error() { let archive = tempfile("referral"); let err = run_basic(&server, &archive).unwrap_err(); match err { - vandelay::error::Error::Connection(msg) => assert!(msg.contains("referral"), "{msg}"), + inbuxa_migrate::error::Error::Connection(msg) => assert!(msg.contains("referral"), "{msg}"), other => panic!("expected Connection, got {other:?}"), } let _ = std::fs::remove_file(&archive); @@ -438,7 +438,7 @@ fn seed_then_run(seed: Script, second: Script) -> (MockSieveServer, PathBuf) { fn run_against( server: &MockSieveServer, archive: &Path, -) -> Result { +) -> Result { run_basic(server, archive) } @@ -935,7 +935,7 @@ fn plain_then_login_both_refused_aborts_run_with_combined_message() { let archive = tempfile("plain_and_login_refused"); let err = run_basic(&server, &archive).unwrap_err(); match err { - vandelay::error::Error::Connection(msg) => { + inbuxa_migrate::error::Error::Connection(msg) => { assert!( msg.contains("LOGIN") && msg.contains("PLAIN"), "expected combined LOGIN/PLAIN error text, got {msg}" diff --git a/tests/mock_sync.rs b/tests/mock_sync.rs index 6396a74..9d469df 100644 --- a/tests/mock_sync.rs +++ b/tests/mock_sync.rs @@ -6,21 +6,21 @@ use std::path::{Path, PathBuf}; +use inbuxa_migrate::db; +use inbuxa_migrate::jmap::account::AccountSelector; +use inbuxa_migrate::jmap::http::Auth; +use inbuxa_migrate::logging::Logger; +use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; +use inbuxa_migrate::types::ObjectType; use mockito::Matcher; use serde_json::{Value, json}; -use vandelay::db; -use vandelay::jmap::account::AccountSelector; -use vandelay::jmap::http::Auth; -use vandelay::logging::Logger; -use vandelay::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; -use vandelay::types::ObjectType; fn tmp() -> PathBuf { static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); let n = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); let mut p = std::env::temp_dir(); p.push(format!( - "vandelay-mocksync-{}-{:?}-{n}.sqlite", + "inbuxa-migrate-mocksync-{}-{:?}-{n}.sqlite", std::process::id(), std::thread::current().id(), )); @@ -1585,8 +1585,8 @@ fn export_missing_target_email_is_created_on_rerun() { let raw = format!("From: a@x\r\nSubject: m{n}\r\nMessage-ID: \r\n\r\nbody {n}",); let blob = db::blobs::intern_blob(&conn, raw.as_bytes()).unwrap(); - let mm = vandelay::sync::keys::index_to_json( - &vandelay::sync::emailmeta::email_index_from_blob(raw.as_bytes()), + let mm = inbuxa_migrate::sync::keys::index_to_json( + &inbuxa_migrate::sync::emailmeta::email_index_from_blob(raw.as_bytes()), ); conn.execute( "INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords,message_match) @@ -1700,8 +1700,8 @@ fn export_email_blake3_fallback_matches_when_no_message_id() { ) .unwrap(); let blob = db::blobs::intern_blob(&conn, raw.as_bytes()).unwrap(); - let mm = vandelay::sync::keys::index_to_json( - &vandelay::sync::emailmeta::email_index_from_blob(raw.as_bytes()), + let mm = inbuxa_migrate::sync::keys::index_to_json( + &inbuxa_migrate::sync::emailmeta::email_index_from_blob(raw.as_bytes()), ); conn.execute( "INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords,message_match) @@ -1711,7 +1711,7 @@ fn export_email_blake3_fallback_matches_when_no_message_id() { .unwrap(); } - let local_idx = vandelay::sync::emailmeta::email_index_from_blob(raw.as_bytes()); + let local_idx = inbuxa_migrate::sync::emailmeta::email_index_from_blob(raw.as_bytes()); assert!(local_idx.mids.is_empty(), "blob must lack Message-ID"); let _root = server.mock("GET", "/").with_status(404).create(); @@ -4144,7 +4144,7 @@ fn export_aborts_with_exit_two_when_the_advertised_api_url_is_unreachable() { .expect_err("an unreachable apiUrl must abort the run"); assert!( - matches!(err, vandelay::error::Error::Connection(_)), + matches!(err, inbuxa_migrate::error::Error::Connection(_)), "a 404 from the advertised apiUrl is a whole-run connection failure, got {err:?}" ); assert_eq!(err.exit_code(), 2, "must not report a partial failure"); @@ -4207,7 +4207,7 @@ fn aborting_import_still_reports_the_types_that_finished() { .as_ref() .expect("a 404 from the api endpoint aborts the run"); assert!( - matches!(err, vandelay::error::Error::Connection(_)), + matches!(err, inbuxa_migrate::error::Error::Connection(_)), "the abort must keep its own exit code, got {err:?}" ); assert_eq!(err.exit_code(), 2); diff --git a/tests/seed_smoke.rs b/tests/seed_smoke.rs index 5475b25..be47a05 100644 --- a/tests/seed_smoke.rs +++ b/tests/seed_smoke.rs @@ -98,7 +98,9 @@ fn verify_account(label: &str, user: &Jmap, account_id: &str, s: &SeedStats) { assert_eq!(list.len(), 2, "{label}: sieve script count"); let primary = list .iter() - .find(|s| s.get("name").and_then(Value::as_str) == Some("vandelay-test-filter")) + .find(|s| { + s.get("name").and_then(Value::as_str) == Some("inbuxa-migrate-test-filter") + }) .expect("primary sieve script present"); assert_eq!( primary.get("isActive").and_then(Value::as_bool), diff --git a/tests/seeder/mod.rs b/tests/seeder/mod.rs index 8b557ab..a016ff2 100644 --- a/tests/seeder/mod.rs +++ b/tests/seeder/mod.rs @@ -17,14 +17,14 @@ use seed::{FileSpec, Layout, MailboxSpec}; pub const ADMIN_USER: &str = "admin"; pub const ADMIN_PASSWORD: &str = "admin"; -pub const DOMAIN: &str = "vandelay.org"; -pub const USER_PASSWORD: &str = "VandelayUser#2026"; -pub const ADMIN_ACCOUNT_PASSWORD: &str = "VandelayAdmin#2026"; +pub const DOMAIN: &str = "inbuxa-migrate.org"; +pub const USER_PASSWORD: &str = "InbuxaUser#2026"; +pub const ADMIN_ACCOUNT_PASSWORD: &str = "InbuxaAdmin#2026"; pub const SYNC_IN: [&str; 3] = ["test1", "test2", "test3"]; pub const SYNC_OUT: [&str; 3] = ["test4", "test5", "test6"]; pub const ADMIN_LOCALPART: &str = "vandeladmin"; -pub const CUSTOM_IDENTITY_NAME: &str = "Vandelay Test (Custom)"; +pub const CUSTOM_IDENTITY_NAME: &str = "Inbuxa Test (Custom)"; macro_rules! mailbox { ($k:expr, $n:expr, $p:expr, $r:expr) => { diff --git a/tests/seeder/seed.rs b/tests/seeder/seed.rs index d93af57..3725ca1 100644 --- a/tests/seeder/seed.rs +++ b/tests/seeder/seed.rs @@ -230,8 +230,8 @@ fn seed_sieve(jmap: &Jmap, account_id: &str, active: bool) -> SeedResult<()> { "SieveScript/set", account_id, json!({ - "s0": { "name": "vandelay-test-filter", "blobId": primary_blob }, - "s1": { "name": "vandelay-vacation", "blobId": secondary_blob } + "s0": { "name": "inbuxa-migrate-test-filter", "blobId": primary_blob }, + "s1": { "name": "inbuxa-migrate-vacation", "blobId": secondary_blob } }), &extra, )?; @@ -247,8 +247,8 @@ fn seed_identity(jmap: &Jmap, account_id: &str, account_email: &str) -> SeedResu "i0": { "name": super::CUSTOM_IDENTITY_NAME, "email": account_email, - "textSignature": "-- \nVandelay Industries", - "htmlSignature": "

Vandelay Industries

" + "textSignature": "-- \nInbuxa Industries", + "htmlSignature": "

Inbuxa Industries

" } }), &[], @@ -339,7 +339,7 @@ fn seed_contacts_calendars( &[CORE, CONTACTS], "AddressBook/set", account_id, - json!({ "ab": { "name": "Vandelay Work Contacts" } }), + json!({ "ab": { "name": "Inbuxa Work Contacts" } }), &[], )?; if let Some(id) = response @@ -359,7 +359,7 @@ fn seed_contacts_calendars( &[CORE, CALENDARS], "Calendar/set", account_id, - json!({ "cal": { "name": "Vandelay Team Calendar", "color": "#3366cc" } }), + json!({ "cal": { "name": "Inbuxa Team Calendar", "color": "#3366cc" } }), &[], )?; if let Some(id) = response diff --git a/tests/sync_caldav.rs b/tests/sync_caldav.rs index 2559816..a068152 100644 --- a/tests/sync_caldav.rs +++ b/tests/sync_caldav.rs @@ -16,16 +16,16 @@ use rusqlite::Connection; fn base_url() -> &'static str { shared_stalwart().base_url() } -use vandelay::jmap::account::AccountSelector; -use vandelay::jmap::http::Auth; -use vandelay::logging::Logger; -use vandelay::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; -use vandelay::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; +use inbuxa_migrate::jmap::account::AccountSelector; +use inbuxa_migrate::jmap::http::Auth; +use inbuxa_migrate::logging::Logger; +use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; +use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; fn tmp_archive(tag: &str) -> PathBuf { let mut p = std::env::temp_dir(); p.push(format!( - "vandelay-{tag}-{}-{}.sqlite", + "inbuxa-migrate-{tag}-{}-{}.sqlite", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) diff --git a/tests/sync_carddav.rs b/tests/sync_carddav.rs index 49fd10e..09071fb 100644 --- a/tests/sync_carddav.rs +++ b/tests/sync_carddav.rs @@ -10,22 +10,22 @@ mod seeder; use std::collections::HashSet; use std::path::{Path, PathBuf}; +use inbuxa_migrate::jmap::account::AccountSelector; +use inbuxa_migrate::jmap::http::Auth; +use inbuxa_migrate::logging::Logger; use integration::stalwart::shared as shared_stalwart; use rusqlite::Connection; -use vandelay::jmap::account::AccountSelector; -use vandelay::jmap::http::Auth; -use vandelay::logging::Logger; fn base_url() -> &'static str { shared_stalwart().base_url() } -use vandelay::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; -use vandelay::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; +use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; +use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; fn tmp_archive(tag: &str) -> PathBuf { let mut p = std::env::temp_dir(); p.push(format!( - "vandelay-{tag}-{}-{}.sqlite", + "inbuxa-migrate-{tag}-{}-{}.sqlite", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) diff --git a/tests/sync_exchange_graph.rs b/tests/sync_exchange_graph.rs index 1d16a7e..acb0713 100644 --- a/tests/sync_exchange_graph.rs +++ b/tests/sync_exchange_graph.rs @@ -9,15 +9,15 @@ mod seeder; use std::path::PathBuf; +use inbuxa_migrate::exchange_graph::types::{EventBodyFormat, MailboxKind, Surfaces}; +use inbuxa_migrate::jmap::account::AccountSelector; +use inbuxa_migrate::jmap::http::Auth; +use inbuxa_migrate::logging::Logger; +use inbuxa_migrate::sync::import_exchange_graph::{GraphAuth, GraphImportConfig}; +use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig}; use integration::stalwart::shared as shared_stalwart; use mockito::{Matcher, Server}; use serde_json::{Value, json}; -use vandelay::exchange_graph::types::{EventBodyFormat, MailboxKind, Surfaces}; -use vandelay::jmap::account::AccountSelector; -use vandelay::jmap::http::Auth; -use vandelay::logging::Logger; -use vandelay::sync::import_exchange_graph::{GraphAuth, GraphImportConfig}; -use vandelay::sync::{self, CommonConfig, ConnectConfig, ExportConfig}; fn base_url() -> &'static str { shared_stalwart().base_url() @@ -26,7 +26,7 @@ fn base_url() -> &'static str { fn tmp_archive(tag: &str) -> PathBuf { let mut p = std::env::temp_dir(); p.push(format!( - "vandelay-{tag}-{}-{}.sqlite", + "inbuxa-migrate-{tag}-{}-{}.sqlite", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -96,9 +96,9 @@ fn graph_fixture(server: &mut Server) { .with_status(200) .with_header("content-type", "text/plain") .with_body( - "From: art@vandelay.example\r\nTo: alice@x.com\r\n\ + "From: art@inbuxa-migrate.example\r\nTo: alice@x.com\r\n\ Subject: Graph round trip\r\nDate: Tue, 01 Sep 2026 10:00:00 +0000\r\n\ - Message-ID: \r\n\ + Message-ID: \r\n\ MIME-Version: 1.0\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n\ Grüße aus Köln 🎉\r\n", ) @@ -219,8 +219,8 @@ fn graph_fixture(server: &mut Server) { server, "/me/contacts/C1", r#"{"id":"C1","displayName":"Graph Contact","givenName":"Graph","surname":"Contact", - "categories":["Work","VIP"],"imAddresses":["sip:graph@vandelay.example"], - "emailAddresses":[{"address":"graph@vandelay.example"}]}"#, + "categories":["Work","VIP"],"imAddresses":["sip:graph@inbuxa-migrate.example"], + "emailAddresses":[{"address":"graph@inbuxa-migrate.example"}]}"#, ); server .mock("GET", "/me/contacts/C1/photo/$value") @@ -393,7 +393,7 @@ fn graph_import_survives_a_full_export_to_stalwart() { .as_object() .and_then(|m| m.values().next()) .and_then(|s| s["uri"].as_str()), - Some("sip:graph@vandelay.example"), + Some("sip:graph@inbuxa-migrate.example"), "a URI-shaped IM address must land in uri: Stalwart drops an OnlineService \ that carries only user, even though RFC 9553 permits it" ); diff --git a/tests/sync_imap.rs b/tests/sync_imap.rs index 5eaf6f8..6c4a777 100644 --- a/tests/sync_imap.rs +++ b/tests/sync_imap.rs @@ -22,16 +22,16 @@ fn imaps_url() -> String { format!("imaps://{}:{}", s.host, s.imaps_port) } -use vandelay::jmap::account::AccountSelector; -use vandelay::jmap::http::Auth; -use vandelay::logging::Logger; -use vandelay::sync::import_imap::{ImapAuth, ImapImportConfig}; -use vandelay::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; +use inbuxa_migrate::jmap::account::AccountSelector; +use inbuxa_migrate::jmap::http::Auth; +use inbuxa_migrate::logging::Logger; +use inbuxa_migrate::sync::import_imap::{ImapAuth, ImapImportConfig}; +use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; fn tmp_archive(tag: &str) -> PathBuf { let mut p = std::env::temp_dir(); p.push(format!( - "vandelay-imap-{tag}-{}-{}.sqlite", + "inbuxa-migrate-imap-{tag}-{}-{}.sqlite", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) diff --git a/tests/sync_jmap.rs b/tests/sync_jmap.rs index bd51ef4..b50b38e 100644 --- a/tests/sync_jmap.rs +++ b/tests/sync_jmap.rs @@ -10,15 +10,15 @@ mod seeder; use std::path::{Path, PathBuf}; use encodify::base64::STANDARD; +use inbuxa_migrate::jmap::account::{self, AccountSelector}; +use inbuxa_migrate::jmap::http::{Auth, HttpClient, RetryPolicy}; +use inbuxa_migrate::jmap::request::Request; +use inbuxa_migrate::jmap::session::Session; +use inbuxa_migrate::logging::Logger; +use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; use integration::stalwart::shared as shared_stalwart; use rusqlite::Connection; use serde_json::{Map, Value, json}; -use vandelay::jmap::account::{self, AccountSelector}; -use vandelay::jmap::http::{Auth, HttpClient, RetryPolicy}; -use vandelay::jmap::request::Request; -use vandelay::jmap::session::Session; -use vandelay::logging::Logger; -use vandelay::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; fn base_url() -> &'static str { shared_stalwart().base_url() @@ -27,7 +27,7 @@ fn base_url() -> &'static str { fn tmp_archive(tag: &str) -> PathBuf { let mut p = std::env::temp_dir(); p.push(format!( - "vandelay-{tag}-{}-{}.sqlite", + "inbuxa-migrate-{tag}-{}-{}.sqlite", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -181,7 +181,7 @@ fn import_resolves_account_via_admin_principal_path() { }, account: AccountSelector::Name(target.email.clone()), }, - objects: Some(vec![vandelay::types::ObjectType::Mailbox]), + objects: Some(vec![inbuxa_migrate::types::ObjectType::Mailbox]), allow_source_change: false, }; let summary = sync::import_jmap::run(common(&archive, false), cfg) @@ -390,7 +390,7 @@ fn import_export_deeply_nested_mailbox_tree() { auth: basic("test4"), account: AccountSelector::Id(src.account_id.clone()), }, - objects: Some(vec![vandelay::types::ObjectType::Mailbox]), + objects: Some(vec![inbuxa_migrate::types::ObjectType::Mailbox]), allow_source_change: false, }; let s = sync::import_jmap::run(common(&archive, false), imp).expect("import"); @@ -432,7 +432,7 @@ fn import_export_deeply_nested_mailbox_tree() { auth: basic("test5"), account: AccountSelector::Id(tgt.account_id.clone()), }, - objects: Some(vec![vandelay::types::ObjectType::Mailbox]), + objects: Some(vec![inbuxa_migrate::types::ObjectType::Mailbox]), allow_source_change: false, }; sync::import_jmap::run(common(&again_archive, false), imp_again) @@ -470,7 +470,7 @@ fn import_export_deeply_nested_filenode_tree() { auth: basic("test4"), account: AccountSelector::Id(src.account_id.clone()), }, - objects: Some(vec![vandelay::types::ObjectType::FileNode]), + objects: Some(vec![inbuxa_migrate::types::ObjectType::FileNode]), allow_source_change: false, }; let s = sync::import_jmap::run(common(&archive, false), imp).expect("import"); @@ -514,7 +514,7 @@ fn import_export_deeply_nested_filenode_tree() { auth: basic("test5"), account: AccountSelector::Id(tgt.account_id.clone()), }, - objects: Some(vec![vandelay::types::ObjectType::FileNode]), + objects: Some(vec![inbuxa_migrate::types::ObjectType::FileNode]), allow_source_change: false, }, ) @@ -554,7 +554,7 @@ fn prune_multilevel_mailbox_tree_destroys_leaf_first() { auth: basic("test4"), account: AccountSelector::Id(src.account_id.clone()), }, - objects: Some(vec![vandelay::types::ObjectType::Mailbox]), + objects: Some(vec![inbuxa_migrate::types::ObjectType::Mailbox]), allow_source_change: false, }, ) @@ -567,7 +567,7 @@ fn prune_multilevel_mailbox_tree_destroys_leaf_first() { let arch_empty = tmp_archive("prune_deep_empty"); { - let _ = vandelay::db::init::open(&arch_empty).expect("init empty archive"); + let _ = inbuxa_migrate::db::init::open(&arch_empty).expect("init empty archive"); } let small_jmap = jmap_for(&fx.account("test6").expect("test6").localpart); @@ -595,7 +595,7 @@ fn prune_multilevel_mailbox_tree_destroys_leaf_first() { auth: basic("test6"), account: AccountSelector::Id(fx.account("test6").unwrap().account_id.clone()), }, - objects: Some(vec![vandelay::types::ObjectType::Mailbox]), + objects: Some(vec![inbuxa_migrate::types::ObjectType::Mailbox]), allow_source_change: false, }, ) @@ -644,8 +644,8 @@ fn prune_non_tree_type_contact_card() { account: AccountSelector::Id(src.account_id.clone()), }, objects: Some(vec![ - vandelay::types::ObjectType::AddressBook, - vandelay::types::ObjectType::ContactCard, + inbuxa_migrate::types::ObjectType::AddressBook, + inbuxa_migrate::types::ObjectType::ContactCard, ]), allow_source_change: false, }, @@ -659,7 +659,7 @@ fn prune_non_tree_type_contact_card() { let arch_empty = tmp_archive("prune_cc_empty"); { - let conn = vandelay::db::init::open(&arch_empty).expect("init empty archive"); + let conn = inbuxa_migrate::db::init::open(&arch_empty).expect("init empty archive"); conn.execute( "INSERT INTO address_books (id,name,is_default) VALUES (1,'AB',1)", [], @@ -690,8 +690,8 @@ fn prune_non_tree_type_contact_card() { seeder::teardown(base_url()).expect("teardown"); } -const ISSUE30_CARD_UID: &str = "vandelay-issue30-card"; -const ISSUE30_EVENT_UID: &str = "vandelay-issue30-event"; +const ISSUE30_CARD_UID: &str = "inbuxa-migrate-issue30-card"; +const ISSUE30_EVENT_UID: &str = "inbuxa-migrate-issue30-event"; fn data_uri_bytes(resource: &Value, uri_key: &str, expect_media_type: &str) -> Vec { let uri = resource @@ -716,10 +716,10 @@ fn export_inlines_contact_and_event_blobs_instead_of_blob_ids() { let tgt = fx.account("test6").expect("test6"); let archive = tmp_archive("issue30"); - let photo: &[u8] = b"\x89PNG\r\n\x1a\nvandelay issue 30 contact photo bytes"; - let agenda: &[u8] = b"vandelay issue 30 calendar enclosure bytes"; + let photo: &[u8] = b"\x89PNG\r\n\x1a\ninbuxa-migrate issue 30 contact photo bytes"; + let agenda: &[u8] = b"inbuxa-migrate issue 30 calendar enclosure bytes"; { - let conn = vandelay::db::init::open(&archive).expect("init archive"); + let conn = inbuxa_migrate::db::init::open(&archive).expect("init archive"); conn.execute( "INSERT INTO address_books (id,name,is_default) VALUES (1,'Issue30 Book',1)", [], @@ -730,8 +730,8 @@ fn export_inlines_contact_and_event_blobs_instead_of_blob_ids() { [], ) .unwrap(); - let photo_blob = vandelay::db::blobs::intern_blob(&conn, photo).unwrap(); - let agenda_blob = vandelay::db::blobs::intern_blob(&conn, agenda).unwrap(); + let photo_blob = inbuxa_migrate::db::blobs::intern_blob(&conn, photo).unwrap(); + let agenda_blob = inbuxa_migrate::db::blobs::intern_blob(&conn, agenda).unwrap(); let card = json!({ "@type": "Card", "version": "1.0", @@ -944,7 +944,7 @@ fn export_prune_destroys_unmatched() { auth: basic("test1"), account: AccountSelector::Id(big.account_id.clone()), }, - objects: Some(vec![vandelay::types::ObjectType::Mailbox]), + objects: Some(vec![inbuxa_migrate::types::ObjectType::Mailbox]), allow_source_change: false, }, ) @@ -964,7 +964,7 @@ fn export_prune_destroys_unmatched() { auth: basic("test3"), account: AccountSelector::Id(small.account_id.clone()), }, - objects: Some(vec![vandelay::types::ObjectType::Mailbox]), + objects: Some(vec![inbuxa_migrate::types::ObjectType::Mailbox]), allow_source_change: false, }, ) @@ -1004,7 +1004,7 @@ fn import_resume_converges_after_partial() { }; let mut first = mk(); - first.objects = Some(vec![vandelay::types::ObjectType::Mailbox]); + first.objects = Some(vec![inbuxa_migrate::types::ObjectType::Mailbox]); sync::import_jmap::run(common(&archive, false), first).expect("partial import"); let conn = Connection::open(&archive).unwrap(); @@ -1038,10 +1038,10 @@ fn import_resume_converges_after_partial() { #[test] #[ignore = "requires Docker"] fn live_burst_exceeds_concurrent_requests_and_recovers() { + use inbuxa_migrate::jmap::http::{HttpClient, RetryPolicy}; + use inbuxa_migrate::jmap::request::{Request, check_method_error}; + use inbuxa_migrate::jmap::session::{Limits, Session}; use std::sync::Arc; - use vandelay::jmap::http::{HttpClient, RetryPolicy}; - use vandelay::jmap::request::{Request, check_method_error}; - use vandelay::jmap::session::{Limits, Session}; let fx = seeder::provision(base_url()).expect("provision"); let acc = fx.account("test1").expect("test1"); @@ -1131,7 +1131,7 @@ fn live_burst_exceeds_concurrent_requests_and_recovers() { } struct JmapSettingsGuard { - admin: vandelay::jmap::http::HttpClient, + admin: inbuxa_migrate::jmap::http::HttpClient, admin_api: String, admin_account: String, previous: serde_json::Map, @@ -1139,8 +1139,8 @@ struct JmapSettingsGuard { impl JmapSettingsGuard { fn override_settings(updates: serde_json::Map) -> Self { - use vandelay::jmap::http::{Auth, HttpClient, RetryPolicy}; - use vandelay::jmap::session::Session; + use inbuxa_migrate::jmap::http::{Auth, HttpClient, RetryPolicy}; + use inbuxa_migrate::jmap::session::Session; let admin = HttpClient::new( Auth::Basic { user: seeder::ADMIN_USER.to_owned(), @@ -1188,7 +1188,7 @@ impl Drop for JmapSettingsGuard { } fn read_jmap_settings( - admin: &vandelay::jmap::http::HttpClient, + admin: &inbuxa_migrate::jmap::http::HttpClient, api: &str, account: &str, properties: &[&str], @@ -1217,7 +1217,7 @@ fn read_jmap_settings( } fn apply_jmap_settings( - admin: &vandelay::jmap::http::HttpClient, + admin: &inbuxa_migrate::jmap::http::HttpClient, api: &str, account: &str, updates: &serde_json::Map, @@ -1252,9 +1252,9 @@ fn apply_jmap_settings( #[test] #[ignore = "requires Docker"] fn live_blob_quota_429_triggers_retry_after_then_succeeds() { - use vandelay::jmap::blobxfer; - use vandelay::jmap::http::{HttpClient, RetryPolicy}; - use vandelay::jmap::session::Session; + use inbuxa_migrate::jmap::blobxfer; + use inbuxa_migrate::jmap::http::{HttpClient, RetryPolicy}; + use inbuxa_migrate::jmap::session::Session; let fx = seeder::provision(base_url()).expect("provision"); let acc = fx.account("test1").expect("test1"); diff --git a/tests/sync_maildir.rs b/tests/sync_maildir.rs index ad570e0..120548c 100644 --- a/tests/sync_maildir.rs +++ b/tests/sync_maildir.rs @@ -19,16 +19,16 @@ use rusqlite::Connection; fn base_url() -> &'static str { shared_stalwart().base_url() } -use vandelay::jmap::account::AccountSelector; -use vandelay::jmap::http::Auth; -use vandelay::logging::Logger; -use vandelay::sync::import_maildir::{MaildirImportConfig, run as run_maildir}; -use vandelay::sync::{self, CommonConfig, ConnectConfig, ImportConfig}; +use inbuxa_migrate::jmap::account::AccountSelector; +use inbuxa_migrate::jmap::http::Auth; +use inbuxa_migrate::logging::Logger; +use inbuxa_migrate::sync::import_maildir::{MaildirImportConfig, run as run_maildir}; +use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ImportConfig}; fn tmp_archive(tag: &str) -> PathBuf { let mut p = std::env::temp_dir(); p.push(format!( - "vandelay-sync-maildir-{tag}-{}-{}.sqlite", + "inbuxa-migrate-sync-maildir-{tag}-{}-{}.sqlite", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -551,8 +551,8 @@ fn maildir_and_jmap_imports_share_blob_bytes() { account: AccountSelector::Id(acc.account_id.clone()), }, objects: Some(vec![ - vandelay::types::ObjectType::Mailbox, - vandelay::types::ObjectType::Email, + inbuxa_migrate::types::ObjectType::Mailbox, + inbuxa_migrate::types::ObjectType::Email, ]), allow_source_change: false, }; @@ -627,8 +627,8 @@ fn maildir_message_count_matches_jmap_for_same_corpus() { account: AccountSelector::Id(acc.account_id.clone()), }, objects: Some(vec![ - vandelay::types::ObjectType::Mailbox, - vandelay::types::ObjectType::Email, + inbuxa_migrate::types::ObjectType::Mailbox, + inbuxa_migrate::types::ObjectType::Email, ]), allow_source_change: false, }; diff --git a/tests/sync_managesieve.rs b/tests/sync_managesieve.rs index 4050740..9f92ce3 100644 --- a/tests/sync_managesieve.rs +++ b/tests/sync_managesieve.rs @@ -25,16 +25,16 @@ fn sieves_url() -> String { let s = shared_stalwart(); format!("sieves://{}:{}", s.host, s.sieve_port) } -use vandelay::jmap::account::AccountSelector; -use vandelay::jmap::http::Auth; -use vandelay::logging::Logger; -use vandelay::sync::import_managesieve::{ManageSieveAuth, ManageSieveImportConfig}; -use vandelay::sync::{self, CommonConfig, ConnectConfig, ImportConfig}; +use inbuxa_migrate::jmap::account::AccountSelector; +use inbuxa_migrate::jmap::http::Auth; +use inbuxa_migrate::logging::Logger; +use inbuxa_migrate::sync::import_managesieve::{ManageSieveAuth, ManageSieveImportConfig}; +use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ImportConfig}; fn tmp_archive(tag: &str) -> PathBuf { let mut p = std::env::temp_dir(); p.push(format!( - "vandelay-managesieve-{tag}-{}-{}.sqlite", + "inbuxa-migrate-managesieve-{tag}-{}-{}.sqlite", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -226,7 +226,7 @@ fn managesieve_and_jmap_imports_share_blob_bytes() { }, account: AccountSelector::Id(acc.account_id.clone()), }, - objects: Some(vec![vandelay::types::ObjectType::SieveScript]), + objects: Some(vec![inbuxa_migrate::types::ObjectType::SieveScript]), allow_source_change: false, }; let jmap_common = CommonConfig { @@ -319,7 +319,7 @@ fn managesieve_source_change_protection_refuses_second_account() { let err = sync::import_managesieve::run(common(&archive), basic_config(&acc2.localpart)) .expect_err("should refuse the second source without override"); - assert!(matches!(err, vandelay::error::Error::SourceChange(_))); + assert!(matches!(err, inbuxa_migrate::error::Error::SourceChange(_))); let _ = std::fs::remove_file(&archive); seeder::teardown(base_url()).expect("teardown"); @@ -336,7 +336,7 @@ fn managesieve_implicit_tls_path_succeeds_when_offered() { cfg.url = sieves_url(); let result = sync::import_managesieve::run(common(&archive), cfg); - if let Err(vandelay::error::Error::Connection(msg)) = &result { + if let Err(inbuxa_migrate::error::Error::Connection(msg)) = &result { eprintln!("(expected on cleartext-only deployments) {msg}"); } @@ -382,7 +382,7 @@ fn managesieve_round_trip_via_jmap_export_converges() { allow_invalid_certs: true, logger: Logger::from_flags(false, 0), }; - let export_cfg = vandelay::sync::ExportConfig { + let export_cfg = inbuxa_migrate::sync::ExportConfig { connect: ConnectConfig { url: fx.base_url.clone(), auth: Auth::Basic { @@ -391,7 +391,7 @@ fn managesieve_round_trip_via_jmap_export_converges() { }, account: AccountSelector::Id(dst.account_id.clone()), }, - objects: Some(vec![vandelay::types::ObjectType::SieveScript]), + objects: Some(vec![inbuxa_migrate::types::ObjectType::SieveScript]), prune: false, yes: false, }; @@ -416,7 +416,7 @@ fn managesieve_round_trip_via_jmap_export_converges() { allow_invalid_certs: true, logger: Logger::from_flags(false, 0), }; - let export_cfg2 = vandelay::sync::ExportConfig { + let export_cfg2 = inbuxa_migrate::sync::ExportConfig { connect: ConnectConfig { url: fx.base_url.clone(), auth: Auth::Basic { @@ -425,7 +425,7 @@ fn managesieve_round_trip_via_jmap_export_converges() { }, account: AccountSelector::Id(dst.account_id.clone()), }, - objects: Some(vec![vandelay::types::ObjectType::SieveScript]), + objects: Some(vec![inbuxa_migrate::types::ObjectType::SieveScript]), prune: false, yes: false, }; diff --git a/tests/sync_takeout.rs b/tests/sync_takeout.rs index 4d4a6b0..d43f2a3 100644 --- a/tests/sync_takeout.rs +++ b/tests/sync_takeout.rs @@ -11,9 +11,9 @@ use std::path::{Path, PathBuf}; use rusqlite::Connection; use tempfile::TempDir; -use vandelay::logging::Logger; -use vandelay::sync::CommonConfig; -use vandelay::sync::import_takeout::{TakeoutImportConfig, run}; +use inbuxa_migrate::logging::Logger; +use inbuxa_migrate::sync::CommonConfig; +use inbuxa_migrate::sync::import_takeout::{TakeoutImportConfig, run}; const RESOURCES: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/resources"); @@ -230,7 +230,7 @@ fn empty_directory_fails_with_usage() { let archive = td.path().join("a.sqlite"); let err = run(common_for(&archive), config_for(td.path().to_path_buf())).unwrap_err(); assert!(format!("{err}").contains("no .mbox")); - assert!(matches!(err, vandelay::error::Error::Usage(_))); + assert!(matches!(err, inbuxa_migrate::error::Error::Usage(_))); assert_eq!(err.exit_code(), 1); } @@ -240,7 +240,7 @@ fn nonexistent_path_fails_with_usage() { let archive = td.path().join("a.sqlite"); let missing = td.path().join("does-not-exist"); let err = run(common_for(&archive), config_for(missing)).unwrap_err(); - assert!(matches!(err, vandelay::error::Error::Usage(_))); + assert!(matches!(err, inbuxa_migrate::error::Error::Usage(_))); assert_eq!(err.exit_code(), 1); } @@ -450,7 +450,7 @@ fn contacts_dedup_by_uid_across_two_vcf_files() { assert_eq!(with_uid, 2); let synthetic_uids: i64 = conn .query_row( - "SELECT COUNT(*) FROM contact_cards WHERE uid LIKE 'vandelay-syn-%'", + "SELECT COUNT(*) FROM contact_cards WHERE uid LIKE 'inbuxa-migrate-syn-%'", [], |r| r.get(0), ) @@ -543,7 +543,7 @@ fn source_change_protection_refuses_different_path() { let archive = td.path().join("a.sqlite"); run(common_for(&archive), config_for(a.clone())).unwrap(); let err = run(common_for(&archive), config_for(b.clone())).unwrap_err(); - assert!(matches!(err, vandelay::error::Error::SourceChange(_))); + assert!(matches!(err, inbuxa_migrate::error::Error::SourceChange(_))); let mut cfg2 = config_for(b); cfg2.allow_source_change = true; diff --git a/tests/sync_webdav.rs b/tests/sync_webdav.rs index e3e724e..2806523 100644 --- a/tests/sync_webdav.rs +++ b/tests/sync_webdav.rs @@ -16,16 +16,16 @@ use rusqlite::Connection; fn base_url() -> &'static str { shared_stalwart().base_url() } -use vandelay::jmap::account::AccountSelector; -use vandelay::jmap::http::Auth; -use vandelay::logging::Logger; -use vandelay::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; -use vandelay::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; +use inbuxa_migrate::jmap::account::AccountSelector; +use inbuxa_migrate::jmap::http::Auth; +use inbuxa_migrate::logging::Logger; +use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; +use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; fn tmp_archive(tag: &str) -> PathBuf { let mut p = std::env::temp_dir(); p.push(format!( - "vandelay-{tag}-{}-{}.sqlite", + "inbuxa-migrate-{tag}-{}-{}.sqlite", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) diff --git a/wix/main.wxs b/wix/main.wxs deleted file mode 100644 index 1715d41..0000000 --- a/wix/main.wxs +++ /dev/null @@ -1,228 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1 - 1 - - - - - - - - - - - - - - - - - - -- 2.54.0 From 568ada35b603199273084a6dd997aa17b2613f4e Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 30 Oct 2026 15:56:59 -0700 Subject: [PATCH 2/2] Import upstream v1.0.13, renamed Upstream commit 7e7a66a47d14dac9946d4fc1bc29e63c839a8cc3, renamed by scripts/upstream-import.py. --- Cargo.lock | 269 +++--- Cargo.toml | 3 +- src/cli.rs | 213 ++++- src/db/init.rs | 33 +- src/db/mod.rs | 1 + src/db/schema.sql | 12 + src/db/sources.rs | 37 + src/db/vacation_response.rs | 66 ++ src/exchange_graph/oauth.rs | 21 +- src/jmap/account.rs | 48 +- src/jmap/request.rs | 3 + src/jmap/session.rs | 17 + src/jmap/wire/calendar.rs | 230 +++++ src/jmap/wire/file_node.rs | 79 +- src/jmap/wire/mod.rs | 1 + src/jmap/wire/vacation_response.rs | 50 ++ src/sync/export.rs | 71 +- src/sync/export/blobpool.rs | 167 ++++ src/sync/export/email.rs | 502 ++++++++--- src/sync/export/flat.rs | 13 +- src/sync/export/keyed.rs | 50 +- src/sync/export/sieve.rs | 23 +- src/sync/export/tree.rs | 43 +- src/sync/export/vacation.rs | 50 ++ src/sync/import_dav.rs | 3 +- src/sync/import_dav/calcard.rs | 129 ++- src/sync/import_dav/coordinator.rs | 31 +- src/sync/import_dav/replace.rs | 255 ++++++ src/sync/import_imap/coordinator.rs | 29 +- src/sync/import_imap/folders.rs | 213 ++++- src/sync/import_jmap.rs | 187 ++-- src/sync/import_jmap/mapping.rs | 211 +++-- src/sync/import_jmap/pool.rs | 42 +- tests/integration/dovecot.rs | 17 + tests/integration/imap_client.rs | 6 + tests/integration_baikal.rs | 8 +- tests/integration_radicale.rs | 4 +- tests/integration_webdav.rs | 4 +- tests/mock_dav.rs | 317 ++++++- tests/mock_imap.rs | 244 +++++ tests/mock_jmap.rs | 5 +- tests/mock_sync.rs | 1286 +++++++++++++++++++++------ tests/seed_smoke.rs | 11 +- tests/seeder/admin.rs | 26 + tests/seeder/mod.rs | 1 + tests/seeder/seed.rs | 9 +- tests/sync_caldav.rs | 4 +- tests/sync_carddav.rs | 4 +- tests/sync_exchange_graph.rs | 48 +- tests/sync_imap.rs | 170 +++- tests/sync_jmap.rs | 135 ++- tests/sync_webdav.rs | 4 +- 52 files changed, 4501 insertions(+), 904 deletions(-) create mode 100644 src/db/vacation_response.rs create mode 100644 src/jmap/wire/vacation_response.rs create mode 100644 src/sync/export/blobpool.rs create mode 100644 src/sync/export/vacation.rs create mode 100644 src/sync/import_dav/replace.rs diff --git a/Cargo.lock b/Cargo.lock index f22f4bd..9f34e2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -157,7 +157,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -174,9 +174,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.18.1" +version = "1.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" +checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" dependencies = [ "aws-lc-sys", "zeroize", @@ -184,9 +184,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" +checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" dependencies = [ "cc", "cmake", @@ -389,9 +389,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.5.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" dependencies = [ "find-msvc-tools", "jobserver", @@ -471,7 +471,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -506,9 +506,9 @@ dependencies = [ [[package]] name = "combine" -version = "4.6.8" +version = "4.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92" dependencies = [ "bytes", "memchr", @@ -576,9 +576,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" dependencies = [ "cfg-if", ] @@ -618,7 +618,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -629,14 +629,14 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ "darling_core", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "defmt" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" dependencies = [ "bitflags 1.3.2", "defmt-macros", @@ -644,30 +644,30 @@ dependencies = [ [[package]] name = "defmt-macros" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" dependencies = [ "defmt-parser", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] name = "defmt-parser" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" dependencies = [ "thiserror", ] [[package]] name = "deranged" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" dependencies = [ "serde_core", ] @@ -680,7 +680,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -717,15 +717,15 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "either" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" [[package]] name = "encodify" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fda1e5c788c2b59096e152ad1a76ce72a2ab80c75985eb94ca82de0b6555db7" +checksum = "e95d6e27686e1d6991a3077f4dbec9d33d18903466d234165ad86cd08c3ebc79" dependencies = [ "memchr", "simdutf8", @@ -733,9 +733,9 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" [[package]] name = "errno" @@ -788,9 +788,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" [[package]] name = "flate2" @@ -886,7 +886,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -955,9 +955,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" dependencies = [ "atomic-waker", "bytes", @@ -1091,9 +1091,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", @@ -1215,9 +1215,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" dependencies = [ "displaydoc", "potential_utf", @@ -1229,9 +1229,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" dependencies = [ "displaydoc", "litemap", @@ -1242,9 +1242,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1256,15 +1256,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" [[package]] name = "icu_properties" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" dependencies = [ "displaydoc", "icu_collections", @@ -1277,15 +1277,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" [[package]] name = "icu_provider" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" dependencies = [ "displaydoc", "icu_locale_core", @@ -1369,9 +1369,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.37" +version = "0.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1baf72f08796de0260609515130699b890ac25f30e610ad894bc5856cafdb" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" dependencies = [ "defmt", "jiff-core", @@ -1396,9 +1396,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.37" +version = "0.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378268a1116ad67ae6228701118ac9f491d78fda38a40a1f1a9e1348de6f7212" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" dependencies = [ "jiff-core", "proc-macro2", @@ -1408,9 +1408,9 @@ dependencies = [ [[package]] name = "jiff-tzdb" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" [[package]] name = "jiff-tzdb-platform" @@ -1504,15 +1504,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.189" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "libsqlite3-sys" -version = "0.38.2" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +checksum = "14a5f01b87b92705d07b433c06f96cbc764637d81a0bffb5734ce17cf0f9a2b3" dependencies = [ "cc", "pkg-config", @@ -1533,9 +1533,9 @@ checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "litrs" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" +checksum = "4744e383959f0db86ede514b809b1c53251889093803c05267acc7d4e7030d70" [[package]] name = "lock_api" @@ -1597,9 +1597,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", "wasi", @@ -1867,9 +1867,9 @@ dependencies = [ [[package]] name = "powerfmt" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" [[package]] name = "ppv-lite86" @@ -1923,9 +1923,9 @@ dependencies = [ [[package]] name = "psl" -version = "2.1.238" +version = "2.1.245" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e6aa2cd4e8e062e78ac5d7df6206e2f8e39b624f658efd6f34f9aa1928ac5da" +checksum = "46cce9bc841c0eeb395275a775d11e3cc91b904e6ab370a9a42369e111b3038e" dependencies = [ "psl-types", ] @@ -2038,7 +2038,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -2096,9 +2096,9 @@ dependencies = [ [[package]] name = "rusqlite" -version = "0.40.2" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +checksum = "9c026b4444579e8d754556c6c75d040d69c4a44283cdb648ab8a8d5965c2cfa8" dependencies = [ "bitflags 2.13.2", "fallible-iterator", @@ -2139,9 +2139,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", @@ -2155,9 +2155,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +checksum = "4a073f5dc7469f984c52ad2752b63b0807745133b6de880b7b64c1ac4c48aec4" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -2203,9 +2203,9 @@ checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -2236,9 +2236,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5" dependencies = [ "windows-sys 0.61.2", ] @@ -2257,9 +2257,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" dependencies = [ "dyn-clone", "ref-cast", @@ -2329,14 +2329,14 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" dependencies = [ "itoa", "memchr", @@ -2353,7 +2353,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -2382,7 +2382,7 @@ dependencies = [ "indexmap 2.14.2", "jiff", "schemars 0.9.0", - "schemars 1.2.2", + "schemars 1.2.3", "serde_core", "serde_json", "serde_with_macros", @@ -2398,7 +2398,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -2465,9 +2465,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.16.2" +version = "1.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" [[package]] name = "socket2" @@ -2545,9 +2545,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.6" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" dependencies = [ "proc-macro2", "quote", @@ -2568,7 +2568,7 @@ checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -2632,7 +2632,7 @@ checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -2667,9 +2667,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" dependencies = [ "displaydoc", "zerovec", @@ -2683,9 +2683,9 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] name = "tokio" -version = "1.53.1" +version = "1.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" dependencies = [ "bytes", "libc", @@ -2706,14 +2706,14 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "tokio-rustls" -version = "0.26.5" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" dependencies = [ "rustls", "tokio", @@ -2732,9 +2732,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.19" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" dependencies = [ "bytes", "futures-core", @@ -2854,9 +2854,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "unicode-ident" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" [[package]] name = "untrusted" @@ -2866,9 +2866,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "3.4.2" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a7ac20be9b7726e0bbdbf974c059676d9acb1cd414961f570a4e8231cacd7fc" +checksum = "cde93ff05758a36a851a398a772b0b79a82fbe3c186a46f2696baaeaf84c966b" dependencies = [ "base64 0.23.1", "cookie_store", @@ -2886,9 +2886,9 @@ dependencies = [ [[package]] name = "ureq-proto" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f86fd172ccca569e458f61b6bdd6220965a9ef36e672a6852953b51a0e1583be" +checksum = "841583fc36089d7d59419568ae157f43a5a03a2e5a81ba87c77d1596ef46e922" dependencies = [ "base64 0.23.1", "http", @@ -2929,16 +2929,16 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.26.1" +version = "1.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" +checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" dependencies = [ "sha1_smol", ] [[package]] name = "inbuxa-migrate" -version = "1.0.11" +version = "1.0.13" dependencies = [ "blake3", "calcard", @@ -2948,6 +2948,7 @@ dependencies = [ "crossbeam-channel", "encodify", "flate2", + "hashify", "indexmap 2.14.2", "mail-parser", "mockito", @@ -2992,9 +2993,9 @@ dependencies = [ [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] @@ -3046,7 +3047,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "wasm-bindgen-shared", ] @@ -3071,9 +3072,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a" dependencies = [ "rustls-pki-types", ] @@ -3274,9 +3275,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -3285,30 +3286,30 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.59" +version = "0.8.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.59" +version = "0.8.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" dependencies = [ "proc-macro2", "quote", @@ -3332,15 +3333,15 @@ checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "synstructure", ] [[package]] name = "zeroize" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" [[package]] name = "zerotrie" @@ -3355,9 +3356,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" dependencies = [ "yoke", "zerofrom", @@ -3366,13 +3367,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 402db81..b7de5e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ bundled-sqlite = ["rusqlite/bundled"] serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0" indexmap = { version = "2.14", features = ["serde"] } -rusqlite = { version = "0.40", features = ["blob"] } +rusqlite = { version = "0.41", features = ["blob"] } blake3 = "1.8.5" time = { version = "0.3.47", features = ["serde-well-known", "macros"] } url= "2.5.8" @@ -40,6 +40,7 @@ flate2 = "1.1.9" quick-xml = "0.42" chrono = { version = "0.4", default-features = false, features = ["std"] } chrono-tz = { version = "0.10", default-features = false } +hashify = "0.2" psl = "2.1.226" [dev-dependencies] diff --git a/src/cli.rs b/src/cli.rs index 680f07f..7d336df 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -18,7 +18,7 @@ use crate::jmap::account::AccountSelector; use crate::jmap::http::Auth; use crate::logging::Logger; use crate::secret; -use crate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; +use crate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange}; use crate::sync::import_exchange_ews::{EwsAuth, EwsImportConfig}; use crate::sync::import_exchange_graph::{GraphAuth, GraphImportConfig}; use crate::sync::import_imap::{ImapAuth, ImapImportConfig}; @@ -494,6 +494,13 @@ pub struct DavImportArgs { )] allow_source_change: bool, + #[arg( + long, + conflicts_with = "allow_source_change", + help = "Remove other recorded sources of this kind, and their items, before importing" + )] + replace_source: bool, + #[command(flatten)] global: GlobalArgs, @@ -715,7 +722,11 @@ fn resolve_dav_import(args: DavImportArgs, kind: DavKindArg) -> Result SourceChange::Replace, + (false, true) => SourceChange::Allow, + (false, false) => SourceChange::Reject, + }, }, )) } @@ -1050,38 +1061,45 @@ fn resolve_exchange_ews_auth(args: &ExchangeEwsImportArgs) -> Result Result { + let mut argv = vec!["inbuxa-migrate", "import", "exchange-ews", "archive.sqlite"]; + argv.extend_from_slice(extra); + let cli = Cli::try_parse_from(argv).unwrap_or_else(|e| panic!("clap rejected args: {e}")); + match cli.resolve()? { + Action::ImportExchangeEws(_, config) => Ok(config.auth), + _ => panic!("expected ImportExchangeEws"), + } + } + + fn set_ews_env(token: Option<&str>, client_secret: Option<&str>) { + unsafe { + match token { + Some(t) => env::set_var("INBUXA_MIGRATE_TOKEN", t), + None => env::remove_var("INBUXA_MIGRATE_TOKEN"), + } + match client_secret { + Some(s) => env::set_var("INBUXA_MIGRATE_EWS_CLIENT_SECRET", s), + None => env::remove_var("INBUXA_MIGRATE_EWS_CLIENT_SECRET"), + } + } + } + + // A bare --auth-bearer takes INBUXA_MIGRATE_TOKEN without needing --ews-client-id. + #[test] + fn ews_bare_bearer_uses_env_token_without_client_id() { + let _g = lock(); + set_ews_env(Some("envtok"), None); + let auth = ews_auth(&["--auth-bearer"]); + set_ews_env(None, None); + assert!(matches!(auth, Ok(EwsAuth::Bearer { token }) if token == "envtok")); + } + + // INBUXA_MIGRATE_TOKEN wins over client credentials taken from the environment. + #[test] + fn ews_env_token_beats_env_client_secret() { + let _g = lock(); + set_ews_env(Some("envtok"), Some("sec")); + let auth = ews_auth(&["--auth-bearer", "--ews-client-id", "cid"]); + set_ews_env(None, None); + assert!(matches!(auth, Ok(EwsAuth::Bearer { token }) if token == "envtok")); + } + + // The env client secret is used when no token is exported. + #[test] + fn ews_env_client_secret_without_token_uses_client_credentials() { + let _g = lock(); + set_ews_env(None, Some("sec")); + let auth = ews_auth(&["--auth-bearer", "--ews-client-id", "cid"]); + set_ews_env(None, None); + assert!(matches!( + auth, + Ok(EwsAuth::OAuth(OAuthFlow::ClientCredentials { client_id, client_secret, .. })) + if client_id == "cid" && client_secret == "sec" + )); + } + + // An explicit --ews-device-code is not overridden by an exported token. + #[test] + fn ews_device_code_flag_beats_env_token() { + let _g = lock(); + set_ews_env(Some("envtok"), None); + let auth = ews_auth(&[ + "--auth-bearer", + "--ews-client-id", + "cid", + "--ews-device-code", + ]); + set_ews_env(None, None); + assert!(matches!( + auth, + Ok(EwsAuth::OAuth(OAuthFlow::DeviceCode { tenant, client_id })) + if tenant == "common" && client_id == "cid" + )); + } + + // --ews-device-code still needs a client id. + #[test] + fn ews_device_code_without_client_id_is_usage_error() { + let _g = lock(); + set_ews_env(Some("envtok"), None); + let auth = ews_auth(&["--auth-bearer", "--ews-device-code"]); + set_ews_env(None, None); + match auth { + Err(Error::Usage(m)) => assert!(m.contains("--ews-client-id"), "msg was: {m}"), + other => panic!("expected Usage error, got {other:?}"), + } + } + + // An inline token wins over everything else. + #[test] + fn ews_inline_token_beats_env_and_flows() { + let _g = lock(); + set_ews_env(Some("envtok"), None); + let auth = ews_auth(&[ + "--auth-bearer=inline", + "--ews-client-id", + "cid", + "--ews-device-code", + ]); + set_ews_env(None, None); + assert!(matches!(auth, Ok(EwsAuth::Bearer { token }) if token == "inline")); + } + fn graph_config(objects: &[&str]) -> Result { let mut argv = vec![ "inbuxa-migrate", @@ -1449,4 +1571,39 @@ mod tests { other => panic!("expected Usage error, got {:?}", other.map(|_| ())), } } + + fn webdav_source_change(extra: &[&str]) -> Result { + let mut argv = vec![ + "inbuxa-migrate", + "import", + "webdav", + "--url", + "https://dav.example/files/u/", + "--auth-bearer", + "tok", + ]; + argv.extend_from_slice(extra); + argv.push("archive.sqlite"); + let cli = Cli::try_parse_from(argv)?; + match cli.resolve() { + Ok(Action::ImportDav(_, config)) => Ok(config.source_change), + _ => panic!("expected ImportDav"), + } + } + + // --replace-source maps to Replace and cannot be combined with --allow-source-change. + #[test] + fn dav_source_change_flags_resolve() { + let _g = lock(); + assert_eq!(webdav_source_change(&[]).unwrap(), SourceChange::Reject); + assert_eq!( + webdav_source_change(&["--allow-source-change"]).unwrap(), + SourceChange::Allow + ); + assert_eq!( + webdav_source_change(&["--replace-source"]).unwrap(), + SourceChange::Replace + ); + assert!(webdav_source_change(&["--replace-source", "--allow-source-change"]).is_err()); + } } diff --git a/src/db/init.rs b/src/db/init.rs index 1e15265..84b8d16 100644 --- a/src/db/init.rs +++ b/src/db/init.rs @@ -19,20 +19,41 @@ pub fn apply_schema(conn: &Connection) -> Result<(), OpenError> { let tx = conn.unchecked_transaction()?; tx.execute_batch(SCHEMA_SQL)?; ensure_calendar_events_data_type(&tx)?; + ensure_file_nodes_executable(&tx)?; ensure_graph_ids_accept_file_nodes(&tx)?; tx.commit()?; Ok(()) } fn ensure_calendar_events_data_type(conn: &Connection) -> Result<(), OpenError> { - let mut stmt = conn.prepare("PRAGMA table_info(calendar_events)")?; + ensure_column( + conn, + "calendar_events", + "data_type", + "ALTER TABLE calendar_events ADD COLUMN data_type TEXT NOT NULL DEFAULT 'Event'", + ) +} + +fn ensure_file_nodes_executable(conn: &Connection) -> Result<(), OpenError> { + ensure_column( + conn, + "file_nodes", + "executable", + "ALTER TABLE file_nodes ADD COLUMN executable INTEGER NOT NULL DEFAULT 0", + ) +} + +fn ensure_column( + conn: &Connection, + table: &str, + column: &str, + alter: &str, +) -> Result<(), OpenError> { + let mut stmt = conn.prepare(&format!("PRAGMA table_info({table})"))?; let rows = stmt.query_map([], |row| row.get::<_, String>(1))?; - let has_column = rows.filter_map(|r| r.ok()).any(|name| name == "data_type"); + let has_column = rows.filter_map(|r| r.ok()).any(|name| name == column); if !has_column { - conn.execute( - "ALTER TABLE calendar_events ADD COLUMN data_type TEXT NOT NULL DEFAULT 'Event'", - [], - )?; + conn.execute(alter, [])?; } Ok(()) } diff --git a/src/db/mod.rs b/src/db/mod.rs index c638fc3..85dadab 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -19,3 +19,4 @@ pub mod roles; pub mod sources; pub mod sync_state_jmap; pub mod takeout_ids; +pub mod vacation_response; diff --git a/src/db/schema.sql b/src/db/schema.sql index 5ed1f37..4069be3 100644 --- a/src/db/schema.sql +++ b/src/db/schema.sql @@ -171,6 +171,17 @@ CREATE TABLE IF NOT EXISTS sieve_scripts ( CREATE UNIQUE INDEX IF NOT EXISTS sieve_scripts_one_active_idx ON sieve_scripts (is_active) WHERE is_active = 1; +CREATE TABLE IF NOT EXISTS vacation_response ( + id INTEGER PRIMARY KEY CHECK (id = 1), + source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + is_enabled INTEGER NOT NULL DEFAULT 0, + from_date TEXT, + to_date TEXT, + subject TEXT, + text_body TEXT, + html_body TEXT +); + CREATE TABLE IF NOT EXISTS address_books ( id INTEGER PRIMARY KEY, name TEXT NOT NULL, @@ -235,6 +246,7 @@ CREATE TABLE IF NOT EXISTS file_nodes ( modified TEXT, is_subscribed INTEGER NOT NULL DEFAULT 1, role TEXT, + executable INTEGER NOT NULL DEFAULT 0, UNIQUE (parent_id, name) ); diff --git a/src/db/sources.rs b/src/db/sources.rs index 8a2600a..f55bd9b 100644 --- a/src/db/sources.rs +++ b/src/db/sources.rs @@ -37,6 +37,27 @@ pub fn conflicting_source( .optional() } +pub fn other_sources( + conn: &Connection, + kind: &str, + session_url: &str, + account_id: &str, +) -> Result, rusqlite::Error> { + let mut stmt = conn.prepare( + "SELECT id, session_url, account_id FROM sources + WHERE kind = ?1 AND NOT (session_url = ?2 AND account_id = ?3) ORDER BY id", + )?; + stmt.query_map(params![kind, session_url, account_id], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) + })? + .collect() +} + +pub fn delete_source(conn: &Connection, id: i64) -> Result<(), rusqlite::Error> { + conn.execute("DELETE FROM sources WHERE id = ?1", params![id])?; + Ok(()) +} + pub fn upsert_source( conn: &Connection, key: &SourceKey, @@ -102,4 +123,20 @@ mod tests { .unwrap(); assert_eq!(conflict, ("https://a/jmap".to_owned(), "w".to_owned())); } + + #[test] + fn other_sources_lists_every_other_source_of_the_kind() { + let c = mem(); + let a = upsert_source(&c, &key("https://a/jmap", "w"), None, "alice").unwrap(); + let b = upsert_source(&c, &key("https://b/jmap", "w"), None, "alice").unwrap(); + upsert_source(&c, &key("https://c/jmap", "w"), None, "alice").unwrap(); + let others = other_sources(&c, "jmap", "https://c/jmap", "w").unwrap(); + let ids: Vec = others.iter().map(|(id, _, _)| *id).collect(); + assert_eq!(ids, [a, b]); + assert!( + other_sources(&c, "imap", "https://c/jmap", "w") + .unwrap() + .is_empty() + ); + } } diff --git a/src/db/vacation_response.rs b/src/db/vacation_response.rs new file mode 100644 index 0000000..18a6607 --- /dev/null +++ b/src/db/vacation_response.rs @@ -0,0 +1,66 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::{Connection, OptionalExtension, params}; + +use crate::jmap::wire::vacation_response::VacationResponse; + +pub fn store( + conn: &Connection, + source_id: i64, + vacation: &VacationResponse, +) -> Result<(), rusqlite::Error> { + conn.execute( + "INSERT OR REPLACE INTO vacation_response + (id, source_id, is_enabled, from_date, to_date, subject, text_body, html_body) + VALUES (1, ?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![ + source_id, + vacation.is_enabled, + vacation.from_date, + vacation.to_date, + vacation.subject, + vacation.text_body, + vacation.html_body, + ], + )?; + Ok(()) +} + +pub fn clear(conn: &Connection, source_id: i64) -> Result<(), rusqlite::Error> { + conn.execute( + "DELETE FROM vacation_response WHERE source_id = ?1", + params![source_id], + )?; + Ok(()) +} + +pub fn exists(conn: &Connection) -> Result { + conn.query_row( + "SELECT EXISTS(SELECT 1 FROM vacation_response)", + [], + |row| row.get(0), + ) +} + +pub fn load(conn: &Connection) -> Result, rusqlite::Error> { + conn.query_row( + "SELECT is_enabled, from_date, to_date, subject, text_body, html_body + FROM vacation_response WHERE id = 1", + [], + |row| { + Ok(VacationResponse { + is_enabled: row.get(0)?, + from_date: row.get(1)?, + to_date: row.get(2)?, + subject: row.get(3)?, + text_body: row.get(4)?, + html_body: row.get(5)?, + }) + }, + ) + .optional() +} diff --git a/src/exchange_graph/oauth.rs b/src/exchange_graph/oauth.rs index b240111..bf66605 100644 --- a/src/exchange_graph/oauth.rs +++ b/src/exchange_graph/oauth.rs @@ -14,8 +14,7 @@ use ureq::tls::{RootCerts, TlsConfig}; use crate::exchange_graph::error::GraphError; -pub const SCOPES: &str = - "offline_access User.Read Mail.Read MailboxSettings.Read Calendars.Read Contacts.Read"; +pub const SCOPES: &str = "offline_access User.Read Mail.Read MailboxSettings.Read Calendars.Read Contacts.Read Files.Read"; const JWT_SEGMENT: Base64 = URL_SAFE.with_padding(Padding::Optional).any_alphabet(); @@ -426,6 +425,24 @@ mod tests { format!("{header}.{payload}.") } + // Device code consent covers every --objects surface, including the drive. + #[test] + fn scopes_cover_every_surface() { + for required in [ + "offline_access", + "Mail.Read", + "MailboxSettings.Read", + "Calendars.Read", + "Contacts.Read", + "Files.Read", + ] { + assert!( + SCOPES.split_whitespace().any(|s| s == required), + "missing {required}" + ); + } + } + #[test] fn jwt_claim_extraction() { let token = make_jwt("t-1", "alice@x.com", 9999999999); diff --git a/src/jmap/account.rs b/src/jmap/account.rs index e2ff271..ce43ac4 100644 --- a/src/jmap/account.rs +++ b/src/jmap/account.rs @@ -18,7 +18,6 @@ pub enum AccountSelector { Name(String), } -const OWNER_URN: &str = "urn:ietf:params:jmap:principals:owner"; const PRINCIPALS_URN: &str = "urn:ietf:params:jmap:principals"; pub fn resolve( @@ -139,21 +138,15 @@ fn unsupported_principals(name: &str, session: &Session) -> Error { fn extract_account_id(principal: &Value, name: &str) -> Result { if let Some(accounts) = principal.get("accounts").and_then(Value::as_object) { - for data in accounts.values() { - if let Some(owner) = data.get(OWNER_URN) - && let Some(id) = owner.get("accountIdForPrincipal").and_then(Value::as_str) - { - return Ok(id.to_owned()); - } + let mut keys = accounts.keys(); + if let (Some(only), None) = (keys.next(), keys.next()) { + return Ok(only.clone()); } - for data in accounts.values() { - if let Some(cap) = data.as_object() { - for v in cap.values() { - if let Some(id) = v.get("accountId").and_then(Value::as_str) { - return Ok(id.to_owned()); - } - } - } + let named = accounts + .iter() + .find(|(_, account)| account.get("name").and_then(Value::as_str) == Some(name)); + if let Some((id, _)) = named { + return Ok(id.clone()); } } principal @@ -254,12 +247,33 @@ mod tests { assert_eq!(extract_account_id(&p, "bob").unwrap(), "acc-7"); } + // RFC 9670: the data account is the key of "accounts", not accountIdForPrincipal. #[test] - fn extract_uses_per_capability_account_id() { + fn extract_uses_accounts_key_not_account_id_for_principal() { + let p = json!({ + "id": "px", + "name": "bob@example.org", + "accounts": { "data-9": { + "name": "bob@example.org", + "isPersonal": false, + "accountCapabilities": { + "urn:ietf:params:jmap:principals:owner": + { "accountIdForPrincipal": "admin-1", "principalId": "px" } + } + } } + }); + assert_eq!(extract_account_id(&p, "bob@example.org").unwrap(), "data-9"); + } + + #[test] + fn extract_picks_the_named_account_among_several() { let p = json!({ "id": "px", "name": "bob", - "accounts": { "z": { "urn:ietf:params:jmap:mail": { "accountId": "data-9" } } } + "accounts": { + "shared-1": { "name": "team" }, + "data-9": { "name": "bob" } + } }); assert_eq!(extract_account_id(&p, "bob").unwrap(), "data-9"); } diff --git a/src/jmap/request.rs b/src/jmap/request.rs index 9cec161..cdd514e 100644 --- a/src/jmap/request.rs +++ b/src/jmap/request.rs @@ -17,6 +17,8 @@ use crate::logging::Logger; pub const URN_CORE: &str = "urn:ietf:params:jmap:core"; +pub const URN_VACATION_RESPONSE: &str = "urn:ietf:params:jmap:vacationresponse"; + pub fn using_urn(method: &str) -> &'static str { let prefix = method.split('/').next().unwrap_or(method); match prefix { @@ -27,6 +29,7 @@ pub fn using_urn(method: &str) -> &'static str { "Calendar" | "CalendarEvent" | "ParticipantIdentity" => "urn:ietf:params:jmap:calendars", "FileNode" => "urn:ietf:params:jmap:filenode", "Principal" => "urn:ietf:params:jmap:principals", + "VacationResponse" => URN_VACATION_RESPONSE, "x:Account" | "x:Domain" => "urn:stalwart:jmap", _ => URN_CORE, } diff --git a/src/jmap/session.rs b/src/jmap/session.rs index 86433bd..3fac838 100644 --- a/src/jmap/session.rs +++ b/src/jmap/session.rs @@ -249,6 +249,13 @@ impl Session { .unwrap_or(false) } + pub fn may_support(&self, account_id: &str, capability_urn: &str) -> bool { + match self.account_capabilities(account_id) { + Some(caps) => caps.contains_key(capability_urn), + None => self.capabilities.contains_key(capability_urn), + } + } + pub fn upload_url_for(&self, account_id: &str) -> String { self.upload_url .replace("{accountId}", &encode_segment(account_id)) @@ -332,6 +339,16 @@ mod tests { assert!(!session.supports("missing", "urn:ietf:params:jmap:mail")); } + // Listed accounts use their own capabilities; unlisted ones fall back to the session's. + #[test] + fn may_support_falls_back_to_session_capabilities_for_unlisted_account() { + let session = parse_session(raw_session()).unwrap(); + assert!(session.may_support("w", "urn:ietf:params:jmap:mail")); + assert!(!session.may_support("w", "urn:ietf:params:jmap:core")); + assert!(session.may_support("missing", "urn:ietf:params:jmap:core")); + assert!(!session.may_support("missing", "urn:ietf:params:jmap:mail")); + } + #[test] fn templates_upload_and_download_urls_with_encoding() { let session = parse_session(raw_session()).unwrap(); diff --git a/src/jmap/wire/calendar.rs b/src/jmap/wire/calendar.rs index d7cd7dc..d59b658 100644 --- a/src/jmap/wire/calendar.rs +++ b/src/jmap/wire/calendar.rs @@ -49,6 +49,183 @@ pub struct Calendar { pub time_zone: Option, } +impl Calendar { + pub fn normalize_color(&mut self) -> Option { + let color = self.color.as_mut()?; + let trimmed = color.trim(); + let kept = match trimmed.strip_prefix('#') { + Some(hex) if hex.bytes().all(|byte| byte.is_ascii_hexdigit()) => match hex.len() { + 3 | 4 => trimmed.get(..=3), + 6 | 8 => trimmed.get(..=6), + _ => None, + }, + Some(_) => None, + None => is_css3_color_name(trimmed).then_some(trimmed), + }; + match kept { + Some(kept) if kept.len() == color.len() => None, + Some(kept) => { + *color = kept.to_owned(); + None + } + None => self.color.take(), + } + } +} + +fn is_css3_color_name(name: &str) -> bool { + hashify::set_ignore_case!( + name.as_bytes(), + "aliceblue", + "antiquewhite", + "aqua", + "aquamarine", + "azure", + "beige", + "bisque", + "black", + "blanchedalmond", + "blue", + "blueviolet", + "brown", + "burlywood", + "cadetblue", + "chartreuse", + "chocolate", + "coral", + "cornflowerblue", + "cornsilk", + "crimson", + "cyan", + "darkblue", + "darkcyan", + "darkgoldenrod", + "darkgray", + "darkgreen", + "darkgrey", + "darkkhaki", + "darkmagenta", + "darkolivegreen", + "darkorange", + "darkorchid", + "darkred", + "darksalmon", + "darkseagreen", + "darkslateblue", + "darkslategray", + "darkslategrey", + "darkturquoise", + "darkviolet", + "deeppink", + "deepskyblue", + "dimgray", + "dimgrey", + "dodgerblue", + "firebrick", + "floralwhite", + "forestgreen", + "fuchsia", + "gainsboro", + "ghostwhite", + "gold", + "goldenrod", + "gray", + "green", + "greenyellow", + "grey", + "honeydew", + "hotpink", + "indianred", + "indigo", + "ivory", + "khaki", + "lavender", + "lavenderblush", + "lawngreen", + "lemonchiffon", + "lightblue", + "lightcoral", + "lightcyan", + "lightgoldenrodyellow", + "lightgray", + "lightgreen", + "lightgrey", + "lightpink", + "lightsalmon", + "lightseagreen", + "lightskyblue", + "lightslategray", + "lightslategrey", + "lightsteelblue", + "lightyellow", + "lime", + "limegreen", + "linen", + "magenta", + "maroon", + "mediumaquamarine", + "mediumblue", + "mediumorchid", + "mediumpurple", + "mediumseagreen", + "mediumslateblue", + "mediumspringgreen", + "mediumturquoise", + "mediumvioletred", + "midnightblue", + "mintcream", + "mistyrose", + "moccasin", + "navajowhite", + "navy", + "oldlace", + "olive", + "olivedrab", + "orange", + "orangered", + "orchid", + "palegoldenrod", + "palegreen", + "paleturquoise", + "palevioletred", + "papayawhip", + "peachpuff", + "peru", + "pink", + "plum", + "powderblue", + "purple", + "red", + "rosybrown", + "royalblue", + "saddlebrown", + "salmon", + "sandybrown", + "seagreen", + "seashell", + "sienna", + "silver", + "skyblue", + "slateblue", + "slategray", + "slategrey", + "snow", + "springgreen", + "steelblue", + "tan", + "teal", + "thistle", + "tomato", + "turquoise", + "violet", + "wheat", + "white", + "whitesmoke", + "yellow", + "yellowgreen", + ) +} + fn default_true() -> bool { true } @@ -56,3 +233,56 @@ fn default_true() -> bool { fn default_include_in_availability() -> String { "all".to_owned() } + +#[cfg(test)] +mod tests { + use super::*; + + fn normalized(color: &str) -> (Option, Option) { + let mut calendar: Calendar = + serde_json::from_value(serde_json::json!({ "name": "C", "color": color })) + .expect("calendar"); + let dropped = calendar.normalize_color(); + (calendar.color, dropped) + } + + // Alpha is stripped, valid colors pass through, anything else is dropped. + #[test] + fn calendar_colors_are_normalized_for_jmap() { + for (input, expected) in [ + ("#AAAAAAFF", "#AAAAAA"), + ("#3366ccfe", "#3366cc"), + ("#ABCD", "#ABC"), + ("#abc", "#abc"), + ("#3366CC", "#3366CC"), + (" #3366cc\n", "#3366cc"), + ("CornflowerBlue", "CornflowerBlue"), + ("grey", "grey"), + ] { + assert_eq!( + normalized(input), + (Some(expected.to_owned()), None), + "{input:?}" + ); + } + for input in [ + "", + "#", + "#12345", + "#GGGGGG", + "#1234567", + "rgb(1,2,3)", + "notacolor", + ] { + assert_eq!( + normalized(input), + (None, Some(input.to_owned())), + "{input:?}" + ); + } + let mut calendar: Calendar = + serde_json::from_value(serde_json::json!({ "name": "C" })).expect("calendar"); + assert_eq!(calendar.normalize_color(), None); + assert_eq!(calendar.color, None); + } +} diff --git a/src/jmap/wire/file_node.rs b/src/jmap/wire/file_node.rs index d70acdf..9380ad5 100644 --- a/src/jmap/wire/file_node.rs +++ b/src/jmap/wire/file_node.rs @@ -17,20 +17,8 @@ pub struct FileNode { #[serde(default, skip_serializing_if = "Option::is_none")] pub parent_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub node_type: Option, - - #[serde(default, skip_serializing_if = "Option::is_none")] - pub blob_id: Option, - - #[serde(default, skip_serializing_if = "Option::is_none")] - pub target: Option>, - pub name: String, - #[serde(rename = "type", default, skip_serializing_if = "Option::is_none")] - pub media_type: Option, - #[serde(with = "time::serde::rfc3339")] pub created: UtcDate, @@ -41,30 +29,61 @@ pub struct FileNode { )] pub modified: Option, - #[serde(default = "default_true", deserialize_with = "bool_or_true")] - pub is_subscribed: bool, - - #[serde(default, skip_serializing_if = "Option::is_none")] - pub role: Option, + #[serde(flatten)] + pub kind: NodeKind, } -impl FileNode { - pub fn effective_node_type(&self) -> NodeType { - match self.node_type { - Some(t) => t, - None if self.blob_id.is_some() => NodeType::File, - None if self.target.is_some() => NodeType::Symlink, - None => NodeType::Directory, +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "@type")] +pub enum NodeKind { + File(FileProps), + Directory(DirectoryProps), + SymLink(SymLinkProps), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FileProps { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub blob_id: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub media_type: Option, + + #[serde(default, deserialize_with = "bool_or_false")] + pub executable: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DirectoryProps { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + + #[serde(default = "default_true", deserialize_with = "bool_or_true")] + pub is_subscribed: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SymLinkProps { + pub target: Vec, +} + +impl NodeKind { + pub fn db_name(&self) -> &'static str { + match self { + NodeKind::File(_) => "file", + NodeKind::Directory(_) => "directory", + NodeKind::SymLink(_) => "symlink", } } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum NodeType { - File, - Directory, - Symlink, +fn bool_or_false<'de, D>(d: D) -> Result +where + D: serde::Deserializer<'de>, +{ + Ok(Option::::deserialize(d)?.unwrap_or_default()) } fn default_true() -> bool { diff --git a/src/jmap/wire/mod.rs b/src/jmap/wire/mod.rs index 2250540..d034364 100644 --- a/src/jmap/wire/mod.rs +++ b/src/jmap/wire/mod.rs @@ -16,5 +16,6 @@ pub mod identity; pub mod mailbox; pub mod participant_identity; pub mod sieve_script; +pub mod vacation_response; pub use common::{Date, EmailAddress, JmapId, UtcDate}; diff --git a/src/jmap/wire/vacation_response.rs b/src/jmap/wire/vacation_response.rs new file mode 100644 index 0000000..f2e0d73 --- /dev/null +++ b/src/jmap/wire/vacation_response.rs @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde::{Deserialize, Serialize}; + +pub const TYPE_NAME: &str = "VacationResponse"; + +pub const SINGLETON_ID: &str = "singleton"; + +pub const RESERVED_SCRIPT_NAME: &str = "vacation"; + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct VacationResponse { + #[serde(default)] + pub is_enabled: bool, + + #[serde(default)] + pub from_date: Option, + + #[serde(default)] + pub to_date: Option, + + #[serde(default)] + pub subject: Option, + + #[serde(default)] + pub text_body: Option, + + #[serde(default)] + pub html_body: Option, +} + +impl VacationResponse { + pub fn is_unset(&self) -> bool { + !self.is_enabled + && self.from_date.is_none() + && self.to_date.is_none() + && self.subject.is_none() + && self.text_body.is_none() + && self.html_body.is_none() + } + + pub fn is_reserved_script_name(name: &str) -> bool { + name.eq_ignore_ascii_case(RESERVED_SCRIPT_NAME) + } +} diff --git a/src/sync/export.rs b/src/sync/export.rs index 12ec7f2..d5dc162 100644 --- a/src/sync/export.rs +++ b/src/sync/export.rs @@ -16,22 +16,24 @@ use crate::jmap::blobxfer; use crate::jmap::connect::{self, Connected}; use crate::jmap::error::JmapError; use crate::jmap::http::HttpClient; -use crate::jmap::request::{Request, SetRequest, get_all, get_objects, query_all_ids, set_call}; +use crate::jmap::request::{ + Request, SetRequest, URN_VACATION_RESPONSE, get_all, get_objects, query_all_ids, set_call, +}; use crate::jmap::session::{Limits, Session}; use crate::jmap::wire::JmapId; +use crate::jmap::wire::vacation_response; use crate::logging::{LEVEL_DEFAULT, Logger}; use crate::sync::import_jmap::mapping::{BlobBytes, TargetResolver}; use crate::sync::{CommonConfig, Context, ExportConfig, Summary, TypeCounts}; use crate::types::ObjectType; -const EXPORT_ORDER: [ObjectType; 10] = [ +const EXPORT_ORDER: [ObjectType; 9] = [ ObjectType::Mailbox, ObjectType::AddressBook, ObjectType::Calendar, ObjectType::FileNode, ObjectType::Identity, ObjectType::SieveScript, - ObjectType::ParticipantIdentity, ObjectType::Email, ObjectType::ContactCard, ObjectType::CalendarEvent, @@ -97,6 +99,18 @@ impl<'a> Uploader<'a> { Ok(id) } + fn touch(&mut self, local_id: i64) { + self.touched.push(local_id); + } + + fn cached(&self, local_id: i64) -> Option<&JmapId> { + self.cache.get(&local_id) + } + + fn store(&mut self, local_id: i64, id: JmapId) { + self.cache.insert(local_id, id); + } + fn invalidate(&mut self, local_id: i64) { self.cache.remove(&local_id); } @@ -125,6 +139,7 @@ struct Net { limits: Limits, session: Session, dry_run: bool, + threads: usize, } fn has_rows(conn: &Connection, ty: ObjectType) -> bool { @@ -148,6 +163,7 @@ pub fn run(common: CommonConfig, config: ExportConfig) -> Result limits: connected.limits, session: connected.session.clone(), dry_run: ctx.dry_run(), + threads: ctx.common.threads, }; let work = work_list(&ctx.conn, &config, &connected, &logger); @@ -202,6 +218,35 @@ pub fn run(common: CommonConfig, config: ExportConfig) -> Result } } + if config + .objects + .as_ref() + .is_none_or(|s| s.contains(&ObjectType::SieveScript)) + && net.session.may_support(&net.account, URN_VACATION_RESPONSE) + { + let mut counts = TypeCounts::default(); + let present = match vacation::reconcile(&ctx, &net, &mut counts, &logger) { + Ok(present) => present, + Err(e) if e.aborts_run() => return Err(e), + Err(e) => { + logger.warn(&format!( + "type {} aborted: {e}", + vacation_response::TYPE_NAME + )); + counts.failed += 1; + true + } + }; + if present { + if ctx.dry_run() { + dry_rows.push((vacation_response::TYPE_NAME, counts.updated, 0, 0)); + } + summary + .per_type + .push((vacation_response::TYPE_NAME, counts)); + } + } + if ctx.dry_run() { print_dry_run(&dry_rows, config.prune); return Ok(Summary::default()); @@ -271,6 +316,16 @@ fn work_list( logger: &Logger, ) -> Vec { let selected = config.objects.as_ref(); + let participant = ObjectType::ParticipantIdentity; + if selected.is_none_or(|s| s.contains(&participant)) + && has_rows(conn, participant) + && logger.enabled(LEVEL_DEFAULT) + { + eprintln!( + "export: {} not exported; the target creates participant identities from the account's addresses", + participant.jmap_name() + ); + } EXPORT_ORDER .into_iter() .filter(|ty| selected.map(|s| s.contains(ty)).unwrap_or(true)) @@ -307,9 +362,7 @@ fn reconcile_type( flat::reconcile(ctx, net, ty, maps, counts, logger) } ObjectType::Identity => keyed::reconcile_identity(ctx, net, maps, counts, logger), - ObjectType::ParticipantIdentity => { - keyed::reconcile_participant(ctx, net, maps, counts, logger) - } + ObjectType::ParticipantIdentity => Ok(Plan::default()), ObjectType::SieveScript => sieve::reconcile(ctx, net, maps, counts, logger), ObjectType::ContactCard | ObjectType::CalendarEvent => { uidtype::reconcile(ctx, net, ty, maps, counts, logger) @@ -336,7 +389,7 @@ pub struct Plan { } fn do_destroy(net: &Net, ty: ObjectType, plan: &Plan, logger: &Logger, counts: &mut TypeCounts) { - if ty == ObjectType::SieveScript { + if ty == ObjectType::SieveScript && plan.active_sieve_target.is_none() { deactivate_active_sieve_script(net, logger); } let destroy = Value::Array( @@ -435,10 +488,14 @@ mod keyed; mod sieve; +mod vacation; + mod uidtype; mod email; +mod blobpool; + mod common { use super::*; diff --git a/src/sync/export/blobpool.rs b/src/sync/export/blobpool.rs new file mode 100644 index 0000000..cb11704 --- /dev/null +++ b/src/sync/export/blobpool.rs @@ -0,0 +1,167 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use std::collections::HashMap; + +use super::{Net, Uploader}; +use crate::jmap::blobxfer; +use crate::jmap::error::JmapError; +use crate::jmap::wire::JmapId; +use crate::sync::import_jmap::mapping::BlobBytes; +use crate::sync::import_jmap::pool::{Pool, effective_workers}; + +pub const MAX_INFLIGHT_BLOBS: usize = 64; +pub const MAX_INFLIGHT_BYTES: u64 = 256 * 1024 * 1024; + +type UploadJob = (i64, Vec); +type UploadDone = (i64, u64, Result); + +#[derive(Debug)] +pub struct Budget { + count: usize, + bytes: u64, + max_count: usize, + max_bytes: u64, +} + +impl Budget { + pub fn new(max_count: usize, max_bytes: u64) -> Budget { + Budget { + count: 0, + bytes: 0, + max_count: max_count.max(1), + max_bytes, + } + } + + pub fn admits(&self, len: u64) -> bool { + self.count == 0 + || (self.count < self.max_count && self.bytes.saturating_add(len) <= self.max_bytes) + } + + pub fn take(&mut self, len: u64) { + self.count += 1; + self.bytes = self.bytes.saturating_add(len); + } + + pub fn release(&mut self, len: u64) { + self.count = self.count.saturating_sub(1); + self.bytes = self.bytes.saturating_sub(len); + } + + pub fn is_idle(&self) -> bool { + self.count == 0 + } +} + +pub struct UploadPool { + pool: Pool, +} + +impl UploadPool { + pub fn new(net: &Net, threads: usize, content_type: &'static str) -> UploadPool { + let workers = effective_workers(threads, &net.limits, true); + let net = net.clone(); + let pool = Pool::new(workers, move |(local_id, bytes): UploadJob| { + let len = bytes.len() as u64; + let res = blobxfer::upload_bytes( + &net.client, + &net.session, + &net.account, + content_type, + &bytes, + ); + (local_id, len, res) + }); + UploadPool { pool } + } + + pub fn upload(&self, uploader: &mut Uploader<'_>, blobs: &[i64]) -> HashMap { + let mut failed = HashMap::new(); + let mut budget = Budget::new(MAX_INFLIGHT_BLOBS, MAX_INFLIGHT_BYTES); + for &local_id in blobs { + if uploader.cached(local_id).is_some() { + continue; + } + let bytes = match uploader.bytes(local_id) { + Ok(b) => b, + Err(e) => { + failed.insert(local_id, e); + continue; + } + }; + let len = bytes.len() as u64; + while !budget.admits(len) { + if !self.collect_one(uploader, &mut budget, &mut failed) { + break; + } + } + budget.take(len); + self.pool.submit((local_id, bytes)); + } + while !budget.is_idle() { + if !self.collect_one(uploader, &mut budget, &mut failed) { + break; + } + } + failed + } + + fn collect_one( + &self, + uploader: &mut Uploader<'_>, + budget: &mut Budget, + failed: &mut HashMap, + ) -> bool { + match self.pool.results().recv() { + Ok((local_id, len, res)) => { + budget.release(len); + match res { + Ok(id) => uploader.store(local_id, id), + Err(e) => { + failed.insert(local_id, e); + } + } + true + } + Err(_) => { + *budget = Budget::new(MAX_INFLIGHT_BLOBS, MAX_INFLIGHT_BYTES); + false + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn budget_bounds_count_and_bytes() { + let mut b = Budget::new(2, 100); + assert!(b.admits(500)); + b.take(60); + assert!(!b.admits(41)); + assert!(b.admits(40)); + b.take(40); + assert!(!b.admits(0)); + b.release(60); + assert!(b.admits(60)); + assert!(!b.admits(61)); + b.release(40); + assert!(b.is_idle()); + } + + #[test] + fn budget_admits_oversize_blob_only_when_idle() { + let mut b = Budget::new(8, 10); + assert!(b.admits(1_000)); + b.take(1_000); + assert!(!b.admits(1)); + b.release(1_000); + assert!(b.admits(1)); + } +} diff --git a/src/sync/export/email.rs b/src/sync/export/email.rs index 1d8c7ab..b9a270c 100644 --- a/src/sync/export/email.rs +++ b/src/sync/export/email.rs @@ -8,14 +8,14 @@ use std::collections::{HashMap, HashSet}; use serde_json::{Map, Value, json}; +use super::blobpool::UploadPool; use super::common::{jid, target_query_get}; use super::{Maps, Net, Plan, Uploader}; use crate::error::Error; use crate::jmap::error::JmapError; -use crate::jmap::request::{ - MethodCall, Request, check_method_error, get_objects, retry_method_call, -}; +use crate::jmap::request::{Request, check_method_error, get_objects, retry_method_call}; use crate::jmap::retry::MethodCallKind; +use crate::jmap::session::Limits; use crate::jmap::wire::JmapId; use crate::logging::Logger; use crate::sync::import_jmap::mapping::{EMAIL_SELECT, EmailRow, TargetResolver, row_to_email}; @@ -117,18 +117,49 @@ pub fn reconcile( let local_keys = email_keys(&local_indices); let mut uploader = Uploader::new(net, &ctx.conn); - for (i, key) in local_keys.iter().enumerate() { - if target_keys.contains(key) { - counts.skipped += 1; - continue; - } - let (local_id, row) = &local[i]; - export_one(net, &mut uploader, maps, *local_id, row, counts, logger); + let pool = (!net.dry_run).then(|| UploadPool::new(net, net.threads, EMAIL_MIME)); + let unmatched: Vec<&(i64, EmailRow)> = local_keys + .iter() + .zip(local.iter()) + .filter_map(|(key, entry)| { + if target_keys.contains(key) { + counts.skipped += 1; + None + } else { + Some(entry) + } + }) + .collect(); + + let mut exporter = Exporter { + net, + uploader: &mut uploader, + pool: pool.as_ref(), + maps, + counts, + logger, + batch: import_batch_size(&net.limits), + }; + let mut rest = unmatched.as_slice(); + while !rest.is_empty() { + let (chunk, tail) = rest.split_at(exporter.batch.min(rest.len())); + exporter.export_chunk(chunk)?; + rest = tail; } Ok(Plan::default()) } +const EMAIL_MIME: &str = "message/rfc822"; + +pub fn import_batch_size(limits: &Limits) -> usize { + usize::try_from(limits.max_objects_in_set.max(1)).unwrap_or(usize::MAX) +} + +pub fn reupload_batch_size(previous: usize) -> usize { + (previous / 2).max(1) +} + fn build_mailbox_ids(row: &EmailRow, maps: &Maps) -> Option> { let mut mids = Map::new(); for ml in &row.mailbox_locals { @@ -184,120 +215,207 @@ fn import_item( }) } -fn export_one( - net: &Net, - uploader: &mut Uploader, - maps: &Maps, - local_id: i64, - row: &EmailRow, - counts: &mut TypeCounts, - logger: &Logger, -) { - let cid = format!("e{local_id}"); - let mids = match build_mailbox_ids(row, maps) { - Some(m) => m, - None => { - logger.warn(&format!( - "Email/import {cid} ({}) skipped: mailbox not on target", - blob_hint(uploader, row) - )); - counts.failed += 1; - return; +struct Pending<'r> { + cid: String, + row: &'r EmailRow, + mids: Map, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Attempt { + First, + Reupload, + LastReupload, +} + +impl Attempt { + fn suffix(self) -> &'static str { + match self { + Attempt::First => "", + Attempt::Reupload | Attempt::LastReupload => " after blob re-upload", } - }; - let blob = match uploader.upload_with(row.blob_local_id, "message/rfc822") { - Ok(b) => b.0, - Err(e) => { - logger.warn(&format!( - "Email/import {cid} ({}) blob upload failed: {e}{}", - blob_hint(uploader, row), - size_note(&e) - )); - counts.failed += 1; - return; - } - }; - if net.dry_run { - counts.created += 1; - return; } - let item = import_item(blob, mids, build_keywords(row), &row.received_at); - match send_single_import(net, &cid, item, logger) { - Ok(SingleImport::Created) => counts.created += 1, - Ok(SingleImport::Skipped) => counts.skipped += 1, - Ok(SingleImport::NotCreated { error_type, .. }) if error_type == "blobNotFound" => { - retry_after_reupload(net, uploader, maps, &cid, row, counts, logger); - } - Ok(SingleImport::NotCreated { detail, .. }) => { - logger.warn(&format!( - "Email/import {cid} ({}) failed: {detail}", - blob_hint(uploader, row) - )); - counts.failed += 1; - } - Err(e) => { - logger.warn(&format!( - "Email/import {cid} ({}) send failed: {e}{}", - blob_hint(uploader, row), - size_note(&e) - )); - counts.failed += 1; + + fn upload_verb(self) -> &'static str { + match self { + Attempt::First => "blob upload", + Attempt::Reupload | Attempt::LastReupload => "blob re-upload", } } } -fn retry_after_reupload( - net: &Net, - uploader: &mut Uploader, - maps: &Maps, - cid: &str, - row: &EmailRow, - counts: &mut TypeCounts, - logger: &Logger, -) { - uploader.invalidate(row.blob_local_id); - let blob = match uploader.upload_with(row.blob_local_id, "message/rfc822") { - Ok(b) => b.0, - Err(e) => { - logger.warn(&format!( - "Email/import {cid} ({}) blob re-upload failed: {e}{}", - blob_hint(uploader, row), - size_note(&e) - )); - counts.failed += 1; - return; +struct Exporter<'a, 'u> { + net: &'a Net, + uploader: &'a mut Uploader<'u>, + pool: Option<&'a UploadPool>, + maps: &'a Maps, + counts: &'a mut TypeCounts, + logger: &'a Logger, + batch: usize, +} + +impl<'r> Exporter<'_, '_> { + fn export_chunk(&mut self, chunk: &[&'r (i64, EmailRow)]) -> Result<(), Error> { + let mut pending: Vec> = Vec::with_capacity(chunk.len()); + for (local_id, row) in chunk.iter().copied() { + let cid = format!("e{local_id}"); + match build_mailbox_ids(row, self.maps) { + Some(mids) => pending.push(Pending { cid, row, mids }), + None => { + self.logger.warn(&format!( + "Email/import {cid} ({}) skipped: mailbox not on target", + blob_hint(self.uploader, row) + )); + self.counts.failed += 1; + } + } } - }; - let mids = match build_mailbox_ids(row, maps) { - Some(m) => m, - None => { - logger.warn(&format!( - "Email/import {cid} ({}) skipped: mailbox not on target", - blob_hint(uploader, row) - )); - counts.failed += 1; - return; + let mut size = pending.len(); + let mut retry = self.import(pending, Attempt::First)?; + if !retry.is_empty() { + self.batch = reupload_batch_size(self.batch); } - }; - let item = import_item(blob, mids, build_keywords(row), &row.received_at); - match send_single_import(net, cid, item, logger) { - Ok(SingleImport::Created) => counts.created += 1, - Ok(SingleImport::Skipped) => counts.skipped += 1, - Ok(SingleImport::NotCreated { detail, .. }) => { - logger.warn(&format!( - "Email/import {cid} ({}) failed after blob re-upload: {detail}", - blob_hint(uploader, row) - )); - counts.failed += 1; + while !retry.is_empty() { + size = reupload_batch_size(size); + let attempt = if size == 1 { + Attempt::LastReupload + } else { + Attempt::Reupload + }; + for p in &retry { + self.uploader.invalidate(p.row.blob_local_id); + } + let mut again = Vec::new(); + while !retry.is_empty() { + let tail = retry.split_off(size.min(retry.len())); + let head = std::mem::replace(&mut retry, tail); + again.extend(self.import(head, attempt)?); + } + retry = again; } - Err(e) => { - logger.warn(&format!( - "Email/import {cid} ({}) send failed after blob re-upload: {e}{}", - blob_hint(uploader, row), - size_note(&e) - )); - counts.failed += 1; + Ok(()) + } + + fn import( + &mut self, + pending: Vec>, + attempt: Attempt, + ) -> Result>, Error> { + let mut failed = self.upload(&pending); + let mut ready: Vec> = Vec::with_capacity(pending.len()); + let mut items: Vec<(String, Value)> = Vec::with_capacity(pending.len()); + for p in pending { + let blob = match failed.remove(&p.row.blob_local_id) { + Some(e) => Err(e), + None => self + .uploader + .cached(p.row.blob_local_id) + .map(|b| b.0.clone()) + .ok_or_else(|| JmapError::malformed("blob upload did not complete")), + }; + let blob = match blob { + Ok(b) => b, + Err(e) => { + self.logger.warn(&format!( + "Email/import {} ({}) {} failed: {e}{}", + p.cid, + blob_hint(self.uploader, p.row), + attempt.upload_verb(), + size_note(&e) + )); + self.counts.failed += 1; + continue; + } + }; + if self.net.dry_run { + self.counts.created += 1; + continue; + } + let item = import_item( + blob, + p.mids.clone(), + build_keywords(p.row), + &p.row.received_at, + ); + items.push((p.cid.clone(), item)); + ready.push(p); } + if items.is_empty() { + return Ok(Vec::new()); + } + + let mut results = Vec::with_capacity(items.len()); + if let Err(e) = send_import(self.net, &items, self.logger, &mut results) { + let mapped = Error::from(e); + if mapped.aborts_run() { + return Err(mapped); + } + for p in ready.iter().skip(results.len()) { + self.logger.warn(&format!( + "Email/import {} ({}) send failed{}: {mapped}", + p.cid, + blob_hint(self.uploader, p.row), + attempt.suffix() + )); + self.counts.failed += 1; + } + ready.truncate(results.len()); + } + + let mut retry = Vec::new(); + for (p, res) in ready.into_iter().zip(results) { + match res { + Ok(SingleImport::Created) => self.counts.created += 1, + Ok(SingleImport::Skipped) => self.counts.skipped += 1, + Ok(SingleImport::NotCreated { error_type, .. }) + if attempt != Attempt::LastReupload && error_type == "blobNotFound" => + { + retry.push(p); + } + Ok(SingleImport::NotCreated { detail, .. }) => { + self.logger.warn(&format!( + "Email/import {} ({}) failed{}: {detail}", + p.cid, + blob_hint(self.uploader, p.row), + attempt.suffix() + )); + self.counts.failed += 1; + } + Err(e) => { + self.logger.warn(&format!( + "Email/import {} ({}) send failed{}: {e}{}", + p.cid, + blob_hint(self.uploader, p.row), + attempt.suffix(), + size_note(&e) + )); + self.counts.failed += 1; + } + } + } + Ok(retry) + } + + fn upload(&mut self, pending: &[Pending<'_>]) -> HashMap { + let Some(pool) = self.pool else { + let mut failed = HashMap::new(); + for p in pending { + if let Err(e) = self.uploader.upload_with(p.row.blob_local_id, EMAIL_MIME) { + failed.insert(p.row.blob_local_id, e); + } + } + return failed; + }; + let mut seen: HashSet = HashSet::with_capacity(pending.len()); + let mut queue: Vec = Vec::with_capacity(pending.len()); + for p in pending { + let local_id = p.row.blob_local_id; + self.uploader.touch(local_id); + if self.uploader.cached(local_id).is_none() && seen.insert(local_id) { + queue.push(local_id); + } + } + pool.upload(self.uploader, &queue) } } @@ -307,33 +425,61 @@ enum SingleImport { NotCreated { error_type: String, detail: String }, } -fn send_single_import( - net: &Net, - cid: &str, - item: Value, - logger: &Logger, -) -> Result { - let mut emails = Map::new(); - emails.insert(cid.to_owned(), item); +fn import_request(net: &Net, items: &[(String, Value)]) -> Request { + let emails: Map = items.iter().cloned().collect(); let mut req = Request::new(); req.call( "Email/import", json!({ "accountId": net.account, "emails": Value::Object(emails) }), "i", ); - req.fits(&net.limits)?; - retry_method_call( - &net.client, - MethodCallKind::SingleObjectWrite, - logger, - || interpret_import(req.send(&net.client, &net.api)?.first()?, cid), - ) + req } -fn interpret_import(mr: &MethodCall, cid: &str) -> Result { - check_method_error(mr)?; - if let Some(err) = mr - .args +fn send_import( + net: &Net, + items: &[(String, Value)], + logger: &Logger, + out: &mut Vec>, +) -> Result<(), JmapError> { + let req = import_request(net, items); + let sent = match req.fits(&net.limits) { + Ok(()) => retry_method_call( + &net.client, + MethodCallKind::SingleObjectWrite, + logger, + || { + let resp = req.send(&net.client, &net.api)?; + let mr = resp.first()?; + check_method_error(mr)?; + Ok(items + .iter() + .map(|(cid, _)| item_outcome(&mr.args, cid)) + .collect::>()) + }, + ), + Err(e) => Err(e), + }; + match sent { + Ok(outcomes) => { + out.extend(outcomes.into_iter().map(Ok)); + Ok(()) + } + Err(JmapError::RequestTooLarge) if items.len() > 1 => { + let (left, right) = items.split_at(items.len() / 2); + send_import(net, left, logger, out)?; + send_import(net, right, logger, out) + } + Err(e @ JmapError::RequestTooLarge) => { + out.push(Err(e)); + Ok(()) + } + Err(e) => Err(e), + } +} + +fn item_outcome(args: &Value, cid: &str) -> SingleImport { + if let Some(err) = args .get("notCreated") .and_then(Value::as_object) .and_then(|nc| nc.get(cid)) @@ -344,23 +490,95 @@ fn interpret_import(mr: &MethodCall, cid: &str) -> Result Limits { + Limits { + max_objects_in_get: 500, + max_objects_in_set: set, + max_calls_in_request: 16, + max_concurrent_requests: 4, + max_concurrent_upload: 4, + max_size_request: 10_000_000, + max_size_upload: 50_000_000, + } + } + + #[test] + fn batch_size_follows_max_objects_in_set() { + assert_eq!(import_batch_size(&limits(500)), 500); + assert_eq!(import_batch_size(&limits(7)), 7); + assert_eq!(import_batch_size(&limits(0)), 1); + } + + #[test] + fn reupload_batches_halve_down_to_one() { + let mut size = 500; + let mut steps = Vec::new(); + while size > 1 { + size = reupload_batch_size(size); + steps.push(size); + } + assert_eq!(steps, vec![250, 125, 62, 31, 15, 7, 3, 1]); + assert_eq!(reupload_batch_size(1), 1); + assert_eq!(reupload_batch_size(0), 1); + } + + #[test] + fn chunking_yields_ceil_n_over_limit_batches() { + let ids: Vec = (0..1001).collect(); + let sizes: Vec = ids + .chunks(import_batch_size(&limits(500))) + .map(<[u32]>::len) + .collect(); + assert_eq!(sizes, vec![500, 500, 1]); + } + + #[test] + fn item_outcome_reads_each_creation_id() { + let args = json!({ + "created": { "e1": { "id": "M1" } }, + "notCreated": { + "e2": { "type": "alreadyExists", "existingId": "M9" }, + "e3": { "type": "blobNotFound" }, + "e4": { "type": "overQuota" } + } + }); + assert!(matches!(item_outcome(&args, "e1"), SingleImport::Created)); + assert!(matches!(item_outcome(&args, "e2"), SingleImport::Skipped)); + assert!(matches!( + item_outcome(&args, "e3"), + SingleImport::NotCreated { ref error_type, .. } if error_type == "blobNotFound" + )); + assert!(matches!( + item_outcome(&args, "e4"), + SingleImport::NotCreated { ref error_type, .. } if error_type == "overQuota" + )); + assert!(matches!( + item_outcome(&args, "e5"), + SingleImport::NotCreated { ref error_type, .. } if error_type.is_empty() + )); + } } diff --git a/src/sync/export/flat.rs b/src/sync/export/flat.rs index 673fe83..68f9da5 100644 --- a/src/sync/export/flat.rs +++ b/src/sync/export/flat.rs @@ -11,7 +11,7 @@ use super::common::{create_batch, jid, target_get_all}; use super::{Maps, Net, Plan}; use crate::error::Error; use crate::jmap::wire::JmapId; -use crate::logging::Logger; +use crate::logging::{LEVEL_PROGRESS, Logger}; use crate::sync::import_jmap::mapping::{ ADDRESS_BOOK_SELECT, CALENDAR_SELECT, row_to_address_book, row_to_calendar, }; @@ -82,7 +82,16 @@ pub fn reconcile( Ok(if ty == ObjectType::AddressBook { row_to_address_book(row).map(|w| serde_json::to_value(&w)) } else { - row_to_calendar(row).map(|w| serde_json::to_value(&w)) + row_to_calendar(row).map(|mut w| { + if let Some(dropped) = w.normalize_color() + && logger.enabled(LEVEL_PROGRESS) + { + eprintln!( + "Calendar {local}: dropping color {dropped:?}, not a CSS color name or #RGB/#RRGGBB value" + ); + } + serde_json::to_value(&w) + }) }) }) .map_err(|e| Error::Partial(e.to_string()))? diff --git a/src/sync/export/keyed.rs b/src/sync/export/keyed.rs index c1e565f..3d45853 100644 --- a/src/sync/export/keyed.rs +++ b/src/sync/export/keyed.rs @@ -12,10 +12,8 @@ use super::common::{create_batch, target_get_all}; use super::{Maps, Net, Plan}; use crate::error::Error; use crate::logging::Logger; -use crate::sync::import_jmap::mapping::{ - IDENTITY_SELECT, PARTICIPANT_IDENTITY_SELECT, row_to_identity, row_to_participant_identity, -}; -use crate::sync::keys::{identity_key, participant_identity_key}; +use crate::sync::import_jmap::mapping::{IDENTITY_SELECT, row_to_identity}; +use crate::sync::keys::identity_key; use crate::sync::{Context, TypeCounts}; use crate::types::ObjectType; @@ -99,47 +97,3 @@ pub fn reconcile_identity( rows, ) } - -pub fn reconcile_participant( - ctx: &Context, - net: &Net, - _maps: &mut Maps, - counts: &mut TypeCounts, - logger: &Logger, -) -> Result { - let mut stmt = ctx - .conn - .prepare(PARTICIPANT_IDENTITY_SELECT) - .map_err(|e| Error::Partial(e.to_string()))?; - let rows: Vec<(i64, [u8; 32], Value)> = stmt - .query_map([], |row| { - let id: i64 = row.get(0)?; - Ok((id, row_to_participant_identity(row))) - }) - .and_then(|m| m.collect::, _>>()) - .map_err(|e| Error::Partial(e.to_string()))? - .into_iter() - .map(|(id, w)| { - let w = w.map_err(Error::from)?; - let key = participant_identity_key(&w.calendar_address, &w.name); - let v = serde_json::to_value(&w).map_err(|e| Error::Partial(e.to_string()))?; - Ok((id, key, v)) - }) - .collect::>()?; - drop(stmt); - run( - net, - ObjectType::ParticipantIdentity, - counts, - logger, - |t| { - Some(participant_identity_key( - t.get("calendarAddress") - .and_then(Value::as_str) - .unwrap_or(""), - t.get("name").and_then(Value::as_str).unwrap_or(""), - )) - }, - rows, - ) -} diff --git a/src/sync/export/sieve.rs b/src/sync/export/sieve.rs index 8928202..f44125c 100644 --- a/src/sync/export/sieve.rs +++ b/src/sync/export/sieve.rs @@ -10,8 +10,10 @@ use serde_json::{Value, json}; use super::common::{create_batch, jid, retry_if_blob_missing, target_get_all}; use super::{Maps, Net, Plan, Uploader}; +use crate::db; use crate::error::Error; -use crate::jmap::request::Request; +use crate::jmap::request::{Request, URN_VACATION_RESPONSE}; +use crate::jmap::wire::vacation_response::VacationResponse; use crate::logging::Logger; use crate::sync::import_jmap::mapping::{SIEVE_SELECT, row_to_sieve_script}; use crate::sync::{Context, TypeCounts}; @@ -35,7 +37,11 @@ pub fn reconcile( target_by_name.insert(name.to_owned(), id); } - let locals: Vec<(i64, Option, bool, i64)> = { + let target_has_vacation = net.session.may_support(&net.account, URN_VACATION_RESPONSE); + let vacation_exported = target_has_vacation + && db::vacation_response::exists(&ctx.conn).map_err(|e| Error::Partial(e.to_string()))?; + + let mut locals: Vec<(i64, Option, bool, i64)> = { let mut stmt = ctx .conn .prepare(SIEVE_SELECT) @@ -54,6 +60,16 @@ pub fn reconcile( .collect::>()? }; + if vacation_exported { + let before = locals.len(); + locals.retain(|(_, name, _, _)| { + !name + .as_deref() + .is_some_and(VacationResponse::is_reserved_script_name) + }); + counts.skipped += (before - locals.len()) as u64; + } + let mut active_target: Option = None; let mut deactivate = false; let mut uploader = Uploader::new(net, &ctx.conn); @@ -127,6 +143,9 @@ pub fn reconcile( let mut prune_candidates: Vec = target_by_name .iter() .filter(|(name, _)| !local_names.contains(*name)) + .filter(|(name, _)| { + !(target_has_vacation && VacationResponse::is_reserved_script_name(name)) + }) .map(|(_, id)| id.clone()) .collect(); prune_candidates.sort(); diff --git a/src/sync/export/tree.rs b/src/sync/export/tree.rs index 188f55e..d83dec3 100644 --- a/src/sync/export/tree.rs +++ b/src/sync/export/tree.rs @@ -13,6 +13,7 @@ use super::common::{create_batch, jid, retry_if_blob_missing, target_query_get}; use super::{Net, Plan, Uploader}; use crate::error::Error; use crate::jmap::wire::JmapId; +use crate::jmap::wire::file_node::NodeKind; use crate::logging::Logger; use crate::sync::import_jmap::mapping::{ FILE_NODE_SELECT, MAILBOX_SELECT, TargetResolver, row_to_file_node, row_to_mailbox, @@ -61,19 +62,33 @@ fn load_local(ctx: &Context, ty: ObjectType) -> Result, Error> { fn load_target(net: &Net, ty: ObjectType) -> Result, Error> { let props: &[&str] = &["role", "name", "parentId", "myRights"]; let list = target_query_get(net, ty, Some(props)).map_err(Error::from)?; + let mut own_rights: HashMap<&str, bool> = HashMap::with_capacity(list.len()); + for v in &list { + if let (Some(id), Some(may)) = ( + v.get("id").and_then(Value::as_str), + v.get("myRights") + .and_then(|r| r.get("mayDelete")) + .and_then(Value::as_bool), + ) { + own_rights.insert(id, may); + } + } Ok(list .iter() .filter_map(|v| { + let id = jid(v)?; + let parent = v.get("parentId").and_then(Value::as_str); + let may_delete = own_rights + .get(id.as_str()) + .or_else(|| parent.and_then(|p| own_rights.get(p))) + .copied() + .unwrap_or(true); Some(TargetNode { - id: jid(v)?, - parent: v.get("parentId").and_then(Value::as_str).map(str::to_owned), + id, + parent: parent.map(str::to_owned), name: v.get("name").and_then(Value::as_str)?.to_owned(), role: v.get("role").and_then(Value::as_str).map(str::to_owned), - may_delete: v - .get("myRights") - .and_then(|r| r.get("mayDelete")) - .and_then(Value::as_bool) - .unwrap_or(true), + may_delete, }) }) .collect()) @@ -441,15 +456,13 @@ fn build_create( .map_err(|e| Error::Partial(e.to_string()))? .map_err(Error::from)?; let mut wire = fnrow.wire; - if let (crate::jmap::wire::file_node::NodeType::File, Some(blob_local)) = - (wire.effective_node_type(), fnrow.blob_local_id) - { - let ct = wire + if let (NodeKind::File(file), Some(blob_local)) = (&mut wire.kind, fnrow.blob_local_id) { + let ct = file .media_type - .clone() - .unwrap_or_else(|| "application/octet-stream".to_owned()); - let id = uploader.upload_with(blob_local, &ct).map_err(Error::from)?; - wire.blob_id = Some(id); + .as_deref() + .unwrap_or("application/octet-stream"); + let id = uploader.upload_with(blob_local, ct).map_err(Error::from)?; + file.blob_id = Some(id); } let mut obj = serde_json::to_value(&wire).map_err(|e| Error::Partial(e.to_string()))?; if let Value::Object(m) = &mut obj { diff --git a/src/sync/export/vacation.rs b/src/sync/export/vacation.rs new file mode 100644 index 0000000..1c9035c --- /dev/null +++ b/src/sync/export/vacation.rs @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use serde_json::json; + +use super::Net; +use crate::db; +use crate::error::Error; +use crate::jmap::request::{SetRequest, set_call}; +use crate::jmap::wire::vacation_response::{SINGLETON_ID, TYPE_NAME}; +use crate::logging::Logger; +use crate::sync::{Context, TypeCounts}; + +pub fn reconcile( + ctx: &Context, + net: &Net, + counts: &mut TypeCounts, + logger: &Logger, +) -> Result { + let Some(vacation) = + db::vacation_response::load(&ctx.conn).map_err(|e| Error::Partial(e.to_string()))? + else { + return Ok(false); + }; + if net.dry_run { + counts.updated += 1; + return Ok(true); + } + let outcome = set_call( + &net.client, + &net.api, + &net.account, + TYPE_NAME, + SetRequest { + update: Some(json!({ SINGLETON_ID: vacation })), + ..Default::default() + }, + &net.limits, + ) + .map_err(Error::from)?; + counts.updated += outcome.updated.len() as u64; + for (id, err) in &outcome.not_updated { + logger.warn(&format!("{TYPE_NAME} {id} not updated: {err}")); + counts.failed += 1; + } + Ok(true) +} diff --git a/src/sync/import_dav.rs b/src/sync/import_dav.rs index 0b2af29..81779d1 100644 --- a/src/sync/import_dav.rs +++ b/src/sync/import_dav.rs @@ -8,9 +8,10 @@ pub mod calcard; pub mod collections; pub mod coordinator; pub mod items; +pub mod replace; pub mod tree; -pub use coordinator::{DavAuth, DavImportConfig, DavKindArg, run, run_reporting}; +pub use coordinator::{DavAuth, DavImportConfig, DavKindArg, SourceChange, run, run_reporting}; use crate::error::Error; use crate::jmap::error::JmapError; diff --git a/src/sync/import_dav/calcard.rs b/src/sync/import_dav/calcard.rs index 88b90bb..a7ccb1f 100644 --- a/src/sync/import_dav/calcard.rs +++ b/src/sync/import_dav/calcard.rs @@ -4,16 +4,22 @@ * SPDX-License-Identifier: Apache-2.0 OR MIT */ +use std::borrow::Cow; +use std::fmt::Write; + +use calcard::Entry; use calcard::icalendar::ICalendar; use calcard::vcard::VCard; use serde_json::{Map, Value}; +const MAX_INVALID_LINE_CHARS: usize = 80; + #[derive(Debug, thiserror::Error)] pub enum CalcardError { - #[error("vcard parse failed")] - VCardParse, - #[error("icalendar parse failed")] - ICalParse, + #[error("vcard parse failed: {0}")] + VCardParse(String), + #[error("icalendar parse failed: {0}")] + ICalParse(String), #[error("jscontact serialisation: {0}")] Json(#[from] serde_json::Error), #[error("no parseable entries in iCalendar")] @@ -29,7 +35,8 @@ pub struct JsContact { } pub fn vcard_to_jscontact(text: &str, item_href: &str) -> Result { - let vcard = VCard::parse(text).map_err(|_| CalcardError::VCardParse)?; + let vcard = VCard::parse(normalize_line_breaks(text)) + .map_err(|e| CalcardError::VCardParse(describe_parse_error(e)))?; let js = vcard.into_jscontact::(); let value: Value = serde_json::from_str(&js.to_string_pretty())?; let uid = value @@ -81,7 +88,8 @@ impl EntryType { } pub fn ical_to_jscalendar_entries(text: &str) -> Result, CalcardError> { - let ical = ICalendar::parse(text).map_err(|_| CalcardError::ICalParse)?; + let ical = ICalendar::parse(normalize_line_breaks(text)) + .map_err(|e| CalcardError::ICalParse(describe_parse_error(e)))?; let js = ical.into_jscalendar::(); let value: Value = serde_json::from_str(&js.to_string_pretty())?; let mut out = Vec::new(); @@ -92,6 +100,67 @@ pub fn ical_to_jscalendar_entries(text: &str) -> Result, Ca Ok(out) } +fn normalize_line_breaks(text: &str) -> Cow<'_, str> { + let mut after_cr = text.split('\r').skip(1).peekable(); + let mut has_bare_cr = false; + while let Some(rest) = after_cr.next() { + if is_bare_cr(rest, after_cr.peek().is_some()) { + has_bare_cr = true; + break; + } + } + if !has_bare_cr { + return Cow::Borrowed(text); + } + let mut parts = text.split('\r').peekable(); + let mut out = String::with_capacity(text.len()); + out.push_str(parts.next().unwrap_or_default()); + while let Some(rest) = parts.next() { + if is_bare_cr(rest, parts.peek().is_some()) { + out.push('\n'); + } + out.push_str(rest); + } + Cow::Owned(out) +} + +fn is_bare_cr(rest: &str, more: bool) -> bool { + let followed_by_cr = rest.is_empty() && more; + !followed_by_cr && !rest.starts_with('\n') +} + +fn describe_parse_error(entry: Entry) -> String { + let mut out = String::new(); + match entry { + Entry::InvalidLine(line) => { + out.push_str("invalid line "); + let mut chars = line.chars(); + let head: String = chars.by_ref().take(MAX_INVALID_LINE_CHARS).collect(); + let _ = write!(out, "{head:?}"); + if chars.next().is_some() { + out.push_str(" (truncated)"); + } + } + Entry::UnexpectedComponentEnd { expected, found } => { + let _ = write!( + out, + "found END:{} where END:{} was expected", + found.as_str(), + expected.as_str() + ); + } + Entry::UnterminatedComponent(name) => { + let _ = write!(out, "{name} is never closed"); + } + Entry::TooManyComponents => out.push_str("too many components"), + Entry::Eof => out.push_str("no BEGIN line found"), + Entry::VCard(_) => out.push_str("found a vCard where an iCalendar was expected"), + Entry::ICalendar(_) => out.push_str("found an iCalendar where a vCard was expected"), + _ => out.push_str("unrecognised content"), + } + out +} + fn collect_entries(value: Value, out: &mut Vec) { match value { Value::Object(map) => { @@ -227,7 +296,53 @@ END:VCALENDAR\r\n"; #[test] fn vcard_garbage_fails_parse() { let err = vcard_to_jscontact("not a vcard", "/x").unwrap_err(); - assert!(matches!(err, CalcardError::VCardParse)); + assert!(matches!(err, CalcardError::VCardParse(_))); + } + + // Kerio Connect sends calendar-data with bare CR line breaks (issue #48). + #[test] + fn ical_with_bare_cr_line_breaks_parses() { + let ical = SAMPLE_ICAL.replace("\r\n", "\r"); + let entries = ical_to_jscalendar_entries(&ical).expect("bare CR parses"); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].data["title"], "Test Event"); + } + + // A folded line split by a bare CR still unfolds. + #[test] + fn ical_with_bare_cr_unfolds_continuation_lines() { + let ical = "BEGIN:VCALENDAR\rVERSION:2.0\rBEGIN:VEVENT\rUID:k1@example\r\ + DTSTAMP:20260101T000000Z\rDTSTART:20260101T090000Z\r\ + SUMMARY:Quarterly\r review\rEND:VEVENT\rEND:VCALENDAR\r"; + let entries = ical_to_jscalendar_entries(ical).expect("parse"); + assert_eq!(entries[0].data["title"], "Quarterly review"); + } + + #[test] + fn vcard_with_bare_cr_line_breaks_parses() { + let vcard = SAMPLE_VCARD.replace("\r\n", "\r"); + let jc = vcard_to_jscontact(&vcard, "/x").expect("bare CR parses"); + assert_eq!(jc.uid, "abc-123"); + } + + #[test] + fn line_break_normalization_borrows_unless_bare_cr() { + for text in ["A\nB\n", "A\r\nB\r\n", "A\r\r\nB\r\r\n", "no breaks"] { + assert!( + matches!(normalize_line_breaks(text), Cow::Borrowed(_)), + "{text:?}" + ); + } + assert_eq!(normalize_line_breaks("A\rB\r\nC\r"), "A\nB\nC\n"); + } + + #[test] + fn ical_parse_error_names_the_offending_line() { + let err = ical_to_jscalendar_entries("Content-Type: text/calendar\r\nBEGIN:VCALENDAR\r\n") + .unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("invalid line"), "{msg}"); + assert!(msg.contains("Content-Type"), "{msg}"); } #[test] diff --git a/src/sync/import_dav/coordinator.rs b/src/sync/import_dav/coordinator.rs index 81db85a..c4336db 100644 --- a/src/sync/import_dav/coordinator.rs +++ b/src/sync/import_dav/coordinator.rs @@ -17,6 +17,7 @@ use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts}; use super::collections; use super::items; +use super::replace::{self, Removed}; use super::tree; #[derive(Debug, Clone, Copy)] @@ -71,6 +72,13 @@ impl DavAuth { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceChange { + Reject, + Allow, + Replace, +} + #[derive(Debug)] pub struct DavImportConfig { pub kind: DavKindArg, @@ -79,7 +87,7 @@ pub struct DavImportConfig { pub allow_cleartext: bool, pub dav_connections: usize, pub multiget_batch: usize, - pub allow_source_change: bool, + pub source_change: SourceChange, } pub fn run(common: CommonConfig, config: DavImportConfig) -> Result { @@ -126,6 +134,7 @@ fn run_into( }; if !common.dry_run + && config.source_change == SourceChange::Reject && let Some((url, acc)) = db::sources::conflicting_source( &conn, config.kind.source_kind(), @@ -133,11 +142,11 @@ fn run_into( &account_id, ) .map_err(|e| Error::Partial(e.to_string()))? - && !config.allow_source_change { return Err(Error::SourceChange(format!( "archive already records {} source ({url}, account {acc}); \ - pass --allow-source-change to import a different account", + pass --allow-source-change to import a different account, \ + or --replace-source to drop the recorded one", config.kind.source_kind() ))); } @@ -152,12 +161,27 @@ fn run_into( } if common.dry_run { + if config.source_change == SourceChange::Replace && logger.enabled(LEVEL_DEFAULT) { + for (_, url, acc) in + db::sources::other_sources(&conn, &key.kind, &session_url, &account_id)? + { + eprintln!( + "dry-run: would remove {} source {url} (account {acc})", + key.kind + ); + } + } let result = run_dry_diff(&conn, &client, &discovery, &config, logger, summary); summary.retries_observed = client.retries_observed(); summary.retry_after_sleeps = client.retry_after_sleeps(); return result; } + let removed = match config.source_change { + SourceChange::Replace => replace::remove_other_sources(&conn, &key, logger)?, + SourceChange::Reject | SourceChange::Allow => Removed::default(), + }; + let username = config.auth.username(); let source_id = db::sources::upsert_source(&conn, &key, Some(&account_id), &username) .map_err(|e| Error::Partial(e.to_string()))?; @@ -175,6 +199,7 @@ fn run_into( }; *summary = phase.summary; + removed.add_to(summary); summary.retries_observed = client.retries_observed(); summary.retry_after_sleeps = client.retry_after_sleeps(); if let Some(e) = phase.error { diff --git a/src/sync/import_dav/replace.rs b/src/sync/import_dav/replace.rs new file mode 100644 index 0000000..4456ee2 --- /dev/null +++ b/src/sync/import_dav/replace.rs @@ -0,0 +1,255 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use rusqlite::{Connection, params}; + +use crate::db::dav_ids::{ADDRESS_BOOK, CALENDAR, CALENDAR_EVENT, CONTACT_CARD, FILE_NODE}; +use crate::db::sources::{self, SourceKey}; +use crate::error::Error; +use crate::logging::{LEVEL_DEFAULT, Logger}; +use crate::sync::Summary; + +const DELETE_MAPPED: [(&str, &str); 4] = [ + ( + CALENDAR_EVENT, + "DELETE FROM calendar_events WHERE id IN + (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)", + ), + ( + CONTACT_CARD, + "DELETE FROM contact_cards WHERE id IN + (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)", + ), + ( + CALENDAR, + "DELETE FROM calendars WHERE id IN + (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)", + ), + ( + ADDRESS_BOOK, + "DELETE FROM address_books WHERE id IN + (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)", + ), +]; + +const DELETE_LEAF_FILE_NODES: &str = "DELETE FROM file_nodes WHERE id IN + (SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2) + AND NOT EXISTS (SELECT 1 FROM file_nodes AS child WHERE child.parent_id = file_nodes.id)"; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct Removed { + pub calendar_events: u64, + pub contact_cards: u64, + pub calendars: u64, + pub address_books: u64, + pub file_nodes: u64, +} + +impl Removed { + fn slot(&mut self, type_name: &str) -> Option<&mut u64> { + match type_name { + CALENDAR_EVENT => Some(&mut self.calendar_events), + CONTACT_CARD => Some(&mut self.contact_cards), + CALENDAR => Some(&mut self.calendars), + ADDRESS_BOOK => Some(&mut self.address_books), + FILE_NODE => Some(&mut self.file_nodes), + _ => None, + } + } + + pub fn add_to(mut self, summary: &mut Summary) { + for (label, counts) in &mut summary.per_type { + if let Some(n) = self.slot(label) { + counts.deleted += std::mem::take(n); + } + } + } +} + +pub fn remove_other_sources( + conn: &Connection, + key: &SourceKey, + logger: Logger, +) -> Result { + let others = sources::other_sources(conn, &key.kind, &key.session_url, &key.account_id)?; + let mut removed = Removed::default(); + if others.is_empty() { + return Ok(removed); + } + let tx = conn.unchecked_transaction()?; + for (source_id, session_url, account_id) in &others { + remove_source(&tx, *source_id, &mut removed)?; + if logger.enabled(LEVEL_DEFAULT) { + eprintln!( + "import: removed {} source {session_url} (account {account_id})", + key.kind + ); + } + } + tx.commit()?; + Ok(removed) +} + +fn remove_source( + conn: &Connection, + source_id: i64, + removed: &mut Removed, +) -> Result<(), rusqlite::Error> { + for (type_name, sql) in DELETE_MAPPED { + let n = conn.execute(sql, params![source_id, type_name])?; + if let Some(slot) = removed.slot(type_name) { + *slot += n as u64; + } + } + let mut leaves = conn.prepare(DELETE_LEAF_FILE_NODES)?; + loop { + let n = leaves.execute(params![source_id, FILE_NODE])?; + if n == 0 { + break; + } + removed.file_nodes += n as u64; + } + sources::delete_source(conn, source_id) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::{dav_ids, init}; + use crate::sync::TypeCounts; + + fn mem() -> Connection { + let c = Connection::open_in_memory().unwrap(); + init::apply_schema(&c).unwrap(); + c.pragma_update(None, "foreign_keys", "ON").unwrap(); + c + } + + fn source(c: &Connection, kind: &str, url: &str) -> (SourceKey, i64) { + let key = SourceKey { + kind: kind.to_owned(), + session_url: url.to_owned(), + account_id: format!("{url}/u/"), + }; + let id = sources::upsert_source(c, &key, None, "u").unwrap(); + (key, id) + } + + fn file_node(c: &Connection, sid: i64, parent: Option, name: &str, href: &str) -> i64 { + c.execute( + "INSERT INTO file_nodes (parent_id, node_type, name, created) + VALUES (?1, 'directory', ?2, '2026-01-01T00:00:00Z')", + params![parent, name], + ) + .unwrap(); + let id = c.last_insert_rowid(); + dav_ids::insert(c, sid, FILE_NODE, href, href, "", id).unwrap(); + id + } + + fn count(c: &Connection, table: &str) -> i64 { + c.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0)) + .unwrap() + } + + #[test] + fn replacing_a_webdav_source_drops_the_old_tree_leaf_first() { + let c = mem(); + let (_, old) = source(&c, "webdav", "https://old"); + let a = file_node(&c, old, None, "a", "/a/"); + let b = file_node(&c, old, Some(a), "b", "/a/b/"); + file_node(&c, old, Some(b), "c", "/a/b/c/"); + let (new_key, new) = source(&c, "webdav", "https://new"); + file_node(&c, new, None, "a", "/a/"); + + let removed = remove_other_sources(&c, &new_key, Logger::from_flags(true, 0)).unwrap(); + + assert_eq!(removed.file_nodes, 3); + assert_eq!(count(&c, "file_nodes"), 1); + assert_eq!(count(&c, "sources"), 1); + assert_eq!(count(&c, "sync_id_dav"), 1); + let kept: i64 = c + .query_row("SELECT source_id FROM sync_id_dav", [], |r| r.get(0)) + .unwrap(); + assert_eq!(kept, new); + } + + #[test] + fn replacing_leaves_other_kinds_untouched() { + let c = mem(); + let (_, cal) = source(&c, "caldav", "https://old"); + c.execute("INSERT INTO calendars (name) VALUES ('Work')", []) + .unwrap(); + let cal_local = c.last_insert_rowid(); + dav_ids::insert(&c, cal, CALENDAR, "/c/", "/c/", "", cal_local).unwrap(); + c.execute( + "INSERT INTO calendar_events (calendar_ids, data) VALUES (?1, '{}')", + params![format!("[{cal_local}]")], + ) + .unwrap(); + let ev = c.last_insert_rowid(); + dav_ids::insert(&c, cal, CALENDAR_EVENT, "/c/", "/c/1.ics", "", ev).unwrap(); + let (_, old_web) = source(&c, "webdav", "https://old"); + file_node(&c, old_web, None, "a", "/a/"); + let (new_web, _) = source(&c, "webdav", "https://new"); + + let removed = remove_other_sources(&c, &new_web, Logger::from_flags(true, 0)).unwrap(); + + assert_eq!(removed.file_nodes, 1); + assert_eq!(removed.calendars + removed.calendar_events, 0); + assert_eq!(count(&c, "calendars"), 1); + assert_eq!(count(&c, "calendar_events"), 1); + assert_eq!(count(&c, "file_nodes"), 0); + } + + #[test] + fn replacing_a_caldav_source_drops_its_calendars_and_events() { + let c = mem(); + let (_, old) = source(&c, "caldav", "https://old"); + c.execute("INSERT INTO calendars (name) VALUES ('Work')", []) + .unwrap(); + let cal_local = c.last_insert_rowid(); + dav_ids::insert(&c, old, CALENDAR, "/c/", "/c/", "", cal_local).unwrap(); + c.execute( + "INSERT INTO calendar_events (calendar_ids, data) VALUES (?1, '{}')", + params![format!("[{cal_local}]")], + ) + .unwrap(); + let ev = c.last_insert_rowid(); + dav_ids::insert(&c, old, CALENDAR_EVENT, "/c/", "/c/1.ics", "", ev).unwrap(); + let (new_key, _) = source(&c, "caldav", "https://new"); + + let removed = remove_other_sources(&c, &new_key, Logger::from_flags(true, 0)).unwrap(); + + assert_eq!((removed.calendars, removed.calendar_events), (1, 1)); + assert_eq!(count(&c, "calendars"), 0); + assert_eq!(count(&c, "calendar_events"), 0); + assert_eq!(count(&c, "sync_id_dav"), 0); + } + + #[test] + fn no_other_source_is_a_no_op() { + let c = mem(); + let (key, _) = source(&c, "webdav", "https://only"); + let removed = remove_other_sources(&c, &key, Logger::from_flags(true, 0)).unwrap(); + assert_eq!(removed, Removed::default()); + assert_eq!(count(&c, "sources"), 1); + } + + #[test] + fn removed_counts_land_on_matching_summary_rows() { + let mut summary = Summary::default(); + summary.per_type.push((FILE_NODE, TypeCounts::default())); + Removed { + file_nodes: 4, + calendars: 9, + ..Removed::default() + } + .add_to(&mut summary); + assert_eq!(summary.per_type.len(), 1); + assert_eq!(summary.per_type[0].1.deleted, 4); + } +} diff --git a/src/sync/import_imap/coordinator.rs b/src/sync/import_imap/coordinator.rs index b5e4c33..3643e5a 100644 --- a/src/sync/import_imap/coordinator.rs +++ b/src/sync/import_imap/coordinator.rs @@ -379,7 +379,7 @@ fn run_into( let local_mailboxes = db::imap_ids::mailbox_folders(&conn, source_id)?; let server_folder_set: HashSet = resolved.iter().map(|f| f.name.clone()).collect(); - upsert_mailboxes(&mut conn, source_id, &resolved, &mut mailbox_counts)?; + upsert_mailboxes(&mut conn, source_id, &resolved, &mut mailbox_counts, logger)?; let server_delimiter = resolved.iter().find_map(|f| f.delimiter).unwrap_or('/'); @@ -418,7 +418,7 @@ fn run_into( ) .map_err(|e| Error::Connection(format!("worker pool: {e}")))?; - for (i, folder) in resolved.iter().enumerate() { + for (i, folder) in resolved.iter().filter(|f| f.selectable).enumerate() { if i > 0 { let _ = control_run_collect(&mut client, &control_ctx, "NOOP"); } @@ -585,16 +585,29 @@ fn upsert_mailboxes( source_id: i64, folders: &[ResolvedFolder], counts: &mut TypeCounts, + logger: Logger, ) -> Result<(), Error> { let tx = conn.transaction()?; let mut local_ids: HashMap = HashMap::new(); for folder in folders { let parent_local = match &folder.parent_path { - Some(p) => local_ids.get(p.as_str()).copied().or_else(|| { - db::imap_ids::local_for_mailbox(&tx, source_id, p) - .ok() - .flatten() - }), + Some(p) => { + let found = match local_ids.get(p.as_str()) { + Some(id) => Some(*id), + None => db::imap_ids::local_for_mailbox(&tx, source_id, p)?, + }; + if found.is_none() { + log_at( + logger, + LEVEL_DEFAULT, + &format!( + "folder {:?}: parent {p:?} could not be resolved; storing it at the top level", + folder.name + ), + ); + } + found + } None => None, }; let existing = db::imap_ids::local_for_mailbox(&tx, source_id, &folder.name)?; @@ -1129,7 +1142,7 @@ fn dry_run_summary( ); } - for folder in folders { + for folder in folders.iter().filter(|f| f.selectable) { let select_resp = match select_folder(client, control_ctx, folder) { Ok(r) => r, Err(e) => { diff --git a/src/sync/import_imap/folders.rs b/src/sync/import_imap/folders.rs index cea6a18..9020a41 100644 --- a/src/sync/import_imap/folders.rs +++ b/src/sync/import_imap/folders.rs @@ -39,6 +39,7 @@ pub struct ResolvedFolder { pub delimiter: Option, pub role: Option<&'static str>, pub subscribed: bool, + pub selectable: bool, pub status: Option, } @@ -131,16 +132,16 @@ pub fn apply_filters( folders: Vec, filters: &FolderFilters, ) -> Vec { - let mut keep: Vec = folders - .into_iter() - .filter(|f| f.selectable) - .filter(|f| match_filters(&f.name, filters)) - .collect(); - if filters.subscribed_only { - keep.retain(|f| f.subscribed); - } - let mut resolved: Vec = Vec::with_capacity(keep.len()); - for f in keep { + let mut resolved: Vec = Vec::with_capacity(folders.len()); + let mut others: HashMap = HashMap::new(); + for f in folders { + if !f.selectable + || !match_filters(&f.name, filters) + || (filters.subscribed_only && !f.subscribed) + { + others.insert(f.name.clone(), f); + continue; + } let role = automap::role_for_folder( &f.name, &f.attributes, @@ -153,21 +154,77 @@ pub fn apply_filters( .iter() .any(|s| s.eq_ignore_ascii_case(r)) { + others.insert(f.name.clone(), f); continue; } - let delim = f.delimiter; - let (leaf, parent_path) = split_parent(&f.name, delim); + let (leaf, parent_path) = split_parent(&f.name, f.delimiter); resolved.push(ResolvedFolder { name: f.name, wire_name: f.wire_name, leaf, parent_path, - delimiter: delim, + delimiter: f.delimiter, role, subscribed: f.subscribed, + selectable: true, status: f.status, }); } + let mut present: HashSet = resolved.iter().map(|f| f.name.clone()).collect(); + let mut structural: Vec = Vec::new(); + for child in &resolved { + let mut parent = child.parent_path.clone(); + let mut child_wire = child.wire_name.clone(); + let mut delim = child.delimiter; + while let Some(name) = parent { + if present.contains(&name) { + break; + } + let ancestor = match others.remove(&name) { + Some(d) => { + let delimiter = d.delimiter.or(delim); + let (leaf, parent_path) = split_parent(&d.name, delimiter); + ResolvedFolder { + name: d.name, + wire_name: d.wire_name, + leaf, + parent_path, + delimiter, + role: None, + subscribed: d.subscribed, + selectable: false, + status: None, + } + } + None => { + let wire_name = match delim { + Some(d) => child_wire + .rsplit_once(d) + .map_or_else(|| name.clone(), |(p, _)| p.to_owned()), + None => name.clone(), + }; + let (leaf, parent_path) = split_parent(&name, delim); + ResolvedFolder { + name, + wire_name, + leaf, + parent_path, + delimiter: delim, + role: None, + subscribed: false, + selectable: false, + status: None, + } + } + }; + present.insert(ancestor.name.clone()); + parent = ancestor.parent_path.clone(); + child_wire.clone_from(&ancestor.wire_name); + delim = ancestor.delimiter; + structural.push(ancestor); + } + } + resolved.extend(structural); resolved } @@ -447,6 +504,7 @@ mod tests { delimiter: Some('/'), role: None, subscribed: false, + selectable: true, status: None, }, ResolvedFolder { @@ -457,6 +515,7 @@ mod tests { delimiter: Some('/'), role: Some("inbox"), subscribed: true, + selectable: true, status: None, }, ResolvedFolder { @@ -467,6 +526,7 @@ mod tests { delimiter: Some('/'), role: None, subscribed: false, + selectable: true, status: None, }, ]; @@ -508,4 +568,131 @@ mod tests { let folders = collect_from_list(&resps, false).unwrap(); assert_eq!(folders.len(), 1); } + + fn find<'a>(res: &'a [ResolvedFolder], name: &str) -> &'a ResolvedFolder { + res.iter().find(|f| f.name == name).unwrap() + } + + #[test] + fn noselect_parent_is_kept_as_structural() { + let folders = collect_from_list( + &[ + lst("INBOX", "/", &["\\HasChildren"]), + lst("INBOX/Projects", "/", &["\\Noselect", "\\HasChildren"]), + lst("INBOX/Projects/Invoices", "/", &["\\HasNoChildren"]), + ], + false, + ) + .unwrap(); + let mut res = apply_filters(folders, &filters_default()); + sort_by_depth(&mut res); + let names: Vec<&str> = res.iter().map(|f| f.name.as_str()).collect(); + assert_eq!( + names, + ["INBOX", "INBOX/Projects", "INBOX/Projects/Invoices"] + ); + let projects = find(&res, "INBOX/Projects"); + assert!(!projects.selectable); + assert_eq!(projects.role, None); + assert_eq!(projects.leaf, "Projects"); + assert_eq!(projects.parent_path.as_deref(), Some("INBOX")); + assert!(find(&res, "INBOX/Projects/Invoices").selectable); + } + + #[test] + fn noselect_without_kept_children_is_dropped() { + let folders = collect_from_list( + &[lst("INBOX", "/", &[]), lst("Empty", "/", &["\\Noselect"])], + false, + ) + .unwrap(); + let res = apply_filters(folders, &filters_default()); + assert_eq!(res.len(), 1); + } + + #[test] + fn consecutive_structural_ancestors_and_synthesised_ones() { + let folders = collect_from_list( + &[lst("A", "/", &["\\NonExistent"]), lst("A/B/C/D", "/", &[])], + false, + ) + .unwrap(); + let mut res = apply_filters(folders, &filters_default()); + sort_by_depth(&mut res); + let names: Vec<&str> = res.iter().map(|f| f.name.as_str()).collect(); + assert_eq!(names, ["A", "A/B", "A/B/C", "A/B/C/D"]); + assert!( + res.iter() + .take(3) + .all(|f| !f.selectable && f.role.is_none()) + ); + assert_eq!(find(&res, "A/B/C").wire_name, "A/B/C"); + assert_eq!(find(&res, "A/B").leaf, "B"); + } + + #[test] + fn trailing_space_and_same_leaf_under_different_parents() { + let folders = collect_from_list( + &[ + lst("Work ", ".", &["\\Noselect"]), + lst("Work", ".", &[]), + lst("Work .Archive", ".", &[]), + lst("Work.Archive", ".", &[]), + ], + false, + ) + .unwrap(); + let res = apply_filters(folders, &filters_default()); + assert_eq!(res.len(), 4); + let spaced = find(&res, "Work "); + assert!(!spaced.selectable); + assert_eq!(spaced.leaf, "Work "); + assert_eq!( + find(&res, "Work .Archive").parent_path.as_deref(), + Some("Work ") + ); + assert_eq!( + find(&res, "Work.Archive").parent_path.as_deref(), + Some("Work") + ); + assert!(find(&res, "Work").selectable); + } + + #[test] + fn synthesised_ancestor_derives_wire_name_from_child() { + let folders = collect_from_list(&[lst("&ZeVnLIqe-/Sub", "/", &[])], false).unwrap(); + let res = apply_filters(folders, &filters_default()); + let parent = find(&res, "日本語"); + assert_eq!(parent.wire_name, "&ZeVnLIqe-"); + assert!(!parent.selectable); + } + + #[test] + fn filters_keep_needed_ancestors_as_structural() { + let list = [ + lst("INBOX", "/", &[]), + lst("Projects", "/", &[]), + lst("Projects/Alpha", "/", &["\\Noselect"]), + lst("Projects/Alpha/Docs", "/", &["\\Subscribed"]), + lst("Other", "/", &[]), + ]; + let check = |res: Vec| { + assert_eq!(res.len(), 3, "{res:?}"); + assert!(!find(&res, "Projects").selectable); + assert!(!find(&res, "Projects/Alpha").selectable); + assert!(find(&res, "Projects/Alpha/Docs").selectable); + }; + let mut f = filters_default(); + f.include = vec![Regex::new("Docs$").unwrap()]; + check(apply_filters(collect_from_list(&list, false).unwrap(), &f)); + let mut f = filters_default(); + f.exclude = vec![Regex::new("^(INBOX|Other|Projects)$").unwrap()]; + check(apply_filters(collect_from_list(&list, false).unwrap(), &f)); + let mut f = filters_default(); + f.explicit = vec!["Projects/Alpha/Docs".to_owned()]; + check(apply_filters(collect_from_list(&list, false).unwrap(), &f)); + let mut f = filters_default(); + f.subscribed_only = true; + check(apply_filters(collect_from_list(&list, false).unwrap(), &f)); + } } diff --git a/src/sync/import_jmap.rs b/src/sync/import_jmap.rs index d9e96a1..d15714f 100644 --- a/src/sync/import_jmap.rs +++ b/src/sync/import_jmap.rs @@ -26,14 +26,19 @@ use crate::jmap::blobxfer; use crate::jmap::connect::{self, Connected}; use crate::jmap::error::JmapError; use crate::jmap::http::{Auth, HttpClient}; -use crate::jmap::request::{get_all, get_changes, get_objects, get_state, query_all_ids}; +use crate::jmap::request::{ + URN_VACATION_RESPONSE, get_all, get_changes, get_objects, get_state, query_all_ids, +}; use crate::jmap::session::{Limits, Session}; use crate::jmap::wire::JmapId; use crate::jmap::wire::common::parse_utc_date; use crate::jmap::wire::email::Email; -use crate::jmap::wire::file_node::{FileNode, NodeType}; +use crate::jmap::wire::file_node::{FileNode, FileProps, NodeKind}; use crate::jmap::wire::sieve_script::SieveScript; +use crate::jmap::wire::vacation_response::{self, VacationResponse}; use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; +use crate::sync::emailmeta::email_meta_from_blob; +use crate::sync::keys::index_to_json; use crate::sync::{CommonConfig, Context, ImportConfig, RunOutcome, Summary, TypeCounts}; use crate::types::ObjectType; @@ -66,12 +71,13 @@ fn get_props(ty: ObjectType) -> Option<&'static [&'static str]> { ObjectType::Email => Some(&["blobId", "receivedAt", "mailboxIds", "keywords"]), ObjectType::SieveScript => Some(&["name", "isActive", "blobId"]), ObjectType::FileNode => Some(&[ + "@type", "parentId", - "nodeType", "blobId", "target", "name", - "type", + "mediaType", + "executable", "created", "modified", "isSubscribed", @@ -107,17 +113,33 @@ impl LocalResolver for DbResolver<'_> { } } -struct PrefetchedBlobs<'a> { - conn: &'a Connection, - bytes: &'a HashMap>, +struct EmailMeta { + match_key: String, + date_header: Option, } -impl BlobIntern for PrefetchedBlobs<'_> { +struct StoredBlob { + local_id: i64, + email: Option, +} + +#[derive(Default)] +struct StoredBlobs(HashMap); + +impl StoredBlobs { + fn get(&self, jmap_blob_id: &str, kind: &str) -> Result<&StoredBlob, JmapError> { + self.0 + .get(jmap_blob_id) + .ok_or_else(|| JmapError::malformed(format!("{kind} blob missing"))) + } +} + +impl BlobIntern for &StoredBlobs { fn intern(&mut self, jmap_blob_id: &str) -> Result { - let data = self.bytes.get(jmap_blob_id).ok_or_else(|| { - JmapError::malformed(format!("blob {jmap_blob_id} was not prefetched")) - })?; - Ok(db::blobs::intern_blob(self.conn, data)?) + self.0 + .get(jmap_blob_id) + .map(|b| b.local_id) + .ok_or_else(|| JmapError::malformed(format!("blob {jmap_blob_id} was not prefetched"))) } } @@ -242,6 +264,34 @@ fn run_into( summary.per_type.push((ty.jmap_name(), counts)); } + if let Some(source_id) = source_id + && !ctx.dry_run() + && config + .objects + .as_ref() + .is_none_or(|s| s.contains(&ObjectType::SieveScript)) + && net.session.may_support(&net.account, URN_VACATION_RESPONSE) + { + if logger.enabled(LEVEL_DEFAULT) { + eprintln!("import: {} ...", vacation_response::TYPE_NAME); + } + let mut counts = TypeCounts::default(); + match import_vacation_response(&ctx.conn, &net, source_id) { + Ok(fetched) => counts.fetched = fetched, + Err(e) if e.aborts_run() => return Err(e), + Err(e) => { + logger.warn(&format!( + "type {} aborted: {e}; continuing (run will exit 5)", + vacation_response::TYPE_NAME + )); + counts.failed += 1; + } + } + summary + .per_type + .push((vacation_response::TYPE_NAME, counts)); + } + if ctx.dry_run() { print_dry_run(&dry_rows); *summary = Summary::default(); @@ -259,6 +309,28 @@ fn run_into( Ok(()) } +fn import_vacation_response(conn: &Connection, net: &Net, source_id: i64) -> Result { + let got = get_all::( + &net.client, + &net.api, + &net.account, + vacation_response::TYPE_NAME, + ) + .map_err(Error::from)?; + match got.list.into_iter().find(|v| !v.is_unset()) { + Some(vacation) => { + db::vacation_response::store(conn, source_id, &vacation) + .map_err(|e| Error::Partial(e.to_string()))?; + Ok(1) + } + None => { + db::vacation_response::clear(conn, source_id) + .map_err(|e| Error::Partial(e.to_string()))?; + Ok(0) + } + } +} + fn work_list(config: &ImportConfig, connected: &Connected) -> Vec { let selected = config.objects.as_ref(); IMPORT_ORDER @@ -462,9 +534,9 @@ fn insert_objects( for batch in ordered.chunks(200) { let blob_refs = blob_references(ty, batch.iter().map(|&i| &objects[i])); let blobs = if blob_refs.is_empty() { - HashMap::new() + StoredBlobs::default() } else { - download_blobs(net, blob_refs, threads, logger) + download_blobs(&ctx.conn, net, ty, blob_refs, threads, logger)? }; let tx = ctx .conn @@ -532,7 +604,7 @@ fn blob_references<'a>(ty: ObjectType, objects: impl Iterator) && seen.insert(b.to_owned()) { let mt = o - .get("type") + .get("mediaType") .and_then(Value::as_str) .unwrap_or("application/octet-stream") .to_owned(); @@ -562,11 +634,13 @@ fn blob_references<'a>(ty: ObjectType, objects: impl Iterator) } fn download_blobs( + conn: &Connection, net: &Net, + ty: ObjectType, refs: Vec, threads: usize, logger: &Logger, -) -> HashMap> { +) -> Result { let workers = effective_workers(threads, &net.limits, true); let net_arc = Arc::new(net.clone()); let pool: Pool = Pool::new(workers, { @@ -587,11 +661,23 @@ fn download_blobs( for r in refs { pool.submit(r); } - let mut out = HashMap::new(); - for (blob_id, res) in pool.finish() { + let mut out = HashMap::with_capacity(total); + for _ in 0..total { + let Ok((blob_id, res)) = pool.results().recv() else { + break; + }; match res { Ok(bytes) => { - out.insert(blob_id, bytes); + let local_id = db::blobs::intern_blob(conn, &bytes) + .map_err(|e| Error::Partial(e.to_string()))?; + let email = (ty == ObjectType::Email).then(|| { + let (index, date_header) = email_meta_from_blob(&bytes); + EmailMeta { + match_key: index_to_json(&index), + date_header, + } + }); + out.insert(blob_id, StoredBlob { local_id, email }); } Err(e) => { logger.warn(&format!("blob {blob_id} download failed: {e}")); @@ -601,7 +687,7 @@ fn download_blobs( if logger.enabled(LEVEL_PROGRESS) { eprintln!("import: downloaded {}/{} blobs", out.len(), total); } - out + Ok(StoredBlobs(out)) } fn order_objects( @@ -649,7 +735,7 @@ fn insert_one( ty: ObjectType, source_id: i64, obj: &Value, - blobs: &HashMap>, + blobs: &StoredBlobs, logger: &Logger, ) -> Result { let resolver = DbResolver { conn, source_id }; @@ -676,14 +762,13 @@ fn insert_one( } ObjectType::Email => { let mut w: Email = serde_json::from_value(obj.clone())?; - let data = blobs - .get(&w.blob_id.0) + let stored = blobs.get(&w.blob_id.0, "email")?; + let meta = stored + .email + .as_ref() .ok_or_else(|| JmapError::malformed("email blob missing"))?; - let blob_local = db::blobs::intern_blob(conn, data)?; - let (index, date_header) = crate::sync::emailmeta::email_meta_from_blob(data); - let mm = crate::sync::keys::index_to_json(&index); if w.received_at.is_none() { - w.received_at = date_header.as_deref().and_then(parse_utc_date); + w.received_at = meta.date_header.as_deref().and_then(parse_utc_date); let id = obj.get("id").and_then(Value::as_str).unwrap_or("?"); let raw = obj .get("receivedAt") @@ -697,37 +782,31 @@ fn insert_one( } )); } - mapping::insert_email(conn, &w, blob_local, &mm, &resolver) + mapping::insert_email(conn, &w, stored.local_id, &meta.match_key, &resolver) } ObjectType::SieveScript => { let w: SieveScript = serde_json::from_value(obj.clone())?; - let data = blobs - .get(&w.blob_id.0) - .ok_or_else(|| JmapError::malformed("sieve blob missing"))?; - let blob_local = db::blobs::intern_blob(conn, data)?; + let blob_local = blobs.get(&w.blob_id.0, "sieve")?.local_id; mapping::insert_sieve_script(conn, &w, blob_local) } ObjectType::FileNode => { let w: FileNode = serde_json::from_value(obj.clone())?; - let blob_local = match (w.effective_node_type(), &w.blob_id) { - (NodeType::File, Some(b)) => { - let data = blobs - .get(&b.0) - .ok_or_else(|| JmapError::malformed("file blob missing"))?; - Some(db::blobs::intern_blob(conn, data)?) - } + let blob_local = match &w.kind { + NodeKind::File(FileProps { + blob_id: Some(b), .. + }) => Some(blobs.get(&b.0, "file")?.local_id), _ => None, }; mapping::insert_file_node(conn, &w, blob_local, &resolver) } ObjectType::ContactCard => { let w = serde_json::from_value(obj.clone())?; - let mut bi = PrefetchedBlobs { conn, bytes: blobs }; + let mut bi = blobs; mapping::insert_contact_card(conn, &w, &resolver, &mut bi) } ObjectType::CalendarEvent => { let w = serde_json::from_value(obj.clone())?; - let mut bi = PrefetchedBlobs { conn, bytes: blobs }; + let mut bi = blobs; mapping::insert_calendar_event(conn, &w, &resolver, &mut bi) } } @@ -876,9 +955,9 @@ fn update_objects( for batch in objects.chunks(200) { let blob_refs = blob_references(ty, batch.iter()); let blobs = if blob_refs.is_empty() { - HashMap::new() + StoredBlobs::default() } else { - download_blobs(net, blob_refs, threads, logger) + download_blobs(&ctx.conn, net, ty, blob_refs, threads, logger)? }; let tx = ctx .conn @@ -915,7 +994,7 @@ fn update_one( source_id: i64, local_id: i64, obj: &Value, - blobs: &HashMap>, + blobs: &StoredBlobs, ) -> Result { let resolver = DbResolver { conn, source_id }; match ty { @@ -942,33 +1021,27 @@ fn update_one( ObjectType::Email => mapping::update_email(conn, local_id, obj, &resolver), ObjectType::SieveScript => { let w: SieveScript = serde_json::from_value(obj.clone())?; - let data = blobs - .get(&w.blob_id.0) - .ok_or_else(|| JmapError::malformed("sieve blob missing"))?; - let blob_local = db::blobs::intern_blob(conn, data)?; + let blob_local = blobs.get(&w.blob_id.0, "sieve")?.local_id; mapping::update_sieve_script(conn, local_id, &w, blob_local) } ObjectType::FileNode => { let w: FileNode = serde_json::from_value(obj.clone())?; - let blob_local = match (w.effective_node_type(), &w.blob_id) { - (NodeType::File, Some(b)) => { - let data = blobs - .get(&b.0) - .ok_or_else(|| JmapError::malformed("file blob missing"))?; - Some(db::blobs::intern_blob(conn, data)?) - } + let blob_local = match &w.kind { + NodeKind::File(FileProps { + blob_id: Some(b), .. + }) => Some(blobs.get(&b.0, "file")?.local_id), _ => None, }; mapping::update_file_node(conn, local_id, &w, blob_local, &resolver) } ObjectType::ContactCard => { let w = serde_json::from_value(obj.clone())?; - let mut bi = PrefetchedBlobs { conn, bytes: blobs }; + let mut bi = blobs; mapping::update_contact_card(conn, local_id, &w, &resolver, &mut bi) } ObjectType::CalendarEvent => { let w = serde_json::from_value(obj.clone())?; - let mut bi = PrefetchedBlobs { conn, bytes: blobs }; + let mut bi = blobs; mapping::update_calendar_event(conn, local_id, &w, &resolver, &mut bi) } } diff --git a/src/sync/import_jmap/mapping.rs b/src/sync/import_jmap/mapping.rs index 78ca2ee..ce9327f 100644 --- a/src/sync/import_jmap/mapping.rs +++ b/src/sync/import_jmap/mapping.rs @@ -17,7 +17,7 @@ use crate::jmap::wire::calendar::Calendar; use crate::jmap::wire::calendar_event::CalendarEvent; use crate::jmap::wire::contact_card::ContactCard; use crate::jmap::wire::email::Email; -use crate::jmap::wire::file_node::FileNode; +use crate::jmap::wire::file_node::{DirectoryProps, FileNode, FileProps, NodeKind, SymLinkProps}; use crate::jmap::wire::identity::Identity; use crate::jmap::wire::mailbox::Mailbox; use crate::jmap::wire::participant_identity::ParticipantIdentity; @@ -318,36 +318,62 @@ pub fn insert_file_node( resolver: &impl LocalResolver, ) -> Result { let parent = opt_parent(resolver, ObjectType::FileNode, &wire.parent_id); - let node_type = serde_json::to_value(wire.effective_node_type())? - .as_str() - .unwrap_or("file") - .to_owned(); - let target = match &wire.target { - Some(t) => Some(serde_json::to_string(t)?), - None => None, - }; + let cols = FileNodeColumns::new(&wire.kind)?; conn.execute( "INSERT INTO file_nodes (parent_id, node_type, blob_id, target, name, media_type, - created, modified, is_subscribed, role) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", + created, modified, is_subscribed, role, executable) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", params![ parent, - node_type, + wire.kind.db_name(), blob_local_id, - target, + cols.target, wire.name, - wire.media_type, + cols.media_type, format_utc(&wire.created)?, opt_format_utc(&wire.modified)?, - wire.is_subscribed as i64, - wire.role + cols.is_subscribed as i64, + cols.role, + cols.executable as i64 ], )?; Ok(conn.last_insert_rowid()) } +struct FileNodeColumns<'a> { + target: Option, + media_type: Option<&'a str>, + is_subscribed: bool, + role: Option<&'a str>, + executable: bool, +} + +impl<'a> FileNodeColumns<'a> { + fn new(kind: &'a NodeKind) -> Result { + let mut cols = FileNodeColumns { + target: None, + media_type: None, + is_subscribed: true, + role: None, + executable: false, + }; + match kind { + NodeKind::File(f) => { + cols.media_type = f.media_type.as_deref(); + cols.executable = f.executable; + } + NodeKind::Directory(d) => { + cols.role = d.role.as_deref(); + cols.is_subscribed = d.is_subscribed; + } + NodeKind::SymLink(l) => cols.target = Some(serde_json::to_string(&l.target)?), + } + Ok(cols) + } +} + pub const FILE_NODE_SELECT: &str = "SELECT id, parent_id, node_type, blob_id, target, name, \ - media_type, created, modified, is_subscribed, role FROM file_nodes"; + media_type, created, modified, is_subscribed, role, executable FROM file_nodes"; pub fn insert_contact_card( conn: &Connection, @@ -587,31 +613,27 @@ pub fn update_file_node( resolver: &impl LocalResolver, ) -> Result { let parent = opt_parent(resolver, ObjectType::FileNode, &wire.parent_id); - let node_type = serde_json::to_value(wire.effective_node_type())? - .as_str() - .unwrap_or("file") - .to_owned(); - let target = match &wire.target { - Some(t) => Some(serde_json::to_string(t)?), - None => None, - }; + let cols = FileNodeColumns::new(&wire.kind)?; let n = conn.execute( "UPDATE file_nodes SET parent_id = ?1, node_type = ?2, blob_id = ?3, target = ?4, - name = ?5, media_type = ?6, created = ?7, modified = ?8, is_subscribed = ?9, role = ?10 - WHERE id = ?11 AND (parent_id IS NOT ?1 OR node_type IS NOT ?2 OR blob_id IS NOT ?3 + name = ?5, media_type = ?6, created = ?7, modified = ?8, is_subscribed = ?9, + role = ?10, executable = ?11 + WHERE id = ?12 AND (parent_id IS NOT ?1 OR node_type IS NOT ?2 OR blob_id IS NOT ?3 OR target IS NOT ?4 OR name IS NOT ?5 OR media_type IS NOT ?6 OR created IS NOT ?7 - OR modified IS NOT ?8 OR is_subscribed IS NOT ?9 OR role IS NOT ?10)", + OR modified IS NOT ?8 OR is_subscribed IS NOT ?9 OR role IS NOT ?10 + OR executable IS NOT ?11)", params![ parent, - node_type, + wire.kind.db_name(), blob_local_id, - target, + cols.target, wire.name, - wire.media_type, + cols.media_type, format_utc(&wire.created)?, opt_format_utc(&wire.modified)?, - wire.is_subscribed as i64, - wire.role, + cols.is_subscribed as i64, + cols.role, + cols.executable as i64, local_id ], )?; @@ -845,26 +867,36 @@ pub fn row_to_file_node( ), None => None, }; - let node_type = Some(serde_json::from_value(Value::from( - row.get::<_, String>(2)?, - ))?); - let target: Option> = from_opt_json(row.get::<_, Option>(4)?)?; - let blob_local_id: Option = row.get(3)?; + let node_type: String = row.get(2)?; + let kind = match node_type.as_str() { + "file" => NodeKind::File(FileProps { + blob_id: None, + media_type: row.get(6)?, + executable: row.get::<_, i64>(11)? != 0, + }), + "directory" => NodeKind::Directory(DirectoryProps { + role: row.get(10)?, + is_subscribed: row.get::<_, i64>(9)? != 0, + }), + "symlink" => NodeKind::SymLink(SymLinkProps { + target: from_opt_json(row.get::<_, Option>(4)?)?.unwrap_or_default(), + }), + other => { + return Err(JmapError::malformed(format!( + "unknown file_node type {other}" + ))); + } + }; Ok(FileNodeRow { wire: FileNode { id: None, parent_id, - node_type, - blob_id: None, - target, name: row.get(5)?, - media_type: row.get(6)?, created: parse_utc(&row.get::<_, String>(7)?)?, modified: opt_parse_utc(row.get::<_, Option>(8)?)?, - is_subscribed: row.get::<_, i64>(9)? != 0, - role: row.get(10)?, + kind, }, - blob_local_id, + blob_local_id: row.get(3)?, }) } @@ -1301,7 +1333,7 @@ mod tests { }; let node: FileNode = serde_json::from_value(serde_json::json!({ "id": "F1", - "nodeType": "symlink", + "@type": "SymLink", "target": ["..", "etc", "hosts"], "name": "link", "created": "2022-02-02T02:02:02Z" @@ -1410,7 +1442,7 @@ mod tests { to_target: HashMap::new(), }; let dir: FileNode = serde_json::from_value(serde_json::json!({ - "id": "D1", "nodeType": "directory", "name": "Docs", + "id": "D1", "@type": "Directory", "name": "Docs", "created": "2022-02-02T02:02:02Z" })) .unwrap(); @@ -1418,8 +1450,8 @@ mod tests { res.to_local .insert((ObjectType::FileNode, "D1".to_owned()), dirl); let file: FileNode = serde_json::from_value(serde_json::json!({ - "id": "F1", "nodeType": "file", "parentId": "D1", "name": "a.bin", - "type": "application/octet-stream", "created": "2022-02-02T02:02:02Z" + "id": "F1", "@type": "File", "parentId": "D1", "name": "a.bin", + "mediaType": "application/octet-stream", "created": "2022-02-02T02:02:02Z" })) .unwrap(); let blob = crate::db::blobs::intern_blob(&c, b"bytes").unwrap(); @@ -1439,6 +1471,72 @@ mod tests { assert_eq!(got.blob_local_id, Some(blob)); } + // Each @type serializes only the properties the draft defines for it. + #[test] + fn file_node_row_to_wire_emits_only_type_specific_properties() { + let c = mem(); + let mut res = MapResolver { + to_local: HashMap::new(), + to_target: HashMap::new(), + }; + let nodes = [ + serde_json::json!({"id": "D1", "@type": "Directory", "name": "Docs", + "role": "documents", "isSubscribed": false, + "created": "2022-02-02T02:02:02Z"}), + serde_json::json!({"id": "F1", "@type": "File", "name": "run.sh", + "blobId": "B1", "mediaType": "text/x-sh", "executable": true, "size": 3, + "created": "2022-02-02T02:02:02Z"}), + serde_json::json!({"id": "L1", "@type": "SymLink", "name": "latest", + "target": ["", "Docs"], "created": "2022-02-02T02:02:02Z"}), + ]; + let blob = crate::db::blobs::intern_blob(&c, b"#!\n").unwrap(); + let mut out = Vec::new(); + for (i, n) in nodes.into_iter().enumerate() { + let w: FileNode = serde_json::from_value(n).unwrap(); + let blob_local = matches!(w.kind, NodeKind::File(_)).then_some(blob); + let l = insert_file_node(&c, &w, blob_local, &res).unwrap(); + res.to_target + .insert((ObjectType::FileNode, l), format!("T{i}")); + let row = c + .query_row( + &format!("{FILE_NODE_SELECT} WHERE id = ?1"), + params![l], + |r| Ok(row_to_file_node(r, &res)), + ) + .unwrap() + .unwrap(); + out.push(serde_json::to_value(&row.wire).unwrap()); + } + let keys = |v: &Value| { + let mut k: Vec = v.as_object().unwrap().keys().cloned().collect(); + k.sort_unstable(); + k + }; + assert_eq!( + keys(&out[0]), + ["@type", "created", "isSubscribed", "name", "role"] + ); + assert_eq!(out[0]["@type"], "Directory"); + assert_eq!(out[0]["isSubscribed"], false); + assert_eq!( + keys(&out[1]), + ["@type", "created", "executable", "mediaType", "name"] + ); + assert_eq!(out[1]["executable"], true); + assert_eq!(out[1]["mediaType"], "text/x-sh"); + assert_eq!(keys(&out[2]), ["@type", "created", "name", "target"]); + assert_eq!(out[2]["@type"], "SymLink"); + assert_eq!(out[2]["target"], serde_json::json!(["", "Docs"])); + } + + #[test] + fn file_node_rejects_unknown_type() { + let r: Result = serde_json::from_value(serde_json::json!({ + "id": "X1", "@type": "Socket", "name": "s", "created": "2022-02-02T02:02:02Z" + })); + assert!(r.is_err()); + } + fn one_row(c: &Connection, table: &str) -> i64 { c.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0)) .unwrap() @@ -1653,30 +1751,31 @@ mod tests { to_target: HashMap::new(), }; let file: FileNode = serde_json::from_value(serde_json::json!({ - "id":"F1","nodeType":"file","name":"a.bin", - "type":"application/octet-stream","created":"2022-02-02T02:02:02Z" + "id":"F1","@type":"File","name":"a.bin", + "mediaType":"application/octet-stream","created":"2022-02-02T02:02:02Z" })) .unwrap(); let blob1 = crate::db::blobs::intern_blob(&c, b"v1").unwrap(); let local = insert_file_node(&c, &file, Some(blob1), &res).unwrap(); let blob2 = crate::db::blobs::intern_blob(&c, b"v2").unwrap(); let changed: FileNode = serde_json::from_value(serde_json::json!({ - "id":"F1","nodeType":"file","name":"renamed.bin", - "type":"text/plain","created":"2022-02-02T02:02:02Z" + "id":"F1","@type":"File","name":"renamed.bin", + "mediaType":"text/plain","executable":true,"created":"2022-02-02T02:02:02Z" })) .unwrap(); update_file_node(&c, local, &changed, Some(blob2), &res).unwrap(); assert_eq!(one_row(&c, "file_nodes"), 1); - let (name, mt, b): (String, Option, Option) = c + let (name, mt, b, x): (String, Option, Option, i64) = c .query_row( - "SELECT name, media_type, blob_id FROM file_nodes WHERE id=?1", + "SELECT name, media_type, blob_id, executable FROM file_nodes WHERE id=?1", params![local], - |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)), ) .unwrap(); assert_eq!(name, "renamed.bin"); assert_eq!(mt.as_deref(), Some("text/plain")); assert_eq!(b, Some(blob2)); + assert_eq!(x, 1); } #[test] diff --git a/src/sync/import_jmap/pool.rs b/src/sync/import_jmap/pool.rs index ca86bbc..bb58a12 100644 --- a/src/sync/import_jmap/pool.rs +++ b/src/sync/import_jmap/pool.rs @@ -6,7 +6,7 @@ use std::thread::JoinHandle; -use crossbeam_channel::{Receiver, Sender, unbounded}; +use crossbeam_channel::{Receiver, Sender, bounded, unbounded}; use crate::jmap::session::Limits; @@ -41,7 +41,7 @@ where { let threads = threads.max(1); let (job_tx, job_rx) = unbounded::(); - let (result_tx, result_rx) = unbounded::(); + let (result_tx, result_rx) = bounded::(threads); let worker = std::sync::Arc::new(worker); let mut workers = Vec::with_capacity(threads); for _ in 0..threads { @@ -75,10 +75,11 @@ where pub fn finish(mut self) -> Vec { drop(self.job_tx.take()); + let out = self.result_rx.iter().collect(); for handle in self.workers.drain(..) { let _ = handle.join(); } - self.result_rx.try_iter().collect() + out } } @@ -89,6 +90,7 @@ where { fn drop(&mut self) { drop(self.job_tx.take()); + for _ in self.result_rx.iter() {} for handle in self.workers.drain(..) { let _ = handle.join(); } @@ -131,4 +133,38 @@ mod tests { let expected: Vec = (0..100).map(|n| n * n).collect(); assert_eq!(got, expected); } + + #[test] + fn pool_holds_at_most_one_unread_result_per_worker() { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + // Unread results back-pressure the workers instead of piling up. + let done = std::sync::Arc::new(AtomicUsize::new(0)); + let pool: Pool = Pool::new(2, { + let done = done.clone(); + move |n| { + done.fetch_add(1, Ordering::SeqCst); + n + } + }); + for n in 0..50 { + pool.submit(n); + } + std::thread::sleep(Duration::from_millis(200)); + assert!(done.load(Ordering::SeqCst) <= 4); + let mut got = pool.finish(); + got.sort_unstable(); + assert_eq!(got, (0..50).collect::>()); + } + + #[test] + fn dropping_pool_with_unread_results_does_not_hang() { + let pool: Pool = Pool::new(2, |n| n); + for n in 0..50 { + pool.submit(n); + } + let _ = pool.results().recv(); + drop(pool); + } } diff --git a/tests/integration/dovecot.rs b/tests/integration/dovecot.rs index a1c17cd..7bd4d6b 100644 --- a/tests/integration/dovecot.rs +++ b/tests/integration/dovecot.rs @@ -358,6 +358,23 @@ impl Dovecot { Ok(()) } + pub fn create_leaves_without_parents( + &self, + account: &Account, + leaves: &[&str], + message: &[u8], + ) -> ContainerResult> { + let mut client = ImapSeed::connect(&self.imap.host, self.imap.port)?; + client.login(&account.username, &account.password)?; + for name in leaves { + client.create(name)?; + client.append_with_flags(name, &[], message)?; + } + let listed = client.list_raw()?; + client.logout()?; + Ok(listed) + } + pub fn delete_first_inbox_message(&self, account: &Account) -> ContainerResult<()> { let mut client = ImapSeed::connect(&self.imap.host, self.imap.port)?; client.login(&account.username, &account.password)?; diff --git a/tests/integration/imap_client.rs b/tests/integration/imap_client.rs index a2a770a..3e2b23b 100644 --- a/tests/integration/imap_client.rs +++ b/tests/integration/imap_client.rs @@ -189,6 +189,12 @@ impl ImapSeed { Ok(names) } + pub fn list_raw(&mut self) -> ContainerResult> { + let tag = self.next_tag(); + self.send(&format!(r#"{tag} LIST "" "*""#))?; + self.read_tagged(&tag) + } + pub fn select(&mut self, mailbox: &str) -> ContainerResult { let tag = self.next_tag(); self.send(&format!("{tag} SELECT {}", quote(mailbox)))?; diff --git a/tests/integration_baikal.rs b/tests/integration_baikal.rs index f5f59c1..ea067f7 100644 --- a/tests/integration_baikal.rs +++ b/tests/integration_baikal.rs @@ -18,7 +18,7 @@ use integration::validate::{ use inbuxa_migrate::error::Error; use inbuxa_migrate::sync::import_dav; -use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; +use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange}; use rusqlite::Connection; use serde_json::Value; @@ -38,7 +38,7 @@ fn dav_config(kind: DavKindArg, account: &Account, dav_root: &str) -> DavImportC allow_cleartext: true, dav_connections: 2, multiget_batch: 25, - allow_source_change: false, + source_change: SourceChange::Reject, } } @@ -57,7 +57,7 @@ fn dav_config_via_discovery( allow_cleartext: true, dav_connections: 2, multiget_batch: 25, - allow_source_change: false, + source_change: SourceChange::Reject, } } @@ -411,7 +411,7 @@ fn baikal_carddav_preserves_apple_item_labels() { allow_cleartext: true, dav_connections: 2, multiget_batch: 25, - allow_source_change: false, + source_change: SourceChange::Reject, }; let summary = import_dav::run(common(&archive), cfg).expect("carddav import"); assert!( diff --git a/tests/integration_radicale.rs b/tests/integration_radicale.rs index 91a108e..71fb24b 100644 --- a/tests/integration_radicale.rs +++ b/tests/integration_radicale.rs @@ -18,7 +18,7 @@ use integration::{Account, Endpoint}; use inbuxa_migrate::error::Error; use inbuxa_migrate::sync::import_dav; -use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; +use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange}; use rusqlite::Connection; use serde_json::Value; @@ -33,7 +33,7 @@ fn dav_config(kind: DavKindArg, account: &Account, http: &Endpoint) -> DavImport allow_cleartext: true, dav_connections: 2, multiget_batch: 25, - allow_source_change: false, + source_change: SourceChange::Reject, } } diff --git a/tests/integration_webdav.rs b/tests/integration_webdav.rs index ed3ff13..afe9111 100644 --- a/tests/integration_webdav.rs +++ b/tests/integration_webdav.rs @@ -17,7 +17,7 @@ use integration::{Account, Endpoint}; use inbuxa_migrate::error::Error; use inbuxa_migrate::sync::import_dav; -use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; +use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange}; fn webdav_config(account: &Account, http: &Endpoint) -> DavImportConfig { DavImportConfig { @@ -30,7 +30,7 @@ fn webdav_config(account: &Account, http: &Endpoint) -> DavImportConfig { allow_cleartext: true, dav_connections: 2, multiget_batch: 25, - allow_source_change: false, + source_change: SourceChange::Reject, } } diff --git a/tests/mock_dav.rs b/tests/mock_dav.rs index c04f513..26cc517 100644 --- a/tests/mock_dav.rs +++ b/tests/mock_dav.rs @@ -1338,7 +1338,9 @@ fn webdav_display_or_basename_test_via_parser_keeps_basename_semantics() { fn dry_run_writes_nothing_but_emits_per_collection_counts() { use inbuxa_migrate::logging::Logger; use inbuxa_migrate::sync::CommonConfig; - use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, run}; + use inbuxa_migrate::sync::import_dav::{ + DavAuth, DavImportConfig, DavKindArg, SourceChange, run, + }; use std::path::PathBuf; let mut server = mockito::Server::new(); @@ -1407,7 +1409,7 @@ fn dry_run_writes_nothing_but_emits_per_collection_counts() { allow_cleartext: true, dav_connections: 1, multiget_batch: 50, - allow_source_change: false, + source_change: SourceChange::Reject, }; let summary = run(common, config).expect("dry-run ok"); @@ -1451,7 +1453,9 @@ fn dav_source_change_protection_fires_across_users_on_same_root() { use encodify::base64::STANDARD; use inbuxa_migrate::logging::Logger; use inbuxa_migrate::sync::CommonConfig; - use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, run}; + use inbuxa_migrate::sync::import_dav::{ + DavAuth, DavImportConfig, DavKindArg, SourceChange, run, + }; use std::path::PathBuf; let mut server = mockito::Server::new(); @@ -1566,7 +1570,7 @@ fn dav_source_change_protection_fires_across_users_on_same_root() { allow_cleartext: true, dav_connections: 1, multiget_batch: 50, - allow_source_change: false, + source_change: SourceChange::Reject, }; run( @@ -1696,7 +1700,7 @@ fn caldav_import_config(url: &str) -> inbuxa_migrate::sync::import_dav::DavImpor allow_cleartext: true, dav_connections: 1, multiget_batch: 8, - allow_source_change: false, + source_change: inbuxa_migrate::sync::import_dav::SourceChange::Reject, } } @@ -1828,6 +1832,122 @@ END:VCALENDAR let _ = std::fs::remove_file(&archive); } +// Kerio Connect sends calendar-data with bare CR line breaks, raw or as (issue #48). +#[test] +fn kerio_bare_cr_calendar_data_imports() { + let mut server = mockito::Server::new(); + let url = server.url(); + let home = "/full-calendars/domain.ext/user/"; + let coll = "/full-calendars/domain.ext/user/ceb087f2-0604-4c31-a70e-9f68e90600c6/"; + + let home_set = format!( + r#" + + + {coll} + + + + Calendar + + HTTP/1.1 200 OK + + +"# + ); + let _discovery = multistatus_response(&mut server, "PROPFIND", home, &home_set); + + let item = |n: u32| format!("{coll}ceb087f2-0604-4c31-a70e-9f68e90600c6-0000000{n}.eml"); + let listing = format!( + r#" + + + {} + "k1" + HTTP/1.1 200 OK + + + {} + "k2" + HTTP/1.1 200 OK + +"#, + item(1), + item(2) + ); + let _enumeration = multistatus_response(&mut server, "PROPFIND", coll, &listing); + + let ical = |uid: &str, summary: &str, eol: &str| { + [ + "BEGIN:VCALENDAR", + "PRODID:-//Kerio Technologies//Kerio Connect//EN", + "VERSION:2.0", + "BEGIN:VEVENT", + &format!("UID:{uid}"), + "DTSTAMP:20260101T000000Z", + "DTSTART:20260101T090000Z", + "DTEND:20260101T100000Z", + &format!("SUMMARY:{summary}"), + "X-LABEL:0", + "END:VEVENT", + "END:VCALENDAR", + "", + ] + .join(eol) + }; + let data = format!( + r#" + + + {} + "k1" + + HTTP/1.1 200 OK + + + {} + "k2" + {} + HTTP/1.1 200 OK + +"#, + item(1), + ical("k1@kerio", "Raw CR", "\r"), + item(2), + ical("k2@kerio", "Entity CR", " "), + ); + let _multiget = multistatus_response(&mut server, "REPORT", coll, &data); + + let archive = dav_archive(); + let summary = inbuxa_migrate::sync::import_dav::run( + dav_common(&archive), + caldav_import_config(&format!("{url}{home}")), + ) + .expect("import"); + + let events = summary + .per_type + .iter() + .find(|(t, _)| *t == "calendarevent") + .map(|(_, c)| c.clone()) + .expect("calendarevent counts"); + assert_eq!(events.failed, 0, "{events:?}"); + assert_eq!(events.fetched, 2, "{events:?}"); + + let conn = rusqlite::Connection::open(&archive).expect("open archive"); + let mut titles: Vec = conn + .prepare("SELECT json_extract(data, '$.title') FROM calendar_events") + .unwrap() + .query_map([], |r| r.get(0)) + .unwrap() + .map(Result::unwrap) + .collect(); + titles.sort_unstable(); + assert_eq!(titles, ["Entity CR", "Raw CR"]); + drop(conn); + let _ = std::fs::remove_file(&archive); +} + #[test] fn credentials_rejected_during_discovery_still_aborts_the_run() { let mut server = mockito::Server::new(); @@ -1902,3 +2022,190 @@ fn dav_import_reports_both_phases_when_every_collection_fails() { ); let _ = std::fs::remove_file(&archive); } + +fn webdav_share(server: &mut mockito::Server, prefix: &str, body: &str) -> Vec { + let url = server.url(); + let root = format!( + r#" + + + {url}{prefix} + + share + HTTP/1.1 200 OK + + + + {url}{prefix}docs/ + + docs + HTTP/1.1 200 OK + + +"# + ); + let docs = format!( + r#" + + + {url}{prefix}docs/ + + docs + HTTP/1.1 200 OK + + + + {url}{prefix}docs/a.txt + + text/plain"a1" + HTTP/1.1 200 OK + + +"# + ); + vec![ + multistatus_response(server, "PROPFIND", prefix, &root), + multistatus_response(server, "PROPFIND", &format!("{prefix}docs/"), &docs), + server + .mock("GET", format!("{prefix}docs/a.txt").as_str()) + .with_status(200) + .with_header("content-type", "text/plain") + .with_body(body) + .create(), + ] +} + +fn webdav_import_config( + url: &str, + source_change: inbuxa_migrate::sync::import_dav::SourceChange, +) -> inbuxa_migrate::sync::import_dav::DavImportConfig { + inbuxa_migrate::sync::import_dav::DavImportConfig { + kind: inbuxa_migrate::sync::import_dav::DavKindArg::Webdav, + source_change, + ..caldav_import_config(url) + } +} + +#[test] +fn allow_source_change_keeps_the_moved_share_old_tree() { + use inbuxa_migrate::sync::import_dav::{SourceChange, run}; + + let mut server = mockito::Server::new(); + let url = server.url(); + let _old = webdav_share(&mut server, "/old/u/", "old\n"); + let _new = webdav_share(&mut server, "/new/u/", "new\n"); + let archive = dav_archive(); + + run( + dav_common(&archive), + webdav_import_config(&format!("{url}/old/u/"), SourceChange::Reject), + ) + .expect("old share import"); + run( + dav_common(&archive), + webdav_import_config(&format!("{url}/new/u/"), SourceChange::Allow), + ) + .expect("new share import"); + + let (nodes, sources) = ( + row_count(&archive, "file_nodes"), + row_count(&archive, "sources"), + ); + let _ = std::fs::remove_file(&archive); + assert_eq!(nodes, 4, "both trees stay side by side"); + assert_eq!(sources, 2); +} + +#[test] +fn replace_source_drops_the_moved_share_old_tree() { + use inbuxa_migrate::sync::import_dav::{SourceChange, run}; + + let mut server = mockito::Server::new(); + let url = server.url(); + let _old = webdav_share(&mut server, "/old/u/", "old\n"); + let _new = webdav_share(&mut server, "/new/u/", "new\n"); + let archive = dav_archive(); + + run( + dav_common(&archive), + webdav_import_config(&format!("{url}/old/u/"), SourceChange::Reject), + ) + .expect("old share import"); + let summary = run( + dav_common(&archive), + webdav_import_config(&format!("{url}/new/u/"), SourceChange::Replace), + ) + .expect("new share import"); + let rerun = run( + dav_common(&archive), + webdav_import_config(&format!("{url}/new/u/"), SourceChange::Reject), + ); + + let conn = rusqlite::Connection::open(&archive).unwrap(); + let account: String = conn + .query_row("SELECT account_id FROM sources", [], |r| r.get(0)) + .unwrap(); + let orphans: i64 = conn + .query_row( + "SELECT count(*) FROM file_nodes + WHERE id NOT IN (SELECT local_id FROM sync_id_dav)", + [], + |r| r.get(0), + ) + .unwrap(); + drop(conn); + let (nodes, sources, blobs) = ( + row_count(&archive, "file_nodes"), + row_count(&archive, "sources"), + row_count(&archive, "blobs"), + ); + let _ = std::fs::remove_file(&archive); + + assert_eq!(nodes, 2, "only the new tree remains"); + assert_eq!(orphans, 0); + assert_eq!(sources, 1); + assert!(account.ends_with("/new/u/"), "kept source is {account}"); + assert_eq!(blobs, 1, "the old file's blob is collected"); + let filenode = summary + .per_type + .iter() + .find(|(name, _)| *name == "filenode") + .expect("filenode counts"); + assert_eq!(filenode.1.deleted, 2); + assert!( + rerun.is_ok(), + "once replaced, the archive records a single source: {rerun:?}" + ); +} + +#[test] +fn dry_run_replace_source_keeps_the_old_tree() { + use inbuxa_migrate::sync::import_dav::{SourceChange, run}; + + let mut server = mockito::Server::new(); + let url = server.url(); + let _old = webdav_share(&mut server, "/old/u/", "old\n"); + let _new = webdav_share(&mut server, "/new/u/", "new\n"); + let archive = dav_archive(); + + run( + dav_common(&archive), + webdav_import_config(&format!("{url}/old/u/"), SourceChange::Reject), + ) + .expect("old share import"); + let mut common = dav_common(&archive); + common.dry_run = true; + run( + common, + webdav_import_config(&format!("{url}/new/u/"), SourceChange::Replace), + ) + .expect("dry-run"); + + let (nodes, sources) = ( + row_count(&archive, "file_nodes"), + row_count(&archive, "sources"), + ); + let _ = std::fs::remove_file(&archive); + assert_eq!(nodes, 2); + assert_eq!(sources, 1); +} diff --git a/tests/mock_imap.rs b/tests/mock_imap.rs index d913fb0..d1b0916 100644 --- a/tests/mock_imap.rs +++ b/tests/mock_imap.rs @@ -2004,3 +2004,247 @@ fn assert_name_selected_as_listed(listed: &'static str, stored: &str, archive_na ); let _ = std::fs::remove_file(&archive); } + +const NOSELECT_LIST: &[&str] = &[ + "* LIST (\\HasChildren) \"/\" \"INBOX\"", + "* LIST (\\Noselect \\HasChildren) \"/\" \"INBOX/Projects\"", + "* LIST (\\HasNoChildren) \"/\" \"INBOX/Projects/Invoices\"", + "* LIST (\\Noselect \\HasChildren) \"/\" \"A\"", + "* LIST (\\HasNoChildren) \"/\" \"A/B/Reports\"", + "* LIST (\\Noselect \\HasChildren) \"/\" \"Clients \"", + "* LIST (\\HasNoChildren) \"/\" \"Clients /Reports\"", + "* LIST (\\HasChildren) \"/\" \"Clients\"", + "* LIST (\\HasNoChildren) \"/\" \"Clients/Reports\"", +]; + +const NOSELECT_SELECTABLE: &[&str] = &[ + "INBOX", + "INBOX/Projects/Invoices", + "A/B/Reports", + "Clients /Reports", + "Clients", + "Clients/Reports", +]; + +fn noselect_control_script(selected: std::sync::Arc>>) -> Script { + Box::new(move |conn: &mut MockConn| -> std::io::Result<()> { + auth_preamble(conn, "IMAP4rev2 LITERAL+ AUTH=PLAIN")?; + while let Ok((tag, cmd)) = conn.read_command() { + if cmd.starts_with("LIST") { + for line in NOSELECT_LIST { + conn.write_line(line)?; + } + conn.write_line(&format!("{tag} OK LIST done"))?; + } else if let Some(name) = cmd + .strip_prefix("SELECT ") + .or_else(|| cmd.strip_prefix("EXAMINE ")) + { + selected + .lock() + .expect("selected") + .push(name.trim_matches('"').to_owned()); + write_select(conn, &tag, 7, 1, 0)?; + } else if cmd.starts_with("UID SEARCH") { + conn.write_line("* SEARCH")?; + conn.write_line(&format!("{tag} OK SEARCH done"))?; + } else if cmd.starts_with("LOGOUT") { + conn.write_line("* BYE bye")?; + conn.write_line(&format!("{tag} OK"))?; + break; + } else { + assert!( + !cmd.starts_with("STATUS") && !cmd.contains("FETCH"), + "unexpected {cmd}" + ); + conn.write_line(&format!("{tag} OK"))?; + } + } + Ok(()) + }) +} + +fn mailbox_tree(conn: &Connection) -> Vec<(i64, String, Option)> { + let mut stmt = conn + .prepare( + "SELECT m.id, s.folder, p.name FROM mailboxes m + JOIN sync_id_imap s ON s.local_id = m.id AND s.type_name = 'mailbox' + LEFT JOIN mailboxes p ON p.id = m.parent_id + ORDER BY s.folder", + ) + .unwrap(); + stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?))) + .unwrap() + .collect::>() + .unwrap() +} + +fn assert_noselect_tree(conn: &Connection) { + let tree = mailbox_tree(conn); + let pairs: Vec<(&str, Option<&str>)> = tree + .iter() + .map(|(_, f, p)| (f.as_str(), p.as_deref())) + .collect(); + assert_eq!( + pairs, + [ + ("A", None), + ("A/B", Some("A")), + ("A/B/Reports", Some("B")), + ("Clients", None), + ("Clients ", None), + ("Clients /Reports", Some("Clients ")), + ("Clients/Reports", Some("Clients")), + ("INBOX", None), + ("INBOX/Projects", Some("INBOX")), + ("INBOX/Projects/Invoices", Some("Projects")), + ] + ); + let roles: i64 = conn + .query_row( + "SELECT COUNT(*) FROM mailboxes WHERE role IS NOT NULL AND name IN ('Projects', 'A', 'B', 'Clients ')", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(roles, 0); +} + +fn assert_only_selectable_selected(selected: &Mutex>) { + let mut got = selected.lock().expect("selected").clone(); + got.sort_unstable(); + let mut want: Vec = NOSELECT_SELECTABLE + .iter() + .map(|s| (*s).to_owned()) + .collect(); + want.sort_unstable(); + assert_eq!(got, want); +} + +#[test] +fn coordinator_keeps_noselect_ancestors_and_never_selects_them() { + let first = std::sync::Arc::new(Mutex::new(Vec::new())); + let second = std::sync::Arc::new(Mutex::new(Vec::new())); + let server = MockImap::start_scripts(vec![ + noselect_control_script(first.clone()), + noselect_control_script(second.clone()), + ]); + let archive = tempfile("noselect"); + let summary = run_import(&server, "alice", archive.clone(), |_| {}).expect("import"); + let mailbox = summary + .per_type + .iter() + .find(|(k, _)| *k == "mailbox") + .unwrap(); + assert_eq!(mailbox.1.created, 10); + assert_eq!(mailbox.1.failed, 0); + assert_only_selectable_selected(&first); + let before = { + let conn = Connection::open(&archive).unwrap(); + assert_noselect_tree(&conn); + mailbox_tree(&conn) + }; + + let summary = run_import(&server, "alice", archive.clone(), |_| {}).expect("reimport"); + let mailbox = summary + .per_type + .iter() + .find(|(k, _)| *k == "mailbox") + .unwrap(); + assert_eq!((mailbox.1.created, mailbox.1.deleted), (0, 0)); + assert_only_selectable_selected(&second); + let conn = Connection::open(&archive).unwrap(); + assert_eq!(mailbox_tree(&conn), before); +} + +#[test] +fn coordinator_heals_archive_missing_noselect_parents() { + let first = std::sync::Arc::new(Mutex::new(Vec::new())); + let second = std::sync::Arc::new(Mutex::new(Vec::new())); + let server = MockImap::start_scripts(vec![ + noselect_control_script(first), + noselect_control_script(second), + ]); + let archive = tempfile("noselect_heal"); + run_import(&server, "alice", archive.clone(), |_| {}).expect("import"); + let child_id = { + let conn = Connection::open(&archive).unwrap(); + let tree = mailbox_tree(&conn); + let child = tree + .iter() + .find(|(_, f, _)| f == "INBOX/Projects/Invoices") + .unwrap() + .0; + // simulate an archive written before structural parents were kept + conn.execute( + "UPDATE mailboxes SET parent_id = NULL WHERE id = ?1", + [child], + ) + .unwrap(); + let parent: i64 = conn + .query_row( + "SELECT local_id FROM sync_id_imap WHERE type_name = 'mailbox' AND folder = 'INBOX/Projects'", + [], + |r| r.get(0), + ) + .unwrap(); + conn.execute( + "DELETE FROM sync_id_imap WHERE type_name = 'mailbox' AND folder = 'INBOX/Projects'", + [], + ) + .unwrap(); + conn.execute("DELETE FROM mailboxes WHERE id = ?1", [parent]) + .unwrap(); + child + }; + run_import(&server, "alice", archive.clone(), |_| {}).expect("reimport"); + let conn = Connection::open(&archive).unwrap(); + assert_noselect_tree(&conn); + let healed = mailbox_tree(&conn) + .into_iter() + .find(|(_, f, _)| f == "INBOX/Projects/Invoices") + .unwrap() + .0; + assert_eq!(healed, child_id); +} + +#[test] +fn dry_run_never_selects_noselect_ancestors() { + let selected = std::sync::Arc::new(Mutex::new(Vec::new())); + let server = MockImap::start_scripts(vec![noselect_control_script(selected.clone())]); + let archive = tempfile("noselect_dry"); + let common = CommonConfig { + archive: archive.clone(), + threads: 1, + dry_run: true, + max_retries: 1, + allow_invalid_certs: false, + logger: Logger::from_flags(true, 0), + }; + let config = ImapImportConfig { + url: server.url(), + auth: ImapAuth::Basic { + user: "alice".to_owned(), + password: "p@ss".to_owned(), + }, + allow_cleartext: true, + compress: false, + include: Vec::new(), + exclude: Vec::new(), + exclude_special: Vec::new(), + folder: Vec::new(), + subscribed_only: false, + automap: true, + include_deleted: false, + fetch_batch: 256, + imap_connections: 1, + allow_source_change: false, + }; + let summary = run(common, config).expect("dry run"); + let mailbox = summary + .per_type + .iter() + .find(|(k, _)| *k == "mailbox") + .unwrap(); + assert_eq!(mailbox.1.created, 10); + assert_only_selectable_selected(&selected); +} diff --git a/tests/mock_jmap.rs b/tests/mock_jmap.rs index 0d44376..7960e78 100644 --- a/tests/mock_jmap.rs +++ b/tests/mock_jmap.rs @@ -331,8 +331,9 @@ fn principal_substring_prefilter_rejects_near_matches() { ["Principal/get", { "list": [ { "id": "p1", "name": "alice2", "accounts": {} }, { "id": "p2", "name": "alice", "accounts": { - "w": { "urn:ietf:params:jmap:principals:owner": - { "accountIdForPrincipal": "w" } } } } + "w": { "name": "alice", "accountCapabilities": { + "urn:ietf:params:jmap:principals:owner": + { "accountIdForPrincipal": "admin", "principalId": "p2" } } } } } ] }, "g"] ] }) .to_string(), diff --git a/tests/mock_sync.rs b/tests/mock_sync.rs index 9d469df..7060e5f 100644 --- a/tests/mock_sync.rs +++ b/tests/mock_sync.rs @@ -486,223 +486,223 @@ fn export_mailbox_already_exists_maps_existing_id() { let _ = std::fs::remove_file(&archive); } -#[test] -fn email_export_sends_one_email_per_import_call() { - let mut server = mockito::Server::new(); - let base = server.url(); - let api = "/jmap/api"; +fn session_body_set_limit(base: &str, max_objects_in_set: u64) -> String { + json!({ + "apiUrl": format!("{base}/jmap/api"), + "uploadUrl": format!("{base}/jmap/upload/{{accountId}}/"), + "downloadUrl": format!("{base}/jmap/dl/{{accountId}}/{{blobId}}/{{type}}/{{name}}"), + "capabilities": { "urn:ietf:params:jmap:core": { + "maxObjectsInGet": 500, "maxObjectsInSet": max_objects_in_set, "maxCallsInRequest": 16, + "maxConcurrentRequests": 4, "maxConcurrentUpload": 4, + "maxSizeRequest": 10000000, "maxSizeUpload": 50000000 + } }, + "accounts": { "w": { "name": "alice", + "accountCapabilities": { "urn:ietf:params:jmap:mail": {} } } } + }) + .to_string() +} - let archive = tmp(); - { - let conn = db::init::open(&archive).unwrap(); +fn seed_inbox_emails(archive: &Path, n: usize, shared_blob: bool) { + let conn = db::init::open(archive).unwrap(); + conn.execute( + "INSERT INTO mailboxes (id,name,parent_id,role) VALUES (1,'Inbox',NULL,'inbox')", + [], + ) + .unwrap(); + for i in 1..=n { + let raw = if shared_blob { + "From: a@x\r\nSubject: dup\r\nMessage-ID: \r\n\r\nbody".to_owned() + } else { + format!("From: a@x\r\nSubject: m{i}\r\nMessage-ID: \r\n\r\nbody {i}") + }; + let blob = db::blobs::intern_blob(&conn, raw.as_bytes()).unwrap(); conn.execute( - "INSERT INTO mailboxes (id,name,parent_id,role) VALUES (1,'Inbox',NULL,'inbox')", - [], + "INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords) + VALUES (?1,'2020-01-01T00:00:00Z','[1]','[]')", + rusqlite::params![blob], ) .unwrap(); - for n in 1..=2 { - let raw = - format!("From: a@x\r\nSubject: m{n}\r\nMessage-ID: \r\n\r\nbody {n}",); - let blob = db::blobs::intern_blob(&conn, raw.as_bytes()).unwrap(); - conn.execute( - "INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords) - VALUES (?1,'2020-01-01T00:00:00Z','[1]','[]')", - rusqlite::params![blob], - ) - .unwrap(); - } } +} - let _root = server.mock("GET", "/").with_status(404).create(); - let _wk = server - .mock("GET", "/.well-known/jmap") - .with_body(session_body(&base)) +fn mock_empty_target_with_inbox( + server: &mut mockito::Server, + session: String, +) -> Vec { + let api = "/jmap/api"; + vec![ + server.mock("GET", "/").with_status(404).create(), + server + .mock("GET", "/.well-known/jmap") + .with_body(session) + .create(), + server + .mock("POST", api) + .match_body(Matcher::Regex("Mailbox/query".into())) + .with_body( + json!({"methodResponses":[["Mailbox/query", + {"accountId":"w","ids":["t1"]},"q"]]}) + .to_string(), + ) + .expect(1) + .create(), + server + .mock("POST", api) + .match_body(Matcher::Regex("Mailbox/query".into())) + .with_body( + json!({"methodResponses":[["Mailbox/query", + {"accountId":"w","ids":[]},"q"]]}) + .to_string(), + ) + .create(), + server + .mock("POST", api) + .match_body(Matcher::Regex("Mailbox/get".into())) + .with_body( + json!({"methodResponses":[["Mailbox/get",{"accountId":"w","list":[ + {"id":"t1","name":"Inbox","role":"inbox","parentId":null, + "myRights":{"mayDelete":true}}],"notFound":[]},"g"]]}) + .to_string(), + ) + .create(), + server + .mock("POST", api) + .match_body(Matcher::Regex("Email/query".into())) + .with_body( + json!({"methodResponses":[["Email/query", + {"accountId":"w","ids":[]},"q"]]}) + .to_string(), + ) + .create(), + ] +} + +fn import_cids(body: &[u8]) -> Vec { + let v: Value = serde_json::from_slice(body).unwrap(); + v["methodCalls"][0][1]["emails"] + .as_object() + .map(|m| m.keys().cloned().collect()) + .unwrap_or_default() +} + +fn export_email_cfg(base: &str) -> ExportConfig { + export_cfg_objects(base, vec![ObjectType::Mailbox, ObjectType::Email]) +} + +#[test] +fn email_export_batches_imports_by_max_objects_in_set() { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; + + // 5 emails with maxObjectsInSet 2 need ceil(5/2) = 3 Email/import calls, none above 2 items. + let mut server = mockito::Server::new(); + let base = server.url(); + let archive = tmp(); + seed_inbox_emails(&archive, 5, false); + let _target = mock_empty_target_with_inbox(&mut server, session_body_set_limit(&base, 2)); + + let blob_seq = Arc::new(AtomicUsize::new(0)); + let seq = blob_seq.clone(); + let ups = server + .mock("POST", Matcher::Regex("/jmap/upload/".into())) + .with_body_from_request(move |_| { + let n = seq.fetch_add(1, Ordering::Relaxed); + json!({ "blobId": format!("UP{n}") }) + .to_string() + .into_bytes() + }) + .expect(5) .create(); - let _mq = server - .mock("POST", api) - .match_body(Matcher::Regex("Mailbox/query".into())) - .with_body( - json!({"methodResponses":[["Mailbox/query", - {"accountId":"w","ids":["t1"]},"q"]]}) - .to_string(), - ) - .expect_at_least(1) - .create(); - let _mq_empty = server - .mock("POST", api) - .match_body(Matcher::AllOf(vec![ - Matcher::Regex("Mailbox/query".into()), - Matcher::Regex("anchor".into()), - ])) - .with_body( - json!({"methodResponses":[["Mailbox/query", - {"accountId":"w","ids":[]},"q"]]}) - .to_string(), - ) - .create(); - let _mg = server - .mock("POST", api) - .match_body(Matcher::Regex("Mailbox/get".into())) - .with_body( - json!({"methodResponses":[["Mailbox/get",{"accountId":"w","list":[ - {"id":"t1","name":"Inbox","role":"inbox","parentId":null, - "myRights":{"mayDelete":true}}],"notFound":[]},"g"]]}) - .to_string(), - ) - .expect_at_least(1) - .create(); - let _eq = server - .mock("POST", api) - .match_body(Matcher::Regex("Email/query".into())) - .with_body( - json!({"methodResponses":[["Email/query", - {"accountId":"w","ids":[]},"q"]]}) - .to_string(), - ) - .expect_at_least(1) + let sizes = Arc::new(Mutex::new(Vec::new())); + let seen = sizes.clone(); + let imports = server + .mock("POST", "/jmap/api") + .match_body(Matcher::Regex("Email/import".into())) + .with_body_from_request(move |req| { + let cids = import_cids(req.body().unwrap()); + seen.lock().unwrap().push(cids.len()); + let created: serde_json::Map = cids + .into_iter() + .map(|c| { + let id = format!("x-{c}"); + ( + c, + json!({"id": id, "blobId": "b", "threadId": "t", "size": 10}), + ) + }) + .collect(); + json!({"methodResponses":[["Email/import", + {"accountId":"w","created": created},"i"]]}) + .to_string() + .into_bytes() + }) + .expect(3) .create(); + let mut cfg = common(&archive); + cfg.threads = 4; + let summary = sync::export::run(cfg, export_email_cfg(&base)).expect("export run"); + + let email = email_counts(&summary); + assert_eq!(email.created, 5); + assert_eq!(email.failed, 0); + let mut sizes = sizes.lock().unwrap().clone(); + sizes.sort_unstable(); + assert_eq!(sizes, vec![1, 2, 2]); + ups.assert(); + imports.assert(); + let _ = std::fs::remove_file(&archive); +} + +#[test] +fn email_export_counts_mixed_batch_outcomes_per_item() { + // One batch: e1 created, e2 alreadyExists (skipped), e3 overQuota (failed), e4 omitted (failed). + let mut server = mockito::Server::new(); + let base = server.url(); + let archive = tmp(); + seed_inbox_emails(&archive, 4, false); + let _target = mock_empty_target_with_inbox(&mut server, session_body(&base)); + let _ups = server .mock("POST", Matcher::Regex("/jmap/upload/".into())) - .with_body(json!({"blobId":"UP1"}).to_string()) - .expect(2) + .with_body(json!({"blobId":"UP"}).to_string()) + .expect(4) .create(); - - let single_only = server - .mock("POST", api) - .match_body(Matcher::AllOf(vec![ - Matcher::Regex("Email/import".into()), - Matcher::Regex("e1".into()), - Matcher::Regex("e2".into()), - ])) - .expect(0) - .create(); - let imports = server - .mock("POST", api) + .mock("POST", "/jmap/api") .match_body(Matcher::Regex("Email/import".into())) .with_body( - json!({"methodResponses":[["Email/import", - {"accountId":"w","created":{"e":{"id":"x","blobId":"b","threadId":"t","size":10}}},"i"]]}) + json!({"methodResponses":[["Email/import",{"accountId":"w", + "created":{"e1":{"id":"x1","blobId":"UP","threadId":"t","size":10}}, + "notCreated":{ + "e2":{"type":"alreadyExists","existingId":"x9"}, + "e3":{"type":"overQuota"} + }},"i"]]}) .to_string(), ) - .expect(2) + .expect(1) .create(); - let summary = sync::export::run( - CommonConfig { - archive: archive.clone(), - threads: 1, - dry_run: false, - max_retries: 0, - allow_invalid_certs: false, - logger: Logger::from_flags(true, 0), - }, - ExportConfig { - connect: ConnectConfig { - url: base.clone(), - auth: Auth::Basic { - user: "u".into(), - password: "p".into(), - }, - account: AccountSelector::Id("w".into()), - }, - objects: None, - prune: false, - yes: true, - }, - ) - .expect("export run"); + let summary = sync::export::run(common(&archive), export_email_cfg(&base)).expect("export run"); - let email = summary - .per_type - .iter() - .find(|(t, _)| *t == "Email") - .map(|(_, c)| c.clone()) - .expect("email counts"); - assert_eq!(email.created, 2, "both emails imported in per-item rounds"); - assert_eq!(email.failed, 0, "no per-unit failure"); - assert!(!summary.any_failed(), "no whole-run failure"); - - single_only.assert(); + let email = email_counts(&summary); + assert_eq!(email.created, 1); + assert_eq!(email.skipped, 1); + assert_eq!(email.failed, 2); imports.assert(); let _ = std::fs::remove_file(&archive); } #[test] fn export_email_blob_not_found_reuploads_and_retries() { + // Two emails share one blob; e2 gets blobNotFound in the batch, so only e2 is re-uploaded and retried. let mut server = mockito::Server::new(); let base = server.url(); let api = "/jmap/api"; - let archive = tmp(); - { - let conn = db::init::open(&archive).unwrap(); - conn.execute( - "INSERT INTO mailboxes (id,name,parent_id,role) VALUES (1,'Inbox',NULL,'inbox')", - [], - ) - .unwrap(); - let raw = b"From: a@x\r\nSubject: dup\r\nMessage-ID: \r\n\r\nbody"; - let blob = db::blobs::intern_blob(&conn, raw).unwrap(); - for _ in 0..2 { - conn.execute( - "INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords) - VALUES (?1,'2020-01-01T00:00:00Z','[1]','[]')", - rusqlite::params![blob], - ) - .unwrap(); - } - } - - let _root = server.mock("GET", "/").with_status(404).create(); - let _wk = server - .mock("GET", "/.well-known/jmap") - .with_body(session_body(&base)) - .create(); - - let _mq1 = server - .mock("POST", api) - .match_body(Matcher::Regex("Mailbox/query".into())) - .with_body( - json!({"methodResponses":[["Mailbox/query", - {"accountId":"w","ids":["t1"]},"q"]]}) - .to_string(), - ) - .expect(1) - .create(); - let _mq2 = server - .mock("POST", api) - .match_body(Matcher::Regex("Mailbox/query".into())) - .with_body( - json!({"methodResponses":[["Mailbox/query", - {"accountId":"w","ids":[]},"q"]]}) - .to_string(), - ) - .expect(1) - .create(); - let _mg = server - .mock("POST", api) - .match_body(Matcher::Regex("Mailbox/get".into())) - .with_body( - json!({"methodResponses":[["Mailbox/get",{"accountId":"w","list":[ - {"id":"t1","name":"Inbox","role":"inbox","parentId":null, - "myRights":{"mayDelete":true}}],"notFound":[]},"g"]]}) - .to_string(), - ) - .expect(1) - .create(); - let _eq = server - .mock("POST", api) - .match_body(Matcher::Regex("Email/query".into())) - .with_body( - json!({"methodResponses":[["Email/query", - {"accountId":"w","ids":[]},"q"]]}) - .to_string(), - ) - .expect(1) - .create(); + seed_inbox_emails(&archive, 2, true); + let _target = mock_empty_target_with_inbox(&mut server, session_body(&base)); let up1 = server .mock("POST", Matcher::Regex("/jmap/upload/".into())) @@ -715,79 +715,44 @@ fn export_email_blob_not_found_reuploads_and_retries() { .expect(1) .create(); - let imp_e1 = server + let batch = server .mock("POST", api) .match_body(Matcher::AllOf(vec![ Matcher::Regex("Email/import".into()), - Matcher::Regex("e1".into()), - ])) - .with_body( - json!({"methodResponses":[["Email/import",{"accountId":"w", - "created":{"e1":{"id":"x1","blobId":"UP1","threadId":"t","size":10}}},"i"]]}) - .to_string(), - ) - .expect(1) - .create(); - let imp_e2_stale = server - .mock("POST", api) - .match_body(Matcher::AllOf(vec![ - Matcher::Regex("Email/import".into()), - Matcher::Regex("e2".into()), Matcher::Regex("UP1".into()), ])) .with_body( json!({"methodResponses":[["Email/import",{"accountId":"w", + "created":{"e1":{"id":"x1","blobId":"UP1","threadId":"t","size":10}}, "notCreated":{"e2":{"type":"blobNotFound"}}},"i"]]}) .to_string(), ) .expect(1) .create(); - let imp_e2_fresh = server + let retry = server .mock("POST", api) .match_body(Matcher::AllOf(vec![ Matcher::Regex("Email/import".into()), - Matcher::Regex("e2".into()), Matcher::Regex("UP2".into()), ])) - .with_body( + .with_body_from_request(|req| { + let cids = import_cids(req.body().unwrap()); + assert_eq!( + cids, + vec!["e2".to_owned()], + "only the affected item is retried" + ); json!({"methodResponses":[["Email/import",{"accountId":"w", "created":{"e2":{"id":"x2","blobId":"UP2","threadId":"t","size":10}}},"i"]]}) - .to_string(), - ) + .to_string() + .into_bytes() + }) .expect(1) .create(); - let summary = sync::export::run( - CommonConfig { - archive: archive.clone(), - threads: 1, - dry_run: false, - max_retries: 1, - allow_invalid_certs: false, - logger: Logger::from_flags(true, 0), - }, - ExportConfig { - connect: ConnectConfig { - url: base.clone(), - auth: Auth::Basic { - user: "u".into(), - password: "p".into(), - }, - account: AccountSelector::Id("w".into()), - }, - objects: None, - prune: false, - yes: true, - }, - ) - .expect("export run"); + let summary = sync::export::run(common(&archive), export_email_cfg(&base)).expect("export run"); - let email = summary - .per_type - .iter() - .find(|(t, _)| *t == "Email") - .map(|(_, c)| c.clone()) - .expect("email counts"); + let email = email_counts(&summary); assert_eq!(email.created, 2, "both emails end up created"); assert_eq!(email.failed, 0, "blobNotFound self-heals, not a failure"); assert_eq!(email.skipped, 0); @@ -795,9 +760,8 @@ fn export_email_blob_not_found_reuploads_and_retries() { up1.assert(); up2.assert(); - imp_e1.assert(); - imp_e2_stale.assert(); - imp_e2_fresh.assert(); + batch.assert(); + retry.assert(); let _ = std::fs::remove_file(&archive); } @@ -1382,8 +1346,9 @@ fn import_unusable_received_at_falls_back_instead_of_dropping_the_email() { let _ = std::fs::remove_file(&archive); } +// Draft -18 FileNodes import by @type; an unrecognised @type fails only that node. #[test] -fn import_file_node_without_node_type_is_inferred_not_skipped() { +fn import_file_node_by_type_and_skips_unknown_type() { let mut server = mockito::Server::new(); let base = server.url(); let api = "/jmap/api"; @@ -1402,7 +1367,7 @@ fn import_file_node_without_node_type_is_inferred_not_skipped() { .match_body(Matcher::Regex("FileNode/query".into())) .with_body( json!({"methodResponses":[["FileNode/query", - {"accountId":"w","ids":["F1","F2"]},"q"]]}) + {"accountId":"w","ids":["F1","F2","F3","F4"]},"q"]]}) .to_string(), ) .create(); @@ -1411,8 +1376,15 @@ fn import_file_node_without_node_type_is_inferred_not_skipped() { .match_body(Matcher::Regex("FileNode/get".into())) .with_body( json!({"methodResponses":[["FileNode/get",{"accountId":"w","state":"sf1","list":[ - {"id":"F1","parentId":null,"name":"docs","created":"2026-01-01T00:00:00Z"}, - {"id":"F2","parentId":"F1","name":"a.txt","blobId":"BLBF","type":"text/plain","created":"2026-01-02T00:00:00Z"} + {"@type":"Directory","id":"F1","parentId":null,"name":"docs","role":null, + "isSubscribed":true,"created":"2026-01-01T00:00:00Z"}, + {"@type":"File","id":"F2","parentId":"F1","name":"a.sh","blobId":"BLBF", + "size":10,"mediaType":"text/x-sh","executable":true, + "created":"2026-01-02T00:00:00Z"}, + {"@type":"SymLink","id":"F3","parentId":null,"name":"latest", + "target":["docs","a.sh"],"created":"2026-01-03T00:00:00Z"}, + {"@type":"Socket","id":"F4","parentId":null,"name":"sock", + "created":"2026-01-04T00:00:00Z"} ],"notFound":[]},"g"]]}) .to_string(), ) @@ -1422,30 +1394,33 @@ fn import_file_node_without_node_type_is_inferred_not_skipped() { .with_body("file bytes") .create(); - let summary = sync::import_jmap::run( + let outcome = sync::import_jmap::run_reporting( common(&archive), import_cfg_objects(&base, vec![ObjectType::FileNode]), - ) - .expect("import does not abort when nodeType is absent"); + ); + assert!(outcome.error.is_none(), "{:?}", outcome.error); - let nodes = summary + let nodes = outcome + .summary .per_type .iter() .find(|(t, _)| *t == "FileNode") .map(|(_, c)| c.clone()) .expect("file node counts"); - assert_eq!( - nodes.fetched, 2, - "a server that omits nodeType still imports" - ); - assert_eq!(nodes.failed, 0); + assert_eq!(nodes.fetched, 3); + assert_eq!(nodes.failed, 1); let conn = rusqlite::Connection::open(&archive).unwrap(); let mut stmt = conn - .prepare("SELECT name, node_type FROM file_nodes ORDER BY name") + .prepare( + "SELECT name, node_type, media_type, executable, target FROM file_nodes ORDER BY name", + ) .unwrap(); - let rows: Vec<(String, String)> = stmt - .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))) + type Row = (String, String, Option, i64, Option); + let rows: Vec = stmt + .query_map([], |r| { + Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)) + }) .unwrap() .map(Result::unwrap) .collect(); @@ -1453,10 +1428,22 @@ fn import_file_node_without_node_type_is_inferred_not_skipped() { assert_eq!( rows, vec![ - ("a.txt".to_owned(), "file".to_owned()), - ("docs".to_owned(), "directory".to_owned()), - ], - "nodeType is inferred from the presence of a blobId" + ( + "a.sh".to_owned(), + "file".to_owned(), + Some("text/x-sh".to_owned()), + 1, + None + ), + ("docs".to_owned(), "directory".to_owned(), None, 0, None), + ( + "latest".to_owned(), + "symlink".to_owned(), + None, + 0, + Some(r#"["docs","a.sh"]"#.to_owned()) + ), + ] ); let _ = std::fs::remove_file(&archive); } @@ -1568,6 +1555,118 @@ fn import_missing_email_blob_is_skipped_and_counted_once() { let _ = std::fs::remove_file(&archive); } +#[test] +fn import_writes_each_email_blob_before_later_downloads_complete() { + // Each download waits for the previous blob to reach the archive WAL; collecting a page first would time out. + let mut server = mockito::Server::new(); + let base = server.url(); + let api = "/jmap/api"; + let archive = tmp(); + let wal = PathBuf::from(format!("{}-wal", archive.display())); + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_full(&base)) + .expect_at_least(1) + .create(); + let _mbterm = anchor_terminator(&mut server, api, "Mailbox"); + let _emterm = anchor_terminator(&mut server, api, "Email"); + let _mbq = server + .mock("POST", api) + .match_body(Matcher::Regex("Mailbox/query".into())) + .with_body( + json!({"methodResponses":[["Mailbox/query", + {"accountId":"w","ids":["MX"]},"q"]]}) + .to_string(), + ) + .create(); + let _mbg = server + .mock("POST", api) + .match_body(Matcher::Regex("Mailbox/get".into())) + .with_body( + json!({"methodResponses":[["Mailbox/get",{"accountId":"w","state":"sm1","list":[ + {"id":"MX","name":"Inbox","parentId":null,"role":"inbox","sortOrder":0,"isSubscribed":true} + ],"notFound":[]},"g"]]}) + .to_string(), + ) + .create(); + let _eq = server + .mock("POST", api) + .match_body(Matcher::Regex("Email/query".into())) + .with_body( + json!({"methodResponses":[["Email/query", + {"accountId":"w","ids":["E1","E2","E3"]},"q"]]}) + .to_string(), + ) + .create(); + let list: Vec = (1..=3) + .map(|n| { + json!({"id":format!("E{n}"),"blobId":format!("BLB{n}"), + "receivedAt":"2020-01-01T00:00:00Z","mailboxIds":{"MX":true},"keywords":{}}) + }) + .collect(); + let _eg = server + .mock("POST", api) + .match_body(Matcher::Regex("Email/get".into())) + .with_body( + json!({"methodResponses":[["Email/get",{"accountId":"w","state":"se1", + "list":list,"notFound":[]},"g"]]}) + .to_string(), + ) + .create(); + + let wal_seen = std::sync::Arc::new(std::sync::Mutex::new(Vec::::new())); + let wal_len = { + let wal = wal.clone(); + move || std::fs::metadata(&wal).map(|m| m.len()).unwrap_or(0) + }; + let mut downloads = Vec::new(); + for n in 1..=3u8 { + let seen = wal_seen.clone(); + let wal_len = wal_len.clone(); + downloads.push( + server + .mock("GET", Matcher::Regex(format!("/jmap/dl/w/BLB{n}/.*"))) + .with_body_from_request(move |_| { + let floor = seen.lock().unwrap().last().copied(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3); + let mut len = wal_len(); + while floor.is_some_and(|f| len <= f) && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(10)); + len = wal_len(); + } + seen.lock().unwrap().push(len); + let mut body = format!("From: a@x\r\nMessage-ID: <{n}@h>\r\n\r\n").into_bytes(); + body.resize(body.len() + 64 * 1024, b'a' + n); + body + }) + .create(), + ); + } + + let summary = sync::import_jmap::run( + common(&archive), + import_cfg_objects(&base, vec![ObjectType::Mailbox, ObjectType::Email]), + ) + .expect("import succeeds"); + assert!(!summary.any_failed()); + + let seen = wal_seen.lock().unwrap().clone(); + assert_eq!(seen.len(), 3); + assert!( + seen.windows(2).all(|w| w[1] > w[0]), + "each blob reaches the archive before the next download finishes: {seen:?}" + ); + let conn = rusqlite::Connection::open(&archive).unwrap(); + assert_eq!( + conn.query_row::("SELECT count(*) FROM emails", [], |r| r.get(0)) + .unwrap(), + 3 + ); + let _ = std::fs::remove_file(&archive); +} + #[test] fn export_missing_target_email_is_created_on_rerun() { let mut server = mockito::Server::new(); @@ -2143,6 +2242,73 @@ fn export_calendar_creates_only_missing() { let _ = std::fs::remove_file(&archive); } +// DAV alpha colors are sent without alpha and unusable colors are dropped. +#[test] +fn export_calendar_normalizes_colors() { + let mut server = mockito::Server::new(); + let base = server.url(); + let api = "/jmap/api"; + let archive = tmp(); + { + let conn = db::init::open(&archive).unwrap(); + conn.execute( + "INSERT INTO calendars (id,name,color,is_default) VALUES + (1,'Sogo','#AAAAAAFF',0), (2,'Short','#abcd',0), (3,'Odd','rgba(1,2,3,0.5)',0)", + [], + ) + .unwrap(); + } + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_full(&base)) + .expect_at_least(1) + .create(); + let _g = server + .mock("POST", api) + .match_body(Matcher::Regex("Calendar/get".into())) + .with_body( + json!({"methodResponses":[["Calendar/get",{"accountId":"w","list":[], + "notFound":[]},"g"]]}) + .to_string(), + ) + .expect(1) + .create(); + let create = server + .mock("POST", api) + .match_body(Matcher::Regex("Calendar/set".into())) + .with_body_from_request(|req| { + let body: Value = serde_json::from_slice(req.body().unwrap()).unwrap(); + let create = &body["methodCalls"][0][1]["create"]; + assert_eq!(create["c1"]["color"], json!("#AAAAAA")); + assert_eq!(create["c2"]["color"], json!("#abc")); + assert!(create["c3"].get("color").is_none(), "{create}"); + json!({"methodResponses":[["Calendar/set",{"accountId":"w", + "created":{"c1":{"id":"T1"},"c2":{"id":"T2"},"c3":{"id":"T3"}}},"s"]]}) + .to_string() + .into_bytes() + }) + .expect(1) + .create(); + + let summary = sync::export::run( + common(&archive), + export_cfg_objects(&base, vec![ObjectType::Calendar]), + ) + .expect("export"); + create.assert(); + let counts = summary + .per_type + .iter() + .find(|(t, _)| *t == "Calendar") + .map(|(_, c)| c.clone()) + .expect("calendar counts"); + assert_eq!(counts.created, 3); + assert_eq!(counts.failed, 0); + let _ = std::fs::remove_file(&archive); +} + #[test] fn export_contact_and_event_blobs_are_inlined_never_blob_ids() { let mut server = mockito::Server::new(); @@ -2586,6 +2752,72 @@ fn export_archive_read_failure_while_inlining_exits_seven() { let _ = std::fs::remove_file(&archive); } +// Archived participant identities are never sent to the target, even with prune, and nothing fails. +#[test] +fn export_skips_participant_identities() { + let mut server = mockito::Server::new(); + let base = server.url(); + let api = "/jmap/api"; + let archive = tmp(); + { + let conn = db::init::open(&archive).unwrap(); + conn.execute( + "INSERT INTO participant_identities (id,name,calendar_address,is_default) + VALUES (1,'Bob','mailto:bob@example.org',1)", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO calendars (id,name,is_default) VALUES (1,'Family',1)", + [], + ) + .unwrap(); + } + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_full(&base)) + .expect_at_least(1) + .create(); + let participant = server + .mock("POST", api) + .match_body(Matcher::Regex("ParticipantIdentity/".into())) + .with_status(500) + .expect(0) + .create(); + let calendars = server + .mock("POST", api) + .match_body(Matcher::Regex("Calendar/get".into())) + .with_body( + json!({"methodResponses":[["Calendar/get",{"accountId":"w","list":[ + {"id":"F","name":"Family","isDefault":true} + ],"notFound":[]},"g"]]}) + .to_string(), + ) + .expect(1) + .create(); + + for objects in [ + vec![ObjectType::ParticipantIdentity, ObjectType::Calendar], + vec![ObjectType::ParticipantIdentity], + ] { + let mut config = export_cfg_objects(&base, objects); + config.prune = true; + let summary = sync::export::run(common(&archive), config).expect("export"); + assert!(!summary.any_failed(), "{:?}", summary.per_type); + assert!( + summary + .per_type + .iter() + .all(|(t, _)| *t != "ParticipantIdentity") + ); + } + participant.assert(); + calendars.assert(); + let _ = std::fs::remove_file(&archive); +} + #[test] fn export_sieve_script_matches_by_name_not_content() { let mut server = mockito::Server::new(); @@ -2682,6 +2914,92 @@ fn export_sieve_script_matches_by_name_not_content() { let _ = std::fs::remove_file(&archive); } +// Pruning stale scripts leaves the archive's active script active on the target. +#[test] +fn export_sieve_prune_keeps_archive_active_script_active() { + prune_sieve_with_archive_active(true, Some("M1")); +} + +// With no active script in the archive, the stale active target script is deactivated and pruned. +#[test] +fn export_sieve_prune_destroys_stale_active_script() { + prune_sieve_with_archive_active(false, None); +} + +fn prune_sieve_with_archive_active(archive_active: bool, expected_active: Option<&str>) { + let mut server = mockito::Server::new(); + let base = server.url(); + let archive = tmp(); + { + let conn = db::init::open(&archive).unwrap(); + let blob = db::blobs::intern_blob(&conn, b"keep;\n").unwrap(); + conn.execute( + "INSERT INTO sieve_scripts (id,name,is_active,blob_id) VALUES (1,'main',?1,?2)", + rusqlite::params![archive_active, blob], + ) + .unwrap(); + } + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_full(&base)) + .expect_at_least(1) + .create(); + let active = std::sync::Arc::new(std::sync::Mutex::new(Some("O1".to_owned()))); + let target_active = active.clone(); + let log = CallLog::default(); + let _api = jmap_dispatch(&mut server, &log, move |name, args| { + let mut active = target_active.lock().unwrap(); + match name { + "SieveScript/get" => json!({"accountId":"w","state":"x","notFound":[],"list":[ + {"id":"M1","name":"main","isActive":active.as_deref() == Some("M1"),"blobId":"BM"}, + {"id":"O1","name":"old","isActive":active.as_deref() == Some("O1"),"blobId":"BO"} + ]}), + "SieveScript/set" => { + if args["onSuccessDeactivateScript"] == json!(true) { + *active = None; + } + if let Some(id) = args["onSuccessActivateScript"].as_str() { + *active = Some(id.to_owned()); + } + let mut response = json!({"accountId":"w"}); + if let Some(ids) = args["destroy"].as_array() { + let (blocked, destroyed): (Vec<&Value>, Vec<&Value>) = + ids.iter().partition(|id| id.as_str() == active.as_deref()); + response["destroyed"] = json!(destroyed); + response["notDestroyed"] = blocked + .iter() + .map(|id| { + ( + id.as_str().unwrap().to_owned(), + json!({"type":"scriptIsActive"}), + ) + }) + .collect::>() + .into(); + } + response + } + other => panic!("unexpected {other}"), + } + }); + + let mut config = export_cfg_objects(&base, vec![ObjectType::SieveScript]); + config.prune = true; + let summary = sync::export::run(common(&archive), config).expect("export"); + assert!(!summary.any_failed(), "{:?}", summary.per_type); + let destroyed: Vec = log + .lock() + .unwrap() + .iter() + .filter_map(|(_, args)| args.get("destroy").cloned()) + .collect(); + assert_eq!(destroyed, vec![json!(["O1"])]); + assert_eq!(active.lock().unwrap().as_deref(), expected_active); + let _ = std::fs::remove_file(&archive); +} + #[test] fn export_sieve_scripts_identical_content_different_names_both_created() { let mut server = mockito::Server::new(); @@ -4458,3 +4776,453 @@ fn export_email_fatal_method_error_is_not_retried() { let _ = std::fs::remove_file(&archive); } + +fn session_body_vacation(base: &str) -> String { + let mut session: Value = serde_json::from_str(&session_body_full(base)).unwrap(); + session["accounts"]["w"]["accountCapabilities"]["urn:ietf:params:jmap:vacationresponse"] = + json!({}); + session.to_string() +} + +type CallLog = std::sync::Arc>>; + +fn jmap_dispatch( + server: &mut mockito::Server, + log: &CallLog, + respond: impl Fn(&str, &Value) -> Value + Send + Sync + 'static, +) -> mockito::Mock { + let log = log.clone(); + server + .mock("POST", "/jmap/api") + .with_body_from_request(move |req| { + let body: Value = serde_json::from_slice(req.body().unwrap()).unwrap(); + let mut responses = Vec::new(); + for call in body["methodCalls"].as_array().unwrap() { + let name = call[0].as_str().unwrap(); + log.lock().unwrap().push((name.to_owned(), call[1].clone())); + responses.push(json!([name, respond(name, &call[1]), call[2]])); + } + json!({ "methodResponses": responses }) + .to_string() + .into_bytes() + }) + .expect_at_least(1) + .create() +} + +fn vacation_row(archive: &Path) -> Option<(bool, Option, Option)> { + let conn = db::init::open(archive).unwrap(); + conn.query_row( + "SELECT is_enabled, subject, from_date FROM vacation_response", + [], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .ok() +} + +// A 0.16 source keeps the vacation script and stores the VacationResponse; catch-up refreshes it. +#[test] +fn import_vacation_response_from_stalwart_016_source() { + let mut server = mockito::Server::new(); + let base = server.url(); + let archive = tmp(); + let vacation = std::sync::Arc::new(std::sync::Mutex::new(json!({ + "id": "singleton", "isEnabled": true, "fromDate": "2026-10-01T00:00:00Z", + "toDate": null, "subject": "Away", "textBody": "Back soon", "htmlBody": null + }))); + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_vacation(&base)) + .expect_at_least(1) + .create(); + let _dl1 = server + .mock("GET", Matcher::Regex("/jmap/dl/w/B1/.*".into())) + .with_body("keep;\n") + .create(); + let _dl2 = server + .mock("GET", Matcher::Regex("/jmap/dl/w/B2/.*".into())) + .with_body("require [\"vacation\"];\nvacation \"Back soon\";\n") + .create(); + let log = CallLog::default(); + let current = vacation.clone(); + let _api = jmap_dispatch(&mut server, &log, move |name, args| match name { + "SieveScript/query" if args.get("anchor").is_some() => json!({"accountId":"w","ids":[]}), + "SieveScript/query" => json!({"accountId":"w","ids":["S1","S2"]}), + "SieveScript/get" => json!({"accountId":"w","state":"x","notFound":[],"list":[ + {"id":"S1","name":"filters","isActive":false,"blobId":"B1"}, + {"id":"S2","name":"vacation","isActive":true,"blobId":"B2"} + ]}), + "VacationResponse/get" => json!({"accountId":"w","state":"x","notFound":[], + "list":[current.lock().unwrap().clone()]}), + other => panic!("unexpected {other}"), + }); + + let summary = sync::import_jmap::run( + common(&archive), + import_cfg_objects(&base, vec![ObjectType::SieveScript]), + ) + .expect("import"); + assert!(!summary.any_failed()); + let counts = summary + .per_type + .iter() + .find(|(t, _)| *t == "VacationResponse") + .map(|(_, c)| c.clone()) + .expect("vacation counts"); + assert_eq!(counts.fetched, 1); + assert_eq!( + vacation_row(&archive), + Some(( + true, + Some("Away".to_owned()), + Some("2026-10-01T00:00:00Z".to_owned()) + )) + ); + let scripts: i64 = db::init::open(&archive) + .unwrap() + .query_row("SELECT count(*) FROM sieve_scripts", [], |r| r.get(0)) + .unwrap(); + assert_eq!(scripts, 2, "the vacation script is still archived"); + + *vacation.lock().unwrap() = json!({"id": "singleton", "isEnabled": false, "subject": "Later"}); + sync::import_jmap::run( + common(&archive), + import_cfg_objects(&base, vec![ObjectType::SieveScript]), + ) + .expect("catch-up import"); + assert_eq!( + vacation_row(&archive), + Some((false, Some("Later".to_owned()), None)) + ); + + *vacation.lock().unwrap() = json!({"id": "singleton", "isEnabled": false}); + sync::import_jmap::run( + common(&archive), + import_cfg_objects(&base, vec![ObjectType::SieveScript]), + ) + .expect("second catch-up import"); + assert_eq!( + vacation_row(&archive), + None, + "an unset vacation clears the row" + ); + let _ = std::fs::remove_file(&archive); +} + +fn seed_vacation_archive(archive: &Path) { + let conn = db::init::open(archive).unwrap(); + let filters = db::blobs::intern_blob(&conn, b"keep;\n").unwrap(); + let vacation = db::blobs::intern_blob(&conn, b"vacation \"Back soon\";\n").unwrap(); + conn.execute( + "INSERT INTO sources (id,kind,session_url,account_id,username) + VALUES (1,'jmap','http://src','a','bob')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO sieve_scripts (id,name,is_active,blob_id) VALUES + (1,'filters',0,?1), (2,'vacation',1,?2)", + rusqlite::params![filters, vacation], + ) + .unwrap(); + conn.execute( + "INSERT INTO vacation_response (id,source_id,is_enabled,subject,text_body) + VALUES (1,1,1,'Away','Back soon')", + [], + ) + .unwrap(); +} + +// The vacation script becomes a VacationResponse update, sent last and never pruned. +#[test] +fn export_vacation_script_becomes_vacation_response() { + let mut server = mockito::Server::new(); + let base = server.url(); + let archive = tmp(); + seed_vacation_archive(&archive); + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_vacation(&base)) + .expect_at_least(1) + .create(); + let _up = server + .mock("POST", Matcher::Regex("/jmap/upload/".into())) + .with_body(json!({"blobId":"UP1"}).to_string()) + .create(); + let log = CallLog::default(); + let second_run = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let target_state = second_run.clone(); + let _api = jmap_dispatch(&mut server, &log, move |name, args| match name { + "SieveScript/get" if target_state.load(std::sync::atomic::Ordering::Relaxed) => { + json!({"accountId":"w","state":"x","notFound":[],"list":[ + {"id":"V1","name":"vacation","isActive":true,"blobId":"BV"}, + {"id":"S1","name":"filters","isActive":false,"blobId":"UP1"} + ]}) + } + "SieveScript/get" => json!({"accountId":"w","state":"x","notFound":[],"list":[ + {"id":"O1","name":"old","isActive":false,"blobId":"BO"} + ]}), + "SieveScript/set" if args.get("create").is_some() => { + json!({"accountId":"w","created":{"c1":{"id":"S1"}}}) + } + "SieveScript/set" if args.get("destroy").is_some() => { + json!({"accountId":"w","destroyed":args["destroy"].clone()}) + } + "SieveScript/set" => json!({"accountId":"w"}), + "VacationResponse/set" => json!({"accountId":"w","updated":{"singleton":null}}), + other => panic!("unexpected {other}"), + }); + let export = || { + let mut config = export_cfg_objects(&base, vec![ObjectType::SieveScript]); + config.prune = true; + let summary = sync::export::run(common(&archive), config).expect("export"); + assert!(!summary.any_failed(), "{:?}", summary.per_type); + let calls = std::mem::take(&mut *log.lock().unwrap()); + (summary, calls) + }; + let args_of = |calls: &[(String, Value)], key: &str| -> Vec { + calls + .iter() + .filter_map(|(_, args)| args.get(key).cloned()) + .collect() + }; + let expected_vacation = json!({"isEnabled": true, "fromDate": null, "toDate": null, + "subject": "Away", "textBody": "Back soon", "htmlBody": null}); + + let (summary, calls) = export(); + let creates = args_of(&calls, "create"); + assert_eq!(creates.len(), 1); + assert_eq!(creates[0]["c1"]["name"], json!("filters")); + assert!( + creates[0].get("c2").is_none(), + "vacation is not created as a script" + ); + assert_eq!(args_of(&calls, "destroy"), vec![json!(["O1"])]); + let (last_name, last_args) = calls.last().unwrap(); + assert_eq!(last_name, "VacationResponse/set"); + assert_eq!(last_args["update"]["singleton"], expected_vacation); + let counts = |ty: &str| { + summary + .per_type + .iter() + .find(|(t, _)| *t == ty) + .map(|(_, c)| c.clone()) + .unwrap() + }; + assert_eq!(counts("SieveScript").created, 1); + assert_eq!(counts("SieveScript").skipped, 1); + assert_eq!(counts("VacationResponse").updated, 1); + + second_run.store(true, std::sync::atomic::Ordering::Relaxed); + let (_, calls) = export(); + assert!(args_of(&calls, "create").is_empty()); + assert!( + args_of(&calls, "destroy").is_empty(), + "the target vacation script is not pruned" + ); + let (last_name, last_args) = calls.last().unwrap(); + assert_eq!(last_name, "VacationResponse/set"); + assert_eq!(last_args["update"]["singleton"], expected_vacation); + let _ = std::fs::remove_file(&archive); +} + +// Without VacationResponse on the target, the vacation script is exported as before. +#[test] +fn export_vacation_script_unchanged_without_target_capability() { + let mut server = mockito::Server::new(); + let base = server.url(); + let archive = tmp(); + seed_vacation_archive(&archive); + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_full(&base)) + .expect_at_least(1) + .create(); + let _up = server + .mock("POST", Matcher::Regex("/jmap/upload/".into())) + .with_body(json!({"blobId":"UP1"}).to_string()) + .create(); + let log = CallLog::default(); + let _api = jmap_dispatch(&mut server, &log, |name, args| match name { + "SieveScript/get" => json!({"accountId":"w","state":"x","notFound":[],"list":[]}), + "SieveScript/set" if args.get("create").is_some() => { + let id = args["create"] + .as_object() + .unwrap() + .keys() + .next() + .unwrap() + .clone(); + json!({"accountId":"w","created":{ id.clone(): {"id": format!("T{id}")} }}) + } + "SieveScript/set" => json!({"accountId":"w"}), + other => panic!("unexpected {other}"), + }); + + let summary = sync::export::run( + common(&archive), + export_cfg_objects(&base, vec![ObjectType::SieveScript]), + ) + .expect("export"); + assert!(!summary.any_failed(), "{:?}", summary.per_type); + let calls = log.lock().unwrap().clone(); + let names: Vec<&Value> = calls + .iter() + .filter_map(|(_, args)| args.get("create")) + .flat_map(|c| c.as_object().unwrap().values().map(|v| &v["name"])) + .collect(); + assert_eq!(names, vec![&json!("filters"), &json!("vacation")]); + assert!( + calls + .iter() + .all(|(n, _)| !n.starts_with("VacationResponse")) + ); + assert!( + summary + .per_type + .iter() + .all(|(t, _)| *t != "VacationResponse") + ); + let _ = std::fs::remove_file(&archive); +} + +fn session_body_admin_vacation(base: &str) -> String { + let mut session: Value = serde_json::from_str(&session_body_full(base)).unwrap(); + let caps = &mut session["capabilities"]; + caps["urn:ietf:params:jmap:principals"] = json!({}); + caps["urn:ietf:params:jmap:sieve"] = json!({}); + caps["urn:ietf:params:jmap:vacationresponse"] = json!({}); + let mut admin = session["accounts"]["w"].take(); + admin["name"] = json!("admin"); + session["accounts"] = json!({ "a": admin }); + session.to_string() +} + +fn principal_bob(name: &str) -> Option { + match name { + "Principal/query" => Some(json!({"accountId":"a","ids":["p2"]})), + "Principal/get" => Some(json!({"accountId":"a","state":"x","notFound":[],"list":[ + {"id":"p2","name":"bob","accounts":{"w":{"name":"bob"}}} + ]})), + _ => None, + } +} + +// Admin credentials with --account-name: the target is not listed in the session, the session advertises VacationResponse. +#[test] +fn export_vacation_response_for_unlisted_account_of_admin_session() { + let mut server = mockito::Server::new(); + let base = server.url(); + let archive = tmp(); + seed_vacation_archive(&archive); + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_admin_vacation(&base)) + .expect_at_least(1) + .create(); + let _up = server + .mock("POST", Matcher::Regex("/jmap/upload/w/".into())) + .with_body(json!({"blobId":"UP1"}).to_string()) + .create(); + let log = CallLog::default(); + let _api = jmap_dispatch(&mut server, &log, |name, args| { + if let Some(reply) = principal_bob(name) { + return reply; + } + match name { + "SieveScript/get" => json!({"accountId":"w","state":"x","notFound":[],"list":[]}), + "SieveScript/set" if args.get("create").is_some() => { + json!({"accountId":"w","created":{"c1":{"id":"S1"}}}) + } + "SieveScript/set" => json!({"accountId":"w"}), + "VacationResponse/set" => json!({"accountId":"w","updated":{"singleton":null}}), + other => panic!("unexpected {other}"), + } + }); + + let mut config = export_cfg_objects(&base, vec![ObjectType::SieveScript]); + config.connect.account = AccountSelector::Name("bob".into()); + let summary = sync::export::run(common(&archive), config).expect("export"); + assert!(!summary.any_failed(), "{:?}", summary.per_type); + let calls = log.lock().unwrap().clone(); + let names: Vec<&Value> = calls + .iter() + .filter_map(|(_, args)| args.get("create")) + .flat_map(|c| c.as_object().unwrap().values().map(|v| &v["name"])) + .collect(); + assert_eq!(names, vec![&json!("filters")], "vacation is not created"); + let (last_name, last_args) = calls.last().unwrap(); + assert_eq!(last_name, "VacationResponse/set"); + assert_eq!(last_args["accountId"], json!("w")); + assert_eq!(last_args["update"]["singleton"]["subject"], json!("Away")); + let _ = std::fs::remove_file(&archive); +} + +// Admin credentials with --account-name: the import stores the target account's VacationResponse. +#[test] +fn import_vacation_response_for_unlisted_account_of_admin_session() { + let mut server = mockito::Server::new(); + let base = server.url(); + let archive = tmp(); + + let _root = server.mock("GET", "/").with_status(404).create(); + let _wk = server + .mock("GET", "/.well-known/jmap") + .with_body(session_body_admin_vacation(&base)) + .expect_at_least(1) + .create(); + let _dl = server + .mock("GET", Matcher::Regex("/jmap/dl/w/B1/.*".into())) + .with_body("require [\"vacation\"];\nvacation \"Back soon\";\n") + .create(); + let log = CallLog::default(); + let _api = jmap_dispatch(&mut server, &log, |name, args| { + if let Some(reply) = principal_bob(name) { + return reply; + } + match name { + "SieveScript/query" if args.get("anchor").is_some() => { + json!({"accountId":"w","ids":[]}) + } + "SieveScript/query" => json!({"accountId":"w","ids":["S1"]}), + "SieveScript/get" => json!({"accountId":"w","state":"x","notFound":[],"list":[ + {"id":"S1","name":"vacation","isActive":true,"blobId":"B1"} + ]}), + "VacationResponse/get" => json!({"accountId":"w","state":"x","notFound":[],"list":[ + {"id":"singleton","isEnabled":true,"fromDate":null,"toDate":null, + "subject":"Away","textBody":"Back soon","htmlBody":null} + ]}), + other => panic!("unexpected {other}"), + } + }); + + let mut config = import_cfg_objects(&base, vec![ObjectType::SieveScript]); + config.connect.account = AccountSelector::Name("bob".into()); + let summary = sync::import_jmap::run(common(&archive), config).expect("import"); + assert!(!summary.any_failed(), "{:?}", summary.per_type); + let fetched = summary + .per_type + .iter() + .find(|(t, _)| *t == "VacationResponse") + .map(|(_, c)| c.fetched); + assert_eq!(fetched, Some(1)); + assert_eq!( + vacation_row(&archive), + Some((true, Some("Away".to_owned()), None)) + ); + assert!( + log.lock() + .unwrap() + .iter() + .any(|(n, args)| n == "VacationResponse/get" && args["accountId"] == json!("w")) + ); + let _ = std::fs::remove_file(&archive); +} diff --git a/tests/seed_smoke.rs b/tests/seed_smoke.rs index be47a05..63c835b 100644 --- a/tests/seed_smoke.rs +++ b/tests/seed_smoke.rs @@ -267,14 +267,9 @@ fn provision_seed_verify_and_teardown() { let owner = exact[0] .get("accounts") .and_then(Value::as_object) - .and_then(|m| { - m.iter().find_map(|(_, v)| { - v.get("urn:ietf:params:jmap:principals:owner") - .and_then(|o| o.get("accountIdForPrincipal")) - .and_then(Value::as_str) - }) - }) - .expect("accountIdForPrincipal"); + .and_then(|m| m.keys().next()) + .map(String::as_str) + .expect("principal accounts key"); assert_eq!(owner, test1.account_id, "resolved account id matches test1"); seeder::teardown(stalwart.base_url()).expect("teardown"); diff --git a/tests/seeder/admin.rs b/tests/seeder/admin.rs index 5a3f954..7c591eb 100644 --- a/tests/seeder/admin.rs +++ b/tests/seeder/admin.rs @@ -121,6 +121,32 @@ impl Admin { Ok(()) } + pub fn enable_directory_queries(&self) -> SeedResult<()> { + let response = self.jmap.call( + STALWART, + "x:Sharing/set", + &self.account_id, + json!({ "update": { "singleton": { "allowDirectoryQueries": true } } }), + )?; + if response.pointer("/updated/singleton").is_none() { + return Err(SeedError::Shape(format!( + "x:Sharing/set not applied: {response}" + ))); + } + let response = self.jmap.call( + STALWART, + "x:Action/set", + &self.account_id, + json!({ "create": { "r": { "@type": "ReloadSettings" } } }), + )?; + if response.pointer("/created/r").is_none() { + return Err(SeedError::Shape(format!( + "ReloadSettings failed: {response}" + ))); + } + Ok(()) + } + pub fn teardown_domain(&self, name: &str) -> SeedResult<()> { let Some(domain_id) = self.domain_id(name)? else { return Ok(()); diff --git a/tests/seeder/mod.rs b/tests/seeder/mod.rs index a016ff2..a0a1860 100644 --- a/tests/seeder/mod.rs +++ b/tests/seeder/mod.rs @@ -176,6 +176,7 @@ pub fn teardown(base_url: &str) -> SeedResult<()> { pub fn provision(base_url: &str) -> SeedResult { let admin = Admin::connect(base_url, ADMIN_USER, ADMIN_PASSWORD)?; + admin.enable_directory_queries()?; admin.teardown_domain(DOMAIN)?; admin.invalidate_caches()?; let domain_id = admin.ensure_domain(DOMAIN)?; diff --git a/tests/seeder/seed.rs b/tests/seeder/seed.rs index 3725ca1..913bd63 100644 --- a/tests/seeder/seed.rs +++ b/tests/seeder/seed.rs @@ -272,14 +272,14 @@ fn seed_filenodes(jmap: &Jmap, account_id: &str, tree: &[FileSpec]) -> SeedResul obj.insert("parentId".to_owned(), Value::String(format!("#{parent}"))); } if spec.directory { - obj.insert("nodeType".to_owned(), Value::String("directory".to_owned())); + obj.insert("@type".to_owned(), Value::String("Directory".to_owned())); } else { let bytes = random_bytes(128 + i * 17)?; let blob_id = jmap.upload(account_id, "application/octet-stream", &bytes)?; - obj.insert("nodeType".to_owned(), Value::String("file".to_owned())); + obj.insert("@type".to_owned(), Value::String("File".to_owned())); obj.insert("blobId".to_owned(), Value::String(blob_id)); obj.insert( - "type".to_owned(), + "mediaType".to_owned(), Value::String("application/octet-stream".to_owned()), ); } @@ -399,6 +399,9 @@ fn seed_contacts_calendars( for immutable in ["method", "prodId"] { obj.remove(immutable); } + if obj.contains_key("recurrenceRule") || obj.contains_key("recurrenceOverrides") { + obj.remove("recurrenceId"); + } let calendar = &calendars[i % calendars.len()]; obj.insert("calendarIds".to_owned(), json!({ calendar.as_str(): true })); jmap.set_create( diff --git a/tests/sync_caldav.rs b/tests/sync_caldav.rs index a068152..3ed6ff3 100644 --- a/tests/sync_caldav.rs +++ b/tests/sync_caldav.rs @@ -19,7 +19,7 @@ fn base_url() -> &'static str { use inbuxa_migrate::jmap::account::AccountSelector; use inbuxa_migrate::jmap::http::Auth; use inbuxa_migrate::logging::Logger; -use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; +use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange}; use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; fn tmp_archive(tag: &str) -> PathBuf { @@ -58,7 +58,7 @@ fn caldav_cfg(base_url: &str, domain: &str, localpart: &str) -> DavImportConfig allow_cleartext: false, dav_connections: 4, multiget_batch: 50, - allow_source_change: false, + source_change: SourceChange::Reject, } } diff --git a/tests/sync_carddav.rs b/tests/sync_carddav.rs index 09071fb..e79cc30 100644 --- a/tests/sync_carddav.rs +++ b/tests/sync_carddav.rs @@ -19,7 +19,7 @@ use rusqlite::Connection; fn base_url() -> &'static str { shared_stalwart().base_url() } -use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; +use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange}; use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; fn tmp_archive(tag: &str) -> PathBuf { @@ -58,7 +58,7 @@ fn carddav_cfg(base_url: &str, domain: &str, localpart: &str) -> DavImportConfig allow_cleartext: false, dav_connections: 4, multiget_batch: 50, - allow_source_change: false, + source_change: SourceChange::Reject, } } diff --git a/tests/sync_exchange_graph.rs b/tests/sync_exchange_graph.rs index acb0713..2a7b081 100644 --- a/tests/sync_exchange_graph.rs +++ b/tests/sync_exchange_graph.rs @@ -376,16 +376,28 @@ fn graph_import_survives_a_full_export_to_stalwart() { .find(|c| c["name"]["full"] == "Graph Contact") .expect("the imported card reached the target"); - let uri = card["media"] + let photo = card["media"] .as_object() .and_then(|m| m.values().next()) - .and_then(|m| m["uri"].as_str()) .expect("the contact photo survived as a Media resource"); - assert!( - uri.starts_with("data:image/png;base64,"), + let media_type = match photo["uri"].as_str() { + Some(uri) => uri + .strip_prefix("data:") + .and_then(|rest| rest.split_once(';')) + .map(|(mt, _)| mt), + None => { + assert!( + photo["blobId"].is_string(), + "the photo is either an inline data URI or an embedded blob: {photo}" + ); + photo["mediaType"].as_str() + } + }; + assert_eq!( + media_type, + Some("image/png"), "Graph reports image/jpeg for every contact photo, so the media type must come \ - from the bytes; got {}", - &uri[..uri.len().min(40)] + from the bytes; got {photo}" ); assert_eq!(card["keywords"]["work"], true, "categories became keywords"); assert_eq!( @@ -408,16 +420,24 @@ fn graph_import_survives_a_full_export_to_stalwart() { .iter() .find(|e| e["title"] == "Event with an enclosure") .expect("the event with an attachment reached the target"); - let href = enclosure["links"] + let link = enclosure["links"] .as_object() .and_then(|m| m.values().next()) - .and_then(|l| l["href"].as_str()) .expect("the attachment survived as a Link"); - assert!( - href.starts_with("data:text/plain;base64,"), - "got {}", - &href[..href.len().min(40)] - ); + match link["href"].as_str() { + Some(href) => assert!( + href.starts_with("data:text/plain;base64,"), + "got {}", + &href[..href.len().min(40)] + ), + None => { + assert!( + link["blobId"].is_string(), + "the attachment is either an inline data URI or an embedded blob: {link}" + ); + assert_eq!(link["contentType"], "text/plain", "got {link}"); + } + } let series = events .iter() @@ -448,7 +468,7 @@ fn graph_import_survives_a_full_export_to_stalwart() { assert!( files .iter() - .any(|f| f["name"] == "report.txt" && f["blobId"].is_string()), + .any(|f| f["name"] == "report.txt" && f["@type"] == "File" && f["blobId"].is_string()), "the drive file reached the target with a blob" ); assert!( diff --git a/tests/sync_imap.rs b/tests/sync_imap.rs index 6c4a777..b2f1130 100644 --- a/tests/sync_imap.rs +++ b/tests/sync_imap.rs @@ -7,11 +7,15 @@ mod integration; mod seeder; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; +use integration::Account; +use integration::dovecot::Dovecot; use integration::stalwart::shared as shared_stalwart; +use integration::validate::{emails_in_mailbox, mailbox_id_by_path, mailbox_path, open_archive}; use rusqlite::Connection; +use serde_json::json; fn base_url() -> &'static str { shared_stalwart().base_url() @@ -75,6 +79,28 @@ fn imap_basic_config(localpart: &str) -> ImapImportConfig { } } +fn dovecot_config(account: &Account, imap: &integration::Endpoint) -> ImapImportConfig { + ImapImportConfig { + url: format!("imap://{}:{}", imap.host, imap.port), + auth: ImapAuth::Basic { + user: account.username.clone(), + password: account.password.clone(), + }, + allow_cleartext: false, + compress: false, + include: Vec::new(), + exclude: Vec::new(), + exclude_special: Vec::new(), + folder: Vec::new(), + subscribed_only: false, + automap: true, + include_deleted: false, + fetch_batch: 64, + imap_connections: 2, + allow_source_change: false, + } +} + fn count(conn: &Connection, table: &str) -> i64 { conn.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0)) .unwrap() @@ -393,3 +419,145 @@ fn imap_imported_archive_exports_via_jmap() { let _ = std::fs::remove_file(&archive); seeder::teardown(base_url()).expect("teardown"); } + +#[test] +#[ignore = "requires Docker"] +fn dovecot_noselect_parents_keep_the_tree_through_export() { + let d = Dovecot::start().expect("dovecot start"); + let account = d.accounts.first().expect("account").clone(); + let message = b"From: tree@inbuxa-migrate.test\r\n\ + To: user1@inbuxa-migrate.test\r\n\ + Subject: Tree\r\n\ + Message-ID: \r\n\ + Date: Wed, 01 Jan 2025 12:00:00 +0000\r\n\ + \r\n\ + Structural parent probe.\r\n"; + let leaves = [ + "Projects/Invoices", + "Clients/Acme/Invoices", + "Archive", + "Archive/2024", + "Archive /2024", + ]; + let structural = ["Projects", "Clients", "Clients/Acme", "Archive "]; + let listed = d + .create_leaves_without_parents(&account, &leaves, message) + .expect("create hierarchy"); + for name in structural { + let quoted = format!("\"{name}\""); + let line = listed + .iter() + .find(|l| l.ends_with("ed) || l.ends_with(&format!(" {name}"))) + .unwrap_or_else(|| panic!("{name:?} missing from LIST: {listed:?}")); + let lower = line.to_ascii_lowercase(); + assert!( + lower.contains("\\noselect") || lower.contains("\\nonexistent"), + "{name:?} should be non-selectable: {line}" + ); + } + + let archive = tmp_archive("dovecot-noselect"); + for pass in 0..2 { + let summary = sync::import_imap::run(common(&archive), dovecot_config(&account, &d.imap)) + .expect("imap import"); + assert!(!summary.any_failed(), "pass {pass}: {summary:?}"); + } + let conn = open_archive(&archive); + for path in leaves.iter().chain(&structural) { + let id = mailbox_id_by_path(&conn, path, '/') + .unwrap_or_else(|| panic!("{path:?} not in the archive tree")); + assert_eq!(mailbox_path(&conn, id, '/'), *path); + } + for path in structural { + let id = mailbox_id_by_path(&conn, path, '/').expect("structural"); + assert_eq!(emails_in_mailbox(&conn, id), 0, "{path:?}"); + } + for path in leaves { + let id = mailbox_id_by_path(&conn, path, '/').expect("leaf"); + assert_eq!(emails_in_mailbox(&conn, id), 1, "{path:?}"); + } + drop(conn); + + let fx = seeder::provision(base_url()).expect("provision"); + let dst = fx.account(seeder::SYNC_OUT[0]).expect("sync-out target"); + let export = sync::export::run( + CommonConfig { + archive: archive.clone(), + threads: 4, + dry_run: false, + max_retries: 5, + allow_invalid_certs: true, + logger: Logger::from_flags(false, 0), + }, + ExportConfig { + connect: ConnectConfig { + url: fx.base_url.clone(), + auth: Auth::Basic { + user: dst.email.clone(), + password: dst.password.clone(), + }, + account: AccountSelector::Id(dst.account_id.clone()), + }, + objects: None, + prune: false, + yes: true, + }, + ) + .expect("export"); + assert!(!export.any_failed(), "export reported failures: {export:?}"); + + let jmap = + seeder::jmap::Jmap::connect(&fx.base_url, &dst.email, &dst.password).expect("session"); + let resp = jmap + .request( + &["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"], + json!([["Mailbox/get", {"accountId": dst.account_id, "ids": null}, "g"]]), + ) + .expect("Mailbox/get"); + let list = resp["methodResponses"][0][1]["list"] + .as_array() + .cloned() + .unwrap_or_default(); + let by_id: HashMap<&str, (&str, Option<&str>)> = list + .iter() + .filter_map(|m| { + Some(( + m["id"].as_str()?, + (m["name"].as_str()?, m["parentId"].as_str()), + )) + }) + .collect(); + let paths: HashSet = by_id + .keys() + .map(|id| { + let mut parts = Vec::new(); + let mut cur = Some(*id); + while let Some(c) = cur { + let (name, parent) = by_id[c]; + parts.push(name); + cur = parent; + } + parts.reverse(); + parts.join("/") + }) + .collect(); + for path in leaves + .iter() + .chain(&structural) + .filter(|p| !p.contains(' ')) + { + assert!( + paths.contains(*path), + "{path:?} missing on target: {paths:?}" + ); + } + for orphan in ["Invoices", "Acme", "2024"] { + assert!( + !paths.contains(orphan), + "{orphan:?} leaked to the top level" + ); + } + + let _ = std::fs::remove_file(&archive); + seeder::teardown(&fx.base_url).expect("teardown"); +} diff --git a/tests/sync_jmap.rs b/tests/sync_jmap.rs index b50b38e..73a9994 100644 --- a/tests/sync_jmap.rs +++ b/tests/sync_jmap.rs @@ -295,8 +295,8 @@ fn seed_deep_filenode_chain(jmap: &seeder::jmap::Jmap, account_id: &str, depth: serde_json::Value::String(format!("dir-{i}")), ); obj.insert( - "nodeType".to_owned(), - serde_json::Value::String("directory".to_owned()), + "@type".to_owned(), + serde_json::Value::String("Directory".to_owned()), ); if i > 0 { obj.insert( @@ -693,20 +693,44 @@ fn prune_non_tree_type_contact_card() { const ISSUE30_CARD_UID: &str = "inbuxa-migrate-issue30-card"; const ISSUE30_EVENT_UID: &str = "inbuxa-migrate-issue30-event"; -fn data_uri_bytes(resource: &Value, uri_key: &str, expect_media_type: &str) -> Vec { - let uri = resource - .get(uri_key) +struct Target<'a> { + client: &'a HttpClient, + session: &'a Session, + account: &'a str, +} + +fn resource_bytes( + target: &Target<'_>, + resource: &Value, + uri_key: &str, + type_key: &str, + expect_media_type: &str, +) -> Vec { + if let Some(uri) = resource.get(uri_key).and_then(Value::as_str) { + let prefix = format!("data:{expect_media_type};base64,"); + let payload = uri + .strip_prefix(&prefix) + .unwrap_or_else(|| panic!("{uri_key} is not a {prefix}... data URI: {resource}")); + return STANDARD.decode(payload).expect("base64 payload"); + } + let blob_id = resource + .get("blobId") .and_then(Value::as_str) - .unwrap_or_else(|| panic!("no {uri_key} on {resource}")); - assert!( - resource.get("blobId").is_none(), - "target must not hold a blobId for {uri_key}: {resource}" + .unwrap_or_else(|| panic!("neither {uri_key} nor blobId on {resource}")); + assert_eq!( + resource.get(type_key).and_then(Value::as_str), + Some(expect_media_type), + "embedded blob keeps its media type: {resource}" ); - let prefix = format!("data:{expect_media_type};base64,"); - let payload = uri - .strip_prefix(&prefix) - .unwrap_or_else(|| panic!("{uri_key} is not a {prefix}... data URI: {resource}")); - STANDARD.decode(payload).expect("base64 payload") + inbuxa_migrate::jmap::blobxfer::download_bytes( + target.client, + target.session, + target.account, + blob_id, + expect_media_type, + "blob", + ) + .unwrap_or_else(|e| panic!("target blobId {blob_id} does not download: {e}")) } #[test] @@ -809,8 +833,13 @@ fn export_inlines_contact_and_event_blobs_instead_of_blob_ids() { let photo_resource = card .pointer("/media/photo") .unwrap_or_else(|| panic!("card has no media.photo: {card}")); + let target = Target { + client: &client, + session: &session, + account: &tgt.account_id, + }; assert_eq!( - data_uri_bytes(photo_resource, "uri", "image/png"), + resource_bytes(&target, photo_resource, "uri", "mediaType", "image/png"), photo, "contact photo bytes survived the export: {card}" ); @@ -839,7 +868,7 @@ fn export_inlines_contact_and_event_blobs_instead_of_blob_ids() { .and_then(|m| m.values().next()) .unwrap_or_else(|| panic!("event has no links: {event}")); assert_eq!( - data_uri_bytes(link, "href", "text/plain"), + resource_bytes(&target, link, "href", "contentType", "text/plain"), agenda, "calendar enclosure bytes survived the export: {event}" ); @@ -927,6 +956,80 @@ fn export_round_trip_and_convergence() { seeder::teardown(base_url()).expect("teardown"); } +#[test] +#[ignore = "requires Docker"] +fn export_emails_in_multiple_import_batches() { + // With maxSetObjects lowered to 2 the email export spans several Email/import batches and still converges. + let fx = seeder::provision(base_url()).expect("provision"); + let src = fx.account("test1").expect("test1"); + let tgt = fx.account("test4").expect("test4"); + let archive = tmp_archive("batches"); + + let imp = ImportConfig { + connect: ConnectConfig { + url: base_url().to_owned(), + auth: basic("test1"), + account: AccountSelector::Id(src.account_id.clone()), + }, + objects: None, + allow_source_change: false, + }; + let s = sync::import_jmap::run(common(&archive, false), imp).expect("import"); + assert!(!s.any_failed()); + let local_emails = { + let conn = Connection::open(&archive).unwrap(); + count(&conn, "emails") + }; + assert!( + local_emails > 2, + "need more than one batch of emails: {local_emails}" + ); + + let mut small = Map::new(); + small.insert("maxSetObjects".to_owned(), json!(2)); + let _guard = JmapSettingsGuard::override_settings(small); + + let probe = HttpClient::new(basic("test4"), RetryPolicy::new(5), true); + let session = Session::discover(&probe, base_url()).expect("discover"); + let limits = session.core_limits().expect("core limits"); + assert_eq!( + limits.max_objects_in_set, 2, + "maxSetObjects override is advertised" + ); + + let e1 = sync::export::run( + common(&archive, false), + export_cfg("test4", &tgt.account_id, false), + ) + .expect("export 1"); + assert!( + !e1.per_type.iter().any(|(_, c)| c.failed > 0), + "export 1 had failures: {e1:?}" + ); + let emails = e1 + .per_type + .iter() + .find(|(t, _)| *t == "Email") + .map(|(_, c)| c.created) + .unwrap_or(0); + assert_eq!( + emails as i64, local_emails, + "every email created across batches" + ); + + let e2 = sync::export::run( + common(&archive, false), + export_cfg("test4", &tgt.account_id, false), + ) + .expect("export 2"); + let created_2: u64 = e2.per_type.iter().map(|(_, c)| c.created).sum(); + assert_eq!(created_2, 0, "second export is convergent: {e2:?}"); + + drop(_guard); + let _ = std::fs::remove_file(&archive); + seeder::teardown(base_url()).expect("teardown"); +} + #[test] #[ignore = "requires Docker"] fn export_prune_destroys_unmatched() { diff --git a/tests/sync_webdav.rs b/tests/sync_webdav.rs index 2806523..196e64d 100644 --- a/tests/sync_webdav.rs +++ b/tests/sync_webdav.rs @@ -19,7 +19,7 @@ fn base_url() -> &'static str { use inbuxa_migrate::jmap::account::AccountSelector; use inbuxa_migrate::jmap::http::Auth; use inbuxa_migrate::logging::Logger; -use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg}; +use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange}; use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig}; fn tmp_archive(tag: &str) -> PathBuf { @@ -58,7 +58,7 @@ fn webdav_cfg(root_url: String, domain: &str, localpart: &str) -> DavImportConfi allow_cleartext: false, dav_connections: 4, multiget_batch: 50, - allow_source_change: false, + source_change: SourceChange::Reject, } } -- 2.54.0