From 210c04997c1b87d453696f7fbb4b183882612a01 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Wed, 30 Sep 2026 10:23:30 -0700 Subject: [PATCH] Track the Gitea workflows and docs/usage.md The .gitignore inherited from upstream ignores every dotfile except .github, and every Markdown file except README and CHANGELOG. The Gitea workflows and docs/usage.md, which the README links to, were therefore never committed. Both are now excepted and added. --- .gitea/workflows/announce.yml | 20 +++ .gitea/workflows/ci.yml | 99 +++++++++++++ .gitignore | 2 + docs/usage.md | 253 ++++++++++++++++++++++++++++++++++ 4 files changed, 374 insertions(+) create mode 100644 .gitea/workflows/announce.yml create mode 100644 .gitea/workflows/ci.yml create mode 100644 docs/usage.md diff --git a/.gitea/workflows/announce.yml b/.gitea/workflows/announce.yml new file mode 100644 index 0000000..14dabb4 --- /dev/null +++ b/.gitea/workflows/announce.yml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: 2026 John Coffey +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# Announce each published release on the community forum, in this project's +# Announcements category (coffey-labs/actions discourse-release; the repo -> +# category map is its release-map.json). Safe to re-run: one topic per tag. +name: announce + +on: + release: + types: [published] + +jobs: + announce: + runs-on: light + steps: + - uses: coffey-labs/actions/discourse-release@baedbb0e89336e49dd5105a22f8ea4f712b453ad + with: + api-key: ${{ secrets.DISCOURSE_RELEASE_KEY }} + discord-webhook: ${{ secrets.DISCORD_RELEASE_WEBHOOK }} diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..1b8ca9b --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: 2026 John Coffey +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# CI on the self-hosted Gitea. Gitea reads .gitea/workflows and ignores +# .github/ once this directory exists; .github/workflows is GitHub's side of +# the switch below. +# +# Every job runs in an image pinned by digest (tag in the trailing comment), +# and the only actions used are coffey-labs/actions ones pinned by SHA. The +# instance resolves short `uses:` against itself, never GitHub, so nothing +# unreviewed can be pulled in. +# +# BUILD ON GITHUB. The org variable BUILD_ON decides which forge builds. +# Set to 'github', the test job below skips and .github/workflows/ci.yml does +# the work on GitHub's runners -- GitHub holds a push mirror of this +# repository, updated on every commit -- and reports back as the commit status +# "github/ci (branch)" or "github/ci (tag)". The `github` job waits for that +# status and passes or fails with it, so a run here still says whether the +# commit is good. Unset (or anything but 'github'), the tests run here. +# +# Releases are built only on GitHub: each architecture is compiled on a +# native runner there, and these runners are amd64 only. With BUILD_ON unset +# a v* tag is tested here but not released. +name: ci + +on: + push: + branches: [main] + tags: ["v*"] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # The unit and mock tests; the #[ignore]d ones need live servers or + # containers and run by hand (README, "Testing"). + test: + if: ${{ vars.BUILD_ON != 'github' }} + runs-on: light + container: + image: rust:1-bookworm@sha256:93ce27a88655056a51dbdd8f5f2d7ddc071c7b0070fb288a37b5a285fc83971e # 1-bookworm + steps: + - uses: coffey-labs/actions/checkout@fab0c4d45e0162963965f1555df27b7bed5e20ec + - run: cargo test --locked + + # Stands in for test and release while GitHub builds: waits for the status + # GitHub's ci.yml posts on this commit and passes or fails with it. A pull + # request is checked at its head commit, which is what GitHub built when + # the branch reached the mirror. Two and a half hours covers a slow queue; + # a timeout here with BUILD_ON=github usually means GitHub never got the + # push -- check the mirror's last error in the repository settings. + github: + if: ${{ vars.BUILD_ON == 'github' }} + # Its own runner label with plenty of slots: this job only polls, but holds a slot + # for as long as the GitHub build takes, and must not starve the build runners. + runs-on: wait + timeout-minutes: 150 + container: + image: rust:1-bookworm@sha256:93ce27a88655056a51dbdd8f5f2d7ddc071c7b0070fb288a37b5a285fc83971e # 1-bookworm + steps: + - shell: bash + env: + TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + SHA: ${{ github.event.pull_request.head.sha || github.sha }} + IS_TAG: ${{ startsWith(github.ref, 'refs/tags/') }} + run: | + set -euo pipefail + apt-get update -qq && apt-get install -y -qq --no-install-recommends jq >/dev/null + ctx="github/ci (branch)"; [ "$IS_TAG" = true ] && ctx="github/ci (tag)" + echo "waiting for '$ctx' on $SHA" + while :; do + s="$(curl -fsS -H "Authorization: token $TOKEN" \ + "$CI_SERVER_INTERNAL/api/v1/repos/$REPO/commits/$SHA/statuses?limit=50" \ + | jq -c --arg c "$ctx" '[.[] | select(.context == $c)] | sort_by(.id) | last // empty')" || s="" + state="$(printf '%s' "$s" | jq -r '.status // .state // empty')" + case "$state" in + success) echo "GitHub: success"; exit 0 ;; + failure|error) echo "GitHub: $state -- $(jq -r '.target_url' <<<"$s")"; exit 1 ;; + esac + sleep 20 + done + + # GitHub makes the Release and publishes it only once its files are + # attached; this waits for GitHub's success and announces then. The action + # makes one topic per tag, so announce.yml firing for the same Release is a + # no-op. + announce: + needs: [github] + if: ${{ startsWith(github.ref, 'refs/tags/v') && needs.github.result == 'success' }} + runs-on: light + steps: + - uses: coffey-labs/actions/discourse-release@baedbb0e89336e49dd5105a22f8ea4f712b453ad + with: + api-key: ${{ secrets.DISCOURSE_RELEASE_KEY }} + discord-webhook: ${{ secrets.DISCORD_RELEASE_WEBHOOK }} + tag: ${{ github.ref_name }} diff --git a/.gitignore b/.gitignore index b52a849..d2d71e3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ target !CHANGELOG.md .* !.github +!.gitea +!/docs/*.md diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 0000000..b6982c3 --- /dev/null +++ b/docs/usage.md @@ -0,0 +1,253 @@ + + +# Using inbuxa-migrate + +The full command reference. The [README](../README.md) says what the tool is +and how to install it. + +## Commands and global flags + +``` +inbuxa-migrate [args...] +``` + +Every command takes a few global flags -- verbosity, worker pool size, retry +policy, TLS handling -- besides its own. The ones that matter most: + +| Flag | Purpose | +| --- | --- | +| `-j, --threads ` | Worker pool size (default: logical CPUs). | +| `--dry-run` | Compute the full plan; perform no writes. | +| `-v`, `-vv`, `-vvv` | Increase log verbosity. | +| `-q, --quiet` | Warnings and errors only. | +| `--max-retries ` | Max retries per request on transient failures (default 5). | +| `--allow-invalid-certs` | Accept self-signed / invalid TLS certs. | + +Secrets come from the `INBUXA_MIGRATE_*` environment variables or a prompt; +see [Credentials](../README.md#credentials). The command line takes them too, +but should not be given them. + +## Import + +``` +inbuxa-migrate import [source-args...] +``` + +Reads a source account into the SQLite `ARCHIVE`, creating it if it is absent. +An archive remembers which account filled it; every importer takes +`--allow-source-change` to fill it from a different one anyway. + +### JMAP + +``` +inbuxa-migrate import jmap \ + --url \ + (--auth-basic [--auth-password ] | --auth-bearer [TOKEN]) \ + (--account-id | --account-name ) \ + [--objects ] \ + +``` + +Imports one JMAP account, from inbuxa, Stalwart or any other JMAP server. `--objects` accepts a comma-separated list of object tokens (`mailbox,email,calendar,calendarevent,addressbook,contactcard,identity,sievescript,participantidentity,filenode`). The default is everything the server advertises. + +### IMAP + +``` +inbuxa-migrate import imap \ + --url imap(s)://host[:port] \ + (--auth-basic [--auth-password ] | --auth-bearer [TOKEN] --auth-user ) \ + [--include ...] [--exclude ...] [--exclude-special ...] \ + [--folder ...] [--subscribed-only] [--noautomap] \ + [--include-deleted] [--allow-cleartext] [--compress] \ + [--fetch-batch ] [--imap-connections <1..8>] \ + +``` + +Imports mail, and only mail, from any IMAP server. Folders are chosen with +`--include` and `--exclude` patterns, or by exact name with `--folder`, but +not both. `--exclude-special` drops folders by SPECIAL-USE role. + +### CalDAV + +``` +inbuxa-migrate import caldav \ + --url \ + (--auth-basic [--auth-password ] | --auth-bearer [TOKEN]) \ + [--allow-cleartext] [--dav-connections <1..8>] [--multiget-batch ] \ + +``` + +Discovers the user's CalDAV principal (or accepts a URL pointing straight at a calendar-home or calendar), then imports calendars and events. + +### CardDAV + +``` +inbuxa-migrate import carddav \ + --url \ + (--auth-basic [--auth-password ] | --auth-bearer [TOKEN]) \ + [--allow-cleartext] [--dav-connections <1..8>] [--multiget-batch ] \ + +``` + +Same shape as `caldav`, but for address books and contacts. + +### WebDAV + +``` +inbuxa-migrate import webdav \ + --url \ + (--auth-basic [--auth-password ] | --auth-bearer [TOKEN]) \ + [--allow-cleartext] [--dav-connections <1..8>] [--multiget-batch ] \ + +``` + +Imports a plain WebDAV file collection as a JMAP `FileNode` tree. + +### ManageSieve + +``` +inbuxa-migrate import managesieve \ + --url sieve(s)://host[:port] \ + (--auth-basic [--auth-password ] | --auth-bearer [TOKEN] --auth-user ) \ + [--allow-cleartext] \ + +``` + +Imports Sieve scripts only. Each script is stored once by content, and the +active one is recorded. + +### Maildir + +``` +inbuxa-migrate import maildir \ + [--include ...] [--exclude ...] [--folder ...] \ + [--noautomap] [--include-deleted] +``` + +Reads a local Maildir++ tree: a directory with `cur/`, `new/` and `tmp/`. No +network. Folders are chosen as for IMAP. + +### Google Takeout + +``` +inbuxa-migrate import takeout [--noautomap] +``` + +Finds every `.mbox`, `.ics` and `.vcf` file under a directory and imports it. +It is shaped for Google Takeout but reads any such tree. `--noautomap` stops +it giving Gmail's system labels their mailbox roles. + +### Microsoft Exchange (EWS) + +``` +inbuxa-migrate import exchange-ews \ + [--url ] [--mailbox ] \ + [--mailbox-kind primary|archive|public-folders] \ + (--auth-basic [--auth-password ] \ + | --auth-bearer [TOKEN] [--ews-tenant --ews-client-id \ + (--ews-device-code | --ews-client-secret )]) \ + [--ews-connections <1..8>] [--ews-getitem-batch ] [--ews-attachment-batch ] \ + [--ews-no-syncfolderitems] \ + +``` + +Imports a mailbox from an on-premises Exchange Server through EWS. Without +`--url` it uses Autodiscover, and then needs `--mailbox`. It signs in with +Basic, with a bearer token acquired beforehand, with OAuth's interactive +device-code flow, or with app-only client credentials. + +For Exchange Online, use `exchange-graph` instead. Microsoft is retiring EWS in Exchange Online: from October 1, 2026 it is blocked unless a tenant administrator sets `EwsEnabled` to `True` and adds the client id to `EwsAllowedAppIDs`, and on April 1, 2027 it is switched off for every tenant. On-premises Exchange Server is not affected. + +### Microsoft Exchange (Graph) + +``` +inbuxa-migrate import exchange-graph \ + (--client-id [--tenant ] | --access-token [TOKEN]) \ + [--user ] \ + [--mailbox-kind primary|archive] \ + [--objects mail,calendar,contacts] \ + [--event-body-format text|html] \ + [--graph-connections <1..16>] [--top <1..1000>] \ + +``` + +Imports a mailbox from Exchange Online through Microsoft Graph. Without +`--access-token` it signs in with the interactive device-code flow. `public-folders` is rejected here: Graph does not expose public folders, so they can only be imported with `exchange-ews`, which for Exchange Online is subject to the retirement described above. + +## Export + +``` +inbuxa-migrate export \ + --url \ + (--auth-basic [--auth-password ] | --auth-bearer [TOKEN]) \ + (--account-id | --account-name ) \ + [--objects ] [--prune [--yes]] \ + +``` + +Writes `ARCHIVE` into an account on a JMAP server, usually inbuxa. It keeps no +state of its own: every run matches the archive against the target afresh. +By default it only adds and updates -- items that match are updated, the +rest are created, and anything already on the target that the archive does +not cover is left alone. + +`--prune` also deletes what is on the target and not in the archive. It asks +first; `--yes` answers for it, for scripts. Export speaks JMAP only. + +## Inspect + +``` +inbuxa-migrate inspect [TYPE] [--limit ] [--offset ] +``` + +Read-only dump of a local archive. This command never opens a network connection and never writes to the archive. + +- Omit `TYPE` for a per-type summary (counts of every object kind plus blob storage stats). +- Pass an object type to dump it: `mailbox`, `email`, `identity`, `sievescript`, `addressbook`, `contactcard`, `calendar`, `calendarevent`, `participantidentity`, `filenode`. +- `mailbox` and `filenode` render as a tree (`--limit`/`--offset` are ignored); all other types use a paginated list and respect `--limit` and `--offset`. + + +## Live tests + +The tests against real servers -- a Stalwart server as a JMAP source, and +Dovecot, Cyrus, Radicale, Baikal and Apache `mod_dav` -- are marked +`#[ignore]`. Each test binary boots its own throwaway container through +`testcontainers`, pulling the image on first use, so Docker must be running +(`docker info` succeeds) before they start. + +Run them one binary at a time, always with one test thread: + +```sh +cargo test --test sync_jmap -- --ignored --test-threads=1 # live JMAP import/export/convergence/prune +cargo test --test sync_imap -- --ignored --test-threads=1 +cargo test --test sync_managesieve -- --ignored --test-threads=1 +cargo test --test sync_maildir -- --ignored --test-threads=1 +cargo test --test sync_caldav -- --ignored --test-threads=1 +cargo test --test sync_carddav -- --ignored --test-threads=1 +cargo test --test sync_webdav -- --ignored --test-threads=1 +cargo test --test live_stalwart -- --ignored --test-threads=1 +cargo test --test seed_smoke -- --ignored --test-threads=1 +cargo test --test seed_only -- --ignored --test-threads=1 + +# Third-party-server tests (one container each): +cargo test --test integration_radicale -- --ignored --test-threads=1 +cargo test --test integration_baikal -- --ignored --test-threads=1 +cargo test --test integration_webdav -- --ignored --test-threads=1 +cargo test --test integration_dovecot -- --ignored --test-threads=1 +cargo test --test integration_cyrus -- --ignored --test-threads=1 + +# Slow tests +cargo test --test mock_jmap -- --ignored +``` + +One thread is required, not advised. The tests in a binary share its +container, and each one creates and removes the same disposable domain, +`inbuxa-migrate.org`, and opens its archive with SQLite's `EXCLUSIVE` lock. +Separate binaries each boot their own container on their own ports, so +running them one after another is safe. +