Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 jcoffey-dev
|
db76a1044f |
@@ -150,14 +150,6 @@ jobs:
|
||||
env:
|
||||
TAG: ${{ github.ref_name }}
|
||||
steps:
|
||||
# The release notes are this version's section of CHANGELOG.md, so the
|
||||
# release, and the announcement made from it, say what changed. Checked
|
||||
# out first: a checkout cleans the workspace, and would take dist/ with
|
||||
# it if it ran after the download.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
sparse-checkout: CHANGELOG.md
|
||||
sparse-checkout-cone-mode: false
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: dist
|
||||
@@ -172,22 +164,14 @@ jobs:
|
||||
set -euo pipefail
|
||||
API="$GITEA_URL/api/v1/repos/$GITHUB_REPOSITORY"
|
||||
auth=(-H "Authorization: token $GITEA_TOKEN")
|
||||
# Everything the release carries has to be here before a release is made.
|
||||
for f in inbuxa-migrate-linux-amd64.tar.gz inbuxa-migrate-linux-arm64.tar.gz SHA256SUMS; do
|
||||
[ -s "dist/$f" ] || { echo "::error::dist/$f is missing; not creating a release"; exit 1; }
|
||||
done
|
||||
files="Binaries for linux/amd64 and linux/arm64. Verify with SHA256SUMS."
|
||||
notes="$(awk -v v="${TAG#v}" 'index($0, "## [" v "]") == 1 {f = 1; next}
|
||||
f && (/^## / || /^---$/) {exit}
|
||||
f' CHANGELOG.md | sed -e '/./,$!d')"
|
||||
if [ -n "$notes" ]; then notes="$notes"$'\n\n'"$files"; else notes="$files"; fi
|
||||
# Reuse the Release if the tag already has one (a re-run), else make a
|
||||
# draft of our own.
|
||||
created=0
|
||||
id="$(curl -fsS "${auth[@]}" "$API/releases/tags/$TAG" 2>/dev/null | jq -r '.id // empty' || true)"
|
||||
if [ -z "$id" ]; then
|
||||
id="$(curl -fsS "${auth[@]}" -H 'Content-Type: application/json' \
|
||||
-d "$(jq -n --arg t "$TAG" --arg b "$notes" '{tag_name:$t, name:$t, draft:true, body:$b}')" \
|
||||
-d "$(jq -n --arg t "$TAG" '{tag_name:$t, name:$t, draft:true,
|
||||
body:"Binaries for linux/amd64 and linux/arm64. Verify with SHA256SUMS."}')" \
|
||||
"$API/releases" | jq -r '.id // empty')"
|
||||
[ -n "$id" ] || { echo "::error::could not create the release"; exit 1; }
|
||||
created=1
|
||||
|
||||
+1
-31
@@ -3,9 +3,7 @@
|
||||
All notable changes to this project are recorded here. Versions are dates:
|
||||
release `v2026.9.30` is version `2026.9.30`.
|
||||
|
||||
## [2026.9.30] -- 2026-09-30
|
||||
|
||||
The first release of inbuxa-migrate.
|
||||
## [Unreleased] -- 2026.9.30
|
||||
|
||||
### Changed
|
||||
- Renamed to inbuxa-migrate: the binary, the crate, and the credential
|
||||
@@ -19,34 +17,6 @@ The first release of inbuxa-migrate.
|
||||
- Released as Linux archives for amd64 and arm64 on the Gitea release page,
|
||||
with `SHA256SUMS`. The npm, Homebrew, shell and PowerShell installers and
|
||||
the MSI are gone.
|
||||
- A second export brings matched items up to date instead of skipping them:
|
||||
flags and folders on mail, changed contacts and events, and edited Sieve
|
||||
scripts.
|
||||
- A message filed in several folders is written once, in every one of them,
|
||||
and a resumed export adds any folder it was still missing.
|
||||
- Sieve scripts from a Stalwart server keep working on inbuxa: their
|
||||
`vnd.stalwart.*` extension names are renamed to inbuxa's, each rename is
|
||||
logged, and a script that can't be activated fails the export instead of
|
||||
leaving filtering quietly off.
|
||||
- `--allow-invalid-certs` covers only the server named with `--url`. Microsoft
|
||||
and Google sign-in are always verified.
|
||||
- Export imports mail in batches, uploads in parallel within the target's
|
||||
limits, and prints progress: count, rate and time left.
|
||||
- `export --dry-run` predicts what would fail -- messages over the target's
|
||||
upload limit, objects over its request limit, Sieve scripts it would reject
|
||||
-- lists what would be created, updated and skipped, and exits as the real
|
||||
run would.
|
||||
|
||||
### Fixed
|
||||
- Every connection times out instead of waiting forever on a dropped link,
|
||||
and a timeout is retried like any other transient error.
|
||||
- An interrupted IMAP or Maildir import keeps what it wrote, and a message
|
||||
that can't be read is recorded and skipped rather than ending the folder.
|
||||
- IMAP and EWS imports hold a bounded amount in memory, in batches capped by
|
||||
size as well as count.
|
||||
- The archive is created readable by its owner only, and an existing archive
|
||||
others can read is reported.
|
||||
- A dry run no longer writes to the target's address books.
|
||||
|
||||
---
|
||||
|
||||
|
||||
+3
-70
@@ -26,26 +26,7 @@ policy, TLS handling -- besides its own. The ones that matter most:
|
||||
| `-v`, `-vv`, `-vvv` | Increase log verbosity. |
|
||||
| `-q, --quiet` | Warnings and errors only. |
|
||||
| `--max-retries <N>` | Max retries per request on transient failures (default 5). |
|
||||
| `--allow-invalid-certs` | Accept a self-signed or otherwise invalid certificate from the server named by `--url` (see below). |
|
||||
|
||||
### Invalid certificates
|
||||
|
||||
`--allow-invalid-certs` is for a server with a self-signed certificate, and
|
||||
it applies to that server only: the host in `--url`, whether that is the
|
||||
source of an import or the target of an export. For an Exchange import with
|
||||
no `--url`, it covers the mailbox's own domain and the hosts under it, which
|
||||
is where on-premises Autodiscover looks, and then only the EWS endpoint
|
||||
Autodiscover finds.
|
||||
|
||||
Every other host is verified as usual, including a host the server
|
||||
redirects to or names for its API, uploads or downloads. The Microsoft and
|
||||
Google sign-in and cloud endpoints are always verified, with or without the
|
||||
flag: a certificate that fails there is an attack or a broken network, never
|
||||
a server to trust.
|
||||
|
||||
Connections time out rather than wait forever: 30 seconds to connect, 5
|
||||
minutes for the server's first byte, and 30 minutes to read a whole
|
||||
response. A timed-out request is retried like any other transient failure.
|
||||
| `--allow-invalid-certs` | Accept self-signed / invalid TLS certs. |
|
||||
|
||||
Secrets come from the `INBUXA_MIGRATE_*` environment variables or a prompt;
|
||||
see [Credentials](../README.md#credentials). The command line takes them too,
|
||||
@@ -83,7 +64,7 @@ inbuxa-migrate import imap \
|
||||
[--include <REGEX>...] [--exclude <REGEX>...] [--exclude-special <ROLE>...] \
|
||||
[--folder <NAME>...] [--subscribed-only] [--noautomap] \
|
||||
[--include-deleted] [--allow-cleartext] [--compress] \
|
||||
[--fetch-batch <N>] [--fetch-batch-mib <MIB>] [--imap-connections <1..8>] \
|
||||
[--fetch-batch <N>] [--imap-connections <1..8>] \
|
||||
<ARCHIVE>
|
||||
```
|
||||
|
||||
@@ -91,14 +72,6 @@ Imports mail, and only mail, from any IMAP server. Folders are chosen with
|
||||
`--include` and `--exclude` patterns, or by exact name with `--folder`, but
|
||||
not both. `--exclude-special` drops folders by SPECIAL-USE role.
|
||||
|
||||
Messages are fetched in chunks of at most `--fetch-batch` messages and
|
||||
`--fetch-batch-mib` MiB (32 by default), so a folder of large attachments is
|
||||
fetched a little at a time, like any other. A single message larger than the
|
||||
cap is fetched on its own. Each chunk is written to the archive as it
|
||||
arrives: an interrupted import keeps what it fetched, and the next run picks
|
||||
up from there. A message that can't be imported is reported with its folder
|
||||
and UID, and the rest of the folder carries on.
|
||||
|
||||
### CalDAV
|
||||
|
||||
```
|
||||
@@ -178,8 +151,7 @@ inbuxa-migrate import exchange-ews \
|
||||
(--auth-basic <USER> [--auth-password <PASS>] \
|
||||
| --auth-bearer [TOKEN] [--ews-tenant <T> --ews-client-id <ID> \
|
||||
(--ews-device-code | --ews-client-secret <SECRET>)]) \
|
||||
[--ews-connections <1..8>] [--ews-getitem-batch <N>] [--ews-getitem-batch-mib <MIB>] \
|
||||
[--ews-attachment-batch <N>] \
|
||||
[--ews-connections <1..8>] [--ews-getitem-batch <N>] [--ews-attachment-batch <N>] \
|
||||
[--ews-no-syncfolderitems] \
|
||||
<ARCHIVE>
|
||||
```
|
||||
@@ -189,12 +161,6 @@ Imports a mailbox from an on-premises Exchange Server through EWS. Without
|
||||
Basic, with a bearer token acquired beforehand, with OAuth's interactive
|
||||
device-code flow, or with app-only client credentials.
|
||||
|
||||
Items are fetched in GetItem batches of at most `--ews-getitem-batch` items
|
||||
and `--ews-getitem-batch-mib` MiB (32 by default), `--ews-connections` at a
|
||||
time. The byte cap applies where Exchange reports each item's size, which it
|
||||
does on a full listing; items found through an incremental sync are batched
|
||||
by count.
|
||||
|
||||
For Exchange Online, use `exchange-graph` instead. Microsoft is retiring EWS in Exchange Online: from October 1, 2026 it is blocked unless a tenant administrator sets `EwsEnabled` to `True` and adds the client id to `EwsAllowedAppIDs`, and on April 1, 2027 it is switched off for every tenant. On-premises Exchange Server is not affected.
|
||||
|
||||
### Microsoft Exchange (Graph)
|
||||
@@ -244,42 +210,9 @@ what changed at the source in between.
|
||||
- **Sieve scripts** are matched by name, and the target's is replaced when
|
||||
its content differs from the archive's.
|
||||
|
||||
Messages go in batches, as many to an `Email/import` as the target's
|
||||
`maxObjectsInSet` allows, up to 50, with their blobs uploaded several at a
|
||||
time -- the target's `maxConcurrentUpload`, and no more than `--threads`.
|
||||
Each message is still counted on its own: one the target rejects fails
|
||||
alone, and the rest of its batch lands. A batch that ends without a clear
|
||||
answer -- a dropped connection, a gateway timeout -- is never sent again as
|
||||
it was. The target is read first, the messages that arrived are counted as
|
||||
created, and only the rest are imported again, so none is ever doubled.
|
||||
|
||||
While it runs, export prints a line every few seconds for mail, contacts and
|
||||
events: how many of how many, how fast, and about how long is left. Each
|
||||
type ends with a line of what was created, updated, left unchanged and
|
||||
failed.
|
||||
|
||||
`--prune` also deletes what is on the target and not in the archive. It asks
|
||||
first; `--yes` answers for it, for scripts. Export speaks JMAP only.
|
||||
|
||||
With `--dry-run`, export reads the target and writes nothing to the account.
|
||||
It prints the plan in plain words: for each type, how much would be
|
||||
created, updated, left unchanged or deleted, and what would fail. It
|
||||
catches in advance the failures a real run would hit:
|
||||
|
||||
- a message larger than the target's `maxSizeUpload`;
|
||||
- a contact, event or other object too large for one request under its
|
||||
`maxSizeRequest`, as happens when a photo is carried inline;
|
||||
- a Sieve script the target would reject. Each script that would be written
|
||||
is checked with `SieveScript/validate`, after any `vnd.stalwart.*` names
|
||||
are renamed, so what is checked is what would be uploaded. The check needs
|
||||
the script as a blob, so the dry run uploads Sieve scripts, and only them;
|
||||
a blob that nothing uses is discarded by the server. A target without
|
||||
`SieveScript/validate` gets one warning, and its scripts are not checked.
|
||||
|
||||
The plan lists each predicted failure and its reason. When anything would
|
||||
fail, the dry run exits 5, as the real run would, so a script can stop
|
||||
before it starts.
|
||||
|
||||
## Inspect
|
||||
|
||||
```
|
||||
|
||||
+1
-25
@@ -122,10 +122,7 @@ struct GlobalArgs {
|
||||
)]
|
||||
max_retries: u32,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
help = "Accept an invalid TLS certificate from the --url host only; sign-in endpoints are always verified"
|
||||
)]
|
||||
#[arg(long, help = "Accept self-signed / invalid TLS certificates")]
|
||||
allow_invalid_certs: bool,
|
||||
}
|
||||
|
||||
@@ -277,14 +274,6 @@ struct ImapImportArgs {
|
||||
)]
|
||||
fetch_batch: usize,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
value_name = "MIB",
|
||||
default_value_t = 32,
|
||||
help = "Most message bytes per body FETCH chunk, in MiB (one larger message goes alone)"
|
||||
)]
|
||||
fetch_batch_mib: u64,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
value_name = "N",
|
||||
@@ -812,7 +801,6 @@ fn resolve_imap_import(args: ImapImportArgs) -> Result<Action, Error> {
|
||||
automap: !args.noautomap,
|
||||
include_deleted: args.include_deleted,
|
||||
fetch_batch: args.fetch_batch,
|
||||
fetch_batch_bytes: args.fetch_batch_mib.max(1).saturating_mul(1024 * 1024),
|
||||
imap_connections,
|
||||
allow_source_change: args.allow_source_change,
|
||||
},
|
||||
@@ -982,14 +970,6 @@ pub struct ExchangeEwsImportArgs {
|
||||
)]
|
||||
ews_getitem_batch: usize,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
value_name = "MIB",
|
||||
default_value_t = 32,
|
||||
help = "Most item bytes per GetItem batch, in MiB (one larger item goes alone)"
|
||||
)]
|
||||
ews_getitem_batch_mib: u64,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
value_name = "N",
|
||||
@@ -1039,10 +1019,6 @@ fn resolve_exchange_ews_import(args: ExchangeEwsImportArgs) -> Result<Action, Er
|
||||
auth,
|
||||
ews_connections,
|
||||
getitem_batch: args.ews_getitem_batch.max(1),
|
||||
getitem_batch_bytes: args
|
||||
.ews_getitem_batch_mib
|
||||
.max(1)
|
||||
.saturating_mul(1024 * 1024),
|
||||
attachment_batch: args.ews_attachment_batch.max(1),
|
||||
use_syncfolderitems: !args.ews_no_syncfolderitems,
|
||||
allow_source_change: args.allow_source_change,
|
||||
|
||||
+15
-47
@@ -14,6 +14,7 @@ use ureq::Agent;
|
||||
use ureq::Body;
|
||||
use ureq::config::{Config, RedirectAuthHeaders};
|
||||
use ureq::http::{Method, Request, Response};
|
||||
use ureq::tls::{RootCerts, TlsConfig};
|
||||
|
||||
use crate::dav::parse::{ControlStrippingReader, DavResponse, parse_multistatus};
|
||||
use crate::dav::retry::{DavOutcome, classify};
|
||||
@@ -21,7 +22,6 @@ use crate::jmap::error::JmapError;
|
||||
use crate::jmap::http::{Auth, RetryPolicy, retry_after_header};
|
||||
use crate::jmap::retry::{self, RateLimitState};
|
||||
use crate::logging::{HttpCall, LEVEL_BODIES, LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
||||
use crate::net::{CertOverride, tls, with_timeouts};
|
||||
|
||||
const MAX_BODY: u64 = 512 * 1024 * 1024;
|
||||
const LONG_RETRY_THRESHOLD: Duration = Duration::from_secs(10);
|
||||
@@ -48,8 +48,6 @@ pub struct MultiStatus {
|
||||
|
||||
struct Inner {
|
||||
agent: Agent,
|
||||
lax_agent: Option<Agent>,
|
||||
certs: CertOverride,
|
||||
auth: Auth,
|
||||
retry: RetryPolicy,
|
||||
rate_limit: RateLimitState,
|
||||
@@ -59,42 +57,31 @@ struct Inner {
|
||||
user_agent: String,
|
||||
}
|
||||
|
||||
impl Inner {
|
||||
/// The agent for `url`: the one that accepts invalid certificates only for
|
||||
/// a host `--allow-invalid-certs` covers, and the verifying one otherwise.
|
||||
fn agent_for(&self, url: &str) -> &Agent {
|
||||
match &self.lax_agent {
|
||||
Some(lax) if self.certs.allows(url) => lax,
|
||||
_ => &self.agent,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct DavClient {
|
||||
inner: Arc<Inner>,
|
||||
}
|
||||
|
||||
impl DavClient {
|
||||
pub fn new(auth: Auth, retry: RetryPolicy, certs: CertOverride) -> Self {
|
||||
let build = |accept_invalid: bool| -> Agent {
|
||||
let config: Config = with_timeouts!(
|
||||
Config::builder()
|
||||
pub fn new(auth: Auth, retry: RetryPolicy, allow_invalid_certs: bool) -> Self {
|
||||
let config: Config = Config::builder()
|
||||
.http_status_as_error(false)
|
||||
.allow_non_standard_methods(true)
|
||||
.max_redirects(0)
|
||||
.redirect_auth_headers(RedirectAuthHeaders::SameHost)
|
||||
.tls_config(tls(accept_invalid))
|
||||
.tls_config(
|
||||
TlsConfig::builder()
|
||||
.unversioned_rustls_crypto_provider(std::sync::Arc::new(
|
||||
rustls::crypto::aws_lc_rs::default_provider(),
|
||||
))
|
||||
.root_certs(RootCerts::PlatformVerifier)
|
||||
.disable_verification(allow_invalid_certs)
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
config.new_agent()
|
||||
};
|
||||
let lax_agent = certs.is_active().then(|| build(true));
|
||||
DavClient {
|
||||
inner: Arc::new(Inner {
|
||||
agent: build(false),
|
||||
lax_agent,
|
||||
certs,
|
||||
agent: config.new_agent(),
|
||||
auth,
|
||||
retry,
|
||||
rate_limit: RateLimitState::new(),
|
||||
@@ -835,7 +822,7 @@ impl DavClient {
|
||||
let request = builder
|
||||
.body(payload)
|
||||
.map_err(|e| ureq::Error::Other(Box::new(std::io::Error::other(e))))?;
|
||||
self.inner.agent_for(req.url).run(request)
|
||||
self.inner.agent.run(request)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -954,25 +941,6 @@ fn truncate(body: &[u8]) -> String {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::net::CertOverride;
|
||||
|
||||
#[test]
|
||||
fn every_timeout_is_a_retryable_transport_error() {
|
||||
for t in [
|
||||
ureq::Timeout::Connect,
|
||||
ureq::Timeout::SendRequest,
|
||||
ureq::Timeout::SendBody,
|
||||
ureq::Timeout::RecvResponse,
|
||||
ureq::Timeout::RecvBody,
|
||||
] {
|
||||
let err = map_ureq_error(ureq::Error::Timeout(t));
|
||||
assert!(matches!(err, JmapError::Transport(_)), "{t:?} -> {err:?}");
|
||||
assert!(
|
||||
matches!(transport_disposition(&err), retry::Disposition::Retryable),
|
||||
"{t:?} must be retried"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_constructs_cleanly() {
|
||||
@@ -982,7 +950,7 @@ mod tests {
|
||||
password: "p".into(),
|
||||
},
|
||||
RetryPolicy::new(3),
|
||||
CertOverride::none(),
|
||||
false,
|
||||
);
|
||||
assert_eq!(c.retries_observed(), 0);
|
||||
assert_eq!(c.retry_after_sleeps(), 0);
|
||||
@@ -995,7 +963,7 @@ mod tests {
|
||||
token: "abc".into(),
|
||||
},
|
||||
RetryPolicy::new(0),
|
||||
CertOverride::none(),
|
||||
false,
|
||||
);
|
||||
let logger = c.logger();
|
||||
assert_eq!(logger.level(), LEVEL_DEFAULT);
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -10,7 +9,6 @@ use rusqlite::Connection;
|
||||
pub const SCHEMA_SQL: &str = include_str!("schema.sql");
|
||||
|
||||
pub fn open(path: &std::path::Path) -> Result<Connection, OpenError> {
|
||||
private_archive(path)?;
|
||||
let conn = Connection::open(path)?;
|
||||
apply_pragmas(&conn)?;
|
||||
apply_schema(&conn)?;
|
||||
@@ -74,50 +72,6 @@ fn ensure_graph_ids_accept_file_nodes(conn: &Connection) -> Result<(), OpenError
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// An archive holds a whole mailbox, so it is created readable by its owner
|
||||
/// only. SQLite gives its `-wal` and `-shm` files the database file's mode,
|
||||
/// so they follow. An existing archive others can read is left as it is,
|
||||
/// with a warning and the command that fixes it.
|
||||
#[cfg(unix)]
|
||||
fn private_archive(path: &std::path::Path) -> Result<(), OpenError> {
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
match std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.mode(0o600)
|
||||
.open(path)
|
||||
{
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
|
||||
if let Some(warning) = permission_warning(path) {
|
||||
eprintln!("warning: {warning}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => Err(OpenError::Create(e)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn private_archive(_path: &std::path::Path) -> Result<(), OpenError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The warning for an archive that someone other than its owner can read.
|
||||
#[cfg(unix)]
|
||||
fn permission_warning(path: &std::path::Path) -> Option<String> {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let mode = std::fs::metadata(path).ok()?.permissions().mode();
|
||||
(mode & 0o077 != 0).then(|| {
|
||||
format!(
|
||||
"archive {} can be read by other users (mode {:o}); run: chmod 600 {}",
|
||||
path.display(),
|
||||
mode & 0o777,
|
||||
path.display()
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn apply_pragmas(conn: &Connection) -> Result<(), OpenError> {
|
||||
conn.pragma_update(None, "journal_mode", "WAL")?;
|
||||
conn.pragma_update(None, "foreign_keys", "ON")?;
|
||||
@@ -129,53 +83,4 @@ fn apply_pragmas(conn: &Connection) -> Result<(), OpenError> {
|
||||
pub enum OpenError {
|
||||
#[error("sqlite error: {0}")]
|
||||
Sqlite(#[from] rusqlite::Error),
|
||||
#[error("cannot create the archive: {0}")]
|
||||
Create(std::io::Error),
|
||||
}
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
mod permission_tests {
|
||||
use super::*;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
fn mode(p: &std::path::Path) -> u32 {
|
||||
std::fs::metadata(p).unwrap().permissions().mode() & 0o777
|
||||
}
|
||||
|
||||
fn scratch(name: &str) -> std::path::PathBuf {
|
||||
let dir =
|
||||
std::env::temp_dir().join(format!("inbuxa-migrate-perm-{}-{name}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
dir.join("a.sqlite")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_new_archive_and_its_wal_are_private() {
|
||||
let path = scratch("new");
|
||||
let conn = open(&path).unwrap();
|
||||
conn.execute_batch("CREATE TABLE t(x); INSERT INTO t VALUES (1);")
|
||||
.unwrap();
|
||||
assert_eq!(mode(&path), 0o600);
|
||||
let wal = path.with_extension("sqlite-wal");
|
||||
assert!(wal.exists(), "WAL mode writes a -wal file");
|
||||
assert_eq!(mode(&wal), 0o600);
|
||||
drop(conn);
|
||||
let _ = std::fs::remove_dir_all(path.parent().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_existing_readable_archive_is_warned_about_not_changed() {
|
||||
let path = scratch("existing");
|
||||
drop(open(&path).unwrap());
|
||||
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap();
|
||||
let warning = permission_warning(&path).expect("warns");
|
||||
assert!(warning.contains("chmod 600"), "{warning}");
|
||||
assert!(warning.contains("644"), "{warning}");
|
||||
drop(open(&path).unwrap());
|
||||
assert_eq!(mode(&path), 0o644, "left as it is");
|
||||
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap();
|
||||
assert!(permission_warning(&path).is_none());
|
||||
let _ = std::fs::remove_dir_all(path.parent().unwrap());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -10,10 +9,10 @@ use quick_xml::events::Event;
|
||||
use serde_json::Value;
|
||||
use ureq::Agent;
|
||||
use ureq::config::Config;
|
||||
use ureq::tls::{RootCerts, TlsConfig};
|
||||
|
||||
use crate::exchange_ews::error::EwsError;
|
||||
use crate::exchange_ews::parse::entity_to_char;
|
||||
use crate::net::{CertOverride, tls, with_timeouts};
|
||||
|
||||
const V2_HOST: &str = "https://outlook.office365.com";
|
||||
const POX_REQ_NS: &str =
|
||||
@@ -49,7 +48,7 @@ pub fn discover(
|
||||
supplied_url: Option<&str>,
|
||||
email: Option<&str>,
|
||||
auth_header: Option<&str>,
|
||||
certs: &CertOverride,
|
||||
allow_invalid_certs: bool,
|
||||
) -> Result<DiscoveryResult, EwsError> {
|
||||
if let Some(url) = supplied_url
|
||||
&& is_fully_qualified_ews_url(url)
|
||||
@@ -64,17 +63,8 @@ pub fn discover(
|
||||
"either a fully-qualified --url or --mailbox is required".to_owned(),
|
||||
));
|
||||
};
|
||||
// Autodiscover v2 is Microsoft's own service and is always verified; a v1
|
||||
// candidate gets the relaxed agent only if `--allow-invalid-certs` covers it.
|
||||
let strict = build_agent(false);
|
||||
let lax = certs.is_active().then(|| build_agent(true));
|
||||
let pick = |url: &str| -> &Agent {
|
||||
match &lax {
|
||||
Some(agent) if certs.allows(url) => agent,
|
||||
_ => &strict,
|
||||
}
|
||||
};
|
||||
if let Ok(url) = autodiscover_v2(&strict, email) {
|
||||
let agent = build_agent(allow_invalid_certs);
|
||||
if let Ok(url) = autodiscover_v2(&agent, email) {
|
||||
return Ok(DiscoveryResult {
|
||||
ews_url: url,
|
||||
source: DiscoverySource::V2,
|
||||
@@ -92,7 +82,7 @@ pub fn discover(
|
||||
let candidates = pox_candidates(domain);
|
||||
for candidate in &candidates {
|
||||
tried.push(candidate.clone());
|
||||
match autodiscover_v1(pick(candidate), candidate, ¤t_email, auth_header) {
|
||||
match autodiscover_v1(&agent, candidate, ¤t_email, auth_header) {
|
||||
Ok(PoxOutcome::EwsUrl(url)) => {
|
||||
return Ok(DiscoveryResult {
|
||||
ews_url: url,
|
||||
@@ -114,7 +104,7 @@ pub fn discover(
|
||||
url_redirects += 1;
|
||||
tried.push(url.clone());
|
||||
if let Ok(PoxOutcome::EwsUrl(u)) =
|
||||
autodiscover_v1(pick(&url), &url, ¤t_email, auth_header)
|
||||
autodiscover_v1(&agent, &url, ¤t_email, auth_header)
|
||||
{
|
||||
return Ok(DiscoveryResult {
|
||||
ews_url: u,
|
||||
@@ -140,11 +130,17 @@ pub fn discover(
|
||||
)))
|
||||
}
|
||||
|
||||
fn build_agent(accept_invalid: bool) -> Agent {
|
||||
let config: Config = with_timeouts!(
|
||||
Config::builder()
|
||||
fn build_agent(allow_invalid_certs: bool) -> Agent {
|
||||
let config: Config = Config::builder()
|
||||
.http_status_as_error(false)
|
||||
.tls_config(tls(accept_invalid))
|
||||
.tls_config(
|
||||
TlsConfig::builder()
|
||||
.unversioned_rustls_crypto_provider(std::sync::Arc::new(
|
||||
rustls::crypto::aws_lc_rs::default_provider(),
|
||||
))
|
||||
.root_certs(RootCerts::PlatformVerifier)
|
||||
.disable_verification(allow_invalid_certs)
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
config.new_agent()
|
||||
|
||||
+16
-45
@@ -12,6 +12,7 @@ use std::time::{Duration, Instant};
|
||||
|
||||
use ureq::Agent;
|
||||
use ureq::config::{Config, RedirectAuthHeaders};
|
||||
use ureq::tls::{RootCerts, TlsConfig};
|
||||
|
||||
use crate::exchange_ews::error::EwsError;
|
||||
use crate::exchange_ews::parse::{EnvelopeKind, SoapFault, read_envelope_summary};
|
||||
@@ -21,15 +22,12 @@ use crate::exchange_ews::types::ServerVersion;
|
||||
use crate::jmap::http::{Auth, RetryPolicy, retry_after_header};
|
||||
use crate::jmap::retry::{self, Disposition, RateLimitState};
|
||||
use crate::logging::{HttpCall, LEVEL_BODIES, LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
||||
use crate::net::{CertOverride, tls, with_timeouts};
|
||||
|
||||
const MAX_BODY: u64 = 2 * 1024 * 1024 * 1024;
|
||||
const LONG_RETRY_THRESHOLD: Duration = Duration::from_secs(10);
|
||||
|
||||
struct Inner {
|
||||
agent: Agent,
|
||||
lax_agent: Option<Agent>,
|
||||
certs: CertOverride,
|
||||
auth: Mutex<Auth>,
|
||||
impersonated_smtp: Mutex<Option<String>>,
|
||||
anchor_mailbox: Mutex<Option<String>>,
|
||||
@@ -44,17 +42,6 @@ struct Inner {
|
||||
user_agent: String,
|
||||
}
|
||||
|
||||
impl Inner {
|
||||
/// The agent for `url`: the one that accepts invalid certificates only for
|
||||
/// a host `--allow-invalid-certs` covers, and the verifying one otherwise.
|
||||
fn agent_for(&self, url: &str) -> &Agent {
|
||||
match &self.lax_agent {
|
||||
Some(lax) if self.certs.allows(url) => lax,
|
||||
_ => &self.agent,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct EwsClient {
|
||||
inner: Arc<Inner>,
|
||||
@@ -67,23 +54,23 @@ pub struct SoapResponse {
|
||||
}
|
||||
|
||||
impl EwsClient {
|
||||
pub fn new(auth: Auth, retry: RetryPolicy, certs: CertOverride) -> EwsClient {
|
||||
let build = |accept_invalid: bool| -> Agent {
|
||||
let config: Config = with_timeouts!(
|
||||
Config::builder()
|
||||
pub fn new(auth: Auth, retry: RetryPolicy, allow_invalid_certs: bool) -> EwsClient {
|
||||
let config: Config = Config::builder()
|
||||
.http_status_as_error(false)
|
||||
.redirect_auth_headers(RedirectAuthHeaders::SameHost)
|
||||
.tls_config(tls(accept_invalid))
|
||||
.tls_config(
|
||||
TlsConfig::builder()
|
||||
.unversioned_rustls_crypto_provider(std::sync::Arc::new(
|
||||
rustls::crypto::aws_lc_rs::default_provider(),
|
||||
))
|
||||
.root_certs(RootCerts::PlatformVerifier)
|
||||
.disable_verification(allow_invalid_certs)
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
config.new_agent()
|
||||
};
|
||||
let lax_agent = certs.is_active().then(|| build(true));
|
||||
EwsClient {
|
||||
inner: Arc::new(Inner {
|
||||
agent: build(false),
|
||||
lax_agent,
|
||||
certs,
|
||||
agent: config.new_agent(),
|
||||
auth: Mutex::new(auth),
|
||||
impersonated_smtp: Mutex::new(None),
|
||||
anchor_mailbox: Mutex::new(None),
|
||||
@@ -421,7 +408,7 @@ impl EwsClient {
|
||||
fn one_attempt(&self, url: &str, body: &str, action: &str) -> AttemptOutcome {
|
||||
let mut req = self
|
||||
.inner
|
||||
.agent_for(url)
|
||||
.agent
|
||||
.post(url)
|
||||
.header("Authorization", self.auth_header())
|
||||
.header("Content-Type", "text/xml; charset=utf-8")
|
||||
@@ -548,29 +535,13 @@ fn truncate(body: &[u8]) -> String {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::net::CertOverride;
|
||||
|
||||
#[test]
|
||||
fn every_timeout_is_a_transport_error_and_so_retried() {
|
||||
// Every EwsError::Transport goes round the retry loop in `execute`.
|
||||
for t in [
|
||||
ureq::Timeout::Connect,
|
||||
ureq::Timeout::SendRequest,
|
||||
ureq::Timeout::SendBody,
|
||||
ureq::Timeout::RecvResponse,
|
||||
ureq::Timeout::RecvBody,
|
||||
] {
|
||||
let err = map_ureq_error(ureq::Error::Timeout(t));
|
||||
assert!(matches!(err, EwsError::Transport(_)), "{t:?} -> {err:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_constructs_with_defaults() {
|
||||
let c = EwsClient::new(
|
||||
Auth::Bearer { token: "t".into() },
|
||||
RetryPolicy::new(3),
|
||||
CertOverride::none(),
|
||||
false,
|
||||
);
|
||||
assert_eq!(c.server_version(), ServerVersion::Exchange2013Sp1);
|
||||
assert_eq!(c.retries_observed(), 0);
|
||||
@@ -582,7 +553,7 @@ mod tests {
|
||||
let c = EwsClient::new(
|
||||
Auth::Bearer { token: "t".into() },
|
||||
RetryPolicy::new(0),
|
||||
CertOverride::none(),
|
||||
false,
|
||||
);
|
||||
c.set_server_version(ServerVersion::Exchange2019);
|
||||
assert_eq!(c.server_version(), ServerVersion::Exchange2019);
|
||||
@@ -593,7 +564,7 @@ mod tests {
|
||||
let c = EwsClient::new(
|
||||
Auth::Bearer { token: "t".into() },
|
||||
RetryPolicy::new(0),
|
||||
CertOverride::none(),
|
||||
false,
|
||||
);
|
||||
c.set_anchor_mailbox(Some("alice@x".to_owned()));
|
||||
assert_eq!(c.anchor_header().as_deref(), Some("alice@x"));
|
||||
|
||||
+31
-15
@@ -1,6 +1,5 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -11,9 +10,9 @@ use std::time::Duration;
|
||||
use encodify::base64::{Base64, Padding, URL_SAFE};
|
||||
use serde_json::Value;
|
||||
use ureq::config::Config;
|
||||
use ureq::tls::{RootCerts, TlsConfig};
|
||||
|
||||
use crate::exchange_ews::error::EwsError;
|
||||
use crate::net::{tls, with_timeouts};
|
||||
|
||||
pub const SCOPE_APP_ONLY: &str = "https://outlook.office365.com/.default";
|
||||
pub const SCOPE_DELEGATED: &str =
|
||||
@@ -47,7 +46,7 @@ pub enum OAuthFlow {
|
||||
},
|
||||
}
|
||||
|
||||
pub fn acquire(flow: &OAuthFlow) -> Result<AcquiredToken, EwsError> {
|
||||
pub fn acquire(flow: &OAuthFlow, allow_invalid_certs: bool) -> Result<AcquiredToken, EwsError> {
|
||||
match flow {
|
||||
OAuthFlow::PreAcquired { token } => {
|
||||
let claims = decode_jwt_claims(token).unwrap_or_default();
|
||||
@@ -64,8 +63,10 @@ pub fn acquire(flow: &OAuthFlow) -> Result<AcquiredToken, EwsError> {
|
||||
tenant,
|
||||
client_id,
|
||||
client_secret,
|
||||
} => client_credentials(tenant, client_id, client_secret),
|
||||
OAuthFlow::DeviceCode { tenant, client_id } => device_code_flow(tenant, client_id),
|
||||
} => client_credentials(tenant, client_id, client_secret, allow_invalid_certs),
|
||||
OAuthFlow::DeviceCode { tenant, client_id } => {
|
||||
device_code_flow(tenant, client_id, allow_invalid_certs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,11 +104,17 @@ fn device_code_endpoint(tenant: &str) -> String {
|
||||
format!("https://login.microsoftonline.com/{tenant}/oauth2/v2.0/devicecode")
|
||||
}
|
||||
|
||||
fn build_agent() -> ureq::Agent {
|
||||
let config: Config = with_timeouts!(
|
||||
Config::builder()
|
||||
fn build_agent(allow_invalid_certs: bool) -> ureq::Agent {
|
||||
let config: Config = Config::builder()
|
||||
.http_status_as_error(false)
|
||||
.tls_config(tls(false))
|
||||
.tls_config(
|
||||
TlsConfig::builder()
|
||||
.unversioned_rustls_crypto_provider(std::sync::Arc::new(
|
||||
rustls::crypto::aws_lc_rs::default_provider(),
|
||||
))
|
||||
.root_certs(RootCerts::PlatformVerifier)
|
||||
.disable_verification(allow_invalid_certs)
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
config.new_agent()
|
||||
@@ -117,8 +124,9 @@ fn client_credentials(
|
||||
tenant: &str,
|
||||
client_id: &str,
|
||||
client_secret: &str,
|
||||
allow_invalid_certs: bool,
|
||||
) -> Result<AcquiredToken, EwsError> {
|
||||
let agent = build_agent();
|
||||
let agent = build_agent(allow_invalid_certs);
|
||||
let body = form_encode(&[
|
||||
("client_id", client_id),
|
||||
("client_secret", client_secret),
|
||||
@@ -134,8 +142,12 @@ fn client_credentials(
|
||||
parse_token_response(resp)
|
||||
}
|
||||
|
||||
fn device_code_flow(tenant: &str, client_id: &str) -> Result<AcquiredToken, EwsError> {
|
||||
let agent = build_agent();
|
||||
fn device_code_flow(
|
||||
tenant: &str,
|
||||
client_id: &str,
|
||||
allow_invalid_certs: bool,
|
||||
) -> Result<AcquiredToken, EwsError> {
|
||||
let agent = build_agent(allow_invalid_certs);
|
||||
let body = form_encode(&[("client_id", client_id), ("scope", SCOPE_DELEGATED)]);
|
||||
let endpoint = device_code_endpoint(tenant);
|
||||
let mut resp = agent
|
||||
@@ -267,8 +279,9 @@ pub fn refresh_with_token(
|
||||
tenant: &str,
|
||||
client_id: &str,
|
||||
refresh_token: &str,
|
||||
allow_invalid_certs: bool,
|
||||
) -> Result<AcquiredToken, EwsError> {
|
||||
let agent = build_agent();
|
||||
let agent = build_agent(allow_invalid_certs);
|
||||
let body = form_encode(&[
|
||||
("client_id", client_id),
|
||||
("grant_type", "refresh_token"),
|
||||
@@ -353,9 +366,12 @@ mod tests {
|
||||
#[test]
|
||||
fn pre_acquired_flow_decodes_claims() {
|
||||
let token = make_jwt("t-2", "bob@x", 9999999999);
|
||||
let acq = acquire(&OAuthFlow::PreAcquired {
|
||||
let acq = acquire(
|
||||
&OAuthFlow::PreAcquired {
|
||||
token: token.clone(),
|
||||
})
|
||||
},
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(acq.access_token, token);
|
||||
assert_eq!(acq.tenant_id.as_deref(), Some("t-2"));
|
||||
|
||||
@@ -503,8 +503,6 @@ pub struct FindItemResponse {
|
||||
pub struct ItemEntry {
|
||||
pub element: String,
|
||||
pub id: ItemId,
|
||||
/// `item:Size` in bytes, when the server returned it.
|
||||
pub size: Option<u64>,
|
||||
}
|
||||
|
||||
pub fn parse_find_item_response(body: &[u8]) -> Result<FindItemResponse, EwsError> {
|
||||
@@ -513,7 +511,6 @@ pub fn parse_find_item_response(body: &[u8]) -> Result<FindItemResponse, EwsErro
|
||||
let mut buf = Vec::new();
|
||||
let mut out = FindItemResponse::default();
|
||||
let mut in_root = false;
|
||||
let mut reading_size = false;
|
||||
loop {
|
||||
buf.clear();
|
||||
let (ns, ev) = xml.read_resolved_event_into(&mut buf)?;
|
||||
@@ -534,13 +531,7 @@ pub fn parse_find_item_response(body: &[u8]) -> Result<FindItemResponse, EwsErro
|
||||
out.items.push(ItemEntry {
|
||||
element: local.clone(),
|
||||
id: ItemId::default(),
|
||||
size: None,
|
||||
});
|
||||
} else if local.eq_ignore_ascii_case("Size")
|
||||
&& matches!(ev, Event::Start(_))
|
||||
&& !out.items.is_empty()
|
||||
{
|
||||
reading_size = true;
|
||||
} else if local.eq_ignore_ascii_case("ItemId")
|
||||
&& let Some(last) = out.items.last_mut()
|
||||
{
|
||||
@@ -548,18 +539,10 @@ pub fn parse_find_item_response(body: &[u8]) -> Result<FindItemResponse, EwsErro
|
||||
}
|
||||
}
|
||||
}
|
||||
Event::Text(ref t) if reading_size => {
|
||||
if let Some(last) = out.items.last_mut() {
|
||||
let text: &str = t;
|
||||
last.size = text.trim().parse().ok();
|
||||
}
|
||||
}
|
||||
Event::End(e) => {
|
||||
let local = e.local_name().as_ref().to_owned();
|
||||
if local.eq_ignore_ascii_case("RootFolder") {
|
||||
in_root = false;
|
||||
} else if local.eq_ignore_ascii_case("Size") {
|
||||
reading_size = false;
|
||||
}
|
||||
}
|
||||
Event::Eof => break,
|
||||
|
||||
+1
-16
@@ -1,6 +1,5 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -84,12 +83,7 @@ pub fn find_item_body(
|
||||
let mut out = String::with_capacity(512);
|
||||
out.push_str("<m:FindItem Traversal=\"");
|
||||
out.push_str(traversal.as_str());
|
||||
// item:Size lets GetItem batches be split by bytes as well as by count.
|
||||
out.push_str(
|
||||
"\"><m:ItemShape><t:BaseShape>IdOnly</t:BaseShape>\
|
||||
<t:AdditionalProperties><t:FieldURI FieldURI=\"item:Size\"/></t:AdditionalProperties>\
|
||||
</m:ItemShape>",
|
||||
);
|
||||
out.push_str("\"><m:ItemShape><t:BaseShape>IdOnly</t:BaseShape></m:ItemShape>");
|
||||
out.push_str("<m:IndexedPageItemView MaxEntriesReturned=\"");
|
||||
out.push_str(&page_size.to_string());
|
||||
out.push_str("\" Offset=\"");
|
||||
@@ -296,15 +290,6 @@ mod tests {
|
||||
assert!(body.contains("<t:DistinguishedFolderId Id=\"archiveroot\"/>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_item_asks_for_item_size() {
|
||||
let folder = FolderId::new("FID", "FCK");
|
||||
let body = find_item_body(FolderRef::Concrete(&folder), Traversal::Shallow, 0, 50);
|
||||
assert!(body.contains(
|
||||
"<t:BaseShape>IdOnly</t:BaseShape><t:AdditionalProperties><t:FieldURI FieldURI=\"item:Size\"/></t:AdditionalProperties></m:ItemShape>"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_item_paginates_with_offset_and_page_size() {
|
||||
let folder = FolderId::new("FID", "FCK");
|
||||
|
||||
@@ -13,6 +13,7 @@ use std::time::{Duration, Instant};
|
||||
use serde_json::Value;
|
||||
use ureq::Agent;
|
||||
use ureq::config::{Config, RedirectAuthHeaders};
|
||||
use ureq::tls::{RootCerts, TlsConfig};
|
||||
use ureq::{ResponseExt, http::Uri};
|
||||
|
||||
use crate::exchange_graph::error::GraphError;
|
||||
@@ -20,7 +21,6 @@ use crate::exchange_graph::retry::{HttpClass, classify_http_status, is_throttled
|
||||
use crate::jmap::http::{RetryPolicy, cross_host, retry_after_header};
|
||||
use crate::jmap::retry::{self, RateLimitState};
|
||||
use crate::logging::{HttpCall, LEVEL_BODIES, LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
||||
use crate::net::{CertOverride, tls, with_timeouts};
|
||||
|
||||
const MAX_BODY: u64 = 256 * 1024 * 1024;
|
||||
const LONG_RETRY_THRESHOLD: Duration = Duration::from_secs(10);
|
||||
@@ -63,8 +63,6 @@ impl GraphResponse {
|
||||
|
||||
struct Inner {
|
||||
agent: Agent,
|
||||
lax_agent: Option<Agent>,
|
||||
certs: CertOverride,
|
||||
bearer: Mutex<String>,
|
||||
retry: RetryPolicy,
|
||||
rate_limit: RateLimitState,
|
||||
@@ -75,17 +73,6 @@ struct Inner {
|
||||
user_agent: String,
|
||||
}
|
||||
|
||||
impl Inner {
|
||||
/// The agent for `url`: the one that accepts invalid certificates only for
|
||||
/// a host `--allow-invalid-certs` covers, and the verifying one otherwise.
|
||||
fn agent_for(&self, url: &str) -> &Agent {
|
||||
match &self.lax_agent {
|
||||
Some(lax) if self.certs.allows(url) => lax,
|
||||
_ => &self.agent,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct GraphClient {
|
||||
inner: Arc<Inner>,
|
||||
@@ -102,23 +89,23 @@ enum Attempt {
|
||||
}
|
||||
|
||||
impl GraphClient {
|
||||
pub fn new(bearer: String, retry: RetryPolicy, certs: CertOverride) -> GraphClient {
|
||||
let build = |accept_invalid: bool| -> Agent {
|
||||
let config: Config = with_timeouts!(
|
||||
Config::builder()
|
||||
pub fn new(bearer: String, retry: RetryPolicy, allow_invalid_certs: bool) -> GraphClient {
|
||||
let config: Config = Config::builder()
|
||||
.http_status_as_error(false)
|
||||
.redirect_auth_headers(RedirectAuthHeaders::SameHost)
|
||||
.tls_config(tls(accept_invalid))
|
||||
.tls_config(
|
||||
TlsConfig::builder()
|
||||
.unversioned_rustls_crypto_provider(std::sync::Arc::new(
|
||||
rustls::crypto::aws_lc_rs::default_provider(),
|
||||
))
|
||||
.root_certs(RootCerts::PlatformVerifier)
|
||||
.disable_verification(allow_invalid_certs)
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
config.new_agent()
|
||||
};
|
||||
let lax_agent = certs.is_active().then(|| build(true));
|
||||
GraphClient {
|
||||
inner: Arc::new(Inner {
|
||||
agent: build(false),
|
||||
lax_agent,
|
||||
certs,
|
||||
agent: config.new_agent(),
|
||||
bearer: Mutex::new(bearer),
|
||||
retry,
|
||||
rate_limit: RateLimitState::new(),
|
||||
@@ -323,7 +310,7 @@ impl GraphClient {
|
||||
extra_prefer: &[&str],
|
||||
) -> Attempt {
|
||||
let mut req = match method {
|
||||
"GET" => self.inner.agent_for(url).get(url),
|
||||
"GET" => self.inner.agent.get(url),
|
||||
other => {
|
||||
return Attempt::Transport(GraphError::Connect(format!(
|
||||
"unsupported method {other} (graph importer is read-only)"
|
||||
@@ -487,30 +474,10 @@ fn format_retry_wait(d: Duration) -> String {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::net::CertOverride;
|
||||
|
||||
#[test]
|
||||
fn every_timeout_is_a_transport_error_and_so_retried() {
|
||||
// `execute` retries every GraphError::Transport; only Connect is fatal.
|
||||
for t in [
|
||||
ureq::Timeout::Connect,
|
||||
ureq::Timeout::SendRequest,
|
||||
ureq::Timeout::SendBody,
|
||||
ureq::Timeout::RecvResponse,
|
||||
ureq::Timeout::RecvBody,
|
||||
] {
|
||||
let err = map_ureq_error(ureq::Error::Timeout(t));
|
||||
assert!(matches!(err, GraphError::Transport(_)), "{t:?} -> {err:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn defaults_construct() {
|
||||
let c = GraphClient::new(
|
||||
"token".to_owned(),
|
||||
RetryPolicy::new(3),
|
||||
CertOverride::none(),
|
||||
);
|
||||
let c = GraphClient::new("token".to_owned(), RetryPolicy::new(3), false);
|
||||
assert_eq!(c.retries_observed(), 0);
|
||||
assert_eq!(c.retry_after_sleeps(), 0);
|
||||
assert_eq!(c.requests_observed(), 0);
|
||||
@@ -519,7 +486,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn bearer_can_be_swapped_at_runtime() {
|
||||
let c = GraphClient::new("old".to_owned(), RetryPolicy::new(0), CertOverride::none());
|
||||
let c = GraphClient::new("old".to_owned(), RetryPolicy::new(0), false);
|
||||
c.set_bearer("new".to_owned());
|
||||
assert_eq!(c.auth_header(), "Bearer new");
|
||||
}
|
||||
|
||||
+21
-11
@@ -1,6 +1,5 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -11,9 +10,9 @@ use std::time::{Duration, Instant};
|
||||
use encodify::base64::{Base64, Padding, URL_SAFE};
|
||||
use serde_json::Value;
|
||||
use ureq::config::Config;
|
||||
use ureq::tls::{RootCerts, TlsConfig};
|
||||
|
||||
use crate::exchange_graph::error::GraphError;
|
||||
use crate::net::{tls, with_timeouts};
|
||||
|
||||
pub const SCOPES: &str =
|
||||
"offline_access User.Read Mail.Read MailboxSettings.Read Calendars.Read Contacts.Read";
|
||||
@@ -79,23 +78,29 @@ pub struct AcquiredToken {
|
||||
pub name: Option<String>,
|
||||
}
|
||||
|
||||
fn build_agent() -> ureq::Agent {
|
||||
let config: Config = with_timeouts!(
|
||||
Config::builder()
|
||||
fn build_agent(allow_invalid_certs: bool) -> ureq::Agent {
|
||||
let config: Config = Config::builder()
|
||||
.http_status_as_error(false)
|
||||
.tls_config(tls(false))
|
||||
.tls_config(
|
||||
TlsConfig::builder()
|
||||
.unversioned_rustls_crypto_provider(std::sync::Arc::new(
|
||||
rustls::crypto::aws_lc_rs::default_provider(),
|
||||
))
|
||||
.root_certs(RootCerts::PlatformVerifier)
|
||||
.disable_verification(allow_invalid_certs)
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
config.new_agent()
|
||||
}
|
||||
|
||||
pub fn acquire(flow: &OAuthFlow) -> Result<AcquiredToken, GraphError> {
|
||||
pub fn acquire(flow: &OAuthFlow, allow_invalid_certs: bool) -> Result<AcquiredToken, GraphError> {
|
||||
match flow {
|
||||
OAuthFlow::PreAcquired { token } => Ok(token_from_string(token.clone())),
|
||||
OAuthFlow::DeviceCode {
|
||||
authority,
|
||||
client_id,
|
||||
} => device_code_flow(authority, client_id),
|
||||
} => device_code_flow(authority, client_id, allow_invalid_certs),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -208,8 +213,12 @@ pub fn parse_token_response(status: u16, json: &Value) -> TokenResponse {
|
||||
}
|
||||
}
|
||||
|
||||
fn device_code_flow(authority: &str, client_id: &str) -> Result<AcquiredToken, GraphError> {
|
||||
let agent = build_agent();
|
||||
fn device_code_flow(
|
||||
authority: &str,
|
||||
client_id: &str,
|
||||
allow_invalid_certs: bool,
|
||||
) -> Result<AcquiredToken, GraphError> {
|
||||
let agent = build_agent(allow_invalid_certs);
|
||||
let body = form_encode(&[("client_id", client_id), ("scope", SCOPES)]);
|
||||
let endpoint = device_code_endpoint(authority);
|
||||
let mut resp = agent
|
||||
@@ -285,8 +294,9 @@ pub fn refresh_access_token(
|
||||
authority: &str,
|
||||
client_id: &str,
|
||||
refresh_token: &str,
|
||||
allow_invalid_certs: bool,
|
||||
) -> Result<AcquiredToken, GraphError> {
|
||||
let agent = build_agent();
|
||||
let agent = build_agent(allow_invalid_certs);
|
||||
let body = form_encode(&[
|
||||
("client_id", client_id),
|
||||
("grant_type", "refresh_token"),
|
||||
|
||||
+1
-3
@@ -1,6 +1,5 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -171,7 +170,6 @@ fn extract_account_id(principal: &Value, name: &str) -> Result<String, Error> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::net::CertOverride;
|
||||
|
||||
fn session_with(name: &str, id: &str) -> Session {
|
||||
let raw = serde_json::json!({
|
||||
@@ -188,7 +186,7 @@ mod tests {
|
||||
HttpClient::new(
|
||||
crate::jmap::http::Auth::Bearer { token: "t".into() },
|
||||
crate::jmap::http::RetryPolicy::new(0),
|
||||
CertOverride::none(),
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
+27
-216
@@ -1,6 +1,5 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -14,6 +13,7 @@ use encodify::base64::STANDARD;
|
||||
use serde_json::Value;
|
||||
use ureq::Agent;
|
||||
use ureq::config::{Config, RedirectAuthHeaders};
|
||||
use ureq::tls::{RootCerts, TlsConfig};
|
||||
use ureq::{ResponseExt, http::Uri};
|
||||
|
||||
use crate::jmap::error::JmapError;
|
||||
@@ -21,7 +21,6 @@ use crate::jmap::inflight::{Permit, Semaphore};
|
||||
use crate::jmap::retry::{self, Disposition, RateLimitState};
|
||||
use crate::jmap::session::Limits;
|
||||
use crate::logging::{HttpCall, LEVEL_BODIES, LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
||||
use crate::net::{CertOverride, send_body_budget, tls, with_timeouts};
|
||||
|
||||
const MAX_BODY: u64 = 512 * 1024 * 1024;
|
||||
|
||||
@@ -70,10 +69,9 @@ impl RetryPolicy {
|
||||
|
||||
struct Inner {
|
||||
agent: Agent,
|
||||
lax_agent: Option<Agent>,
|
||||
certs: CertOverride,
|
||||
auth: Auth,
|
||||
retry: RetryPolicy,
|
||||
allow_invalid_certs: bool,
|
||||
rate_limit: RateLimitState,
|
||||
log_level: AtomicU8,
|
||||
requests_gate: OnceLock<Semaphore>,
|
||||
@@ -83,24 +81,9 @@ struct Inner {
|
||||
retry_after_sleeps: AtomicU64,
|
||||
}
|
||||
|
||||
impl Inner {
|
||||
/// The agent for `url`: the one that accepts invalid certificates only for
|
||||
/// a host `--allow-invalid-certs` covers, and the verifying one otherwise.
|
||||
fn agent_for(&self, url: &str) -> &Agent {
|
||||
match &self.lax_agent {
|
||||
Some(lax) if self.certs.allows(url) => lax,
|
||||
_ => &self.agent,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum Kind {
|
||||
Api,
|
||||
/// An API call that must not be sent twice: a write the server may
|
||||
/// already have applied when the connection failed. A transport error is
|
||||
/// returned to the caller, which checks the target instead of resending.
|
||||
ApiOnce,
|
||||
Upload,
|
||||
}
|
||||
|
||||
@@ -120,25 +103,26 @@ enum Attempt {
|
||||
}
|
||||
|
||||
impl HttpClient {
|
||||
pub fn new(auth: Auth, retry: RetryPolicy, certs: CertOverride) -> Self {
|
||||
let build = |accept_invalid: bool| -> Agent {
|
||||
let config: Config = with_timeouts!(
|
||||
Config::builder()
|
||||
pub fn new(auth: Auth, retry: RetryPolicy, allow_invalid_certs: bool) -> Self {
|
||||
let config: Config = Config::builder()
|
||||
.http_status_as_error(false)
|
||||
.redirect_auth_headers(RedirectAuthHeaders::SameHost)
|
||||
.tls_config(tls(accept_invalid))
|
||||
.tls_config(
|
||||
TlsConfig::builder()
|
||||
.unversioned_rustls_crypto_provider(std::sync::Arc::new(
|
||||
rustls::crypto::aws_lc_rs::default_provider(),
|
||||
))
|
||||
.root_certs(RootCerts::PlatformVerifier)
|
||||
.disable_verification(allow_invalid_certs)
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
config.new_agent()
|
||||
};
|
||||
let lax_agent = certs.is_active().then(|| build(true));
|
||||
HttpClient {
|
||||
inner: Arc::new(Inner {
|
||||
agent: build(false),
|
||||
lax_agent,
|
||||
certs,
|
||||
agent: config.new_agent(),
|
||||
auth,
|
||||
retry,
|
||||
allow_invalid_certs,
|
||||
rate_limit: RateLimitState::new(),
|
||||
log_level: AtomicU8::new(LEVEL_DEFAULT),
|
||||
requests_gate: OnceLock::new(),
|
||||
@@ -188,6 +172,10 @@ impl HttpClient {
|
||||
&self.inner.retry
|
||||
}
|
||||
|
||||
pub fn allow_invalid_certs(&self) -> bool {
|
||||
self.inner.allow_invalid_certs
|
||||
}
|
||||
|
||||
pub fn rate_limit(&self) -> &RateLimitState {
|
||||
&self.inner.rate_limit
|
||||
}
|
||||
@@ -224,23 +212,6 @@ impl HttpClient {
|
||||
.map_err(|e| JmapError::Malformed(format!("response is not valid json: {e}")))
|
||||
}
|
||||
|
||||
/// As `post_json`, but a transport failure is not retried: the request
|
||||
/// may have reached the server, and sending it again could apply it twice.
|
||||
/// Throttling and `503` answers, which mean it was not processed, are still
|
||||
/// retried.
|
||||
pub fn post_json_once(&self, url: &str, body: &Value) -> Result<Value, JmapError> {
|
||||
let payload = serde_json::to_vec(body)?;
|
||||
let raw = self.execute(
|
||||
Kind::ApiOnce,
|
||||
"POST",
|
||||
url,
|
||||
Some(&payload),
|
||||
Some("application/json"),
|
||||
)?;
|
||||
serde_json::from_slice(&raw)
|
||||
.map_err(|e| JmapError::Malformed(format!("response is not valid json: {e}")))
|
||||
}
|
||||
|
||||
pub fn upload(
|
||||
&self,
|
||||
upload_url: &str,
|
||||
@@ -319,16 +290,6 @@ impl HttpClient {
|
||||
body: truncate(&body),
|
||||
});
|
||||
}
|
||||
// A gateway error may come back after the server
|
||||
// behind it applied the write: not safe to resend.
|
||||
StatusOutcome::Retryable
|
||||
if matches!(kind, Kind::ApiOnce) && matches!(status, 502 | 504) =>
|
||||
{
|
||||
return Err(JmapError::HttpStatus {
|
||||
status,
|
||||
body: truncate(&body),
|
||||
});
|
||||
}
|
||||
StatusOutcome::Retryable => {
|
||||
attempt += 1;
|
||||
self.inner.retries_total.fetch_add(1, Ordering::Relaxed);
|
||||
@@ -375,7 +336,6 @@ impl HttpClient {
|
||||
}
|
||||
}
|
||||
}
|
||||
Attempt::Transport(err) if matches!(kind, Kind::ApiOnce) => return Err(err),
|
||||
Attempt::Transport(err) => match transport_disposition(&err) {
|
||||
Disposition::Fatal => return Err(err),
|
||||
Disposition::Retryable => {
|
||||
@@ -426,22 +386,17 @@ impl HttpClient {
|
||||
let result = if let Some(payload) = body {
|
||||
let mut req = self
|
||||
.inner
|
||||
.agent_for(url)
|
||||
.agent
|
||||
.post(url)
|
||||
.header("Authorization", auth)
|
||||
.header("Accept", "application/json");
|
||||
if let Some(ct) = content_type {
|
||||
req = req.header("Content-Type", ct);
|
||||
}
|
||||
// A blob upload can run to hundreds of megabytes, so its send
|
||||
// budget grows with its size instead of the agent's flat default.
|
||||
req.config()
|
||||
.timeout_send_body(Some(send_body_budget(payload.len())))
|
||||
.build()
|
||||
.send(payload)
|
||||
req.send(payload)
|
||||
} else {
|
||||
self.inner
|
||||
.agent_for(url)
|
||||
.agent
|
||||
.get(url)
|
||||
.header("Authorization", auth)
|
||||
.header("Accept", "application/json")
|
||||
@@ -689,150 +644,6 @@ pub fn format_retry_wait(d: Duration) -> String {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::net::CertOverride;
|
||||
|
||||
/// A server that reads each request and hangs up without answering: a
|
||||
/// transport failure after the request has been sent. Returns its URL and
|
||||
/// the count of requests it has seen.
|
||||
fn hang_up_server() -> (String, Arc<AtomicU64>) {
|
||||
use std::io::Read;
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let url = format!("http://{}/jmap/api", listener.local_addr().unwrap());
|
||||
let seen = Arc::new(AtomicU64::new(0));
|
||||
let counter = seen.clone();
|
||||
std::thread::spawn(move || {
|
||||
for stream in listener.incoming() {
|
||||
let Ok(mut stream) = stream else { break };
|
||||
let mut buf = [0u8; 4096];
|
||||
let _ = stream.read(&mut buf);
|
||||
counter.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
});
|
||||
(url, seen)
|
||||
}
|
||||
|
||||
fn quick_retries(max_retries: u32) -> RetryPolicy {
|
||||
RetryPolicy {
|
||||
max_retries,
|
||||
base: Duration::from_millis(1),
|
||||
cap: Duration::from_millis(2),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_write_sent_once_is_not_resent_after_a_transport_failure() {
|
||||
let (url, seen) = hang_up_server();
|
||||
let client = HttpClient::new(
|
||||
Auth::Bearer { token: "t".into() },
|
||||
quick_retries(2),
|
||||
CertOverride::none(),
|
||||
);
|
||||
let err = client
|
||||
.post_json_once(&url, &serde_json::json!({}))
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, JmapError::Transport(_)), "{err}");
|
||||
assert_eq!(seen.load(Ordering::SeqCst), 1, "sent exactly once");
|
||||
|
||||
let (url, seen) = hang_up_server();
|
||||
let _ = client.post_json(&url, &serde_json::json!({}));
|
||||
assert_eq!(
|
||||
seen.load(Ordering::SeqCst),
|
||||
3,
|
||||
"an ordinary call retries twice"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_write_sent_once_is_not_resent_after_a_gateway_timeout_but_is_after_503() {
|
||||
let mut server = mockito::Server::new();
|
||||
let url = format!("{}/jmap/api", server.url());
|
||||
let client = HttpClient::new(
|
||||
Auth::Bearer { token: "t".into() },
|
||||
quick_retries(2),
|
||||
CertOverride::none(),
|
||||
);
|
||||
let gateway = server
|
||||
.mock("POST", "/jmap/api")
|
||||
.with_status(504)
|
||||
.expect(1)
|
||||
.create();
|
||||
let err = client
|
||||
.post_json_once(&url, &serde_json::json!({}))
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
matches!(err, JmapError::HttpStatus { status: 504, .. }),
|
||||
"{err}"
|
||||
);
|
||||
gateway.assert();
|
||||
gateway.remove();
|
||||
|
||||
let busy = server
|
||||
.mock("POST", "/jmap/api")
|
||||
.with_status(503)
|
||||
.expect(3)
|
||||
.create();
|
||||
let _ = client.post_json_once(&url, &serde_json::json!({}));
|
||||
busy.assert();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_certificates_are_accepted_only_for_the_named_host() {
|
||||
let client = HttpClient::new(
|
||||
Auth::Bearer {
|
||||
token: "t".to_owned(),
|
||||
},
|
||||
RetryPolicy::new(0),
|
||||
CertOverride::for_url(true, "https://mail.example.test/.well-known/jmap"),
|
||||
);
|
||||
let inner = &client.inner;
|
||||
let lax = inner.lax_agent.as_ref().expect("a relaxed agent exists");
|
||||
assert!(std::ptr::eq(
|
||||
inner.agent_for("https://mail.example.test/api"),
|
||||
lax
|
||||
));
|
||||
assert!(std::ptr::eq(
|
||||
inner.agent_for("https://files.example.test/upload"),
|
||||
&inner.agent
|
||||
));
|
||||
assert!(std::ptr::eq(
|
||||
inner.agent_for("https://login.microsoftonline.com/common/oauth2/v2.0/token"),
|
||||
&inner.agent
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_the_flag_there_is_no_relaxed_agent() {
|
||||
let client = HttpClient::new(
|
||||
Auth::Bearer {
|
||||
token: "t".to_owned(),
|
||||
},
|
||||
RetryPolicy::new(0),
|
||||
CertOverride::for_url(false, "https://mail.example.test/"),
|
||||
);
|
||||
assert!(client.inner.lax_agent.is_none());
|
||||
assert!(std::ptr::eq(
|
||||
client.inner.agent_for("https://mail.example.test/api"),
|
||||
&client.inner.agent
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_timeout_is_a_retryable_transport_error() {
|
||||
for t in [
|
||||
ureq::Timeout::Connect,
|
||||
ureq::Timeout::SendRequest,
|
||||
ureq::Timeout::SendBody,
|
||||
ureq::Timeout::RecvResponse,
|
||||
ureq::Timeout::RecvBody,
|
||||
] {
|
||||
let err = map_ureq_error(ureq::Error::Timeout(t));
|
||||
assert!(matches!(err, JmapError::Transport(_)), "{t:?} -> {err:?}");
|
||||
assert!(
|
||||
matches!(transport_disposition(&err), Disposition::Retryable),
|
||||
"{t:?} must be retried"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn basic_header_matches_rfc7617_example() {
|
||||
@@ -900,7 +711,7 @@ mod tests {
|
||||
token: "t".to_owned(),
|
||||
},
|
||||
RetryPolicy::new(0),
|
||||
CertOverride::none(),
|
||||
false,
|
||||
);
|
||||
let body = br#"{"type":"urn:ietf:params:jmap:error:limit","limit":"someServerLimit"}"#;
|
||||
assert!(matches!(
|
||||
@@ -916,7 +727,7 @@ mod tests {
|
||||
token: "t".to_owned(),
|
||||
},
|
||||
RetryPolicy::new(0),
|
||||
CertOverride::none(),
|
||||
false,
|
||||
);
|
||||
let body = br#"{"type":"urn:ietf:params:jmap:error:limit","limit":"maxSizeRequest"}"#;
|
||||
assert!(matches!(
|
||||
@@ -932,7 +743,7 @@ mod tests {
|
||||
token: "t".to_owned(),
|
||||
},
|
||||
RetryPolicy::new(0),
|
||||
CertOverride::none(),
|
||||
false,
|
||||
);
|
||||
let body =
|
||||
br#"{"type":"urn:ietf:params:jmap:error:limit","limit":"maxConcurrentRequests"}"#;
|
||||
@@ -961,7 +772,7 @@ mod tests {
|
||||
token: "t".to_owned(),
|
||||
},
|
||||
RetryPolicy::new(0),
|
||||
CertOverride::none(),
|
||||
false,
|
||||
);
|
||||
client.set_limits(&limits_with(10, 4, 4));
|
||||
let err = client
|
||||
@@ -985,7 +796,7 @@ mod tests {
|
||||
token: "t".to_owned(),
|
||||
},
|
||||
RetryPolicy::new(0),
|
||||
CertOverride::none(),
|
||||
false,
|
||||
);
|
||||
client.set_limits(&limits_with(1024, 4, 4));
|
||||
let err = client
|
||||
@@ -1037,7 +848,7 @@ mod tests {
|
||||
token: "t".to_owned(),
|
||||
},
|
||||
RetryPolicy::new(0),
|
||||
CertOverride::none(),
|
||||
false,
|
||||
);
|
||||
assert_eq!(client.retries_observed(), 0);
|
||||
assert_eq!(client.retry_after_sleeps(), 0);
|
||||
|
||||
+1
-10
@@ -131,14 +131,6 @@ impl Request {
|
||||
let value = client.post_json(api_url, &self.envelope()?)?;
|
||||
Response::parse(value)
|
||||
}
|
||||
|
||||
/// As `send`, for a write that must not be applied twice: a transport
|
||||
/// failure comes back as an error instead of being resent (see
|
||||
/// `HttpClient::post_json_once`).
|
||||
pub fn send_once(&self, client: &HttpClient, api_url: &str) -> Result<Response, JmapError> {
|
||||
let value = client.post_json_once(api_url, &self.envelope()?)?;
|
||||
Response::parse(value)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -794,7 +786,6 @@ fn decode_set(mr: &MethodCall) -> SetOutcome {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::net::CertOverride;
|
||||
use std::cell::Cell;
|
||||
|
||||
use super::*;
|
||||
@@ -807,7 +798,7 @@ mod tests {
|
||||
token: "t".to_owned(),
|
||||
},
|
||||
RetryPolicy::new(max_retries),
|
||||
CertOverride::none(),
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -17,7 +16,6 @@ pub mod inspect;
|
||||
pub mod jmap;
|
||||
pub mod logging;
|
||||
pub mod managesieve;
|
||||
pub mod net;
|
||||
pub mod secret;
|
||||
pub mod sync;
|
||||
pub mod types;
|
||||
|
||||
@@ -31,7 +31,6 @@ fn run() -> i32 {
|
||||
Err(err) => return fail(&err),
|
||||
};
|
||||
|
||||
let mut quiet_report = false;
|
||||
let (outcome, logger) = match action {
|
||||
Action::Import(common, config) => {
|
||||
let logger = common.logger;
|
||||
@@ -76,8 +75,6 @@ fn run() -> i32 {
|
||||
}
|
||||
Action::Export(common, config) => {
|
||||
let logger = common.logger;
|
||||
// A dry run prints its own plan; the counts below would repeat it.
|
||||
quiet_report = common.dry_run;
|
||||
(
|
||||
RunOutcome::from_result(sync::export::run(common, config)),
|
||||
logger,
|
||||
@@ -91,9 +88,7 @@ fn run() -> i32 {
|
||||
}
|
||||
};
|
||||
|
||||
if !quiet_report {
|
||||
report(&outcome.summary);
|
||||
}
|
||||
match outcome.error {
|
||||
Some(err) => fail(&err),
|
||||
None => {
|
||||
|
||||
-284
@@ -1,284 +0,0 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
//! Settings every HTTP agent shares: timeouts, TLS, and which hosts, if any,
|
||||
//! may present a certificate that does not verify.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use ureq::tls::{RootCerts, TlsConfig};
|
||||
|
||||
/// Opening the socket and completing any TLS handshake.
|
||||
pub const CONNECT: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Writing the request line and headers.
|
||||
pub const SEND_REQUEST: Duration = Duration::from_secs(60);
|
||||
|
||||
/// Waiting for the response headers once the request is sent. This is the
|
||||
/// server's thinking time: a large `Email/import`, an EWS `FindItem` over a big
|
||||
/// folder or a CalDAV REPORT can legitimately take a while before the first
|
||||
/// byte comes back.
|
||||
pub const RECV_RESPONSE: Duration = Duration::from_secs(5 * 60);
|
||||
|
||||
/// Reading the whole response body. ureq counts this as one budget for the
|
||||
/// entire body, not per read, so it has to cover the largest body a client
|
||||
/// accepts (512 MiB) on a slow link: 30 minutes is about 300 KB/s. A stalled
|
||||
/// transfer is abandoned and retried after at most this long.
|
||||
pub const RECV_BODY: Duration = Duration::from_secs(30 * 60);
|
||||
|
||||
/// Sending a request body when its size is not known in advance. Uploads know
|
||||
/// their size and get [`send_body_budget`] instead.
|
||||
pub const SEND_BODY: Duration = Duration::from_secs(30 * 60);
|
||||
|
||||
/// The slowest upload rate a send budget allows for, in bytes per second.
|
||||
const MIN_UPLOAD_RATE: u64 = 64 * 1024;
|
||||
|
||||
/// The floor under every send budget, so small bodies still get a sensible
|
||||
/// allowance on a slow or busy connection.
|
||||
const SEND_BODY_FLOOR: Duration = Duration::from_secs(2 * 60);
|
||||
|
||||
/// How long sending a body of `len` bytes may take: the floor plus the time it
|
||||
/// takes at [`MIN_UPLOAD_RATE`].
|
||||
pub fn send_body_budget(len: usize) -> Duration {
|
||||
SEND_BODY_FLOOR + Duration::from_secs(len as u64 / MIN_UPLOAD_RATE)
|
||||
}
|
||||
|
||||
/// Applies the shared timeouts to a ureq `ConfigBuilder`. A macro rather than
|
||||
/// a function because ureq keeps the builder's scope types private, so a
|
||||
/// function could not name them.
|
||||
macro_rules! with_timeouts {
|
||||
($builder:expr) => {
|
||||
$builder
|
||||
.timeout_connect(Some($crate::net::CONNECT))
|
||||
.timeout_send_request(Some($crate::net::SEND_REQUEST))
|
||||
.timeout_send_body(Some($crate::net::SEND_BODY))
|
||||
.timeout_recv_response(Some($crate::net::RECV_RESPONSE))
|
||||
.timeout_recv_body(Some($crate::net::RECV_BODY))
|
||||
};
|
||||
}
|
||||
pub(crate) use with_timeouts;
|
||||
|
||||
/// TLS settings for an agent: the platform's roots, and certificate checks off
|
||||
/// only when `accept_invalid` is set.
|
||||
pub fn tls(accept_invalid: bool) -> TlsConfig {
|
||||
TlsConfig::builder()
|
||||
.unversioned_rustls_crypto_provider(std::sync::Arc::new(
|
||||
rustls::crypto::aws_lc_rs::default_provider(),
|
||||
))
|
||||
.root_certs(RootCerts::PlatformVerifier)
|
||||
.disable_verification(accept_invalid)
|
||||
.build()
|
||||
}
|
||||
|
||||
/// Hosts that are always verified, whatever `--allow-invalid-certs` says:
|
||||
/// the Microsoft and Google sign-in and cloud endpoints. A certificate that
|
||||
/// fails there is an attack or a broken network, never a self-signed server
|
||||
/// the user meant to trust. Matched as a suffix on a label boundary.
|
||||
const ALWAYS_VERIFY: &[&str] = &[
|
||||
"microsoftonline.com",
|
||||
"microsoftonline.us",
|
||||
"microsoft.com",
|
||||
"microsoft.us",
|
||||
"office365.com",
|
||||
"office.com",
|
||||
"outlook.com",
|
||||
"chinacloudapi.cn",
|
||||
"partner.outlook.cn",
|
||||
"google.com",
|
||||
"googleapis.com",
|
||||
"gmail.com",
|
||||
];
|
||||
|
||||
/// Where `--allow-invalid-certs` applies: the host the user named, or, for
|
||||
/// Exchange Autodiscover without a `--url`, the mailbox's own domain and its
|
||||
/// subdomains. Everything else, including any host a server redirects or
|
||||
/// points to, is verified as usual.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct CertOverride {
|
||||
hosts: Vec<String>,
|
||||
domains: Vec<String>,
|
||||
}
|
||||
|
||||
impl CertOverride {
|
||||
/// Verify everything.
|
||||
pub fn none() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// When `enabled`, accept invalid certificates from the host of `url`.
|
||||
pub fn for_url(enabled: bool, url: &str) -> Self {
|
||||
match (enabled, host_of(url)) {
|
||||
(true, Some(host)) if !always_verified(&host) => CertOverride {
|
||||
hosts: vec![host],
|
||||
domains: Vec::new(),
|
||||
},
|
||||
_ => Self::none(),
|
||||
}
|
||||
}
|
||||
|
||||
/// When `enabled`, accept invalid certificates from `domain` and every
|
||||
/// host under it.
|
||||
pub fn for_domain(enabled: bool, domain: &str) -> Self {
|
||||
let domain = domain.trim_end_matches('.').to_ascii_lowercase();
|
||||
if enabled && !domain.is_empty() && !always_verified(&domain) {
|
||||
CertOverride {
|
||||
hosts: Vec::new(),
|
||||
domains: vec![domain],
|
||||
}
|
||||
} else {
|
||||
Self::none()
|
||||
}
|
||||
}
|
||||
|
||||
/// The same override, narrowed to the host of `url`, if `url` is one this
|
||||
/// override already covers. Used once Autodiscover has found the real
|
||||
/// endpoint.
|
||||
pub fn narrowed_to(&self, url: &str) -> Self {
|
||||
match host_of(url) {
|
||||
Some(host) if self.allows_host(&host) => CertOverride {
|
||||
hosts: vec![host],
|
||||
domains: Vec::new(),
|
||||
},
|
||||
_ => Self::none(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this override covers anything at all.
|
||||
pub fn is_active(&self) -> bool {
|
||||
!self.hosts.is_empty() || !self.domains.is_empty()
|
||||
}
|
||||
|
||||
/// Whether a certificate that does not verify is accepted for `url`.
|
||||
pub fn allows(&self, url: &str) -> bool {
|
||||
host_of(url).is_some_and(|host| self.allows_host(&host))
|
||||
}
|
||||
|
||||
fn allows_host(&self, host: &str) -> bool {
|
||||
if always_verified(host) {
|
||||
return false;
|
||||
}
|
||||
self.hosts.iter().any(|h| h == host)
|
||||
|| self
|
||||
.domains
|
||||
.iter()
|
||||
.any(|d| host == d || host.ends_with(&format!(".{d}")))
|
||||
}
|
||||
}
|
||||
|
||||
fn host_of(url: &str) -> Option<String> {
|
||||
let parsed = url::Url::parse(url).ok()?;
|
||||
let host = parsed
|
||||
.host_str()?
|
||||
.trim_end_matches('.')
|
||||
.to_ascii_lowercase();
|
||||
Some(
|
||||
host.trim_start_matches('[')
|
||||
.trim_end_matches(']')
|
||||
.to_owned(),
|
||||
)
|
||||
}
|
||||
|
||||
fn always_verified(host: &str) -> bool {
|
||||
ALWAYS_VERIFY
|
||||
.iter()
|
||||
.any(|d| host == *d || host.ends_with(&format!(".{d}")))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn disabled_flag_covers_nothing() {
|
||||
let o = CertOverride::for_url(false, "https://mail.example.test/jmap");
|
||||
assert!(!o.is_active());
|
||||
assert!(!o.allows("https://mail.example.test/jmap"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn covers_only_the_named_host() {
|
||||
let o = CertOverride::for_url(true, "https://Mail.Example.test:8443/.well-known/jmap");
|
||||
assert!(o.is_active());
|
||||
assert!(o.allows("https://mail.example.test/api"));
|
||||
assert!(o.allows("https://MAIL.example.test:9000/upload"));
|
||||
assert!(!o.allows("https://files.example.test/download"));
|
||||
assert!(!o.allows("https://example.test/"));
|
||||
assert!(!o.allows("https://mail.example.test.evil.test/"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sign_in_and_cloud_hosts_are_always_verified() {
|
||||
for url in [
|
||||
"https://login.microsoftonline.com/common/oauth2/v2.0/token",
|
||||
"https://graph.microsoft.com/v1.0/me",
|
||||
"https://outlook.office365.com/EWS/Exchange.asmx",
|
||||
"https://autodiscover-s.outlook.com/autodiscover/autodiscover.xml",
|
||||
"https://oauth2.googleapis.com/token",
|
||||
"https://accounts.google.com/o/oauth2/device/code",
|
||||
] {
|
||||
let o = CertOverride::for_url(true, url);
|
||||
assert!(!o.is_active(), "{url}");
|
||||
assert!(!o.allows(url), "{url}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_domain_covers_its_subdomains_but_not_look_alikes() {
|
||||
let o = CertOverride::for_domain(true, "Corp.Example.");
|
||||
assert!(o.allows("https://autodiscover.corp.example/autodiscover/autodiscover.xml"));
|
||||
assert!(o.allows("https://corp.example/autodiscover/autodiscover.xml"));
|
||||
assert!(!o.allows("https://notcorp.example/"));
|
||||
assert!(!o.allows("https://corp.example.evil.test/"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_domain_override_never_reaches_microsoft() {
|
||||
let o = CertOverride::for_domain(true, "office365.com");
|
||||
assert!(!o.is_active());
|
||||
let corp = CertOverride::for_domain(true, "corp.example");
|
||||
assert!(!corp.allows("https://outlook.office365.com/EWS/Exchange.asmx"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn narrowing_keeps_only_a_covered_endpoint() {
|
||||
let o = CertOverride::for_domain(true, "corp.example");
|
||||
let inside = o.narrowed_to("https://mail.corp.example/EWS/Exchange.asmx");
|
||||
assert!(inside.allows("https://mail.corp.example/EWS/Exchange.asmx"));
|
||||
assert!(!inside.allows("https://autodiscover.corp.example/"));
|
||||
let outside = o.narrowed_to("https://outlook.office365.com/EWS/Exchange.asmx");
|
||||
assert!(!outside.is_active());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ip_literals_are_matched() {
|
||||
let o = CertOverride::for_url(true, "https://[::1]:8443/jmap");
|
||||
assert!(o.allows("https://[::1]:9000/other"));
|
||||
let v4 = CertOverride::for_url(true, "https://192.0.2.10/jmap");
|
||||
assert!(v4.allows("https://192.0.2.10:8443/"));
|
||||
assert!(!v4.allows("https://192.0.2.11/"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_budget_grows_with_size() {
|
||||
assert_eq!(send_body_budget(0), Duration::from_secs(120));
|
||||
assert_eq!(
|
||||
send_body_budget(64 * 1024 * 600),
|
||||
Duration::from_secs(120 + 600)
|
||||
);
|
||||
assert!(send_body_budget(512 * 1024 * 1024) > Duration::from_secs(2 * 60 * 60));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timeouts_are_applied_to_a_config() {
|
||||
let config: ureq::config::Config = with_timeouts!(ureq::config::Config::builder()).build();
|
||||
let t = config.timeouts();
|
||||
assert_eq!(t.connect, Some(CONNECT));
|
||||
assert_eq!(t.send_request, Some(SEND_REQUEST));
|
||||
assert_eq!(t.send_body, Some(SEND_BODY));
|
||||
assert_eq!(t.recv_response, Some(RECV_RESPONSE));
|
||||
assert_eq!(t.recv_body, Some(RECV_BODY));
|
||||
}
|
||||
}
|
||||
@@ -1,92 +0,0 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
//! Fetch batches bounded by bytes as well as by count. A batch of a few
|
||||
//! hundred messages is small for ordinary mail and many gigabytes for a
|
||||
//! mailbox of large attachments; capping the bytes too keeps what a batch
|
||||
//! holds in memory about the same whatever the mail is like.
|
||||
|
||||
/// The default byte cap for one fetch batch.
|
||||
pub const DEFAULT_BATCH_BYTES: u64 = 32 * 1024 * 1024;
|
||||
|
||||
/// Splits `items` into contiguous batches of at most `max_count` items and
|
||||
/// at most `max_bytes` bytes, as reported by `size`. An item whose size is
|
||||
/// unknown counts as 0 bytes, so without sizes this is batching by count. An
|
||||
/// item larger than `max_bytes` goes in a batch of its own: every batch has
|
||||
/// at least one item.
|
||||
pub fn by_count_and_bytes<T>(
|
||||
items: &[T],
|
||||
size: impl Fn(&T) -> u64,
|
||||
max_count: usize,
|
||||
max_bytes: u64,
|
||||
) -> Vec<&[T]> {
|
||||
let max_count = max_count.max(1);
|
||||
let max_bytes = max_bytes.max(1);
|
||||
let mut out = Vec::new();
|
||||
let mut start = 0usize;
|
||||
let mut bytes = 0u64;
|
||||
for (i, item) in items.iter().enumerate() {
|
||||
let s = size(item);
|
||||
let count = i - start;
|
||||
if count > 0 && (count >= max_count || bytes.saturating_add(s) > max_bytes) {
|
||||
out.push(&items[start..i]);
|
||||
start = i;
|
||||
bytes = 0;
|
||||
}
|
||||
bytes = bytes.saturating_add(s);
|
||||
}
|
||||
if start < items.len() {
|
||||
out.push(&items[start..]);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn batches(sizes: &[u64], max_count: usize, max_bytes: u64) -> Vec<Vec<u64>> {
|
||||
by_count_and_bytes(sizes, |s| *s, max_count, max_bytes)
|
||||
.into_iter()
|
||||
.map(|b| b.to_vec())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn respects_the_byte_cap() {
|
||||
assert_eq!(
|
||||
batches(&[10, 10, 10, 10, 10], 100, 25),
|
||||
vec![vec![10, 10], vec![10, 10], vec![10]]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn respects_the_count_cap() {
|
||||
assert_eq!(
|
||||
batches(&[1, 1, 1, 1, 1], 2, 1000),
|
||||
vec![vec![1, 1], vec![1, 1], vec![1]]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_item_over_the_cap_goes_alone() {
|
||||
assert_eq!(
|
||||
batches(&[5, 500, 5, 5], 100, 20),
|
||||
vec![vec![5], vec![500], vec![5, 5]]
|
||||
);
|
||||
assert_eq!(batches(&[500], 100, 20), vec![vec![500]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_sizes_batch_by_count() {
|
||||
assert_eq!(batches(&[0, 0, 0], 2, 1), vec![vec![0, 0], vec![0]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nothing_in_nothing_out() {
|
||||
assert!(batches(&[], 10, 10).is_empty());
|
||||
}
|
||||
}
|
||||
+17
-414
@@ -7,9 +7,6 @@
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::io::{IsTerminal, Write};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use rusqlite::Connection;
|
||||
use serde_json::{Map, Value, json};
|
||||
@@ -91,9 +88,8 @@ impl<'a> Uploader<'a> {
|
||||
return Ok(id.clone());
|
||||
}
|
||||
let id = if self.net.dry_run {
|
||||
let len = db::blobs::blob_len(self.conn, local_id)?
|
||||
let _exists = db::blobs::blob_bytes(self.conn, local_id)?
|
||||
.ok_or_else(|| JmapError::malformed(format!("blob local id {local_id} missing")))?;
|
||||
self.net.check_upload_size(len)?;
|
||||
JmapId(format!("dryrun-blob-{local_id}"))
|
||||
} else {
|
||||
let bytes = db::blobs::blob_bytes(self.conn, local_id)?
|
||||
@@ -110,106 +106,6 @@ impl<'a> Uploader<'a> {
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// As `upload_with`, but sends `bytes` in place of the stored blob: for
|
||||
/// content rewritten on its way to the target. Cached under the same
|
||||
/// local id, so a retry sends the rewritten bytes again.
|
||||
fn upload_bytes_as(
|
||||
&mut self,
|
||||
local_id: i64,
|
||||
content_type: &str,
|
||||
bytes: &[u8],
|
||||
) -> Result<JmapId, JmapError> {
|
||||
self.touched.push(local_id);
|
||||
if let Some(id) = self.cache.get(&local_id) {
|
||||
return Ok(id.clone());
|
||||
}
|
||||
let id = if self.net.dry_run {
|
||||
self.net.check_upload_size(bytes.len() as u64)?;
|
||||
JmapId(format!("dryrun-blob-{local_id}"))
|
||||
} else {
|
||||
blobxfer::upload_bytes(
|
||||
&self.net.client,
|
||||
&self.net.session,
|
||||
&self.net.account,
|
||||
content_type,
|
||||
bytes,
|
||||
)?
|
||||
};
|
||||
self.cache.insert(local_id, id.clone());
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Uploads several stored blobs at once, on up to `Net::upload_workers`
|
||||
/// threads, and returns each one's result in the order given. Each thread
|
||||
/// reads its blobs through its own read-only connection to the archive, so
|
||||
/// at most one blob per thread is held in memory. With one worker, or in a
|
||||
/// dry run, it is `upload_with` in a loop.
|
||||
fn upload_many(
|
||||
&mut self,
|
||||
local_ids: &[i64],
|
||||
content_type: &str,
|
||||
) -> Vec<Result<JmapId, JmapError>> {
|
||||
if self.net.dry_run || self.net.upload_workers <= 1 {
|
||||
return local_ids
|
||||
.iter()
|
||||
.map(|id| self.upload_with(*id, content_type))
|
||||
.collect();
|
||||
}
|
||||
self.touched.extend_from_slice(local_ids);
|
||||
let mut todo: Vec<i64> = Vec::new();
|
||||
for id in local_ids {
|
||||
if !self.cache.contains_key(id) && !todo.contains(id) {
|
||||
todo.push(*id);
|
||||
}
|
||||
}
|
||||
let net = self.net;
|
||||
let results = run_bounded(
|
||||
&todo,
|
||||
net.upload_workers,
|
||||
|| {
|
||||
rusqlite::Connection::open_with_flags(
|
||||
&net.archive,
|
||||
rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
|
||||
)
|
||||
},
|
||||
|conn, local_id| {
|
||||
let conn = conn
|
||||
.as_ref()
|
||||
.map_err(|e| JmapError::malformed(format!("archive not readable: {e}")))?;
|
||||
let bytes = db::blobs::blob_bytes(conn, *local_id)?.ok_or_else(|| {
|
||||
JmapError::malformed(format!("blob local id {local_id} missing"))
|
||||
})?;
|
||||
blobxfer::upload_bytes(
|
||||
&net.client,
|
||||
&net.session,
|
||||
&net.account,
|
||||
content_type,
|
||||
&bytes,
|
||||
)
|
||||
},
|
||||
);
|
||||
let mut failed: HashMap<i64, JmapError> = HashMap::new();
|
||||
for (local_id, result) in todo.into_iter().zip(results) {
|
||||
match result {
|
||||
Ok(id) => {
|
||||
self.cache.insert(local_id, id);
|
||||
}
|
||||
Err(e) => {
|
||||
failed.insert(local_id, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
local_ids
|
||||
.iter()
|
||||
.map(|id| match self.cache.get(id) {
|
||||
Some(blob) => Ok(blob.clone()),
|
||||
None => Err(failed.get(id).map(clone_error).unwrap_or_else(|| {
|
||||
JmapError::malformed(format!("blob local id {id} not uploaded"))
|
||||
})),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn invalidate(&mut self, local_id: i64) {
|
||||
self.cache.remove(&local_id);
|
||||
}
|
||||
@@ -223,58 +119,6 @@ impl<'a> Uploader<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
/// A copy of an upload error for each archive row that shares the blob. The
|
||||
/// errors that carry meaning for the caller -- the size limits -- keep their
|
||||
/// kind; the rest keep their message.
|
||||
fn clone_error(e: &JmapError) -> JmapError {
|
||||
match e {
|
||||
JmapError::RequestTooLarge => JmapError::RequestTooLarge,
|
||||
JmapError::SingleObjectTooLarge(m) => JmapError::SingleObjectTooLarge(m.clone()),
|
||||
other => JmapError::Transport(other.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs `f` over `jobs` on at most `workers` threads and returns the results
|
||||
/// in job order. Each thread builds its own state once with `init`, such as a
|
||||
/// connection of its own to the archive.
|
||||
fn run_bounded<J, S, R>(
|
||||
jobs: &[J],
|
||||
workers: usize,
|
||||
init: impl Fn() -> S + Sync,
|
||||
f: impl Fn(&mut S, &J) -> R + Sync,
|
||||
) -> Vec<R>
|
||||
where
|
||||
J: Sync,
|
||||
R: Send,
|
||||
{
|
||||
let workers = workers.clamp(1, jobs.len().max(1));
|
||||
if workers == 1 {
|
||||
let mut state = init();
|
||||
return jobs.iter().map(|j| f(&mut state, j)).collect();
|
||||
}
|
||||
let next = AtomicUsize::new(0);
|
||||
let slots: Mutex<Vec<Option<R>>> = Mutex::new((0..jobs.len()).map(|_| None).collect());
|
||||
std::thread::scope(|scope| {
|
||||
for _ in 0..workers {
|
||||
scope.spawn(|| {
|
||||
let mut state = init();
|
||||
loop {
|
||||
let i = next.fetch_add(1, Ordering::SeqCst);
|
||||
let Some(job) = jobs.get(i) else { break };
|
||||
let r = f(&mut state, job);
|
||||
slots.lock().expect("result slots")[i] = Some(r);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
slots
|
||||
.into_inner()
|
||||
.expect("result slots")
|
||||
.into_iter()
|
||||
.map(|r| r.expect("every job ran"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
impl BlobBytes for Uploader<'_> {
|
||||
fn bytes(&self, local_id: i64) -> Result<Vec<u8>, JmapError> {
|
||||
db::blobs::blob_bytes(self.conn, local_id)?
|
||||
@@ -290,40 +134,6 @@ struct Net {
|
||||
limits: Limits,
|
||||
session: Session,
|
||||
dry_run: bool,
|
||||
/// The archive's path, for the upload threads' own connections.
|
||||
archive: PathBuf,
|
||||
/// Blobs uploaded at once: the server's `maxConcurrentUpload`, and no
|
||||
/// more than `--threads`.
|
||||
upload_workers: usize,
|
||||
/// In a dry run, what would fail and why, for the plan.
|
||||
would_fail: std::sync::Arc<Mutex<Vec<String>>>,
|
||||
}
|
||||
|
||||
impl Net {
|
||||
/// In a dry run, a blob of `len` bytes that the target would refuse:
|
||||
/// over its `maxSizeUpload`.
|
||||
fn check_upload_size(&self, len: u64) -> Result<(), JmapError> {
|
||||
let cap = self.limits.max_size_upload;
|
||||
if cap > 0 && len > cap {
|
||||
return Err(JmapError::SingleObjectTooLarge(format!(
|
||||
"{} is larger than the target accepts ({} maxSizeUpload)",
|
||||
crate::inspect::format_bytes(len),
|
||||
crate::inspect::format_bytes(cap)
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Notes, in a dry run, that `what` would fail and why. A real run
|
||||
/// reports failures as they happen and keeps no list.
|
||||
fn would_fail(&self, what: impl Into<String>) {
|
||||
if self.dry_run {
|
||||
self.would_fail
|
||||
.lock()
|
||||
.expect("would-fail list")
|
||||
.push(what.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn has_rows(conn: &Connection, ty: ObjectType) -> bool {
|
||||
@@ -347,11 +157,6 @@ pub fn run(common: CommonConfig, config: ExportConfig) -> Result<Summary, Error>
|
||||
limits: connected.limits,
|
||||
session: connected.session.clone(),
|
||||
dry_run: ctx.dry_run(),
|
||||
archive: ctx.common.archive.clone(),
|
||||
upload_workers: (connected.limits.max_concurrent_upload as usize)
|
||||
.min(ctx.common.threads)
|
||||
.max(1),
|
||||
would_fail: Default::default(),
|
||||
};
|
||||
|
||||
let work = work_list(&ctx.conn, &config, &connected, &logger);
|
||||
@@ -365,7 +170,6 @@ pub fn run(common: CommonConfig, config: ExportConfig) -> Result<Summary, Error>
|
||||
if logger.enabled(LEVEL_DEFAULT) {
|
||||
eprintln!("export: {} ...", ty.jmap_name());
|
||||
}
|
||||
let started = std::time::Instant::now();
|
||||
let mut counts = TypeCounts::default();
|
||||
let res = reconcile_type(
|
||||
&ctx,
|
||||
@@ -385,16 +189,6 @@ pub fn run(common: CommonConfig, config: ExportConfig) -> Result<Summary, Error>
|
||||
Plan::default()
|
||||
}
|
||||
};
|
||||
if logger.enabled(LEVEL_DEFAULT) && !ctx.dry_run() {
|
||||
eprintln!(
|
||||
"{}",
|
||||
crate::sync::progress::done_line(
|
||||
&format!("export: {}", ty.jmap_name()),
|
||||
&counts,
|
||||
started.elapsed()
|
||||
)
|
||||
);
|
||||
}
|
||||
plans.insert(*ty, plan);
|
||||
counts_per_type.insert(*ty, counts);
|
||||
}
|
||||
@@ -418,9 +212,8 @@ pub fn run(common: CommonConfig, config: ExportConfig) -> Result<Summary, Error>
|
||||
}
|
||||
|
||||
if ctx.dry_run() {
|
||||
let would_fail = net.would_fail.lock().expect("would-fail list").clone();
|
||||
print_plan(&summary, &dry_rows, &would_fail, config.prune);
|
||||
return Ok(summary);
|
||||
print_dry_run(&dry_rows, config.prune);
|
||||
return Ok(Summary::default());
|
||||
}
|
||||
summary.retries_observed = ctx.client.retries_observed();
|
||||
summary.retry_after_sleeps = ctx.client.retry_after_sleeps();
|
||||
@@ -626,69 +419,21 @@ fn sample(ids: &[String]) -> String {
|
||||
ids[..n].join(", ")
|
||||
}
|
||||
|
||||
/// The dry run's report, in plain words: per type, what would be created,
|
||||
/// updated, left as it is and would fail; then why each failure would happen.
|
||||
fn print_plan(
|
||||
summary: &Summary,
|
||||
dry_rows: &[(&'static str, u64, u64, u64)],
|
||||
would_fail: &[String],
|
||||
prune: bool,
|
||||
) {
|
||||
print!("{}", plan_text(summary, dry_rows, would_fail, prune));
|
||||
}
|
||||
|
||||
fn plan_text(
|
||||
summary: &Summary,
|
||||
dry_rows: &[(&'static str, u64, u64, u64)],
|
||||
would_fail: &[String],
|
||||
prune: bool,
|
||||
) -> String {
|
||||
use crate::sync::progress::thousands;
|
||||
let mut out = String::from("Dry run: nothing was written to the target. The plan:\n");
|
||||
for (ty, c) in &summary.per_type {
|
||||
let mut parts: Vec<String> = Vec::new();
|
||||
if c.created > 0 {
|
||||
parts.push(format!("{} to create", thousands(c.created)));
|
||||
}
|
||||
if c.updated > 0 {
|
||||
parts.push(format!("{} to update", thousands(c.updated)));
|
||||
}
|
||||
if c.skipped > 0 {
|
||||
parts.push(format!("{} unchanged", thousands(c.skipped)));
|
||||
}
|
||||
if c.failed > 0 {
|
||||
parts.push(format!("{} would fail", thousands(c.failed)));
|
||||
}
|
||||
fn print_dry_run(rows: &[(&'static str, u64, u64, u64)], prune: bool) {
|
||||
if prune {
|
||||
let gone = dry_rows
|
||||
.iter()
|
||||
.find(|(t, ..)| t == ty)
|
||||
.map(|(.., d)| *d)
|
||||
.unwrap_or(0);
|
||||
if gone > 0 {
|
||||
parts.push(format!("{} to delete (--prune)", thousands(gone)));
|
||||
println!(
|
||||
"{:<22} {:>10} {:>10} {:>12}",
|
||||
"TYPE", "CREATE", "MATCHED", "WOULD-DESTROY"
|
||||
);
|
||||
for (ty, c, m, d) in rows {
|
||||
println!("{ty:<22} {c:>10} {m:>10} {d:>12}");
|
||||
}
|
||||
} else {
|
||||
println!("{:<22} {:>10} {:>10}", "TYPE", "CREATE", "MATCHED");
|
||||
for (ty, c, m, _) in rows {
|
||||
println!("{ty:<22} {c:>10} {m:>10}");
|
||||
}
|
||||
}
|
||||
if parts.is_empty() {
|
||||
parts.push("nothing to do".to_owned());
|
||||
}
|
||||
out.push_str(&format!(" {ty:<20} {}\n", parts.join(", ")));
|
||||
}
|
||||
let failed: u64 = summary.per_type.iter().map(|(_, c)| c.failed).sum();
|
||||
if failed > 0 {
|
||||
out.push_str("Would fail:\n");
|
||||
for line in would_fail {
|
||||
out.push_str(&format!(" {line}\n"));
|
||||
}
|
||||
let unexplained = failed.saturating_sub(would_fail.len() as u64);
|
||||
if unexplained > 0 {
|
||||
out.push_str(&format!(
|
||||
" {} more; the warnings above say why\n",
|
||||
thousands(unexplained)
|
||||
));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
mod tree;
|
||||
@@ -699,8 +444,6 @@ mod keyed;
|
||||
|
||||
mod sieve;
|
||||
|
||||
mod sieve_names;
|
||||
|
||||
mod uidtype;
|
||||
|
||||
mod email;
|
||||
@@ -749,7 +492,7 @@ mod common {
|
||||
creates: Vec<(String, Value)>,
|
||||
) -> Result<crate::jmap::request::SetOutcome, JmapError> {
|
||||
if net.dry_run {
|
||||
return Ok(synthesize_dry_run_outcome(net, ty, &creates));
|
||||
return Ok(synthesize_dry_run_outcome(ty, &creates));
|
||||
}
|
||||
let mut map = Map::new();
|
||||
for (cid, obj) in creates {
|
||||
@@ -848,35 +591,12 @@ mod common {
|
||||
create_batch(net, ty, vec![(cid.to_owned(), wire)]).map_err(Error::from)
|
||||
}
|
||||
|
||||
/// Room left in a request for everything but the object itself: the
|
||||
/// envelope, the method name and the arguments around it.
|
||||
const REQUEST_OVERHEAD: u64 = 512;
|
||||
|
||||
/// What a dry run predicts for `creates`: each one created, except an
|
||||
/// object too big to fit in one request under the target's
|
||||
/// `maxSizeRequest`, which a real run could not send either.
|
||||
fn synthesize_dry_run_outcome(
|
||||
net: &Net,
|
||||
ty: ObjectType,
|
||||
creates: &[(String, Value)],
|
||||
) -> crate::jmap::request::SetOutcome {
|
||||
let mut outcome = crate::jmap::request::SetOutcome::default();
|
||||
let cap = net.limits.max_size_request;
|
||||
for (cid, obj) in creates {
|
||||
let size = serde_json::to_vec(obj).map(|v| v.len() as u64).unwrap_or(0);
|
||||
if cap > 0 && size + REQUEST_OVERHEAD > cap {
|
||||
let why = format!(
|
||||
"{} is larger than one request to the target may be ({} maxSizeRequest)",
|
||||
crate::inspect::format_bytes(size),
|
||||
crate::inspect::format_bytes(cap)
|
||||
);
|
||||
net.would_fail(format!("{} {cid}: {why}", ty.jmap_name()));
|
||||
outcome.not_created.push((
|
||||
cid.clone(),
|
||||
serde_json::json!({ "type": "tooLarge", "description": why }),
|
||||
));
|
||||
continue;
|
||||
}
|
||||
for (cid, _) in creates {
|
||||
let synthetic = serde_json::json!({
|
||||
"id": format!("dryrun-{}-{cid}", ty.jmap_name())
|
||||
});
|
||||
@@ -885,120 +605,3 @@ mod common {
|
||||
outcome
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod pool_tests {
|
||||
use super::run_bounded;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
fn never_more_workers_at_once_than_the_cap_and_results_keep_job_order() {
|
||||
let jobs: Vec<usize> = (0..24).collect();
|
||||
let running = AtomicUsize::new(0);
|
||||
let most = AtomicUsize::new(0);
|
||||
let inits = AtomicUsize::new(0);
|
||||
let out = run_bounded(
|
||||
&jobs,
|
||||
3,
|
||||
|| inits.fetch_add(1, Ordering::SeqCst),
|
||||
|_, j| {
|
||||
let now = running.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
most.fetch_max(now, Ordering::SeqCst);
|
||||
std::thread::sleep(Duration::from_millis(5));
|
||||
running.fetch_sub(1, Ordering::SeqCst);
|
||||
j * 2
|
||||
},
|
||||
);
|
||||
assert_eq!(out, jobs.iter().map(|j| j * 2).collect::<Vec<_>>());
|
||||
let most = most.load(Ordering::SeqCst);
|
||||
assert!(most <= 3, "{most} ran at once");
|
||||
assert!(most > 1, "work ran in parallel");
|
||||
assert_eq!(
|
||||
inits.load(Ordering::SeqCst),
|
||||
3,
|
||||
"state built once per worker"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_worker_or_one_job_runs_in_place() {
|
||||
let out = run_bounded(&[1, 2, 3], 1, || (), |_, j| j + 1);
|
||||
assert_eq!(out, vec![2, 3, 4]);
|
||||
let out = run_bounded(&[7], 8, || (), |_, j| j + 1);
|
||||
assert_eq!(out, vec![8]);
|
||||
let out: Vec<i32> = run_bounded(&[], 4, || (), |_, j: &i32| *j);
|
||||
assert!(out.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod plan_tests {
|
||||
use super::plan_text;
|
||||
use crate::sync::{Summary, TypeCounts};
|
||||
|
||||
fn summary(rows: &[(&'static str, u64, u64, u64, u64)]) -> Summary {
|
||||
Summary {
|
||||
per_type: rows
|
||||
.iter()
|
||||
.map(|(t, created, updated, skipped, failed)| {
|
||||
(
|
||||
*t,
|
||||
TypeCounts {
|
||||
created: *created,
|
||||
updated: *updated,
|
||||
skipped: *skipped,
|
||||
failed: *failed,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_plan_reads_in_plain_words_and_says_why_things_would_fail() {
|
||||
let s = summary(&[
|
||||
("Mailbox", 0, 0, 12, 0),
|
||||
("Email", 1200, 40, 5000, 2),
|
||||
("SieveScript", 0, 0, 0, 0),
|
||||
]);
|
||||
let text = plan_text(
|
||||
&s,
|
||||
&[],
|
||||
&["Email e7 (message-id <a@b>): 61 MB is larger than the target accepts".to_owned()],
|
||||
false,
|
||||
);
|
||||
assert!(
|
||||
text.starts_with("Dry run: nothing was written to the target."),
|
||||
"{text}"
|
||||
);
|
||||
assert!(text.contains("Mailbox 12 unchanged"), "{text}");
|
||||
assert!(
|
||||
text.contains(
|
||||
"Email 1,200 to create, 40 to update, 5,000 unchanged, 2 would fail"
|
||||
),
|
||||
"{text}"
|
||||
);
|
||||
assert!(
|
||||
text.contains("SieveScript nothing to do"),
|
||||
"{text}"
|
||||
);
|
||||
assert!(text.contains("Would fail:\n Email e7"), "{text}");
|
||||
assert!(
|
||||
text.contains("1 more; the warnings above say why"),
|
||||
"{text}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prune_counts_appear_only_with_prune() {
|
||||
let s = summary(&[("ContactCard", 0, 0, 3, 0)]);
|
||||
let rows = [("ContactCard", 0, 3, 4)];
|
||||
assert!(plan_text(&s, &rows, &[], true).contains("3 unchanged, 4 to delete (--prune)"));
|
||||
assert!(!plan_text(&s, &rows, &[], false).contains("delete"));
|
||||
assert!(!plan_text(&s, &rows, &[], false).contains("Would fail"));
|
||||
}
|
||||
}
|
||||
|
||||
+57
-348
@@ -21,7 +21,6 @@ use crate::jmap::wire::JmapId;
|
||||
use crate::logging::Logger;
|
||||
use crate::sync::import_jmap::mapping::{EMAIL_SELECT, EmailRow, TargetResolver, row_to_email};
|
||||
use crate::sync::keys::{EmailIndex, EmailKey, email_index, email_keys, index_from_json};
|
||||
use crate::sync::progress::Progress;
|
||||
use crate::sync::{Context, TypeCounts};
|
||||
use crate::types::ObjectType;
|
||||
|
||||
@@ -62,7 +61,45 @@ pub fn reconcile(
|
||||
logger: &Logger,
|
||||
) -> Result<Plan, Error> {
|
||||
let ty = ObjectType::Email;
|
||||
let (targets, target_keys) = target_emails(net).map_err(Error::from)?;
|
||||
|
||||
let target_min = target_query_get(
|
||||
net,
|
||||
ty,
|
||||
Some(&["messageId", "size", "mailboxIds", "keywords"]),
|
||||
)
|
||||
.map_err(Error::from)?;
|
||||
let mut indices: Vec<EmailIndex> = target_min.iter().map(server_index).collect();
|
||||
|
||||
let fallback_ids: Vec<JmapId> = target_min
|
||||
.iter()
|
||||
.zip(indices.iter())
|
||||
.filter(|(_, i)| i.mids.is_empty())
|
||||
.filter_map(|(v, _)| jid(v).map(JmapId))
|
||||
.collect();
|
||||
if !fallback_ids.is_empty() {
|
||||
let got = get_objects::<Value>(
|
||||
&net.client,
|
||||
&net.api,
|
||||
&net.account,
|
||||
ty.jmap_name(),
|
||||
&fallback_ids,
|
||||
Some(&["messageId", "from", "subject", "sentAt", "to"]),
|
||||
&net.limits,
|
||||
)
|
||||
.map_err(Error::from)?;
|
||||
let by_id: HashMap<String, &Value> = got
|
||||
.list
|
||||
.iter()
|
||||
.filter_map(|v| jid(v).map(|i| (i, v)))
|
||||
.collect();
|
||||
for (v, slot) in target_min.iter().zip(indices.iter_mut()) {
|
||||
if let Some(full) = jid(v).and_then(|i| by_id.get(&i)) {
|
||||
*slot = server_index(full);
|
||||
}
|
||||
}
|
||||
}
|
||||
let targets: Vec<TargetEmail> = target_min.iter().map(TargetEmail::from_value).collect();
|
||||
let target_keys = email_keys(&indices);
|
||||
|
||||
let mut local: Vec<(i64, EmailRow)> = {
|
||||
let mut stmt = ctx
|
||||
@@ -97,343 +134,27 @@ pub fn reconcile(
|
||||
|
||||
let migrated = maps.targets_of(ObjectType::Mailbox);
|
||||
let mut updates: Vec<(String, Value)> = Vec::new();
|
||||
let mut creates: Vec<usize> = Vec::new();
|
||||
for (i, unit) in units.iter().enumerate() {
|
||||
match pairs[i] {
|
||||
Some(t) => match email_patch(&unit.row, &targets[t], maps, &migrated) {
|
||||
Some(patch) => updates.push((targets[t].id.clone(), patch)),
|
||||
None => counts.skipped += 1,
|
||||
},
|
||||
None => creates.push(i),
|
||||
}
|
||||
}
|
||||
let mut progress = Progress::new("export: Email", creates.len() as u64, logger);
|
||||
import_units(
|
||||
None => export_one(
|
||||
net,
|
||||
&mut uploader,
|
||||
maps,
|
||||
&units,
|
||||
&creates,
|
||||
counts,
|
||||
logger,
|
||||
&mut progress,
|
||||
);
|
||||
update_batch(net, ty, updates, counts, logger);
|
||||
|
||||
Ok(Plan::default())
|
||||
}
|
||||
|
||||
/// The emails already on the target, and the key each one matches by.
|
||||
fn target_emails(net: &Net) -> Result<(Vec<TargetEmail>, Vec<EmailKey>), JmapError> {
|
||||
let ty = ObjectType::Email;
|
||||
let target_min = target_query_get(
|
||||
net,
|
||||
ty,
|
||||
Some(&["messageId", "size", "mailboxIds", "keywords"]),
|
||||
)?;
|
||||
let mut indices: Vec<EmailIndex> = target_min.iter().map(server_index).collect();
|
||||
|
||||
let fallback_ids: Vec<JmapId> = target_min
|
||||
.iter()
|
||||
.zip(indices.iter())
|
||||
.filter(|(_, i)| i.mids.is_empty())
|
||||
.filter_map(|(v, _)| jid(v).map(JmapId))
|
||||
.collect();
|
||||
if !fallback_ids.is_empty() {
|
||||
let got = get_objects::<Value>(
|
||||
&net.client,
|
||||
&net.api,
|
||||
&net.account,
|
||||
ty.jmap_name(),
|
||||
&fallback_ids,
|
||||
Some(&["messageId", "from", "subject", "sentAt", "to"]),
|
||||
&net.limits,
|
||||
)?;
|
||||
let by_id: HashMap<String, &Value> = got
|
||||
.list
|
||||
.iter()
|
||||
.filter_map(|v| jid(v).map(|i| (i, v)))
|
||||
.collect();
|
||||
for (v, slot) in target_min.iter().zip(indices.iter_mut()) {
|
||||
if let Some(full) = jid(v).and_then(|i| by_id.get(&i)) {
|
||||
*slot = server_index(full);
|
||||
}
|
||||
}
|
||||
}
|
||||
let targets: Vec<TargetEmail> = target_min.iter().map(TargetEmail::from_value).collect();
|
||||
let keys = email_keys(&indices);
|
||||
Ok((targets, keys))
|
||||
}
|
||||
|
||||
/// The most emails one `Email/import` carries: the server's
|
||||
/// `maxObjectsInSet`, but no more than this, so that a request that fails
|
||||
/// without a clear answer leaves few messages to check.
|
||||
const IMPORT_BATCH_CAP: usize = 50;
|
||||
|
||||
/// One message ready to import: its creation id, its place in `units`, and
|
||||
/// the `Email/import` entry.
|
||||
struct Pending {
|
||||
cid: String,
|
||||
unit: usize,
|
||||
item: Value,
|
||||
}
|
||||
|
||||
/// Writes the messages the target does not have yet. They go in batches of
|
||||
/// up to `maxObjectsInSet` (capped by `IMPORT_BATCH_CAP`), their blobs
|
||||
/// uploaded at once up to `maxConcurrentUpload`. Each message is still
|
||||
/// counted on its own: one rejected in a batch fails alone. A batch that
|
||||
/// fails without a clear answer is never resent blindly -- the target is
|
||||
/// checked first, and only what did not arrive is imported again.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn import_units(
|
||||
net: &Net,
|
||||
uploader: &mut Uploader,
|
||||
maps: &Maps,
|
||||
units: &[Unit],
|
||||
creates: &[usize],
|
||||
counts: &mut TypeCounts,
|
||||
logger: &Logger,
|
||||
progress: &mut Progress,
|
||||
) {
|
||||
let batch = (net.limits.max_objects_in_set as usize).clamp(1, IMPORT_BATCH_CAP);
|
||||
let mut unclear: Vec<Pending> = Vec::new();
|
||||
for chunk in creates.chunks(batch) {
|
||||
let mut ready: Vec<(usize, Map<String, Value>)> = Vec::new();
|
||||
for &i in chunk {
|
||||
let row = &units[i].row;
|
||||
match build_mailbox_ids(row, maps) {
|
||||
Some(mids) => ready.push((i, mids)),
|
||||
None => {
|
||||
logger.warn(&format!(
|
||||
"Email/import e{} ({}) skipped: mailbox not on target",
|
||||
units[i].local_id,
|
||||
blob_hint(uploader, row)
|
||||
));
|
||||
net.would_fail(format!(
|
||||
"Email e{} ({}): its folder is not on the target",
|
||||
units[i].local_id,
|
||||
blob_hint(uploader, row)
|
||||
));
|
||||
counts.failed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
let blobs: Vec<i64> = ready
|
||||
.iter()
|
||||
.map(|(i, _)| units[*i].row.blob_local_id)
|
||||
.collect();
|
||||
let uploaded = uploader.upload_many(&blobs, "message/rfc822");
|
||||
let mut pending: Vec<Pending> = Vec::new();
|
||||
for ((i, mids), result) in ready.into_iter().zip(uploaded) {
|
||||
let row = &units[i].row;
|
||||
let cid = format!("e{}", units[i].local_id);
|
||||
match result {
|
||||
Ok(blob) => pending.push(Pending {
|
||||
cid,
|
||||
unit: i,
|
||||
item: import_item(blob.0, mids, build_keywords(row), &row.received_at),
|
||||
}),
|
||||
Err(e) => {
|
||||
logger.warn(&format!(
|
||||
"Email/import {cid} ({}) blob upload failed: {e}{}",
|
||||
blob_hint(uploader, row),
|
||||
size_note(&e)
|
||||
));
|
||||
net.would_fail(format!(
|
||||
"Email {cid} ({}): {}",
|
||||
blob_hint(uploader, row),
|
||||
plain_reason(&e)
|
||||
));
|
||||
counts.failed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if net.dry_run {
|
||||
counts.created += pending.len() as u64;
|
||||
} else {
|
||||
send_batch(
|
||||
net,
|
||||
uploader,
|
||||
maps,
|
||||
units,
|
||||
pending,
|
||||
counts,
|
||||
logger,
|
||||
&mut unclear,
|
||||
);
|
||||
}
|
||||
progress.add(chunk.len() as u64);
|
||||
}
|
||||
if !unclear.is_empty() {
|
||||
settle_unclear(net, uploader, maps, units, unclear, counts, logger);
|
||||
}
|
||||
}
|
||||
|
||||
/// Sends one `Email/import` for `batch` and counts each message's outcome.
|
||||
/// A request too large for the server is split in two; a method error that
|
||||
/// rejects the whole call is retried one message at a time, so the one at
|
||||
/// fault fails alone. Anything that leaves it unclear whether the server
|
||||
/// applied the call goes to `unclear`.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn send_batch(
|
||||
net: &Net,
|
||||
uploader: &mut Uploader,
|
||||
maps: &Maps,
|
||||
units: &[Unit],
|
||||
batch: Vec<Pending>,
|
||||
counts: &mut TypeCounts,
|
||||
logger: &Logger,
|
||||
unclear: &mut Vec<Pending>,
|
||||
) {
|
||||
if batch.is_empty() {
|
||||
return;
|
||||
}
|
||||
let mut emails = Map::new();
|
||||
for p in &batch {
|
||||
emails.insert(p.cid.clone(), p.item.clone());
|
||||
}
|
||||
let mut req = Request::new();
|
||||
req.call(
|
||||
"Email/import",
|
||||
json!({ "accountId": net.account, "emails": Value::Object(emails) }),
|
||||
"i",
|
||||
);
|
||||
let cids: Vec<&str> = batch.iter().map(|p| p.cid.as_str()).collect();
|
||||
let sent = req.fits(&net.limits).and_then(|()| {
|
||||
retry_method_call(
|
||||
&net.client,
|
||||
MethodCallKind::SingleObjectWrite,
|
||||
logger,
|
||||
|| {
|
||||
let resp = req.send_once(&net.client, &net.api)?;
|
||||
let mr = resp.first()?;
|
||||
check_method_error(mr)?;
|
||||
Ok(cids
|
||||
.iter()
|
||||
.map(|cid| interpret_import_for(mr, cid, cids.len()))
|
||||
.collect::<Vec<_>>())
|
||||
},
|
||||
)
|
||||
});
|
||||
match sent {
|
||||
Ok(outcomes) => {
|
||||
for (p, outcome) in batch.into_iter().zip(outcomes) {
|
||||
let row = &units[p.unit].row;
|
||||
match outcome {
|
||||
SingleImport::Created => counts.created += 1,
|
||||
SingleImport::Skipped => counts.skipped += 1,
|
||||
SingleImport::NotCreated { error_type, .. } if error_type == "blobNotFound" => {
|
||||
retry_after_reupload(net, uploader, maps, &p.cid, row, counts, logger);
|
||||
}
|
||||
SingleImport::NotCreated { detail, .. } => {
|
||||
logger.warn(&format!(
|
||||
"Email/import {} ({}) failed: {detail}",
|
||||
p.cid,
|
||||
blob_hint(uploader, row)
|
||||
));
|
||||
counts.failed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(JmapError::RequestTooLarge | JmapError::SingleObjectTooLarge(_)) if batch.len() > 1 => {
|
||||
let mut batch = batch;
|
||||
let second = batch.split_off(batch.len() / 2);
|
||||
send_batch(net, uploader, maps, units, batch, counts, logger, unclear);
|
||||
send_batch(net, uploader, maps, units, second, counts, logger, unclear);
|
||||
}
|
||||
Err(e) if applied_unknown(&e) => {
|
||||
logger.warn(&format!(
|
||||
"Email/import of {} message(s) ended without a clear answer ({e}); the target is checked before any is sent again",
|
||||
batch.len()
|
||||
));
|
||||
unclear.extend(batch);
|
||||
}
|
||||
Err(JmapError::Method { .. }) if batch.len() > 1 => {
|
||||
for p in batch {
|
||||
send_batch(net, uploader, maps, units, vec![p], counts, logger, unclear);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
for p in batch {
|
||||
logger.warn(&format!(
|
||||
"Email/import {} ({}) send failed: {e}{}",
|
||||
p.cid,
|
||||
blob_hint(uploader, &units[p.unit].row),
|
||||
size_note(&e)
|
||||
));
|
||||
counts.failed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the server may have applied a call that failed with `e`: the
|
||||
/// connection broke after the request was sent, the answer was unreadable, or
|
||||
/// the server said it applied part of it.
|
||||
fn applied_unknown(e: &JmapError) -> bool {
|
||||
match e {
|
||||
JmapError::Transport(_)
|
||||
| JmapError::RetriesExhausted(_)
|
||||
| JmapError::Malformed(_)
|
||||
| JmapError::HttpStatus { .. } => true,
|
||||
JmapError::Method { error_type, .. } => error_type == "serverPartialFail",
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Settles messages whose import ended without a clear answer: reads the
|
||||
/// target again, counts those that arrived as created, and imports the rest
|
||||
/// one at a time. If the target cannot be read, they are counted as failed --
|
||||
/// the next export matches whatever did arrive, so none is ever doubled.
|
||||
fn settle_unclear(
|
||||
net: &Net,
|
||||
uploader: &mut Uploader,
|
||||
maps: &Maps,
|
||||
units: &[Unit],
|
||||
unclear: Vec<Pending>,
|
||||
counts: &mut TypeCounts,
|
||||
logger: &Logger,
|
||||
) {
|
||||
let (targets, target_keys) = match target_emails(net) {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
logger.warn(&format!(
|
||||
"could not read the target to settle {} message(s) ({e}); they count as failed, and the next export matches whatever arrived",
|
||||
unclear.len()
|
||||
));
|
||||
counts.failed += unclear.len() as u64;
|
||||
return;
|
||||
}
|
||||
};
|
||||
let keys: Vec<EmailKey> = email_keys(
|
||||
&unclear
|
||||
.iter()
|
||||
.map(|p| index_from_json(&units[p.unit].row.message_match))
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
let sizes: Vec<Option<u64>> = unclear
|
||||
.iter()
|
||||
.map(|p| uploader.blob_len(units[p.unit].row.blob_local_id))
|
||||
.collect();
|
||||
let pairs = pair_with_targets(&keys, &sizes, &target_keys, &targets);
|
||||
for (p, found) in unclear.into_iter().zip(pairs) {
|
||||
if found.is_some() {
|
||||
counts.created += 1;
|
||||
continue;
|
||||
}
|
||||
let unit = &units[p.unit];
|
||||
export_one(
|
||||
net,
|
||||
uploader,
|
||||
maps,
|
||||
unit.local_id,
|
||||
&unit.row,
|
||||
counts,
|
||||
logger,
|
||||
);
|
||||
),
|
||||
}
|
||||
}
|
||||
update_batch(net, ty, updates, counts, logger);
|
||||
|
||||
Ok(Plan::default())
|
||||
}
|
||||
|
||||
/// One message to write: the archive rows that hold the same bytes, folded
|
||||
/// together. A source that files one message in several folders (IMAP and
|
||||
@@ -627,15 +348,6 @@ fn blob_hint(uploader: &Uploader, row: &EmailRow) -> String {
|
||||
s
|
||||
}
|
||||
|
||||
/// A failure reason for the dry-run plan: the size message on its own, or
|
||||
/// the error as it is.
|
||||
fn plain_reason(e: &JmapError) -> String {
|
||||
match e {
|
||||
JmapError::SingleObjectTooLarge(m) => m.clone(),
|
||||
other => other.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn size_note(e: &JmapError) -> &'static str {
|
||||
if matches!(
|
||||
e,
|
||||
@@ -809,13 +521,6 @@ fn send_single_import(
|
||||
|
||||
fn interpret_import(mr: &MethodCall, cid: &str) -> Result<SingleImport, JmapError> {
|
||||
check_method_error(mr)?;
|
||||
Ok(interpret_import_for(mr, cid, 1))
|
||||
}
|
||||
|
||||
/// One message's outcome in an `Email/import` answer. With a single message
|
||||
/// in the call, any `created` entry is taken as its own, as servers may key it
|
||||
/// differently.
|
||||
fn interpret_import_for(mr: &MethodCall, cid: &str, in_call: usize) -> SingleImport {
|
||||
if let Some(err) = mr
|
||||
.args
|
||||
.get("notCreated")
|
||||
@@ -828,21 +533,25 @@ fn interpret_import_for(mr: &MethodCall, cid: &str, in_call: usize) -> SingleImp
|
||||
.unwrap_or("")
|
||||
.to_owned();
|
||||
if error_type == "alreadyExists" {
|
||||
return SingleImport::Skipped;
|
||||
return Ok(SingleImport::Skipped);
|
||||
}
|
||||
return SingleImport::NotCreated {
|
||||
return Ok(SingleImport::NotCreated {
|
||||
error_type,
|
||||
detail: err.to_string(),
|
||||
};
|
||||
});
|
||||
}
|
||||
let created = mr.args.get("created").and_then(Value::as_object);
|
||||
if created.is_some_and(|c| c.contains_key(cid) || (in_call == 1 && !c.is_empty())) {
|
||||
return SingleImport::Created;
|
||||
if mr
|
||||
.args
|
||||
.get("created")
|
||||
.and_then(Value::as_object)
|
||||
.is_some_and(|c| !c.is_empty())
|
||||
{
|
||||
return Ok(SingleImport::Created);
|
||||
}
|
||||
SingleImport::NotCreated {
|
||||
Ok(SingleImport::NotCreated {
|
||||
error_type: String::new(),
|
||||
detail: format!("Email/import returned neither created nor notCreated for {cid}"),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -109,10 +108,6 @@ pub fn reconcile(
|
||||
ty.jmap_name(),
|
||||
id
|
||||
));
|
||||
} else if net.dry_run {
|
||||
// The object only exists in the plan; claiming the
|
||||
// default would be a write.
|
||||
default_claimed = true;
|
||||
} else {
|
||||
let mut req = crate::jmap::request::Request::new();
|
||||
req.call(
|
||||
|
||||
+25
-216
@@ -10,14 +10,12 @@ use std::collections::{HashMap, HashSet};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::common::{create_batch, jid, retry_if_blob_missing, target_get_all, update_batch};
|
||||
use super::sieve_names;
|
||||
use super::{Maps, Net, Plan, Uploader};
|
||||
use crate::db;
|
||||
use crate::error::Error;
|
||||
use crate::jmap::blobxfer;
|
||||
use crate::jmap::error::JmapError;
|
||||
use crate::jmap::request::{Request, check_method_error};
|
||||
use crate::logging::{LEVEL_DEFAULT, Logger};
|
||||
use crate::sync::import_jmap::mapping::BlobBytes;
|
||||
use crate::jmap::request::Request;
|
||||
use crate::logging::Logger;
|
||||
use crate::sync::import_jmap::mapping::{SIEVE_SELECT, row_to_sieve_script};
|
||||
use crate::sync::{Context, TypeCounts};
|
||||
use crate::types::ObjectType;
|
||||
@@ -66,77 +64,31 @@ pub fn reconcile(
|
||||
let mut active_target: Option<String> = None;
|
||||
let mut deactivate = false;
|
||||
let mut uploader = Uploader::new(net, &ctx.conn);
|
||||
let rename_vendor = sieve_names::target_uses_inbuxa_names(&target_sieve_extensions(net));
|
||||
let wanted_active = locals
|
||||
.iter()
|
||||
.find(|(_, _, a, _)| *a)
|
||||
.map(|(_, n, _, _)| n.clone().unwrap_or_default());
|
||||
|
||||
let mut updates: Vec<(String, Value)> = Vec::new();
|
||||
let mut validator = Validator::default();
|
||||
|
||||
for (local, name, is_active, blob_local) in &locals {
|
||||
let matched = name.as_ref().and_then(|n| target_by_name.get(n)).cloned();
|
||||
let label = name.as_deref().unwrap_or("(unnamed)");
|
||||
let rewritten = if rename_vendor {
|
||||
renamed_script(&uploader, *blob_local)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let target_id = if let Some(id) = matched {
|
||||
// Compare what would be written -- the renamed bytes where the
|
||||
// script needed renaming -- so an unchanged script stays unchanged.
|
||||
let ours = match &rewritten {
|
||||
Some((bytes, _)) => bytes.clone(),
|
||||
None => uploader.bytes(*blob_local).map_err(Error::from)?,
|
||||
};
|
||||
match content_differs(net, &ours, target_blob.get(&id)) {
|
||||
match content_differs(ctx, net, *blob_local, target_blob.get(&id)) {
|
||||
Ok(false) => counts.skipped += 1,
|
||||
Ok(true) if !validator.accepts(net, label, &ours, counts, logger) => {}
|
||||
Ok(true) => {
|
||||
let blob = match &rewritten {
|
||||
Some((bytes, renamed)) => {
|
||||
log_renames(label, renamed, logger);
|
||||
uploader.upload_bytes_as(*blob_local, "application/sieve", bytes)
|
||||
}
|
||||
None => uploader.upload_with(*blob_local, "application/sieve"),
|
||||
};
|
||||
match blob {
|
||||
Ok(b) => updates.push((id.clone(), json!({ "blobId": b.0 }))),
|
||||
Ok(true) => match uploader.upload_with(*blob_local, "application/sieve") {
|
||||
Ok(blob) => updates.push((id.clone(), json!({ "blobId": blob.0 }))),
|
||||
Err(e) => {
|
||||
logger.warn(&format!(
|
||||
"SieveScript {label}: upload for update failed: {e}"
|
||||
));
|
||||
logger.warn(&format!("SieveScript {id}: upload for update failed: {e}"));
|
||||
counts.failed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
logger.warn(&format!("SieveScript {label}: not compared: {e}"));
|
||||
logger.warn(&format!("SieveScript {id}: not compared: {e}"));
|
||||
counts.skipped += 1;
|
||||
}
|
||||
}
|
||||
id
|
||||
} else {
|
||||
let cid = format!("c{local}");
|
||||
if net.dry_run {
|
||||
let ours = match &rewritten {
|
||||
Some((bytes, _)) => bytes.clone(),
|
||||
None => uploader.bytes(*blob_local).map_err(Error::from)?,
|
||||
};
|
||||
if !validator.accepts(net, label, &ours, counts, logger) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if let Some((_, renamed)) = &rewritten {
|
||||
log_renames(label, renamed, logger);
|
||||
}
|
||||
let rewritten = rewritten.as_ref().map(|(bytes, _)| bytes);
|
||||
let build = |up: &mut Uploader<'_>| -> Result<Value, Error> {
|
||||
let blob_id = match &rewritten {
|
||||
Some(bytes) => up.upload_bytes_as(*blob_local, "application/sieve", bytes),
|
||||
None => up.upload_with(*blob_local, "application/sieve"),
|
||||
}
|
||||
let blob_id = up
|
||||
.upload_with(*blob_local, "application/sieve")
|
||||
.map_err(Error::from)?;
|
||||
let mut obj = serde_json::Map::new();
|
||||
if let Some(n) = name {
|
||||
@@ -161,7 +113,7 @@ pub fn reconcile(
|
||||
}
|
||||
None => {
|
||||
for (cid, err) in &outcome.not_created {
|
||||
logger.warn(&format!("SieveScript {label} ({cid}) not created: {err}"));
|
||||
logger.warn(&format!("SieveScript {cid} not created: {err}"));
|
||||
}
|
||||
counts.failed += 1;
|
||||
continue;
|
||||
@@ -178,15 +130,6 @@ pub fn reconcile(
|
||||
if active_target.is_none() && locals.iter().all(|(_, _, a, _)| !*a) {
|
||||
deactivate = true;
|
||||
}
|
||||
if let (Some(name), None) = (&wanted_active, &active_target) {
|
||||
// The script that was active at the source never made it to the
|
||||
// target (its creation failure is already counted): say plainly that
|
||||
// the account now has no filtering, rather than leave it to a warning.
|
||||
logger.error(&format!(
|
||||
"the active Sieve script \"{name}\" could not be created on the target; \
|
||||
no filtering is active there"
|
||||
));
|
||||
}
|
||||
|
||||
if !net.dry_run {
|
||||
let mut req = Request::new();
|
||||
@@ -198,20 +141,8 @@ pub fn reconcile(
|
||||
json!({ "accountId": net.account })
|
||||
};
|
||||
req.call("SieveScript/set", args, "a");
|
||||
let result = req
|
||||
.send(&net.client, &net.api)
|
||||
.and_then(|resp| resp.by_call_id("a").cloned())
|
||||
.and_then(|mr| check_method_error(&mr));
|
||||
if let Err(e) = result {
|
||||
match (&active_target, &wanted_active) {
|
||||
(Some(_), Some(name)) => {
|
||||
logger.error(&format!(
|
||||
"the Sieve script \"{name}\" was created but could not be activated: {e}"
|
||||
));
|
||||
counts.failed += 1;
|
||||
}
|
||||
_ => logger.warn(&format!("SieveScript activation failed: {e}")),
|
||||
}
|
||||
if let Err(e) = req.send(&net.client, &net.api) {
|
||||
logger.warn(&format!("SieveScript activation failed: {e}"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -229,142 +160,20 @@ pub fn reconcile(
|
||||
})
|
||||
}
|
||||
|
||||
/// Checks, in a dry run, that the target would accept each script about to
|
||||
/// be written. A script too large to upload, or one `SieveScript/validate`
|
||||
/// rejects, is counted as a failure and listed in the plan. A real run
|
||||
/// checks nothing here: the target's own answer to the write is the check.
|
||||
#[derive(Default)]
|
||||
struct Validator {
|
||||
unsupported: bool,
|
||||
}
|
||||
|
||||
impl Validator {
|
||||
/// Whether the script may be written. Always true outside a dry run.
|
||||
fn accepts(
|
||||
&mut self,
|
||||
/// Whether the target's copy of a script differs from the archive's. A target
|
||||
/// that reports no blob is taken as different, so the archive's is written.
|
||||
fn content_differs(
|
||||
ctx: &Context,
|
||||
net: &Net,
|
||||
label: &str,
|
||||
bytes: &[u8],
|
||||
counts: &mut TypeCounts,
|
||||
logger: &Logger,
|
||||
) -> bool {
|
||||
if !net.dry_run || self.unsupported {
|
||||
return true;
|
||||
}
|
||||
let why = match net.check_upload_size(bytes.len() as u64) {
|
||||
Err(JmapError::SingleObjectTooLarge(m)) => Some(m),
|
||||
Err(e) => Some(e.to_string()),
|
||||
Ok(()) => match validate(net, bytes) {
|
||||
Ok(why) => why,
|
||||
Err(e) if is_unknown_method(&e) => {
|
||||
logger.warn("the target cannot validate Sieve scripts; they are not checked");
|
||||
self.unsupported = true;
|
||||
None
|
||||
}
|
||||
Err(e) => {
|
||||
logger.warn(&format!("SieveScript {label}: not validated: {e}"));
|
||||
None
|
||||
}
|
||||
},
|
||||
};
|
||||
match why {
|
||||
None => true,
|
||||
Some(why) => {
|
||||
logger.warn(&format!(
|
||||
"SieveScript {label}: the target would reject it: {why}"
|
||||
));
|
||||
net.would_fail(format!(
|
||||
"SieveScript \"{label}\": the target would reject it: {why}"
|
||||
));
|
||||
counts.failed += 1;
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Asks the target whether it would accept `bytes` as a Sieve script: `None`
|
||||
/// if it would, or its reason. The script goes up as a blob, which the
|
||||
/// server keeps only for a while; nothing is created in the account.
|
||||
fn validate(net: &Net, bytes: &[u8]) -> Result<Option<String>, JmapError> {
|
||||
let blob = blobxfer::upload_bytes(
|
||||
&net.client,
|
||||
&net.session,
|
||||
&net.account,
|
||||
"application/sieve",
|
||||
bytes,
|
||||
)?;
|
||||
let mut req = Request::new();
|
||||
req.call(
|
||||
"SieveScript/validate",
|
||||
json!({ "accountId": net.account, "blobId": blob.0 }),
|
||||
"v",
|
||||
);
|
||||
let resp = req.send(&net.client, &net.api)?;
|
||||
let mr = resp.by_call_id("v")?;
|
||||
check_method_error(mr)?;
|
||||
Ok(match mr.args.get("error") {
|
||||
None | Some(Value::Null) => None,
|
||||
Some(err) => Some(
|
||||
err.get("description")
|
||||
.and_then(Value::as_str)
|
||||
.or_else(|| err.get("type").and_then(Value::as_str))
|
||||
.unwrap_or("rejected")
|
||||
.to_owned(),
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
fn is_unknown_method(e: &JmapError) -> bool {
|
||||
match e {
|
||||
JmapError::UnknownMethod => true,
|
||||
JmapError::Method { error_type, .. } => error_type == "unknownMethod",
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The target's `sieveExtensions`, from its Sieve account capability.
|
||||
fn target_sieve_extensions(net: &Net) -> Vec<String> {
|
||||
net.session
|
||||
.account_capabilities(&net.account)
|
||||
.and_then(|caps| caps.get("urn:ietf:params:jmap:sieve"))
|
||||
.and_then(|c| c.get("sieveExtensions"))
|
||||
.and_then(Value::as_array)
|
||||
.map(|a| {
|
||||
a.iter()
|
||||
.filter_map(Value::as_str)
|
||||
.map(str::to_owned)
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// A script's bytes after renaming, and the names that were renamed.
|
||||
type Renamed = (Vec<u8>, Vec<String>);
|
||||
|
||||
/// The script's bytes with Stalwart's vendor names renamed for an inbuxa
|
||||
/// target, and the names renamed, or `None` when it needs no change.
|
||||
fn renamed_script(uploader: &Uploader<'_>, blob_local: i64) -> Result<Option<Renamed>, Error> {
|
||||
let bytes = uploader.bytes(blob_local).map_err(Error::from)?;
|
||||
Ok(sieve_names::rewrite(&bytes))
|
||||
}
|
||||
|
||||
/// Prints each rename made to a script about to be written.
|
||||
fn log_renames(label: &str, renamed: &[String], logger: &Logger) {
|
||||
for old in renamed {
|
||||
let new = old.replacen("vnd.stalwart.", "vnd.inbuxa.", 1);
|
||||
if logger.enabled(LEVEL_DEFAULT) {
|
||||
eprintln!("export: SieveScript {label}: renamed {old} to {new}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the target's copy of a script differs from `ours`. A target that
|
||||
/// reports no blob is taken as different, so ours is written.
|
||||
fn content_differs(net: &Net, ours: &[u8], target_blob: Option<&String>) -> Result<bool, Error> {
|
||||
blob_local: i64,
|
||||
target_blob: Option<&String>,
|
||||
) -> Result<bool, Error> {
|
||||
let Some(target_blob) = target_blob else {
|
||||
return Ok(true);
|
||||
};
|
||||
let ours = db::blobs::blob_bytes(&ctx.conn, blob_local)
|
||||
.map_err(|e| Error::Partial(e.to_string()))?
|
||||
.ok_or_else(|| Error::Partial(format!("blob local id {blob_local} missing")))?;
|
||||
let theirs = blobxfer::download_bytes(
|
||||
&net.client,
|
||||
&net.session,
|
||||
@@ -374,5 +183,5 @@ fn content_differs(net: &Net, ours: &[u8], target_blob: Option<&String>) -> Resu
|
||||
"script.sieve",
|
||||
)
|
||||
.map_err(Error::from)?;
|
||||
Ok(ours != theirs.as_slice())
|
||||
Ok(ours != theirs)
|
||||
}
|
||||
|
||||
@@ -1,342 +0,0 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
//! Stalwart's vendor Sieve names, renamed for inbuxa.
|
||||
//!
|
||||
//! inbuxa accepts `vnd.inbuxa.while` and `vnd.inbuxa.expressions` where
|
||||
//! Stalwart accepted `vnd.stalwart.*`, with no alias, and names its
|
||||
//! environment items the same way. A script carried over unchanged fails to
|
||||
//! compile on inbuxa, so export renames those names -- and only those names:
|
||||
//! the strings of a `require` list, the name argument of an `environment`
|
||||
//! test, and `${env.vnd.stalwart.…}` references inside strings. Everything
|
||||
//! else in the script, including other strings that happen to contain the
|
||||
//! text, is copied byte for byte.
|
||||
|
||||
const OLD: &str = "vnd.stalwart.";
|
||||
const NEW: &str = "vnd.inbuxa.";
|
||||
const OLD_ENV_REF: &str = "${env.vnd.stalwart.";
|
||||
const NEW_ENV_REF: &str = "${env.vnd.inbuxa.";
|
||||
|
||||
/// Whether the target advertises inbuxa's vendor extensions, from the
|
||||
/// `sieveExtensions` list of its `urn:ietf:params:jmap:sieve` account
|
||||
/// capability.
|
||||
pub fn target_uses_inbuxa_names(sieve_extensions: &[String]) -> bool {
|
||||
sieve_extensions.iter().any(|e| e.starts_with(NEW))
|
||||
}
|
||||
|
||||
/// The script with Stalwart's vendor names renamed, and the old names that
|
||||
/// were changed, in order. `None` when nothing needed renaming, or when the
|
||||
/// script is not UTF-8 (left alone rather than guessed at).
|
||||
pub fn rewrite(script: &[u8]) -> Option<(Vec<u8>, Vec<String>)> {
|
||||
let text = std::str::from_utf8(script).ok()?;
|
||||
if !text.contains(OLD) {
|
||||
return None;
|
||||
}
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let mut renamed = Vec::new();
|
||||
let mut copied = 0;
|
||||
let mut context = Context::None;
|
||||
for tok in Tokens::new(text) {
|
||||
match tok.kind {
|
||||
Kind::Word => {
|
||||
context = match tok.text(text).to_ascii_lowercase().as_str() {
|
||||
"require" => Context::Require,
|
||||
"environment" => Context::Environment,
|
||||
_ if context == Context::Environment => Context::Environment,
|
||||
_ => Context::None,
|
||||
};
|
||||
}
|
||||
Kind::Tag => {
|
||||
// `environment :comparator "i;octet"`: the comparator's own
|
||||
// string is not the item name.
|
||||
if context == Context::Environment
|
||||
&& tok.text(text).eq_ignore_ascii_case(":comparator")
|
||||
{
|
||||
context = Context::EnvironmentComparator;
|
||||
}
|
||||
}
|
||||
Kind::Quoted | Kind::Multiline => {
|
||||
let (start, end) = tok.content;
|
||||
let content = &text[start..end];
|
||||
let mut replacement: Option<String> = None;
|
||||
let whole_name = matches!(context, Context::Require | Context::Environment);
|
||||
if whole_name && content.starts_with(OLD) {
|
||||
replacement = Some(format!("{NEW}{}", &content[OLD.len()..]));
|
||||
renamed.push(content.to_owned());
|
||||
}
|
||||
let current = replacement.as_deref().unwrap_or(content);
|
||||
if current.contains(OLD_ENV_REF) {
|
||||
let mut n = 0;
|
||||
let mut rest = current;
|
||||
while let Some(i) = rest.find(OLD_ENV_REF) {
|
||||
let tail = &rest[i + 2..];
|
||||
let name_end = tail.find('}').unwrap_or(tail.len());
|
||||
renamed.push(tail[4..name_end].to_owned());
|
||||
rest = &rest[i + OLD_ENV_REF.len()..];
|
||||
n += 1;
|
||||
}
|
||||
if n > 0 {
|
||||
replacement = Some(current.replace(OLD_ENV_REF, NEW_ENV_REF));
|
||||
}
|
||||
}
|
||||
if let Some(r) = replacement {
|
||||
out.push_str(&text[copied..start]);
|
||||
out.push_str(&r);
|
||||
copied = end;
|
||||
}
|
||||
context = match context {
|
||||
Context::Require => Context::Require,
|
||||
Context::EnvironmentComparator => Context::Environment,
|
||||
_ => Context::None,
|
||||
};
|
||||
}
|
||||
Kind::Punct(';') | Kind::Punct('{') | Kind::Punct('}') => context = Context::None,
|
||||
Kind::Punct(_) => {}
|
||||
}
|
||||
}
|
||||
if renamed.is_empty() {
|
||||
return None;
|
||||
}
|
||||
out.push_str(&text[copied..]);
|
||||
Some((out.into_bytes(), renamed))
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum Context {
|
||||
None,
|
||||
Require,
|
||||
Environment,
|
||||
EnvironmentComparator,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum Kind {
|
||||
Word,
|
||||
Tag,
|
||||
Quoted,
|
||||
Multiline,
|
||||
Punct(char),
|
||||
}
|
||||
|
||||
struct Token {
|
||||
kind: Kind,
|
||||
span: (usize, usize),
|
||||
/// The string's content, without quotes or the `text:` framing.
|
||||
content: (usize, usize),
|
||||
}
|
||||
|
||||
impl Token {
|
||||
fn text<'a>(&self, src: &'a str) -> &'a str {
|
||||
&src[self.span.0..self.span.1]
|
||||
}
|
||||
}
|
||||
|
||||
/// Just enough of RFC 5228's lexer to find strings and the words before
|
||||
/// them: comments are skipped, and quoted strings and `text:` blocks are
|
||||
/// read whole, so nothing inside them is mistaken for a command.
|
||||
struct Tokens<'a> {
|
||||
src: &'a str,
|
||||
pos: usize,
|
||||
}
|
||||
|
||||
impl<'a> Tokens<'a> {
|
||||
fn new(src: &'a str) -> Self {
|
||||
Tokens { src, pos: 0 }
|
||||
}
|
||||
}
|
||||
|
||||
impl Iterator for Tokens<'_> {
|
||||
type Item = Token;
|
||||
|
||||
fn next(&mut self) -> Option<Token> {
|
||||
let b = self.src.as_bytes();
|
||||
loop {
|
||||
while self.pos < b.len() && b[self.pos].is_ascii_whitespace() {
|
||||
self.pos += 1;
|
||||
}
|
||||
if self.pos >= b.len() {
|
||||
return None;
|
||||
}
|
||||
if b[self.pos] == b'#' {
|
||||
while self.pos < b.len() && b[self.pos] != b'\n' {
|
||||
self.pos += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if b[self.pos..].starts_with(b"/*") {
|
||||
self.pos = match self.src[self.pos + 2..].find("*/") {
|
||||
Some(i) => self.pos + 2 + i + 2,
|
||||
None => b.len(),
|
||||
};
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
let start = self.pos;
|
||||
let c = b[start];
|
||||
if c == b'"' {
|
||||
let mut i = start + 1;
|
||||
while i < b.len() && b[i] != b'"' {
|
||||
i += if b[i] == b'\\' { 2 } else { 1 };
|
||||
}
|
||||
let end = i.min(b.len());
|
||||
self.pos = (end + 1).min(b.len());
|
||||
return Some(Token {
|
||||
kind: Kind::Quoted,
|
||||
span: (start, self.pos),
|
||||
content: (start + 1, end),
|
||||
});
|
||||
}
|
||||
if c.is_ascii_alphabetic() || c == b'_' || c == b':' {
|
||||
let mut i = start + 1;
|
||||
while i < b.len() && (b[i].is_ascii_alphanumeric() || b[i] == b'_') {
|
||||
i += 1;
|
||||
}
|
||||
let word = &self.src[start..i];
|
||||
if word.eq_ignore_ascii_case("text:")
|
||||
|| (word.eq_ignore_ascii_case("text") && b.get(i) == Some(&b':'))
|
||||
{
|
||||
let after = if b.get(i) == Some(&b':') { i + 1 } else { i };
|
||||
// The body starts after the rest of the `text:` line and runs
|
||||
// to a line holding a single dot.
|
||||
let body = match self.src[after..].find('\n') {
|
||||
Some(n) => after + n + 1,
|
||||
None => b.len(),
|
||||
};
|
||||
let (body_end, next) = find_dot_line(self.src, body);
|
||||
self.pos = next;
|
||||
return Some(Token {
|
||||
kind: Kind::Multiline,
|
||||
span: (start, next),
|
||||
content: (body, body_end),
|
||||
});
|
||||
}
|
||||
self.pos = i;
|
||||
let kind = if c == b':' { Kind::Tag } else { Kind::Word };
|
||||
return Some(Token {
|
||||
kind,
|
||||
span: (start, i),
|
||||
content: (start, i),
|
||||
});
|
||||
}
|
||||
let ch = self.src[start..].chars().next().unwrap_or('\0');
|
||||
self.pos = start + ch.len_utf8();
|
||||
Some(Token {
|
||||
kind: Kind::Punct(ch),
|
||||
span: (start, self.pos),
|
||||
content: (start, self.pos),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// End of a `text:` body (the start of its closing dot line) and the offset
|
||||
/// after that line.
|
||||
fn find_dot_line(src: &str, from: usize) -> (usize, usize) {
|
||||
let mut line_start = from;
|
||||
while line_start < src.len() {
|
||||
let line_end = src[line_start..]
|
||||
.find('\n')
|
||||
.map(|n| line_start + n)
|
||||
.unwrap_or(src.len());
|
||||
if src[line_start..line_end].trim_end_matches('\r') == "." {
|
||||
return (line_start, (line_end + 1).min(src.len()));
|
||||
}
|
||||
line_start = line_end + 1;
|
||||
}
|
||||
(src.len(), src.len())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn run(s: &str) -> Option<(String, Vec<String>)> {
|
||||
rewrite(s.as_bytes()).map(|(b, r)| (String::from_utf8(b).unwrap(), r))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renames_a_require_list() {
|
||||
let (out, renamed) =
|
||||
run("require [\"fileinto\", \"vnd.stalwart.while\", \"vnd.stalwart.expressions\"];\n")
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
out,
|
||||
"require [\"fileinto\", \"vnd.inbuxa.while\", \"vnd.inbuxa.expressions\"];\n"
|
||||
);
|
||||
assert_eq!(renamed, ["vnd.stalwart.while", "vnd.stalwart.expressions"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renames_a_single_require_string() {
|
||||
let (out, _) = run("REQUIRE \"vnd.stalwart.while\";").unwrap();
|
||||
assert_eq!(out, "REQUIRE \"vnd.inbuxa.while\";");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renames_the_environment_item_name_only() {
|
||||
let src = "if environment :comparator \"i;octet\" :is \"vnd.stalwart.username\" \"vnd.stalwart.x\" { keep; }";
|
||||
let (out, renamed) = run(src).unwrap();
|
||||
assert_eq!(
|
||||
out,
|
||||
"if environment :comparator \"i;octet\" :is \"vnd.inbuxa.username\" \"vnd.stalwart.x\" { keep; }"
|
||||
);
|
||||
assert_eq!(renamed, ["vnd.stalwart.username"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renames_env_references_inside_strings() {
|
||||
let src = "set \"box\" \"${env.vnd.stalwart.default_mailbox}/Archive\";";
|
||||
let (out, renamed) = run(src).unwrap();
|
||||
assert_eq!(
|
||||
out,
|
||||
"set \"box\" \"${env.vnd.inbuxa.default_mailbox}/Archive\";"
|
||||
);
|
||||
assert_eq!(renamed, ["vnd.stalwart.default_mailbox"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renames_env_references_in_text_blocks() {
|
||||
let src = "vacation text:\nHi ${env.vnd.stalwart.username}.\n.\n;\n";
|
||||
let (out, _) = run(src).unwrap();
|
||||
assert_eq!(
|
||||
out,
|
||||
"vacation text:\nHi ${env.vnd.inbuxa.username}.\n.\n;\n"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leaves_unrelated_strings_comments_and_text_alone() {
|
||||
let src = "# vnd.stalwart.while is old\n/* \"vnd.stalwart.x\" */\n\
|
||||
if header :contains \"subject\" \"vnd.stalwart.while\" { fileinto \"vnd.stalwart.box\"; }\n\
|
||||
vacation text:\nrequire \"vnd.stalwart.while\";\n.\n;\n";
|
||||
assert!(run(src).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn require_context_ends_at_the_semicolon() {
|
||||
let src = "require \"fileinto\"; fileinto \"vnd.stalwart.folder\";";
|
||||
assert!(run(src).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nothing_to_do_is_none() {
|
||||
assert!(run("require \"fileinto\";\nkeep;\n").is_none());
|
||||
assert!(rewrite(&[0xff, 0xfe, b'v']).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn target_detection() {
|
||||
assert!(target_uses_inbuxa_names(&[
|
||||
"fileinto".to_owned(),
|
||||
"vnd.inbuxa.while".to_owned()
|
||||
]));
|
||||
assert!(!target_uses_inbuxa_names(&[
|
||||
"fileinto".to_owned(),
|
||||
"vnd.stalwart.while".to_owned()
|
||||
]));
|
||||
assert!(!target_uses_inbuxa_names(&[]));
|
||||
}
|
||||
}
|
||||
@@ -17,7 +17,6 @@ use crate::logging::Logger;
|
||||
use crate::sync::import_jmap::mapping::{
|
||||
CALENDAR_EVENT_SELECT, CONTACT_CARD_SELECT, calendar_event_to_wire, contact_card_to_wire,
|
||||
};
|
||||
use crate::sync::progress::Progress;
|
||||
use crate::sync::prune::{TargetObj, candidates};
|
||||
use crate::sync::{Context, TypeCounts};
|
||||
use crate::types::ObjectType;
|
||||
@@ -80,13 +79,7 @@ pub fn reconcile(
|
||||
let mut matched_uids: HashSet<String> = HashSet::new();
|
||||
let mut updates: Vec<(String, Value)> = Vec::new();
|
||||
let blobs = Uploader::new(net, &ctx.conn);
|
||||
let mut progress = Progress::new(
|
||||
format!("export: {}", ty.jmap_name()),
|
||||
rows.len() as u64,
|
||||
logger,
|
||||
);
|
||||
for (local, uid) in &rows {
|
||||
progress.add(1);
|
||||
if let Some((tid, existing)) = by_uid.get(uid) {
|
||||
maps.insert(ty, *local, crate::jmap::wire::JmapId(tid.clone()));
|
||||
matched_uids.insert(uid.clone());
|
||||
@@ -109,7 +102,6 @@ pub fn reconcile(
|
||||
Err(e) if e.aborts_run() => return Err(e),
|
||||
Err(e) => {
|
||||
logger.warn(&format!("{} skipped: {e}", describe(ty, *local, uid)));
|
||||
net.would_fail(format!("{}: {e}", describe(ty, *local, uid)));
|
||||
counts.failed += 1;
|
||||
continue;
|
||||
}
|
||||
@@ -195,19 +187,13 @@ fn build_wire(
|
||||
}
|
||||
}
|
||||
|
||||
/// An `updated` value as a point in time, so that offsets and fractional
|
||||
/// seconds compare as the same instant. `None` if it does not parse.
|
||||
fn parse_updated(s: &str) -> Option<time::OffsetDateTime> {
|
||||
time::OffsetDateTime::parse(s, &time::format_description::well_known::Rfc3339).ok()
|
||||
}
|
||||
|
||||
/// The update that makes `target` match the archive's `wire` object, or
|
||||
/// `None` when nothing changed. When both carry `updated`, it decides: the
|
||||
/// archive's copy wins only if it is newer. Otherwise each property the
|
||||
/// archive writes is compared, and those that differ are sent whole.
|
||||
fn changed_properties(wire: &Value, target: &Value) -> Option<Value> {
|
||||
let wire = wire.as_object()?;
|
||||
let stamp = |v: Option<&Value>| v.and_then(Value::as_str).and_then(parse_updated);
|
||||
let stamp = |v: Option<&Value>| v.and_then(Value::as_str).map(str::to_owned);
|
||||
if let (Some(ours), Some(theirs)) = (stamp(wire.get("updated")), stamp(target.get("updated")))
|
||||
&& ours <= theirs
|
||||
{
|
||||
@@ -249,16 +235,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn updated_compares_instants_not_strings() {
|
||||
let target = json!({"uid": "u", "title": "old", "updated": "2026-01-02T00:00:00Z"});
|
||||
let same_instant = json!({"uid": "u", "title": "new",
|
||||
"updated": "2026-01-02T01:00:00.000+01:00"});
|
||||
assert_eq!(changed_properties(&same_instant, &target), None);
|
||||
let later = json!({"uid": "u", "title": "new", "updated": "2026-01-02T00:00:00.5Z"});
|
||||
assert!(changed_properties(&later, &target).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn updated_decides_when_both_sides_carry_it() {
|
||||
let target = json!({"uid": "u", "title": "old", "updated": "2026-01-02T00:00:00Z"});
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -19,7 +18,6 @@ use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts};
|
||||
use super::collections;
|
||||
use super::items;
|
||||
use super::tree;
|
||||
use crate::net::CertOverride;
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum DavKindArg {
|
||||
@@ -107,7 +105,7 @@ fn run_into(
|
||||
let client = DavClient::new(
|
||||
config.auth.to_jmap_auth(),
|
||||
RetryPolicy::new(common.max_retries),
|
||||
CertOverride::for_url(common.allow_invalid_certs, &config.url),
|
||||
common.allow_invalid_certs,
|
||||
);
|
||||
client.set_logger(logger);
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -105,12 +104,7 @@ fn reconcile_one(
|
||||
to_fetch.push(id.clone());
|
||||
}
|
||||
if !to_fetch.is_empty() {
|
||||
let failed_items = for_each_fetched_item(
|
||||
ctx,
|
||||
ItemShape::CalendarItem,
|
||||
&to_fetch,
|
||||
&outcome.sizes,
|
||||
|msg| {
|
||||
let failed_items = for_each_fetched_item(ctx, ItemShape::CalendarItem, &to_fetch, |msg| {
|
||||
if !msg.success {
|
||||
if matches!(
|
||||
msg.response_code,
|
||||
@@ -150,8 +144,7 @@ fn reconcile_one(
|
||||
existing,
|
||||
counts,
|
||||
)
|
||||
},
|
||||
)?;
|
||||
})?;
|
||||
counts.failed += failed_items;
|
||||
}
|
||||
delete_vanished(
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -97,8 +96,7 @@ fn reconcile_one(
|
||||
to_fetch.push(id.clone());
|
||||
}
|
||||
if !to_fetch.is_empty() {
|
||||
let failed_items =
|
||||
for_each_fetched_item(ctx, ItemShape::Contact, &to_fetch, &outcome.sizes, |msg| {
|
||||
let failed_items = for_each_fetched_item(ctx, ItemShape::Contact, &to_fetch, |msg| {
|
||||
if !msg.success {
|
||||
if matches!(
|
||||
msg.response_code,
|
||||
|
||||
@@ -20,7 +20,6 @@ use crate::sync::{CommonConfig, Summary, TypeCounts};
|
||||
|
||||
use super::folders::{self, plan_folders};
|
||||
use super::{calendar, contacts, messages};
|
||||
use crate::net::CertOverride;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum EwsAuth {
|
||||
@@ -37,8 +36,6 @@ pub struct EwsImportConfig {
|
||||
pub auth: EwsAuth,
|
||||
pub ews_connections: usize,
|
||||
pub getitem_batch: usize,
|
||||
/// Byte cap for one GetItem batch; see `sync::batch`.
|
||||
pub getitem_batch_bytes: u64,
|
||||
pub attachment_batch: usize,
|
||||
pub use_syncfolderitems: bool,
|
||||
pub allow_source_change: bool,
|
||||
@@ -48,8 +45,8 @@ pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result<Summary, Err
|
||||
let logger = common.logger;
|
||||
let mut conn = db::init::open(&common.archive)?;
|
||||
|
||||
let (auth, acquired) = resolve_auth(&config.auth)?;
|
||||
let (discovery, certs) = run_autodiscover(&config, &acquired, common.allow_invalid_certs)?;
|
||||
let (auth, acquired) = resolve_auth(&config.auth, common.allow_invalid_certs)?;
|
||||
let discovery = run_autodiscover(&config, &acquired, common.allow_invalid_certs)?;
|
||||
if logger.enabled(LEVEL_PROGRESS) {
|
||||
eprintln!(
|
||||
"EWS discovery: url={} source={:?}",
|
||||
@@ -80,7 +77,7 @@ pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result<Summary, Err
|
||||
let client = EwsClient::new(
|
||||
auth,
|
||||
RetryPolicy::new(common.max_retries),
|
||||
certs.narrowed_to(&discovery.ews_url),
|
||||
common.allow_invalid_certs,
|
||||
);
|
||||
client.set_logger(logger);
|
||||
if matches!(config.mailbox_kind, MailboxKind::PublicFolders) {
|
||||
@@ -91,7 +88,13 @@ pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result<Summary, Err
|
||||
if let EwsAuth::OAuth(OAuthFlow::ClientCredentials { .. }) = &config.auth {
|
||||
client.set_impersonation(Some(mailbox.clone()));
|
||||
}
|
||||
spawn_token_refresher(&client, &config.auth, &acquired, logger);
|
||||
spawn_token_refresher(
|
||||
&client,
|
||||
&config.auth,
|
||||
&acquired,
|
||||
common.allow_invalid_certs,
|
||||
logger,
|
||||
);
|
||||
|
||||
let username = match &config.auth {
|
||||
EwsAuth::Basic { user, .. } => user.clone(),
|
||||
@@ -149,7 +152,6 @@ pub fn run(common: CommonConfig, config: EwsImportConfig) -> Result<Summary, Err
|
||||
url: &session_url,
|
||||
source_id,
|
||||
batch_size: config.getitem_batch.max(1),
|
||||
batch_bytes: config.getitem_batch_bytes.max(1),
|
||||
attachment_batch: config.attachment_batch.max(1),
|
||||
connections: config.ews_connections.clamp(1, 8),
|
||||
use_syncfolderitems: config.use_syncfolderitems,
|
||||
@@ -209,7 +211,10 @@ fn run_dry(
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
fn resolve_auth(auth: &EwsAuth) -> Result<(Auth, Option<AcquiredToken>), Error> {
|
||||
fn resolve_auth(
|
||||
auth: &EwsAuth,
|
||||
allow_invalid_certs: bool,
|
||||
) -> Result<(Auth, Option<AcquiredToken>), Error> {
|
||||
match auth {
|
||||
EwsAuth::Basic { user, password } => Ok((
|
||||
Auth::Basic {
|
||||
@@ -219,9 +224,12 @@ fn resolve_auth(auth: &EwsAuth) -> Result<(Auth, Option<AcquiredToken>), Error>
|
||||
None,
|
||||
)),
|
||||
EwsAuth::Bearer { token } => {
|
||||
let acq = acquire(&OAuthFlow::PreAcquired {
|
||||
let acq = acquire(
|
||||
&OAuthFlow::PreAcquired {
|
||||
token: token.clone(),
|
||||
})
|
||||
},
|
||||
allow_invalid_certs,
|
||||
)
|
||||
.map_err(Error::from)?;
|
||||
Ok((
|
||||
Auth::Bearer {
|
||||
@@ -231,7 +239,7 @@ fn resolve_auth(auth: &EwsAuth) -> Result<(Auth, Option<AcquiredToken>), Error>
|
||||
))
|
||||
}
|
||||
EwsAuth::OAuth(flow) => {
|
||||
let acq = acquire(flow).map_err(Error::from)?;
|
||||
let acq = acquire(flow, allow_invalid_certs).map_err(Error::from)?;
|
||||
Ok((
|
||||
Auth::Bearer {
|
||||
token: acq.access_token.clone(),
|
||||
@@ -246,36 +254,19 @@ fn run_autodiscover(
|
||||
config: &EwsImportConfig,
|
||||
acquired: &Option<AcquiredToken>,
|
||||
allow_invalid_certs: bool,
|
||||
) -> Result<(DiscoveryResult, CertOverride), Error> {
|
||||
) -> Result<DiscoveryResult, Error> {
|
||||
let email = config
|
||||
.mailbox
|
||||
.clone()
|
||||
.or_else(|| acquired.as_ref().and_then(|a| a.upn.clone()));
|
||||
let certs =
|
||||
autodiscover_cert_override(config.url.as_deref(), email.as_deref(), allow_invalid_certs);
|
||||
let result =
|
||||
discover(config.url.as_deref(), email.as_deref(), None, &certs).map_err(Error::from)?;
|
||||
Ok((result, certs))
|
||||
}
|
||||
|
||||
/// Where `--allow-invalid-certs` applies for an EWS import: the host of
|
||||
/// `--url` when one is given, and otherwise the mailbox's own domain, which is
|
||||
/// where on-premises Autodiscover looks. Microsoft's hosts are never covered.
|
||||
fn autodiscover_cert_override(
|
||||
url: Option<&str>,
|
||||
email: Option<&str>,
|
||||
enabled: bool,
|
||||
) -> CertOverride {
|
||||
match (
|
||||
url,
|
||||
email
|
||||
.and_then(|e| e.rsplit_once('@'))
|
||||
.map(|(_, domain)| domain),
|
||||
) {
|
||||
(Some(url), _) => CertOverride::for_url(enabled, url),
|
||||
(None, Some(domain)) => CertOverride::for_domain(enabled, domain),
|
||||
(None, None) => CertOverride::none(),
|
||||
}
|
||||
let result = discover(
|
||||
config.url.as_deref(),
|
||||
email.as_deref(),
|
||||
None,
|
||||
allow_invalid_certs,
|
||||
)
|
||||
.map_err(Error::from)?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn resolve_mailbox(
|
||||
@@ -379,6 +370,7 @@ fn spawn_token_refresher(
|
||||
client: &EwsClient,
|
||||
auth: &EwsAuth,
|
||||
initial: &Option<AcquiredToken>,
|
||||
allow_invalid_certs: bool,
|
||||
logger: crate::logging::Logger,
|
||||
) {
|
||||
let flow = match auth {
|
||||
@@ -410,9 +402,14 @@ fn spawn_token_refresher(
|
||||
let result = if let (Some(rt), OAuthFlow::DeviceCode { tenant, client_id }) =
|
||||
(refresh_token.as_deref(), &flow)
|
||||
{
|
||||
crate::exchange_ews::oauth::refresh_with_token(tenant, client_id, rt)
|
||||
crate::exchange_ews::oauth::refresh_with_token(
|
||||
tenant,
|
||||
client_id,
|
||||
rt,
|
||||
allow_invalid_certs,
|
||||
)
|
||||
} else {
|
||||
crate::exchange_ews::oauth::acquire(&flow)
|
||||
crate::exchange_ews::oauth::acquire(&flow, allow_invalid_certs)
|
||||
};
|
||||
match result {
|
||||
Ok(tok) => {
|
||||
@@ -454,26 +451,6 @@ fn run_gc(conn: &Connection) -> Result<(), Error> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn cert_override_follows_url_then_mailbox_domain() {
|
||||
let by_url = autodiscover_cert_override(
|
||||
Some("https://mail.corp.example/EWS/Exchange.asmx"),
|
||||
Some("[email protected]"),
|
||||
true,
|
||||
);
|
||||
assert!(by_url.allows("https://mail.corp.example/EWS/Exchange.asmx"));
|
||||
assert!(!by_url.allows("https://autodiscover.corp.example/"));
|
||||
|
||||
let by_domain = autodiscover_cert_override(None, Some("[email protected]"), true);
|
||||
assert!(
|
||||
by_domain.allows("https://autodiscover.corp.example/autodiscover/autodiscover.xml")
|
||||
);
|
||||
assert!(!by_domain.allows("https://outlook.office365.com/EWS/Exchange.asmx"));
|
||||
|
||||
assert!(!autodiscover_cert_override(None, Some("[email protected]"), false).is_active());
|
||||
assert!(!autodiscover_cert_override(None, None, true).is_active());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn synthetic_account_id_uses_smtp_for_primary() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -24,8 +23,6 @@ pub struct ItemRunCtx<'a> {
|
||||
pub url: &'a str,
|
||||
pub source_id: i64,
|
||||
pub batch_size: usize,
|
||||
/// Byte cap for one GetItem batch, by `item:Size`; see `sync::batch`.
|
||||
pub batch_bytes: u64,
|
||||
pub attachment_batch: usize,
|
||||
pub connections: usize,
|
||||
pub use_syncfolderitems: bool,
|
||||
@@ -50,9 +47,6 @@ pub struct EnumeratedItem {
|
||||
pub struct EnumerationOutcome {
|
||||
pub items: Vec<EnumeratedItem>,
|
||||
pub mode: EnumerationMode,
|
||||
/// `item:Size` by item id, where the listing gave it. FindItem does;
|
||||
/// SyncFolderItems changes do not, and are batched by count.
|
||||
pub sizes: HashMap<String, u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -76,10 +70,9 @@ pub fn enumerate_folder(
|
||||
{
|
||||
return Ok(outcome);
|
||||
}
|
||||
enumerate_via_find_item(ctx, folder).map(|(items, sizes)| EnumerationOutcome {
|
||||
enumerate_via_find_item(ctx, folder).map(|items| EnumerationOutcome {
|
||||
items,
|
||||
mode: EnumerationMode::Full,
|
||||
sizes,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -158,16 +151,14 @@ fn try_sync_folder_items(
|
||||
deletions,
|
||||
new_sync_state: sync_state,
|
||||
},
|
||||
sizes: HashMap::new(),
|
||||
}))
|
||||
}
|
||||
|
||||
fn enumerate_via_find_item(
|
||||
ctx: &ItemRunCtx<'_>,
|
||||
folder: &FolderId,
|
||||
) -> Result<(Vec<EnumeratedItem>, HashMap<String, u64>), EwsError> {
|
||||
) -> Result<Vec<EnumeratedItem>, EwsError> {
|
||||
let mut items: Vec<EnumeratedItem> = Vec::new();
|
||||
let mut sizes: HashMap<String, u64> = HashMap::new();
|
||||
let mut offset: u32 = 0;
|
||||
let page_size: u32 = 500;
|
||||
loop {
|
||||
@@ -181,9 +172,6 @@ fn enumerate_via_find_item(
|
||||
let parsed = parse_find_item_response(&resp.body)?;
|
||||
let returned = parsed.items.len() as u32;
|
||||
for entry in parsed.items {
|
||||
if let Some(size) = entry.size {
|
||||
sizes.insert(entry.id.id.clone(), size);
|
||||
}
|
||||
items.push(EnumeratedItem {
|
||||
element: entry.element,
|
||||
id: entry.id,
|
||||
@@ -200,7 +188,7 @@ fn enumerate_via_find_item(
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok((items, sizes))
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -297,22 +285,13 @@ pub fn get_items(
|
||||
shape: ItemShape,
|
||||
ids: &[ItemId],
|
||||
) -> Result<GetItemBatchOutcome, EwsError> {
|
||||
let batches: Vec<&[ItemId]> = ids.chunks(ctx.batch_size.max(1)).collect();
|
||||
get_item_batches(ctx, shape, &batches)
|
||||
}
|
||||
|
||||
/// Runs one GetItem per batch, over up to `connections` connections.
|
||||
pub fn get_item_batches(
|
||||
ctx: &ItemRunCtx<'_>,
|
||||
shape: ItemShape,
|
||||
batches: &[&[ItemId]],
|
||||
) -> Result<GetItemBatchOutcome, EwsError> {
|
||||
let batch = ctx.batch_size.max(1);
|
||||
let workers = ctx.connections.clamp(1, 8);
|
||||
let version = ctx.client.server_version();
|
||||
let mut failed_items: u64 = 0;
|
||||
if workers <= 1 || batches.len() <= 1 {
|
||||
if workers <= 1 || ids.len() <= batch {
|
||||
let mut all = Vec::new();
|
||||
for chunk in batches.iter().copied() {
|
||||
for chunk in ids.chunks(batch) {
|
||||
let body = get_item_body(shape, chunk, version);
|
||||
match ctx.client.call(ctx.url, "GetItem", &body) {
|
||||
Ok(resp) => match parse_response_messages(&resp.body, "GetItemResponseMessage") {
|
||||
@@ -358,7 +337,7 @@ pub fn get_item_batches(
|
||||
(n, result)
|
||||
});
|
||||
let mut submitted = 0usize;
|
||||
for chunk in batches {
|
||||
for chunk in ids.chunks(batch) {
|
||||
pool.submit(chunk.to_vec());
|
||||
submitted += 1;
|
||||
}
|
||||
@@ -389,31 +368,21 @@ pub fn get_item_batches(
|
||||
})
|
||||
}
|
||||
|
||||
/// Fetches `ids` in GetItem batches of at most `batch_size` items and, where
|
||||
/// `sizes` knows them, at most `batch_bytes` bytes, and hands each message
|
||||
/// on as it is parsed. Batches are fetched `connections` at a time, so what
|
||||
/// is held at once is about `batch_bytes` per connection, however large the
|
||||
/// mail is.
|
||||
pub fn for_each_fetched_item<F>(
|
||||
ctx: &ItemRunCtx<'_>,
|
||||
shape: ItemShape,
|
||||
ids: &[ItemId],
|
||||
sizes: &HashMap<String, u64>,
|
||||
mut on_message: F,
|
||||
) -> Result<u64, Error>
|
||||
where
|
||||
F: FnMut(crate::exchange_ews::parse::ResponseMessage) -> Result<(), Error>,
|
||||
{
|
||||
let batch = ctx.batch_size.max(1);
|
||||
let workers = ctx.connections.clamp(1, 8);
|
||||
let batches = crate::sync::batch::by_count_and_bytes(
|
||||
ids,
|
||||
|id| sizes.get(&id.id).copied().unwrap_or(0),
|
||||
ctx.batch_size,
|
||||
ctx.batch_bytes,
|
||||
);
|
||||
let window = batch.saturating_mul(workers).max(batch);
|
||||
let mut failed_items = 0u64;
|
||||
for win in batches.chunks(workers) {
|
||||
let outcome = get_item_batches(ctx, shape, win).map_err(Error::from)?;
|
||||
for win in ids.chunks(window) {
|
||||
let outcome = get_items(ctx, shape, win).map_err(Error::from)?;
|
||||
failed_items = failed_items.saturating_add(outcome.failed_items);
|
||||
for msg in outcome.messages {
|
||||
on_message(msg)?;
|
||||
@@ -475,7 +444,6 @@ mod tests {
|
||||
element: "Message".to_owned(),
|
||||
id: ItemId::new("A", "ck-2"),
|
||||
}],
|
||||
sizes: HashMap::new(),
|
||||
mode: EnumerationMode::Delta {
|
||||
deletions: vec!["Z".to_owned()],
|
||||
new_sync_state: "STATE2".to_owned(),
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -96,8 +95,7 @@ fn reconcile_one_folder(
|
||||
to_fetch.push(id.clone());
|
||||
}
|
||||
if !to_fetch.is_empty() {
|
||||
let failed_items =
|
||||
for_each_fetched_item(ctx, ItemShape::Message, &to_fetch, &outcome.sizes, |msg| {
|
||||
let failed_items = for_each_fetched_item(ctx, ItemShape::Message, &to_fetch, |msg| {
|
||||
if !msg.success {
|
||||
if matches!(
|
||||
msg.response_code,
|
||||
|
||||
@@ -20,7 +20,6 @@ use crate::exchange_graph::oauth::{
|
||||
use crate::exchange_graph::types::{EventBodyFormat, MailboxKind, Surfaces, synthetic_account_id};
|
||||
use crate::jmap::http::RetryPolicy;
|
||||
use crate::logging::LEVEL_DEFAULT;
|
||||
use crate::net::CertOverride;
|
||||
use crate::sync::{CommonConfig, Summary, TypeCounts};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -69,11 +68,11 @@ pub fn run(common: CommonConfig, config: GraphImportConfig) -> Result<Summary, E
|
||||
let logger = common.logger;
|
||||
let mut conn = db::init::open(&common.archive)?;
|
||||
|
||||
let acquired = acquire_with_flow(&config.auth)?;
|
||||
let acquired = acquire_with_flow(&config.auth, common.allow_invalid_certs)?;
|
||||
let client = GraphClient::new(
|
||||
acquired.access_token.clone(),
|
||||
RetryPolicy::new(common.max_retries),
|
||||
CertOverride::for_url(common.allow_invalid_certs, &config.api_base),
|
||||
common.allow_invalid_certs,
|
||||
);
|
||||
client.set_logger(logger);
|
||||
|
||||
@@ -122,7 +121,13 @@ pub fn run(common: CommonConfig, config: GraphImportConfig) -> Result<Summary, E
|
||||
&principal.user_principal_name,
|
||||
)?;
|
||||
|
||||
let _refresher = spawn_token_refresher(&client, &config.auth, &acquired, logger);
|
||||
let _refresher = spawn_token_refresher(
|
||||
&client,
|
||||
&config.auth,
|
||||
&acquired,
|
||||
common.allow_invalid_certs,
|
||||
logger,
|
||||
);
|
||||
|
||||
let mut summary = Summary::default();
|
||||
let mut mailbox_counts = TypeCounts::default();
|
||||
@@ -277,7 +282,7 @@ pub fn enumerate_mail_folders(
|
||||
Ok(all)
|
||||
}
|
||||
|
||||
fn acquire_with_flow(auth: &GraphAuth) -> Result<AcquiredToken, Error> {
|
||||
fn acquire_with_flow(auth: &GraphAuth, allow_invalid_certs: bool) -> Result<AcquiredToken, Error> {
|
||||
let flow = match auth {
|
||||
GraphAuth::PreAcquired { token } => OAuthFlow::PreAcquired {
|
||||
token: token.clone(),
|
||||
@@ -290,7 +295,7 @@ fn acquire_with_flow(auth: &GraphAuth) -> Result<AcquiredToken, Error> {
|
||||
client_id: client_id.clone(),
|
||||
},
|
||||
};
|
||||
acquire(&flow).map_err(Error::from)
|
||||
acquire(&flow, allow_invalid_certs).map_err(Error::from)
|
||||
}
|
||||
|
||||
fn resolve_endpoints(config: &GraphImportConfig, client: &GraphClient) -> Result<Endpoints, Error> {
|
||||
@@ -374,6 +379,7 @@ fn spawn_token_refresher(
|
||||
client: &GraphClient,
|
||||
auth: &GraphAuth,
|
||||
initial: &AcquiredToken,
|
||||
allow_invalid_certs: bool,
|
||||
logger: crate::logging::Logger,
|
||||
) -> Option<TokenRefresher> {
|
||||
let (authority, client_id) = match auth {
|
||||
@@ -410,7 +416,12 @@ fn spawn_token_refresher(
|
||||
break;
|
||||
}
|
||||
}
|
||||
match refresh_access_token(&authority, &client_id, &refresh_token) {
|
||||
match refresh_access_token(
|
||||
&authority,
|
||||
&client_id,
|
||||
&refresh_token,
|
||||
allow_invalid_certs,
|
||||
) {
|
||||
Ok(tok) => {
|
||||
client.set_bearer(tok.access_token.clone());
|
||||
if let Some(new_refresh) = tok.refresh_token {
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -46,11 +45,7 @@ const EMAIL_TYPE: &str = "email";
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) struct RunOpts {
|
||||
source_id: i64,
|
||||
/// The folder generation this folder's fetch jobs carry; see
|
||||
/// `WorkerPool::cancel_before`.
|
||||
generation: u64,
|
||||
fetch_batch: usize,
|
||||
fetch_batch_bytes: u64,
|
||||
include_deleted: bool,
|
||||
logger: Logger,
|
||||
}
|
||||
@@ -200,8 +195,6 @@ pub struct ImapImportConfig {
|
||||
pub automap: bool,
|
||||
pub include_deleted: bool,
|
||||
pub fetch_batch: usize,
|
||||
/// Byte cap for one fetch chunk, by RFC822.SIZE; see `sync::batch`.
|
||||
pub fetch_batch_bytes: u64,
|
||||
pub imap_connections: usize,
|
||||
pub allow_source_change: bool,
|
||||
}
|
||||
@@ -405,9 +398,7 @@ fn run_into(
|
||||
|
||||
let opts = RunOpts {
|
||||
source_id,
|
||||
generation: 0,
|
||||
fetch_batch: config.fetch_batch.max(1),
|
||||
fetch_batch_bytes: config.fetch_batch_bytes.max(1),
|
||||
include_deleted: config.include_deleted,
|
||||
logger,
|
||||
};
|
||||
@@ -431,17 +422,13 @@ fn run_into(
|
||||
if i > 0 {
|
||||
let _ = control_run_collect(&mut client, &control_ctx, "NOOP");
|
||||
}
|
||||
// Each folder gets a new generation; anything still queued or in
|
||||
// flight for an earlier folder is skipped or dropped from here on.
|
||||
let generation = i as u64 + 1;
|
||||
pool.cancel_before(generation);
|
||||
match reconcile_folder(
|
||||
&mut conn,
|
||||
&mut client,
|
||||
&control_ctx,
|
||||
&pool,
|
||||
folder,
|
||||
RunOpts { generation, ..opts },
|
||||
opts,
|
||||
&mut email_counts,
|
||||
) {
|
||||
Ok(()) => {}
|
||||
@@ -708,9 +695,7 @@ fn reconcile_folder(
|
||||
) -> Result<(), Error> {
|
||||
let RunOpts {
|
||||
source_id,
|
||||
generation,
|
||||
fetch_batch,
|
||||
fetch_batch_bytes: _,
|
||||
include_deleted: _,
|
||||
logger,
|
||||
} = opts;
|
||||
@@ -816,17 +801,10 @@ fn reconcile_folder(
|
||||
folder.name
|
||||
))
|
||||
})?;
|
||||
let sizes = fetch_sizes(client, control_ctx, &folder.name, &diff.new, logger);
|
||||
let batches: Vec<&[u32]> = crate::sync::batch::by_count_and_bytes(
|
||||
&diff.new,
|
||||
|uid| sizes.get(uid).copied().unwrap_or(0),
|
||||
fetch_batch,
|
||||
opts.fetch_batch_bytes,
|
||||
);
|
||||
let batches: Vec<&[u32]> = chunks(&diff.new, fetch_batch);
|
||||
let n_batches = batches.len();
|
||||
for batch in &batches {
|
||||
pool.submit(FetchJob {
|
||||
generation,
|
||||
folder: folder.name.clone(),
|
||||
wire_name: folder.wire_name.clone(),
|
||||
uidvalidity,
|
||||
@@ -835,16 +813,12 @@ fn reconcile_folder(
|
||||
}
|
||||
let keepalive_interval = std::time::Duration::from_secs(45);
|
||||
let mut chunks_done: usize = 0;
|
||||
let tx = conn.transaction()?;
|
||||
let target = FetchTarget {
|
||||
folder: folder.name.as_str(),
|
||||
uidvalidity,
|
||||
mailbox_local,
|
||||
};
|
||||
// Committed after every chunk, not once per folder: each message is
|
||||
// written whole (see `insert_recording_failure`), so what is
|
||||
// committed is always consistent, a crash keeps it, and the next run
|
||||
// fetches only the UIDs that are still missing.
|
||||
let mut tx = conn.transaction()?;
|
||||
while chunks_done < n_batches {
|
||||
let event = loop {
|
||||
match pool.recv_timeout(keepalive_interval) {
|
||||
@@ -857,15 +831,15 @@ fn reconcile_folder(
|
||||
}
|
||||
}
|
||||
};
|
||||
match route_event(event, generation) {
|
||||
Routed::Stale => {}
|
||||
Routed::Item(attrs) => {
|
||||
insert_recording_failure(&mut tx, &target, &attrs, opts, counts)?;
|
||||
match event {
|
||||
FetchEvent::Item { attrs, .. } => {
|
||||
insert_single_message(&tx, &target, &attrs, opts, counts)?;
|
||||
}
|
||||
Routed::ChunkDone {
|
||||
FetchEvent::ChunkDone {
|
||||
folder: chunk_folder,
|
||||
uids_requested,
|
||||
outcome,
|
||||
uids_requested,
|
||||
..
|
||||
} => {
|
||||
chunks_done += 1;
|
||||
if let Err(e) = outcome {
|
||||
@@ -876,8 +850,6 @@ fn reconcile_folder(
|
||||
);
|
||||
counts.failed += uids_requested.len() as u64;
|
||||
}
|
||||
tx.commit()?;
|
||||
tx = conn.transaction()?;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -903,47 +875,6 @@ fn reconcile_folder(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RFC822.SIZE for each of `uids`, fetched on the control connection in
|
||||
/// large metadata-only chunks, so the body fetch can be split by bytes.
|
||||
/// Best effort: a server that won't answer just leaves the chunks sized by
|
||||
/// count.
|
||||
fn fetch_sizes(
|
||||
client: &mut ImapClient,
|
||||
ctx: &ControlCtx,
|
||||
folder: &str,
|
||||
uids: &[u32],
|
||||
logger: Logger,
|
||||
) -> HashMap<u32, u64> {
|
||||
const SIZE_CHUNK: usize = 1000;
|
||||
let mut sizes = HashMap::with_capacity(uids.len());
|
||||
for chunk in chunks(uids, SIZE_CHUNK) {
|
||||
let set = command::format_uid_set(chunk, true);
|
||||
let cmd = command::uid_fetch(&set, &["UID", "RFC822.SIZE"]);
|
||||
match call_with_retry(client, ctx, |c| c.run_collect(&cmd)) {
|
||||
Ok(resp) => {
|
||||
for u in &resp.untagged {
|
||||
if let Some(attrs) = fetch::extract(u)
|
||||
&& let (Some(uid), Some(size)) = (attrs.uid, attrs.size)
|
||||
{
|
||||
sizes.insert(uid, size);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
log_at(
|
||||
logger,
|
||||
LEVEL_DEFAULT,
|
||||
&format!(
|
||||
"folder {folder:?}: message sizes unavailable ({e}); fetching in chunks by count only"
|
||||
),
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
sizes
|
||||
}
|
||||
|
||||
fn wipe_folder_emails(
|
||||
conn: &mut Connection,
|
||||
source_id: i64,
|
||||
@@ -989,78 +920,6 @@ fn delete_vanished_emails(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) enum Routed {
|
||||
/// From an earlier folder generation: dropped, never filed here.
|
||||
Stale,
|
||||
Item(fetch::FetchAttrs),
|
||||
ChunkDone {
|
||||
folder: String,
|
||||
uids_requested: Vec<u32>,
|
||||
outcome: Result<(), ImapError>,
|
||||
},
|
||||
}
|
||||
|
||||
pub(super) fn route_event(event: FetchEvent, generation: u64) -> Routed {
|
||||
match event {
|
||||
FetchEvent::Item {
|
||||
generation: g,
|
||||
attrs,
|
||||
..
|
||||
} if g == generation => Routed::Item(attrs),
|
||||
FetchEvent::ChunkDone {
|
||||
generation: g,
|
||||
folder,
|
||||
uids_requested,
|
||||
outcome,
|
||||
..
|
||||
} if g == generation => Routed::ChunkDone {
|
||||
folder,
|
||||
uids_requested,
|
||||
outcome,
|
||||
},
|
||||
_ => Routed::Stale,
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes one message inside its own savepoint. A message that cannot be
|
||||
/// imported -- an INTERNALDATE that will not parse, say -- is rolled back
|
||||
/// on its own, logged with its folder and UID, counted as failed, and the
|
||||
/// folder carries on; it stays out of the UID map, so the next run tries
|
||||
/// it again. Archive and I/O errors still stop the run.
|
||||
fn insert_recording_failure(
|
||||
tx: &mut rusqlite::Transaction<'_>,
|
||||
target: &FetchTarget<'_>,
|
||||
attrs: &fetch::FetchAttrs,
|
||||
opts: RunOpts,
|
||||
counts: &mut TypeCounts,
|
||||
) -> Result<(), Error> {
|
||||
let sp = tx.savepoint()?;
|
||||
match insert_single_message(&sp, target, attrs, opts, counts) {
|
||||
Ok(()) => {
|
||||
sp.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
Err(e) if e.aborts_run() => Err(e),
|
||||
Err(e) => {
|
||||
drop(sp);
|
||||
log_at(
|
||||
opts.logger,
|
||||
LEVEL_DEFAULT,
|
||||
&format!(
|
||||
"folder {:?} uid {}: not imported: {e}",
|
||||
target.folder,
|
||||
attrs
|
||||
.uid
|
||||
.map(|u| u.to_string())
|
||||
.unwrap_or_else(|| "?".to_owned())
|
||||
),
|
||||
);
|
||||
counts.failed += 1;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct FetchTarget<'a> {
|
||||
pub folder: &'a str,
|
||||
pub uidvalidity: u32,
|
||||
@@ -1068,7 +927,7 @@ pub(super) struct FetchTarget<'a> {
|
||||
}
|
||||
|
||||
fn insert_single_message(
|
||||
tx: &Connection,
|
||||
tx: &rusqlite::Transaction<'_>,
|
||||
target: &FetchTarget<'_>,
|
||||
attrs: &fetch::FetchAttrs,
|
||||
opts: RunOpts,
|
||||
@@ -1076,9 +935,7 @@ fn insert_single_message(
|
||||
) -> Result<(), Error> {
|
||||
let RunOpts {
|
||||
source_id,
|
||||
generation: _,
|
||||
fetch_batch: _,
|
||||
fetch_batch_bytes: _,
|
||||
include_deleted,
|
||||
logger,
|
||||
} = opts;
|
||||
@@ -1155,9 +1012,7 @@ fn refresh_present_flags(
|
||||
) -> Result<u64, Error> {
|
||||
let RunOpts {
|
||||
source_id,
|
||||
generation: _,
|
||||
fetch_batch,
|
||||
fetch_batch_bytes: _,
|
||||
include_deleted,
|
||||
logger: _,
|
||||
} = opts;
|
||||
@@ -1382,36 +1237,6 @@ fn dry_run_summary(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn item(generation: u64, uid: u32) -> FetchEvent {
|
||||
FetchEvent::Item {
|
||||
generation,
|
||||
folder: "F".to_owned(),
|
||||
uidvalidity: 1,
|
||||
attrs: fetch::FetchAttrs {
|
||||
uid: Some(uid),
|
||||
..Default::default()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn events_from_another_generation_are_dropped() {
|
||||
assert!(matches!(route_event(item(1, 5), 2), Routed::Stale));
|
||||
assert!(matches!(route_event(item(3, 5), 2), Routed::Stale));
|
||||
match route_event(item(2, 5), 2) {
|
||||
Routed::Item(attrs) => assert_eq!(attrs.uid, Some(5)),
|
||||
_ => panic!("current-generation item was not routed"),
|
||||
}
|
||||
let stale_done = FetchEvent::ChunkDone {
|
||||
generation: 1,
|
||||
folder: "Old".to_owned(),
|
||||
uidvalidity: 1,
|
||||
uids_requested: vec![5],
|
||||
outcome: Ok(()),
|
||||
};
|
||||
assert!(matches!(route_event(stale_done, 2), Routed::Stale));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_endpoint_imaps_defaults_to_993() {
|
||||
let e = parse_endpoint("imaps://mail.example.com").unwrap();
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -59,22 +58,20 @@ pub fn imap_internaldate_to_rfc3339(s: &str) -> Result<String, Error> {
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
// RFC 3501 spells the month "Jan", but servers are not all that careful,
|
||||
// and a date is not worth losing a message over: match any case.
|
||||
fn month_to_num(s: &str) -> Result<u32, Error> {
|
||||
let m = match s.to_ascii_lowercase().as_str() {
|
||||
"jan" => 1,
|
||||
"feb" => 2,
|
||||
"mar" => 3,
|
||||
"apr" => 4,
|
||||
"may" => 5,
|
||||
"jun" => 6,
|
||||
"jul" => 7,
|
||||
"aug" => 8,
|
||||
"sep" => 9,
|
||||
"oct" => 10,
|
||||
"nov" => 11,
|
||||
"dec" => 12,
|
||||
let m = match s {
|
||||
"Jan" => 1,
|
||||
"Feb" => 2,
|
||||
"Mar" => 3,
|
||||
"Apr" => 4,
|
||||
"May" => 5,
|
||||
"Jun" => 6,
|
||||
"Jul" => 7,
|
||||
"Aug" => 8,
|
||||
"Sep" => 9,
|
||||
"Oct" => 10,
|
||||
"Nov" => 11,
|
||||
"Dec" => 12,
|
||||
other => return Err(Error::Partial(format!("INTERNALDATE month {other:?}"))),
|
||||
};
|
||||
Ok(m)
|
||||
@@ -102,22 +99,6 @@ fn parse_zone(s: &str) -> Result<(char, u32, u32), Error> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn month_matches_any_case() {
|
||||
for d in [
|
||||
"12-May-2025 10:00:00 +0000",
|
||||
"12-may-2025 10:00:00 +0000",
|
||||
"12-MAY-2025 10:00:00 +0000",
|
||||
] {
|
||||
assert_eq!(
|
||||
imap_internaldate_to_rfc3339(d).unwrap(),
|
||||
"2025-05-12T10:00:00Z",
|
||||
"{d}"
|
||||
);
|
||||
}
|
||||
assert!(imap_internaldate_to_rfc3339("12-Mai-2025 10:00:00 +0000").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utc_zone_becomes_z() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -1,16 +1,14 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
use std::panic::{AssertUnwindSafe, catch_unwind};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::thread;
|
||||
|
||||
use crossbeam_channel::{Receiver, Sender, bounded, unbounded};
|
||||
use crossbeam_channel::{Receiver, Sender, unbounded};
|
||||
|
||||
use crate::imap::client::{ConnectMode, ImapClient};
|
||||
use crate::imap::command;
|
||||
@@ -25,12 +23,7 @@ use super::fetch::FetchAttrs;
|
||||
|
||||
pub const HARD_CAP: usize = 8;
|
||||
|
||||
/// A job or event from an older folder generation than the one the
|
||||
/// coordinator is working on belongs to a folder it has already given up
|
||||
/// on. Workers skip such jobs without fetching, and the coordinator drops
|
||||
/// such events, so nothing from one folder can be filed into the next.
|
||||
pub struct FetchJob {
|
||||
pub generation: u64,
|
||||
pub folder: String,
|
||||
pub wire_name: String,
|
||||
pub uidvalidity: u32,
|
||||
@@ -39,13 +32,11 @@ pub struct FetchJob {
|
||||
|
||||
pub enum FetchEvent {
|
||||
Item {
|
||||
generation: u64,
|
||||
folder: String,
|
||||
uidvalidity: u32,
|
||||
attrs: FetchAttrs,
|
||||
},
|
||||
ChunkDone {
|
||||
generation: u64,
|
||||
folder: String,
|
||||
uidvalidity: u32,
|
||||
uids_requested: Vec<u32>,
|
||||
@@ -68,29 +59,22 @@ pub struct WorkerPool {
|
||||
job_tx: Sender<FetchJob>,
|
||||
event_rx: Receiver<FetchEvent>,
|
||||
handles: Vec<thread::JoinHandle<()>>,
|
||||
cancel_below: Arc<AtomicU64>,
|
||||
}
|
||||
|
||||
impl WorkerPool {
|
||||
pub fn start(args: WorkerArgs, pool_size: usize) -> Result<WorkerPool, ImapError> {
|
||||
let size = pool_size.clamp(1, HARD_CAP);
|
||||
let (job_tx, job_rx) = unbounded::<FetchJob>();
|
||||
// Jobs are only lists of UIDs, but each event carries a whole message:
|
||||
// bounding the events stops fast workers from running ahead of the
|
||||
// single archive writer, so memory holds at most a couple of messages
|
||||
// per worker rather than whole folders.
|
||||
let (event_tx, event_rx) = bounded::<FetchEvent>(size * 2);
|
||||
let (event_tx, event_rx) = unbounded::<FetchEvent>();
|
||||
let mut handles = Vec::with_capacity(size);
|
||||
let args = Arc::new(args);
|
||||
let cancel_below = Arc::new(AtomicU64::new(0));
|
||||
|
||||
for _ in 0..size {
|
||||
let args = args.clone();
|
||||
let job_rx = job_rx.clone();
|
||||
let event_tx = event_tx.clone();
|
||||
let cancel_below = cancel_below.clone();
|
||||
let handle = thread::spawn(move || {
|
||||
worker_loop(args, job_rx, event_tx, cancel_below);
|
||||
worker_loop(args, job_rx, event_tx);
|
||||
});
|
||||
handles.push(handle);
|
||||
}
|
||||
@@ -99,17 +83,9 @@ impl WorkerPool {
|
||||
job_tx,
|
||||
event_rx,
|
||||
handles,
|
||||
cancel_below,
|
||||
})
|
||||
}
|
||||
|
||||
/// Jobs of any generation below `generation` are skipped from now on:
|
||||
/// called when the coordinator moves to a new folder, so work still
|
||||
/// queued for one it abandoned is not fetched.
|
||||
pub fn cancel_before(&self, generation: u64) {
|
||||
self.cancel_below.fetch_max(generation, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn submit(&self, job: FetchJob) {
|
||||
let _ = self.job_tx.send(job);
|
||||
}
|
||||
@@ -125,43 +101,21 @@ impl WorkerPool {
|
||||
self.event_rx.recv_timeout(timeout)
|
||||
}
|
||||
|
||||
/// Stops the workers. Events still in flight are drained and dropped
|
||||
/// first: a worker blocked handing over an event the coordinator will
|
||||
/// never read (after a folder was abandoned) would otherwise never
|
||||
/// finish, and joining it would hang.
|
||||
pub fn shutdown(self) {
|
||||
self.cancel_before(u64::MAX);
|
||||
drop(self.job_tx);
|
||||
while self.event_rx.recv().is_ok() {}
|
||||
for h in self.handles {
|
||||
let _ = h.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn worker_loop(
|
||||
args: Arc<WorkerArgs>,
|
||||
job_rx: Receiver<FetchJob>,
|
||||
event_tx: Sender<FetchEvent>,
|
||||
cancel_below: Arc<AtomicU64>,
|
||||
) {
|
||||
fn worker_loop(args: Arc<WorkerArgs>, job_rx: Receiver<FetchJob>, event_tx: Sender<FetchEvent>) {
|
||||
let mut client: Option<ImapClient> = None;
|
||||
let mut current_folder: Option<String> = None;
|
||||
while let Ok(job) = job_rx.recv() {
|
||||
let job_gen = job.generation;
|
||||
let job_folder = job.folder.clone();
|
||||
let job_uv = job.uidvalidity;
|
||||
let job_uids = job.uids.clone();
|
||||
if job_gen < cancel_below.load(Ordering::SeqCst) {
|
||||
let _ = event_tx.send(FetchEvent::ChunkDone {
|
||||
generation: job_gen,
|
||||
folder: job_folder,
|
||||
uidvalidity: job_uv,
|
||||
uids_requested: job_uids,
|
||||
outcome: Err(ImapError::Protocol("cancelled: folder abandoned".into())),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
let event_tx_for_job = event_tx.clone();
|
||||
let outcome = match catch_unwind(AssertUnwindSafe(|| {
|
||||
run_job_with_retry(
|
||||
@@ -180,7 +134,6 @@ fn worker_loop(
|
||||
}
|
||||
};
|
||||
let _ = event_tx.send(FetchEvent::ChunkDone {
|
||||
generation: job_gen,
|
||||
folder: job_folder,
|
||||
uidvalidity: job_uv,
|
||||
uids_requested: job_uids,
|
||||
@@ -282,7 +235,6 @@ fn run_one_job(
|
||||
*current_folder = Some(job.folder.clone());
|
||||
}
|
||||
let set = command::format_uid_set(&job.uids, true);
|
||||
let generation = job.generation;
|
||||
let folder = job.folder.clone();
|
||||
let uv = job.uidvalidity;
|
||||
client.run_streamed(
|
||||
@@ -295,7 +247,6 @@ fn run_one_job(
|
||||
&& let Some(attrs) = super::fetch::extract(&u)
|
||||
{
|
||||
let _ = event_tx.send(FetchEvent::Item {
|
||||
generation,
|
||||
folder: folder.clone(),
|
||||
uidvalidity: uv,
|
||||
attrs,
|
||||
@@ -305,76 +256,3 @@ fn run_one_job(
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::Duration;
|
||||
|
||||
fn unreachable_args() -> WorkerArgs {
|
||||
WorkerArgs {
|
||||
connector: Arc::new(Connector::new(false).expect("connector")),
|
||||
endpoint: Arc::new(Endpoint {
|
||||
host: "127.0.0.1".to_owned(),
|
||||
port: 1,
|
||||
implicit_tls: false,
|
||||
}),
|
||||
mode: ConnectMode::Plain,
|
||||
auth: ImapAuth::Basic {
|
||||
user: "u".to_owned(),
|
||||
password: "p".to_owned(),
|
||||
},
|
||||
compress: false,
|
||||
policy: RetryPolicy::new(0),
|
||||
backoff: BackoffState::new(),
|
||||
logger: Logger::from_flags(false, 0),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_job_from_an_abandoned_generation_is_skipped_without_fetching() {
|
||||
// Port 1 refuses connections: a job that were actually run would come
|
||||
// back as a connection error, not as a cancellation.
|
||||
let pool = WorkerPool::start(unreachable_args(), 1).expect("pool");
|
||||
pool.cancel_before(2);
|
||||
pool.submit(FetchJob {
|
||||
generation: 1,
|
||||
folder: "Old".to_owned(),
|
||||
wire_name: "Old".to_owned(),
|
||||
uidvalidity: 7,
|
||||
uids: vec![1, 2, 3],
|
||||
});
|
||||
match pool.recv_timeout(Duration::from_secs(5)).expect("event") {
|
||||
FetchEvent::ChunkDone {
|
||||
generation,
|
||||
folder,
|
||||
uids_requested,
|
||||
outcome,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(generation, 1);
|
||||
assert_eq!(folder, "Old");
|
||||
assert_eq!(uids_requested, vec![1, 2, 3]);
|
||||
let err = outcome.expect_err("cancelled");
|
||||
assert!(err.to_string().contains("cancelled"), "{err}");
|
||||
}
|
||||
FetchEvent::Item { .. } => panic!("a cancelled job fetched something"),
|
||||
}
|
||||
pool.shutdown();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shutdown_returns_with_work_still_queued() {
|
||||
let pool = WorkerPool::start(unreachable_args(), 2).expect("pool");
|
||||
for g in 0..20u64 {
|
||||
pool.submit(FetchJob {
|
||||
generation: g,
|
||||
folder: format!("F{g}"),
|
||||
wire_name: format!("F{g}"),
|
||||
uidvalidity: 1,
|
||||
uids: vec![1],
|
||||
});
|
||||
}
|
||||
pool.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -141,7 +140,7 @@ pub struct InsertContext<'a> {
|
||||
}
|
||||
|
||||
pub fn insert_new(
|
||||
tx: &Connection,
|
||||
tx: &rusqlite::Transaction<'_>,
|
||||
ctx: InsertContext<'_>,
|
||||
entry: &DiskEntry,
|
||||
) -> Result<Option<i64>, InsertError> {
|
||||
@@ -269,10 +268,6 @@ pub fn delete_vanished(
|
||||
|
||||
const PROGRESS_TICK: u64 = 1000;
|
||||
|
||||
/// New messages are committed in groups of this many rather than once per
|
||||
/// folder, so an interrupted import of a large folder keeps what it wrote.
|
||||
const COMMIT_EVERY: u64 = 500;
|
||||
|
||||
pub fn apply_folder(
|
||||
conn: &mut Connection,
|
||||
ctx: InsertContext<'_>,
|
||||
@@ -282,23 +277,15 @@ pub fn apply_folder(
|
||||
) -> Result<(), crate::error::Error> {
|
||||
let stored_keywords =
|
||||
load_present_keywords(conn, ctx.source_id, ctx.folder).unwrap_or_default();
|
||||
let mut tx = conn.transaction()?;
|
||||
let tx = conn.transaction()?;
|
||||
let total_new = diff.new.len() as u64;
|
||||
let mut inserted: u64 = 0;
|
||||
for entry in &diff.new {
|
||||
// Each message in its own savepoint: one that fails part-way leaves
|
||||
// nothing behind, not an email row without its id mapping.
|
||||
let sp = tx.savepoint()?;
|
||||
match insert_new(&sp, ctx, entry) {
|
||||
match insert_new(&tx, ctx, entry) {
|
||||
Ok(Some(_)) => {
|
||||
sp.commit()?;
|
||||
counts.created += 1;
|
||||
counts.fetched += 1;
|
||||
inserted += 1;
|
||||
if inserted.is_multiple_of(COMMIT_EVERY) {
|
||||
tx.commit()?;
|
||||
tx = conn.transaction()?;
|
||||
}
|
||||
if inserted.is_multiple_of(PROGRESS_TICK)
|
||||
&& logger.enabled(crate::logging::LEVEL_PROGRESS)
|
||||
{
|
||||
@@ -309,11 +296,9 @@ pub fn apply_folder(
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
sp.commit()?;
|
||||
counts.skipped += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
drop(sp);
|
||||
logger.warn(&format!(
|
||||
"maildir {folder:?}/{name}: {e}",
|
||||
folder = ctx.folder,
|
||||
@@ -438,43 +423,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_message_that_cannot_be_read_is_recorded_and_the_rest_are_imported() {
|
||||
let td = tempfile::tempdir().unwrap();
|
||||
ensure_folder_skel(td.path());
|
||||
write_maildir_message(td.path(), "cur", "1.M0.host:2,S", b"Subject: a\r\n\r\na");
|
||||
let gone = write_maildir_message(td.path(), "cur", "2.M0.host:2,S", b"Subject: b\r\n\r\nb");
|
||||
write_maildir_message(td.path(), "cur", "3.M0.host:2,S", b"Subject: c\r\n\r\nc");
|
||||
let listing = list_folder(td.path()).unwrap();
|
||||
// Gone between the listing and the read, as a file being moved is.
|
||||
fs::remove_file(gone).unwrap();
|
||||
let (mut c, sid) = fresh_archive();
|
||||
let d = diff(listing.entries, &HashMap::new());
|
||||
let ctx = InsertContext {
|
||||
source_id: sid,
|
||||
folder: "INBOX",
|
||||
mailbox_local: 1,
|
||||
include_deleted: false,
|
||||
};
|
||||
let mut counts = TypeCounts::default();
|
||||
apply_folder(
|
||||
&mut c,
|
||||
ctx,
|
||||
d,
|
||||
&mut counts,
|
||||
crate::logging::Logger::from_flags(false, 0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!((counts.created, counts.failed), (2, 1));
|
||||
let emails: i64 = c
|
||||
.query_row("SELECT COUNT(*) FROM emails", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
let mapped: i64 = c
|
||||
.query_row("SELECT COUNT(*) FROM sync_id_maildir", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!((emails, mapped), (2, 2), "no email row without its mapping");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_folder_returns_cur_and_new_skipping_tmp() {
|
||||
let td = tempfile::tempdir().unwrap();
|
||||
|
||||
+1
-5
@@ -1,11 +1,9 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
pub mod batch;
|
||||
pub mod emailmeta;
|
||||
pub mod export;
|
||||
pub mod import_dav;
|
||||
@@ -17,7 +15,6 @@ pub mod import_maildir;
|
||||
pub mod import_managesieve;
|
||||
pub mod import_takeout;
|
||||
pub mod keys;
|
||||
pub mod progress;
|
||||
pub mod prune;
|
||||
|
||||
use std::path::PathBuf;
|
||||
@@ -44,7 +41,6 @@ pub(crate) fn table_name(ty: ObjectType) -> &'static str {
|
||||
use crate::jmap::account::AccountSelector;
|
||||
use crate::jmap::http::{Auth, HttpClient, RetryPolicy};
|
||||
use crate::logging::Logger;
|
||||
use crate::net::CertOverride;
|
||||
use crate::types::ObjectType;
|
||||
|
||||
pub struct CommonConfig {
|
||||
@@ -138,7 +134,7 @@ impl Context {
|
||||
let client = HttpClient::new(
|
||||
connect.auth.clone(),
|
||||
RetryPolicy::new(common.max_retries),
|
||||
CertOverride::for_url(common.allow_invalid_certs, &connect.url),
|
||||
common.allow_invalid_certs,
|
||||
);
|
||||
Ok(Context {
|
||||
conn,
|
||||
|
||||
@@ -1,196 +0,0 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
//! A progress line for long runs: how many of how many, how fast, and about
|
||||
//! how long is left. Printed to stderr at the default log level, at most once
|
||||
//! per interval, so a large export shows it is moving without flooding the
|
||||
//! terminal.
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::logging::{LEVEL_DEFAULT, Logger};
|
||||
|
||||
/// How often a progress line is printed while work continues.
|
||||
pub const PROGRESS_INTERVAL: Duration = Duration::from_secs(5);
|
||||
|
||||
pub struct Progress {
|
||||
label: String,
|
||||
total: u64,
|
||||
done: u64,
|
||||
started: Instant,
|
||||
last: Instant,
|
||||
interval: Duration,
|
||||
enabled: bool,
|
||||
}
|
||||
|
||||
impl Progress {
|
||||
/// `label` names the work, e.g. "export: Email". Nothing is printed when
|
||||
/// the logger is quiet or there is nothing to do.
|
||||
pub fn new(label: impl Into<String>, total: u64, logger: &Logger) -> Progress {
|
||||
let now = Instant::now();
|
||||
Progress {
|
||||
label: label.into(),
|
||||
total,
|
||||
done: 0,
|
||||
started: now,
|
||||
last: now,
|
||||
interval: PROGRESS_INTERVAL,
|
||||
enabled: logger.enabled(LEVEL_DEFAULT) && total > 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Records `n` more items done, and prints a line once the interval has
|
||||
/// passed since the last one.
|
||||
pub fn add(&mut self, n: u64) {
|
||||
self.done = (self.done + n).min(self.total);
|
||||
if !self.enabled {
|
||||
return;
|
||||
}
|
||||
let now = Instant::now();
|
||||
if now.duration_since(self.last) >= self.interval {
|
||||
self.last = now;
|
||||
eprintln!("{}", self.line(now.duration_since(self.started)));
|
||||
}
|
||||
}
|
||||
|
||||
fn line(&self, elapsed: Duration) -> String {
|
||||
progress_line(&self.label, self.done, self.total, elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
/// `export: Email 1,200/5,000 (24%), 40/s, about 1m35s left`. The rate and
|
||||
/// the time left are left out until there is enough to estimate them from.
|
||||
pub fn progress_line(label: &str, done: u64, total: u64, elapsed: Duration) -> String {
|
||||
let pct = (done * 100).checked_div(total).unwrap_or(100);
|
||||
let mut out = format!("{label} {}/{} ({pct}%)", thousands(done), thousands(total));
|
||||
let secs = elapsed.as_secs_f64();
|
||||
if done > 0 && secs >= 1.0 {
|
||||
let rate = done as f64 / secs;
|
||||
out.push_str(&format!(", {}/s", format_rate(rate)));
|
||||
let left = (total - done) as f64 / rate;
|
||||
if total > done && left.is_finite() {
|
||||
out.push_str(&format!(
|
||||
", about {} left",
|
||||
format_duration(Duration::from_secs_f64(left))
|
||||
));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// A duration as `45s`, `1m35s` or `2h03m`.
|
||||
pub fn format_duration(d: Duration) -> String {
|
||||
let total = d.as_secs();
|
||||
if total < 60 {
|
||||
format!("{total}s")
|
||||
} else if total < 3600 {
|
||||
format!("{}m{:02}s", total / 60, total % 60)
|
||||
} else {
|
||||
format!("{}h{:02}m", total / 3600, (total % 3600) / 60)
|
||||
}
|
||||
}
|
||||
|
||||
/// The line printed when a type is finished: `export: Email done: 120
|
||||
/// created, 3 updated, 5,000 unchanged, 0 failed (2m03s)`.
|
||||
pub fn done_line(label: &str, counts: &crate::sync::TypeCounts, elapsed: Duration) -> String {
|
||||
format!(
|
||||
"{label} done: {} created, {} updated, {} unchanged, {} failed ({})",
|
||||
thousands(counts.created),
|
||||
thousands(counts.updated),
|
||||
thousands(counts.skipped),
|
||||
thousands(counts.failed),
|
||||
format_duration(elapsed)
|
||||
)
|
||||
}
|
||||
|
||||
fn format_rate(rate: f64) -> String {
|
||||
if rate >= 10.0 {
|
||||
format!("{:.0}", rate)
|
||||
} else {
|
||||
format!("{:.1}", rate)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn thousands(n: u64) -> String {
|
||||
let digits = n.to_string();
|
||||
let mut out = String::with_capacity(digits.len() + digits.len() / 3);
|
||||
for (i, c) in digits.chars().enumerate() {
|
||||
if i > 0 && (digits.len() - i).is_multiple_of(3) {
|
||||
out.push(',');
|
||||
}
|
||||
out.push(c);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_line_has_count_rate_and_time_left() {
|
||||
let line = progress_line("export: Email", 1200, 5000, Duration::from_secs(30));
|
||||
assert!(
|
||||
line.starts_with("export: Email 1,200/5,000 (24%)"),
|
||||
"{line}"
|
||||
);
|
||||
assert!(line.contains("40/s"), "{line}");
|
||||
assert!(line.contains("about 1m35s left"), "{line}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_estimate_before_there_is_something_to_estimate_from() {
|
||||
assert_eq!(
|
||||
progress_line("export: Email", 0, 10, Duration::from_secs(5)),
|
||||
"export: Email 0/10 (0%)"
|
||||
);
|
||||
assert_eq!(
|
||||
progress_line("export: Email", 3, 10, Duration::from_millis(200)),
|
||||
"export: Email 3/10 (30%)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_finished_run_shows_no_time_left() {
|
||||
let line = progress_line("export: Email", 10, 10, Duration::from_secs(4));
|
||||
assert!(!line.contains("left"), "{line}");
|
||||
assert!(line.contains("(100%)"), "{line}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn durations_and_counts_read_naturally() {
|
||||
assert_eq!(format_duration(Duration::from_secs(45)), "45s");
|
||||
assert_eq!(format_duration(Duration::from_secs(95)), "1m35s");
|
||||
assert_eq!(format_duration(Duration::from_secs(7380)), "2h03m");
|
||||
assert_eq!(thousands(0), "0");
|
||||
assert_eq!(thousands(999), "999");
|
||||
assert_eq!(thousands(1_000), "1,000");
|
||||
assert_eq!(thousands(1_234_567), "1,234,567");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_done_line_names_every_count() {
|
||||
let counts = crate::sync::TypeCounts {
|
||||
created: 120,
|
||||
updated: 3,
|
||||
skipped: 5000,
|
||||
failed: 1,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
done_line("export: Email", &counts, Duration::from_secs(123)),
|
||||
"export: Email done: 120 created, 3 updated, 5,000 unchanged, 1 failed (2m03s)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_quiet_logger_or_empty_job_prints_nothing() {
|
||||
let p = Progress::new("x", 10, &Logger::from_flags(true, 0));
|
||||
assert!(!p.enabled);
|
||||
let p = Progress::new("x", 0, &Logger::from_flags(false, 0));
|
||||
assert!(!p.enabled);
|
||||
}
|
||||
}
|
||||
@@ -38,7 +38,6 @@ fn imap_config(account: &Account, imap: &Endpoint) -> ImapImportConfig {
|
||||
automap: true,
|
||||
include_deleted: false,
|
||||
fetch_batch: 64,
|
||||
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
|
||||
imap_connections: 2,
|
||||
allow_source_change: false,
|
||||
}
|
||||
|
||||
@@ -43,7 +43,6 @@ fn imap_config(account: &Account, imap: &integration::Endpoint) -> ImapImportCon
|
||||
automap: true,
|
||||
include_deleted: false,
|
||||
fetch_batch: 64,
|
||||
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
|
||||
imap_connections: 2,
|
||||
allow_source_change: false,
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ mod seeder;
|
||||
use inbuxa_migrate::jmap::account::{self, AccountSelector};
|
||||
use inbuxa_migrate::jmap::http::{Auth, HttpClient, RetryPolicy};
|
||||
use inbuxa_migrate::jmap::session::Session;
|
||||
use inbuxa_migrate::net::CertOverride;
|
||||
use integration::stalwart::shared as shared_stalwart;
|
||||
|
||||
fn admin_client() -> HttpClient {
|
||||
@@ -21,7 +20,7 @@ fn admin_client() -> HttpClient {
|
||||
password: seeder::ADMIN_PASSWORD.into(),
|
||||
},
|
||||
RetryPolicy::new(5),
|
||||
CertOverride::for_url(true, shared_stalwart().base_url()),
|
||||
true,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
+1
-2
@@ -13,7 +13,6 @@ use inbuxa_migrate::dav::parse::{parse_multistatus, strip_ascii_control_chars};
|
||||
use inbuxa_migrate::dav::xml;
|
||||
use inbuxa_migrate::jmap::error::JmapError;
|
||||
use inbuxa_migrate::jmap::http::{Auth, RetryPolicy};
|
||||
use inbuxa_migrate::net::CertOverride;
|
||||
|
||||
fn client(retries: u32) -> DavClient {
|
||||
DavClient::new(
|
||||
@@ -22,7 +21,7 @@ fn client(retries: u32) -> DavClient {
|
||||
password: "p".into(),
|
||||
},
|
||||
RetryPolicy::new(retries),
|
||||
CertOverride::none(),
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -22,7 +22,6 @@ use inbuxa_migrate::exchange_ews::xml::{
|
||||
get_item_body, sync_folder_items_body,
|
||||
};
|
||||
use inbuxa_migrate::jmap::http::{Auth, RetryPolicy};
|
||||
use inbuxa_migrate::net::CertOverride;
|
||||
use mockito::Matcher;
|
||||
|
||||
const TXT_XML: &str = "text/xml; charset=utf-8";
|
||||
@@ -34,7 +33,7 @@ fn client(retries: u32) -> EwsClient {
|
||||
token: "t".to_owned(),
|
||||
},
|
||||
RetryPolicy::new(retries),
|
||||
CertOverride::none(),
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -64,7 +63,7 @@ fn autodiscover_v2_returns_global_endpoint() {
|
||||
let _ = server;
|
||||
let url = "https://outlook.office365.com/EWS/Exchange.asmx";
|
||||
assert!(inbuxa_migrate::exchange_ews::autodiscover::is_fully_qualified_ews_url(url));
|
||||
let r = discover(Some(url), None, None, &CertOverride::none()).unwrap();
|
||||
let r = discover(Some(url), None, None, false).unwrap();
|
||||
assert_eq!(r.source, DiscoverySource::SuppliedUrl);
|
||||
assert_eq!(r.ews_url, url);
|
||||
}
|
||||
@@ -864,7 +863,6 @@ fn for_each_fetched_item_streams_every_id_across_windows() {
|
||||
url: &url,
|
||||
source_id: 1,
|
||||
batch_size: 1,
|
||||
batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
|
||||
attachment_batch: 1,
|
||||
connections: 2,
|
||||
use_syncfolderitems: false,
|
||||
@@ -874,8 +872,7 @@ fn for_each_fetched_item_streams_every_id_across_windows() {
|
||||
let ids: Vec<ItemId> = (0..5).map(|i| ItemId::new(format!("I{i}"), "K")).collect();
|
||||
|
||||
let mut delivered = 0usize;
|
||||
let failed =
|
||||
for_each_fetched_item(&ctx, ItemShape::Message, &ids, &Default::default(), |msg| {
|
||||
let failed = for_each_fetched_item(&ctx, ItemShape::Message, &ids, |msg| {
|
||||
assert!(msg.success);
|
||||
delivered += 1;
|
||||
Ok(())
|
||||
@@ -887,73 +884,6 @@ fn for_each_fetched_item_streams_every_id_across_windows() {
|
||||
_m.assert();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn getitem_batches_are_split_by_bytes_when_sizes_are_known() {
|
||||
use inbuxa_migrate::logging::Logger;
|
||||
use inbuxa_migrate::sync::import_exchange_ews::items::{ItemRunCtx, for_each_fetched_item};
|
||||
use std::collections::HashMap;
|
||||
|
||||
let mut server = mockito::Server::new();
|
||||
let url = format!("{}/EWS/Exchange.asmx", server.url());
|
||||
let one_message = envelope(&format!(
|
||||
"<m:GetItemResponse{NS}><m:ResponseMessages><m:GetItemResponseMessage ResponseClass=\"Success\">\
|
||||
<m:ResponseCode>NoError</m:ResponseCode><m:Items><t:Message><t:ItemId Id=\"X\" ChangeKey=\"K\"/></t:Message></m:Items>\
|
||||
</m:GetItemResponseMessage></m:ResponseMessages></m:GetItemResponse>"
|
||||
));
|
||||
// Ten items fit one batch by count, but at 20 bytes each and a 30-byte
|
||||
// cap every item goes alone: four GetItem calls, not one.
|
||||
let m = server
|
||||
.mock("POST", "/EWS/Exchange.asmx")
|
||||
.with_status(200)
|
||||
.with_header("content-type", TXT_XML)
|
||||
.with_body(&one_message)
|
||||
.expect(4)
|
||||
.create();
|
||||
let c = client(0);
|
||||
let ctx = ItemRunCtx {
|
||||
client: &c,
|
||||
url: &url,
|
||||
source_id: 1,
|
||||
batch_size: 10,
|
||||
batch_bytes: 30,
|
||||
attachment_batch: 1,
|
||||
connections: 2,
|
||||
use_syncfolderitems: false,
|
||||
sync_batch: 512,
|
||||
logger: Logger::new(0),
|
||||
};
|
||||
let ids: Vec<ItemId> = (0..4).map(|i| ItemId::new(format!("I{i}"), "K")).collect();
|
||||
let sizes: HashMap<String, u64> = ids.iter().map(|id| (id.id.clone(), 20)).collect();
|
||||
let mut delivered = 0usize;
|
||||
for_each_fetched_item(&ctx, ItemShape::Message, &ids, &sizes, |_| {
|
||||
delivered += 1;
|
||||
Ok(())
|
||||
})
|
||||
.expect("fetch");
|
||||
assert_eq!(delivered, 4);
|
||||
m.assert();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_item_reports_each_items_size() {
|
||||
use inbuxa_migrate::exchange_ews::parse::parse_find_item_response;
|
||||
let body = envelope(&format!(
|
||||
"<m:FindItemResponse{NS}><m:ResponseMessages><m:FindItemResponseMessage ResponseClass=\"Success\">\
|
||||
<m:ResponseCode>NoError</m:ResponseCode>\
|
||||
<m:RootFolder TotalItemsInView=\"2\" IncludesLastItemInRange=\"true\"><t:Items>\
|
||||
<t:Message><t:ItemId Id=\"A\" ChangeKey=\"K\"/><t:Size>1234</t:Size></t:Message>\
|
||||
<t:Message><t:ItemId Id=\"B\" ChangeKey=\"K\"/></t:Message>\
|
||||
</t:Items></m:RootFolder></m:FindItemResponseMessage></m:ResponseMessages></m:FindItemResponse>"
|
||||
));
|
||||
let r = parse_find_item_response(body.as_bytes()).unwrap();
|
||||
assert_eq!(r.items.len(), 2);
|
||||
assert_eq!(
|
||||
(r.items[0].id.id.as_str(), r.items[0].size),
|
||||
("A", Some(1234))
|
||||
);
|
||||
assert_eq!((r.items[1].id.id.as_str(), r.items[1].size), ("B", None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn warning_response_class_is_treated_as_success_in_mock() {
|
||||
let body = envelope(&format!(
|
||||
|
||||
@@ -21,7 +21,6 @@ use inbuxa_migrate::exchange_graph::recurrence::convert_patterned_recurrence;
|
||||
use inbuxa_migrate::exchange_graph::retry::{HttpClass, classify_http_status};
|
||||
use inbuxa_migrate::exchange_graph::types::Surfaces;
|
||||
use inbuxa_migrate::jmap::http::RetryPolicy;
|
||||
use inbuxa_migrate::net::CertOverride;
|
||||
use mockito::{Matcher, Server};
|
||||
use serde_json::json;
|
||||
|
||||
@@ -29,11 +28,7 @@ static INIT: Once = Once::new();
|
||||
|
||||
fn client_with_retries(retries: u32) -> GraphClient {
|
||||
INIT.call_once(|| {});
|
||||
GraphClient::new(
|
||||
"BEARER".to_owned(),
|
||||
RetryPolicy::new(retries),
|
||||
CertOverride::none(),
|
||||
)
|
||||
GraphClient::new("BEARER".to_owned(), RetryPolicy::new(retries), false)
|
||||
}
|
||||
|
||||
fn url_message_collection(server_url: &str, folder: &str, top: usize) -> String {
|
||||
@@ -1686,11 +1681,7 @@ fn graph_client_retries_after_401_when_bearer_is_swapped() {
|
||||
.expect(1)
|
||||
.create();
|
||||
let base = server.url();
|
||||
let client = GraphClient::new(
|
||||
"EXPIRED".to_owned(),
|
||||
RetryPolicy::new(0),
|
||||
CertOverride::none(),
|
||||
);
|
||||
let client = GraphClient::new("EXPIRED".to_owned(), RetryPolicy::new(0), false);
|
||||
let url = format!("{base}/me");
|
||||
let err = client.get(&url, Accept::Json).unwrap_err();
|
||||
assert!(matches!(err, GraphError::Auth(_)));
|
||||
|
||||
@@ -1,342 +0,0 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
//! Batched `Email/import` on export: batches honor the server's limits, one
|
||||
//! rejected message fails alone, and a batch that ends without a clear answer
|
||||
//! is settled against the target instead of being sent again.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use inbuxa_migrate::db;
|
||||
use inbuxa_migrate::jmap::account::AccountSelector;
|
||||
use inbuxa_migrate::jmap::http::Auth;
|
||||
use inbuxa_migrate::logging::Logger;
|
||||
use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, TypeCounts};
|
||||
use mockito::Matcher;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
const API: &str = "/jmap/api";
|
||||
|
||||
fn tmp() -> PathBuf {
|
||||
static SEQ: AtomicUsize = AtomicUsize::new(0);
|
||||
let n = SEQ.fetch_add(1, Ordering::Relaxed);
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"inbuxa-migrate-exportbatch-{}-{:?}-{n}.sqlite",
|
||||
std::process::id(),
|
||||
std::thread::current().id(),
|
||||
));
|
||||
let _ = std::fs::remove_file(&p);
|
||||
p
|
||||
}
|
||||
|
||||
fn session(base: &str, max_objects_in_set: u64, max_concurrent_upload: u64) -> String {
|
||||
json!({
|
||||
"apiUrl": format!("{base}{API}"),
|
||||
"uploadUrl": format!("{base}/jmap/upload/{{accountId}}/"),
|
||||
"downloadUrl": format!("{base}/jmap/dl/{{accountId}}/{{blobId}}/{{type}}/{{name}}"),
|
||||
"capabilities": { "urn:ietf:params:jmap:core": {
|
||||
"maxObjectsInGet": 500, "maxObjectsInSet": max_objects_in_set,
|
||||
"maxCallsInRequest": 16, "maxConcurrentRequests": 4,
|
||||
"maxConcurrentUpload": max_concurrent_upload,
|
||||
"maxSizeRequest": 10000000, "maxSizeUpload": 50000000
|
||||
} },
|
||||
"accounts": { "w": { "name": "alice",
|
||||
"accountCapabilities": { "urn:ietf:params:jmap:mail": {} } } }
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// An archive with one Inbox and `n` distinct messages, `<m-1@h>` .. `<m-n@h>`.
|
||||
fn seed(n: usize) -> PathBuf {
|
||||
let archive = tmp();
|
||||
let conn = db::init::open(&archive).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO mailboxes (id,name,parent_id,role) VALUES (1,'Inbox',NULL,'inbox')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
for i in 1..=n {
|
||||
let raw = format!("From: a@x\r\nSubject: m{i}\r\nMessage-ID: <m-{i}@h>\r\n\r\nbody {i}");
|
||||
let blob = db::blobs::intern_blob(&conn, raw.as_bytes()).unwrap();
|
||||
let mm = inbuxa_migrate::sync::keys::index_to_json(
|
||||
&inbuxa_migrate::sync::emailmeta::email_index_from_blob(raw.as_bytes()),
|
||||
);
|
||||
conn.execute(
|
||||
"INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords,message_match)
|
||||
VALUES (?1,'2020-01-01T00:00:00Z','[1]','[]',?2)",
|
||||
rusqlite::params![blob, mm],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
archive
|
||||
}
|
||||
|
||||
/// The session, an Inbox already on the target, and uploads. The target's
|
||||
/// email list is left to each test.
|
||||
fn mock_target(server: &mut mockito::ServerGuard, session_body: String) -> Vec<mockito::Mock> {
|
||||
vec![
|
||||
server.mock("GET", "/").with_status(404).create(),
|
||||
server
|
||||
.mock("GET", "/.well-known/jmap")
|
||||
.with_body(session_body)
|
||||
.create(),
|
||||
server
|
||||
.mock("POST", API)
|
||||
.match_body(Matcher::Regex("Mailbox/query".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Mailbox/query",
|
||||
{"accountId":"w","ids":["t1"]},"q"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.create(),
|
||||
server
|
||||
.mock("POST", API)
|
||||
.match_body(Matcher::AllOf(vec![
|
||||
Matcher::Regex("Mailbox/query".into()),
|
||||
Matcher::Regex("anchor".into()),
|
||||
]))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Mailbox/query",
|
||||
{"accountId":"w","ids":[]},"q"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.create(),
|
||||
server
|
||||
.mock("POST", API)
|
||||
.match_body(Matcher::Regex("Mailbox/get".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Mailbox/get",{"accountId":"w","list":[
|
||||
{"id":"t1","name":"Inbox","role":"inbox","parentId":null,
|
||||
"myRights":{"mayDelete":true}}],"notFound":[]},"g"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.create(),
|
||||
server
|
||||
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
|
||||
.with_body(json!({"blobId":"UP"}).to_string())
|
||||
.create(),
|
||||
]
|
||||
}
|
||||
|
||||
fn empty_email_query(server: &mut mockito::ServerGuard) -> mockito::Mock {
|
||||
server
|
||||
.mock("POST", API)
|
||||
.match_body(Matcher::Regex("Email/query".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Email/query",{"accountId":"w","ids":[]},"q"]]}).to_string(),
|
||||
)
|
||||
.create()
|
||||
}
|
||||
|
||||
/// The creation ids of the emails in an `Email/import` request body.
|
||||
fn import_cids(body: &[u8]) -> Vec<String> {
|
||||
let v: Value = serde_json::from_slice(body).unwrap_or(Value::Null);
|
||||
v["methodCalls"][0][1]["emails"]
|
||||
.as_object()
|
||||
.map(|m| m.keys().cloned().collect())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// An `Email/import` answer: each id in `cids` created, except those in
|
||||
/// `rejected`, which come back as `invalidEmail`.
|
||||
fn import_answer(cids: &[String], rejected: &[&str]) -> Vec<u8> {
|
||||
let mut created = serde_json::Map::new();
|
||||
let mut not_created = serde_json::Map::new();
|
||||
for cid in cids {
|
||||
if rejected.contains(&cid.as_str()) {
|
||||
not_created.insert(cid.clone(), json!({"type":"invalidEmail"}));
|
||||
} else {
|
||||
created.insert(
|
||||
cid.clone(),
|
||||
json!({"id": format!("T{cid}"), "blobId":"b","threadId":"t","size":10}),
|
||||
);
|
||||
}
|
||||
}
|
||||
json!({"methodResponses":[["Email/import",
|
||||
{"accountId":"w","created":created,"notCreated":not_created},"i"]]})
|
||||
.to_string()
|
||||
.into_bytes()
|
||||
}
|
||||
|
||||
fn run_export(archive: &Path, base: &str, threads: usize) -> TypeCounts {
|
||||
let summary = sync::export::run(
|
||||
CommonConfig {
|
||||
archive: archive.to_path_buf(),
|
||||
threads,
|
||||
dry_run: false,
|
||||
max_retries: 1,
|
||||
allow_invalid_certs: false,
|
||||
logger: Logger::from_flags(true, 0),
|
||||
},
|
||||
ExportConfig {
|
||||
connect: ConnectConfig {
|
||||
url: base.to_owned(),
|
||||
auth: Auth::Basic {
|
||||
user: "u".into(),
|
||||
password: "p".into(),
|
||||
},
|
||||
account: AccountSelector::Id("w".into()),
|
||||
},
|
||||
objects: None,
|
||||
prune: false,
|
||||
yes: true,
|
||||
},
|
||||
)
|
||||
.expect("export run");
|
||||
summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(t, _)| *t == "Email")
|
||||
.map(|(_, c)| c.clone())
|
||||
.expect("email counts")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn imports_are_batched_up_to_max_objects_in_set() {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let archive = seed(5);
|
||||
let _base_mocks = mock_target(&mut server, session(&base, 2, 4));
|
||||
let _eq = empty_email_query(&mut server);
|
||||
|
||||
let sizes = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let seen = sizes.clone();
|
||||
let imports = server
|
||||
.mock("POST", API)
|
||||
.match_body(Matcher::Regex("Email/import".into()))
|
||||
.with_body_from_request(move |req| {
|
||||
let cids = import_cids(req.body().unwrap());
|
||||
seen.lock().unwrap().push(cids.len());
|
||||
import_answer(&cids, &[])
|
||||
})
|
||||
.expect(3)
|
||||
.create();
|
||||
|
||||
let email = run_export(&archive, &base, 4);
|
||||
assert_eq!(email.created, 5);
|
||||
assert_eq!(email.failed, 0);
|
||||
imports.assert();
|
||||
let mut sizes = sizes.lock().unwrap().clone();
|
||||
sizes.sort_unstable();
|
||||
assert_eq!(
|
||||
sizes,
|
||||
vec![1, 2, 2],
|
||||
"no call carries more than maxObjectsInSet"
|
||||
);
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_rejected_message_in_a_batch_fails_alone() {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let archive = seed(3);
|
||||
let _base_mocks = mock_target(&mut server, session(&base, 50, 4));
|
||||
let _eq = empty_email_query(&mut server);
|
||||
|
||||
let rejected = Arc::new(std::sync::Mutex::new(String::new()));
|
||||
let pick = rejected.clone();
|
||||
let imports = server
|
||||
.mock("POST", API)
|
||||
.match_body(Matcher::Regex("Email/import".into()))
|
||||
.with_body_from_request(move |req| {
|
||||
let cids = import_cids(req.body().unwrap());
|
||||
let mut sorted = cids.clone();
|
||||
sorted.sort();
|
||||
let middle = sorted[1].clone();
|
||||
*pick.lock().unwrap() = middle.clone();
|
||||
import_answer(&cids, &[middle.as_str()])
|
||||
})
|
||||
.expect(1)
|
||||
.create();
|
||||
|
||||
let email = run_export(&archive, &base, 1);
|
||||
assert_eq!(email.created, 2, "the other two land");
|
||||
assert_eq!(email.failed, 1, "only {} fails", rejected.lock().unwrap());
|
||||
imports.assert();
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unclear_batch_is_settled_against_the_target_not_resent() {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let archive = seed(2);
|
||||
let _base_mocks = mock_target(&mut server, session(&base, 50, 4));
|
||||
|
||||
// First look: the target is empty. After the unclear batch: m-1 arrived.
|
||||
let queries = Arc::new(AtomicUsize::new(0));
|
||||
let q = queries.clone();
|
||||
let _eq = server
|
||||
.mock("POST", API)
|
||||
.match_body(Matcher::Regex("Email/query".into()))
|
||||
.with_body_from_request(move |req| {
|
||||
// A page after the first (it carries an anchor) is empty.
|
||||
let paged = String::from_utf8_lossy(req.body().unwrap()).contains("anchor");
|
||||
let ids = if paged || q.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||
json!([])
|
||||
} else {
|
||||
json!(["arrived"])
|
||||
};
|
||||
json!({"methodResponses":[["Email/query",{"accountId":"w","ids":ids},"q"]]})
|
||||
.to_string()
|
||||
.into_bytes()
|
||||
})
|
||||
.create();
|
||||
let _eg = server
|
||||
.mock("POST", API)
|
||||
.match_body(Matcher::Regex("Email/get".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Email/get",{"accountId":"w","list":[
|
||||
{"id":"arrived","messageId":["m-1@h"],"mailboxIds":{"t1":true},"keywords":{}}
|
||||
],"notFound":[]},"g"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.create();
|
||||
|
||||
// The batch carries both messages and gets a gateway timeout, so it may
|
||||
// or may not have been applied. mockito answers with the first matching
|
||||
// mock still owed hits, so this answers the first import only; every
|
||||
// later import is created by the next mock, which records what it sees.
|
||||
let gateway = server
|
||||
.mock("POST", API)
|
||||
.match_body(Matcher::Regex("Email/import".into()))
|
||||
.with_status(504)
|
||||
.expect(1)
|
||||
.create();
|
||||
let later = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let later_seen = later.clone();
|
||||
let _created = server
|
||||
.mock("POST", API)
|
||||
.match_body(Matcher::Regex("Email/import".into()))
|
||||
.with_body_from_request(move |req| {
|
||||
let cids = import_cids(req.body().unwrap());
|
||||
later_seen.lock().unwrap().push(cids.clone());
|
||||
import_answer(&cids, &[])
|
||||
})
|
||||
.create();
|
||||
|
||||
let email = run_export(&archive, &base, 1);
|
||||
gateway.assert();
|
||||
assert!(
|
||||
queries.load(Ordering::SeqCst) >= 2,
|
||||
"the target is read again before anything is resent"
|
||||
);
|
||||
assert_eq!(
|
||||
*later.lock().unwrap(),
|
||||
vec![vec!["e2".to_owned()]],
|
||||
"only the message that did not arrive is sent again, on its own"
|
||||
);
|
||||
assert_eq!(
|
||||
email.created, 2,
|
||||
"one found on the target, one imported again"
|
||||
);
|
||||
assert_eq!(email.failed, 0);
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
@@ -1,325 +0,0 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 John Coffey <johnellis@linux.com>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
//! `export --dry-run` predicts what a real run would fail on: a message
|
||||
//! larger than the target accepts, an object too big for one request, and a
|
||||
//! Sieve script the target rejects. It keeps the counts, so the run exits
|
||||
//! non-zero just as the real one would, and it writes nothing.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use inbuxa_migrate::db;
|
||||
use inbuxa_migrate::jmap::account::AccountSelector;
|
||||
use inbuxa_migrate::jmap::http::Auth;
|
||||
use inbuxa_migrate::logging::Logger;
|
||||
use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, Summary, TypeCounts};
|
||||
use mockito::Matcher;
|
||||
use serde_json::json;
|
||||
|
||||
const API: &str = "/jmap/api";
|
||||
|
||||
fn tmp() -> PathBuf {
|
||||
static SEQ: AtomicUsize = AtomicUsize::new(0);
|
||||
let n = SEQ.fetch_add(1, Ordering::Relaxed);
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"inbuxa-migrate-exportdry-{}-{:?}-{n}.sqlite",
|
||||
std::process::id(),
|
||||
std::thread::current().id(),
|
||||
));
|
||||
let _ = std::fs::remove_file(&p);
|
||||
p
|
||||
}
|
||||
|
||||
fn session(base: &str, max_size_upload: u64, max_size_request: u64, sieve: &[&str]) -> String {
|
||||
json!({
|
||||
"apiUrl": format!("{base}{API}"),
|
||||
"uploadUrl": format!("{base}/jmap/upload/{{accountId}}/"),
|
||||
"downloadUrl": format!("{base}/jmap/dl/{{accountId}}/{{blobId}}/{{type}}/{{name}}"),
|
||||
"capabilities": { "urn:ietf:params:jmap:core": {
|
||||
"maxObjectsInGet": 500, "maxObjectsInSet": 500, "maxCallsInRequest": 16,
|
||||
"maxConcurrentRequests": 4, "maxConcurrentUpload": 4,
|
||||
"maxSizeRequest": max_size_request, "maxSizeUpload": max_size_upload
|
||||
} },
|
||||
"accounts": { "w": { "name": "alice",
|
||||
"accountCapabilities": {
|
||||
"urn:ietf:params:jmap:mail": {},
|
||||
"urn:ietf:params:jmap:contacts": {},
|
||||
"urn:ietf:params:jmap:sieve": { "sieveExtensions": sieve }
|
||||
} } }
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn dry_run(archive: &Path, base: &str) -> Summary {
|
||||
sync::export::run(
|
||||
CommonConfig {
|
||||
archive: archive.to_path_buf(),
|
||||
threads: 2,
|
||||
dry_run: true,
|
||||
max_retries: 0,
|
||||
allow_invalid_certs: false,
|
||||
logger: Logger::from_flags(true, 0),
|
||||
},
|
||||
ExportConfig {
|
||||
connect: ConnectConfig {
|
||||
url: base.to_owned(),
|
||||
auth: Auth::Basic {
|
||||
user: "u".into(),
|
||||
password: "p".into(),
|
||||
},
|
||||
account: AccountSelector::Id("w".into()),
|
||||
},
|
||||
objects: None,
|
||||
prune: false,
|
||||
yes: true,
|
||||
},
|
||||
)
|
||||
.expect("a dry run returns its plan")
|
||||
}
|
||||
|
||||
fn counts(summary: &Summary, ty: &str) -> TypeCounts {
|
||||
summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(t, _)| *t == ty)
|
||||
.map(|(_, c)| c.clone())
|
||||
.unwrap_or_else(|| panic!("no counts for {ty}: {summary:?}"))
|
||||
}
|
||||
|
||||
fn root_and_session(server: &mut mockito::ServerGuard, body: String) -> Vec<mockito::Mock> {
|
||||
vec![
|
||||
server.mock("GET", "/").with_status(404).create(),
|
||||
server
|
||||
.mock("GET", "/.well-known/jmap")
|
||||
.with_body(body)
|
||||
.create(),
|
||||
]
|
||||
}
|
||||
|
||||
/// Nothing that writes to the account: no `/set`, no import. Returns the
|
||||
/// mocks, each expecting no calls.
|
||||
fn no_writes(server: &mut mockito::ServerGuard) -> Vec<mockito::Mock> {
|
||||
["/set\"", "Email/import"]
|
||||
.into_iter()
|
||||
.map(|m| {
|
||||
server
|
||||
.mock("POST", API)
|
||||
.match_body(Matcher::Regex(m.into()))
|
||||
.expect(0)
|
||||
.create()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn empty(
|
||||
server: &mut mockito::ServerGuard,
|
||||
method: &str,
|
||||
reply: serde_json::Value,
|
||||
) -> mockito::Mock {
|
||||
server
|
||||
.mock("POST", API)
|
||||
.match_body(Matcher::Regex(method.into()))
|
||||
.with_body(json!({ "methodResponses": [[method, reply, "x"]] }).to_string())
|
||||
.create()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_message_too_large_to_upload_is_predicted_to_fail() {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let archive = tmp();
|
||||
{
|
||||
let conn = db::init::open(&archive).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO mailboxes (id,name,parent_id,role) VALUES (1,'Inbox',NULL,'inbox')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
for body in ["short".to_owned(), "x".repeat(4000)] {
|
||||
let raw = format!(
|
||||
"From: a@x\r\nSubject: s\r\nMessage-ID: <{}@h>\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
let blob = db::blobs::intern_blob(&conn, raw.as_bytes()).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords)
|
||||
VALUES (?1,'2020-01-01T00:00:00Z','[1]','[]')",
|
||||
rusqlite::params![blob],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
let _s = root_and_session(&mut server, session(&base, 1000, 10_000_000, &[]));
|
||||
let _mq = empty(
|
||||
&mut server,
|
||||
"Mailbox/query",
|
||||
json!({"accountId":"w","ids":[]}),
|
||||
);
|
||||
let _eq = empty(
|
||||
&mut server,
|
||||
"Email/query",
|
||||
json!({"accountId":"w","ids":[]}),
|
||||
);
|
||||
let no_upload = server
|
||||
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
|
||||
.expect(0)
|
||||
.create();
|
||||
let writes = no_writes(&mut server);
|
||||
|
||||
let summary = dry_run(&archive, &base);
|
||||
let email = counts(&summary, "Email");
|
||||
assert_eq!(email.created, 1, "the short one would be created");
|
||||
assert_eq!(email.failed, 1, "the long one is over maxSizeUpload");
|
||||
assert!(
|
||||
summary.any_failed(),
|
||||
"so the dry run exits non-zero, like a real run"
|
||||
);
|
||||
no_upload.assert();
|
||||
for w in writes {
|
||||
w.assert();
|
||||
}
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_contact_too_large_for_one_request_is_predicted_to_fail() {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let archive = tmp();
|
||||
{
|
||||
let conn = db::init::open(&archive).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO address_books (id,name,is_default) VALUES (1,'Personal',1)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
let photo = db::blobs::intern_blob(&conn, &vec![b'P'; 8000]).unwrap();
|
||||
let huge = json!({ "@type": "Card", "name": { "full": "Photo Person" },
|
||||
"media": { "photo": { "@type": "Media", "kind": "photo",
|
||||
"@blob": photo, "mediaType": "image/png" } } })
|
||||
.to_string();
|
||||
let small = json!({ "@type": "Card", "name": { "full": "Small Person" } }).to_string();
|
||||
for (id, uid, data) in [(1, "huge-card", &huge), (2, "small-card", &small)] {
|
||||
conn.execute(
|
||||
"INSERT INTO contact_cards (id,uid,address_book_ids,data) VALUES (?1,?2,'[1]',?3)",
|
||||
rusqlite::params![id, uid, data],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
let _s = root_and_session(&mut server, session(&base, 50_000_000, 4000, &[]));
|
||||
let _ab = empty(
|
||||
&mut server,
|
||||
"AddressBook/get",
|
||||
json!({"accountId":"w","list":[],"notFound":[]}),
|
||||
);
|
||||
let _cq = empty(
|
||||
&mut server,
|
||||
"ContactCard/query",
|
||||
json!({"accountId":"w","ids":[]}),
|
||||
);
|
||||
let writes = no_writes(&mut server);
|
||||
|
||||
let summary = dry_run(&archive, &base);
|
||||
let cards = counts(&summary, "ContactCard");
|
||||
assert_eq!(cards.created, 1, "the small card would be created");
|
||||
assert_eq!(
|
||||
cards.failed, 1,
|
||||
"the card with the photo inlined is over maxSizeRequest"
|
||||
);
|
||||
assert!(summary.any_failed());
|
||||
for w in writes {
|
||||
w.assert();
|
||||
}
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
/// One active script with Stalwart's `vnd.stalwart.while`, against a target
|
||||
/// that names it `vnd.inbuxa.while`; `validate` is the answer to
|
||||
/// `SieveScript/validate`. Checks that the script sent for validation is the
|
||||
/// renamed one, and returns the run's counts.
|
||||
fn dry_run_one_script(validate: serde_json::Value) -> Summary {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let archive = tmp();
|
||||
{
|
||||
let conn = db::init::open(&archive).unwrap();
|
||||
let blob = db::blobs::intern_blob(
|
||||
&conn,
|
||||
b"require [\"fileinto\", \"vnd.stalwart.while\"];\nkeep;\n",
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO sieve_scripts (id,name,is_active,blob_id) VALUES (1,'main',1,?1)",
|
||||
rusqlite::params![blob],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let _s = root_and_session(
|
||||
&mut server,
|
||||
session(
|
||||
&base,
|
||||
50_000_000,
|
||||
10_000_000,
|
||||
&["fileinto", "vnd.inbuxa.while"],
|
||||
),
|
||||
);
|
||||
let _get = empty(
|
||||
&mut server,
|
||||
"SieveScript/get",
|
||||
json!({"accountId":"w","list":[],"notFound":[]}),
|
||||
);
|
||||
let upload = server
|
||||
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
|
||||
.match_body(Matcher::Regex("vnd\\.inbuxa\\.while".into()))
|
||||
.with_body(json!({"blobId":"TMP"}).to_string())
|
||||
.expect(1)
|
||||
.create();
|
||||
let _validate = server
|
||||
.mock("POST", API)
|
||||
.match_body(Matcher::Regex("SieveScript/validate".into()))
|
||||
.with_body(json!({"methodResponses":[validate]}).to_string())
|
||||
.create();
|
||||
let writes = no_writes(&mut server);
|
||||
let summary = dry_run(&archive, &base);
|
||||
for w in writes {
|
||||
w.assert();
|
||||
}
|
||||
upload.assert();
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
summary
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_sieve_script_the_target_rejects_is_predicted_to_fail() {
|
||||
let summary = dry_run_one_script(json!(["SieveScript/validate",
|
||||
{"accountId":"w","error":{"type":"invalidScript","description":"unknown test"}},"v"]));
|
||||
let sieve = counts(&summary, "SieveScript");
|
||||
assert_eq!(sieve.failed, 1);
|
||||
assert_eq!(sieve.created, 0);
|
||||
assert!(summary.any_failed());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_valid_sieve_script_is_validated_after_renaming_and_nothing_is_written() {
|
||||
let summary = dry_run_one_script(json!(["SieveScript/validate",
|
||||
{"accountId":"w","error":null},"v"]));
|
||||
let sieve = counts(&summary, "SieveScript");
|
||||
assert_eq!(sieve.created, 1, "it would be created");
|
||||
assert_eq!(sieve.failed, 0);
|
||||
assert!(!summary.any_failed());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_target_without_validate_still_gets_a_plan() {
|
||||
let summary = dry_run_one_script(json!(["error",
|
||||
{"type":"unknownMethod"},"v"]));
|
||||
let sieve = counts(&summary, "SieveScript");
|
||||
assert_eq!(sieve.created, 1, "not checked, so planned as written");
|
||||
assert_eq!(sieve.failed, 0);
|
||||
}
|
||||
@@ -169,7 +169,6 @@ fn run_import(
|
||||
automap: true,
|
||||
include_deleted: false,
|
||||
fetch_batch: 256,
|
||||
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
|
||||
imap_connections: 1,
|
||||
allow_source_change: false,
|
||||
};
|
||||
@@ -2006,218 +2005,3 @@ fn assert_name_selected_as_listed(listed: &'static str, stored: &str, archive_na
|
||||
);
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
fn write_fetch_message_dated(
|
||||
conn: &mut MockConn,
|
||||
seq: u32,
|
||||
uid: u32,
|
||||
internaldate: &str,
|
||||
body: &[u8],
|
||||
) -> std::io::Result<()> {
|
||||
let header = format!(
|
||||
"* {seq} FETCH (UID {uid} FLAGS (\\Seen) INTERNALDATE \"{internaldate}\" RFC822.SIZE {} BODY[] {{{}}}\r\n",
|
||||
body.len(),
|
||||
body.len()
|
||||
);
|
||||
conn.write_raw(header.as_bytes())?;
|
||||
conn.write_raw(body)?;
|
||||
conn.write_raw(b")\r\n")
|
||||
}
|
||||
|
||||
const BODY_A: &[u8] = b"From: a@b\r\nMessage-ID: <a@h>\r\nSubject: a\r\n\r\none";
|
||||
const BODY_B: &[u8] = b"From: a@b\r\nMessage-ID: <b@h>\r\nSubject: b\r\n\r\ntwo";
|
||||
const BODY_C: &[u8] = b"From: a@b\r\nMessage-ID: <c@h>\r\nSubject: c\r\n\r\nthree";
|
||||
|
||||
fn email_counts(summary: &inbuxa_migrate::sync::Summary) -> inbuxa_migrate::sync::TypeCounts {
|
||||
summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(k, _)| *k == "email")
|
||||
.map(|(_, c)| c.clone())
|
||||
.expect("email counts")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_message_that_will_not_import_is_recorded_and_the_folder_carries_on() {
|
||||
let worker: Script = Box::new(|conn: &mut MockConn| -> std::io::Result<()> {
|
||||
auth_preamble(conn, "IMAP4rev2 LITERAL+ AUTH=PLAIN")?;
|
||||
let (tag, _) = conn.read_command()?;
|
||||
write_select(conn, &tag, 12345, 4, 3)?;
|
||||
let (tag, cmd) = conn.read_command()?;
|
||||
assert!(cmd.starts_with("UID FETCH"), "got {cmd}");
|
||||
write_fetch_message_dated(conn, 1, 1, "12-May-2025 10:00:00 +0000", BODY_A)?;
|
||||
// No such month: this one cannot be imported.
|
||||
write_fetch_message_dated(conn, 2, 2, "12-Mai-2025 10:00:00 +0000", BODY_B)?;
|
||||
// Lower case is only untidy, and is imported.
|
||||
write_fetch_message_dated(conn, 3, 3, "12-may-2025 10:00:00 +0000", BODY_C)?;
|
||||
conn.write_line(&format!("{tag} OK"))?;
|
||||
drain_until_close(conn);
|
||||
Ok(())
|
||||
});
|
||||
let server = MockImap::start_scripts(vec![
|
||||
control_script_one_folder(12345, 4, &[1, 2, 3]),
|
||||
worker,
|
||||
]);
|
||||
let archive = tempfile("bad-message");
|
||||
let summary = run_import(&server, "alice", archive.clone(), |_| {}).expect("import");
|
||||
let email = email_counts(&summary);
|
||||
assert_eq!(email.created, 2, "summary={summary:?}");
|
||||
assert_eq!(email.failed, 1, "summary={summary:?}");
|
||||
let conn = Connection::open(&archive).unwrap();
|
||||
db::init::apply_schema(&conn).unwrap();
|
||||
assert_eq!(count(&conn, "emails"), 2);
|
||||
let uids: Vec<i64> = conn
|
||||
.prepare("SELECT uid FROM sync_id_imap WHERE type_name = 'email' ORDER BY uid")
|
||||
.unwrap()
|
||||
.query_map([], |r| r.get(0))
|
||||
.unwrap()
|
||||
.collect::<Result<_, _>>()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
uids,
|
||||
vec![1, 3],
|
||||
"the failed message must stay out of the UID map"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_chunk_keeps_the_ones_before_it_and_a_rerun_fetches_only_what_is_missing() {
|
||||
// The archive is opened with an exclusive lock, so the commit after each
|
||||
// chunk cannot be watched from outside while a run is going; what can be
|
||||
// checked is its effect. First run, one UID per chunk: chunk 1 arrives,
|
||||
// chunk 2 fails the way a dying server would.
|
||||
let archive = tempfile("chunk-commit");
|
||||
let worker_1: Script = Box::new(|conn: &mut MockConn| -> std::io::Result<()> {
|
||||
auth_preamble(conn, "IMAP4rev2 LITERAL+ AUTH=PLAIN")?;
|
||||
let (tag, _) = conn.read_command()?;
|
||||
write_select(conn, &tag, 777, 3, 2)?;
|
||||
let (tag, cmd) = conn.read_command()?;
|
||||
assert!(cmd.starts_with("UID FETCH 1 "), "got {cmd}");
|
||||
write_fetch_message(conn, 1, 1, BODY_A)?;
|
||||
conn.write_line(&format!("{tag} OK"))?;
|
||||
let (tag, cmd) = conn.read_command()?;
|
||||
assert!(cmd.starts_with("UID FETCH 2 "), "got {cmd}");
|
||||
conn.write_line(&format!("{tag} NO [SERVERBUG] gone"))?;
|
||||
drain_until_close(conn);
|
||||
Ok(())
|
||||
});
|
||||
// Second run: only UID 2 is missing, so only UID 2 may be fetched.
|
||||
let worker_2: Script = Box::new(|conn: &mut MockConn| -> std::io::Result<()> {
|
||||
auth_preamble(conn, "IMAP4rev2 LITERAL+ AUTH=PLAIN")?;
|
||||
let (tag, _) = conn.read_command()?;
|
||||
write_select(conn, &tag, 777, 3, 2)?;
|
||||
let (tag, cmd) = conn.read_command()?;
|
||||
assert!(
|
||||
cmd.starts_with("UID FETCH 2 "),
|
||||
"rerun refetched more than UID 2: {cmd}"
|
||||
);
|
||||
write_fetch_message(conn, 2, 2, BODY_B)?;
|
||||
conn.write_line(&format!("{tag} OK"))?;
|
||||
drain_until_close(conn);
|
||||
Ok(())
|
||||
});
|
||||
let server = MockImap::start_scripts(vec![
|
||||
control_script_one_folder(777, 3, &[1, 2]),
|
||||
worker_1,
|
||||
control_script_one_folder(777, 3, &[1, 2]),
|
||||
worker_2,
|
||||
]);
|
||||
|
||||
let first =
|
||||
run_import(&server, "alice", archive.clone(), |c| c.fetch_batch = 1).expect("first run");
|
||||
let e1 = email_counts(&first);
|
||||
assert_eq!((e1.created, e1.failed), (1, 1), "first={first:?}");
|
||||
|
||||
let second =
|
||||
run_import(&server, "alice", archive.clone(), |c| c.fetch_batch = 1).expect("second run");
|
||||
let e2 = email_counts(&second);
|
||||
assert_eq!((e2.created, e2.failed), (1, 0), "second={second:?}");
|
||||
let conn = Connection::open(&archive).unwrap();
|
||||
db::init::apply_schema(&conn).unwrap();
|
||||
assert_eq!(count(&conn, "emails"), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn body_fetches_are_split_by_message_size() {
|
||||
// Three 20-byte messages under a 30-byte cap: the control connection
|
||||
// learns the sizes first, and each body is then fetched on its own.
|
||||
let control: Script = Box::new(|conn: &mut MockConn| -> std::io::Result<()> {
|
||||
auth_preamble(conn, "IMAP4rev2 LITERAL+ AUTH=PLAIN")?;
|
||||
let (tag, _) = conn.read_command()?;
|
||||
conn.write_line("* LIST () \"/\" \"INBOX\"")?;
|
||||
conn.write_line(&format!("{tag} OK"))?;
|
||||
let (tag, _) = conn.read_command()?;
|
||||
conn.write_line(&format!("{tag} OK"))?;
|
||||
let (tag, _) = conn.read_command()?;
|
||||
write_select(conn, &tag, 555, 4, 3)?;
|
||||
let (tag, _) = conn.read_command()?;
|
||||
conn.write_line("* SEARCH 1 2 3")?;
|
||||
conn.write_line(&format!("{tag} OK"))?;
|
||||
let (tag, cmd) = conn.read_command()?;
|
||||
assert_eq!(cmd, "UID FETCH 1:3 (UID RFC822.SIZE)");
|
||||
for uid in 1..=3 {
|
||||
conn.write_line(&format!("* {uid} FETCH (UID {uid} RFC822.SIZE 20)"))?;
|
||||
}
|
||||
conn.write_line(&format!("{tag} OK"))?;
|
||||
drain_until_close(conn);
|
||||
Ok(())
|
||||
});
|
||||
let fetches = std::sync::Arc::new(Mutex::new(Vec::<String>::new()));
|
||||
let worker: Script = {
|
||||
let fetches = fetches.clone();
|
||||
Box::new(move |conn: &mut MockConn| -> std::io::Result<()> {
|
||||
auth_preamble(conn, "IMAP4rev2 LITERAL+ AUTH=PLAIN")?;
|
||||
let (tag, _) = conn.read_command()?;
|
||||
write_select(conn, &tag, 555, 4, 3)?;
|
||||
let bodies: [&[u8]; 3] = [BODY_A, BODY_B, BODY_C];
|
||||
for _ in 0..3 {
|
||||
let (tag, cmd) = conn.read_command()?;
|
||||
let uid: u32 = cmd
|
||||
.strip_prefix("UID FETCH ")
|
||||
.and_then(|r| r.split(' ').next())
|
||||
.and_then(|n| n.parse().ok())
|
||||
.unwrap_or_else(|| panic!("expected a single-UID fetch, got {cmd}"));
|
||||
fetches.lock().unwrap().push(cmd.clone());
|
||||
write_fetch_message(conn, uid, uid, bodies[(uid - 1) as usize])?;
|
||||
conn.write_line(&format!("{tag} OK"))?;
|
||||
}
|
||||
drain_until_close(conn);
|
||||
Ok(())
|
||||
})
|
||||
};
|
||||
let server = MockImap::start_scripts(vec![control, worker]);
|
||||
let archive = tempfile("byte-chunks");
|
||||
let summary =
|
||||
run_import(&server, "alice", archive, |c| c.fetch_batch_bytes = 30).expect("import");
|
||||
assert_eq!(email_counts(&summary).created, 3, "summary={summary:?}");
|
||||
assert_eq!(
|
||||
fetches.lock().unwrap().len(),
|
||||
3,
|
||||
"one body fetch per message"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_chunk_larger_than_the_event_queue_completes() {
|
||||
// One worker holds at most two events in flight; a ten-message chunk
|
||||
// has to wait on the archive writer, not deadlock on it.
|
||||
const UIDS: &[u32] = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
|
||||
let worker: Script = Box::new(|conn: &mut MockConn| -> std::io::Result<()> {
|
||||
auth_preamble(conn, "IMAP4rev2 LITERAL+ AUTH=PLAIN")?;
|
||||
let (tag, _) = conn.read_command()?;
|
||||
write_select(conn, &tag, 999, 11, 10)?;
|
||||
let (tag, cmd) = conn.read_command()?;
|
||||
assert!(cmd.starts_with("UID FETCH 1:10 "), "got {cmd}");
|
||||
for uid in UIDS {
|
||||
let body = format!("From: a@b\r\nMessage-ID: <{uid}@h>\r\n\r\nbody {uid}");
|
||||
write_fetch_message(conn, *uid, *uid, body.as_bytes())?;
|
||||
}
|
||||
conn.write_line(&format!("{tag} OK"))?;
|
||||
drain_until_close(conn);
|
||||
Ok(())
|
||||
});
|
||||
let server = MockImap::start_scripts(vec![control_script_one_folder(999, 11, UIDS), worker]);
|
||||
let archive = tempfile("backpressure");
|
||||
let summary = run_import(&server, "alice", archive, |_| {}).expect("import");
|
||||
assert_eq!(email_counts(&summary).created, 10, "summary={summary:?}");
|
||||
}
|
||||
|
||||
+1
-2
@@ -17,7 +17,6 @@ use inbuxa_migrate::jmap::session::{Limits, Session};
|
||||
use inbuxa_migrate::jmap::wire::JmapId;
|
||||
use inbuxa_migrate::jmap::wire::identity::Identity;
|
||||
use inbuxa_migrate::jmap::wire::mailbox::Mailbox;
|
||||
use inbuxa_migrate::net::CertOverride;
|
||||
use serde_json::json;
|
||||
|
||||
fn client(retries: u32) -> HttpClient {
|
||||
@@ -27,7 +26,7 @@ fn client(retries: u32) -> HttpClient {
|
||||
password: "p".into(),
|
||||
},
|
||||
RetryPolicy::new(retries),
|
||||
CertOverride::none(),
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
+144
-309
@@ -487,6 +487,150 @@ fn export_mailbox_already_exists_maps_existing_id() {
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn email_export_sends_one_email_per_import_call() {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let api = "/jmap/api";
|
||||
|
||||
let archive = tmp();
|
||||
{
|
||||
let conn = db::init::open(&archive).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO mailboxes (id,name,parent_id,role) VALUES (1,'Inbox',NULL,'inbox')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
for n in 1..=2 {
|
||||
let raw =
|
||||
format!("From: a@x\r\nSubject: m{n}\r\nMessage-ID: <m-{n}@h>\r\n\r\nbody {n}",);
|
||||
let blob = db::blobs::intern_blob(&conn, raw.as_bytes()).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords)
|
||||
VALUES (?1,'2020-01-01T00:00:00Z','[1]','[]')",
|
||||
rusqlite::params![blob],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
let _root = server.mock("GET", "/").with_status(404).create();
|
||||
let _wk = server
|
||||
.mock("GET", "/.well-known/jmap")
|
||||
.with_body(session_body(&base))
|
||||
.create();
|
||||
|
||||
let _mq = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Mailbox/query".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Mailbox/query",
|
||||
{"accountId":"w","ids":["t1"]},"q"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let _mq_empty = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::AllOf(vec![
|
||||
Matcher::Regex("Mailbox/query".into()),
|
||||
Matcher::Regex("anchor".into()),
|
||||
]))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Mailbox/query",
|
||||
{"accountId":"w","ids":[]},"q"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.create();
|
||||
let _mg = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Mailbox/get".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Mailbox/get",{"accountId":"w","list":[
|
||||
{"id":"t1","name":"Inbox","role":"inbox","parentId":null,
|
||||
"myRights":{"mayDelete":true}}],"notFound":[]},"g"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let _eq = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Email/query".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Email/query",
|
||||
{"accountId":"w","ids":[]},"q"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
|
||||
let _ups = server
|
||||
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
|
||||
.with_body(json!({"blobId":"UP1"}).to_string())
|
||||
.expect(2)
|
||||
.create();
|
||||
|
||||
let single_only = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::AllOf(vec![
|
||||
Matcher::Regex("Email/import".into()),
|
||||
Matcher::Regex("e1".into()),
|
||||
Matcher::Regex("e2".into()),
|
||||
]))
|
||||
.expect(0)
|
||||
.create();
|
||||
|
||||
let imports = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Email/import".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Email/import",
|
||||
{"accountId":"w","created":{"e":{"id":"x","blobId":"b","threadId":"t","size":10}}},"i"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(2)
|
||||
.create();
|
||||
|
||||
let summary = sync::export::run(
|
||||
CommonConfig {
|
||||
archive: archive.clone(),
|
||||
threads: 1,
|
||||
dry_run: false,
|
||||
max_retries: 0,
|
||||
allow_invalid_certs: false,
|
||||
logger: Logger::from_flags(true, 0),
|
||||
},
|
||||
ExportConfig {
|
||||
connect: ConnectConfig {
|
||||
url: base.clone(),
|
||||
auth: Auth::Basic {
|
||||
user: "u".into(),
|
||||
password: "p".into(),
|
||||
},
|
||||
account: AccountSelector::Id("w".into()),
|
||||
},
|
||||
objects: None,
|
||||
prune: false,
|
||||
yes: true,
|
||||
},
|
||||
)
|
||||
.expect("export run");
|
||||
|
||||
let email = summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(t, _)| *t == "Email")
|
||||
.map(|(_, c)| c.clone())
|
||||
.expect("email counts");
|
||||
assert_eq!(email.created, 2, "both emails imported in per-item rounds");
|
||||
assert_eq!(email.failed, 0, "no per-unit failure");
|
||||
assert!(!summary.any_failed(), "no whole-run failure");
|
||||
|
||||
single_only.assert();
|
||||
imports.assert();
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_email_blob_not_found_reuploads_and_retries() {
|
||||
let mut server = mockito::Server::new();
|
||||
@@ -2644,205 +2788,6 @@ fn export_sieve_script_matched_by_name_with_the_same_content_is_left_alone() {
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
/// A session like `session_body_full`, whose Sieve capability lists
|
||||
/// `extensions` in `sieveExtensions`.
|
||||
fn session_body_sieve(base: &str, extensions: &[&str]) -> String {
|
||||
let mut v: serde_json::Value = serde_json::from_str(&session_body_full(base)).unwrap();
|
||||
v["accounts"]["w"]["accountCapabilities"]["urn:ietf:params:jmap:sieve"] =
|
||||
json!({ "sieveExtensions": extensions });
|
||||
v.to_string()
|
||||
}
|
||||
|
||||
/// One active script `name` in a fresh archive, with `body` as its content.
|
||||
fn archive_with_active_sieve(name: &str, body: &[u8]) -> PathBuf {
|
||||
let archive = tmp();
|
||||
let conn = db::init::open(&archive).unwrap();
|
||||
let blob = db::blobs::intern_blob(&conn, body).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO sieve_scripts (id,name,is_active,blob_id) VALUES (1,?1,1,?2)",
|
||||
rusqlite::params![name, blob],
|
||||
)
|
||||
.unwrap();
|
||||
archive
|
||||
}
|
||||
|
||||
/// Mocks for exporting one script to an empty target: get, upload (matched
|
||||
/// by `upload_body`), create, and activation answered with `activation`.
|
||||
/// Returns the upload mock and the activation mock.
|
||||
fn mock_sieve_export(
|
||||
server: &mut mockito::ServerGuard,
|
||||
session: String,
|
||||
upload_body: Matcher,
|
||||
create_ok: bool,
|
||||
activation: serde_json::Value,
|
||||
) -> (mockito::Mock, mockito::Mock, Vec<mockito::Mock>) {
|
||||
let api = "/jmap/api";
|
||||
let mut keep = vec![
|
||||
server.mock("GET", "/").with_status(404).create(),
|
||||
server
|
||||
.mock("GET", "/.well-known/jmap")
|
||||
.with_body(session)
|
||||
.expect_at_least(1)
|
||||
.create(),
|
||||
server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("SieveScript/get".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["SieveScript/get",
|
||||
{"accountId":"w","list":[],"notFound":[]},"g"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.create(),
|
||||
];
|
||||
let upload = server
|
||||
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
|
||||
.match_body(upload_body)
|
||||
.with_body(json!({"blobId":"UPN"}).to_string())
|
||||
.expect(1)
|
||||
.create();
|
||||
let created = if create_ok {
|
||||
json!({"accountId":"w","created":{"c1":{"id":"S1"}}})
|
||||
} else {
|
||||
json!({"accountId":"w","notCreated":{"c1":{"type":"invalidScript",
|
||||
"description":"unknown extension"}}})
|
||||
};
|
||||
keep.push(
|
||||
server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::AllOf(vec![
|
||||
Matcher::Regex("SieveScript/set".into()),
|
||||
Matcher::Regex("\"create\"".into()),
|
||||
]))
|
||||
.with_body(json!({"methodResponses":[["SieveScript/set", created, "s"]]}).to_string())
|
||||
.create(),
|
||||
);
|
||||
let activate = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::AllOf(vec![
|
||||
Matcher::Regex("SieveScript/set".into()),
|
||||
Matcher::Regex("onSuccess".into()),
|
||||
]))
|
||||
.with_body(json!({"methodResponses":[activation]}).to_string())
|
||||
.expect(1)
|
||||
.create();
|
||||
(upload, activate, keep)
|
||||
}
|
||||
|
||||
fn sieve_counts(summary: &sync::Summary) -> sync::TypeCounts {
|
||||
summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(t, _)| *t == "SieveScript")
|
||||
.map(|(_, c)| c.clone())
|
||||
.expect("sieve counts")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_sieve_renames_stalwart_names_for_an_inbuxa_target() {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let archive = archive_with_active_sieve(
|
||||
"loop",
|
||||
b"require [\"fileinto\", \"vnd.stalwart.while\"];\nkeep;\n",
|
||||
);
|
||||
let (upload, activate, _keep) = mock_sieve_export(
|
||||
&mut server,
|
||||
session_body_sieve(
|
||||
&base,
|
||||
&["fileinto", "vnd.inbuxa.while", "vnd.inbuxa.expressions"],
|
||||
),
|
||||
Matcher::Exact("require [\"fileinto\", \"vnd.inbuxa.while\"];\nkeep;\n".into()),
|
||||
true,
|
||||
json!(["SieveScript/set", {"accountId":"w"}, "a"]),
|
||||
);
|
||||
let summary = sync::export::run(
|
||||
common(&archive),
|
||||
export_cfg_objects(&base, vec![ObjectType::SieveScript]),
|
||||
)
|
||||
.expect("export");
|
||||
upload.assert();
|
||||
activate.assert();
|
||||
let c = sieve_counts(&summary);
|
||||
assert_eq!((c.created, c.failed), (1, 0));
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_sieve_keeps_stalwart_names_for_a_target_without_inbuxa_names() {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let archive = archive_with_active_sieve(
|
||||
"loop",
|
||||
b"require [\"fileinto\", \"vnd.stalwart.while\"];\nkeep;\n",
|
||||
);
|
||||
let (upload, _activate, _keep) = mock_sieve_export(
|
||||
&mut server,
|
||||
session_body_sieve(&base, &["fileinto", "vnd.stalwart.while"]),
|
||||
Matcher::Regex("vnd\\.stalwart\\.while".into()),
|
||||
true,
|
||||
json!(["SieveScript/set", {"accountId":"w"}, "a"]),
|
||||
);
|
||||
let summary = sync::export::run(
|
||||
common(&archive),
|
||||
export_cfg_objects(&base, vec![ObjectType::SieveScript]),
|
||||
)
|
||||
.expect("export");
|
||||
upload.assert();
|
||||
assert_eq!(sieve_counts(&summary).failed, 0);
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_sieve_activation_error_is_a_failure() {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let archive = archive_with_active_sieve("main", b"require [\"fileinto\"];\nkeep;\n");
|
||||
let (_upload, activate, _keep) = mock_sieve_export(
|
||||
&mut server,
|
||||
session_body_full(&base),
|
||||
Matcher::Any,
|
||||
true,
|
||||
json!(["error", {"type":"invalidArguments","description":"cannot activate"}, "a"]),
|
||||
);
|
||||
let summary = sync::export::run(
|
||||
common(&archive),
|
||||
export_cfg_objects(&base, vec![ObjectType::SieveScript]),
|
||||
)
|
||||
.expect("export");
|
||||
activate.assert();
|
||||
let c = sieve_counts(&summary);
|
||||
assert_eq!(c.created, 1);
|
||||
assert_eq!(
|
||||
c.failed, 1,
|
||||
"a failed activation is a failure, not a warning"
|
||||
);
|
||||
assert!(summary.any_failed(), "so export exits non-zero");
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_sieve_active_script_not_created_is_a_failure() {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let archive = archive_with_active_sieve("main", b"require [\"nope\"];\nkeep;\n");
|
||||
let (_upload, _activate, _keep) = mock_sieve_export(
|
||||
&mut server,
|
||||
session_body_full(&base),
|
||||
Matcher::Any,
|
||||
false,
|
||||
json!(["SieveScript/set", {"accountId":"w"}, "a"]),
|
||||
);
|
||||
let summary = sync::export::run(
|
||||
common(&archive),
|
||||
export_cfg_objects(&base, vec![ObjectType::SieveScript]),
|
||||
)
|
||||
.expect("export");
|
||||
let c = sieve_counts(&summary);
|
||||
assert_eq!((c.created, c.failed), (0, 1));
|
||||
assert!(summary.any_failed());
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_sieve_scripts_identical_content_different_names_both_created() {
|
||||
let mut server = mockito::Server::new();
|
||||
@@ -5046,113 +4991,3 @@ fn export_rerun_leaves_a_contact_alone_when_the_target_is_newer() {
|
||||
);
|
||||
assert_eq!(counts.skipped, 1);
|
||||
}
|
||||
|
||||
fn event_rerun(local_updated: &str, updates_sent: usize) -> inbuxa_migrate::sync::TypeCounts {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let api = "/jmap/api";
|
||||
let archive = tmp();
|
||||
{
|
||||
let conn = db::init::open(&archive).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO calendars (id,name,is_default) VALUES (1,'Work',1)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
let event = json!({"@type":"Event","uid":"ev1","title":"Planning, moved",
|
||||
"start":"2026-03-02T10:00:00","duration":"PT1H",
|
||||
"updated":local_updated});
|
||||
conn.execute(
|
||||
"INSERT INTO calendar_events (id,calendar_ids,is_draft,use_default_alerts,data)
|
||||
VALUES (1,'[1]',0,0,?1)",
|
||||
rusqlite::params![event.to_string()],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let _root = server.mock("GET", "/").with_status(404).create();
|
||||
let _wk = server
|
||||
.mock("GET", "/.well-known/jmap")
|
||||
.with_body(session_body_full(&base))
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let _calg = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Calendar/get".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Calendar/get",{"accountId":"w","list":[
|
||||
{"id":"K","name":"Work","isDefault":true,"myRights":{"mayDelete":false}}
|
||||
],"notFound":[]},"g"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let _term = anchor_terminator(&mut server, api, "CalendarEvent");
|
||||
let _eq = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("CalendarEvent/query".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["CalendarEvent/query",{"accountId":"w","ids":["E1"]},"q"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(1)
|
||||
.create();
|
||||
let _eg = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("CalendarEvent/get".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["CalendarEvent/get",{"accountId":"w","list":[
|
||||
{"id":"E1","@type":"Event","uid":"ev1","title":"Planning",
|
||||
"start":"2026-03-01T10:00:00","duration":"PT1H",
|
||||
"calendarIds":{"K":true},"isDraft":false,"useDefaultAlerts":false,
|
||||
"updated":"2026-02-01T00:00:00Z"}
|
||||
],"notFound":[]},"g"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(1)
|
||||
.create();
|
||||
let update = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::AllOf(vec![
|
||||
Matcher::Regex("CalendarEvent/set".into()),
|
||||
Matcher::Regex("\"update\"".into()),
|
||||
Matcher::Regex("Planning, moved".into()),
|
||||
]))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["CalendarEvent/set",{"accountId":"w","updated":{"E1":null}},"s"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(updates_sent)
|
||||
.create();
|
||||
|
||||
let summary = sync::export::run(
|
||||
common(&archive),
|
||||
export_cfg_objects(&base, vec![ObjectType::Calendar, ObjectType::CalendarEvent]),
|
||||
)
|
||||
.expect("export");
|
||||
let counts = summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(t, _)| *t == "CalendarEvent")
|
||||
.map(|(_, c)| c.clone())
|
||||
.expect("event counts");
|
||||
update.assert();
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
counts
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_rerun_updates_an_event_moved_at_the_source() {
|
||||
let counts = event_rerun("2026-03-01T00:00:00Z", 1);
|
||||
assert_eq!(counts.updated, 1, "the newer archive copy is written");
|
||||
assert_eq!(counts.failed, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_rerun_leaves_an_event_alone_when_nothing_is_newer() {
|
||||
let counts = event_rerun("2026-02-01T01:00:00+01:00", 0);
|
||||
assert_eq!(
|
||||
counts.updated, 0,
|
||||
"same instant as the target's, written another way"
|
||||
);
|
||||
assert_eq!(counts.skipped, 1);
|
||||
}
|
||||
|
||||
@@ -71,7 +71,6 @@ fn imap_basic_config(localpart: &str) -> ImapImportConfig {
|
||||
automap: true,
|
||||
include_deleted: false,
|
||||
fetch_batch: 256,
|
||||
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
|
||||
imap_connections: 4,
|
||||
allow_source_change: false,
|
||||
}
|
||||
|
||||
+5
-22
@@ -16,7 +16,6 @@ use inbuxa_migrate::jmap::http::{Auth, HttpClient, RetryPolicy};
|
||||
use inbuxa_migrate::jmap::request::Request;
|
||||
use inbuxa_migrate::jmap::session::Session;
|
||||
use inbuxa_migrate::logging::Logger;
|
||||
use inbuxa_migrate::net::CertOverride;
|
||||
use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig};
|
||||
use integration::stalwart::shared as shared_stalwart;
|
||||
use rusqlite::Connection;
|
||||
@@ -786,11 +785,7 @@ fn export_inlines_contact_and_event_blobs_instead_of_blob_ids() {
|
||||
assert_eq!(counts.failed, 0, "{name} had no failures: {counts:?}");
|
||||
}
|
||||
|
||||
let client = HttpClient::new(
|
||||
basic("test6"),
|
||||
RetryPolicy::new(5),
|
||||
CertOverride::for_url(true, base_url()),
|
||||
);
|
||||
let client = HttpClient::new(basic("test6"), RetryPolicy::new(5), true);
|
||||
let session = Session::discover(&client, base_url()).expect("discover target session");
|
||||
let api = session.api_url.clone();
|
||||
|
||||
@@ -1052,11 +1047,7 @@ fn live_burst_exceeds_concurrent_requests_and_recovers() {
|
||||
let fx = seeder::provision(base_url()).expect("provision");
|
||||
let acc = fx.account("test1").expect("test1");
|
||||
|
||||
let client = HttpClient::new(
|
||||
basic("test1"),
|
||||
RetryPolicy::new(20),
|
||||
CertOverride::for_url(true, base_url()),
|
||||
);
|
||||
let client = HttpClient::new(basic("test1"), RetryPolicy::new(20), true);
|
||||
let session = Session::discover(&client, base_url()).expect("discover session");
|
||||
let server_limits = session.core_limits().expect("core limits");
|
||||
|
||||
@@ -1157,7 +1148,7 @@ impl JmapSettingsGuard {
|
||||
password: seeder::ADMIN_PASSWORD.to_owned(),
|
||||
},
|
||||
RetryPolicy::new(5),
|
||||
CertOverride::for_url(true, base_url()),
|
||||
true,
|
||||
);
|
||||
let session = Session::discover(&admin, base_url()).expect("admin discover");
|
||||
let admin_account = session
|
||||
@@ -1278,11 +1269,7 @@ fn live_blob_quota_429_triggers_retry_after_then_succeeds() {
|
||||
);
|
||||
let _ttl_guard = JmapSettingsGuard::override_settings(updates);
|
||||
|
||||
let client = HttpClient::new(
|
||||
basic("test1"),
|
||||
RetryPolicy::new(20),
|
||||
CertOverride::for_url(true, base_url()),
|
||||
);
|
||||
let client = HttpClient::new(basic("test1"), RetryPolicy::new(20), true);
|
||||
let session = Session::discover(&client, base_url()).expect("discover session");
|
||||
let limits = session.core_limits().expect("core limits");
|
||||
client.set_limits(&limits);
|
||||
@@ -1359,11 +1346,7 @@ fn import_delta_propagates_email_keyword_change_via_changes() {
|
||||
.expect("an unflagged email exists in the archive")
|
||||
};
|
||||
|
||||
let client = HttpClient::new(
|
||||
basic("test1"),
|
||||
RetryPolicy::new(5),
|
||||
CertOverride::for_url(true, base_url()),
|
||||
);
|
||||
let client = HttpClient::new(basic("test1"), RetryPolicy::new(5), true);
|
||||
let session = Session::discover(&client, base_url()).expect("session discovered");
|
||||
let account = account::resolve(
|
||||
&AccountSelector::Id(acc.account_id.clone()),
|
||||
|
||||
Reference in New Issue
Block a user