export: a dry run that predicts failures and exits as the real run would #11
@@ -246,6 +246,25 @@ failed.
|
|||||||
`--prune` also deletes what is on the target and not in the archive. It asks
|
`--prune` also deletes what is on the target and not in the archive. It asks
|
||||||
first; `--yes` answers for it, for scripts. Export speaks JMAP only.
|
first; `--yes` answers for it, for scripts. Export speaks JMAP only.
|
||||||
|
|
||||||
|
With `--dry-run`, export reads the target and writes nothing to the account.
|
||||||
|
It prints the plan in plain words: for each type, how much would be
|
||||||
|
created, updated, left unchanged or deleted, and what would fail. It
|
||||||
|
catches in advance the failures a real run would hit:
|
||||||
|
|
||||||
|
- a message larger than the target's `maxSizeUpload`;
|
||||||
|
- a contact, event or other object too large for one request under its
|
||||||
|
`maxSizeRequest`, as happens when a photo is carried inline;
|
||||||
|
- a Sieve script the target would reject. Each script that would be written
|
||||||
|
is checked with `SieveScript/validate`, after any `vnd.stalwart.*` names
|
||||||
|
are renamed, so what is checked is what would be uploaded. The check needs
|
||||||
|
the script as a blob, so the dry run uploads Sieve scripts, and only them;
|
||||||
|
a blob that nothing uses is discarded by the server. A target without
|
||||||
|
`SieveScript/validate` gets one warning, and its scripts are not checked.
|
||||||
|
|
||||||
|
The plan lists each predicted failure and its reason. When anything would
|
||||||
|
fail, the dry run exits 5, as the real run would, so a script can stop
|
||||||
|
before it starts.
|
||||||
|
|
||||||
## Inspect
|
## Inspect
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|||||||
+6
-1
@@ -31,6 +31,7 @@ fn run() -> i32 {
|
|||||||
Err(err) => return fail(&err),
|
Err(err) => return fail(&err),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let mut quiet_report = false;
|
||||||
let (outcome, logger) = match action {
|
let (outcome, logger) = match action {
|
||||||
Action::Import(common, config) => {
|
Action::Import(common, config) => {
|
||||||
let logger = common.logger;
|
let logger = common.logger;
|
||||||
@@ -75,6 +76,8 @@ fn run() -> i32 {
|
|||||||
}
|
}
|
||||||
Action::Export(common, config) => {
|
Action::Export(common, config) => {
|
||||||
let logger = common.logger;
|
let logger = common.logger;
|
||||||
|
// A dry run prints its own plan; the counts below would repeat it.
|
||||||
|
quiet_report = common.dry_run;
|
||||||
(
|
(
|
||||||
RunOutcome::from_result(sync::export::run(common, config)),
|
RunOutcome::from_result(sync::export::run(common, config)),
|
||||||
logger,
|
logger,
|
||||||
@@ -88,7 +91,9 @@ fn run() -> i32 {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
report(&outcome.summary);
|
if !quiet_report {
|
||||||
|
report(&outcome.summary);
|
||||||
|
}
|
||||||
match outcome.error {
|
match outcome.error {
|
||||||
Some(err) => fail(&err),
|
Some(err) => fail(&err),
|
||||||
None => {
|
None => {
|
||||||
|
|||||||
+192
-17
@@ -91,8 +91,9 @@ impl<'a> Uploader<'a> {
|
|||||||
return Ok(id.clone());
|
return Ok(id.clone());
|
||||||
}
|
}
|
||||||
let id = if self.net.dry_run {
|
let id = if self.net.dry_run {
|
||||||
let _exists = db::blobs::blob_bytes(self.conn, local_id)?
|
let len = db::blobs::blob_len(self.conn, local_id)?
|
||||||
.ok_or_else(|| JmapError::malformed(format!("blob local id {local_id} missing")))?;
|
.ok_or_else(|| JmapError::malformed(format!("blob local id {local_id} missing")))?;
|
||||||
|
self.net.check_upload_size(len)?;
|
||||||
JmapId(format!("dryrun-blob-{local_id}"))
|
JmapId(format!("dryrun-blob-{local_id}"))
|
||||||
} else {
|
} else {
|
||||||
let bytes = db::blobs::blob_bytes(self.conn, local_id)?
|
let bytes = db::blobs::blob_bytes(self.conn, local_id)?
|
||||||
@@ -123,6 +124,7 @@ impl<'a> Uploader<'a> {
|
|||||||
return Ok(id.clone());
|
return Ok(id.clone());
|
||||||
}
|
}
|
||||||
let id = if self.net.dry_run {
|
let id = if self.net.dry_run {
|
||||||
|
self.net.check_upload_size(bytes.len() as u64)?;
|
||||||
JmapId(format!("dryrun-blob-{local_id}"))
|
JmapId(format!("dryrun-blob-{local_id}"))
|
||||||
} else {
|
} else {
|
||||||
blobxfer::upload_bytes(
|
blobxfer::upload_bytes(
|
||||||
@@ -293,6 +295,35 @@ struct Net {
|
|||||||
/// Blobs uploaded at once: the server's `maxConcurrentUpload`, and no
|
/// Blobs uploaded at once: the server's `maxConcurrentUpload`, and no
|
||||||
/// more than `--threads`.
|
/// more than `--threads`.
|
||||||
upload_workers: usize,
|
upload_workers: usize,
|
||||||
|
/// In a dry run, what would fail and why, for the plan.
|
||||||
|
would_fail: std::sync::Arc<Mutex<Vec<String>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Net {
|
||||||
|
/// In a dry run, a blob of `len` bytes that the target would refuse:
|
||||||
|
/// over its `maxSizeUpload`.
|
||||||
|
fn check_upload_size(&self, len: u64) -> Result<(), JmapError> {
|
||||||
|
let cap = self.limits.max_size_upload;
|
||||||
|
if cap > 0 && len > cap {
|
||||||
|
return Err(JmapError::SingleObjectTooLarge(format!(
|
||||||
|
"{} is larger than the target accepts ({} maxSizeUpload)",
|
||||||
|
crate::inspect::format_bytes(len),
|
||||||
|
crate::inspect::format_bytes(cap)
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Notes, in a dry run, that `what` would fail and why. A real run
|
||||||
|
/// reports failures as they happen and keeps no list.
|
||||||
|
fn would_fail(&self, what: impl Into<String>) {
|
||||||
|
if self.dry_run {
|
||||||
|
self.would_fail
|
||||||
|
.lock()
|
||||||
|
.expect("would-fail list")
|
||||||
|
.push(what.into());
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn has_rows(conn: &Connection, ty: ObjectType) -> bool {
|
fn has_rows(conn: &Connection, ty: ObjectType) -> bool {
|
||||||
@@ -320,6 +351,7 @@ pub fn run(common: CommonConfig, config: ExportConfig) -> Result<Summary, Error>
|
|||||||
upload_workers: (connected.limits.max_concurrent_upload as usize)
|
upload_workers: (connected.limits.max_concurrent_upload as usize)
|
||||||
.min(ctx.common.threads)
|
.min(ctx.common.threads)
|
||||||
.max(1),
|
.max(1),
|
||||||
|
would_fail: Default::default(),
|
||||||
};
|
};
|
||||||
|
|
||||||
let work = work_list(&ctx.conn, &config, &connected, &logger);
|
let work = work_list(&ctx.conn, &config, &connected, &logger);
|
||||||
@@ -386,8 +418,9 @@ pub fn run(common: CommonConfig, config: ExportConfig) -> Result<Summary, Error>
|
|||||||
}
|
}
|
||||||
|
|
||||||
if ctx.dry_run() {
|
if ctx.dry_run() {
|
||||||
print_dry_run(&dry_rows, config.prune);
|
let would_fail = net.would_fail.lock().expect("would-fail list").clone();
|
||||||
return Ok(Summary::default());
|
print_plan(&summary, &dry_rows, &would_fail, config.prune);
|
||||||
|
return Ok(summary);
|
||||||
}
|
}
|
||||||
summary.retries_observed = ctx.client.retries_observed();
|
summary.retries_observed = ctx.client.retries_observed();
|
||||||
summary.retry_after_sleeps = ctx.client.retry_after_sleeps();
|
summary.retry_after_sleeps = ctx.client.retry_after_sleeps();
|
||||||
@@ -593,21 +626,69 @@ fn sample(ids: &[String]) -> String {
|
|||||||
ids[..n].join(", ")
|
ids[..n].join(", ")
|
||||||
}
|
}
|
||||||
|
|
||||||
fn print_dry_run(rows: &[(&'static str, u64, u64, u64)], prune: bool) {
|
/// The dry run's report, in plain words: per type, what would be created,
|
||||||
if prune {
|
/// updated, left as it is and would fail; then why each failure would happen.
|
||||||
println!(
|
fn print_plan(
|
||||||
"{:<22} {:>10} {:>10} {:>12}",
|
summary: &Summary,
|
||||||
"TYPE", "CREATE", "MATCHED", "WOULD-DESTROY"
|
dry_rows: &[(&'static str, u64, u64, u64)],
|
||||||
);
|
would_fail: &[String],
|
||||||
for (ty, c, m, d) in rows {
|
prune: bool,
|
||||||
println!("{ty:<22} {c:>10} {m:>10} {d:>12}");
|
) {
|
||||||
|
print!("{}", plan_text(summary, dry_rows, would_fail, prune));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn plan_text(
|
||||||
|
summary: &Summary,
|
||||||
|
dry_rows: &[(&'static str, u64, u64, u64)],
|
||||||
|
would_fail: &[String],
|
||||||
|
prune: bool,
|
||||||
|
) -> String {
|
||||||
|
use crate::sync::progress::thousands;
|
||||||
|
let mut out = String::from("Dry run: nothing was written to the target. The plan:\n");
|
||||||
|
for (ty, c) in &summary.per_type {
|
||||||
|
let mut parts: Vec<String> = Vec::new();
|
||||||
|
if c.created > 0 {
|
||||||
|
parts.push(format!("{} to create", thousands(c.created)));
|
||||||
}
|
}
|
||||||
} else {
|
if c.updated > 0 {
|
||||||
println!("{:<22} {:>10} {:>10}", "TYPE", "CREATE", "MATCHED");
|
parts.push(format!("{} to update", thousands(c.updated)));
|
||||||
for (ty, c, m, _) in rows {
|
}
|
||||||
println!("{ty:<22} {c:>10} {m:>10}");
|
if c.skipped > 0 {
|
||||||
|
parts.push(format!("{} unchanged", thousands(c.skipped)));
|
||||||
|
}
|
||||||
|
if c.failed > 0 {
|
||||||
|
parts.push(format!("{} would fail", thousands(c.failed)));
|
||||||
|
}
|
||||||
|
if prune {
|
||||||
|
let gone = dry_rows
|
||||||
|
.iter()
|
||||||
|
.find(|(t, ..)| t == ty)
|
||||||
|
.map(|(.., d)| *d)
|
||||||
|
.unwrap_or(0);
|
||||||
|
if gone > 0 {
|
||||||
|
parts.push(format!("{} to delete (--prune)", thousands(gone)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if parts.is_empty() {
|
||||||
|
parts.push("nothing to do".to_owned());
|
||||||
|
}
|
||||||
|
out.push_str(&format!(" {ty:<20} {}\n", parts.join(", ")));
|
||||||
|
}
|
||||||
|
let failed: u64 = summary.per_type.iter().map(|(_, c)| c.failed).sum();
|
||||||
|
if failed > 0 {
|
||||||
|
out.push_str("Would fail:\n");
|
||||||
|
for line in would_fail {
|
||||||
|
out.push_str(&format!(" {line}\n"));
|
||||||
|
}
|
||||||
|
let unexplained = failed.saturating_sub(would_fail.len() as u64);
|
||||||
|
if unexplained > 0 {
|
||||||
|
out.push_str(&format!(
|
||||||
|
" {} more; the warnings above say why\n",
|
||||||
|
thousands(unexplained)
|
||||||
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
mod tree;
|
mod tree;
|
||||||
@@ -668,7 +749,7 @@ mod common {
|
|||||||
creates: Vec<(String, Value)>,
|
creates: Vec<(String, Value)>,
|
||||||
) -> Result<crate::jmap::request::SetOutcome, JmapError> {
|
) -> Result<crate::jmap::request::SetOutcome, JmapError> {
|
||||||
if net.dry_run {
|
if net.dry_run {
|
||||||
return Ok(synthesize_dry_run_outcome(ty, &creates));
|
return Ok(synthesize_dry_run_outcome(net, ty, &creates));
|
||||||
}
|
}
|
||||||
let mut map = Map::new();
|
let mut map = Map::new();
|
||||||
for (cid, obj) in creates {
|
for (cid, obj) in creates {
|
||||||
@@ -767,12 +848,35 @@ mod common {
|
|||||||
create_batch(net, ty, vec![(cid.to_owned(), wire)]).map_err(Error::from)
|
create_batch(net, ty, vec![(cid.to_owned(), wire)]).map_err(Error::from)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Room left in a request for everything but the object itself: the
|
||||||
|
/// envelope, the method name and the arguments around it.
|
||||||
|
const REQUEST_OVERHEAD: u64 = 512;
|
||||||
|
|
||||||
|
/// What a dry run predicts for `creates`: each one created, except an
|
||||||
|
/// object too big to fit in one request under the target's
|
||||||
|
/// `maxSizeRequest`, which a real run could not send either.
|
||||||
fn synthesize_dry_run_outcome(
|
fn synthesize_dry_run_outcome(
|
||||||
|
net: &Net,
|
||||||
ty: ObjectType,
|
ty: ObjectType,
|
||||||
creates: &[(String, Value)],
|
creates: &[(String, Value)],
|
||||||
) -> crate::jmap::request::SetOutcome {
|
) -> crate::jmap::request::SetOutcome {
|
||||||
let mut outcome = crate::jmap::request::SetOutcome::default();
|
let mut outcome = crate::jmap::request::SetOutcome::default();
|
||||||
for (cid, _) in creates {
|
let cap = net.limits.max_size_request;
|
||||||
|
for (cid, obj) in creates {
|
||||||
|
let size = serde_json::to_vec(obj).map(|v| v.len() as u64).unwrap_or(0);
|
||||||
|
if cap > 0 && size + REQUEST_OVERHEAD > cap {
|
||||||
|
let why = format!(
|
||||||
|
"{} is larger than one request to the target may be ({} maxSizeRequest)",
|
||||||
|
crate::inspect::format_bytes(size),
|
||||||
|
crate::inspect::format_bytes(cap)
|
||||||
|
);
|
||||||
|
net.would_fail(format!("{} {cid}: {why}", ty.jmap_name()));
|
||||||
|
outcome.not_created.push((
|
||||||
|
cid.clone(),
|
||||||
|
serde_json::json!({ "type": "tooLarge", "description": why }),
|
||||||
|
));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
let synthetic = serde_json::json!({
|
let synthetic = serde_json::json!({
|
||||||
"id": format!("dryrun-{}-{cid}", ty.jmap_name())
|
"id": format!("dryrun-{}-{cid}", ty.jmap_name())
|
||||||
});
|
});
|
||||||
@@ -827,3 +931,74 @@ mod pool_tests {
|
|||||||
assert!(out.is_empty());
|
assert!(out.is_empty());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod plan_tests {
|
||||||
|
use super::plan_text;
|
||||||
|
use crate::sync::{Summary, TypeCounts};
|
||||||
|
|
||||||
|
fn summary(rows: &[(&'static str, u64, u64, u64, u64)]) -> Summary {
|
||||||
|
Summary {
|
||||||
|
per_type: rows
|
||||||
|
.iter()
|
||||||
|
.map(|(t, created, updated, skipped, failed)| {
|
||||||
|
(
|
||||||
|
*t,
|
||||||
|
TypeCounts {
|
||||||
|
created: *created,
|
||||||
|
updated: *updated,
|
||||||
|
skipped: *skipped,
|
||||||
|
failed: *failed,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_plan_reads_in_plain_words_and_says_why_things_would_fail() {
|
||||||
|
let s = summary(&[
|
||||||
|
("Mailbox", 0, 0, 12, 0),
|
||||||
|
("Email", 1200, 40, 5000, 2),
|
||||||
|
("SieveScript", 0, 0, 0, 0),
|
||||||
|
]);
|
||||||
|
let text = plan_text(
|
||||||
|
&s,
|
||||||
|
&[],
|
||||||
|
&["Email e7 (message-id <a@b>): 61 MB is larger than the target accepts".to_owned()],
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
text.starts_with("Dry run: nothing was written to the target."),
|
||||||
|
"{text}"
|
||||||
|
);
|
||||||
|
assert!(text.contains("Mailbox 12 unchanged"), "{text}");
|
||||||
|
assert!(
|
||||||
|
text.contains(
|
||||||
|
"Email 1,200 to create, 40 to update, 5,000 unchanged, 2 would fail"
|
||||||
|
),
|
||||||
|
"{text}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
text.contains("SieveScript nothing to do"),
|
||||||
|
"{text}"
|
||||||
|
);
|
||||||
|
assert!(text.contains("Would fail:\n Email e7"), "{text}");
|
||||||
|
assert!(
|
||||||
|
text.contains("1 more; the warnings above say why"),
|
||||||
|
"{text}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn prune_counts_appear_only_with_prune() {
|
||||||
|
let s = summary(&[("ContactCard", 0, 0, 3, 0)]);
|
||||||
|
let rows = [("ContactCard", 0, 3, 4)];
|
||||||
|
assert!(plan_text(&s, &rows, &[], true).contains("3 unchanged, 4 to delete (--prune)"));
|
||||||
|
assert!(!plan_text(&s, &rows, &[], false).contains("delete"));
|
||||||
|
assert!(!plan_text(&s, &rows, &[], false).contains("Would fail"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -209,6 +209,11 @@ fn import_units(
|
|||||||
units[i].local_id,
|
units[i].local_id,
|
||||||
blob_hint(uploader, row)
|
blob_hint(uploader, row)
|
||||||
));
|
));
|
||||||
|
net.would_fail(format!(
|
||||||
|
"Email e{} ({}): its folder is not on the target",
|
||||||
|
units[i].local_id,
|
||||||
|
blob_hint(uploader, row)
|
||||||
|
));
|
||||||
counts.failed += 1;
|
counts.failed += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -234,6 +239,11 @@ fn import_units(
|
|||||||
blob_hint(uploader, row),
|
blob_hint(uploader, row),
|
||||||
size_note(&e)
|
size_note(&e)
|
||||||
));
|
));
|
||||||
|
net.would_fail(format!(
|
||||||
|
"Email {cid} ({}): {}",
|
||||||
|
blob_hint(uploader, row),
|
||||||
|
plain_reason(&e)
|
||||||
|
));
|
||||||
counts.failed += 1;
|
counts.failed += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -617,6 +627,15 @@ fn blob_hint(uploader: &Uploader, row: &EmailRow) -> String {
|
|||||||
s
|
s
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A failure reason for the dry-run plan: the size message on its own, or
|
||||||
|
/// the error as it is.
|
||||||
|
fn plain_reason(e: &JmapError) -> String {
|
||||||
|
match e {
|
||||||
|
JmapError::SingleObjectTooLarge(m) => m.clone(),
|
||||||
|
other => other.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn size_note(e: &JmapError) -> &'static str {
|
fn size_note(e: &JmapError) -> &'static str {
|
||||||
if matches!(
|
if matches!(
|
||||||
e,
|
e,
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
/*
|
/*
|
||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
|
* SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||||
*/
|
*/
|
||||||
@@ -108,6 +109,10 @@ pub fn reconcile(
|
|||||||
ty.jmap_name(),
|
ty.jmap_name(),
|
||||||
id
|
id
|
||||||
));
|
));
|
||||||
|
} else if net.dry_run {
|
||||||
|
// The object only exists in the plan; claiming the
|
||||||
|
// default would be a write.
|
||||||
|
default_claimed = true;
|
||||||
} else {
|
} else {
|
||||||
let mut req = crate::jmap::request::Request::new();
|
let mut req = crate::jmap::request::Request::new();
|
||||||
req.call(
|
req.call(
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ use super::sieve_names;
|
|||||||
use super::{Maps, Net, Plan, Uploader};
|
use super::{Maps, Net, Plan, Uploader};
|
||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
use crate::jmap::blobxfer;
|
use crate::jmap::blobxfer;
|
||||||
|
use crate::jmap::error::JmapError;
|
||||||
use crate::jmap::request::{Request, check_method_error};
|
use crate::jmap::request::{Request, check_method_error};
|
||||||
use crate::logging::{LEVEL_DEFAULT, Logger};
|
use crate::logging::{LEVEL_DEFAULT, Logger};
|
||||||
use crate::sync::import_jmap::mapping::BlobBytes;
|
use crate::sync::import_jmap::mapping::BlobBytes;
|
||||||
@@ -72,6 +73,7 @@ pub fn reconcile(
|
|||||||
.map(|(_, n, _, _)| n.clone().unwrap_or_default());
|
.map(|(_, n, _, _)| n.clone().unwrap_or_default());
|
||||||
|
|
||||||
let mut updates: Vec<(String, Value)> = Vec::new();
|
let mut updates: Vec<(String, Value)> = Vec::new();
|
||||||
|
let mut validator = Validator::default();
|
||||||
|
|
||||||
for (local, name, is_active, blob_local) in &locals {
|
for (local, name, is_active, blob_local) in &locals {
|
||||||
let matched = name.as_ref().and_then(|n| target_by_name.get(n)).cloned();
|
let matched = name.as_ref().and_then(|n| target_by_name.get(n)).cloned();
|
||||||
@@ -90,6 +92,7 @@ pub fn reconcile(
|
|||||||
};
|
};
|
||||||
match content_differs(net, &ours, target_blob.get(&id)) {
|
match content_differs(net, &ours, target_blob.get(&id)) {
|
||||||
Ok(false) => counts.skipped += 1,
|
Ok(false) => counts.skipped += 1,
|
||||||
|
Ok(true) if !validator.accepts(net, label, &ours, counts, logger) => {}
|
||||||
Ok(true) => {
|
Ok(true) => {
|
||||||
let blob = match &rewritten {
|
let blob = match &rewritten {
|
||||||
Some((bytes, renamed)) => {
|
Some((bytes, renamed)) => {
|
||||||
@@ -116,6 +119,15 @@ pub fn reconcile(
|
|||||||
id
|
id
|
||||||
} else {
|
} else {
|
||||||
let cid = format!("c{local}");
|
let cid = format!("c{local}");
|
||||||
|
if net.dry_run {
|
||||||
|
let ours = match &rewritten {
|
||||||
|
Some((bytes, _)) => bytes.clone(),
|
||||||
|
None => uploader.bytes(*blob_local).map_err(Error::from)?,
|
||||||
|
};
|
||||||
|
if !validator.accepts(net, label, &ours, counts, logger) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
if let Some((_, renamed)) = &rewritten {
|
if let Some((_, renamed)) = &rewritten {
|
||||||
log_renames(label, renamed, logger);
|
log_renames(label, renamed, logger);
|
||||||
}
|
}
|
||||||
@@ -217,6 +229,100 @@ pub fn reconcile(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Checks, in a dry run, that the target would accept each script about to
|
||||||
|
/// be written. A script too large to upload, or one `SieveScript/validate`
|
||||||
|
/// rejects, is counted as a failure and listed in the plan. A real run
|
||||||
|
/// checks nothing here: the target's own answer to the write is the check.
|
||||||
|
#[derive(Default)]
|
||||||
|
struct Validator {
|
||||||
|
unsupported: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Validator {
|
||||||
|
/// Whether the script may be written. Always true outside a dry run.
|
||||||
|
fn accepts(
|
||||||
|
&mut self,
|
||||||
|
net: &Net,
|
||||||
|
label: &str,
|
||||||
|
bytes: &[u8],
|
||||||
|
counts: &mut TypeCounts,
|
||||||
|
logger: &Logger,
|
||||||
|
) -> bool {
|
||||||
|
if !net.dry_run || self.unsupported {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
let why = match net.check_upload_size(bytes.len() as u64) {
|
||||||
|
Err(JmapError::SingleObjectTooLarge(m)) => Some(m),
|
||||||
|
Err(e) => Some(e.to_string()),
|
||||||
|
Ok(()) => match validate(net, bytes) {
|
||||||
|
Ok(why) => why,
|
||||||
|
Err(e) if is_unknown_method(&e) => {
|
||||||
|
logger.warn("the target cannot validate Sieve scripts; they are not checked");
|
||||||
|
self.unsupported = true;
|
||||||
|
None
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
logger.warn(&format!("SieveScript {label}: not validated: {e}"));
|
||||||
|
None
|
||||||
|
}
|
||||||
|
},
|
||||||
|
};
|
||||||
|
match why {
|
||||||
|
None => true,
|
||||||
|
Some(why) => {
|
||||||
|
logger.warn(&format!(
|
||||||
|
"SieveScript {label}: the target would reject it: {why}"
|
||||||
|
));
|
||||||
|
net.would_fail(format!(
|
||||||
|
"SieveScript \"{label}\": the target would reject it: {why}"
|
||||||
|
));
|
||||||
|
counts.failed += 1;
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Asks the target whether it would accept `bytes` as a Sieve script: `None`
|
||||||
|
/// if it would, or its reason. The script goes up as a blob, which the
|
||||||
|
/// server keeps only for a while; nothing is created in the account.
|
||||||
|
fn validate(net: &Net, bytes: &[u8]) -> Result<Option<String>, JmapError> {
|
||||||
|
let blob = blobxfer::upload_bytes(
|
||||||
|
&net.client,
|
||||||
|
&net.session,
|
||||||
|
&net.account,
|
||||||
|
"application/sieve",
|
||||||
|
bytes,
|
||||||
|
)?;
|
||||||
|
let mut req = Request::new();
|
||||||
|
req.call(
|
||||||
|
"SieveScript/validate",
|
||||||
|
json!({ "accountId": net.account, "blobId": blob.0 }),
|
||||||
|
"v",
|
||||||
|
);
|
||||||
|
let resp = req.send(&net.client, &net.api)?;
|
||||||
|
let mr = resp.by_call_id("v")?;
|
||||||
|
check_method_error(mr)?;
|
||||||
|
Ok(match mr.args.get("error") {
|
||||||
|
None | Some(Value::Null) => None,
|
||||||
|
Some(err) => Some(
|
||||||
|
err.get("description")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.or_else(|| err.get("type").and_then(Value::as_str))
|
||||||
|
.unwrap_or("rejected")
|
||||||
|
.to_owned(),
|
||||||
|
),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_unknown_method(e: &JmapError) -> bool {
|
||||||
|
match e {
|
||||||
|
JmapError::UnknownMethod => true,
|
||||||
|
JmapError::Method { error_type, .. } => error_type == "unknownMethod",
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// The target's `sieveExtensions`, from its Sieve account capability.
|
/// The target's `sieveExtensions`, from its Sieve account capability.
|
||||||
fn target_sieve_extensions(net: &Net) -> Vec<String> {
|
fn target_sieve_extensions(net: &Net) -> Vec<String> {
|
||||||
net.session
|
net.session
|
||||||
|
|||||||
@@ -109,6 +109,7 @@ pub fn reconcile(
|
|||||||
Err(e) if e.aborts_run() => return Err(e),
|
Err(e) if e.aborts_run() => return Err(e),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
logger.warn(&format!("{} skipped: {e}", describe(ty, *local, uid)));
|
logger.warn(&format!("{} skipped: {e}", describe(ty, *local, uid)));
|
||||||
|
net.would_fail(format!("{}: {e}", describe(ty, *local, uid)));
|
||||||
counts.failed += 1;
|
counts.failed += 1;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,325 @@
|
|||||||
|
/*
|
||||||
|
* SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||||
|
*/
|
||||||
|
|
||||||
|
//! `export --dry-run` predicts what a real run would fail on: a message
|
||||||
|
//! larger than the target accepts, an object too big for one request, and a
|
||||||
|
//! Sieve script the target rejects. It keeps the counts, so the run exits
|
||||||
|
//! non-zero just as the real one would, and it writes nothing.
|
||||||
|
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
|
||||||
|
use inbuxa_migrate::db;
|
||||||
|
use inbuxa_migrate::jmap::account::AccountSelector;
|
||||||
|
use inbuxa_migrate::jmap::http::Auth;
|
||||||
|
use inbuxa_migrate::logging::Logger;
|
||||||
|
use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, Summary, TypeCounts};
|
||||||
|
use mockito::Matcher;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
const API: &str = "/jmap/api";
|
||||||
|
|
||||||
|
fn tmp() -> PathBuf {
|
||||||
|
static SEQ: AtomicUsize = AtomicUsize::new(0);
|
||||||
|
let n = SEQ.fetch_add(1, Ordering::Relaxed);
|
||||||
|
let mut p = std::env::temp_dir();
|
||||||
|
p.push(format!(
|
||||||
|
"inbuxa-migrate-exportdry-{}-{:?}-{n}.sqlite",
|
||||||
|
std::process::id(),
|
||||||
|
std::thread::current().id(),
|
||||||
|
));
|
||||||
|
let _ = std::fs::remove_file(&p);
|
||||||
|
p
|
||||||
|
}
|
||||||
|
|
||||||
|
fn session(base: &str, max_size_upload: u64, max_size_request: u64, sieve: &[&str]) -> String {
|
||||||
|
json!({
|
||||||
|
"apiUrl": format!("{base}{API}"),
|
||||||
|
"uploadUrl": format!("{base}/jmap/upload/{{accountId}}/"),
|
||||||
|
"downloadUrl": format!("{base}/jmap/dl/{{accountId}}/{{blobId}}/{{type}}/{{name}}"),
|
||||||
|
"capabilities": { "urn:ietf:params:jmap:core": {
|
||||||
|
"maxObjectsInGet": 500, "maxObjectsInSet": 500, "maxCallsInRequest": 16,
|
||||||
|
"maxConcurrentRequests": 4, "maxConcurrentUpload": 4,
|
||||||
|
"maxSizeRequest": max_size_request, "maxSizeUpload": max_size_upload
|
||||||
|
} },
|
||||||
|
"accounts": { "w": { "name": "alice",
|
||||||
|
"accountCapabilities": {
|
||||||
|
"urn:ietf:params:jmap:mail": {},
|
||||||
|
"urn:ietf:params:jmap:contacts": {},
|
||||||
|
"urn:ietf:params:jmap:sieve": { "sieveExtensions": sieve }
|
||||||
|
} } }
|
||||||
|
})
|
||||||
|
.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dry_run(archive: &Path, base: &str) -> Summary {
|
||||||
|
sync::export::run(
|
||||||
|
CommonConfig {
|
||||||
|
archive: archive.to_path_buf(),
|
||||||
|
threads: 2,
|
||||||
|
dry_run: true,
|
||||||
|
max_retries: 0,
|
||||||
|
allow_invalid_certs: false,
|
||||||
|
logger: Logger::from_flags(true, 0),
|
||||||
|
},
|
||||||
|
ExportConfig {
|
||||||
|
connect: ConnectConfig {
|
||||||
|
url: base.to_owned(),
|
||||||
|
auth: Auth::Basic {
|
||||||
|
user: "u".into(),
|
||||||
|
password: "p".into(),
|
||||||
|
},
|
||||||
|
account: AccountSelector::Id("w".into()),
|
||||||
|
},
|
||||||
|
objects: None,
|
||||||
|
prune: false,
|
||||||
|
yes: true,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("a dry run returns its plan")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn counts(summary: &Summary, ty: &str) -> TypeCounts {
|
||||||
|
summary
|
||||||
|
.per_type
|
||||||
|
.iter()
|
||||||
|
.find(|(t, _)| *t == ty)
|
||||||
|
.map(|(_, c)| c.clone())
|
||||||
|
.unwrap_or_else(|| panic!("no counts for {ty}: {summary:?}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn root_and_session(server: &mut mockito::ServerGuard, body: String) -> Vec<mockito::Mock> {
|
||||||
|
vec![
|
||||||
|
server.mock("GET", "/").with_status(404).create(),
|
||||||
|
server
|
||||||
|
.mock("GET", "/.well-known/jmap")
|
||||||
|
.with_body(body)
|
||||||
|
.create(),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Nothing that writes to the account: no `/set`, no import. Returns the
|
||||||
|
/// mocks, each expecting no calls.
|
||||||
|
fn no_writes(server: &mut mockito::ServerGuard) -> Vec<mockito::Mock> {
|
||||||
|
["/set\"", "Email/import"]
|
||||||
|
.into_iter()
|
||||||
|
.map(|m| {
|
||||||
|
server
|
||||||
|
.mock("POST", API)
|
||||||
|
.match_body(Matcher::Regex(m.into()))
|
||||||
|
.expect(0)
|
||||||
|
.create()
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn empty(
|
||||||
|
server: &mut mockito::ServerGuard,
|
||||||
|
method: &str,
|
||||||
|
reply: serde_json::Value,
|
||||||
|
) -> mockito::Mock {
|
||||||
|
server
|
||||||
|
.mock("POST", API)
|
||||||
|
.match_body(Matcher::Regex(method.into()))
|
||||||
|
.with_body(json!({ "methodResponses": [[method, reply, "x"]] }).to_string())
|
||||||
|
.create()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_message_too_large_to_upload_is_predicted_to_fail() {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let base = server.url();
|
||||||
|
let archive = tmp();
|
||||||
|
{
|
||||||
|
let conn = db::init::open(&archive).unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO mailboxes (id,name,parent_id,role) VALUES (1,'Inbox',NULL,'inbox')",
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
for body in ["short".to_owned(), "x".repeat(4000)] {
|
||||||
|
let raw = format!(
|
||||||
|
"From: a@x\r\nSubject: s\r\nMessage-ID: <{}@h>\r\n\r\n{body}",
|
||||||
|
body.len()
|
||||||
|
);
|
||||||
|
let blob = db::blobs::intern_blob(&conn, raw.as_bytes()).unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords)
|
||||||
|
VALUES (?1,'2020-01-01T00:00:00Z','[1]','[]')",
|
||||||
|
rusqlite::params![blob],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let _s = root_and_session(&mut server, session(&base, 1000, 10_000_000, &[]));
|
||||||
|
let _mq = empty(
|
||||||
|
&mut server,
|
||||||
|
"Mailbox/query",
|
||||||
|
json!({"accountId":"w","ids":[]}),
|
||||||
|
);
|
||||||
|
let _eq = empty(
|
||||||
|
&mut server,
|
||||||
|
"Email/query",
|
||||||
|
json!({"accountId":"w","ids":[]}),
|
||||||
|
);
|
||||||
|
let no_upload = server
|
||||||
|
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
|
||||||
|
.expect(0)
|
||||||
|
.create();
|
||||||
|
let writes = no_writes(&mut server);
|
||||||
|
|
||||||
|
let summary = dry_run(&archive, &base);
|
||||||
|
let email = counts(&summary, "Email");
|
||||||
|
assert_eq!(email.created, 1, "the short one would be created");
|
||||||
|
assert_eq!(email.failed, 1, "the long one is over maxSizeUpload");
|
||||||
|
assert!(
|
||||||
|
summary.any_failed(),
|
||||||
|
"so the dry run exits non-zero, like a real run"
|
||||||
|
);
|
||||||
|
no_upload.assert();
|
||||||
|
for w in writes {
|
||||||
|
w.assert();
|
||||||
|
}
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_contact_too_large_for_one_request_is_predicted_to_fail() {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let base = server.url();
|
||||||
|
let archive = tmp();
|
||||||
|
{
|
||||||
|
let conn = db::init::open(&archive).unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO address_books (id,name,is_default) VALUES (1,'Personal',1)",
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let photo = db::blobs::intern_blob(&conn, &vec![b'P'; 8000]).unwrap();
|
||||||
|
let huge = json!({ "@type": "Card", "name": { "full": "Photo Person" },
|
||||||
|
"media": { "photo": { "@type": "Media", "kind": "photo",
|
||||||
|
"@blob": photo, "mediaType": "image/png" } } })
|
||||||
|
.to_string();
|
||||||
|
let small = json!({ "@type": "Card", "name": { "full": "Small Person" } }).to_string();
|
||||||
|
for (id, uid, data) in [(1, "huge-card", &huge), (2, "small-card", &small)] {
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO contact_cards (id,uid,address_book_ids,data) VALUES (?1,?2,'[1]',?3)",
|
||||||
|
rusqlite::params![id, uid, data],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let _s = root_and_session(&mut server, session(&base, 50_000_000, 4000, &[]));
|
||||||
|
let _ab = empty(
|
||||||
|
&mut server,
|
||||||
|
"AddressBook/get",
|
||||||
|
json!({"accountId":"w","list":[],"notFound":[]}),
|
||||||
|
);
|
||||||
|
let _cq = empty(
|
||||||
|
&mut server,
|
||||||
|
"ContactCard/query",
|
||||||
|
json!({"accountId":"w","ids":[]}),
|
||||||
|
);
|
||||||
|
let writes = no_writes(&mut server);
|
||||||
|
|
||||||
|
let summary = dry_run(&archive, &base);
|
||||||
|
let cards = counts(&summary, "ContactCard");
|
||||||
|
assert_eq!(cards.created, 1, "the small card would be created");
|
||||||
|
assert_eq!(
|
||||||
|
cards.failed, 1,
|
||||||
|
"the card with the photo inlined is over maxSizeRequest"
|
||||||
|
);
|
||||||
|
assert!(summary.any_failed());
|
||||||
|
for w in writes {
|
||||||
|
w.assert();
|
||||||
|
}
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One active script with Stalwart's `vnd.stalwart.while`, against a target
|
||||||
|
/// that names it `vnd.inbuxa.while`; `validate` is the answer to
|
||||||
|
/// `SieveScript/validate`. Checks that the script sent for validation is the
|
||||||
|
/// renamed one, and returns the run's counts.
|
||||||
|
fn dry_run_one_script(validate: serde_json::Value) -> Summary {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let base = server.url();
|
||||||
|
let archive = tmp();
|
||||||
|
{
|
||||||
|
let conn = db::init::open(&archive).unwrap();
|
||||||
|
let blob = db::blobs::intern_blob(
|
||||||
|
&conn,
|
||||||
|
b"require [\"fileinto\", \"vnd.stalwart.while\"];\nkeep;\n",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO sieve_scripts (id,name,is_active,blob_id) VALUES (1,'main',1,?1)",
|
||||||
|
rusqlite::params![blob],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let _s = root_and_session(
|
||||||
|
&mut server,
|
||||||
|
session(
|
||||||
|
&base,
|
||||||
|
50_000_000,
|
||||||
|
10_000_000,
|
||||||
|
&["fileinto", "vnd.inbuxa.while"],
|
||||||
|
),
|
||||||
|
);
|
||||||
|
let _get = empty(
|
||||||
|
&mut server,
|
||||||
|
"SieveScript/get",
|
||||||
|
json!({"accountId":"w","list":[],"notFound":[]}),
|
||||||
|
);
|
||||||
|
let upload = server
|
||||||
|
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
|
||||||
|
.match_body(Matcher::Regex("vnd\\.inbuxa\\.while".into()))
|
||||||
|
.with_body(json!({"blobId":"TMP"}).to_string())
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let _validate = server
|
||||||
|
.mock("POST", API)
|
||||||
|
.match_body(Matcher::Regex("SieveScript/validate".into()))
|
||||||
|
.with_body(json!({"methodResponses":[validate]}).to_string())
|
||||||
|
.create();
|
||||||
|
let writes = no_writes(&mut server);
|
||||||
|
let summary = dry_run(&archive, &base);
|
||||||
|
for w in writes {
|
||||||
|
w.assert();
|
||||||
|
}
|
||||||
|
upload.assert();
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
summary
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_sieve_script_the_target_rejects_is_predicted_to_fail() {
|
||||||
|
let summary = dry_run_one_script(json!(["SieveScript/validate",
|
||||||
|
{"accountId":"w","error":{"type":"invalidScript","description":"unknown test"}},"v"]));
|
||||||
|
let sieve = counts(&summary, "SieveScript");
|
||||||
|
assert_eq!(sieve.failed, 1);
|
||||||
|
assert_eq!(sieve.created, 0);
|
||||||
|
assert!(summary.any_failed());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_valid_sieve_script_is_validated_after_renaming_and_nothing_is_written() {
|
||||||
|
let summary = dry_run_one_script(json!(["SieveScript/validate",
|
||||||
|
{"accountId":"w","error":null},"v"]));
|
||||||
|
let sieve = counts(&summary, "SieveScript");
|
||||||
|
assert_eq!(sieve.created, 1, "it would be created");
|
||||||
|
assert_eq!(sieve.failed, 0);
|
||||||
|
assert!(!summary.any_failed());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_target_without_validate_still_gets_a_plan() {
|
||||||
|
let summary = dry_run_one_script(json!(["error",
|
||||||
|
{"type":"unknownMethod"},"v"]));
|
||||||
|
let sieve = counts(&summary, "SieveScript");
|
||||||
|
assert_eq!(sieve.created, 1, "not checked, so planned as written");
|
||||||
|
assert_eq!(sieve.failed, 0);
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user