v1.0.8 (fixes #29 fixes #30 fixes #31)

This commit is contained in:
Maurus Decimus
2026-08-16 12:29:05 +02:00
parent c4e06bd87d
commit 1cc633f39c
43 changed files with 4082 additions and 686 deletions
+2 -1
View File
@@ -7,4 +7,5 @@ target
*.md *.md
!README.md !README.md
!CHANGELOG.md !CHANGELOG.md
specs/ .*
!.github
+15
View File
@@ -2,6 +2,21 @@
All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/).
## [1.0.8] - 2026-08-15
### Added
- A warning when the session advertises `apiUrl`, `uploadUrl` or `downloadUrl` on a different origin than the one connected to, naming both origins (#29).
- The per-type report is printed when a run aborts.
### Changed
### Fixed
- A transient `serverUnavailable` from `Email/import` permanently dropped that message from the migration instead of retrying it.
- The Exchange Graph importer stored a whole converted event as a `recurrenceOverrides` value (#31).
- Exporting a ContactCard carrying a photo blob failed against Stalwart, and a CalendarEvent enclosure sent as a `blobId` was silently dropped by the server; the bytes of both are now inlined as a base64 `data:` URI (#30).
- A connection failure against the session-advertised `apiUrl` was reported as a per-type partial failure (exit 5, "consistent and resumable") instead of aborting (#29).
- Session `apiUrl`, `uploadUrl` and `downloadUrl` given as relative references were used unresolved (#29).
## [1.0.7] - 2026-07-26 ## [1.0.7] - 2026-07-26
### Added ### Added
Generated
+298 -319
View File
File diff suppressed because it is too large Load Diff
+3 -3
View File
@@ -1,7 +1,7 @@
[package] [package]
name = "vandelay" name = "vandelay"
description = "JMAP account migration utility" description = "JMAP account migration utility"
version = "1.0.7" version = "1.0.8"
authors = ["Stalwart Labs LLC <[email protected]>"] authors = ["Stalwart Labs LLC <[email protected]>"]
license = "Apache-2.0 OR MIT" license = "Apache-2.0 OR MIT"
repository = "https://github.com/stalwartlabs/vandelay" repository = "https://github.com/stalwartlabs/vandelay"
@@ -36,7 +36,7 @@ ureq = { version = "3.3.0", default-features = false, features = ["json", "gzip"
clap = { version = "4.6.1", features = ["derive"] } clap = { version = "4.6.1", features = ["derive"] }
crossbeam-channel = "0.5.15" crossbeam-channel = "0.5.15"
num_cpus = "1.17.0" num_cpus = "1.17.0"
base64 = "0.22.1" base64 = "0.23"
mail-parser = "0.11.3" mail-parser = "0.11.3"
calcard = "0.3" calcard = "0.3"
rustls = { version = "0.23.40", default-features = false, features = ["aws-lc-rs", "std", "tls12", "logging"] } rustls = { version = "0.23.40", default-features = false, features = ["aws-lc-rs", "std", "tls12", "logging"] }
@@ -50,7 +50,7 @@ chrono-tz = { version = "0.10", default-features = false }
[dev-dependencies] [dev-dependencies]
mockito = "1.7.2" mockito = "1.7.2"
tempfile = "3.13" tempfile = "3.13"
testcontainers = { version = "0.27", features = ["blocking"] } testcontainers = { version = "0.28", features = ["blocking"] }
[profile.dist] [profile.dist]
inherits = "release" inherits = "release"
+60 -6
View File
@@ -12,7 +12,7 @@ use regex::Regex;
use crate::error::Error; use crate::error::Error;
use crate::exchange_ews::oauth::OAuthFlow; use crate::exchange_ews::oauth::OAuthFlow;
use crate::exchange_ews::types::MailboxKind; use crate::exchange_ews::types::MailboxKind;
use crate::exchange_graph::types::{EventBodyFormat, MailboxKind as GraphMailboxKind}; use crate::exchange_graph::types::{EventBodyFormat, MailboxKind as GraphMailboxKind, Surfaces};
use crate::inspect::InspectConfig; use crate::inspect::InspectConfig;
use crate::jmap::account::AccountSelector; use crate::jmap::account::AccountSelector;
use crate::jmap::http::Auth; use crate::jmap::http::Auth;
@@ -1132,7 +1132,7 @@ pub struct ExchangeGraphImportArgs {
#[arg( #[arg(
long, long,
value_name = "LIST", value_name = "LIST",
help = "Comma-separated surface list (mail,calendar,contacts; default all)" help = "Comma-separated surface list: mail | calendar | contacts (default: all three)"
)] )]
objects: Option<String>, objects: Option<String>,
@@ -1203,11 +1203,12 @@ fn resolve_exchange_graph_import(args: ExchangeGraphImportArgs) -> Result<Action
args.client_id.as_deref(), args.client_id.as_deref(),
&args.tenant, &args.tenant,
)?; )?;
let objects = args let surfaces = args
.objects .objects
.as_deref() .as_deref()
.map(crate::types::parse_object_list) .map(Surfaces::parse_list)
.transpose()?; .transpose()?
.unwrap_or_default();
let top = args.top.clamp(1, 1000); let top = args.top.clamp(1, 1000);
let graph_connections = args let graph_connections = args
.graph_connections .graph_connections
@@ -1220,7 +1221,7 @@ fn resolve_exchange_graph_import(args: ExchangeGraphImportArgs) -> Result<Action
api_base: args.api_base, api_base: args.api_base,
user_target: args.user, user_target: args.user,
mailbox_kind, mailbox_kind,
objects, surfaces,
event_body_format, event_body_format,
graph_connections, graph_connections,
top, top,
@@ -1361,4 +1362,57 @@ mod tests {
let auth = resolve_auth(None, None, Some(&bearer)).unwrap(); let auth = resolve_auth(None, None, Some(&bearer)).unwrap();
assert!(matches!(auth, Auth::Bearer { token } if token == "t")); assert!(matches!(auth, Auth::Bearer { token } if token == "t"));
} }
fn graph_config(objects: &[&str]) -> Result<GraphImportConfig, Error> {
let mut argv = vec![
"vandelay",
"import",
"exchange-graph",
"--access-token",
"tok",
];
argv.extend_from_slice(objects);
argv.push("archive.sqlite");
let cli = Cli::try_parse_from(argv).unwrap_or_else(|e| panic!("clap rejected args: {e}"));
match cli.resolve()? {
Action::ImportExchangeGraph(_, config) => Ok(config),
_ => panic!("expected ImportExchangeGraph"),
}
}
#[test]
fn graph_objects_contacts_selects_only_contacts() {
let _g = lock();
let config = graph_config(&["--objects", "contacts"]).unwrap();
assert_eq!(
config.surfaces,
Surfaces {
mail: false,
calendar: false,
contacts: true
}
);
}
#[test]
fn graph_objects_absent_selects_every_surface() {
let _g = lock();
let config = graph_config(&[]).unwrap();
assert_eq!(config.surfaces, Surfaces::ALL);
}
#[test]
fn graph_objects_rejects_jmap_type_names() {
let _g = lock();
match graph_config(&["--objects", "contactcard"]) {
Err(Error::Usage(m)) => {
assert!(m.contains("unknown surface: contactcard"), "msg was: {m}");
assert!(
m.contains("valid: mail, calendar, contacts"),
"msg was: {m}"
);
}
other => panic!("expected Usage error, got {:?}", other.map(|_| ())),
}
}
} }
+47
View File
@@ -43,6 +43,20 @@ impl From<rusqlite::Error> for Error {
} }
impl Error { impl Error {
pub fn aborts_run(&self) -> bool {
match self {
Error::Partial(_) => false,
Error::Usage(_)
| Error::Connection(_)
| Error::Account(_)
| Error::SourceChange(_)
| Error::PruneAborted
| Error::Unimplemented(_)
| Error::Db(_)
| Error::Io(_) => true,
}
}
pub fn exit_code(&self) -> i32 { pub fn exit_code(&self) -> i32 {
match self { match self {
Error::Usage(_) => 1, Error::Usage(_) => 1,
@@ -74,4 +88,37 @@ mod tests {
let io = Error::Io(std::io::Error::other("disk")); let io = Error::Io(std::io::Error::other("disk"));
assert_eq!(io.exit_code(), 7); assert_eq!(io.exit_code(), 7);
} }
#[test]
fn only_partial_is_a_per_unit_failure() {
assert!(!Error::Partial("one object".into()).aborts_run());
}
#[test]
fn whole_run_conditions_abort() {
assert!(Error::Connection("http status 404".into()).aborts_run());
assert!(Error::Account("ambiguous".into()).aborts_run());
assert!(Error::SourceChange("other account".into()).aborts_run());
assert!(Error::Usage("bad flag".into()).aborts_run());
assert!(Error::Unimplemented("x").aborts_run());
assert!(Error::PruneAborted.aborts_run());
assert!(Error::Db(OpenError::Sqlite(rusqlite::Error::QueryReturnedNoRows)).aborts_run());
assert!(Error::Io(std::io::Error::other("disk")).aborts_run());
}
#[test]
fn aborting_errors_do_not_map_to_the_partial_exit_code() {
for e in [
Error::Connection("x".into()),
Error::Account("x".into()),
Error::SourceChange("x".into()),
Error::Db(OpenError::Sqlite(rusqlite::Error::QueryReturnedNoRows)),
Error::Io(std::io::Error::other("disk")),
] {
assert!(e.aborts_run(), "{e} must abort the run");
assert_ne!(e.exit_code(), 5, "{e} must not report a partial failure");
}
assert_eq!(Error::Connection("x".into()).exit_code(), 2);
assert_eq!(Error::Partial("x".into()).exit_code(), 5);
}
} }
+210
View File
@@ -0,0 +1,210 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*/
use serde_json::{Map, Value};
const IGNORED_EXACT: &[&str] = &[
"@type",
"method",
"organizerCalendarAddress",
"privacy",
"prodId",
"recurrenceId",
"recurrenceIdTimeZone",
"sentBy",
"uid",
];
const IGNORED_FIRST_TOKEN: &[&str] = &["recurrenceOverrides", "recurrenceRule", "relatedTo"];
const CALENDAR_ADDRESS_DEPENDENTS: &[&str] = &[
"calendarAddress",
"delegatedFrom",
"delegatedTo",
"email",
"expectReply",
"kind",
"memberOf",
"participationStatus",
"progress",
"roles",
"sentBy",
];
pub fn synthetic_attendee_address(identifier: &str) -> String {
format!(
"urn:x-vandelay:attendee:{}",
blake3::hash(identifier.as_bytes()).to_hex()
)
}
pub fn drop_calendar_address_dependents(participants: &mut Map<String, Value>) {
for participant in participants.values_mut() {
let Some(object) = participant.as_object_mut() else {
continue;
};
for key in CALENDAR_ADDRESS_DEPENDENTS {
object.remove(*key);
}
}
}
pub fn is_override_ignored(pointer: &str) -> bool {
if IGNORED_EXACT.contains(&pointer) || is_participant_calendar_address(pointer) {
return true;
}
let first = pointer.split('/').next().unwrap_or(pointer);
IGNORED_FIRST_TOKEN.contains(&first)
}
fn is_participant_calendar_address(pointer: &str) -> bool {
let mut tokens = pointer.split('/');
tokens.next() == Some("participants")
&& tokens.next().is_some()
&& tokens.next() == Some("calendarAddress")
&& tokens.next().is_none()
}
pub fn override_patch_from_event(event: &Value) -> Value {
let Some(object) = event.as_object() else {
return event.clone();
};
let patch: Map<String, Value> = object
.iter()
.filter(|(key, _)| !is_override_ignored(key))
.map(|(key, value)| (key.clone(), value.clone()))
.collect();
Value::Object(patch)
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn exact_pointers_are_ignored() {
for pointer in [
"@type",
"method",
"organizerCalendarAddress",
"privacy",
"prodId",
"recurrenceId",
"recurrenceIdTimeZone",
"sentBy",
"uid",
] {
assert!(is_override_ignored(pointer), "{pointer} must be ignored");
}
}
#[test]
fn first_token_pointers_are_ignored_at_any_depth() {
assert!(is_override_ignored("recurrenceRule"));
assert!(is_override_ignored("recurrenceRule/frequency"));
assert!(is_override_ignored(
"recurrenceOverrides/2026-01-01T09:00:00"
));
assert!(is_override_ignored("relatedTo/uid-1/relation"));
}
#[test]
fn participant_calendar_address_is_ignored_only_at_its_own_depth() {
assert!(is_override_ignored("participants/att-1/calendarAddress"));
assert!(!is_override_ignored("participants"));
assert!(!is_override_ignored("participants/att-1"));
assert!(!is_override_ignored(
"participants/att-1/participationStatus"
));
assert!(!is_override_ignored(
"participants/att-1/calendarAddress/extra"
));
}
#[test]
fn patchable_pointers_survive() {
for pointer in [
"start",
"duration",
"title",
"excluded",
"participants",
"created",
"updated",
"status",
] {
assert!(!is_override_ignored(pointer), "{pointer} must survive");
}
}
#[test]
fn an_event_is_reduced_to_a_patch() {
let event = json!({
"@type": "Event",
"uid": "uid-1",
"prodId": "vandelay",
"privacy": "public",
"organizerCalendarAddress": "mailto:[email protected]",
"recurrenceRule": {"frequency": "daily"},
"start": "2026-03-04T09:00:00",
"duration": "PT1H",
"title": "Moved"
});
let patch = override_patch_from_event(&event);
let patch = patch.as_object().expect("patch object");
assert_eq!(patch.len(), 3);
assert_eq!(patch["start"], "2026-03-04T09:00:00");
assert_eq!(patch["duration"], "PT1H");
assert_eq!(patch["title"], "Moved");
}
#[test]
fn an_exclusion_patch_is_preserved() {
let patch = override_patch_from_event(&json!({"excluded": true}));
assert_eq!(patch, json!({"excluded": true}));
}
#[test]
fn a_synthetic_address_is_stable_per_identifier_and_never_a_mailto() {
let addr = synthetic_attendee_address("Jane Doe");
assert_eq!(addr, synthetic_attendee_address("Jane Doe"));
assert_ne!(addr, synthetic_attendee_address("John Doe"));
assert!(addr.starts_with("urn:x-vandelay:attendee:"));
assert!(
!addr.starts_with("mailto:"),
"export must never invite a fabricated address"
);
}
#[test]
fn calendar_address_dependents_are_dropped_and_identity_is_kept() {
let mut participants = json!({
"1": {
"@type": "Participant",
"name": "Room 4",
"calendarAddress": "urn:x-vandelay:attendee:abc",
"email": "[email protected]",
"roles": {"required": true},
"kind": "resource",
"participationStatus": "accepted",
"expectReply": true,
"description": "Third floor"
}
})
.as_object()
.expect("participants object")
.clone();
drop_calendar_address_dependents(&mut participants);
let p = &participants["1"];
assert_eq!(p["@type"], "Participant");
assert_eq!(p["name"], "Room 4");
assert_eq!(p["description"], "Third floor");
for key in CALENDAR_ADDRESS_DEPENDENTS {
assert!(p.get(*key).is_none(), "{key} must be dropped");
}
}
}
+1
View File
@@ -4,4 +4,5 @@
* SPDX-License-Identifier: Apache-2.0 OR MIT * SPDX-License-Identifier: Apache-2.0 OR MIT
*/ */
pub mod jscalendar;
pub mod tz; pub mod tz;
+48 -25
View File
@@ -6,6 +6,9 @@
use serde_json::{Map, Value, json}; use serde_json::{Map, Value, json};
use crate::exchange::jscalendar::{
drop_calendar_address_dependents, is_override_ignored, synthetic_attendee_address,
};
use crate::exchange_ews::parse::{CalendarItemRaw, RawAttendee, RawOccurrence}; use crate::exchange_ews::parse::{CalendarItemRaw, RawAttendee, RawOccurrence};
use crate::exchange_ews::recurrence::to_jscalendar_rule; use crate::exchange_ews::recurrence::to_jscalendar_rule;
use crate::exchange_ews::tz::resolve_to_iana; use crate::exchange_ews::tz::resolve_to_iana;
@@ -181,6 +184,9 @@ fn build_event_map(raw: &CalendarItemRaw, iana: Option<&str>) -> Map<String, Val
AttendeeRole::Resource, AttendeeRole::Resource,
); );
if !participants.is_empty() { if !participants.is_empty() {
if !event.contains_key("organizerCalendarAddress") {
drop_calendar_address_dependents(&mut participants);
}
event.insert("participants".to_owned(), Value::Object(participants)); event.insert("participants".to_owned(), Value::Object(participants));
} }
if let Some(rec) = raw.recurrence.as_ref() if let Some(rec) = raw.recurrence.as_ref()
@@ -229,13 +235,6 @@ fn build_event_map(raw: &CalendarItemRaw, iana: Option<&str>) -> Map<String, Val
event event
} }
fn synthetic_attendee_address(identifier: &str) -> String {
format!(
"urn:x-vandelay:attendee:{}",
blake3::hash(identifier.as_bytes()).to_hex()
)
}
fn is_smtp_routing(routing_type: Option<&str>) -> bool { fn is_smtp_routing(routing_type: Option<&str>) -> bool {
routing_type.is_none_or(|rt| rt.eq_ignore_ascii_case("SMTP")) routing_type.is_none_or(|rt| rt.eq_ignore_ascii_case("SMTP"))
} }
@@ -320,21 +319,11 @@ fn add_attendees(
} }
} }
const OVERRIDE_IGNORED_POINTERS: &[&str] = &[ const NOT_PATCHED_PER_OCCURRENCE: &[&str] = &["created", "updated"];
"@type",
"uid", fn is_override_excluded(key: &str) -> bool {
"recurrenceRule", is_override_ignored(key) || NOT_PATCHED_PER_OCCURRENCE.contains(&key)
"recurrenceOverrides", }
"recurrenceId",
"recurrenceIdTimeZone",
"method",
"organizerCalendarAddress",
"privacy",
"prodId",
"relatedTo",
"created",
"updated",
];
fn build_recurrence_overrides( fn build_recurrence_overrides(
base_event: &Map<String, Value>, base_event: &Map<String, Value>,
@@ -388,7 +377,7 @@ fn override_patch(
let mut patch = Map::new(); let mut patch = Map::new();
let inherited_start = Value::String(recurrence_id.to_owned()); let inherited_start = Value::String(recurrence_id.to_owned());
for (k, v) in occ_event { for (k, v) in occ_event {
if OVERRIDE_IGNORED_POINTERS.contains(&k.as_str()) { if is_override_excluded(k) {
continue; continue;
} }
let baseline = if k == "start" { let baseline = if k == "start" {
@@ -401,7 +390,7 @@ fn override_patch(
} }
} }
for k in base_event.keys() { for k in base_event.keys() {
if k == "start" || OVERRIDE_IGNORED_POINTERS.contains(&k.as_str()) { if k == "start" || is_override_excluded(k) {
continue; continue;
} }
if !occ_event.contains_key(k) { if !occ_event.contains_key(k) {
@@ -710,6 +699,7 @@ mod tests {
uid: Some("u".to_owned()), uid: Some("u".to_owned()),
start: Some("2025-06-15T14:00:00Z".to_owned()), start: Some("2025-06-15T14:00:00Z".to_owned()),
end: Some("2025-06-15T15:00:00Z".to_owned()), end: Some("2025-06-15T15:00:00Z".to_owned()),
organizer_smtp: Some("[email protected]".to_owned()),
required_attendees: vec![crate::exchange_ews::parse::RawAttendee { required_attendees: vec![crate::exchange_ews::parse::RawAttendee {
email: None, email: None,
routing_type: None, routing_type: None,
@@ -796,6 +786,7 @@ mod tests {
uid: Some("u".to_owned()), uid: Some("u".to_owned()),
start: Some("2025-06-15T14:00:00Z".to_owned()), start: Some("2025-06-15T14:00:00Z".to_owned()),
end: Some("2025-06-15T15:00:00Z".to_owned()), end: Some("2025-06-15T15:00:00Z".to_owned()),
organizer_smtp: Some("[email protected]".to_owned()),
required_attendees: vec![RawAttendee { required_attendees: vec![RawAttendee {
email: Some("[email protected]".to_owned()), email: Some("[email protected]".to_owned()),
routing_type: Some("SMTP".to_owned()), routing_type: Some("SMTP".to_owned()),
@@ -809,7 +800,7 @@ mod tests {
.as_object() .as_object()
.unwrap() .unwrap()
.values() .values()
.next() .find(|p| p["email"] == "[email protected]")
.unwrap(); .unwrap();
assert_eq!(att["calendarAddress"], "mailto:[email protected]"); assert_eq!(att["calendarAddress"], "mailto:[email protected]");
assert_eq!(att["email"], "[email protected]"); assert_eq!(att["email"], "[email protected]");
@@ -984,6 +975,7 @@ mod tests {
uid: Some("uid-res".to_owned()), uid: Some("uid-res".to_owned()),
start: Some("2025-06-15T14:00:00Z".to_owned()), start: Some("2025-06-15T14:00:00Z".to_owned()),
end: Some("2025-06-15T15:00:00Z".to_owned()), end: Some("2025-06-15T15:00:00Z".to_owned()),
organizer_smtp: Some("[email protected]".to_owned()),
resources: vec![crate::exchange_ews::parse::RawAttendee { resources: vec![crate::exchange_ews::parse::RawAttendee {
email: Some("room-7@x".to_owned()), email: Some("room-7@x".to_owned()),
routing_type: None, routing_type: None,
@@ -1011,6 +1003,7 @@ mod tests {
uid: Some("uid-rsvp".to_owned()), uid: Some("uid-rsvp".to_owned()),
start: Some("2025-06-15T14:00:00Z".to_owned()), start: Some("2025-06-15T14:00:00Z".to_owned()),
end: Some("2025-06-15T15:00:00Z".to_owned()), end: Some("2025-06-15T15:00:00Z".to_owned()),
organizer_smtp: Some("[email protected]".to_owned()),
required_attendees: vec![crate::exchange_ews::parse::RawAttendee { required_attendees: vec![crate::exchange_ews::parse::RawAttendee {
email: Some("bob@x".to_owned()), email: Some("bob@x".to_owned()),
routing_type: None, routing_type: None,
@@ -1031,6 +1024,36 @@ mod tests {
assert_eq!(bob["roles"]["required"], true); assert_eq!(bob["roles"]["required"], true);
} }
#[test]
fn attendees_lose_calendar_addresses_when_no_organizer_is_known() {
let raw = CalendarItemRaw {
uid: Some("uid-no-organizer".to_owned()),
start: Some("2025-06-15T14:00:00Z".to_owned()),
end: Some("2025-06-15T15:00:00Z".to_owned()),
required_attendees: vec![crate::exchange_ews::parse::RawAttendee {
email: Some("bob@x".to_owned()),
routing_type: None,
name: Some("Bob".to_owned()),
response_type: Some("Accept".to_owned()),
}],
..CalendarItemRaw::default()
};
let v = to_jscalendar(&raw).data;
assert!(v.get("organizerCalendarAddress").is_none());
let bob = v["participants"]
.as_object()
.unwrap()
.values()
.find(|p| p["name"] == "Bob")
.unwrap();
for key in ["calendarAddress", "email", "roles", "participationStatus"] {
assert!(
bob.get(key).is_none(),
"{key} requires calendarAddress, which requires organizerCalendarAddress"
);
}
}
#[test] #[test]
fn all_day_span_helper_is_inclusive_of_start() { fn all_day_span_helper_is_inclusive_of_start() {
assert_eq!(all_day_span_days("2025-07-04", "2025-07-04"), 1); assert_eq!(all_day_span_days("2025-07-04", "2025-07-04"), 1);
+130 -32
View File
@@ -6,6 +6,7 @@
use serde_json::{Map, Value, json}; use serde_json::{Map, Value, json};
use crate::exchange::jscalendar::{drop_calendar_address_dependents, synthetic_attendee_address};
use crate::exchange::tz::resolve_to_iana; use crate::exchange::tz::resolve_to_iana;
use crate::exchange_graph::error::GraphError; use crate::exchange_graph::error::GraphError;
use crate::exchange_graph::recurrence::convert_patterned_recurrence_rule; use crate::exchange_graph::recurrence::convert_patterned_recurrence_rule;
@@ -242,13 +243,8 @@ pub fn convert_event(
.get("organizer") .get("organizer")
.and_then(|o| o.get("emailAddress")) .and_then(|o| o.get("emailAddress"))
{ {
if let Some(addr) = org.get("address").and_then(Value::as_str) if let Some((cal_addr, _)) = resolve_calendar_address(org) {
&& !addr.is_empty() card.insert("organizerCalendarAddress".to_owned(), Value::from(cal_addr));
{
card.insert(
"organizerCalendarAddress".to_owned(),
Value::from(format!("mailto:{addr}")),
);
} }
let p = build_participant(org, &["owner", "chair"], None, None); let p = build_participant(org, &["owner", "chair"], None, None);
participants.push(("organizer".to_owned(), p)); participants.push(("organizer".to_owned(), p));
@@ -275,7 +271,10 @@ pub fn convert_event(
} }
} }
if !participants.is_empty() { if !participants.is_empty() {
let map: Map<String, Value> = participants.into_iter().collect(); let mut map: Map<String, Value> = participants.into_iter().collect();
if !card.contains_key("organizerCalendarAddress") {
drop_calendar_address_dependents(&mut map);
}
card.insert("participants".to_owned(), Value::Object(map)); card.insert("participants".to_owned(), Value::Object(map));
} }
@@ -346,6 +345,23 @@ fn extract_local_datetime(slot: Option<&Value>) -> Option<String> {
Some(dt.to_owned()) Some(dt.to_owned())
} }
fn non_empty(email: &Value, key: &str) -> Option<String> {
email
.get(key)
.and_then(Value::as_str)
.map(str::trim)
.filter(|s| !s.is_empty())
.map(str::to_owned)
}
fn resolve_calendar_address(email: &Value) -> Option<(String, Option<String>)> {
if let Some(addr) = non_empty(email, "address") {
let cal_addr = format!("mailto:{addr}");
return Some((cal_addr, Some(addr)));
}
non_empty(email, "name").map(|name| (synthetic_attendee_address(&name), None))
}
fn build_participant( fn build_participant(
email: &Value, email: &Value,
roles: &[&str], roles: &[&str],
@@ -354,31 +370,28 @@ fn build_participant(
) -> Value { ) -> Value {
let mut map = Map::new(); let mut map = Map::new();
map.insert("@type".to_owned(), Value::from("Participant")); map.insert("@type".to_owned(), Value::from("Participant"));
if let Some(addr) = email.get("address").and_then(Value::as_str) { if let Some(name) = non_empty(email, "name") {
map.insert( map.insert("name".to_owned(), Value::from(name));
"calendarAddress".to_owned(),
Value::from(format!("mailto:{addr}")),
);
map.insert("email".to_owned(), Value::from(addr.to_owned()));
} }
if let Some(name) = email.get("name").and_then(Value::as_str) if let Some((cal_addr, smtp)) = resolve_calendar_address(email) {
&& !name.is_empty() map.insert("calendarAddress".to_owned(), Value::from(cal_addr));
{ if let Some(addr) = smtp {
map.insert("name".to_owned(), Value::from(name.to_owned())); map.insert("email".to_owned(), Value::from(addr));
} }
let role_map: Map<String, Value> = roles let role_map: Map<String, Value> = roles
.iter() .iter()
.map(|r| ((*r).to_owned(), Value::Bool(true))) .map(|r| ((*r).to_owned(), Value::Bool(true)))
.collect(); .collect();
map.insert("roles".to_owned(), Value::Object(role_map)); map.insert("roles".to_owned(), Value::Object(role_map));
if let Some(status) = participation_status { if let Some(status) = participation_status {
map.insert( map.insert(
"participationStatus".to_owned(), "participationStatus".to_owned(),
Value::from(status.to_owned()), Value::from(status.to_owned()),
); );
} }
if let Some(b) = expect_reply { if let Some(b) = expect_reply {
map.insert("expectReply".to_owned(), Value::Bool(b)); map.insert("expectReply".to_owned(), Value::Bool(b));
}
} }
Value::Object(map) Value::Object(map)
} }
@@ -650,6 +663,91 @@ mod tests {
assert_eq!(participants["att-1"]["roles"]["required"], true); assert_eq!(participants["att-1"]["roles"]["required"], true);
} }
#[test]
fn a_name_only_attendee_gets_a_stable_synthetic_calendar_address() {
let mut v = sample();
v["attendees"] = json!([
{
"emailAddress": {"name": "Room 4"},
"type": "resource",
"status": {"response": "accepted"}
}
]);
let conv = convert_event(&v, None).unwrap();
let p = &conv.data["participants"]["att-1"];
let addr = p["calendarAddress"].as_str().unwrap();
assert!(
addr.starts_with("urn:x-vandelay:attendee:"),
"a name-only attendee needs a synthetic calendarAddress, got {addr}"
);
assert!(
p.get("email").is_none(),
"there is no SMTP address to report as email"
);
assert_eq!(p["participationStatus"], "accepted");
assert_eq!(p["roles"]["informational"], true);
assert_eq!(
addr,
convert_event(&v, None).unwrap().data["participants"]["att-1"]["calendarAddress"]
);
}
#[test]
fn an_unidentifiable_attendee_keeps_no_calendar_address_dependent_property() {
let mut v = sample();
v["attendees"] = json!([
{"emailAddress": {}, "type": "required", "status": {"response": "accepted"}}
]);
let conv = convert_event(&v, None).unwrap();
let p = &conv.data["participants"]["att-1"];
for key in [
"calendarAddress",
"email",
"roles",
"participationStatus",
"expectReply",
] {
assert!(
p.get(key).is_none(),
"{key} requires calendarAddress (jscalendarbis 3.4.6)"
);
}
assert_eq!(p["@type"], "Participant");
}
#[test]
fn an_organizer_without_an_address_still_yields_a_calendar_address() {
let mut v = sample();
v["organizer"] = json!({"emailAddress": {"name": "Alice Example"}});
let conv = convert_event(&v, None).unwrap();
let organizer_address = conv.data["organizerCalendarAddress"].as_str().unwrap();
assert!(organizer_address.starts_with("urn:x-vandelay:attendee:"));
assert_eq!(
conv.data["participants"]["organizer"]["calendarAddress"],
organizer_address
);
}
#[test]
fn participants_drop_calendar_addresses_when_no_organizer_is_known() {
let mut v = sample();
v.as_object_mut().unwrap().remove("organizer");
let conv = convert_event(&v, None).unwrap();
assert!(
conv.data.get("organizerCalendarAddress").is_none(),
"no organizer is derivable from the source event"
);
let participants = conv.data["participants"].as_object().unwrap();
assert!(!participants.is_empty());
for (key, participant) in participants {
assert!(
participant.get("calendarAddress").is_none(),
"{key} may not carry a calendarAddress without organizerCalendarAddress"
);
assert!(participant.get("roles").is_none(), "{key} kept roles");
}
}
#[test] #[test]
fn optional_attendee_keeps_role_and_uses_status_when_present() { fn optional_attendee_keeps_role_and_uses_status_when_present() {
let mut v = sample(); let mut v = sample();
+133
View File
@@ -59,6 +59,59 @@ impl EventBodyFormat {
} }
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Surfaces {
pub mail: bool,
pub calendar: bool,
pub contacts: bool,
}
impl Default for Surfaces {
fn default() -> Self {
Surfaces::ALL
}
}
impl Surfaces {
pub const ALL: Surfaces = Surfaces {
mail: true,
calendar: true,
contacts: true,
};
pub const NONE: Surfaces = Surfaces {
mail: false,
calendar: false,
contacts: false,
};
pub fn parse_list(list: &str) -> Result<Surfaces, Error> {
let mut selected = Surfaces::NONE;
for token in list.split(',') {
let token = token.trim();
if token.is_empty() {
continue;
}
match token.to_ascii_lowercase().as_str() {
"mail" => selected.mail = true,
"calendar" => selected.calendar = true,
"contacts" => selected.contacts = true,
_ => {
return Err(Error::Usage(format!(
"unknown surface: {token} (valid: mail, calendar, contacts)"
)));
}
}
}
if selected == Surfaces::NONE {
return Err(Error::Usage(
"--objects given but resolved to an empty surface list".to_owned(),
));
}
Ok(selected)
}
}
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct ResolvedPrincipal { pub struct ResolvedPrincipal {
pub id: String, pub id: String,
@@ -102,6 +155,86 @@ mod tests {
); );
} }
#[test]
fn surfaces_default_is_all_three() {
let all = Surfaces::default();
assert!(all.mail && all.calendar && all.contacts);
}
#[test]
fn surfaces_parse_each_name() {
assert_eq!(
Surfaces::parse_list("mail").unwrap(),
Surfaces {
mail: true,
calendar: false,
contacts: false
}
);
assert_eq!(
Surfaces::parse_list("calendar").unwrap(),
Surfaces {
mail: false,
calendar: true,
contacts: false
}
);
assert_eq!(
Surfaces::parse_list("contacts").unwrap(),
Surfaces {
mail: false,
calendar: false,
contacts: true
}
);
}
#[test]
fn surfaces_parse_is_case_insensitive() {
assert_eq!(
Surfaces::parse_list("Mail,CALENDAR,Contacts").unwrap(),
Surfaces::ALL
);
}
#[test]
fn surfaces_parse_dedups_and_trims() {
assert_eq!(
Surfaces::parse_list(" mail , contacts ,mail,, ").unwrap(),
Surfaces {
mail: true,
calendar: false,
contacts: true
}
);
}
#[test]
fn surfaces_parse_rejects_unknown_name() {
let err = Surfaces::parse_list("mail,contact").unwrap_err();
let msg = err.to_string();
assert!(msg.contains("unknown surface: contact"), "msg was: {msg}");
assert!(
msg.contains("valid: mail, calendar, contacts"),
"msg was: {msg}"
);
}
#[test]
fn surfaces_parse_rejects_jmap_type_names() {
assert!(Surfaces::parse_list("mailbox").is_err());
assert!(Surfaces::parse_list("contactcard").is_err());
}
#[test]
fn surfaces_parse_rejects_empty_list() {
let err = Surfaces::parse_list(" , ").unwrap_err();
assert!(
err.to_string().contains("empty surface list"),
"msg was: {err}"
);
}
#[test] #[test]
fn body_format_prefer_value() { fn body_format_prefer_value() {
assert_eq!( assert_eq!(
+437 -10
View File
@@ -4,10 +4,32 @@
* SPDX-License-Identifier: Apache-2.0 OR MIT * SPDX-License-Identifier: Apache-2.0 OR MIT
*/ */
use base64::Engine;
use base64::engine::general_purpose::STANDARD;
use serde_json::{Map, Value}; use serde_json::{Map, Value};
use crate::jmap::error::JmapError;
pub const SENTINEL_KEY: &str = "@blob"; pub const SENTINEL_KEY: &str = "@blob";
const DEFAULT_MEDIA_TYPE: &str = "application/octet-stream";
const MEDIA_TYPE_KEYS: [&str; 2] = ["mediaType", "contentType"];
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum InlineShape {
JsContactResource,
JsCalendarLink,
}
impl InlineShape {
fn uri_key(self) -> &'static str {
match self {
InlineShape::JsContactResource => "uri",
InlineShape::JsCalendarLink => "href",
}
}
}
pub fn import_blob_ids<F>(value: &mut Value, mut resolve: F) -> Result<(), BlobWalkError> pub fn import_blob_ids<F>(value: &mut Value, mut resolve: F) -> Result<(), BlobWalkError>
where where
F: FnMut(&str) -> Result<i64, BlobWalkError>, F: FnMut(&str) -> Result<i64, BlobWalkError>,
@@ -39,31 +61,36 @@ where
Ok(()) Ok(())
} }
pub fn export_blob_ids<F>(value: &mut Value, mut resolve: F) -> Result<(), BlobWalkError> pub fn inline_blob_data_uris<F>(
value: &mut Value,
shape: InlineShape,
mut fetch: F,
) -> Result<(), BlobWalkError>
where where
F: FnMut(i64) -> Result<String, BlobWalkError>, F: FnMut(i64) -> Result<Vec<u8>, BlobWalkError>,
{ {
walk_export(value, &mut resolve) walk_inline(value, shape, &mut fetch)
} }
fn walk_export<F>(value: &mut Value, resolve: &mut F) -> Result<(), BlobWalkError> fn walk_inline<F>(value: &mut Value, shape: InlineShape, fetch: &mut F) -> Result<(), BlobWalkError>
where where
F: FnMut(i64) -> Result<String, BlobWalkError>, F: FnMut(i64) -> Result<Vec<u8>, BlobWalkError>,
{ {
match value { match value {
Value::Object(map) => { Value::Object(map) => {
if let Some(sentinel) = map.remove(SENTINEL_KEY) { if let Some(sentinel) = map.remove(SENTINEL_KEY) {
let local_id = sentinel.as_i64().ok_or(BlobWalkError::MalformedSentinel)?; let local_id = sentinel.as_i64().ok_or(BlobWalkError::MalformedSentinel)?;
let target_jmap_id = resolve(local_id)?; let bytes = fetch(local_id)?;
map.insert("blobId".to_owned(), Value::String(target_jmap_id)); let uri = data_uri(map, &bytes);
map.insert(shape.uri_key().to_owned(), Value::String(uri));
} }
for child in map.values_mut() { for child in map.values_mut() {
walk_export(child, resolve)?; walk_inline(child, shape, fetch)?;
} }
} }
Value::Array(items) => { Value::Array(items) => {
for item in items { for item in items {
walk_export(item, resolve)?; walk_inline(item, shape, fetch)?;
} }
} }
_ => {} _ => {}
@@ -71,6 +98,70 @@ where
Ok(()) Ok(())
} }
fn data_uri(map: &Map<String, Value>, bytes: &[u8]) -> String {
let media_type = MEDIA_TYPE_KEYS
.iter()
.find_map(|k| map.get(*k).and_then(Value::as_str).and_then(uri_media_type))
.unwrap_or_else(|| DEFAULT_MEDIA_TYPE.to_owned());
let mut uri = String::with_capacity(13 + media_type.len() + bytes.len().div_ceil(3) * 4);
uri.push_str("data:");
uri.push_str(&media_type);
uri.push_str(";base64,");
STANDARD.encode_string(bytes, &mut uri);
uri
}
fn uri_media_type(raw: &str) -> Option<String> {
let mut parts = raw.split(';').map(str::trim);
let essence = parts.next()?;
let (ty, subtype) = essence.split_once('/')?;
if ty.is_empty() || subtype.is_empty() || subtype.contains('/') {
return None;
}
let mut out = String::with_capacity(raw.len());
escape_into(&mut out, essence);
for parameter in parts.filter(|p| !p.is_empty()) {
out.push(';');
escape_into(&mut out, parameter);
}
Some(out)
}
fn escape_into(out: &mut String, part: &str) {
const HEX: &[u8; 16] = b"0123456789ABCDEF";
for byte in part.bytes() {
if is_media_type_uri_byte(byte) {
out.push(byte as char);
} else {
out.push('%');
out.push(HEX[(byte >> 4) as usize] as char);
out.push(HEX[(byte & 0x0F) as usize] as char);
}
}
}
fn is_media_type_uri_byte(byte: u8) -> bool {
byte.is_ascii_alphanumeric()
|| matches!(
byte,
b'!' | b'$'
| b'&'
| b'\''
| b'('
| b')'
| b'*'
| b'+'
| b'-'
| b'.'
| b'/'
| b':'
| b'='
| b'@'
| b'_'
| b'~'
)
}
pub fn prepend_property(target: &mut Map<String, Value>, key: &str, value: Value) { pub fn prepend_property(target: &mut Map<String, Value>, key: &str, value: Value) {
let existing: Vec<(String, Value)> = let existing: Vec<(String, Value)> =
target.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); target.iter().map(|(k, v)| (k.clone(), v.clone())).collect();
@@ -88,5 +179,341 @@ pub enum BlobWalkError {
#[error("malformed @blob sentinel: expected integer local id")] #[error("malformed @blob sentinel: expected integer local id")]
MalformedSentinel, MalformedSentinel,
#[error("blob resolver failed: {0}")] #[error("blob resolver failed: {0}")]
Resolver(String), Resolver(Box<JmapError>),
}
impl BlobWalkError {
pub fn resolver(source: JmapError) -> BlobWalkError {
BlobWalkError::Resolver(Box::new(source))
}
pub fn into_source(self) -> JmapError {
match self {
BlobWalkError::Resolver(source) => *source,
other => JmapError::Blob(other),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn fetch_ok(local_id: i64) -> Result<Vec<u8>, BlobWalkError> {
Ok(format!("bytes-{local_id}").into_bytes())
}
fn decode(uri: &str, expect_media_type: &str) -> Vec<u8> {
assert_well_formed(uri);
let prefix = format!("data:{expect_media_type};base64,");
let payload = uri
.strip_prefix(&prefix)
.unwrap_or_else(|| panic!("{uri} does not start with {prefix}"));
STANDARD.decode(payload).expect("base64 payload")
}
fn assert_well_formed(uri: &str) {
let rest = uri
.strip_prefix("data:")
.unwrap_or_else(|| panic!("{uri} is not a data URI"));
let (media_type, payload) = rest
.split_once(";base64,")
.unwrap_or_else(|| panic!("{uri} carries no ;base64, separator"));
let bytes = media_type.as_bytes();
let mut i = 0;
while i < bytes.len() {
let b = bytes[i];
if b == b'%' {
assert!(
i + 2 < bytes.len()
&& bytes[i + 1].is_ascii_hexdigit()
&& bytes[i + 2].is_ascii_hexdigit(),
"truncated percent escape in {uri}"
);
i += 3;
continue;
}
assert!(
is_media_type_uri_byte(b) || b == b';',
"byte {b:#04x} is not allowed unescaped in {uri}"
);
i += 1;
}
assert!(
payload
.bytes()
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'/' | b'=')),
"non-base64 payload in {uri}"
);
}
fn assert_no_blob_id(value: &Value) {
match value {
Value::Object(map) => {
assert!(map.get("blobId").is_none(), "blobId survived in {value}");
assert!(map.get(SENTINEL_KEY).is_none(), "@blob survived in {value}");
for child in map.values() {
assert_no_blob_id(child);
}
}
Value::Array(items) => items.iter().for_each(assert_no_blob_id),
_ => {}
}
}
#[test]
fn media_sentinel_becomes_a_uri_property() {
let mut card = json!({
"@type": "Card",
"media": { "photo": {
"@type": "Media", "kind": "photo",
"@blob": 7, "mediaType": "image/png"
} }
});
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap();
let photo = &card["media"]["photo"];
assert_eq!(
decode(photo["uri"].as_str().unwrap(), "image/png"),
b"bytes-7"
);
assert_eq!(photo["mediaType"], json!("image/png"));
assert!(photo.get("href").is_none());
assert_no_blob_id(&card);
}
#[test]
fn calendar_link_sentinel_becomes_an_href_property() {
let mut event = json!({
"@type": "Event",
"links": { "1": {
"@type": "Link", "rel": "enclosure",
"@blob": 3, "contentType": "text/plain", "title": "note.txt"
} }
});
inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap();
let link = &event["links"]["1"];
assert_eq!(
decode(link["href"].as_str().unwrap(), "text/plain"),
b"bytes-3"
);
assert_eq!(link["contentType"], json!("text/plain"));
assert_eq!(link["title"], json!("note.txt"));
assert!(link.get("uri").is_none());
assert_no_blob_id(&event);
}
#[test]
fn jscontact_link_keeps_the_resource_uri_property() {
let mut card = json!({
"@type": "Card",
"links": { "l1": { "@type": "Link", "@blob": 1, "mediaType": "application/pdf" } }
});
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap();
let link = &card["links"]["l1"];
assert!(link.get("href").is_none(), "JSContact Link uses uri");
assert_eq!(
decode(link["uri"].as_str().unwrap(), "application/pdf"),
b"bytes-1"
);
}
#[test]
fn absent_media_type_defaults_to_octet_stream() {
let mut card = json!({ "media": { "photo": { "@blob": 2 } } });
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap();
let uri = card["media"]["photo"]["uri"].as_str().unwrap();
assert_eq!(decode(uri, DEFAULT_MEDIA_TYPE), b"bytes-2");
}
#[test]
fn empty_media_type_defaults_to_octet_stream() {
let mut event = json!({ "links": { "1": { "@blob": 4, "contentType": "" } } });
inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap();
let uri = event["links"]["1"]["href"].as_str().unwrap();
assert_eq!(decode(uri, DEFAULT_MEDIA_TYPE), b"bytes-4");
}
#[test]
fn empty_media_type_falls_through_to_the_next_key() {
let mut card = json!({ "media": { "photo": {
"@blob": 4, "mediaType": "", "contentType": "image/jpeg"
} } });
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap();
let uri = card["media"]["photo"]["uri"].as_str().unwrap();
assert_eq!(decode(uri, "image/jpeg"), b"bytes-4");
}
#[test]
fn media_type_parameters_are_kept_without_whitespace() {
let mut event = json!({ "links": { "1": {
"@blob": 4, "contentType": "text/plain; charset=utf-8"
} } });
inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap();
let uri = event["links"]["1"]["href"].as_str().unwrap();
assert!(!uri.contains(' '), "{uri} carries a raw space");
assert_eq!(decode(uri, "text/plain;charset=utf-8"), b"bytes-4");
}
#[test]
fn media_type_keeps_several_parameters_and_drops_empty_ones() {
let mut event = json!({ "links": { "1": {
"@blob": 4, "contentType": " text/plain ;; charset=utf-8 ; format=flowed "
} } });
inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap();
let uri = event["links"]["1"]["href"].as_str().unwrap();
assert_eq!(
decode(uri, "text/plain;charset=utf-8;format=flowed"),
b"bytes-4"
);
}
#[test]
fn media_type_characters_illegal_in_a_uri_are_percent_encoded() {
let mut event = json!({ "links": { "1": {
"@blob": 4, "contentType": "text/plain; name=\"rep ort,v2#1\u{e9}.txt\""
} } });
inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap();
let uri = event["links"]["1"]["href"].as_str().unwrap();
assert_eq!(
decode(uri, "text/plain;name=%22rep%20ort%2Cv2%231%C3%A9.txt%22"),
b"bytes-4"
);
}
#[test]
fn a_percent_in_the_media_type_is_itself_escaped() {
let mut card = json!({ "media": { "photo": {
"@blob": 4, "mediaType": "image/png; note=100%"
} } });
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap();
let uri = card["media"]["photo"]["uri"].as_str().unwrap();
assert_eq!(decode(uri, "image/png;note=100%25"), b"bytes-4");
}
#[test]
fn a_media_type_that_is_not_type_slash_subtype_defaults_to_octet_stream() {
for bogus in ["png", "image/", "/png", "image/png/extra", ";charset=utf-8"] {
let mut card = json!({ "media": { "photo": { "@blob": 4, "mediaType": bogus } } });
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap();
let uri = card["media"]["photo"]["uri"].as_str().unwrap();
assert_eq!(decode(uri, DEFAULT_MEDIA_TYPE), b"bytes-4", "{bogus}");
}
}
#[test]
fn inlined_bytes_round_trip_exactly() {
let payload: Vec<u8> = (0u16..=255).map(|b| b as u8).collect();
let mut card = json!({ "media": { "photo": {
"@blob": 1, "mediaType": "application/octet-stream; charset=binary"
} } });
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, |_| {
Ok(payload.clone())
})
.unwrap();
let uri = card["media"]["photo"]["uri"].as_str().unwrap();
assert_eq!(
decode(uri, "application/octet-stream;charset=binary"),
payload
);
}
#[test]
fn sentinels_nested_in_arrays_and_subobjects_are_all_inlined() {
let mut value = json!({
"locations": { "loc1": { "links": [
{ "@blob": 5, "contentType": "image/gif" },
{ "@blob": 6 }
] } },
"links": { "top": { "@blob": 8, "contentType": "image/gif" } }
});
inline_blob_data_uris(&mut value, InlineShape::JsCalendarLink, fetch_ok).unwrap();
let arr = value["locations"]["loc1"]["links"].as_array().unwrap();
assert_eq!(
decode(arr[0]["href"].as_str().unwrap(), "image/gif"),
b"bytes-5"
);
assert_eq!(
decode(arr[1]["href"].as_str().unwrap(), DEFAULT_MEDIA_TYPE),
b"bytes-6"
);
assert_eq!(
decode(value["links"]["top"]["href"].as_str().unwrap(), "image/gif"),
b"bytes-8"
);
assert_no_blob_id(&value);
}
#[test]
fn malformed_sentinel_is_rejected() {
let mut value = json!({ "media": { "photo": { "@blob": "not-an-id" } } });
let err = inline_blob_data_uris(&mut value, InlineShape::JsContactResource, fetch_ok)
.expect_err("malformed sentinel");
assert!(matches!(err, BlobWalkError::MalformedSentinel), "{err:?}");
}
#[test]
fn resolver_failure_propagates() {
let mut value = json!({ "media": { "photo": { "@blob": 9 } } });
let err = inline_blob_data_uris(&mut value, InlineShape::JsContactResource, |_| {
Err(BlobWalkError::resolver(JmapError::malformed("gone")))
})
.expect_err("resolver failure");
assert!(matches!(err, BlobWalkError::Resolver(_)), "{err:?}");
}
#[test]
fn the_resolver_error_kind_survives_the_walk() {
let mut value = json!({ "media": { "photo": { "@blob": 9 } } });
let err = inline_blob_data_uris(&mut value, InlineShape::JsContactResource, |_| {
Err(BlobWalkError::resolver(JmapError::Sqlite(
rusqlite::Error::QueryReturnedNoRows,
)))
})
.expect_err("resolver failure")
.into_source();
assert!(matches!(err, JmapError::Sqlite(_)), "{err:?}");
let mapped = crate::error::Error::from(err);
assert!(mapped.aborts_run(), "{mapped} must abort the run");
assert_eq!(mapped.exit_code(), 7);
}
#[test]
fn a_malformed_sentinel_stays_a_blob_walk_error() {
let err = BlobWalkError::MalformedSentinel.into_source();
assert!(
matches!(err, JmapError::Blob(BlobWalkError::MalformedSentinel)),
"{err:?}"
);
assert_eq!(crate::error::Error::from(err).exit_code(), 5);
}
#[test]
fn values_without_a_sentinel_are_untouched() {
let original = json!({
"@type": "Card",
"media": { "photo": { "@type": "Media", "kind": "photo",
"uri": "https://example.test/p.png", "mediaType": "image/png" } },
"nickNames": { "n1": { "name": "Jay" } }
});
let mut value = original.clone();
inline_blob_data_uris(&mut value, InlineShape::JsContactResource, |_| {
panic!("fetch must not be called")
})
.unwrap();
assert_eq!(value, original);
}
#[test]
fn import_walker_still_produces_the_sentinel() {
let mut value =
json!({ "media": { "photo": { "blobId": "B1", "mediaType": "image/png" } } });
import_blob_ids(&mut value, |id| {
assert_eq!(id, "B1");
Ok(11)
})
.unwrap();
assert_eq!(value["media"]["photo"][SENTINEL_KEY], json!(11));
assert!(value["media"]["photo"].get("blobId").is_none());
}
} }
+3
View File
@@ -31,6 +31,9 @@ pub fn prepare(ctx: &Context, connect: &ConnectConfig) -> Result<Connected, Erro
ctx.client.set_logger(ctx.common.logger); ctx.client.set_logger(ctx.common.logger);
let session = Session::discover(&ctx.client, &connect.url)?; let session = Session::discover(&ctx.client, &connect.url)?;
for mismatch in session.origin_mismatches(&connect.url) {
ctx.common.logger.warn(&mismatch.to_string());
}
let limits = session.core_limits()?; let limits = session.core_limits()?;
ctx.client.set_limits(&limits); ctx.client.set_limits(&limits);
let account_id = account::resolve(&connect.account, &session, &ctx.client)?; let account_id = account::resolve(&connect.account, &session, &ctx.client)?;
+57 -1
View File
@@ -4,6 +4,7 @@
* SPDX-License-Identifier: Apache-2.0 OR MIT * SPDX-License-Identifier: Apache-2.0 OR MIT
*/ */
use crate::db::init::OpenError;
use crate::error::Error; use crate::error::Error;
use crate::jmap::blob::BlobWalkError; use crate::jmap::blob::BlobWalkError;
@@ -70,7 +71,62 @@ impl From<JmapError> for Error {
match value { match value {
JmapError::Connect(m) | JmapError::Transport(m) => Error::Connection(m), JmapError::Connect(m) | JmapError::Transport(m) => Error::Connection(m),
JmapError::Auth(m) => Error::Connection(format!("authentication rejected: {m}")), JmapError::Auth(m) => Error::Connection(format!("authentication rejected: {m}")),
other => Error::Connection(other.to_string()), JmapError::Sqlite(e) => Error::Db(OpenError::Sqlite(e)),
reached @ (JmapError::HttpStatus { .. } | JmapError::RetriesExhausted(_)) => {
Error::Connection(reached.to_string())
}
per_unit => Error::Partial(per_unit.to_string()),
} }
} }
} }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn unreachable_endpoint_aborts_the_run() {
for e in [
JmapError::Connect("refused".to_owned()),
JmapError::Transport("io: broken pipe".to_owned()),
JmapError::Auth("bad password".to_owned()),
JmapError::HttpStatus {
status: 404,
body: "<html>404 Not Found</html>".to_owned(),
},
JmapError::RetriesExhausted("gave up".to_owned()),
] {
let mapped = Error::from(e);
assert!(mapped.aborts_run(), "{mapped} must abort the run");
assert_eq!(mapped.exit_code(), 2);
}
}
#[test]
fn protocol_level_errors_stay_per_unit() {
for e in [
JmapError::Method {
call_id: "q".to_owned(),
error_type: "unknownMethod".to_owned(),
description: None,
},
JmapError::UnknownMethod,
JmapError::CannotCalculateChanges,
JmapError::AnchorNotFound,
JmapError::RequestTooLarge,
JmapError::SingleObjectTooLarge("one blob".to_owned()),
JmapError::Malformed("not a list".to_owned()),
] {
let mapped = Error::from(e);
assert!(!mapped.aborts_run(), "{mapped} must not abort the run");
assert_eq!(mapped.exit_code(), 5);
}
}
#[test]
fn archive_failures_are_local_io() {
let mapped = Error::from(JmapError::Sqlite(rusqlite::Error::QueryReturnedNoRows));
assert!(mapped.aborts_run());
assert_eq!(mapped.exit_code(), 7);
}
}
+5 -2
View File
@@ -617,9 +617,11 @@ fn format_rate_headers(headers: &[(String, String)]) -> Option<String> {
Some(parts.join("; ")) Some(parts.join("; "))
} }
fn format_retry_wait(d: Duration) -> String { pub fn format_retry_wait(d: Duration) -> String {
let total = d.as_secs(); let total = d.as_secs();
if total < 60 { if total == 0 {
format!("{}ms", d.as_millis())
} else if total < 60 {
format!("{total}s") format!("{total}s")
} else if total < 3600 { } else if total < 3600 {
format!("{}m{:02}s", total / 60, total % 60) format!("{}m{:02}s", total / 60, total % 60)
@@ -826,6 +828,7 @@ mod tests {
#[test] #[test]
fn format_retry_wait_short_medium_long() { fn format_retry_wait_short_medium_long() {
assert_eq!(format_retry_wait(Duration::from_millis(250)), "250ms");
assert_eq!(format_retry_wait(Duration::from_secs(7)), "7s"); assert_eq!(format_retry_wait(Duration::from_secs(7)), "7s");
assert_eq!(format_retry_wait(Duration::from_secs(67)), "1m07s"); assert_eq!(format_retry_wait(Duration::from_secs(67)), "1m07s");
assert_eq!(format_retry_wait(Duration::from_secs(1427)), "23m47s"); assert_eq!(format_retry_wait(Duration::from_secs(1427)), "23m47s");
+129 -1
View File
@@ -9,9 +9,11 @@ use serde::de::DeserializeOwned;
use serde_json::{Map, Value, json}; use serde_json::{Map, Value, json};
use crate::jmap::error::JmapError; use crate::jmap::error::JmapError;
use crate::jmap::http::HttpClient; use crate::jmap::http::{HttpClient, format_retry_wait};
use crate::jmap::retry::{Disposition, MethodCallKind, backoff_delay, jmap_method_disposition};
use crate::jmap::session::Limits; use crate::jmap::session::Limits;
use crate::jmap::wire::JmapId; use crate::jmap::wire::JmapId;
use crate::logging::Logger;
pub const URN_CORE: &str = "urn:ietf:params:jmap:core"; pub const URN_CORE: &str = "urn:ietf:params:jmap:core";
@@ -184,6 +186,43 @@ pub fn check_method_error(mr: &MethodCall) -> Result<(), JmapError> {
}) })
} }
fn method_error_is_retryable(err: &JmapError, kind: MethodCallKind) -> bool {
match err {
JmapError::Method { error_type, .. } => {
jmap_method_disposition(error_type, kind) == Disposition::Retryable
}
_ => false,
}
}
pub fn retry_method_call<T>(
client: &HttpClient,
kind: MethodCallKind,
logger: &Logger,
mut attempt: impl FnMut() -> Result<T, JmapError>,
) -> Result<T, JmapError> {
let policy = *client.retry();
let mut tries = 0u32;
loop {
let err = match attempt() {
Ok(value) => return Ok(value),
Err(e) => e,
};
if tries >= policy.max_retries || !method_error_is_retryable(&err, kind) {
return Err(err);
}
tries += 1;
let delay = backoff_delay(&policy, tries);
client.rate_limit().cooldown().arm(delay);
logger.warn(&format!(
"{err}; retrying in {} ({tries}/{})",
format_retry_wait(delay),
policy.max_retries
));
std::thread::sleep(delay);
}
}
fn ids_array(ids: &[JmapId]) -> Value { fn ids_array(ids: &[JmapId]) -> Value {
Value::Array(ids.iter().map(|i| Value::String(i.0.clone())).collect()) Value::Array(ids.iter().map(|i| Value::String(i.0.clone())).collect())
} }
@@ -720,7 +759,96 @@ fn decode_set(mr: &MethodCall) -> SetOutcome {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use std::cell::Cell;
use super::*; use super::*;
use crate::error::Error;
use crate::jmap::http::{Auth, RetryPolicy};
fn test_client(max_retries: u32) -> HttpClient {
HttpClient::new(
Auth::Bearer {
token: "t".to_owned(),
},
RetryPolicy::new(max_retries),
false,
)
}
fn method_error(error_type: &str) -> JmapError {
JmapError::Method {
call_id: "i".to_owned(),
error_type: error_type.to_owned(),
description: None,
}
}
fn drive(
max_retries: u32,
outcome: impl Fn(u32) -> Result<&'static str, JmapError>,
) -> (Result<&'static str, JmapError>, u32) {
let client = test_client(max_retries);
let calls = Cell::new(0u32);
let result = retry_method_call(
&client,
MethodCallKind::SingleObjectWrite,
&Logger::from_flags(true, 0),
|| {
calls.set(calls.get() + 1);
outcome(calls.get())
},
);
(result, calls.get())
}
#[test]
fn transient_method_error_is_retried_until_it_succeeds() {
let (result, calls) = drive(3, |n| {
if n < 3 {
Err(method_error("serverUnavailable"))
} else {
Ok("created")
}
});
assert_eq!(result.unwrap(), "created");
assert_eq!(calls, 3);
}
#[test]
fn exhausting_the_budget_returns_the_original_method_error() {
let (result, calls) = drive(2, |_| Err(method_error("serverUnavailable")));
assert_eq!(calls, 3, "one initial attempt plus max_retries");
let err = result.unwrap_err();
assert!(
matches!(&err, JmapError::Method { error_type, .. } if error_type == "serverUnavailable"),
"the caller must still see the method error, not RetriesExhausted: {err}"
);
assert!(
!Error::from(err).aborts_run(),
"an exhausted method retry stays a per-unit failure"
);
}
#[test]
fn permanent_method_error_consumes_no_attempt() {
let (result, calls) = drive(5, |_| Err(method_error("invalidArguments")));
assert!(result.is_err());
assert_eq!(calls, 1);
}
#[test]
fn partial_fail_is_not_retried_for_a_write() {
let (result, calls) = drive(5, |_| Err(method_error("serverPartialFail")));
assert!(result.is_err());
assert_eq!(calls, 1, "retrying a partial write could duplicate it");
}
#[test]
fn non_method_errors_are_left_to_the_http_layer() {
let (result, calls) = drive(5, |_| Err(JmapError::Transport("reset".to_owned())));
assert!(result.is_err());
assert_eq!(calls, 1);
}
#[test] #[test]
fn using_union_includes_core_and_dedups() { fn using_union_includes_core_and_dedups() {
+35 -4
View File
@@ -27,9 +27,19 @@ pub fn classify_http_status(status: u16) -> Disposition {
} }
} }
pub fn jmap_method_disposition(error_type: &str) -> Disposition { #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MethodCallKind {
Read,
SingleObjectWrite,
}
pub fn jmap_method_disposition(error_type: &str, kind: MethodCallKind) -> Disposition {
match error_type { match error_type {
"serverUnavailable" | "serverPartialFail" => Disposition::Retryable, "serverUnavailable" => Disposition::Retryable,
"serverPartialFail" => match kind {
MethodCallKind::Read => Disposition::Retryable,
MethodCallKind::SingleObjectWrite => Disposition::Fatal,
},
_ => Disposition::Fatal, _ => Disposition::Fatal,
} }
} }
@@ -231,12 +241,33 @@ mod tests {
#[test] #[test]
fn method_classification() { fn method_classification() {
for kind in [MethodCallKind::Read, MethodCallKind::SingleObjectWrite] {
assert_eq!(
jmap_method_disposition("serverUnavailable", kind),
Disposition::Retryable,
"{kind:?}"
);
assert_eq!(
jmap_method_disposition("invalidArguments", kind),
Disposition::Fatal,
"{kind:?}"
);
assert_eq!(
jmap_method_disposition("forbidden", kind),
Disposition::Fatal,
"{kind:?}"
);
}
}
#[test]
fn partial_fail_is_never_retried_for_a_write() {
assert_eq!( assert_eq!(
jmap_method_disposition("serverUnavailable"), jmap_method_disposition("serverPartialFail", MethodCallKind::Read),
Disposition::Retryable Disposition::Retryable
); );
assert_eq!( assert_eq!(
jmap_method_disposition("invalidArguments"), jmap_method_disposition("serverPartialFail", MethodCallKind::SingleObjectWrite),
Disposition::Fatal Disposition::Fatal
); );
} }
+265 -2
View File
@@ -58,12 +58,100 @@ fn well_known_url(base: &str) -> String {
format!("{trimmed}/.well-known/jmap") format!("{trimmed}/.well-known/jmap")
} }
fn scheme_end(url: &str) -> Option<usize> {
let bytes = url.as_bytes();
if !bytes.first()?.is_ascii_alphabetic() {
return None;
}
for (index, byte) in bytes.iter().copied().enumerate() {
if byte == b':' {
return Some(index);
}
if !(byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'-' | b'.')) {
return None;
}
}
None
}
pub fn resolve_reference(base: &str, reference: &str) -> String {
if reference.is_empty() {
return base.to_owned();
}
if scheme_end(reference).is_some() {
return reference.to_owned();
}
let Some(base_scheme_end) = scheme_end(base) else {
return reference.to_owned();
};
let scheme = &base[..=base_scheme_end];
let Some(authority_and_path) = base[base_scheme_end + 1..].strip_prefix("//") else {
return reference.to_owned();
};
let mut out = String::with_capacity(scheme.len() + 2 + base.len() + reference.len());
out.push_str(scheme);
out.push_str("//");
if let Some(rest) = reference.strip_prefix("//") {
out.push_str(rest);
return out;
}
let authority_len = authority_and_path
.find(['/', '?', '#'])
.unwrap_or(authority_and_path.len());
out.push_str(&authority_and_path[..authority_len]);
if reference.starts_with('/') {
out.push_str(reference);
return out;
}
let path = &authority_and_path[authority_len..];
let path = path.split(['?', '#']).next().unwrap_or("");
match path.rfind('/') {
Some(last) => out.push_str(&path[..=last]),
None => out.push('/'),
}
out.push_str(reference);
out
}
pub fn origin_of(url: &str) -> Option<String> {
let origin = url::Url::parse(url).ok()?.origin();
if !origin.is_tuple() {
return None;
}
Some(origin.ascii_serialization())
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OriginMismatch {
pub fields: Vec<&'static str>,
pub session_origin: String,
pub connect_origin: String,
}
impl std::fmt::Display for OriginMismatch {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("session advertises ")?;
for (index, field) in self.fields.iter().enumerate() {
if index > 0 {
f.write_str(", ")?;
}
f.write_str(field)?;
}
write!(
f,
" on {} (connected to {}); vandelay must use the advertised URL. \
If this is wrong, fix the server's advertised HTTP URL setting.",
self.session_origin, self.connect_origin
)
}
}
impl Session { impl Session {
pub fn discover(client: &HttpClient, url: &str) -> Result<Session, JmapError> { pub fn discover(client: &HttpClient, url: &str) -> Result<Session, JmapError> {
match client.get(url) { match client.get(url) {
Ok(direct) => { Ok(direct) => {
if let Some(session) = parse_session(&direct) { if let Some(session) = parse_session(&direct) {
return session.ensure_authenticated(); return session.rebased_on(url).ensure_authenticated();
} }
} }
Err(JmapError::Auth(m)) => return Err(JmapError::Auth(m)), Err(JmapError::Auth(m)) => return Err(JmapError::Auth(m)),
@@ -73,7 +161,7 @@ impl Session {
match client.get(&well_known) { match client.get(&well_known) {
Ok(body) => { Ok(body) => {
if let Some(session) = parse_session(&body) { if let Some(session) = parse_session(&body) {
return session.ensure_authenticated(); return session.rebased_on(&well_known).ensure_authenticated();
} }
Err(JmapError::Connect(format!( Err(JmapError::Connect(format!(
"no JMAP Session object at {url} or {well_known}" "no JMAP Session object at {url} or {well_known}"
@@ -86,6 +174,44 @@ impl Session {
} }
} }
fn rebased_on(mut self, session_resource_url: &str) -> Session {
self.api_url = resolve_reference(session_resource_url, &self.api_url);
self.upload_url = resolve_reference(session_resource_url, &self.upload_url);
self.download_url = resolve_reference(session_resource_url, &self.download_url);
self
}
pub fn origin_mismatches(&self, connect_url: &str) -> Vec<OriginMismatch> {
let Some(connect_origin) = origin_of(connect_url) else {
return Vec::new();
};
let mut mismatches: Vec<OriginMismatch> = Vec::new();
for (field, value) in [
("apiUrl", &self.api_url),
("uploadUrl", &self.upload_url),
("downloadUrl", &self.download_url),
] {
let Some(session_origin) = origin_of(value) else {
continue;
};
if session_origin == connect_origin {
continue;
}
match mismatches
.iter_mut()
.find(|m| m.session_origin == session_origin)
{
Some(existing) => existing.fields.push(field),
None => mismatches.push(OriginMismatch {
fields: vec![field],
session_origin,
connect_origin: connect_origin.clone(),
}),
}
}
mismatches
}
fn ensure_authenticated(self) -> Result<Session, JmapError> { fn ensure_authenticated(self) -> Result<Session, JmapError> {
if self.accounts.is_empty() { if self.accounts.is_empty() {
return Err(JmapError::Auth( return Err(JmapError::Auth(
@@ -236,6 +362,143 @@ mod tests {
)); ));
} }
#[test]
fn origin_treats_the_default_port_as_equivalent() {
assert_eq!(
origin_of("https://mail.example/jmap"),
origin_of("https://mail.example:443/jmap")
);
assert_eq!(
origin_of("http://mail.example/jmap"),
origin_of("http://mail.example:80/jmap")
);
}
#[test]
fn origin_distinguishes_port_host_and_scheme() {
let base = origin_of("https://mail.example:10443/jmap");
assert_ne!(base, origin_of("https://mail.example/jmap"));
assert_ne!(base, origin_of("https://other.example:10443/jmap"));
assert_ne!(base, origin_of("http://mail.example:10443/jmap"));
}
fn session_on(base: &str) -> Session {
let raw = format!(
r#"{{"apiUrl":"{base}/jmap/","uploadUrl":"{base}/jmap/upload/{{accountId}}/",
"downloadUrl":"{base}/jmap/dl/{{accountId}}/{{blobId}}/{{type}}/{{name}}",
"capabilities":{{}},"accounts":{{}}}}"#
);
parse_session(&raw).expect("session")
}
#[test]
fn matching_origin_reports_no_mismatch() {
let session = session_on("https://mail.example:10443");
assert!(
session
.origin_mismatches("https://mail.example:10443")
.is_empty()
);
assert!(
session_on("https://mail.example")
.origin_mismatches("https://mail.example:443/")
.is_empty()
);
}
#[test]
fn dropped_port_is_reported_once_naming_every_field() {
let session = session_on("https://mail.example");
let mismatches = session.origin_mismatches("https://mail.example:10443");
assert_eq!(mismatches.len(), 1, "one warning per distinct origin");
assert_eq!(
mismatches[0].fields,
vec!["apiUrl", "uploadUrl", "downloadUrl"]
);
assert_eq!(mismatches[0].session_origin, "https://mail.example");
assert_eq!(mismatches[0].connect_origin, "https://mail.example:10443");
let warning = mismatches[0].to_string();
assert!(warning.contains("session advertises apiUrl"), "{warning}");
assert!(
warning.contains("on https://mail.example (connected to https://mail.example:10443)"),
"{warning}"
);
assert!(warning.contains("advertised HTTP URL setting"), "{warning}");
}
#[test]
fn a_split_upload_host_is_reported_separately_from_the_api_host() {
let raw = r#"{"apiUrl":"https://mail.example:10443/jmap/",
"uploadUrl":"https://blobs.example/upload/{accountId}/",
"downloadUrl":"https://blobs.example/dl/{accountId}/{blobId}/{type}/{name}",
"capabilities":{},"accounts":{}}"#;
let session = parse_session(raw).expect("session");
let mismatches = session.origin_mismatches("https://mail.example:10443");
assert_eq!(mismatches.len(), 1);
assert_eq!(mismatches[0].fields, vec!["uploadUrl", "downloadUrl"]);
assert_eq!(mismatches[0].session_origin, "https://blobs.example");
}
#[test]
fn resolve_reference_keeps_absolute_urls_verbatim() {
assert_eq!(
resolve_reference(
"https://h.example/.well-known/jmap",
"https://other.example:8443/jmap/api"
),
"https://other.example:8443/jmap/api"
);
}
#[test]
fn resolve_reference_handles_scheme_relative_absolute_and_relative_paths() {
let base = "https://h.example:10443/.well-known/jmap";
assert_eq!(
resolve_reference(base, "//other.example/jmap/api"),
"https://other.example/jmap/api"
);
assert_eq!(
resolve_reference(base, "/jmap/api"),
"https://h.example:10443/jmap/api"
);
assert_eq!(
resolve_reference(base, "api"),
"https://h.example:10443/.well-known/api"
);
}
#[test]
fn resolve_reference_preserves_uri_template_braces() {
assert_eq!(
resolve_reference(
"https://h.example/.well-known/jmap",
"/jmap/upload/{accountId}/"
),
"https://h.example/jmap/upload/{accountId}/"
);
}
#[test]
fn relative_session_urls_are_resolved_against_the_session_resource() {
let raw = r#"{"apiUrl":"/jmap/","uploadUrl":"/jmap/upload/{accountId}/",
"downloadUrl":"/jmap/download/{accountId}/{blobId}/{type}/{name}",
"capabilities":{},"accounts":{}}"#;
let session = parse_session(raw)
.expect("session")
.rebased_on("https://h.example:10443/.well-known/jmap");
assert_eq!(session.api_url, "https://h.example:10443/jmap/");
assert_eq!(
session.upload_url_for("w"),
"https://h.example:10443/jmap/upload/w/"
);
assert!(
session
.origin_mismatches("https://h.example:10443")
.is_empty(),
"a resolved relative URL shares the connect origin"
);
}
#[test] #[test]
fn well_known_appends_correctly() { fn well_known_appends_correctly() {
assert_eq!( assert_eq!(
+28 -16
View File
@@ -9,7 +9,7 @@ use clap::Parser;
use vandelay::cli::{Action, Cli}; use vandelay::cli::{Action, Cli};
use vandelay::error::Error; use vandelay::error::Error;
use vandelay::inspect; use vandelay::inspect;
use vandelay::sync::{self, Summary}; use vandelay::sync::{self, RunOutcome, Summary};
fn main() { fn main() {
let code = run(); let code = run();
@@ -30,42 +30,54 @@ fn run() -> i32 {
Err(err) => return fail(&err), Err(err) => return fail(&err),
}; };
let result = match action { let (outcome, logger) = match action {
Action::Import(common, config) => { Action::Import(common, config) => {
let logger = common.logger; let logger = common.logger;
sync::import_jmap::run(common, config).map(|s| (s, logger)) (sync::import_jmap::run_reporting(common, config), logger)
} }
Action::ImportImap(common, config) => { Action::ImportImap(common, config) => {
let logger = common.logger; let logger = common.logger;
sync::import_imap::run(common, config).map(|s| (s, logger)) (sync::import_imap::run_reporting(common, config), logger)
} }
Action::ImportDav(common, config) => { Action::ImportDav(common, config) => {
let logger = common.logger; let logger = common.logger;
sync::import_dav::run(common, config).map(|s| (s, logger)) (sync::import_dav::run_reporting(common, config), logger)
} }
Action::ImportManageSieve(common, config) => { Action::ImportManageSieve(common, config) => {
let logger = common.logger; let logger = common.logger;
sync::import_managesieve::run(common, config).map(|s| (s, logger)) (
RunOutcome::from_result(sync::import_managesieve::run(common, config)),
logger,
)
} }
Action::ImportMaildir(common, config) => { Action::ImportMaildir(common, config) => {
let logger = common.logger; let logger = common.logger;
sync::import_maildir::run(common, config).map(|s| (s, logger)) (sync::import_maildir::run_reporting(common, config), logger)
} }
Action::ImportTakeout(common, config) => { Action::ImportTakeout(common, config) => {
let logger = common.logger; let logger = common.logger;
sync::import_takeout::run(common, config).map(|s| (s, logger)) (sync::import_takeout::run_reporting(common, config), logger)
} }
Action::ImportExchangeEws(common, config) => { Action::ImportExchangeEws(common, config) => {
let logger = common.logger; let logger = common.logger;
sync::import_exchange_ews::run(common, config).map(|s| (s, logger)) (
RunOutcome::from_result(sync::import_exchange_ews::run(common, config)),
logger,
)
} }
Action::ImportExchangeGraph(common, config) => { Action::ImportExchangeGraph(common, config) => {
let logger = common.logger; let logger = common.logger;
sync::import_exchange_graph::run(common, config).map(|s| (s, logger)) (
RunOutcome::from_result(sync::import_exchange_graph::run(common, config)),
logger,
)
} }
Action::Export(common, config) => { Action::Export(common, config) => {
let logger = common.logger; let logger = common.logger;
sync::export::run(common, config).map(|s| (s, logger)) (
RunOutcome::from_result(sync::export::run(common, config)),
logger,
)
} }
Action::Inspect(config) => { Action::Inspect(config) => {
return match inspect::run(config) { return match inspect::run(config) {
@@ -75,17 +87,17 @@ fn run() -> i32 {
} }
}; };
match result { report(&outcome.summary);
Ok((summary, logger)) => { match outcome.error {
report(&summary); Some(err) => fail(&err),
if summary.any_failed() { None => {
if outcome.summary.any_failed() {
logger.error("some objects failed; the archive is consistent and resumable"); logger.error("some objects failed; the archive is consistent and resumable");
5 5
} else { } else {
0 0
} }
} }
Err(err) => fail(&err),
} }
} }
+6 -4
View File
@@ -20,7 +20,7 @@ use crate::jmap::request::{Request, SetRequest, get_all, get_objects, query_all_
use crate::jmap::session::{Limits, Session}; use crate::jmap::session::{Limits, Session};
use crate::jmap::wire::JmapId; use crate::jmap::wire::JmapId;
use crate::logging::{LEVEL_DEFAULT, Logger}; use crate::logging::{LEVEL_DEFAULT, Logger};
use crate::sync::import_jmap::mapping::{BlobUpload, TargetResolver}; use crate::sync::import_jmap::mapping::{BlobBytes, TargetResolver};
use crate::sync::{CommonConfig, Context, ExportConfig, Summary, TypeCounts}; use crate::sync::{CommonConfig, Context, ExportConfig, Summary, TypeCounts};
use crate::types::ObjectType; use crate::types::ObjectType;
@@ -110,9 +110,10 @@ impl<'a> Uploader<'a> {
} }
} }
impl BlobUpload for Uploader<'_> { impl BlobBytes for Uploader<'_> {
fn upload(&mut self, local_id: i64) -> Result<JmapId, JmapError> { fn bytes(&self, local_id: i64) -> Result<Vec<u8>, JmapError> {
self.upload_with(local_id, "application/octet-stream") db::blobs::blob_bytes(self.conn, local_id)?
.ok_or_else(|| JmapError::malformed(format!("blob local id {local_id} missing")))
} }
} }
@@ -172,6 +173,7 @@ pub fn run(common: CommonConfig, config: ExportConfig) -> Result<Summary, Error>
); );
let plan = match res { let plan = match res {
Ok(p) => p, Ok(p) => p,
Err(e) if e.aborts_run() => return Err(e),
Err(e) => { Err(e) => {
logger.warn(&format!("type {} aborted: {e}", ty.jmap_name())); logger.warn(&format!("type {} aborted: {e}", ty.jmap_name()));
counts.failed += 1; counts.failed += 1;
+21 -6
View File
@@ -12,7 +12,10 @@ use super::common::{jid, target_query_get};
use super::{Maps, Net, Plan, Uploader}; use super::{Maps, Net, Plan, Uploader};
use crate::error::Error; use crate::error::Error;
use crate::jmap::error::JmapError; use crate::jmap::error::JmapError;
use crate::jmap::request::{Request, check_method_error, get_objects}; use crate::jmap::request::{
MethodCall, Request, check_method_error, get_objects, retry_method_call,
};
use crate::jmap::retry::MethodCallKind;
use crate::jmap::wire::JmapId; use crate::jmap::wire::JmapId;
use crate::logging::Logger; use crate::logging::Logger;
use crate::sync::import_jmap::mapping::{EMAIL_SELECT, EmailRow, TargetResolver, row_to_email}; use crate::sync::import_jmap::mapping::{EMAIL_SELECT, EmailRow, TargetResolver, row_to_email};
@@ -219,7 +222,7 @@ fn export_one(
return; return;
} }
let item = import_item(blob, mids, build_keywords(row), &row.received_at); let item = import_item(blob, mids, build_keywords(row), &row.received_at);
match send_single_import(net, &cid, item) { match send_single_import(net, &cid, item, logger) {
Ok(SingleImport::Created) => counts.created += 1, Ok(SingleImport::Created) => counts.created += 1,
Ok(SingleImport::Skipped) => counts.skipped += 1, Ok(SingleImport::Skipped) => counts.skipped += 1,
Ok(SingleImport::NotCreated { error_type, .. }) if error_type == "blobNotFound" => { Ok(SingleImport::NotCreated { error_type, .. }) if error_type == "blobNotFound" => {
@@ -277,7 +280,7 @@ fn retry_after_reupload(
} }
}; };
let item = import_item(blob, mids, build_keywords(row), &row.received_at); let item = import_item(blob, mids, build_keywords(row), &row.received_at);
match send_single_import(net, cid, item) { match send_single_import(net, cid, item, logger) {
Ok(SingleImport::Created) => counts.created += 1, Ok(SingleImport::Created) => counts.created += 1,
Ok(SingleImport::Skipped) => counts.skipped += 1, Ok(SingleImport::Skipped) => counts.skipped += 1,
Ok(SingleImport::NotCreated { detail, .. }) => { Ok(SingleImport::NotCreated { detail, .. }) => {
@@ -304,7 +307,12 @@ enum SingleImport {
NotCreated { error_type: String, detail: String }, NotCreated { error_type: String, detail: String },
} }
fn send_single_import(net: &Net, cid: &str, item: Value) -> Result<SingleImport, JmapError> { fn send_single_import(
net: &Net,
cid: &str,
item: Value,
logger: &Logger,
) -> Result<SingleImport, JmapError> {
let mut emails = Map::new(); let mut emails = Map::new();
emails.insert(cid.to_owned(), item); emails.insert(cid.to_owned(), item);
let mut req = Request::new(); let mut req = Request::new();
@@ -314,8 +322,15 @@ fn send_single_import(net: &Net, cid: &str, item: Value) -> Result<SingleImport,
"i", "i",
); );
req.fits(&net.limits)?; req.fits(&net.limits)?;
let resp = req.send(&net.client, &net.api)?; retry_method_call(
let mr = resp.first()?; &net.client,
MethodCallKind::SingleObjectWrite,
logger,
|| interpret_import(req.send(&net.client, &net.api)?.first()?, cid),
)
}
fn interpret_import(mr: &MethodCall, cid: &str) -> Result<SingleImport, JmapError> {
check_method_error(mr)?; check_method_error(mr)?;
if let Some(err) = mr if let Some(err) = mr
.args .args
+32 -20
View File
@@ -5,10 +5,11 @@
*/ */
use std::collections::HashSet; use std::collections::HashSet;
use std::fmt::Write as _;
use serde_json::Value; use serde_json::Value;
use super::common::{create_batch, jid, retry_if_blob_missing, target_query_get}; use super::common::{create_batch, jid, target_query_get};
use super::{Maps, Net, Plan, Uploader}; use super::{Maps, Net, Plan, Uploader};
use crate::error::Error; use crate::error::Error;
use crate::logging::Logger; use crate::logging::Logger;
@@ -23,6 +24,15 @@ fn target_uid(v: &Value) -> Option<String> {
v.get("uid").and_then(Value::as_str).map(str::to_owned) v.get("uid").and_then(Value::as_str).map(str::to_owned)
} }
fn describe(ty: ObjectType, local: i64, uid: &str) -> String {
let mut out = String::new();
let _ = write!(out, "{} local {local}", ty.jmap_name());
if !uid.is_empty() {
let _ = write!(out, " (uid {uid})");
}
out
}
pub fn reconcile( pub fn reconcile(
ctx: &Context, ctx: &Context,
net: &Net, net: &Net,
@@ -66,7 +76,7 @@ pub fn reconcile(
}; };
let mut matched_uids: HashSet<String> = HashSet::new(); let mut matched_uids: HashSet<String> = HashSet::new();
let mut uploader = Uploader::new(net, &ctx.conn); let blobs = Uploader::new(net, &ctx.conn);
for (local, uid) in &rows { for (local, uid) in &rows {
if let Some(tid) = by_uid.get(uid) { if let Some(tid) = by_uid.get(uid) {
maps.insert(ty, *local, crate::jmap::wire::JmapId(tid.clone())); maps.insert(ty, *local, crate::jmap::wire::JmapId(tid.clone()));
@@ -75,28 +85,30 @@ pub fn reconcile(
continue; continue;
} }
let cid = format!("c{local}"); let cid = format!("c{local}");
let _ = uploader.take_touched(); let wire = match build_wire(ctx, ty, *local, maps, &blobs) {
let wire = match build_wire(ctx, ty, *local, maps, &mut uploader) {
Ok(w) => w, Ok(w) => w,
Err(e) if e.aborts_run() => return Err(e),
Err(e) => { Err(e) => {
logger.warn(&format!("{} local {local} skipped: {e}", ty.jmap_name())); logger.warn(&format!("{} skipped: {e}", describe(ty, *local, uid)));
counts.failed += 1; counts.failed += 1;
continue; continue;
} }
}; };
let touched = uploader.take_touched(); let outcome = match create_batch(net, ty, vec![(cid.clone(), wire)]) {
let outcome = create_batch(net, ty, vec![(cid.clone(), wire)]).map_err(Error::from)?; Ok(o) => o,
let outcome = Err(e) => {
match retry_if_blob_missing(net, ty, &cid, &mut uploader, touched, outcome, |up| { let mapped = Error::from(e);
build_wire(ctx, ty, *local, maps, up) if mapped.aborts_run() {
}) { return Err(mapped);
Ok(o) => o,
Err(e) => {
logger.warn(&format!("{} local {local} skipped: {e}", ty.jmap_name()));
counts.failed += 1;
continue;
} }
}; logger.warn(&format!(
"{} not created: {mapped}",
describe(ty, *local, uid)
));
counts.failed += 1;
continue;
}
};
for (cid, v) in &outcome.created { for (cid, v) in &outcome.created {
if let Some(parsed) = cid.strip_prefix('c').and_then(|s| s.parse::<i64>().ok()) if let Some(parsed) = cid.strip_prefix('c').and_then(|s| s.parse::<i64>().ok())
&& let Some(id) = jid(v) && let Some(id) = jid(v)
@@ -136,7 +148,7 @@ fn build_wire(
ty: ObjectType, ty: ObjectType,
local: i64, local: i64,
maps: &Maps, maps: &Maps,
up: &mut Uploader<'_>, blobs: &Uploader<'_>,
) -> Result<Value, Error> { ) -> Result<Value, Error> {
if ty == ObjectType::ContactCard { if ty == ObjectType::ContactCard {
let (uid, abids, data): (String, String, String) = ctx let (uid, abids, data): (String, String, String) = ctx
@@ -147,7 +159,7 @@ fn build_wire(
|r| Ok((r.get(1)?, r.get(2)?, r.get(3)?)), |r| Ok((r.get(1)?, r.get(2)?, r.get(3)?)),
) )
.map_err(|e| Error::Partial(e.to_string()))?; .map_err(|e| Error::Partial(e.to_string()))?;
contact_card_to_wire(&uid, &abids, &data, maps, up).map_err(Error::from) contact_card_to_wire(&uid, &abids, &data, maps, blobs).map_err(Error::from)
} else { } else {
let (cal, dr, ud, data): (String, i64, i64, String) = ctx let (cal, dr, ud, data): (String, i64, i64, String) = ctx
.conn .conn
@@ -157,6 +169,6 @@ fn build_wire(
|r| Ok((r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)), |r| Ok((r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)),
) )
.map_err(|e| Error::Partial(e.to_string()))?; .map_err(|e| Error::Partial(e.to_string()))?;
calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, maps, up).map_err(Error::from) calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, maps, blobs).map_err(Error::from)
} }
} }
+45 -1
View File
@@ -10,4 +10,48 @@ pub mod coordinator;
pub mod items; pub mod items;
pub mod tree; pub mod tree;
pub use coordinator::{DavAuth, DavImportConfig, DavKindArg, run}; pub use coordinator::{DavAuth, DavImportConfig, DavKindArg, run, run_reporting};
use crate::error::Error;
use crate::jmap::error::JmapError;
pub(crate) fn per_collection_failure(err: JmapError) -> Error {
match err {
JmapError::Sqlite(e) => Error::Db(crate::db::init::OpenError::Sqlite(e)),
scoped => Error::Partial(scoped.to_string()),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_forbidden_collection_is_a_per_unit_failure() {
for e in [
JmapError::Auth("server returned 403: forbidden".to_owned()),
JmapError::HttpStatus {
status: 405,
body: "method not allowed".to_owned(),
},
JmapError::RetriesExhausted("PROPFIND kept returning 503".to_owned()),
JmapError::Transport("io: connection reset".to_owned()),
JmapError::Malformed("multistatus parse".to_owned()),
] {
let mapped = per_collection_failure(e);
assert!(
!mapped.aborts_run(),
"{mapped} is scoped to one collection and must not abort the run"
);
assert_eq!(mapped.exit_code(), 5);
}
}
#[test]
fn an_archive_failure_still_aborts() {
let mapped =
per_collection_failure(JmapError::Sqlite(rusqlite::Error::QueryReturnedNoRows));
assert!(mapped.aborts_run());
assert_eq!(mapped.exit_code(), 7);
}
}
+142 -55
View File
@@ -13,7 +13,7 @@ use crate::db::sources::SourceKey;
use crate::error::Error; use crate::error::Error;
use crate::jmap::http::{Auth, RetryPolicy}; use crate::jmap::http::{Auth, RetryPolicy};
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
use crate::sync::{CommonConfig, Summary, TypeCounts}; use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts};
use super::collections; use super::collections;
use super::items; use super::items;
@@ -83,6 +83,20 @@ pub struct DavImportConfig {
} }
pub fn run(common: CommonConfig, config: DavImportConfig) -> Result<Summary, Error> { pub fn run(common: CommonConfig, config: DavImportConfig) -> Result<Summary, Error> {
run_reporting(common, config).into_result()
}
pub fn run_reporting(common: CommonConfig, config: DavImportConfig) -> RunOutcome {
let mut summary = Summary::default();
let error = run_into(common, config, &mut summary).err();
RunOutcome { summary, error }
}
fn run_into(
common: CommonConfig,
config: DavImportConfig,
summary: &mut Summary,
) -> Result<(), Error> {
let logger = common.logger; let logger = common.logger;
enforce_tls_policy(&config.url, config.allow_cleartext)?; enforce_tls_policy(&config.url, config.allow_cleartext)?;
@@ -138,38 +152,42 @@ pub fn run(common: CommonConfig, config: DavImportConfig) -> Result<Summary, Err
} }
if common.dry_run { if common.dry_run {
let mut summary = run_dry_diff(&conn, &client, &discovery, &config, logger)?; let result = run_dry_diff(&conn, &client, &discovery, &config, logger, summary);
summary.retries_observed = client.retries_observed(); summary.retries_observed = client.retries_observed();
summary.retry_after_sleeps = client.retry_after_sleeps(); summary.retry_after_sleeps = client.retry_after_sleeps();
return Ok(summary); return result;
} }
let username = config.auth.username(); let username = config.auth.username();
let source_id = db::sources::upsert_source(&conn, &key, Some(&account_id), &username) let source_id = db::sources::upsert_source(&conn, &key, Some(&account_id), &username)
.map_err(|e| Error::Partial(e.to_string()))?; .map_err(|e| Error::Partial(e.to_string()))?;
let summary = match config.kind { let phase = match config.kind {
DavKindArg::Caldav => { DavKindArg::Caldav => {
run_caldav(&mut conn, &client, source_id, &discovery, &config, logger)? run_caldav(&mut conn, &client, source_id, &discovery, &config, logger)
} }
DavKindArg::Carddav => { DavKindArg::Carddav => {
run_carddav(&mut conn, &client, source_id, &discovery, &config, logger)? run_carddav(&mut conn, &client, source_id, &discovery, &config, logger)
} }
DavKindArg::Webdav => { DavKindArg::Webdav => {
run_webdav(&mut conn, &client, source_id, &discovery, &config, logger)? run_webdav(&mut conn, &client, source_id, &discovery, &config, logger)
} }
}; };
*summary = phase.summary;
summary.retries_observed = client.retries_observed();
summary.retry_after_sleeps = client.retry_after_sleeps();
if let Some(e) = phase.error {
return Err(e);
}
if !summary.any_failed() if !summary.any_failed()
&& let Err(e) = run_gc(&conn) && let Err(e) = run_gc(&conn)
{ {
logger.warn(&format!("blob GC skipped: {e}")); logger.warn(&format!("blob GC skipped: {e}"));
} }
let mut summary = summary; Ok(())
summary.retries_observed = client.retries_observed();
summary.retry_after_sleeps = client.retry_after_sleeps();
Ok(summary)
} }
type ReconcileCollections = fn( type ReconcileCollections = fn(
@@ -198,25 +216,71 @@ fn run_collection_phase(
config: &DavImportConfig, config: &DavImportConfig,
logger: Logger, logger: Logger,
phase: ItemPhase, phase: ItemPhase,
) -> Result<Summary, Error> { ) -> RunOutcome {
let mut summary = Summary::default(); let mut summary = Summary::default();
let mut container_counts = TypeCounts::default(); let mut counts = PhaseCounts::default();
let mut item_counts = TypeCounts::default(); let error = collection_phase_into(
conn,
client,
source_id,
PhaseInput {
discovery,
config,
logger,
phase: &phase,
},
&mut counts,
)
.err();
summary
.per_type
.push((phase.container_label, counts.container));
summary.per_type.push((phase.item_label, counts.items));
RunOutcome { summary, error }
}
#[derive(Default)]
struct PhaseCounts {
container: TypeCounts,
items: TypeCounts,
}
struct PhaseInput<'a> {
discovery: &'a Discovery,
config: &'a DavImportConfig,
logger: Logger,
phase: &'a ItemPhase,
}
fn collection_phase_into(
conn: &mut Connection,
client: &DavClient,
source_id: i64,
input: PhaseInput<'_>,
counts: &mut PhaseCounts,
) -> Result<(), Error> {
let PhaseInput {
discovery,
config,
logger,
phase,
} = input;
let upserted = (phase.reconcile_collections)( let upserted = (phase.reconcile_collections)(
conn, conn,
source_id, source_id,
&discovery.collections, &discovery.collections,
&mut container_counts, &mut counts.container,
logger, logger,
)?; )?;
if logger.enabled(LEVEL_DEFAULT) { if logger.enabled(LEVEL_DEFAULT) {
eprintln!( eprintln!(
"import: {} done (upserted={} deleted={} failed={})", "import: {} done (upserted={} deleted={} failed={})",
phase.container_label, phase.container_label,
container_counts.created + container_counts.fetched, counts.container.created + counts.container.fetched,
container_counts.deleted, counts.container.deleted,
container_counts.failed counts.container.failed
); );
} }
@@ -229,23 +293,19 @@ fn run_collection_phase(
logger, logger,
}; };
for (collection_href, local_id) in &upserted { for (collection_href, local_id) in &upserted {
match (phase.reconcile_items)(conn, &ctx, collection_href, *local_id, &mut item_counts) { match (phase.reconcile_items)(conn, &ctx, collection_href, *local_id, &mut counts.items) {
Ok(()) => {} Ok(()) => {}
Err(e) if e.aborts_run() => return Err(e),
Err(e) => { Err(e) => {
logger.warn(&format!( logger.warn(&format!(
"{} {collection_href:?}: items failed: {e}", "{} {collection_href:?}: items failed: {e}",
phase.container_label phase.container_label
)); ));
item_counts.failed += 1; counts.items.failed += 1;
} }
} }
} }
Ok(())
summary
.per_type
.push((phase.container_label, container_counts));
summary.per_type.push((phase.item_label, item_counts));
Ok(summary)
} }
fn run_dry_diff( fn run_dry_diff(
@@ -254,11 +314,11 @@ fn run_dry_diff(
discovery: &Discovery, discovery: &Discovery,
config: &DavImportConfig, config: &DavImportConfig,
logger: Logger, logger: Logger,
) -> Result<Summary, Error> { summary: &mut Summary,
) -> Result<(), Error> {
use crate::dav::href::join_absolute; use crate::dav::href::join_absolute;
use crate::dav::xml; use crate::dav::xml;
use crate::db::dav_ids; use crate::db::dav_ids;
let mut summary = Summary::default();
let (container_label, item_label, container_type, item_type) = match config.kind { let (container_label, item_label, container_type, item_type) = match config.kind {
DavKindArg::Caldav => ( DavKindArg::Caldav => (
"calendar", "calendar",
@@ -295,33 +355,57 @@ fn run_dry_diff(
for coll in &discovery.collections { for coll in &discovery.collections {
let url = join_absolute(&discovery.home_set_url, coll.href.as_str()) let url = join_absolute(&discovery.home_set_url, coll.href.as_str())
.map_err(|e| Error::Partial(e.to_string()))?; .map_err(|e| Error::Partial(e.to_string()))?;
let ms = client match client
.propfind_responses(&url, 1, &xml::propfind_dav_items(), &url) .propfind_responses(&url, 1, &xml::propfind_dav_items(), &url)
.map_err(Error::from)?; .map_err(super::per_collection_failure)
let new_count = ms {
.responses Ok(ms) => {
.iter() let new_count = ms
.filter(|r| !r.props.is_collection) .responses
.count(); .iter()
item_counts.created += new_count as u64; .filter(|r| !r.props.is_collection)
if logger.enabled(LEVEL_DEFAULT) { .count();
eprintln!(" {} items: {new_count}", coll.href.as_str()); item_counts.created += new_count as u64;
if logger.enabled(LEVEL_DEFAULT) {
eprintln!(" {} items: {new_count}", coll.href.as_str());
}
}
Err(e) if e.aborts_run() => return Err(e),
Err(e) => {
logger.warn(&format!(
"{container_label} {:?}: enumeration failed: {e}",
coll.href.as_str()
));
item_counts.failed += 1;
}
} }
} }
} else if let Some(root) = discovery.collections.first() { } else if let Some(root) = discovery.collections.first() {
let url = join_absolute(&discovery.home_set_url, root.href.as_str()) let url = join_absolute(&discovery.home_set_url, root.href.as_str())
.map_err(|e| Error::Partial(e.to_string()))?; .map_err(|e| Error::Partial(e.to_string()))?;
let ms = client match client
.propfind_responses(&url, 1, &xml::propfind_webdav_listing(), &url) .propfind_responses(&url, 1, &xml::propfind_webdav_listing(), &url)
.map_err(Error::from)?; .map_err(super::per_collection_failure)
let new_count = ms {
.responses Ok(ms) => {
.iter() let new_count = ms
.filter(|r| !r.props.is_collection) .responses
.count(); .iter()
item_counts.created += new_count as u64; .filter(|r| !r.props.is_collection)
if logger.enabled(LEVEL_DEFAULT) { .count();
eprintln!(" root {} files: {new_count}", root.href.as_str()); item_counts.created += new_count as u64;
if logger.enabled(LEVEL_DEFAULT) {
eprintln!(" root {} files: {new_count}", root.href.as_str());
}
}
Err(e) if e.aborts_run() => return Err(e),
Err(e) => {
logger.warn(&format!(
"{container_label} {:?}: enumeration failed: {e}",
root.href.as_str()
));
container_counts.failed += 1;
}
} }
} }
@@ -332,7 +416,7 @@ fn run_dry_diff(
if !matches!(config.kind, DavKindArg::Webdav) { if !matches!(config.kind, DavKindArg::Webdav) {
summary.per_type.push((item_label, item_counts)); summary.per_type.push((item_label, item_counts));
} }
Ok(summary) Ok(())
} }
fn run_caldav( fn run_caldav(
@@ -342,7 +426,7 @@ fn run_caldav(
discovery: &Discovery, discovery: &Discovery,
config: &DavImportConfig, config: &DavImportConfig,
logger: Logger, logger: Logger,
) -> Result<Summary, Error> { ) -> RunOutcome {
run_collection_phase( run_collection_phase(
conn, conn,
client, client,
@@ -366,7 +450,7 @@ fn run_carddav(
discovery: &Discovery, discovery: &Discovery,
config: &DavImportConfig, config: &DavImportConfig,
logger: Logger, logger: Logger,
) -> Result<Summary, Error> { ) -> RunOutcome {
run_collection_phase( run_collection_phase(
conn, conn,
client, client,
@@ -390,13 +474,16 @@ fn run_webdav(
discovery: &Discovery, discovery: &Discovery,
config: &DavImportConfig, config: &DavImportConfig,
logger: Logger, logger: Logger,
) -> Result<Summary, Error> { ) -> RunOutcome {
let mut summary = Summary::default(); let mut summary = Summary::default();
let mut file_counts = TypeCounts::default(); let mut file_counts = TypeCounts::default();
if discovery.collections.is_empty() { if discovery.collections.is_empty() {
summary.per_type.push(("filenode", file_counts)); summary.per_type.push(("filenode", file_counts));
return Ok(summary); return RunOutcome {
summary,
error: None,
};
} }
let root = &discovery.collections[0]; let root = &discovery.collections[0];
let ctx = tree::WebDavCtx { let ctx = tree::WebDavCtx {
@@ -406,10 +493,10 @@ fn run_webdav(
dav_connections: config.dav_connections, dav_connections: config.dav_connections,
logger, logger,
}; };
tree::reconcile_filenodes(conn, &ctx, root, &mut file_counts)?; let error = tree::reconcile_filenodes(conn, &ctx, root, &mut file_counts).err();
summary.per_type.push(("filenode", file_counts)); summary.per_type.push(("filenode", file_counts));
Ok(summary) RunOutcome { summary, error }
} }
fn map_discovery_error(err: DiscoveryError) -> Error { fn map_discovery_error(err: DiscoveryError) -> Error {
+1 -1
View File
@@ -113,7 +113,7 @@ fn enumerate_items(client: &DavClient, url: &str) -> Result<Vec<ServerItem>, Err
let body = xml::propfind_dav_items(); let body = xml::propfind_dav_items();
let ms = client let ms = client
.propfind_responses(url, 1, &body, url) .propfind_responses(url, 1, &body, url)
.map_err(Error::from)?; .map_err(super::per_collection_failure)?;
if ms.status >= 400 { if ms.status >= 400 {
return Err(Error::Partial(format!( return Err(Error::Partial(format!(
"enumerate {url}: http {}", "enumerate {url}: http {}",
+2 -1
View File
@@ -115,6 +115,7 @@ pub fn reconcile_filenodes(
} }
} }
} }
Err(e) if e.aborts_run() => return Err(e),
Err(e) => { Err(e) => {
logger.warn(&format!("PROPFIND {url}: {e}")); logger.warn(&format!("PROPFIND {url}: {e}"));
counts.failed += 1; counts.failed += 1;
@@ -394,7 +395,7 @@ fn walk_one(
let body = xml::propfind_webdav_listing(); let body = xml::propfind_webdav_listing();
let ms = client let ms = client
.propfind_responses(url, 1, &body, url) .propfind_responses(url, 1, &body, url)
.map_err(Error::from)?; .map_err(super::per_collection_failure)?;
if ms.status >= 400 { if ms.status >= 400 {
return Err(Error::Partial(format!("http {}", ms.status))); return Err(Error::Partial(format!("http {}", ms.status)));
} }
+3 -2
View File
@@ -11,6 +11,7 @@ use serde_json::{Value, json};
use crate::db::exchange_graph_ids; use crate::db::exchange_graph_ids;
use crate::error::Error; use crate::error::Error;
use crate::exchange::jscalendar::override_patch_from_event;
use crate::exchange_graph::api::{self, PREFER_BODY_HTML, PREFER_BODY_TEXT, PREFER_TIMEZONE_UTC}; use crate::exchange_graph::api::{self, PREFER_BODY_HTML, PREFER_BODY_TEXT, PREFER_TIMEZONE_UTC};
use crate::exchange_graph::calendar_map::{ use crate::exchange_graph::calendar_map::{
ConvertedEvent, EventType, classify_event_type, convert_event, ConvertedEvent, EventType, classify_event_type, convert_event,
@@ -185,7 +186,7 @@ fn merge_exception_into(master: &mut ConvertedEvent, ex: &ConvertedEvent) {
let Value::Object(overrides) = overrides else { let Value::Object(overrides) = overrides else {
return; return;
}; };
overrides.insert(key, ex.data.clone()); overrides.insert(key, override_patch_from_event(&ex.data));
} }
fn merge_exception_into_existing( fn merge_exception_into_existing(
@@ -253,7 +254,7 @@ fn merge_persisted_master(
.entry("recurrenceOverrides".to_owned()) .entry("recurrenceOverrides".to_owned())
.or_insert_with(|| Value::Object(serde_json::Map::new())); .or_insert_with(|| Value::Object(serde_json::Map::new()));
if let Value::Object(overrides) = entry { if let Value::Object(overrides) = entry {
overrides.insert(key, ex_data.clone()); overrides.insert(key, override_patch_from_event(ex_data));
} }
} }
tx.execute( tx.execute(
+7 -30
View File
@@ -16,11 +16,10 @@ use crate::exchange_graph::error::GraphError;
use crate::exchange_graph::oauth::{ use crate::exchange_graph::oauth::{
AcquiredToken, OAuthFlow, acquire, default_authority, refresh_access_token, AcquiredToken, OAuthFlow, acquire, default_authority, refresh_access_token,
}; };
use crate::exchange_graph::types::{EventBodyFormat, MailboxKind, synthetic_account_id}; use crate::exchange_graph::types::{EventBodyFormat, MailboxKind, Surfaces, synthetic_account_id};
use crate::jmap::http::RetryPolicy; use crate::jmap::http::RetryPolicy;
use crate::logging::LEVEL_DEFAULT; use crate::logging::LEVEL_DEFAULT;
use crate::sync::{CommonConfig, Summary, TypeCounts}; use crate::sync::{CommonConfig, Summary, TypeCounts};
use crate::types::ObjectType;
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub enum GraphAuth { pub enum GraphAuth {
@@ -39,7 +38,7 @@ pub struct GraphImportConfig {
pub api_base: String, pub api_base: String,
pub user_target: Option<String>, pub user_target: Option<String>,
pub mailbox_kind: MailboxKind, pub mailbox_kind: MailboxKind,
pub objects: Option<Vec<ObjectType>>, pub surfaces: Surfaces,
pub event_body_format: EventBodyFormat, pub event_body_format: EventBodyFormat,
pub graph_connections: usize, pub graph_connections: usize,
pub top: usize, pub top: usize,
@@ -141,32 +140,10 @@ pub fn run(common: CommonConfig, config: GraphImportConfig) -> Result<Summary, E
event_body_format: config.event_body_format, event_body_format: config.event_body_format,
}; };
let want_mail = config let primary = !matches!(config.mailbox_kind, MailboxKind::Archive);
.objects let want_mail = config.surfaces.mail;
.as_ref() let want_calendar = config.surfaces.calendar && primary;
.map(|set| { let want_contacts = config.surfaces.contacts && primary;
set.iter()
.any(|o| matches!(o, ObjectType::Mailbox | ObjectType::Email))
})
.unwrap_or(true);
let want_calendar = config
.objects
.as_ref()
.map(|set| {
set.iter()
.any(|o| matches!(o, ObjectType::Calendar | ObjectType::CalendarEvent))
})
.unwrap_or(true)
&& !matches!(config.mailbox_kind, MailboxKind::Archive);
let want_contacts = config
.objects
.as_ref()
.map(|set| {
set.iter()
.any(|o| matches!(o, ObjectType::AddressBook | ObjectType::ContactCard))
})
.unwrap_or(true)
&& !matches!(config.mailbox_kind, MailboxKind::Archive);
if want_mail { if want_mail {
let folders = super::folders::reconcile_mail( let folders = super::folders::reconcile_mail(
@@ -509,7 +486,7 @@ mod tests {
api_base: "https://graph.microsoft.com/v1.0".to_owned(), api_base: "https://graph.microsoft.com/v1.0".to_owned(),
user_target: None, user_target: None,
mailbox_kind: MailboxKind::Primary, mailbox_kind: MailboxKind::Primary,
objects: None, surfaces: Surfaces::ALL,
event_body_format: EventBodyFormat::Text, event_body_format: EventBodyFormat::Text,
graph_connections: 4, graph_connections: 4,
top: 100, top: 100,
+1 -1
View File
@@ -13,4 +13,4 @@ pub mod pool;
pub mod coordinator; pub mod coordinator;
pub use coordinator::{ImapAuth, ImapImportConfig, run}; pub use coordinator::{ImapAuth, ImapImportConfig, run, run_reporting};
+31 -5
View File
@@ -29,7 +29,7 @@ use crate::imap::transport::Connector;
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
use crate::sync::emailmeta::email_index_from_blob; use crate::sync::emailmeta::email_index_from_blob;
use crate::sync::keys::index_to_json; use crate::sync::keys::index_to_json;
use crate::sync::{CommonConfig, Summary, TypeCounts}; use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts};
use super::fetch; use super::fetch;
use super::folders::{ use super::folders::{
@@ -199,6 +199,20 @@ pub enum ImapAuth {
} }
pub fn run(common: CommonConfig, config: ImapImportConfig) -> Result<Summary, Error> { pub fn run(common: CommonConfig, config: ImapImportConfig) -> Result<Summary, Error> {
run_reporting(common, config).into_result()
}
pub fn run_reporting(common: CommonConfig, config: ImapImportConfig) -> RunOutcome {
let mut summary = Summary::default();
let error = run_into(common, config, &mut summary).err();
RunOutcome { summary, error }
}
fn run_into(
common: CommonConfig,
config: ImapImportConfig,
summary: &mut Summary,
) -> Result<(), Error> {
let logger = common.logger; let logger = common.logger;
let mut conn = db::init::open(&common.archive)?; let mut conn = db::init::open(&common.archive)?;
@@ -341,14 +355,15 @@ pub fn run(common: CommonConfig, config: ImapImportConfig) -> Result<Summary, Er
if common.dry_run { if common.dry_run {
let existing_source = db::sources::find_source(&conn, &source_key)?; let existing_source = db::sources::find_source(&conn, &source_key)?;
return dry_run_summary( *summary = dry_run_summary(
&conn, &conn,
existing_source, existing_source,
&mut client, &mut client,
&control_ctx, &control_ctx,
&resolved, &resolved,
logger, logger,
); )?;
return Ok(());
} }
let mut mailbox_counts = TypeCounts::default(); let mut mailbox_counts = TypeCounts::default();
@@ -410,6 +425,16 @@ pub fn run(common: CommonConfig, config: ImapImportConfig) -> Result<Summary, Er
&mut email_counts, &mut email_counts,
) { ) {
Ok(()) => {} Ok(()) => {}
Err(e) if e.aborts_run() => {
pool.shutdown();
let _ = client.logout();
*summary = Summary {
per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)],
retries_observed: backoff.total_retries(),
retry_after_sleeps: backoff.transient_retries() as u64,
};
return Err(e);
}
Err(e) => { Err(e) => {
log_at( log_at(
logger, logger,
@@ -424,11 +449,12 @@ pub fn run(common: CommonConfig, config: ImapImportConfig) -> Result<Summary, Er
pool.shutdown(); pool.shutdown();
let _ = client.logout(); let _ = client.logout();
Ok(Summary { *summary = Summary {
per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)], per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)],
retries_observed: backoff.total_retries(), retries_observed: backoff.total_retries(),
retry_after_sleeps: backoff.transient_retries() as u64, retry_after_sleeps: backoff.transient_retries() as u64,
}) };
Ok(())
} }
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
+19 -4
View File
@@ -33,7 +33,7 @@ use crate::jmap::wire::email::Email;
use crate::jmap::wire::file_node::{FileNode, NodeType}; use crate::jmap::wire::file_node::{FileNode, NodeType};
use crate::jmap::wire::sieve_script::SieveScript; use crate::jmap::wire::sieve_script::SieveScript;
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
use crate::sync::{CommonConfig, Context, ImportConfig, Summary, TypeCounts}; use crate::sync::{CommonConfig, Context, ImportConfig, RunOutcome, Summary, TypeCounts};
use crate::types::ObjectType; use crate::types::ObjectType;
const IMPORT_ORDER: [ObjectType; 10] = [ const IMPORT_ORDER: [ObjectType; 10] = [
@@ -147,6 +147,20 @@ fn username_of(auth: &Auth) -> String {
} }
pub fn run(common: CommonConfig, config: ImportConfig) -> Result<Summary, Error> { pub fn run(common: CommonConfig, config: ImportConfig) -> Result<Summary, Error> {
run_reporting(common, config).into_result()
}
pub fn run_reporting(common: CommonConfig, config: ImportConfig) -> RunOutcome {
let mut summary = Summary::default();
let error = run_into(common, config, &mut summary).err();
RunOutcome { summary, error }
}
fn run_into(
common: CommonConfig,
config: ImportConfig,
summary: &mut Summary,
) -> Result<(), Error> {
let logger = common.logger; let logger = common.logger;
let ctx = Context::open(common, &config.connect)?; let ctx = Context::open(common, &config.connect)?;
let connected = connect::prepare(&ctx, &config.connect)?; let connected = connect::prepare(&ctx, &config.connect)?;
@@ -196,7 +210,6 @@ pub fn run(common: CommonConfig, config: ImportConfig) -> Result<Summary, Error>
session: connected.session.clone(), session: connected.session.clone(),
}; };
let mut summary = Summary::default();
let mut dry_rows: Vec<(&'static str, u64, u64, u64)> = Vec::new(); let mut dry_rows: Vec<(&'static str, u64, u64, u64)> = Vec::new();
let threads = ctx.common.threads; let threads = ctx.common.threads;
@@ -216,6 +229,7 @@ pub fn run(common: CommonConfig, config: ImportConfig) -> Result<Summary, Error>
&mut dry_rows, &mut dry_rows,
) { ) {
Ok(()) => {} Ok(()) => {}
Err(e) if e.aborts_run() => return Err(e),
Err(e) => { Err(e) => {
logger.warn(&format!( logger.warn(&format!(
"type {} aborted: {e}; continuing (run will exit 5)", "type {} aborted: {e}; continuing (run will exit 5)",
@@ -229,7 +243,8 @@ pub fn run(common: CommonConfig, config: ImportConfig) -> Result<Summary, Error>
if ctx.dry_run() { if ctx.dry_run() {
print_dry_run(&dry_rows); print_dry_run(&dry_rows);
return Ok(Summary::default()); *summary = Summary::default();
return Ok(());
} }
if !summary.any_failed() if !summary.any_failed()
@@ -240,7 +255,7 @@ pub fn run(common: CommonConfig, config: ImportConfig) -> Result<Summary, Error>
summary.retries_observed = ctx.client.retries_observed(); summary.retries_observed = ctx.client.retries_observed();
summary.retry_after_sleeps = ctx.client.retry_after_sleeps(); summary.retry_after_sleeps = ctx.client.retry_after_sleeps();
Ok(summary) Ok(())
} }
fn work_list(config: &ImportConfig, connected: &Connected) -> Vec<ObjectType> { fn work_list(config: &ImportConfig, connected: &Connected) -> Vec<ObjectType> {
+171 -30
View File
@@ -8,7 +8,7 @@ use indexmap::IndexMap;
use rusqlite::{Connection, Row, params}; use rusqlite::{Connection, Row, params};
use serde_json::{Map, Value}; use serde_json::{Map, Value};
use crate::jmap::blob::{export_blob_ids, import_blob_ids}; use crate::jmap::blob::{BlobWalkError, InlineShape, import_blob_ids, inline_blob_data_uris};
use crate::jmap::error::JmapError; use crate::jmap::error::JmapError;
use crate::jmap::wire::JmapId; use crate::jmap::wire::JmapId;
use crate::jmap::wire::address_book::AddressBook; use crate::jmap::wire::address_book::AddressBook;
@@ -35,8 +35,8 @@ pub trait BlobIntern {
fn intern(&mut self, jmap_blob_id: &str) -> Result<i64, JmapError>; fn intern(&mut self, jmap_blob_id: &str) -> Result<i64, JmapError>;
} }
pub trait BlobUpload { pub trait BlobBytes {
fn upload(&mut self, local_id: i64) -> Result<JmapId, JmapError>; fn bytes(&self, local_id: i64) -> Result<Vec<u8>, JmapError>;
} }
fn translate_in( fn translate_in(
@@ -662,10 +662,10 @@ pub fn contact_card_to_wire(
address_book_ids: &str, address_book_ids: &str,
data: &str, data: &str,
resolver: &impl TargetResolver, resolver: &impl TargetResolver,
blobs: &mut impl BlobUpload, blobs: &impl BlobBytes,
) -> Result<Value, JmapError> { ) -> Result<Value, JmapError> {
let mut value: Value = serde_json::from_str(data)?; let mut value: Value = serde_json::from_str(data)?;
restore_blobs_in(&mut value, blobs)?; inline_blobs_in(&mut value, InlineShape::JsContactResource, blobs)?;
let locals = parse_local_id_array(address_book_ids)?; let locals = parse_local_id_array(address_book_ids)?;
let abids = translate_out(&locals, ObjectType::AddressBook, resolver)?; let abids = translate_out(&locals, ObjectType::AddressBook, resolver)?;
if let Value::Object(map) = &mut value { if let Value::Object(map) = &mut value {
@@ -681,10 +681,10 @@ pub fn calendar_event_to_wire(
use_default_alerts: bool, use_default_alerts: bool,
data: &str, data: &str,
resolver: &impl TargetResolver, resolver: &impl TargetResolver,
blobs: &mut impl BlobUpload, blobs: &impl BlobBytes,
) -> Result<Value, JmapError> { ) -> Result<Value, JmapError> {
let mut value: Value = serde_json::from_str(data)?; let mut value: Value = serde_json::from_str(data)?;
restore_blobs_in(&mut value, blobs)?; inline_blobs_in(&mut value, InlineShape::JsCalendarLink, blobs)?;
let locals = parse_local_id_array(calendar_ids)?; let locals = parse_local_id_array(calendar_ids)?;
let calids = translate_out(&locals, ObjectType::Calendar, resolver)?; let calids = translate_out(&locals, ObjectType::Calendar, resolver)?;
if let Value::Object(map) = &mut value { if let Value::Object(map) = &mut value {
@@ -725,21 +725,20 @@ fn take_string(value: &mut Value, key: &str) -> Option<String> {
fn rewrite_blobs_in(data: &mut Value, blobs: &mut impl BlobIntern) -> Result<(), JmapError> { fn rewrite_blobs_in(data: &mut Value, blobs: &mut impl BlobIntern) -> Result<(), JmapError> {
import_blob_ids(data, |jmap_blob_id| { import_blob_ids(data, |jmap_blob_id| {
blobs blobs.intern(jmap_blob_id).map_err(BlobWalkError::resolver)
.intern(jmap_blob_id)
.map_err(|e| crate::jmap::blob::BlobWalkError::Resolver(e.to_string()))
}) })
.map_err(JmapError::from) .map_err(BlobWalkError::into_source)
} }
fn restore_blobs_in(data: &mut Value, blobs: &mut impl BlobUpload) -> Result<(), JmapError> { fn inline_blobs_in(
export_blob_ids(data, |local_id| { data: &mut Value,
blobs shape: InlineShape,
.upload(local_id) blobs: &impl BlobBytes,
.map(|id| id.0) ) -> Result<(), JmapError> {
.map_err(|e| crate::jmap::blob::BlobWalkError::Resolver(e.to_string())) inline_blob_data_uris(data, shape, |local_id| {
blobs.bytes(local_id).map_err(BlobWalkError::resolver)
}) })
.map_err(JmapError::from) .map_err(BlobWalkError::into_source)
} }
fn opt_json<T: serde::Serialize>(value: &Option<T>) -> Result<Option<String>, JmapError> { fn opt_json<T: serde::Serialize>(value: &Option<T>) -> Result<Option<String>, JmapError> {
@@ -904,9 +903,35 @@ mod tests {
Ok(42) Ok(42)
} }
} }
impl BlobUpload for FakeBlobs { impl BlobBytes for FakeBlobs {
fn upload(&mut self, local_id: i64) -> Result<JmapId, JmapError> { fn bytes(&self, local_id: i64) -> Result<Vec<u8>, JmapError> {
Ok(JmapId(format!("T{local_id}"))) Ok(format!("payload-{local_id}").into_bytes())
}
}
fn decode_data_uri(value: &Value, media_type: &str) -> Vec<u8> {
use base64::Engine;
let uri = value.as_str().unwrap_or_else(|| panic!("{value} is a URI"));
let prefix = format!("data:{media_type};base64,");
let payload = uri
.strip_prefix(&prefix)
.unwrap_or_else(|| panic!("{uri} does not start with {prefix}"));
base64::engine::general_purpose::STANDARD
.decode(payload)
.expect("base64 payload")
}
fn assert_no_blob_id(value: &Value) {
match value {
Value::Object(map) => {
assert!(map.get("blobId").is_none(), "blobId present in {value}");
assert!(map.get("@blob").is_none(), "@blob present in {value}");
for child in map.values() {
assert_no_blob_id(child);
}
}
Value::Array(items) => items.iter().for_each(assert_no_blob_id),
_ => {}
} }
} }
@@ -1032,7 +1057,7 @@ mod tests {
} }
#[test] #[test]
fn contact_card_strips_uid_and_rewrites_blob() { fn contact_card_strips_uid_and_inlines_media_as_data_uri() {
let c = mem(); let c = mem();
let res = MapResolver { let res = MapResolver {
to_local: HashMap::from([((ObjectType::AddressBook, "AB".to_owned()), 1)]), to_local: HashMap::from([((ObjectType::AddressBook, "AB".to_owned()), 1)]),
@@ -1043,7 +1068,10 @@ mod tests {
"addressBookIds": { "AB": true }, "addressBookIds": { "AB": true },
"uid": "urn:uuid:42", "uid": "urn:uuid:42",
"name": { "full": "Jane" }, "name": { "full": "Jane" },
"photos": { "p1": { "blobId": "PB", "mediaType": "image/png" } } "media": { "photo": {
"@type": "Media", "kind": "photo",
"blobId": "PB", "mediaType": "image/png"
} }
})) }))
.unwrap(); .unwrap();
let mut blobs = FakeBlobs; let mut blobs = FakeBlobs;
@@ -1058,14 +1086,127 @@ mod tests {
assert_eq!(uid, "urn:uuid:42"); assert_eq!(uid, "urn:uuid:42");
let stored: Value = serde_json::from_str(&data).unwrap(); let stored: Value = serde_json::from_str(&data).unwrap();
assert!(stored.get("uid").is_none()); assert!(stored.get("uid").is_none());
assert_eq!(stored["photos"]["p1"]["@blob"], Value::from(42)); assert_eq!(stored["media"]["photo"]["@blob"], Value::from(42));
let mut up = FakeBlobs; let up = FakeBlobs;
let wire = contact_card_to_wire(&uid, &abids, &data, &res, &mut up).unwrap(); let wire = contact_card_to_wire(&uid, &abids, &data, &res, &up).unwrap();
assert_eq!(wire["uid"], Value::from("urn:uuid:42")); assert_eq!(wire["uid"], Value::from("urn:uuid:42"));
assert_eq!(wire["addressBookIds"]["TAB"], Value::Bool(true)); assert_eq!(wire["addressBookIds"]["TAB"], Value::Bool(true));
assert_eq!(wire["photos"]["p1"]["blobId"], Value::from("T42")); assert_eq!(
assert!(wire["photos"]["p1"].get("@blob").is_none()); decode_data_uri(&wire["media"]["photo"]["uri"], "image/png"),
b"payload-42"
);
assert_eq!(
wire["media"]["photo"]["mediaType"],
Value::from("image/png")
);
assert!(wire["media"]["photo"].get("href").is_none());
assert_no_blob_id(&wire);
}
#[test]
fn calendar_event_inlines_link_enclosure_as_data_uri() {
let c = mem();
let res = MapResolver {
to_local: HashMap::from([((ObjectType::Calendar, "CAL".to_owned()), 5)]),
to_target: HashMap::from([((ObjectType::Calendar, 5), "TCAL".to_owned())]),
};
let ev: CalendarEvent = serde_json::from_value(serde_json::json!({
"id": "EV1",
"calendarIds": { "CAL": true },
"uid": "ev-with-enclosure",
"title": "Review",
"@type": "Event",
"links": { "1": {
"@type": "Link", "rel": "enclosure",
"blobId": "AB", "contentType": "text/plain", "title": "agenda.txt"
} }
}))
.unwrap();
let mut blobs = FakeBlobs;
let local = insert_calendar_event(&c, &ev, &res, &mut blobs).unwrap();
let (cal, dr, ud, data): (String, i64, i64, String) = c
.query_row(
&format!("{CALENDAR_EVENT_SELECT} AND id = ?1"),
params![local],
|r| Ok((r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)),
)
.unwrap();
let stored: Value = serde_json::from_str(&data).unwrap();
assert_eq!(stored["links"]["1"]["@blob"], Value::from(42));
let up = FakeBlobs;
let wire = calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, &res, &up).unwrap();
assert_eq!(
decode_data_uri(&wire["links"]["1"]["href"], "text/plain"),
b"payload-42"
);
assert_eq!(wire["links"]["1"]["contentType"], Value::from("text/plain"));
assert_eq!(wire["links"]["1"]["rel"], Value::from("enclosure"));
assert!(wire["links"]["1"].get("uri").is_none());
assert_no_blob_id(&wire);
}
#[test]
fn contact_card_media_without_media_type_defaults_to_octet_stream() {
let c = mem();
let res = MapResolver {
to_local: HashMap::from([((ObjectType::AddressBook, "AB".to_owned()), 1)]),
to_target: HashMap::from([((ObjectType::AddressBook, 1), "TAB".to_owned())]),
};
c.execute(
"INSERT INTO contact_cards (id,uid,address_book_ids,data)
VALUES (1,'u-1','[1]',?1)",
params![
serde_json::json!({ "@type": "Card", "media": { "photo": { "@blob": 9 } } })
.to_string()
],
)
.unwrap();
let (uid, abids, data): (String, String, String) = c
.query_row(&format!("{CONTACT_CARD_SELECT} WHERE id = 1"), [], |r| {
Ok((r.get(1)?, r.get(2)?, r.get(3)?))
})
.unwrap();
let up = FakeBlobs;
let wire = contact_card_to_wire(&uid, &abids, &data, &res, &up).unwrap();
assert_eq!(
decode_data_uri(&wire["media"]["photo"]["uri"], "application/octet-stream"),
b"payload-9"
);
assert_no_blob_id(&wire);
}
#[test]
fn an_archive_read_failure_while_inlining_is_an_archive_error_not_a_unit_failure() {
struct BrokenBlobs;
impl BlobBytes for BrokenBlobs {
fn bytes(&self, _local_id: i64) -> Result<Vec<u8>, JmapError> {
Err(JmapError::Sqlite(rusqlite::Error::QueryReturnedNoRows))
}
}
let res = MapResolver {
to_local: HashMap::new(),
to_target: HashMap::from([
((ObjectType::AddressBook, 1), "TAB".to_owned()),
((ObjectType::Calendar, 1), "TCAL".to_owned()),
]),
};
let card = serde_json::json!({ "@type": "Card", "media": { "photo": { "@blob": 9 } } })
.to_string();
let event =
serde_json::json!({ "@type": "Event", "links": { "1": { "@blob": 9 } } }).to_string();
let failures = [
contact_card_to_wire("u-1", "[1]", &card, &res, &BrokenBlobs).expect_err("card"),
calendar_event_to_wire("[1]", false, false, &event, &res, &BrokenBlobs)
.expect_err("event"),
];
for err in failures {
assert!(matches!(err, JmapError::Sqlite(_)), "{err:?}");
let mapped = crate::error::Error::from(err);
assert!(mapped.aborts_run(), "{mapped} must abort the run");
assert_eq!(mapped.exit_code(), 7);
}
} }
#[test] #[test]
@@ -1103,8 +1244,8 @@ mod tests {
assert!(stored.get("calendarIds").is_none()); assert!(stored.get("calendarIds").is_none());
assert_eq!(stored["title"], Value::from("Sprint")); assert_eq!(stored["title"], Value::from("Sprint"));
let mut up = FakeBlobs; let up = FakeBlobs;
let wire = calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, &res, &mut up).unwrap(); let wire = calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, &res, &up).unwrap();
assert_eq!(wire["calendarIds"]["TCAL"], Value::Bool(true)); assert_eq!(wire["calendarIds"]["TCAL"], Value::Bool(true));
assert_eq!(wire["isDraft"], Value::Bool(true)); assert_eq!(wire["isDraft"], Value::Bool(true));
assert_eq!(wire["title"], Value::from("Sprint")); assert_eq!(wire["title"], Value::from("Sprint"));
+1 -1
View File
@@ -9,4 +9,4 @@ pub mod keywords;
pub mod messages; pub mod messages;
pub mod tree; pub mod tree;
pub use coordinator::{MaildirImportConfig, run}; pub use coordinator::{MaildirImportConfig, run, run_reporting};
+28 -5
View File
@@ -15,7 +15,7 @@ use crate::db;
use crate::db::sources::SourceKey; use crate::db::sources::SourceKey;
use crate::error::Error; use crate::error::Error;
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
use crate::sync::{CommonConfig, Summary, TypeCounts}; use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts};
use super::messages; use super::messages;
use super::tree; use super::tree;
@@ -32,6 +32,20 @@ pub struct MaildirImportConfig {
} }
pub fn run(common: CommonConfig, config: MaildirImportConfig) -> Result<Summary, Error> { pub fn run(common: CommonConfig, config: MaildirImportConfig) -> Result<Summary, Error> {
run_reporting(common, config).into_result()
}
pub fn run_reporting(common: CommonConfig, config: MaildirImportConfig) -> RunOutcome {
let mut summary = Summary::default();
let error = run_into(common, config, &mut summary).err();
RunOutcome { summary, error }
}
fn run_into(
common: CommonConfig,
config: MaildirImportConfig,
summary: &mut Summary,
) -> Result<(), Error> {
let logger = common.logger; let logger = common.logger;
if common.threads > 1 { if common.threads > 1 {
log_at( log_at(
@@ -121,8 +135,8 @@ pub fn run(common: CommonConfig, config: MaildirImportConfig) -> Result<Summary,
); );
if common.dry_run { if common.dry_run {
let summary = build_dry_run_summary(&conn, source_id, &resolved, logger)?; *summary = build_dry_run_summary(&conn, source_id, &resolved, logger)?;
return Ok(summary); return Ok(());
} }
let mut mailbox_counts = TypeCounts::default(); let mut mailbox_counts = TypeCounts::default();
@@ -174,6 +188,14 @@ pub fn run(common: CommonConfig, config: MaildirImportConfig) -> Result<Summary,
&mut email_counts, &mut email_counts,
logger, logger,
) { ) {
if e.aborts_run() {
*summary = Summary {
per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)],
retries_observed: 0,
retry_after_sleeps: 0,
};
return Err(e);
}
log_at( log_at(
logger, logger,
LEVEL_DEFAULT, LEVEL_DEFAULT,
@@ -207,11 +229,12 @@ pub fn run(common: CommonConfig, config: MaildirImportConfig) -> Result<Summary,
), ),
); );
Ok(Summary { *summary = Summary {
per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)], per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)],
retries_observed: 0, retries_observed: 0,
retry_after_sleeps: 0, retry_after_sleeps: 0,
}) };
Ok(())
} }
struct RunFlags { struct RunFlags {
+1 -1
View File
@@ -13,4 +13,4 @@ pub mod mbox;
pub mod tree; pub mod tree;
pub mod walk; pub mod walk;
pub use coordinator::{TakeoutImportConfig, run}; pub use coordinator::{TakeoutImportConfig, run, run_reporting};
+61 -23
View File
@@ -15,7 +15,7 @@ use crate::db::sources::SourceKey;
use crate::db::takeout_ids; use crate::db::takeout_ids;
use crate::error::Error; use crate::error::Error;
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger}; use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
use crate::sync::{CommonConfig, Summary, TypeCounts}; use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts};
use super::calendar; use super::calendar;
use super::contacts; use super::contacts;
@@ -31,6 +31,20 @@ pub struct TakeoutImportConfig {
} }
pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary, Error> { pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary, Error> {
run_reporting(common, config).into_result()
}
pub fn run_reporting(common: CommonConfig, config: TakeoutImportConfig) -> RunOutcome {
let mut summary = Summary::default();
let error = run_into(common, config, &mut summary).err();
RunOutcome { summary, error }
}
fn run_into(
common: CommonConfig,
config: TakeoutImportConfig,
summary: &mut Summary,
) -> Result<(), Error> {
let logger = common.logger; let logger = common.logger;
if common.threads > 1 && logger.enabled(LEVEL_PROGRESS) { if common.threads > 1 && logger.enabled(LEVEL_PROGRESS) {
eprintln!("takeout importer is single-threaded; --threads value will be ignored"); eprintln!("takeout importer is single-threaded; --threads value will be ignored");
@@ -90,7 +104,8 @@ pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary,
}; };
if common.dry_run { if common.dry_run {
return Ok(dry_run_summary(&conn, source_id, &walk_result, logger)); *summary = dry_run_summary(&conn, source_id, &walk_result, logger);
return Ok(());
} }
let options = MappingOptions { let options = MappingOptions {
@@ -107,7 +122,8 @@ pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary,
let mut mailbox_cache: HashMap<String, i64> = let mut mailbox_cache: HashMap<String, i64> =
takeout_ids::all_for_type(&conn, source_id, takeout_ids::MAILBOX)?; takeout_ids::all_for_type(&conn, source_id, takeout_ids::MAILBOX)?;
process_mbox_files( let mut aborted: Option<Error> = None;
if let Err(e) = process_mbox_files(
&mut conn, &mut conn,
source_id, source_id,
&walk_result, &walk_result,
@@ -116,27 +132,38 @@ pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary,
&mut mailbox_counts, &mut mailbox_counts,
&mut email_counts, &mut email_counts,
logger, logger,
)?; ) {
aborted = Some(e);
}
process_ics_files( if aborted.is_none()
&mut conn, && let Err(e) = process_ics_files(
source_id, &mut conn,
&walk_result, source_id,
&mut calendar_counts, &walk_result,
&mut calendar_event_counts, &mut calendar_counts,
logger, &mut calendar_event_counts,
)?; logger,
)
{
aborted = Some(e);
}
process_vcf_files( if aborted.is_none()
&mut conn, && let Err(e) = process_vcf_files(
source_id, &mut conn,
&walk_result, source_id,
&mut book_counts, &walk_result,
&mut card_counts, &mut book_counts,
logger, &mut card_counts,
)?; logger,
)
{
aborted = Some(e);
}
let no_failures = mailbox_counts.failed == 0 let no_failures = aborted.is_none()
&& mailbox_counts.failed == 0
&& email_counts.failed == 0 && email_counts.failed == 0
&& calendar_counts.failed == 0 && calendar_counts.failed == 0
&& calendar_event_counts.failed == 0 && calendar_event_counts.failed == 0
@@ -171,7 +198,7 @@ pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary,
); );
} }
Ok(Summary { *summary = Summary {
per_type: vec![ per_type: vec![
("mailbox", mailbox_counts), ("mailbox", mailbox_counts),
("email", email_counts), ("email", email_counts),
@@ -182,7 +209,11 @@ pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary,
], ],
retries_observed: 0, retries_observed: 0,
retry_after_sleeps: 0, retry_after_sleeps: 0,
}) };
match aborted {
Some(e) => Err(e),
None => Ok(()),
}
} }
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
@@ -212,6 +243,7 @@ fn process_mbox_files(
}; };
match mail::process_file(conn, &file.path, ctx, mailbox_counts, email_counts, logger) { match mail::process_file(conn, &file.path, ctx, mailbox_counts, email_counts, logger) {
Ok(()) => {} Ok(()) => {}
Err(e) if e.aborts_run() => return Err(e),
Err(e) => { Err(e) => {
logger.warn(&format!("mbox {:?}: {e}", file.path)); logger.warn(&format!("mbox {:?}: {e}", file.path));
email_counts.failed += 1; email_counts.failed += 1;
@@ -244,6 +276,9 @@ fn process_ics_files(
event_counts, event_counts,
logger, logger,
) { ) {
if e.aborts_run() {
return Err(e);
}
logger.warn(&format!("ics {:?}: {e}", file.path)); logger.warn(&format!("ics {:?}: {e}", file.path));
event_counts.failed += 1; event_counts.failed += 1;
} }
@@ -274,6 +309,9 @@ fn process_vcf_files(
card_counts, card_counts,
logger, logger,
) { ) {
if e.aborts_run() {
return Err(e);
}
logger.warn(&format!("vcf {:?}: {e}", file.path)); logger.warn(&format!("vcf {:?}: {e}", file.path));
card_counts.failed += 1; card_counts.failed += 1;
} }
+53
View File
@@ -94,6 +94,34 @@ impl Summary {
} }
} }
#[derive(Debug)]
pub struct RunOutcome {
pub summary: Summary,
pub error: Option<Error>,
}
impl RunOutcome {
pub fn from_result(result: Result<Summary, Error>) -> RunOutcome {
match result {
Ok(summary) => RunOutcome {
summary,
error: None,
},
Err(error) => RunOutcome {
summary: Summary::default(),
error: Some(error),
},
}
}
pub fn into_result(self) -> Result<Summary, Error> {
match self.error {
Some(error) => Err(error),
None => Ok(self.summary),
}
}
}
pub struct Context { pub struct Context {
pub conn: Connection, pub conn: Connection,
pub client: HttpClient, pub client: HttpClient,
@@ -119,3 +147,28 @@ impl Context {
self.common.dry_run self.common.dry_run
} }
} }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn an_aborted_run_keeps_its_error_and_its_partial_counts() {
let mut summary = Summary::default();
summary.per_type.push(("Mailbox", TypeCounts::default()));
let outcome = RunOutcome {
summary,
error: Some(Error::Connection("http status 404".to_owned())),
};
assert_eq!(outcome.summary.per_type.len(), 1);
let err = outcome.into_result().expect_err("aborted");
assert_eq!(err.exit_code(), 2);
}
#[test]
fn a_result_without_a_native_outcome_reports_nothing_on_abort() {
let outcome = RunOutcome::from_result(Err(Error::Partial("one object".to_owned())));
assert!(outcome.summary.per_type.is_empty());
assert!(outcome.error.is_some());
}
}
+4 -1
View File
@@ -151,7 +151,10 @@ fn sweep_abandoned_containers() {
return; return;
} }
eprintln!("removing {} abandoned test container(s)", ids.len()); eprintln!("removing {} abandoned test container(s)", ids.len());
let _ = Command::new("docker").args(["rm", "-f"]).args(&ids).output(); let _ = Command::new("docker")
.args(["rm", "-f"])
.args(&ids)
.output();
} }
fn pick_free_port() -> ContainerResult<u16> { fn pick_free_port() -> ContainerResult<u16> {
+231
View File
@@ -1595,3 +1595,234 @@ fn mixed_source_archive_allows_distinct_kinds_against_same_url() {
let no_conflict = sources::conflicting_source(&conn, "caldav", "https://x", "alice").unwrap(); let no_conflict = sources::conflicting_source(&conn, "caldav", "https://x", "alice").unwrap();
assert!(no_conflict.is_none()); assert!(no_conflict.is_none());
} }
fn dav_archive() -> std::path::PathBuf {
static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let n = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let mut p = std::env::temp_dir();
p.push(format!(
"vandelay-mockdav-{}-{:?}-{n}.sqlite",
std::process::id(),
std::thread::current().id(),
));
let _ = std::fs::remove_file(&p);
p
}
fn caldav_import_config(url: &str) -> vandelay::sync::import_dav::DavImportConfig {
vandelay::sync::import_dav::DavImportConfig {
kind: vandelay::sync::import_dav::DavKindArg::Caldav,
url: url.to_owned(),
auth: vandelay::sync::import_dav::DavAuth::Basic {
user: "u".to_owned(),
password: "p".to_owned(),
},
allow_cleartext: true,
dav_connections: 1,
multiget_batch: 8,
allow_source_change: false,
}
}
fn dav_common(archive: &std::path::Path) -> vandelay::sync::CommonConfig {
vandelay::sync::CommonConfig {
archive: archive.to_path_buf(),
threads: 1,
dry_run: false,
max_retries: 0,
allow_invalid_certs: false,
logger: vandelay::logging::Logger::from_flags(true, 0),
}
}
fn row_count(archive: &std::path::Path, table: &str) -> i64 {
let conn = rusqlite::Connection::open(archive).expect("open archive");
conn.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0))
.expect("count")
}
#[test]
fn forbidden_collection_is_a_per_unit_failure_and_the_others_still_import() {
let mut server = mockito::Server::new();
let url = server.url();
let home_set = format!(
r#"<?xml version="1.0"?>
<d:multistatus xmlns:d="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav">
<d:response>
<d:href>{url}/dav/cal/u/work/</d:href>
<d:propstat>
<d:prop>
<d:resourcetype><d:collection/><c:calendar/></d:resourcetype>
<d:displayname>Work</d:displayname>
</d:prop>
<d:status>HTTP/1.1 200 OK</d:status>
</d:propstat>
</d:response>
<d:response>
<d:href>{url}/dav/cal/u/shared/</d:href>
<d:propstat>
<d:prop>
<d:resourcetype><d:collection/><c:calendar/></d:resourcetype>
<d:displayname>Shared</d:displayname>
</d:prop>
<d:status>HTTP/1.1 200 OK</d:status>
</d:propstat>
</d:response>
</d:multistatus>"#
);
let _discovery = multistatus_response(&mut server, "PROPFIND", "/dav/cal/u/", &home_set);
let work_items = format!(
r#"<?xml version="1.0"?>
<d:multistatus xmlns:d="DAV:">
<d:response>
<d:href>{url}/dav/cal/u/work/e1.ics</d:href>
<d:propstat>
<d:prop><d:resourcetype/><d:getetag>"v1"</d:getetag></d:prop>
<d:status>HTTP/1.1 200 OK</d:status>
</d:propstat>
</d:response>
</d:multistatus>"#
);
let _work_enumeration =
multistatus_response(&mut server, "PROPFIND", "/dav/cal/u/work/", &work_items);
let work_data = format!(
r#"<?xml version="1.0"?>
<d:multistatus xmlns:d="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav">
<d:response>
<d:href>{url}/dav/cal/u/work/e1.ics</d:href>
<d:propstat>
<d:prop>
<d:getetag>"v1"</d:getetag>
<c:calendar-data>BEGIN:VCALENDAR
VERSION:2.0
PRODID:-//Test//EN
BEGIN:VEVENT
UID:e1@example.com
DTSTAMP:20260101T000000Z
DTSTART:20260101T090000Z
DTEND:20260101T100000Z
SUMMARY:Standup
END:VEVENT
END:VCALENDAR
</c:calendar-data>
</d:prop>
<d:status>HTTP/1.1 200 OK</d:status>
</d:propstat>
</d:response>
</d:multistatus>"#
);
let _work_multiget =
multistatus_response(&mut server, "REPORT", "/dav/cal/u/work/", &work_data);
let forbidden = server
.mock("PROPFIND", "/dav/cal/u/shared/")
.with_status(403)
.with_header("content-type", "text/plain")
.with_body("you may not read this calendar")
.expect_at_least(1)
.create();
let archive = dav_archive();
let summary = vandelay::sync::import_dav::run(
dav_common(&archive),
caldav_import_config(&format!("{url}/dav/cal/u/")),
)
.expect("a forbidden collection must not abort the run");
forbidden.assert();
let events = summary
.per_type
.iter()
.find(|(t, _)| *t == "calendarevent")
.map(|(_, c)| c.clone())
.expect("calendarevent counts");
assert_eq!(events.failed, 1, "the forbidden calendar counts as failed");
assert_eq!(events.fetched, 1, "the readable calendar still imported");
assert!(
summary.any_failed(),
"a per-unit failure ends the run at exit 5"
);
assert_eq!(row_count(&archive, "calendars"), 2);
assert_eq!(row_count(&archive, "calendar_events"), 1);
let _ = std::fs::remove_file(&archive);
}
#[test]
fn credentials_rejected_during_discovery_still_aborts_the_run() {
let mut server = mockito::Server::new();
let url = server.url();
let _rejected = server
.mock("PROPFIND", mockito::Matcher::Any)
.with_status(401)
.with_header("content-type", "text/plain")
.with_body("bad password")
.expect_at_least(1)
.create();
let archive = dav_archive();
let err = vandelay::sync::import_dav::run(
dav_common(&archive),
caldav_import_config(&format!("{url}/dav/cal/u/")),
)
.expect_err("credentials rejected for the principal aborts the run");
assert!(
matches!(err, vandelay::error::Error::Connection(_)),
"discovery-level auth rejection is a whole-run failure, got {err:?}"
);
assert_eq!(err.exit_code(), 2);
let _ = std::fs::remove_file(&archive);
}
#[test]
fn dav_import_reports_both_phases_when_every_collection_fails() {
let mut server = mockito::Server::new();
let url = server.url();
let home_set = format!(
r#"<?xml version="1.0"?>
<d:multistatus xmlns:d="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav">
<d:response>
<d:href>{url}/dav/cal/u/work/</d:href>
<d:propstat>
<d:prop>
<d:resourcetype><d:collection/><c:calendar/></d:resourcetype>
<d:displayname>Work</d:displayname>
</d:prop>
<d:status>HTTP/1.1 200 OK</d:status>
</d:propstat>
</d:response>
</d:multistatus>"#
);
let _discovery = multistatus_response(&mut server, "PROPFIND", "/dav/cal/u/", &home_set);
let _forbidden = server
.mock("PROPFIND", "/dav/cal/u/work/")
.with_status(403)
.with_body("nope")
.expect_at_least(1)
.create();
let archive = dav_archive();
let outcome = vandelay::sync::import_dav::run_reporting(
dav_common(&archive),
caldav_import_config(&format!("{url}/dav/cal/u/")),
);
assert!(outcome.error.is_none());
assert_eq!(
outcome
.summary
.per_type
.iter()
.map(|(t, _)| *t)
.collect::<Vec<_>>(),
vec!["calendar", "calendarevent"],
"both phases are reported even when every collection failed"
);
let _ = std::fs::remove_file(&archive);
}
+252 -73
View File
@@ -20,6 +20,7 @@ use vandelay::exchange_graph::oauth::{
}; };
use vandelay::exchange_graph::recurrence::convert_patterned_recurrence; use vandelay::exchange_graph::recurrence::convert_patterned_recurrence;
use vandelay::exchange_graph::retry::{HttpClass, classify_http_status}; use vandelay::exchange_graph::retry::{HttpClass, classify_http_status};
use vandelay::exchange_graph::types::Surfaces;
use vandelay::jmap::http::RetryPolicy; use vandelay::jmap::http::RetryPolicy;
static INIT: Once = Once::new(); static INIT: Once = Once::new();
@@ -584,7 +585,7 @@ fn integration_dry_run_against_mock_server_lists_three_surfaces() {
api_base: base.clone(), api_base: base.clone(),
user_target: None, user_target: None,
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
objects: None, surfaces: Surfaces::ALL,
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2, graph_connections: 2,
top: 100, top: 100,
@@ -693,10 +694,7 @@ fn integration_full_run_mail_only_imports_mime_via_value() {
api_base: server.url(), api_base: server.url(),
user_target: None, user_target: None,
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
objects: Some(vec![ surfaces: surfaces("mail"),
vandelay::types::ObjectType::Mailbox,
vandelay::types::ObjectType::Email,
]),
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2, graph_connections: 2,
top: 100, top: 100,
@@ -792,10 +790,7 @@ fn integration_duplicate_message_id_does_not_abort_run() {
api_base: server.url(), api_base: server.url(),
user_target: None, user_target: None,
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
objects: Some(vec![ surfaces: surfaces("mail"),
vandelay::types::ObjectType::Mailbox,
vandelay::types::ObjectType::Email,
]),
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2, graph_connections: 2,
top: 100, top: 100,
@@ -884,10 +879,7 @@ fn integration_full_run_is_convergent_on_second_invocation() {
api_base: api_base.clone(), api_base: api_base.clone(),
user_target: None, user_target: None,
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
objects: Some(vec![ surfaces: surfaces("mail"),
vandelay::types::ObjectType::Mailbox,
vandelay::types::ObjectType::Email,
]),
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2, graph_connections: 2,
top: 100, top: 100,
@@ -966,7 +958,7 @@ fn source_change_protection_refuses_a_different_account() {
api_base: server.url(), api_base: server.url(),
user_target: None, user_target: None,
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
objects: None, surfaces: Surfaces::ALL,
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2, graph_connections: 2,
top: 100, top: 100,
@@ -1016,10 +1008,14 @@ fn make_common(archive: std::path::PathBuf) -> vandelay::sync::CommonConfig {
} }
} }
fn surfaces(list: &str) -> vandelay::exchange_graph::types::Surfaces {
vandelay::exchange_graph::types::Surfaces::parse_list(list).unwrap()
}
fn make_config( fn make_config(
api_base: String, api_base: String,
user_target: Option<String>, user_target: Option<String>,
objects: Option<Vec<vandelay::types::ObjectType>>, surfaces: vandelay::exchange_graph::types::Surfaces,
) -> vandelay::sync::import_exchange_graph::GraphImportConfig { ) -> vandelay::sync::import_exchange_graph::GraphImportConfig {
vandelay::sync::import_exchange_graph::GraphImportConfig { vandelay::sync::import_exchange_graph::GraphImportConfig {
auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired { auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired {
@@ -1028,7 +1024,7 @@ fn make_config(
api_base, api_base,
user_target, user_target,
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
objects, surfaces,
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2, graph_connections: 2,
top: 100, top: 100,
@@ -1094,7 +1090,7 @@ fn users_upn_routes_through_users_segment_not_me() {
}; };
let summary = vandelay::sync::import_exchange_graph::run( let summary = vandelay::sync::import_exchange_graph::run(
common, common,
make_config(base, Some("[email protected]".to_owned()), None), make_config(base, Some("[email protected]".to_owned()), Surfaces::ALL),
) )
.unwrap(); .unwrap();
let mailbox = summary let mailbox = summary
@@ -1193,14 +1189,7 @@ fn series_master_with_exception_merges_into_recurrence_overrides() {
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
let summary = vandelay::sync::import_exchange_graph::run( let summary = vandelay::sync::import_exchange_graph::run(
make_common(archive.clone()), make_common(archive.clone()),
make_config( make_config(base, None, surfaces("calendar")),
base,
None,
Some(vec![
vandelay::types::ObjectType::Calendar,
vandelay::types::ObjectType::CalendarEvent,
]),
),
) )
.unwrap(); .unwrap();
let events = summary let events = summary
@@ -1227,6 +1216,26 @@ fn series_master_with_exception_merges_into_recurrence_overrides() {
); );
let override_data = &overrides["2026-05-11T15:00:00"]; let override_data = &overrides["2026-05-11T15:00:00"];
assert_eq!(override_data["title"], "Moved sync"); assert_eq!(override_data["title"], "Moved sync");
assert_eq!(override_data["start"], "2026-05-11T16:00:00");
for ignored in [
"@type",
"uid",
"method",
"organizerCalendarAddress",
"privacy",
"prodId",
"recurrenceId",
"recurrenceIdTimeZone",
"sentBy",
"recurrenceRule",
"recurrenceOverrides",
"relatedTo",
] {
assert!(
override_data.get(ignored).is_none(),
"{ignored} must not appear in a PatchObject (jscalendarbis 3.3.4); got {override_data}"
);
}
} }
#[test] #[test]
@@ -1285,14 +1294,7 @@ fn occurrence_event_is_skipped_no_row_no_id_mapping() {
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
let summary = vandelay::sync::import_exchange_graph::run( let summary = vandelay::sync::import_exchange_graph::run(
make_common(archive.clone()), make_common(archive.clone()),
make_config( make_config(base, None, surfaces("calendar")),
base,
None,
Some(vec![
vandelay::types::ObjectType::Calendar,
vandelay::types::ObjectType::CalendarEvent,
]),
),
) )
.unwrap(); .unwrap();
let events = summary let events = summary
@@ -1372,14 +1374,7 @@ fn hidden_mail_folder_has_is_subscribed_zero() {
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
let _ = vandelay::sync::import_exchange_graph::run( let _ = vandelay::sync::import_exchange_graph::run(
make_common(archive.clone()), make_common(archive.clone()),
make_config( make_config(base, None, surfaces("mail")),
base,
None,
Some(vec![
vandelay::types::ObjectType::Mailbox,
vandelay::types::ObjectType::Email,
]),
),
) )
.unwrap(); .unwrap();
let conn = vandelay::db::init::open(&archive).unwrap(); let conn = vandelay::db::init::open(&archive).unwrap();
@@ -1473,14 +1468,7 @@ fn well_known_folder_probes_assign_jmap_roles() {
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
let _ = vandelay::sync::import_exchange_graph::run( let _ = vandelay::sync::import_exchange_graph::run(
make_common(archive.clone()), make_common(archive.clone()),
make_config( make_config(base, None, surfaces("mail")),
base,
None,
Some(vec![
vandelay::types::ObjectType::Mailbox,
vandelay::types::ObjectType::Email,
]),
),
) )
.unwrap(); .unwrap();
let conn = vandelay::db::init::open(&archive).unwrap(); let conn = vandelay::db::init::open(&archive).unwrap();
@@ -1557,14 +1545,7 @@ SGVsbG8=\r\n--X--\r\n";
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
let summary = vandelay::sync::import_exchange_graph::run( let summary = vandelay::sync::import_exchange_graph::run(
make_common(archive), make_common(archive),
make_config( make_config(base, None, surfaces("mail")),
base,
None,
Some(vec![
vandelay::types::ObjectType::Mailbox,
vandelay::types::ObjectType::Email,
]),
),
) )
.unwrap(); .unwrap();
let emails = summary let emails = summary
@@ -1652,14 +1633,7 @@ fn event_get_carries_outlook_timezone_and_body_content_type_prefer() {
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned(); let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
let _ = vandelay::sync::import_exchange_graph::run( let _ = vandelay::sync::import_exchange_graph::run(
make_common(archive), make_common(archive),
make_config( make_config(base, None, surfaces("calendar")),
base,
None,
Some(vec![
vandelay::types::ObjectType::Calendar,
vandelay::types::ObjectType::CalendarEvent,
]),
),
) )
.unwrap(); .unwrap();
} }
@@ -1758,7 +1732,7 @@ fn full_run_records_graph_id_in_sync_id_exchange_graph_with_padding() {
api_base: server.url(), api_base: server.url(),
user_target: None, user_target: None,
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
objects: Some(vec![vandelay::types::ObjectType::Mailbox]), surfaces: surfaces("mail"),
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2, graph_connections: 2,
top: 100, top: 100,
@@ -1828,7 +1802,7 @@ fn archive_mailbox_kind_encodes_synthetic_suffix_in_account_id() {
api_base: server.url(), api_base: server.url(),
user_target: None, user_target: None,
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Archive, mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Archive,
objects: Some(vec![vandelay::types::ObjectType::Mailbox]), surfaces: surfaces("mail"),
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2, graph_connections: 2,
top: 100, top: 100,
@@ -1893,7 +1867,7 @@ fn allow_source_change_permits_overwriting_a_different_account() {
api_base: server.url(), api_base: server.url(),
user_target: None, user_target: None,
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
objects: Some(vec![vandelay::types::ObjectType::Mailbox]), surfaces: surfaces("mail"),
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2, graph_connections: 2,
top: 100, top: 100,
@@ -1974,7 +1948,7 @@ fn dry_run_makes_no_per_item_get_and_no_sqlite_writes() {
api_base: server.url(), api_base: server.url(),
user_target: None, user_target: None,
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
objects: None, surfaces: Surfaces::ALL,
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2, graph_connections: 2,
top: 100, top: 100,
@@ -2142,10 +2116,7 @@ fn folder_enumeration_failure_skips_vanished_deletion() {
api_base: server.url(), api_base: server.url(),
user_target: None, user_target: None,
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary, mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
objects: Some(vec![ surfaces: surfaces("mail"),
vandelay::types::ObjectType::Mailbox,
vandelay::types::ObjectType::Email,
]),
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text, event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
graph_connections: 2, graph_connections: 2,
top: 100, top: 100,
@@ -2163,3 +2134,211 @@ fn folder_enumeration_failure_skips_vanished_deletion() {
"transient folder-enumeration failure must NOT delete the locally-known message" "transient folder-enumeration failure must NOT delete the locally-known message"
); );
} }
fn row_count(conn: &rusqlite::Connection, table: &str) -> i64 {
conn.query_row(&format!("SELECT count(*) FROM {table}"), [], |row| {
row.get(0)
})
.unwrap()
}
fn stub_contacts_surface(server: &mut Server) {
server
.mock("GET", "/me/contactFolders?$top=100")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(r#"{"value":[{"id":"CON1","displayName":"Contacts"}]}"#)
.expect_at_least(0)
.create();
server
.mock("GET", "/me/contactFolders/CON1/childFolders?$top=100")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(r#"{"value":[]}"#)
.expect_at_least(0)
.create();
server
.mock(
"GET",
"/me/contactFolders/CON1/contacts?$top=100&$select=id",
)
.with_status(200)
.with_header("content-type", "application/json")
.with_body(r#"{"value":[{"id":"CT1"}]}"#)
.expect_at_least(0)
.create();
server
.mock("GET", "/me/contacts/CT1")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
r#"{"id":"CT1","displayName":"Alice Liddell","givenName":"Alice",
"surname":"Liddell","emailAddresses":[{"name":"Alice","address":"[email protected]"}]}"#,
)
.expect_at_least(0)
.create();
}
fn stub_mail_surface(server: &mut Server) {
server
.mock("GET", "/me/mailFolders?$top=100&includeHiddenFolders=true")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(r#"{"value":[{"id":"FMAIL","displayName":"Inbox","isHidden":false}]}"#)
.expect_at_least(0)
.create();
server
.mock(
"GET",
"/me/mailFolders/FMAIL/childFolders?$top=100&includeHiddenFolders=true",
)
.with_status(200)
.with_header("content-type", "application/json")
.with_body(r#"{"value":[]}"#)
.expect_at_least(0)
.create();
stub_well_known_folders(server, "FMAIL");
server
.mock("GET", "/me/mailFolders/FMAIL/messages?$top=100&$select=id")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(r#"{"value":[{"id":"MSG-S"}]}"#)
.expect_at_least(0)
.create();
server
.mock("GET", "/me/messages/MSG-S/$value")
.with_status(200)
.with_header("content-type", "text/plain")
.with_body(
"From: a@x\r\nTo: b@x\r\nSubject: surface\r\n\
Date: Tue, 27 May 2026 10:00:00 +0000\r\nMessage-ID: <surface@x>\r\n\r\nbody",
)
.expect_at_least(0)
.create();
}
fn stub_calendar_surface(server: &mut Server) {
server
.mock("GET", "/me/mailboxSettings?$select=timeZone")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(r#"{"timeZone":"UTC"}"#)
.expect_at_least(0)
.create();
server
.mock("GET", "/me/calendars?$top=100")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(r#"{"value":[{"id":"CAL1","name":"Calendar","isDefaultCalendar":true}]}"#)
.expect_at_least(0)
.create();
server
.mock(
"GET",
"/me/calendars/CAL1/events?$top=100&$select=id,type,seriesMasterId",
)
.with_status(200)
.with_header("content-type", "application/json")
.with_body(r#"{"value":[{"id":"EV1","type":"singleInstance","iCalUId":"uid-ev1"}]}"#)
.expect_at_least(0)
.create();
server
.mock("GET", "/me/events/EV1")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
r#"{"id":"EV1","iCalUId":"uid-ev1","type":"singleInstance","subject":"Standup",
"start":{"dateTime":"2026-05-04T15:00:00.0000000","timeZone":"UTC"},
"end":{"dateTime":"2026-05-04T16:00:00.0000000","timeZone":"UTC"}}"#,
)
.expect_at_least(0)
.create();
}
#[test]
fn contacts_surface_imports_only_address_books_and_cards() {
let mut server = Server::new();
server
.mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned()))
.with_status(200)
.with_header("content-type", "application/json")
.with_body(r#"{"id":"uid-surface-c","userPrincipalName":"[email protected]"}"#)
.create();
stub_contacts_surface(&mut server);
stub_calendar_surface(&mut server);
stub_mail_surface(&mut server);
let no_mail = server
.mock("GET", Matcher::Regex(r"^/me/mailFolders\?".to_owned()))
.with_status(500)
.with_body("MUST NOT BE CALLED")
.expect(0)
.create();
let no_calendars = server
.mock("GET", Matcher::Regex(r"^/me/calendars\?".to_owned()))
.with_status(500)
.with_body("MUST NOT BE CALLED")
.expect(0)
.create();
let base = server.url();
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
vandelay::sync::import_exchange_graph::run(
make_common(archive.clone()),
make_config(base, None, surfaces("contacts")),
)
.unwrap();
no_mail.assert();
no_calendars.assert();
let conn = vandelay::db::init::open(&archive).unwrap();
assert_eq!(row_count(&conn, "address_books"), 1);
assert_eq!(row_count(&conn, "contact_cards"), 1);
assert_eq!(row_count(&conn, "mailboxes"), 0);
assert_eq!(row_count(&conn, "emails"), 0);
assert_eq!(row_count(&conn, "calendars"), 0);
assert_eq!(row_count(&conn, "calendar_events"), 0);
}
#[test]
fn mail_surface_imports_only_mailboxes_and_emails() {
let mut server = Server::new();
server
.mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned()))
.with_status(200)
.with_header("content-type", "application/json")
.with_body(r#"{"id":"uid-surface-m","userPrincipalName":"[email protected]"}"#)
.create();
stub_mail_surface(&mut server);
stub_calendar_surface(&mut server);
stub_contacts_surface(&mut server);
let no_calendars = server
.mock("GET", Matcher::Regex(r"^/me/calendars\?".to_owned()))
.with_status(500)
.with_body("MUST NOT BE CALLED")
.expect(0)
.create();
let no_contact_folders = server
.mock("GET", Matcher::Regex(r"^/me/contactFolders\?".to_owned()))
.with_status(500)
.with_body("MUST NOT BE CALLED")
.expect(0)
.create();
let base = server.url();
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
vandelay::sync::import_exchange_graph::run(
make_common(archive.clone()),
make_config(base, None, surfaces("mail")),
)
.unwrap();
no_calendars.assert();
no_contact_folders.assert();
let conn = vandelay::db::init::open(&archive).unwrap();
assert_eq!(row_count(&conn, "mailboxes"), 1);
assert_eq!(row_count(&conn, "emails"), 1);
assert_eq!(row_count(&conn, "address_books"), 0);
assert_eq!(row_count(&conn, "contact_cards"), 0);
assert_eq!(row_count(&conn, "calendars"), 0);
assert_eq!(row_count(&conn, "calendar_events"), 0);
}
+94
View File
@@ -1201,3 +1201,97 @@ fn get_state_reads_state_from_empty_get() {
let st = get_state(&client(0), &url, "w", "Email").expect("get_state"); let st = get_state(&client(0), &url, "w", "Email").expect("get_state");
assert_eq!(st.as_deref(), Some("snap-1")); assert_eq!(st.as_deref(), Some("snap-1"));
} }
fn archive_path() -> std::path::PathBuf {
static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let n = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let mut p = std::env::temp_dir();
p.push(format!(
"vandelay-mockjmap-{}-{n}.sqlite",
std::process::id()
));
let _ = std::fs::remove_file(&p);
p
}
#[test]
fn session_urls_on_a_foreign_origin_are_reported_as_a_mismatch() {
let mut server = mockito::Server::new();
let base = server.url();
let elsewhere = "https://mail.example".to_owned();
let _root = server.mock("GET", "/").with_status(404).create();
let _wk = server
.mock("GET", "/.well-known/jmap")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(session_json(&elsewhere))
.create();
let session = Session::discover(&client(0), &base).expect("session discovered");
let mismatches = session.origin_mismatches(&base);
assert_eq!(mismatches.len(), 1, "one warning per distinct origin");
assert_eq!(
mismatches[0].fields,
vec!["apiUrl", "uploadUrl", "downloadUrl"]
);
let warning = mismatches[0].to_string();
assert!(warning.contains("session advertises apiUrl"), "{warning}");
assert!(warning.contains("connected to "), "{warning}");
assert!(warning.contains("advertised HTTP URL setting"), "{warning}");
}
#[test]
fn import_aborts_with_exit_two_when_the_advertised_api_url_is_unreachable() {
let mut api_host = mockito::Server::new();
let mut session_host = mockito::Server::new();
let session_base = session_host.url();
let api_base = api_host.url();
let _root = session_host.mock("GET", "/").with_status(404).create();
let _wk = session_host
.mock("GET", "/.well-known/jmap")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(session_json(&api_base))
.create();
let _nginx = api_host
.mock("POST", "/jmap/api")
.with_status(404)
.with_header("content-type", "text/html")
.with_body(
"<html><head><title>404 Not Found</title></head><body>404 Not Found</body></html>",
)
.create();
let archive = archive_path();
let err = vandelay::sync::import_jmap::run(
vandelay::sync::CommonConfig {
archive: archive.clone(),
threads: 1,
dry_run: false,
max_retries: 0,
allow_invalid_certs: false,
logger: vandelay::logging::Logger::from_flags(true, 0),
},
vandelay::sync::ImportConfig {
connect: vandelay::sync::ConnectConfig {
url: session_base.clone(),
auth: Auth::Basic {
user: "u".into(),
password: "p".into(),
},
account: AccountSelector::Id("w".into()),
},
objects: None,
allow_source_change: false,
},
)
.expect_err("an unreachable apiUrl must abort the run");
assert!(
matches!(err, vandelay::error::Error::Connection(_)),
"a 404 from the advertised apiUrl is a whole-run connection failure, got {err:?}"
);
assert_eq!(err.exit_code(), 2, "must not report a partial failure");
let _ = std::fs::remove_file(&archive);
}
+809
View File
@@ -1761,6 +1761,449 @@ fn export_calendar_creates_only_missing() {
let _ = std::fs::remove_file(&archive); let _ = std::fs::remove_file(&archive);
} }
#[test]
fn export_contact_and_event_blobs_are_inlined_never_blob_ids() {
let mut server = mockito::Server::new();
let base = server.url();
let api = "/jmap/api";
let archive = tmp();
let photo: &[u8] = b"\x89PNG\r\n\x1a\nfake-photo-bytes";
let agenda: &[u8] = b"agenda for the review meeting";
{
let conn = db::init::open(&archive).unwrap();
conn.execute(
"INSERT INTO address_books (id,name,is_default) VALUES (1,'Personal',1)",
[],
)
.unwrap();
conn.execute(
"INSERT INTO calendars (id,name,is_default) VALUES (1,'Work',1)",
[],
)
.unwrap();
let photo_blob = db::blobs::intern_blob(&conn, photo).unwrap();
let agenda_blob = db::blobs::intern_blob(&conn, agenda).unwrap();
let card = json!({
"@type": "Card",
"version": "1.0",
"name": { "full": "Photo Person" },
"media": { "photo": {
"@type": "Media", "kind": "photo",
"@blob": photo_blob, "mediaType": "image/png"
} }
})
.to_string();
conn.execute(
"INSERT INTO contact_cards (id,uid,address_book_ids,data)
VALUES (1,'card-uid-1','[1]',?1)",
[card.as_str()],
)
.unwrap();
let event = json!({
"@type": "Event",
"uid": "event-uid-1",
"title": "Review",
"start": "2026-03-01T09:00:00",
"duration": "PT1H",
"timeZone": "Etc/UTC",
"links": { "1": {
"@type": "Link", "rel": "enclosure",
"@blob": agenda_blob, "contentType": "text/plain", "title": "agenda.txt"
} }
})
.to_string();
conn.execute(
"INSERT INTO calendar_events (id,calendar_ids,is_draft,use_default_alerts,data)
VALUES (1,'[1]',0,0,?1)",
[event.as_str()],
)
.unwrap();
}
let _root = server.mock("GET", "/").with_status(404).create();
let _wk = server
.mock("GET", "/.well-known/jmap")
.with_body(session_body_full(&base))
.expect_at_least(1)
.create();
let no_blob_id_in_card = server
.mock("POST", api)
.match_body(Matcher::AllOf(vec![
Matcher::Regex("ContactCard/set".into()),
Matcher::Regex("blobId".into()),
]))
.with_body(
json!({"methodResponses":[["ContactCard/set",{"accountId":"w",
"created":{"c1":{"id":"TCARD"}}},"s"]]})
.to_string(),
)
.expect(0)
.create();
let no_blob_id_in_event = server
.mock("POST", api)
.match_body(Matcher::AllOf(vec![
Matcher::Regex("CalendarEvent/set".into()),
Matcher::Regex("blobId".into()),
]))
.with_body(
json!({"methodResponses":[["CalendarEvent/set",{"accountId":"w",
"created":{"c1":{"id":"TEVENT"}}},"s"]]})
.to_string(),
)
.expect(0)
.create();
let _ab_get = server
.mock("POST", api)
.match_body(Matcher::Regex("AddressBook/get".into()))
.with_body(
json!({"methodResponses":[["AddressBook/get",{"accountId":"w","list":[],
"notFound":[]},"g"]]})
.to_string(),
)
.expect_at_least(1)
.create();
let _ab_set = server
.mock("POST", api)
.match_body(Matcher::Regex("AddressBook/set".into()))
.with_body(
json!({"methodResponses":[["AddressBook/set",{"accountId":"w",
"created":{"c1":{"id":"TAB"}}},"s"]]})
.to_string(),
)
.expect(1)
.create();
let _cal_get = server
.mock("POST", api)
.match_body(Matcher::Regex("Calendar/get".into()))
.with_body(
json!({"methodResponses":[["Calendar/get",{"accountId":"w","list":[],
"notFound":[]},"g"]]})
.to_string(),
)
.expect_at_least(1)
.create();
let _cal_set = server
.mock("POST", api)
.match_body(Matcher::Regex("Calendar/set".into()))
.with_body(
json!({"methodResponses":[["Calendar/set",{"accountId":"w",
"created":{"c1":{"id":"TCAL"}}},"s"]]})
.to_string(),
)
.expect(1)
.create();
let _cc_query = server
.mock("POST", api)
.match_body(Matcher::Regex("ContactCard/query".into()))
.with_body(
json!({"methodResponses":[["ContactCard/query",{"accountId":"w","ids":[]},"q"]]})
.to_string(),
)
.expect_at_least(1)
.create();
let _ce_query = server
.mock("POST", api)
.match_body(Matcher::Regex("CalendarEvent/query".into()))
.with_body(
json!({"methodResponses":[["CalendarEvent/query",{"accountId":"w","ids":[]},"q"]]})
.to_string(),
)
.expect_at_least(1)
.create();
let card_inlined = server
.mock("POST", api)
.match_body(Matcher::AllOf(vec![
Matcher::Regex("ContactCard/set".into()),
Matcher::Regex("data:image/png;base64,".into()),
]))
.with_body(
json!({"methodResponses":[["ContactCard/set",{"accountId":"w",
"created":{"c1":{"id":"TCARD"}}},"s"]]})
.to_string(),
)
.expect(1)
.create();
let event_inlined = server
.mock("POST", api)
.match_body(Matcher::AllOf(vec![
Matcher::Regex("CalendarEvent/set".into()),
Matcher::Regex("data:text/plain;base64,".into()),
]))
.with_body(
json!({"methodResponses":[["CalendarEvent/set",{"accountId":"w",
"created":{"c1":{"id":"TEVENT"}}},"s"]]})
.to_string(),
)
.expect(1)
.create();
let summary = sync::export::run(
common(&archive),
export_cfg_objects(
&base,
vec![
ObjectType::AddressBook,
ObjectType::Calendar,
ObjectType::ContactCard,
ObjectType::CalendarEvent,
],
),
)
.expect("export");
no_blob_id_in_card.assert();
no_blob_id_in_event.assert();
card_inlined.assert();
event_inlined.assert();
for name in ["ContactCard", "CalendarEvent"] {
let counts = summary
.per_type
.iter()
.find(|(t, _)| *t == name)
.map(|(_, c)| c.clone())
.unwrap_or_else(|| panic!("{name} counts"));
assert_eq!(counts.created, 1, "{name} created: {counts:?}");
assert_eq!(counts.failed, 0, "{name} failed: {counts:?}");
}
let _ = std::fs::remove_file(&archive);
}
fn session_body_small_request(base: &str, max_size_request: u64) -> String {
json!({
"apiUrl": format!("{base}/jmap/api"),
"uploadUrl": format!("{base}/jmap/upload/{{accountId}}/"),
"downloadUrl": format!("{base}/jmap/dl/{{accountId}}/{{blobId}}/{{type}}/{{name}}"),
"capabilities": { "urn:ietf:params:jmap:core": {
"maxObjectsInGet": 500, "maxObjectsInSet": 500, "maxCallsInRequest": 16,
"maxConcurrentRequests": 4, "maxConcurrentUpload": 4,
"maxSizeRequest": max_size_request, "maxSizeUpload": 50000000
} },
"accounts": { "w": { "name": "alice",
"accountCapabilities": {
"urn:ietf:params:jmap:mail": {},
"urn:ietf:params:jmap:contacts": {},
"urn:ietf:params:jmap:calendars": {}
} } }
})
.to_string()
}
#[test]
fn export_oversized_contact_card_fails_alone_and_the_type_keeps_going() {
let mut server = mockito::Server::new();
let base = server.url();
let api = "/jmap/api";
let archive = tmp();
{
let conn = db::init::open(&archive).unwrap();
conn.execute(
"INSERT INTO address_books (id,name,is_default) VALUES (1,'Personal',1)",
[],
)
.unwrap();
let photo = vec![b'P'; 8000];
let photo_blob = db::blobs::intern_blob(&conn, &photo).unwrap();
let huge = json!({
"@type": "Card",
"name": { "full": "Photo Person" },
"media": { "photo": {
"@type": "Media", "kind": "photo",
"@blob": photo_blob, "mediaType": "image/png"
} }
})
.to_string();
conn.execute(
"INSERT INTO contact_cards (id,uid,address_book_ids,data)
VALUES (1,'huge-card','[1]',?1)",
[huge.as_str()],
)
.unwrap();
let small = json!({ "@type": "Card", "name": { "full": "Small Person" } }).to_string();
conn.execute(
"INSERT INTO contact_cards (id,uid,address_book_ids,data)
VALUES (2,'small-card','[1]',?1)",
[small.as_str()],
)
.unwrap();
}
let _root = server.mock("GET", "/").with_status(404).create();
let _wk = server
.mock("GET", "/.well-known/jmap")
.with_body(session_body_small_request(&base, 4000))
.expect_at_least(1)
.create();
let _ccterm = anchor_terminator(&mut server, api, "ContactCard");
let _ab_get = server
.mock("POST", api)
.match_body(Matcher::Regex("AddressBook/get".into()))
.with_body(
json!({"methodResponses":[["AddressBook/get",{"accountId":"w","list":[],
"notFound":[]},"g"]]})
.to_string(),
)
.expect_at_least(1)
.create();
let _ab_set = server
.mock("POST", api)
.match_body(Matcher::Regex("AddressBook/set".into()))
.with_body(
json!({"methodResponses":[["AddressBook/set",{"accountId":"w",
"created":{"c1":{"id":"TAB"}}},"s"]]})
.to_string(),
)
.expect(1)
.create();
let _cc_query = server
.mock("POST", api)
.match_body(Matcher::Regex("ContactCard/query".into()))
.with_body(
json!({"methodResponses":[["ContactCard/query",{"accountId":"w",
"ids":["TSTALE"]},"q"]]})
.to_string(),
)
.expect_at_least(1)
.create();
let _cc_get = server
.mock("POST", api)
.match_body(Matcher::Regex("ContactCard/get".into()))
.with_body(
json!({"methodResponses":[["ContactCard/get",{"accountId":"w","list":[
{"id":"TSTALE","uid":"stale-card"}
],"notFound":[]},"g"]]})
.to_string(),
)
.expect_at_least(1)
.create();
let destroyed = server
.mock("POST", api)
.match_body(Matcher::AllOf(vec![
Matcher::Regex("ContactCard/set".into()),
Matcher::Regex("destroy".into()),
]))
.with_body(
json!({"methodResponses":[["ContactCard/set",{"accountId":"w",
"destroyed":["TSTALE"]},"s"]]})
.to_string(),
)
.expect(1)
.create();
let created = server
.mock("POST", api)
.match_body(Matcher::AllOf(vec![
Matcher::Regex("ContactCard/set".into()),
Matcher::Regex("Small Person".into()),
]))
.with_body(
json!({"methodResponses":[["ContactCard/set",{"accountId":"w",
"created":{"c2":{"id":"TSMALL"}}},"s"]]})
.to_string(),
)
.expect(1)
.create();
let summary = sync::export::run(
common(&archive),
ExportConfig {
prune: true,
..export_cfg_objects(
&base,
vec![ObjectType::AddressBook, ObjectType::ContactCard],
)
},
)
.expect("the oversized card must not abort the run");
created.assert();
destroyed.assert();
let counts = summary
.per_type
.iter()
.find(|(t, _)| *t == "ContactCard")
.map(|(_, c)| c.clone())
.expect("ContactCard counts");
assert_eq!(counts.created, 1, "{counts:?}");
assert_eq!(counts.failed, 1, "{counts:?}");
assert_eq!(
counts.deleted, 1,
"the prune plan stays complete: {counts:?}"
);
let _ = std::fs::remove_file(&archive);
}
#[test]
fn export_archive_read_failure_while_inlining_exits_seven() {
let mut server = mockito::Server::new();
let base = server.url();
let api = "/jmap/api";
let archive = tmp();
{
let conn = db::init::open(&archive).unwrap();
conn.execute(
"INSERT INTO address_books (id,name,is_default) VALUES (1,'Personal',1)",
[],
)
.unwrap();
let photo_blob = db::blobs::intern_blob(&conn, b"photo-bytes").unwrap();
let card = json!({
"@type": "Card",
"media": { "photo": { "@type": "Media", "kind": "photo",
"@blob": photo_blob, "mediaType": "image/png" } }
})
.to_string();
conn.execute(
"INSERT INTO contact_cards (id,uid,address_book_ids,data)
VALUES (1,'card-uid-1','[1]',?1)",
[card.as_str()],
)
.unwrap();
conn.execute("ALTER TABLE blobs RENAME COLUMN data TO payload", [])
.unwrap();
}
let _root = server.mock("GET", "/").with_status(404).create();
let _wk = server
.mock("GET", "/.well-known/jmap")
.with_body(session_body_full(&base))
.expect_at_least(1)
.create();
let _cc_query = server
.mock("POST", api)
.match_body(Matcher::Regex("ContactCard/query".into()))
.with_body(
json!({"methodResponses":[["ContactCard/query",{"accountId":"w","ids":[]},"q"]]})
.to_string(),
)
.expect_at_least(1)
.create();
let no_create = server
.mock("POST", api)
.match_body(Matcher::Regex("ContactCard/set".into()))
.with_body(
json!({"methodResponses":[["ContactCard/set",{"accountId":"w",
"created":{"c1":{"id":"TCARD"}}},"s"]]})
.to_string(),
)
.expect(0)
.create();
let err = sync::export::run(
common(&archive),
export_cfg_objects(&base, vec![ObjectType::ContactCard]),
)
.expect_err("an unreadable blobs table must abort the run");
no_create.assert();
assert!(err.aborts_run(), "{err}");
assert_eq!(err.exit_code(), 7, "{err}");
let _ = std::fs::remove_file(&archive);
}
#[test] #[test]
fn export_sieve_script_matches_by_name_not_content() { fn export_sieve_script_matches_by_name_not_content() {
let mut server = mockito::Server::new(); let mut server = mockito::Server::new();
@@ -3264,3 +3707,369 @@ fn import_jmap_duplicate_role_is_deduplicated_to_single_mailbox() {
drop(conn); drop(conn);
let _ = std::fs::remove_file(&archive); let _ = std::fs::remove_file(&archive);
} }
#[test]
fn export_aborts_with_exit_two_when_the_advertised_api_url_is_unreachable() {
let mut session_host = mockito::Server::new();
let mut api_host = mockito::Server::new();
let session_base = session_host.url();
let api_base = api_host.url();
let archive = tmp();
{
let conn = db::init::open(&archive).unwrap();
conn.execute(
"INSERT INTO mailboxes (id,name,parent_id,role,sort_order,is_subscribed)
VALUES (1,'Inbox',NULL,'inbox',0,1)",
[],
)
.unwrap();
conn.execute(
"INSERT INTO address_books (id,name,description,is_default)
VALUES (1,'Personal',NULL,1)",
[],
)
.unwrap();
}
let _root = session_host.mock("GET", "/").with_status(404).create();
let _wk = session_host
.mock("GET", "/.well-known/jmap")
.with_body(session_body_full(&api_base))
.expect_at_least(1)
.create();
let nginx = api_host
.mock("POST", "/jmap/api")
.with_status(404)
.with_header("content-type", "text/html")
.with_body("<html><head><title>404 Not Found</title></head><body>nginx</body></html>")
.expect(1)
.create();
let err = sync::export::run(
CommonConfig {
archive: archive.clone(),
threads: 1,
dry_run: false,
max_retries: 0,
allow_invalid_certs: false,
logger: Logger::from_flags(true, 0),
},
export_cfg_objects(
&session_base,
vec![ObjectType::Mailbox, ObjectType::AddressBook],
),
)
.expect_err("an unreachable apiUrl must abort the run");
assert!(
matches!(err, vandelay::error::Error::Connection(_)),
"a 404 from the advertised apiUrl is a whole-run connection failure, got {err:?}"
);
assert_eq!(err.exit_code(), 2, "must not report a partial failure");
nginx.assert();
let _ = std::fs::remove_file(&archive);
}
#[test]
fn aborting_import_still_reports_the_types_that_finished() {
let mut server = mockito::Server::new();
let base = server.url();
let api = "/jmap/api";
let archive = tmp();
let _root = server.mock("GET", "/").with_status(404).create();
let _wk = server
.mock("GET", "/.well-known/jmap")
.with_body(session_body_full(&base))
.expect_at_least(1)
.create();
let _term = anchor_terminator(&mut server, api, "Mailbox");
let _mq = server
.mock("POST", api)
.match_body(Matcher::Regex("Mailbox/query".into()))
.with_body(
json!({"methodResponses":[["Mailbox/query",
{"accountId":"w","ids":["A"]},"q"]]})
.to_string(),
)
.expect_at_least(1)
.create();
let _mg = server
.mock("POST", api)
.match_body(Matcher::Regex("Mailbox/get".into()))
.with_body(
json!({"methodResponses":[["Mailbox/get",{"accountId":"w","state":"s1","list":[
{"id":"A","name":"alpha","parentId":null,"role":null,"sortOrder":0,"isSubscribed":true}
],"notFound":[]},"g"]]})
.to_string(),
)
.expect_at_least(1)
.create();
let _dead_endpoint = server
.mock("POST", api)
.match_body(Matcher::Regex("Email/query".into()))
.with_status(404)
.with_header("content-type", "text/html")
.with_body("<html><head><title>404 Not Found</title></head><body>nginx</body></html>")
.expect_at_least(1)
.create();
let outcome = sync::import_jmap::run_reporting(
common(&archive),
import_cfg_objects(&base, vec![ObjectType::Mailbox, ObjectType::Email]),
);
let err = outcome
.error
.as_ref()
.expect("a 404 from the api endpoint aborts the run");
assert!(
matches!(err, vandelay::error::Error::Connection(_)),
"the abort must keep its own exit code, got {err:?}"
);
assert_eq!(err.exit_code(), 2);
let mailbox = outcome
.summary
.per_type
.iter()
.find(|(t, _)| *t == "Mailbox")
.map(|(_, c)| c.clone())
.expect("the type that finished before the abort must still be reported");
assert_eq!(mailbox.fetched, 1);
assert!(
!outcome.summary.per_type.iter().any(|(t, _)| *t == "Email"),
"the type that aborted contributes no counts"
);
let _ = std::fs::remove_file(&archive);
}
fn seed_single_email(archive: &Path) {
let conn = db::init::open(archive).unwrap();
conn.execute(
"INSERT INTO mailboxes (id,name,parent_id,role) VALUES (1,'Inbox',NULL,'inbox')",
[],
)
.unwrap();
let blob = db::blobs::intern_blob(
&conn,
b"From: a@x\r\nSubject: hi\r\nMessage-ID: <m-9@h>\r\n\r\nbody",
)
.unwrap();
conn.execute(
"INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords)
VALUES (?1,'2020-01-01T00:00:00Z','[1]','[\"$seen\"]')",
rusqlite::params![blob],
)
.unwrap();
}
fn mock_reconcile_for_single_email(server: &mut mockito::ServerGuard, base: &str, api: &str) {
server.mock("GET", "/").with_status(404).create();
server
.mock("GET", "/.well-known/jmap")
.with_body(session_body(base))
.create();
server
.mock("POST", api)
.match_body(Matcher::Regex("Mailbox/query".into()))
.with_body(
json!({"methodResponses":[["Mailbox/query",
{"accountId":"w","ids":["t1"]},"q"]]})
.to_string(),
)
.expect(1)
.create();
server
.mock("POST", api)
.match_body(Matcher::Regex("Mailbox/query".into()))
.with_body(
json!({"methodResponses":[["Mailbox/query",
{"accountId":"w","ids":[]},"q"]]})
.to_string(),
)
.expect(1)
.create();
server
.mock("POST", api)
.match_body(Matcher::Regex("Mailbox/get".into()))
.with_body(
json!({"methodResponses":[["Mailbox/get",{"accountId":"w","list":[
{"id":"t1","name":"Inbox","role":"inbox","parentId":null,
"myRights":{"mayDelete":true}}],"notFound":[]},"g"]]})
.to_string(),
)
.expect(1)
.create();
server
.mock("POST", api)
.match_body(Matcher::Regex("Email/query".into()))
.with_body(
json!({"methodResponses":[["Email/query",
{"accountId":"w","ids":[]},"q"]]})
.to_string(),
)
.expect(1)
.create();
}
fn run_export(archive: &Path, base: &str, max_retries: u32) -> sync::Summary {
sync::export::run(
CommonConfig {
archive: archive.to_path_buf(),
threads: 1,
dry_run: false,
max_retries,
allow_invalid_certs: false,
logger: Logger::from_flags(true, 0),
},
ExportConfig {
connect: ConnectConfig {
url: base.to_owned(),
auth: Auth::Basic {
user: "u".into(),
password: "p".into(),
},
account: AccountSelector::Id("w".into()),
},
objects: None,
prune: false,
yes: true,
},
)
.expect("export run")
}
fn email_counts(summary: &sync::Summary) -> sync::TypeCounts {
summary
.per_type
.iter()
.find(|(t, _)| *t == "Email")
.map(|(_, c)| c.clone())
.expect("email counts")
}
fn method_error_body(error_type: &str) -> String {
json!({"methodResponses":[["error",
{"type": error_type,
"description":"This server is temporarily unavailable. \
Attempting this same operation later may succeed."},
"i"]]})
.to_string()
}
#[test]
fn export_email_server_unavailable_is_retried_not_dropped() {
let mut server = mockito::Server::new();
let base = server.url();
let api = "/jmap/api";
let archive = tmp();
seed_single_email(&archive);
mock_reconcile_for_single_email(&mut server, &base, api);
let up = server
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
.with_body(json!({"blobId":"UPLOADED"}).to_string())
.expect(1)
.create();
let failing = server
.mock("POST", api)
.match_body(Matcher::Regex("Email/import".into()))
.with_body(method_error_body("serverUnavailable"))
.expect(2)
.create();
let succeeding = server
.mock("POST", api)
.match_body(Matcher::Regex("Email/import".into()))
.with_body(
json!({"methodResponses":[["Email/import",{"accountId":"w",
"created":{"e1":{"id":"E1","blobId":"UPLOADED","threadId":"T1","size":48}}},"i"]]})
.to_string(),
)
.expect(1)
.create();
let summary = run_export(&archive, &base, 3);
let email = email_counts(&summary);
failing.assert();
succeeding.assert();
up.assert();
assert_eq!(email.created, 1, "the message must survive a transient error");
assert_eq!(email.failed, 0, "a retried message is not a failure");
assert!(!summary.any_failed());
let _ = std::fs::remove_file(&archive);
}
#[test]
fn export_email_exhausted_method_retries_stay_per_message() {
let mut server = mockito::Server::new();
let base = server.url();
let api = "/jmap/api";
let archive = tmp();
seed_single_email(&archive);
mock_reconcile_for_single_email(&mut server, &base, api);
let _up = server
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
.with_body(json!({"blobId":"UPLOADED"}).to_string())
.expect(1)
.create();
let failing = server
.mock("POST", api)
.match_body(Matcher::Regex("Email/import".into()))
.with_body(method_error_body("serverUnavailable"))
.expect(3)
.create();
let summary = run_export(&archive, &base, 2);
let email = email_counts(&summary);
failing.assert();
assert_eq!(email.created, 0);
assert_eq!(
email.failed, 1,
"exhausting the retries stays a per-message failure, not a run abort"
);
assert!(summary.any_failed(), "the run must still exit 5");
let _ = std::fs::remove_file(&archive);
}
#[test]
fn export_email_fatal_method_error_is_not_retried() {
let mut server = mockito::Server::new();
let base = server.url();
let api = "/jmap/api";
let archive = tmp();
seed_single_email(&archive);
mock_reconcile_for_single_email(&mut server, &base, api);
let _up = server
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
.with_body(json!({"blobId":"UPLOADED"}).to_string())
.expect(1)
.create();
let failing = server
.mock("POST", api)
.match_body(Matcher::Regex("Email/import".into()))
.with_body(method_error_body("invalidArguments"))
.expect(1)
.create();
let summary = run_export(&archive, &base, 5);
let email = email_counts(&summary);
failing.assert();
assert_eq!(
email.failed, 1,
"a permanent method error must not consume retries"
);
let _ = std::fs::remove_file(&archive);
}
+161
View File
@@ -689,6 +689,167 @@ fn prune_non_tree_type_contact_card() {
seeder::teardown(base_url()).expect("teardown"); seeder::teardown(base_url()).expect("teardown");
} }
const ISSUE30_CARD_UID: &str = "vandelay-issue30-card";
const ISSUE30_EVENT_UID: &str = "vandelay-issue30-event";
fn data_uri_bytes(resource: &Value, uri_key: &str, expect_media_type: &str) -> Vec<u8> {
use base64::Engine;
let uri = resource
.get(uri_key)
.and_then(Value::as_str)
.unwrap_or_else(|| panic!("no {uri_key} on {resource}"));
assert!(
resource.get("blobId").is_none(),
"target must not hold a blobId for {uri_key}: {resource}"
);
let prefix = format!("data:{expect_media_type};base64,");
let payload = uri
.strip_prefix(&prefix)
.unwrap_or_else(|| panic!("{uri_key} is not a {prefix}... data URI: {resource}"));
base64::engine::general_purpose::STANDARD
.decode(payload)
.expect("base64 payload")
}
#[test]
#[ignore = "requires Docker"]
fn export_inlines_contact_and_event_blobs_instead_of_blob_ids() {
let fx = seeder::provision(base_url()).expect("provision");
let tgt = fx.account("test6").expect("test6");
let archive = tmp_archive("issue30");
let photo: &[u8] = b"\x89PNG\r\n\x1a\nvandelay issue 30 contact photo bytes";
let agenda: &[u8] = b"vandelay issue 30 calendar enclosure bytes";
{
let conn = vandelay::db::init::open(&archive).expect("init archive");
conn.execute(
"INSERT INTO address_books (id,name,is_default) VALUES (1,'Issue30 Book',1)",
[],
)
.unwrap();
conn.execute(
"INSERT INTO calendars (id,name,is_default) VALUES (1,'Issue30 Calendar',1)",
[],
)
.unwrap();
let photo_blob = vandelay::db::blobs::intern_blob(&conn, photo).unwrap();
let agenda_blob = vandelay::db::blobs::intern_blob(&conn, agenda).unwrap();
let card = json!({
"@type": "Card",
"version": "1.0",
"name": { "full": "Issue Thirty" },
"media": { "photo": {
"@type": "Media", "kind": "photo",
"@blob": photo_blob, "mediaType": "image/png"
} }
})
.to_string();
conn.execute(
"INSERT INTO contact_cards (id,uid,address_book_ids,data) VALUES (1,?1,'[1]',?2)",
[ISSUE30_CARD_UID, card.as_str()],
)
.unwrap();
let event = json!({
"@type": "Event",
"uid": ISSUE30_EVENT_UID,
"title": "Issue 30 review",
"start": "2026-03-01T09:00:00",
"duration": "PT1H",
"timeZone": "Etc/UTC",
"links": { "1": {
"@type": "Link", "rel": "enclosure",
"@blob": agenda_blob, "contentType": "text/plain", "title": "agenda.txt"
} }
})
.to_string();
conn.execute(
"INSERT INTO calendar_events (id,calendar_ids,is_draft,use_default_alerts,data)
VALUES (1,'[1]',0,0,?1)",
[event.as_str()],
)
.unwrap();
}
let summary = sync::export::run(
common(&archive, false),
export_cfg("test6", &tgt.account_id, false),
)
.expect("export");
for name in ["ContactCard", "CalendarEvent"] {
let counts = summary
.per_type
.iter()
.find(|(t, _)| *t == name)
.map(|(_, c)| c.clone())
.unwrap_or_else(|| panic!("{name} counts missing: {summary:?}"));
assert_eq!(counts.created, 1, "{name} created exactly one: {counts:?}");
assert_eq!(counts.failed, 0, "{name} had no failures: {counts:?}");
}
let client = HttpClient::new(basic("test6"), RetryPolicy::new(5), true);
let session = Session::discover(&client, base_url()).expect("discover target session");
let api = session.api_url.clone();
let cards = client
.post_json(
&api,
&json!({
"using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:contacts"],
"methodCalls": [["ContactCard/get",
{ "accountId": tgt.account_id, "ids": Value::Null }, "g"]]
}),
)
.expect("ContactCard/get");
let card = cards
.pointer("/methodResponses/0/1/list")
.and_then(Value::as_array)
.expect("ContactCard list")
.iter()
.find(|c| c.get("uid").and_then(Value::as_str) == Some(ISSUE30_CARD_UID))
.unwrap_or_else(|| panic!("exported card not found on the target: {cards}"))
.clone();
let photo_resource = card
.pointer("/media/photo")
.unwrap_or_else(|| panic!("card has no media.photo: {card}"));
assert_eq!(
data_uri_bytes(photo_resource, "uri", "image/png"),
photo,
"contact photo bytes survived the export: {card}"
);
let events = client
.post_json(
&api,
&json!({
"using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:calendars"],
"methodCalls": [["CalendarEvent/get",
{ "accountId": tgt.account_id, "ids": Value::Null }, "g"]]
}),
)
.expect("CalendarEvent/get");
let event = events
.pointer("/methodResponses/0/1/list")
.and_then(Value::as_array)
.expect("CalendarEvent list")
.iter()
.find(|e| e.get("uid").and_then(Value::as_str) == Some(ISSUE30_EVENT_UID))
.unwrap_or_else(|| panic!("exported event not found on the target: {events}"))
.clone();
let link = event
.get("links")
.and_then(Value::as_object)
.and_then(|m| m.values().next())
.unwrap_or_else(|| panic!("event has no links: {event}"));
assert_eq!(
data_uri_bytes(link, "href", "text/plain"),
agenda,
"calendar enclosure bytes survived the export: {event}"
);
let _ = std::fs::remove_file(&archive);
seeder::teardown(base_url()).expect("teardown");
}
#[test] #[test]
#[ignore = "requires Docker"] #[ignore = "requires Docker"]
fn export_round_trip_and_convergence() { fn export_round_trip_and_convergence() {