+2
-1
@@ -7,4 +7,5 @@ target
|
||||
*.md
|
||||
!README.md
|
||||
!CHANGELOG.md
|
||||
specs/
|
||||
.*
|
||||
!.github
|
||||
|
||||
@@ -2,6 +2,21 @@
|
||||
|
||||
All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/).
|
||||
|
||||
## [1.0.8] - 2026-08-15
|
||||
|
||||
### Added
|
||||
- A warning when the session advertises `apiUrl`, `uploadUrl` or `downloadUrl` on a different origin than the one connected to, naming both origins (#29).
|
||||
- The per-type report is printed when a run aborts.
|
||||
|
||||
### Changed
|
||||
|
||||
### Fixed
|
||||
- A transient `serverUnavailable` from `Email/import` permanently dropped that message from the migration instead of retrying it.
|
||||
- The Exchange Graph importer stored a whole converted event as a `recurrenceOverrides` value (#31).
|
||||
- Exporting a ContactCard carrying a photo blob failed against Stalwart, and a CalendarEvent enclosure sent as a `blobId` was silently dropped by the server; the bytes of both are now inlined as a base64 `data:` URI (#30).
|
||||
- A connection failure against the session-advertised `apiUrl` was reported as a per-type partial failure (exit 5, "consistent and resumable") instead of aborting (#29).
|
||||
- Session `apiUrl`, `uploadUrl` and `downloadUrl` given as relative references were used unresolved (#29).
|
||||
|
||||
## [1.0.7] - 2026-07-26
|
||||
|
||||
### Added
|
||||
|
||||
Generated
+298
-319
File diff suppressed because it is too large
Load Diff
+3
-3
@@ -1,7 +1,7 @@
|
||||
[package]
|
||||
name = "vandelay"
|
||||
description = "JMAP account migration utility"
|
||||
version = "1.0.7"
|
||||
version = "1.0.8"
|
||||
authors = ["Stalwart Labs LLC <[email protected]>"]
|
||||
license = "Apache-2.0 OR MIT"
|
||||
repository = "https://github.com/stalwartlabs/vandelay"
|
||||
@@ -36,7 +36,7 @@ ureq = { version = "3.3.0", default-features = false, features = ["json", "gzip"
|
||||
clap = { version = "4.6.1", features = ["derive"] }
|
||||
crossbeam-channel = "0.5.15"
|
||||
num_cpus = "1.17.0"
|
||||
base64 = "0.22.1"
|
||||
base64 = "0.23"
|
||||
mail-parser = "0.11.3"
|
||||
calcard = "0.3"
|
||||
rustls = { version = "0.23.40", default-features = false, features = ["aws-lc-rs", "std", "tls12", "logging"] }
|
||||
@@ -50,7 +50,7 @@ chrono-tz = { version = "0.10", default-features = false }
|
||||
[dev-dependencies]
|
||||
mockito = "1.7.2"
|
||||
tempfile = "3.13"
|
||||
testcontainers = { version = "0.27", features = ["blocking"] }
|
||||
testcontainers = { version = "0.28", features = ["blocking"] }
|
||||
|
||||
[profile.dist]
|
||||
inherits = "release"
|
||||
|
||||
+60
-6
@@ -12,7 +12,7 @@ use regex::Regex;
|
||||
use crate::error::Error;
|
||||
use crate::exchange_ews::oauth::OAuthFlow;
|
||||
use crate::exchange_ews::types::MailboxKind;
|
||||
use crate::exchange_graph::types::{EventBodyFormat, MailboxKind as GraphMailboxKind};
|
||||
use crate::exchange_graph::types::{EventBodyFormat, MailboxKind as GraphMailboxKind, Surfaces};
|
||||
use crate::inspect::InspectConfig;
|
||||
use crate::jmap::account::AccountSelector;
|
||||
use crate::jmap::http::Auth;
|
||||
@@ -1132,7 +1132,7 @@ pub struct ExchangeGraphImportArgs {
|
||||
#[arg(
|
||||
long,
|
||||
value_name = "LIST",
|
||||
help = "Comma-separated surface list (mail,calendar,contacts; default all)"
|
||||
help = "Comma-separated surface list: mail | calendar | contacts (default: all three)"
|
||||
)]
|
||||
objects: Option<String>,
|
||||
|
||||
@@ -1203,11 +1203,12 @@ fn resolve_exchange_graph_import(args: ExchangeGraphImportArgs) -> Result<Action
|
||||
args.client_id.as_deref(),
|
||||
&args.tenant,
|
||||
)?;
|
||||
let objects = args
|
||||
let surfaces = args
|
||||
.objects
|
||||
.as_deref()
|
||||
.map(crate::types::parse_object_list)
|
||||
.transpose()?;
|
||||
.map(Surfaces::parse_list)
|
||||
.transpose()?
|
||||
.unwrap_or_default();
|
||||
let top = args.top.clamp(1, 1000);
|
||||
let graph_connections = args
|
||||
.graph_connections
|
||||
@@ -1220,7 +1221,7 @@ fn resolve_exchange_graph_import(args: ExchangeGraphImportArgs) -> Result<Action
|
||||
api_base: args.api_base,
|
||||
user_target: args.user,
|
||||
mailbox_kind,
|
||||
objects,
|
||||
surfaces,
|
||||
event_body_format,
|
||||
graph_connections,
|
||||
top,
|
||||
@@ -1361,4 +1362,57 @@ mod tests {
|
||||
let auth = resolve_auth(None, None, Some(&bearer)).unwrap();
|
||||
assert!(matches!(auth, Auth::Bearer { token } if token == "t"));
|
||||
}
|
||||
|
||||
fn graph_config(objects: &[&str]) -> Result<GraphImportConfig, Error> {
|
||||
let mut argv = vec![
|
||||
"vandelay",
|
||||
"import",
|
||||
"exchange-graph",
|
||||
"--access-token",
|
||||
"tok",
|
||||
];
|
||||
argv.extend_from_slice(objects);
|
||||
argv.push("archive.sqlite");
|
||||
let cli = Cli::try_parse_from(argv).unwrap_or_else(|e| panic!("clap rejected args: {e}"));
|
||||
match cli.resolve()? {
|
||||
Action::ImportExchangeGraph(_, config) => Ok(config),
|
||||
_ => panic!("expected ImportExchangeGraph"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn graph_objects_contacts_selects_only_contacts() {
|
||||
let _g = lock();
|
||||
let config = graph_config(&["--objects", "contacts"]).unwrap();
|
||||
assert_eq!(
|
||||
config.surfaces,
|
||||
Surfaces {
|
||||
mail: false,
|
||||
calendar: false,
|
||||
contacts: true
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn graph_objects_absent_selects_every_surface() {
|
||||
let _g = lock();
|
||||
let config = graph_config(&[]).unwrap();
|
||||
assert_eq!(config.surfaces, Surfaces::ALL);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn graph_objects_rejects_jmap_type_names() {
|
||||
let _g = lock();
|
||||
match graph_config(&["--objects", "contactcard"]) {
|
||||
Err(Error::Usage(m)) => {
|
||||
assert!(m.contains("unknown surface: contactcard"), "msg was: {m}");
|
||||
assert!(
|
||||
m.contains("valid: mail, calendar, contacts"),
|
||||
"msg was: {m}"
|
||||
);
|
||||
}
|
||||
other => panic!("expected Usage error, got {:?}", other.map(|_| ())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,6 +43,20 @@ impl From<rusqlite::Error> for Error {
|
||||
}
|
||||
|
||||
impl Error {
|
||||
pub fn aborts_run(&self) -> bool {
|
||||
match self {
|
||||
Error::Partial(_) => false,
|
||||
Error::Usage(_)
|
||||
| Error::Connection(_)
|
||||
| Error::Account(_)
|
||||
| Error::SourceChange(_)
|
||||
| Error::PruneAborted
|
||||
| Error::Unimplemented(_)
|
||||
| Error::Db(_)
|
||||
| Error::Io(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn exit_code(&self) -> i32 {
|
||||
match self {
|
||||
Error::Usage(_) => 1,
|
||||
@@ -74,4 +88,37 @@ mod tests {
|
||||
let io = Error::Io(std::io::Error::other("disk"));
|
||||
assert_eq!(io.exit_code(), 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_partial_is_a_per_unit_failure() {
|
||||
assert!(!Error::Partial("one object".into()).aborts_run());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whole_run_conditions_abort() {
|
||||
assert!(Error::Connection("http status 404".into()).aborts_run());
|
||||
assert!(Error::Account("ambiguous".into()).aborts_run());
|
||||
assert!(Error::SourceChange("other account".into()).aborts_run());
|
||||
assert!(Error::Usage("bad flag".into()).aborts_run());
|
||||
assert!(Error::Unimplemented("x").aborts_run());
|
||||
assert!(Error::PruneAborted.aborts_run());
|
||||
assert!(Error::Db(OpenError::Sqlite(rusqlite::Error::QueryReturnedNoRows)).aborts_run());
|
||||
assert!(Error::Io(std::io::Error::other("disk")).aborts_run());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aborting_errors_do_not_map_to_the_partial_exit_code() {
|
||||
for e in [
|
||||
Error::Connection("x".into()),
|
||||
Error::Account("x".into()),
|
||||
Error::SourceChange("x".into()),
|
||||
Error::Db(OpenError::Sqlite(rusqlite::Error::QueryReturnedNoRows)),
|
||||
Error::Io(std::io::Error::other("disk")),
|
||||
] {
|
||||
assert!(e.aborts_run(), "{e} must abort the run");
|
||||
assert_ne!(e.exit_code(), 5, "{e} must not report a partial failure");
|
||||
}
|
||||
assert_eq!(Error::Connection("x".into()).exit_code(), 2);
|
||||
assert_eq!(Error::Partial("x".into()).exit_code(), 5);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
const IGNORED_EXACT: &[&str] = &[
|
||||
"@type",
|
||||
"method",
|
||||
"organizerCalendarAddress",
|
||||
"privacy",
|
||||
"prodId",
|
||||
"recurrenceId",
|
||||
"recurrenceIdTimeZone",
|
||||
"sentBy",
|
||||
"uid",
|
||||
];
|
||||
|
||||
const IGNORED_FIRST_TOKEN: &[&str] = &["recurrenceOverrides", "recurrenceRule", "relatedTo"];
|
||||
|
||||
const CALENDAR_ADDRESS_DEPENDENTS: &[&str] = &[
|
||||
"calendarAddress",
|
||||
"delegatedFrom",
|
||||
"delegatedTo",
|
||||
"email",
|
||||
"expectReply",
|
||||
"kind",
|
||||
"memberOf",
|
||||
"participationStatus",
|
||||
"progress",
|
||||
"roles",
|
||||
"sentBy",
|
||||
];
|
||||
|
||||
pub fn synthetic_attendee_address(identifier: &str) -> String {
|
||||
format!(
|
||||
"urn:x-vandelay:attendee:{}",
|
||||
blake3::hash(identifier.as_bytes()).to_hex()
|
||||
)
|
||||
}
|
||||
|
||||
pub fn drop_calendar_address_dependents(participants: &mut Map<String, Value>) {
|
||||
for participant in participants.values_mut() {
|
||||
let Some(object) = participant.as_object_mut() else {
|
||||
continue;
|
||||
};
|
||||
for key in CALENDAR_ADDRESS_DEPENDENTS {
|
||||
object.remove(*key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_override_ignored(pointer: &str) -> bool {
|
||||
if IGNORED_EXACT.contains(&pointer) || is_participant_calendar_address(pointer) {
|
||||
return true;
|
||||
}
|
||||
let first = pointer.split('/').next().unwrap_or(pointer);
|
||||
IGNORED_FIRST_TOKEN.contains(&first)
|
||||
}
|
||||
|
||||
fn is_participant_calendar_address(pointer: &str) -> bool {
|
||||
let mut tokens = pointer.split('/');
|
||||
tokens.next() == Some("participants")
|
||||
&& tokens.next().is_some()
|
||||
&& tokens.next() == Some("calendarAddress")
|
||||
&& tokens.next().is_none()
|
||||
}
|
||||
|
||||
pub fn override_patch_from_event(event: &Value) -> Value {
|
||||
let Some(object) = event.as_object() else {
|
||||
return event.clone();
|
||||
};
|
||||
let patch: Map<String, Value> = object
|
||||
.iter()
|
||||
.filter(|(key, _)| !is_override_ignored(key))
|
||||
.map(|(key, value)| (key.clone(), value.clone()))
|
||||
.collect();
|
||||
Value::Object(patch)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn exact_pointers_are_ignored() {
|
||||
for pointer in [
|
||||
"@type",
|
||||
"method",
|
||||
"organizerCalendarAddress",
|
||||
"privacy",
|
||||
"prodId",
|
||||
"recurrenceId",
|
||||
"recurrenceIdTimeZone",
|
||||
"sentBy",
|
||||
"uid",
|
||||
] {
|
||||
assert!(is_override_ignored(pointer), "{pointer} must be ignored");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_token_pointers_are_ignored_at_any_depth() {
|
||||
assert!(is_override_ignored("recurrenceRule"));
|
||||
assert!(is_override_ignored("recurrenceRule/frequency"));
|
||||
assert!(is_override_ignored(
|
||||
"recurrenceOverrides/2026-01-01T09:00:00"
|
||||
));
|
||||
assert!(is_override_ignored("relatedTo/uid-1/relation"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn participant_calendar_address_is_ignored_only_at_its_own_depth() {
|
||||
assert!(is_override_ignored("participants/att-1/calendarAddress"));
|
||||
assert!(!is_override_ignored("participants"));
|
||||
assert!(!is_override_ignored("participants/att-1"));
|
||||
assert!(!is_override_ignored(
|
||||
"participants/att-1/participationStatus"
|
||||
));
|
||||
assert!(!is_override_ignored(
|
||||
"participants/att-1/calendarAddress/extra"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn patchable_pointers_survive() {
|
||||
for pointer in [
|
||||
"start",
|
||||
"duration",
|
||||
"title",
|
||||
"excluded",
|
||||
"participants",
|
||||
"created",
|
||||
"updated",
|
||||
"status",
|
||||
] {
|
||||
assert!(!is_override_ignored(pointer), "{pointer} must survive");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_event_is_reduced_to_a_patch() {
|
||||
let event = json!({
|
||||
"@type": "Event",
|
||||
"uid": "uid-1",
|
||||
"prodId": "vandelay",
|
||||
"privacy": "public",
|
||||
"organizerCalendarAddress": "mailto:[email protected]",
|
||||
"recurrenceRule": {"frequency": "daily"},
|
||||
"start": "2026-03-04T09:00:00",
|
||||
"duration": "PT1H",
|
||||
"title": "Moved"
|
||||
});
|
||||
let patch = override_patch_from_event(&event);
|
||||
let patch = patch.as_object().expect("patch object");
|
||||
assert_eq!(patch.len(), 3);
|
||||
assert_eq!(patch["start"], "2026-03-04T09:00:00");
|
||||
assert_eq!(patch["duration"], "PT1H");
|
||||
assert_eq!(patch["title"], "Moved");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_exclusion_patch_is_preserved() {
|
||||
let patch = override_patch_from_event(&json!({"excluded": true}));
|
||||
assert_eq!(patch, json!({"excluded": true}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_synthetic_address_is_stable_per_identifier_and_never_a_mailto() {
|
||||
let addr = synthetic_attendee_address("Jane Doe");
|
||||
assert_eq!(addr, synthetic_attendee_address("Jane Doe"));
|
||||
assert_ne!(addr, synthetic_attendee_address("John Doe"));
|
||||
assert!(addr.starts_with("urn:x-vandelay:attendee:"));
|
||||
assert!(
|
||||
!addr.starts_with("mailto:"),
|
||||
"export must never invite a fabricated address"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn calendar_address_dependents_are_dropped_and_identity_is_kept() {
|
||||
let mut participants = json!({
|
||||
"1": {
|
||||
"@type": "Participant",
|
||||
"name": "Room 4",
|
||||
"calendarAddress": "urn:x-vandelay:attendee:abc",
|
||||
"email": "[email protected]",
|
||||
"roles": {"required": true},
|
||||
"kind": "resource",
|
||||
"participationStatus": "accepted",
|
||||
"expectReply": true,
|
||||
"description": "Third floor"
|
||||
}
|
||||
})
|
||||
.as_object()
|
||||
.expect("participants object")
|
||||
.clone();
|
||||
drop_calendar_address_dependents(&mut participants);
|
||||
let p = &participants["1"];
|
||||
assert_eq!(p["@type"], "Participant");
|
||||
assert_eq!(p["name"], "Room 4");
|
||||
assert_eq!(p["description"], "Third floor");
|
||||
for key in CALENDAR_ADDRESS_DEPENDENTS {
|
||||
assert!(p.get(*key).is_none(), "{key} must be dropped");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,4 +4,5 @@
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
pub mod jscalendar;
|
||||
pub mod tz;
|
||||
|
||||
@@ -6,6 +6,9 @@
|
||||
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use crate::exchange::jscalendar::{
|
||||
drop_calendar_address_dependents, is_override_ignored, synthetic_attendee_address,
|
||||
};
|
||||
use crate::exchange_ews::parse::{CalendarItemRaw, RawAttendee, RawOccurrence};
|
||||
use crate::exchange_ews::recurrence::to_jscalendar_rule;
|
||||
use crate::exchange_ews::tz::resolve_to_iana;
|
||||
@@ -181,6 +184,9 @@ fn build_event_map(raw: &CalendarItemRaw, iana: Option<&str>) -> Map<String, Val
|
||||
AttendeeRole::Resource,
|
||||
);
|
||||
if !participants.is_empty() {
|
||||
if !event.contains_key("organizerCalendarAddress") {
|
||||
drop_calendar_address_dependents(&mut participants);
|
||||
}
|
||||
event.insert("participants".to_owned(), Value::Object(participants));
|
||||
}
|
||||
if let Some(rec) = raw.recurrence.as_ref()
|
||||
@@ -229,13 +235,6 @@ fn build_event_map(raw: &CalendarItemRaw, iana: Option<&str>) -> Map<String, Val
|
||||
event
|
||||
}
|
||||
|
||||
fn synthetic_attendee_address(identifier: &str) -> String {
|
||||
format!(
|
||||
"urn:x-vandelay:attendee:{}",
|
||||
blake3::hash(identifier.as_bytes()).to_hex()
|
||||
)
|
||||
}
|
||||
|
||||
fn is_smtp_routing(routing_type: Option<&str>) -> bool {
|
||||
routing_type.is_none_or(|rt| rt.eq_ignore_ascii_case("SMTP"))
|
||||
}
|
||||
@@ -320,21 +319,11 @@ fn add_attendees(
|
||||
}
|
||||
}
|
||||
|
||||
const OVERRIDE_IGNORED_POINTERS: &[&str] = &[
|
||||
"@type",
|
||||
"uid",
|
||||
"recurrenceRule",
|
||||
"recurrenceOverrides",
|
||||
"recurrenceId",
|
||||
"recurrenceIdTimeZone",
|
||||
"method",
|
||||
"organizerCalendarAddress",
|
||||
"privacy",
|
||||
"prodId",
|
||||
"relatedTo",
|
||||
"created",
|
||||
"updated",
|
||||
];
|
||||
const NOT_PATCHED_PER_OCCURRENCE: &[&str] = &["created", "updated"];
|
||||
|
||||
fn is_override_excluded(key: &str) -> bool {
|
||||
is_override_ignored(key) || NOT_PATCHED_PER_OCCURRENCE.contains(&key)
|
||||
}
|
||||
|
||||
fn build_recurrence_overrides(
|
||||
base_event: &Map<String, Value>,
|
||||
@@ -388,7 +377,7 @@ fn override_patch(
|
||||
let mut patch = Map::new();
|
||||
let inherited_start = Value::String(recurrence_id.to_owned());
|
||||
for (k, v) in occ_event {
|
||||
if OVERRIDE_IGNORED_POINTERS.contains(&k.as_str()) {
|
||||
if is_override_excluded(k) {
|
||||
continue;
|
||||
}
|
||||
let baseline = if k == "start" {
|
||||
@@ -401,7 +390,7 @@ fn override_patch(
|
||||
}
|
||||
}
|
||||
for k in base_event.keys() {
|
||||
if k == "start" || OVERRIDE_IGNORED_POINTERS.contains(&k.as_str()) {
|
||||
if k == "start" || is_override_excluded(k) {
|
||||
continue;
|
||||
}
|
||||
if !occ_event.contains_key(k) {
|
||||
@@ -710,6 +699,7 @@ mod tests {
|
||||
uid: Some("u".to_owned()),
|
||||
start: Some("2025-06-15T14:00:00Z".to_owned()),
|
||||
end: Some("2025-06-15T15:00:00Z".to_owned()),
|
||||
organizer_smtp: Some("[email protected]".to_owned()),
|
||||
required_attendees: vec![crate::exchange_ews::parse::RawAttendee {
|
||||
email: None,
|
||||
routing_type: None,
|
||||
@@ -796,6 +786,7 @@ mod tests {
|
||||
uid: Some("u".to_owned()),
|
||||
start: Some("2025-06-15T14:00:00Z".to_owned()),
|
||||
end: Some("2025-06-15T15:00:00Z".to_owned()),
|
||||
organizer_smtp: Some("[email protected]".to_owned()),
|
||||
required_attendees: vec![RawAttendee {
|
||||
email: Some("[email protected]".to_owned()),
|
||||
routing_type: Some("SMTP".to_owned()),
|
||||
@@ -809,7 +800,7 @@ mod tests {
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.values()
|
||||
.next()
|
||||
.find(|p| p["email"] == "[email protected]")
|
||||
.unwrap();
|
||||
assert_eq!(att["calendarAddress"], "mailto:[email protected]");
|
||||
assert_eq!(att["email"], "[email protected]");
|
||||
@@ -984,6 +975,7 @@ mod tests {
|
||||
uid: Some("uid-res".to_owned()),
|
||||
start: Some("2025-06-15T14:00:00Z".to_owned()),
|
||||
end: Some("2025-06-15T15:00:00Z".to_owned()),
|
||||
organizer_smtp: Some("[email protected]".to_owned()),
|
||||
resources: vec![crate::exchange_ews::parse::RawAttendee {
|
||||
email: Some("room-7@x".to_owned()),
|
||||
routing_type: None,
|
||||
@@ -1011,6 +1003,7 @@ mod tests {
|
||||
uid: Some("uid-rsvp".to_owned()),
|
||||
start: Some("2025-06-15T14:00:00Z".to_owned()),
|
||||
end: Some("2025-06-15T15:00:00Z".to_owned()),
|
||||
organizer_smtp: Some("[email protected]".to_owned()),
|
||||
required_attendees: vec![crate::exchange_ews::parse::RawAttendee {
|
||||
email: Some("bob@x".to_owned()),
|
||||
routing_type: None,
|
||||
@@ -1031,6 +1024,36 @@ mod tests {
|
||||
assert_eq!(bob["roles"]["required"], true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attendees_lose_calendar_addresses_when_no_organizer_is_known() {
|
||||
let raw = CalendarItemRaw {
|
||||
uid: Some("uid-no-organizer".to_owned()),
|
||||
start: Some("2025-06-15T14:00:00Z".to_owned()),
|
||||
end: Some("2025-06-15T15:00:00Z".to_owned()),
|
||||
required_attendees: vec![crate::exchange_ews::parse::RawAttendee {
|
||||
email: Some("bob@x".to_owned()),
|
||||
routing_type: None,
|
||||
name: Some("Bob".to_owned()),
|
||||
response_type: Some("Accept".to_owned()),
|
||||
}],
|
||||
..CalendarItemRaw::default()
|
||||
};
|
||||
let v = to_jscalendar(&raw).data;
|
||||
assert!(v.get("organizerCalendarAddress").is_none());
|
||||
let bob = v["participants"]
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.values()
|
||||
.find(|p| p["name"] == "Bob")
|
||||
.unwrap();
|
||||
for key in ["calendarAddress", "email", "roles", "participationStatus"] {
|
||||
assert!(
|
||||
bob.get(key).is_none(),
|
||||
"{key} requires calendarAddress, which requires organizerCalendarAddress"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_day_span_helper_is_inclusive_of_start() {
|
||||
assert_eq!(all_day_span_days("2025-07-04", "2025-07-04"), 1);
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use crate::exchange::jscalendar::{drop_calendar_address_dependents, synthetic_attendee_address};
|
||||
use crate::exchange::tz::resolve_to_iana;
|
||||
use crate::exchange_graph::error::GraphError;
|
||||
use crate::exchange_graph::recurrence::convert_patterned_recurrence_rule;
|
||||
@@ -242,13 +243,8 @@ pub fn convert_event(
|
||||
.get("organizer")
|
||||
.and_then(|o| o.get("emailAddress"))
|
||||
{
|
||||
if let Some(addr) = org.get("address").and_then(Value::as_str)
|
||||
&& !addr.is_empty()
|
||||
{
|
||||
card.insert(
|
||||
"organizerCalendarAddress".to_owned(),
|
||||
Value::from(format!("mailto:{addr}")),
|
||||
);
|
||||
if let Some((cal_addr, _)) = resolve_calendar_address(org) {
|
||||
card.insert("organizerCalendarAddress".to_owned(), Value::from(cal_addr));
|
||||
}
|
||||
let p = build_participant(org, &["owner", "chair"], None, None);
|
||||
participants.push(("organizer".to_owned(), p));
|
||||
@@ -275,7 +271,10 @@ pub fn convert_event(
|
||||
}
|
||||
}
|
||||
if !participants.is_empty() {
|
||||
let map: Map<String, Value> = participants.into_iter().collect();
|
||||
let mut map: Map<String, Value> = participants.into_iter().collect();
|
||||
if !card.contains_key("organizerCalendarAddress") {
|
||||
drop_calendar_address_dependents(&mut map);
|
||||
}
|
||||
card.insert("participants".to_owned(), Value::Object(map));
|
||||
}
|
||||
|
||||
@@ -346,6 +345,23 @@ fn extract_local_datetime(slot: Option<&Value>) -> Option<String> {
|
||||
Some(dt.to_owned())
|
||||
}
|
||||
|
||||
fn non_empty(email: &Value, key: &str) -> Option<String> {
|
||||
email
|
||||
.get(key)
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_owned)
|
||||
}
|
||||
|
||||
fn resolve_calendar_address(email: &Value) -> Option<(String, Option<String>)> {
|
||||
if let Some(addr) = non_empty(email, "address") {
|
||||
let cal_addr = format!("mailto:{addr}");
|
||||
return Some((cal_addr, Some(addr)));
|
||||
}
|
||||
non_empty(email, "name").map(|name| (synthetic_attendee_address(&name), None))
|
||||
}
|
||||
|
||||
fn build_participant(
|
||||
email: &Value,
|
||||
roles: &[&str],
|
||||
@@ -354,17 +370,13 @@ fn build_participant(
|
||||
) -> Value {
|
||||
let mut map = Map::new();
|
||||
map.insert("@type".to_owned(), Value::from("Participant"));
|
||||
if let Some(addr) = email.get("address").and_then(Value::as_str) {
|
||||
map.insert(
|
||||
"calendarAddress".to_owned(),
|
||||
Value::from(format!("mailto:{addr}")),
|
||||
);
|
||||
map.insert("email".to_owned(), Value::from(addr.to_owned()));
|
||||
if let Some(name) = non_empty(email, "name") {
|
||||
map.insert("name".to_owned(), Value::from(name));
|
||||
}
|
||||
if let Some(name) = email.get("name").and_then(Value::as_str)
|
||||
&& !name.is_empty()
|
||||
{
|
||||
map.insert("name".to_owned(), Value::from(name.to_owned()));
|
||||
if let Some((cal_addr, smtp)) = resolve_calendar_address(email) {
|
||||
map.insert("calendarAddress".to_owned(), Value::from(cal_addr));
|
||||
if let Some(addr) = smtp {
|
||||
map.insert("email".to_owned(), Value::from(addr));
|
||||
}
|
||||
let role_map: Map<String, Value> = roles
|
||||
.iter()
|
||||
@@ -380,6 +392,7 @@ fn build_participant(
|
||||
if let Some(b) = expect_reply {
|
||||
map.insert("expectReply".to_owned(), Value::Bool(b));
|
||||
}
|
||||
}
|
||||
Value::Object(map)
|
||||
}
|
||||
|
||||
@@ -650,6 +663,91 @@ mod tests {
|
||||
assert_eq!(participants["att-1"]["roles"]["required"], true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_name_only_attendee_gets_a_stable_synthetic_calendar_address() {
|
||||
let mut v = sample();
|
||||
v["attendees"] = json!([
|
||||
{
|
||||
"emailAddress": {"name": "Room 4"},
|
||||
"type": "resource",
|
||||
"status": {"response": "accepted"}
|
||||
}
|
||||
]);
|
||||
let conv = convert_event(&v, None).unwrap();
|
||||
let p = &conv.data["participants"]["att-1"];
|
||||
let addr = p["calendarAddress"].as_str().unwrap();
|
||||
assert!(
|
||||
addr.starts_with("urn:x-vandelay:attendee:"),
|
||||
"a name-only attendee needs a synthetic calendarAddress, got {addr}"
|
||||
);
|
||||
assert!(
|
||||
p.get("email").is_none(),
|
||||
"there is no SMTP address to report as email"
|
||||
);
|
||||
assert_eq!(p["participationStatus"], "accepted");
|
||||
assert_eq!(p["roles"]["informational"], true);
|
||||
assert_eq!(
|
||||
addr,
|
||||
convert_event(&v, None).unwrap().data["participants"]["att-1"]["calendarAddress"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unidentifiable_attendee_keeps_no_calendar_address_dependent_property() {
|
||||
let mut v = sample();
|
||||
v["attendees"] = json!([
|
||||
{"emailAddress": {}, "type": "required", "status": {"response": "accepted"}}
|
||||
]);
|
||||
let conv = convert_event(&v, None).unwrap();
|
||||
let p = &conv.data["participants"]["att-1"];
|
||||
for key in [
|
||||
"calendarAddress",
|
||||
"email",
|
||||
"roles",
|
||||
"participationStatus",
|
||||
"expectReply",
|
||||
] {
|
||||
assert!(
|
||||
p.get(key).is_none(),
|
||||
"{key} requires calendarAddress (jscalendarbis 3.4.6)"
|
||||
);
|
||||
}
|
||||
assert_eq!(p["@type"], "Participant");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_organizer_without_an_address_still_yields_a_calendar_address() {
|
||||
let mut v = sample();
|
||||
v["organizer"] = json!({"emailAddress": {"name": "Alice Example"}});
|
||||
let conv = convert_event(&v, None).unwrap();
|
||||
let organizer_address = conv.data["organizerCalendarAddress"].as_str().unwrap();
|
||||
assert!(organizer_address.starts_with("urn:x-vandelay:attendee:"));
|
||||
assert_eq!(
|
||||
conv.data["participants"]["organizer"]["calendarAddress"],
|
||||
organizer_address
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn participants_drop_calendar_addresses_when_no_organizer_is_known() {
|
||||
let mut v = sample();
|
||||
v.as_object_mut().unwrap().remove("organizer");
|
||||
let conv = convert_event(&v, None).unwrap();
|
||||
assert!(
|
||||
conv.data.get("organizerCalendarAddress").is_none(),
|
||||
"no organizer is derivable from the source event"
|
||||
);
|
||||
let participants = conv.data["participants"].as_object().unwrap();
|
||||
assert!(!participants.is_empty());
|
||||
for (key, participant) in participants {
|
||||
assert!(
|
||||
participant.get("calendarAddress").is_none(),
|
||||
"{key} may not carry a calendarAddress without organizerCalendarAddress"
|
||||
);
|
||||
assert!(participant.get("roles").is_none(), "{key} kept roles");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn optional_attendee_keeps_role_and_uses_status_when_present() {
|
||||
let mut v = sample();
|
||||
|
||||
@@ -59,6 +59,59 @@ impl EventBodyFormat {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Surfaces {
|
||||
pub mail: bool,
|
||||
pub calendar: bool,
|
||||
pub contacts: bool,
|
||||
}
|
||||
|
||||
impl Default for Surfaces {
|
||||
fn default() -> Self {
|
||||
Surfaces::ALL
|
||||
}
|
||||
}
|
||||
|
||||
impl Surfaces {
|
||||
pub const ALL: Surfaces = Surfaces {
|
||||
mail: true,
|
||||
calendar: true,
|
||||
contacts: true,
|
||||
};
|
||||
|
||||
pub const NONE: Surfaces = Surfaces {
|
||||
mail: false,
|
||||
calendar: false,
|
||||
contacts: false,
|
||||
};
|
||||
|
||||
pub fn parse_list(list: &str) -> Result<Surfaces, Error> {
|
||||
let mut selected = Surfaces::NONE;
|
||||
for token in list.split(',') {
|
||||
let token = token.trim();
|
||||
if token.is_empty() {
|
||||
continue;
|
||||
}
|
||||
match token.to_ascii_lowercase().as_str() {
|
||||
"mail" => selected.mail = true,
|
||||
"calendar" => selected.calendar = true,
|
||||
"contacts" => selected.contacts = true,
|
||||
_ => {
|
||||
return Err(Error::Usage(format!(
|
||||
"unknown surface: {token} (valid: mail, calendar, contacts)"
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
if selected == Surfaces::NONE {
|
||||
return Err(Error::Usage(
|
||||
"--objects given but resolved to an empty surface list".to_owned(),
|
||||
));
|
||||
}
|
||||
Ok(selected)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ResolvedPrincipal {
|
||||
pub id: String,
|
||||
@@ -102,6 +155,86 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn surfaces_default_is_all_three() {
|
||||
let all = Surfaces::default();
|
||||
assert!(all.mail && all.calendar && all.contacts);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn surfaces_parse_each_name() {
|
||||
assert_eq!(
|
||||
Surfaces::parse_list("mail").unwrap(),
|
||||
Surfaces {
|
||||
mail: true,
|
||||
calendar: false,
|
||||
contacts: false
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
Surfaces::parse_list("calendar").unwrap(),
|
||||
Surfaces {
|
||||
mail: false,
|
||||
calendar: true,
|
||||
contacts: false
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
Surfaces::parse_list("contacts").unwrap(),
|
||||
Surfaces {
|
||||
mail: false,
|
||||
calendar: false,
|
||||
contacts: true
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn surfaces_parse_is_case_insensitive() {
|
||||
assert_eq!(
|
||||
Surfaces::parse_list("Mail,CALENDAR,Contacts").unwrap(),
|
||||
Surfaces::ALL
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn surfaces_parse_dedups_and_trims() {
|
||||
assert_eq!(
|
||||
Surfaces::parse_list(" mail , contacts ,mail,, ").unwrap(),
|
||||
Surfaces {
|
||||
mail: true,
|
||||
calendar: false,
|
||||
contacts: true
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn surfaces_parse_rejects_unknown_name() {
|
||||
let err = Surfaces::parse_list("mail,contact").unwrap_err();
|
||||
let msg = err.to_string();
|
||||
assert!(msg.contains("unknown surface: contact"), "msg was: {msg}");
|
||||
assert!(
|
||||
msg.contains("valid: mail, calendar, contacts"),
|
||||
"msg was: {msg}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn surfaces_parse_rejects_jmap_type_names() {
|
||||
assert!(Surfaces::parse_list("mailbox").is_err());
|
||||
assert!(Surfaces::parse_list("contactcard").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn surfaces_parse_rejects_empty_list() {
|
||||
let err = Surfaces::parse_list(" , ").unwrap_err();
|
||||
assert!(
|
||||
err.to_string().contains("empty surface list"),
|
||||
"msg was: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn body_format_prefer_value() {
|
||||
assert_eq!(
|
||||
|
||||
+437
-10
@@ -4,10 +4,32 @@
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::jmap::error::JmapError;
|
||||
|
||||
pub const SENTINEL_KEY: &str = "@blob";
|
||||
|
||||
const DEFAULT_MEDIA_TYPE: &str = "application/octet-stream";
|
||||
const MEDIA_TYPE_KEYS: [&str; 2] = ["mediaType", "contentType"];
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum InlineShape {
|
||||
JsContactResource,
|
||||
JsCalendarLink,
|
||||
}
|
||||
|
||||
impl InlineShape {
|
||||
fn uri_key(self) -> &'static str {
|
||||
match self {
|
||||
InlineShape::JsContactResource => "uri",
|
||||
InlineShape::JsCalendarLink => "href",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn import_blob_ids<F>(value: &mut Value, mut resolve: F) -> Result<(), BlobWalkError>
|
||||
where
|
||||
F: FnMut(&str) -> Result<i64, BlobWalkError>,
|
||||
@@ -39,31 +61,36 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn export_blob_ids<F>(value: &mut Value, mut resolve: F) -> Result<(), BlobWalkError>
|
||||
pub fn inline_blob_data_uris<F>(
|
||||
value: &mut Value,
|
||||
shape: InlineShape,
|
||||
mut fetch: F,
|
||||
) -> Result<(), BlobWalkError>
|
||||
where
|
||||
F: FnMut(i64) -> Result<String, BlobWalkError>,
|
||||
F: FnMut(i64) -> Result<Vec<u8>, BlobWalkError>,
|
||||
{
|
||||
walk_export(value, &mut resolve)
|
||||
walk_inline(value, shape, &mut fetch)
|
||||
}
|
||||
|
||||
fn walk_export<F>(value: &mut Value, resolve: &mut F) -> Result<(), BlobWalkError>
|
||||
fn walk_inline<F>(value: &mut Value, shape: InlineShape, fetch: &mut F) -> Result<(), BlobWalkError>
|
||||
where
|
||||
F: FnMut(i64) -> Result<String, BlobWalkError>,
|
||||
F: FnMut(i64) -> Result<Vec<u8>, BlobWalkError>,
|
||||
{
|
||||
match value {
|
||||
Value::Object(map) => {
|
||||
if let Some(sentinel) = map.remove(SENTINEL_KEY) {
|
||||
let local_id = sentinel.as_i64().ok_or(BlobWalkError::MalformedSentinel)?;
|
||||
let target_jmap_id = resolve(local_id)?;
|
||||
map.insert("blobId".to_owned(), Value::String(target_jmap_id));
|
||||
let bytes = fetch(local_id)?;
|
||||
let uri = data_uri(map, &bytes);
|
||||
map.insert(shape.uri_key().to_owned(), Value::String(uri));
|
||||
}
|
||||
for child in map.values_mut() {
|
||||
walk_export(child, resolve)?;
|
||||
walk_inline(child, shape, fetch)?;
|
||||
}
|
||||
}
|
||||
Value::Array(items) => {
|
||||
for item in items {
|
||||
walk_export(item, resolve)?;
|
||||
walk_inline(item, shape, fetch)?;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
@@ -71,6 +98,70 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn data_uri(map: &Map<String, Value>, bytes: &[u8]) -> String {
|
||||
let media_type = MEDIA_TYPE_KEYS
|
||||
.iter()
|
||||
.find_map(|k| map.get(*k).and_then(Value::as_str).and_then(uri_media_type))
|
||||
.unwrap_or_else(|| DEFAULT_MEDIA_TYPE.to_owned());
|
||||
let mut uri = String::with_capacity(13 + media_type.len() + bytes.len().div_ceil(3) * 4);
|
||||
uri.push_str("data:");
|
||||
uri.push_str(&media_type);
|
||||
uri.push_str(";base64,");
|
||||
STANDARD.encode_string(bytes, &mut uri);
|
||||
uri
|
||||
}
|
||||
|
||||
fn uri_media_type(raw: &str) -> Option<String> {
|
||||
let mut parts = raw.split(';').map(str::trim);
|
||||
let essence = parts.next()?;
|
||||
let (ty, subtype) = essence.split_once('/')?;
|
||||
if ty.is_empty() || subtype.is_empty() || subtype.contains('/') {
|
||||
return None;
|
||||
}
|
||||
let mut out = String::with_capacity(raw.len());
|
||||
escape_into(&mut out, essence);
|
||||
for parameter in parts.filter(|p| !p.is_empty()) {
|
||||
out.push(';');
|
||||
escape_into(&mut out, parameter);
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
fn escape_into(out: &mut String, part: &str) {
|
||||
const HEX: &[u8; 16] = b"0123456789ABCDEF";
|
||||
for byte in part.bytes() {
|
||||
if is_media_type_uri_byte(byte) {
|
||||
out.push(byte as char);
|
||||
} else {
|
||||
out.push('%');
|
||||
out.push(HEX[(byte >> 4) as usize] as char);
|
||||
out.push(HEX[(byte & 0x0F) as usize] as char);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_media_type_uri_byte(byte: u8) -> bool {
|
||||
byte.is_ascii_alphanumeric()
|
||||
|| matches!(
|
||||
byte,
|
||||
b'!' | b'$'
|
||||
| b'&'
|
||||
| b'\''
|
||||
| b'('
|
||||
| b')'
|
||||
| b'*'
|
||||
| b'+'
|
||||
| b'-'
|
||||
| b'.'
|
||||
| b'/'
|
||||
| b':'
|
||||
| b'='
|
||||
| b'@'
|
||||
| b'_'
|
||||
| b'~'
|
||||
)
|
||||
}
|
||||
|
||||
pub fn prepend_property(target: &mut Map<String, Value>, key: &str, value: Value) {
|
||||
let existing: Vec<(String, Value)> =
|
||||
target.iter().map(|(k, v)| (k.clone(), v.clone())).collect();
|
||||
@@ -88,5 +179,341 @@ pub enum BlobWalkError {
|
||||
#[error("malformed @blob sentinel: expected integer local id")]
|
||||
MalformedSentinel,
|
||||
#[error("blob resolver failed: {0}")]
|
||||
Resolver(String),
|
||||
Resolver(Box<JmapError>),
|
||||
}
|
||||
|
||||
impl BlobWalkError {
|
||||
pub fn resolver(source: JmapError) -> BlobWalkError {
|
||||
BlobWalkError::Resolver(Box::new(source))
|
||||
}
|
||||
|
||||
pub fn into_source(self) -> JmapError {
|
||||
match self {
|
||||
BlobWalkError::Resolver(source) => *source,
|
||||
other => JmapError::Blob(other),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn fetch_ok(local_id: i64) -> Result<Vec<u8>, BlobWalkError> {
|
||||
Ok(format!("bytes-{local_id}").into_bytes())
|
||||
}
|
||||
|
||||
fn decode(uri: &str, expect_media_type: &str) -> Vec<u8> {
|
||||
assert_well_formed(uri);
|
||||
let prefix = format!("data:{expect_media_type};base64,");
|
||||
let payload = uri
|
||||
.strip_prefix(&prefix)
|
||||
.unwrap_or_else(|| panic!("{uri} does not start with {prefix}"));
|
||||
STANDARD.decode(payload).expect("base64 payload")
|
||||
}
|
||||
|
||||
fn assert_well_formed(uri: &str) {
|
||||
let rest = uri
|
||||
.strip_prefix("data:")
|
||||
.unwrap_or_else(|| panic!("{uri} is not a data URI"));
|
||||
let (media_type, payload) = rest
|
||||
.split_once(";base64,")
|
||||
.unwrap_or_else(|| panic!("{uri} carries no ;base64, separator"));
|
||||
let bytes = media_type.as_bytes();
|
||||
let mut i = 0;
|
||||
while i < bytes.len() {
|
||||
let b = bytes[i];
|
||||
if b == b'%' {
|
||||
assert!(
|
||||
i + 2 < bytes.len()
|
||||
&& bytes[i + 1].is_ascii_hexdigit()
|
||||
&& bytes[i + 2].is_ascii_hexdigit(),
|
||||
"truncated percent escape in {uri}"
|
||||
);
|
||||
i += 3;
|
||||
continue;
|
||||
}
|
||||
assert!(
|
||||
is_media_type_uri_byte(b) || b == b';',
|
||||
"byte {b:#04x} is not allowed unescaped in {uri}"
|
||||
);
|
||||
i += 1;
|
||||
}
|
||||
assert!(
|
||||
payload
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'/' | b'=')),
|
||||
"non-base64 payload in {uri}"
|
||||
);
|
||||
}
|
||||
|
||||
fn assert_no_blob_id(value: &Value) {
|
||||
match value {
|
||||
Value::Object(map) => {
|
||||
assert!(map.get("blobId").is_none(), "blobId survived in {value}");
|
||||
assert!(map.get(SENTINEL_KEY).is_none(), "@blob survived in {value}");
|
||||
for child in map.values() {
|
||||
assert_no_blob_id(child);
|
||||
}
|
||||
}
|
||||
Value::Array(items) => items.iter().for_each(assert_no_blob_id),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn media_sentinel_becomes_a_uri_property() {
|
||||
let mut card = json!({
|
||||
"@type": "Card",
|
||||
"media": { "photo": {
|
||||
"@type": "Media", "kind": "photo",
|
||||
"@blob": 7, "mediaType": "image/png"
|
||||
} }
|
||||
});
|
||||
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap();
|
||||
let photo = &card["media"]["photo"];
|
||||
assert_eq!(
|
||||
decode(photo["uri"].as_str().unwrap(), "image/png"),
|
||||
b"bytes-7"
|
||||
);
|
||||
assert_eq!(photo["mediaType"], json!("image/png"));
|
||||
assert!(photo.get("href").is_none());
|
||||
assert_no_blob_id(&card);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn calendar_link_sentinel_becomes_an_href_property() {
|
||||
let mut event = json!({
|
||||
"@type": "Event",
|
||||
"links": { "1": {
|
||||
"@type": "Link", "rel": "enclosure",
|
||||
"@blob": 3, "contentType": "text/plain", "title": "note.txt"
|
||||
} }
|
||||
});
|
||||
inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap();
|
||||
let link = &event["links"]["1"];
|
||||
assert_eq!(
|
||||
decode(link["href"].as_str().unwrap(), "text/plain"),
|
||||
b"bytes-3"
|
||||
);
|
||||
assert_eq!(link["contentType"], json!("text/plain"));
|
||||
assert_eq!(link["title"], json!("note.txt"));
|
||||
assert!(link.get("uri").is_none());
|
||||
assert_no_blob_id(&event);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jscontact_link_keeps_the_resource_uri_property() {
|
||||
let mut card = json!({
|
||||
"@type": "Card",
|
||||
"links": { "l1": { "@type": "Link", "@blob": 1, "mediaType": "application/pdf" } }
|
||||
});
|
||||
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap();
|
||||
let link = &card["links"]["l1"];
|
||||
assert!(link.get("href").is_none(), "JSContact Link uses uri");
|
||||
assert_eq!(
|
||||
decode(link["uri"].as_str().unwrap(), "application/pdf"),
|
||||
b"bytes-1"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_media_type_defaults_to_octet_stream() {
|
||||
let mut card = json!({ "media": { "photo": { "@blob": 2 } } });
|
||||
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap();
|
||||
let uri = card["media"]["photo"]["uri"].as_str().unwrap();
|
||||
assert_eq!(decode(uri, DEFAULT_MEDIA_TYPE), b"bytes-2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_media_type_defaults_to_octet_stream() {
|
||||
let mut event = json!({ "links": { "1": { "@blob": 4, "contentType": "" } } });
|
||||
inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap();
|
||||
let uri = event["links"]["1"]["href"].as_str().unwrap();
|
||||
assert_eq!(decode(uri, DEFAULT_MEDIA_TYPE), b"bytes-4");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_media_type_falls_through_to_the_next_key() {
|
||||
let mut card = json!({ "media": { "photo": {
|
||||
"@blob": 4, "mediaType": "", "contentType": "image/jpeg"
|
||||
} } });
|
||||
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap();
|
||||
let uri = card["media"]["photo"]["uri"].as_str().unwrap();
|
||||
assert_eq!(decode(uri, "image/jpeg"), b"bytes-4");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn media_type_parameters_are_kept_without_whitespace() {
|
||||
let mut event = json!({ "links": { "1": {
|
||||
"@blob": 4, "contentType": "text/plain; charset=utf-8"
|
||||
} } });
|
||||
inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap();
|
||||
let uri = event["links"]["1"]["href"].as_str().unwrap();
|
||||
assert!(!uri.contains(' '), "{uri} carries a raw space");
|
||||
assert_eq!(decode(uri, "text/plain;charset=utf-8"), b"bytes-4");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn media_type_keeps_several_parameters_and_drops_empty_ones() {
|
||||
let mut event = json!({ "links": { "1": {
|
||||
"@blob": 4, "contentType": " text/plain ;; charset=utf-8 ; format=flowed "
|
||||
} } });
|
||||
inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap();
|
||||
let uri = event["links"]["1"]["href"].as_str().unwrap();
|
||||
assert_eq!(
|
||||
decode(uri, "text/plain;charset=utf-8;format=flowed"),
|
||||
b"bytes-4"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn media_type_characters_illegal_in_a_uri_are_percent_encoded() {
|
||||
let mut event = json!({ "links": { "1": {
|
||||
"@blob": 4, "contentType": "text/plain; name=\"rep ort,v2#1\u{e9}.txt\""
|
||||
} } });
|
||||
inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap();
|
||||
let uri = event["links"]["1"]["href"].as_str().unwrap();
|
||||
assert_eq!(
|
||||
decode(uri, "text/plain;name=%22rep%20ort%2Cv2%231%C3%A9.txt%22"),
|
||||
b"bytes-4"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_percent_in_the_media_type_is_itself_escaped() {
|
||||
let mut card = json!({ "media": { "photo": {
|
||||
"@blob": 4, "mediaType": "image/png; note=100%"
|
||||
} } });
|
||||
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap();
|
||||
let uri = card["media"]["photo"]["uri"].as_str().unwrap();
|
||||
assert_eq!(decode(uri, "image/png;note=100%25"), b"bytes-4");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_media_type_that_is_not_type_slash_subtype_defaults_to_octet_stream() {
|
||||
for bogus in ["png", "image/", "/png", "image/png/extra", ";charset=utf-8"] {
|
||||
let mut card = json!({ "media": { "photo": { "@blob": 4, "mediaType": bogus } } });
|
||||
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap();
|
||||
let uri = card["media"]["photo"]["uri"].as_str().unwrap();
|
||||
assert_eq!(decode(uri, DEFAULT_MEDIA_TYPE), b"bytes-4", "{bogus}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inlined_bytes_round_trip_exactly() {
|
||||
let payload: Vec<u8> = (0u16..=255).map(|b| b as u8).collect();
|
||||
let mut card = json!({ "media": { "photo": {
|
||||
"@blob": 1, "mediaType": "application/octet-stream; charset=binary"
|
||||
} } });
|
||||
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, |_| {
|
||||
Ok(payload.clone())
|
||||
})
|
||||
.unwrap();
|
||||
let uri = card["media"]["photo"]["uri"].as_str().unwrap();
|
||||
assert_eq!(
|
||||
decode(uri, "application/octet-stream;charset=binary"),
|
||||
payload
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sentinels_nested_in_arrays_and_subobjects_are_all_inlined() {
|
||||
let mut value = json!({
|
||||
"locations": { "loc1": { "links": [
|
||||
{ "@blob": 5, "contentType": "image/gif" },
|
||||
{ "@blob": 6 }
|
||||
] } },
|
||||
"links": { "top": { "@blob": 8, "contentType": "image/gif" } }
|
||||
});
|
||||
inline_blob_data_uris(&mut value, InlineShape::JsCalendarLink, fetch_ok).unwrap();
|
||||
let arr = value["locations"]["loc1"]["links"].as_array().unwrap();
|
||||
assert_eq!(
|
||||
decode(arr[0]["href"].as_str().unwrap(), "image/gif"),
|
||||
b"bytes-5"
|
||||
);
|
||||
assert_eq!(
|
||||
decode(arr[1]["href"].as_str().unwrap(), DEFAULT_MEDIA_TYPE),
|
||||
b"bytes-6"
|
||||
);
|
||||
assert_eq!(
|
||||
decode(value["links"]["top"]["href"].as_str().unwrap(), "image/gif"),
|
||||
b"bytes-8"
|
||||
);
|
||||
assert_no_blob_id(&value);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_sentinel_is_rejected() {
|
||||
let mut value = json!({ "media": { "photo": { "@blob": "not-an-id" } } });
|
||||
let err = inline_blob_data_uris(&mut value, InlineShape::JsContactResource, fetch_ok)
|
||||
.expect_err("malformed sentinel");
|
||||
assert!(matches!(err, BlobWalkError::MalformedSentinel), "{err:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolver_failure_propagates() {
|
||||
let mut value = json!({ "media": { "photo": { "@blob": 9 } } });
|
||||
let err = inline_blob_data_uris(&mut value, InlineShape::JsContactResource, |_| {
|
||||
Err(BlobWalkError::resolver(JmapError::malformed("gone")))
|
||||
})
|
||||
.expect_err("resolver failure");
|
||||
assert!(matches!(err, BlobWalkError::Resolver(_)), "{err:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_resolver_error_kind_survives_the_walk() {
|
||||
let mut value = json!({ "media": { "photo": { "@blob": 9 } } });
|
||||
let err = inline_blob_data_uris(&mut value, InlineShape::JsContactResource, |_| {
|
||||
Err(BlobWalkError::resolver(JmapError::Sqlite(
|
||||
rusqlite::Error::QueryReturnedNoRows,
|
||||
)))
|
||||
})
|
||||
.expect_err("resolver failure")
|
||||
.into_source();
|
||||
assert!(matches!(err, JmapError::Sqlite(_)), "{err:?}");
|
||||
let mapped = crate::error::Error::from(err);
|
||||
assert!(mapped.aborts_run(), "{mapped} must abort the run");
|
||||
assert_eq!(mapped.exit_code(), 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_malformed_sentinel_stays_a_blob_walk_error() {
|
||||
let err = BlobWalkError::MalformedSentinel.into_source();
|
||||
assert!(
|
||||
matches!(err, JmapError::Blob(BlobWalkError::MalformedSentinel)),
|
||||
"{err:?}"
|
||||
);
|
||||
assert_eq!(crate::error::Error::from(err).exit_code(), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn values_without_a_sentinel_are_untouched() {
|
||||
let original = json!({
|
||||
"@type": "Card",
|
||||
"media": { "photo": { "@type": "Media", "kind": "photo",
|
||||
"uri": "https://example.test/p.png", "mediaType": "image/png" } },
|
||||
"nickNames": { "n1": { "name": "Jay" } }
|
||||
});
|
||||
let mut value = original.clone();
|
||||
inline_blob_data_uris(&mut value, InlineShape::JsContactResource, |_| {
|
||||
panic!("fetch must not be called")
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(value, original);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn import_walker_still_produces_the_sentinel() {
|
||||
let mut value =
|
||||
json!({ "media": { "photo": { "blobId": "B1", "mediaType": "image/png" } } });
|
||||
import_blob_ids(&mut value, |id| {
|
||||
assert_eq!(id, "B1");
|
||||
Ok(11)
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(value["media"]["photo"][SENTINEL_KEY], json!(11));
|
||||
assert!(value["media"]["photo"].get("blobId").is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,6 +31,9 @@ pub fn prepare(ctx: &Context, connect: &ConnectConfig) -> Result<Connected, Erro
|
||||
ctx.client.set_logger(ctx.common.logger);
|
||||
|
||||
let session = Session::discover(&ctx.client, &connect.url)?;
|
||||
for mismatch in session.origin_mismatches(&connect.url) {
|
||||
ctx.common.logger.warn(&mismatch.to_string());
|
||||
}
|
||||
let limits = session.core_limits()?;
|
||||
ctx.client.set_limits(&limits);
|
||||
let account_id = account::resolve(&connect.account, &session, &ctx.client)?;
|
||||
|
||||
+57
-1
@@ -4,6 +4,7 @@
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
use crate::db::init::OpenError;
|
||||
use crate::error::Error;
|
||||
use crate::jmap::blob::BlobWalkError;
|
||||
|
||||
@@ -70,7 +71,62 @@ impl From<JmapError> for Error {
|
||||
match value {
|
||||
JmapError::Connect(m) | JmapError::Transport(m) => Error::Connection(m),
|
||||
JmapError::Auth(m) => Error::Connection(format!("authentication rejected: {m}")),
|
||||
other => Error::Connection(other.to_string()),
|
||||
JmapError::Sqlite(e) => Error::Db(OpenError::Sqlite(e)),
|
||||
reached @ (JmapError::HttpStatus { .. } | JmapError::RetriesExhausted(_)) => {
|
||||
Error::Connection(reached.to_string())
|
||||
}
|
||||
per_unit => Error::Partial(per_unit.to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn unreachable_endpoint_aborts_the_run() {
|
||||
for e in [
|
||||
JmapError::Connect("refused".to_owned()),
|
||||
JmapError::Transport("io: broken pipe".to_owned()),
|
||||
JmapError::Auth("bad password".to_owned()),
|
||||
JmapError::HttpStatus {
|
||||
status: 404,
|
||||
body: "<html>404 Not Found</html>".to_owned(),
|
||||
},
|
||||
JmapError::RetriesExhausted("gave up".to_owned()),
|
||||
] {
|
||||
let mapped = Error::from(e);
|
||||
assert!(mapped.aborts_run(), "{mapped} must abort the run");
|
||||
assert_eq!(mapped.exit_code(), 2);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn protocol_level_errors_stay_per_unit() {
|
||||
for e in [
|
||||
JmapError::Method {
|
||||
call_id: "q".to_owned(),
|
||||
error_type: "unknownMethod".to_owned(),
|
||||
description: None,
|
||||
},
|
||||
JmapError::UnknownMethod,
|
||||
JmapError::CannotCalculateChanges,
|
||||
JmapError::AnchorNotFound,
|
||||
JmapError::RequestTooLarge,
|
||||
JmapError::SingleObjectTooLarge("one blob".to_owned()),
|
||||
JmapError::Malformed("not a list".to_owned()),
|
||||
] {
|
||||
let mapped = Error::from(e);
|
||||
assert!(!mapped.aborts_run(), "{mapped} must not abort the run");
|
||||
assert_eq!(mapped.exit_code(), 5);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_failures_are_local_io() {
|
||||
let mapped = Error::from(JmapError::Sqlite(rusqlite::Error::QueryReturnedNoRows));
|
||||
assert!(mapped.aborts_run());
|
||||
assert_eq!(mapped.exit_code(), 7);
|
||||
}
|
||||
}
|
||||
|
||||
+5
-2
@@ -617,9 +617,11 @@ fn format_rate_headers(headers: &[(String, String)]) -> Option<String> {
|
||||
Some(parts.join("; "))
|
||||
}
|
||||
|
||||
fn format_retry_wait(d: Duration) -> String {
|
||||
pub fn format_retry_wait(d: Duration) -> String {
|
||||
let total = d.as_secs();
|
||||
if total < 60 {
|
||||
if total == 0 {
|
||||
format!("{}ms", d.as_millis())
|
||||
} else if total < 60 {
|
||||
format!("{total}s")
|
||||
} else if total < 3600 {
|
||||
format!("{}m{:02}s", total / 60, total % 60)
|
||||
@@ -826,6 +828,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn format_retry_wait_short_medium_long() {
|
||||
assert_eq!(format_retry_wait(Duration::from_millis(250)), "250ms");
|
||||
assert_eq!(format_retry_wait(Duration::from_secs(7)), "7s");
|
||||
assert_eq!(format_retry_wait(Duration::from_secs(67)), "1m07s");
|
||||
assert_eq!(format_retry_wait(Duration::from_secs(1427)), "23m47s");
|
||||
|
||||
+129
-1
@@ -9,9 +9,11 @@ use serde::de::DeserializeOwned;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use crate::jmap::error::JmapError;
|
||||
use crate::jmap::http::HttpClient;
|
||||
use crate::jmap::http::{HttpClient, format_retry_wait};
|
||||
use crate::jmap::retry::{Disposition, MethodCallKind, backoff_delay, jmap_method_disposition};
|
||||
use crate::jmap::session::Limits;
|
||||
use crate::jmap::wire::JmapId;
|
||||
use crate::logging::Logger;
|
||||
|
||||
pub const URN_CORE: &str = "urn:ietf:params:jmap:core";
|
||||
|
||||
@@ -184,6 +186,43 @@ pub fn check_method_error(mr: &MethodCall) -> Result<(), JmapError> {
|
||||
})
|
||||
}
|
||||
|
||||
fn method_error_is_retryable(err: &JmapError, kind: MethodCallKind) -> bool {
|
||||
match err {
|
||||
JmapError::Method { error_type, .. } => {
|
||||
jmap_method_disposition(error_type, kind) == Disposition::Retryable
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn retry_method_call<T>(
|
||||
client: &HttpClient,
|
||||
kind: MethodCallKind,
|
||||
logger: &Logger,
|
||||
mut attempt: impl FnMut() -> Result<T, JmapError>,
|
||||
) -> Result<T, JmapError> {
|
||||
let policy = *client.retry();
|
||||
let mut tries = 0u32;
|
||||
loop {
|
||||
let err = match attempt() {
|
||||
Ok(value) => return Ok(value),
|
||||
Err(e) => e,
|
||||
};
|
||||
if tries >= policy.max_retries || !method_error_is_retryable(&err, kind) {
|
||||
return Err(err);
|
||||
}
|
||||
tries += 1;
|
||||
let delay = backoff_delay(&policy, tries);
|
||||
client.rate_limit().cooldown().arm(delay);
|
||||
logger.warn(&format!(
|
||||
"{err}; retrying in {} ({tries}/{})",
|
||||
format_retry_wait(delay),
|
||||
policy.max_retries
|
||||
));
|
||||
std::thread::sleep(delay);
|
||||
}
|
||||
}
|
||||
|
||||
fn ids_array(ids: &[JmapId]) -> Value {
|
||||
Value::Array(ids.iter().map(|i| Value::String(i.0.clone())).collect())
|
||||
}
|
||||
@@ -720,7 +759,96 @@ fn decode_set(mr: &MethodCall) -> SetOutcome {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::cell::Cell;
|
||||
|
||||
use super::*;
|
||||
use crate::error::Error;
|
||||
use crate::jmap::http::{Auth, RetryPolicy};
|
||||
|
||||
fn test_client(max_retries: u32) -> HttpClient {
|
||||
HttpClient::new(
|
||||
Auth::Bearer {
|
||||
token: "t".to_owned(),
|
||||
},
|
||||
RetryPolicy::new(max_retries),
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
fn method_error(error_type: &str) -> JmapError {
|
||||
JmapError::Method {
|
||||
call_id: "i".to_owned(),
|
||||
error_type: error_type.to_owned(),
|
||||
description: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn drive(
|
||||
max_retries: u32,
|
||||
outcome: impl Fn(u32) -> Result<&'static str, JmapError>,
|
||||
) -> (Result<&'static str, JmapError>, u32) {
|
||||
let client = test_client(max_retries);
|
||||
let calls = Cell::new(0u32);
|
||||
let result = retry_method_call(
|
||||
&client,
|
||||
MethodCallKind::SingleObjectWrite,
|
||||
&Logger::from_flags(true, 0),
|
||||
|| {
|
||||
calls.set(calls.get() + 1);
|
||||
outcome(calls.get())
|
||||
},
|
||||
);
|
||||
(result, calls.get())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transient_method_error_is_retried_until_it_succeeds() {
|
||||
let (result, calls) = drive(3, |n| {
|
||||
if n < 3 {
|
||||
Err(method_error("serverUnavailable"))
|
||||
} else {
|
||||
Ok("created")
|
||||
}
|
||||
});
|
||||
assert_eq!(result.unwrap(), "created");
|
||||
assert_eq!(calls, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exhausting_the_budget_returns_the_original_method_error() {
|
||||
let (result, calls) = drive(2, |_| Err(method_error("serverUnavailable")));
|
||||
assert_eq!(calls, 3, "one initial attempt plus max_retries");
|
||||
let err = result.unwrap_err();
|
||||
assert!(
|
||||
matches!(&err, JmapError::Method { error_type, .. } if error_type == "serverUnavailable"),
|
||||
"the caller must still see the method error, not RetriesExhausted: {err}"
|
||||
);
|
||||
assert!(
|
||||
!Error::from(err).aborts_run(),
|
||||
"an exhausted method retry stays a per-unit failure"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn permanent_method_error_consumes_no_attempt() {
|
||||
let (result, calls) = drive(5, |_| Err(method_error("invalidArguments")));
|
||||
assert!(result.is_err());
|
||||
assert_eq!(calls, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_fail_is_not_retried_for_a_write() {
|
||||
let (result, calls) = drive(5, |_| Err(method_error("serverPartialFail")));
|
||||
assert!(result.is_err());
|
||||
assert_eq!(calls, 1, "retrying a partial write could duplicate it");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_method_errors_are_left_to_the_http_layer() {
|
||||
let (result, calls) = drive(5, |_| Err(JmapError::Transport("reset".to_owned())));
|
||||
assert!(result.is_err());
|
||||
assert_eq!(calls, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn using_union_includes_core_and_dedups() {
|
||||
|
||||
+35
-4
@@ -27,9 +27,19 @@ pub fn classify_http_status(status: u16) -> Disposition {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn jmap_method_disposition(error_type: &str) -> Disposition {
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum MethodCallKind {
|
||||
Read,
|
||||
SingleObjectWrite,
|
||||
}
|
||||
|
||||
pub fn jmap_method_disposition(error_type: &str, kind: MethodCallKind) -> Disposition {
|
||||
match error_type {
|
||||
"serverUnavailable" | "serverPartialFail" => Disposition::Retryable,
|
||||
"serverUnavailable" => Disposition::Retryable,
|
||||
"serverPartialFail" => match kind {
|
||||
MethodCallKind::Read => Disposition::Retryable,
|
||||
MethodCallKind::SingleObjectWrite => Disposition::Fatal,
|
||||
},
|
||||
_ => Disposition::Fatal,
|
||||
}
|
||||
}
|
||||
@@ -231,12 +241,33 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn method_classification() {
|
||||
for kind in [MethodCallKind::Read, MethodCallKind::SingleObjectWrite] {
|
||||
assert_eq!(
|
||||
jmap_method_disposition("serverUnavailable"),
|
||||
jmap_method_disposition("serverUnavailable", kind),
|
||||
Disposition::Retryable,
|
||||
"{kind:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
jmap_method_disposition("invalidArguments", kind),
|
||||
Disposition::Fatal,
|
||||
"{kind:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
jmap_method_disposition("forbidden", kind),
|
||||
Disposition::Fatal,
|
||||
"{kind:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_fail_is_never_retried_for_a_write() {
|
||||
assert_eq!(
|
||||
jmap_method_disposition("serverPartialFail", MethodCallKind::Read),
|
||||
Disposition::Retryable
|
||||
);
|
||||
assert_eq!(
|
||||
jmap_method_disposition("invalidArguments"),
|
||||
jmap_method_disposition("serverPartialFail", MethodCallKind::SingleObjectWrite),
|
||||
Disposition::Fatal
|
||||
);
|
||||
}
|
||||
|
||||
+265
-2
@@ -58,12 +58,100 @@ fn well_known_url(base: &str) -> String {
|
||||
format!("{trimmed}/.well-known/jmap")
|
||||
}
|
||||
|
||||
fn scheme_end(url: &str) -> Option<usize> {
|
||||
let bytes = url.as_bytes();
|
||||
if !bytes.first()?.is_ascii_alphabetic() {
|
||||
return None;
|
||||
}
|
||||
for (index, byte) in bytes.iter().copied().enumerate() {
|
||||
if byte == b':' {
|
||||
return Some(index);
|
||||
}
|
||||
if !(byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'-' | b'.')) {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub fn resolve_reference(base: &str, reference: &str) -> String {
|
||||
if reference.is_empty() {
|
||||
return base.to_owned();
|
||||
}
|
||||
if scheme_end(reference).is_some() {
|
||||
return reference.to_owned();
|
||||
}
|
||||
let Some(base_scheme_end) = scheme_end(base) else {
|
||||
return reference.to_owned();
|
||||
};
|
||||
let scheme = &base[..=base_scheme_end];
|
||||
let Some(authority_and_path) = base[base_scheme_end + 1..].strip_prefix("//") else {
|
||||
return reference.to_owned();
|
||||
};
|
||||
let mut out = String::with_capacity(scheme.len() + 2 + base.len() + reference.len());
|
||||
out.push_str(scheme);
|
||||
out.push_str("//");
|
||||
if let Some(rest) = reference.strip_prefix("//") {
|
||||
out.push_str(rest);
|
||||
return out;
|
||||
}
|
||||
let authority_len = authority_and_path
|
||||
.find(['/', '?', '#'])
|
||||
.unwrap_or(authority_and_path.len());
|
||||
out.push_str(&authority_and_path[..authority_len]);
|
||||
if reference.starts_with('/') {
|
||||
out.push_str(reference);
|
||||
return out;
|
||||
}
|
||||
let path = &authority_and_path[authority_len..];
|
||||
let path = path.split(['?', '#']).next().unwrap_or("");
|
||||
match path.rfind('/') {
|
||||
Some(last) => out.push_str(&path[..=last]),
|
||||
None => out.push('/'),
|
||||
}
|
||||
out.push_str(reference);
|
||||
out
|
||||
}
|
||||
|
||||
pub fn origin_of(url: &str) -> Option<String> {
|
||||
let origin = url::Url::parse(url).ok()?.origin();
|
||||
if !origin.is_tuple() {
|
||||
return None;
|
||||
}
|
||||
Some(origin.ascii_serialization())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct OriginMismatch {
|
||||
pub fields: Vec<&'static str>,
|
||||
pub session_origin: String,
|
||||
pub connect_origin: String,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for OriginMismatch {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str("session advertises ")?;
|
||||
for (index, field) in self.fields.iter().enumerate() {
|
||||
if index > 0 {
|
||||
f.write_str(", ")?;
|
||||
}
|
||||
f.write_str(field)?;
|
||||
}
|
||||
write!(
|
||||
f,
|
||||
" on {} (connected to {}); vandelay must use the advertised URL. \
|
||||
If this is wrong, fix the server's advertised HTTP URL setting.",
|
||||
self.session_origin, self.connect_origin
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl Session {
|
||||
pub fn discover(client: &HttpClient, url: &str) -> Result<Session, JmapError> {
|
||||
match client.get(url) {
|
||||
Ok(direct) => {
|
||||
if let Some(session) = parse_session(&direct) {
|
||||
return session.ensure_authenticated();
|
||||
return session.rebased_on(url).ensure_authenticated();
|
||||
}
|
||||
}
|
||||
Err(JmapError::Auth(m)) => return Err(JmapError::Auth(m)),
|
||||
@@ -73,7 +161,7 @@ impl Session {
|
||||
match client.get(&well_known) {
|
||||
Ok(body) => {
|
||||
if let Some(session) = parse_session(&body) {
|
||||
return session.ensure_authenticated();
|
||||
return session.rebased_on(&well_known).ensure_authenticated();
|
||||
}
|
||||
Err(JmapError::Connect(format!(
|
||||
"no JMAP Session object at {url} or {well_known}"
|
||||
@@ -86,6 +174,44 @@ impl Session {
|
||||
}
|
||||
}
|
||||
|
||||
fn rebased_on(mut self, session_resource_url: &str) -> Session {
|
||||
self.api_url = resolve_reference(session_resource_url, &self.api_url);
|
||||
self.upload_url = resolve_reference(session_resource_url, &self.upload_url);
|
||||
self.download_url = resolve_reference(session_resource_url, &self.download_url);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn origin_mismatches(&self, connect_url: &str) -> Vec<OriginMismatch> {
|
||||
let Some(connect_origin) = origin_of(connect_url) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut mismatches: Vec<OriginMismatch> = Vec::new();
|
||||
for (field, value) in [
|
||||
("apiUrl", &self.api_url),
|
||||
("uploadUrl", &self.upload_url),
|
||||
("downloadUrl", &self.download_url),
|
||||
] {
|
||||
let Some(session_origin) = origin_of(value) else {
|
||||
continue;
|
||||
};
|
||||
if session_origin == connect_origin {
|
||||
continue;
|
||||
}
|
||||
match mismatches
|
||||
.iter_mut()
|
||||
.find(|m| m.session_origin == session_origin)
|
||||
{
|
||||
Some(existing) => existing.fields.push(field),
|
||||
None => mismatches.push(OriginMismatch {
|
||||
fields: vec![field],
|
||||
session_origin,
|
||||
connect_origin: connect_origin.clone(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
mismatches
|
||||
}
|
||||
|
||||
fn ensure_authenticated(self) -> Result<Session, JmapError> {
|
||||
if self.accounts.is_empty() {
|
||||
return Err(JmapError::Auth(
|
||||
@@ -236,6 +362,143 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn origin_treats_the_default_port_as_equivalent() {
|
||||
assert_eq!(
|
||||
origin_of("https://mail.example/jmap"),
|
||||
origin_of("https://mail.example:443/jmap")
|
||||
);
|
||||
assert_eq!(
|
||||
origin_of("http://mail.example/jmap"),
|
||||
origin_of("http://mail.example:80/jmap")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn origin_distinguishes_port_host_and_scheme() {
|
||||
let base = origin_of("https://mail.example:10443/jmap");
|
||||
assert_ne!(base, origin_of("https://mail.example/jmap"));
|
||||
assert_ne!(base, origin_of("https://other.example:10443/jmap"));
|
||||
assert_ne!(base, origin_of("http://mail.example:10443/jmap"));
|
||||
}
|
||||
|
||||
fn session_on(base: &str) -> Session {
|
||||
let raw = format!(
|
||||
r#"{{"apiUrl":"{base}/jmap/","uploadUrl":"{base}/jmap/upload/{{accountId}}/",
|
||||
"downloadUrl":"{base}/jmap/dl/{{accountId}}/{{blobId}}/{{type}}/{{name}}",
|
||||
"capabilities":{{}},"accounts":{{}}}}"#
|
||||
);
|
||||
parse_session(&raw).expect("session")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matching_origin_reports_no_mismatch() {
|
||||
let session = session_on("https://mail.example:10443");
|
||||
assert!(
|
||||
session
|
||||
.origin_mismatches("https://mail.example:10443")
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
session_on("https://mail.example")
|
||||
.origin_mismatches("https://mail.example:443/")
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dropped_port_is_reported_once_naming_every_field() {
|
||||
let session = session_on("https://mail.example");
|
||||
let mismatches = session.origin_mismatches("https://mail.example:10443");
|
||||
assert_eq!(mismatches.len(), 1, "one warning per distinct origin");
|
||||
assert_eq!(
|
||||
mismatches[0].fields,
|
||||
vec!["apiUrl", "uploadUrl", "downloadUrl"]
|
||||
);
|
||||
assert_eq!(mismatches[0].session_origin, "https://mail.example");
|
||||
assert_eq!(mismatches[0].connect_origin, "https://mail.example:10443");
|
||||
let warning = mismatches[0].to_string();
|
||||
assert!(warning.contains("session advertises apiUrl"), "{warning}");
|
||||
assert!(
|
||||
warning.contains("on https://mail.example (connected to https://mail.example:10443)"),
|
||||
"{warning}"
|
||||
);
|
||||
assert!(warning.contains("advertised HTTP URL setting"), "{warning}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_split_upload_host_is_reported_separately_from_the_api_host() {
|
||||
let raw = r#"{"apiUrl":"https://mail.example:10443/jmap/",
|
||||
"uploadUrl":"https://blobs.example/upload/{accountId}/",
|
||||
"downloadUrl":"https://blobs.example/dl/{accountId}/{blobId}/{type}/{name}",
|
||||
"capabilities":{},"accounts":{}}"#;
|
||||
let session = parse_session(raw).expect("session");
|
||||
let mismatches = session.origin_mismatches("https://mail.example:10443");
|
||||
assert_eq!(mismatches.len(), 1);
|
||||
assert_eq!(mismatches[0].fields, vec!["uploadUrl", "downloadUrl"]);
|
||||
assert_eq!(mismatches[0].session_origin, "https://blobs.example");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_reference_keeps_absolute_urls_verbatim() {
|
||||
assert_eq!(
|
||||
resolve_reference(
|
||||
"https://h.example/.well-known/jmap",
|
||||
"https://other.example:8443/jmap/api"
|
||||
),
|
||||
"https://other.example:8443/jmap/api"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_reference_handles_scheme_relative_absolute_and_relative_paths() {
|
||||
let base = "https://h.example:10443/.well-known/jmap";
|
||||
assert_eq!(
|
||||
resolve_reference(base, "//other.example/jmap/api"),
|
||||
"https://other.example/jmap/api"
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_reference(base, "/jmap/api"),
|
||||
"https://h.example:10443/jmap/api"
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_reference(base, "api"),
|
||||
"https://h.example:10443/.well-known/api"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_reference_preserves_uri_template_braces() {
|
||||
assert_eq!(
|
||||
resolve_reference(
|
||||
"https://h.example/.well-known/jmap",
|
||||
"/jmap/upload/{accountId}/"
|
||||
),
|
||||
"https://h.example/jmap/upload/{accountId}/"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relative_session_urls_are_resolved_against_the_session_resource() {
|
||||
let raw = r#"{"apiUrl":"/jmap/","uploadUrl":"/jmap/upload/{accountId}/",
|
||||
"downloadUrl":"/jmap/download/{accountId}/{blobId}/{type}/{name}",
|
||||
"capabilities":{},"accounts":{}}"#;
|
||||
let session = parse_session(raw)
|
||||
.expect("session")
|
||||
.rebased_on("https://h.example:10443/.well-known/jmap");
|
||||
assert_eq!(session.api_url, "https://h.example:10443/jmap/");
|
||||
assert_eq!(
|
||||
session.upload_url_for("w"),
|
||||
"https://h.example:10443/jmap/upload/w/"
|
||||
);
|
||||
assert!(
|
||||
session
|
||||
.origin_mismatches("https://h.example:10443")
|
||||
.is_empty(),
|
||||
"a resolved relative URL shares the connect origin"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn well_known_appends_correctly() {
|
||||
assert_eq!(
|
||||
|
||||
+28
-16
@@ -9,7 +9,7 @@ use clap::Parser;
|
||||
use vandelay::cli::{Action, Cli};
|
||||
use vandelay::error::Error;
|
||||
use vandelay::inspect;
|
||||
use vandelay::sync::{self, Summary};
|
||||
use vandelay::sync::{self, RunOutcome, Summary};
|
||||
|
||||
fn main() {
|
||||
let code = run();
|
||||
@@ -30,42 +30,54 @@ fn run() -> i32 {
|
||||
Err(err) => return fail(&err),
|
||||
};
|
||||
|
||||
let result = match action {
|
||||
let (outcome, logger) = match action {
|
||||
Action::Import(common, config) => {
|
||||
let logger = common.logger;
|
||||
sync::import_jmap::run(common, config).map(|s| (s, logger))
|
||||
(sync::import_jmap::run_reporting(common, config), logger)
|
||||
}
|
||||
Action::ImportImap(common, config) => {
|
||||
let logger = common.logger;
|
||||
sync::import_imap::run(common, config).map(|s| (s, logger))
|
||||
(sync::import_imap::run_reporting(common, config), logger)
|
||||
}
|
||||
Action::ImportDav(common, config) => {
|
||||
let logger = common.logger;
|
||||
sync::import_dav::run(common, config).map(|s| (s, logger))
|
||||
(sync::import_dav::run_reporting(common, config), logger)
|
||||
}
|
||||
Action::ImportManageSieve(common, config) => {
|
||||
let logger = common.logger;
|
||||
sync::import_managesieve::run(common, config).map(|s| (s, logger))
|
||||
(
|
||||
RunOutcome::from_result(sync::import_managesieve::run(common, config)),
|
||||
logger,
|
||||
)
|
||||
}
|
||||
Action::ImportMaildir(common, config) => {
|
||||
let logger = common.logger;
|
||||
sync::import_maildir::run(common, config).map(|s| (s, logger))
|
||||
(sync::import_maildir::run_reporting(common, config), logger)
|
||||
}
|
||||
Action::ImportTakeout(common, config) => {
|
||||
let logger = common.logger;
|
||||
sync::import_takeout::run(common, config).map(|s| (s, logger))
|
||||
(sync::import_takeout::run_reporting(common, config), logger)
|
||||
}
|
||||
Action::ImportExchangeEws(common, config) => {
|
||||
let logger = common.logger;
|
||||
sync::import_exchange_ews::run(common, config).map(|s| (s, logger))
|
||||
(
|
||||
RunOutcome::from_result(sync::import_exchange_ews::run(common, config)),
|
||||
logger,
|
||||
)
|
||||
}
|
||||
Action::ImportExchangeGraph(common, config) => {
|
||||
let logger = common.logger;
|
||||
sync::import_exchange_graph::run(common, config).map(|s| (s, logger))
|
||||
(
|
||||
RunOutcome::from_result(sync::import_exchange_graph::run(common, config)),
|
||||
logger,
|
||||
)
|
||||
}
|
||||
Action::Export(common, config) => {
|
||||
let logger = common.logger;
|
||||
sync::export::run(common, config).map(|s| (s, logger))
|
||||
(
|
||||
RunOutcome::from_result(sync::export::run(common, config)),
|
||||
logger,
|
||||
)
|
||||
}
|
||||
Action::Inspect(config) => {
|
||||
return match inspect::run(config) {
|
||||
@@ -75,17 +87,17 @@ fn run() -> i32 {
|
||||
}
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok((summary, logger)) => {
|
||||
report(&summary);
|
||||
if summary.any_failed() {
|
||||
report(&outcome.summary);
|
||||
match outcome.error {
|
||||
Some(err) => fail(&err),
|
||||
None => {
|
||||
if outcome.summary.any_failed() {
|
||||
logger.error("some objects failed; the archive is consistent and resumable");
|
||||
5
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
Err(err) => fail(&err),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+6
-4
@@ -20,7 +20,7 @@ use crate::jmap::request::{Request, SetRequest, get_all, get_objects, query_all_
|
||||
use crate::jmap::session::{Limits, Session};
|
||||
use crate::jmap::wire::JmapId;
|
||||
use crate::logging::{LEVEL_DEFAULT, Logger};
|
||||
use crate::sync::import_jmap::mapping::{BlobUpload, TargetResolver};
|
||||
use crate::sync::import_jmap::mapping::{BlobBytes, TargetResolver};
|
||||
use crate::sync::{CommonConfig, Context, ExportConfig, Summary, TypeCounts};
|
||||
use crate::types::ObjectType;
|
||||
|
||||
@@ -110,9 +110,10 @@ impl<'a> Uploader<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
impl BlobUpload for Uploader<'_> {
|
||||
fn upload(&mut self, local_id: i64) -> Result<JmapId, JmapError> {
|
||||
self.upload_with(local_id, "application/octet-stream")
|
||||
impl BlobBytes for Uploader<'_> {
|
||||
fn bytes(&self, local_id: i64) -> Result<Vec<u8>, JmapError> {
|
||||
db::blobs::blob_bytes(self.conn, local_id)?
|
||||
.ok_or_else(|| JmapError::malformed(format!("blob local id {local_id} missing")))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,6 +173,7 @@ pub fn run(common: CommonConfig, config: ExportConfig) -> Result<Summary, Error>
|
||||
);
|
||||
let plan = match res {
|
||||
Ok(p) => p,
|
||||
Err(e) if e.aborts_run() => return Err(e),
|
||||
Err(e) => {
|
||||
logger.warn(&format!("type {} aborted: {e}", ty.jmap_name()));
|
||||
counts.failed += 1;
|
||||
|
||||
@@ -12,7 +12,10 @@ use super::common::{jid, target_query_get};
|
||||
use super::{Maps, Net, Plan, Uploader};
|
||||
use crate::error::Error;
|
||||
use crate::jmap::error::JmapError;
|
||||
use crate::jmap::request::{Request, check_method_error, get_objects};
|
||||
use crate::jmap::request::{
|
||||
MethodCall, Request, check_method_error, get_objects, retry_method_call,
|
||||
};
|
||||
use crate::jmap::retry::MethodCallKind;
|
||||
use crate::jmap::wire::JmapId;
|
||||
use crate::logging::Logger;
|
||||
use crate::sync::import_jmap::mapping::{EMAIL_SELECT, EmailRow, TargetResolver, row_to_email};
|
||||
@@ -219,7 +222,7 @@ fn export_one(
|
||||
return;
|
||||
}
|
||||
let item = import_item(blob, mids, build_keywords(row), &row.received_at);
|
||||
match send_single_import(net, &cid, item) {
|
||||
match send_single_import(net, &cid, item, logger) {
|
||||
Ok(SingleImport::Created) => counts.created += 1,
|
||||
Ok(SingleImport::Skipped) => counts.skipped += 1,
|
||||
Ok(SingleImport::NotCreated { error_type, .. }) if error_type == "blobNotFound" => {
|
||||
@@ -277,7 +280,7 @@ fn retry_after_reupload(
|
||||
}
|
||||
};
|
||||
let item = import_item(blob, mids, build_keywords(row), &row.received_at);
|
||||
match send_single_import(net, cid, item) {
|
||||
match send_single_import(net, cid, item, logger) {
|
||||
Ok(SingleImport::Created) => counts.created += 1,
|
||||
Ok(SingleImport::Skipped) => counts.skipped += 1,
|
||||
Ok(SingleImport::NotCreated { detail, .. }) => {
|
||||
@@ -304,7 +307,12 @@ enum SingleImport {
|
||||
NotCreated { error_type: String, detail: String },
|
||||
}
|
||||
|
||||
fn send_single_import(net: &Net, cid: &str, item: Value) -> Result<SingleImport, JmapError> {
|
||||
fn send_single_import(
|
||||
net: &Net,
|
||||
cid: &str,
|
||||
item: Value,
|
||||
logger: &Logger,
|
||||
) -> Result<SingleImport, JmapError> {
|
||||
let mut emails = Map::new();
|
||||
emails.insert(cid.to_owned(), item);
|
||||
let mut req = Request::new();
|
||||
@@ -314,8 +322,15 @@ fn send_single_import(net: &Net, cid: &str, item: Value) -> Result<SingleImport,
|
||||
"i",
|
||||
);
|
||||
req.fits(&net.limits)?;
|
||||
let resp = req.send(&net.client, &net.api)?;
|
||||
let mr = resp.first()?;
|
||||
retry_method_call(
|
||||
&net.client,
|
||||
MethodCallKind::SingleObjectWrite,
|
||||
logger,
|
||||
|| interpret_import(req.send(&net.client, &net.api)?.first()?, cid),
|
||||
)
|
||||
}
|
||||
|
||||
fn interpret_import(mr: &MethodCall, cid: &str) -> Result<SingleImport, JmapError> {
|
||||
check_method_error(mr)?;
|
||||
if let Some(err) = mr
|
||||
.args
|
||||
|
||||
+27
-15
@@ -5,10 +5,11 @@
|
||||
*/
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::fmt::Write as _;
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use super::common::{create_batch, jid, retry_if_blob_missing, target_query_get};
|
||||
use super::common::{create_batch, jid, target_query_get};
|
||||
use super::{Maps, Net, Plan, Uploader};
|
||||
use crate::error::Error;
|
||||
use crate::logging::Logger;
|
||||
@@ -23,6 +24,15 @@ fn target_uid(v: &Value) -> Option<String> {
|
||||
v.get("uid").and_then(Value::as_str).map(str::to_owned)
|
||||
}
|
||||
|
||||
fn describe(ty: ObjectType, local: i64, uid: &str) -> String {
|
||||
let mut out = String::new();
|
||||
let _ = write!(out, "{} local {local}", ty.jmap_name());
|
||||
if !uid.is_empty() {
|
||||
let _ = write!(out, " (uid {uid})");
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub fn reconcile(
|
||||
ctx: &Context,
|
||||
net: &Net,
|
||||
@@ -66,7 +76,7 @@ pub fn reconcile(
|
||||
};
|
||||
|
||||
let mut matched_uids: HashSet<String> = HashSet::new();
|
||||
let mut uploader = Uploader::new(net, &ctx.conn);
|
||||
let blobs = Uploader::new(net, &ctx.conn);
|
||||
for (local, uid) in &rows {
|
||||
if let Some(tid) = by_uid.get(uid) {
|
||||
maps.insert(ty, *local, crate::jmap::wire::JmapId(tid.clone()));
|
||||
@@ -75,24 +85,26 @@ pub fn reconcile(
|
||||
continue;
|
||||
}
|
||||
let cid = format!("c{local}");
|
||||
let _ = uploader.take_touched();
|
||||
let wire = match build_wire(ctx, ty, *local, maps, &mut uploader) {
|
||||
let wire = match build_wire(ctx, ty, *local, maps, &blobs) {
|
||||
Ok(w) => w,
|
||||
Err(e) if e.aborts_run() => return Err(e),
|
||||
Err(e) => {
|
||||
logger.warn(&format!("{} local {local} skipped: {e}", ty.jmap_name()));
|
||||
logger.warn(&format!("{} skipped: {e}", describe(ty, *local, uid)));
|
||||
counts.failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let touched = uploader.take_touched();
|
||||
let outcome = create_batch(net, ty, vec![(cid.clone(), wire)]).map_err(Error::from)?;
|
||||
let outcome =
|
||||
match retry_if_blob_missing(net, ty, &cid, &mut uploader, touched, outcome, |up| {
|
||||
build_wire(ctx, ty, *local, maps, up)
|
||||
}) {
|
||||
let outcome = match create_batch(net, ty, vec![(cid.clone(), wire)]) {
|
||||
Ok(o) => o,
|
||||
Err(e) => {
|
||||
logger.warn(&format!("{} local {local} skipped: {e}", ty.jmap_name()));
|
||||
let mapped = Error::from(e);
|
||||
if mapped.aborts_run() {
|
||||
return Err(mapped);
|
||||
}
|
||||
logger.warn(&format!(
|
||||
"{} not created: {mapped}",
|
||||
describe(ty, *local, uid)
|
||||
));
|
||||
counts.failed += 1;
|
||||
continue;
|
||||
}
|
||||
@@ -136,7 +148,7 @@ fn build_wire(
|
||||
ty: ObjectType,
|
||||
local: i64,
|
||||
maps: &Maps,
|
||||
up: &mut Uploader<'_>,
|
||||
blobs: &Uploader<'_>,
|
||||
) -> Result<Value, Error> {
|
||||
if ty == ObjectType::ContactCard {
|
||||
let (uid, abids, data): (String, String, String) = ctx
|
||||
@@ -147,7 +159,7 @@ fn build_wire(
|
||||
|r| Ok((r.get(1)?, r.get(2)?, r.get(3)?)),
|
||||
)
|
||||
.map_err(|e| Error::Partial(e.to_string()))?;
|
||||
contact_card_to_wire(&uid, &abids, &data, maps, up).map_err(Error::from)
|
||||
contact_card_to_wire(&uid, &abids, &data, maps, blobs).map_err(Error::from)
|
||||
} else {
|
||||
let (cal, dr, ud, data): (String, i64, i64, String) = ctx
|
||||
.conn
|
||||
@@ -157,6 +169,6 @@ fn build_wire(
|
||||
|r| Ok((r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)),
|
||||
)
|
||||
.map_err(|e| Error::Partial(e.to_string()))?;
|
||||
calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, maps, up).map_err(Error::from)
|
||||
calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, maps, blobs).map_err(Error::from)
|
||||
}
|
||||
}
|
||||
|
||||
+45
-1
@@ -10,4 +10,48 @@ pub mod coordinator;
|
||||
pub mod items;
|
||||
pub mod tree;
|
||||
|
||||
pub use coordinator::{DavAuth, DavImportConfig, DavKindArg, run};
|
||||
pub use coordinator::{DavAuth, DavImportConfig, DavKindArg, run, run_reporting};
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::jmap::error::JmapError;
|
||||
|
||||
pub(crate) fn per_collection_failure(err: JmapError) -> Error {
|
||||
match err {
|
||||
JmapError::Sqlite(e) => Error::Db(crate::db::init::OpenError::Sqlite(e)),
|
||||
scoped => Error::Partial(scoped.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_forbidden_collection_is_a_per_unit_failure() {
|
||||
for e in [
|
||||
JmapError::Auth("server returned 403: forbidden".to_owned()),
|
||||
JmapError::HttpStatus {
|
||||
status: 405,
|
||||
body: "method not allowed".to_owned(),
|
||||
},
|
||||
JmapError::RetriesExhausted("PROPFIND kept returning 503".to_owned()),
|
||||
JmapError::Transport("io: connection reset".to_owned()),
|
||||
JmapError::Malformed("multistatus parse".to_owned()),
|
||||
] {
|
||||
let mapped = per_collection_failure(e);
|
||||
assert!(
|
||||
!mapped.aborts_run(),
|
||||
"{mapped} is scoped to one collection and must not abort the run"
|
||||
);
|
||||
assert_eq!(mapped.exit_code(), 5);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_archive_failure_still_aborts() {
|
||||
let mapped =
|
||||
per_collection_failure(JmapError::Sqlite(rusqlite::Error::QueryReturnedNoRows));
|
||||
assert!(mapped.aborts_run());
|
||||
assert_eq!(mapped.exit_code(), 7);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ use crate::db::sources::SourceKey;
|
||||
use crate::error::Error;
|
||||
use crate::jmap::http::{Auth, RetryPolicy};
|
||||
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
||||
use crate::sync::{CommonConfig, Summary, TypeCounts};
|
||||
use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts};
|
||||
|
||||
use super::collections;
|
||||
use super::items;
|
||||
@@ -83,6 +83,20 @@ pub struct DavImportConfig {
|
||||
}
|
||||
|
||||
pub fn run(common: CommonConfig, config: DavImportConfig) -> Result<Summary, Error> {
|
||||
run_reporting(common, config).into_result()
|
||||
}
|
||||
|
||||
pub fn run_reporting(common: CommonConfig, config: DavImportConfig) -> RunOutcome {
|
||||
let mut summary = Summary::default();
|
||||
let error = run_into(common, config, &mut summary).err();
|
||||
RunOutcome { summary, error }
|
||||
}
|
||||
|
||||
fn run_into(
|
||||
common: CommonConfig,
|
||||
config: DavImportConfig,
|
||||
summary: &mut Summary,
|
||||
) -> Result<(), Error> {
|
||||
let logger = common.logger;
|
||||
enforce_tls_policy(&config.url, config.allow_cleartext)?;
|
||||
|
||||
@@ -138,38 +152,42 @@ pub fn run(common: CommonConfig, config: DavImportConfig) -> Result<Summary, Err
|
||||
}
|
||||
|
||||
if common.dry_run {
|
||||
let mut summary = run_dry_diff(&conn, &client, &discovery, &config, logger)?;
|
||||
let result = run_dry_diff(&conn, &client, &discovery, &config, logger, summary);
|
||||
summary.retries_observed = client.retries_observed();
|
||||
summary.retry_after_sleeps = client.retry_after_sleeps();
|
||||
return Ok(summary);
|
||||
return result;
|
||||
}
|
||||
|
||||
let username = config.auth.username();
|
||||
let source_id = db::sources::upsert_source(&conn, &key, Some(&account_id), &username)
|
||||
.map_err(|e| Error::Partial(e.to_string()))?;
|
||||
|
||||
let summary = match config.kind {
|
||||
let phase = match config.kind {
|
||||
DavKindArg::Caldav => {
|
||||
run_caldav(&mut conn, &client, source_id, &discovery, &config, logger)?
|
||||
run_caldav(&mut conn, &client, source_id, &discovery, &config, logger)
|
||||
}
|
||||
DavKindArg::Carddav => {
|
||||
run_carddav(&mut conn, &client, source_id, &discovery, &config, logger)?
|
||||
run_carddav(&mut conn, &client, source_id, &discovery, &config, logger)
|
||||
}
|
||||
DavKindArg::Webdav => {
|
||||
run_webdav(&mut conn, &client, source_id, &discovery, &config, logger)?
|
||||
run_webdav(&mut conn, &client, source_id, &discovery, &config, logger)
|
||||
}
|
||||
};
|
||||
|
||||
*summary = phase.summary;
|
||||
summary.retries_observed = client.retries_observed();
|
||||
summary.retry_after_sleeps = client.retry_after_sleeps();
|
||||
if let Some(e) = phase.error {
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
if !summary.any_failed()
|
||||
&& let Err(e) = run_gc(&conn)
|
||||
{
|
||||
logger.warn(&format!("blob GC skipped: {e}"));
|
||||
}
|
||||
|
||||
let mut summary = summary;
|
||||
summary.retries_observed = client.retries_observed();
|
||||
summary.retry_after_sleeps = client.retry_after_sleeps();
|
||||
Ok(summary)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
type ReconcileCollections = fn(
|
||||
@@ -198,25 +216,71 @@ fn run_collection_phase(
|
||||
config: &DavImportConfig,
|
||||
logger: Logger,
|
||||
phase: ItemPhase,
|
||||
) -> Result<Summary, Error> {
|
||||
) -> RunOutcome {
|
||||
let mut summary = Summary::default();
|
||||
let mut container_counts = TypeCounts::default();
|
||||
let mut item_counts = TypeCounts::default();
|
||||
let mut counts = PhaseCounts::default();
|
||||
let error = collection_phase_into(
|
||||
conn,
|
||||
client,
|
||||
source_id,
|
||||
PhaseInput {
|
||||
discovery,
|
||||
config,
|
||||
logger,
|
||||
phase: &phase,
|
||||
},
|
||||
&mut counts,
|
||||
)
|
||||
.err();
|
||||
|
||||
summary
|
||||
.per_type
|
||||
.push((phase.container_label, counts.container));
|
||||
summary.per_type.push((phase.item_label, counts.items));
|
||||
RunOutcome { summary, error }
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct PhaseCounts {
|
||||
container: TypeCounts,
|
||||
items: TypeCounts,
|
||||
}
|
||||
|
||||
struct PhaseInput<'a> {
|
||||
discovery: &'a Discovery,
|
||||
config: &'a DavImportConfig,
|
||||
logger: Logger,
|
||||
phase: &'a ItemPhase,
|
||||
}
|
||||
|
||||
fn collection_phase_into(
|
||||
conn: &mut Connection,
|
||||
client: &DavClient,
|
||||
source_id: i64,
|
||||
input: PhaseInput<'_>,
|
||||
counts: &mut PhaseCounts,
|
||||
) -> Result<(), Error> {
|
||||
let PhaseInput {
|
||||
discovery,
|
||||
config,
|
||||
logger,
|
||||
phase,
|
||||
} = input;
|
||||
|
||||
let upserted = (phase.reconcile_collections)(
|
||||
conn,
|
||||
source_id,
|
||||
&discovery.collections,
|
||||
&mut container_counts,
|
||||
&mut counts.container,
|
||||
logger,
|
||||
)?;
|
||||
if logger.enabled(LEVEL_DEFAULT) {
|
||||
eprintln!(
|
||||
"import: {} done (upserted={} deleted={} failed={})",
|
||||
phase.container_label,
|
||||
container_counts.created + container_counts.fetched,
|
||||
container_counts.deleted,
|
||||
container_counts.failed
|
||||
counts.container.created + counts.container.fetched,
|
||||
counts.container.deleted,
|
||||
counts.container.failed
|
||||
);
|
||||
}
|
||||
|
||||
@@ -229,23 +293,19 @@ fn run_collection_phase(
|
||||
logger,
|
||||
};
|
||||
for (collection_href, local_id) in &upserted {
|
||||
match (phase.reconcile_items)(conn, &ctx, collection_href, *local_id, &mut item_counts) {
|
||||
match (phase.reconcile_items)(conn, &ctx, collection_href, *local_id, &mut counts.items) {
|
||||
Ok(()) => {}
|
||||
Err(e) if e.aborts_run() => return Err(e),
|
||||
Err(e) => {
|
||||
logger.warn(&format!(
|
||||
"{} {collection_href:?}: items failed: {e}",
|
||||
phase.container_label
|
||||
));
|
||||
item_counts.failed += 1;
|
||||
counts.items.failed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
summary
|
||||
.per_type
|
||||
.push((phase.container_label, container_counts));
|
||||
summary.per_type.push((phase.item_label, item_counts));
|
||||
Ok(summary)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run_dry_diff(
|
||||
@@ -254,11 +314,11 @@ fn run_dry_diff(
|
||||
discovery: &Discovery,
|
||||
config: &DavImportConfig,
|
||||
logger: Logger,
|
||||
) -> Result<Summary, Error> {
|
||||
summary: &mut Summary,
|
||||
) -> Result<(), Error> {
|
||||
use crate::dav::href::join_absolute;
|
||||
use crate::dav::xml;
|
||||
use crate::db::dav_ids;
|
||||
let mut summary = Summary::default();
|
||||
let (container_label, item_label, container_type, item_type) = match config.kind {
|
||||
DavKindArg::Caldav => (
|
||||
"calendar",
|
||||
@@ -295,9 +355,11 @@ fn run_dry_diff(
|
||||
for coll in &discovery.collections {
|
||||
let url = join_absolute(&discovery.home_set_url, coll.href.as_str())
|
||||
.map_err(|e| Error::Partial(e.to_string()))?;
|
||||
let ms = client
|
||||
match client
|
||||
.propfind_responses(&url, 1, &xml::propfind_dav_items(), &url)
|
||||
.map_err(Error::from)?;
|
||||
.map_err(super::per_collection_failure)
|
||||
{
|
||||
Ok(ms) => {
|
||||
let new_count = ms
|
||||
.responses
|
||||
.iter()
|
||||
@@ -308,12 +370,24 @@ fn run_dry_diff(
|
||||
eprintln!(" {} items: {new_count}", coll.href.as_str());
|
||||
}
|
||||
}
|
||||
Err(e) if e.aborts_run() => return Err(e),
|
||||
Err(e) => {
|
||||
logger.warn(&format!(
|
||||
"{container_label} {:?}: enumeration failed: {e}",
|
||||
coll.href.as_str()
|
||||
));
|
||||
item_counts.failed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if let Some(root) = discovery.collections.first() {
|
||||
let url = join_absolute(&discovery.home_set_url, root.href.as_str())
|
||||
.map_err(|e| Error::Partial(e.to_string()))?;
|
||||
let ms = client
|
||||
match client
|
||||
.propfind_responses(&url, 1, &xml::propfind_webdav_listing(), &url)
|
||||
.map_err(Error::from)?;
|
||||
.map_err(super::per_collection_failure)
|
||||
{
|
||||
Ok(ms) => {
|
||||
let new_count = ms
|
||||
.responses
|
||||
.iter()
|
||||
@@ -324,6 +398,16 @@ fn run_dry_diff(
|
||||
eprintln!(" root {} files: {new_count}", root.href.as_str());
|
||||
}
|
||||
}
|
||||
Err(e) if e.aborts_run() => return Err(e),
|
||||
Err(e) => {
|
||||
logger.warn(&format!(
|
||||
"{container_label} {:?}: enumeration failed: {e}",
|
||||
root.href.as_str()
|
||||
));
|
||||
container_counts.failed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let _ = container_type;
|
||||
let _ = item_type;
|
||||
@@ -332,7 +416,7 @@ fn run_dry_diff(
|
||||
if !matches!(config.kind, DavKindArg::Webdav) {
|
||||
summary.per_type.push((item_label, item_counts));
|
||||
}
|
||||
Ok(summary)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run_caldav(
|
||||
@@ -342,7 +426,7 @@ fn run_caldav(
|
||||
discovery: &Discovery,
|
||||
config: &DavImportConfig,
|
||||
logger: Logger,
|
||||
) -> Result<Summary, Error> {
|
||||
) -> RunOutcome {
|
||||
run_collection_phase(
|
||||
conn,
|
||||
client,
|
||||
@@ -366,7 +450,7 @@ fn run_carddav(
|
||||
discovery: &Discovery,
|
||||
config: &DavImportConfig,
|
||||
logger: Logger,
|
||||
) -> Result<Summary, Error> {
|
||||
) -> RunOutcome {
|
||||
run_collection_phase(
|
||||
conn,
|
||||
client,
|
||||
@@ -390,13 +474,16 @@ fn run_webdav(
|
||||
discovery: &Discovery,
|
||||
config: &DavImportConfig,
|
||||
logger: Logger,
|
||||
) -> Result<Summary, Error> {
|
||||
) -> RunOutcome {
|
||||
let mut summary = Summary::default();
|
||||
let mut file_counts = TypeCounts::default();
|
||||
|
||||
if discovery.collections.is_empty() {
|
||||
summary.per_type.push(("filenode", file_counts));
|
||||
return Ok(summary);
|
||||
return RunOutcome {
|
||||
summary,
|
||||
error: None,
|
||||
};
|
||||
}
|
||||
let root = &discovery.collections[0];
|
||||
let ctx = tree::WebDavCtx {
|
||||
@@ -406,10 +493,10 @@ fn run_webdav(
|
||||
dav_connections: config.dav_connections,
|
||||
logger,
|
||||
};
|
||||
tree::reconcile_filenodes(conn, &ctx, root, &mut file_counts)?;
|
||||
let error = tree::reconcile_filenodes(conn, &ctx, root, &mut file_counts).err();
|
||||
|
||||
summary.per_type.push(("filenode", file_counts));
|
||||
Ok(summary)
|
||||
RunOutcome { summary, error }
|
||||
}
|
||||
|
||||
fn map_discovery_error(err: DiscoveryError) -> Error {
|
||||
|
||||
@@ -113,7 +113,7 @@ fn enumerate_items(client: &DavClient, url: &str) -> Result<Vec<ServerItem>, Err
|
||||
let body = xml::propfind_dav_items();
|
||||
let ms = client
|
||||
.propfind_responses(url, 1, &body, url)
|
||||
.map_err(Error::from)?;
|
||||
.map_err(super::per_collection_failure)?;
|
||||
if ms.status >= 400 {
|
||||
return Err(Error::Partial(format!(
|
||||
"enumerate {url}: http {}",
|
||||
|
||||
@@ -115,6 +115,7 @@ pub fn reconcile_filenodes(
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) if e.aborts_run() => return Err(e),
|
||||
Err(e) => {
|
||||
logger.warn(&format!("PROPFIND {url}: {e}"));
|
||||
counts.failed += 1;
|
||||
@@ -394,7 +395,7 @@ fn walk_one(
|
||||
let body = xml::propfind_webdav_listing();
|
||||
let ms = client
|
||||
.propfind_responses(url, 1, &body, url)
|
||||
.map_err(Error::from)?;
|
||||
.map_err(super::per_collection_failure)?;
|
||||
if ms.status >= 400 {
|
||||
return Err(Error::Partial(format!("http {}", ms.status)));
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ use serde_json::{Value, json};
|
||||
|
||||
use crate::db::exchange_graph_ids;
|
||||
use crate::error::Error;
|
||||
use crate::exchange::jscalendar::override_patch_from_event;
|
||||
use crate::exchange_graph::api::{self, PREFER_BODY_HTML, PREFER_BODY_TEXT, PREFER_TIMEZONE_UTC};
|
||||
use crate::exchange_graph::calendar_map::{
|
||||
ConvertedEvent, EventType, classify_event_type, convert_event,
|
||||
@@ -185,7 +186,7 @@ fn merge_exception_into(master: &mut ConvertedEvent, ex: &ConvertedEvent) {
|
||||
let Value::Object(overrides) = overrides else {
|
||||
return;
|
||||
};
|
||||
overrides.insert(key, ex.data.clone());
|
||||
overrides.insert(key, override_patch_from_event(&ex.data));
|
||||
}
|
||||
|
||||
fn merge_exception_into_existing(
|
||||
@@ -253,7 +254,7 @@ fn merge_persisted_master(
|
||||
.entry("recurrenceOverrides".to_owned())
|
||||
.or_insert_with(|| Value::Object(serde_json::Map::new()));
|
||||
if let Value::Object(overrides) = entry {
|
||||
overrides.insert(key, ex_data.clone());
|
||||
overrides.insert(key, override_patch_from_event(ex_data));
|
||||
}
|
||||
}
|
||||
tx.execute(
|
||||
|
||||
@@ -16,11 +16,10 @@ use crate::exchange_graph::error::GraphError;
|
||||
use crate::exchange_graph::oauth::{
|
||||
AcquiredToken, OAuthFlow, acquire, default_authority, refresh_access_token,
|
||||
};
|
||||
use crate::exchange_graph::types::{EventBodyFormat, MailboxKind, synthetic_account_id};
|
||||
use crate::exchange_graph::types::{EventBodyFormat, MailboxKind, Surfaces, synthetic_account_id};
|
||||
use crate::jmap::http::RetryPolicy;
|
||||
use crate::logging::LEVEL_DEFAULT;
|
||||
use crate::sync::{CommonConfig, Summary, TypeCounts};
|
||||
use crate::types::ObjectType;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum GraphAuth {
|
||||
@@ -39,7 +38,7 @@ pub struct GraphImportConfig {
|
||||
pub api_base: String,
|
||||
pub user_target: Option<String>,
|
||||
pub mailbox_kind: MailboxKind,
|
||||
pub objects: Option<Vec<ObjectType>>,
|
||||
pub surfaces: Surfaces,
|
||||
pub event_body_format: EventBodyFormat,
|
||||
pub graph_connections: usize,
|
||||
pub top: usize,
|
||||
@@ -141,32 +140,10 @@ pub fn run(common: CommonConfig, config: GraphImportConfig) -> Result<Summary, E
|
||||
event_body_format: config.event_body_format,
|
||||
};
|
||||
|
||||
let want_mail = config
|
||||
.objects
|
||||
.as_ref()
|
||||
.map(|set| {
|
||||
set.iter()
|
||||
.any(|o| matches!(o, ObjectType::Mailbox | ObjectType::Email))
|
||||
})
|
||||
.unwrap_or(true);
|
||||
let want_calendar = config
|
||||
.objects
|
||||
.as_ref()
|
||||
.map(|set| {
|
||||
set.iter()
|
||||
.any(|o| matches!(o, ObjectType::Calendar | ObjectType::CalendarEvent))
|
||||
})
|
||||
.unwrap_or(true)
|
||||
&& !matches!(config.mailbox_kind, MailboxKind::Archive);
|
||||
let want_contacts = config
|
||||
.objects
|
||||
.as_ref()
|
||||
.map(|set| {
|
||||
set.iter()
|
||||
.any(|o| matches!(o, ObjectType::AddressBook | ObjectType::ContactCard))
|
||||
})
|
||||
.unwrap_or(true)
|
||||
&& !matches!(config.mailbox_kind, MailboxKind::Archive);
|
||||
let primary = !matches!(config.mailbox_kind, MailboxKind::Archive);
|
||||
let want_mail = config.surfaces.mail;
|
||||
let want_calendar = config.surfaces.calendar && primary;
|
||||
let want_contacts = config.surfaces.contacts && primary;
|
||||
|
||||
if want_mail {
|
||||
let folders = super::folders::reconcile_mail(
|
||||
@@ -509,7 +486,7 @@ mod tests {
|
||||
api_base: "https://graph.microsoft.com/v1.0".to_owned(),
|
||||
user_target: None,
|
||||
mailbox_kind: MailboxKind::Primary,
|
||||
objects: None,
|
||||
surfaces: Surfaces::ALL,
|
||||
event_body_format: EventBodyFormat::Text,
|
||||
graph_connections: 4,
|
||||
top: 100,
|
||||
|
||||
@@ -13,4 +13,4 @@ pub mod pool;
|
||||
|
||||
pub mod coordinator;
|
||||
|
||||
pub use coordinator::{ImapAuth, ImapImportConfig, run};
|
||||
pub use coordinator::{ImapAuth, ImapImportConfig, run, run_reporting};
|
||||
|
||||
@@ -29,7 +29,7 @@ use crate::imap::transport::Connector;
|
||||
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
||||
use crate::sync::emailmeta::email_index_from_blob;
|
||||
use crate::sync::keys::index_to_json;
|
||||
use crate::sync::{CommonConfig, Summary, TypeCounts};
|
||||
use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts};
|
||||
|
||||
use super::fetch;
|
||||
use super::folders::{
|
||||
@@ -199,6 +199,20 @@ pub enum ImapAuth {
|
||||
}
|
||||
|
||||
pub fn run(common: CommonConfig, config: ImapImportConfig) -> Result<Summary, Error> {
|
||||
run_reporting(common, config).into_result()
|
||||
}
|
||||
|
||||
pub fn run_reporting(common: CommonConfig, config: ImapImportConfig) -> RunOutcome {
|
||||
let mut summary = Summary::default();
|
||||
let error = run_into(common, config, &mut summary).err();
|
||||
RunOutcome { summary, error }
|
||||
}
|
||||
|
||||
fn run_into(
|
||||
common: CommonConfig,
|
||||
config: ImapImportConfig,
|
||||
summary: &mut Summary,
|
||||
) -> Result<(), Error> {
|
||||
let logger = common.logger;
|
||||
let mut conn = db::init::open(&common.archive)?;
|
||||
|
||||
@@ -341,14 +355,15 @@ pub fn run(common: CommonConfig, config: ImapImportConfig) -> Result<Summary, Er
|
||||
|
||||
if common.dry_run {
|
||||
let existing_source = db::sources::find_source(&conn, &source_key)?;
|
||||
return dry_run_summary(
|
||||
*summary = dry_run_summary(
|
||||
&conn,
|
||||
existing_source,
|
||||
&mut client,
|
||||
&control_ctx,
|
||||
&resolved,
|
||||
logger,
|
||||
);
|
||||
)?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut mailbox_counts = TypeCounts::default();
|
||||
@@ -410,6 +425,16 @@ pub fn run(common: CommonConfig, config: ImapImportConfig) -> Result<Summary, Er
|
||||
&mut email_counts,
|
||||
) {
|
||||
Ok(()) => {}
|
||||
Err(e) if e.aborts_run() => {
|
||||
pool.shutdown();
|
||||
let _ = client.logout();
|
||||
*summary = Summary {
|
||||
per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)],
|
||||
retries_observed: backoff.total_retries(),
|
||||
retry_after_sleeps: backoff.transient_retries() as u64,
|
||||
};
|
||||
return Err(e);
|
||||
}
|
||||
Err(e) => {
|
||||
log_at(
|
||||
logger,
|
||||
@@ -424,11 +449,12 @@ pub fn run(common: CommonConfig, config: ImapImportConfig) -> Result<Summary, Er
|
||||
pool.shutdown();
|
||||
let _ = client.logout();
|
||||
|
||||
Ok(Summary {
|
||||
*summary = Summary {
|
||||
per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)],
|
||||
retries_observed: backoff.total_retries(),
|
||||
retry_after_sleeps: backoff.transient_retries() as u64,
|
||||
})
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
|
||||
+19
-4
@@ -33,7 +33,7 @@ use crate::jmap::wire::email::Email;
|
||||
use crate::jmap::wire::file_node::{FileNode, NodeType};
|
||||
use crate::jmap::wire::sieve_script::SieveScript;
|
||||
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
||||
use crate::sync::{CommonConfig, Context, ImportConfig, Summary, TypeCounts};
|
||||
use crate::sync::{CommonConfig, Context, ImportConfig, RunOutcome, Summary, TypeCounts};
|
||||
use crate::types::ObjectType;
|
||||
|
||||
const IMPORT_ORDER: [ObjectType; 10] = [
|
||||
@@ -147,6 +147,20 @@ fn username_of(auth: &Auth) -> String {
|
||||
}
|
||||
|
||||
pub fn run(common: CommonConfig, config: ImportConfig) -> Result<Summary, Error> {
|
||||
run_reporting(common, config).into_result()
|
||||
}
|
||||
|
||||
pub fn run_reporting(common: CommonConfig, config: ImportConfig) -> RunOutcome {
|
||||
let mut summary = Summary::default();
|
||||
let error = run_into(common, config, &mut summary).err();
|
||||
RunOutcome { summary, error }
|
||||
}
|
||||
|
||||
fn run_into(
|
||||
common: CommonConfig,
|
||||
config: ImportConfig,
|
||||
summary: &mut Summary,
|
||||
) -> Result<(), Error> {
|
||||
let logger = common.logger;
|
||||
let ctx = Context::open(common, &config.connect)?;
|
||||
let connected = connect::prepare(&ctx, &config.connect)?;
|
||||
@@ -196,7 +210,6 @@ pub fn run(common: CommonConfig, config: ImportConfig) -> Result<Summary, Error>
|
||||
session: connected.session.clone(),
|
||||
};
|
||||
|
||||
let mut summary = Summary::default();
|
||||
let mut dry_rows: Vec<(&'static str, u64, u64, u64)> = Vec::new();
|
||||
let threads = ctx.common.threads;
|
||||
|
||||
@@ -216,6 +229,7 @@ pub fn run(common: CommonConfig, config: ImportConfig) -> Result<Summary, Error>
|
||||
&mut dry_rows,
|
||||
) {
|
||||
Ok(()) => {}
|
||||
Err(e) if e.aborts_run() => return Err(e),
|
||||
Err(e) => {
|
||||
logger.warn(&format!(
|
||||
"type {} aborted: {e}; continuing (run will exit 5)",
|
||||
@@ -229,7 +243,8 @@ pub fn run(common: CommonConfig, config: ImportConfig) -> Result<Summary, Error>
|
||||
|
||||
if ctx.dry_run() {
|
||||
print_dry_run(&dry_rows);
|
||||
return Ok(Summary::default());
|
||||
*summary = Summary::default();
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if !summary.any_failed()
|
||||
@@ -240,7 +255,7 @@ pub fn run(common: CommonConfig, config: ImportConfig) -> Result<Summary, Error>
|
||||
|
||||
summary.retries_observed = ctx.client.retries_observed();
|
||||
summary.retry_after_sleeps = ctx.client.retry_after_sleeps();
|
||||
Ok(summary)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn work_list(config: &ImportConfig, connected: &Connected) -> Vec<ObjectType> {
|
||||
|
||||
+171
-30
@@ -8,7 +8,7 @@ use indexmap::IndexMap;
|
||||
use rusqlite::{Connection, Row, params};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::jmap::blob::{export_blob_ids, import_blob_ids};
|
||||
use crate::jmap::blob::{BlobWalkError, InlineShape, import_blob_ids, inline_blob_data_uris};
|
||||
use crate::jmap::error::JmapError;
|
||||
use crate::jmap::wire::JmapId;
|
||||
use crate::jmap::wire::address_book::AddressBook;
|
||||
@@ -35,8 +35,8 @@ pub trait BlobIntern {
|
||||
fn intern(&mut self, jmap_blob_id: &str) -> Result<i64, JmapError>;
|
||||
}
|
||||
|
||||
pub trait BlobUpload {
|
||||
fn upload(&mut self, local_id: i64) -> Result<JmapId, JmapError>;
|
||||
pub trait BlobBytes {
|
||||
fn bytes(&self, local_id: i64) -> Result<Vec<u8>, JmapError>;
|
||||
}
|
||||
|
||||
fn translate_in(
|
||||
@@ -662,10 +662,10 @@ pub fn contact_card_to_wire(
|
||||
address_book_ids: &str,
|
||||
data: &str,
|
||||
resolver: &impl TargetResolver,
|
||||
blobs: &mut impl BlobUpload,
|
||||
blobs: &impl BlobBytes,
|
||||
) -> Result<Value, JmapError> {
|
||||
let mut value: Value = serde_json::from_str(data)?;
|
||||
restore_blobs_in(&mut value, blobs)?;
|
||||
inline_blobs_in(&mut value, InlineShape::JsContactResource, blobs)?;
|
||||
let locals = parse_local_id_array(address_book_ids)?;
|
||||
let abids = translate_out(&locals, ObjectType::AddressBook, resolver)?;
|
||||
if let Value::Object(map) = &mut value {
|
||||
@@ -681,10 +681,10 @@ pub fn calendar_event_to_wire(
|
||||
use_default_alerts: bool,
|
||||
data: &str,
|
||||
resolver: &impl TargetResolver,
|
||||
blobs: &mut impl BlobUpload,
|
||||
blobs: &impl BlobBytes,
|
||||
) -> Result<Value, JmapError> {
|
||||
let mut value: Value = serde_json::from_str(data)?;
|
||||
restore_blobs_in(&mut value, blobs)?;
|
||||
inline_blobs_in(&mut value, InlineShape::JsCalendarLink, blobs)?;
|
||||
let locals = parse_local_id_array(calendar_ids)?;
|
||||
let calids = translate_out(&locals, ObjectType::Calendar, resolver)?;
|
||||
if let Value::Object(map) = &mut value {
|
||||
@@ -725,21 +725,20 @@ fn take_string(value: &mut Value, key: &str) -> Option<String> {
|
||||
|
||||
fn rewrite_blobs_in(data: &mut Value, blobs: &mut impl BlobIntern) -> Result<(), JmapError> {
|
||||
import_blob_ids(data, |jmap_blob_id| {
|
||||
blobs
|
||||
.intern(jmap_blob_id)
|
||||
.map_err(|e| crate::jmap::blob::BlobWalkError::Resolver(e.to_string()))
|
||||
blobs.intern(jmap_blob_id).map_err(BlobWalkError::resolver)
|
||||
})
|
||||
.map_err(JmapError::from)
|
||||
.map_err(BlobWalkError::into_source)
|
||||
}
|
||||
|
||||
fn restore_blobs_in(data: &mut Value, blobs: &mut impl BlobUpload) -> Result<(), JmapError> {
|
||||
export_blob_ids(data, |local_id| {
|
||||
blobs
|
||||
.upload(local_id)
|
||||
.map(|id| id.0)
|
||||
.map_err(|e| crate::jmap::blob::BlobWalkError::Resolver(e.to_string()))
|
||||
fn inline_blobs_in(
|
||||
data: &mut Value,
|
||||
shape: InlineShape,
|
||||
blobs: &impl BlobBytes,
|
||||
) -> Result<(), JmapError> {
|
||||
inline_blob_data_uris(data, shape, |local_id| {
|
||||
blobs.bytes(local_id).map_err(BlobWalkError::resolver)
|
||||
})
|
||||
.map_err(JmapError::from)
|
||||
.map_err(BlobWalkError::into_source)
|
||||
}
|
||||
|
||||
fn opt_json<T: serde::Serialize>(value: &Option<T>) -> Result<Option<String>, JmapError> {
|
||||
@@ -904,9 +903,35 @@ mod tests {
|
||||
Ok(42)
|
||||
}
|
||||
}
|
||||
impl BlobUpload for FakeBlobs {
|
||||
fn upload(&mut self, local_id: i64) -> Result<JmapId, JmapError> {
|
||||
Ok(JmapId(format!("T{local_id}")))
|
||||
impl BlobBytes for FakeBlobs {
|
||||
fn bytes(&self, local_id: i64) -> Result<Vec<u8>, JmapError> {
|
||||
Ok(format!("payload-{local_id}").into_bytes())
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_data_uri(value: &Value, media_type: &str) -> Vec<u8> {
|
||||
use base64::Engine;
|
||||
let uri = value.as_str().unwrap_or_else(|| panic!("{value} is a URI"));
|
||||
let prefix = format!("data:{media_type};base64,");
|
||||
let payload = uri
|
||||
.strip_prefix(&prefix)
|
||||
.unwrap_or_else(|| panic!("{uri} does not start with {prefix}"));
|
||||
base64::engine::general_purpose::STANDARD
|
||||
.decode(payload)
|
||||
.expect("base64 payload")
|
||||
}
|
||||
|
||||
fn assert_no_blob_id(value: &Value) {
|
||||
match value {
|
||||
Value::Object(map) => {
|
||||
assert!(map.get("blobId").is_none(), "blobId present in {value}");
|
||||
assert!(map.get("@blob").is_none(), "@blob present in {value}");
|
||||
for child in map.values() {
|
||||
assert_no_blob_id(child);
|
||||
}
|
||||
}
|
||||
Value::Array(items) => items.iter().for_each(assert_no_blob_id),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1032,7 +1057,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn contact_card_strips_uid_and_rewrites_blob() {
|
||||
fn contact_card_strips_uid_and_inlines_media_as_data_uri() {
|
||||
let c = mem();
|
||||
let res = MapResolver {
|
||||
to_local: HashMap::from([((ObjectType::AddressBook, "AB".to_owned()), 1)]),
|
||||
@@ -1043,7 +1068,10 @@ mod tests {
|
||||
"addressBookIds": { "AB": true },
|
||||
"uid": "urn:uuid:42",
|
||||
"name": { "full": "Jane" },
|
||||
"photos": { "p1": { "blobId": "PB", "mediaType": "image/png" } }
|
||||
"media": { "photo": {
|
||||
"@type": "Media", "kind": "photo",
|
||||
"blobId": "PB", "mediaType": "image/png"
|
||||
} }
|
||||
}))
|
||||
.unwrap();
|
||||
let mut blobs = FakeBlobs;
|
||||
@@ -1058,14 +1086,127 @@ mod tests {
|
||||
assert_eq!(uid, "urn:uuid:42");
|
||||
let stored: Value = serde_json::from_str(&data).unwrap();
|
||||
assert!(stored.get("uid").is_none());
|
||||
assert_eq!(stored["photos"]["p1"]["@blob"], Value::from(42));
|
||||
assert_eq!(stored["media"]["photo"]["@blob"], Value::from(42));
|
||||
|
||||
let mut up = FakeBlobs;
|
||||
let wire = contact_card_to_wire(&uid, &abids, &data, &res, &mut up).unwrap();
|
||||
let up = FakeBlobs;
|
||||
let wire = contact_card_to_wire(&uid, &abids, &data, &res, &up).unwrap();
|
||||
assert_eq!(wire["uid"], Value::from("urn:uuid:42"));
|
||||
assert_eq!(wire["addressBookIds"]["TAB"], Value::Bool(true));
|
||||
assert_eq!(wire["photos"]["p1"]["blobId"], Value::from("T42"));
|
||||
assert!(wire["photos"]["p1"].get("@blob").is_none());
|
||||
assert_eq!(
|
||||
decode_data_uri(&wire["media"]["photo"]["uri"], "image/png"),
|
||||
b"payload-42"
|
||||
);
|
||||
assert_eq!(
|
||||
wire["media"]["photo"]["mediaType"],
|
||||
Value::from("image/png")
|
||||
);
|
||||
assert!(wire["media"]["photo"].get("href").is_none());
|
||||
assert_no_blob_id(&wire);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn calendar_event_inlines_link_enclosure_as_data_uri() {
|
||||
let c = mem();
|
||||
let res = MapResolver {
|
||||
to_local: HashMap::from([((ObjectType::Calendar, "CAL".to_owned()), 5)]),
|
||||
to_target: HashMap::from([((ObjectType::Calendar, 5), "TCAL".to_owned())]),
|
||||
};
|
||||
let ev: CalendarEvent = serde_json::from_value(serde_json::json!({
|
||||
"id": "EV1",
|
||||
"calendarIds": { "CAL": true },
|
||||
"uid": "ev-with-enclosure",
|
||||
"title": "Review",
|
||||
"@type": "Event",
|
||||
"links": { "1": {
|
||||
"@type": "Link", "rel": "enclosure",
|
||||
"blobId": "AB", "contentType": "text/plain", "title": "agenda.txt"
|
||||
} }
|
||||
}))
|
||||
.unwrap();
|
||||
let mut blobs = FakeBlobs;
|
||||
let local = insert_calendar_event(&c, &ev, &res, &mut blobs).unwrap();
|
||||
let (cal, dr, ud, data): (String, i64, i64, String) = c
|
||||
.query_row(
|
||||
&format!("{CALENDAR_EVENT_SELECT} AND id = ?1"),
|
||||
params![local],
|
||||
|r| Ok((r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)),
|
||||
)
|
||||
.unwrap();
|
||||
let stored: Value = serde_json::from_str(&data).unwrap();
|
||||
assert_eq!(stored["links"]["1"]["@blob"], Value::from(42));
|
||||
|
||||
let up = FakeBlobs;
|
||||
let wire = calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, &res, &up).unwrap();
|
||||
assert_eq!(
|
||||
decode_data_uri(&wire["links"]["1"]["href"], "text/plain"),
|
||||
b"payload-42"
|
||||
);
|
||||
assert_eq!(wire["links"]["1"]["contentType"], Value::from("text/plain"));
|
||||
assert_eq!(wire["links"]["1"]["rel"], Value::from("enclosure"));
|
||||
assert!(wire["links"]["1"].get("uri").is_none());
|
||||
assert_no_blob_id(&wire);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn contact_card_media_without_media_type_defaults_to_octet_stream() {
|
||||
let c = mem();
|
||||
let res = MapResolver {
|
||||
to_local: HashMap::from([((ObjectType::AddressBook, "AB".to_owned()), 1)]),
|
||||
to_target: HashMap::from([((ObjectType::AddressBook, 1), "TAB".to_owned())]),
|
||||
};
|
||||
c.execute(
|
||||
"INSERT INTO contact_cards (id,uid,address_book_ids,data)
|
||||
VALUES (1,'u-1','[1]',?1)",
|
||||
params![
|
||||
serde_json::json!({ "@type": "Card", "media": { "photo": { "@blob": 9 } } })
|
||||
.to_string()
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
let (uid, abids, data): (String, String, String) = c
|
||||
.query_row(&format!("{CONTACT_CARD_SELECT} WHERE id = 1"), [], |r| {
|
||||
Ok((r.get(1)?, r.get(2)?, r.get(3)?))
|
||||
})
|
||||
.unwrap();
|
||||
let up = FakeBlobs;
|
||||
let wire = contact_card_to_wire(&uid, &abids, &data, &res, &up).unwrap();
|
||||
assert_eq!(
|
||||
decode_data_uri(&wire["media"]["photo"]["uri"], "application/octet-stream"),
|
||||
b"payload-9"
|
||||
);
|
||||
assert_no_blob_id(&wire);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_archive_read_failure_while_inlining_is_an_archive_error_not_a_unit_failure() {
|
||||
struct BrokenBlobs;
|
||||
impl BlobBytes for BrokenBlobs {
|
||||
fn bytes(&self, _local_id: i64) -> Result<Vec<u8>, JmapError> {
|
||||
Err(JmapError::Sqlite(rusqlite::Error::QueryReturnedNoRows))
|
||||
}
|
||||
}
|
||||
let res = MapResolver {
|
||||
to_local: HashMap::new(),
|
||||
to_target: HashMap::from([
|
||||
((ObjectType::AddressBook, 1), "TAB".to_owned()),
|
||||
((ObjectType::Calendar, 1), "TCAL".to_owned()),
|
||||
]),
|
||||
};
|
||||
let card = serde_json::json!({ "@type": "Card", "media": { "photo": { "@blob": 9 } } })
|
||||
.to_string();
|
||||
let event =
|
||||
serde_json::json!({ "@type": "Event", "links": { "1": { "@blob": 9 } } }).to_string();
|
||||
let failures = [
|
||||
contact_card_to_wire("u-1", "[1]", &card, &res, &BrokenBlobs).expect_err("card"),
|
||||
calendar_event_to_wire("[1]", false, false, &event, &res, &BrokenBlobs)
|
||||
.expect_err("event"),
|
||||
];
|
||||
for err in failures {
|
||||
assert!(matches!(err, JmapError::Sqlite(_)), "{err:?}");
|
||||
let mapped = crate::error::Error::from(err);
|
||||
assert!(mapped.aborts_run(), "{mapped} must abort the run");
|
||||
assert_eq!(mapped.exit_code(), 7);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1103,8 +1244,8 @@ mod tests {
|
||||
assert!(stored.get("calendarIds").is_none());
|
||||
assert_eq!(stored["title"], Value::from("Sprint"));
|
||||
|
||||
let mut up = FakeBlobs;
|
||||
let wire = calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, &res, &mut up).unwrap();
|
||||
let up = FakeBlobs;
|
||||
let wire = calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, &res, &up).unwrap();
|
||||
assert_eq!(wire["calendarIds"]["TCAL"], Value::Bool(true));
|
||||
assert_eq!(wire["isDraft"], Value::Bool(true));
|
||||
assert_eq!(wire["title"], Value::from("Sprint"));
|
||||
|
||||
@@ -9,4 +9,4 @@ pub mod keywords;
|
||||
pub mod messages;
|
||||
pub mod tree;
|
||||
|
||||
pub use coordinator::{MaildirImportConfig, run};
|
||||
pub use coordinator::{MaildirImportConfig, run, run_reporting};
|
||||
|
||||
@@ -15,7 +15,7 @@ use crate::db;
|
||||
use crate::db::sources::SourceKey;
|
||||
use crate::error::Error;
|
||||
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
||||
use crate::sync::{CommonConfig, Summary, TypeCounts};
|
||||
use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts};
|
||||
|
||||
use super::messages;
|
||||
use super::tree;
|
||||
@@ -32,6 +32,20 @@ pub struct MaildirImportConfig {
|
||||
}
|
||||
|
||||
pub fn run(common: CommonConfig, config: MaildirImportConfig) -> Result<Summary, Error> {
|
||||
run_reporting(common, config).into_result()
|
||||
}
|
||||
|
||||
pub fn run_reporting(common: CommonConfig, config: MaildirImportConfig) -> RunOutcome {
|
||||
let mut summary = Summary::default();
|
||||
let error = run_into(common, config, &mut summary).err();
|
||||
RunOutcome { summary, error }
|
||||
}
|
||||
|
||||
fn run_into(
|
||||
common: CommonConfig,
|
||||
config: MaildirImportConfig,
|
||||
summary: &mut Summary,
|
||||
) -> Result<(), Error> {
|
||||
let logger = common.logger;
|
||||
if common.threads > 1 {
|
||||
log_at(
|
||||
@@ -121,8 +135,8 @@ pub fn run(common: CommonConfig, config: MaildirImportConfig) -> Result<Summary,
|
||||
);
|
||||
|
||||
if common.dry_run {
|
||||
let summary = build_dry_run_summary(&conn, source_id, &resolved, logger)?;
|
||||
return Ok(summary);
|
||||
*summary = build_dry_run_summary(&conn, source_id, &resolved, logger)?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut mailbox_counts = TypeCounts::default();
|
||||
@@ -174,6 +188,14 @@ pub fn run(common: CommonConfig, config: MaildirImportConfig) -> Result<Summary,
|
||||
&mut email_counts,
|
||||
logger,
|
||||
) {
|
||||
if e.aborts_run() {
|
||||
*summary = Summary {
|
||||
per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)],
|
||||
retries_observed: 0,
|
||||
retry_after_sleeps: 0,
|
||||
};
|
||||
return Err(e);
|
||||
}
|
||||
log_at(
|
||||
logger,
|
||||
LEVEL_DEFAULT,
|
||||
@@ -207,11 +229,12 @@ pub fn run(common: CommonConfig, config: MaildirImportConfig) -> Result<Summary,
|
||||
),
|
||||
);
|
||||
|
||||
Ok(Summary {
|
||||
*summary = Summary {
|
||||
per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)],
|
||||
retries_observed: 0,
|
||||
retry_after_sleeps: 0,
|
||||
})
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct RunFlags {
|
||||
|
||||
@@ -13,4 +13,4 @@ pub mod mbox;
|
||||
pub mod tree;
|
||||
pub mod walk;
|
||||
|
||||
pub use coordinator::{TakeoutImportConfig, run};
|
||||
pub use coordinator::{TakeoutImportConfig, run, run_reporting};
|
||||
|
||||
@@ -15,7 +15,7 @@ use crate::db::sources::SourceKey;
|
||||
use crate::db::takeout_ids;
|
||||
use crate::error::Error;
|
||||
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
||||
use crate::sync::{CommonConfig, Summary, TypeCounts};
|
||||
use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts};
|
||||
|
||||
use super::calendar;
|
||||
use super::contacts;
|
||||
@@ -31,6 +31,20 @@ pub struct TakeoutImportConfig {
|
||||
}
|
||||
|
||||
pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary, Error> {
|
||||
run_reporting(common, config).into_result()
|
||||
}
|
||||
|
||||
pub fn run_reporting(common: CommonConfig, config: TakeoutImportConfig) -> RunOutcome {
|
||||
let mut summary = Summary::default();
|
||||
let error = run_into(common, config, &mut summary).err();
|
||||
RunOutcome { summary, error }
|
||||
}
|
||||
|
||||
fn run_into(
|
||||
common: CommonConfig,
|
||||
config: TakeoutImportConfig,
|
||||
summary: &mut Summary,
|
||||
) -> Result<(), Error> {
|
||||
let logger = common.logger;
|
||||
if common.threads > 1 && logger.enabled(LEVEL_PROGRESS) {
|
||||
eprintln!("takeout importer is single-threaded; --threads value will be ignored");
|
||||
@@ -90,7 +104,8 @@ pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary,
|
||||
};
|
||||
|
||||
if common.dry_run {
|
||||
return Ok(dry_run_summary(&conn, source_id, &walk_result, logger));
|
||||
*summary = dry_run_summary(&conn, source_id, &walk_result, logger);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let options = MappingOptions {
|
||||
@@ -107,7 +122,8 @@ pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary,
|
||||
let mut mailbox_cache: HashMap<String, i64> =
|
||||
takeout_ids::all_for_type(&conn, source_id, takeout_ids::MAILBOX)?;
|
||||
|
||||
process_mbox_files(
|
||||
let mut aborted: Option<Error> = None;
|
||||
if let Err(e) = process_mbox_files(
|
||||
&mut conn,
|
||||
source_id,
|
||||
&walk_result,
|
||||
@@ -116,27 +132,38 @@ pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary,
|
||||
&mut mailbox_counts,
|
||||
&mut email_counts,
|
||||
logger,
|
||||
)?;
|
||||
) {
|
||||
aborted = Some(e);
|
||||
}
|
||||
|
||||
process_ics_files(
|
||||
if aborted.is_none()
|
||||
&& let Err(e) = process_ics_files(
|
||||
&mut conn,
|
||||
source_id,
|
||||
&walk_result,
|
||||
&mut calendar_counts,
|
||||
&mut calendar_event_counts,
|
||||
logger,
|
||||
)?;
|
||||
)
|
||||
{
|
||||
aborted = Some(e);
|
||||
}
|
||||
|
||||
process_vcf_files(
|
||||
if aborted.is_none()
|
||||
&& let Err(e) = process_vcf_files(
|
||||
&mut conn,
|
||||
source_id,
|
||||
&walk_result,
|
||||
&mut book_counts,
|
||||
&mut card_counts,
|
||||
logger,
|
||||
)?;
|
||||
)
|
||||
{
|
||||
aborted = Some(e);
|
||||
}
|
||||
|
||||
let no_failures = mailbox_counts.failed == 0
|
||||
let no_failures = aborted.is_none()
|
||||
&& mailbox_counts.failed == 0
|
||||
&& email_counts.failed == 0
|
||||
&& calendar_counts.failed == 0
|
||||
&& calendar_event_counts.failed == 0
|
||||
@@ -171,7 +198,7 @@ pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary,
|
||||
);
|
||||
}
|
||||
|
||||
Ok(Summary {
|
||||
*summary = Summary {
|
||||
per_type: vec![
|
||||
("mailbox", mailbox_counts),
|
||||
("email", email_counts),
|
||||
@@ -182,7 +209,11 @@ pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary,
|
||||
],
|
||||
retries_observed: 0,
|
||||
retry_after_sleeps: 0,
|
||||
})
|
||||
};
|
||||
match aborted {
|
||||
Some(e) => Err(e),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -212,6 +243,7 @@ fn process_mbox_files(
|
||||
};
|
||||
match mail::process_file(conn, &file.path, ctx, mailbox_counts, email_counts, logger) {
|
||||
Ok(()) => {}
|
||||
Err(e) if e.aborts_run() => return Err(e),
|
||||
Err(e) => {
|
||||
logger.warn(&format!("mbox {:?}: {e}", file.path));
|
||||
email_counts.failed += 1;
|
||||
@@ -244,6 +276,9 @@ fn process_ics_files(
|
||||
event_counts,
|
||||
logger,
|
||||
) {
|
||||
if e.aborts_run() {
|
||||
return Err(e);
|
||||
}
|
||||
logger.warn(&format!("ics {:?}: {e}", file.path));
|
||||
event_counts.failed += 1;
|
||||
}
|
||||
@@ -274,6 +309,9 @@ fn process_vcf_files(
|
||||
card_counts,
|
||||
logger,
|
||||
) {
|
||||
if e.aborts_run() {
|
||||
return Err(e);
|
||||
}
|
||||
logger.warn(&format!("vcf {:?}: {e}", file.path));
|
||||
card_counts.failed += 1;
|
||||
}
|
||||
|
||||
@@ -94,6 +94,34 @@ impl Summary {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct RunOutcome {
|
||||
pub summary: Summary,
|
||||
pub error: Option<Error>,
|
||||
}
|
||||
|
||||
impl RunOutcome {
|
||||
pub fn from_result(result: Result<Summary, Error>) -> RunOutcome {
|
||||
match result {
|
||||
Ok(summary) => RunOutcome {
|
||||
summary,
|
||||
error: None,
|
||||
},
|
||||
Err(error) => RunOutcome {
|
||||
summary: Summary::default(),
|
||||
error: Some(error),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn into_result(self) -> Result<Summary, Error> {
|
||||
match self.error {
|
||||
Some(error) => Err(error),
|
||||
None => Ok(self.summary),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Context {
|
||||
pub conn: Connection,
|
||||
pub client: HttpClient,
|
||||
@@ -119,3 +147,28 @@ impl Context {
|
||||
self.common.dry_run
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn an_aborted_run_keeps_its_error_and_its_partial_counts() {
|
||||
let mut summary = Summary::default();
|
||||
summary.per_type.push(("Mailbox", TypeCounts::default()));
|
||||
let outcome = RunOutcome {
|
||||
summary,
|
||||
error: Some(Error::Connection("http status 404".to_owned())),
|
||||
};
|
||||
assert_eq!(outcome.summary.per_type.len(), 1);
|
||||
let err = outcome.into_result().expect_err("aborted");
|
||||
assert_eq!(err.exit_code(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_result_without_a_native_outcome_reports_nothing_on_abort() {
|
||||
let outcome = RunOutcome::from_result(Err(Error::Partial("one object".to_owned())));
|
||||
assert!(outcome.summary.per_type.is_empty());
|
||||
assert!(outcome.error.is_some());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,7 +151,10 @@ fn sweep_abandoned_containers() {
|
||||
return;
|
||||
}
|
||||
eprintln!("removing {} abandoned test container(s)", ids.len());
|
||||
let _ = Command::new("docker").args(["rm", "-f"]).args(&ids).output();
|
||||
let _ = Command::new("docker")
|
||||
.args(["rm", "-f"])
|
||||
.args(&ids)
|
||||
.output();
|
||||
}
|
||||
|
||||
fn pick_free_port() -> ContainerResult<u16> {
|
||||
|
||||
@@ -1595,3 +1595,234 @@ fn mixed_source_archive_allows_distinct_kinds_against_same_url() {
|
||||
let no_conflict = sources::conflicting_source(&conn, "caldav", "https://x", "alice").unwrap();
|
||||
assert!(no_conflict.is_none());
|
||||
}
|
||||
|
||||
fn dav_archive() -> std::path::PathBuf {
|
||||
static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
|
||||
let n = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"vandelay-mockdav-{}-{:?}-{n}.sqlite",
|
||||
std::process::id(),
|
||||
std::thread::current().id(),
|
||||
));
|
||||
let _ = std::fs::remove_file(&p);
|
||||
p
|
||||
}
|
||||
|
||||
fn caldav_import_config(url: &str) -> vandelay::sync::import_dav::DavImportConfig {
|
||||
vandelay::sync::import_dav::DavImportConfig {
|
||||
kind: vandelay::sync::import_dav::DavKindArg::Caldav,
|
||||
url: url.to_owned(),
|
||||
auth: vandelay::sync::import_dav::DavAuth::Basic {
|
||||
user: "u".to_owned(),
|
||||
password: "p".to_owned(),
|
||||
},
|
||||
allow_cleartext: true,
|
||||
dav_connections: 1,
|
||||
multiget_batch: 8,
|
||||
allow_source_change: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn dav_common(archive: &std::path::Path) -> vandelay::sync::CommonConfig {
|
||||
vandelay::sync::CommonConfig {
|
||||
archive: archive.to_path_buf(),
|
||||
threads: 1,
|
||||
dry_run: false,
|
||||
max_retries: 0,
|
||||
allow_invalid_certs: false,
|
||||
logger: vandelay::logging::Logger::from_flags(true, 0),
|
||||
}
|
||||
}
|
||||
|
||||
fn row_count(archive: &std::path::Path, table: &str) -> i64 {
|
||||
let conn = rusqlite::Connection::open(archive).expect("open archive");
|
||||
conn.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0))
|
||||
.expect("count")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn forbidden_collection_is_a_per_unit_failure_and_the_others_still_import() {
|
||||
let mut server = mockito::Server::new();
|
||||
let url = server.url();
|
||||
|
||||
let home_set = format!(
|
||||
r#"<?xml version="1.0"?>
|
||||
<d:multistatus xmlns:d="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav">
|
||||
<d:response>
|
||||
<d:href>{url}/dav/cal/u/work/</d:href>
|
||||
<d:propstat>
|
||||
<d:prop>
|
||||
<d:resourcetype><d:collection/><c:calendar/></d:resourcetype>
|
||||
<d:displayname>Work</d:displayname>
|
||||
</d:prop>
|
||||
<d:status>HTTP/1.1 200 OK</d:status>
|
||||
</d:propstat>
|
||||
</d:response>
|
||||
<d:response>
|
||||
<d:href>{url}/dav/cal/u/shared/</d:href>
|
||||
<d:propstat>
|
||||
<d:prop>
|
||||
<d:resourcetype><d:collection/><c:calendar/></d:resourcetype>
|
||||
<d:displayname>Shared</d:displayname>
|
||||
</d:prop>
|
||||
<d:status>HTTP/1.1 200 OK</d:status>
|
||||
</d:propstat>
|
||||
</d:response>
|
||||
</d:multistatus>"#
|
||||
);
|
||||
let _discovery = multistatus_response(&mut server, "PROPFIND", "/dav/cal/u/", &home_set);
|
||||
|
||||
let work_items = format!(
|
||||
r#"<?xml version="1.0"?>
|
||||
<d:multistatus xmlns:d="DAV:">
|
||||
<d:response>
|
||||
<d:href>{url}/dav/cal/u/work/e1.ics</d:href>
|
||||
<d:propstat>
|
||||
<d:prop><d:resourcetype/><d:getetag>"v1"</d:getetag></d:prop>
|
||||
<d:status>HTTP/1.1 200 OK</d:status>
|
||||
</d:propstat>
|
||||
</d:response>
|
||||
</d:multistatus>"#
|
||||
);
|
||||
let _work_enumeration =
|
||||
multistatus_response(&mut server, "PROPFIND", "/dav/cal/u/work/", &work_items);
|
||||
|
||||
let work_data = format!(
|
||||
r#"<?xml version="1.0"?>
|
||||
<d:multistatus xmlns:d="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav">
|
||||
<d:response>
|
||||
<d:href>{url}/dav/cal/u/work/e1.ics</d:href>
|
||||
<d:propstat>
|
||||
<d:prop>
|
||||
<d:getetag>"v1"</d:getetag>
|
||||
<c:calendar-data>BEGIN:VCALENDAR
|
||||
VERSION:2.0
|
||||
PRODID:-//Test//EN
|
||||
BEGIN:VEVENT
|
||||
UID:e1@example.com
|
||||
DTSTAMP:20260101T000000Z
|
||||
DTSTART:20260101T090000Z
|
||||
DTEND:20260101T100000Z
|
||||
SUMMARY:Standup
|
||||
END:VEVENT
|
||||
END:VCALENDAR
|
||||
</c:calendar-data>
|
||||
</d:prop>
|
||||
<d:status>HTTP/1.1 200 OK</d:status>
|
||||
</d:propstat>
|
||||
</d:response>
|
||||
</d:multistatus>"#
|
||||
);
|
||||
let _work_multiget =
|
||||
multistatus_response(&mut server, "REPORT", "/dav/cal/u/work/", &work_data);
|
||||
|
||||
let forbidden = server
|
||||
.mock("PROPFIND", "/dav/cal/u/shared/")
|
||||
.with_status(403)
|
||||
.with_header("content-type", "text/plain")
|
||||
.with_body("you may not read this calendar")
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
|
||||
let archive = dav_archive();
|
||||
let summary = vandelay::sync::import_dav::run(
|
||||
dav_common(&archive),
|
||||
caldav_import_config(&format!("{url}/dav/cal/u/")),
|
||||
)
|
||||
.expect("a forbidden collection must not abort the run");
|
||||
|
||||
forbidden.assert();
|
||||
|
||||
let events = summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(t, _)| *t == "calendarevent")
|
||||
.map(|(_, c)| c.clone())
|
||||
.expect("calendarevent counts");
|
||||
assert_eq!(events.failed, 1, "the forbidden calendar counts as failed");
|
||||
assert_eq!(events.fetched, 1, "the readable calendar still imported");
|
||||
assert!(
|
||||
summary.any_failed(),
|
||||
"a per-unit failure ends the run at exit 5"
|
||||
);
|
||||
|
||||
assert_eq!(row_count(&archive, "calendars"), 2);
|
||||
assert_eq!(row_count(&archive, "calendar_events"), 1);
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credentials_rejected_during_discovery_still_aborts_the_run() {
|
||||
let mut server = mockito::Server::new();
|
||||
let url = server.url();
|
||||
|
||||
let _rejected = server
|
||||
.mock("PROPFIND", mockito::Matcher::Any)
|
||||
.with_status(401)
|
||||
.with_header("content-type", "text/plain")
|
||||
.with_body("bad password")
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
|
||||
let archive = dav_archive();
|
||||
let err = vandelay::sync::import_dav::run(
|
||||
dav_common(&archive),
|
||||
caldav_import_config(&format!("{url}/dav/cal/u/")),
|
||||
)
|
||||
.expect_err("credentials rejected for the principal aborts the run");
|
||||
|
||||
assert!(
|
||||
matches!(err, vandelay::error::Error::Connection(_)),
|
||||
"discovery-level auth rejection is a whole-run failure, got {err:?}"
|
||||
);
|
||||
assert_eq!(err.exit_code(), 2);
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dav_import_reports_both_phases_when_every_collection_fails() {
|
||||
let mut server = mockito::Server::new();
|
||||
let url = server.url();
|
||||
|
||||
let home_set = format!(
|
||||
r#"<?xml version="1.0"?>
|
||||
<d:multistatus xmlns:d="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav">
|
||||
<d:response>
|
||||
<d:href>{url}/dav/cal/u/work/</d:href>
|
||||
<d:propstat>
|
||||
<d:prop>
|
||||
<d:resourcetype><d:collection/><c:calendar/></d:resourcetype>
|
||||
<d:displayname>Work</d:displayname>
|
||||
</d:prop>
|
||||
<d:status>HTTP/1.1 200 OK</d:status>
|
||||
</d:propstat>
|
||||
</d:response>
|
||||
</d:multistatus>"#
|
||||
);
|
||||
let _discovery = multistatus_response(&mut server, "PROPFIND", "/dav/cal/u/", &home_set);
|
||||
let _forbidden = server
|
||||
.mock("PROPFIND", "/dav/cal/u/work/")
|
||||
.with_status(403)
|
||||
.with_body("nope")
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
|
||||
let archive = dav_archive();
|
||||
let outcome = vandelay::sync::import_dav::run_reporting(
|
||||
dav_common(&archive),
|
||||
caldav_import_config(&format!("{url}/dav/cal/u/")),
|
||||
);
|
||||
assert!(outcome.error.is_none());
|
||||
assert_eq!(
|
||||
outcome
|
||||
.summary
|
||||
.per_type
|
||||
.iter()
|
||||
.map(|(t, _)| *t)
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["calendar", "calendarevent"],
|
||||
"both phases are reported even when every collection failed"
|
||||
);
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
+252
-73
@@ -20,6 +20,7 @@ use vandelay::exchange_graph::oauth::{
|
||||
};
|
||||
use vandelay::exchange_graph::recurrence::convert_patterned_recurrence;
|
||||
use vandelay::exchange_graph::retry::{HttpClass, classify_http_status};
|
||||
use vandelay::exchange_graph::types::Surfaces;
|
||||
use vandelay::jmap::http::RetryPolicy;
|
||||
|
||||
static INIT: Once = Once::new();
|
||||
@@ -584,7 +585,7 @@ fn integration_dry_run_against_mock_server_lists_three_surfaces() {
|
||||
api_base: base.clone(),
|
||||
user_target: None,
|
||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||
objects: None,
|
||||
surfaces: Surfaces::ALL,
|
||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||
graph_connections: 2,
|
||||
top: 100,
|
||||
@@ -693,10 +694,7 @@ fn integration_full_run_mail_only_imports_mime_via_value() {
|
||||
api_base: server.url(),
|
||||
user_target: None,
|
||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||
objects: Some(vec![
|
||||
vandelay::types::ObjectType::Mailbox,
|
||||
vandelay::types::ObjectType::Email,
|
||||
]),
|
||||
surfaces: surfaces("mail"),
|
||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||
graph_connections: 2,
|
||||
top: 100,
|
||||
@@ -792,10 +790,7 @@ fn integration_duplicate_message_id_does_not_abort_run() {
|
||||
api_base: server.url(),
|
||||
user_target: None,
|
||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||
objects: Some(vec![
|
||||
vandelay::types::ObjectType::Mailbox,
|
||||
vandelay::types::ObjectType::Email,
|
||||
]),
|
||||
surfaces: surfaces("mail"),
|
||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||
graph_connections: 2,
|
||||
top: 100,
|
||||
@@ -884,10 +879,7 @@ fn integration_full_run_is_convergent_on_second_invocation() {
|
||||
api_base: api_base.clone(),
|
||||
user_target: None,
|
||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||
objects: Some(vec![
|
||||
vandelay::types::ObjectType::Mailbox,
|
||||
vandelay::types::ObjectType::Email,
|
||||
]),
|
||||
surfaces: surfaces("mail"),
|
||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||
graph_connections: 2,
|
||||
top: 100,
|
||||
@@ -966,7 +958,7 @@ fn source_change_protection_refuses_a_different_account() {
|
||||
api_base: server.url(),
|
||||
user_target: None,
|
||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||
objects: None,
|
||||
surfaces: Surfaces::ALL,
|
||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||
graph_connections: 2,
|
||||
top: 100,
|
||||
@@ -1016,10 +1008,14 @@ fn make_common(archive: std::path::PathBuf) -> vandelay::sync::CommonConfig {
|
||||
}
|
||||
}
|
||||
|
||||
fn surfaces(list: &str) -> vandelay::exchange_graph::types::Surfaces {
|
||||
vandelay::exchange_graph::types::Surfaces::parse_list(list).unwrap()
|
||||
}
|
||||
|
||||
fn make_config(
|
||||
api_base: String,
|
||||
user_target: Option<String>,
|
||||
objects: Option<Vec<vandelay::types::ObjectType>>,
|
||||
surfaces: vandelay::exchange_graph::types::Surfaces,
|
||||
) -> vandelay::sync::import_exchange_graph::GraphImportConfig {
|
||||
vandelay::sync::import_exchange_graph::GraphImportConfig {
|
||||
auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired {
|
||||
@@ -1028,7 +1024,7 @@ fn make_config(
|
||||
api_base,
|
||||
user_target,
|
||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||
objects,
|
||||
surfaces,
|
||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||
graph_connections: 2,
|
||||
top: 100,
|
||||
@@ -1094,7 +1090,7 @@ fn users_upn_routes_through_users_segment_not_me() {
|
||||
};
|
||||
let summary = vandelay::sync::import_exchange_graph::run(
|
||||
common,
|
||||
make_config(base, Some("[email protected]".to_owned()), None),
|
||||
make_config(base, Some("[email protected]".to_owned()), Surfaces::ALL),
|
||||
)
|
||||
.unwrap();
|
||||
let mailbox = summary
|
||||
@@ -1193,14 +1189,7 @@ fn series_master_with_exception_merges_into_recurrence_overrides() {
|
||||
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
||||
let summary = vandelay::sync::import_exchange_graph::run(
|
||||
make_common(archive.clone()),
|
||||
make_config(
|
||||
base,
|
||||
None,
|
||||
Some(vec![
|
||||
vandelay::types::ObjectType::Calendar,
|
||||
vandelay::types::ObjectType::CalendarEvent,
|
||||
]),
|
||||
),
|
||||
make_config(base, None, surfaces("calendar")),
|
||||
)
|
||||
.unwrap();
|
||||
let events = summary
|
||||
@@ -1227,6 +1216,26 @@ fn series_master_with_exception_merges_into_recurrence_overrides() {
|
||||
);
|
||||
let override_data = &overrides["2026-05-11T15:00:00"];
|
||||
assert_eq!(override_data["title"], "Moved sync");
|
||||
assert_eq!(override_data["start"], "2026-05-11T16:00:00");
|
||||
for ignored in [
|
||||
"@type",
|
||||
"uid",
|
||||
"method",
|
||||
"organizerCalendarAddress",
|
||||
"privacy",
|
||||
"prodId",
|
||||
"recurrenceId",
|
||||
"recurrenceIdTimeZone",
|
||||
"sentBy",
|
||||
"recurrenceRule",
|
||||
"recurrenceOverrides",
|
||||
"relatedTo",
|
||||
] {
|
||||
assert!(
|
||||
override_data.get(ignored).is_none(),
|
||||
"{ignored} must not appear in a PatchObject (jscalendarbis 3.3.4); got {override_data}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1285,14 +1294,7 @@ fn occurrence_event_is_skipped_no_row_no_id_mapping() {
|
||||
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
||||
let summary = vandelay::sync::import_exchange_graph::run(
|
||||
make_common(archive.clone()),
|
||||
make_config(
|
||||
base,
|
||||
None,
|
||||
Some(vec![
|
||||
vandelay::types::ObjectType::Calendar,
|
||||
vandelay::types::ObjectType::CalendarEvent,
|
||||
]),
|
||||
),
|
||||
make_config(base, None, surfaces("calendar")),
|
||||
)
|
||||
.unwrap();
|
||||
let events = summary
|
||||
@@ -1372,14 +1374,7 @@ fn hidden_mail_folder_has_is_subscribed_zero() {
|
||||
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
||||
let _ = vandelay::sync::import_exchange_graph::run(
|
||||
make_common(archive.clone()),
|
||||
make_config(
|
||||
base,
|
||||
None,
|
||||
Some(vec![
|
||||
vandelay::types::ObjectType::Mailbox,
|
||||
vandelay::types::ObjectType::Email,
|
||||
]),
|
||||
),
|
||||
make_config(base, None, surfaces("mail")),
|
||||
)
|
||||
.unwrap();
|
||||
let conn = vandelay::db::init::open(&archive).unwrap();
|
||||
@@ -1473,14 +1468,7 @@ fn well_known_folder_probes_assign_jmap_roles() {
|
||||
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
||||
let _ = vandelay::sync::import_exchange_graph::run(
|
||||
make_common(archive.clone()),
|
||||
make_config(
|
||||
base,
|
||||
None,
|
||||
Some(vec![
|
||||
vandelay::types::ObjectType::Mailbox,
|
||||
vandelay::types::ObjectType::Email,
|
||||
]),
|
||||
),
|
||||
make_config(base, None, surfaces("mail")),
|
||||
)
|
||||
.unwrap();
|
||||
let conn = vandelay::db::init::open(&archive).unwrap();
|
||||
@@ -1557,14 +1545,7 @@ SGVsbG8=\r\n--X--\r\n";
|
||||
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
||||
let summary = vandelay::sync::import_exchange_graph::run(
|
||||
make_common(archive),
|
||||
make_config(
|
||||
base,
|
||||
None,
|
||||
Some(vec![
|
||||
vandelay::types::ObjectType::Mailbox,
|
||||
vandelay::types::ObjectType::Email,
|
||||
]),
|
||||
),
|
||||
make_config(base, None, surfaces("mail")),
|
||||
)
|
||||
.unwrap();
|
||||
let emails = summary
|
||||
@@ -1652,14 +1633,7 @@ fn event_get_carries_outlook_timezone_and_body_content_type_prefer() {
|
||||
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
||||
let _ = vandelay::sync::import_exchange_graph::run(
|
||||
make_common(archive),
|
||||
make_config(
|
||||
base,
|
||||
None,
|
||||
Some(vec![
|
||||
vandelay::types::ObjectType::Calendar,
|
||||
vandelay::types::ObjectType::CalendarEvent,
|
||||
]),
|
||||
),
|
||||
make_config(base, None, surfaces("calendar")),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
@@ -1758,7 +1732,7 @@ fn full_run_records_graph_id_in_sync_id_exchange_graph_with_padding() {
|
||||
api_base: server.url(),
|
||||
user_target: None,
|
||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||
objects: Some(vec![vandelay::types::ObjectType::Mailbox]),
|
||||
surfaces: surfaces("mail"),
|
||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||
graph_connections: 2,
|
||||
top: 100,
|
||||
@@ -1828,7 +1802,7 @@ fn archive_mailbox_kind_encodes_synthetic_suffix_in_account_id() {
|
||||
api_base: server.url(),
|
||||
user_target: None,
|
||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Archive,
|
||||
objects: Some(vec![vandelay::types::ObjectType::Mailbox]),
|
||||
surfaces: surfaces("mail"),
|
||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||
graph_connections: 2,
|
||||
top: 100,
|
||||
@@ -1893,7 +1867,7 @@ fn allow_source_change_permits_overwriting_a_different_account() {
|
||||
api_base: server.url(),
|
||||
user_target: None,
|
||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||
objects: Some(vec![vandelay::types::ObjectType::Mailbox]),
|
||||
surfaces: surfaces("mail"),
|
||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||
graph_connections: 2,
|
||||
top: 100,
|
||||
@@ -1974,7 +1948,7 @@ fn dry_run_makes_no_per_item_get_and_no_sqlite_writes() {
|
||||
api_base: server.url(),
|
||||
user_target: None,
|
||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||
objects: None,
|
||||
surfaces: Surfaces::ALL,
|
||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||
graph_connections: 2,
|
||||
top: 100,
|
||||
@@ -2142,10 +2116,7 @@ fn folder_enumeration_failure_skips_vanished_deletion() {
|
||||
api_base: server.url(),
|
||||
user_target: None,
|
||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||
objects: Some(vec![
|
||||
vandelay::types::ObjectType::Mailbox,
|
||||
vandelay::types::ObjectType::Email,
|
||||
]),
|
||||
surfaces: surfaces("mail"),
|
||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||
graph_connections: 2,
|
||||
top: 100,
|
||||
@@ -2163,3 +2134,211 @@ fn folder_enumeration_failure_skips_vanished_deletion() {
|
||||
"transient folder-enumeration failure must NOT delete the locally-known message"
|
||||
);
|
||||
}
|
||||
|
||||
fn row_count(conn: &rusqlite::Connection, table: &str) -> i64 {
|
||||
conn.query_row(&format!("SELECT count(*) FROM {table}"), [], |row| {
|
||||
row.get(0)
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn stub_contacts_surface(server: &mut Server) {
|
||||
server
|
||||
.mock("GET", "/me/contactFolders?$top=100")
|
||||
.with_status(200)
|
||||
.with_header("content-type", "application/json")
|
||||
.with_body(r#"{"value":[{"id":"CON1","displayName":"Contacts"}]}"#)
|
||||
.expect_at_least(0)
|
||||
.create();
|
||||
server
|
||||
.mock("GET", "/me/contactFolders/CON1/childFolders?$top=100")
|
||||
.with_status(200)
|
||||
.with_header("content-type", "application/json")
|
||||
.with_body(r#"{"value":[]}"#)
|
||||
.expect_at_least(0)
|
||||
.create();
|
||||
server
|
||||
.mock(
|
||||
"GET",
|
||||
"/me/contactFolders/CON1/contacts?$top=100&$select=id",
|
||||
)
|
||||
.with_status(200)
|
||||
.with_header("content-type", "application/json")
|
||||
.with_body(r#"{"value":[{"id":"CT1"}]}"#)
|
||||
.expect_at_least(0)
|
||||
.create();
|
||||
server
|
||||
.mock("GET", "/me/contacts/CT1")
|
||||
.with_status(200)
|
||||
.with_header("content-type", "application/json")
|
||||
.with_body(
|
||||
r#"{"id":"CT1","displayName":"Alice Liddell","givenName":"Alice",
|
||||
"surname":"Liddell","emailAddresses":[{"name":"Alice","address":"[email protected]"}]}"#,
|
||||
)
|
||||
.expect_at_least(0)
|
||||
.create();
|
||||
}
|
||||
|
||||
fn stub_mail_surface(server: &mut Server) {
|
||||
server
|
||||
.mock("GET", "/me/mailFolders?$top=100&includeHiddenFolders=true")
|
||||
.with_status(200)
|
||||
.with_header("content-type", "application/json")
|
||||
.with_body(r#"{"value":[{"id":"FMAIL","displayName":"Inbox","isHidden":false}]}"#)
|
||||
.expect_at_least(0)
|
||||
.create();
|
||||
server
|
||||
.mock(
|
||||
"GET",
|
||||
"/me/mailFolders/FMAIL/childFolders?$top=100&includeHiddenFolders=true",
|
||||
)
|
||||
.with_status(200)
|
||||
.with_header("content-type", "application/json")
|
||||
.with_body(r#"{"value":[]}"#)
|
||||
.expect_at_least(0)
|
||||
.create();
|
||||
stub_well_known_folders(server, "FMAIL");
|
||||
server
|
||||
.mock("GET", "/me/mailFolders/FMAIL/messages?$top=100&$select=id")
|
||||
.with_status(200)
|
||||
.with_header("content-type", "application/json")
|
||||
.with_body(r#"{"value":[{"id":"MSG-S"}]}"#)
|
||||
.expect_at_least(0)
|
||||
.create();
|
||||
server
|
||||
.mock("GET", "/me/messages/MSG-S/$value")
|
||||
.with_status(200)
|
||||
.with_header("content-type", "text/plain")
|
||||
.with_body(
|
||||
"From: a@x\r\nTo: b@x\r\nSubject: surface\r\n\
|
||||
Date: Tue, 27 May 2026 10:00:00 +0000\r\nMessage-ID: <surface@x>\r\n\r\nbody",
|
||||
)
|
||||
.expect_at_least(0)
|
||||
.create();
|
||||
}
|
||||
|
||||
fn stub_calendar_surface(server: &mut Server) {
|
||||
server
|
||||
.mock("GET", "/me/mailboxSettings?$select=timeZone")
|
||||
.with_status(200)
|
||||
.with_header("content-type", "application/json")
|
||||
.with_body(r#"{"timeZone":"UTC"}"#)
|
||||
.expect_at_least(0)
|
||||
.create();
|
||||
server
|
||||
.mock("GET", "/me/calendars?$top=100")
|
||||
.with_status(200)
|
||||
.with_header("content-type", "application/json")
|
||||
.with_body(r#"{"value":[{"id":"CAL1","name":"Calendar","isDefaultCalendar":true}]}"#)
|
||||
.expect_at_least(0)
|
||||
.create();
|
||||
server
|
||||
.mock(
|
||||
"GET",
|
||||
"/me/calendars/CAL1/events?$top=100&$select=id,type,seriesMasterId",
|
||||
)
|
||||
.with_status(200)
|
||||
.with_header("content-type", "application/json")
|
||||
.with_body(r#"{"value":[{"id":"EV1","type":"singleInstance","iCalUId":"uid-ev1"}]}"#)
|
||||
.expect_at_least(0)
|
||||
.create();
|
||||
server
|
||||
.mock("GET", "/me/events/EV1")
|
||||
.with_status(200)
|
||||
.with_header("content-type", "application/json")
|
||||
.with_body(
|
||||
r#"{"id":"EV1","iCalUId":"uid-ev1","type":"singleInstance","subject":"Standup",
|
||||
"start":{"dateTime":"2026-05-04T15:00:00.0000000","timeZone":"UTC"},
|
||||
"end":{"dateTime":"2026-05-04T16:00:00.0000000","timeZone":"UTC"}}"#,
|
||||
)
|
||||
.expect_at_least(0)
|
||||
.create();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn contacts_surface_imports_only_address_books_and_cards() {
|
||||
let mut server = Server::new();
|
||||
server
|
||||
.mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned()))
|
||||
.with_status(200)
|
||||
.with_header("content-type", "application/json")
|
||||
.with_body(r#"{"id":"uid-surface-c","userPrincipalName":"[email protected]"}"#)
|
||||
.create();
|
||||
stub_contacts_surface(&mut server);
|
||||
stub_calendar_surface(&mut server);
|
||||
stub_mail_surface(&mut server);
|
||||
let no_mail = server
|
||||
.mock("GET", Matcher::Regex(r"^/me/mailFolders\?".to_owned()))
|
||||
.with_status(500)
|
||||
.with_body("MUST NOT BE CALLED")
|
||||
.expect(0)
|
||||
.create();
|
||||
let no_calendars = server
|
||||
.mock("GET", Matcher::Regex(r"^/me/calendars\?".to_owned()))
|
||||
.with_status(500)
|
||||
.with_body("MUST NOT BE CALLED")
|
||||
.expect(0)
|
||||
.create();
|
||||
|
||||
let base = server.url();
|
||||
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
||||
vandelay::sync::import_exchange_graph::run(
|
||||
make_common(archive.clone()),
|
||||
make_config(base, None, surfaces("contacts")),
|
||||
)
|
||||
.unwrap();
|
||||
no_mail.assert();
|
||||
no_calendars.assert();
|
||||
|
||||
let conn = vandelay::db::init::open(&archive).unwrap();
|
||||
assert_eq!(row_count(&conn, "address_books"), 1);
|
||||
assert_eq!(row_count(&conn, "contact_cards"), 1);
|
||||
assert_eq!(row_count(&conn, "mailboxes"), 0);
|
||||
assert_eq!(row_count(&conn, "emails"), 0);
|
||||
assert_eq!(row_count(&conn, "calendars"), 0);
|
||||
assert_eq!(row_count(&conn, "calendar_events"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mail_surface_imports_only_mailboxes_and_emails() {
|
||||
let mut server = Server::new();
|
||||
server
|
||||
.mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned()))
|
||||
.with_status(200)
|
||||
.with_header("content-type", "application/json")
|
||||
.with_body(r#"{"id":"uid-surface-m","userPrincipalName":"[email protected]"}"#)
|
||||
.create();
|
||||
stub_mail_surface(&mut server);
|
||||
stub_calendar_surface(&mut server);
|
||||
stub_contacts_surface(&mut server);
|
||||
let no_calendars = server
|
||||
.mock("GET", Matcher::Regex(r"^/me/calendars\?".to_owned()))
|
||||
.with_status(500)
|
||||
.with_body("MUST NOT BE CALLED")
|
||||
.expect(0)
|
||||
.create();
|
||||
let no_contact_folders = server
|
||||
.mock("GET", Matcher::Regex(r"^/me/contactFolders\?".to_owned()))
|
||||
.with_status(500)
|
||||
.with_body("MUST NOT BE CALLED")
|
||||
.expect(0)
|
||||
.create();
|
||||
|
||||
let base = server.url();
|
||||
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
||||
vandelay::sync::import_exchange_graph::run(
|
||||
make_common(archive.clone()),
|
||||
make_config(base, None, surfaces("mail")),
|
||||
)
|
||||
.unwrap();
|
||||
no_calendars.assert();
|
||||
no_contact_folders.assert();
|
||||
|
||||
let conn = vandelay::db::init::open(&archive).unwrap();
|
||||
assert_eq!(row_count(&conn, "mailboxes"), 1);
|
||||
assert_eq!(row_count(&conn, "emails"), 1);
|
||||
assert_eq!(row_count(&conn, "address_books"), 0);
|
||||
assert_eq!(row_count(&conn, "contact_cards"), 0);
|
||||
assert_eq!(row_count(&conn, "calendars"), 0);
|
||||
assert_eq!(row_count(&conn, "calendar_events"), 0);
|
||||
}
|
||||
|
||||
@@ -1201,3 +1201,97 @@ fn get_state_reads_state_from_empty_get() {
|
||||
let st = get_state(&client(0), &url, "w", "Email").expect("get_state");
|
||||
assert_eq!(st.as_deref(), Some("snap-1"));
|
||||
}
|
||||
|
||||
fn archive_path() -> std::path::PathBuf {
|
||||
static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
|
||||
let n = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"vandelay-mockjmap-{}-{n}.sqlite",
|
||||
std::process::id()
|
||||
));
|
||||
let _ = std::fs::remove_file(&p);
|
||||
p
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_urls_on_a_foreign_origin_are_reported_as_a_mismatch() {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let elsewhere = "https://mail.example".to_owned();
|
||||
let _root = server.mock("GET", "/").with_status(404).create();
|
||||
let _wk = server
|
||||
.mock("GET", "/.well-known/jmap")
|
||||
.with_status(200)
|
||||
.with_header("content-type", "application/json")
|
||||
.with_body(session_json(&elsewhere))
|
||||
.create();
|
||||
|
||||
let session = Session::discover(&client(0), &base).expect("session discovered");
|
||||
let mismatches = session.origin_mismatches(&base);
|
||||
assert_eq!(mismatches.len(), 1, "one warning per distinct origin");
|
||||
assert_eq!(
|
||||
mismatches[0].fields,
|
||||
vec!["apiUrl", "uploadUrl", "downloadUrl"]
|
||||
);
|
||||
let warning = mismatches[0].to_string();
|
||||
assert!(warning.contains("session advertises apiUrl"), "{warning}");
|
||||
assert!(warning.contains("connected to "), "{warning}");
|
||||
assert!(warning.contains("advertised HTTP URL setting"), "{warning}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn import_aborts_with_exit_two_when_the_advertised_api_url_is_unreachable() {
|
||||
let mut api_host = mockito::Server::new();
|
||||
let mut session_host = mockito::Server::new();
|
||||
let session_base = session_host.url();
|
||||
let api_base = api_host.url();
|
||||
|
||||
let _root = session_host.mock("GET", "/").with_status(404).create();
|
||||
let _wk = session_host
|
||||
.mock("GET", "/.well-known/jmap")
|
||||
.with_status(200)
|
||||
.with_header("content-type", "application/json")
|
||||
.with_body(session_json(&api_base))
|
||||
.create();
|
||||
let _nginx = api_host
|
||||
.mock("POST", "/jmap/api")
|
||||
.with_status(404)
|
||||
.with_header("content-type", "text/html")
|
||||
.with_body(
|
||||
"<html><head><title>404 Not Found</title></head><body>404 Not Found</body></html>",
|
||||
)
|
||||
.create();
|
||||
|
||||
let archive = archive_path();
|
||||
let err = vandelay::sync::import_jmap::run(
|
||||
vandelay::sync::CommonConfig {
|
||||
archive: archive.clone(),
|
||||
threads: 1,
|
||||
dry_run: false,
|
||||
max_retries: 0,
|
||||
allow_invalid_certs: false,
|
||||
logger: vandelay::logging::Logger::from_flags(true, 0),
|
||||
},
|
||||
vandelay::sync::ImportConfig {
|
||||
connect: vandelay::sync::ConnectConfig {
|
||||
url: session_base.clone(),
|
||||
auth: Auth::Basic {
|
||||
user: "u".into(),
|
||||
password: "p".into(),
|
||||
},
|
||||
account: AccountSelector::Id("w".into()),
|
||||
},
|
||||
objects: None,
|
||||
allow_source_change: false,
|
||||
},
|
||||
)
|
||||
.expect_err("an unreachable apiUrl must abort the run");
|
||||
|
||||
assert!(
|
||||
matches!(err, vandelay::error::Error::Connection(_)),
|
||||
"a 404 from the advertised apiUrl is a whole-run connection failure, got {err:?}"
|
||||
);
|
||||
assert_eq!(err.exit_code(), 2, "must not report a partial failure");
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
@@ -1761,6 +1761,449 @@ fn export_calendar_creates_only_missing() {
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_contact_and_event_blobs_are_inlined_never_blob_ids() {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let api = "/jmap/api";
|
||||
let archive = tmp();
|
||||
let photo: &[u8] = b"\x89PNG\r\n\x1a\nfake-photo-bytes";
|
||||
let agenda: &[u8] = b"agenda for the review meeting";
|
||||
{
|
||||
let conn = db::init::open(&archive).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO address_books (id,name,is_default) VALUES (1,'Personal',1)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO calendars (id,name,is_default) VALUES (1,'Work',1)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
let photo_blob = db::blobs::intern_blob(&conn, photo).unwrap();
|
||||
let agenda_blob = db::blobs::intern_blob(&conn, agenda).unwrap();
|
||||
let card = json!({
|
||||
"@type": "Card",
|
||||
"version": "1.0",
|
||||
"name": { "full": "Photo Person" },
|
||||
"media": { "photo": {
|
||||
"@type": "Media", "kind": "photo",
|
||||
"@blob": photo_blob, "mediaType": "image/png"
|
||||
} }
|
||||
})
|
||||
.to_string();
|
||||
conn.execute(
|
||||
"INSERT INTO contact_cards (id,uid,address_book_ids,data)
|
||||
VALUES (1,'card-uid-1','[1]',?1)",
|
||||
[card.as_str()],
|
||||
)
|
||||
.unwrap();
|
||||
let event = json!({
|
||||
"@type": "Event",
|
||||
"uid": "event-uid-1",
|
||||
"title": "Review",
|
||||
"start": "2026-03-01T09:00:00",
|
||||
"duration": "PT1H",
|
||||
"timeZone": "Etc/UTC",
|
||||
"links": { "1": {
|
||||
"@type": "Link", "rel": "enclosure",
|
||||
"@blob": agenda_blob, "contentType": "text/plain", "title": "agenda.txt"
|
||||
} }
|
||||
})
|
||||
.to_string();
|
||||
conn.execute(
|
||||
"INSERT INTO calendar_events (id,calendar_ids,is_draft,use_default_alerts,data)
|
||||
VALUES (1,'[1]',0,0,?1)",
|
||||
[event.as_str()],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let _root = server.mock("GET", "/").with_status(404).create();
|
||||
let _wk = server
|
||||
.mock("GET", "/.well-known/jmap")
|
||||
.with_body(session_body_full(&base))
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
|
||||
let no_blob_id_in_card = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::AllOf(vec![
|
||||
Matcher::Regex("ContactCard/set".into()),
|
||||
Matcher::Regex("blobId".into()),
|
||||
]))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["ContactCard/set",{"accountId":"w",
|
||||
"created":{"c1":{"id":"TCARD"}}},"s"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(0)
|
||||
.create();
|
||||
let no_blob_id_in_event = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::AllOf(vec![
|
||||
Matcher::Regex("CalendarEvent/set".into()),
|
||||
Matcher::Regex("blobId".into()),
|
||||
]))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["CalendarEvent/set",{"accountId":"w",
|
||||
"created":{"c1":{"id":"TEVENT"}}},"s"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(0)
|
||||
.create();
|
||||
|
||||
let _ab_get = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("AddressBook/get".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["AddressBook/get",{"accountId":"w","list":[],
|
||||
"notFound":[]},"g"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let _ab_set = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("AddressBook/set".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["AddressBook/set",{"accountId":"w",
|
||||
"created":{"c1":{"id":"TAB"}}},"s"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(1)
|
||||
.create();
|
||||
let _cal_get = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Calendar/get".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Calendar/get",{"accountId":"w","list":[],
|
||||
"notFound":[]},"g"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let _cal_set = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Calendar/set".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Calendar/set",{"accountId":"w",
|
||||
"created":{"c1":{"id":"TCAL"}}},"s"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(1)
|
||||
.create();
|
||||
let _cc_query = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("ContactCard/query".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["ContactCard/query",{"accountId":"w","ids":[]},"q"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let _ce_query = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("CalendarEvent/query".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["CalendarEvent/query",{"accountId":"w","ids":[]},"q"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
|
||||
let card_inlined = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::AllOf(vec![
|
||||
Matcher::Regex("ContactCard/set".into()),
|
||||
Matcher::Regex("data:image/png;base64,".into()),
|
||||
]))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["ContactCard/set",{"accountId":"w",
|
||||
"created":{"c1":{"id":"TCARD"}}},"s"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(1)
|
||||
.create();
|
||||
let event_inlined = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::AllOf(vec![
|
||||
Matcher::Regex("CalendarEvent/set".into()),
|
||||
Matcher::Regex("data:text/plain;base64,".into()),
|
||||
]))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["CalendarEvent/set",{"accountId":"w",
|
||||
"created":{"c1":{"id":"TEVENT"}}},"s"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(1)
|
||||
.create();
|
||||
|
||||
let summary = sync::export::run(
|
||||
common(&archive),
|
||||
export_cfg_objects(
|
||||
&base,
|
||||
vec![
|
||||
ObjectType::AddressBook,
|
||||
ObjectType::Calendar,
|
||||
ObjectType::ContactCard,
|
||||
ObjectType::CalendarEvent,
|
||||
],
|
||||
),
|
||||
)
|
||||
.expect("export");
|
||||
|
||||
no_blob_id_in_card.assert();
|
||||
no_blob_id_in_event.assert();
|
||||
card_inlined.assert();
|
||||
event_inlined.assert();
|
||||
|
||||
for name in ["ContactCard", "CalendarEvent"] {
|
||||
let counts = summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(t, _)| *t == name)
|
||||
.map(|(_, c)| c.clone())
|
||||
.unwrap_or_else(|| panic!("{name} counts"));
|
||||
assert_eq!(counts.created, 1, "{name} created: {counts:?}");
|
||||
assert_eq!(counts.failed, 0, "{name} failed: {counts:?}");
|
||||
}
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
fn session_body_small_request(base: &str, max_size_request: u64) -> String {
|
||||
json!({
|
||||
"apiUrl": format!("{base}/jmap/api"),
|
||||
"uploadUrl": format!("{base}/jmap/upload/{{accountId}}/"),
|
||||
"downloadUrl": format!("{base}/jmap/dl/{{accountId}}/{{blobId}}/{{type}}/{{name}}"),
|
||||
"capabilities": { "urn:ietf:params:jmap:core": {
|
||||
"maxObjectsInGet": 500, "maxObjectsInSet": 500, "maxCallsInRequest": 16,
|
||||
"maxConcurrentRequests": 4, "maxConcurrentUpload": 4,
|
||||
"maxSizeRequest": max_size_request, "maxSizeUpload": 50000000
|
||||
} },
|
||||
"accounts": { "w": { "name": "alice",
|
||||
"accountCapabilities": {
|
||||
"urn:ietf:params:jmap:mail": {},
|
||||
"urn:ietf:params:jmap:contacts": {},
|
||||
"urn:ietf:params:jmap:calendars": {}
|
||||
} } }
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_oversized_contact_card_fails_alone_and_the_type_keeps_going() {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let api = "/jmap/api";
|
||||
let archive = tmp();
|
||||
{
|
||||
let conn = db::init::open(&archive).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO address_books (id,name,is_default) VALUES (1,'Personal',1)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
let photo = vec![b'P'; 8000];
|
||||
let photo_blob = db::blobs::intern_blob(&conn, &photo).unwrap();
|
||||
let huge = json!({
|
||||
"@type": "Card",
|
||||
"name": { "full": "Photo Person" },
|
||||
"media": { "photo": {
|
||||
"@type": "Media", "kind": "photo",
|
||||
"@blob": photo_blob, "mediaType": "image/png"
|
||||
} }
|
||||
})
|
||||
.to_string();
|
||||
conn.execute(
|
||||
"INSERT INTO contact_cards (id,uid,address_book_ids,data)
|
||||
VALUES (1,'huge-card','[1]',?1)",
|
||||
[huge.as_str()],
|
||||
)
|
||||
.unwrap();
|
||||
let small = json!({ "@type": "Card", "name": { "full": "Small Person" } }).to_string();
|
||||
conn.execute(
|
||||
"INSERT INTO contact_cards (id,uid,address_book_ids,data)
|
||||
VALUES (2,'small-card','[1]',?1)",
|
||||
[small.as_str()],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let _root = server.mock("GET", "/").with_status(404).create();
|
||||
let _wk = server
|
||||
.mock("GET", "/.well-known/jmap")
|
||||
.with_body(session_body_small_request(&base, 4000))
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let _ccterm = anchor_terminator(&mut server, api, "ContactCard");
|
||||
|
||||
let _ab_get = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("AddressBook/get".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["AddressBook/get",{"accountId":"w","list":[],
|
||||
"notFound":[]},"g"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let _ab_set = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("AddressBook/set".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["AddressBook/set",{"accountId":"w",
|
||||
"created":{"c1":{"id":"TAB"}}},"s"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(1)
|
||||
.create();
|
||||
let _cc_query = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("ContactCard/query".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["ContactCard/query",{"accountId":"w",
|
||||
"ids":["TSTALE"]},"q"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let _cc_get = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("ContactCard/get".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["ContactCard/get",{"accountId":"w","list":[
|
||||
{"id":"TSTALE","uid":"stale-card"}
|
||||
],"notFound":[]},"g"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let destroyed = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::AllOf(vec![
|
||||
Matcher::Regex("ContactCard/set".into()),
|
||||
Matcher::Regex("destroy".into()),
|
||||
]))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["ContactCard/set",{"accountId":"w",
|
||||
"destroyed":["TSTALE"]},"s"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(1)
|
||||
.create();
|
||||
let created = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::AllOf(vec![
|
||||
Matcher::Regex("ContactCard/set".into()),
|
||||
Matcher::Regex("Small Person".into()),
|
||||
]))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["ContactCard/set",{"accountId":"w",
|
||||
"created":{"c2":{"id":"TSMALL"}}},"s"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(1)
|
||||
.create();
|
||||
|
||||
let summary = sync::export::run(
|
||||
common(&archive),
|
||||
ExportConfig {
|
||||
prune: true,
|
||||
..export_cfg_objects(
|
||||
&base,
|
||||
vec![ObjectType::AddressBook, ObjectType::ContactCard],
|
||||
)
|
||||
},
|
||||
)
|
||||
.expect("the oversized card must not abort the run");
|
||||
|
||||
created.assert();
|
||||
destroyed.assert();
|
||||
let counts = summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(t, _)| *t == "ContactCard")
|
||||
.map(|(_, c)| c.clone())
|
||||
.expect("ContactCard counts");
|
||||
assert_eq!(counts.created, 1, "{counts:?}");
|
||||
assert_eq!(counts.failed, 1, "{counts:?}");
|
||||
assert_eq!(
|
||||
counts.deleted, 1,
|
||||
"the prune plan stays complete: {counts:?}"
|
||||
);
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_archive_read_failure_while_inlining_exits_seven() {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let api = "/jmap/api";
|
||||
let archive = tmp();
|
||||
{
|
||||
let conn = db::init::open(&archive).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO address_books (id,name,is_default) VALUES (1,'Personal',1)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
let photo_blob = db::blobs::intern_blob(&conn, b"photo-bytes").unwrap();
|
||||
let card = json!({
|
||||
"@type": "Card",
|
||||
"media": { "photo": { "@type": "Media", "kind": "photo",
|
||||
"@blob": photo_blob, "mediaType": "image/png" } }
|
||||
})
|
||||
.to_string();
|
||||
conn.execute(
|
||||
"INSERT INTO contact_cards (id,uid,address_book_ids,data)
|
||||
VALUES (1,'card-uid-1','[1]',?1)",
|
||||
[card.as_str()],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute("ALTER TABLE blobs RENAME COLUMN data TO payload", [])
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let _root = server.mock("GET", "/").with_status(404).create();
|
||||
let _wk = server
|
||||
.mock("GET", "/.well-known/jmap")
|
||||
.with_body(session_body_full(&base))
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let _cc_query = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("ContactCard/query".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["ContactCard/query",{"accountId":"w","ids":[]},"q"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let no_create = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("ContactCard/set".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["ContactCard/set",{"accountId":"w",
|
||||
"created":{"c1":{"id":"TCARD"}}},"s"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(0)
|
||||
.create();
|
||||
|
||||
let err = sync::export::run(
|
||||
common(&archive),
|
||||
export_cfg_objects(&base, vec![ObjectType::ContactCard]),
|
||||
)
|
||||
.expect_err("an unreadable blobs table must abort the run");
|
||||
|
||||
no_create.assert();
|
||||
assert!(err.aborts_run(), "{err}");
|
||||
assert_eq!(err.exit_code(), 7, "{err}");
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_sieve_script_matches_by_name_not_content() {
|
||||
let mut server = mockito::Server::new();
|
||||
@@ -3264,3 +3707,369 @@ fn import_jmap_duplicate_role_is_deduplicated_to_single_mailbox() {
|
||||
drop(conn);
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_aborts_with_exit_two_when_the_advertised_api_url_is_unreachable() {
|
||||
let mut session_host = mockito::Server::new();
|
||||
let mut api_host = mockito::Server::new();
|
||||
let session_base = session_host.url();
|
||||
let api_base = api_host.url();
|
||||
let archive = tmp();
|
||||
{
|
||||
let conn = db::init::open(&archive).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO mailboxes (id,name,parent_id,role,sort_order,is_subscribed)
|
||||
VALUES (1,'Inbox',NULL,'inbox',0,1)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO address_books (id,name,description,is_default)
|
||||
VALUES (1,'Personal',NULL,1)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let _root = session_host.mock("GET", "/").with_status(404).create();
|
||||
let _wk = session_host
|
||||
.mock("GET", "/.well-known/jmap")
|
||||
.with_body(session_body_full(&api_base))
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let nginx = api_host
|
||||
.mock("POST", "/jmap/api")
|
||||
.with_status(404)
|
||||
.with_header("content-type", "text/html")
|
||||
.with_body("<html><head><title>404 Not Found</title></head><body>nginx</body></html>")
|
||||
.expect(1)
|
||||
.create();
|
||||
|
||||
let err = sync::export::run(
|
||||
CommonConfig {
|
||||
archive: archive.clone(),
|
||||
threads: 1,
|
||||
dry_run: false,
|
||||
max_retries: 0,
|
||||
allow_invalid_certs: false,
|
||||
logger: Logger::from_flags(true, 0),
|
||||
},
|
||||
export_cfg_objects(
|
||||
&session_base,
|
||||
vec![ObjectType::Mailbox, ObjectType::AddressBook],
|
||||
),
|
||||
)
|
||||
.expect_err("an unreachable apiUrl must abort the run");
|
||||
|
||||
assert!(
|
||||
matches!(err, vandelay::error::Error::Connection(_)),
|
||||
"a 404 from the advertised apiUrl is a whole-run connection failure, got {err:?}"
|
||||
);
|
||||
assert_eq!(err.exit_code(), 2, "must not report a partial failure");
|
||||
nginx.assert();
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aborting_import_still_reports_the_types_that_finished() {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let api = "/jmap/api";
|
||||
let archive = tmp();
|
||||
|
||||
let _root = server.mock("GET", "/").with_status(404).create();
|
||||
let _wk = server
|
||||
.mock("GET", "/.well-known/jmap")
|
||||
.with_body(session_body_full(&base))
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let _term = anchor_terminator(&mut server, api, "Mailbox");
|
||||
|
||||
let _mq = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Mailbox/query".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Mailbox/query",
|
||||
{"accountId":"w","ids":["A"]},"q"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let _mg = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Mailbox/get".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Mailbox/get",{"accountId":"w","state":"s1","list":[
|
||||
{"id":"A","name":"alpha","parentId":null,"role":null,"sortOrder":0,"isSubscribed":true}
|
||||
],"notFound":[]},"g"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
let _dead_endpoint = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Email/query".into()))
|
||||
.with_status(404)
|
||||
.with_header("content-type", "text/html")
|
||||
.with_body("<html><head><title>404 Not Found</title></head><body>nginx</body></html>")
|
||||
.expect_at_least(1)
|
||||
.create();
|
||||
|
||||
let outcome = sync::import_jmap::run_reporting(
|
||||
common(&archive),
|
||||
import_cfg_objects(&base, vec![ObjectType::Mailbox, ObjectType::Email]),
|
||||
);
|
||||
|
||||
let err = outcome
|
||||
.error
|
||||
.as_ref()
|
||||
.expect("a 404 from the api endpoint aborts the run");
|
||||
assert!(
|
||||
matches!(err, vandelay::error::Error::Connection(_)),
|
||||
"the abort must keep its own exit code, got {err:?}"
|
||||
);
|
||||
assert_eq!(err.exit_code(), 2);
|
||||
|
||||
let mailbox = outcome
|
||||
.summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(t, _)| *t == "Mailbox")
|
||||
.map(|(_, c)| c.clone())
|
||||
.expect("the type that finished before the abort must still be reported");
|
||||
assert_eq!(mailbox.fetched, 1);
|
||||
assert!(
|
||||
!outcome.summary.per_type.iter().any(|(t, _)| *t == "Email"),
|
||||
"the type that aborted contributes no counts"
|
||||
);
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
fn seed_single_email(archive: &Path) {
|
||||
let conn = db::init::open(archive).unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO mailboxes (id,name,parent_id,role) VALUES (1,'Inbox',NULL,'inbox')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
let blob = db::blobs::intern_blob(
|
||||
&conn,
|
||||
b"From: a@x\r\nSubject: hi\r\nMessage-ID: <m-9@h>\r\n\r\nbody",
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords)
|
||||
VALUES (?1,'2020-01-01T00:00:00Z','[1]','[\"$seen\"]')",
|
||||
rusqlite::params![blob],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn mock_reconcile_for_single_email(server: &mut mockito::ServerGuard, base: &str, api: &str) {
|
||||
server.mock("GET", "/").with_status(404).create();
|
||||
server
|
||||
.mock("GET", "/.well-known/jmap")
|
||||
.with_body(session_body(base))
|
||||
.create();
|
||||
server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Mailbox/query".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Mailbox/query",
|
||||
{"accountId":"w","ids":["t1"]},"q"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(1)
|
||||
.create();
|
||||
server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Mailbox/query".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Mailbox/query",
|
||||
{"accountId":"w","ids":[]},"q"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(1)
|
||||
.create();
|
||||
server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Mailbox/get".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Mailbox/get",{"accountId":"w","list":[
|
||||
{"id":"t1","name":"Inbox","role":"inbox","parentId":null,
|
||||
"myRights":{"mayDelete":true}}],"notFound":[]},"g"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(1)
|
||||
.create();
|
||||
server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Email/query".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Email/query",
|
||||
{"accountId":"w","ids":[]},"q"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(1)
|
||||
.create();
|
||||
}
|
||||
|
||||
fn run_export(archive: &Path, base: &str, max_retries: u32) -> sync::Summary {
|
||||
sync::export::run(
|
||||
CommonConfig {
|
||||
archive: archive.to_path_buf(),
|
||||
threads: 1,
|
||||
dry_run: false,
|
||||
max_retries,
|
||||
allow_invalid_certs: false,
|
||||
logger: Logger::from_flags(true, 0),
|
||||
},
|
||||
ExportConfig {
|
||||
connect: ConnectConfig {
|
||||
url: base.to_owned(),
|
||||
auth: Auth::Basic {
|
||||
user: "u".into(),
|
||||
password: "p".into(),
|
||||
},
|
||||
account: AccountSelector::Id("w".into()),
|
||||
},
|
||||
objects: None,
|
||||
prune: false,
|
||||
yes: true,
|
||||
},
|
||||
)
|
||||
.expect("export run")
|
||||
}
|
||||
|
||||
fn email_counts(summary: &sync::Summary) -> sync::TypeCounts {
|
||||
summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(t, _)| *t == "Email")
|
||||
.map(|(_, c)| c.clone())
|
||||
.expect("email counts")
|
||||
}
|
||||
|
||||
fn method_error_body(error_type: &str) -> String {
|
||||
json!({"methodResponses":[["error",
|
||||
{"type": error_type,
|
||||
"description":"This server is temporarily unavailable. \
|
||||
Attempting this same operation later may succeed."},
|
||||
"i"]]})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_email_server_unavailable_is_retried_not_dropped() {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let api = "/jmap/api";
|
||||
|
||||
let archive = tmp();
|
||||
seed_single_email(&archive);
|
||||
mock_reconcile_for_single_email(&mut server, &base, api);
|
||||
|
||||
let up = server
|
||||
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
|
||||
.with_body(json!({"blobId":"UPLOADED"}).to_string())
|
||||
.expect(1)
|
||||
.create();
|
||||
let failing = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Email/import".into()))
|
||||
.with_body(method_error_body("serverUnavailable"))
|
||||
.expect(2)
|
||||
.create();
|
||||
let succeeding = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Email/import".into()))
|
||||
.with_body(
|
||||
json!({"methodResponses":[["Email/import",{"accountId":"w",
|
||||
"created":{"e1":{"id":"E1","blobId":"UPLOADED","threadId":"T1","size":48}}},"i"]]})
|
||||
.to_string(),
|
||||
)
|
||||
.expect(1)
|
||||
.create();
|
||||
|
||||
let summary = run_export(&archive, &base, 3);
|
||||
let email = email_counts(&summary);
|
||||
|
||||
failing.assert();
|
||||
succeeding.assert();
|
||||
up.assert();
|
||||
assert_eq!(email.created, 1, "the message must survive a transient error");
|
||||
assert_eq!(email.failed, 0, "a retried message is not a failure");
|
||||
assert!(!summary.any_failed());
|
||||
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_email_exhausted_method_retries_stay_per_message() {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let api = "/jmap/api";
|
||||
|
||||
let archive = tmp();
|
||||
seed_single_email(&archive);
|
||||
mock_reconcile_for_single_email(&mut server, &base, api);
|
||||
|
||||
let _up = server
|
||||
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
|
||||
.with_body(json!({"blobId":"UPLOADED"}).to_string())
|
||||
.expect(1)
|
||||
.create();
|
||||
let failing = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Email/import".into()))
|
||||
.with_body(method_error_body("serverUnavailable"))
|
||||
.expect(3)
|
||||
.create();
|
||||
|
||||
let summary = run_export(&archive, &base, 2);
|
||||
let email = email_counts(&summary);
|
||||
|
||||
failing.assert();
|
||||
assert_eq!(email.created, 0);
|
||||
assert_eq!(
|
||||
email.failed, 1,
|
||||
"exhausting the retries stays a per-message failure, not a run abort"
|
||||
);
|
||||
assert!(summary.any_failed(), "the run must still exit 5");
|
||||
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_email_fatal_method_error_is_not_retried() {
|
||||
let mut server = mockito::Server::new();
|
||||
let base = server.url();
|
||||
let api = "/jmap/api";
|
||||
|
||||
let archive = tmp();
|
||||
seed_single_email(&archive);
|
||||
mock_reconcile_for_single_email(&mut server, &base, api);
|
||||
|
||||
let _up = server
|
||||
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
|
||||
.with_body(json!({"blobId":"UPLOADED"}).to_string())
|
||||
.expect(1)
|
||||
.create();
|
||||
let failing = server
|
||||
.mock("POST", api)
|
||||
.match_body(Matcher::Regex("Email/import".into()))
|
||||
.with_body(method_error_body("invalidArguments"))
|
||||
.expect(1)
|
||||
.create();
|
||||
|
||||
let summary = run_export(&archive, &base, 5);
|
||||
let email = email_counts(&summary);
|
||||
|
||||
failing.assert();
|
||||
assert_eq!(
|
||||
email.failed, 1,
|
||||
"a permanent method error must not consume retries"
|
||||
);
|
||||
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
@@ -689,6 +689,167 @@ fn prune_non_tree_type_contact_card() {
|
||||
seeder::teardown(base_url()).expect("teardown");
|
||||
}
|
||||
|
||||
const ISSUE30_CARD_UID: &str = "vandelay-issue30-card";
|
||||
const ISSUE30_EVENT_UID: &str = "vandelay-issue30-event";
|
||||
|
||||
fn data_uri_bytes(resource: &Value, uri_key: &str, expect_media_type: &str) -> Vec<u8> {
|
||||
use base64::Engine;
|
||||
let uri = resource
|
||||
.get(uri_key)
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_else(|| panic!("no {uri_key} on {resource}"));
|
||||
assert!(
|
||||
resource.get("blobId").is_none(),
|
||||
"target must not hold a blobId for {uri_key}: {resource}"
|
||||
);
|
||||
let prefix = format!("data:{expect_media_type};base64,");
|
||||
let payload = uri
|
||||
.strip_prefix(&prefix)
|
||||
.unwrap_or_else(|| panic!("{uri_key} is not a {prefix}... data URI: {resource}"));
|
||||
base64::engine::general_purpose::STANDARD
|
||||
.decode(payload)
|
||||
.expect("base64 payload")
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires Docker"]
|
||||
fn export_inlines_contact_and_event_blobs_instead_of_blob_ids() {
|
||||
let fx = seeder::provision(base_url()).expect("provision");
|
||||
let tgt = fx.account("test6").expect("test6");
|
||||
let archive = tmp_archive("issue30");
|
||||
|
||||
let photo: &[u8] = b"\x89PNG\r\n\x1a\nvandelay issue 30 contact photo bytes";
|
||||
let agenda: &[u8] = b"vandelay issue 30 calendar enclosure bytes";
|
||||
{
|
||||
let conn = vandelay::db::init::open(&archive).expect("init archive");
|
||||
conn.execute(
|
||||
"INSERT INTO address_books (id,name,is_default) VALUES (1,'Issue30 Book',1)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO calendars (id,name,is_default) VALUES (1,'Issue30 Calendar',1)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
let photo_blob = vandelay::db::blobs::intern_blob(&conn, photo).unwrap();
|
||||
let agenda_blob = vandelay::db::blobs::intern_blob(&conn, agenda).unwrap();
|
||||
let card = json!({
|
||||
"@type": "Card",
|
||||
"version": "1.0",
|
||||
"name": { "full": "Issue Thirty" },
|
||||
"media": { "photo": {
|
||||
"@type": "Media", "kind": "photo",
|
||||
"@blob": photo_blob, "mediaType": "image/png"
|
||||
} }
|
||||
})
|
||||
.to_string();
|
||||
conn.execute(
|
||||
"INSERT INTO contact_cards (id,uid,address_book_ids,data) VALUES (1,?1,'[1]',?2)",
|
||||
[ISSUE30_CARD_UID, card.as_str()],
|
||||
)
|
||||
.unwrap();
|
||||
let event = json!({
|
||||
"@type": "Event",
|
||||
"uid": ISSUE30_EVENT_UID,
|
||||
"title": "Issue 30 review",
|
||||
"start": "2026-03-01T09:00:00",
|
||||
"duration": "PT1H",
|
||||
"timeZone": "Etc/UTC",
|
||||
"links": { "1": {
|
||||
"@type": "Link", "rel": "enclosure",
|
||||
"@blob": agenda_blob, "contentType": "text/plain", "title": "agenda.txt"
|
||||
} }
|
||||
})
|
||||
.to_string();
|
||||
conn.execute(
|
||||
"INSERT INTO calendar_events (id,calendar_ids,is_draft,use_default_alerts,data)
|
||||
VALUES (1,'[1]',0,0,?1)",
|
||||
[event.as_str()],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let summary = sync::export::run(
|
||||
common(&archive, false),
|
||||
export_cfg("test6", &tgt.account_id, false),
|
||||
)
|
||||
.expect("export");
|
||||
for name in ["ContactCard", "CalendarEvent"] {
|
||||
let counts = summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(t, _)| *t == name)
|
||||
.map(|(_, c)| c.clone())
|
||||
.unwrap_or_else(|| panic!("{name} counts missing: {summary:?}"));
|
||||
assert_eq!(counts.created, 1, "{name} created exactly one: {counts:?}");
|
||||
assert_eq!(counts.failed, 0, "{name} had no failures: {counts:?}");
|
||||
}
|
||||
|
||||
let client = HttpClient::new(basic("test6"), RetryPolicy::new(5), true);
|
||||
let session = Session::discover(&client, base_url()).expect("discover target session");
|
||||
let api = session.api_url.clone();
|
||||
|
||||
let cards = client
|
||||
.post_json(
|
||||
&api,
|
||||
&json!({
|
||||
"using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:contacts"],
|
||||
"methodCalls": [["ContactCard/get",
|
||||
{ "accountId": tgt.account_id, "ids": Value::Null }, "g"]]
|
||||
}),
|
||||
)
|
||||
.expect("ContactCard/get");
|
||||
let card = cards
|
||||
.pointer("/methodResponses/0/1/list")
|
||||
.and_then(Value::as_array)
|
||||
.expect("ContactCard list")
|
||||
.iter()
|
||||
.find(|c| c.get("uid").and_then(Value::as_str) == Some(ISSUE30_CARD_UID))
|
||||
.unwrap_or_else(|| panic!("exported card not found on the target: {cards}"))
|
||||
.clone();
|
||||
let photo_resource = card
|
||||
.pointer("/media/photo")
|
||||
.unwrap_or_else(|| panic!("card has no media.photo: {card}"));
|
||||
assert_eq!(
|
||||
data_uri_bytes(photo_resource, "uri", "image/png"),
|
||||
photo,
|
||||
"contact photo bytes survived the export: {card}"
|
||||
);
|
||||
|
||||
let events = client
|
||||
.post_json(
|
||||
&api,
|
||||
&json!({
|
||||
"using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:calendars"],
|
||||
"methodCalls": [["CalendarEvent/get",
|
||||
{ "accountId": tgt.account_id, "ids": Value::Null }, "g"]]
|
||||
}),
|
||||
)
|
||||
.expect("CalendarEvent/get");
|
||||
let event = events
|
||||
.pointer("/methodResponses/0/1/list")
|
||||
.and_then(Value::as_array)
|
||||
.expect("CalendarEvent list")
|
||||
.iter()
|
||||
.find(|e| e.get("uid").and_then(Value::as_str) == Some(ISSUE30_EVENT_UID))
|
||||
.unwrap_or_else(|| panic!("exported event not found on the target: {events}"))
|
||||
.clone();
|
||||
let link = event
|
||||
.get("links")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|m| m.values().next())
|
||||
.unwrap_or_else(|| panic!("event has no links: {event}"));
|
||||
assert_eq!(
|
||||
data_uri_bytes(link, "href", "text/plain"),
|
||||
agenda,
|
||||
"calendar enclosure bytes survived the export: {event}"
|
||||
);
|
||||
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
seeder::teardown(base_url()).expect("teardown");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires Docker"]
|
||||
fn export_round_trip_and_convergence() {
|
||||
|
||||
Reference in New Issue
Block a user