+2
-1
@@ -7,4 +7,5 @@ target
|
|||||||
*.md
|
*.md
|
||||||
!README.md
|
!README.md
|
||||||
!CHANGELOG.md
|
!CHANGELOG.md
|
||||||
specs/
|
.*
|
||||||
|
!.github
|
||||||
|
|||||||
@@ -2,6 +2,21 @@
|
|||||||
|
|
||||||
All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/).
|
All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/).
|
||||||
|
|
||||||
|
## [1.0.8] - 2026-08-15
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- A warning when the session advertises `apiUrl`, `uploadUrl` or `downloadUrl` on a different origin than the one connected to, naming both origins (#29).
|
||||||
|
- The per-type report is printed when a run aborts.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- A transient `serverUnavailable` from `Email/import` permanently dropped that message from the migration instead of retrying it.
|
||||||
|
- The Exchange Graph importer stored a whole converted event as a `recurrenceOverrides` value (#31).
|
||||||
|
- Exporting a ContactCard carrying a photo blob failed against Stalwart, and a CalendarEvent enclosure sent as a `blobId` was silently dropped by the server; the bytes of both are now inlined as a base64 `data:` URI (#30).
|
||||||
|
- A connection failure against the session-advertised `apiUrl` was reported as a per-type partial failure (exit 5, "consistent and resumable") instead of aborting (#29).
|
||||||
|
- Session `apiUrl`, `uploadUrl` and `downloadUrl` given as relative references were used unresolved (#29).
|
||||||
|
|
||||||
## [1.0.7] - 2026-07-26
|
## [1.0.7] - 2026-07-26
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|||||||
Generated
+298
-319
File diff suppressed because it is too large
Load Diff
+3
-3
@@ -1,7 +1,7 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "vandelay"
|
name = "vandelay"
|
||||||
description = "JMAP account migration utility"
|
description = "JMAP account migration utility"
|
||||||
version = "1.0.7"
|
version = "1.0.8"
|
||||||
authors = ["Stalwart Labs LLC <[email protected]>"]
|
authors = ["Stalwart Labs LLC <[email protected]>"]
|
||||||
license = "Apache-2.0 OR MIT"
|
license = "Apache-2.0 OR MIT"
|
||||||
repository = "https://github.com/stalwartlabs/vandelay"
|
repository = "https://github.com/stalwartlabs/vandelay"
|
||||||
@@ -36,7 +36,7 @@ ureq = { version = "3.3.0", default-features = false, features = ["json", "gzip"
|
|||||||
clap = { version = "4.6.1", features = ["derive"] }
|
clap = { version = "4.6.1", features = ["derive"] }
|
||||||
crossbeam-channel = "0.5.15"
|
crossbeam-channel = "0.5.15"
|
||||||
num_cpus = "1.17.0"
|
num_cpus = "1.17.0"
|
||||||
base64 = "0.22.1"
|
base64 = "0.23"
|
||||||
mail-parser = "0.11.3"
|
mail-parser = "0.11.3"
|
||||||
calcard = "0.3"
|
calcard = "0.3"
|
||||||
rustls = { version = "0.23.40", default-features = false, features = ["aws-lc-rs", "std", "tls12", "logging"] }
|
rustls = { version = "0.23.40", default-features = false, features = ["aws-lc-rs", "std", "tls12", "logging"] }
|
||||||
@@ -50,7 +50,7 @@ chrono-tz = { version = "0.10", default-features = false }
|
|||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
mockito = "1.7.2"
|
mockito = "1.7.2"
|
||||||
tempfile = "3.13"
|
tempfile = "3.13"
|
||||||
testcontainers = { version = "0.27", features = ["blocking"] }
|
testcontainers = { version = "0.28", features = ["blocking"] }
|
||||||
|
|
||||||
[profile.dist]
|
[profile.dist]
|
||||||
inherits = "release"
|
inherits = "release"
|
||||||
|
|||||||
+60
-6
@@ -12,7 +12,7 @@ use regex::Regex;
|
|||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
use crate::exchange_ews::oauth::OAuthFlow;
|
use crate::exchange_ews::oauth::OAuthFlow;
|
||||||
use crate::exchange_ews::types::MailboxKind;
|
use crate::exchange_ews::types::MailboxKind;
|
||||||
use crate::exchange_graph::types::{EventBodyFormat, MailboxKind as GraphMailboxKind};
|
use crate::exchange_graph::types::{EventBodyFormat, MailboxKind as GraphMailboxKind, Surfaces};
|
||||||
use crate::inspect::InspectConfig;
|
use crate::inspect::InspectConfig;
|
||||||
use crate::jmap::account::AccountSelector;
|
use crate::jmap::account::AccountSelector;
|
||||||
use crate::jmap::http::Auth;
|
use crate::jmap::http::Auth;
|
||||||
@@ -1132,7 +1132,7 @@ pub struct ExchangeGraphImportArgs {
|
|||||||
#[arg(
|
#[arg(
|
||||||
long,
|
long,
|
||||||
value_name = "LIST",
|
value_name = "LIST",
|
||||||
help = "Comma-separated surface list (mail,calendar,contacts; default all)"
|
help = "Comma-separated surface list: mail | calendar | contacts (default: all three)"
|
||||||
)]
|
)]
|
||||||
objects: Option<String>,
|
objects: Option<String>,
|
||||||
|
|
||||||
@@ -1203,11 +1203,12 @@ fn resolve_exchange_graph_import(args: ExchangeGraphImportArgs) -> Result<Action
|
|||||||
args.client_id.as_deref(),
|
args.client_id.as_deref(),
|
||||||
&args.tenant,
|
&args.tenant,
|
||||||
)?;
|
)?;
|
||||||
let objects = args
|
let surfaces = args
|
||||||
.objects
|
.objects
|
||||||
.as_deref()
|
.as_deref()
|
||||||
.map(crate::types::parse_object_list)
|
.map(Surfaces::parse_list)
|
||||||
.transpose()?;
|
.transpose()?
|
||||||
|
.unwrap_or_default();
|
||||||
let top = args.top.clamp(1, 1000);
|
let top = args.top.clamp(1, 1000);
|
||||||
let graph_connections = args
|
let graph_connections = args
|
||||||
.graph_connections
|
.graph_connections
|
||||||
@@ -1220,7 +1221,7 @@ fn resolve_exchange_graph_import(args: ExchangeGraphImportArgs) -> Result<Action
|
|||||||
api_base: args.api_base,
|
api_base: args.api_base,
|
||||||
user_target: args.user,
|
user_target: args.user,
|
||||||
mailbox_kind,
|
mailbox_kind,
|
||||||
objects,
|
surfaces,
|
||||||
event_body_format,
|
event_body_format,
|
||||||
graph_connections,
|
graph_connections,
|
||||||
top,
|
top,
|
||||||
@@ -1361,4 +1362,57 @@ mod tests {
|
|||||||
let auth = resolve_auth(None, None, Some(&bearer)).unwrap();
|
let auth = resolve_auth(None, None, Some(&bearer)).unwrap();
|
||||||
assert!(matches!(auth, Auth::Bearer { token } if token == "t"));
|
assert!(matches!(auth, Auth::Bearer { token } if token == "t"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn graph_config(objects: &[&str]) -> Result<GraphImportConfig, Error> {
|
||||||
|
let mut argv = vec![
|
||||||
|
"vandelay",
|
||||||
|
"import",
|
||||||
|
"exchange-graph",
|
||||||
|
"--access-token",
|
||||||
|
"tok",
|
||||||
|
];
|
||||||
|
argv.extend_from_slice(objects);
|
||||||
|
argv.push("archive.sqlite");
|
||||||
|
let cli = Cli::try_parse_from(argv).unwrap_or_else(|e| panic!("clap rejected args: {e}"));
|
||||||
|
match cli.resolve()? {
|
||||||
|
Action::ImportExchangeGraph(_, config) => Ok(config),
|
||||||
|
_ => panic!("expected ImportExchangeGraph"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn graph_objects_contacts_selects_only_contacts() {
|
||||||
|
let _g = lock();
|
||||||
|
let config = graph_config(&["--objects", "contacts"]).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
config.surfaces,
|
||||||
|
Surfaces {
|
||||||
|
mail: false,
|
||||||
|
calendar: false,
|
||||||
|
contacts: true
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn graph_objects_absent_selects_every_surface() {
|
||||||
|
let _g = lock();
|
||||||
|
let config = graph_config(&[]).unwrap();
|
||||||
|
assert_eq!(config.surfaces, Surfaces::ALL);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn graph_objects_rejects_jmap_type_names() {
|
||||||
|
let _g = lock();
|
||||||
|
match graph_config(&["--objects", "contactcard"]) {
|
||||||
|
Err(Error::Usage(m)) => {
|
||||||
|
assert!(m.contains("unknown surface: contactcard"), "msg was: {m}");
|
||||||
|
assert!(
|
||||||
|
m.contains("valid: mail, calendar, contacts"),
|
||||||
|
"msg was: {m}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
other => panic!("expected Usage error, got {:?}", other.map(|_| ())),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -43,6 +43,20 @@ impl From<rusqlite::Error> for Error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl Error {
|
impl Error {
|
||||||
|
pub fn aborts_run(&self) -> bool {
|
||||||
|
match self {
|
||||||
|
Error::Partial(_) => false,
|
||||||
|
Error::Usage(_)
|
||||||
|
| Error::Connection(_)
|
||||||
|
| Error::Account(_)
|
||||||
|
| Error::SourceChange(_)
|
||||||
|
| Error::PruneAborted
|
||||||
|
| Error::Unimplemented(_)
|
||||||
|
| Error::Db(_)
|
||||||
|
| Error::Io(_) => true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn exit_code(&self) -> i32 {
|
pub fn exit_code(&self) -> i32 {
|
||||||
match self {
|
match self {
|
||||||
Error::Usage(_) => 1,
|
Error::Usage(_) => 1,
|
||||||
@@ -74,4 +88,37 @@ mod tests {
|
|||||||
let io = Error::Io(std::io::Error::other("disk"));
|
let io = Error::Io(std::io::Error::other("disk"));
|
||||||
assert_eq!(io.exit_code(), 7);
|
assert_eq!(io.exit_code(), 7);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn only_partial_is_a_per_unit_failure() {
|
||||||
|
assert!(!Error::Partial("one object".into()).aborts_run());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn whole_run_conditions_abort() {
|
||||||
|
assert!(Error::Connection("http status 404".into()).aborts_run());
|
||||||
|
assert!(Error::Account("ambiguous".into()).aborts_run());
|
||||||
|
assert!(Error::SourceChange("other account".into()).aborts_run());
|
||||||
|
assert!(Error::Usage("bad flag".into()).aborts_run());
|
||||||
|
assert!(Error::Unimplemented("x").aborts_run());
|
||||||
|
assert!(Error::PruneAborted.aborts_run());
|
||||||
|
assert!(Error::Db(OpenError::Sqlite(rusqlite::Error::QueryReturnedNoRows)).aborts_run());
|
||||||
|
assert!(Error::Io(std::io::Error::other("disk")).aborts_run());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn aborting_errors_do_not_map_to_the_partial_exit_code() {
|
||||||
|
for e in [
|
||||||
|
Error::Connection("x".into()),
|
||||||
|
Error::Account("x".into()),
|
||||||
|
Error::SourceChange("x".into()),
|
||||||
|
Error::Db(OpenError::Sqlite(rusqlite::Error::QueryReturnedNoRows)),
|
||||||
|
Error::Io(std::io::Error::other("disk")),
|
||||||
|
] {
|
||||||
|
assert!(e.aborts_run(), "{e} must abort the run");
|
||||||
|
assert_ne!(e.exit_code(), 5, "{e} must not report a partial failure");
|
||||||
|
}
|
||||||
|
assert_eq!(Error::Connection("x".into()).exit_code(), 2);
|
||||||
|
assert_eq!(Error::Partial("x".into()).exit_code(), 5);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,210 @@
|
|||||||
|
/*
|
||||||
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||||
|
*/
|
||||||
|
|
||||||
|
use serde_json::{Map, Value};
|
||||||
|
|
||||||
|
const IGNORED_EXACT: &[&str] = &[
|
||||||
|
"@type",
|
||||||
|
"method",
|
||||||
|
"organizerCalendarAddress",
|
||||||
|
"privacy",
|
||||||
|
"prodId",
|
||||||
|
"recurrenceId",
|
||||||
|
"recurrenceIdTimeZone",
|
||||||
|
"sentBy",
|
||||||
|
"uid",
|
||||||
|
];
|
||||||
|
|
||||||
|
const IGNORED_FIRST_TOKEN: &[&str] = &["recurrenceOverrides", "recurrenceRule", "relatedTo"];
|
||||||
|
|
||||||
|
const CALENDAR_ADDRESS_DEPENDENTS: &[&str] = &[
|
||||||
|
"calendarAddress",
|
||||||
|
"delegatedFrom",
|
||||||
|
"delegatedTo",
|
||||||
|
"email",
|
||||||
|
"expectReply",
|
||||||
|
"kind",
|
||||||
|
"memberOf",
|
||||||
|
"participationStatus",
|
||||||
|
"progress",
|
||||||
|
"roles",
|
||||||
|
"sentBy",
|
||||||
|
];
|
||||||
|
|
||||||
|
pub fn synthetic_attendee_address(identifier: &str) -> String {
|
||||||
|
format!(
|
||||||
|
"urn:x-vandelay:attendee:{}",
|
||||||
|
blake3::hash(identifier.as_bytes()).to_hex()
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn drop_calendar_address_dependents(participants: &mut Map<String, Value>) {
|
||||||
|
for participant in participants.values_mut() {
|
||||||
|
let Some(object) = participant.as_object_mut() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
for key in CALENDAR_ADDRESS_DEPENDENTS {
|
||||||
|
object.remove(*key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_override_ignored(pointer: &str) -> bool {
|
||||||
|
if IGNORED_EXACT.contains(&pointer) || is_participant_calendar_address(pointer) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
let first = pointer.split('/').next().unwrap_or(pointer);
|
||||||
|
IGNORED_FIRST_TOKEN.contains(&first)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_participant_calendar_address(pointer: &str) -> bool {
|
||||||
|
let mut tokens = pointer.split('/');
|
||||||
|
tokens.next() == Some("participants")
|
||||||
|
&& tokens.next().is_some()
|
||||||
|
&& tokens.next() == Some("calendarAddress")
|
||||||
|
&& tokens.next().is_none()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn override_patch_from_event(event: &Value) -> Value {
|
||||||
|
let Some(object) = event.as_object() else {
|
||||||
|
return event.clone();
|
||||||
|
};
|
||||||
|
let patch: Map<String, Value> = object
|
||||||
|
.iter()
|
||||||
|
.filter(|(key, _)| !is_override_ignored(key))
|
||||||
|
.map(|(key, value)| (key.clone(), value.clone()))
|
||||||
|
.collect();
|
||||||
|
Value::Object(patch)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn exact_pointers_are_ignored() {
|
||||||
|
for pointer in [
|
||||||
|
"@type",
|
||||||
|
"method",
|
||||||
|
"organizerCalendarAddress",
|
||||||
|
"privacy",
|
||||||
|
"prodId",
|
||||||
|
"recurrenceId",
|
||||||
|
"recurrenceIdTimeZone",
|
||||||
|
"sentBy",
|
||||||
|
"uid",
|
||||||
|
] {
|
||||||
|
assert!(is_override_ignored(pointer), "{pointer} must be ignored");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn first_token_pointers_are_ignored_at_any_depth() {
|
||||||
|
assert!(is_override_ignored("recurrenceRule"));
|
||||||
|
assert!(is_override_ignored("recurrenceRule/frequency"));
|
||||||
|
assert!(is_override_ignored(
|
||||||
|
"recurrenceOverrides/2026-01-01T09:00:00"
|
||||||
|
));
|
||||||
|
assert!(is_override_ignored("relatedTo/uid-1/relation"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn participant_calendar_address_is_ignored_only_at_its_own_depth() {
|
||||||
|
assert!(is_override_ignored("participants/att-1/calendarAddress"));
|
||||||
|
assert!(!is_override_ignored("participants"));
|
||||||
|
assert!(!is_override_ignored("participants/att-1"));
|
||||||
|
assert!(!is_override_ignored(
|
||||||
|
"participants/att-1/participationStatus"
|
||||||
|
));
|
||||||
|
assert!(!is_override_ignored(
|
||||||
|
"participants/att-1/calendarAddress/extra"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn patchable_pointers_survive() {
|
||||||
|
for pointer in [
|
||||||
|
"start",
|
||||||
|
"duration",
|
||||||
|
"title",
|
||||||
|
"excluded",
|
||||||
|
"participants",
|
||||||
|
"created",
|
||||||
|
"updated",
|
||||||
|
"status",
|
||||||
|
] {
|
||||||
|
assert!(!is_override_ignored(pointer), "{pointer} must survive");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_event_is_reduced_to_a_patch() {
|
||||||
|
let event = json!({
|
||||||
|
"@type": "Event",
|
||||||
|
"uid": "uid-1",
|
||||||
|
"prodId": "vandelay",
|
||||||
|
"privacy": "public",
|
||||||
|
"organizerCalendarAddress": "mailto:[email protected]",
|
||||||
|
"recurrenceRule": {"frequency": "daily"},
|
||||||
|
"start": "2026-03-04T09:00:00",
|
||||||
|
"duration": "PT1H",
|
||||||
|
"title": "Moved"
|
||||||
|
});
|
||||||
|
let patch = override_patch_from_event(&event);
|
||||||
|
let patch = patch.as_object().expect("patch object");
|
||||||
|
assert_eq!(patch.len(), 3);
|
||||||
|
assert_eq!(patch["start"], "2026-03-04T09:00:00");
|
||||||
|
assert_eq!(patch["duration"], "PT1H");
|
||||||
|
assert_eq!(patch["title"], "Moved");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_exclusion_patch_is_preserved() {
|
||||||
|
let patch = override_patch_from_event(&json!({"excluded": true}));
|
||||||
|
assert_eq!(patch, json!({"excluded": true}));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_synthetic_address_is_stable_per_identifier_and_never_a_mailto() {
|
||||||
|
let addr = synthetic_attendee_address("Jane Doe");
|
||||||
|
assert_eq!(addr, synthetic_attendee_address("Jane Doe"));
|
||||||
|
assert_ne!(addr, synthetic_attendee_address("John Doe"));
|
||||||
|
assert!(addr.starts_with("urn:x-vandelay:attendee:"));
|
||||||
|
assert!(
|
||||||
|
!addr.starts_with("mailto:"),
|
||||||
|
"export must never invite a fabricated address"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn calendar_address_dependents_are_dropped_and_identity_is_kept() {
|
||||||
|
let mut participants = json!({
|
||||||
|
"1": {
|
||||||
|
"@type": "Participant",
|
||||||
|
"name": "Room 4",
|
||||||
|
"calendarAddress": "urn:x-vandelay:attendee:abc",
|
||||||
|
"email": "[email protected]",
|
||||||
|
"roles": {"required": true},
|
||||||
|
"kind": "resource",
|
||||||
|
"participationStatus": "accepted",
|
||||||
|
"expectReply": true,
|
||||||
|
"description": "Third floor"
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.as_object()
|
||||||
|
.expect("participants object")
|
||||||
|
.clone();
|
||||||
|
drop_calendar_address_dependents(&mut participants);
|
||||||
|
let p = &participants["1"];
|
||||||
|
assert_eq!(p["@type"], "Participant");
|
||||||
|
assert_eq!(p["name"], "Room 4");
|
||||||
|
assert_eq!(p["description"], "Third floor");
|
||||||
|
for key in CALENDAR_ADDRESS_DEPENDENTS {
|
||||||
|
assert!(p.get(*key).is_none(), "{key} must be dropped");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,4 +4,5 @@
|
|||||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
pub mod jscalendar;
|
||||||
pub mod tz;
|
pub mod tz;
|
||||||
|
|||||||
@@ -6,6 +6,9 @@
|
|||||||
|
|
||||||
use serde_json::{Map, Value, json};
|
use serde_json::{Map, Value, json};
|
||||||
|
|
||||||
|
use crate::exchange::jscalendar::{
|
||||||
|
drop_calendar_address_dependents, is_override_ignored, synthetic_attendee_address,
|
||||||
|
};
|
||||||
use crate::exchange_ews::parse::{CalendarItemRaw, RawAttendee, RawOccurrence};
|
use crate::exchange_ews::parse::{CalendarItemRaw, RawAttendee, RawOccurrence};
|
||||||
use crate::exchange_ews::recurrence::to_jscalendar_rule;
|
use crate::exchange_ews::recurrence::to_jscalendar_rule;
|
||||||
use crate::exchange_ews::tz::resolve_to_iana;
|
use crate::exchange_ews::tz::resolve_to_iana;
|
||||||
@@ -181,6 +184,9 @@ fn build_event_map(raw: &CalendarItemRaw, iana: Option<&str>) -> Map<String, Val
|
|||||||
AttendeeRole::Resource,
|
AttendeeRole::Resource,
|
||||||
);
|
);
|
||||||
if !participants.is_empty() {
|
if !participants.is_empty() {
|
||||||
|
if !event.contains_key("organizerCalendarAddress") {
|
||||||
|
drop_calendar_address_dependents(&mut participants);
|
||||||
|
}
|
||||||
event.insert("participants".to_owned(), Value::Object(participants));
|
event.insert("participants".to_owned(), Value::Object(participants));
|
||||||
}
|
}
|
||||||
if let Some(rec) = raw.recurrence.as_ref()
|
if let Some(rec) = raw.recurrence.as_ref()
|
||||||
@@ -229,13 +235,6 @@ fn build_event_map(raw: &CalendarItemRaw, iana: Option<&str>) -> Map<String, Val
|
|||||||
event
|
event
|
||||||
}
|
}
|
||||||
|
|
||||||
fn synthetic_attendee_address(identifier: &str) -> String {
|
|
||||||
format!(
|
|
||||||
"urn:x-vandelay:attendee:{}",
|
|
||||||
blake3::hash(identifier.as_bytes()).to_hex()
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn is_smtp_routing(routing_type: Option<&str>) -> bool {
|
fn is_smtp_routing(routing_type: Option<&str>) -> bool {
|
||||||
routing_type.is_none_or(|rt| rt.eq_ignore_ascii_case("SMTP"))
|
routing_type.is_none_or(|rt| rt.eq_ignore_ascii_case("SMTP"))
|
||||||
}
|
}
|
||||||
@@ -320,21 +319,11 @@ fn add_attendees(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const OVERRIDE_IGNORED_POINTERS: &[&str] = &[
|
const NOT_PATCHED_PER_OCCURRENCE: &[&str] = &["created", "updated"];
|
||||||
"@type",
|
|
||||||
"uid",
|
fn is_override_excluded(key: &str) -> bool {
|
||||||
"recurrenceRule",
|
is_override_ignored(key) || NOT_PATCHED_PER_OCCURRENCE.contains(&key)
|
||||||
"recurrenceOverrides",
|
}
|
||||||
"recurrenceId",
|
|
||||||
"recurrenceIdTimeZone",
|
|
||||||
"method",
|
|
||||||
"organizerCalendarAddress",
|
|
||||||
"privacy",
|
|
||||||
"prodId",
|
|
||||||
"relatedTo",
|
|
||||||
"created",
|
|
||||||
"updated",
|
|
||||||
];
|
|
||||||
|
|
||||||
fn build_recurrence_overrides(
|
fn build_recurrence_overrides(
|
||||||
base_event: &Map<String, Value>,
|
base_event: &Map<String, Value>,
|
||||||
@@ -388,7 +377,7 @@ fn override_patch(
|
|||||||
let mut patch = Map::new();
|
let mut patch = Map::new();
|
||||||
let inherited_start = Value::String(recurrence_id.to_owned());
|
let inherited_start = Value::String(recurrence_id.to_owned());
|
||||||
for (k, v) in occ_event {
|
for (k, v) in occ_event {
|
||||||
if OVERRIDE_IGNORED_POINTERS.contains(&k.as_str()) {
|
if is_override_excluded(k) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let baseline = if k == "start" {
|
let baseline = if k == "start" {
|
||||||
@@ -401,7 +390,7 @@ fn override_patch(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
for k in base_event.keys() {
|
for k in base_event.keys() {
|
||||||
if k == "start" || OVERRIDE_IGNORED_POINTERS.contains(&k.as_str()) {
|
if k == "start" || is_override_excluded(k) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if !occ_event.contains_key(k) {
|
if !occ_event.contains_key(k) {
|
||||||
@@ -710,6 +699,7 @@ mod tests {
|
|||||||
uid: Some("u".to_owned()),
|
uid: Some("u".to_owned()),
|
||||||
start: Some("2025-06-15T14:00:00Z".to_owned()),
|
start: Some("2025-06-15T14:00:00Z".to_owned()),
|
||||||
end: Some("2025-06-15T15:00:00Z".to_owned()),
|
end: Some("2025-06-15T15:00:00Z".to_owned()),
|
||||||
|
organizer_smtp: Some("[email protected]".to_owned()),
|
||||||
required_attendees: vec![crate::exchange_ews::parse::RawAttendee {
|
required_attendees: vec![crate::exchange_ews::parse::RawAttendee {
|
||||||
email: None,
|
email: None,
|
||||||
routing_type: None,
|
routing_type: None,
|
||||||
@@ -796,6 +786,7 @@ mod tests {
|
|||||||
uid: Some("u".to_owned()),
|
uid: Some("u".to_owned()),
|
||||||
start: Some("2025-06-15T14:00:00Z".to_owned()),
|
start: Some("2025-06-15T14:00:00Z".to_owned()),
|
||||||
end: Some("2025-06-15T15:00:00Z".to_owned()),
|
end: Some("2025-06-15T15:00:00Z".to_owned()),
|
||||||
|
organizer_smtp: Some("[email protected]".to_owned()),
|
||||||
required_attendees: vec![RawAttendee {
|
required_attendees: vec![RawAttendee {
|
||||||
email: Some("[email protected]".to_owned()),
|
email: Some("[email protected]".to_owned()),
|
||||||
routing_type: Some("SMTP".to_owned()),
|
routing_type: Some("SMTP".to_owned()),
|
||||||
@@ -809,7 +800,7 @@ mod tests {
|
|||||||
.as_object()
|
.as_object()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.values()
|
.values()
|
||||||
.next()
|
.find(|p| p["email"] == "[email protected]")
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(att["calendarAddress"], "mailto:[email protected]");
|
assert_eq!(att["calendarAddress"], "mailto:[email protected]");
|
||||||
assert_eq!(att["email"], "[email protected]");
|
assert_eq!(att["email"], "[email protected]");
|
||||||
@@ -984,6 +975,7 @@ mod tests {
|
|||||||
uid: Some("uid-res".to_owned()),
|
uid: Some("uid-res".to_owned()),
|
||||||
start: Some("2025-06-15T14:00:00Z".to_owned()),
|
start: Some("2025-06-15T14:00:00Z".to_owned()),
|
||||||
end: Some("2025-06-15T15:00:00Z".to_owned()),
|
end: Some("2025-06-15T15:00:00Z".to_owned()),
|
||||||
|
organizer_smtp: Some("[email protected]".to_owned()),
|
||||||
resources: vec![crate::exchange_ews::parse::RawAttendee {
|
resources: vec![crate::exchange_ews::parse::RawAttendee {
|
||||||
email: Some("room-7@x".to_owned()),
|
email: Some("room-7@x".to_owned()),
|
||||||
routing_type: None,
|
routing_type: None,
|
||||||
@@ -1011,6 +1003,7 @@ mod tests {
|
|||||||
uid: Some("uid-rsvp".to_owned()),
|
uid: Some("uid-rsvp".to_owned()),
|
||||||
start: Some("2025-06-15T14:00:00Z".to_owned()),
|
start: Some("2025-06-15T14:00:00Z".to_owned()),
|
||||||
end: Some("2025-06-15T15:00:00Z".to_owned()),
|
end: Some("2025-06-15T15:00:00Z".to_owned()),
|
||||||
|
organizer_smtp: Some("[email protected]".to_owned()),
|
||||||
required_attendees: vec![crate::exchange_ews::parse::RawAttendee {
|
required_attendees: vec![crate::exchange_ews::parse::RawAttendee {
|
||||||
email: Some("bob@x".to_owned()),
|
email: Some("bob@x".to_owned()),
|
||||||
routing_type: None,
|
routing_type: None,
|
||||||
@@ -1031,6 +1024,36 @@ mod tests {
|
|||||||
assert_eq!(bob["roles"]["required"], true);
|
assert_eq!(bob["roles"]["required"], true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn attendees_lose_calendar_addresses_when_no_organizer_is_known() {
|
||||||
|
let raw = CalendarItemRaw {
|
||||||
|
uid: Some("uid-no-organizer".to_owned()),
|
||||||
|
start: Some("2025-06-15T14:00:00Z".to_owned()),
|
||||||
|
end: Some("2025-06-15T15:00:00Z".to_owned()),
|
||||||
|
required_attendees: vec![crate::exchange_ews::parse::RawAttendee {
|
||||||
|
email: Some("bob@x".to_owned()),
|
||||||
|
routing_type: None,
|
||||||
|
name: Some("Bob".to_owned()),
|
||||||
|
response_type: Some("Accept".to_owned()),
|
||||||
|
}],
|
||||||
|
..CalendarItemRaw::default()
|
||||||
|
};
|
||||||
|
let v = to_jscalendar(&raw).data;
|
||||||
|
assert!(v.get("organizerCalendarAddress").is_none());
|
||||||
|
let bob = v["participants"]
|
||||||
|
.as_object()
|
||||||
|
.unwrap()
|
||||||
|
.values()
|
||||||
|
.find(|p| p["name"] == "Bob")
|
||||||
|
.unwrap();
|
||||||
|
for key in ["calendarAddress", "email", "roles", "participationStatus"] {
|
||||||
|
assert!(
|
||||||
|
bob.get(key).is_none(),
|
||||||
|
"{key} requires calendarAddress, which requires organizerCalendarAddress"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn all_day_span_helper_is_inclusive_of_start() {
|
fn all_day_span_helper_is_inclusive_of_start() {
|
||||||
assert_eq!(all_day_span_days("2025-07-04", "2025-07-04"), 1);
|
assert_eq!(all_day_span_days("2025-07-04", "2025-07-04"), 1);
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
|
|
||||||
use serde_json::{Map, Value, json};
|
use serde_json::{Map, Value, json};
|
||||||
|
|
||||||
|
use crate::exchange::jscalendar::{drop_calendar_address_dependents, synthetic_attendee_address};
|
||||||
use crate::exchange::tz::resolve_to_iana;
|
use crate::exchange::tz::resolve_to_iana;
|
||||||
use crate::exchange_graph::error::GraphError;
|
use crate::exchange_graph::error::GraphError;
|
||||||
use crate::exchange_graph::recurrence::convert_patterned_recurrence_rule;
|
use crate::exchange_graph::recurrence::convert_patterned_recurrence_rule;
|
||||||
@@ -242,13 +243,8 @@ pub fn convert_event(
|
|||||||
.get("organizer")
|
.get("organizer")
|
||||||
.and_then(|o| o.get("emailAddress"))
|
.and_then(|o| o.get("emailAddress"))
|
||||||
{
|
{
|
||||||
if let Some(addr) = org.get("address").and_then(Value::as_str)
|
if let Some((cal_addr, _)) = resolve_calendar_address(org) {
|
||||||
&& !addr.is_empty()
|
card.insert("organizerCalendarAddress".to_owned(), Value::from(cal_addr));
|
||||||
{
|
|
||||||
card.insert(
|
|
||||||
"organizerCalendarAddress".to_owned(),
|
|
||||||
Value::from(format!("mailto:{addr}")),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
let p = build_participant(org, &["owner", "chair"], None, None);
|
let p = build_participant(org, &["owner", "chair"], None, None);
|
||||||
participants.push(("organizer".to_owned(), p));
|
participants.push(("organizer".to_owned(), p));
|
||||||
@@ -275,7 +271,10 @@ pub fn convert_event(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !participants.is_empty() {
|
if !participants.is_empty() {
|
||||||
let map: Map<String, Value> = participants.into_iter().collect();
|
let mut map: Map<String, Value> = participants.into_iter().collect();
|
||||||
|
if !card.contains_key("organizerCalendarAddress") {
|
||||||
|
drop_calendar_address_dependents(&mut map);
|
||||||
|
}
|
||||||
card.insert("participants".to_owned(), Value::Object(map));
|
card.insert("participants".to_owned(), Value::Object(map));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -346,6 +345,23 @@ fn extract_local_datetime(slot: Option<&Value>) -> Option<String> {
|
|||||||
Some(dt.to_owned())
|
Some(dt.to_owned())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn non_empty(email: &Value, key: &str) -> Option<String> {
|
||||||
|
email
|
||||||
|
.get(key)
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.map(str::to_owned)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolve_calendar_address(email: &Value) -> Option<(String, Option<String>)> {
|
||||||
|
if let Some(addr) = non_empty(email, "address") {
|
||||||
|
let cal_addr = format!("mailto:{addr}");
|
||||||
|
return Some((cal_addr, Some(addr)));
|
||||||
|
}
|
||||||
|
non_empty(email, "name").map(|name| (synthetic_attendee_address(&name), None))
|
||||||
|
}
|
||||||
|
|
||||||
fn build_participant(
|
fn build_participant(
|
||||||
email: &Value,
|
email: &Value,
|
||||||
roles: &[&str],
|
roles: &[&str],
|
||||||
@@ -354,31 +370,28 @@ fn build_participant(
|
|||||||
) -> Value {
|
) -> Value {
|
||||||
let mut map = Map::new();
|
let mut map = Map::new();
|
||||||
map.insert("@type".to_owned(), Value::from("Participant"));
|
map.insert("@type".to_owned(), Value::from("Participant"));
|
||||||
if let Some(addr) = email.get("address").and_then(Value::as_str) {
|
if let Some(name) = non_empty(email, "name") {
|
||||||
map.insert(
|
map.insert("name".to_owned(), Value::from(name));
|
||||||
"calendarAddress".to_owned(),
|
|
||||||
Value::from(format!("mailto:{addr}")),
|
|
||||||
);
|
|
||||||
map.insert("email".to_owned(), Value::from(addr.to_owned()));
|
|
||||||
}
|
}
|
||||||
if let Some(name) = email.get("name").and_then(Value::as_str)
|
if let Some((cal_addr, smtp)) = resolve_calendar_address(email) {
|
||||||
&& !name.is_empty()
|
map.insert("calendarAddress".to_owned(), Value::from(cal_addr));
|
||||||
{
|
if let Some(addr) = smtp {
|
||||||
map.insert("name".to_owned(), Value::from(name.to_owned()));
|
map.insert("email".to_owned(), Value::from(addr));
|
||||||
}
|
}
|
||||||
let role_map: Map<String, Value> = roles
|
let role_map: Map<String, Value> = roles
|
||||||
.iter()
|
.iter()
|
||||||
.map(|r| ((*r).to_owned(), Value::Bool(true)))
|
.map(|r| ((*r).to_owned(), Value::Bool(true)))
|
||||||
.collect();
|
.collect();
|
||||||
map.insert("roles".to_owned(), Value::Object(role_map));
|
map.insert("roles".to_owned(), Value::Object(role_map));
|
||||||
if let Some(status) = participation_status {
|
if let Some(status) = participation_status {
|
||||||
map.insert(
|
map.insert(
|
||||||
"participationStatus".to_owned(),
|
"participationStatus".to_owned(),
|
||||||
Value::from(status.to_owned()),
|
Value::from(status.to_owned()),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if let Some(b) = expect_reply {
|
if let Some(b) = expect_reply {
|
||||||
map.insert("expectReply".to_owned(), Value::Bool(b));
|
map.insert("expectReply".to_owned(), Value::Bool(b));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Value::Object(map)
|
Value::Object(map)
|
||||||
}
|
}
|
||||||
@@ -650,6 +663,91 @@ mod tests {
|
|||||||
assert_eq!(participants["att-1"]["roles"]["required"], true);
|
assert_eq!(participants["att-1"]["roles"]["required"], true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_name_only_attendee_gets_a_stable_synthetic_calendar_address() {
|
||||||
|
let mut v = sample();
|
||||||
|
v["attendees"] = json!([
|
||||||
|
{
|
||||||
|
"emailAddress": {"name": "Room 4"},
|
||||||
|
"type": "resource",
|
||||||
|
"status": {"response": "accepted"}
|
||||||
|
}
|
||||||
|
]);
|
||||||
|
let conv = convert_event(&v, None).unwrap();
|
||||||
|
let p = &conv.data["participants"]["att-1"];
|
||||||
|
let addr = p["calendarAddress"].as_str().unwrap();
|
||||||
|
assert!(
|
||||||
|
addr.starts_with("urn:x-vandelay:attendee:"),
|
||||||
|
"a name-only attendee needs a synthetic calendarAddress, got {addr}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
p.get("email").is_none(),
|
||||||
|
"there is no SMTP address to report as email"
|
||||||
|
);
|
||||||
|
assert_eq!(p["participationStatus"], "accepted");
|
||||||
|
assert_eq!(p["roles"]["informational"], true);
|
||||||
|
assert_eq!(
|
||||||
|
addr,
|
||||||
|
convert_event(&v, None).unwrap().data["participants"]["att-1"]["calendarAddress"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_unidentifiable_attendee_keeps_no_calendar_address_dependent_property() {
|
||||||
|
let mut v = sample();
|
||||||
|
v["attendees"] = json!([
|
||||||
|
{"emailAddress": {}, "type": "required", "status": {"response": "accepted"}}
|
||||||
|
]);
|
||||||
|
let conv = convert_event(&v, None).unwrap();
|
||||||
|
let p = &conv.data["participants"]["att-1"];
|
||||||
|
for key in [
|
||||||
|
"calendarAddress",
|
||||||
|
"email",
|
||||||
|
"roles",
|
||||||
|
"participationStatus",
|
||||||
|
"expectReply",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
p.get(key).is_none(),
|
||||||
|
"{key} requires calendarAddress (jscalendarbis 3.4.6)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(p["@type"], "Participant");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_organizer_without_an_address_still_yields_a_calendar_address() {
|
||||||
|
let mut v = sample();
|
||||||
|
v["organizer"] = json!({"emailAddress": {"name": "Alice Example"}});
|
||||||
|
let conv = convert_event(&v, None).unwrap();
|
||||||
|
let organizer_address = conv.data["organizerCalendarAddress"].as_str().unwrap();
|
||||||
|
assert!(organizer_address.starts_with("urn:x-vandelay:attendee:"));
|
||||||
|
assert_eq!(
|
||||||
|
conv.data["participants"]["organizer"]["calendarAddress"],
|
||||||
|
organizer_address
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn participants_drop_calendar_addresses_when_no_organizer_is_known() {
|
||||||
|
let mut v = sample();
|
||||||
|
v.as_object_mut().unwrap().remove("organizer");
|
||||||
|
let conv = convert_event(&v, None).unwrap();
|
||||||
|
assert!(
|
||||||
|
conv.data.get("organizerCalendarAddress").is_none(),
|
||||||
|
"no organizer is derivable from the source event"
|
||||||
|
);
|
||||||
|
let participants = conv.data["participants"].as_object().unwrap();
|
||||||
|
assert!(!participants.is_empty());
|
||||||
|
for (key, participant) in participants {
|
||||||
|
assert!(
|
||||||
|
participant.get("calendarAddress").is_none(),
|
||||||
|
"{key} may not carry a calendarAddress without organizerCalendarAddress"
|
||||||
|
);
|
||||||
|
assert!(participant.get("roles").is_none(), "{key} kept roles");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn optional_attendee_keeps_role_and_uses_status_when_present() {
|
fn optional_attendee_keeps_role_and_uses_status_when_present() {
|
||||||
let mut v = sample();
|
let mut v = sample();
|
||||||
|
|||||||
@@ -59,6 +59,59 @@ impl EventBodyFormat {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub struct Surfaces {
|
||||||
|
pub mail: bool,
|
||||||
|
pub calendar: bool,
|
||||||
|
pub contacts: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for Surfaces {
|
||||||
|
fn default() -> Self {
|
||||||
|
Surfaces::ALL
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Surfaces {
|
||||||
|
pub const ALL: Surfaces = Surfaces {
|
||||||
|
mail: true,
|
||||||
|
calendar: true,
|
||||||
|
contacts: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub const NONE: Surfaces = Surfaces {
|
||||||
|
mail: false,
|
||||||
|
calendar: false,
|
||||||
|
contacts: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub fn parse_list(list: &str) -> Result<Surfaces, Error> {
|
||||||
|
let mut selected = Surfaces::NONE;
|
||||||
|
for token in list.split(',') {
|
||||||
|
let token = token.trim();
|
||||||
|
if token.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
match token.to_ascii_lowercase().as_str() {
|
||||||
|
"mail" => selected.mail = true,
|
||||||
|
"calendar" => selected.calendar = true,
|
||||||
|
"contacts" => selected.contacts = true,
|
||||||
|
_ => {
|
||||||
|
return Err(Error::Usage(format!(
|
||||||
|
"unknown surface: {token} (valid: mail, calendar, contacts)"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if selected == Surfaces::NONE {
|
||||||
|
return Err(Error::Usage(
|
||||||
|
"--objects given but resolved to an empty surface list".to_owned(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(selected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct ResolvedPrincipal {
|
pub struct ResolvedPrincipal {
|
||||||
pub id: String,
|
pub id: String,
|
||||||
@@ -102,6 +155,86 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn surfaces_default_is_all_three() {
|
||||||
|
let all = Surfaces::default();
|
||||||
|
assert!(all.mail && all.calendar && all.contacts);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn surfaces_parse_each_name() {
|
||||||
|
assert_eq!(
|
||||||
|
Surfaces::parse_list("mail").unwrap(),
|
||||||
|
Surfaces {
|
||||||
|
mail: true,
|
||||||
|
calendar: false,
|
||||||
|
contacts: false
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
Surfaces::parse_list("calendar").unwrap(),
|
||||||
|
Surfaces {
|
||||||
|
mail: false,
|
||||||
|
calendar: true,
|
||||||
|
contacts: false
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
Surfaces::parse_list("contacts").unwrap(),
|
||||||
|
Surfaces {
|
||||||
|
mail: false,
|
||||||
|
calendar: false,
|
||||||
|
contacts: true
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn surfaces_parse_is_case_insensitive() {
|
||||||
|
assert_eq!(
|
||||||
|
Surfaces::parse_list("Mail,CALENDAR,Contacts").unwrap(),
|
||||||
|
Surfaces::ALL
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn surfaces_parse_dedups_and_trims() {
|
||||||
|
assert_eq!(
|
||||||
|
Surfaces::parse_list(" mail , contacts ,mail,, ").unwrap(),
|
||||||
|
Surfaces {
|
||||||
|
mail: true,
|
||||||
|
calendar: false,
|
||||||
|
contacts: true
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn surfaces_parse_rejects_unknown_name() {
|
||||||
|
let err = Surfaces::parse_list("mail,contact").unwrap_err();
|
||||||
|
let msg = err.to_string();
|
||||||
|
assert!(msg.contains("unknown surface: contact"), "msg was: {msg}");
|
||||||
|
assert!(
|
||||||
|
msg.contains("valid: mail, calendar, contacts"),
|
||||||
|
"msg was: {msg}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn surfaces_parse_rejects_jmap_type_names() {
|
||||||
|
assert!(Surfaces::parse_list("mailbox").is_err());
|
||||||
|
assert!(Surfaces::parse_list("contactcard").is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn surfaces_parse_rejects_empty_list() {
|
||||||
|
let err = Surfaces::parse_list(" , ").unwrap_err();
|
||||||
|
assert!(
|
||||||
|
err.to_string().contains("empty surface list"),
|
||||||
|
"msg was: {err}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn body_format_prefer_value() {
|
fn body_format_prefer_value() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|||||||
+437
-10
@@ -4,10 +4,32 @@
|
|||||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
use base64::Engine;
|
||||||
|
use base64::engine::general_purpose::STANDARD;
|
||||||
use serde_json::{Map, Value};
|
use serde_json::{Map, Value};
|
||||||
|
|
||||||
|
use crate::jmap::error::JmapError;
|
||||||
|
|
||||||
pub const SENTINEL_KEY: &str = "@blob";
|
pub const SENTINEL_KEY: &str = "@blob";
|
||||||
|
|
||||||
|
const DEFAULT_MEDIA_TYPE: &str = "application/octet-stream";
|
||||||
|
const MEDIA_TYPE_KEYS: [&str; 2] = ["mediaType", "contentType"];
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub enum InlineShape {
|
||||||
|
JsContactResource,
|
||||||
|
JsCalendarLink,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl InlineShape {
|
||||||
|
fn uri_key(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
InlineShape::JsContactResource => "uri",
|
||||||
|
InlineShape::JsCalendarLink => "href",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn import_blob_ids<F>(value: &mut Value, mut resolve: F) -> Result<(), BlobWalkError>
|
pub fn import_blob_ids<F>(value: &mut Value, mut resolve: F) -> Result<(), BlobWalkError>
|
||||||
where
|
where
|
||||||
F: FnMut(&str) -> Result<i64, BlobWalkError>,
|
F: FnMut(&str) -> Result<i64, BlobWalkError>,
|
||||||
@@ -39,31 +61,36 @@ where
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn export_blob_ids<F>(value: &mut Value, mut resolve: F) -> Result<(), BlobWalkError>
|
pub fn inline_blob_data_uris<F>(
|
||||||
|
value: &mut Value,
|
||||||
|
shape: InlineShape,
|
||||||
|
mut fetch: F,
|
||||||
|
) -> Result<(), BlobWalkError>
|
||||||
where
|
where
|
||||||
F: FnMut(i64) -> Result<String, BlobWalkError>,
|
F: FnMut(i64) -> Result<Vec<u8>, BlobWalkError>,
|
||||||
{
|
{
|
||||||
walk_export(value, &mut resolve)
|
walk_inline(value, shape, &mut fetch)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn walk_export<F>(value: &mut Value, resolve: &mut F) -> Result<(), BlobWalkError>
|
fn walk_inline<F>(value: &mut Value, shape: InlineShape, fetch: &mut F) -> Result<(), BlobWalkError>
|
||||||
where
|
where
|
||||||
F: FnMut(i64) -> Result<String, BlobWalkError>,
|
F: FnMut(i64) -> Result<Vec<u8>, BlobWalkError>,
|
||||||
{
|
{
|
||||||
match value {
|
match value {
|
||||||
Value::Object(map) => {
|
Value::Object(map) => {
|
||||||
if let Some(sentinel) = map.remove(SENTINEL_KEY) {
|
if let Some(sentinel) = map.remove(SENTINEL_KEY) {
|
||||||
let local_id = sentinel.as_i64().ok_or(BlobWalkError::MalformedSentinel)?;
|
let local_id = sentinel.as_i64().ok_or(BlobWalkError::MalformedSentinel)?;
|
||||||
let target_jmap_id = resolve(local_id)?;
|
let bytes = fetch(local_id)?;
|
||||||
map.insert("blobId".to_owned(), Value::String(target_jmap_id));
|
let uri = data_uri(map, &bytes);
|
||||||
|
map.insert(shape.uri_key().to_owned(), Value::String(uri));
|
||||||
}
|
}
|
||||||
for child in map.values_mut() {
|
for child in map.values_mut() {
|
||||||
walk_export(child, resolve)?;
|
walk_inline(child, shape, fetch)?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Value::Array(items) => {
|
Value::Array(items) => {
|
||||||
for item in items {
|
for item in items {
|
||||||
walk_export(item, resolve)?;
|
walk_inline(item, shape, fetch)?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ => {}
|
_ => {}
|
||||||
@@ -71,6 +98,70 @@ where
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn data_uri(map: &Map<String, Value>, bytes: &[u8]) -> String {
|
||||||
|
let media_type = MEDIA_TYPE_KEYS
|
||||||
|
.iter()
|
||||||
|
.find_map(|k| map.get(*k).and_then(Value::as_str).and_then(uri_media_type))
|
||||||
|
.unwrap_or_else(|| DEFAULT_MEDIA_TYPE.to_owned());
|
||||||
|
let mut uri = String::with_capacity(13 + media_type.len() + bytes.len().div_ceil(3) * 4);
|
||||||
|
uri.push_str("data:");
|
||||||
|
uri.push_str(&media_type);
|
||||||
|
uri.push_str(";base64,");
|
||||||
|
STANDARD.encode_string(bytes, &mut uri);
|
||||||
|
uri
|
||||||
|
}
|
||||||
|
|
||||||
|
fn uri_media_type(raw: &str) -> Option<String> {
|
||||||
|
let mut parts = raw.split(';').map(str::trim);
|
||||||
|
let essence = parts.next()?;
|
||||||
|
let (ty, subtype) = essence.split_once('/')?;
|
||||||
|
if ty.is_empty() || subtype.is_empty() || subtype.contains('/') {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let mut out = String::with_capacity(raw.len());
|
||||||
|
escape_into(&mut out, essence);
|
||||||
|
for parameter in parts.filter(|p| !p.is_empty()) {
|
||||||
|
out.push(';');
|
||||||
|
escape_into(&mut out, parameter);
|
||||||
|
}
|
||||||
|
Some(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn escape_into(out: &mut String, part: &str) {
|
||||||
|
const HEX: &[u8; 16] = b"0123456789ABCDEF";
|
||||||
|
for byte in part.bytes() {
|
||||||
|
if is_media_type_uri_byte(byte) {
|
||||||
|
out.push(byte as char);
|
||||||
|
} else {
|
||||||
|
out.push('%');
|
||||||
|
out.push(HEX[(byte >> 4) as usize] as char);
|
||||||
|
out.push(HEX[(byte & 0x0F) as usize] as char);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_media_type_uri_byte(byte: u8) -> bool {
|
||||||
|
byte.is_ascii_alphanumeric()
|
||||||
|
|| matches!(
|
||||||
|
byte,
|
||||||
|
b'!' | b'$'
|
||||||
|
| b'&'
|
||||||
|
| b'\''
|
||||||
|
| b'('
|
||||||
|
| b')'
|
||||||
|
| b'*'
|
||||||
|
| b'+'
|
||||||
|
| b'-'
|
||||||
|
| b'.'
|
||||||
|
| b'/'
|
||||||
|
| b':'
|
||||||
|
| b'='
|
||||||
|
| b'@'
|
||||||
|
| b'_'
|
||||||
|
| b'~'
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn prepend_property(target: &mut Map<String, Value>, key: &str, value: Value) {
|
pub fn prepend_property(target: &mut Map<String, Value>, key: &str, value: Value) {
|
||||||
let existing: Vec<(String, Value)> =
|
let existing: Vec<(String, Value)> =
|
||||||
target.iter().map(|(k, v)| (k.clone(), v.clone())).collect();
|
target.iter().map(|(k, v)| (k.clone(), v.clone())).collect();
|
||||||
@@ -88,5 +179,341 @@ pub enum BlobWalkError {
|
|||||||
#[error("malformed @blob sentinel: expected integer local id")]
|
#[error("malformed @blob sentinel: expected integer local id")]
|
||||||
MalformedSentinel,
|
MalformedSentinel,
|
||||||
#[error("blob resolver failed: {0}")]
|
#[error("blob resolver failed: {0}")]
|
||||||
Resolver(String),
|
Resolver(Box<JmapError>),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BlobWalkError {
|
||||||
|
pub fn resolver(source: JmapError) -> BlobWalkError {
|
||||||
|
BlobWalkError::Resolver(Box::new(source))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn into_source(self) -> JmapError {
|
||||||
|
match self {
|
||||||
|
BlobWalkError::Resolver(source) => *source,
|
||||||
|
other => JmapError::Blob(other),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
fn fetch_ok(local_id: i64) -> Result<Vec<u8>, BlobWalkError> {
|
||||||
|
Ok(format!("bytes-{local_id}").into_bytes())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode(uri: &str, expect_media_type: &str) -> Vec<u8> {
|
||||||
|
assert_well_formed(uri);
|
||||||
|
let prefix = format!("data:{expect_media_type};base64,");
|
||||||
|
let payload = uri
|
||||||
|
.strip_prefix(&prefix)
|
||||||
|
.unwrap_or_else(|| panic!("{uri} does not start with {prefix}"));
|
||||||
|
STANDARD.decode(payload).expect("base64 payload")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_well_formed(uri: &str) {
|
||||||
|
let rest = uri
|
||||||
|
.strip_prefix("data:")
|
||||||
|
.unwrap_or_else(|| panic!("{uri} is not a data URI"));
|
||||||
|
let (media_type, payload) = rest
|
||||||
|
.split_once(";base64,")
|
||||||
|
.unwrap_or_else(|| panic!("{uri} carries no ;base64, separator"));
|
||||||
|
let bytes = media_type.as_bytes();
|
||||||
|
let mut i = 0;
|
||||||
|
while i < bytes.len() {
|
||||||
|
let b = bytes[i];
|
||||||
|
if b == b'%' {
|
||||||
|
assert!(
|
||||||
|
i + 2 < bytes.len()
|
||||||
|
&& bytes[i + 1].is_ascii_hexdigit()
|
||||||
|
&& bytes[i + 2].is_ascii_hexdigit(),
|
||||||
|
"truncated percent escape in {uri}"
|
||||||
|
);
|
||||||
|
i += 3;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
is_media_type_uri_byte(b) || b == b';',
|
||||||
|
"byte {b:#04x} is not allowed unescaped in {uri}"
|
||||||
|
);
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
payload
|
||||||
|
.bytes()
|
||||||
|
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'/' | b'=')),
|
||||||
|
"non-base64 payload in {uri}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_no_blob_id(value: &Value) {
|
||||||
|
match value {
|
||||||
|
Value::Object(map) => {
|
||||||
|
assert!(map.get("blobId").is_none(), "blobId survived in {value}");
|
||||||
|
assert!(map.get(SENTINEL_KEY).is_none(), "@blob survived in {value}");
|
||||||
|
for child in map.values() {
|
||||||
|
assert_no_blob_id(child);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Value::Array(items) => items.iter().for_each(assert_no_blob_id),
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn media_sentinel_becomes_a_uri_property() {
|
||||||
|
let mut card = json!({
|
||||||
|
"@type": "Card",
|
||||||
|
"media": { "photo": {
|
||||||
|
"@type": "Media", "kind": "photo",
|
||||||
|
"@blob": 7, "mediaType": "image/png"
|
||||||
|
} }
|
||||||
|
});
|
||||||
|
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap();
|
||||||
|
let photo = &card["media"]["photo"];
|
||||||
|
assert_eq!(
|
||||||
|
decode(photo["uri"].as_str().unwrap(), "image/png"),
|
||||||
|
b"bytes-7"
|
||||||
|
);
|
||||||
|
assert_eq!(photo["mediaType"], json!("image/png"));
|
||||||
|
assert!(photo.get("href").is_none());
|
||||||
|
assert_no_blob_id(&card);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn calendar_link_sentinel_becomes_an_href_property() {
|
||||||
|
let mut event = json!({
|
||||||
|
"@type": "Event",
|
||||||
|
"links": { "1": {
|
||||||
|
"@type": "Link", "rel": "enclosure",
|
||||||
|
"@blob": 3, "contentType": "text/plain", "title": "note.txt"
|
||||||
|
} }
|
||||||
|
});
|
||||||
|
inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap();
|
||||||
|
let link = &event["links"]["1"];
|
||||||
|
assert_eq!(
|
||||||
|
decode(link["href"].as_str().unwrap(), "text/plain"),
|
||||||
|
b"bytes-3"
|
||||||
|
);
|
||||||
|
assert_eq!(link["contentType"], json!("text/plain"));
|
||||||
|
assert_eq!(link["title"], json!("note.txt"));
|
||||||
|
assert!(link.get("uri").is_none());
|
||||||
|
assert_no_blob_id(&event);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn jscontact_link_keeps_the_resource_uri_property() {
|
||||||
|
let mut card = json!({
|
||||||
|
"@type": "Card",
|
||||||
|
"links": { "l1": { "@type": "Link", "@blob": 1, "mediaType": "application/pdf" } }
|
||||||
|
});
|
||||||
|
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap();
|
||||||
|
let link = &card["links"]["l1"];
|
||||||
|
assert!(link.get("href").is_none(), "JSContact Link uses uri");
|
||||||
|
assert_eq!(
|
||||||
|
decode(link["uri"].as_str().unwrap(), "application/pdf"),
|
||||||
|
b"bytes-1"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn absent_media_type_defaults_to_octet_stream() {
|
||||||
|
let mut card = json!({ "media": { "photo": { "@blob": 2 } } });
|
||||||
|
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap();
|
||||||
|
let uri = card["media"]["photo"]["uri"].as_str().unwrap();
|
||||||
|
assert_eq!(decode(uri, DEFAULT_MEDIA_TYPE), b"bytes-2");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn empty_media_type_defaults_to_octet_stream() {
|
||||||
|
let mut event = json!({ "links": { "1": { "@blob": 4, "contentType": "" } } });
|
||||||
|
inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap();
|
||||||
|
let uri = event["links"]["1"]["href"].as_str().unwrap();
|
||||||
|
assert_eq!(decode(uri, DEFAULT_MEDIA_TYPE), b"bytes-4");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn empty_media_type_falls_through_to_the_next_key() {
|
||||||
|
let mut card = json!({ "media": { "photo": {
|
||||||
|
"@blob": 4, "mediaType": "", "contentType": "image/jpeg"
|
||||||
|
} } });
|
||||||
|
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap();
|
||||||
|
let uri = card["media"]["photo"]["uri"].as_str().unwrap();
|
||||||
|
assert_eq!(decode(uri, "image/jpeg"), b"bytes-4");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn media_type_parameters_are_kept_without_whitespace() {
|
||||||
|
let mut event = json!({ "links": { "1": {
|
||||||
|
"@blob": 4, "contentType": "text/plain; charset=utf-8"
|
||||||
|
} } });
|
||||||
|
inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap();
|
||||||
|
let uri = event["links"]["1"]["href"].as_str().unwrap();
|
||||||
|
assert!(!uri.contains(' '), "{uri} carries a raw space");
|
||||||
|
assert_eq!(decode(uri, "text/plain;charset=utf-8"), b"bytes-4");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn media_type_keeps_several_parameters_and_drops_empty_ones() {
|
||||||
|
let mut event = json!({ "links": { "1": {
|
||||||
|
"@blob": 4, "contentType": " text/plain ;; charset=utf-8 ; format=flowed "
|
||||||
|
} } });
|
||||||
|
inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap();
|
||||||
|
let uri = event["links"]["1"]["href"].as_str().unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
decode(uri, "text/plain;charset=utf-8;format=flowed"),
|
||||||
|
b"bytes-4"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn media_type_characters_illegal_in_a_uri_are_percent_encoded() {
|
||||||
|
let mut event = json!({ "links": { "1": {
|
||||||
|
"@blob": 4, "contentType": "text/plain; name=\"rep ort,v2#1\u{e9}.txt\""
|
||||||
|
} } });
|
||||||
|
inline_blob_data_uris(&mut event, InlineShape::JsCalendarLink, fetch_ok).unwrap();
|
||||||
|
let uri = event["links"]["1"]["href"].as_str().unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
decode(uri, "text/plain;name=%22rep%20ort%2Cv2%231%C3%A9.txt%22"),
|
||||||
|
b"bytes-4"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_percent_in_the_media_type_is_itself_escaped() {
|
||||||
|
let mut card = json!({ "media": { "photo": {
|
||||||
|
"@blob": 4, "mediaType": "image/png; note=100%"
|
||||||
|
} } });
|
||||||
|
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap();
|
||||||
|
let uri = card["media"]["photo"]["uri"].as_str().unwrap();
|
||||||
|
assert_eq!(decode(uri, "image/png;note=100%25"), b"bytes-4");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_media_type_that_is_not_type_slash_subtype_defaults_to_octet_stream() {
|
||||||
|
for bogus in ["png", "image/", "/png", "image/png/extra", ";charset=utf-8"] {
|
||||||
|
let mut card = json!({ "media": { "photo": { "@blob": 4, "mediaType": bogus } } });
|
||||||
|
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, fetch_ok).unwrap();
|
||||||
|
let uri = card["media"]["photo"]["uri"].as_str().unwrap();
|
||||||
|
assert_eq!(decode(uri, DEFAULT_MEDIA_TYPE), b"bytes-4", "{bogus}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn inlined_bytes_round_trip_exactly() {
|
||||||
|
let payload: Vec<u8> = (0u16..=255).map(|b| b as u8).collect();
|
||||||
|
let mut card = json!({ "media": { "photo": {
|
||||||
|
"@blob": 1, "mediaType": "application/octet-stream; charset=binary"
|
||||||
|
} } });
|
||||||
|
inline_blob_data_uris(&mut card, InlineShape::JsContactResource, |_| {
|
||||||
|
Ok(payload.clone())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let uri = card["media"]["photo"]["uri"].as_str().unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
decode(uri, "application/octet-stream;charset=binary"),
|
||||||
|
payload
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sentinels_nested_in_arrays_and_subobjects_are_all_inlined() {
|
||||||
|
let mut value = json!({
|
||||||
|
"locations": { "loc1": { "links": [
|
||||||
|
{ "@blob": 5, "contentType": "image/gif" },
|
||||||
|
{ "@blob": 6 }
|
||||||
|
] } },
|
||||||
|
"links": { "top": { "@blob": 8, "contentType": "image/gif" } }
|
||||||
|
});
|
||||||
|
inline_blob_data_uris(&mut value, InlineShape::JsCalendarLink, fetch_ok).unwrap();
|
||||||
|
let arr = value["locations"]["loc1"]["links"].as_array().unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
decode(arr[0]["href"].as_str().unwrap(), "image/gif"),
|
||||||
|
b"bytes-5"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
decode(arr[1]["href"].as_str().unwrap(), DEFAULT_MEDIA_TYPE),
|
||||||
|
b"bytes-6"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
decode(value["links"]["top"]["href"].as_str().unwrap(), "image/gif"),
|
||||||
|
b"bytes-8"
|
||||||
|
);
|
||||||
|
assert_no_blob_id(&value);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn malformed_sentinel_is_rejected() {
|
||||||
|
let mut value = json!({ "media": { "photo": { "@blob": "not-an-id" } } });
|
||||||
|
let err = inline_blob_data_uris(&mut value, InlineShape::JsContactResource, fetch_ok)
|
||||||
|
.expect_err("malformed sentinel");
|
||||||
|
assert!(matches!(err, BlobWalkError::MalformedSentinel), "{err:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolver_failure_propagates() {
|
||||||
|
let mut value = json!({ "media": { "photo": { "@blob": 9 } } });
|
||||||
|
let err = inline_blob_data_uris(&mut value, InlineShape::JsContactResource, |_| {
|
||||||
|
Err(BlobWalkError::resolver(JmapError::malformed("gone")))
|
||||||
|
})
|
||||||
|
.expect_err("resolver failure");
|
||||||
|
assert!(matches!(err, BlobWalkError::Resolver(_)), "{err:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_resolver_error_kind_survives_the_walk() {
|
||||||
|
let mut value = json!({ "media": { "photo": { "@blob": 9 } } });
|
||||||
|
let err = inline_blob_data_uris(&mut value, InlineShape::JsContactResource, |_| {
|
||||||
|
Err(BlobWalkError::resolver(JmapError::Sqlite(
|
||||||
|
rusqlite::Error::QueryReturnedNoRows,
|
||||||
|
)))
|
||||||
|
})
|
||||||
|
.expect_err("resolver failure")
|
||||||
|
.into_source();
|
||||||
|
assert!(matches!(err, JmapError::Sqlite(_)), "{err:?}");
|
||||||
|
let mapped = crate::error::Error::from(err);
|
||||||
|
assert!(mapped.aborts_run(), "{mapped} must abort the run");
|
||||||
|
assert_eq!(mapped.exit_code(), 7);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_malformed_sentinel_stays_a_blob_walk_error() {
|
||||||
|
let err = BlobWalkError::MalformedSentinel.into_source();
|
||||||
|
assert!(
|
||||||
|
matches!(err, JmapError::Blob(BlobWalkError::MalformedSentinel)),
|
||||||
|
"{err:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(crate::error::Error::from(err).exit_code(), 5);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn values_without_a_sentinel_are_untouched() {
|
||||||
|
let original = json!({
|
||||||
|
"@type": "Card",
|
||||||
|
"media": { "photo": { "@type": "Media", "kind": "photo",
|
||||||
|
"uri": "https://example.test/p.png", "mediaType": "image/png" } },
|
||||||
|
"nickNames": { "n1": { "name": "Jay" } }
|
||||||
|
});
|
||||||
|
let mut value = original.clone();
|
||||||
|
inline_blob_data_uris(&mut value, InlineShape::JsContactResource, |_| {
|
||||||
|
panic!("fetch must not be called")
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(value, original);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn import_walker_still_produces_the_sentinel() {
|
||||||
|
let mut value =
|
||||||
|
json!({ "media": { "photo": { "blobId": "B1", "mediaType": "image/png" } } });
|
||||||
|
import_blob_ids(&mut value, |id| {
|
||||||
|
assert_eq!(id, "B1");
|
||||||
|
Ok(11)
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(value["media"]["photo"][SENTINEL_KEY], json!(11));
|
||||||
|
assert!(value["media"]["photo"].get("blobId").is_none());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,6 +31,9 @@ pub fn prepare(ctx: &Context, connect: &ConnectConfig) -> Result<Connected, Erro
|
|||||||
ctx.client.set_logger(ctx.common.logger);
|
ctx.client.set_logger(ctx.common.logger);
|
||||||
|
|
||||||
let session = Session::discover(&ctx.client, &connect.url)?;
|
let session = Session::discover(&ctx.client, &connect.url)?;
|
||||||
|
for mismatch in session.origin_mismatches(&connect.url) {
|
||||||
|
ctx.common.logger.warn(&mismatch.to_string());
|
||||||
|
}
|
||||||
let limits = session.core_limits()?;
|
let limits = session.core_limits()?;
|
||||||
ctx.client.set_limits(&limits);
|
ctx.client.set_limits(&limits);
|
||||||
let account_id = account::resolve(&connect.account, &session, &ctx.client)?;
|
let account_id = account::resolve(&connect.account, &session, &ctx.client)?;
|
||||||
|
|||||||
+57
-1
@@ -4,6 +4,7 @@
|
|||||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
use crate::db::init::OpenError;
|
||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
use crate::jmap::blob::BlobWalkError;
|
use crate::jmap::blob::BlobWalkError;
|
||||||
|
|
||||||
@@ -70,7 +71,62 @@ impl From<JmapError> for Error {
|
|||||||
match value {
|
match value {
|
||||||
JmapError::Connect(m) | JmapError::Transport(m) => Error::Connection(m),
|
JmapError::Connect(m) | JmapError::Transport(m) => Error::Connection(m),
|
||||||
JmapError::Auth(m) => Error::Connection(format!("authentication rejected: {m}")),
|
JmapError::Auth(m) => Error::Connection(format!("authentication rejected: {m}")),
|
||||||
other => Error::Connection(other.to_string()),
|
JmapError::Sqlite(e) => Error::Db(OpenError::Sqlite(e)),
|
||||||
|
reached @ (JmapError::HttpStatus { .. } | JmapError::RetriesExhausted(_)) => {
|
||||||
|
Error::Connection(reached.to_string())
|
||||||
|
}
|
||||||
|
per_unit => Error::Partial(per_unit.to_string()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unreachable_endpoint_aborts_the_run() {
|
||||||
|
for e in [
|
||||||
|
JmapError::Connect("refused".to_owned()),
|
||||||
|
JmapError::Transport("io: broken pipe".to_owned()),
|
||||||
|
JmapError::Auth("bad password".to_owned()),
|
||||||
|
JmapError::HttpStatus {
|
||||||
|
status: 404,
|
||||||
|
body: "<html>404 Not Found</html>".to_owned(),
|
||||||
|
},
|
||||||
|
JmapError::RetriesExhausted("gave up".to_owned()),
|
||||||
|
] {
|
||||||
|
let mapped = Error::from(e);
|
||||||
|
assert!(mapped.aborts_run(), "{mapped} must abort the run");
|
||||||
|
assert_eq!(mapped.exit_code(), 2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn protocol_level_errors_stay_per_unit() {
|
||||||
|
for e in [
|
||||||
|
JmapError::Method {
|
||||||
|
call_id: "q".to_owned(),
|
||||||
|
error_type: "unknownMethod".to_owned(),
|
||||||
|
description: None,
|
||||||
|
},
|
||||||
|
JmapError::UnknownMethod,
|
||||||
|
JmapError::CannotCalculateChanges,
|
||||||
|
JmapError::AnchorNotFound,
|
||||||
|
JmapError::RequestTooLarge,
|
||||||
|
JmapError::SingleObjectTooLarge("one blob".to_owned()),
|
||||||
|
JmapError::Malformed("not a list".to_owned()),
|
||||||
|
] {
|
||||||
|
let mapped = Error::from(e);
|
||||||
|
assert!(!mapped.aborts_run(), "{mapped} must not abort the run");
|
||||||
|
assert_eq!(mapped.exit_code(), 5);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn archive_failures_are_local_io() {
|
||||||
|
let mapped = Error::from(JmapError::Sqlite(rusqlite::Error::QueryReturnedNoRows));
|
||||||
|
assert!(mapped.aborts_run());
|
||||||
|
assert_eq!(mapped.exit_code(), 7);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+5
-2
@@ -617,9 +617,11 @@ fn format_rate_headers(headers: &[(String, String)]) -> Option<String> {
|
|||||||
Some(parts.join("; "))
|
Some(parts.join("; "))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn format_retry_wait(d: Duration) -> String {
|
pub fn format_retry_wait(d: Duration) -> String {
|
||||||
let total = d.as_secs();
|
let total = d.as_secs();
|
||||||
if total < 60 {
|
if total == 0 {
|
||||||
|
format!("{}ms", d.as_millis())
|
||||||
|
} else if total < 60 {
|
||||||
format!("{total}s")
|
format!("{total}s")
|
||||||
} else if total < 3600 {
|
} else if total < 3600 {
|
||||||
format!("{}m{:02}s", total / 60, total % 60)
|
format!("{}m{:02}s", total / 60, total % 60)
|
||||||
@@ -826,6 +828,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn format_retry_wait_short_medium_long() {
|
fn format_retry_wait_short_medium_long() {
|
||||||
|
assert_eq!(format_retry_wait(Duration::from_millis(250)), "250ms");
|
||||||
assert_eq!(format_retry_wait(Duration::from_secs(7)), "7s");
|
assert_eq!(format_retry_wait(Duration::from_secs(7)), "7s");
|
||||||
assert_eq!(format_retry_wait(Duration::from_secs(67)), "1m07s");
|
assert_eq!(format_retry_wait(Duration::from_secs(67)), "1m07s");
|
||||||
assert_eq!(format_retry_wait(Duration::from_secs(1427)), "23m47s");
|
assert_eq!(format_retry_wait(Duration::from_secs(1427)), "23m47s");
|
||||||
|
|||||||
+129
-1
@@ -9,9 +9,11 @@ use serde::de::DeserializeOwned;
|
|||||||
use serde_json::{Map, Value, json};
|
use serde_json::{Map, Value, json};
|
||||||
|
|
||||||
use crate::jmap::error::JmapError;
|
use crate::jmap::error::JmapError;
|
||||||
use crate::jmap::http::HttpClient;
|
use crate::jmap::http::{HttpClient, format_retry_wait};
|
||||||
|
use crate::jmap::retry::{Disposition, MethodCallKind, backoff_delay, jmap_method_disposition};
|
||||||
use crate::jmap::session::Limits;
|
use crate::jmap::session::Limits;
|
||||||
use crate::jmap::wire::JmapId;
|
use crate::jmap::wire::JmapId;
|
||||||
|
use crate::logging::Logger;
|
||||||
|
|
||||||
pub const URN_CORE: &str = "urn:ietf:params:jmap:core";
|
pub const URN_CORE: &str = "urn:ietf:params:jmap:core";
|
||||||
|
|
||||||
@@ -184,6 +186,43 @@ pub fn check_method_error(mr: &MethodCall) -> Result<(), JmapError> {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn method_error_is_retryable(err: &JmapError, kind: MethodCallKind) -> bool {
|
||||||
|
match err {
|
||||||
|
JmapError::Method { error_type, .. } => {
|
||||||
|
jmap_method_disposition(error_type, kind) == Disposition::Retryable
|
||||||
|
}
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn retry_method_call<T>(
|
||||||
|
client: &HttpClient,
|
||||||
|
kind: MethodCallKind,
|
||||||
|
logger: &Logger,
|
||||||
|
mut attempt: impl FnMut() -> Result<T, JmapError>,
|
||||||
|
) -> Result<T, JmapError> {
|
||||||
|
let policy = *client.retry();
|
||||||
|
let mut tries = 0u32;
|
||||||
|
loop {
|
||||||
|
let err = match attempt() {
|
||||||
|
Ok(value) => return Ok(value),
|
||||||
|
Err(e) => e,
|
||||||
|
};
|
||||||
|
if tries >= policy.max_retries || !method_error_is_retryable(&err, kind) {
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
|
tries += 1;
|
||||||
|
let delay = backoff_delay(&policy, tries);
|
||||||
|
client.rate_limit().cooldown().arm(delay);
|
||||||
|
logger.warn(&format!(
|
||||||
|
"{err}; retrying in {} ({tries}/{})",
|
||||||
|
format_retry_wait(delay),
|
||||||
|
policy.max_retries
|
||||||
|
));
|
||||||
|
std::thread::sleep(delay);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn ids_array(ids: &[JmapId]) -> Value {
|
fn ids_array(ids: &[JmapId]) -> Value {
|
||||||
Value::Array(ids.iter().map(|i| Value::String(i.0.clone())).collect())
|
Value::Array(ids.iter().map(|i| Value::String(i.0.clone())).collect())
|
||||||
}
|
}
|
||||||
@@ -720,7 +759,96 @@ fn decode_set(mr: &MethodCall) -> SetOutcome {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
use std::cell::Cell;
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use crate::error::Error;
|
||||||
|
use crate::jmap::http::{Auth, RetryPolicy};
|
||||||
|
|
||||||
|
fn test_client(max_retries: u32) -> HttpClient {
|
||||||
|
HttpClient::new(
|
||||||
|
Auth::Bearer {
|
||||||
|
token: "t".to_owned(),
|
||||||
|
},
|
||||||
|
RetryPolicy::new(max_retries),
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn method_error(error_type: &str) -> JmapError {
|
||||||
|
JmapError::Method {
|
||||||
|
call_id: "i".to_owned(),
|
||||||
|
error_type: error_type.to_owned(),
|
||||||
|
description: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drive(
|
||||||
|
max_retries: u32,
|
||||||
|
outcome: impl Fn(u32) -> Result<&'static str, JmapError>,
|
||||||
|
) -> (Result<&'static str, JmapError>, u32) {
|
||||||
|
let client = test_client(max_retries);
|
||||||
|
let calls = Cell::new(0u32);
|
||||||
|
let result = retry_method_call(
|
||||||
|
&client,
|
||||||
|
MethodCallKind::SingleObjectWrite,
|
||||||
|
&Logger::from_flags(true, 0),
|
||||||
|
|| {
|
||||||
|
calls.set(calls.get() + 1);
|
||||||
|
outcome(calls.get())
|
||||||
|
},
|
||||||
|
);
|
||||||
|
(result, calls.get())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn transient_method_error_is_retried_until_it_succeeds() {
|
||||||
|
let (result, calls) = drive(3, |n| {
|
||||||
|
if n < 3 {
|
||||||
|
Err(method_error("serverUnavailable"))
|
||||||
|
} else {
|
||||||
|
Ok("created")
|
||||||
|
}
|
||||||
|
});
|
||||||
|
assert_eq!(result.unwrap(), "created");
|
||||||
|
assert_eq!(calls, 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn exhausting_the_budget_returns_the_original_method_error() {
|
||||||
|
let (result, calls) = drive(2, |_| Err(method_error("serverUnavailable")));
|
||||||
|
assert_eq!(calls, 3, "one initial attempt plus max_retries");
|
||||||
|
let err = result.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
matches!(&err, JmapError::Method { error_type, .. } if error_type == "serverUnavailable"),
|
||||||
|
"the caller must still see the method error, not RetriesExhausted: {err}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!Error::from(err).aborts_run(),
|
||||||
|
"an exhausted method retry stays a per-unit failure"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn permanent_method_error_consumes_no_attempt() {
|
||||||
|
let (result, calls) = drive(5, |_| Err(method_error("invalidArguments")));
|
||||||
|
assert!(result.is_err());
|
||||||
|
assert_eq!(calls, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn partial_fail_is_not_retried_for_a_write() {
|
||||||
|
let (result, calls) = drive(5, |_| Err(method_error("serverPartialFail")));
|
||||||
|
assert!(result.is_err());
|
||||||
|
assert_eq!(calls, 1, "retrying a partial write could duplicate it");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn non_method_errors_are_left_to_the_http_layer() {
|
||||||
|
let (result, calls) = drive(5, |_| Err(JmapError::Transport("reset".to_owned())));
|
||||||
|
assert!(result.is_err());
|
||||||
|
assert_eq!(calls, 1);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn using_union_includes_core_and_dedups() {
|
fn using_union_includes_core_and_dedups() {
|
||||||
|
|||||||
+35
-4
@@ -27,9 +27,19 @@ pub fn classify_http_status(status: u16) -> Disposition {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn jmap_method_disposition(error_type: &str) -> Disposition {
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum MethodCallKind {
|
||||||
|
Read,
|
||||||
|
SingleObjectWrite,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn jmap_method_disposition(error_type: &str, kind: MethodCallKind) -> Disposition {
|
||||||
match error_type {
|
match error_type {
|
||||||
"serverUnavailable" | "serverPartialFail" => Disposition::Retryable,
|
"serverUnavailable" => Disposition::Retryable,
|
||||||
|
"serverPartialFail" => match kind {
|
||||||
|
MethodCallKind::Read => Disposition::Retryable,
|
||||||
|
MethodCallKind::SingleObjectWrite => Disposition::Fatal,
|
||||||
|
},
|
||||||
_ => Disposition::Fatal,
|
_ => Disposition::Fatal,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -231,12 +241,33 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn method_classification() {
|
fn method_classification() {
|
||||||
|
for kind in [MethodCallKind::Read, MethodCallKind::SingleObjectWrite] {
|
||||||
|
assert_eq!(
|
||||||
|
jmap_method_disposition("serverUnavailable", kind),
|
||||||
|
Disposition::Retryable,
|
||||||
|
"{kind:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
jmap_method_disposition("invalidArguments", kind),
|
||||||
|
Disposition::Fatal,
|
||||||
|
"{kind:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
jmap_method_disposition("forbidden", kind),
|
||||||
|
Disposition::Fatal,
|
||||||
|
"{kind:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn partial_fail_is_never_retried_for_a_write() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
jmap_method_disposition("serverUnavailable"),
|
jmap_method_disposition("serverPartialFail", MethodCallKind::Read),
|
||||||
Disposition::Retryable
|
Disposition::Retryable
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
jmap_method_disposition("invalidArguments"),
|
jmap_method_disposition("serverPartialFail", MethodCallKind::SingleObjectWrite),
|
||||||
Disposition::Fatal
|
Disposition::Fatal
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
+265
-2
@@ -58,12 +58,100 @@ fn well_known_url(base: &str) -> String {
|
|||||||
format!("{trimmed}/.well-known/jmap")
|
format!("{trimmed}/.well-known/jmap")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn scheme_end(url: &str) -> Option<usize> {
|
||||||
|
let bytes = url.as_bytes();
|
||||||
|
if !bytes.first()?.is_ascii_alphabetic() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
for (index, byte) in bytes.iter().copied().enumerate() {
|
||||||
|
if byte == b':' {
|
||||||
|
return Some(index);
|
||||||
|
}
|
||||||
|
if !(byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'-' | b'.')) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn resolve_reference(base: &str, reference: &str) -> String {
|
||||||
|
if reference.is_empty() {
|
||||||
|
return base.to_owned();
|
||||||
|
}
|
||||||
|
if scheme_end(reference).is_some() {
|
||||||
|
return reference.to_owned();
|
||||||
|
}
|
||||||
|
let Some(base_scheme_end) = scheme_end(base) else {
|
||||||
|
return reference.to_owned();
|
||||||
|
};
|
||||||
|
let scheme = &base[..=base_scheme_end];
|
||||||
|
let Some(authority_and_path) = base[base_scheme_end + 1..].strip_prefix("//") else {
|
||||||
|
return reference.to_owned();
|
||||||
|
};
|
||||||
|
let mut out = String::with_capacity(scheme.len() + 2 + base.len() + reference.len());
|
||||||
|
out.push_str(scheme);
|
||||||
|
out.push_str("//");
|
||||||
|
if let Some(rest) = reference.strip_prefix("//") {
|
||||||
|
out.push_str(rest);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
let authority_len = authority_and_path
|
||||||
|
.find(['/', '?', '#'])
|
||||||
|
.unwrap_or(authority_and_path.len());
|
||||||
|
out.push_str(&authority_and_path[..authority_len]);
|
||||||
|
if reference.starts_with('/') {
|
||||||
|
out.push_str(reference);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
let path = &authority_and_path[authority_len..];
|
||||||
|
let path = path.split(['?', '#']).next().unwrap_or("");
|
||||||
|
match path.rfind('/') {
|
||||||
|
Some(last) => out.push_str(&path[..=last]),
|
||||||
|
None => out.push('/'),
|
||||||
|
}
|
||||||
|
out.push_str(reference);
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn origin_of(url: &str) -> Option<String> {
|
||||||
|
let origin = url::Url::parse(url).ok()?.origin();
|
||||||
|
if !origin.is_tuple() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(origin.ascii_serialization())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct OriginMismatch {
|
||||||
|
pub fields: Vec<&'static str>,
|
||||||
|
pub session_origin: String,
|
||||||
|
pub connect_origin: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for OriginMismatch {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.write_str("session advertises ")?;
|
||||||
|
for (index, field) in self.fields.iter().enumerate() {
|
||||||
|
if index > 0 {
|
||||||
|
f.write_str(", ")?;
|
||||||
|
}
|
||||||
|
f.write_str(field)?;
|
||||||
|
}
|
||||||
|
write!(
|
||||||
|
f,
|
||||||
|
" on {} (connected to {}); vandelay must use the advertised URL. \
|
||||||
|
If this is wrong, fix the server's advertised HTTP URL setting.",
|
||||||
|
self.session_origin, self.connect_origin
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl Session {
|
impl Session {
|
||||||
pub fn discover(client: &HttpClient, url: &str) -> Result<Session, JmapError> {
|
pub fn discover(client: &HttpClient, url: &str) -> Result<Session, JmapError> {
|
||||||
match client.get(url) {
|
match client.get(url) {
|
||||||
Ok(direct) => {
|
Ok(direct) => {
|
||||||
if let Some(session) = parse_session(&direct) {
|
if let Some(session) = parse_session(&direct) {
|
||||||
return session.ensure_authenticated();
|
return session.rebased_on(url).ensure_authenticated();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(JmapError::Auth(m)) => return Err(JmapError::Auth(m)),
|
Err(JmapError::Auth(m)) => return Err(JmapError::Auth(m)),
|
||||||
@@ -73,7 +161,7 @@ impl Session {
|
|||||||
match client.get(&well_known) {
|
match client.get(&well_known) {
|
||||||
Ok(body) => {
|
Ok(body) => {
|
||||||
if let Some(session) = parse_session(&body) {
|
if let Some(session) = parse_session(&body) {
|
||||||
return session.ensure_authenticated();
|
return session.rebased_on(&well_known).ensure_authenticated();
|
||||||
}
|
}
|
||||||
Err(JmapError::Connect(format!(
|
Err(JmapError::Connect(format!(
|
||||||
"no JMAP Session object at {url} or {well_known}"
|
"no JMAP Session object at {url} or {well_known}"
|
||||||
@@ -86,6 +174,44 @@ impl Session {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn rebased_on(mut self, session_resource_url: &str) -> Session {
|
||||||
|
self.api_url = resolve_reference(session_resource_url, &self.api_url);
|
||||||
|
self.upload_url = resolve_reference(session_resource_url, &self.upload_url);
|
||||||
|
self.download_url = resolve_reference(session_resource_url, &self.download_url);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn origin_mismatches(&self, connect_url: &str) -> Vec<OriginMismatch> {
|
||||||
|
let Some(connect_origin) = origin_of(connect_url) else {
|
||||||
|
return Vec::new();
|
||||||
|
};
|
||||||
|
let mut mismatches: Vec<OriginMismatch> = Vec::new();
|
||||||
|
for (field, value) in [
|
||||||
|
("apiUrl", &self.api_url),
|
||||||
|
("uploadUrl", &self.upload_url),
|
||||||
|
("downloadUrl", &self.download_url),
|
||||||
|
] {
|
||||||
|
let Some(session_origin) = origin_of(value) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if session_origin == connect_origin {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
match mismatches
|
||||||
|
.iter_mut()
|
||||||
|
.find(|m| m.session_origin == session_origin)
|
||||||
|
{
|
||||||
|
Some(existing) => existing.fields.push(field),
|
||||||
|
None => mismatches.push(OriginMismatch {
|
||||||
|
fields: vec![field],
|
||||||
|
session_origin,
|
||||||
|
connect_origin: connect_origin.clone(),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
mismatches
|
||||||
|
}
|
||||||
|
|
||||||
fn ensure_authenticated(self) -> Result<Session, JmapError> {
|
fn ensure_authenticated(self) -> Result<Session, JmapError> {
|
||||||
if self.accounts.is_empty() {
|
if self.accounts.is_empty() {
|
||||||
return Err(JmapError::Auth(
|
return Err(JmapError::Auth(
|
||||||
@@ -236,6 +362,143 @@ mod tests {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn origin_treats_the_default_port_as_equivalent() {
|
||||||
|
assert_eq!(
|
||||||
|
origin_of("https://mail.example/jmap"),
|
||||||
|
origin_of("https://mail.example:443/jmap")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
origin_of("http://mail.example/jmap"),
|
||||||
|
origin_of("http://mail.example:80/jmap")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn origin_distinguishes_port_host_and_scheme() {
|
||||||
|
let base = origin_of("https://mail.example:10443/jmap");
|
||||||
|
assert_ne!(base, origin_of("https://mail.example/jmap"));
|
||||||
|
assert_ne!(base, origin_of("https://other.example:10443/jmap"));
|
||||||
|
assert_ne!(base, origin_of("http://mail.example:10443/jmap"));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn session_on(base: &str) -> Session {
|
||||||
|
let raw = format!(
|
||||||
|
r#"{{"apiUrl":"{base}/jmap/","uploadUrl":"{base}/jmap/upload/{{accountId}}/",
|
||||||
|
"downloadUrl":"{base}/jmap/dl/{{accountId}}/{{blobId}}/{{type}}/{{name}}",
|
||||||
|
"capabilities":{{}},"accounts":{{}}}}"#
|
||||||
|
);
|
||||||
|
parse_session(&raw).expect("session")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn matching_origin_reports_no_mismatch() {
|
||||||
|
let session = session_on("https://mail.example:10443");
|
||||||
|
assert!(
|
||||||
|
session
|
||||||
|
.origin_mismatches("https://mail.example:10443")
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
session_on("https://mail.example")
|
||||||
|
.origin_mismatches("https://mail.example:443/")
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dropped_port_is_reported_once_naming_every_field() {
|
||||||
|
let session = session_on("https://mail.example");
|
||||||
|
let mismatches = session.origin_mismatches("https://mail.example:10443");
|
||||||
|
assert_eq!(mismatches.len(), 1, "one warning per distinct origin");
|
||||||
|
assert_eq!(
|
||||||
|
mismatches[0].fields,
|
||||||
|
vec!["apiUrl", "uploadUrl", "downloadUrl"]
|
||||||
|
);
|
||||||
|
assert_eq!(mismatches[0].session_origin, "https://mail.example");
|
||||||
|
assert_eq!(mismatches[0].connect_origin, "https://mail.example:10443");
|
||||||
|
let warning = mismatches[0].to_string();
|
||||||
|
assert!(warning.contains("session advertises apiUrl"), "{warning}");
|
||||||
|
assert!(
|
||||||
|
warning.contains("on https://mail.example (connected to https://mail.example:10443)"),
|
||||||
|
"{warning}"
|
||||||
|
);
|
||||||
|
assert!(warning.contains("advertised HTTP URL setting"), "{warning}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_split_upload_host_is_reported_separately_from_the_api_host() {
|
||||||
|
let raw = r#"{"apiUrl":"https://mail.example:10443/jmap/",
|
||||||
|
"uploadUrl":"https://blobs.example/upload/{accountId}/",
|
||||||
|
"downloadUrl":"https://blobs.example/dl/{accountId}/{blobId}/{type}/{name}",
|
||||||
|
"capabilities":{},"accounts":{}}"#;
|
||||||
|
let session = parse_session(raw).expect("session");
|
||||||
|
let mismatches = session.origin_mismatches("https://mail.example:10443");
|
||||||
|
assert_eq!(mismatches.len(), 1);
|
||||||
|
assert_eq!(mismatches[0].fields, vec!["uploadUrl", "downloadUrl"]);
|
||||||
|
assert_eq!(mismatches[0].session_origin, "https://blobs.example");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolve_reference_keeps_absolute_urls_verbatim() {
|
||||||
|
assert_eq!(
|
||||||
|
resolve_reference(
|
||||||
|
"https://h.example/.well-known/jmap",
|
||||||
|
"https://other.example:8443/jmap/api"
|
||||||
|
),
|
||||||
|
"https://other.example:8443/jmap/api"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolve_reference_handles_scheme_relative_absolute_and_relative_paths() {
|
||||||
|
let base = "https://h.example:10443/.well-known/jmap";
|
||||||
|
assert_eq!(
|
||||||
|
resolve_reference(base, "//other.example/jmap/api"),
|
||||||
|
"https://other.example/jmap/api"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
resolve_reference(base, "/jmap/api"),
|
||||||
|
"https://h.example:10443/jmap/api"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
resolve_reference(base, "api"),
|
||||||
|
"https://h.example:10443/.well-known/api"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolve_reference_preserves_uri_template_braces() {
|
||||||
|
assert_eq!(
|
||||||
|
resolve_reference(
|
||||||
|
"https://h.example/.well-known/jmap",
|
||||||
|
"/jmap/upload/{accountId}/"
|
||||||
|
),
|
||||||
|
"https://h.example/jmap/upload/{accountId}/"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn relative_session_urls_are_resolved_against_the_session_resource() {
|
||||||
|
let raw = r#"{"apiUrl":"/jmap/","uploadUrl":"/jmap/upload/{accountId}/",
|
||||||
|
"downloadUrl":"/jmap/download/{accountId}/{blobId}/{type}/{name}",
|
||||||
|
"capabilities":{},"accounts":{}}"#;
|
||||||
|
let session = parse_session(raw)
|
||||||
|
.expect("session")
|
||||||
|
.rebased_on("https://h.example:10443/.well-known/jmap");
|
||||||
|
assert_eq!(session.api_url, "https://h.example:10443/jmap/");
|
||||||
|
assert_eq!(
|
||||||
|
session.upload_url_for("w"),
|
||||||
|
"https://h.example:10443/jmap/upload/w/"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
session
|
||||||
|
.origin_mismatches("https://h.example:10443")
|
||||||
|
.is_empty(),
|
||||||
|
"a resolved relative URL shares the connect origin"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn well_known_appends_correctly() {
|
fn well_known_appends_correctly() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|||||||
+28
-16
@@ -9,7 +9,7 @@ use clap::Parser;
|
|||||||
use vandelay::cli::{Action, Cli};
|
use vandelay::cli::{Action, Cli};
|
||||||
use vandelay::error::Error;
|
use vandelay::error::Error;
|
||||||
use vandelay::inspect;
|
use vandelay::inspect;
|
||||||
use vandelay::sync::{self, Summary};
|
use vandelay::sync::{self, RunOutcome, Summary};
|
||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
let code = run();
|
let code = run();
|
||||||
@@ -30,42 +30,54 @@ fn run() -> i32 {
|
|||||||
Err(err) => return fail(&err),
|
Err(err) => return fail(&err),
|
||||||
};
|
};
|
||||||
|
|
||||||
let result = match action {
|
let (outcome, logger) = match action {
|
||||||
Action::Import(common, config) => {
|
Action::Import(common, config) => {
|
||||||
let logger = common.logger;
|
let logger = common.logger;
|
||||||
sync::import_jmap::run(common, config).map(|s| (s, logger))
|
(sync::import_jmap::run_reporting(common, config), logger)
|
||||||
}
|
}
|
||||||
Action::ImportImap(common, config) => {
|
Action::ImportImap(common, config) => {
|
||||||
let logger = common.logger;
|
let logger = common.logger;
|
||||||
sync::import_imap::run(common, config).map(|s| (s, logger))
|
(sync::import_imap::run_reporting(common, config), logger)
|
||||||
}
|
}
|
||||||
Action::ImportDav(common, config) => {
|
Action::ImportDav(common, config) => {
|
||||||
let logger = common.logger;
|
let logger = common.logger;
|
||||||
sync::import_dav::run(common, config).map(|s| (s, logger))
|
(sync::import_dav::run_reporting(common, config), logger)
|
||||||
}
|
}
|
||||||
Action::ImportManageSieve(common, config) => {
|
Action::ImportManageSieve(common, config) => {
|
||||||
let logger = common.logger;
|
let logger = common.logger;
|
||||||
sync::import_managesieve::run(common, config).map(|s| (s, logger))
|
(
|
||||||
|
RunOutcome::from_result(sync::import_managesieve::run(common, config)),
|
||||||
|
logger,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
Action::ImportMaildir(common, config) => {
|
Action::ImportMaildir(common, config) => {
|
||||||
let logger = common.logger;
|
let logger = common.logger;
|
||||||
sync::import_maildir::run(common, config).map(|s| (s, logger))
|
(sync::import_maildir::run_reporting(common, config), logger)
|
||||||
}
|
}
|
||||||
Action::ImportTakeout(common, config) => {
|
Action::ImportTakeout(common, config) => {
|
||||||
let logger = common.logger;
|
let logger = common.logger;
|
||||||
sync::import_takeout::run(common, config).map(|s| (s, logger))
|
(sync::import_takeout::run_reporting(common, config), logger)
|
||||||
}
|
}
|
||||||
Action::ImportExchangeEws(common, config) => {
|
Action::ImportExchangeEws(common, config) => {
|
||||||
let logger = common.logger;
|
let logger = common.logger;
|
||||||
sync::import_exchange_ews::run(common, config).map(|s| (s, logger))
|
(
|
||||||
|
RunOutcome::from_result(sync::import_exchange_ews::run(common, config)),
|
||||||
|
logger,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
Action::ImportExchangeGraph(common, config) => {
|
Action::ImportExchangeGraph(common, config) => {
|
||||||
let logger = common.logger;
|
let logger = common.logger;
|
||||||
sync::import_exchange_graph::run(common, config).map(|s| (s, logger))
|
(
|
||||||
|
RunOutcome::from_result(sync::import_exchange_graph::run(common, config)),
|
||||||
|
logger,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
Action::Export(common, config) => {
|
Action::Export(common, config) => {
|
||||||
let logger = common.logger;
|
let logger = common.logger;
|
||||||
sync::export::run(common, config).map(|s| (s, logger))
|
(
|
||||||
|
RunOutcome::from_result(sync::export::run(common, config)),
|
||||||
|
logger,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
Action::Inspect(config) => {
|
Action::Inspect(config) => {
|
||||||
return match inspect::run(config) {
|
return match inspect::run(config) {
|
||||||
@@ -75,17 +87,17 @@ fn run() -> i32 {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
match result {
|
report(&outcome.summary);
|
||||||
Ok((summary, logger)) => {
|
match outcome.error {
|
||||||
report(&summary);
|
Some(err) => fail(&err),
|
||||||
if summary.any_failed() {
|
None => {
|
||||||
|
if outcome.summary.any_failed() {
|
||||||
logger.error("some objects failed; the archive is consistent and resumable");
|
logger.error("some objects failed; the archive is consistent and resumable");
|
||||||
5
|
5
|
||||||
} else {
|
} else {
|
||||||
0
|
0
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(err) => fail(&err),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+6
-4
@@ -20,7 +20,7 @@ use crate::jmap::request::{Request, SetRequest, get_all, get_objects, query_all_
|
|||||||
use crate::jmap::session::{Limits, Session};
|
use crate::jmap::session::{Limits, Session};
|
||||||
use crate::jmap::wire::JmapId;
|
use crate::jmap::wire::JmapId;
|
||||||
use crate::logging::{LEVEL_DEFAULT, Logger};
|
use crate::logging::{LEVEL_DEFAULT, Logger};
|
||||||
use crate::sync::import_jmap::mapping::{BlobUpload, TargetResolver};
|
use crate::sync::import_jmap::mapping::{BlobBytes, TargetResolver};
|
||||||
use crate::sync::{CommonConfig, Context, ExportConfig, Summary, TypeCounts};
|
use crate::sync::{CommonConfig, Context, ExportConfig, Summary, TypeCounts};
|
||||||
use crate::types::ObjectType;
|
use crate::types::ObjectType;
|
||||||
|
|
||||||
@@ -110,9 +110,10 @@ impl<'a> Uploader<'a> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl BlobUpload for Uploader<'_> {
|
impl BlobBytes for Uploader<'_> {
|
||||||
fn upload(&mut self, local_id: i64) -> Result<JmapId, JmapError> {
|
fn bytes(&self, local_id: i64) -> Result<Vec<u8>, JmapError> {
|
||||||
self.upload_with(local_id, "application/octet-stream")
|
db::blobs::blob_bytes(self.conn, local_id)?
|
||||||
|
.ok_or_else(|| JmapError::malformed(format!("blob local id {local_id} missing")))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -172,6 +173,7 @@ pub fn run(common: CommonConfig, config: ExportConfig) -> Result<Summary, Error>
|
|||||||
);
|
);
|
||||||
let plan = match res {
|
let plan = match res {
|
||||||
Ok(p) => p,
|
Ok(p) => p,
|
||||||
|
Err(e) if e.aborts_run() => return Err(e),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
logger.warn(&format!("type {} aborted: {e}", ty.jmap_name()));
|
logger.warn(&format!("type {} aborted: {e}", ty.jmap_name()));
|
||||||
counts.failed += 1;
|
counts.failed += 1;
|
||||||
|
|||||||
@@ -12,7 +12,10 @@ use super::common::{jid, target_query_get};
|
|||||||
use super::{Maps, Net, Plan, Uploader};
|
use super::{Maps, Net, Plan, Uploader};
|
||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
use crate::jmap::error::JmapError;
|
use crate::jmap::error::JmapError;
|
||||||
use crate::jmap::request::{Request, check_method_error, get_objects};
|
use crate::jmap::request::{
|
||||||
|
MethodCall, Request, check_method_error, get_objects, retry_method_call,
|
||||||
|
};
|
||||||
|
use crate::jmap::retry::MethodCallKind;
|
||||||
use crate::jmap::wire::JmapId;
|
use crate::jmap::wire::JmapId;
|
||||||
use crate::logging::Logger;
|
use crate::logging::Logger;
|
||||||
use crate::sync::import_jmap::mapping::{EMAIL_SELECT, EmailRow, TargetResolver, row_to_email};
|
use crate::sync::import_jmap::mapping::{EMAIL_SELECT, EmailRow, TargetResolver, row_to_email};
|
||||||
@@ -219,7 +222,7 @@ fn export_one(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let item = import_item(blob, mids, build_keywords(row), &row.received_at);
|
let item = import_item(blob, mids, build_keywords(row), &row.received_at);
|
||||||
match send_single_import(net, &cid, item) {
|
match send_single_import(net, &cid, item, logger) {
|
||||||
Ok(SingleImport::Created) => counts.created += 1,
|
Ok(SingleImport::Created) => counts.created += 1,
|
||||||
Ok(SingleImport::Skipped) => counts.skipped += 1,
|
Ok(SingleImport::Skipped) => counts.skipped += 1,
|
||||||
Ok(SingleImport::NotCreated { error_type, .. }) if error_type == "blobNotFound" => {
|
Ok(SingleImport::NotCreated { error_type, .. }) if error_type == "blobNotFound" => {
|
||||||
@@ -277,7 +280,7 @@ fn retry_after_reupload(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
let item = import_item(blob, mids, build_keywords(row), &row.received_at);
|
let item = import_item(blob, mids, build_keywords(row), &row.received_at);
|
||||||
match send_single_import(net, cid, item) {
|
match send_single_import(net, cid, item, logger) {
|
||||||
Ok(SingleImport::Created) => counts.created += 1,
|
Ok(SingleImport::Created) => counts.created += 1,
|
||||||
Ok(SingleImport::Skipped) => counts.skipped += 1,
|
Ok(SingleImport::Skipped) => counts.skipped += 1,
|
||||||
Ok(SingleImport::NotCreated { detail, .. }) => {
|
Ok(SingleImport::NotCreated { detail, .. }) => {
|
||||||
@@ -304,7 +307,12 @@ enum SingleImport {
|
|||||||
NotCreated { error_type: String, detail: String },
|
NotCreated { error_type: String, detail: String },
|
||||||
}
|
}
|
||||||
|
|
||||||
fn send_single_import(net: &Net, cid: &str, item: Value) -> Result<SingleImport, JmapError> {
|
fn send_single_import(
|
||||||
|
net: &Net,
|
||||||
|
cid: &str,
|
||||||
|
item: Value,
|
||||||
|
logger: &Logger,
|
||||||
|
) -> Result<SingleImport, JmapError> {
|
||||||
let mut emails = Map::new();
|
let mut emails = Map::new();
|
||||||
emails.insert(cid.to_owned(), item);
|
emails.insert(cid.to_owned(), item);
|
||||||
let mut req = Request::new();
|
let mut req = Request::new();
|
||||||
@@ -314,8 +322,15 @@ fn send_single_import(net: &Net, cid: &str, item: Value) -> Result<SingleImport,
|
|||||||
"i",
|
"i",
|
||||||
);
|
);
|
||||||
req.fits(&net.limits)?;
|
req.fits(&net.limits)?;
|
||||||
let resp = req.send(&net.client, &net.api)?;
|
retry_method_call(
|
||||||
let mr = resp.first()?;
|
&net.client,
|
||||||
|
MethodCallKind::SingleObjectWrite,
|
||||||
|
logger,
|
||||||
|
|| interpret_import(req.send(&net.client, &net.api)?.first()?, cid),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn interpret_import(mr: &MethodCall, cid: &str) -> Result<SingleImport, JmapError> {
|
||||||
check_method_error(mr)?;
|
check_method_error(mr)?;
|
||||||
if let Some(err) = mr
|
if let Some(err) = mr
|
||||||
.args
|
.args
|
||||||
|
|||||||
+32
-20
@@ -5,10 +5,11 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
|
use std::fmt::Write as _;
|
||||||
|
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
||||||
use super::common::{create_batch, jid, retry_if_blob_missing, target_query_get};
|
use super::common::{create_batch, jid, target_query_get};
|
||||||
use super::{Maps, Net, Plan, Uploader};
|
use super::{Maps, Net, Plan, Uploader};
|
||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
use crate::logging::Logger;
|
use crate::logging::Logger;
|
||||||
@@ -23,6 +24,15 @@ fn target_uid(v: &Value) -> Option<String> {
|
|||||||
v.get("uid").and_then(Value::as_str).map(str::to_owned)
|
v.get("uid").and_then(Value::as_str).map(str::to_owned)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn describe(ty: ObjectType, local: i64, uid: &str) -> String {
|
||||||
|
let mut out = String::new();
|
||||||
|
let _ = write!(out, "{} local {local}", ty.jmap_name());
|
||||||
|
if !uid.is_empty() {
|
||||||
|
let _ = write!(out, " (uid {uid})");
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
pub fn reconcile(
|
pub fn reconcile(
|
||||||
ctx: &Context,
|
ctx: &Context,
|
||||||
net: &Net,
|
net: &Net,
|
||||||
@@ -66,7 +76,7 @@ pub fn reconcile(
|
|||||||
};
|
};
|
||||||
|
|
||||||
let mut matched_uids: HashSet<String> = HashSet::new();
|
let mut matched_uids: HashSet<String> = HashSet::new();
|
||||||
let mut uploader = Uploader::new(net, &ctx.conn);
|
let blobs = Uploader::new(net, &ctx.conn);
|
||||||
for (local, uid) in &rows {
|
for (local, uid) in &rows {
|
||||||
if let Some(tid) = by_uid.get(uid) {
|
if let Some(tid) = by_uid.get(uid) {
|
||||||
maps.insert(ty, *local, crate::jmap::wire::JmapId(tid.clone()));
|
maps.insert(ty, *local, crate::jmap::wire::JmapId(tid.clone()));
|
||||||
@@ -75,28 +85,30 @@ pub fn reconcile(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let cid = format!("c{local}");
|
let cid = format!("c{local}");
|
||||||
let _ = uploader.take_touched();
|
let wire = match build_wire(ctx, ty, *local, maps, &blobs) {
|
||||||
let wire = match build_wire(ctx, ty, *local, maps, &mut uploader) {
|
|
||||||
Ok(w) => w,
|
Ok(w) => w,
|
||||||
|
Err(e) if e.aborts_run() => return Err(e),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
logger.warn(&format!("{} local {local} skipped: {e}", ty.jmap_name()));
|
logger.warn(&format!("{} skipped: {e}", describe(ty, *local, uid)));
|
||||||
counts.failed += 1;
|
counts.failed += 1;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let touched = uploader.take_touched();
|
let outcome = match create_batch(net, ty, vec![(cid.clone(), wire)]) {
|
||||||
let outcome = create_batch(net, ty, vec![(cid.clone(), wire)]).map_err(Error::from)?;
|
Ok(o) => o,
|
||||||
let outcome =
|
Err(e) => {
|
||||||
match retry_if_blob_missing(net, ty, &cid, &mut uploader, touched, outcome, |up| {
|
let mapped = Error::from(e);
|
||||||
build_wire(ctx, ty, *local, maps, up)
|
if mapped.aborts_run() {
|
||||||
}) {
|
return Err(mapped);
|
||||||
Ok(o) => o,
|
|
||||||
Err(e) => {
|
|
||||||
logger.warn(&format!("{} local {local} skipped: {e}", ty.jmap_name()));
|
|
||||||
counts.failed += 1;
|
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
};
|
logger.warn(&format!(
|
||||||
|
"{} not created: {mapped}",
|
||||||
|
describe(ty, *local, uid)
|
||||||
|
));
|
||||||
|
counts.failed += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
for (cid, v) in &outcome.created {
|
for (cid, v) in &outcome.created {
|
||||||
if let Some(parsed) = cid.strip_prefix('c').and_then(|s| s.parse::<i64>().ok())
|
if let Some(parsed) = cid.strip_prefix('c').and_then(|s| s.parse::<i64>().ok())
|
||||||
&& let Some(id) = jid(v)
|
&& let Some(id) = jid(v)
|
||||||
@@ -136,7 +148,7 @@ fn build_wire(
|
|||||||
ty: ObjectType,
|
ty: ObjectType,
|
||||||
local: i64,
|
local: i64,
|
||||||
maps: &Maps,
|
maps: &Maps,
|
||||||
up: &mut Uploader<'_>,
|
blobs: &Uploader<'_>,
|
||||||
) -> Result<Value, Error> {
|
) -> Result<Value, Error> {
|
||||||
if ty == ObjectType::ContactCard {
|
if ty == ObjectType::ContactCard {
|
||||||
let (uid, abids, data): (String, String, String) = ctx
|
let (uid, abids, data): (String, String, String) = ctx
|
||||||
@@ -147,7 +159,7 @@ fn build_wire(
|
|||||||
|r| Ok((r.get(1)?, r.get(2)?, r.get(3)?)),
|
|r| Ok((r.get(1)?, r.get(2)?, r.get(3)?)),
|
||||||
)
|
)
|
||||||
.map_err(|e| Error::Partial(e.to_string()))?;
|
.map_err(|e| Error::Partial(e.to_string()))?;
|
||||||
contact_card_to_wire(&uid, &abids, &data, maps, up).map_err(Error::from)
|
contact_card_to_wire(&uid, &abids, &data, maps, blobs).map_err(Error::from)
|
||||||
} else {
|
} else {
|
||||||
let (cal, dr, ud, data): (String, i64, i64, String) = ctx
|
let (cal, dr, ud, data): (String, i64, i64, String) = ctx
|
||||||
.conn
|
.conn
|
||||||
@@ -157,6 +169,6 @@ fn build_wire(
|
|||||||
|r| Ok((r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)),
|
|r| Ok((r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)),
|
||||||
)
|
)
|
||||||
.map_err(|e| Error::Partial(e.to_string()))?;
|
.map_err(|e| Error::Partial(e.to_string()))?;
|
||||||
calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, maps, up).map_err(Error::from)
|
calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, maps, blobs).map_err(Error::from)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+45
-1
@@ -10,4 +10,48 @@ pub mod coordinator;
|
|||||||
pub mod items;
|
pub mod items;
|
||||||
pub mod tree;
|
pub mod tree;
|
||||||
|
|
||||||
pub use coordinator::{DavAuth, DavImportConfig, DavKindArg, run};
|
pub use coordinator::{DavAuth, DavImportConfig, DavKindArg, run, run_reporting};
|
||||||
|
|
||||||
|
use crate::error::Error;
|
||||||
|
use crate::jmap::error::JmapError;
|
||||||
|
|
||||||
|
pub(crate) fn per_collection_failure(err: JmapError) -> Error {
|
||||||
|
match err {
|
||||||
|
JmapError::Sqlite(e) => Error::Db(crate::db::init::OpenError::Sqlite(e)),
|
||||||
|
scoped => Error::Partial(scoped.to_string()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_forbidden_collection_is_a_per_unit_failure() {
|
||||||
|
for e in [
|
||||||
|
JmapError::Auth("server returned 403: forbidden".to_owned()),
|
||||||
|
JmapError::HttpStatus {
|
||||||
|
status: 405,
|
||||||
|
body: "method not allowed".to_owned(),
|
||||||
|
},
|
||||||
|
JmapError::RetriesExhausted("PROPFIND kept returning 503".to_owned()),
|
||||||
|
JmapError::Transport("io: connection reset".to_owned()),
|
||||||
|
JmapError::Malformed("multistatus parse".to_owned()),
|
||||||
|
] {
|
||||||
|
let mapped = per_collection_failure(e);
|
||||||
|
assert!(
|
||||||
|
!mapped.aborts_run(),
|
||||||
|
"{mapped} is scoped to one collection and must not abort the run"
|
||||||
|
);
|
||||||
|
assert_eq!(mapped.exit_code(), 5);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_archive_failure_still_aborts() {
|
||||||
|
let mapped =
|
||||||
|
per_collection_failure(JmapError::Sqlite(rusqlite::Error::QueryReturnedNoRows));
|
||||||
|
assert!(mapped.aborts_run());
|
||||||
|
assert_eq!(mapped.exit_code(), 7);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ use crate::db::sources::SourceKey;
|
|||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
use crate::jmap::http::{Auth, RetryPolicy};
|
use crate::jmap::http::{Auth, RetryPolicy};
|
||||||
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
||||||
use crate::sync::{CommonConfig, Summary, TypeCounts};
|
use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts};
|
||||||
|
|
||||||
use super::collections;
|
use super::collections;
|
||||||
use super::items;
|
use super::items;
|
||||||
@@ -83,6 +83,20 @@ pub struct DavImportConfig {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn run(common: CommonConfig, config: DavImportConfig) -> Result<Summary, Error> {
|
pub fn run(common: CommonConfig, config: DavImportConfig) -> Result<Summary, Error> {
|
||||||
|
run_reporting(common, config).into_result()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn run_reporting(common: CommonConfig, config: DavImportConfig) -> RunOutcome {
|
||||||
|
let mut summary = Summary::default();
|
||||||
|
let error = run_into(common, config, &mut summary).err();
|
||||||
|
RunOutcome { summary, error }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_into(
|
||||||
|
common: CommonConfig,
|
||||||
|
config: DavImportConfig,
|
||||||
|
summary: &mut Summary,
|
||||||
|
) -> Result<(), Error> {
|
||||||
let logger = common.logger;
|
let logger = common.logger;
|
||||||
enforce_tls_policy(&config.url, config.allow_cleartext)?;
|
enforce_tls_policy(&config.url, config.allow_cleartext)?;
|
||||||
|
|
||||||
@@ -138,38 +152,42 @@ pub fn run(common: CommonConfig, config: DavImportConfig) -> Result<Summary, Err
|
|||||||
}
|
}
|
||||||
|
|
||||||
if common.dry_run {
|
if common.dry_run {
|
||||||
let mut summary = run_dry_diff(&conn, &client, &discovery, &config, logger)?;
|
let result = run_dry_diff(&conn, &client, &discovery, &config, logger, summary);
|
||||||
summary.retries_observed = client.retries_observed();
|
summary.retries_observed = client.retries_observed();
|
||||||
summary.retry_after_sleeps = client.retry_after_sleeps();
|
summary.retry_after_sleeps = client.retry_after_sleeps();
|
||||||
return Ok(summary);
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
let username = config.auth.username();
|
let username = config.auth.username();
|
||||||
let source_id = db::sources::upsert_source(&conn, &key, Some(&account_id), &username)
|
let source_id = db::sources::upsert_source(&conn, &key, Some(&account_id), &username)
|
||||||
.map_err(|e| Error::Partial(e.to_string()))?;
|
.map_err(|e| Error::Partial(e.to_string()))?;
|
||||||
|
|
||||||
let summary = match config.kind {
|
let phase = match config.kind {
|
||||||
DavKindArg::Caldav => {
|
DavKindArg::Caldav => {
|
||||||
run_caldav(&mut conn, &client, source_id, &discovery, &config, logger)?
|
run_caldav(&mut conn, &client, source_id, &discovery, &config, logger)
|
||||||
}
|
}
|
||||||
DavKindArg::Carddav => {
|
DavKindArg::Carddav => {
|
||||||
run_carddav(&mut conn, &client, source_id, &discovery, &config, logger)?
|
run_carddav(&mut conn, &client, source_id, &discovery, &config, logger)
|
||||||
}
|
}
|
||||||
DavKindArg::Webdav => {
|
DavKindArg::Webdav => {
|
||||||
run_webdav(&mut conn, &client, source_id, &discovery, &config, logger)?
|
run_webdav(&mut conn, &client, source_id, &discovery, &config, logger)
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
*summary = phase.summary;
|
||||||
|
summary.retries_observed = client.retries_observed();
|
||||||
|
summary.retry_after_sleeps = client.retry_after_sleeps();
|
||||||
|
if let Some(e) = phase.error {
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
|
|
||||||
if !summary.any_failed()
|
if !summary.any_failed()
|
||||||
&& let Err(e) = run_gc(&conn)
|
&& let Err(e) = run_gc(&conn)
|
||||||
{
|
{
|
||||||
logger.warn(&format!("blob GC skipped: {e}"));
|
logger.warn(&format!("blob GC skipped: {e}"));
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut summary = summary;
|
Ok(())
|
||||||
summary.retries_observed = client.retries_observed();
|
|
||||||
summary.retry_after_sleeps = client.retry_after_sleeps();
|
|
||||||
Ok(summary)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type ReconcileCollections = fn(
|
type ReconcileCollections = fn(
|
||||||
@@ -198,25 +216,71 @@ fn run_collection_phase(
|
|||||||
config: &DavImportConfig,
|
config: &DavImportConfig,
|
||||||
logger: Logger,
|
logger: Logger,
|
||||||
phase: ItemPhase,
|
phase: ItemPhase,
|
||||||
) -> Result<Summary, Error> {
|
) -> RunOutcome {
|
||||||
let mut summary = Summary::default();
|
let mut summary = Summary::default();
|
||||||
let mut container_counts = TypeCounts::default();
|
let mut counts = PhaseCounts::default();
|
||||||
let mut item_counts = TypeCounts::default();
|
let error = collection_phase_into(
|
||||||
|
conn,
|
||||||
|
client,
|
||||||
|
source_id,
|
||||||
|
PhaseInput {
|
||||||
|
discovery,
|
||||||
|
config,
|
||||||
|
logger,
|
||||||
|
phase: &phase,
|
||||||
|
},
|
||||||
|
&mut counts,
|
||||||
|
)
|
||||||
|
.err();
|
||||||
|
|
||||||
|
summary
|
||||||
|
.per_type
|
||||||
|
.push((phase.container_label, counts.container));
|
||||||
|
summary.per_type.push((phase.item_label, counts.items));
|
||||||
|
RunOutcome { summary, error }
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct PhaseCounts {
|
||||||
|
container: TypeCounts,
|
||||||
|
items: TypeCounts,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct PhaseInput<'a> {
|
||||||
|
discovery: &'a Discovery,
|
||||||
|
config: &'a DavImportConfig,
|
||||||
|
logger: Logger,
|
||||||
|
phase: &'a ItemPhase,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collection_phase_into(
|
||||||
|
conn: &mut Connection,
|
||||||
|
client: &DavClient,
|
||||||
|
source_id: i64,
|
||||||
|
input: PhaseInput<'_>,
|
||||||
|
counts: &mut PhaseCounts,
|
||||||
|
) -> Result<(), Error> {
|
||||||
|
let PhaseInput {
|
||||||
|
discovery,
|
||||||
|
config,
|
||||||
|
logger,
|
||||||
|
phase,
|
||||||
|
} = input;
|
||||||
|
|
||||||
let upserted = (phase.reconcile_collections)(
|
let upserted = (phase.reconcile_collections)(
|
||||||
conn,
|
conn,
|
||||||
source_id,
|
source_id,
|
||||||
&discovery.collections,
|
&discovery.collections,
|
||||||
&mut container_counts,
|
&mut counts.container,
|
||||||
logger,
|
logger,
|
||||||
)?;
|
)?;
|
||||||
if logger.enabled(LEVEL_DEFAULT) {
|
if logger.enabled(LEVEL_DEFAULT) {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"import: {} done (upserted={} deleted={} failed={})",
|
"import: {} done (upserted={} deleted={} failed={})",
|
||||||
phase.container_label,
|
phase.container_label,
|
||||||
container_counts.created + container_counts.fetched,
|
counts.container.created + counts.container.fetched,
|
||||||
container_counts.deleted,
|
counts.container.deleted,
|
||||||
container_counts.failed
|
counts.container.failed
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -229,23 +293,19 @@ fn run_collection_phase(
|
|||||||
logger,
|
logger,
|
||||||
};
|
};
|
||||||
for (collection_href, local_id) in &upserted {
|
for (collection_href, local_id) in &upserted {
|
||||||
match (phase.reconcile_items)(conn, &ctx, collection_href, *local_id, &mut item_counts) {
|
match (phase.reconcile_items)(conn, &ctx, collection_href, *local_id, &mut counts.items) {
|
||||||
Ok(()) => {}
|
Ok(()) => {}
|
||||||
|
Err(e) if e.aborts_run() => return Err(e),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
logger.warn(&format!(
|
logger.warn(&format!(
|
||||||
"{} {collection_href:?}: items failed: {e}",
|
"{} {collection_href:?}: items failed: {e}",
|
||||||
phase.container_label
|
phase.container_label
|
||||||
));
|
));
|
||||||
item_counts.failed += 1;
|
counts.items.failed += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Ok(())
|
||||||
summary
|
|
||||||
.per_type
|
|
||||||
.push((phase.container_label, container_counts));
|
|
||||||
summary.per_type.push((phase.item_label, item_counts));
|
|
||||||
Ok(summary)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn run_dry_diff(
|
fn run_dry_diff(
|
||||||
@@ -254,11 +314,11 @@ fn run_dry_diff(
|
|||||||
discovery: &Discovery,
|
discovery: &Discovery,
|
||||||
config: &DavImportConfig,
|
config: &DavImportConfig,
|
||||||
logger: Logger,
|
logger: Logger,
|
||||||
) -> Result<Summary, Error> {
|
summary: &mut Summary,
|
||||||
|
) -> Result<(), Error> {
|
||||||
use crate::dav::href::join_absolute;
|
use crate::dav::href::join_absolute;
|
||||||
use crate::dav::xml;
|
use crate::dav::xml;
|
||||||
use crate::db::dav_ids;
|
use crate::db::dav_ids;
|
||||||
let mut summary = Summary::default();
|
|
||||||
let (container_label, item_label, container_type, item_type) = match config.kind {
|
let (container_label, item_label, container_type, item_type) = match config.kind {
|
||||||
DavKindArg::Caldav => (
|
DavKindArg::Caldav => (
|
||||||
"calendar",
|
"calendar",
|
||||||
@@ -295,33 +355,57 @@ fn run_dry_diff(
|
|||||||
for coll in &discovery.collections {
|
for coll in &discovery.collections {
|
||||||
let url = join_absolute(&discovery.home_set_url, coll.href.as_str())
|
let url = join_absolute(&discovery.home_set_url, coll.href.as_str())
|
||||||
.map_err(|e| Error::Partial(e.to_string()))?;
|
.map_err(|e| Error::Partial(e.to_string()))?;
|
||||||
let ms = client
|
match client
|
||||||
.propfind_responses(&url, 1, &xml::propfind_dav_items(), &url)
|
.propfind_responses(&url, 1, &xml::propfind_dav_items(), &url)
|
||||||
.map_err(Error::from)?;
|
.map_err(super::per_collection_failure)
|
||||||
let new_count = ms
|
{
|
||||||
.responses
|
Ok(ms) => {
|
||||||
.iter()
|
let new_count = ms
|
||||||
.filter(|r| !r.props.is_collection)
|
.responses
|
||||||
.count();
|
.iter()
|
||||||
item_counts.created += new_count as u64;
|
.filter(|r| !r.props.is_collection)
|
||||||
if logger.enabled(LEVEL_DEFAULT) {
|
.count();
|
||||||
eprintln!(" {} items: {new_count}", coll.href.as_str());
|
item_counts.created += new_count as u64;
|
||||||
|
if logger.enabled(LEVEL_DEFAULT) {
|
||||||
|
eprintln!(" {} items: {new_count}", coll.href.as_str());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) if e.aborts_run() => return Err(e),
|
||||||
|
Err(e) => {
|
||||||
|
logger.warn(&format!(
|
||||||
|
"{container_label} {:?}: enumeration failed: {e}",
|
||||||
|
coll.href.as_str()
|
||||||
|
));
|
||||||
|
item_counts.failed += 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else if let Some(root) = discovery.collections.first() {
|
} else if let Some(root) = discovery.collections.first() {
|
||||||
let url = join_absolute(&discovery.home_set_url, root.href.as_str())
|
let url = join_absolute(&discovery.home_set_url, root.href.as_str())
|
||||||
.map_err(|e| Error::Partial(e.to_string()))?;
|
.map_err(|e| Error::Partial(e.to_string()))?;
|
||||||
let ms = client
|
match client
|
||||||
.propfind_responses(&url, 1, &xml::propfind_webdav_listing(), &url)
|
.propfind_responses(&url, 1, &xml::propfind_webdav_listing(), &url)
|
||||||
.map_err(Error::from)?;
|
.map_err(super::per_collection_failure)
|
||||||
let new_count = ms
|
{
|
||||||
.responses
|
Ok(ms) => {
|
||||||
.iter()
|
let new_count = ms
|
||||||
.filter(|r| !r.props.is_collection)
|
.responses
|
||||||
.count();
|
.iter()
|
||||||
item_counts.created += new_count as u64;
|
.filter(|r| !r.props.is_collection)
|
||||||
if logger.enabled(LEVEL_DEFAULT) {
|
.count();
|
||||||
eprintln!(" root {} files: {new_count}", root.href.as_str());
|
item_counts.created += new_count as u64;
|
||||||
|
if logger.enabled(LEVEL_DEFAULT) {
|
||||||
|
eprintln!(" root {} files: {new_count}", root.href.as_str());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) if e.aborts_run() => return Err(e),
|
||||||
|
Err(e) => {
|
||||||
|
logger.warn(&format!(
|
||||||
|
"{container_label} {:?}: enumeration failed: {e}",
|
||||||
|
root.href.as_str()
|
||||||
|
));
|
||||||
|
container_counts.failed += 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -332,7 +416,7 @@ fn run_dry_diff(
|
|||||||
if !matches!(config.kind, DavKindArg::Webdav) {
|
if !matches!(config.kind, DavKindArg::Webdav) {
|
||||||
summary.per_type.push((item_label, item_counts));
|
summary.per_type.push((item_label, item_counts));
|
||||||
}
|
}
|
||||||
Ok(summary)
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn run_caldav(
|
fn run_caldav(
|
||||||
@@ -342,7 +426,7 @@ fn run_caldav(
|
|||||||
discovery: &Discovery,
|
discovery: &Discovery,
|
||||||
config: &DavImportConfig,
|
config: &DavImportConfig,
|
||||||
logger: Logger,
|
logger: Logger,
|
||||||
) -> Result<Summary, Error> {
|
) -> RunOutcome {
|
||||||
run_collection_phase(
|
run_collection_phase(
|
||||||
conn,
|
conn,
|
||||||
client,
|
client,
|
||||||
@@ -366,7 +450,7 @@ fn run_carddav(
|
|||||||
discovery: &Discovery,
|
discovery: &Discovery,
|
||||||
config: &DavImportConfig,
|
config: &DavImportConfig,
|
||||||
logger: Logger,
|
logger: Logger,
|
||||||
) -> Result<Summary, Error> {
|
) -> RunOutcome {
|
||||||
run_collection_phase(
|
run_collection_phase(
|
||||||
conn,
|
conn,
|
||||||
client,
|
client,
|
||||||
@@ -390,13 +474,16 @@ fn run_webdav(
|
|||||||
discovery: &Discovery,
|
discovery: &Discovery,
|
||||||
config: &DavImportConfig,
|
config: &DavImportConfig,
|
||||||
logger: Logger,
|
logger: Logger,
|
||||||
) -> Result<Summary, Error> {
|
) -> RunOutcome {
|
||||||
let mut summary = Summary::default();
|
let mut summary = Summary::default();
|
||||||
let mut file_counts = TypeCounts::default();
|
let mut file_counts = TypeCounts::default();
|
||||||
|
|
||||||
if discovery.collections.is_empty() {
|
if discovery.collections.is_empty() {
|
||||||
summary.per_type.push(("filenode", file_counts));
|
summary.per_type.push(("filenode", file_counts));
|
||||||
return Ok(summary);
|
return RunOutcome {
|
||||||
|
summary,
|
||||||
|
error: None,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
let root = &discovery.collections[0];
|
let root = &discovery.collections[0];
|
||||||
let ctx = tree::WebDavCtx {
|
let ctx = tree::WebDavCtx {
|
||||||
@@ -406,10 +493,10 @@ fn run_webdav(
|
|||||||
dav_connections: config.dav_connections,
|
dav_connections: config.dav_connections,
|
||||||
logger,
|
logger,
|
||||||
};
|
};
|
||||||
tree::reconcile_filenodes(conn, &ctx, root, &mut file_counts)?;
|
let error = tree::reconcile_filenodes(conn, &ctx, root, &mut file_counts).err();
|
||||||
|
|
||||||
summary.per_type.push(("filenode", file_counts));
|
summary.per_type.push(("filenode", file_counts));
|
||||||
Ok(summary)
|
RunOutcome { summary, error }
|
||||||
}
|
}
|
||||||
|
|
||||||
fn map_discovery_error(err: DiscoveryError) -> Error {
|
fn map_discovery_error(err: DiscoveryError) -> Error {
|
||||||
|
|||||||
@@ -113,7 +113,7 @@ fn enumerate_items(client: &DavClient, url: &str) -> Result<Vec<ServerItem>, Err
|
|||||||
let body = xml::propfind_dav_items();
|
let body = xml::propfind_dav_items();
|
||||||
let ms = client
|
let ms = client
|
||||||
.propfind_responses(url, 1, &body, url)
|
.propfind_responses(url, 1, &body, url)
|
||||||
.map_err(Error::from)?;
|
.map_err(super::per_collection_failure)?;
|
||||||
if ms.status >= 400 {
|
if ms.status >= 400 {
|
||||||
return Err(Error::Partial(format!(
|
return Err(Error::Partial(format!(
|
||||||
"enumerate {url}: http {}",
|
"enumerate {url}: http {}",
|
||||||
|
|||||||
@@ -115,6 +115,7 @@ pub fn reconcile_filenodes(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Err(e) if e.aborts_run() => return Err(e),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
logger.warn(&format!("PROPFIND {url}: {e}"));
|
logger.warn(&format!("PROPFIND {url}: {e}"));
|
||||||
counts.failed += 1;
|
counts.failed += 1;
|
||||||
@@ -394,7 +395,7 @@ fn walk_one(
|
|||||||
let body = xml::propfind_webdav_listing();
|
let body = xml::propfind_webdav_listing();
|
||||||
let ms = client
|
let ms = client
|
||||||
.propfind_responses(url, 1, &body, url)
|
.propfind_responses(url, 1, &body, url)
|
||||||
.map_err(Error::from)?;
|
.map_err(super::per_collection_failure)?;
|
||||||
if ms.status >= 400 {
|
if ms.status >= 400 {
|
||||||
return Err(Error::Partial(format!("http {}", ms.status)));
|
return Err(Error::Partial(format!("http {}", ms.status)));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ use serde_json::{Value, json};
|
|||||||
|
|
||||||
use crate::db::exchange_graph_ids;
|
use crate::db::exchange_graph_ids;
|
||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
|
use crate::exchange::jscalendar::override_patch_from_event;
|
||||||
use crate::exchange_graph::api::{self, PREFER_BODY_HTML, PREFER_BODY_TEXT, PREFER_TIMEZONE_UTC};
|
use crate::exchange_graph::api::{self, PREFER_BODY_HTML, PREFER_BODY_TEXT, PREFER_TIMEZONE_UTC};
|
||||||
use crate::exchange_graph::calendar_map::{
|
use crate::exchange_graph::calendar_map::{
|
||||||
ConvertedEvent, EventType, classify_event_type, convert_event,
|
ConvertedEvent, EventType, classify_event_type, convert_event,
|
||||||
@@ -185,7 +186,7 @@ fn merge_exception_into(master: &mut ConvertedEvent, ex: &ConvertedEvent) {
|
|||||||
let Value::Object(overrides) = overrides else {
|
let Value::Object(overrides) = overrides else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
overrides.insert(key, ex.data.clone());
|
overrides.insert(key, override_patch_from_event(&ex.data));
|
||||||
}
|
}
|
||||||
|
|
||||||
fn merge_exception_into_existing(
|
fn merge_exception_into_existing(
|
||||||
@@ -253,7 +254,7 @@ fn merge_persisted_master(
|
|||||||
.entry("recurrenceOverrides".to_owned())
|
.entry("recurrenceOverrides".to_owned())
|
||||||
.or_insert_with(|| Value::Object(serde_json::Map::new()));
|
.or_insert_with(|| Value::Object(serde_json::Map::new()));
|
||||||
if let Value::Object(overrides) = entry {
|
if let Value::Object(overrides) = entry {
|
||||||
overrides.insert(key, ex_data.clone());
|
overrides.insert(key, override_patch_from_event(ex_data));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
tx.execute(
|
tx.execute(
|
||||||
|
|||||||
@@ -16,11 +16,10 @@ use crate::exchange_graph::error::GraphError;
|
|||||||
use crate::exchange_graph::oauth::{
|
use crate::exchange_graph::oauth::{
|
||||||
AcquiredToken, OAuthFlow, acquire, default_authority, refresh_access_token,
|
AcquiredToken, OAuthFlow, acquire, default_authority, refresh_access_token,
|
||||||
};
|
};
|
||||||
use crate::exchange_graph::types::{EventBodyFormat, MailboxKind, synthetic_account_id};
|
use crate::exchange_graph::types::{EventBodyFormat, MailboxKind, Surfaces, synthetic_account_id};
|
||||||
use crate::jmap::http::RetryPolicy;
|
use crate::jmap::http::RetryPolicy;
|
||||||
use crate::logging::LEVEL_DEFAULT;
|
use crate::logging::LEVEL_DEFAULT;
|
||||||
use crate::sync::{CommonConfig, Summary, TypeCounts};
|
use crate::sync::{CommonConfig, Summary, TypeCounts};
|
||||||
use crate::types::ObjectType;
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub enum GraphAuth {
|
pub enum GraphAuth {
|
||||||
@@ -39,7 +38,7 @@ pub struct GraphImportConfig {
|
|||||||
pub api_base: String,
|
pub api_base: String,
|
||||||
pub user_target: Option<String>,
|
pub user_target: Option<String>,
|
||||||
pub mailbox_kind: MailboxKind,
|
pub mailbox_kind: MailboxKind,
|
||||||
pub objects: Option<Vec<ObjectType>>,
|
pub surfaces: Surfaces,
|
||||||
pub event_body_format: EventBodyFormat,
|
pub event_body_format: EventBodyFormat,
|
||||||
pub graph_connections: usize,
|
pub graph_connections: usize,
|
||||||
pub top: usize,
|
pub top: usize,
|
||||||
@@ -141,32 +140,10 @@ pub fn run(common: CommonConfig, config: GraphImportConfig) -> Result<Summary, E
|
|||||||
event_body_format: config.event_body_format,
|
event_body_format: config.event_body_format,
|
||||||
};
|
};
|
||||||
|
|
||||||
let want_mail = config
|
let primary = !matches!(config.mailbox_kind, MailboxKind::Archive);
|
||||||
.objects
|
let want_mail = config.surfaces.mail;
|
||||||
.as_ref()
|
let want_calendar = config.surfaces.calendar && primary;
|
||||||
.map(|set| {
|
let want_contacts = config.surfaces.contacts && primary;
|
||||||
set.iter()
|
|
||||||
.any(|o| matches!(o, ObjectType::Mailbox | ObjectType::Email))
|
|
||||||
})
|
|
||||||
.unwrap_or(true);
|
|
||||||
let want_calendar = config
|
|
||||||
.objects
|
|
||||||
.as_ref()
|
|
||||||
.map(|set| {
|
|
||||||
set.iter()
|
|
||||||
.any(|o| matches!(o, ObjectType::Calendar | ObjectType::CalendarEvent))
|
|
||||||
})
|
|
||||||
.unwrap_or(true)
|
|
||||||
&& !matches!(config.mailbox_kind, MailboxKind::Archive);
|
|
||||||
let want_contacts = config
|
|
||||||
.objects
|
|
||||||
.as_ref()
|
|
||||||
.map(|set| {
|
|
||||||
set.iter()
|
|
||||||
.any(|o| matches!(o, ObjectType::AddressBook | ObjectType::ContactCard))
|
|
||||||
})
|
|
||||||
.unwrap_or(true)
|
|
||||||
&& !matches!(config.mailbox_kind, MailboxKind::Archive);
|
|
||||||
|
|
||||||
if want_mail {
|
if want_mail {
|
||||||
let folders = super::folders::reconcile_mail(
|
let folders = super::folders::reconcile_mail(
|
||||||
@@ -509,7 +486,7 @@ mod tests {
|
|||||||
api_base: "https://graph.microsoft.com/v1.0".to_owned(),
|
api_base: "https://graph.microsoft.com/v1.0".to_owned(),
|
||||||
user_target: None,
|
user_target: None,
|
||||||
mailbox_kind: MailboxKind::Primary,
|
mailbox_kind: MailboxKind::Primary,
|
||||||
objects: None,
|
surfaces: Surfaces::ALL,
|
||||||
event_body_format: EventBodyFormat::Text,
|
event_body_format: EventBodyFormat::Text,
|
||||||
graph_connections: 4,
|
graph_connections: 4,
|
||||||
top: 100,
|
top: 100,
|
||||||
|
|||||||
@@ -13,4 +13,4 @@ pub mod pool;
|
|||||||
|
|
||||||
pub mod coordinator;
|
pub mod coordinator;
|
||||||
|
|
||||||
pub use coordinator::{ImapAuth, ImapImportConfig, run};
|
pub use coordinator::{ImapAuth, ImapImportConfig, run, run_reporting};
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ use crate::imap::transport::Connector;
|
|||||||
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
||||||
use crate::sync::emailmeta::email_index_from_blob;
|
use crate::sync::emailmeta::email_index_from_blob;
|
||||||
use crate::sync::keys::index_to_json;
|
use crate::sync::keys::index_to_json;
|
||||||
use crate::sync::{CommonConfig, Summary, TypeCounts};
|
use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts};
|
||||||
|
|
||||||
use super::fetch;
|
use super::fetch;
|
||||||
use super::folders::{
|
use super::folders::{
|
||||||
@@ -199,6 +199,20 @@ pub enum ImapAuth {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn run(common: CommonConfig, config: ImapImportConfig) -> Result<Summary, Error> {
|
pub fn run(common: CommonConfig, config: ImapImportConfig) -> Result<Summary, Error> {
|
||||||
|
run_reporting(common, config).into_result()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn run_reporting(common: CommonConfig, config: ImapImportConfig) -> RunOutcome {
|
||||||
|
let mut summary = Summary::default();
|
||||||
|
let error = run_into(common, config, &mut summary).err();
|
||||||
|
RunOutcome { summary, error }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_into(
|
||||||
|
common: CommonConfig,
|
||||||
|
config: ImapImportConfig,
|
||||||
|
summary: &mut Summary,
|
||||||
|
) -> Result<(), Error> {
|
||||||
let logger = common.logger;
|
let logger = common.logger;
|
||||||
let mut conn = db::init::open(&common.archive)?;
|
let mut conn = db::init::open(&common.archive)?;
|
||||||
|
|
||||||
@@ -341,14 +355,15 @@ pub fn run(common: CommonConfig, config: ImapImportConfig) -> Result<Summary, Er
|
|||||||
|
|
||||||
if common.dry_run {
|
if common.dry_run {
|
||||||
let existing_source = db::sources::find_source(&conn, &source_key)?;
|
let existing_source = db::sources::find_source(&conn, &source_key)?;
|
||||||
return dry_run_summary(
|
*summary = dry_run_summary(
|
||||||
&conn,
|
&conn,
|
||||||
existing_source,
|
existing_source,
|
||||||
&mut client,
|
&mut client,
|
||||||
&control_ctx,
|
&control_ctx,
|
||||||
&resolved,
|
&resolved,
|
||||||
logger,
|
logger,
|
||||||
);
|
)?;
|
||||||
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut mailbox_counts = TypeCounts::default();
|
let mut mailbox_counts = TypeCounts::default();
|
||||||
@@ -410,6 +425,16 @@ pub fn run(common: CommonConfig, config: ImapImportConfig) -> Result<Summary, Er
|
|||||||
&mut email_counts,
|
&mut email_counts,
|
||||||
) {
|
) {
|
||||||
Ok(()) => {}
|
Ok(()) => {}
|
||||||
|
Err(e) if e.aborts_run() => {
|
||||||
|
pool.shutdown();
|
||||||
|
let _ = client.logout();
|
||||||
|
*summary = Summary {
|
||||||
|
per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)],
|
||||||
|
retries_observed: backoff.total_retries(),
|
||||||
|
retry_after_sleeps: backoff.transient_retries() as u64,
|
||||||
|
};
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
log_at(
|
log_at(
|
||||||
logger,
|
logger,
|
||||||
@@ -424,11 +449,12 @@ pub fn run(common: CommonConfig, config: ImapImportConfig) -> Result<Summary, Er
|
|||||||
pool.shutdown();
|
pool.shutdown();
|
||||||
let _ = client.logout();
|
let _ = client.logout();
|
||||||
|
|
||||||
Ok(Summary {
|
*summary = Summary {
|
||||||
per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)],
|
per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)],
|
||||||
retries_observed: backoff.total_retries(),
|
retries_observed: backoff.total_retries(),
|
||||||
retry_after_sleeps: backoff.transient_retries() as u64,
|
retry_after_sleeps: backoff.transient_retries() as u64,
|
||||||
})
|
};
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
|
|||||||
+19
-4
@@ -33,7 +33,7 @@ use crate::jmap::wire::email::Email;
|
|||||||
use crate::jmap::wire::file_node::{FileNode, NodeType};
|
use crate::jmap::wire::file_node::{FileNode, NodeType};
|
||||||
use crate::jmap::wire::sieve_script::SieveScript;
|
use crate::jmap::wire::sieve_script::SieveScript;
|
||||||
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
||||||
use crate::sync::{CommonConfig, Context, ImportConfig, Summary, TypeCounts};
|
use crate::sync::{CommonConfig, Context, ImportConfig, RunOutcome, Summary, TypeCounts};
|
||||||
use crate::types::ObjectType;
|
use crate::types::ObjectType;
|
||||||
|
|
||||||
const IMPORT_ORDER: [ObjectType; 10] = [
|
const IMPORT_ORDER: [ObjectType; 10] = [
|
||||||
@@ -147,6 +147,20 @@ fn username_of(auth: &Auth) -> String {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn run(common: CommonConfig, config: ImportConfig) -> Result<Summary, Error> {
|
pub fn run(common: CommonConfig, config: ImportConfig) -> Result<Summary, Error> {
|
||||||
|
run_reporting(common, config).into_result()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn run_reporting(common: CommonConfig, config: ImportConfig) -> RunOutcome {
|
||||||
|
let mut summary = Summary::default();
|
||||||
|
let error = run_into(common, config, &mut summary).err();
|
||||||
|
RunOutcome { summary, error }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_into(
|
||||||
|
common: CommonConfig,
|
||||||
|
config: ImportConfig,
|
||||||
|
summary: &mut Summary,
|
||||||
|
) -> Result<(), Error> {
|
||||||
let logger = common.logger;
|
let logger = common.logger;
|
||||||
let ctx = Context::open(common, &config.connect)?;
|
let ctx = Context::open(common, &config.connect)?;
|
||||||
let connected = connect::prepare(&ctx, &config.connect)?;
|
let connected = connect::prepare(&ctx, &config.connect)?;
|
||||||
@@ -196,7 +210,6 @@ pub fn run(common: CommonConfig, config: ImportConfig) -> Result<Summary, Error>
|
|||||||
session: connected.session.clone(),
|
session: connected.session.clone(),
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut summary = Summary::default();
|
|
||||||
let mut dry_rows: Vec<(&'static str, u64, u64, u64)> = Vec::new();
|
let mut dry_rows: Vec<(&'static str, u64, u64, u64)> = Vec::new();
|
||||||
let threads = ctx.common.threads;
|
let threads = ctx.common.threads;
|
||||||
|
|
||||||
@@ -216,6 +229,7 @@ pub fn run(common: CommonConfig, config: ImportConfig) -> Result<Summary, Error>
|
|||||||
&mut dry_rows,
|
&mut dry_rows,
|
||||||
) {
|
) {
|
||||||
Ok(()) => {}
|
Ok(()) => {}
|
||||||
|
Err(e) if e.aborts_run() => return Err(e),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
logger.warn(&format!(
|
logger.warn(&format!(
|
||||||
"type {} aborted: {e}; continuing (run will exit 5)",
|
"type {} aborted: {e}; continuing (run will exit 5)",
|
||||||
@@ -229,7 +243,8 @@ pub fn run(common: CommonConfig, config: ImportConfig) -> Result<Summary, Error>
|
|||||||
|
|
||||||
if ctx.dry_run() {
|
if ctx.dry_run() {
|
||||||
print_dry_run(&dry_rows);
|
print_dry_run(&dry_rows);
|
||||||
return Ok(Summary::default());
|
*summary = Summary::default();
|
||||||
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
if !summary.any_failed()
|
if !summary.any_failed()
|
||||||
@@ -240,7 +255,7 @@ pub fn run(common: CommonConfig, config: ImportConfig) -> Result<Summary, Error>
|
|||||||
|
|
||||||
summary.retries_observed = ctx.client.retries_observed();
|
summary.retries_observed = ctx.client.retries_observed();
|
||||||
summary.retry_after_sleeps = ctx.client.retry_after_sleeps();
|
summary.retry_after_sleeps = ctx.client.retry_after_sleeps();
|
||||||
Ok(summary)
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn work_list(config: &ImportConfig, connected: &Connected) -> Vec<ObjectType> {
|
fn work_list(config: &ImportConfig, connected: &Connected) -> Vec<ObjectType> {
|
||||||
|
|||||||
+171
-30
@@ -8,7 +8,7 @@ use indexmap::IndexMap;
|
|||||||
use rusqlite::{Connection, Row, params};
|
use rusqlite::{Connection, Row, params};
|
||||||
use serde_json::{Map, Value};
|
use serde_json::{Map, Value};
|
||||||
|
|
||||||
use crate::jmap::blob::{export_blob_ids, import_blob_ids};
|
use crate::jmap::blob::{BlobWalkError, InlineShape, import_blob_ids, inline_blob_data_uris};
|
||||||
use crate::jmap::error::JmapError;
|
use crate::jmap::error::JmapError;
|
||||||
use crate::jmap::wire::JmapId;
|
use crate::jmap::wire::JmapId;
|
||||||
use crate::jmap::wire::address_book::AddressBook;
|
use crate::jmap::wire::address_book::AddressBook;
|
||||||
@@ -35,8 +35,8 @@ pub trait BlobIntern {
|
|||||||
fn intern(&mut self, jmap_blob_id: &str) -> Result<i64, JmapError>;
|
fn intern(&mut self, jmap_blob_id: &str) -> Result<i64, JmapError>;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub trait BlobUpload {
|
pub trait BlobBytes {
|
||||||
fn upload(&mut self, local_id: i64) -> Result<JmapId, JmapError>;
|
fn bytes(&self, local_id: i64) -> Result<Vec<u8>, JmapError>;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn translate_in(
|
fn translate_in(
|
||||||
@@ -662,10 +662,10 @@ pub fn contact_card_to_wire(
|
|||||||
address_book_ids: &str,
|
address_book_ids: &str,
|
||||||
data: &str,
|
data: &str,
|
||||||
resolver: &impl TargetResolver,
|
resolver: &impl TargetResolver,
|
||||||
blobs: &mut impl BlobUpload,
|
blobs: &impl BlobBytes,
|
||||||
) -> Result<Value, JmapError> {
|
) -> Result<Value, JmapError> {
|
||||||
let mut value: Value = serde_json::from_str(data)?;
|
let mut value: Value = serde_json::from_str(data)?;
|
||||||
restore_blobs_in(&mut value, blobs)?;
|
inline_blobs_in(&mut value, InlineShape::JsContactResource, blobs)?;
|
||||||
let locals = parse_local_id_array(address_book_ids)?;
|
let locals = parse_local_id_array(address_book_ids)?;
|
||||||
let abids = translate_out(&locals, ObjectType::AddressBook, resolver)?;
|
let abids = translate_out(&locals, ObjectType::AddressBook, resolver)?;
|
||||||
if let Value::Object(map) = &mut value {
|
if let Value::Object(map) = &mut value {
|
||||||
@@ -681,10 +681,10 @@ pub fn calendar_event_to_wire(
|
|||||||
use_default_alerts: bool,
|
use_default_alerts: bool,
|
||||||
data: &str,
|
data: &str,
|
||||||
resolver: &impl TargetResolver,
|
resolver: &impl TargetResolver,
|
||||||
blobs: &mut impl BlobUpload,
|
blobs: &impl BlobBytes,
|
||||||
) -> Result<Value, JmapError> {
|
) -> Result<Value, JmapError> {
|
||||||
let mut value: Value = serde_json::from_str(data)?;
|
let mut value: Value = serde_json::from_str(data)?;
|
||||||
restore_blobs_in(&mut value, blobs)?;
|
inline_blobs_in(&mut value, InlineShape::JsCalendarLink, blobs)?;
|
||||||
let locals = parse_local_id_array(calendar_ids)?;
|
let locals = parse_local_id_array(calendar_ids)?;
|
||||||
let calids = translate_out(&locals, ObjectType::Calendar, resolver)?;
|
let calids = translate_out(&locals, ObjectType::Calendar, resolver)?;
|
||||||
if let Value::Object(map) = &mut value {
|
if let Value::Object(map) = &mut value {
|
||||||
@@ -725,21 +725,20 @@ fn take_string(value: &mut Value, key: &str) -> Option<String> {
|
|||||||
|
|
||||||
fn rewrite_blobs_in(data: &mut Value, blobs: &mut impl BlobIntern) -> Result<(), JmapError> {
|
fn rewrite_blobs_in(data: &mut Value, blobs: &mut impl BlobIntern) -> Result<(), JmapError> {
|
||||||
import_blob_ids(data, |jmap_blob_id| {
|
import_blob_ids(data, |jmap_blob_id| {
|
||||||
blobs
|
blobs.intern(jmap_blob_id).map_err(BlobWalkError::resolver)
|
||||||
.intern(jmap_blob_id)
|
|
||||||
.map_err(|e| crate::jmap::blob::BlobWalkError::Resolver(e.to_string()))
|
|
||||||
})
|
})
|
||||||
.map_err(JmapError::from)
|
.map_err(BlobWalkError::into_source)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn restore_blobs_in(data: &mut Value, blobs: &mut impl BlobUpload) -> Result<(), JmapError> {
|
fn inline_blobs_in(
|
||||||
export_blob_ids(data, |local_id| {
|
data: &mut Value,
|
||||||
blobs
|
shape: InlineShape,
|
||||||
.upload(local_id)
|
blobs: &impl BlobBytes,
|
||||||
.map(|id| id.0)
|
) -> Result<(), JmapError> {
|
||||||
.map_err(|e| crate::jmap::blob::BlobWalkError::Resolver(e.to_string()))
|
inline_blob_data_uris(data, shape, |local_id| {
|
||||||
|
blobs.bytes(local_id).map_err(BlobWalkError::resolver)
|
||||||
})
|
})
|
||||||
.map_err(JmapError::from)
|
.map_err(BlobWalkError::into_source)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn opt_json<T: serde::Serialize>(value: &Option<T>) -> Result<Option<String>, JmapError> {
|
fn opt_json<T: serde::Serialize>(value: &Option<T>) -> Result<Option<String>, JmapError> {
|
||||||
@@ -904,9 +903,35 @@ mod tests {
|
|||||||
Ok(42)
|
Ok(42)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
impl BlobUpload for FakeBlobs {
|
impl BlobBytes for FakeBlobs {
|
||||||
fn upload(&mut self, local_id: i64) -> Result<JmapId, JmapError> {
|
fn bytes(&self, local_id: i64) -> Result<Vec<u8>, JmapError> {
|
||||||
Ok(JmapId(format!("T{local_id}")))
|
Ok(format!("payload-{local_id}").into_bytes())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_data_uri(value: &Value, media_type: &str) -> Vec<u8> {
|
||||||
|
use base64::Engine;
|
||||||
|
let uri = value.as_str().unwrap_or_else(|| panic!("{value} is a URI"));
|
||||||
|
let prefix = format!("data:{media_type};base64,");
|
||||||
|
let payload = uri
|
||||||
|
.strip_prefix(&prefix)
|
||||||
|
.unwrap_or_else(|| panic!("{uri} does not start with {prefix}"));
|
||||||
|
base64::engine::general_purpose::STANDARD
|
||||||
|
.decode(payload)
|
||||||
|
.expect("base64 payload")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_no_blob_id(value: &Value) {
|
||||||
|
match value {
|
||||||
|
Value::Object(map) => {
|
||||||
|
assert!(map.get("blobId").is_none(), "blobId present in {value}");
|
||||||
|
assert!(map.get("@blob").is_none(), "@blob present in {value}");
|
||||||
|
for child in map.values() {
|
||||||
|
assert_no_blob_id(child);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Value::Array(items) => items.iter().for_each(assert_no_blob_id),
|
||||||
|
_ => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1032,7 +1057,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn contact_card_strips_uid_and_rewrites_blob() {
|
fn contact_card_strips_uid_and_inlines_media_as_data_uri() {
|
||||||
let c = mem();
|
let c = mem();
|
||||||
let res = MapResolver {
|
let res = MapResolver {
|
||||||
to_local: HashMap::from([((ObjectType::AddressBook, "AB".to_owned()), 1)]),
|
to_local: HashMap::from([((ObjectType::AddressBook, "AB".to_owned()), 1)]),
|
||||||
@@ -1043,7 +1068,10 @@ mod tests {
|
|||||||
"addressBookIds": { "AB": true },
|
"addressBookIds": { "AB": true },
|
||||||
"uid": "urn:uuid:42",
|
"uid": "urn:uuid:42",
|
||||||
"name": { "full": "Jane" },
|
"name": { "full": "Jane" },
|
||||||
"photos": { "p1": { "blobId": "PB", "mediaType": "image/png" } }
|
"media": { "photo": {
|
||||||
|
"@type": "Media", "kind": "photo",
|
||||||
|
"blobId": "PB", "mediaType": "image/png"
|
||||||
|
} }
|
||||||
}))
|
}))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let mut blobs = FakeBlobs;
|
let mut blobs = FakeBlobs;
|
||||||
@@ -1058,14 +1086,127 @@ mod tests {
|
|||||||
assert_eq!(uid, "urn:uuid:42");
|
assert_eq!(uid, "urn:uuid:42");
|
||||||
let stored: Value = serde_json::from_str(&data).unwrap();
|
let stored: Value = serde_json::from_str(&data).unwrap();
|
||||||
assert!(stored.get("uid").is_none());
|
assert!(stored.get("uid").is_none());
|
||||||
assert_eq!(stored["photos"]["p1"]["@blob"], Value::from(42));
|
assert_eq!(stored["media"]["photo"]["@blob"], Value::from(42));
|
||||||
|
|
||||||
let mut up = FakeBlobs;
|
let up = FakeBlobs;
|
||||||
let wire = contact_card_to_wire(&uid, &abids, &data, &res, &mut up).unwrap();
|
let wire = contact_card_to_wire(&uid, &abids, &data, &res, &up).unwrap();
|
||||||
assert_eq!(wire["uid"], Value::from("urn:uuid:42"));
|
assert_eq!(wire["uid"], Value::from("urn:uuid:42"));
|
||||||
assert_eq!(wire["addressBookIds"]["TAB"], Value::Bool(true));
|
assert_eq!(wire["addressBookIds"]["TAB"], Value::Bool(true));
|
||||||
assert_eq!(wire["photos"]["p1"]["blobId"], Value::from("T42"));
|
assert_eq!(
|
||||||
assert!(wire["photos"]["p1"].get("@blob").is_none());
|
decode_data_uri(&wire["media"]["photo"]["uri"], "image/png"),
|
||||||
|
b"payload-42"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
wire["media"]["photo"]["mediaType"],
|
||||||
|
Value::from("image/png")
|
||||||
|
);
|
||||||
|
assert!(wire["media"]["photo"].get("href").is_none());
|
||||||
|
assert_no_blob_id(&wire);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn calendar_event_inlines_link_enclosure_as_data_uri() {
|
||||||
|
let c = mem();
|
||||||
|
let res = MapResolver {
|
||||||
|
to_local: HashMap::from([((ObjectType::Calendar, "CAL".to_owned()), 5)]),
|
||||||
|
to_target: HashMap::from([((ObjectType::Calendar, 5), "TCAL".to_owned())]),
|
||||||
|
};
|
||||||
|
let ev: CalendarEvent = serde_json::from_value(serde_json::json!({
|
||||||
|
"id": "EV1",
|
||||||
|
"calendarIds": { "CAL": true },
|
||||||
|
"uid": "ev-with-enclosure",
|
||||||
|
"title": "Review",
|
||||||
|
"@type": "Event",
|
||||||
|
"links": { "1": {
|
||||||
|
"@type": "Link", "rel": "enclosure",
|
||||||
|
"blobId": "AB", "contentType": "text/plain", "title": "agenda.txt"
|
||||||
|
} }
|
||||||
|
}))
|
||||||
|
.unwrap();
|
||||||
|
let mut blobs = FakeBlobs;
|
||||||
|
let local = insert_calendar_event(&c, &ev, &res, &mut blobs).unwrap();
|
||||||
|
let (cal, dr, ud, data): (String, i64, i64, String) = c
|
||||||
|
.query_row(
|
||||||
|
&format!("{CALENDAR_EVENT_SELECT} AND id = ?1"),
|
||||||
|
params![local],
|
||||||
|
|r| Ok((r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let stored: Value = serde_json::from_str(&data).unwrap();
|
||||||
|
assert_eq!(stored["links"]["1"]["@blob"], Value::from(42));
|
||||||
|
|
||||||
|
let up = FakeBlobs;
|
||||||
|
let wire = calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, &res, &up).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
decode_data_uri(&wire["links"]["1"]["href"], "text/plain"),
|
||||||
|
b"payload-42"
|
||||||
|
);
|
||||||
|
assert_eq!(wire["links"]["1"]["contentType"], Value::from("text/plain"));
|
||||||
|
assert_eq!(wire["links"]["1"]["rel"], Value::from("enclosure"));
|
||||||
|
assert!(wire["links"]["1"].get("uri").is_none());
|
||||||
|
assert_no_blob_id(&wire);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn contact_card_media_without_media_type_defaults_to_octet_stream() {
|
||||||
|
let c = mem();
|
||||||
|
let res = MapResolver {
|
||||||
|
to_local: HashMap::from([((ObjectType::AddressBook, "AB".to_owned()), 1)]),
|
||||||
|
to_target: HashMap::from([((ObjectType::AddressBook, 1), "TAB".to_owned())]),
|
||||||
|
};
|
||||||
|
c.execute(
|
||||||
|
"INSERT INTO contact_cards (id,uid,address_book_ids,data)
|
||||||
|
VALUES (1,'u-1','[1]',?1)",
|
||||||
|
params![
|
||||||
|
serde_json::json!({ "@type": "Card", "media": { "photo": { "@blob": 9 } } })
|
||||||
|
.to_string()
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let (uid, abids, data): (String, String, String) = c
|
||||||
|
.query_row(&format!("{CONTACT_CARD_SELECT} WHERE id = 1"), [], |r| {
|
||||||
|
Ok((r.get(1)?, r.get(2)?, r.get(3)?))
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let up = FakeBlobs;
|
||||||
|
let wire = contact_card_to_wire(&uid, &abids, &data, &res, &up).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
decode_data_uri(&wire["media"]["photo"]["uri"], "application/octet-stream"),
|
||||||
|
b"payload-9"
|
||||||
|
);
|
||||||
|
assert_no_blob_id(&wire);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_archive_read_failure_while_inlining_is_an_archive_error_not_a_unit_failure() {
|
||||||
|
struct BrokenBlobs;
|
||||||
|
impl BlobBytes for BrokenBlobs {
|
||||||
|
fn bytes(&self, _local_id: i64) -> Result<Vec<u8>, JmapError> {
|
||||||
|
Err(JmapError::Sqlite(rusqlite::Error::QueryReturnedNoRows))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let res = MapResolver {
|
||||||
|
to_local: HashMap::new(),
|
||||||
|
to_target: HashMap::from([
|
||||||
|
((ObjectType::AddressBook, 1), "TAB".to_owned()),
|
||||||
|
((ObjectType::Calendar, 1), "TCAL".to_owned()),
|
||||||
|
]),
|
||||||
|
};
|
||||||
|
let card = serde_json::json!({ "@type": "Card", "media": { "photo": { "@blob": 9 } } })
|
||||||
|
.to_string();
|
||||||
|
let event =
|
||||||
|
serde_json::json!({ "@type": "Event", "links": { "1": { "@blob": 9 } } }).to_string();
|
||||||
|
let failures = [
|
||||||
|
contact_card_to_wire("u-1", "[1]", &card, &res, &BrokenBlobs).expect_err("card"),
|
||||||
|
calendar_event_to_wire("[1]", false, false, &event, &res, &BrokenBlobs)
|
||||||
|
.expect_err("event"),
|
||||||
|
];
|
||||||
|
for err in failures {
|
||||||
|
assert!(matches!(err, JmapError::Sqlite(_)), "{err:?}");
|
||||||
|
let mapped = crate::error::Error::from(err);
|
||||||
|
assert!(mapped.aborts_run(), "{mapped} must abort the run");
|
||||||
|
assert_eq!(mapped.exit_code(), 7);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1103,8 +1244,8 @@ mod tests {
|
|||||||
assert!(stored.get("calendarIds").is_none());
|
assert!(stored.get("calendarIds").is_none());
|
||||||
assert_eq!(stored["title"], Value::from("Sprint"));
|
assert_eq!(stored["title"], Value::from("Sprint"));
|
||||||
|
|
||||||
let mut up = FakeBlobs;
|
let up = FakeBlobs;
|
||||||
let wire = calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, &res, &mut up).unwrap();
|
let wire = calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, &res, &up).unwrap();
|
||||||
assert_eq!(wire["calendarIds"]["TCAL"], Value::Bool(true));
|
assert_eq!(wire["calendarIds"]["TCAL"], Value::Bool(true));
|
||||||
assert_eq!(wire["isDraft"], Value::Bool(true));
|
assert_eq!(wire["isDraft"], Value::Bool(true));
|
||||||
assert_eq!(wire["title"], Value::from("Sprint"));
|
assert_eq!(wire["title"], Value::from("Sprint"));
|
||||||
|
|||||||
@@ -9,4 +9,4 @@ pub mod keywords;
|
|||||||
pub mod messages;
|
pub mod messages;
|
||||||
pub mod tree;
|
pub mod tree;
|
||||||
|
|
||||||
pub use coordinator::{MaildirImportConfig, run};
|
pub use coordinator::{MaildirImportConfig, run, run_reporting};
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ use crate::db;
|
|||||||
use crate::db::sources::SourceKey;
|
use crate::db::sources::SourceKey;
|
||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
||||||
use crate::sync::{CommonConfig, Summary, TypeCounts};
|
use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts};
|
||||||
|
|
||||||
use super::messages;
|
use super::messages;
|
||||||
use super::tree;
|
use super::tree;
|
||||||
@@ -32,6 +32,20 @@ pub struct MaildirImportConfig {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn run(common: CommonConfig, config: MaildirImportConfig) -> Result<Summary, Error> {
|
pub fn run(common: CommonConfig, config: MaildirImportConfig) -> Result<Summary, Error> {
|
||||||
|
run_reporting(common, config).into_result()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn run_reporting(common: CommonConfig, config: MaildirImportConfig) -> RunOutcome {
|
||||||
|
let mut summary = Summary::default();
|
||||||
|
let error = run_into(common, config, &mut summary).err();
|
||||||
|
RunOutcome { summary, error }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_into(
|
||||||
|
common: CommonConfig,
|
||||||
|
config: MaildirImportConfig,
|
||||||
|
summary: &mut Summary,
|
||||||
|
) -> Result<(), Error> {
|
||||||
let logger = common.logger;
|
let logger = common.logger;
|
||||||
if common.threads > 1 {
|
if common.threads > 1 {
|
||||||
log_at(
|
log_at(
|
||||||
@@ -121,8 +135,8 @@ pub fn run(common: CommonConfig, config: MaildirImportConfig) -> Result<Summary,
|
|||||||
);
|
);
|
||||||
|
|
||||||
if common.dry_run {
|
if common.dry_run {
|
||||||
let summary = build_dry_run_summary(&conn, source_id, &resolved, logger)?;
|
*summary = build_dry_run_summary(&conn, source_id, &resolved, logger)?;
|
||||||
return Ok(summary);
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut mailbox_counts = TypeCounts::default();
|
let mut mailbox_counts = TypeCounts::default();
|
||||||
@@ -174,6 +188,14 @@ pub fn run(common: CommonConfig, config: MaildirImportConfig) -> Result<Summary,
|
|||||||
&mut email_counts,
|
&mut email_counts,
|
||||||
logger,
|
logger,
|
||||||
) {
|
) {
|
||||||
|
if e.aborts_run() {
|
||||||
|
*summary = Summary {
|
||||||
|
per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)],
|
||||||
|
retries_observed: 0,
|
||||||
|
retry_after_sleeps: 0,
|
||||||
|
};
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
log_at(
|
log_at(
|
||||||
logger,
|
logger,
|
||||||
LEVEL_DEFAULT,
|
LEVEL_DEFAULT,
|
||||||
@@ -207,11 +229,12 @@ pub fn run(common: CommonConfig, config: MaildirImportConfig) -> Result<Summary,
|
|||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
Ok(Summary {
|
*summary = Summary {
|
||||||
per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)],
|
per_type: vec![("mailbox", mailbox_counts), ("email", email_counts)],
|
||||||
retries_observed: 0,
|
retries_observed: 0,
|
||||||
retry_after_sleeps: 0,
|
retry_after_sleeps: 0,
|
||||||
})
|
};
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
struct RunFlags {
|
struct RunFlags {
|
||||||
|
|||||||
@@ -13,4 +13,4 @@ pub mod mbox;
|
|||||||
pub mod tree;
|
pub mod tree;
|
||||||
pub mod walk;
|
pub mod walk;
|
||||||
|
|
||||||
pub use coordinator::{TakeoutImportConfig, run};
|
pub use coordinator::{TakeoutImportConfig, run, run_reporting};
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ use crate::db::sources::SourceKey;
|
|||||||
use crate::db::takeout_ids;
|
use crate::db::takeout_ids;
|
||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
||||||
use crate::sync::{CommonConfig, Summary, TypeCounts};
|
use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts};
|
||||||
|
|
||||||
use super::calendar;
|
use super::calendar;
|
||||||
use super::contacts;
|
use super::contacts;
|
||||||
@@ -31,6 +31,20 @@ pub struct TakeoutImportConfig {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary, Error> {
|
pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary, Error> {
|
||||||
|
run_reporting(common, config).into_result()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn run_reporting(common: CommonConfig, config: TakeoutImportConfig) -> RunOutcome {
|
||||||
|
let mut summary = Summary::default();
|
||||||
|
let error = run_into(common, config, &mut summary).err();
|
||||||
|
RunOutcome { summary, error }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_into(
|
||||||
|
common: CommonConfig,
|
||||||
|
config: TakeoutImportConfig,
|
||||||
|
summary: &mut Summary,
|
||||||
|
) -> Result<(), Error> {
|
||||||
let logger = common.logger;
|
let logger = common.logger;
|
||||||
if common.threads > 1 && logger.enabled(LEVEL_PROGRESS) {
|
if common.threads > 1 && logger.enabled(LEVEL_PROGRESS) {
|
||||||
eprintln!("takeout importer is single-threaded; --threads value will be ignored");
|
eprintln!("takeout importer is single-threaded; --threads value will be ignored");
|
||||||
@@ -90,7 +104,8 @@ pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary,
|
|||||||
};
|
};
|
||||||
|
|
||||||
if common.dry_run {
|
if common.dry_run {
|
||||||
return Ok(dry_run_summary(&conn, source_id, &walk_result, logger));
|
*summary = dry_run_summary(&conn, source_id, &walk_result, logger);
|
||||||
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
let options = MappingOptions {
|
let options = MappingOptions {
|
||||||
@@ -107,7 +122,8 @@ pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary,
|
|||||||
let mut mailbox_cache: HashMap<String, i64> =
|
let mut mailbox_cache: HashMap<String, i64> =
|
||||||
takeout_ids::all_for_type(&conn, source_id, takeout_ids::MAILBOX)?;
|
takeout_ids::all_for_type(&conn, source_id, takeout_ids::MAILBOX)?;
|
||||||
|
|
||||||
process_mbox_files(
|
let mut aborted: Option<Error> = None;
|
||||||
|
if let Err(e) = process_mbox_files(
|
||||||
&mut conn,
|
&mut conn,
|
||||||
source_id,
|
source_id,
|
||||||
&walk_result,
|
&walk_result,
|
||||||
@@ -116,27 +132,38 @@ pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary,
|
|||||||
&mut mailbox_counts,
|
&mut mailbox_counts,
|
||||||
&mut email_counts,
|
&mut email_counts,
|
||||||
logger,
|
logger,
|
||||||
)?;
|
) {
|
||||||
|
aborted = Some(e);
|
||||||
|
}
|
||||||
|
|
||||||
process_ics_files(
|
if aborted.is_none()
|
||||||
&mut conn,
|
&& let Err(e) = process_ics_files(
|
||||||
source_id,
|
&mut conn,
|
||||||
&walk_result,
|
source_id,
|
||||||
&mut calendar_counts,
|
&walk_result,
|
||||||
&mut calendar_event_counts,
|
&mut calendar_counts,
|
||||||
logger,
|
&mut calendar_event_counts,
|
||||||
)?;
|
logger,
|
||||||
|
)
|
||||||
|
{
|
||||||
|
aborted = Some(e);
|
||||||
|
}
|
||||||
|
|
||||||
process_vcf_files(
|
if aborted.is_none()
|
||||||
&mut conn,
|
&& let Err(e) = process_vcf_files(
|
||||||
source_id,
|
&mut conn,
|
||||||
&walk_result,
|
source_id,
|
||||||
&mut book_counts,
|
&walk_result,
|
||||||
&mut card_counts,
|
&mut book_counts,
|
||||||
logger,
|
&mut card_counts,
|
||||||
)?;
|
logger,
|
||||||
|
)
|
||||||
|
{
|
||||||
|
aborted = Some(e);
|
||||||
|
}
|
||||||
|
|
||||||
let no_failures = mailbox_counts.failed == 0
|
let no_failures = aborted.is_none()
|
||||||
|
&& mailbox_counts.failed == 0
|
||||||
&& email_counts.failed == 0
|
&& email_counts.failed == 0
|
||||||
&& calendar_counts.failed == 0
|
&& calendar_counts.failed == 0
|
||||||
&& calendar_event_counts.failed == 0
|
&& calendar_event_counts.failed == 0
|
||||||
@@ -171,7 +198,7 @@ pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary,
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(Summary {
|
*summary = Summary {
|
||||||
per_type: vec![
|
per_type: vec![
|
||||||
("mailbox", mailbox_counts),
|
("mailbox", mailbox_counts),
|
||||||
("email", email_counts),
|
("email", email_counts),
|
||||||
@@ -182,7 +209,11 @@ pub fn run(common: CommonConfig, config: TakeoutImportConfig) -> Result<Summary,
|
|||||||
],
|
],
|
||||||
retries_observed: 0,
|
retries_observed: 0,
|
||||||
retry_after_sleeps: 0,
|
retry_after_sleeps: 0,
|
||||||
})
|
};
|
||||||
|
match aborted {
|
||||||
|
Some(e) => Err(e),
|
||||||
|
None => Ok(()),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
@@ -212,6 +243,7 @@ fn process_mbox_files(
|
|||||||
};
|
};
|
||||||
match mail::process_file(conn, &file.path, ctx, mailbox_counts, email_counts, logger) {
|
match mail::process_file(conn, &file.path, ctx, mailbox_counts, email_counts, logger) {
|
||||||
Ok(()) => {}
|
Ok(()) => {}
|
||||||
|
Err(e) if e.aborts_run() => return Err(e),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
logger.warn(&format!("mbox {:?}: {e}", file.path));
|
logger.warn(&format!("mbox {:?}: {e}", file.path));
|
||||||
email_counts.failed += 1;
|
email_counts.failed += 1;
|
||||||
@@ -244,6 +276,9 @@ fn process_ics_files(
|
|||||||
event_counts,
|
event_counts,
|
||||||
logger,
|
logger,
|
||||||
) {
|
) {
|
||||||
|
if e.aborts_run() {
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
logger.warn(&format!("ics {:?}: {e}", file.path));
|
logger.warn(&format!("ics {:?}: {e}", file.path));
|
||||||
event_counts.failed += 1;
|
event_counts.failed += 1;
|
||||||
}
|
}
|
||||||
@@ -274,6 +309,9 @@ fn process_vcf_files(
|
|||||||
card_counts,
|
card_counts,
|
||||||
logger,
|
logger,
|
||||||
) {
|
) {
|
||||||
|
if e.aborts_run() {
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
logger.warn(&format!("vcf {:?}: {e}", file.path));
|
logger.warn(&format!("vcf {:?}: {e}", file.path));
|
||||||
card_counts.failed += 1;
|
card_counts.failed += 1;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -94,6 +94,34 @@ impl Summary {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct RunOutcome {
|
||||||
|
pub summary: Summary,
|
||||||
|
pub error: Option<Error>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RunOutcome {
|
||||||
|
pub fn from_result(result: Result<Summary, Error>) -> RunOutcome {
|
||||||
|
match result {
|
||||||
|
Ok(summary) => RunOutcome {
|
||||||
|
summary,
|
||||||
|
error: None,
|
||||||
|
},
|
||||||
|
Err(error) => RunOutcome {
|
||||||
|
summary: Summary::default(),
|
||||||
|
error: Some(error),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn into_result(self) -> Result<Summary, Error> {
|
||||||
|
match self.error {
|
||||||
|
Some(error) => Err(error),
|
||||||
|
None => Ok(self.summary),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub struct Context {
|
pub struct Context {
|
||||||
pub conn: Connection,
|
pub conn: Connection,
|
||||||
pub client: HttpClient,
|
pub client: HttpClient,
|
||||||
@@ -119,3 +147,28 @@ impl Context {
|
|||||||
self.common.dry_run
|
self.common.dry_run
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_aborted_run_keeps_its_error_and_its_partial_counts() {
|
||||||
|
let mut summary = Summary::default();
|
||||||
|
summary.per_type.push(("Mailbox", TypeCounts::default()));
|
||||||
|
let outcome = RunOutcome {
|
||||||
|
summary,
|
||||||
|
error: Some(Error::Connection("http status 404".to_owned())),
|
||||||
|
};
|
||||||
|
assert_eq!(outcome.summary.per_type.len(), 1);
|
||||||
|
let err = outcome.into_result().expect_err("aborted");
|
||||||
|
assert_eq!(err.exit_code(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_result_without_a_native_outcome_reports_nothing_on_abort() {
|
||||||
|
let outcome = RunOutcome::from_result(Err(Error::Partial("one object".to_owned())));
|
||||||
|
assert!(outcome.summary.per_type.is_empty());
|
||||||
|
assert!(outcome.error.is_some());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -151,7 +151,10 @@ fn sweep_abandoned_containers() {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
eprintln!("removing {} abandoned test container(s)", ids.len());
|
eprintln!("removing {} abandoned test container(s)", ids.len());
|
||||||
let _ = Command::new("docker").args(["rm", "-f"]).args(&ids).output();
|
let _ = Command::new("docker")
|
||||||
|
.args(["rm", "-f"])
|
||||||
|
.args(&ids)
|
||||||
|
.output();
|
||||||
}
|
}
|
||||||
|
|
||||||
fn pick_free_port() -> ContainerResult<u16> {
|
fn pick_free_port() -> ContainerResult<u16> {
|
||||||
|
|||||||
@@ -1595,3 +1595,234 @@ fn mixed_source_archive_allows_distinct_kinds_against_same_url() {
|
|||||||
let no_conflict = sources::conflicting_source(&conn, "caldav", "https://x", "alice").unwrap();
|
let no_conflict = sources::conflicting_source(&conn, "caldav", "https://x", "alice").unwrap();
|
||||||
assert!(no_conflict.is_none());
|
assert!(no_conflict.is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn dav_archive() -> std::path::PathBuf {
|
||||||
|
static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
|
||||||
|
let n = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||||
|
let mut p = std::env::temp_dir();
|
||||||
|
p.push(format!(
|
||||||
|
"vandelay-mockdav-{}-{:?}-{n}.sqlite",
|
||||||
|
std::process::id(),
|
||||||
|
std::thread::current().id(),
|
||||||
|
));
|
||||||
|
let _ = std::fs::remove_file(&p);
|
||||||
|
p
|
||||||
|
}
|
||||||
|
|
||||||
|
fn caldav_import_config(url: &str) -> vandelay::sync::import_dav::DavImportConfig {
|
||||||
|
vandelay::sync::import_dav::DavImportConfig {
|
||||||
|
kind: vandelay::sync::import_dav::DavKindArg::Caldav,
|
||||||
|
url: url.to_owned(),
|
||||||
|
auth: vandelay::sync::import_dav::DavAuth::Basic {
|
||||||
|
user: "u".to_owned(),
|
||||||
|
password: "p".to_owned(),
|
||||||
|
},
|
||||||
|
allow_cleartext: true,
|
||||||
|
dav_connections: 1,
|
||||||
|
multiget_batch: 8,
|
||||||
|
allow_source_change: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dav_common(archive: &std::path::Path) -> vandelay::sync::CommonConfig {
|
||||||
|
vandelay::sync::CommonConfig {
|
||||||
|
archive: archive.to_path_buf(),
|
||||||
|
threads: 1,
|
||||||
|
dry_run: false,
|
||||||
|
max_retries: 0,
|
||||||
|
allow_invalid_certs: false,
|
||||||
|
logger: vandelay::logging::Logger::from_flags(true, 0),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn row_count(archive: &std::path::Path, table: &str) -> i64 {
|
||||||
|
let conn = rusqlite::Connection::open(archive).expect("open archive");
|
||||||
|
conn.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0))
|
||||||
|
.expect("count")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn forbidden_collection_is_a_per_unit_failure_and_the_others_still_import() {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let url = server.url();
|
||||||
|
|
||||||
|
let home_set = format!(
|
||||||
|
r#"<?xml version="1.0"?>
|
||||||
|
<d:multistatus xmlns:d="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav">
|
||||||
|
<d:response>
|
||||||
|
<d:href>{url}/dav/cal/u/work/</d:href>
|
||||||
|
<d:propstat>
|
||||||
|
<d:prop>
|
||||||
|
<d:resourcetype><d:collection/><c:calendar/></d:resourcetype>
|
||||||
|
<d:displayname>Work</d:displayname>
|
||||||
|
</d:prop>
|
||||||
|
<d:status>HTTP/1.1 200 OK</d:status>
|
||||||
|
</d:propstat>
|
||||||
|
</d:response>
|
||||||
|
<d:response>
|
||||||
|
<d:href>{url}/dav/cal/u/shared/</d:href>
|
||||||
|
<d:propstat>
|
||||||
|
<d:prop>
|
||||||
|
<d:resourcetype><d:collection/><c:calendar/></d:resourcetype>
|
||||||
|
<d:displayname>Shared</d:displayname>
|
||||||
|
</d:prop>
|
||||||
|
<d:status>HTTP/1.1 200 OK</d:status>
|
||||||
|
</d:propstat>
|
||||||
|
</d:response>
|
||||||
|
</d:multistatus>"#
|
||||||
|
);
|
||||||
|
let _discovery = multistatus_response(&mut server, "PROPFIND", "/dav/cal/u/", &home_set);
|
||||||
|
|
||||||
|
let work_items = format!(
|
||||||
|
r#"<?xml version="1.0"?>
|
||||||
|
<d:multistatus xmlns:d="DAV:">
|
||||||
|
<d:response>
|
||||||
|
<d:href>{url}/dav/cal/u/work/e1.ics</d:href>
|
||||||
|
<d:propstat>
|
||||||
|
<d:prop><d:resourcetype/><d:getetag>"v1"</d:getetag></d:prop>
|
||||||
|
<d:status>HTTP/1.1 200 OK</d:status>
|
||||||
|
</d:propstat>
|
||||||
|
</d:response>
|
||||||
|
</d:multistatus>"#
|
||||||
|
);
|
||||||
|
let _work_enumeration =
|
||||||
|
multistatus_response(&mut server, "PROPFIND", "/dav/cal/u/work/", &work_items);
|
||||||
|
|
||||||
|
let work_data = format!(
|
||||||
|
r#"<?xml version="1.0"?>
|
||||||
|
<d:multistatus xmlns:d="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav">
|
||||||
|
<d:response>
|
||||||
|
<d:href>{url}/dav/cal/u/work/e1.ics</d:href>
|
||||||
|
<d:propstat>
|
||||||
|
<d:prop>
|
||||||
|
<d:getetag>"v1"</d:getetag>
|
||||||
|
<c:calendar-data>BEGIN:VCALENDAR
|
||||||
|
VERSION:2.0
|
||||||
|
PRODID:-//Test//EN
|
||||||
|
BEGIN:VEVENT
|
||||||
|
UID:[email protected]
|
||||||
|
DTSTAMP:20260101T000000Z
|
||||||
|
DTSTART:20260101T090000Z
|
||||||
|
DTEND:20260101T100000Z
|
||||||
|
SUMMARY:Standup
|
||||||
|
END:VEVENT
|
||||||
|
END:VCALENDAR
|
||||||
|
</c:calendar-data>
|
||||||
|
</d:prop>
|
||||||
|
<d:status>HTTP/1.1 200 OK</d:status>
|
||||||
|
</d:propstat>
|
||||||
|
</d:response>
|
||||||
|
</d:multistatus>"#
|
||||||
|
);
|
||||||
|
let _work_multiget =
|
||||||
|
multistatus_response(&mut server, "REPORT", "/dav/cal/u/work/", &work_data);
|
||||||
|
|
||||||
|
let forbidden = server
|
||||||
|
.mock("PROPFIND", "/dav/cal/u/shared/")
|
||||||
|
.with_status(403)
|
||||||
|
.with_header("content-type", "text/plain")
|
||||||
|
.with_body("you may not read this calendar")
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let archive = dav_archive();
|
||||||
|
let summary = vandelay::sync::import_dav::run(
|
||||||
|
dav_common(&archive),
|
||||||
|
caldav_import_config(&format!("{url}/dav/cal/u/")),
|
||||||
|
)
|
||||||
|
.expect("a forbidden collection must not abort the run");
|
||||||
|
|
||||||
|
forbidden.assert();
|
||||||
|
|
||||||
|
let events = summary
|
||||||
|
.per_type
|
||||||
|
.iter()
|
||||||
|
.find(|(t, _)| *t == "calendarevent")
|
||||||
|
.map(|(_, c)| c.clone())
|
||||||
|
.expect("calendarevent counts");
|
||||||
|
assert_eq!(events.failed, 1, "the forbidden calendar counts as failed");
|
||||||
|
assert_eq!(events.fetched, 1, "the readable calendar still imported");
|
||||||
|
assert!(
|
||||||
|
summary.any_failed(),
|
||||||
|
"a per-unit failure ends the run at exit 5"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(row_count(&archive, "calendars"), 2);
|
||||||
|
assert_eq!(row_count(&archive, "calendar_events"), 1);
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn credentials_rejected_during_discovery_still_aborts_the_run() {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let url = server.url();
|
||||||
|
|
||||||
|
let _rejected = server
|
||||||
|
.mock("PROPFIND", mockito::Matcher::Any)
|
||||||
|
.with_status(401)
|
||||||
|
.with_header("content-type", "text/plain")
|
||||||
|
.with_body("bad password")
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let archive = dav_archive();
|
||||||
|
let err = vandelay::sync::import_dav::run(
|
||||||
|
dav_common(&archive),
|
||||||
|
caldav_import_config(&format!("{url}/dav/cal/u/")),
|
||||||
|
)
|
||||||
|
.expect_err("credentials rejected for the principal aborts the run");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
matches!(err, vandelay::error::Error::Connection(_)),
|
||||||
|
"discovery-level auth rejection is a whole-run failure, got {err:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(err.exit_code(), 2);
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dav_import_reports_both_phases_when_every_collection_fails() {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let url = server.url();
|
||||||
|
|
||||||
|
let home_set = format!(
|
||||||
|
r#"<?xml version="1.0"?>
|
||||||
|
<d:multistatus xmlns:d="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav">
|
||||||
|
<d:response>
|
||||||
|
<d:href>{url}/dav/cal/u/work/</d:href>
|
||||||
|
<d:propstat>
|
||||||
|
<d:prop>
|
||||||
|
<d:resourcetype><d:collection/><c:calendar/></d:resourcetype>
|
||||||
|
<d:displayname>Work</d:displayname>
|
||||||
|
</d:prop>
|
||||||
|
<d:status>HTTP/1.1 200 OK</d:status>
|
||||||
|
</d:propstat>
|
||||||
|
</d:response>
|
||||||
|
</d:multistatus>"#
|
||||||
|
);
|
||||||
|
let _discovery = multistatus_response(&mut server, "PROPFIND", "/dav/cal/u/", &home_set);
|
||||||
|
let _forbidden = server
|
||||||
|
.mock("PROPFIND", "/dav/cal/u/work/")
|
||||||
|
.with_status(403)
|
||||||
|
.with_body("nope")
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let archive = dav_archive();
|
||||||
|
let outcome = vandelay::sync::import_dav::run_reporting(
|
||||||
|
dav_common(&archive),
|
||||||
|
caldav_import_config(&format!("{url}/dav/cal/u/")),
|
||||||
|
);
|
||||||
|
assert!(outcome.error.is_none());
|
||||||
|
assert_eq!(
|
||||||
|
outcome
|
||||||
|
.summary
|
||||||
|
.per_type
|
||||||
|
.iter()
|
||||||
|
.map(|(t, _)| *t)
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
vec!["calendar", "calendarevent"],
|
||||||
|
"both phases are reported even when every collection failed"
|
||||||
|
);
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
}
|
||||||
|
|||||||
+252
-73
@@ -20,6 +20,7 @@ use vandelay::exchange_graph::oauth::{
|
|||||||
};
|
};
|
||||||
use vandelay::exchange_graph::recurrence::convert_patterned_recurrence;
|
use vandelay::exchange_graph::recurrence::convert_patterned_recurrence;
|
||||||
use vandelay::exchange_graph::retry::{HttpClass, classify_http_status};
|
use vandelay::exchange_graph::retry::{HttpClass, classify_http_status};
|
||||||
|
use vandelay::exchange_graph::types::Surfaces;
|
||||||
use vandelay::jmap::http::RetryPolicy;
|
use vandelay::jmap::http::RetryPolicy;
|
||||||
|
|
||||||
static INIT: Once = Once::new();
|
static INIT: Once = Once::new();
|
||||||
@@ -584,7 +585,7 @@ fn integration_dry_run_against_mock_server_lists_three_surfaces() {
|
|||||||
api_base: base.clone(),
|
api_base: base.clone(),
|
||||||
user_target: None,
|
user_target: None,
|
||||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||||
objects: None,
|
surfaces: Surfaces::ALL,
|
||||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||||
graph_connections: 2,
|
graph_connections: 2,
|
||||||
top: 100,
|
top: 100,
|
||||||
@@ -693,10 +694,7 @@ fn integration_full_run_mail_only_imports_mime_via_value() {
|
|||||||
api_base: server.url(),
|
api_base: server.url(),
|
||||||
user_target: None,
|
user_target: None,
|
||||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||||
objects: Some(vec![
|
surfaces: surfaces("mail"),
|
||||||
vandelay::types::ObjectType::Mailbox,
|
|
||||||
vandelay::types::ObjectType::Email,
|
|
||||||
]),
|
|
||||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||||
graph_connections: 2,
|
graph_connections: 2,
|
||||||
top: 100,
|
top: 100,
|
||||||
@@ -792,10 +790,7 @@ fn integration_duplicate_message_id_does_not_abort_run() {
|
|||||||
api_base: server.url(),
|
api_base: server.url(),
|
||||||
user_target: None,
|
user_target: None,
|
||||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||||
objects: Some(vec![
|
surfaces: surfaces("mail"),
|
||||||
vandelay::types::ObjectType::Mailbox,
|
|
||||||
vandelay::types::ObjectType::Email,
|
|
||||||
]),
|
|
||||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||||
graph_connections: 2,
|
graph_connections: 2,
|
||||||
top: 100,
|
top: 100,
|
||||||
@@ -884,10 +879,7 @@ fn integration_full_run_is_convergent_on_second_invocation() {
|
|||||||
api_base: api_base.clone(),
|
api_base: api_base.clone(),
|
||||||
user_target: None,
|
user_target: None,
|
||||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||||
objects: Some(vec![
|
surfaces: surfaces("mail"),
|
||||||
vandelay::types::ObjectType::Mailbox,
|
|
||||||
vandelay::types::ObjectType::Email,
|
|
||||||
]),
|
|
||||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||||
graph_connections: 2,
|
graph_connections: 2,
|
||||||
top: 100,
|
top: 100,
|
||||||
@@ -966,7 +958,7 @@ fn source_change_protection_refuses_a_different_account() {
|
|||||||
api_base: server.url(),
|
api_base: server.url(),
|
||||||
user_target: None,
|
user_target: None,
|
||||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||||
objects: None,
|
surfaces: Surfaces::ALL,
|
||||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||||
graph_connections: 2,
|
graph_connections: 2,
|
||||||
top: 100,
|
top: 100,
|
||||||
@@ -1016,10 +1008,14 @@ fn make_common(archive: std::path::PathBuf) -> vandelay::sync::CommonConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn surfaces(list: &str) -> vandelay::exchange_graph::types::Surfaces {
|
||||||
|
vandelay::exchange_graph::types::Surfaces::parse_list(list).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
fn make_config(
|
fn make_config(
|
||||||
api_base: String,
|
api_base: String,
|
||||||
user_target: Option<String>,
|
user_target: Option<String>,
|
||||||
objects: Option<Vec<vandelay::types::ObjectType>>,
|
surfaces: vandelay::exchange_graph::types::Surfaces,
|
||||||
) -> vandelay::sync::import_exchange_graph::GraphImportConfig {
|
) -> vandelay::sync::import_exchange_graph::GraphImportConfig {
|
||||||
vandelay::sync::import_exchange_graph::GraphImportConfig {
|
vandelay::sync::import_exchange_graph::GraphImportConfig {
|
||||||
auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired {
|
auth: vandelay::sync::import_exchange_graph::GraphAuth::PreAcquired {
|
||||||
@@ -1028,7 +1024,7 @@ fn make_config(
|
|||||||
api_base,
|
api_base,
|
||||||
user_target,
|
user_target,
|
||||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||||
objects,
|
surfaces,
|
||||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||||
graph_connections: 2,
|
graph_connections: 2,
|
||||||
top: 100,
|
top: 100,
|
||||||
@@ -1094,7 +1090,7 @@ fn users_upn_routes_through_users_segment_not_me() {
|
|||||||
};
|
};
|
||||||
let summary = vandelay::sync::import_exchange_graph::run(
|
let summary = vandelay::sync::import_exchange_graph::run(
|
||||||
common,
|
common,
|
||||||
make_config(base, Some("[email protected]".to_owned()), None),
|
make_config(base, Some("[email protected]".to_owned()), Surfaces::ALL),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let mailbox = summary
|
let mailbox = summary
|
||||||
@@ -1193,14 +1189,7 @@ fn series_master_with_exception_merges_into_recurrence_overrides() {
|
|||||||
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
||||||
let summary = vandelay::sync::import_exchange_graph::run(
|
let summary = vandelay::sync::import_exchange_graph::run(
|
||||||
make_common(archive.clone()),
|
make_common(archive.clone()),
|
||||||
make_config(
|
make_config(base, None, surfaces("calendar")),
|
||||||
base,
|
|
||||||
None,
|
|
||||||
Some(vec![
|
|
||||||
vandelay::types::ObjectType::Calendar,
|
|
||||||
vandelay::types::ObjectType::CalendarEvent,
|
|
||||||
]),
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let events = summary
|
let events = summary
|
||||||
@@ -1227,6 +1216,26 @@ fn series_master_with_exception_merges_into_recurrence_overrides() {
|
|||||||
);
|
);
|
||||||
let override_data = &overrides["2026-05-11T15:00:00"];
|
let override_data = &overrides["2026-05-11T15:00:00"];
|
||||||
assert_eq!(override_data["title"], "Moved sync");
|
assert_eq!(override_data["title"], "Moved sync");
|
||||||
|
assert_eq!(override_data["start"], "2026-05-11T16:00:00");
|
||||||
|
for ignored in [
|
||||||
|
"@type",
|
||||||
|
"uid",
|
||||||
|
"method",
|
||||||
|
"organizerCalendarAddress",
|
||||||
|
"privacy",
|
||||||
|
"prodId",
|
||||||
|
"recurrenceId",
|
||||||
|
"recurrenceIdTimeZone",
|
||||||
|
"sentBy",
|
||||||
|
"recurrenceRule",
|
||||||
|
"recurrenceOverrides",
|
||||||
|
"relatedTo",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
override_data.get(ignored).is_none(),
|
||||||
|
"{ignored} must not appear in a PatchObject (jscalendarbis 3.3.4); got {override_data}"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1285,14 +1294,7 @@ fn occurrence_event_is_skipped_no_row_no_id_mapping() {
|
|||||||
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
||||||
let summary = vandelay::sync::import_exchange_graph::run(
|
let summary = vandelay::sync::import_exchange_graph::run(
|
||||||
make_common(archive.clone()),
|
make_common(archive.clone()),
|
||||||
make_config(
|
make_config(base, None, surfaces("calendar")),
|
||||||
base,
|
|
||||||
None,
|
|
||||||
Some(vec![
|
|
||||||
vandelay::types::ObjectType::Calendar,
|
|
||||||
vandelay::types::ObjectType::CalendarEvent,
|
|
||||||
]),
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let events = summary
|
let events = summary
|
||||||
@@ -1372,14 +1374,7 @@ fn hidden_mail_folder_has_is_subscribed_zero() {
|
|||||||
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
||||||
let _ = vandelay::sync::import_exchange_graph::run(
|
let _ = vandelay::sync::import_exchange_graph::run(
|
||||||
make_common(archive.clone()),
|
make_common(archive.clone()),
|
||||||
make_config(
|
make_config(base, None, surfaces("mail")),
|
||||||
base,
|
|
||||||
None,
|
|
||||||
Some(vec![
|
|
||||||
vandelay::types::ObjectType::Mailbox,
|
|
||||||
vandelay::types::ObjectType::Email,
|
|
||||||
]),
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let conn = vandelay::db::init::open(&archive).unwrap();
|
let conn = vandelay::db::init::open(&archive).unwrap();
|
||||||
@@ -1473,14 +1468,7 @@ fn well_known_folder_probes_assign_jmap_roles() {
|
|||||||
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
||||||
let _ = vandelay::sync::import_exchange_graph::run(
|
let _ = vandelay::sync::import_exchange_graph::run(
|
||||||
make_common(archive.clone()),
|
make_common(archive.clone()),
|
||||||
make_config(
|
make_config(base, None, surfaces("mail")),
|
||||||
base,
|
|
||||||
None,
|
|
||||||
Some(vec![
|
|
||||||
vandelay::types::ObjectType::Mailbox,
|
|
||||||
vandelay::types::ObjectType::Email,
|
|
||||||
]),
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let conn = vandelay::db::init::open(&archive).unwrap();
|
let conn = vandelay::db::init::open(&archive).unwrap();
|
||||||
@@ -1557,14 +1545,7 @@ SGVsbG8=\r\n--X--\r\n";
|
|||||||
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
||||||
let summary = vandelay::sync::import_exchange_graph::run(
|
let summary = vandelay::sync::import_exchange_graph::run(
|
||||||
make_common(archive),
|
make_common(archive),
|
||||||
make_config(
|
make_config(base, None, surfaces("mail")),
|
||||||
base,
|
|
||||||
None,
|
|
||||||
Some(vec![
|
|
||||||
vandelay::types::ObjectType::Mailbox,
|
|
||||||
vandelay::types::ObjectType::Email,
|
|
||||||
]),
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let emails = summary
|
let emails = summary
|
||||||
@@ -1652,14 +1633,7 @@ fn event_get_carries_outlook_timezone_and_body_content_type_prefer() {
|
|||||||
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
||||||
let _ = vandelay::sync::import_exchange_graph::run(
|
let _ = vandelay::sync::import_exchange_graph::run(
|
||||||
make_common(archive),
|
make_common(archive),
|
||||||
make_config(
|
make_config(base, None, surfaces("calendar")),
|
||||||
base,
|
|
||||||
None,
|
|
||||||
Some(vec![
|
|
||||||
vandelay::types::ObjectType::Calendar,
|
|
||||||
vandelay::types::ObjectType::CalendarEvent,
|
|
||||||
]),
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
@@ -1758,7 +1732,7 @@ fn full_run_records_graph_id_in_sync_id_exchange_graph_with_padding() {
|
|||||||
api_base: server.url(),
|
api_base: server.url(),
|
||||||
user_target: None,
|
user_target: None,
|
||||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||||
objects: Some(vec![vandelay::types::ObjectType::Mailbox]),
|
surfaces: surfaces("mail"),
|
||||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||||
graph_connections: 2,
|
graph_connections: 2,
|
||||||
top: 100,
|
top: 100,
|
||||||
@@ -1828,7 +1802,7 @@ fn archive_mailbox_kind_encodes_synthetic_suffix_in_account_id() {
|
|||||||
api_base: server.url(),
|
api_base: server.url(),
|
||||||
user_target: None,
|
user_target: None,
|
||||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Archive,
|
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Archive,
|
||||||
objects: Some(vec![vandelay::types::ObjectType::Mailbox]),
|
surfaces: surfaces("mail"),
|
||||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||||
graph_connections: 2,
|
graph_connections: 2,
|
||||||
top: 100,
|
top: 100,
|
||||||
@@ -1893,7 +1867,7 @@ fn allow_source_change_permits_overwriting_a_different_account() {
|
|||||||
api_base: server.url(),
|
api_base: server.url(),
|
||||||
user_target: None,
|
user_target: None,
|
||||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||||
objects: Some(vec![vandelay::types::ObjectType::Mailbox]),
|
surfaces: surfaces("mail"),
|
||||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||||
graph_connections: 2,
|
graph_connections: 2,
|
||||||
top: 100,
|
top: 100,
|
||||||
@@ -1974,7 +1948,7 @@ fn dry_run_makes_no_per_item_get_and_no_sqlite_writes() {
|
|||||||
api_base: server.url(),
|
api_base: server.url(),
|
||||||
user_target: None,
|
user_target: None,
|
||||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||||
objects: None,
|
surfaces: Surfaces::ALL,
|
||||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||||
graph_connections: 2,
|
graph_connections: 2,
|
||||||
top: 100,
|
top: 100,
|
||||||
@@ -2142,10 +2116,7 @@ fn folder_enumeration_failure_skips_vanished_deletion() {
|
|||||||
api_base: server.url(),
|
api_base: server.url(),
|
||||||
user_target: None,
|
user_target: None,
|
||||||
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
mailbox_kind: vandelay::exchange_graph::types::MailboxKind::Primary,
|
||||||
objects: Some(vec![
|
surfaces: surfaces("mail"),
|
||||||
vandelay::types::ObjectType::Mailbox,
|
|
||||||
vandelay::types::ObjectType::Email,
|
|
||||||
]),
|
|
||||||
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
event_body_format: vandelay::exchange_graph::types::EventBodyFormat::Text,
|
||||||
graph_connections: 2,
|
graph_connections: 2,
|
||||||
top: 100,
|
top: 100,
|
||||||
@@ -2163,3 +2134,211 @@ fn folder_enumeration_failure_skips_vanished_deletion() {
|
|||||||
"transient folder-enumeration failure must NOT delete the locally-known message"
|
"transient folder-enumeration failure must NOT delete the locally-known message"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn row_count(conn: &rusqlite::Connection, table: &str) -> i64 {
|
||||||
|
conn.query_row(&format!("SELECT count(*) FROM {table}"), [], |row| {
|
||||||
|
row.get(0)
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stub_contacts_surface(server: &mut Server) {
|
||||||
|
server
|
||||||
|
.mock("GET", "/me/contactFolders?$top=100")
|
||||||
|
.with_status(200)
|
||||||
|
.with_header("content-type", "application/json")
|
||||||
|
.with_body(r#"{"value":[{"id":"CON1","displayName":"Contacts"}]}"#)
|
||||||
|
.expect_at_least(0)
|
||||||
|
.create();
|
||||||
|
server
|
||||||
|
.mock("GET", "/me/contactFolders/CON1/childFolders?$top=100")
|
||||||
|
.with_status(200)
|
||||||
|
.with_header("content-type", "application/json")
|
||||||
|
.with_body(r#"{"value":[]}"#)
|
||||||
|
.expect_at_least(0)
|
||||||
|
.create();
|
||||||
|
server
|
||||||
|
.mock(
|
||||||
|
"GET",
|
||||||
|
"/me/contactFolders/CON1/contacts?$top=100&$select=id",
|
||||||
|
)
|
||||||
|
.with_status(200)
|
||||||
|
.with_header("content-type", "application/json")
|
||||||
|
.with_body(r#"{"value":[{"id":"CT1"}]}"#)
|
||||||
|
.expect_at_least(0)
|
||||||
|
.create();
|
||||||
|
server
|
||||||
|
.mock("GET", "/me/contacts/CT1")
|
||||||
|
.with_status(200)
|
||||||
|
.with_header("content-type", "application/json")
|
||||||
|
.with_body(
|
||||||
|
r#"{"id":"CT1","displayName":"Alice Liddell","givenName":"Alice",
|
||||||
|
"surname":"Liddell","emailAddresses":[{"name":"Alice","address":"[email protected]"}]}"#,
|
||||||
|
)
|
||||||
|
.expect_at_least(0)
|
||||||
|
.create();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stub_mail_surface(server: &mut Server) {
|
||||||
|
server
|
||||||
|
.mock("GET", "/me/mailFolders?$top=100&includeHiddenFolders=true")
|
||||||
|
.with_status(200)
|
||||||
|
.with_header("content-type", "application/json")
|
||||||
|
.with_body(r#"{"value":[{"id":"FMAIL","displayName":"Inbox","isHidden":false}]}"#)
|
||||||
|
.expect_at_least(0)
|
||||||
|
.create();
|
||||||
|
server
|
||||||
|
.mock(
|
||||||
|
"GET",
|
||||||
|
"/me/mailFolders/FMAIL/childFolders?$top=100&includeHiddenFolders=true",
|
||||||
|
)
|
||||||
|
.with_status(200)
|
||||||
|
.with_header("content-type", "application/json")
|
||||||
|
.with_body(r#"{"value":[]}"#)
|
||||||
|
.expect_at_least(0)
|
||||||
|
.create();
|
||||||
|
stub_well_known_folders(server, "FMAIL");
|
||||||
|
server
|
||||||
|
.mock("GET", "/me/mailFolders/FMAIL/messages?$top=100&$select=id")
|
||||||
|
.with_status(200)
|
||||||
|
.with_header("content-type", "application/json")
|
||||||
|
.with_body(r#"{"value":[{"id":"MSG-S"}]}"#)
|
||||||
|
.expect_at_least(0)
|
||||||
|
.create();
|
||||||
|
server
|
||||||
|
.mock("GET", "/me/messages/MSG-S/$value")
|
||||||
|
.with_status(200)
|
||||||
|
.with_header("content-type", "text/plain")
|
||||||
|
.with_body(
|
||||||
|
"From: a@x\r\nTo: b@x\r\nSubject: surface\r\n\
|
||||||
|
Date: Tue, 27 May 2026 10:00:00 +0000\r\nMessage-ID: <surface@x>\r\n\r\nbody",
|
||||||
|
)
|
||||||
|
.expect_at_least(0)
|
||||||
|
.create();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stub_calendar_surface(server: &mut Server) {
|
||||||
|
server
|
||||||
|
.mock("GET", "/me/mailboxSettings?$select=timeZone")
|
||||||
|
.with_status(200)
|
||||||
|
.with_header("content-type", "application/json")
|
||||||
|
.with_body(r#"{"timeZone":"UTC"}"#)
|
||||||
|
.expect_at_least(0)
|
||||||
|
.create();
|
||||||
|
server
|
||||||
|
.mock("GET", "/me/calendars?$top=100")
|
||||||
|
.with_status(200)
|
||||||
|
.with_header("content-type", "application/json")
|
||||||
|
.with_body(r#"{"value":[{"id":"CAL1","name":"Calendar","isDefaultCalendar":true}]}"#)
|
||||||
|
.expect_at_least(0)
|
||||||
|
.create();
|
||||||
|
server
|
||||||
|
.mock(
|
||||||
|
"GET",
|
||||||
|
"/me/calendars/CAL1/events?$top=100&$select=id,type,seriesMasterId",
|
||||||
|
)
|
||||||
|
.with_status(200)
|
||||||
|
.with_header("content-type", "application/json")
|
||||||
|
.with_body(r#"{"value":[{"id":"EV1","type":"singleInstance","iCalUId":"uid-ev1"}]}"#)
|
||||||
|
.expect_at_least(0)
|
||||||
|
.create();
|
||||||
|
server
|
||||||
|
.mock("GET", "/me/events/EV1")
|
||||||
|
.with_status(200)
|
||||||
|
.with_header("content-type", "application/json")
|
||||||
|
.with_body(
|
||||||
|
r#"{"id":"EV1","iCalUId":"uid-ev1","type":"singleInstance","subject":"Standup",
|
||||||
|
"start":{"dateTime":"2026-05-04T15:00:00.0000000","timeZone":"UTC"},
|
||||||
|
"end":{"dateTime":"2026-05-04T16:00:00.0000000","timeZone":"UTC"}}"#,
|
||||||
|
)
|
||||||
|
.expect_at_least(0)
|
||||||
|
.create();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn contacts_surface_imports_only_address_books_and_cards() {
|
||||||
|
let mut server = Server::new();
|
||||||
|
server
|
||||||
|
.mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned()))
|
||||||
|
.with_status(200)
|
||||||
|
.with_header("content-type", "application/json")
|
||||||
|
.with_body(r#"{"id":"uid-surface-c","userPrincipalName":"[email protected]"}"#)
|
||||||
|
.create();
|
||||||
|
stub_contacts_surface(&mut server);
|
||||||
|
stub_calendar_surface(&mut server);
|
||||||
|
stub_mail_surface(&mut server);
|
||||||
|
let no_mail = server
|
||||||
|
.mock("GET", Matcher::Regex(r"^/me/mailFolders\?".to_owned()))
|
||||||
|
.with_status(500)
|
||||||
|
.with_body("MUST NOT BE CALLED")
|
||||||
|
.expect(0)
|
||||||
|
.create();
|
||||||
|
let no_calendars = server
|
||||||
|
.mock("GET", Matcher::Regex(r"^/me/calendars\?".to_owned()))
|
||||||
|
.with_status(500)
|
||||||
|
.with_body("MUST NOT BE CALLED")
|
||||||
|
.expect(0)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let base = server.url();
|
||||||
|
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
||||||
|
vandelay::sync::import_exchange_graph::run(
|
||||||
|
make_common(archive.clone()),
|
||||||
|
make_config(base, None, surfaces("contacts")),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
no_mail.assert();
|
||||||
|
no_calendars.assert();
|
||||||
|
|
||||||
|
let conn = vandelay::db::init::open(&archive).unwrap();
|
||||||
|
assert_eq!(row_count(&conn, "address_books"), 1);
|
||||||
|
assert_eq!(row_count(&conn, "contact_cards"), 1);
|
||||||
|
assert_eq!(row_count(&conn, "mailboxes"), 0);
|
||||||
|
assert_eq!(row_count(&conn, "emails"), 0);
|
||||||
|
assert_eq!(row_count(&conn, "calendars"), 0);
|
||||||
|
assert_eq!(row_count(&conn, "calendar_events"), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn mail_surface_imports_only_mailboxes_and_emails() {
|
||||||
|
let mut server = Server::new();
|
||||||
|
server
|
||||||
|
.mock("GET", Matcher::Regex(r"^/me\?\$select=id".to_owned()))
|
||||||
|
.with_status(200)
|
||||||
|
.with_header("content-type", "application/json")
|
||||||
|
.with_body(r#"{"id":"uid-surface-m","userPrincipalName":"[email protected]"}"#)
|
||||||
|
.create();
|
||||||
|
stub_mail_surface(&mut server);
|
||||||
|
stub_calendar_surface(&mut server);
|
||||||
|
stub_contacts_surface(&mut server);
|
||||||
|
let no_calendars = server
|
||||||
|
.mock("GET", Matcher::Regex(r"^/me/calendars\?".to_owned()))
|
||||||
|
.with_status(500)
|
||||||
|
.with_body("MUST NOT BE CALLED")
|
||||||
|
.expect(0)
|
||||||
|
.create();
|
||||||
|
let no_contact_folders = server
|
||||||
|
.mock("GET", Matcher::Regex(r"^/me/contactFolders\?".to_owned()))
|
||||||
|
.with_status(500)
|
||||||
|
.with_body("MUST NOT BE CALLED")
|
||||||
|
.expect(0)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let base = server.url();
|
||||||
|
let archive = tempfile::NamedTempFile::new().unwrap().path().to_owned();
|
||||||
|
vandelay::sync::import_exchange_graph::run(
|
||||||
|
make_common(archive.clone()),
|
||||||
|
make_config(base, None, surfaces("mail")),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
no_calendars.assert();
|
||||||
|
no_contact_folders.assert();
|
||||||
|
|
||||||
|
let conn = vandelay::db::init::open(&archive).unwrap();
|
||||||
|
assert_eq!(row_count(&conn, "mailboxes"), 1);
|
||||||
|
assert_eq!(row_count(&conn, "emails"), 1);
|
||||||
|
assert_eq!(row_count(&conn, "address_books"), 0);
|
||||||
|
assert_eq!(row_count(&conn, "contact_cards"), 0);
|
||||||
|
assert_eq!(row_count(&conn, "calendars"), 0);
|
||||||
|
assert_eq!(row_count(&conn, "calendar_events"), 0);
|
||||||
|
}
|
||||||
|
|||||||
@@ -1201,3 +1201,97 @@ fn get_state_reads_state_from_empty_get() {
|
|||||||
let st = get_state(&client(0), &url, "w", "Email").expect("get_state");
|
let st = get_state(&client(0), &url, "w", "Email").expect("get_state");
|
||||||
assert_eq!(st.as_deref(), Some("snap-1"));
|
assert_eq!(st.as_deref(), Some("snap-1"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn archive_path() -> std::path::PathBuf {
|
||||||
|
static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
|
||||||
|
let n = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||||
|
let mut p = std::env::temp_dir();
|
||||||
|
p.push(format!(
|
||||||
|
"vandelay-mockjmap-{}-{n}.sqlite",
|
||||||
|
std::process::id()
|
||||||
|
));
|
||||||
|
let _ = std::fs::remove_file(&p);
|
||||||
|
p
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn session_urls_on_a_foreign_origin_are_reported_as_a_mismatch() {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let base = server.url();
|
||||||
|
let elsewhere = "https://mail.example".to_owned();
|
||||||
|
let _root = server.mock("GET", "/").with_status(404).create();
|
||||||
|
let _wk = server
|
||||||
|
.mock("GET", "/.well-known/jmap")
|
||||||
|
.with_status(200)
|
||||||
|
.with_header("content-type", "application/json")
|
||||||
|
.with_body(session_json(&elsewhere))
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let session = Session::discover(&client(0), &base).expect("session discovered");
|
||||||
|
let mismatches = session.origin_mismatches(&base);
|
||||||
|
assert_eq!(mismatches.len(), 1, "one warning per distinct origin");
|
||||||
|
assert_eq!(
|
||||||
|
mismatches[0].fields,
|
||||||
|
vec!["apiUrl", "uploadUrl", "downloadUrl"]
|
||||||
|
);
|
||||||
|
let warning = mismatches[0].to_string();
|
||||||
|
assert!(warning.contains("session advertises apiUrl"), "{warning}");
|
||||||
|
assert!(warning.contains("connected to "), "{warning}");
|
||||||
|
assert!(warning.contains("advertised HTTP URL setting"), "{warning}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn import_aborts_with_exit_two_when_the_advertised_api_url_is_unreachable() {
|
||||||
|
let mut api_host = mockito::Server::new();
|
||||||
|
let mut session_host = mockito::Server::new();
|
||||||
|
let session_base = session_host.url();
|
||||||
|
let api_base = api_host.url();
|
||||||
|
|
||||||
|
let _root = session_host.mock("GET", "/").with_status(404).create();
|
||||||
|
let _wk = session_host
|
||||||
|
.mock("GET", "/.well-known/jmap")
|
||||||
|
.with_status(200)
|
||||||
|
.with_header("content-type", "application/json")
|
||||||
|
.with_body(session_json(&api_base))
|
||||||
|
.create();
|
||||||
|
let _nginx = api_host
|
||||||
|
.mock("POST", "/jmap/api")
|
||||||
|
.with_status(404)
|
||||||
|
.with_header("content-type", "text/html")
|
||||||
|
.with_body(
|
||||||
|
"<html><head><title>404 Not Found</title></head><body>404 Not Found</body></html>",
|
||||||
|
)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let archive = archive_path();
|
||||||
|
let err = vandelay::sync::import_jmap::run(
|
||||||
|
vandelay::sync::CommonConfig {
|
||||||
|
archive: archive.clone(),
|
||||||
|
threads: 1,
|
||||||
|
dry_run: false,
|
||||||
|
max_retries: 0,
|
||||||
|
allow_invalid_certs: false,
|
||||||
|
logger: vandelay::logging::Logger::from_flags(true, 0),
|
||||||
|
},
|
||||||
|
vandelay::sync::ImportConfig {
|
||||||
|
connect: vandelay::sync::ConnectConfig {
|
||||||
|
url: session_base.clone(),
|
||||||
|
auth: Auth::Basic {
|
||||||
|
user: "u".into(),
|
||||||
|
password: "p".into(),
|
||||||
|
},
|
||||||
|
account: AccountSelector::Id("w".into()),
|
||||||
|
},
|
||||||
|
objects: None,
|
||||||
|
allow_source_change: false,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect_err("an unreachable apiUrl must abort the run");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
matches!(err, vandelay::error::Error::Connection(_)),
|
||||||
|
"a 404 from the advertised apiUrl is a whole-run connection failure, got {err:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(err.exit_code(), 2, "must not report a partial failure");
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
}
|
||||||
|
|||||||
@@ -1761,6 +1761,449 @@ fn export_calendar_creates_only_missing() {
|
|||||||
let _ = std::fs::remove_file(&archive);
|
let _ = std::fs::remove_file(&archive);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn export_contact_and_event_blobs_are_inlined_never_blob_ids() {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let base = server.url();
|
||||||
|
let api = "/jmap/api";
|
||||||
|
let archive = tmp();
|
||||||
|
let photo: &[u8] = b"\x89PNG\r\n\x1a\nfake-photo-bytes";
|
||||||
|
let agenda: &[u8] = b"agenda for the review meeting";
|
||||||
|
{
|
||||||
|
let conn = db::init::open(&archive).unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO address_books (id,name,is_default) VALUES (1,'Personal',1)",
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO calendars (id,name,is_default) VALUES (1,'Work',1)",
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let photo_blob = db::blobs::intern_blob(&conn, photo).unwrap();
|
||||||
|
let agenda_blob = db::blobs::intern_blob(&conn, agenda).unwrap();
|
||||||
|
let card = json!({
|
||||||
|
"@type": "Card",
|
||||||
|
"version": "1.0",
|
||||||
|
"name": { "full": "Photo Person" },
|
||||||
|
"media": { "photo": {
|
||||||
|
"@type": "Media", "kind": "photo",
|
||||||
|
"@blob": photo_blob, "mediaType": "image/png"
|
||||||
|
} }
|
||||||
|
})
|
||||||
|
.to_string();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO contact_cards (id,uid,address_book_ids,data)
|
||||||
|
VALUES (1,'card-uid-1','[1]',?1)",
|
||||||
|
[card.as_str()],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let event = json!({
|
||||||
|
"@type": "Event",
|
||||||
|
"uid": "event-uid-1",
|
||||||
|
"title": "Review",
|
||||||
|
"start": "2026-03-01T09:00:00",
|
||||||
|
"duration": "PT1H",
|
||||||
|
"timeZone": "Etc/UTC",
|
||||||
|
"links": { "1": {
|
||||||
|
"@type": "Link", "rel": "enclosure",
|
||||||
|
"@blob": agenda_blob, "contentType": "text/plain", "title": "agenda.txt"
|
||||||
|
} }
|
||||||
|
})
|
||||||
|
.to_string();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO calendar_events (id,calendar_ids,is_draft,use_default_alerts,data)
|
||||||
|
VALUES (1,'[1]',0,0,?1)",
|
||||||
|
[event.as_str()],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
let _root = server.mock("GET", "/").with_status(404).create();
|
||||||
|
let _wk = server
|
||||||
|
.mock("GET", "/.well-known/jmap")
|
||||||
|
.with_body(session_body_full(&base))
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let no_blob_id_in_card = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::AllOf(vec![
|
||||||
|
Matcher::Regex("ContactCard/set".into()),
|
||||||
|
Matcher::Regex("blobId".into()),
|
||||||
|
]))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["ContactCard/set",{"accountId":"w",
|
||||||
|
"created":{"c1":{"id":"TCARD"}}},"s"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(0)
|
||||||
|
.create();
|
||||||
|
let no_blob_id_in_event = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::AllOf(vec![
|
||||||
|
Matcher::Regex("CalendarEvent/set".into()),
|
||||||
|
Matcher::Regex("blobId".into()),
|
||||||
|
]))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["CalendarEvent/set",{"accountId":"w",
|
||||||
|
"created":{"c1":{"id":"TEVENT"}}},"s"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(0)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let _ab_get = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("AddressBook/get".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["AddressBook/get",{"accountId":"w","list":[],
|
||||||
|
"notFound":[]},"g"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
let _ab_set = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("AddressBook/set".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["AddressBook/set",{"accountId":"w",
|
||||||
|
"created":{"c1":{"id":"TAB"}}},"s"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let _cal_get = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("Calendar/get".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["Calendar/get",{"accountId":"w","list":[],
|
||||||
|
"notFound":[]},"g"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
let _cal_set = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("Calendar/set".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["Calendar/set",{"accountId":"w",
|
||||||
|
"created":{"c1":{"id":"TCAL"}}},"s"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let _cc_query = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("ContactCard/query".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["ContactCard/query",{"accountId":"w","ids":[]},"q"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
let _ce_query = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("CalendarEvent/query".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["CalendarEvent/query",{"accountId":"w","ids":[]},"q"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let card_inlined = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::AllOf(vec![
|
||||||
|
Matcher::Regex("ContactCard/set".into()),
|
||||||
|
Matcher::Regex("data:image/png;base64,".into()),
|
||||||
|
]))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["ContactCard/set",{"accountId":"w",
|
||||||
|
"created":{"c1":{"id":"TCARD"}}},"s"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let event_inlined = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::AllOf(vec![
|
||||||
|
Matcher::Regex("CalendarEvent/set".into()),
|
||||||
|
Matcher::Regex("data:text/plain;base64,".into()),
|
||||||
|
]))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["CalendarEvent/set",{"accountId":"w",
|
||||||
|
"created":{"c1":{"id":"TEVENT"}}},"s"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let summary = sync::export::run(
|
||||||
|
common(&archive),
|
||||||
|
export_cfg_objects(
|
||||||
|
&base,
|
||||||
|
vec![
|
||||||
|
ObjectType::AddressBook,
|
||||||
|
ObjectType::Calendar,
|
||||||
|
ObjectType::ContactCard,
|
||||||
|
ObjectType::CalendarEvent,
|
||||||
|
],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.expect("export");
|
||||||
|
|
||||||
|
no_blob_id_in_card.assert();
|
||||||
|
no_blob_id_in_event.assert();
|
||||||
|
card_inlined.assert();
|
||||||
|
event_inlined.assert();
|
||||||
|
|
||||||
|
for name in ["ContactCard", "CalendarEvent"] {
|
||||||
|
let counts = summary
|
||||||
|
.per_type
|
||||||
|
.iter()
|
||||||
|
.find(|(t, _)| *t == name)
|
||||||
|
.map(|(_, c)| c.clone())
|
||||||
|
.unwrap_or_else(|| panic!("{name} counts"));
|
||||||
|
assert_eq!(counts.created, 1, "{name} created: {counts:?}");
|
||||||
|
assert_eq!(counts.failed, 0, "{name} failed: {counts:?}");
|
||||||
|
}
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn session_body_small_request(base: &str, max_size_request: u64) -> String {
|
||||||
|
json!({
|
||||||
|
"apiUrl": format!("{base}/jmap/api"),
|
||||||
|
"uploadUrl": format!("{base}/jmap/upload/{{accountId}}/"),
|
||||||
|
"downloadUrl": format!("{base}/jmap/dl/{{accountId}}/{{blobId}}/{{type}}/{{name}}"),
|
||||||
|
"capabilities": { "urn:ietf:params:jmap:core": {
|
||||||
|
"maxObjectsInGet": 500, "maxObjectsInSet": 500, "maxCallsInRequest": 16,
|
||||||
|
"maxConcurrentRequests": 4, "maxConcurrentUpload": 4,
|
||||||
|
"maxSizeRequest": max_size_request, "maxSizeUpload": 50000000
|
||||||
|
} },
|
||||||
|
"accounts": { "w": { "name": "alice",
|
||||||
|
"accountCapabilities": {
|
||||||
|
"urn:ietf:params:jmap:mail": {},
|
||||||
|
"urn:ietf:params:jmap:contacts": {},
|
||||||
|
"urn:ietf:params:jmap:calendars": {}
|
||||||
|
} } }
|
||||||
|
})
|
||||||
|
.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn export_oversized_contact_card_fails_alone_and_the_type_keeps_going() {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let base = server.url();
|
||||||
|
let api = "/jmap/api";
|
||||||
|
let archive = tmp();
|
||||||
|
{
|
||||||
|
let conn = db::init::open(&archive).unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO address_books (id,name,is_default) VALUES (1,'Personal',1)",
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let photo = vec![b'P'; 8000];
|
||||||
|
let photo_blob = db::blobs::intern_blob(&conn, &photo).unwrap();
|
||||||
|
let huge = json!({
|
||||||
|
"@type": "Card",
|
||||||
|
"name": { "full": "Photo Person" },
|
||||||
|
"media": { "photo": {
|
||||||
|
"@type": "Media", "kind": "photo",
|
||||||
|
"@blob": photo_blob, "mediaType": "image/png"
|
||||||
|
} }
|
||||||
|
})
|
||||||
|
.to_string();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO contact_cards (id,uid,address_book_ids,data)
|
||||||
|
VALUES (1,'huge-card','[1]',?1)",
|
||||||
|
[huge.as_str()],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let small = json!({ "@type": "Card", "name": { "full": "Small Person" } }).to_string();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO contact_cards (id,uid,address_book_ids,data)
|
||||||
|
VALUES (2,'small-card','[1]',?1)",
|
||||||
|
[small.as_str()],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
let _root = server.mock("GET", "/").with_status(404).create();
|
||||||
|
let _wk = server
|
||||||
|
.mock("GET", "/.well-known/jmap")
|
||||||
|
.with_body(session_body_small_request(&base, 4000))
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
let _ccterm = anchor_terminator(&mut server, api, "ContactCard");
|
||||||
|
|
||||||
|
let _ab_get = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("AddressBook/get".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["AddressBook/get",{"accountId":"w","list":[],
|
||||||
|
"notFound":[]},"g"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
let _ab_set = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("AddressBook/set".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["AddressBook/set",{"accountId":"w",
|
||||||
|
"created":{"c1":{"id":"TAB"}}},"s"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let _cc_query = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("ContactCard/query".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["ContactCard/query",{"accountId":"w",
|
||||||
|
"ids":["TSTALE"]},"q"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
let _cc_get = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("ContactCard/get".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["ContactCard/get",{"accountId":"w","list":[
|
||||||
|
{"id":"TSTALE","uid":"stale-card"}
|
||||||
|
],"notFound":[]},"g"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
let destroyed = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::AllOf(vec![
|
||||||
|
Matcher::Regex("ContactCard/set".into()),
|
||||||
|
Matcher::Regex("destroy".into()),
|
||||||
|
]))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["ContactCard/set",{"accountId":"w",
|
||||||
|
"destroyed":["TSTALE"]},"s"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let created = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::AllOf(vec![
|
||||||
|
Matcher::Regex("ContactCard/set".into()),
|
||||||
|
Matcher::Regex("Small Person".into()),
|
||||||
|
]))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["ContactCard/set",{"accountId":"w",
|
||||||
|
"created":{"c2":{"id":"TSMALL"}}},"s"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let summary = sync::export::run(
|
||||||
|
common(&archive),
|
||||||
|
ExportConfig {
|
||||||
|
prune: true,
|
||||||
|
..export_cfg_objects(
|
||||||
|
&base,
|
||||||
|
vec![ObjectType::AddressBook, ObjectType::ContactCard],
|
||||||
|
)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("the oversized card must not abort the run");
|
||||||
|
|
||||||
|
created.assert();
|
||||||
|
destroyed.assert();
|
||||||
|
let counts = summary
|
||||||
|
.per_type
|
||||||
|
.iter()
|
||||||
|
.find(|(t, _)| *t == "ContactCard")
|
||||||
|
.map(|(_, c)| c.clone())
|
||||||
|
.expect("ContactCard counts");
|
||||||
|
assert_eq!(counts.created, 1, "{counts:?}");
|
||||||
|
assert_eq!(counts.failed, 1, "{counts:?}");
|
||||||
|
assert_eq!(
|
||||||
|
counts.deleted, 1,
|
||||||
|
"the prune plan stays complete: {counts:?}"
|
||||||
|
);
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn export_archive_read_failure_while_inlining_exits_seven() {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let base = server.url();
|
||||||
|
let api = "/jmap/api";
|
||||||
|
let archive = tmp();
|
||||||
|
{
|
||||||
|
let conn = db::init::open(&archive).unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO address_books (id,name,is_default) VALUES (1,'Personal',1)",
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let photo_blob = db::blobs::intern_blob(&conn, b"photo-bytes").unwrap();
|
||||||
|
let card = json!({
|
||||||
|
"@type": "Card",
|
||||||
|
"media": { "photo": { "@type": "Media", "kind": "photo",
|
||||||
|
"@blob": photo_blob, "mediaType": "image/png" } }
|
||||||
|
})
|
||||||
|
.to_string();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO contact_cards (id,uid,address_book_ids,data)
|
||||||
|
VALUES (1,'card-uid-1','[1]',?1)",
|
||||||
|
[card.as_str()],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
conn.execute("ALTER TABLE blobs RENAME COLUMN data TO payload", [])
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
let _root = server.mock("GET", "/").with_status(404).create();
|
||||||
|
let _wk = server
|
||||||
|
.mock("GET", "/.well-known/jmap")
|
||||||
|
.with_body(session_body_full(&base))
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
let _cc_query = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("ContactCard/query".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["ContactCard/query",{"accountId":"w","ids":[]},"q"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
let no_create = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("ContactCard/set".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["ContactCard/set",{"accountId":"w",
|
||||||
|
"created":{"c1":{"id":"TCARD"}}},"s"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(0)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let err = sync::export::run(
|
||||||
|
common(&archive),
|
||||||
|
export_cfg_objects(&base, vec![ObjectType::ContactCard]),
|
||||||
|
)
|
||||||
|
.expect_err("an unreadable blobs table must abort the run");
|
||||||
|
|
||||||
|
no_create.assert();
|
||||||
|
assert!(err.aborts_run(), "{err}");
|
||||||
|
assert_eq!(err.exit_code(), 7, "{err}");
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn export_sieve_script_matches_by_name_not_content() {
|
fn export_sieve_script_matches_by_name_not_content() {
|
||||||
let mut server = mockito::Server::new();
|
let mut server = mockito::Server::new();
|
||||||
@@ -3264,3 +3707,369 @@ fn import_jmap_duplicate_role_is_deduplicated_to_single_mailbox() {
|
|||||||
drop(conn);
|
drop(conn);
|
||||||
let _ = std::fs::remove_file(&archive);
|
let _ = std::fs::remove_file(&archive);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn export_aborts_with_exit_two_when_the_advertised_api_url_is_unreachable() {
|
||||||
|
let mut session_host = mockito::Server::new();
|
||||||
|
let mut api_host = mockito::Server::new();
|
||||||
|
let session_base = session_host.url();
|
||||||
|
let api_base = api_host.url();
|
||||||
|
let archive = tmp();
|
||||||
|
{
|
||||||
|
let conn = db::init::open(&archive).unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO mailboxes (id,name,parent_id,role,sort_order,is_subscribed)
|
||||||
|
VALUES (1,'Inbox',NULL,'inbox',0,1)",
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO address_books (id,name,description,is_default)
|
||||||
|
VALUES (1,'Personal',NULL,1)",
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
let _root = session_host.mock("GET", "/").with_status(404).create();
|
||||||
|
let _wk = session_host
|
||||||
|
.mock("GET", "/.well-known/jmap")
|
||||||
|
.with_body(session_body_full(&api_base))
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
let nginx = api_host
|
||||||
|
.mock("POST", "/jmap/api")
|
||||||
|
.with_status(404)
|
||||||
|
.with_header("content-type", "text/html")
|
||||||
|
.with_body("<html><head><title>404 Not Found</title></head><body>nginx</body></html>")
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let err = sync::export::run(
|
||||||
|
CommonConfig {
|
||||||
|
archive: archive.clone(),
|
||||||
|
threads: 1,
|
||||||
|
dry_run: false,
|
||||||
|
max_retries: 0,
|
||||||
|
allow_invalid_certs: false,
|
||||||
|
logger: Logger::from_flags(true, 0),
|
||||||
|
},
|
||||||
|
export_cfg_objects(
|
||||||
|
&session_base,
|
||||||
|
vec![ObjectType::Mailbox, ObjectType::AddressBook],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.expect_err("an unreachable apiUrl must abort the run");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
matches!(err, vandelay::error::Error::Connection(_)),
|
||||||
|
"a 404 from the advertised apiUrl is a whole-run connection failure, got {err:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(err.exit_code(), 2, "must not report a partial failure");
|
||||||
|
nginx.assert();
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn aborting_import_still_reports_the_types_that_finished() {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let base = server.url();
|
||||||
|
let api = "/jmap/api";
|
||||||
|
let archive = tmp();
|
||||||
|
|
||||||
|
let _root = server.mock("GET", "/").with_status(404).create();
|
||||||
|
let _wk = server
|
||||||
|
.mock("GET", "/.well-known/jmap")
|
||||||
|
.with_body(session_body_full(&base))
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
let _term = anchor_terminator(&mut server, api, "Mailbox");
|
||||||
|
|
||||||
|
let _mq = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("Mailbox/query".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["Mailbox/query",
|
||||||
|
{"accountId":"w","ids":["A"]},"q"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
let _mg = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("Mailbox/get".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["Mailbox/get",{"accountId":"w","state":"s1","list":[
|
||||||
|
{"id":"A","name":"alpha","parentId":null,"role":null,"sortOrder":0,"isSubscribed":true}
|
||||||
|
],"notFound":[]},"g"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
let _dead_endpoint = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("Email/query".into()))
|
||||||
|
.with_status(404)
|
||||||
|
.with_header("content-type", "text/html")
|
||||||
|
.with_body("<html><head><title>404 Not Found</title></head><body>nginx</body></html>")
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let outcome = sync::import_jmap::run_reporting(
|
||||||
|
common(&archive),
|
||||||
|
import_cfg_objects(&base, vec![ObjectType::Mailbox, ObjectType::Email]),
|
||||||
|
);
|
||||||
|
|
||||||
|
let err = outcome
|
||||||
|
.error
|
||||||
|
.as_ref()
|
||||||
|
.expect("a 404 from the api endpoint aborts the run");
|
||||||
|
assert!(
|
||||||
|
matches!(err, vandelay::error::Error::Connection(_)),
|
||||||
|
"the abort must keep its own exit code, got {err:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(err.exit_code(), 2);
|
||||||
|
|
||||||
|
let mailbox = outcome
|
||||||
|
.summary
|
||||||
|
.per_type
|
||||||
|
.iter()
|
||||||
|
.find(|(t, _)| *t == "Mailbox")
|
||||||
|
.map(|(_, c)| c.clone())
|
||||||
|
.expect("the type that finished before the abort must still be reported");
|
||||||
|
assert_eq!(mailbox.fetched, 1);
|
||||||
|
assert!(
|
||||||
|
!outcome.summary.per_type.iter().any(|(t, _)| *t == "Email"),
|
||||||
|
"the type that aborted contributes no counts"
|
||||||
|
);
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn seed_single_email(archive: &Path) {
|
||||||
|
let conn = db::init::open(archive).unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO mailboxes (id,name,parent_id,role) VALUES (1,'Inbox',NULL,'inbox')",
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let blob = db::blobs::intern_blob(
|
||||||
|
&conn,
|
||||||
|
b"From: a@x\r\nSubject: hi\r\nMessage-ID: <m-9@h>\r\n\r\nbody",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO emails (blob_id,received_at,mailbox_ids,keywords)
|
||||||
|
VALUES (?1,'2020-01-01T00:00:00Z','[1]','[\"$seen\"]')",
|
||||||
|
rusqlite::params![blob],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn mock_reconcile_for_single_email(server: &mut mockito::ServerGuard, base: &str, api: &str) {
|
||||||
|
server.mock("GET", "/").with_status(404).create();
|
||||||
|
server
|
||||||
|
.mock("GET", "/.well-known/jmap")
|
||||||
|
.with_body(session_body(base))
|
||||||
|
.create();
|
||||||
|
server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("Mailbox/query".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["Mailbox/query",
|
||||||
|
{"accountId":"w","ids":["t1"]},"q"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("Mailbox/query".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["Mailbox/query",
|
||||||
|
{"accountId":"w","ids":[]},"q"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("Mailbox/get".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["Mailbox/get",{"accountId":"w","list":[
|
||||||
|
{"id":"t1","name":"Inbox","role":"inbox","parentId":null,
|
||||||
|
"myRights":{"mayDelete":true}}],"notFound":[]},"g"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("Email/query".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["Email/query",
|
||||||
|
{"accountId":"w","ids":[]},"q"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_export(archive: &Path, base: &str, max_retries: u32) -> sync::Summary {
|
||||||
|
sync::export::run(
|
||||||
|
CommonConfig {
|
||||||
|
archive: archive.to_path_buf(),
|
||||||
|
threads: 1,
|
||||||
|
dry_run: false,
|
||||||
|
max_retries,
|
||||||
|
allow_invalid_certs: false,
|
||||||
|
logger: Logger::from_flags(true, 0),
|
||||||
|
},
|
||||||
|
ExportConfig {
|
||||||
|
connect: ConnectConfig {
|
||||||
|
url: base.to_owned(),
|
||||||
|
auth: Auth::Basic {
|
||||||
|
user: "u".into(),
|
||||||
|
password: "p".into(),
|
||||||
|
},
|
||||||
|
account: AccountSelector::Id("w".into()),
|
||||||
|
},
|
||||||
|
objects: None,
|
||||||
|
prune: false,
|
||||||
|
yes: true,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("export run")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn email_counts(summary: &sync::Summary) -> sync::TypeCounts {
|
||||||
|
summary
|
||||||
|
.per_type
|
||||||
|
.iter()
|
||||||
|
.find(|(t, _)| *t == "Email")
|
||||||
|
.map(|(_, c)| c.clone())
|
||||||
|
.expect("email counts")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn method_error_body(error_type: &str) -> String {
|
||||||
|
json!({"methodResponses":[["error",
|
||||||
|
{"type": error_type,
|
||||||
|
"description":"This server is temporarily unavailable. \
|
||||||
|
Attempting this same operation later may succeed."},
|
||||||
|
"i"]]})
|
||||||
|
.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn export_email_server_unavailable_is_retried_not_dropped() {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let base = server.url();
|
||||||
|
let api = "/jmap/api";
|
||||||
|
|
||||||
|
let archive = tmp();
|
||||||
|
seed_single_email(&archive);
|
||||||
|
mock_reconcile_for_single_email(&mut server, &base, api);
|
||||||
|
|
||||||
|
let up = server
|
||||||
|
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
|
||||||
|
.with_body(json!({"blobId":"UPLOADED"}).to_string())
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let failing = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("Email/import".into()))
|
||||||
|
.with_body(method_error_body("serverUnavailable"))
|
||||||
|
.expect(2)
|
||||||
|
.create();
|
||||||
|
let succeeding = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("Email/import".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["Email/import",{"accountId":"w",
|
||||||
|
"created":{"e1":{"id":"E1","blobId":"UPLOADED","threadId":"T1","size":48}}},"i"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let summary = run_export(&archive, &base, 3);
|
||||||
|
let email = email_counts(&summary);
|
||||||
|
|
||||||
|
failing.assert();
|
||||||
|
succeeding.assert();
|
||||||
|
up.assert();
|
||||||
|
assert_eq!(email.created, 1, "the message must survive a transient error");
|
||||||
|
assert_eq!(email.failed, 0, "a retried message is not a failure");
|
||||||
|
assert!(!summary.any_failed());
|
||||||
|
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn export_email_exhausted_method_retries_stay_per_message() {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let base = server.url();
|
||||||
|
let api = "/jmap/api";
|
||||||
|
|
||||||
|
let archive = tmp();
|
||||||
|
seed_single_email(&archive);
|
||||||
|
mock_reconcile_for_single_email(&mut server, &base, api);
|
||||||
|
|
||||||
|
let _up = server
|
||||||
|
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
|
||||||
|
.with_body(json!({"blobId":"UPLOADED"}).to_string())
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let failing = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("Email/import".into()))
|
||||||
|
.with_body(method_error_body("serverUnavailable"))
|
||||||
|
.expect(3)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let summary = run_export(&archive, &base, 2);
|
||||||
|
let email = email_counts(&summary);
|
||||||
|
|
||||||
|
failing.assert();
|
||||||
|
assert_eq!(email.created, 0);
|
||||||
|
assert_eq!(
|
||||||
|
email.failed, 1,
|
||||||
|
"exhausting the retries stays a per-message failure, not a run abort"
|
||||||
|
);
|
||||||
|
assert!(summary.any_failed(), "the run must still exit 5");
|
||||||
|
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn export_email_fatal_method_error_is_not_retried() {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let base = server.url();
|
||||||
|
let api = "/jmap/api";
|
||||||
|
|
||||||
|
let archive = tmp();
|
||||||
|
seed_single_email(&archive);
|
||||||
|
mock_reconcile_for_single_email(&mut server, &base, api);
|
||||||
|
|
||||||
|
let _up = server
|
||||||
|
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
|
||||||
|
.with_body(json!({"blobId":"UPLOADED"}).to_string())
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let failing = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("Email/import".into()))
|
||||||
|
.with_body(method_error_body("invalidArguments"))
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let summary = run_export(&archive, &base, 5);
|
||||||
|
let email = email_counts(&summary);
|
||||||
|
|
||||||
|
failing.assert();
|
||||||
|
assert_eq!(
|
||||||
|
email.failed, 1,
|
||||||
|
"a permanent method error must not consume retries"
|
||||||
|
);
|
||||||
|
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
}
|
||||||
|
|||||||
@@ -689,6 +689,167 @@ fn prune_non_tree_type_contact_card() {
|
|||||||
seeder::teardown(base_url()).expect("teardown");
|
seeder::teardown(base_url()).expect("teardown");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const ISSUE30_CARD_UID: &str = "vandelay-issue30-card";
|
||||||
|
const ISSUE30_EVENT_UID: &str = "vandelay-issue30-event";
|
||||||
|
|
||||||
|
fn data_uri_bytes(resource: &Value, uri_key: &str, expect_media_type: &str) -> Vec<u8> {
|
||||||
|
use base64::Engine;
|
||||||
|
let uri = resource
|
||||||
|
.get(uri_key)
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or_else(|| panic!("no {uri_key} on {resource}"));
|
||||||
|
assert!(
|
||||||
|
resource.get("blobId").is_none(),
|
||||||
|
"target must not hold a blobId for {uri_key}: {resource}"
|
||||||
|
);
|
||||||
|
let prefix = format!("data:{expect_media_type};base64,");
|
||||||
|
let payload = uri
|
||||||
|
.strip_prefix(&prefix)
|
||||||
|
.unwrap_or_else(|| panic!("{uri_key} is not a {prefix}... data URI: {resource}"));
|
||||||
|
base64::engine::general_purpose::STANDARD
|
||||||
|
.decode(payload)
|
||||||
|
.expect("base64 payload")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[ignore = "requires Docker"]
|
||||||
|
fn export_inlines_contact_and_event_blobs_instead_of_blob_ids() {
|
||||||
|
let fx = seeder::provision(base_url()).expect("provision");
|
||||||
|
let tgt = fx.account("test6").expect("test6");
|
||||||
|
let archive = tmp_archive("issue30");
|
||||||
|
|
||||||
|
let photo: &[u8] = b"\x89PNG\r\n\x1a\nvandelay issue 30 contact photo bytes";
|
||||||
|
let agenda: &[u8] = b"vandelay issue 30 calendar enclosure bytes";
|
||||||
|
{
|
||||||
|
let conn = vandelay::db::init::open(&archive).expect("init archive");
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO address_books (id,name,is_default) VALUES (1,'Issue30 Book',1)",
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO calendars (id,name,is_default) VALUES (1,'Issue30 Calendar',1)",
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let photo_blob = vandelay::db::blobs::intern_blob(&conn, photo).unwrap();
|
||||||
|
let agenda_blob = vandelay::db::blobs::intern_blob(&conn, agenda).unwrap();
|
||||||
|
let card = json!({
|
||||||
|
"@type": "Card",
|
||||||
|
"version": "1.0",
|
||||||
|
"name": { "full": "Issue Thirty" },
|
||||||
|
"media": { "photo": {
|
||||||
|
"@type": "Media", "kind": "photo",
|
||||||
|
"@blob": photo_blob, "mediaType": "image/png"
|
||||||
|
} }
|
||||||
|
})
|
||||||
|
.to_string();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO contact_cards (id,uid,address_book_ids,data) VALUES (1,?1,'[1]',?2)",
|
||||||
|
[ISSUE30_CARD_UID, card.as_str()],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let event = json!({
|
||||||
|
"@type": "Event",
|
||||||
|
"uid": ISSUE30_EVENT_UID,
|
||||||
|
"title": "Issue 30 review",
|
||||||
|
"start": "2026-03-01T09:00:00",
|
||||||
|
"duration": "PT1H",
|
||||||
|
"timeZone": "Etc/UTC",
|
||||||
|
"links": { "1": {
|
||||||
|
"@type": "Link", "rel": "enclosure",
|
||||||
|
"@blob": agenda_blob, "contentType": "text/plain", "title": "agenda.txt"
|
||||||
|
} }
|
||||||
|
})
|
||||||
|
.to_string();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO calendar_events (id,calendar_ids,is_draft,use_default_alerts,data)
|
||||||
|
VALUES (1,'[1]',0,0,?1)",
|
||||||
|
[event.as_str()],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
let summary = sync::export::run(
|
||||||
|
common(&archive, false),
|
||||||
|
export_cfg("test6", &tgt.account_id, false),
|
||||||
|
)
|
||||||
|
.expect("export");
|
||||||
|
for name in ["ContactCard", "CalendarEvent"] {
|
||||||
|
let counts = summary
|
||||||
|
.per_type
|
||||||
|
.iter()
|
||||||
|
.find(|(t, _)| *t == name)
|
||||||
|
.map(|(_, c)| c.clone())
|
||||||
|
.unwrap_or_else(|| panic!("{name} counts missing: {summary:?}"));
|
||||||
|
assert_eq!(counts.created, 1, "{name} created exactly one: {counts:?}");
|
||||||
|
assert_eq!(counts.failed, 0, "{name} had no failures: {counts:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
let client = HttpClient::new(basic("test6"), RetryPolicy::new(5), true);
|
||||||
|
let session = Session::discover(&client, base_url()).expect("discover target session");
|
||||||
|
let api = session.api_url.clone();
|
||||||
|
|
||||||
|
let cards = client
|
||||||
|
.post_json(
|
||||||
|
&api,
|
||||||
|
&json!({
|
||||||
|
"using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:contacts"],
|
||||||
|
"methodCalls": [["ContactCard/get",
|
||||||
|
{ "accountId": tgt.account_id, "ids": Value::Null }, "g"]]
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.expect("ContactCard/get");
|
||||||
|
let card = cards
|
||||||
|
.pointer("/methodResponses/0/1/list")
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.expect("ContactCard list")
|
||||||
|
.iter()
|
||||||
|
.find(|c| c.get("uid").and_then(Value::as_str) == Some(ISSUE30_CARD_UID))
|
||||||
|
.unwrap_or_else(|| panic!("exported card not found on the target: {cards}"))
|
||||||
|
.clone();
|
||||||
|
let photo_resource = card
|
||||||
|
.pointer("/media/photo")
|
||||||
|
.unwrap_or_else(|| panic!("card has no media.photo: {card}"));
|
||||||
|
assert_eq!(
|
||||||
|
data_uri_bytes(photo_resource, "uri", "image/png"),
|
||||||
|
photo,
|
||||||
|
"contact photo bytes survived the export: {card}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let events = client
|
||||||
|
.post_json(
|
||||||
|
&api,
|
||||||
|
&json!({
|
||||||
|
"using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:calendars"],
|
||||||
|
"methodCalls": [["CalendarEvent/get",
|
||||||
|
{ "accountId": tgt.account_id, "ids": Value::Null }, "g"]]
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.expect("CalendarEvent/get");
|
||||||
|
let event = events
|
||||||
|
.pointer("/methodResponses/0/1/list")
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.expect("CalendarEvent list")
|
||||||
|
.iter()
|
||||||
|
.find(|e| e.get("uid").and_then(Value::as_str) == Some(ISSUE30_EVENT_UID))
|
||||||
|
.unwrap_or_else(|| panic!("exported event not found on the target: {events}"))
|
||||||
|
.clone();
|
||||||
|
let link = event
|
||||||
|
.get("links")
|
||||||
|
.and_then(Value::as_object)
|
||||||
|
.and_then(|m| m.values().next())
|
||||||
|
.unwrap_or_else(|| panic!("event has no links: {event}"));
|
||||||
|
assert_eq!(
|
||||||
|
data_uri_bytes(link, "href", "text/plain"),
|
||||||
|
agenda,
|
||||||
|
"calendar enclosure bytes survived the export: {event}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
seeder::teardown(base_url()).expect("teardown");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[ignore = "requires Docker"]
|
#[ignore = "requires Docker"]
|
||||||
fn export_round_trip_and_convergence() {
|
fn export_round_trip_and_convergence() {
|
||||||
|
|||||||
Reference in New Issue
Block a user