export: bring matched items up to date on every run #7
+16
-9
@@ -211,16 +211,23 @@ inbuxa-migrate export \
|
|||||||
|
|
||||||
Writes `ARCHIVE` into an account on a JMAP server, usually inbuxa. It keeps no
|
Writes `ARCHIVE` into an account on a JMAP server, usually inbuxa. It keeps no
|
||||||
state of its own: every run matches the archive against the target afresh.
|
state of its own: every run matches the archive against the target afresh.
|
||||||
By default it only adds: what the archive holds and the target lacks is
|
By default it only adds and updates: what the target lacks is created, what
|
||||||
created, and anything already on the target is left as it is -- an item that
|
it has is brought up to date, and anything on the target that the archive
|
||||||
matches is not updated yet, so a change made at the source after the first
|
does not cover is left alone. So a second run after a later import carries
|
||||||
export does not reach the target on a second one.
|
what changed at the source in between.
|
||||||
|
|
||||||
Email is matched by Message-ID, or without one by sender, subject, date and
|
- **Email** is matched by Message-ID, or without one by sender, subject, date
|
||||||
recipients; where several messages share one, size decides. A message the
|
and recipients; where several messages share one, size decides. A message
|
||||||
source kept in several folders -- IMAP and Maildir copies, Gmail labels -- is
|
the source kept in several folders -- IMAP and Maildir copies, Gmail labels
|
||||||
written once, in all of them, and a later run adds any folder it is still
|
-- is written once, in all of them. On a match, its keywords (read,
|
||||||
missing on the target.
|
flagged and the rest) are set to the archive's, and so are its memberships
|
||||||
|
of folders this run migrated. Folders that exist only on the target are
|
||||||
|
left alone, and a message is never left in no folder.
|
||||||
|
- **Contacts and events** are matched by UID. When both copies carry an
|
||||||
|
`updated` time, the archive's is written only if it is newer; otherwise
|
||||||
|
the properties that differ are written.
|
||||||
|
- **Sieve scripts** are matched by name, and the target's is replaced when
|
||||||
|
its content differs from the archive's.
|
||||||
|
|
||||||
`--prune` also deletes what is on the target and not in the archive. It asks
|
`--prune` also deletes what is on the target and not in the archive. It asks
|
||||||
first; `--yes` answers for it, for scripts. Export speaks JMAP only.
|
first; `--yes` answers for it, for scripts. Export speaks JMAP only.
|
||||||
|
|||||||
+59
-1
@@ -5,7 +5,7 @@
|
|||||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::io::{IsTerminal, Write};
|
use std::io::{IsTerminal, Write};
|
||||||
|
|
||||||
use rusqlite::Connection;
|
use rusqlite::Connection;
|
||||||
@@ -49,6 +49,14 @@ impl Maps {
|
|||||||
fn insert(&mut self, ty: ObjectType, local: i64, target: JmapId) {
|
fn insert(&mut self, ty: ObjectType, local: i64, target: JmapId) {
|
||||||
self.m.entry(ty).or_default().insert(local, target);
|
self.m.entry(ty).or_default().insert(local, target);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Every target id this run mapped for `ty`: the objects it migrated.
|
||||||
|
fn targets_of(&self, ty: ObjectType) -> HashSet<String> {
|
||||||
|
self.m
|
||||||
|
.get(&ty)
|
||||||
|
.map(|m| m.values().map(|id| id.0.clone()).collect())
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl TargetResolver for Maps {
|
impl TargetResolver for Maps {
|
||||||
@@ -533,6 +541,56 @@ mod common {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Sends `updates` (target id, patch) as batched `/set` calls and counts
|
||||||
|
/// the result into `counts`. A dry run counts them as updated and sends
|
||||||
|
/// nothing.
|
||||||
|
pub fn update_batch(
|
||||||
|
net: &Net,
|
||||||
|
ty: ObjectType,
|
||||||
|
updates: Vec<(String, Value)>,
|
||||||
|
counts: &mut TypeCounts,
|
||||||
|
logger: &Logger,
|
||||||
|
) {
|
||||||
|
if updates.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if net.dry_run {
|
||||||
|
counts.updated += updates.len() as u64;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let total = updates.len() as u64;
|
||||||
|
let mut map = Map::new();
|
||||||
|
for (id, patch) in updates {
|
||||||
|
map.insert(id, patch);
|
||||||
|
}
|
||||||
|
match set_call(
|
||||||
|
&net.client,
|
||||||
|
&net.api,
|
||||||
|
&net.account,
|
||||||
|
ty.jmap_name(),
|
||||||
|
SetRequest {
|
||||||
|
update: Some(Value::Object(map)),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
&net.limits,
|
||||||
|
) {
|
||||||
|
Ok(outcome) => {
|
||||||
|
counts.updated += outcome.updated.len() as u64;
|
||||||
|
for (id, err) in &outcome.not_updated {
|
||||||
|
logger.warn(&format!("{}/set {id} not updated: {err}", ty.jmap_name()));
|
||||||
|
counts.failed += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
logger.warn(&format!(
|
||||||
|
"{}/set: updating {total} object(s) failed: {e}",
|
||||||
|
ty.jmap_name()
|
||||||
|
));
|
||||||
|
counts.failed += total;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn blob_not_found(outcome: &crate::jmap::request::SetOutcome, cid: &str) -> bool {
|
fn blob_not_found(outcome: &crate::jmap::request::SetOutcome, cid: &str) -> bool {
|
||||||
outcome.not_created.iter().any(|(c, err)| {
|
outcome.not_created.iter().any(|(c, err)| {
|
||||||
c == cid && err.get("type").and_then(Value::as_str) == Some("blobNotFound")
|
c == cid && err.get("type").and_then(Value::as_str) == Some("blobNotFound")
|
||||||
|
|||||||
+141
-67
@@ -9,12 +9,12 @@ use std::collections::{HashMap, HashSet};
|
|||||||
|
|
||||||
use serde_json::{Map, Value, json};
|
use serde_json::{Map, Value, json};
|
||||||
|
|
||||||
use super::common::{jid, target_query_get};
|
use super::common::{jid, target_query_get, update_batch};
|
||||||
use super::{Maps, Net, Plan, Uploader};
|
use super::{Maps, Net, Plan, Uploader};
|
||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
use crate::jmap::error::JmapError;
|
use crate::jmap::error::JmapError;
|
||||||
use crate::jmap::request::{
|
use crate::jmap::request::{
|
||||||
MethodCall, Request, SetRequest, check_method_error, get_objects, retry_method_call, set_call,
|
MethodCall, Request, check_method_error, get_objects, retry_method_call,
|
||||||
};
|
};
|
||||||
use crate::jmap::retry::MethodCallKind;
|
use crate::jmap::retry::MethodCallKind;
|
||||||
use crate::jmap::wire::JmapId;
|
use crate::jmap::wire::JmapId;
|
||||||
@@ -62,8 +62,12 @@ pub fn reconcile(
|
|||||||
) -> Result<Plan, Error> {
|
) -> Result<Plan, Error> {
|
||||||
let ty = ObjectType::Email;
|
let ty = ObjectType::Email;
|
||||||
|
|
||||||
let target_min = target_query_get(net, ty, Some(&["messageId", "size", "mailboxIds"]))
|
let target_min = target_query_get(
|
||||||
.map_err(Error::from)?;
|
net,
|
||||||
|
ty,
|
||||||
|
Some(&["messageId", "size", "mailboxIds", "keywords"]),
|
||||||
|
)
|
||||||
|
.map_err(Error::from)?;
|
||||||
let mut indices: Vec<EmailIndex> = target_min.iter().map(server_index).collect();
|
let mut indices: Vec<EmailIndex> = target_min.iter().map(server_index).collect();
|
||||||
|
|
||||||
let fallback_ids: Vec<JmapId> = target_min
|
let fallback_ids: Vec<JmapId> = target_min
|
||||||
@@ -128,11 +132,12 @@ pub fn reconcile(
|
|||||||
.collect();
|
.collect();
|
||||||
let pairs = pair_with_targets(&local_keys, &sizes, &target_keys, &targets);
|
let pairs = pair_with_targets(&local_keys, &sizes, &target_keys, &targets);
|
||||||
|
|
||||||
let mut membership_updates: Vec<(String, Value)> = Vec::new();
|
let migrated = maps.targets_of(ObjectType::Mailbox);
|
||||||
|
let mut updates: Vec<(String, Value)> = Vec::new();
|
||||||
for (i, unit) in units.iter().enumerate() {
|
for (i, unit) in units.iter().enumerate() {
|
||||||
match pairs[i] {
|
match pairs[i] {
|
||||||
Some(t) => match missing_memberships(&unit.row, &targets[t], maps) {
|
Some(t) => match email_patch(&unit.row, &targets[t], maps, &migrated) {
|
||||||
Some(patch) => membership_updates.push((targets[t].id.clone(), patch)),
|
Some(patch) => updates.push((targets[t].id.clone(), patch)),
|
||||||
None => counts.skipped += 1,
|
None => counts.skipped += 1,
|
||||||
},
|
},
|
||||||
None => export_one(
|
None => export_one(
|
||||||
@@ -146,7 +151,7 @@ pub fn reconcile(
|
|||||||
),
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
send_membership_updates(net, membership_updates, counts, logger);
|
update_batch(net, ty, updates, counts, logger);
|
||||||
|
|
||||||
Ok(Plan::default())
|
Ok(Plan::default())
|
||||||
}
|
}
|
||||||
@@ -197,6 +202,7 @@ struct TargetEmail {
|
|||||||
id: String,
|
id: String,
|
||||||
size: Option<u64>,
|
size: Option<u64>,
|
||||||
mailboxes: Option<HashSet<String>>,
|
mailboxes: Option<HashSet<String>>,
|
||||||
|
keywords: Option<HashSet<String>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl TargetEmail {
|
impl TargetEmail {
|
||||||
@@ -208,6 +214,10 @@ impl TargetEmail {
|
|||||||
.get("mailboxIds")
|
.get("mailboxIds")
|
||||||
.and_then(Value::as_object)
|
.and_then(Value::as_object)
|
||||||
.map(|m| m.keys().cloned().collect()),
|
.map(|m| m.keys().cloned().collect()),
|
||||||
|
keywords: v
|
||||||
|
.get("keywords")
|
||||||
|
.and_then(Value::as_object)
|
||||||
|
.map(|m| m.keys().map(|k| k.to_lowercase()).collect()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -254,65 +264,58 @@ fn pair_with_targets(
|
|||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The `Email/set` patch adding the folders a matched email is missing on the
|
/// The `Email/set` patch that brings a matched email on the target in line
|
||||||
/// target, or `None` when it is already in all of them. Folders that exist
|
/// with the archive, or `None` when it already is. The source is taken as
|
||||||
/// only on the target are left alone.
|
/// the truth for what it covers: keywords are added and removed to match, and
|
||||||
fn missing_memberships(row: &EmailRow, target: &TargetEmail, maps: &Maps) -> Option<Value> {
|
/// so are memberships of folders this run migrated. Folders that exist only
|
||||||
let have = target.mailboxes.as_ref()?;
|
/// on the target are left alone, an email is never left in no folder, and
|
||||||
|
/// whatever the server did not report is not touched.
|
||||||
|
fn email_patch(
|
||||||
|
row: &EmailRow,
|
||||||
|
target: &TargetEmail,
|
||||||
|
maps: &Maps,
|
||||||
|
migrated: &HashSet<String>,
|
||||||
|
) -> Option<Value> {
|
||||||
let mut patch = Map::new();
|
let mut patch = Map::new();
|
||||||
for ml in &row.mailbox_locals {
|
if let Some(have) = &target.mailboxes {
|
||||||
if let Some(t) = maps.target(ObjectType::Mailbox, *ml)
|
let want: HashSet<String> = row
|
||||||
&& !have.contains(&t.0)
|
.mailbox_locals
|
||||||
{
|
.iter()
|
||||||
patch.insert(format!("mailboxIds/{}", t.0), Value::Bool(true));
|
.filter_map(|ml| maps.target(ObjectType::Mailbox, *ml).map(|t| t.0))
|
||||||
|
.collect();
|
||||||
|
let add: Vec<&String> = want.iter().filter(|t| !have.contains(*t)).collect();
|
||||||
|
let remove: Vec<&String> = have
|
||||||
|
.iter()
|
||||||
|
.filter(|t| migrated.contains(*t) && !want.contains(*t))
|
||||||
|
.collect();
|
||||||
|
let left = have.len() - remove.len() + add.len();
|
||||||
|
for t in add {
|
||||||
|
patch.insert(
|
||||||
|
format!("mailboxIds/{}", pointer_escape(t)),
|
||||||
|
Value::Bool(true),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if left > 0 {
|
||||||
|
for t in remove {
|
||||||
|
patch.insert(format!("mailboxIds/{}", pointer_escape(t)), Value::Null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(have) = &target.keywords {
|
||||||
|
let want: HashSet<String> = row.keywords.iter().map(|k| k.to_lowercase()).collect();
|
||||||
|
for k in want.difference(have) {
|
||||||
|
patch.insert(format!("keywords/{}", pointer_escape(k)), Value::Bool(true));
|
||||||
|
}
|
||||||
|
for k in have.difference(&want) {
|
||||||
|
patch.insert(format!("keywords/{}", pointer_escape(k)), Value::Null);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
(!patch.is_empty()).then_some(Value::Object(patch))
|
(!patch.is_empty()).then_some(Value::Object(patch))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn send_membership_updates(
|
/// Escapes one JSON Pointer segment (RFC 6901), as JMAP patch paths use.
|
||||||
net: &Net,
|
fn pointer_escape(segment: &str) -> String {
|
||||||
updates: Vec<(String, Value)>,
|
segment.replace('~', "~0").replace('/', "~1")
|
||||||
counts: &mut TypeCounts,
|
|
||||||
logger: &Logger,
|
|
||||||
) {
|
|
||||||
if updates.is_empty() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if net.dry_run {
|
|
||||||
counts.updated += updates.len() as u64;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
let total = updates.len() as u64;
|
|
||||||
let mut map = Map::new();
|
|
||||||
for (id, patch) in updates {
|
|
||||||
map.insert(id, patch);
|
|
||||||
}
|
|
||||||
match set_call(
|
|
||||||
&net.client,
|
|
||||||
&net.api,
|
|
||||||
&net.account,
|
|
||||||
ObjectType::Email.jmap_name(),
|
|
||||||
SetRequest {
|
|
||||||
update: Some(Value::Object(map)),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
&net.limits,
|
|
||||||
) {
|
|
||||||
Ok(outcome) => {
|
|
||||||
counts.updated += outcome.updated.len() as u64;
|
|
||||||
for (id, err) in &outcome.not_updated {
|
|
||||||
logger.warn(&format!("Email/set {id}: folders not added: {err}"));
|
|
||||||
counts.failed += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
logger.warn(&format!(
|
|
||||||
"Email/set: adding folders to {total} email(s) failed: {e}"
|
|
||||||
));
|
|
||||||
counts.failed += total;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn build_mailbox_ids(row: &EmailRow, maps: &Maps) -> Option<Map<String, Value>> {
|
fn build_mailbox_ids(row: &EmailRow, maps: &Maps) -> Option<Map<String, Value>> {
|
||||||
@@ -570,9 +573,14 @@ mod tests {
|
|||||||
id: id.to_owned(),
|
id: id.to_owned(),
|
||||||
size,
|
size,
|
||||||
mailboxes: mailboxes.map(|m| m.iter().map(|s| (*s).to_owned()).collect()),
|
mailboxes: mailboxes.map(|m| m.iter().map(|s| (*s).to_owned()).collect()),
|
||||||
|
keywords: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn set(items: &[&str]) -> HashSet<String> {
|
||||||
|
items.iter().map(|s| (*s).to_owned()).collect()
|
||||||
|
}
|
||||||
|
|
||||||
fn mid(m: &str) -> EmailKey {
|
fn mid(m: &str) -> EmailKey {
|
||||||
EmailKey::MessageId(m.to_owned())
|
EmailKey::MessageId(m.to_owned())
|
||||||
}
|
}
|
||||||
@@ -628,19 +636,85 @@ mod tests {
|
|||||||
assert_eq!(pairs, vec![Some(0), None]);
|
assert_eq!(pairs, vec![Some(0), None]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
fn two_folder_maps() -> Maps {
|
||||||
fn missing_memberships_adds_only_the_absent_migrated_folders() {
|
|
||||||
let mut maps = Maps::default();
|
let mut maps = Maps::default();
|
||||||
maps.insert(ObjectType::Mailbox, 1, JmapId("T1".into()));
|
maps.insert(ObjectType::Mailbox, 1, JmapId("T1".into()));
|
||||||
maps.insert(ObjectType::Mailbox, 2, JmapId("T2".into()));
|
maps.insert(ObjectType::Mailbox, 2, JmapId("T2".into()));
|
||||||
|
maps
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn patch_adds_the_absent_migrated_folders() {
|
||||||
|
let maps = two_folder_maps();
|
||||||
let r = row(10, &[1, 2, 3], &[]);
|
let r = row(10, &[1, 2, 3], &[]);
|
||||||
let patch = missing_memberships(&r, &target("E", None, Some(&["T1", "Own"])), &maps)
|
let patch = email_patch(
|
||||||
.expect("T2 is missing");
|
&r,
|
||||||
|
&target("E", None, Some(&["T1", "Own"])),
|
||||||
|
&maps,
|
||||||
|
&set(&["T1", "T2"]),
|
||||||
|
)
|
||||||
|
.expect("T2 is missing");
|
||||||
assert_eq!(patch, json!({ "mailboxIds/T2": true }));
|
assert_eq!(patch, json!({ "mailboxIds/T2": true }));
|
||||||
assert!(missing_memberships(&r, &target("E", None, Some(&["T1", "T2"])), &maps).is_none());
|
|
||||||
assert!(
|
assert!(
|
||||||
missing_memberships(&r, &target("E", None, None), &maps).is_none(),
|
email_patch(
|
||||||
|
&r,
|
||||||
|
&target("E", None, Some(&["T1", "T2"])),
|
||||||
|
&maps,
|
||||||
|
&set(&["T1", "T2"])
|
||||||
|
)
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
email_patch(&r, &target("E", None, None), &maps, &set(&["T1", "T2"])).is_none(),
|
||||||
"unknown membership is left alone"
|
"unknown membership is left alone"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn patch_moves_between_migrated_folders_but_keeps_target_only_ones() {
|
||||||
|
let maps = two_folder_maps();
|
||||||
|
let r = row(10, &[2], &[]);
|
||||||
|
let patch = email_patch(
|
||||||
|
&r,
|
||||||
|
&target("E", None, Some(&["T1", "Own"])),
|
||||||
|
&maps,
|
||||||
|
&set(&["T1", "T2"]),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
patch,
|
||||||
|
json!({ "mailboxIds/T2": true, "mailboxIds/T1": null })
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn patch_never_leaves_an_email_in_no_folder() {
|
||||||
|
let maps = two_folder_maps();
|
||||||
|
let r = row(10, &[9], &[]);
|
||||||
|
assert!(
|
||||||
|
email_patch(
|
||||||
|
&r,
|
||||||
|
&target("E", None, Some(&["T1"])),
|
||||||
|
&maps,
|
||||||
|
&set(&["T1", "T2"])
|
||||||
|
)
|
||||||
|
.is_none(),
|
||||||
|
"the only folder is not removed when nothing replaces it"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn patch_syncs_keywords_both_ways_case_insensitively() {
|
||||||
|
let maps = two_folder_maps();
|
||||||
|
let r = row(10, &[1], &["$Seen", "work/urgent"]);
|
||||||
|
let mut t = target("E", None, Some(&["T1"]));
|
||||||
|
t.keywords = Some(set(&["$seen", "$flagged"]));
|
||||||
|
let patch = email_patch(&r, &t, &maps, &set(&["T1", "T2"])).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
patch,
|
||||||
|
json!({ "keywords/work~1urgent": true, "keywords/$flagged": null })
|
||||||
|
);
|
||||||
|
t.keywords = Some(set(&["$seen", "work/urgent"]));
|
||||||
|
assert!(email_patch(&r, &t, &maps, &set(&["T1", "T2"])).is_none());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+84
-23
@@ -9,10 +9,11 @@ use std::collections::{HashMap, HashSet};
|
|||||||
|
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
use super::common::{create_batch, jid, retry_if_blob_missing, target_get_all};
|
use super::common::{create_batch, jid, retry_if_blob_missing, target_get_all, update_batch};
|
||||||
use super::sieve_names;
|
use super::sieve_names;
|
||||||
use super::{Maps, Net, Plan, Uploader};
|
use super::{Maps, Net, Plan, Uploader};
|
||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
|
use crate::jmap::blobxfer;
|
||||||
use crate::jmap::request::{Request, check_method_error};
|
use crate::jmap::request::{Request, check_method_error};
|
||||||
use crate::logging::{LEVEL_DEFAULT, Logger};
|
use crate::logging::{LEVEL_DEFAULT, Logger};
|
||||||
use crate::sync::import_jmap::mapping::BlobBytes;
|
use crate::sync::import_jmap::mapping::BlobBytes;
|
||||||
@@ -31,10 +32,14 @@ pub fn reconcile(
|
|||||||
let targets = target_get_all(net, ty).map_err(Error::from)?;
|
let targets = target_get_all(net, ty).map_err(Error::from)?;
|
||||||
|
|
||||||
let mut target_by_name: HashMap<String, String> = HashMap::new();
|
let mut target_by_name: HashMap<String, String> = HashMap::new();
|
||||||
|
let mut target_blob: HashMap<String, String> = HashMap::new();
|
||||||
for t in &targets {
|
for t in &targets {
|
||||||
let (Some(id), Some(name)) = (jid(t), t.get("name").and_then(Value::as_str)) else {
|
let (Some(id), Some(name)) = (jid(t), t.get("name").and_then(Value::as_str)) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
|
if let Some(blob) = t.get("blobId").and_then(Value::as_str) {
|
||||||
|
target_blob.insert(id.clone(), blob.to_owned());
|
||||||
|
}
|
||||||
target_by_name.insert(name.to_owned(), id);
|
target_by_name.insert(name.to_owned(), id);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -66,19 +71,55 @@ pub fn reconcile(
|
|||||||
.find(|(_, _, a, _)| *a)
|
.find(|(_, _, a, _)| *a)
|
||||||
.map(|(_, n, _, _)| n.clone().unwrap_or_default());
|
.map(|(_, n, _, _)| n.clone().unwrap_or_default());
|
||||||
|
|
||||||
|
let mut updates: Vec<(String, Value)> = Vec::new();
|
||||||
|
|
||||||
for (local, name, is_active, blob_local) in &locals {
|
for (local, name, is_active, blob_local) in &locals {
|
||||||
let matched = name.as_ref().and_then(|n| target_by_name.get(n)).cloned();
|
let matched = name.as_ref().and_then(|n| target_by_name.get(n)).cloned();
|
||||||
|
let label = name.as_deref().unwrap_or("(unnamed)");
|
||||||
|
let rewritten = if rename_vendor {
|
||||||
|
renamed_script(&uploader, *blob_local)?
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
let target_id = if let Some(id) = matched {
|
let target_id = if let Some(id) = matched {
|
||||||
counts.skipped += 1;
|
// Compare what would be written -- the renamed bytes where the
|
||||||
|
// script needed renaming -- so an unchanged script stays unchanged.
|
||||||
|
let ours = match &rewritten {
|
||||||
|
Some((bytes, _)) => bytes.clone(),
|
||||||
|
None => uploader.bytes(*blob_local).map_err(Error::from)?,
|
||||||
|
};
|
||||||
|
match content_differs(net, &ours, target_blob.get(&id)) {
|
||||||
|
Ok(false) => counts.skipped += 1,
|
||||||
|
Ok(true) => {
|
||||||
|
let blob = match &rewritten {
|
||||||
|
Some((bytes, renamed)) => {
|
||||||
|
log_renames(label, renamed, logger);
|
||||||
|
uploader.upload_bytes_as(*blob_local, "application/sieve", bytes)
|
||||||
|
}
|
||||||
|
None => uploader.upload_with(*blob_local, "application/sieve"),
|
||||||
|
};
|
||||||
|
match blob {
|
||||||
|
Ok(b) => updates.push((id.clone(), json!({ "blobId": b.0 }))),
|
||||||
|
Err(e) => {
|
||||||
|
logger.warn(&format!(
|
||||||
|
"SieveScript {label}: upload for update failed: {e}"
|
||||||
|
));
|
||||||
|
counts.failed += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
logger.warn(&format!("SieveScript {label}: not compared: {e}"));
|
||||||
|
counts.skipped += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
id
|
id
|
||||||
} else {
|
} else {
|
||||||
let cid = format!("c{local}");
|
let cid = format!("c{local}");
|
||||||
let label = name.as_deref().unwrap_or("(unnamed)");
|
if let Some((_, renamed)) = &rewritten {
|
||||||
let rewritten = if rename_vendor {
|
log_renames(label, renamed, logger);
|
||||||
renamed_script(&uploader, *blob_local, label, logger)?
|
}
|
||||||
} else {
|
let rewritten = rewritten.as_ref().map(|(bytes, _)| bytes);
|
||||||
None
|
|
||||||
};
|
|
||||||
let build = |up: &mut Uploader<'_>| -> Result<Value, Error> {
|
let build = |up: &mut Uploader<'_>| -> Result<Value, Error> {
|
||||||
let blob_id = match &rewritten {
|
let blob_id = match &rewritten {
|
||||||
Some(bytes) => up.upload_bytes_as(*blob_local, "application/sieve", bytes),
|
Some(bytes) => up.upload_bytes_as(*blob_local, "application/sieve", bytes),
|
||||||
@@ -120,6 +161,8 @@ pub fn reconcile(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
update_batch(net, ty, updates, counts, logger);
|
||||||
|
|
||||||
if active_target.is_none() && locals.iter().all(|(_, _, a, _)| !*a) {
|
if active_target.is_none() && locals.iter().all(|(_, _, a, _)| !*a) {
|
||||||
deactivate = true;
|
deactivate = true;
|
||||||
}
|
}
|
||||||
@@ -190,22 +233,40 @@ fn target_sieve_extensions(net: &Net) -> Vec<String> {
|
|||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A script's bytes after renaming, and the names that were renamed.
|
||||||
|
type Renamed = (Vec<u8>, Vec<String>);
|
||||||
|
|
||||||
/// The script's bytes with Stalwart's vendor names renamed for an inbuxa
|
/// The script's bytes with Stalwart's vendor names renamed for an inbuxa
|
||||||
/// target, or `None` when it needs no change. Each rename is logged.
|
/// target, and the names renamed, or `None` when it needs no change.
|
||||||
fn renamed_script(
|
fn renamed_script(uploader: &Uploader<'_>, blob_local: i64) -> Result<Option<Renamed>, Error> {
|
||||||
uploader: &Uploader<'_>,
|
|
||||||
blob_local: i64,
|
|
||||||
label: &str,
|
|
||||||
logger: &Logger,
|
|
||||||
) -> Result<Option<Vec<u8>>, Error> {
|
|
||||||
let bytes = uploader.bytes(blob_local).map_err(Error::from)?;
|
let bytes = uploader.bytes(blob_local).map_err(Error::from)?;
|
||||||
Ok(sieve_names::rewrite(&bytes).map(|(out, renamed)| {
|
Ok(sieve_names::rewrite(&bytes))
|
||||||
for old in &renamed {
|
}
|
||||||
let new = old.replacen("vnd.stalwart.", "vnd.inbuxa.", 1);
|
|
||||||
if logger.enabled(LEVEL_DEFAULT) {
|
/// Prints each rename made to a script about to be written.
|
||||||
eprintln!("export: SieveScript {label}: renamed {old} to {new}");
|
fn log_renames(label: &str, renamed: &[String], logger: &Logger) {
|
||||||
}
|
for old in renamed {
|
||||||
|
let new = old.replacen("vnd.stalwart.", "vnd.inbuxa.", 1);
|
||||||
|
if logger.enabled(LEVEL_DEFAULT) {
|
||||||
|
eprintln!("export: SieveScript {label}: renamed {old} to {new}");
|
||||||
}
|
}
|
||||||
out
|
}
|
||||||
}))
|
}
|
||||||
|
|
||||||
|
/// Whether the target's copy of a script differs from `ours`. A target that
|
||||||
|
/// reports no blob is taken as different, so ours is written.
|
||||||
|
fn content_differs(net: &Net, ours: &[u8], target_blob: Option<&String>) -> Result<bool, Error> {
|
||||||
|
let Some(target_blob) = target_blob else {
|
||||||
|
return Ok(true);
|
||||||
|
};
|
||||||
|
let theirs = blobxfer::download_bytes(
|
||||||
|
&net.client,
|
||||||
|
&net.session,
|
||||||
|
&net.account,
|
||||||
|
target_blob,
|
||||||
|
"application/sieve",
|
||||||
|
"script.sieve",
|
||||||
|
)
|
||||||
|
.map_err(Error::from)?;
|
||||||
|
Ok(ours != theirs.as_slice())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,15 +1,16 @@
|
|||||||
/*
|
/*
|
||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
|
* SPDX-FileCopyrightText: 2026 John Coffey <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use std::collections::HashSet;
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::fmt::Write as _;
|
use std::fmt::Write as _;
|
||||||
|
|
||||||
use serde_json::Value;
|
use serde_json::{Map, Value};
|
||||||
|
|
||||||
use super::common::{create_batch, jid, target_query_get};
|
use super::common::{create_batch, jid, target_query_get, update_batch};
|
||||||
use super::{Maps, Net, Plan, Uploader};
|
use super::{Maps, Net, Plan, Uploader};
|
||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
use crate::logging::Logger;
|
use crate::logging::Logger;
|
||||||
@@ -42,10 +43,10 @@ pub fn reconcile(
|
|||||||
logger: &Logger,
|
logger: &Logger,
|
||||||
) -> Result<Plan, Error> {
|
) -> Result<Plan, Error> {
|
||||||
let targets = target_query_get(net, ty, None).map_err(Error::from)?;
|
let targets = target_query_get(net, ty, None).map_err(Error::from)?;
|
||||||
let mut by_uid: std::collections::HashMap<String, String> = std::collections::HashMap::new();
|
let mut by_uid: HashMap<String, (String, &Value)> = HashMap::new();
|
||||||
for t in &targets {
|
for t in &targets {
|
||||||
if let (Some(uid), Some(id)) = (target_uid(t), jid(t)) {
|
if let (Some(uid), Some(id)) = (target_uid(t), jid(t)) {
|
||||||
by_uid.entry(uid).or_insert(id);
|
by_uid.entry(uid).or_insert((id, t));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -76,12 +77,23 @@ pub fn reconcile(
|
|||||||
};
|
};
|
||||||
|
|
||||||
let mut matched_uids: HashSet<String> = HashSet::new();
|
let mut matched_uids: HashSet<String> = HashSet::new();
|
||||||
|
let mut updates: Vec<(String, Value)> = Vec::new();
|
||||||
let blobs = Uploader::new(net, &ctx.conn);
|
let blobs = Uploader::new(net, &ctx.conn);
|
||||||
for (local, uid) in &rows {
|
for (local, uid) in &rows {
|
||||||
if let Some(tid) = by_uid.get(uid) {
|
if let Some((tid, existing)) = by_uid.get(uid) {
|
||||||
maps.insert(ty, *local, crate::jmap::wire::JmapId(tid.clone()));
|
maps.insert(ty, *local, crate::jmap::wire::JmapId(tid.clone()));
|
||||||
matched_uids.insert(uid.clone());
|
matched_uids.insert(uid.clone());
|
||||||
counts.skipped += 1;
|
match build_wire(ctx, ty, *local, maps, &blobs) {
|
||||||
|
Ok(wire) => match changed_properties(&wire, existing) {
|
||||||
|
Some(patch) => updates.push((tid.clone(), patch)),
|
||||||
|
None => counts.skipped += 1,
|
||||||
|
},
|
||||||
|
Err(e) if e.aborts_run() => return Err(e),
|
||||||
|
Err(e) => {
|
||||||
|
logger.warn(&format!("{} not compared: {e}", describe(ty, *local, uid)));
|
||||||
|
counts.skipped += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let cid = format!("c{local}");
|
let cid = format!("c{local}");
|
||||||
@@ -123,6 +135,8 @@ pub fn reconcile(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
update_batch(net, ty, updates, counts, logger);
|
||||||
|
|
||||||
let objs: Vec<TargetObj> = targets
|
let objs: Vec<TargetObj> = targets
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|t| {
|
.filter_map(|t| {
|
||||||
@@ -172,3 +186,80 @@ fn build_wire(
|
|||||||
calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, maps, blobs).map_err(Error::from)
|
calendar_event_to_wire(&cal, dr != 0, ud != 0, &data, maps, blobs).map_err(Error::from)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// An `updated` value as a point in time, so that offsets and fractional
|
||||||
|
/// seconds compare as the same instant. `None` if it does not parse.
|
||||||
|
fn parse_updated(s: &str) -> Option<time::OffsetDateTime> {
|
||||||
|
time::OffsetDateTime::parse(s, &time::format_description::well_known::Rfc3339).ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The update that makes `target` match the archive's `wire` object, or
|
||||||
|
/// `None` when nothing changed. When both carry `updated`, it decides: the
|
||||||
|
/// archive's copy wins only if it is newer. Otherwise each property the
|
||||||
|
/// archive writes is compared, and those that differ are sent whole.
|
||||||
|
fn changed_properties(wire: &Value, target: &Value) -> Option<Value> {
|
||||||
|
let wire = wire.as_object()?;
|
||||||
|
let stamp = |v: Option<&Value>| v.and_then(Value::as_str).and_then(parse_updated);
|
||||||
|
if let (Some(ours), Some(theirs)) = (stamp(wire.get("updated")), stamp(target.get("updated")))
|
||||||
|
&& ours <= theirs
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let mut patch = Map::new();
|
||||||
|
for (k, v) in wire {
|
||||||
|
if k == "uid" || k == "id" {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if target.get(k) != Some(v) {
|
||||||
|
patch.insert(k.clone(), v.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(!patch.is_empty()).then_some(Value::Object(patch))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unchanged_object_needs_no_update() {
|
||||||
|
let wire = json!({"uid": "u", "name": {"full": "Ann"}, "addressBookIds": {"A": true}});
|
||||||
|
let target = json!({"id": "T", "uid": "u", "name": {"full": "Ann"},
|
||||||
|
"addressBookIds": {"A": true}, "extra": 1});
|
||||||
|
assert_eq!(changed_properties(&wire, &target), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn changed_properties_are_sent_whole() {
|
||||||
|
let wire = json!({"uid": "u", "name": {"full": "Ann B"}, "addressBookIds": {"A": true}});
|
||||||
|
let target = json!({"id": "T", "uid": "u", "name": {"full": "Ann"},
|
||||||
|
"addressBookIds": {"A": true}});
|
||||||
|
assert_eq!(
|
||||||
|
changed_properties(&wire, &target),
|
||||||
|
Some(json!({"name": {"full": "Ann B"}}))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn updated_compares_instants_not_strings() {
|
||||||
|
let target = json!({"uid": "u", "title": "old", "updated": "2026-01-02T00:00:00Z"});
|
||||||
|
let same_instant = json!({"uid": "u", "title": "new",
|
||||||
|
"updated": "2026-01-02T01:00:00.000+01:00"});
|
||||||
|
assert_eq!(changed_properties(&same_instant, &target), None);
|
||||||
|
let later = json!({"uid": "u", "title": "new", "updated": "2026-01-02T00:00:00.5Z"});
|
||||||
|
assert!(changed_properties(&later, &target).is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn updated_decides_when_both_sides_carry_it() {
|
||||||
|
let target = json!({"uid": "u", "title": "old", "updated": "2026-01-02T00:00:00Z"});
|
||||||
|
let older = json!({"uid": "u", "title": "new", "updated": "2026-01-01T00:00:00Z"});
|
||||||
|
assert_eq!(changed_properties(&older, &target), None, "target is newer");
|
||||||
|
let newer = json!({"uid": "u", "title": "new", "updated": "2026-01-03T00:00:00Z"});
|
||||||
|
assert_eq!(
|
||||||
|
changed_properties(&newer, &target),
|
||||||
|
Some(json!({"title": "new", "updated": "2026-01-03T00:00:00Z"}))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+405
-5
@@ -2575,7 +2575,7 @@ fn export_archive_read_failure_while_inlining_exits_seven() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn export_sieve_script_matches_by_name_not_content() {
|
fn export_sieve_script_matched_by_name_is_updated_when_its_content_differs() {
|
||||||
let mut server = mockito::Server::new();
|
let mut server = mockito::Server::new();
|
||||||
let base = server.url();
|
let base = server.url();
|
||||||
let api = "/jmap/api";
|
let api = "/jmap/api";
|
||||||
@@ -2615,9 +2615,125 @@ fn export_sieve_script_matches_by_name_not_content() {
|
|||||||
)
|
)
|
||||||
.expect(1)
|
.expect(1)
|
||||||
.create();
|
.create();
|
||||||
let no_download = server
|
let download = server
|
||||||
.mock("GET", Matcher::Regex("/jmap/dl/w/BSRV/.*".into()))
|
.mock("GET", Matcher::Regex("/jmap/dl/w/BSRV/.*".into()))
|
||||||
.with_body(b"unused".as_slice())
|
.with_body(b"keep;\n".as_slice())
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let upload = server
|
||||||
|
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
|
||||||
|
.with_body(json!({"blobId":"UPN"}).to_string())
|
||||||
|
.expect(2)
|
||||||
|
.create();
|
||||||
|
let update = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::AllOf(vec![
|
||||||
|
Matcher::Regex("SieveScript/set".into()),
|
||||||
|
Matcher::Regex("\"update\":\\{\"S1\":\\{\"blobId\":\"UPN\"".into()),
|
||||||
|
]))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["SieveScript/set",{"accountId":"w",
|
||||||
|
"updated":{"S1":null}},"s"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let create = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::AllOf(vec![
|
||||||
|
Matcher::Regex("SieveScript/set".into()),
|
||||||
|
Matcher::Regex("reject".into()),
|
||||||
|
]))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["SieveScript/set",{"accountId":"w",
|
||||||
|
"created":{"c2":{"id":"S2"}}},"s"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let _activate = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("onSuccessActivateScript".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["SieveScript/set",{"accountId":"w"},"a"]]}).to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let summary = sync::export::run(
|
||||||
|
common(&archive),
|
||||||
|
export_cfg_objects(&base, vec![ObjectType::SieveScript]),
|
||||||
|
)
|
||||||
|
.expect("export");
|
||||||
|
upload.assert();
|
||||||
|
create.assert();
|
||||||
|
download.assert();
|
||||||
|
update.assert();
|
||||||
|
let counts = summary
|
||||||
|
.per_type
|
||||||
|
.iter()
|
||||||
|
.find(|(t, _)| *t == "SieveScript")
|
||||||
|
.map(|(_, c)| c.clone())
|
||||||
|
.expect("sieve counts");
|
||||||
|
assert_eq!(
|
||||||
|
counts.updated, 1,
|
||||||
|
"the name-matched script gets the archive's content"
|
||||||
|
);
|
||||||
|
assert_eq!(counts.skipped, 0);
|
||||||
|
assert_eq!(counts.created, 1, "the unmatched name is created");
|
||||||
|
assert_eq!(counts.failed, 0);
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn export_sieve_script_matched_by_name_with_the_same_content_is_left_alone() {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let base = server.url();
|
||||||
|
let api = "/jmap/api";
|
||||||
|
let archive = tmp();
|
||||||
|
let keepall_local = b"require [\"fileinto\"];\nkeep;\n";
|
||||||
|
let reject_local = b"require [\"reject\"];\nreject \"go away\";\n";
|
||||||
|
{
|
||||||
|
let conn = db::init::open(&archive).unwrap();
|
||||||
|
let blob1 = db::blobs::intern_blob(&conn, keepall_local).unwrap();
|
||||||
|
let blob2 = db::blobs::intern_blob(&conn, reject_local).unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO sieve_scripts (id,name,is_active,blob_id) VALUES (1,'keepall',1,?1)",
|
||||||
|
rusqlite::params![blob1],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO sieve_scripts (id,name,is_active,blob_id) VALUES (2,'reject',0,?1)",
|
||||||
|
rusqlite::params![blob2],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
let _root = server.mock("GET", "/").with_status(404).create();
|
||||||
|
let _wk = server
|
||||||
|
.mock("GET", "/.well-known/jmap")
|
||||||
|
.with_body(session_body_full(&base))
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
let _g = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("SieveScript/get".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["SieveScript/get",{"accountId":"w","list":[
|
||||||
|
{"id":"S1","name":"keepall","isActive":false,"blobId":"BSRV"}
|
||||||
|
],"notFound":[]},"g"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let download = server
|
||||||
|
.mock("GET", Matcher::Regex("/jmap/dl/w/BSRV/.*".into()))
|
||||||
|
.with_body(keepall_local.as_slice())
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let no_update = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("\"update\"".into()))
|
||||||
.expect(0)
|
.expect(0)
|
||||||
.create();
|
.create();
|
||||||
let upload = server
|
let upload = server
|
||||||
@@ -2654,7 +2770,8 @@ fn export_sieve_script_matches_by_name_not_content() {
|
|||||||
.expect("export");
|
.expect("export");
|
||||||
upload.assert();
|
upload.assert();
|
||||||
create.assert();
|
create.assert();
|
||||||
no_download.assert();
|
download.assert();
|
||||||
|
no_update.assert();
|
||||||
let counts = summary
|
let counts = summary
|
||||||
.per_type
|
.per_type
|
||||||
.iter()
|
.iter()
|
||||||
@@ -2663,8 +2780,9 @@ fn export_sieve_script_matches_by_name_not_content() {
|
|||||||
.expect("sieve counts");
|
.expect("sieve counts");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
counts.skipped, 1,
|
counts.skipped, 1,
|
||||||
"name-matched script is skipped even though its content differs from the target"
|
"same name and same content: nothing to do"
|
||||||
);
|
);
|
||||||
|
assert_eq!(counts.updated, 0);
|
||||||
assert_eq!(counts.created, 1, "the unmatched name is created");
|
assert_eq!(counts.created, 1, "the unmatched name is created");
|
||||||
assert_eq!(counts.failed, 0);
|
assert_eq!(counts.failed, 0);
|
||||||
let _ = std::fs::remove_file(&archive);
|
let _ = std::fs::remove_file(&archive);
|
||||||
@@ -4900,3 +5018,285 @@ fn export_different_messages_sharing_a_message_id_are_not_merged() {
|
|||||||
no_set.assert();
|
no_set.assert();
|
||||||
let _ = std::fs::remove_file(&archive);
|
let _ = std::fs::remove_file(&archive);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn export_rerun_carries_a_read_flag_set_at_the_source() {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let base = server.url();
|
||||||
|
let api = "/jmap/api";
|
||||||
|
let archive = tmp();
|
||||||
|
let _setup = two_folder_archive_and_target(&mut server, &archive);
|
||||||
|
insert_email_copy(&archive, ONE_MESSAGE, 1);
|
||||||
|
{
|
||||||
|
let conn = db::init::open(&archive).unwrap();
|
||||||
|
conn.execute("UPDATE emails SET keywords='[\"$seen\"]'", [])
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
let _eq = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("Email/query".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["Email/query",{"accountId":"w","ids":["X1"]},"q"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let _eg = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("Email/get".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["Email/get",{"accountId":"w","list":[
|
||||||
|
{"id":"X1","messageId":["both@h"],"size":ONE_MESSAGE.len(),
|
||||||
|
"mailboxIds":{"T1":true},"keywords":{"$flagged":true}}
|
||||||
|
],"notFound":[]},"g"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let set = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::AllOf(vec![
|
||||||
|
Matcher::Regex("Email/set".into()),
|
||||||
|
Matcher::Regex("\"keywords/\\$seen\":true".into()),
|
||||||
|
Matcher::Regex("\"keywords/\\$flagged\":null".into()),
|
||||||
|
]))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["Email/set",{"accountId":"w","updated":{"X1":null}},"s"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let summary = sync::export::run(
|
||||||
|
common(&archive),
|
||||||
|
export_cfg_objects(&base, vec![ObjectType::Mailbox, ObjectType::Email]),
|
||||||
|
)
|
||||||
|
.expect("export");
|
||||||
|
let email = email_counts(&summary);
|
||||||
|
assert_eq!(email.updated, 1);
|
||||||
|
assert_eq!(email.created, 0);
|
||||||
|
assert_eq!(email.failed, 0);
|
||||||
|
set.assert();
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn contact_rerun(local_updated: &str, updates_sent: usize) -> inbuxa_migrate::sync::TypeCounts {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let base = server.url();
|
||||||
|
let api = "/jmap/api";
|
||||||
|
let archive = tmp();
|
||||||
|
{
|
||||||
|
let conn = db::init::open(&archive).unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO address_books (id,name,description,is_default) VALUES (1,'Personal',NULL,1)",
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let card = json!({"@type":"Card","version":"1.0","uid":"u1",
|
||||||
|
"name":{"full":"Ann Brown"},"updated":local_updated});
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO contact_cards (id,uid,address_book_ids,data) VALUES (1,'u1','[1]',?1)",
|
||||||
|
rusqlite::params![card.to_string()],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let _root = server.mock("GET", "/").with_status(404).create();
|
||||||
|
let _wk = server
|
||||||
|
.mock("GET", "/.well-known/jmap")
|
||||||
|
.with_body(session_body_full(&base))
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
let _abg = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("AddressBook/get".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["AddressBook/get",{"accountId":"w","list":[
|
||||||
|
{"id":"P","name":"Personal","isDefault":true,"myRights":{"mayDelete":false}}
|
||||||
|
],"notFound":[]},"g"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
let _term = anchor_terminator(&mut server, api, "ContactCard");
|
||||||
|
let _cq = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("ContactCard/query".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["ContactCard/query",{"accountId":"w","ids":["C1"]},"q"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let _cg = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("ContactCard/get".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["ContactCard/get",{"accountId":"w","list":[
|
||||||
|
{"id":"C1","@type":"Card","version":"1.0","uid":"u1",
|
||||||
|
"name":{"full":"Ann"},"addressBookIds":{"P":true},
|
||||||
|
"updated":"2026-02-01T00:00:00Z"}
|
||||||
|
],"notFound":[]},"g"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let update = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::AllOf(vec![
|
||||||
|
Matcher::Regex("ContactCard/set".into()),
|
||||||
|
Matcher::Regex("\"update\"".into()),
|
||||||
|
Matcher::Regex("Ann Brown".into()),
|
||||||
|
]))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["ContactCard/set",{"accountId":"w","updated":{"C1":null}},"s"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(updates_sent)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let summary = sync::export::run(
|
||||||
|
common(&archive),
|
||||||
|
export_cfg_objects(
|
||||||
|
&base,
|
||||||
|
vec![ObjectType::AddressBook, ObjectType::ContactCard],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.expect("export");
|
||||||
|
let counts = summary
|
||||||
|
.per_type
|
||||||
|
.iter()
|
||||||
|
.find(|(t, _)| *t == "ContactCard")
|
||||||
|
.map(|(_, c)| c.clone())
|
||||||
|
.expect("contact counts");
|
||||||
|
update.assert();
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
counts
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn export_rerun_updates_a_contact_edited_at_the_source() {
|
||||||
|
let counts = contact_rerun("2026-03-01T00:00:00Z", 1);
|
||||||
|
assert_eq!(counts.updated, 1, "the newer archive copy is written");
|
||||||
|
assert_eq!(counts.failed, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn export_rerun_leaves_a_contact_alone_when_the_target_is_newer() {
|
||||||
|
let counts = contact_rerun("2026-01-01T00:00:00Z", 0);
|
||||||
|
assert_eq!(
|
||||||
|
counts.updated, 0,
|
||||||
|
"the target's copy is newer, so nothing is sent"
|
||||||
|
);
|
||||||
|
assert_eq!(counts.skipped, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn event_rerun(local_updated: &str, updates_sent: usize) -> inbuxa_migrate::sync::TypeCounts {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let base = server.url();
|
||||||
|
let api = "/jmap/api";
|
||||||
|
let archive = tmp();
|
||||||
|
{
|
||||||
|
let conn = db::init::open(&archive).unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO calendars (id,name,is_default) VALUES (1,'Work',1)",
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let event = json!({"@type":"Event","uid":"ev1","title":"Planning, moved",
|
||||||
|
"start":"2026-03-02T10:00:00","duration":"PT1H",
|
||||||
|
"updated":local_updated});
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO calendar_events (id,calendar_ids,is_draft,use_default_alerts,data)
|
||||||
|
VALUES (1,'[1]',0,0,?1)",
|
||||||
|
rusqlite::params![event.to_string()],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let _root = server.mock("GET", "/").with_status(404).create();
|
||||||
|
let _wk = server
|
||||||
|
.mock("GET", "/.well-known/jmap")
|
||||||
|
.with_body(session_body_full(&base))
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
let _calg = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("Calendar/get".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["Calendar/get",{"accountId":"w","list":[
|
||||||
|
{"id":"K","name":"Work","isDefault":true,"myRights":{"mayDelete":false}}
|
||||||
|
],"notFound":[]},"g"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
let _term = anchor_terminator(&mut server, api, "CalendarEvent");
|
||||||
|
let _eq = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("CalendarEvent/query".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["CalendarEvent/query",{"accountId":"w","ids":["E1"]},"q"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let _eg = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("CalendarEvent/get".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["CalendarEvent/get",{"accountId":"w","list":[
|
||||||
|
{"id":"E1","@type":"Event","uid":"ev1","title":"Planning",
|
||||||
|
"start":"2026-03-01T10:00:00","duration":"PT1H",
|
||||||
|
"calendarIds":{"K":true},"isDraft":false,"useDefaultAlerts":false,
|
||||||
|
"updated":"2026-02-01T00:00:00Z"}
|
||||||
|
],"notFound":[]},"g"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let update = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::AllOf(vec![
|
||||||
|
Matcher::Regex("CalendarEvent/set".into()),
|
||||||
|
Matcher::Regex("\"update\"".into()),
|
||||||
|
Matcher::Regex("Planning, moved".into()),
|
||||||
|
]))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["CalendarEvent/set",{"accountId":"w","updated":{"E1":null}},"s"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(updates_sent)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let summary = sync::export::run(
|
||||||
|
common(&archive),
|
||||||
|
export_cfg_objects(&base, vec![ObjectType::Calendar, ObjectType::CalendarEvent]),
|
||||||
|
)
|
||||||
|
.expect("export");
|
||||||
|
let counts = summary
|
||||||
|
.per_type
|
||||||
|
.iter()
|
||||||
|
.find(|(t, _)| *t == "CalendarEvent")
|
||||||
|
.map(|(_, c)| c.clone())
|
||||||
|
.expect("event counts");
|
||||||
|
update.assert();
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
counts
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn export_rerun_updates_an_event_moved_at_the_source() {
|
||||||
|
let counts = event_rerun("2026-03-01T00:00:00Z", 1);
|
||||||
|
assert_eq!(counts.updated, 1, "the newer archive copy is written");
|
||||||
|
assert_eq!(counts.failed, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn export_rerun_leaves_an_event_alone_when_nothing_is_newer() {
|
||||||
|
let counts = event_rerun("2026-02-01T01:00:00+01:00", 0);
|
||||||
|
assert_eq!(
|
||||||
|
counts.updated, 0,
|
||||||
|
"same instant as the target's, written another way"
|
||||||
|
);
|
||||||
|
assert_eq!(counts.skipped, 1);
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user