Fix sync issues
This commit is contained in:
@@ -57,6 +57,56 @@ Because the archive is a self-contained SQLite file that fully describes one acc
|
|||||||
- **Source-change protection:** An archive remembers which account it was filled from; pointing it at a different one fails unless explicitly permitted.
|
- **Source-change protection:** An archive remembers which account it was filled from; pointing it at a different one fails unless explicitly permitted.
|
||||||
- **Read-only inspection:** A built-in `inspect` command dumps any object type from an archive for verification.
|
- **Read-only inspection:** A built-in `inspect` command dumps any object type from an archive for verification.
|
||||||
|
|
||||||
|
## Install
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# macOS / Linux
|
||||||
|
curl --proto '=https' --tlsv1.2 -LsSf \
|
||||||
|
https://github.com/stalwartlabs/vandelay/releases/latest/download/vandelay-installer.sh | sh
|
||||||
|
|
||||||
|
# Homebrew
|
||||||
|
brew install stalwartlabs/tap/vandelay
|
||||||
|
|
||||||
|
# Windows
|
||||||
|
powershell -ExecutionPolicy Bypass -c "irm https://github.com/stalwartlabs/vandelay/releases/latest/download/vandelay-installer.ps1 | iex"
|
||||||
|
|
||||||
|
# npm
|
||||||
|
npm install -g @stalwartlabs/vandelay
|
||||||
|
|
||||||
|
# From source
|
||||||
|
cargo install --path .
|
||||||
|
```
|
||||||
|
|
||||||
|
A signed `.msi` is also published with each release.
|
||||||
|
|
||||||
|
## Quick start
|
||||||
|
|
||||||
|
A typical run is two commands, one to capture a source account into a local SQLite archive and one to push that archive into a JMAP target.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# 1. Import an IMAP mailbox into a fresh archive.
|
||||||
|
export VANDELAY_PASSWORD='source-app-password'
|
||||||
|
vandelay import imap \
|
||||||
|
--url imaps://imap.example.com \
|
||||||
|
--auth-basic [email protected] \
|
||||||
|
alice.sqlite
|
||||||
|
|
||||||
|
# 2. Peek at what landed.
|
||||||
|
vandelay inspect alice.sqlite # per-type summary
|
||||||
|
vandelay inspect alice.sqlite mailbox # mailbox tree
|
||||||
|
vandelay inspect alice.sqlite email --limit 5
|
||||||
|
|
||||||
|
# 3. Push the archive into a target JMAP server.
|
||||||
|
export VANDELAY_PASSWORD='target-password'
|
||||||
|
vandelay export \
|
||||||
|
--url https://jmap.example.org \
|
||||||
|
--auth-basic [email protected] \
|
||||||
|
--account-name [email protected] \
|
||||||
|
alice.sqlite
|
||||||
|
```
|
||||||
|
|
||||||
|
Both commands are convergent: rerun either to resume an interrupted run, or rerun `import` later to pick up new mail since the last snapshot. Use `--dry-run` on either side to compute the full plan without writing.
|
||||||
|
|
||||||
## CLI quick reference
|
## CLI quick reference
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|||||||
+31
-1
@@ -136,13 +136,43 @@ fn try_treat_url_as_home_or_collection(
|
|||||||
if collections.is_empty() {
|
if collections.is_empty() {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
let principal_url = resolve_principal_url(client, kind, url)?;
|
||||||
Ok(Some(Discovery {
|
Ok(Some(Discovery {
|
||||||
principal_url: None,
|
principal_url,
|
||||||
home_set_url: url.to_owned(),
|
home_set_url: url.to_owned(),
|
||||||
collections,
|
collections,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn resolve_principal_url(
|
||||||
|
client: &DavClient,
|
||||||
|
kind: DavKind,
|
||||||
|
url: &str,
|
||||||
|
) -> Result<Option<String>, DiscoveryError> {
|
||||||
|
if matches!(kind, DavKind::Webdav) {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let body = xml::propfind_current_user_principal();
|
||||||
|
let ms = match client.propfind_responses(url, 0, &body, url) {
|
||||||
|
Ok(ms) => ms,
|
||||||
|
Err(JmapError::HttpStatus { status, .. }) if (400..600).contains(&status) => {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
Err(JmapError::RetriesExhausted(_)) => return Ok(None),
|
||||||
|
Err(e) => return Err(DiscoveryError::Transport(e)),
|
||||||
|
};
|
||||||
|
let final_url = ms.final_url.clone();
|
||||||
|
let Some(principal_href) = ms
|
||||||
|
.responses
|
||||||
|
.iter()
|
||||||
|
.find_map(|r| r.props.current_user_principal.as_deref())
|
||||||
|
else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let resolved = absolute_url(&final_url, &normalise(&final_url, principal_href)?)?;
|
||||||
|
Ok(Some(resolved))
|
||||||
|
}
|
||||||
|
|
||||||
fn try_via_principal(
|
fn try_via_principal(
|
||||||
client: &DavClient,
|
client: &DavClient,
|
||||||
kind: DavKind,
|
kind: DavKind,
|
||||||
|
|||||||
+2
-2
@@ -98,8 +98,8 @@ fn fail(err: &Error) -> i32 {
|
|||||||
fn report(summary: &Summary) {
|
fn report(summary: &Summary) {
|
||||||
for (type_name, counts) in &summary.per_type {
|
for (type_name, counts) in &summary.per_type {
|
||||||
println!(
|
println!(
|
||||||
"{type_name}: created={} fetched={} deleted={} skipped={} failed={}",
|
"{type_name}: created={} fetched={} updated={} deleted={} skipped={} failed={}",
|
||||||
counts.created, counts.fetched, counts.deleted, counts.skipped, counts.failed
|
counts.created, counts.fetched, counts.updated, counts.deleted, counts.skipped, counts.failed
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+13
-34
@@ -4,19 +4,16 @@
|
|||||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::{HashMap, HashSet};
|
||||||
|
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
use super::common::{create_batch, jid, target_get_all};
|
use super::common::{create_batch, jid, target_get_all};
|
||||||
use super::{Maps, Net, Plan, Uploader};
|
use super::{Maps, Net, Plan, Uploader};
|
||||||
use crate::db;
|
|
||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
use crate::jmap::blobxfer;
|
|
||||||
use crate::jmap::request::Request;
|
use crate::jmap::request::Request;
|
||||||
use crate::logging::Logger;
|
use crate::logging::Logger;
|
||||||
use crate::sync::import_jmap::mapping::{SIEVE_SELECT, row_to_sieve_script};
|
use crate::sync::import_jmap::mapping::{SIEVE_SELECT, row_to_sieve_script};
|
||||||
use crate::sync::keys::blake3_bytes;
|
|
||||||
use crate::sync::{Context, TypeCounts};
|
use crate::sync::{Context, TypeCounts};
|
||||||
use crate::types::ObjectType;
|
use crate::types::ObjectType;
|
||||||
|
|
||||||
@@ -30,21 +27,12 @@ pub fn reconcile(
|
|||||||
let ty = ObjectType::SieveScript;
|
let ty = ObjectType::SieveScript;
|
||||||
let targets = target_get_all(net, ty).map_err(Error::from)?;
|
let targets = target_get_all(net, ty).map_err(Error::from)?;
|
||||||
|
|
||||||
let mut target_by_key: HashMap<[u8; 32], String> = HashMap::new();
|
let mut target_by_name: HashMap<String, String> = HashMap::new();
|
||||||
for t in &targets {
|
for t in &targets {
|
||||||
let (Some(id), Some(blob)) = (jid(t), t.get("blobId").and_then(Value::as_str)) else {
|
let (Some(id), Some(name)) = (jid(t), t.get("name").and_then(Value::as_str)) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
let bytes = blobxfer::download_bytes(
|
target_by_name.insert(name.to_owned(), id);
|
||||||
&net.client,
|
|
||||||
&net.session,
|
|
||||||
&net.account,
|
|
||||||
blob,
|
|
||||||
"application/sieve",
|
|
||||||
"script",
|
|
||||||
)
|
|
||||||
.map_err(Error::from)?;
|
|
||||||
target_by_key.insert(blake3_bytes(&bytes), id);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let locals: Vec<(i64, Option<String>, bool, i64)> = {
|
let locals: Vec<(i64, Option<String>, bool, i64)> = {
|
||||||
@@ -71,13 +59,10 @@ pub fn reconcile(
|
|||||||
let mut uploader = Uploader::new(net, &ctx.conn);
|
let mut uploader = Uploader::new(net, &ctx.conn);
|
||||||
|
|
||||||
for (local, name, is_active, blob_local) in &locals {
|
for (local, name, is_active, blob_local) in &locals {
|
||||||
let bytes = db::blobs::blob_bytes(&ctx.conn, *blob_local)
|
let matched = name.as_ref().and_then(|n| target_by_name.get(n)).cloned();
|
||||||
.map_err(|e| Error::Partial(e.to_string()))?
|
let target_id = if let Some(id) = matched {
|
||||||
.ok_or_else(|| Error::Partial("sieve blob missing".to_owned()))?;
|
|
||||||
let key = blake3_bytes(&bytes);
|
|
||||||
let target_id = if let Some(id) = target_by_key.get(&key) {
|
|
||||||
counts.skipped += 1;
|
counts.skipped += 1;
|
||||||
id.clone()
|
id
|
||||||
} else {
|
} else {
|
||||||
let blob_id = uploader
|
let blob_id = uploader
|
||||||
.upload_with(*blob_local, "application/sieve")
|
.upload_with(*blob_local, "application/sieve")
|
||||||
@@ -92,7 +77,9 @@ pub fn reconcile(
|
|||||||
match outcome.created.first().and_then(|(_, v)| jid(v)) {
|
match outcome.created.first().and_then(|(_, v)| jid(v)) {
|
||||||
Some(id) => {
|
Some(id) => {
|
||||||
counts.created += 1;
|
counts.created += 1;
|
||||||
target_by_key.insert(key, id.clone());
|
if let Some(n) = name {
|
||||||
|
target_by_name.insert(n.clone(), id.clone());
|
||||||
|
}
|
||||||
id
|
id
|
||||||
}
|
}
|
||||||
None => {
|
None => {
|
||||||
@@ -128,18 +115,10 @@ pub fn reconcile(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let local_keys: std::collections::HashSet<[u8; 32]> = locals
|
let local_names: HashSet<String> = locals.iter().filter_map(|(_, n, _, _)| n.clone()).collect();
|
||||||
|
let mut prune_candidates: Vec<String> = target_by_name
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|(_, _, _, b)| {
|
.filter(|(name, _)| !local_names.contains(*name))
|
||||||
db::blobs::blob_bytes(&ctx.conn, *b)
|
|
||||||
.ok()
|
|
||||||
.flatten()
|
|
||||||
.map(|by| blake3_bytes(&by))
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
let mut prune_candidates: Vec<String> = target_by_key
|
|
||||||
.iter()
|
|
||||||
.filter(|(k, _)| !local_keys.contains(*k))
|
|
||||||
.map(|(_, id)| id.clone())
|
.map(|(_, id)| id.clone())
|
||||||
.collect();
|
.collect();
|
||||||
prune_candidates.sort();
|
prune_candidates.sort();
|
||||||
|
|||||||
@@ -826,6 +826,18 @@ fn reconcile_folder(
|
|||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if !diff.present.is_empty() {
|
||||||
|
counts.updated += refresh_present_flags(
|
||||||
|
conn,
|
||||||
|
client,
|
||||||
|
control_ctx,
|
||||||
|
&folder.name,
|
||||||
|
uidvalidity,
|
||||||
|
&diff.present,
|
||||||
|
opts,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
|
||||||
let now = OffsetDateTime::now_utc()
|
let now = OffsetDateTime::now_utc()
|
||||||
.format(&Rfc3339)
|
.format(&Rfc3339)
|
||||||
.unwrap_or_else(|_| String::from("1970-01-01T00:00:00Z"));
|
.unwrap_or_else(|_| String::from("1970-01-01T00:00:00Z"));
|
||||||
@@ -961,6 +973,93 @@ fn insert_single_message(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn refresh_present_flags(
|
||||||
|
conn: &mut Connection,
|
||||||
|
client: &mut ImapClient,
|
||||||
|
control_ctx: &ControlCtx,
|
||||||
|
folder: &str,
|
||||||
|
uidvalidity: u32,
|
||||||
|
present: &[u32],
|
||||||
|
opts: RunOpts,
|
||||||
|
) -> Result<u64, Error> {
|
||||||
|
let RunOpts {
|
||||||
|
source_id,
|
||||||
|
fetch_batch,
|
||||||
|
include_deleted,
|
||||||
|
logger: _,
|
||||||
|
} = opts;
|
||||||
|
let mut updated: u64 = 0;
|
||||||
|
let tx = conn.transaction()?;
|
||||||
|
for batch in chunks(present, fetch_batch.max(1)) {
|
||||||
|
let set = command::format_uid_set(batch, true);
|
||||||
|
let resp = control_run_collect(
|
||||||
|
client,
|
||||||
|
control_ctx,
|
||||||
|
&command::uid_fetch(&set, &["UID", "FLAGS"]),
|
||||||
|
)?;
|
||||||
|
for u in &resp.untagged {
|
||||||
|
let Some(attrs) = fetch::extract(u) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let Some(uid) = attrs.uid else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let translation = translate_flags(&attrs.flags, include_deleted);
|
||||||
|
if translation.has_deleted_flag && !include_deleted {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some((local_id, existing)) =
|
||||||
|
load_email_keywords(&tx, source_id, folder, uidvalidity, uid)?
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if keyword_set_differs(&existing, &translation.keywords) {
|
||||||
|
let json = Value::Array(
|
||||||
|
translation
|
||||||
|
.keywords
|
||||||
|
.iter()
|
||||||
|
.cloned()
|
||||||
|
.map(Value::String)
|
||||||
|
.collect(),
|
||||||
|
);
|
||||||
|
tx.execute(
|
||||||
|
"UPDATE emails SET keywords = ?1 WHERE id = ?2",
|
||||||
|
params![json.to_string(), local_id],
|
||||||
|
)?;
|
||||||
|
updated += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(updated)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn load_email_keywords(
|
||||||
|
tx: &rusqlite::Transaction<'_>,
|
||||||
|
source_id: i64,
|
||||||
|
folder: &str,
|
||||||
|
uidvalidity: u32,
|
||||||
|
uid: u32,
|
||||||
|
) -> Result<Option<(i64, Vec<String>)>, Error> {
|
||||||
|
let Some(local_id) = db::imap_ids::local_for_email(tx, source_id, folder, uidvalidity, uid)?
|
||||||
|
else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let kw_json: String = tx.query_row(
|
||||||
|
"SELECT keywords FROM emails WHERE id = ?1",
|
||||||
|
params![local_id],
|
||||||
|
|r| r.get(0),
|
||||||
|
)?;
|
||||||
|
let kws: Vec<String> = serde_json::from_str(&kw_json).unwrap_or_default();
|
||||||
|
Ok(Some((local_id, kws)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn keyword_set_differs(a: &[String], b: &[String]) -> bool {
|
||||||
|
let sa: HashSet<&str> = a.iter().map(String::as_str).collect();
|
||||||
|
let sb: HashSet<&str> = b.iter().map(String::as_str).collect();
|
||||||
|
sa != sb
|
||||||
|
}
|
||||||
|
|
||||||
fn dry_run_summary(
|
fn dry_run_summary(
|
||||||
conn: &Connection,
|
conn: &Connection,
|
||||||
source_id: Option<i64>,
|
source_id: Option<i64>,
|
||||||
|
|||||||
@@ -75,6 +75,7 @@ pub struct ExportConfig {
|
|||||||
pub struct TypeCounts {
|
pub struct TypeCounts {
|
||||||
pub created: u64,
|
pub created: u64,
|
||||||
pub fetched: u64,
|
pub fetched: u64,
|
||||||
|
pub updated: u64,
|
||||||
pub deleted: u64,
|
pub deleted: u64,
|
||||||
pub skipped: u64,
|
pub skipped: u64,
|
||||||
pub failed: u64,
|
pub failed: u64,
|
||||||
|
|||||||
@@ -66,6 +66,66 @@ fn discovery_uses_url_as_homeset_when_collection_present() {
|
|||||||
assert!(disc.collections[0].props.is_calendar);
|
assert!(disc.collections[0].props.is_calendar);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn discovery_resolves_per_user_principal_even_when_url_lists_collections() {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let url = server.url();
|
||||||
|
|
||||||
|
let collections_body = format!(
|
||||||
|
r#"<?xml version="1.0"?>
|
||||||
|
<d:multistatus xmlns:d="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav">
|
||||||
|
<d:response>
|
||||||
|
<d:href>{url}/dav/cal/[email protected]/default/</d:href>
|
||||||
|
<d:propstat>
|
||||||
|
<d:prop>
|
||||||
|
<d:resourcetype><d:collection/><c:calendar/></d:resourcetype>
|
||||||
|
<d:displayname>Default</d:displayname>
|
||||||
|
</d:prop>
|
||||||
|
<d:status>HTTP/1.1 200 OK</d:status>
|
||||||
|
</d:propstat>
|
||||||
|
</d:response>
|
||||||
|
</d:multistatus>"#
|
||||||
|
);
|
||||||
|
let _listing = server
|
||||||
|
.mock("PROPFIND", "/dav/cal/")
|
||||||
|
.match_header("depth", "1")
|
||||||
|
.with_status(207)
|
||||||
|
.with_header("content-type", "application/xml; charset=utf-8")
|
||||||
|
.with_body(&collections_body)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let principal_body = format!(
|
||||||
|
r#"<?xml version="1.0"?>
|
||||||
|
<d:multistatus xmlns:d="DAV:">
|
||||||
|
<d:response>
|
||||||
|
<d:href>{url}/dav/cal/</d:href>
|
||||||
|
<d:propstat>
|
||||||
|
<d:prop>
|
||||||
|
<d:current-user-principal><d:href>{url}/dav/principals/[email protected]/</d:href></d:current-user-principal>
|
||||||
|
</d:prop>
|
||||||
|
<d:status>HTTP/1.1 200 OK</d:status>
|
||||||
|
</d:propstat>
|
||||||
|
</d:response>
|
||||||
|
</d:multistatus>"#
|
||||||
|
);
|
||||||
|
let _principal = server
|
||||||
|
.mock("PROPFIND", "/dav/cal/")
|
||||||
|
.match_header("depth", "0")
|
||||||
|
.with_status(207)
|
||||||
|
.with_header("content-type", "application/xml; charset=utf-8")
|
||||||
|
.with_body(&principal_body)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let c = client(0);
|
||||||
|
let disc = discover(&c, DavKind::Caldav, &format!("{url}/dav/cal/")).expect("discover");
|
||||||
|
assert_eq!(disc.collections.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
disc.principal_url.as_deref(),
|
||||||
|
Some(format!("{url}/dav/principals/[email protected]/").as_str()),
|
||||||
|
"account identity must be the per-user principal, not the shared base DAV root"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn discovery_falls_through_principal_to_home_set() {
|
fn discovery_falls_through_principal_to_home_set() {
|
||||||
let mut server = mockito::Server::new();
|
let mut server = mockito::Server::new();
|
||||||
@@ -1129,6 +1189,133 @@ fn dry_run_writes_nothing_but_emits_per_collection_counts() {
|
|||||||
assert_eq!(event_counts.1.created, 2);
|
assert_eq!(event_counts.1.created, 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dav_source_change_protection_fires_across_users_on_same_root() {
|
||||||
|
use base64::Engine;
|
||||||
|
use base64::engine::general_purpose::STANDARD;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use vandelay::logging::Logger;
|
||||||
|
use vandelay::sync::CommonConfig;
|
||||||
|
use vandelay::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, run};
|
||||||
|
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let url = server.url();
|
||||||
|
|
||||||
|
let listing = format!(
|
||||||
|
r#"<?xml version="1.0"?>
|
||||||
|
<d:multistatus xmlns:d="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav">
|
||||||
|
<d:response>
|
||||||
|
<d:href>{url}/dav/cal/shared/default/</d:href>
|
||||||
|
<d:propstat>
|
||||||
|
<d:prop>
|
||||||
|
<d:resourcetype><d:collection/><c:calendar/></d:resourcetype>
|
||||||
|
<d:displayname>Default</d:displayname>
|
||||||
|
</d:prop>
|
||||||
|
<d:status>HTTP/1.1 200 OK</d:status>
|
||||||
|
</d:propstat>
|
||||||
|
</d:response>
|
||||||
|
</d:multistatus>"#
|
||||||
|
);
|
||||||
|
let _listing = server
|
||||||
|
.mock("PROPFIND", "/dav/cal/")
|
||||||
|
.match_header("depth", "1")
|
||||||
|
.with_status(207)
|
||||||
|
.with_header("content-type", "application/xml; charset=utf-8")
|
||||||
|
.with_body(&listing)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let empty_items = format!(
|
||||||
|
r#"<?xml version="1.0"?>
|
||||||
|
<d:multistatus xmlns:d="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav">
|
||||||
|
<d:response>
|
||||||
|
<d:href>{url}/dav/cal/shared/default/</d:href>
|
||||||
|
<d:propstat>
|
||||||
|
<d:prop><d:resourcetype><d:collection/><c:calendar/></d:resourcetype></d:prop>
|
||||||
|
<d:status>HTTP/1.1 200 OK</d:status>
|
||||||
|
</d:propstat>
|
||||||
|
</d:response>
|
||||||
|
</d:multistatus>"#
|
||||||
|
);
|
||||||
|
let _items = server
|
||||||
|
.mock("PROPFIND", "/dav/cal/shared/default/")
|
||||||
|
.with_status(207)
|
||||||
|
.with_header("content-type", "application/xml; charset=utf-8")
|
||||||
|
.with_body(&empty_items)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let auth_a = format!("Basic {}", STANDARD.encode("[email protected]:passA"));
|
||||||
|
let auth_b = format!("Basic {}", STANDARD.encode("[email protected]:passB"));
|
||||||
|
let principal = |who: &str| {
|
||||||
|
format!(
|
||||||
|
r#"<?xml version="1.0"?>
|
||||||
|
<d:multistatus xmlns:d="DAV:">
|
||||||
|
<d:response>
|
||||||
|
<d:href>{url}/dav/cal/</d:href>
|
||||||
|
<d:propstat>
|
||||||
|
<d:prop><d:current-user-principal><d:href>{url}/dav/principals/{who}/</d:href></d:current-user-principal></d:prop>
|
||||||
|
<d:status>HTTP/1.1 200 OK</d:status>
|
||||||
|
</d:propstat>
|
||||||
|
</d:response>
|
||||||
|
</d:multistatus>"#
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let _pa = server
|
||||||
|
.mock("PROPFIND", "/dav/cal/")
|
||||||
|
.match_header("depth", "0")
|
||||||
|
.match_header("authorization", auth_a.as_str())
|
||||||
|
.with_status(207)
|
||||||
|
.with_header("content-type", "application/xml; charset=utf-8")
|
||||||
|
.with_body(principal("[email protected]"))
|
||||||
|
.create();
|
||||||
|
let _pb = server
|
||||||
|
.mock("PROPFIND", "/dav/cal/")
|
||||||
|
.match_header("depth", "0")
|
||||||
|
.match_header("authorization", auth_b.as_str())
|
||||||
|
.with_status(207)
|
||||||
|
.with_header("content-type", "application/xml; charset=utf-8")
|
||||||
|
.with_body(principal("[email protected]"))
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let archive: PathBuf = std::env::temp_dir().join(format!(
|
||||||
|
"vandelay-dav-srcchange-{}-{}.sqlite",
|
||||||
|
std::process::id(),
|
||||||
|
std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.unwrap()
|
||||||
|
.as_nanos()
|
||||||
|
));
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
|
||||||
|
let common = |archive: &PathBuf| CommonConfig {
|
||||||
|
archive: archive.clone(),
|
||||||
|
threads: 1,
|
||||||
|
dry_run: false,
|
||||||
|
max_retries: 0,
|
||||||
|
allow_invalid_certs: true,
|
||||||
|
logger: Logger::from_flags(false, 0),
|
||||||
|
};
|
||||||
|
let config = |user: &str, pass: &str| DavImportConfig {
|
||||||
|
kind: DavKindArg::Caldav,
|
||||||
|
url: format!("{url}/dav/cal/"),
|
||||||
|
auth: DavAuth::Basic {
|
||||||
|
user: user.to_owned(),
|
||||||
|
password: pass.to_owned(),
|
||||||
|
},
|
||||||
|
allow_cleartext: true,
|
||||||
|
dav_connections: 1,
|
||||||
|
multiget_batch: 50,
|
||||||
|
allow_source_change: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
run(common(&archive), config("[email protected]", "passA")).expect("user A import ok");
|
||||||
|
let err = run(common(&archive), config("[email protected]", "passB")).unwrap_err();
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
assert!(
|
||||||
|
matches!(err, vandelay::error::Error::SourceChange(_)),
|
||||||
|
"importing a different user into the same archive must trigger source-change protection; got {err:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn source_change_protection_rejects_different_session_url_same_account() {
|
fn source_change_protection_rejects_different_session_url_same_account() {
|
||||||
use rusqlite::Connection;
|
use rusqlite::Connection;
|
||||||
|
|||||||
+110
-1
@@ -511,6 +511,43 @@ fn control_script_one_folder(uidvalidity: u32, uidnext: u32, uids: &'static [u32
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn control_script_present_flags(
|
||||||
|
uidvalidity: u32,
|
||||||
|
uidnext: u32,
|
||||||
|
uids: &'static [u32],
|
||||||
|
flags_reply: &'static [(u32, &'static str)],
|
||||||
|
) -> Script {
|
||||||
|
Box::new(move |conn: &mut MockConn| -> std::io::Result<()> {
|
||||||
|
auth_preamble(conn, "IMAP4rev2 LITERAL+ AUTH=PLAIN")?;
|
||||||
|
let (tag, cmd) = conn.read_command()?;
|
||||||
|
assert_eq!(cmd, "LIST \"\" \"*\"");
|
||||||
|
conn.write_line("* LIST () \"/\" \"INBOX\"")?;
|
||||||
|
conn.write_line(&format!("{tag} OK LIST done"))?;
|
||||||
|
let (tag, cmd) = conn.read_command()?;
|
||||||
|
assert_eq!(cmd, "LSUB \"\" \"*\"");
|
||||||
|
conn.write_line(&format!("{tag} OK LSUB done"))?;
|
||||||
|
let (tag, cmd) = conn.read_command()?;
|
||||||
|
assert_eq!(cmd, "SELECT \"INBOX\"");
|
||||||
|
write_select(conn, &tag, uidvalidity, uidnext, uids.len() as u32)?;
|
||||||
|
let (tag, cmd) = conn.read_command()?;
|
||||||
|
assert_eq!(cmd, "UID SEARCH ALL");
|
||||||
|
let uid_strs: Vec<String> = uids.iter().map(|u| u.to_string()).collect();
|
||||||
|
conn.write_line(&format!("* SEARCH {}", uid_strs.join(" ")))?;
|
||||||
|
conn.write_line(&format!("{tag} OK SEARCH done"))?;
|
||||||
|
let (tag, cmd) = conn.read_command()?;
|
||||||
|
assert!(
|
||||||
|
cmd.starts_with("UID FETCH") && cmd.contains("(UID FLAGS)") && !cmd.contains("BODY"),
|
||||||
|
"expected body-less flags fetch on the present set, got {cmd}"
|
||||||
|
);
|
||||||
|
for (uid, flags) in flags_reply {
|
||||||
|
conn.write_line(&format!("* {uid} FETCH (UID {uid} FLAGS ({flags}))"))?;
|
||||||
|
}
|
||||||
|
conn.write_line(&format!("{tag} OK FETCH done"))?;
|
||||||
|
drain_until_close(conn);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn coordinator_imports_one_folder_one_message() {
|
fn coordinator_imports_one_folder_one_message() {
|
||||||
let server = MockImap::start_scripts(vec![
|
let server = MockImap::start_scripts(vec![
|
||||||
@@ -755,7 +792,7 @@ fn coordinator_present_run_is_convergent() {
|
|||||||
let mut scripts: Vec<Script> = Vec::new();
|
let mut scripts: Vec<Script> = Vec::new();
|
||||||
scripts.extend(single_inbox_scripts(100, 2, MSG_BODY));
|
scripts.extend(single_inbox_scripts(100, 2, MSG_BODY));
|
||||||
|
|
||||||
scripts.push(control_script_one_folder(100, 2, &[1]));
|
scripts.push(control_script_present_flags(100, 2, &[1], &[(1, "\\Seen")]));
|
||||||
scripts.push(worker_idle_script("IMAP4rev2 LITERAL+ AUTH=PLAIN"));
|
scripts.push(worker_idle_script("IMAP4rev2 LITERAL+ AUTH=PLAIN"));
|
||||||
let server = MockImap::start_scripts(scripts);
|
let server = MockImap::start_scripts(scripts);
|
||||||
let archive = tempfile("converge");
|
let archive = tempfile("converge");
|
||||||
@@ -768,6 +805,78 @@ fn coordinator_present_run_is_convergent() {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(email.1.created, 0, "convergent run creates nothing");
|
assert_eq!(email.1.created, 0, "convergent run creates nothing");
|
||||||
assert_eq!(email.1.deleted, 0, "convergent run deletes nothing");
|
assert_eq!(email.1.deleted, 0, "convergent run deletes nothing");
|
||||||
|
assert_eq!(email.1.updated, 0, "unchanged flags update nothing");
|
||||||
|
let dbc = Connection::open(&archive).unwrap();
|
||||||
|
assert_eq!(count(&dbc, "blobs"), 1, "no body re-fetched on a present-only run");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn coordinator_present_flag_change_updates_keywords() {
|
||||||
|
let mut scripts: Vec<Script> = Vec::new();
|
||||||
|
scripts.extend(single_inbox_scripts(100, 2, MSG_BODY));
|
||||||
|
scripts.push(control_script_present_flags(
|
||||||
|
100,
|
||||||
|
2,
|
||||||
|
&[1],
|
||||||
|
&[(1, "\\Seen \\Flagged")],
|
||||||
|
));
|
||||||
|
scripts.push(worker_idle_script("IMAP4rev2 LITERAL+ AUTH=PLAIN"));
|
||||||
|
let server = MockImap::start_scripts(scripts);
|
||||||
|
let archive = tempfile("flagupdate");
|
||||||
|
run_import(&server, "alice", archive.clone(), |_| {}).expect("first import");
|
||||||
|
let summary = run_import(&server, "alice", archive.clone(), |_| {}).expect("second import");
|
||||||
|
let email = summary
|
||||||
|
.per_type
|
||||||
|
.iter()
|
||||||
|
.find(|(k, _)| *k == "email")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(email.1.updated, 1, "a changed flag set is counted as updated");
|
||||||
|
assert_eq!(email.1.created, 0, "present message is not re-created");
|
||||||
|
assert_eq!(email.1.fetched, 0, "no body fetched on a present-only run");
|
||||||
|
let dbc = Connection::open(&archive).unwrap();
|
||||||
|
assert_eq!(count(&dbc, "blobs"), 1, "body not re-fetched");
|
||||||
|
let kw: String = dbc
|
||||||
|
.query_row("SELECT keywords FROM emails LIMIT 1", [], |r| r.get(0))
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
kw.contains("$seen") && kw.contains("$flagged"),
|
||||||
|
"keywords should reflect the new \\Flagged: {kw}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn coordinator_present_newly_deleted_is_left_intact() {
|
||||||
|
let mut scripts: Vec<Script> = Vec::new();
|
||||||
|
scripts.extend(single_inbox_scripts(100, 2, MSG_BODY));
|
||||||
|
scripts.push(control_script_present_flags(
|
||||||
|
100,
|
||||||
|
2,
|
||||||
|
&[1],
|
||||||
|
&[(1, "\\Seen \\Deleted")],
|
||||||
|
));
|
||||||
|
scripts.push(worker_idle_script("IMAP4rev2 LITERAL+ AUTH=PLAIN"));
|
||||||
|
let server = MockImap::start_scripts(scripts);
|
||||||
|
let archive = tempfile("presentdeleted");
|
||||||
|
run_import(&server, "alice", archive.clone(), |_| {}).expect("first import");
|
||||||
|
let summary = run_import(&server, "alice", archive.clone(), |_| {}).expect("second import");
|
||||||
|
let email = summary
|
||||||
|
.per_type
|
||||||
|
.iter()
|
||||||
|
.find(|(k, _)| *k == "email")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
email.1.updated, 0,
|
||||||
|
"a present message that newly gained \\Deleted is skipped, not updated"
|
||||||
|
);
|
||||||
|
let dbc = Connection::open(&archive).unwrap();
|
||||||
|
assert_eq!(count(&dbc, "emails"), 1, "the archived message is preserved");
|
||||||
|
let kw: String = dbc
|
||||||
|
.query_row("SELECT keywords FROM emails LIMIT 1", [], |r| r.get(0))
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
kw.contains("$seen") && !kw.contains("$deleted"),
|
||||||
|
"keywords left intact: {kw}"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
+121
-11
@@ -1098,17 +1098,17 @@ fn export_calendar_creates_only_missing() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn export_sieve_script_skips_matching_blob_content() {
|
fn export_sieve_script_matches_by_name_not_content() {
|
||||||
let mut server = mockito::Server::new();
|
let mut server = mockito::Server::new();
|
||||||
let base = server.url();
|
let base = server.url();
|
||||||
let api = "/jmap/api";
|
let api = "/jmap/api";
|
||||||
let archive = tmp();
|
let archive = tmp();
|
||||||
let local_script = b"require [\"fileinto\"];\nkeep;\n";
|
let keepall_local = b"require [\"fileinto\"];\nkeep;\n";
|
||||||
let new_script = b"require [\"reject\"];\nreject \"go away\";\n";
|
let reject_local = b"require [\"reject\"];\nreject \"go away\";\n";
|
||||||
{
|
{
|
||||||
let conn = db::init::open(&archive).unwrap();
|
let conn = db::init::open(&archive).unwrap();
|
||||||
let blob1 = db::blobs::intern_blob(&conn, local_script).unwrap();
|
let blob1 = db::blobs::intern_blob(&conn, keepall_local).unwrap();
|
||||||
let blob2 = db::blobs::intern_blob(&conn, new_script).unwrap();
|
let blob2 = db::blobs::intern_blob(&conn, reject_local).unwrap();
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT INTO sieve_scripts (id,name,is_active,blob_id) VALUES (1,'keepall',1,?1)",
|
"INSERT INTO sieve_scripts (id,name,is_active,blob_id) VALUES (1,'keepall',1,?1)",
|
||||||
rusqlite::params![blob1],
|
rusqlite::params![blob1],
|
||||||
@@ -1132,16 +1132,16 @@ fn export_sieve_script_skips_matching_blob_content() {
|
|||||||
.match_body(Matcher::Regex("SieveScript/get".into()))
|
.match_body(Matcher::Regex("SieveScript/get".into()))
|
||||||
.with_body(
|
.with_body(
|
||||||
json!({"methodResponses":[["SieveScript/get",{"accountId":"w","list":[
|
json!({"methodResponses":[["SieveScript/get",{"accountId":"w","list":[
|
||||||
{"id":"S1","name":"already-there","isActive":false,"blobId":"BSRV"}
|
{"id":"S1","name":"keepall","isActive":false,"blobId":"BSRV"}
|
||||||
],"notFound":[]},"g"]]})
|
],"notFound":[]},"g"]]})
|
||||||
.to_string(),
|
.to_string(),
|
||||||
)
|
)
|
||||||
.expect(1)
|
.expect(1)
|
||||||
.create();
|
.create();
|
||||||
let _dl = server
|
let no_download = server
|
||||||
.mock("GET", Matcher::Regex("/jmap/dl/w/BSRV/.*".into()))
|
.mock("GET", Matcher::Regex("/jmap/dl/w/BSRV/.*".into()))
|
||||||
.with_body(local_script.as_slice())
|
.with_body(b"unused".as_slice())
|
||||||
.expect(1)
|
.expect(0)
|
||||||
.create();
|
.create();
|
||||||
let upload = server
|
let upload = server
|
||||||
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
|
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
|
||||||
@@ -1177,6 +1177,7 @@ fn export_sieve_script_skips_matching_blob_content() {
|
|||||||
.expect("export");
|
.expect("export");
|
||||||
upload.assert();
|
upload.assert();
|
||||||
create.assert();
|
create.assert();
|
||||||
|
no_download.assert();
|
||||||
let counts = summary
|
let counts = summary
|
||||||
.per_type
|
.per_type
|
||||||
.iter()
|
.iter()
|
||||||
@@ -1185,9 +1186,118 @@ fn export_sieve_script_skips_matching_blob_content() {
|
|||||||
.expect("sieve counts");
|
.expect("sieve counts");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
counts.skipped, 1,
|
counts.skipped, 1,
|
||||||
"matching-blob script is skipped regardless of name"
|
"name-matched script is skipped even though its content differs from the target"
|
||||||
);
|
);
|
||||||
assert_eq!(counts.created, 1, "differing-blob script is created");
|
assert_eq!(counts.created, 1, "the unmatched name is created");
|
||||||
|
assert_eq!(counts.failed, 0);
|
||||||
|
let _ = std::fs::remove_file(&archive);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn export_sieve_scripts_identical_content_different_names_both_created() {
|
||||||
|
let mut server = mockito::Server::new();
|
||||||
|
let base = server.url();
|
||||||
|
let api = "/jmap/api";
|
||||||
|
let archive = tmp();
|
||||||
|
let shared = b"require [\"fileinto\"];\nfileinto \"Archive\";\n";
|
||||||
|
{
|
||||||
|
let conn = db::init::open(&archive).unwrap();
|
||||||
|
let blob = db::blobs::intern_blob(&conn, shared).unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO sieve_scripts (id,name,is_active,blob_id) VALUES (1,'duplicate-A',0,?1)",
|
||||||
|
rusqlite::params![blob],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO sieve_scripts (id,name,is_active,blob_id) VALUES (2,'duplicate-B',0,?1)",
|
||||||
|
rusqlite::params![blob],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let dup_count: i64 = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT count(DISTINCT blob_id) FROM sieve_scripts",
|
||||||
|
[],
|
||||||
|
|r| r.get(0),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(dup_count, 1, "both scripts share one blob (byte-identical)");
|
||||||
|
}
|
||||||
|
|
||||||
|
let _root = server.mock("GET", "/").with_status(404).create();
|
||||||
|
let _wk = server
|
||||||
|
.mock("GET", "/.well-known/jmap")
|
||||||
|
.with_body(session_body_full(&base))
|
||||||
|
.expect_at_least(1)
|
||||||
|
.create();
|
||||||
|
let _g = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("SieveScript/get".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["SieveScript/get",
|
||||||
|
{"accountId":"w","list":[],"notFound":[]},"g"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let upload = server
|
||||||
|
.mock("POST", Matcher::Regex("/jmap/upload/".into()))
|
||||||
|
.with_body(json!({"blobId":"UPN"}).to_string())
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let create_a = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::AllOf(vec![
|
||||||
|
Matcher::Regex("SieveScript/set".into()),
|
||||||
|
Matcher::Regex("duplicate-A".into()),
|
||||||
|
]))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["SieveScript/set",{"accountId":"w",
|
||||||
|
"created":{"c1":{"id":"S1"}}},"s"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let create_b = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::AllOf(vec![
|
||||||
|
Matcher::Regex("SieveScript/set".into()),
|
||||||
|
Matcher::Regex("duplicate-B".into()),
|
||||||
|
]))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["SieveScript/set",{"accountId":"w",
|
||||||
|
"created":{"c2":{"id":"S2"}}},"s"]]})
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
let _deactivate = server
|
||||||
|
.mock("POST", api)
|
||||||
|
.match_body(Matcher::Regex("onSuccessDeactivateScript".into()))
|
||||||
|
.with_body(
|
||||||
|
json!({"methodResponses":[["SieveScript/set",{"accountId":"w"},"a"]]}).to_string(),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.create();
|
||||||
|
|
||||||
|
let summary = sync::export::run(
|
||||||
|
common(&archive),
|
||||||
|
export_cfg_objects(&base, vec![ObjectType::SieveScript]),
|
||||||
|
)
|
||||||
|
.expect("export");
|
||||||
|
upload.assert();
|
||||||
|
create_a.assert();
|
||||||
|
create_b.assert();
|
||||||
|
let counts = summary
|
||||||
|
.per_type
|
||||||
|
.iter()
|
||||||
|
.find(|(t, _)| *t == "SieveScript")
|
||||||
|
.map(|(_, c)| c.clone())
|
||||||
|
.expect("sieve counts");
|
||||||
|
assert_eq!(
|
||||||
|
counts.created, 2,
|
||||||
|
"two scripts with identical content but distinct names must both reach the target"
|
||||||
|
);
|
||||||
|
assert_eq!(counts.skipped, 0, "neither distinct name collapses onto the other");
|
||||||
assert_eq!(counts.failed, 0);
|
assert_eq!(counts.failed, 0);
|
||||||
let _ = std::fs::remove_file(&archive);
|
let _ = std::fs::remove_file(&archive);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user