Import upstream v1.0.13 #49
No files matched your search
@@ -81,7 +81,7 @@ jobs:
|
||||
"git fetch origin upstream:upstream\n"
|
||||
f"scripts/upstream-import.py {tag}\n"
|
||||
f"git switch -c import/upstream-{tag} origin/main\n"
|
||||
"git merge upstream\n"
|
||||
"git merge refs/heads/upstream\n"
|
||||
"```\n\n"
|
||||
"Before reading the changes, scan them for hidden characters and text aimed at "
|
||||
"automated tools. Resolve, run the checks, open a PR, and push `upstream` once it lands.")
|
||||
|
||||
@@ -7,6 +7,41 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
Upstream's 1.0.12 and 1.0.13 are merged in.
|
||||
|
||||
### Added
|
||||
- The out-of-office reply comes across: imported from a server's
|
||||
`VacationResponse`, and exported to the target's, in place of copying the
|
||||
script the server keeps for it.
|
||||
- WebDAV, CalDAV, CardDAV: `--replace-source` removes the other recorded
|
||||
sources of the same kind, and what was imported from them, before
|
||||
importing, so a moved share leaves one tree in the archive.
|
||||
- Files keep their `executable` flag where the server has one. An archive
|
||||
gains the column at its next import; until then `inspect` and the other
|
||||
read-only commands call it an older layout.
|
||||
|
||||
### Changed
|
||||
- FileNodes are read in either shape servers use: `nodeType` and `type`, as
|
||||
inbuxa and other 0.16-line servers spell them, or `@type` and `mediaType`
|
||||
(draft-ietf-jmap-filenode-18, as in Stalwart 1.0). An export asks the
|
||||
target which it takes and writes that one. Upstream dropped the older
|
||||
shape; this fork keeps both.
|
||||
- The Graph device-code sign-in asks for `Files.Read`, so the default run
|
||||
reaches OneDrive.
|
||||
- iCalendar and vCard parse failures say what was wrong and quote the line.
|
||||
|
||||
### Fixed
|
||||
- IMAP: children of a `\Noselect` parent folder kept their place instead of
|
||||
landing at the top level.
|
||||
- JMAP import no longer holds every blob of a page in memory at once, which
|
||||
large messages could exhaust.
|
||||
- CalDAV, CardDAV: items with bare CR line breaks, as Kerio Connect sends,
|
||||
are read.
|
||||
- EWS: a bare `--auth-bearer` takes the API key or token from the
|
||||
environment without asking for `--ews-client-id`.
|
||||
- `--account-name` as an administrator takes the data account from the
|
||||
principal's `accounts`, as RFC 9670 defines it.
|
||||
|
||||
## [27.1-rc.6]
|
||||
|
||||
The sixth release candidate of the 27.1 series: whole domains from Microsoft
|
||||
|
||||
Generated
+5
-4
@@ -1337,6 +1337,7 @@ dependencies = [
|
||||
"encodify",
|
||||
"flate2",
|
||||
"getrandom 0.3.4",
|
||||
"hashify",
|
||||
"indexmap 2.14.2",
|
||||
"mail-parser",
|
||||
"mockito",
|
||||
@@ -1544,9 +1545,9 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
||||
|
||||
[[package]]
|
||||
name = "libsqlite3-sys"
|
||||
version = "0.38.2"
|
||||
version = "0.39.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8"
|
||||
checksum = "14a5f01b87b92705d07b433c06f96cbc764637d81a0bffb5734ce17cf0f9a2b3"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"pkg-config",
|
||||
@@ -2130,9 +2131,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rusqlite"
|
||||
version = "0.40.2"
|
||||
version = "0.41.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3"
|
||||
checksum = "da9bba6f3506135e120a6586df05be18b48d18896600646a4bec8f993d91a594"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"fallible-iterator",
|
||||
|
||||
+2
-1
@@ -21,7 +21,7 @@ bundled-sqlite = ["rusqlite/bundled"]
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
indexmap = { version = "2.14", features = ["serde"] }
|
||||
rusqlite = { version = "0.40", features = ["blob"] }
|
||||
rusqlite = { version = "0.41", features = ["blob"] }
|
||||
blake3 = "1.8.5"
|
||||
time = { version = "0.3.47", features = ["serde-well-known", "macros"] }
|
||||
url= "2.5.8"
|
||||
@@ -40,6 +40,7 @@ flate2 = "1.1.9"
|
||||
quick-xml = "0.42"
|
||||
chrono = { version = "0.4", default-features = false, features = ["std"] }
|
||||
chrono-tz = { version = "0.10", default-features = false }
|
||||
hashify = "0.2"
|
||||
psl = "2.1.226"
|
||||
getrandom = "0.3.4"
|
||||
|
||||
|
||||
+14
-2
@@ -59,7 +59,10 @@ inbuxa-migrate import <source> [source-args...] <ARCHIVE>
|
||||
|
||||
Reads a source account into the SQLite `ARCHIVE`, creating it if it is absent.
|
||||
An archive remembers which account filled it; every importer takes
|
||||
`--allow-source-change` to fill it from a different one anyway.
|
||||
`--allow-source-change` to fill it from a different one anyway. The DAV
|
||||
importers (`caldav`, `carddav`, `webdav`) also take `--replace-source`, which
|
||||
first removes the other sources of the same kind, and what came from them, so
|
||||
a share that moved leaves one tree in the archive, not two.
|
||||
|
||||
### JMAP
|
||||
|
||||
@@ -74,6 +77,11 @@ inbuxa-migrate import jmap \
|
||||
|
||||
Imports one JMAP account, from inbuxa, Stalwart or any other JMAP server. `--objects` accepts a comma-separated list of object tokens (`mailbox,email,calendar,calendarevent,addressbook,contactcard,identity,sievescript,participantidentity,filenode`). The default is everything the server advertises.
|
||||
|
||||
The out-of-office reply, the server's `VacationResponse`, comes with
|
||||
`sievescript`. Files are read in either shape servers spell them: `nodeType`
|
||||
and `type`, as inbuxa and other servers on the 0.16 line do, or `@type` and
|
||||
`mediaType`, as draft-ietf-jmap-filenode-18 servers such as Stalwart 1.0 do.
|
||||
|
||||
### IMAP
|
||||
|
||||
```
|
||||
@@ -285,7 +293,11 @@ what changed at the source in between.
|
||||
`updated` time, the archive's is written only if it is newer; otherwise
|
||||
the properties that differ are written.
|
||||
- **Sieve scripts** are matched by name, and the target's is replaced when
|
||||
its content differs from the archive's.
|
||||
its content differs from the archive's. Where the target has a
|
||||
`VacationResponse`, the out-of-office reply is written there, and the
|
||||
script a server keeps for it is not copied as a script.
|
||||
- **Files** are written in the shape the target takes, which `export` asks
|
||||
it once per run: inbuxa takes `nodeType` and `type`.
|
||||
|
||||
Messages go in batches, as many to an `Email/import` as the target's
|
||||
`maxObjectsInSet` allows, up to 50, with their blobs uploaded several at a
|
||||
|
||||
@@ -22,12 +22,13 @@
|
||||
#
|
||||
# --first starts the branch. Give it the upstream commit the fork was made
|
||||
# from; the snapshot's parent is that commit. Merge it into main once with
|
||||
# `git merge -s ours upstream`, which records that main already has it, and
|
||||
# later imports merge cleanly against it.
|
||||
# `git merge -s ours refs/heads/upstream`, which records that main already
|
||||
# has it, and later imports merge cleanly against it. Name the branch in
|
||||
# full: plain `upstream` is also the remote.
|
||||
#
|
||||
# After an import:
|
||||
# git switch -c import/upstream-REF origin/main
|
||||
# git merge upstream
|
||||
# git merge refs/heads/upstream
|
||||
# (resolve; for Cargo.lock take main's and let `cargo build` add what's new)
|
||||
# then the usual checks and a PR. Push `upstream` to origin only once that PR
|
||||
# has landed. The upstream-watch workflow reads the imported release from the
|
||||
|
||||
+183
-28
@@ -27,7 +27,7 @@ use crate::secret;
|
||||
use crate::sync::export_dav::ExportDavConfig;
|
||||
use crate::sync::export_imap::ExportImapConfig;
|
||||
use crate::sync::export_managesieve::ExportManageSieveConfig;
|
||||
use crate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg};
|
||||
use crate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange};
|
||||
use crate::sync::import_exchange_ews::{EwsAuth, EwsImportConfig};
|
||||
use crate::sync::import_exchange_graph::{GraphAuth, GraphImportConfig};
|
||||
use crate::sync::import_imap::{ImapAuth, ImapImportConfig};
|
||||
@@ -920,6 +920,13 @@ pub struct DavImportArgs {
|
||||
)]
|
||||
allow_source_change: bool,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
conflicts_with = "allow_source_change",
|
||||
help = "Remove other recorded sources of this kind, and their items, before importing"
|
||||
)]
|
||||
replace_source: bool,
|
||||
|
||||
#[command(flatten)]
|
||||
global: GlobalArgs,
|
||||
|
||||
@@ -1159,7 +1166,11 @@ fn resolve_dav_import(args: DavImportArgs, kind: DavKindArg) -> Result<Action, E
|
||||
allow_cleartext: args.allow_cleartext,
|
||||
dav_connections,
|
||||
multiget_batch,
|
||||
allow_source_change: args.allow_source_change,
|
||||
source_change: match (args.replace_source, args.allow_source_change) {
|
||||
(true, _) => SourceChange::Replace,
|
||||
(false, true) => SourceChange::Allow,
|
||||
(false, false) => SourceChange::Reject,
|
||||
},
|
||||
},
|
||||
))
|
||||
}
|
||||
@@ -1648,38 +1659,43 @@ fn resolve_exchange_ews_auth(args: &ExchangeEwsImportArgs) -> Result<EwsAuth, Er
|
||||
if let Some(token) = bearer.clone() {
|
||||
return Ok(EwsAuth::Bearer { token });
|
||||
}
|
||||
let tenant = args
|
||||
.ews_tenant
|
||||
.clone()
|
||||
.unwrap_or_else(|| "common".to_owned());
|
||||
let client_id = args.ews_client_id.clone().ok_or_else(|| {
|
||||
Error::Usage(
|
||||
"--ews-client-id is required when --auth-bearer is used without an inline token"
|
||||
.to_owned(),
|
||||
)
|
||||
})?;
|
||||
let tenant = || {
|
||||
args.ews_tenant
|
||||
.clone()
|
||||
.unwrap_or_else(|| "common".to_owned())
|
||||
};
|
||||
let required_client_id = || {
|
||||
args.ews_client_id.clone().ok_or_else(|| {
|
||||
Error::Usage(
|
||||
"--ews-client-id is required with --ews-device-code or --ews-client-secret"
|
||||
.to_owned(),
|
||||
)
|
||||
})
|
||||
};
|
||||
if args.ews_device_code {
|
||||
return Ok(EwsAuth::OAuth(OAuthFlow::DeviceCode { tenant, client_id }));
|
||||
}
|
||||
if let Some(secret) = args.ews_client_secret.clone() {
|
||||
let resolved = secret::resolve(
|
||||
Some(secret.as_str()),
|
||||
"INBUXA_MIGRATE_EWS_CLIENT_SECRET",
|
||||
"client secret",
|
||||
)?;
|
||||
return Ok(EwsAuth::OAuth(OAuthFlow::ClientCredentials {
|
||||
tenant,
|
||||
client_id,
|
||||
client_secret: resolved,
|
||||
return Ok(EwsAuth::OAuth(OAuthFlow::DeviceCode {
|
||||
tenant: tenant(),
|
||||
client_id: required_client_id()?,
|
||||
}));
|
||||
}
|
||||
if let Ok(env_secret) = std::env::var("INBUXA_MIGRATE_EWS_CLIENT_SECRET")
|
||||
&& !env_secret.is_empty()
|
||||
if let Some(secret) = args.ews_client_secret.as_deref() {
|
||||
return Ok(EwsAuth::OAuth(OAuthFlow::ClientCredentials {
|
||||
tenant: tenant(),
|
||||
client_id: required_client_id()?,
|
||||
client_secret: secret.to_owned(),
|
||||
}));
|
||||
}
|
||||
if let Some(token) = secret::from_env(secret::API_KEY_VARS) {
|
||||
return Ok(EwsAuth::Bearer { token });
|
||||
}
|
||||
if let Some(client_id) = args.ews_client_id.clone()
|
||||
&& let Ok(client_secret) = std::env::var("INBUXA_MIGRATE_EWS_CLIENT_SECRET")
|
||||
&& !client_secret.is_empty()
|
||||
{
|
||||
return Ok(EwsAuth::OAuth(OAuthFlow::ClientCredentials {
|
||||
tenant,
|
||||
tenant: tenant(),
|
||||
client_id,
|
||||
client_secret: env_secret,
|
||||
client_secret,
|
||||
}));
|
||||
}
|
||||
let token = secret::resolve_any(None, secret::API_KEY_VARS, "API key or bearer token")?;
|
||||
@@ -2042,6 +2058,110 @@ mod tests {
|
||||
unsafe { env::remove_var("INBUXA_MIGRATE_TOKEN") };
|
||||
}
|
||||
|
||||
fn ews_auth(extra: &[&str]) -> Result<EwsAuth, Error> {
|
||||
let mut argv = vec!["inbuxa-migrate", "import", "exchange-ews", "archive.sqlite"];
|
||||
argv.extend_from_slice(extra);
|
||||
let cli = Cli::try_parse_from(argv).unwrap_or_else(|e| panic!("clap rejected args: {e}"));
|
||||
match cli.resolve()? {
|
||||
Action::ImportExchangeEws(_, config) => Ok(config.auth),
|
||||
_ => panic!("expected ImportExchangeEws"),
|
||||
}
|
||||
}
|
||||
|
||||
fn set_ews_env(token: Option<&str>, client_secret: Option<&str>) {
|
||||
unsafe {
|
||||
match token {
|
||||
Some(t) => env::set_var("INBUXA_MIGRATE_TOKEN", t),
|
||||
None => env::remove_var("INBUXA_MIGRATE_TOKEN"),
|
||||
}
|
||||
match client_secret {
|
||||
Some(s) => env::set_var("INBUXA_MIGRATE_EWS_CLIENT_SECRET", s),
|
||||
None => env::remove_var("INBUXA_MIGRATE_EWS_CLIENT_SECRET"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A bare --auth-bearer takes INBUXA_MIGRATE_TOKEN without needing --ews-client-id.
|
||||
#[test]
|
||||
fn ews_bare_bearer_uses_env_token_without_client_id() {
|
||||
let _g = lock();
|
||||
set_ews_env(Some("envtok"), None);
|
||||
let auth = ews_auth(&["--auth-bearer"]);
|
||||
set_ews_env(None, None);
|
||||
assert!(matches!(auth, Ok(EwsAuth::Bearer { token }) if token == "envtok"));
|
||||
}
|
||||
|
||||
// INBUXA_MIGRATE_TOKEN wins over client credentials taken from the environment.
|
||||
#[test]
|
||||
fn ews_env_token_beats_env_client_secret() {
|
||||
let _g = lock();
|
||||
set_ews_env(Some("envtok"), Some("sec"));
|
||||
let auth = ews_auth(&["--auth-bearer", "--ews-client-id", "cid"]);
|
||||
set_ews_env(None, None);
|
||||
assert!(matches!(auth, Ok(EwsAuth::Bearer { token }) if token == "envtok"));
|
||||
}
|
||||
|
||||
// The env client secret is used when no token is exported.
|
||||
#[test]
|
||||
fn ews_env_client_secret_without_token_uses_client_credentials() {
|
||||
let _g = lock();
|
||||
set_ews_env(None, Some("sec"));
|
||||
let auth = ews_auth(&["--auth-bearer", "--ews-client-id", "cid"]);
|
||||
set_ews_env(None, None);
|
||||
assert!(matches!(
|
||||
auth,
|
||||
Ok(EwsAuth::OAuth(OAuthFlow::ClientCredentials { client_id, client_secret, .. }))
|
||||
if client_id == "cid" && client_secret == "sec"
|
||||
));
|
||||
}
|
||||
|
||||
// An explicit --ews-device-code is not overridden by an exported token.
|
||||
#[test]
|
||||
fn ews_device_code_flag_beats_env_token() {
|
||||
let _g = lock();
|
||||
set_ews_env(Some("envtok"), None);
|
||||
let auth = ews_auth(&[
|
||||
"--auth-bearer",
|
||||
"--ews-client-id",
|
||||
"cid",
|
||||
"--ews-device-code",
|
||||
]);
|
||||
set_ews_env(None, None);
|
||||
assert!(matches!(
|
||||
auth,
|
||||
Ok(EwsAuth::OAuth(OAuthFlow::DeviceCode { tenant, client_id }))
|
||||
if tenant == "common" && client_id == "cid"
|
||||
));
|
||||
}
|
||||
|
||||
// --ews-device-code still needs a client id.
|
||||
#[test]
|
||||
fn ews_device_code_without_client_id_is_usage_error() {
|
||||
let _g = lock();
|
||||
set_ews_env(Some("envtok"), None);
|
||||
let auth = ews_auth(&["--auth-bearer", "--ews-device-code"]);
|
||||
set_ews_env(None, None);
|
||||
match auth {
|
||||
Err(Error::Usage(m)) => assert!(m.contains("--ews-client-id"), "msg was: {m}"),
|
||||
other => panic!("expected Usage error, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
// An inline token wins over everything else.
|
||||
#[test]
|
||||
fn ews_inline_token_beats_env_and_flows() {
|
||||
let _g = lock();
|
||||
set_ews_env(Some("envtok"), None);
|
||||
let auth = ews_auth(&[
|
||||
"--auth-bearer=inline",
|
||||
"--ews-client-id",
|
||||
"cid",
|
||||
"--ews-device-code",
|
||||
]);
|
||||
set_ews_env(None, None);
|
||||
assert!(matches!(auth, Ok(EwsAuth::Bearer { token }) if token == "inline"));
|
||||
}
|
||||
|
||||
fn graph_config(objects: &[&str]) -> Result<GraphImportConfig, Error> {
|
||||
let mut argv = vec![
|
||||
"inbuxa-migrate",
|
||||
@@ -2162,4 +2282,39 @@ mod tests {
|
||||
_ => panic!("a missing key file is a usage error"),
|
||||
}
|
||||
}
|
||||
|
||||
fn webdav_source_change(extra: &[&str]) -> Result<SourceChange, clap::Error> {
|
||||
let mut argv = vec![
|
||||
"inbuxa-migrate",
|
||||
"import",
|
||||
"webdav",
|
||||
"--url",
|
||||
"https://dav.example/files/u/",
|
||||
"--auth-bearer",
|
||||
"tok",
|
||||
];
|
||||
argv.extend_from_slice(extra);
|
||||
argv.push("archive.sqlite");
|
||||
let cli = Cli::try_parse_from(argv)?;
|
||||
match cli.resolve() {
|
||||
Ok(Action::ImportDav(_, config)) => Ok(config.source_change),
|
||||
_ => panic!("expected ImportDav"),
|
||||
}
|
||||
}
|
||||
|
||||
// --replace-source maps to Replace and cannot be combined with --allow-source-change.
|
||||
#[test]
|
||||
fn dav_source_change_flags_resolve() {
|
||||
let _g = lock();
|
||||
assert_eq!(webdav_source_change(&[]).unwrap(), SourceChange::Reject);
|
||||
assert_eq!(
|
||||
webdav_source_change(&["--allow-source-change"]).unwrap(),
|
||||
SourceChange::Allow
|
||||
);
|
||||
assert_eq!(
|
||||
webdav_source_change(&["--replace-source"]).unwrap(),
|
||||
SourceChange::Replace
|
||||
);
|
||||
assert!(webdav_source_change(&["--replace-source", "--allow-source-change"]).is_err());
|
||||
}
|
||||
}
|
||||
+38
-10
@@ -70,10 +70,17 @@ pub fn open_read_only(path: &std::path::Path) -> Result<Connection, OpenError> {
|
||||
let missing: Vec<&str> = schema_tables()
|
||||
.filter(|t| !present.iter().any(|p| p == t))
|
||||
.collect();
|
||||
let has_data_type = conn
|
||||
.prepare("SELECT 1 FROM pragma_table_info('calendar_events') WHERE name = 'data_type'")?
|
||||
.exists([])?;
|
||||
if !missing.is_empty() || !has_data_type {
|
||||
// Columns added to tables older archives already have.
|
||||
let mut has_columns = true;
|
||||
for (table, column) in [
|
||||
("calendar_events", "data_type"),
|
||||
("file_nodes", "executable"),
|
||||
] {
|
||||
has_columns &= conn
|
||||
.prepare("SELECT 1 FROM pragma_table_info(?1) WHERE name = ?2")?
|
||||
.exists([table, column])?;
|
||||
}
|
||||
if !missing.is_empty() || !has_columns {
|
||||
return Err(OpenError::NotAnArchive(format!(
|
||||
"{} was written by an older inbuxa-migrate and lacks part of today's layout; \
|
||||
the next import into it brings it up to date",
|
||||
@@ -122,6 +129,7 @@ pub fn apply_schema(conn: &Connection) -> Result<(), OpenError> {
|
||||
let tx = conn.unchecked_transaction()?;
|
||||
tx.execute_batch(SCHEMA_SQL)?;
|
||||
ensure_calendar_events_data_type(&tx)?;
|
||||
ensure_file_nodes_executable(&tx)?;
|
||||
ensure_graph_ids_accept_file_nodes(&tx)?;
|
||||
ensure_dav_ids_share_local_rows(&tx)?;
|
||||
tx.commit()?;
|
||||
@@ -129,14 +137,34 @@ pub fn apply_schema(conn: &Connection) -> Result<(), OpenError> {
|
||||
}
|
||||
|
||||
fn ensure_calendar_events_data_type(conn: &Connection) -> Result<(), OpenError> {
|
||||
let mut stmt = conn.prepare("PRAGMA table_info(calendar_events)")?;
|
||||
ensure_column(
|
||||
conn,
|
||||
"calendar_events",
|
||||
"data_type",
|
||||
"ALTER TABLE calendar_events ADD COLUMN data_type TEXT NOT NULL DEFAULT 'Event'",
|
||||
)
|
||||
}
|
||||
|
||||
fn ensure_file_nodes_executable(conn: &Connection) -> Result<(), OpenError> {
|
||||
ensure_column(
|
||||
conn,
|
||||
"file_nodes",
|
||||
"executable",
|
||||
"ALTER TABLE file_nodes ADD COLUMN executable INTEGER NOT NULL DEFAULT 0",
|
||||
)
|
||||
}
|
||||
|
||||
fn ensure_column(
|
||||
conn: &Connection,
|
||||
table: &str,
|
||||
column: &str,
|
||||
alter: &str,
|
||||
) -> Result<(), OpenError> {
|
||||
let mut stmt = conn.prepare(&format!("PRAGMA table_info({table})"))?;
|
||||
let rows = stmt.query_map([], |row| row.get::<_, String>(1))?;
|
||||
let has_column = rows.filter_map(|r| r.ok()).any(|name| name == "data_type");
|
||||
let has_column = rows.filter_map(|r| r.ok()).any(|name| name == column);
|
||||
if !has_column {
|
||||
conn.execute(
|
||||
"ALTER TABLE calendar_events ADD COLUMN data_type TEXT NOT NULL DEFAULT 'Event'",
|
||||
[],
|
||||
)?;
|
||||
conn.execute(alter, [])?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -20,3 +20,4 @@ pub mod roles;
|
||||
pub mod sources;
|
||||
pub mod sync_state_jmap;
|
||||
pub mod takeout_ids;
|
||||
pub mod vacation_response;
|
||||
@@ -172,6 +172,17 @@ CREATE TABLE IF NOT EXISTS sieve_scripts (
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS sieve_scripts_one_active_idx
|
||||
ON sieve_scripts (is_active) WHERE is_active = 1;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS vacation_response (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
||||
is_enabled INTEGER NOT NULL DEFAULT 0,
|
||||
from_date TEXT,
|
||||
to_date TEXT,
|
||||
subject TEXT,
|
||||
text_body TEXT,
|
||||
html_body TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS address_books (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
@@ -236,6 +247,7 @@ CREATE TABLE IF NOT EXISTS file_nodes (
|
||||
modified TEXT,
|
||||
is_subscribed INTEGER NOT NULL DEFAULT 1,
|
||||
role TEXT,
|
||||
executable INTEGER NOT NULL DEFAULT 0,
|
||||
UNIQUE (parent_id, name)
|
||||
);
|
||||
|
||||
|
||||
@@ -37,6 +37,27 @@ pub fn conflicting_source(
|
||||
.optional()
|
||||
}
|
||||
|
||||
pub fn other_sources(
|
||||
conn: &Connection,
|
||||
kind: &str,
|
||||
session_url: &str,
|
||||
account_id: &str,
|
||||
) -> Result<Vec<(i64, String, String)>, rusqlite::Error> {
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT id, session_url, account_id FROM sources
|
||||
WHERE kind = ?1 AND NOT (session_url = ?2 AND account_id = ?3) ORDER BY id",
|
||||
)?;
|
||||
stmt.query_map(params![kind, session_url, account_id], |row| {
|
||||
Ok((row.get(0)?, row.get(1)?, row.get(2)?))
|
||||
})?
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn delete_source(conn: &Connection, id: i64) -> Result<(), rusqlite::Error> {
|
||||
conn.execute("DELETE FROM sources WHERE id = ?1", params![id])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn upsert_source(
|
||||
conn: &Connection,
|
||||
key: &SourceKey,
|
||||
@@ -102,4 +123,20 @@ mod tests {
|
||||
.unwrap();
|
||||
assert_eq!(conflict, ("https://a/jmap".to_owned(), "w".to_owned()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_sources_lists_every_other_source_of_the_kind() {
|
||||
let c = mem();
|
||||
let a = upsert_source(&c, &key("https://a/jmap", "w"), None, "alice").unwrap();
|
||||
let b = upsert_source(&c, &key("https://b/jmap", "w"), None, "alice").unwrap();
|
||||
upsert_source(&c, &key("https://c/jmap", "w"), None, "alice").unwrap();
|
||||
let others = other_sources(&c, "jmap", "https://c/jmap", "w").unwrap();
|
||||
let ids: Vec<i64> = others.iter().map(|(id, _, _)| *id).collect();
|
||||
assert_eq!(ids, [a, b]);
|
||||
assert!(
|
||||
other_sources(&c, "imap", "https://c/jmap", "w")
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
|
||||
use crate::jmap::wire::vacation_response::VacationResponse;
|
||||
|
||||
pub fn store(
|
||||
conn: &Connection,
|
||||
source_id: i64,
|
||||
vacation: &VacationResponse,
|
||||
) -> Result<(), rusqlite::Error> {
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO vacation_response
|
||||
(id, source_id, is_enabled, from_date, to_date, subject, text_body, html_body)
|
||||
VALUES (1, ?1, ?2, ?3, ?4, ?5, ?6, ?7)",
|
||||
params![
|
||||
source_id,
|
||||
vacation.is_enabled,
|
||||
vacation.from_date,
|
||||
vacation.to_date,
|
||||
vacation.subject,
|
||||
vacation.text_body,
|
||||
vacation.html_body,
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn clear(conn: &Connection, source_id: i64) -> Result<(), rusqlite::Error> {
|
||||
conn.execute(
|
||||
"DELETE FROM vacation_response WHERE source_id = ?1",
|
||||
params![source_id],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn exists(conn: &Connection) -> Result<bool, rusqlite::Error> {
|
||||
conn.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM vacation_response)",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn load(conn: &Connection) -> Result<Option<VacationResponse>, rusqlite::Error> {
|
||||
conn.query_row(
|
||||
"SELECT is_enabled, from_date, to_date, subject, text_body, html_body
|
||||
FROM vacation_response WHERE id = 1",
|
||||
[],
|
||||
|row| {
|
||||
Ok(VacationResponse {
|
||||
is_enabled: row.get(0)?,
|
||||
from_date: row.get(1)?,
|
||||
to_date: row.get(2)?,
|
||||
subject: row.get(3)?,
|
||||
text_body: row.get(4)?,
|
||||
html_body: row.get(5)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
.optional()
|
||||
}
|
||||
@@ -15,8 +15,7 @@ use ureq::config::Config;
|
||||
use crate::exchange_graph::error::GraphError;
|
||||
use crate::net::{tls, with_timeouts};
|
||||
|
||||
pub const SCOPES: &str =
|
||||
"offline_access User.Read Mail.Read MailboxSettings.Read Calendars.Read Contacts.Read";
|
||||
pub const SCOPES: &str = "offline_access User.Read Mail.Read MailboxSettings.Read Calendars.Read Contacts.Read Files.Read";
|
||||
|
||||
/// App-only: the application permissions an administrator granted the app.
|
||||
pub const APP_ONLY_SCOPE: &str = "https://graph.microsoft.com/.default";
|
||||
@@ -462,6 +461,24 @@ mod tests {
|
||||
format!("{header}.{payload}.")
|
||||
}
|
||||
|
||||
// Device code consent covers every --objects surface, including the drive.
|
||||
#[test]
|
||||
fn scopes_cover_every_surface() {
|
||||
for required in [
|
||||
"offline_access",
|
||||
"Mail.Read",
|
||||
"MailboxSettings.Read",
|
||||
"Calendars.Read",
|
||||
"Contacts.Read",
|
||||
"Files.Read",
|
||||
] {
|
||||
assert!(
|
||||
SCOPES.split_whitespace().any(|s| s == required),
|
||||
"missing {required}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jwt_claim_extraction() {
|
||||
let token = make_jwt("t-1", "[email protected]", 9999999999);
|
||||
|
||||
+31
-17
@@ -19,7 +19,6 @@ pub enum AccountSelector {
|
||||
Name(String),
|
||||
}
|
||||
|
||||
const OWNER_URN: &str = "urn:ietf:params:jmap:principals:owner";
|
||||
const PRINCIPALS_URN: &str = "urn:ietf:params:jmap:principals";
|
||||
|
||||
pub fn resolve(
|
||||
@@ -140,21 +139,15 @@ fn unsupported_principals(name: &str, session: &Session) -> Error {
|
||||
|
||||
fn extract_account_id(principal: &Value, name: &str) -> Result<String, Error> {
|
||||
if let Some(accounts) = principal.get("accounts").and_then(Value::as_object) {
|
||||
for data in accounts.values() {
|
||||
if let Some(owner) = data.get(OWNER_URN)
|
||||
&& let Some(id) = owner.get("accountIdForPrincipal").and_then(Value::as_str)
|
||||
{
|
||||
return Ok(id.to_owned());
|
||||
}
|
||||
let mut keys = accounts.keys();
|
||||
if let (Some(only), None) = (keys.next(), keys.next()) {
|
||||
return Ok(only.clone());
|
||||
}
|
||||
for data in accounts.values() {
|
||||
if let Some(cap) = data.as_object() {
|
||||
for v in cap.values() {
|
||||
if let Some(id) = v.get("accountId").and_then(Value::as_str) {
|
||||
return Ok(id.to_owned());
|
||||
}
|
||||
}
|
||||
}
|
||||
let named = accounts
|
||||
.iter()
|
||||
.find(|(_, account)| account.get("name").and_then(Value::as_str) == Some(name));
|
||||
if let Some((id, _)) = named {
|
||||
return Ok(id.clone());
|
||||
}
|
||||
}
|
||||
principal
|
||||
@@ -256,12 +249,33 @@ mod tests {
|
||||
assert_eq!(extract_account_id(&p, "bob").unwrap(), "acc-7");
|
||||
}
|
||||
|
||||
// RFC 9670: the data account is the key of "accounts", not accountIdForPrincipal.
|
||||
#[test]
|
||||
fn extract_uses_per_capability_account_id() {
|
||||
fn extract_uses_accounts_key_not_account_id_for_principal() {
|
||||
let p = json!({
|
||||
"id": "px",
|
||||
"name": "[email protected]",
|
||||
"accounts": { "data-9": {
|
||||
"name": "[email protected]",
|
||||
"isPersonal": false,
|
||||
"accountCapabilities": {
|
||||
"urn:ietf:params:jmap:principals:owner":
|
||||
{ "accountIdForPrincipal": "admin-1", "principalId": "px" }
|
||||
}
|
||||
} }
|
||||
});
|
||||
assert_eq!(extract_account_id(&p, "[email protected]").unwrap(), "data-9");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_picks_the_named_account_among_several() {
|
||||
let p = json!({
|
||||
"id": "px",
|
||||
"name": "bob",
|
||||
"accounts": { "z": { "urn:ietf:params:jmap:mail": { "accountId": "data-9" } } }
|
||||
"accounts": {
|
||||
"shared-1": { "name": "team" },
|
||||
"data-9": { "name": "bob" }
|
||||
}
|
||||
});
|
||||
assert_eq!(extract_account_id(&p, "bob").unwrap(), "data-9");
|
||||
}
|
||||
|
||||
@@ -18,6 +18,8 @@ use crate::logging::Logger;
|
||||
|
||||
pub const URN_CORE: &str = "urn:ietf:params:jmap:core";
|
||||
|
||||
pub const URN_VACATION_RESPONSE: &str = "urn:ietf:params:jmap:vacationresponse";
|
||||
|
||||
/// The session capability every inbuxa server has.
|
||||
pub const URN_INBUXA: &str = "urn:inbuxa:jmap";
|
||||
|
||||
@@ -50,6 +52,7 @@ pub fn using_urn(method: &str, registry: Option<&'static str>) -> Result<&'stati
|
||||
"Calendar" | "CalendarEvent" | "ParticipantIdentity" => "urn:ietf:params:jmap:calendars",
|
||||
"FileNode" => "urn:ietf:params:jmap:filenode",
|
||||
"Principal" => "urn:ietf:params:jmap:principals",
|
||||
"VacationResponse" => URN_VACATION_RESPONSE,
|
||||
_ => URN_CORE,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -278,6 +278,13 @@ impl Session {
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
pub fn may_support(&self, account_id: &str, capability_urn: &str) -> bool {
|
||||
match self.account_capabilities(account_id) {
|
||||
Some(caps) => caps.contains_key(capability_urn),
|
||||
None => self.capabilities.contains_key(capability_urn),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn upload_url_for(&self, account_id: &str) -> String {
|
||||
self.upload_url
|
||||
.replace("{accountId}", &encode_segment(account_id))
|
||||
@@ -414,6 +421,16 @@ mod tests {
|
||||
assert!(!session.supports("missing", "urn:ietf:params:jmap:mail"));
|
||||
}
|
||||
|
||||
// Listed accounts use their own capabilities; unlisted ones fall back to the session's.
|
||||
#[test]
|
||||
fn may_support_falls_back_to_session_capabilities_for_unlisted_account() {
|
||||
let session = parse_session(raw_session()).unwrap();
|
||||
assert!(session.may_support("w", "urn:ietf:params:jmap:mail"));
|
||||
assert!(!session.may_support("w", "urn:ietf:params:jmap:core"));
|
||||
assert!(session.may_support("missing", "urn:ietf:params:jmap:core"));
|
||||
assert!(!session.may_support("missing", "urn:ietf:params:jmap:mail"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn templates_upload_and_download_urls_with_encoding() {
|
||||
let session = parse_session(raw_session()).unwrap();
|
||||
|
||||
@@ -49,6 +49,183 @@ pub struct Calendar {
|
||||
pub time_zone: Option<String>,
|
||||
}
|
||||
|
||||
impl Calendar {
|
||||
pub fn normalize_color(&mut self) -> Option<String> {
|
||||
let color = self.color.as_mut()?;
|
||||
let trimmed = color.trim();
|
||||
let kept = match trimmed.strip_prefix('#') {
|
||||
Some(hex) if hex.bytes().all(|byte| byte.is_ascii_hexdigit()) => match hex.len() {
|
||||
3 | 4 => trimmed.get(..=3),
|
||||
6 | 8 => trimmed.get(..=6),
|
||||
_ => None,
|
||||
},
|
||||
Some(_) => None,
|
||||
None => is_css3_color_name(trimmed).then_some(trimmed),
|
||||
};
|
||||
match kept {
|
||||
Some(kept) if kept.len() == color.len() => None,
|
||||
Some(kept) => {
|
||||
*color = kept.to_owned();
|
||||
None
|
||||
}
|
||||
None => self.color.take(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_css3_color_name(name: &str) -> bool {
|
||||
hashify::set_ignore_case!(
|
||||
name.as_bytes(),
|
||||
"aliceblue",
|
||||
"antiquewhite",
|
||||
"aqua",
|
||||
"aquamarine",
|
||||
"azure",
|
||||
"beige",
|
||||
"bisque",
|
||||
"black",
|
||||
"blanchedalmond",
|
||||
"blue",
|
||||
"blueviolet",
|
||||
"brown",
|
||||
"burlywood",
|
||||
"cadetblue",
|
||||
"chartreuse",
|
||||
"chocolate",
|
||||
"coral",
|
||||
"cornflowerblue",
|
||||
"cornsilk",
|
||||
"crimson",
|
||||
"cyan",
|
||||
"darkblue",
|
||||
"darkcyan",
|
||||
"darkgoldenrod",
|
||||
"darkgray",
|
||||
"darkgreen",
|
||||
"darkgrey",
|
||||
"darkkhaki",
|
||||
"darkmagenta",
|
||||
"darkolivegreen",
|
||||
"darkorange",
|
||||
"darkorchid",
|
||||
"darkred",
|
||||
"darksalmon",
|
||||
"darkseagreen",
|
||||
"darkslateblue",
|
||||
"darkslategray",
|
||||
"darkslategrey",
|
||||
"darkturquoise",
|
||||
"darkviolet",
|
||||
"deeppink",
|
||||
"deepskyblue",
|
||||
"dimgray",
|
||||
"dimgrey",
|
||||
"dodgerblue",
|
||||
"firebrick",
|
||||
"floralwhite",
|
||||
"forestgreen",
|
||||
"fuchsia",
|
||||
"gainsboro",
|
||||
"ghostwhite",
|
||||
"gold",
|
||||
"goldenrod",
|
||||
"gray",
|
||||
"green",
|
||||
"greenyellow",
|
||||
"grey",
|
||||
"honeydew",
|
||||
"hotpink",
|
||||
"indianred",
|
||||
"indigo",
|
||||
"ivory",
|
||||
"khaki",
|
||||
"lavender",
|
||||
"lavenderblush",
|
||||
"lawngreen",
|
||||
"lemonchiffon",
|
||||
"lightblue",
|
||||
"lightcoral",
|
||||
"lightcyan",
|
||||
"lightgoldenrodyellow",
|
||||
"lightgray",
|
||||
"lightgreen",
|
||||
"lightgrey",
|
||||
"lightpink",
|
||||
"lightsalmon",
|
||||
"lightseagreen",
|
||||
"lightskyblue",
|
||||
"lightslategray",
|
||||
"lightslategrey",
|
||||
"lightsteelblue",
|
||||
"lightyellow",
|
||||
"lime",
|
||||
"limegreen",
|
||||
"linen",
|
||||
"magenta",
|
||||
"maroon",
|
||||
"mediumaquamarine",
|
||||
"mediumblue",
|
||||
"mediumorchid",
|
||||
"mediumpurple",
|
||||
"mediumseagreen",
|
||||
"mediumslateblue",
|
||||
"mediumspringgreen",
|
||||
"mediumturquoise",
|
||||
"mediumvioletred",
|
||||
"midnightblue",
|
||||
"mintcream",
|
||||
"mistyrose",
|
||||
"moccasin",
|
||||
"navajowhite",
|
||||
"navy",
|
||||
"oldlace",
|
||||
"olive",
|
||||
"olivedrab",
|
||||
"orange",
|
||||
"orangered",
|
||||
"orchid",
|
||||
"palegoldenrod",
|
||||
"palegreen",
|
||||
"paleturquoise",
|
||||
"palevioletred",
|
||||
"papayawhip",
|
||||
"peachpuff",
|
||||
"peru",
|
||||
"pink",
|
||||
"plum",
|
||||
"powderblue",
|
||||
"purple",
|
||||
"red",
|
||||
"rosybrown",
|
||||
"royalblue",
|
||||
"saddlebrown",
|
||||
"salmon",
|
||||
"sandybrown",
|
||||
"seagreen",
|
||||
"seashell",
|
||||
"sienna",
|
||||
"silver",
|
||||
"skyblue",
|
||||
"slateblue",
|
||||
"slategray",
|
||||
"slategrey",
|
||||
"snow",
|
||||
"springgreen",
|
||||
"steelblue",
|
||||
"tan",
|
||||
"teal",
|
||||
"thistle",
|
||||
"tomato",
|
||||
"turquoise",
|
||||
"violet",
|
||||
"wheat",
|
||||
"white",
|
||||
"whitesmoke",
|
||||
"yellow",
|
||||
"yellowgreen",
|
||||
)
|
||||
}
|
||||
|
||||
fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
@@ -56,3 +233,56 @@ fn default_true() -> bool {
|
||||
fn default_include_in_availability() -> String {
|
||||
"all".to_owned()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn normalized(color: &str) -> (Option<String>, Option<String>) {
|
||||
let mut calendar: Calendar =
|
||||
serde_json::from_value(serde_json::json!({ "name": "C", "color": color }))
|
||||
.expect("calendar");
|
||||
let dropped = calendar.normalize_color();
|
||||
(calendar.color, dropped)
|
||||
}
|
||||
|
||||
// Alpha is stripped, valid colors pass through, anything else is dropped.
|
||||
#[test]
|
||||
fn calendar_colors_are_normalized_for_jmap() {
|
||||
for (input, expected) in [
|
||||
("#AAAAAAFF", "#AAAAAA"),
|
||||
("#3366ccfe", "#3366cc"),
|
||||
("#ABCD", "#ABC"),
|
||||
("#abc", "#abc"),
|
||||
("#3366CC", "#3366CC"),
|
||||
(" #3366cc\n", "#3366cc"),
|
||||
("CornflowerBlue", "CornflowerBlue"),
|
||||
("grey", "grey"),
|
||||
] {
|
||||
assert_eq!(
|
||||
normalized(input),
|
||||
(Some(expected.to_owned()), None),
|
||||
"{input:?}"
|
||||
);
|
||||
}
|
||||
for input in [
|
||||
"",
|
||||
"#",
|
||||
"#12345",
|
||||
"#GGGGGG",
|
||||
"#1234567",
|
||||
"rgb(1,2,3)",
|
||||
"notacolor",
|
||||
] {
|
||||
assert_eq!(
|
||||
normalized(input),
|
||||
(None, Some(input.to_owned())),
|
||||
"{input:?}"
|
||||
);
|
||||
}
|
||||
let mut calendar: Calendar =
|
||||
serde_json::from_value(serde_json::json!({ "name": "C" })).expect("calendar");
|
||||
assert_eq!(calendar.normalize_color(), None);
|
||||
assert_eq!(calendar.color, None);
|
||||
}
|
||||
}
|
||||
+200
-31
@@ -1,5 +1,6 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -7,9 +8,66 @@
|
||||
use super::common::bool_or_true;
|
||||
use super::{JmapId, UtcDate};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
/// How a server spells a FileNode. Servers on the 0.16 line, inbuxa among
|
||||
/// them, name the kind `nodeType` (`file`, `directory`, `symlink`) and a
|
||||
/// file's media type `type`. draft-ietf-jmap-filenode-18, which Stalwart 1.0
|
||||
/// follows, says `@type` (`File`, `Directory`, `SymLink`) and `mediaType`,
|
||||
/// and adds `executable`. Each refuses the other's names on a write and
|
||||
/// ignores them on a get, so a get asks for both and a write uses the
|
||||
/// server's own.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Shape {
|
||||
Legacy,
|
||||
Draft18,
|
||||
}
|
||||
|
||||
/// A FileNode, in either shape, read into this one; a node with neither
|
||||
/// `@type` nor `nodeType` is a file if it has a blob, a symlink if it has a
|
||||
/// target, and a directory otherwise.
|
||||
pub fn to_draft18(node: &mut Map<String, Value>) {
|
||||
let kind = match node.remove("@type") {
|
||||
Some(Value::String(t)) => t,
|
||||
_ => match node.remove("nodeType") {
|
||||
Some(Value::String(t)) => t,
|
||||
_ if node.get("blobId").is_some_and(|v| !v.is_null()) => "file".to_owned(),
|
||||
_ if node.get("target").is_some_and(|v| !v.is_null()) => "symlink".to_owned(),
|
||||
_ => "directory".to_owned(),
|
||||
},
|
||||
};
|
||||
node.remove("nodeType");
|
||||
let kind = match kind.to_ascii_lowercase().as_str() {
|
||||
"file" => "File".to_owned(),
|
||||
"directory" => "Directory".to_owned(),
|
||||
"symlink" => "SymLink".to_owned(),
|
||||
_ => kind,
|
||||
};
|
||||
node.insert("@type".to_owned(), Value::String(kind));
|
||||
if let Some(media_type) = node.remove("type") {
|
||||
node.entry("mediaType").or_insert(media_type);
|
||||
}
|
||||
}
|
||||
|
||||
/// A FileNode, or a patch to one, written as `shape` spells it.
|
||||
pub fn for_server(node: &mut Map<String, Value>, shape: Shape) {
|
||||
if shape == Shape::Draft18 {
|
||||
return;
|
||||
}
|
||||
if let Some(Value::String(kind)) = node.remove("@type") {
|
||||
node.insert(
|
||||
"nodeType".to_owned(),
|
||||
Value::String(kind.to_ascii_lowercase()),
|
||||
);
|
||||
}
|
||||
if let Some(media_type) = node.remove("mediaType") {
|
||||
node.insert("type".to_owned(), media_type);
|
||||
}
|
||||
node.remove("executable");
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(rename_all = "camelCase", try_from = "Map<String, Value>")]
|
||||
pub struct FileNode {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub id: Option<JmapId>,
|
||||
@@ -17,20 +75,8 @@ pub struct FileNode {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub parent_id: Option<JmapId>,
|
||||
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub node_type: Option<NodeType>,
|
||||
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub blob_id: Option<JmapId>,
|
||||
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub target: Option<Vec<String>>,
|
||||
|
||||
pub name: String,
|
||||
|
||||
#[serde(rename = "type", default, skip_serializing_if = "Option::is_none")]
|
||||
pub media_type: Option<String>,
|
||||
|
||||
#[serde(with = "time::serde::rfc3339")]
|
||||
pub created: UtcDate,
|
||||
|
||||
@@ -41,32 +87,155 @@ pub struct FileNode {
|
||||
)]
|
||||
pub modified: Option<UtcDate>,
|
||||
|
||||
#[serde(default = "default_true", deserialize_with = "bool_or_true")]
|
||||
pub is_subscribed: bool,
|
||||
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub role: Option<String>,
|
||||
#[serde(flatten)]
|
||||
pub kind: NodeKind,
|
||||
}
|
||||
|
||||
impl FileNode {
|
||||
pub fn effective_node_type(&self) -> NodeType {
|
||||
match self.node_type {
|
||||
Some(t) => t,
|
||||
None if self.blob_id.is_some() => NodeType::File,
|
||||
None if self.target.is_some() => NodeType::Symlink,
|
||||
None => NodeType::Directory,
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "@type")]
|
||||
pub enum NodeKind {
|
||||
File(FileProps),
|
||||
Directory(DirectoryProps),
|
||||
SymLink(SymLinkProps),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct FileProps {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub blob_id: Option<JmapId>,
|
||||
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub media_type: Option<String>,
|
||||
|
||||
#[serde(default, deserialize_with = "bool_or_false")]
|
||||
pub executable: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DirectoryProps {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub role: Option<String>,
|
||||
|
||||
#[serde(default = "default_true", deserialize_with = "bool_or_true")]
|
||||
pub is_subscribed: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SymLinkProps {
|
||||
pub target: Vec<String>,
|
||||
}
|
||||
|
||||
/// `FileNode`'s fields, read once the node is in draft-18's names.
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct Draft18Node {
|
||||
#[serde(default)]
|
||||
id: Option<JmapId>,
|
||||
#[serde(default)]
|
||||
parent_id: Option<JmapId>,
|
||||
name: String,
|
||||
#[serde(with = "time::serde::rfc3339")]
|
||||
created: UtcDate,
|
||||
#[serde(default, with = "time::serde::rfc3339::option")]
|
||||
modified: Option<UtcDate>,
|
||||
#[serde(flatten)]
|
||||
kind: NodeKind,
|
||||
}
|
||||
|
||||
impl TryFrom<Map<String, Value>> for FileNode {
|
||||
type Error = serde_json::Error;
|
||||
|
||||
fn try_from(mut node: Map<String, Value>) -> Result<Self, Self::Error> {
|
||||
to_draft18(&mut node);
|
||||
let n: Draft18Node = serde_json::from_value(Value::Object(node))?;
|
||||
Ok(FileNode {
|
||||
id: n.id,
|
||||
parent_id: n.parent_id,
|
||||
name: n.name,
|
||||
created: n.created,
|
||||
modified: n.modified,
|
||||
kind: n.kind,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl NodeKind {
|
||||
pub fn db_name(&self) -> &'static str {
|
||||
match self {
|
||||
NodeKind::File(_) => "file",
|
||||
NodeKind::Directory(_) => "directory",
|
||||
NodeKind::SymLink(_) => "symlink",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum NodeType {
|
||||
File,
|
||||
Directory,
|
||||
Symlink,
|
||||
fn bool_or_false<'de, D>(d: D) -> Result<bool, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
Ok(Option::<bool>::deserialize(d)?.unwrap_or_default())
|
||||
}
|
||||
|
||||
fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn node(v: Value) -> FileNode {
|
||||
serde_json::from_value(v).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn both_shapes_read_the_same() {
|
||||
let created = "2022-02-02T02:02:02Z";
|
||||
let draft = node(
|
||||
json!({"id": "F1", "@type": "File", "name": "a.sh", "blobId": "B1",
|
||||
"mediaType": "text/x-sh", "executable": true, "created": created}),
|
||||
);
|
||||
let legacy = node(
|
||||
json!({"id": "F1", "nodeType": "file", "name": "a.sh", "blobId": "B1",
|
||||
"type": "text/x-sh", "created": created}),
|
||||
);
|
||||
for n in [&draft, &legacy] {
|
||||
let NodeKind::File(f) = &n.kind else {
|
||||
panic!("{n:?}")
|
||||
};
|
||||
assert_eq!(f.media_type.as_deref(), Some("text/x-sh"));
|
||||
assert_eq!(f.blob_id.as_ref().map(|b| b.0.as_str()), Some("B1"));
|
||||
}
|
||||
let link = node(json!({"id": "L", "nodeType": "symlink", "name": "l",
|
||||
"target": ["a"], "created": created}));
|
||||
assert!(matches!(link.kind, NodeKind::SymLink(_)));
|
||||
let dir = node(json!({"id": "D", "name": "d", "created": created}));
|
||||
assert!(matches!(dir.kind, NodeKind::Directory(_)));
|
||||
assert!(
|
||||
serde_json::from_value::<FileNode>(json!({"id": "S", "nodeType": "socket",
|
||||
"name": "s", "created": created}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_legacy_server_gets_its_own_names() {
|
||||
let mut m = json!({"@type": "File", "name": "a.sh", "blobId": "B1",
|
||||
"mediaType": "text/x-sh", "executable": true});
|
||||
let Value::Object(map) = &mut m else {
|
||||
unreachable!()
|
||||
};
|
||||
let mut draft = map.clone();
|
||||
for_server(&mut draft, Shape::Draft18);
|
||||
assert_eq!(&draft, map);
|
||||
for_server(map, Shape::Legacy);
|
||||
assert_eq!(
|
||||
m,
|
||||
json!({"nodeType": "file", "name": "a.sh", "blobId": "B1",
|
||||
"type": "text/x-sh"})
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -16,5 +16,6 @@ pub mod identity;
|
||||
pub mod mailbox;
|
||||
pub mod participant_identity;
|
||||
pub mod sieve_script;
|
||||
pub mod vacation_response;
|
||||
|
||||
pub use common::{Date, EmailAddress, JmapId, UtcDate};
|
||||
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub const TYPE_NAME: &str = "VacationResponse";
|
||||
|
||||
pub const SINGLETON_ID: &str = "singleton";
|
||||
|
||||
pub const RESERVED_SCRIPT_NAME: &str = "vacation";
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct VacationResponse {
|
||||
#[serde(default)]
|
||||
pub is_enabled: bool,
|
||||
|
||||
#[serde(default)]
|
||||
pub from_date: Option<String>,
|
||||
|
||||
#[serde(default)]
|
||||
pub to_date: Option<String>,
|
||||
|
||||
#[serde(default)]
|
||||
pub subject: Option<String>,
|
||||
|
||||
#[serde(default)]
|
||||
pub text_body: Option<String>,
|
||||
|
||||
#[serde(default)]
|
||||
pub html_body: Option<String>,
|
||||
}
|
||||
|
||||
impl VacationResponse {
|
||||
pub fn is_unset(&self) -> bool {
|
||||
!self.is_enabled
|
||||
&& self.from_date.is_none()
|
||||
&& self.to_date.is_none()
|
||||
&& self.subject.is_none()
|
||||
&& self.text_body.is_none()
|
||||
&& self.html_body.is_none()
|
||||
}
|
||||
|
||||
pub fn is_reserved_script_name(name: &str) -> bool {
|
||||
name.eq_ignore_ascii_case(RESERVED_SCRIPT_NAME)
|
||||
}
|
||||
}
|
||||
+10
-6
@@ -1,5 +1,6 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -29,12 +30,8 @@ pub fn resolve_any(
|
||||
if let Some(value) = flag_value {
|
||||
return Ok(value.to_owned());
|
||||
}
|
||||
for env_var in env_vars {
|
||||
if let Ok(value) = std::env::var(env_var)
|
||||
&& !value.is_empty()
|
||||
{
|
||||
return Ok(value);
|
||||
}
|
||||
if let Some(value) = from_env(env_vars) {
|
||||
return Ok(value);
|
||||
}
|
||||
if std::io::stdin().is_terminal() {
|
||||
return prompt_no_echo(prompt_label);
|
||||
@@ -45,6 +42,13 @@ pub fn resolve_any(
|
||||
)))
|
||||
}
|
||||
|
||||
/// The first of `env_vars` that is set and not empty.
|
||||
pub fn from_env(env_vars: &[&str]) -> Option<String> {
|
||||
env_vars
|
||||
.iter()
|
||||
.find_map(|var| std::env::var(var).ok().filter(|v| !v.is_empty()))
|
||||
}
|
||||
|
||||
fn prompt_no_echo(label: &str) -> Result<String, Error> {
|
||||
let mut stderr = std::io::stderr();
|
||||
write!(stderr, "{label}: ")?;
|
||||
|
||||
+47
-7
@@ -20,22 +20,24 @@ use crate::jmap::blobxfer;
|
||||
use crate::jmap::connect::{self, Connected};
|
||||
use crate::jmap::error::JmapError;
|
||||
use crate::jmap::http::HttpClient;
|
||||
use crate::jmap::request::{Request, SetRequest, get_all, get_objects, query_all_ids, set_call};
|
||||
use crate::jmap::request::{
|
||||
Request, SetRequest, URN_VACATION_RESPONSE, get_all, get_objects, query_all_ids, set_call,
|
||||
};
|
||||
use crate::jmap::session::{Limits, Session};
|
||||
use crate::jmap::wire::JmapId;
|
||||
use crate::jmap::wire::vacation_response;
|
||||
use crate::logging::{LEVEL_DEFAULT, Logger};
|
||||
use crate::sync::import_jmap::mapping::{BlobBytes, TargetResolver};
|
||||
use crate::sync::{CommonConfig, Context, ExportConfig, Summary, TypeCounts};
|
||||
use crate::types::ObjectType;
|
||||
|
||||
const EXPORT_ORDER: [ObjectType; 10] = [
|
||||
const EXPORT_ORDER: [ObjectType; 9] = [
|
||||
ObjectType::Mailbox,
|
||||
ObjectType::AddressBook,
|
||||
ObjectType::Calendar,
|
||||
ObjectType::FileNode,
|
||||
ObjectType::Identity,
|
||||
ObjectType::SieveScript,
|
||||
ObjectType::ParticipantIdentity,
|
||||
ObjectType::Email,
|
||||
ObjectType::ContactCard,
|
||||
ObjectType::CalendarEvent,
|
||||
@@ -417,6 +419,35 @@ pub fn run(common: CommonConfig, config: ExportConfig) -> Result<Summary, Error>
|
||||
}
|
||||
}
|
||||
|
||||
if config
|
||||
.objects
|
||||
.as_ref()
|
||||
.is_none_or(|s| s.contains(&ObjectType::SieveScript))
|
||||
&& net.session.may_support(&net.account, URN_VACATION_RESPONSE)
|
||||
{
|
||||
let mut counts = TypeCounts::default();
|
||||
let present = match vacation::reconcile(&ctx, &net, &mut counts, &logger) {
|
||||
Ok(present) => present,
|
||||
Err(e) if e.aborts_run() => return Err(e),
|
||||
Err(e) => {
|
||||
logger.warn(&format!(
|
||||
"type {} aborted: {e}",
|
||||
vacation_response::TYPE_NAME
|
||||
));
|
||||
counts.failed += 1;
|
||||
true
|
||||
}
|
||||
};
|
||||
if present {
|
||||
if ctx.dry_run() {
|
||||
dry_rows.push((vacation_response::TYPE_NAME, counts.updated, 0, 0));
|
||||
}
|
||||
summary
|
||||
.per_type
|
||||
.push((vacation_response::TYPE_NAME, counts));
|
||||
}
|
||||
}
|
||||
|
||||
if ctx.dry_run() {
|
||||
let would_fail = net.would_fail.lock().expect("would-fail list").clone();
|
||||
print_plan(&summary, &dry_rows, &would_fail, config.prune);
|
||||
@@ -487,6 +518,16 @@ fn work_list(
|
||||
logger: &Logger,
|
||||
) -> Vec<ObjectType> {
|
||||
let selected = config.objects.as_ref();
|
||||
let participant = ObjectType::ParticipantIdentity;
|
||||
if selected.is_none_or(|s| s.contains(&participant))
|
||||
&& has_rows(conn, participant)
|
||||
&& logger.enabled(LEVEL_DEFAULT)
|
||||
{
|
||||
eprintln!(
|
||||
"export: {} not exported; the target creates participant identities from the account's addresses",
|
||||
participant.jmap_name()
|
||||
);
|
||||
}
|
||||
EXPORT_ORDER
|
||||
.into_iter()
|
||||
.filter(|ty| selected.map(|s| s.contains(ty)).unwrap_or(true))
|
||||
@@ -523,9 +564,7 @@ fn reconcile_type(
|
||||
flat::reconcile(ctx, net, ty, maps, counts, logger)
|
||||
}
|
||||
ObjectType::Identity => keyed::reconcile_identity(ctx, net, maps, counts, logger),
|
||||
ObjectType::ParticipantIdentity => {
|
||||
keyed::reconcile_participant(ctx, net, maps, counts, logger)
|
||||
}
|
||||
ObjectType::ParticipantIdentity => Ok(Plan::default()),
|
||||
ObjectType::SieveScript => sieve::reconcile(ctx, net, maps, counts, logger),
|
||||
ObjectType::ContactCard | ObjectType::CalendarEvent => {
|
||||
uidtype::reconcile(ctx, net, ty, maps, counts, logger)
|
||||
@@ -552,7 +591,7 @@ pub struct Plan {
|
||||
}
|
||||
|
||||
fn do_destroy(net: &Net, ty: ObjectType, plan: &Plan, logger: &Logger, counts: &mut TypeCounts) {
|
||||
if ty == ObjectType::SieveScript {
|
||||
if ty == ObjectType::SieveScript && plan.active_sieve_target.is_none() {
|
||||
deactivate_active_sieve_script(net, logger);
|
||||
}
|
||||
let destroy = Value::Array(
|
||||
@@ -700,6 +739,7 @@ mod keyed;
|
||||
mod sieve;
|
||||
|
||||
mod sieve_names;
|
||||
mod vacation;
|
||||
|
||||
mod uidtype;
|
||||
|
||||
|
||||
+11
-2
@@ -12,7 +12,7 @@ use super::common::{create_batch, jid, target_get_all};
|
||||
use super::{Maps, Net, Plan};
|
||||
use crate::error::Error;
|
||||
use crate::jmap::wire::JmapId;
|
||||
use crate::logging::Logger;
|
||||
use crate::logging::{LEVEL_PROGRESS, Logger};
|
||||
use crate::sync::import_jmap::mapping::{
|
||||
ADDRESS_BOOK_SELECT, CALENDAR_SELECT, row_to_address_book, row_to_calendar,
|
||||
};
|
||||
@@ -83,7 +83,16 @@ pub fn reconcile(
|
||||
Ok(if ty == ObjectType::AddressBook {
|
||||
row_to_address_book(row).map(|w| serde_json::to_value(&w))
|
||||
} else {
|
||||
row_to_calendar(row).map(|w| serde_json::to_value(&w))
|
||||
row_to_calendar(row).map(|mut w| {
|
||||
if let Some(dropped) = w.normalize_color()
|
||||
&& logger.enabled(LEVEL_PROGRESS)
|
||||
{
|
||||
eprintln!(
|
||||
"Calendar {local}: dropping color {dropped:?}, not a CSS color name or #RGB/#RRGGBB value"
|
||||
);
|
||||
}
|
||||
serde_json::to_value(&w)
|
||||
})
|
||||
})
|
||||
})
|
||||
.map_err(|e| Error::Partial(e.to_string()))?
|
||||
|
||||
@@ -12,10 +12,8 @@ use super::common::{create_batch, target_get_all};
|
||||
use super::{Maps, Net, Plan};
|
||||
use crate::error::Error;
|
||||
use crate::logging::Logger;
|
||||
use crate::sync::import_jmap::mapping::{
|
||||
IDENTITY_SELECT, PARTICIPANT_IDENTITY_SELECT, row_to_identity, row_to_participant_identity,
|
||||
};
|
||||
use crate::sync::keys::{identity_key, participant_identity_key};
|
||||
use crate::sync::import_jmap::mapping::{IDENTITY_SELECT, row_to_identity};
|
||||
use crate::sync::keys::identity_key;
|
||||
use crate::sync::{Context, TypeCounts};
|
||||
use crate::types::ObjectType;
|
||||
|
||||
@@ -99,47 +97,3 @@ pub fn reconcile_identity(
|
||||
rows,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn reconcile_participant(
|
||||
ctx: &Context,
|
||||
net: &Net,
|
||||
_maps: &mut Maps,
|
||||
counts: &mut TypeCounts,
|
||||
logger: &Logger,
|
||||
) -> Result<Plan, Error> {
|
||||
let mut stmt = ctx
|
||||
.conn
|
||||
.prepare(PARTICIPANT_IDENTITY_SELECT)
|
||||
.map_err(|e| Error::Partial(e.to_string()))?;
|
||||
let rows: Vec<(i64, [u8; 32], Value)> = stmt
|
||||
.query_map([], |row| {
|
||||
let id: i64 = row.get(0)?;
|
||||
Ok((id, row_to_participant_identity(row)))
|
||||
})
|
||||
.and_then(|m| m.collect::<Result<Vec<_>, _>>())
|
||||
.map_err(|e| Error::Partial(e.to_string()))?
|
||||
.into_iter()
|
||||
.map(|(id, w)| {
|
||||
let w = w.map_err(Error::from)?;
|
||||
let key = participant_identity_key(&w.calendar_address, &w.name);
|
||||
let v = serde_json::to_value(&w).map_err(|e| Error::Partial(e.to_string()))?;
|
||||
Ok((id, key, v))
|
||||
})
|
||||
.collect::<Result<_, Error>>()?;
|
||||
drop(stmt);
|
||||
run(
|
||||
net,
|
||||
ObjectType::ParticipantIdentity,
|
||||
counts,
|
||||
logger,
|
||||
|t| {
|
||||
Some(participant_identity_key(
|
||||
t.get("calendarAddress")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or(""),
|
||||
t.get("name").and_then(Value::as_str).unwrap_or(""),
|
||||
))
|
||||
},
|
||||
rows,
|
||||
)
|
||||
}
|
||||
@@ -12,10 +12,12 @@ use serde_json::{Value, json};
|
||||
use super::common::{create_batch, jid, retry_if_blob_missing, target_get_all, update_batch};
|
||||
use super::sieve_names;
|
||||
use super::{Maps, Net, Plan, Uploader};
|
||||
use crate::db;
|
||||
use crate::error::Error;
|
||||
use crate::jmap::blobxfer;
|
||||
use crate::jmap::error::JmapError;
|
||||
use crate::jmap::request::{Request, URN_INBUXA, check_method_error};
|
||||
use crate::jmap::request::{Request, URN_INBUXA, URN_VACATION_RESPONSE, check_method_error};
|
||||
use crate::jmap::wire::vacation_response::VacationResponse;
|
||||
use crate::logging::{LEVEL_DEFAULT, Logger};
|
||||
use crate::sync::import_jmap::mapping::BlobBytes;
|
||||
use crate::sync::import_jmap::mapping::{SIEVE_SELECT, row_to_sieve_script};
|
||||
@@ -44,7 +46,11 @@ pub fn reconcile(
|
||||
target_by_name.insert(name.to_owned(), id);
|
||||
}
|
||||
|
||||
let locals: Vec<(i64, Option<String>, bool, i64)> = {
|
||||
let target_has_vacation = net.session.may_support(&net.account, URN_VACATION_RESPONSE);
|
||||
let vacation_exported = target_has_vacation
|
||||
&& db::vacation_response::exists(&ctx.conn).map_err(|e| Error::Partial(e.to_string()))?;
|
||||
|
||||
let mut locals: Vec<(i64, Option<String>, bool, i64)> = {
|
||||
let mut stmt = ctx
|
||||
.conn
|
||||
.prepare(SIEVE_SELECT)
|
||||
@@ -63,6 +69,16 @@ pub fn reconcile(
|
||||
.collect::<Result<_, Error>>()?
|
||||
};
|
||||
|
||||
if vacation_exported {
|
||||
let before = locals.len();
|
||||
locals.retain(|(_, name, _, _)| {
|
||||
!name
|
||||
.as_deref()
|
||||
.is_some_and(VacationResponse::is_reserved_script_name)
|
||||
});
|
||||
counts.skipped += (before - locals.len()) as u64;
|
||||
}
|
||||
|
||||
let mut active_target: Option<String> = None;
|
||||
let mut deactivate = false;
|
||||
let mut uploader = Uploader::new(net, &ctx.conn);
|
||||
@@ -222,6 +238,9 @@ pub fn reconcile(
|
||||
let mut prune_candidates: Vec<String> = target_by_name
|
||||
.iter()
|
||||
.filter(|(name, _)| !local_names.contains(*name))
|
||||
.filter(|(name, _)| {
|
||||
!(target_has_vacation && VacationResponse::is_reserved_script_name(name))
|
||||
})
|
||||
.map(|(_, id)| id.clone())
|
||||
.collect();
|
||||
prune_candidates.sort();
|
||||
|
||||
+71
-19
@@ -1,5 +1,6 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -7,12 +8,14 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use rusqlite::params;
|
||||
use serde_json::Value;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::common::{create_batch, jid, retry_if_blob_missing, target_query_get};
|
||||
use super::{Net, Plan, Uploader};
|
||||
use crate::error::Error;
|
||||
use crate::jmap::request::{Request, URN_INBUXA};
|
||||
use crate::jmap::wire::JmapId;
|
||||
use crate::jmap::wire::file_node::{NodeKind, Shape, for_server};
|
||||
use crate::logging::Logger;
|
||||
use crate::sync::import_jmap::mapping::{
|
||||
FILE_NODE_SELECT, MAILBOX_SELECT, TargetResolver, row_to_file_node, row_to_mailbox,
|
||||
@@ -61,24 +64,72 @@ fn load_local(ctx: &Context, ty: ObjectType) -> Result<Vec<LocalNode>, Error> {
|
||||
fn load_target(net: &Net, ty: ObjectType) -> Result<Vec<TargetNode>, Error> {
|
||||
let props: &[&str] = &["role", "name", "parentId", "myRights"];
|
||||
let list = target_query_get(net, ty, Some(props)).map_err(Error::from)?;
|
||||
let mut own_rights: HashMap<&str, bool> = HashMap::with_capacity(list.len());
|
||||
for v in &list {
|
||||
if let (Some(id), Some(may)) = (
|
||||
v.get("id").and_then(Value::as_str),
|
||||
v.get("myRights")
|
||||
.and_then(|r| r.get("mayDelete"))
|
||||
.and_then(Value::as_bool),
|
||||
) {
|
||||
own_rights.insert(id, may);
|
||||
}
|
||||
}
|
||||
Ok(list
|
||||
.iter()
|
||||
.filter_map(|v| {
|
||||
let id = jid(v)?;
|
||||
let parent = v.get("parentId").and_then(Value::as_str);
|
||||
let may_delete = own_rights
|
||||
.get(id.as_str())
|
||||
.or_else(|| parent.and_then(|p| own_rights.get(p)))
|
||||
.copied()
|
||||
.unwrap_or(true);
|
||||
Some(TargetNode {
|
||||
id: jid(v)?,
|
||||
parent: v.get("parentId").and_then(Value::as_str).map(str::to_owned),
|
||||
id,
|
||||
parent: parent.map(str::to_owned),
|
||||
name: v.get("name").and_then(Value::as_str)?.to_owned(),
|
||||
role: v.get("role").and_then(Value::as_str).map(str::to_owned),
|
||||
may_delete: v
|
||||
.get("myRights")
|
||||
.and_then(|r| r.get("mayDelete"))
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(true),
|
||||
may_delete,
|
||||
})
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// The FileNode shape the target takes. Nothing in the session says, so this
|
||||
/// asks with a query sorted by `type`: a server on the 0.16 line answers it,
|
||||
/// and a draft-18 one refuses it as `unsupportedSort`, the property being
|
||||
/// gone. It writes nothing. Any other answer leaves it to the session, and
|
||||
/// inbuxa takes the 0.16 shape.
|
||||
fn file_node_shape(net: &Net) -> Shape {
|
||||
let mut req = Request::new();
|
||||
req.call(
|
||||
"FileNode/query",
|
||||
json!({
|
||||
"accountId": net.account,
|
||||
"sort": [{"property": "type", "isAscending": true}],
|
||||
"limit": 1,
|
||||
}),
|
||||
"shape",
|
||||
);
|
||||
let answer = req.send(&net.client, &net.api);
|
||||
let call = answer
|
||||
.as_ref()
|
||||
.ok()
|
||||
.and_then(|r| r.by_call_id("shape").ok());
|
||||
match call {
|
||||
Some(c) if c.name == "FileNode/query" => Shape::Legacy,
|
||||
Some(c)
|
||||
if c.name == "error"
|
||||
&& c.args.get("type").and_then(Value::as_str) == Some("unsupportedSort") =>
|
||||
{
|
||||
Shape::Draft18
|
||||
}
|
||||
_ if net.session.capabilities.contains_key(URN_INBUXA) => Shape::Legacy,
|
||||
_ => Shape::Draft18,
|
||||
}
|
||||
}
|
||||
|
||||
fn depth(local: i64, by: &HashMap<i64, Option<i64>>) -> usize {
|
||||
let mut d = 0;
|
||||
let mut cur = by.get(&local).copied().flatten();
|
||||
@@ -197,6 +248,7 @@ pub fn reconcile(
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
|
||||
let shape = (ty == ObjectType::FileNode && !to_create.is_empty()).then(|| file_node_shape(net));
|
||||
let mut uploader = Uploader::new(net, &ctx.conn);
|
||||
let mut taken_roles: HashSet<String> = targets.iter().filter_map(|t| t.role.clone()).collect();
|
||||
let interleave = ty == ObjectType::FileNode;
|
||||
@@ -240,7 +292,7 @@ pub fn reconcile(
|
||||
}
|
||||
let cid = format!("c{}", n.local);
|
||||
let _ = uploader.take_touched();
|
||||
let obj = match build_create(ctx, ty, n.local, maps, &mut uploader) {
|
||||
let obj = match build_create(ctx, ty, n.local, maps, &mut uploader, shape) {
|
||||
Ok(o) => o,
|
||||
Err(e) => {
|
||||
logger.warn(&format!(
|
||||
@@ -262,7 +314,7 @@ pub fn reconcile(
|
||||
&mut uploader,
|
||||
touched,
|
||||
outcome,
|
||||
|up| build_create(ctx, ty, n.local, maps, up),
|
||||
|up| build_create(ctx, ty, n.local, maps, up, shape),
|
||||
) {
|
||||
Ok(o) => o,
|
||||
Err(e) => {
|
||||
@@ -335,7 +387,7 @@ pub fn reconcile(
|
||||
counts.skipped += 1;
|
||||
continue;
|
||||
}
|
||||
match build_create(ctx, ty, n.local, maps, &mut uploader) {
|
||||
match build_create(ctx, ty, n.local, maps, &mut uploader, shape) {
|
||||
Ok(mut obj) => {
|
||||
if ty == ObjectType::Mailbox
|
||||
&& let Some(r) = n.role.as_deref()
|
||||
@@ -418,6 +470,7 @@ fn build_create(
|
||||
local: i64,
|
||||
maps: &Maps,
|
||||
uploader: &mut Uploader<'_>,
|
||||
shape: Option<Shape>,
|
||||
) -> Result<Value, Error> {
|
||||
if ty == ObjectType::Mailbox {
|
||||
let wire = ctx
|
||||
@@ -441,21 +494,20 @@ fn build_create(
|
||||
.map_err(|e| Error::Partial(e.to_string()))?
|
||||
.map_err(Error::from)?;
|
||||
let mut wire = fnrow.wire;
|
||||
if let (crate::jmap::wire::file_node::NodeType::File, Some(blob_local)) =
|
||||
(wire.effective_node_type(), fnrow.blob_local_id)
|
||||
{
|
||||
let ct = wire
|
||||
if let (NodeKind::File(file), Some(blob_local)) = (&mut wire.kind, fnrow.blob_local_id) {
|
||||
let ct = file
|
||||
.media_type
|
||||
.clone()
|
||||
.unwrap_or_else(|| "application/octet-stream".to_owned());
|
||||
let id = uploader.upload_with(blob_local, &ct).map_err(Error::from)?;
|
||||
wire.blob_id = Some(id);
|
||||
.as_deref()
|
||||
.unwrap_or("application/octet-stream");
|
||||
let id = uploader.upload_with(blob_local, ct).map_err(Error::from)?;
|
||||
file.blob_id = Some(id);
|
||||
}
|
||||
let mut obj = serde_json::to_value(&wire).map_err(|e| Error::Partial(e.to_string()))?;
|
||||
if let Value::Object(m) = &mut obj {
|
||||
m.remove("created");
|
||||
m.remove("modified");
|
||||
m.remove("isSubscribed");
|
||||
for_server(m, shape.unwrap_or(Shape::Draft18));
|
||||
}
|
||||
Ok(obj)
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
use serde_json::json;
|
||||
|
||||
use super::Net;
|
||||
use crate::db;
|
||||
use crate::error::Error;
|
||||
use crate::jmap::request::{SetRequest, set_call};
|
||||
use crate::jmap::wire::vacation_response::{SINGLETON_ID, TYPE_NAME};
|
||||
use crate::logging::Logger;
|
||||
use crate::sync::{Context, TypeCounts};
|
||||
|
||||
pub fn reconcile(
|
||||
ctx: &Context,
|
||||
net: &Net,
|
||||
counts: &mut TypeCounts,
|
||||
logger: &Logger,
|
||||
) -> Result<bool, Error> {
|
||||
let Some(vacation) =
|
||||
db::vacation_response::load(&ctx.conn).map_err(|e| Error::Partial(e.to_string()))?
|
||||
else {
|
||||
return Ok(false);
|
||||
};
|
||||
if net.dry_run {
|
||||
counts.updated += 1;
|
||||
return Ok(true);
|
||||
}
|
||||
let outcome = set_call(
|
||||
&net.client,
|
||||
&net.api,
|
||||
&net.account,
|
||||
TYPE_NAME,
|
||||
SetRequest {
|
||||
update: Some(json!({ SINGLETON_ID: vacation })),
|
||||
..Default::default()
|
||||
},
|
||||
&net.limits,
|
||||
)
|
||||
.map_err(Error::from)?;
|
||||
counts.updated += outcome.updated.len() as u64;
|
||||
for (id, err) in &outcome.not_updated {
|
||||
logger.warn(&format!("{TYPE_NAME} {id} not updated: {err}"));
|
||||
counts.failed += 1;
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
@@ -8,9 +8,10 @@ pub mod calcard;
|
||||
pub mod collections;
|
||||
pub mod coordinator;
|
||||
pub mod items;
|
||||
pub mod replace;
|
||||
pub mod tree;
|
||||
|
||||
pub use coordinator::{DavAuth, DavImportConfig, DavKindArg, run, run_reporting};
|
||||
pub use coordinator::{DavAuth, DavImportConfig, DavKindArg, SourceChange, run, run_reporting};
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::jmap::error::JmapError;
|
||||
|
||||
@@ -5,16 +5,22 @@
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::fmt::Write;
|
||||
|
||||
use calcard::Entry;
|
||||
use calcard::icalendar::ICalendar;
|
||||
use calcard::vcard::VCard;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
const MAX_INVALID_LINE_CHARS: usize = 80;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum CalcardError {
|
||||
#[error("vcard parse failed")]
|
||||
VCardParse,
|
||||
#[error("icalendar parse failed")]
|
||||
ICalParse,
|
||||
#[error("vcard parse failed: {0}")]
|
||||
VCardParse(String),
|
||||
#[error("icalendar parse failed: {0}")]
|
||||
ICalParse(String),
|
||||
#[error("jscontact serialisation: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
#[error("no parseable entries in iCalendar")]
|
||||
@@ -30,7 +36,8 @@ pub struct JsContact {
|
||||
}
|
||||
|
||||
pub fn vcard_to_jscontact(text: &str, item_href: &str) -> Result<JsContact, CalcardError> {
|
||||
let vcard = VCard::parse(text).map_err(|_| CalcardError::VCardParse)?;
|
||||
let vcard = VCard::parse(normalize_line_breaks(text))
|
||||
.map_err(|e| CalcardError::VCardParse(describe_parse_error(e)))?;
|
||||
let js = vcard.into_jscontact::<String, String>();
|
||||
let value: Value = serde_json::from_str(&js.to_string_pretty())?;
|
||||
let uid = value
|
||||
@@ -82,7 +89,8 @@ impl EntryType {
|
||||
}
|
||||
|
||||
pub fn ical_to_jscalendar_entries(text: &str) -> Result<Vec<JsCalendarEntry>, CalcardError> {
|
||||
let ical = ICalendar::parse(text).map_err(|_| CalcardError::ICalParse)?;
|
||||
let ical = ICalendar::parse(normalize_line_breaks(text))
|
||||
.map_err(|e| CalcardError::ICalParse(describe_parse_error(e)))?;
|
||||
let js = ical.into_jscalendar::<String, String>();
|
||||
let value: Value = serde_json::from_str(&js.to_string_pretty())?;
|
||||
let mut out = Vec::new();
|
||||
@@ -93,6 +101,67 @@ pub fn ical_to_jscalendar_entries(text: &str) -> Result<Vec<JsCalendarEntry>, Ca
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn normalize_line_breaks(text: &str) -> Cow<'_, str> {
|
||||
let mut after_cr = text.split('\r').skip(1).peekable();
|
||||
let mut has_bare_cr = false;
|
||||
while let Some(rest) = after_cr.next() {
|
||||
if is_bare_cr(rest, after_cr.peek().is_some()) {
|
||||
has_bare_cr = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !has_bare_cr {
|
||||
return Cow::Borrowed(text);
|
||||
}
|
||||
let mut parts = text.split('\r').peekable();
|
||||
let mut out = String::with_capacity(text.len());
|
||||
out.push_str(parts.next().unwrap_or_default());
|
||||
while let Some(rest) = parts.next() {
|
||||
if is_bare_cr(rest, parts.peek().is_some()) {
|
||||
out.push('\n');
|
||||
}
|
||||
out.push_str(rest);
|
||||
}
|
||||
Cow::Owned(out)
|
||||
}
|
||||
|
||||
fn is_bare_cr(rest: &str, more: bool) -> bool {
|
||||
let followed_by_cr = rest.is_empty() && more;
|
||||
!followed_by_cr && !rest.starts_with('\n')
|
||||
}
|
||||
|
||||
fn describe_parse_error(entry: Entry) -> String {
|
||||
let mut out = String::new();
|
||||
match entry {
|
||||
Entry::InvalidLine(line) => {
|
||||
out.push_str("invalid line ");
|
||||
let mut chars = line.chars();
|
||||
let head: String = chars.by_ref().take(MAX_INVALID_LINE_CHARS).collect();
|
||||
let _ = write!(out, "{head:?}");
|
||||
if chars.next().is_some() {
|
||||
out.push_str(" (truncated)");
|
||||
}
|
||||
}
|
||||
Entry::UnexpectedComponentEnd { expected, found } => {
|
||||
let _ = write!(
|
||||
out,
|
||||
"found END:{} where END:{} was expected",
|
||||
found.as_str(),
|
||||
expected.as_str()
|
||||
);
|
||||
}
|
||||
Entry::UnterminatedComponent(name) => {
|
||||
let _ = write!(out, "{name} is never closed");
|
||||
}
|
||||
Entry::TooManyComponents => out.push_str("too many components"),
|
||||
Entry::Eof => out.push_str("no BEGIN line found"),
|
||||
Entry::VCard(_) => out.push_str("found a vCard where an iCalendar was expected"),
|
||||
Entry::ICalendar(_) => out.push_str("found an iCalendar where a vCard was expected"),
|
||||
_ => out.push_str("unrecognised content"),
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn collect_entries(value: Value, out: &mut Vec<JsCalendarEntry>) {
|
||||
match value {
|
||||
Value::Object(map) => {
|
||||
@@ -228,7 +297,53 @@ END:VCALENDAR\r\n";
|
||||
#[test]
|
||||
fn vcard_garbage_fails_parse() {
|
||||
let err = vcard_to_jscontact("not a vcard", "/x").unwrap_err();
|
||||
assert!(matches!(err, CalcardError::VCardParse));
|
||||
assert!(matches!(err, CalcardError::VCardParse(_)));
|
||||
}
|
||||
|
||||
// Kerio Connect sends calendar-data with bare CR line breaks (issue #48).
|
||||
#[test]
|
||||
fn ical_with_bare_cr_line_breaks_parses() {
|
||||
let ical = SAMPLE_ICAL.replace("\r\n", "\r");
|
||||
let entries = ical_to_jscalendar_entries(&ical).expect("bare CR parses");
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].data["title"], "Test Event");
|
||||
}
|
||||
|
||||
// A folded line split by a bare CR still unfolds.
|
||||
#[test]
|
||||
fn ical_with_bare_cr_unfolds_continuation_lines() {
|
||||
let ical = "BEGIN:VCALENDAR\rVERSION:2.0\rBEGIN:VEVENT\rUID:k1@example\r\
|
||||
DTSTAMP:20260101T000000Z\rDTSTART:20260101T090000Z\r\
|
||||
SUMMARY:Quarterly\r review\rEND:VEVENT\rEND:VCALENDAR\r";
|
||||
let entries = ical_to_jscalendar_entries(ical).expect("parse");
|
||||
assert_eq!(entries[0].data["title"], "Quarterly review");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vcard_with_bare_cr_line_breaks_parses() {
|
||||
let vcard = SAMPLE_VCARD.replace("\r\n", "\r");
|
||||
let jc = vcard_to_jscontact(&vcard, "/x").expect("bare CR parses");
|
||||
assert_eq!(jc.uid, "abc-123");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn line_break_normalization_borrows_unless_bare_cr() {
|
||||
for text in ["A\nB\n", "A\r\nB\r\n", "A\r\r\nB\r\r\n", "no breaks"] {
|
||||
assert!(
|
||||
matches!(normalize_line_breaks(text), Cow::Borrowed(_)),
|
||||
"{text:?}"
|
||||
);
|
||||
}
|
||||
assert_eq!(normalize_line_breaks("A\rB\r\nC\r"), "A\nB\nC\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ical_parse_error_names_the_offending_line() {
|
||||
let err = ical_to_jscalendar_entries("Content-Type: text/calendar\r\nBEGIN:VCALENDAR\r\n")
|
||||
.unwrap_err();
|
||||
let msg = err.to_string();
|
||||
assert!(msg.contains("invalid line"), "{msg}");
|
||||
assert!(msg.contains("Content-Type"), "{msg}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -18,6 +18,7 @@ use crate::sync::{CommonConfig, RunOutcome, Summary, TypeCounts};
|
||||
|
||||
use super::collections;
|
||||
use super::items;
|
||||
use super::replace::{self, Removed};
|
||||
use super::tree;
|
||||
use crate::net::CertOverride;
|
||||
|
||||
@@ -73,6 +74,13 @@ impl DavAuth {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum SourceChange {
|
||||
Reject,
|
||||
Allow,
|
||||
Replace,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct DavImportConfig {
|
||||
pub kind: DavKindArg,
|
||||
@@ -81,7 +89,7 @@ pub struct DavImportConfig {
|
||||
pub allow_cleartext: bool,
|
||||
pub dav_connections: usize,
|
||||
pub multiget_batch: usize,
|
||||
pub allow_source_change: bool,
|
||||
pub source_change: SourceChange,
|
||||
}
|
||||
|
||||
pub fn run(common: CommonConfig, config: DavImportConfig) -> Result<Summary, Error> {
|
||||
@@ -128,6 +136,7 @@ fn run_into(
|
||||
};
|
||||
|
||||
if !common.dry_run
|
||||
&& config.source_change == SourceChange::Reject
|
||||
&& let Some((url, acc)) = db::sources::conflicting_source(
|
||||
&conn,
|
||||
config.kind.source_kind(),
|
||||
@@ -135,11 +144,11 @@ fn run_into(
|
||||
&account_id,
|
||||
)
|
||||
.map_err(|e| Error::Partial(e.to_string()))?
|
||||
&& !config.allow_source_change
|
||||
{
|
||||
return Err(Error::SourceChange(format!(
|
||||
"archive already records {} source ({url}, account {acc}); \
|
||||
pass --allow-source-change to import a different account",
|
||||
pass --allow-source-change to import a different account, \
|
||||
or --replace-source to drop the recorded one",
|
||||
config.kind.source_kind()
|
||||
)));
|
||||
}
|
||||
@@ -154,12 +163,27 @@ fn run_into(
|
||||
}
|
||||
|
||||
if common.dry_run {
|
||||
if config.source_change == SourceChange::Replace && logger.enabled(LEVEL_DEFAULT) {
|
||||
for (_, url, acc) in
|
||||
db::sources::other_sources(&conn, &key.kind, &session_url, &account_id)?
|
||||
{
|
||||
eprintln!(
|
||||
"dry-run: would remove {} source {url} (account {acc})",
|
||||
key.kind
|
||||
);
|
||||
}
|
||||
}
|
||||
let result = run_dry_diff(&conn, &client, &discovery, &config, logger, summary);
|
||||
summary.retries_observed = client.retries_observed();
|
||||
summary.retry_after_sleeps = client.retry_after_sleeps();
|
||||
return result;
|
||||
}
|
||||
|
||||
let removed = match config.source_change {
|
||||
SourceChange::Replace => replace::remove_other_sources(&conn, &key, logger)?,
|
||||
SourceChange::Reject | SourceChange::Allow => Removed::default(),
|
||||
};
|
||||
|
||||
let username = config.auth.username();
|
||||
let source_id = db::sources::upsert_source(&conn, &key, Some(&account_id), &username)
|
||||
.map_err(|e| Error::Partial(e.to_string()))?;
|
||||
@@ -177,6 +201,7 @@ fn run_into(
|
||||
};
|
||||
|
||||
*summary = phase.summary;
|
||||
removed.add_to(summary);
|
||||
summary.retries_observed = client.retries_observed();
|
||||
summary.retry_after_sleeps = client.retry_after_sleeps();
|
||||
if let Some(e) = phase.error {
|
||||
|
||||
@@ -0,0 +1,255 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
|
||||
use rusqlite::{Connection, params};
|
||||
|
||||
use crate::db::dav_ids::{ADDRESS_BOOK, CALENDAR, CALENDAR_EVENT, CONTACT_CARD, FILE_NODE};
|
||||
use crate::db::sources::{self, SourceKey};
|
||||
use crate::error::Error;
|
||||
use crate::logging::{LEVEL_DEFAULT, Logger};
|
||||
use crate::sync::Summary;
|
||||
|
||||
const DELETE_MAPPED: [(&str, &str); 4] = [
|
||||
(
|
||||
CALENDAR_EVENT,
|
||||
"DELETE FROM calendar_events WHERE id IN
|
||||
(SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)",
|
||||
),
|
||||
(
|
||||
CONTACT_CARD,
|
||||
"DELETE FROM contact_cards WHERE id IN
|
||||
(SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)",
|
||||
),
|
||||
(
|
||||
CALENDAR,
|
||||
"DELETE FROM calendars WHERE id IN
|
||||
(SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)",
|
||||
),
|
||||
(
|
||||
ADDRESS_BOOK,
|
||||
"DELETE FROM address_books WHERE id IN
|
||||
(SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)",
|
||||
),
|
||||
];
|
||||
|
||||
const DELETE_LEAF_FILE_NODES: &str = "DELETE FROM file_nodes WHERE id IN
|
||||
(SELECT local_id FROM sync_id_dav WHERE source_id = ?1 AND type_name = ?2)
|
||||
AND NOT EXISTS (SELECT 1 FROM file_nodes AS child WHERE child.parent_id = file_nodes.id)";
|
||||
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Removed {
|
||||
pub calendar_events: u64,
|
||||
pub contact_cards: u64,
|
||||
pub calendars: u64,
|
||||
pub address_books: u64,
|
||||
pub file_nodes: u64,
|
||||
}
|
||||
|
||||
impl Removed {
|
||||
fn slot(&mut self, type_name: &str) -> Option<&mut u64> {
|
||||
match type_name {
|
||||
CALENDAR_EVENT => Some(&mut self.calendar_events),
|
||||
CONTACT_CARD => Some(&mut self.contact_cards),
|
||||
CALENDAR => Some(&mut self.calendars),
|
||||
ADDRESS_BOOK => Some(&mut self.address_books),
|
||||
FILE_NODE => Some(&mut self.file_nodes),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_to(mut self, summary: &mut Summary) {
|
||||
for (label, counts) in &mut summary.per_type {
|
||||
if let Some(n) = self.slot(label) {
|
||||
counts.deleted += std::mem::take(n);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn remove_other_sources(
|
||||
conn: &Connection,
|
||||
key: &SourceKey,
|
||||
logger: Logger,
|
||||
) -> Result<Removed, Error> {
|
||||
let others = sources::other_sources(conn, &key.kind, &key.session_url, &key.account_id)?;
|
||||
let mut removed = Removed::default();
|
||||
if others.is_empty() {
|
||||
return Ok(removed);
|
||||
}
|
||||
let tx = conn.unchecked_transaction()?;
|
||||
for (source_id, session_url, account_id) in &others {
|
||||
remove_source(&tx, *source_id, &mut removed)?;
|
||||
if logger.enabled(LEVEL_DEFAULT) {
|
||||
eprintln!(
|
||||
"import: removed {} source {session_url} (account {account_id})",
|
||||
key.kind
|
||||
);
|
||||
}
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
fn remove_source(
|
||||
conn: &Connection,
|
||||
source_id: i64,
|
||||
removed: &mut Removed,
|
||||
) -> Result<(), rusqlite::Error> {
|
||||
for (type_name, sql) in DELETE_MAPPED {
|
||||
let n = conn.execute(sql, params![source_id, type_name])?;
|
||||
if let Some(slot) = removed.slot(type_name) {
|
||||
*slot += n as u64;
|
||||
}
|
||||
}
|
||||
let mut leaves = conn.prepare(DELETE_LEAF_FILE_NODES)?;
|
||||
loop {
|
||||
let n = leaves.execute(params![source_id, FILE_NODE])?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
removed.file_nodes += n as u64;
|
||||
}
|
||||
sources::delete_source(conn, source_id)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::db::{dav_ids, init};
|
||||
use crate::sync::TypeCounts;
|
||||
|
||||
fn mem() -> Connection {
|
||||
let c = Connection::open_in_memory().unwrap();
|
||||
init::apply_schema(&c).unwrap();
|
||||
c.pragma_update(None, "foreign_keys", "ON").unwrap();
|
||||
c
|
||||
}
|
||||
|
||||
fn source(c: &Connection, kind: &str, url: &str) -> (SourceKey, i64) {
|
||||
let key = SourceKey {
|
||||
kind: kind.to_owned(),
|
||||
session_url: url.to_owned(),
|
||||
account_id: format!("{url}/u/"),
|
||||
};
|
||||
let id = sources::upsert_source(c, &key, None, "u").unwrap();
|
||||
(key, id)
|
||||
}
|
||||
|
||||
fn file_node(c: &Connection, sid: i64, parent: Option<i64>, name: &str, href: &str) -> i64 {
|
||||
c.execute(
|
||||
"INSERT INTO file_nodes (parent_id, node_type, name, created)
|
||||
VALUES (?1, 'directory', ?2, '2026-01-01T00:00:00Z')",
|
||||
params![parent, name],
|
||||
)
|
||||
.unwrap();
|
||||
let id = c.last_insert_rowid();
|
||||
dav_ids::insert(c, sid, FILE_NODE, href, href, "", id).unwrap();
|
||||
id
|
||||
}
|
||||
|
||||
fn count(c: &Connection, table: &str) -> i64 {
|
||||
c.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replacing_a_webdav_source_drops_the_old_tree_leaf_first() {
|
||||
let c = mem();
|
||||
let (_, old) = source(&c, "webdav", "https://old");
|
||||
let a = file_node(&c, old, None, "a", "/a/");
|
||||
let b = file_node(&c, old, Some(a), "b", "/a/b/");
|
||||
file_node(&c, old, Some(b), "c", "/a/b/c/");
|
||||
let (new_key, new) = source(&c, "webdav", "https://new");
|
||||
file_node(&c, new, None, "a", "/a/");
|
||||
|
||||
let removed = remove_other_sources(&c, &new_key, Logger::from_flags(true, 0)).unwrap();
|
||||
|
||||
assert_eq!(removed.file_nodes, 3);
|
||||
assert_eq!(count(&c, "file_nodes"), 1);
|
||||
assert_eq!(count(&c, "sources"), 1);
|
||||
assert_eq!(count(&c, "sync_id_dav"), 1);
|
||||
let kept: i64 = c
|
||||
.query_row("SELECT source_id FROM sync_id_dav", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(kept, new);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replacing_leaves_other_kinds_untouched() {
|
||||
let c = mem();
|
||||
let (_, cal) = source(&c, "caldav", "https://old");
|
||||
c.execute("INSERT INTO calendars (name) VALUES ('Work')", [])
|
||||
.unwrap();
|
||||
let cal_local = c.last_insert_rowid();
|
||||
dav_ids::insert(&c, cal, CALENDAR, "/c/", "/c/", "", cal_local).unwrap();
|
||||
c.execute(
|
||||
"INSERT INTO calendar_events (calendar_ids, data) VALUES (?1, '{}')",
|
||||
params![format!("[{cal_local}]")],
|
||||
)
|
||||
.unwrap();
|
||||
let ev = c.last_insert_rowid();
|
||||
dav_ids::insert(&c, cal, CALENDAR_EVENT, "/c/", "/c/1.ics", "", ev).unwrap();
|
||||
let (_, old_web) = source(&c, "webdav", "https://old");
|
||||
file_node(&c, old_web, None, "a", "/a/");
|
||||
let (new_web, _) = source(&c, "webdav", "https://new");
|
||||
|
||||
let removed = remove_other_sources(&c, &new_web, Logger::from_flags(true, 0)).unwrap();
|
||||
|
||||
assert_eq!(removed.file_nodes, 1);
|
||||
assert_eq!(removed.calendars + removed.calendar_events, 0);
|
||||
assert_eq!(count(&c, "calendars"), 1);
|
||||
assert_eq!(count(&c, "calendar_events"), 1);
|
||||
assert_eq!(count(&c, "file_nodes"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replacing_a_caldav_source_drops_its_calendars_and_events() {
|
||||
let c = mem();
|
||||
let (_, old) = source(&c, "caldav", "https://old");
|
||||
c.execute("INSERT INTO calendars (name) VALUES ('Work')", [])
|
||||
.unwrap();
|
||||
let cal_local = c.last_insert_rowid();
|
||||
dav_ids::insert(&c, old, CALENDAR, "/c/", "/c/", "", cal_local).unwrap();
|
||||
c.execute(
|
||||
"INSERT INTO calendar_events (calendar_ids, data) VALUES (?1, '{}')",
|
||||
params![format!("[{cal_local}]")],
|
||||
)
|
||||
.unwrap();
|
||||
let ev = c.last_insert_rowid();
|
||||
dav_ids::insert(&c, old, CALENDAR_EVENT, "/c/", "/c/1.ics", "", ev).unwrap();
|
||||
let (new_key, _) = source(&c, "caldav", "https://new");
|
||||
|
||||
let removed = remove_other_sources(&c, &new_key, Logger::from_flags(true, 0)).unwrap();
|
||||
|
||||
assert_eq!((removed.calendars, removed.calendar_events), (1, 1));
|
||||
assert_eq!(count(&c, "calendars"), 0);
|
||||
assert_eq!(count(&c, "calendar_events"), 0);
|
||||
assert_eq!(count(&c, "sync_id_dav"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_other_source_is_a_no_op() {
|
||||
let c = mem();
|
||||
let (key, _) = source(&c, "webdav", "https://only");
|
||||
let removed = remove_other_sources(&c, &key, Logger::from_flags(true, 0)).unwrap();
|
||||
assert_eq!(removed, Removed::default());
|
||||
assert_eq!(count(&c, "sources"), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn removed_counts_land_on_matching_summary_rows() {
|
||||
let mut summary = Summary::default();
|
||||
summary.per_type.push((FILE_NODE, TypeCounts::default()));
|
||||
Removed {
|
||||
file_nodes: 4,
|
||||
calendars: 9,
|
||||
..Removed::default()
|
||||
}
|
||||
.add_to(&mut summary);
|
||||
assert_eq!(summary.per_type.len(), 1);
|
||||
assert_eq!(summary.per_type[0].1.deleted, 4);
|
||||
}
|
||||
}
|
||||
@@ -395,7 +395,7 @@ fn run_into(
|
||||
let local_mailboxes = db::imap_ids::mailbox_folders(&conn, source_id)?;
|
||||
let server_folder_set: HashSet<String> = resolved.iter().map(|f| f.name.clone()).collect();
|
||||
|
||||
upsert_mailboxes(&mut conn, source_id, &resolved, &mut mailbox_counts)?;
|
||||
upsert_mailboxes(&mut conn, source_id, &resolved, &mut mailbox_counts, logger)?;
|
||||
|
||||
let server_delimiter = resolved.iter().find_map(|f| f.delimiter).unwrap_or('/');
|
||||
|
||||
@@ -436,7 +436,7 @@ fn run_into(
|
||||
)
|
||||
.map_err(|e| Error::Connection(format!("worker pool: {e}")))?;
|
||||
|
||||
for (i, folder) in resolved.iter().enumerate() {
|
||||
for (i, folder) in resolved.iter().filter(|f| f.selectable).enumerate() {
|
||||
if i > 0 {
|
||||
let _ = control_run_collect(&mut client, &control_ctx, "NOOP");
|
||||
}
|
||||
@@ -614,16 +614,29 @@ fn upsert_mailboxes(
|
||||
source_id: i64,
|
||||
folders: &[ResolvedFolder],
|
||||
counts: &mut TypeCounts,
|
||||
logger: Logger,
|
||||
) -> Result<(), Error> {
|
||||
let tx = conn.transaction()?;
|
||||
let mut local_ids: HashMap<String, i64> = HashMap::new();
|
||||
for folder in folders {
|
||||
let parent_local = match &folder.parent_path {
|
||||
Some(p) => local_ids.get(p.as_str()).copied().or_else(|| {
|
||||
db::imap_ids::local_for_mailbox(&tx, source_id, p)
|
||||
.ok()
|
||||
.flatten()
|
||||
}),
|
||||
Some(p) => {
|
||||
let found = match local_ids.get(p.as_str()) {
|
||||
Some(id) => Some(*id),
|
||||
None => db::imap_ids::local_for_mailbox(&tx, source_id, p)?,
|
||||
};
|
||||
if found.is_none() {
|
||||
log_at(
|
||||
logger,
|
||||
LEVEL_DEFAULT,
|
||||
&format!(
|
||||
"folder {:?}: parent {p:?} could not be resolved; storing it at the top level",
|
||||
folder.name
|
||||
),
|
||||
);
|
||||
}
|
||||
found
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let existing = db::imap_ids::local_for_mailbox(&tx, source_id, &folder.name)?;
|
||||
@@ -1290,7 +1303,7 @@ fn dry_run_summary(
|
||||
);
|
||||
}
|
||||
|
||||
for folder in folders {
|
||||
for folder in folders.iter().filter(|f| f.selectable) {
|
||||
let select_resp = match select_folder(client, control_ctx, folder) {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
|
||||
+200
-13
@@ -39,6 +39,7 @@ pub struct ResolvedFolder {
|
||||
pub delimiter: Option<char>,
|
||||
pub role: Option<&'static str>,
|
||||
pub subscribed: bool,
|
||||
pub selectable: bool,
|
||||
pub status: Option<FolderStatus>,
|
||||
}
|
||||
|
||||
@@ -131,16 +132,16 @@ pub fn apply_filters(
|
||||
folders: Vec<DiscoveredFolder>,
|
||||
filters: &FolderFilters,
|
||||
) -> Vec<ResolvedFolder> {
|
||||
let mut keep: Vec<DiscoveredFolder> = folders
|
||||
.into_iter()
|
||||
.filter(|f| f.selectable)
|
||||
.filter(|f| match_filters(&f.name, filters))
|
||||
.collect();
|
||||
if filters.subscribed_only {
|
||||
keep.retain(|f| f.subscribed);
|
||||
}
|
||||
let mut resolved: Vec<ResolvedFolder> = Vec::with_capacity(keep.len());
|
||||
for f in keep {
|
||||
let mut resolved: Vec<ResolvedFolder> = Vec::with_capacity(folders.len());
|
||||
let mut others: HashMap<String, DiscoveredFolder> = HashMap::new();
|
||||
for f in folders {
|
||||
if !f.selectable
|
||||
|| !match_filters(&f.name, filters)
|
||||
|| (filters.subscribed_only && !f.subscribed)
|
||||
{
|
||||
others.insert(f.name.clone(), f);
|
||||
continue;
|
||||
}
|
||||
let role = automap::role_for_folder(
|
||||
&f.name,
|
||||
&f.attributes,
|
||||
@@ -153,21 +154,77 @@ pub fn apply_filters(
|
||||
.iter()
|
||||
.any(|s| s.eq_ignore_ascii_case(r))
|
||||
{
|
||||
others.insert(f.name.clone(), f);
|
||||
continue;
|
||||
}
|
||||
let delim = f.delimiter;
|
||||
let (leaf, parent_path) = split_parent(&f.name, delim);
|
||||
let (leaf, parent_path) = split_parent(&f.name, f.delimiter);
|
||||
resolved.push(ResolvedFolder {
|
||||
name: f.name,
|
||||
wire_name: f.wire_name,
|
||||
leaf,
|
||||
parent_path,
|
||||
delimiter: delim,
|
||||
delimiter: f.delimiter,
|
||||
role,
|
||||
subscribed: f.subscribed,
|
||||
selectable: true,
|
||||
status: f.status,
|
||||
});
|
||||
}
|
||||
let mut present: HashSet<String> = resolved.iter().map(|f| f.name.clone()).collect();
|
||||
let mut structural: Vec<ResolvedFolder> = Vec::new();
|
||||
for child in &resolved {
|
||||
let mut parent = child.parent_path.clone();
|
||||
let mut child_wire = child.wire_name.clone();
|
||||
let mut delim = child.delimiter;
|
||||
while let Some(name) = parent {
|
||||
if present.contains(&name) {
|
||||
break;
|
||||
}
|
||||
let ancestor = match others.remove(&name) {
|
||||
Some(d) => {
|
||||
let delimiter = d.delimiter.or(delim);
|
||||
let (leaf, parent_path) = split_parent(&d.name, delimiter);
|
||||
ResolvedFolder {
|
||||
name: d.name,
|
||||
wire_name: d.wire_name,
|
||||
leaf,
|
||||
parent_path,
|
||||
delimiter,
|
||||
role: None,
|
||||
subscribed: d.subscribed,
|
||||
selectable: false,
|
||||
status: None,
|
||||
}
|
||||
}
|
||||
None => {
|
||||
let wire_name = match delim {
|
||||
Some(d) => child_wire
|
||||
.rsplit_once(d)
|
||||
.map_or_else(|| name.clone(), |(p, _)| p.to_owned()),
|
||||
None => name.clone(),
|
||||
};
|
||||
let (leaf, parent_path) = split_parent(&name, delim);
|
||||
ResolvedFolder {
|
||||
name,
|
||||
wire_name,
|
||||
leaf,
|
||||
parent_path,
|
||||
delimiter: delim,
|
||||
role: None,
|
||||
subscribed: false,
|
||||
selectable: false,
|
||||
status: None,
|
||||
}
|
||||
}
|
||||
};
|
||||
present.insert(ancestor.name.clone());
|
||||
parent = ancestor.parent_path.clone();
|
||||
child_wire.clone_from(&ancestor.wire_name);
|
||||
delim = ancestor.delimiter;
|
||||
structural.push(ancestor);
|
||||
}
|
||||
}
|
||||
resolved.extend(structural);
|
||||
resolved
|
||||
}
|
||||
|
||||
@@ -447,6 +504,7 @@ mod tests {
|
||||
delimiter: Some('/'),
|
||||
role: None,
|
||||
subscribed: false,
|
||||
selectable: true,
|
||||
status: None,
|
||||
},
|
||||
ResolvedFolder {
|
||||
@@ -457,6 +515,7 @@ mod tests {
|
||||
delimiter: Some('/'),
|
||||
role: Some("inbox"),
|
||||
subscribed: true,
|
||||
selectable: true,
|
||||
status: None,
|
||||
},
|
||||
ResolvedFolder {
|
||||
@@ -467,6 +526,7 @@ mod tests {
|
||||
delimiter: Some('/'),
|
||||
role: None,
|
||||
subscribed: false,
|
||||
selectable: true,
|
||||
status: None,
|
||||
},
|
||||
];
|
||||
@@ -508,4 +568,131 @@ mod tests {
|
||||
let folders = collect_from_list(&resps, false).unwrap();
|
||||
assert_eq!(folders.len(), 1);
|
||||
}
|
||||
|
||||
fn find<'a>(res: &'a [ResolvedFolder], name: &str) -> &'a ResolvedFolder {
|
||||
res.iter().find(|f| f.name == name).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn noselect_parent_is_kept_as_structural() {
|
||||
let folders = collect_from_list(
|
||||
&[
|
||||
lst("INBOX", "/", &["\\HasChildren"]),
|
||||
lst("INBOX/Projects", "/", &["\\Noselect", "\\HasChildren"]),
|
||||
lst("INBOX/Projects/Invoices", "/", &["\\HasNoChildren"]),
|
||||
],
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
let mut res = apply_filters(folders, &filters_default());
|
||||
sort_by_depth(&mut res);
|
||||
let names: Vec<&str> = res.iter().map(|f| f.name.as_str()).collect();
|
||||
assert_eq!(
|
||||
names,
|
||||
["INBOX", "INBOX/Projects", "INBOX/Projects/Invoices"]
|
||||
);
|
||||
let projects = find(&res, "INBOX/Projects");
|
||||
assert!(!projects.selectable);
|
||||
assert_eq!(projects.role, None);
|
||||
assert_eq!(projects.leaf, "Projects");
|
||||
assert_eq!(projects.parent_path.as_deref(), Some("INBOX"));
|
||||
assert!(find(&res, "INBOX/Projects/Invoices").selectable);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn noselect_without_kept_children_is_dropped() {
|
||||
let folders = collect_from_list(
|
||||
&[lst("INBOX", "/", &[]), lst("Empty", "/", &["\\Noselect"])],
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
let res = apply_filters(folders, &filters_default());
|
||||
assert_eq!(res.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consecutive_structural_ancestors_and_synthesised_ones() {
|
||||
let folders = collect_from_list(
|
||||
&[lst("A", "/", &["\\NonExistent"]), lst("A/B/C/D", "/", &[])],
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
let mut res = apply_filters(folders, &filters_default());
|
||||
sort_by_depth(&mut res);
|
||||
let names: Vec<&str> = res.iter().map(|f| f.name.as_str()).collect();
|
||||
assert_eq!(names, ["A", "A/B", "A/B/C", "A/B/C/D"]);
|
||||
assert!(
|
||||
res.iter()
|
||||
.take(3)
|
||||
.all(|f| !f.selectable && f.role.is_none())
|
||||
);
|
||||
assert_eq!(find(&res, "A/B/C").wire_name, "A/B/C");
|
||||
assert_eq!(find(&res, "A/B").leaf, "B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trailing_space_and_same_leaf_under_different_parents() {
|
||||
let folders = collect_from_list(
|
||||
&[
|
||||
lst("Work ", ".", &["\\Noselect"]),
|
||||
lst("Work", ".", &[]),
|
||||
lst("Work .Archive", ".", &[]),
|
||||
lst("Work.Archive", ".", &[]),
|
||||
],
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
let res = apply_filters(folders, &filters_default());
|
||||
assert_eq!(res.len(), 4);
|
||||
let spaced = find(&res, "Work ");
|
||||
assert!(!spaced.selectable);
|
||||
assert_eq!(spaced.leaf, "Work ");
|
||||
assert_eq!(
|
||||
find(&res, "Work .Archive").parent_path.as_deref(),
|
||||
Some("Work ")
|
||||
);
|
||||
assert_eq!(
|
||||
find(&res, "Work.Archive").parent_path.as_deref(),
|
||||
Some("Work")
|
||||
);
|
||||
assert!(find(&res, "Work").selectable);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn synthesised_ancestor_derives_wire_name_from_child() {
|
||||
let folders = collect_from_list(&[lst("&ZeVnLIqe-/Sub", "/", &[])], false).unwrap();
|
||||
let res = apply_filters(folders, &filters_default());
|
||||
let parent = find(&res, "日本語");
|
||||
assert_eq!(parent.wire_name, "&ZeVnLIqe-");
|
||||
assert!(!parent.selectable);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filters_keep_needed_ancestors_as_structural() {
|
||||
let list = [
|
||||
lst("INBOX", "/", &[]),
|
||||
lst("Projects", "/", &[]),
|
||||
lst("Projects/Alpha", "/", &["\\Noselect"]),
|
||||
lst("Projects/Alpha/Docs", "/", &["\\Subscribed"]),
|
||||
lst("Other", "/", &[]),
|
||||
];
|
||||
let check = |res: Vec<ResolvedFolder>| {
|
||||
assert_eq!(res.len(), 3, "{res:?}");
|
||||
assert!(!find(&res, "Projects").selectable);
|
||||
assert!(!find(&res, "Projects/Alpha").selectable);
|
||||
assert!(find(&res, "Projects/Alpha/Docs").selectable);
|
||||
};
|
||||
let mut f = filters_default();
|
||||
f.include = vec![Regex::new("Docs$").unwrap()];
|
||||
check(apply_filters(collect_from_list(&list, false).unwrap(), &f));
|
||||
let mut f = filters_default();
|
||||
f.exclude = vec![Regex::new("^(INBOX|Other|Projects)$").unwrap()];
|
||||
check(apply_filters(collect_from_list(&list, false).unwrap(), &f));
|
||||
let mut f = filters_default();
|
||||
f.explicit = vec!["Projects/Alpha/Docs".to_owned()];
|
||||
check(apply_filters(collect_from_list(&list, false).unwrap(), &f));
|
||||
let mut f = filters_default();
|
||||
f.subscribed_only = true;
|
||||
check(apply_filters(collect_from_list(&list, false).unwrap(), &f));
|
||||
}
|
||||
}
|
||||
+135
-56
@@ -1,5 +1,6 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC <legal@coffeylabs.org>
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||
*/
|
||||
@@ -26,14 +27,19 @@ use crate::jmap::blobxfer;
|
||||
use crate::jmap::connect::{self, Connected};
|
||||
use crate::jmap::error::JmapError;
|
||||
use crate::jmap::http::{Auth, HttpClient};
|
||||
use crate::jmap::request::{get_all, get_changes, get_objects, get_state, query_all_ids};
|
||||
use crate::jmap::request::{
|
||||
URN_VACATION_RESPONSE, get_all, get_changes, get_objects, get_state, query_all_ids,
|
||||
};
|
||||
use crate::jmap::session::{Limits, Session};
|
||||
use crate::jmap::wire::JmapId;
|
||||
use crate::jmap::wire::common::parse_utc_date;
|
||||
use crate::jmap::wire::email::Email;
|
||||
use crate::jmap::wire::file_node::{FileNode, NodeType};
|
||||
use crate::jmap::wire::file_node::{FileNode, FileProps, NodeKind};
|
||||
use crate::jmap::wire::sieve_script::SieveScript;
|
||||
use crate::jmap::wire::vacation_response::{self, VacationResponse};
|
||||
use crate::logging::{LEVEL_DEFAULT, LEVEL_PROGRESS, Logger};
|
||||
use crate::sync::emailmeta::email_meta_from_blob;
|
||||
use crate::sync::keys::index_to_json;
|
||||
use crate::sync::{CommonConfig, Context, ImportConfig, RunOutcome, Summary, TypeCounts};
|
||||
use crate::types::ObjectType;
|
||||
|
||||
@@ -65,13 +71,18 @@ fn get_props(ty: ObjectType) -> Option<&'static [&'static str]> {
|
||||
ObjectType::Mailbox => Some(&["name", "parentId", "role", "sortOrder", "isSubscribed"]),
|
||||
ObjectType::Email => Some(&["blobId", "receivedAt", "mailboxIds", "keywords"]),
|
||||
ObjectType::SieveScript => Some(&["name", "isActive", "blobId"]),
|
||||
// Both shapes' names (wire::file_node::Shape): each server ignores
|
||||
// the other's.
|
||||
ObjectType::FileNode => Some(&[
|
||||
"parentId",
|
||||
"@type",
|
||||
"nodeType",
|
||||
"parentId",
|
||||
"blobId",
|
||||
"target",
|
||||
"name",
|
||||
"mediaType",
|
||||
"type",
|
||||
"executable",
|
||||
"created",
|
||||
"modified",
|
||||
"isSubscribed",
|
||||
@@ -107,17 +118,33 @@ impl LocalResolver for DbResolver<'_> {
|
||||
}
|
||||
}
|
||||
|
||||
struct PrefetchedBlobs<'a> {
|
||||
conn: &'a Connection,
|
||||
bytes: &'a HashMap<String, Vec<u8>>,
|
||||
struct EmailMeta {
|
||||
match_key: String,
|
||||
date_header: Option<String>,
|
||||
}
|
||||
|
||||
impl BlobIntern for PrefetchedBlobs<'_> {
|
||||
struct StoredBlob {
|
||||
local_id: i64,
|
||||
email: Option<EmailMeta>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct StoredBlobs(HashMap<String, StoredBlob>);
|
||||
|
||||
impl StoredBlobs {
|
||||
fn get(&self, jmap_blob_id: &str, kind: &str) -> Result<&StoredBlob, JmapError> {
|
||||
self.0
|
||||
.get(jmap_blob_id)
|
||||
.ok_or_else(|| JmapError::malformed(format!("{kind} blob missing")))
|
||||
}
|
||||
}
|
||||
|
||||
impl BlobIntern for &StoredBlobs {
|
||||
fn intern(&mut self, jmap_blob_id: &str) -> Result<i64, JmapError> {
|
||||
let data = self.bytes.get(jmap_blob_id).ok_or_else(|| {
|
||||
JmapError::malformed(format!("blob {jmap_blob_id} was not prefetched"))
|
||||
})?;
|
||||
Ok(db::blobs::intern_blob(self.conn, data)?)
|
||||
self.0
|
||||
.get(jmap_blob_id)
|
||||
.map(|b| b.local_id)
|
||||
.ok_or_else(|| JmapError::malformed(format!("blob {jmap_blob_id} was not prefetched")))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -242,6 +269,34 @@ fn run_into(
|
||||
summary.per_type.push((ty.jmap_name(), counts));
|
||||
}
|
||||
|
||||
if let Some(source_id) = source_id
|
||||
&& !ctx.dry_run()
|
||||
&& config
|
||||
.objects
|
||||
.as_ref()
|
||||
.is_none_or(|s| s.contains(&ObjectType::SieveScript))
|
||||
&& net.session.may_support(&net.account, URN_VACATION_RESPONSE)
|
||||
{
|
||||
if logger.enabled(LEVEL_DEFAULT) {
|
||||
eprintln!("import: {} ...", vacation_response::TYPE_NAME);
|
||||
}
|
||||
let mut counts = TypeCounts::default();
|
||||
match import_vacation_response(&ctx.conn, &net, source_id) {
|
||||
Ok(fetched) => counts.fetched = fetched,
|
||||
Err(e) if e.aborts_run() => return Err(e),
|
||||
Err(e) => {
|
||||
logger.warn(&format!(
|
||||
"type {} aborted: {e}; continuing (run will exit 5)",
|
||||
vacation_response::TYPE_NAME
|
||||
));
|
||||
counts.failed += 1;
|
||||
}
|
||||
}
|
||||
summary
|
||||
.per_type
|
||||
.push((vacation_response::TYPE_NAME, counts));
|
||||
}
|
||||
|
||||
if ctx.dry_run() {
|
||||
print_dry_run(&dry_rows);
|
||||
*summary = Summary::default();
|
||||
@@ -259,6 +314,28 @@ fn run_into(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn import_vacation_response(conn: &Connection, net: &Net, source_id: i64) -> Result<u64, Error> {
|
||||
let got = get_all::<VacationResponse>(
|
||||
&net.client,
|
||||
&net.api,
|
||||
&net.account,
|
||||
vacation_response::TYPE_NAME,
|
||||
)
|
||||
.map_err(Error::from)?;
|
||||
match got.list.into_iter().find(|v| !v.is_unset()) {
|
||||
Some(vacation) => {
|
||||
db::vacation_response::store(conn, source_id, &vacation)
|
||||
.map_err(|e| Error::Partial(e.to_string()))?;
|
||||
Ok(1)
|
||||
}
|
||||
None => {
|
||||
db::vacation_response::clear(conn, source_id)
|
||||
.map_err(|e| Error::Partial(e.to_string()))?;
|
||||
Ok(0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn work_list(config: &ImportConfig, connected: &Connected) -> Vec<ObjectType> {
|
||||
let selected = config.objects.as_ref();
|
||||
IMPORT_ORDER
|
||||
@@ -462,9 +539,9 @@ fn insert_objects(
|
||||
for batch in ordered.chunks(200) {
|
||||
let blob_refs = blob_references(ty, batch.iter().map(|&i| &objects[i]));
|
||||
let blobs = if blob_refs.is_empty() {
|
||||
HashMap::new()
|
||||
StoredBlobs::default()
|
||||
} else {
|
||||
download_blobs(net, blob_refs, threads, logger)
|
||||
download_blobs(&ctx.conn, net, ty, blob_refs, threads, logger)?
|
||||
};
|
||||
let tx = ctx
|
||||
.conn
|
||||
@@ -532,7 +609,8 @@ fn blob_references<'a>(ty: ObjectType, objects: impl Iterator<Item = &'a Value>)
|
||||
&& seen.insert(b.to_owned())
|
||||
{
|
||||
let mt = o
|
||||
.get("type")
|
||||
.get("mediaType")
|
||||
.or_else(|| o.get("type"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("application/octet-stream")
|
||||
.to_owned();
|
||||
@@ -562,11 +640,13 @@ fn blob_references<'a>(ty: ObjectType, objects: impl Iterator<Item = &'a Value>)
|
||||
}
|
||||
|
||||
fn download_blobs(
|
||||
conn: &Connection,
|
||||
net: &Net,
|
||||
ty: ObjectType,
|
||||
refs: Vec<BlobRef>,
|
||||
threads: usize,
|
||||
logger: &Logger,
|
||||
) -> HashMap<String, Vec<u8>> {
|
||||
) -> Result<StoredBlobs, Error> {
|
||||
let workers = effective_workers(threads, &net.limits, true);
|
||||
let net_arc = Arc::new(net.clone());
|
||||
let pool: Pool<BlobRef, DlMsg> = Pool::new(workers, {
|
||||
@@ -587,11 +667,23 @@ fn download_blobs(
|
||||
for r in refs {
|
||||
pool.submit(r);
|
||||
}
|
||||
let mut out = HashMap::new();
|
||||
for (blob_id, res) in pool.finish() {
|
||||
let mut out = HashMap::with_capacity(total);
|
||||
for _ in 0..total {
|
||||
let Ok((blob_id, res)) = pool.results().recv() else {
|
||||
break;
|
||||
};
|
||||
match res {
|
||||
Ok(bytes) => {
|
||||
out.insert(blob_id, bytes);
|
||||
let local_id = db::blobs::intern_blob(conn, &bytes)
|
||||
.map_err(|e| Error::Partial(e.to_string()))?;
|
||||
let email = (ty == ObjectType::Email).then(|| {
|
||||
let (index, date_header) = email_meta_from_blob(&bytes);
|
||||
EmailMeta {
|
||||
match_key: index_to_json(&index),
|
||||
date_header,
|
||||
}
|
||||
});
|
||||
out.insert(blob_id, StoredBlob { local_id, email });
|
||||
}
|
||||
Err(e) => {
|
||||
logger.warn(&format!("blob {blob_id} download failed: {e}"));
|
||||
@@ -601,7 +693,7 @@ fn download_blobs(
|
||||
if logger.enabled(LEVEL_PROGRESS) {
|
||||
eprintln!("import: downloaded {}/{} blobs", out.len(), total);
|
||||
}
|
||||
out
|
||||
Ok(StoredBlobs(out))
|
||||
}
|
||||
|
||||
fn order_objects(
|
||||
@@ -649,7 +741,7 @@ fn insert_one(
|
||||
ty: ObjectType,
|
||||
source_id: i64,
|
||||
obj: &Value,
|
||||
blobs: &HashMap<String, Vec<u8>>,
|
||||
blobs: &StoredBlobs,
|
||||
logger: &Logger,
|
||||
) -> Result<i64, JmapError> {
|
||||
let resolver = DbResolver { conn, source_id };
|
||||
@@ -676,14 +768,13 @@ fn insert_one(
|
||||
}
|
||||
ObjectType::Email => {
|
||||
let mut w: Email = serde_json::from_value(obj.clone())?;
|
||||
let data = blobs
|
||||
.get(&w.blob_id.0)
|
||||
let stored = blobs.get(&w.blob_id.0, "email")?;
|
||||
let meta = stored
|
||||
.email
|
||||
.as_ref()
|
||||
.ok_or_else(|| JmapError::malformed("email blob missing"))?;
|
||||
let blob_local = db::blobs::intern_blob(conn, data)?;
|
||||
let (index, date_header) = crate::sync::emailmeta::email_meta_from_blob(data);
|
||||
let mm = crate::sync::keys::index_to_json(&index);
|
||||
if w.received_at.is_none() {
|
||||
w.received_at = date_header.as_deref().and_then(parse_utc_date);
|
||||
w.received_at = meta.date_header.as_deref().and_then(parse_utc_date);
|
||||
let id = obj.get("id").and_then(Value::as_str).unwrap_or("?");
|
||||
let raw = obj
|
||||
.get("receivedAt")
|
||||
@@ -697,37 +788,31 @@ fn insert_one(
|
||||
}
|
||||
));
|
||||
}
|
||||
mapping::insert_email(conn, &w, blob_local, &mm, &resolver)
|
||||
mapping::insert_email(conn, &w, stored.local_id, &meta.match_key, &resolver)
|
||||
}
|
||||
ObjectType::SieveScript => {
|
||||
let w: SieveScript = serde_json::from_value(obj.clone())?;
|
||||
let data = blobs
|
||||
.get(&w.blob_id.0)
|
||||
.ok_or_else(|| JmapError::malformed("sieve blob missing"))?;
|
||||
let blob_local = db::blobs::intern_blob(conn, data)?;
|
||||
let blob_local = blobs.get(&w.blob_id.0, "sieve")?.local_id;
|
||||
mapping::insert_sieve_script(conn, &w, blob_local)
|
||||
}
|
||||
ObjectType::FileNode => {
|
||||
let w: FileNode = serde_json::from_value(obj.clone())?;
|
||||
let blob_local = match (w.effective_node_type(), &w.blob_id) {
|
||||
(NodeType::File, Some(b)) => {
|
||||
let data = blobs
|
||||
.get(&b.0)
|
||||
.ok_or_else(|| JmapError::malformed("file blob missing"))?;
|
||||
Some(db::blobs::intern_blob(conn, data)?)
|
||||
}
|
||||
let blob_local = match &w.kind {
|
||||
NodeKind::File(FileProps {
|
||||
blob_id: Some(b), ..
|
||||
}) => Some(blobs.get(&b.0, "file")?.local_id),
|
||||
_ => None,
|
||||
};
|
||||
mapping::insert_file_node(conn, &w, blob_local, &resolver)
|
||||
}
|
||||
ObjectType::ContactCard => {
|
||||
let w = serde_json::from_value(obj.clone())?;
|
||||
let mut bi = PrefetchedBlobs { conn, bytes: blobs };
|
||||
let mut bi = blobs;
|
||||
mapping::insert_contact_card(conn, &w, &resolver, &mut bi)
|
||||
}
|
||||
ObjectType::CalendarEvent => {
|
||||
let w = serde_json::from_value(obj.clone())?;
|
||||
let mut bi = PrefetchedBlobs { conn, bytes: blobs };
|
||||
let mut bi = blobs;
|
||||
mapping::insert_calendar_event(conn, &w, &resolver, &mut bi)
|
||||
}
|
||||
}
|
||||
@@ -876,9 +961,9 @@ fn update_objects(
|
||||
for batch in objects.chunks(200) {
|
||||
let blob_refs = blob_references(ty, batch.iter());
|
||||
let blobs = if blob_refs.is_empty() {
|
||||
HashMap::new()
|
||||
StoredBlobs::default()
|
||||
} else {
|
||||
download_blobs(net, blob_refs, threads, logger)
|
||||
download_blobs(&ctx.conn, net, ty, blob_refs, threads, logger)?
|
||||
};
|
||||
let tx = ctx
|
||||
.conn
|
||||
@@ -915,7 +1000,7 @@ fn update_one(
|
||||
source_id: i64,
|
||||
local_id: i64,
|
||||
obj: &Value,
|
||||
blobs: &HashMap<String, Vec<u8>>,
|
||||
blobs: &StoredBlobs,
|
||||
) -> Result<bool, JmapError> {
|
||||
let resolver = DbResolver { conn, source_id };
|
||||
match ty {
|
||||
@@ -942,33 +1027,27 @@ fn update_one(
|
||||
ObjectType::Email => mapping::update_email(conn, local_id, obj, &resolver),
|
||||
ObjectType::SieveScript => {
|
||||
let w: SieveScript = serde_json::from_value(obj.clone())?;
|
||||
let data = blobs
|
||||
.get(&w.blob_id.0)
|
||||
.ok_or_else(|| JmapError::malformed("sieve blob missing"))?;
|
||||
let blob_local = db::blobs::intern_blob(conn, data)?;
|
||||
let blob_local = blobs.get(&w.blob_id.0, "sieve")?.local_id;
|
||||
mapping::update_sieve_script(conn, local_id, &w, blob_local)
|
||||
}
|
||||
ObjectType::FileNode => {
|
||||
let w: FileNode = serde_json::from_value(obj.clone())?;
|
||||
let blob_local = match (w.effective_node_type(), &w.blob_id) {
|
||||
(NodeType::File, Some(b)) => {
|
||||
let data = blobs
|
||||
.get(&b.0)
|
||||
.ok_or_else(|| JmapError::malformed("file blob missing"))?;
|
||||
Some(db::blobs::intern_blob(conn, data)?)
|
||||
}
|
||||
let blob_local = match &w.kind {
|
||||
NodeKind::File(FileProps {
|
||||
blob_id: Some(b), ..
|
||||
}) => Some(blobs.get(&b.0, "file")?.local_id),
|
||||
_ => None,
|
||||
};
|
||||
mapping::update_file_node(conn, local_id, &w, blob_local, &resolver)
|
||||
}
|
||||
ObjectType::ContactCard => {
|
||||
let w = serde_json::from_value(obj.clone())?;
|
||||
let mut bi = PrefetchedBlobs { conn, bytes: blobs };
|
||||
let mut bi = blobs;
|
||||
mapping::update_contact_card(conn, local_id, &w, &resolver, &mut bi)
|
||||
}
|
||||
ObjectType::CalendarEvent => {
|
||||
let w = serde_json::from_value(obj.clone())?;
|
||||
let mut bi = PrefetchedBlobs { conn, bytes: blobs };
|
||||
let mut bi = blobs;
|
||||
mapping::update_calendar_event(conn, local_id, &w, &resolver, &mut bi)
|
||||
}
|
||||
}
|
||||
|
||||
+155
-56
@@ -17,7 +17,7 @@ use crate::jmap::wire::calendar::Calendar;
|
||||
use crate::jmap::wire::calendar_event::CalendarEvent;
|
||||
use crate::jmap::wire::contact_card::ContactCard;
|
||||
use crate::jmap::wire::email::Email;
|
||||
use crate::jmap::wire::file_node::FileNode;
|
||||
use crate::jmap::wire::file_node::{DirectoryProps, FileNode, FileProps, NodeKind, SymLinkProps};
|
||||
use crate::jmap::wire::identity::Identity;
|
||||
use crate::jmap::wire::mailbox::Mailbox;
|
||||
use crate::jmap::wire::participant_identity::ParticipantIdentity;
|
||||
@@ -318,36 +318,62 @@ pub fn insert_file_node(
|
||||
resolver: &impl LocalResolver,
|
||||
) -> Result<i64, JmapError> {
|
||||
let parent = opt_parent(resolver, ObjectType::FileNode, &wire.parent_id);
|
||||
let node_type = serde_json::to_value(wire.effective_node_type())?
|
||||
.as_str()
|
||||
.unwrap_or("file")
|
||||
.to_owned();
|
||||
let target = match &wire.target {
|
||||
Some(t) => Some(serde_json::to_string(t)?),
|
||||
None => None,
|
||||
};
|
||||
let cols = FileNodeColumns::new(&wire.kind)?;
|
||||
conn.execute(
|
||||
"INSERT INTO file_nodes (parent_id, node_type, blob_id, target, name, media_type,
|
||||
created, modified, is_subscribed, role)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
|
||||
created, modified, is_subscribed, role, executable)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)",
|
||||
params![
|
||||
parent,
|
||||
node_type,
|
||||
wire.kind.db_name(),
|
||||
blob_local_id,
|
||||
target,
|
||||
cols.target,
|
||||
wire.name,
|
||||
wire.media_type,
|
||||
cols.media_type,
|
||||
format_utc(&wire.created)?,
|
||||
opt_format_utc(&wire.modified)?,
|
||||
wire.is_subscribed as i64,
|
||||
wire.role
|
||||
cols.is_subscribed as i64,
|
||||
cols.role,
|
||||
cols.executable as i64
|
||||
],
|
||||
)?;
|
||||
Ok(conn.last_insert_rowid())
|
||||
}
|
||||
|
||||
struct FileNodeColumns<'a> {
|
||||
target: Option<String>,
|
||||
media_type: Option<&'a str>,
|
||||
is_subscribed: bool,
|
||||
role: Option<&'a str>,
|
||||
executable: bool,
|
||||
}
|
||||
|
||||
impl<'a> FileNodeColumns<'a> {
|
||||
fn new(kind: &'a NodeKind) -> Result<Self, JmapError> {
|
||||
let mut cols = FileNodeColumns {
|
||||
target: None,
|
||||
media_type: None,
|
||||
is_subscribed: true,
|
||||
role: None,
|
||||
executable: false,
|
||||
};
|
||||
match kind {
|
||||
NodeKind::File(f) => {
|
||||
cols.media_type = f.media_type.as_deref();
|
||||
cols.executable = f.executable;
|
||||
}
|
||||
NodeKind::Directory(d) => {
|
||||
cols.role = d.role.as_deref();
|
||||
cols.is_subscribed = d.is_subscribed;
|
||||
}
|
||||
NodeKind::SymLink(l) => cols.target = Some(serde_json::to_string(&l.target)?),
|
||||
}
|
||||
Ok(cols)
|
||||
}
|
||||
}
|
||||
|
||||
pub const FILE_NODE_SELECT: &str = "SELECT id, parent_id, node_type, blob_id, target, name, \
|
||||
media_type, created, modified, is_subscribed, role FROM file_nodes";
|
||||
media_type, created, modified, is_subscribed, role, executable FROM file_nodes";
|
||||
|
||||
pub fn insert_contact_card(
|
||||
conn: &Connection,
|
||||
@@ -587,31 +613,27 @@ pub fn update_file_node(
|
||||
resolver: &impl LocalResolver,
|
||||
) -> Result<bool, JmapError> {
|
||||
let parent = opt_parent(resolver, ObjectType::FileNode, &wire.parent_id);
|
||||
let node_type = serde_json::to_value(wire.effective_node_type())?
|
||||
.as_str()
|
||||
.unwrap_or("file")
|
||||
.to_owned();
|
||||
let target = match &wire.target {
|
||||
Some(t) => Some(serde_json::to_string(t)?),
|
||||
None => None,
|
||||
};
|
||||
let cols = FileNodeColumns::new(&wire.kind)?;
|
||||
let n = conn.execute(
|
||||
"UPDATE file_nodes SET parent_id = ?1, node_type = ?2, blob_id = ?3, target = ?4,
|
||||
name = ?5, media_type = ?6, created = ?7, modified = ?8, is_subscribed = ?9, role = ?10
|
||||
WHERE id = ?11 AND (parent_id IS NOT ?1 OR node_type IS NOT ?2 OR blob_id IS NOT ?3
|
||||
name = ?5, media_type = ?6, created = ?7, modified = ?8, is_subscribed = ?9,
|
||||
role = ?10, executable = ?11
|
||||
WHERE id = ?12 AND (parent_id IS NOT ?1 OR node_type IS NOT ?2 OR blob_id IS NOT ?3
|
||||
OR target IS NOT ?4 OR name IS NOT ?5 OR media_type IS NOT ?6 OR created IS NOT ?7
|
||||
OR modified IS NOT ?8 OR is_subscribed IS NOT ?9 OR role IS NOT ?10)",
|
||||
OR modified IS NOT ?8 OR is_subscribed IS NOT ?9 OR role IS NOT ?10
|
||||
OR executable IS NOT ?11)",
|
||||
params![
|
||||
parent,
|
||||
node_type,
|
||||
wire.kind.db_name(),
|
||||
blob_local_id,
|
||||
target,
|
||||
cols.target,
|
||||
wire.name,
|
||||
wire.media_type,
|
||||
cols.media_type,
|
||||
format_utc(&wire.created)?,
|
||||
opt_format_utc(&wire.modified)?,
|
||||
wire.is_subscribed as i64,
|
||||
wire.role,
|
||||
cols.is_subscribed as i64,
|
||||
cols.role,
|
||||
cols.executable as i64,
|
||||
local_id
|
||||
],
|
||||
)?;
|
||||
@@ -845,26 +867,36 @@ pub fn row_to_file_node(
|
||||
),
|
||||
None => None,
|
||||
};
|
||||
let node_type = Some(serde_json::from_value(Value::from(
|
||||
row.get::<_, String>(2)?,
|
||||
))?);
|
||||
let target: Option<Vec<String>> = from_opt_json(row.get::<_, Option<String>>(4)?)?;
|
||||
let blob_local_id: Option<i64> = row.get(3)?;
|
||||
let node_type: String = row.get(2)?;
|
||||
let kind = match node_type.as_str() {
|
||||
"file" => NodeKind::File(FileProps {
|
||||
blob_id: None,
|
||||
media_type: row.get(6)?,
|
||||
executable: row.get::<_, i64>(11)? != 0,
|
||||
}),
|
||||
"directory" => NodeKind::Directory(DirectoryProps {
|
||||
role: row.get(10)?,
|
||||
is_subscribed: row.get::<_, i64>(9)? != 0,
|
||||
}),
|
||||
"symlink" => NodeKind::SymLink(SymLinkProps {
|
||||
target: from_opt_json(row.get::<_, Option<String>>(4)?)?.unwrap_or_default(),
|
||||
}),
|
||||
other => {
|
||||
return Err(JmapError::malformed(format!(
|
||||
"unknown file_node type {other}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
Ok(FileNodeRow {
|
||||
wire: FileNode {
|
||||
id: None,
|
||||
parent_id,
|
||||
node_type,
|
||||
blob_id: None,
|
||||
target,
|
||||
name: row.get(5)?,
|
||||
media_type: row.get(6)?,
|
||||
created: parse_utc(&row.get::<_, String>(7)?)?,
|
||||
modified: opt_parse_utc(row.get::<_, Option<String>>(8)?)?,
|
||||
is_subscribed: row.get::<_, i64>(9)? != 0,
|
||||
role: row.get(10)?,
|
||||
kind,
|
||||
},
|
||||
blob_local_id,
|
||||
blob_local_id: row.get(3)?,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1301,7 +1333,7 @@ mod tests {
|
||||
};
|
||||
let node: FileNode = serde_json::from_value(serde_json::json!({
|
||||
"id": "F1",
|
||||
"nodeType": "symlink",
|
||||
"@type": "SymLink",
|
||||
"target": ["..", "etc", "hosts"],
|
||||
"name": "link",
|
||||
"created": "2022-02-02T02:02:02Z"
|
||||
@@ -1410,7 +1442,7 @@ mod tests {
|
||||
to_target: HashMap::new(),
|
||||
};
|
||||
let dir: FileNode = serde_json::from_value(serde_json::json!({
|
||||
"id": "D1", "nodeType": "directory", "name": "Docs",
|
||||
"id": "D1", "@type": "Directory", "name": "Docs",
|
||||
"created": "2022-02-02T02:02:02Z"
|
||||
}))
|
||||
.unwrap();
|
||||
@@ -1418,8 +1450,8 @@ mod tests {
|
||||
res.to_local
|
||||
.insert((ObjectType::FileNode, "D1".to_owned()), dirl);
|
||||
let file: FileNode = serde_json::from_value(serde_json::json!({
|
||||
"id": "F1", "nodeType": "file", "parentId": "D1", "name": "a.bin",
|
||||
"type": "application/octet-stream", "created": "2022-02-02T02:02:02Z"
|
||||
"id": "F1", "@type": "File", "parentId": "D1", "name": "a.bin",
|
||||
"mediaType": "application/octet-stream", "created": "2022-02-02T02:02:02Z"
|
||||
}))
|
||||
.unwrap();
|
||||
let blob = crate::db::blobs::intern_blob(&c, b"bytes").unwrap();
|
||||
@@ -1439,6 +1471,72 @@ mod tests {
|
||||
assert_eq!(got.blob_local_id, Some(blob));
|
||||
}
|
||||
|
||||
// Each @type serializes only the properties the draft defines for it.
|
||||
#[test]
|
||||
fn file_node_row_to_wire_emits_only_type_specific_properties() {
|
||||
let c = mem();
|
||||
let mut res = MapResolver {
|
||||
to_local: HashMap::new(),
|
||||
to_target: HashMap::new(),
|
||||
};
|
||||
let nodes = [
|
||||
serde_json::json!({"id": "D1", "@type": "Directory", "name": "Docs",
|
||||
"role": "documents", "isSubscribed": false,
|
||||
"created": "2022-02-02T02:02:02Z"}),
|
||||
serde_json::json!({"id": "F1", "@type": "File", "name": "run.sh",
|
||||
"blobId": "B1", "mediaType": "text/x-sh", "executable": true, "size": 3,
|
||||
"created": "2022-02-02T02:02:02Z"}),
|
||||
serde_json::json!({"id": "L1", "@type": "SymLink", "name": "latest",
|
||||
"target": ["", "Docs"], "created": "2022-02-02T02:02:02Z"}),
|
||||
];
|
||||
let blob = crate::db::blobs::intern_blob(&c, b"#!\n").unwrap();
|
||||
let mut out = Vec::new();
|
||||
for (i, n) in nodes.into_iter().enumerate() {
|
||||
let w: FileNode = serde_json::from_value(n).unwrap();
|
||||
let blob_local = matches!(w.kind, NodeKind::File(_)).then_some(blob);
|
||||
let l = insert_file_node(&c, &w, blob_local, &res).unwrap();
|
||||
res.to_target
|
||||
.insert((ObjectType::FileNode, l), format!("T{i}"));
|
||||
let row = c
|
||||
.query_row(
|
||||
&format!("{FILE_NODE_SELECT} WHERE id = ?1"),
|
||||
params![l],
|
||||
|r| Ok(row_to_file_node(r, &res)),
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
out.push(serde_json::to_value(&row.wire).unwrap());
|
||||
}
|
||||
let keys = |v: &Value| {
|
||||
let mut k: Vec<String> = v.as_object().unwrap().keys().cloned().collect();
|
||||
k.sort_unstable();
|
||||
k
|
||||
};
|
||||
assert_eq!(
|
||||
keys(&out[0]),
|
||||
["@type", "created", "isSubscribed", "name", "role"]
|
||||
);
|
||||
assert_eq!(out[0]["@type"], "Directory");
|
||||
assert_eq!(out[0]["isSubscribed"], false);
|
||||
assert_eq!(
|
||||
keys(&out[1]),
|
||||
["@type", "created", "executable", "mediaType", "name"]
|
||||
);
|
||||
assert_eq!(out[1]["executable"], true);
|
||||
assert_eq!(out[1]["mediaType"], "text/x-sh");
|
||||
assert_eq!(keys(&out[2]), ["@type", "created", "name", "target"]);
|
||||
assert_eq!(out[2]["@type"], "SymLink");
|
||||
assert_eq!(out[2]["target"], serde_json::json!(["", "Docs"]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_node_rejects_unknown_type() {
|
||||
let r: Result<FileNode, _> = serde_json::from_value(serde_json::json!({
|
||||
"id": "X1", "@type": "Socket", "name": "s", "created": "2022-02-02T02:02:02Z"
|
||||
}));
|
||||
assert!(r.is_err());
|
||||
}
|
||||
|
||||
fn one_row(c: &Connection, table: &str) -> i64 {
|
||||
c.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0))
|
||||
.unwrap()
|
||||
@@ -1653,30 +1751,31 @@ mod tests {
|
||||
to_target: HashMap::new(),
|
||||
};
|
||||
let file: FileNode = serde_json::from_value(serde_json::json!({
|
||||
"id":"F1","nodeType":"file","name":"a.bin",
|
||||
"type":"application/octet-stream","created":"2022-02-02T02:02:02Z"
|
||||
"id":"F1","@type":"File","name":"a.bin",
|
||||
"mediaType":"application/octet-stream","created":"2022-02-02T02:02:02Z"
|
||||
}))
|
||||
.unwrap();
|
||||
let blob1 = crate::db::blobs::intern_blob(&c, b"v1").unwrap();
|
||||
let local = insert_file_node(&c, &file, Some(blob1), &res).unwrap();
|
||||
let blob2 = crate::db::blobs::intern_blob(&c, b"v2").unwrap();
|
||||
let changed: FileNode = serde_json::from_value(serde_json::json!({
|
||||
"id":"F1","nodeType":"file","name":"renamed.bin",
|
||||
"type":"text/plain","created":"2022-02-02T02:02:02Z"
|
||||
"id":"F1","@type":"File","name":"renamed.bin",
|
||||
"mediaType":"text/plain","executable":true,"created":"2022-02-02T02:02:02Z"
|
||||
}))
|
||||
.unwrap();
|
||||
update_file_node(&c, local, &changed, Some(blob2), &res).unwrap();
|
||||
assert_eq!(one_row(&c, "file_nodes"), 1);
|
||||
let (name, mt, b): (String, Option<String>, Option<i64>) = c
|
||||
let (name, mt, b, x): (String, Option<String>, Option<i64>, i64) = c
|
||||
.query_row(
|
||||
"SELECT name, media_type, blob_id FROM file_nodes WHERE id=?1",
|
||||
"SELECT name, media_type, blob_id, executable FROM file_nodes WHERE id=?1",
|
||||
params![local],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(name, "renamed.bin");
|
||||
assert_eq!(mt.as_deref(), Some("text/plain"));
|
||||
assert_eq!(b, Some(blob2));
|
||||
assert_eq!(x, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
use std::thread::JoinHandle;
|
||||
|
||||
use crossbeam_channel::{Receiver, Sender, unbounded};
|
||||
use crossbeam_channel::{Receiver, Sender, bounded, unbounded};
|
||||
|
||||
use crate::jmap::session::Limits;
|
||||
|
||||
@@ -41,7 +41,7 @@ where
|
||||
{
|
||||
let threads = threads.max(1);
|
||||
let (job_tx, job_rx) = unbounded::<J>();
|
||||
let (result_tx, result_rx) = unbounded::<R>();
|
||||
let (result_tx, result_rx) = bounded::<R>(threads);
|
||||
let worker = std::sync::Arc::new(worker);
|
||||
let mut workers = Vec::with_capacity(threads);
|
||||
for _ in 0..threads {
|
||||
@@ -75,10 +75,11 @@ where
|
||||
|
||||
pub fn finish(mut self) -> Vec<R> {
|
||||
drop(self.job_tx.take());
|
||||
let out = self.result_rx.iter().collect();
|
||||
for handle in self.workers.drain(..) {
|
||||
let _ = handle.join();
|
||||
}
|
||||
self.result_rx.try_iter().collect()
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,6 +90,7 @@ where
|
||||
{
|
||||
fn drop(&mut self) {
|
||||
drop(self.job_tx.take());
|
||||
for _ in self.result_rx.iter() {}
|
||||
for handle in self.workers.drain(..) {
|
||||
let _ = handle.join();
|
||||
}
|
||||
@@ -131,4 +133,38 @@ mod tests {
|
||||
let expected: Vec<u64> = (0..100).map(|n| n * n).collect();
|
||||
assert_eq!(got, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_holds_at_most_one_unread_result_per_worker() {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
// Unread results back-pressure the workers instead of piling up.
|
||||
let done = std::sync::Arc::new(AtomicUsize::new(0));
|
||||
let pool: Pool<u64, u64> = Pool::new(2, {
|
||||
let done = done.clone();
|
||||
move |n| {
|
||||
done.fetch_add(1, Ordering::SeqCst);
|
||||
n
|
||||
}
|
||||
});
|
||||
for n in 0..50 {
|
||||
pool.submit(n);
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(200));
|
||||
assert!(done.load(Ordering::SeqCst) <= 4);
|
||||
let mut got = pool.finish();
|
||||
got.sort_unstable();
|
||||
assert_eq!(got, (0..50).collect::<Vec<u64>>());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dropping_pool_with_unread_results_does_not_hang() {
|
||||
let pool: Pool<u64, u64> = Pool::new(2, |n| n);
|
||||
for n in 0..50 {
|
||||
pool.submit(n);
|
||||
}
|
||||
let _ = pool.results().recv();
|
||||
drop(pool);
|
||||
}
|
||||
}
|
||||
@@ -359,6 +359,23 @@ impl Dovecot {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn create_leaves_without_parents(
|
||||
&self,
|
||||
account: &Account,
|
||||
leaves: &[&str],
|
||||
message: &[u8],
|
||||
) -> ContainerResult<Vec<String>> {
|
||||
let mut client = ImapSeed::connect(&self.imap.host, self.imap.port)?;
|
||||
client.login(&account.username, &account.password)?;
|
||||
for name in leaves {
|
||||
client.create(name)?;
|
||||
client.append_with_flags(name, &[], message)?;
|
||||
}
|
||||
let listed = client.list_raw()?;
|
||||
client.logout()?;
|
||||
Ok(listed)
|
||||
}
|
||||
|
||||
pub fn delete_first_inbox_message(&self, account: &Account) -> ContainerResult<()> {
|
||||
let mut client = ImapSeed::connect(&self.imap.host, self.imap.port)?;
|
||||
client.login(&account.username, &account.password)?;
|
||||
|
||||
@@ -189,6 +189,12 @@ impl ImapSeed {
|
||||
Ok(names)
|
||||
}
|
||||
|
||||
pub fn list_raw(&mut self) -> ContainerResult<Vec<String>> {
|
||||
let tag = self.next_tag();
|
||||
self.send(&format!(r#"{tag} LIST "" "*""#))?;
|
||||
self.read_tagged(&tag)
|
||||
}
|
||||
|
||||
pub fn select(&mut self, mailbox: &str) -> ContainerResult<usize> {
|
||||
let tag = self.next_tag();
|
||||
self.send(&format!("{tag} SELECT {}", quote(mailbox)))?;
|
||||
|
||||
@@ -243,6 +243,7 @@ fn jmap(base_url: &str, auth: &str, calls: Value) -> ContainerResult<Value> {
|
||||
"urn:ietf:params:jmap:core",
|
||||
"urn:ietf:params:jmap:mail",
|
||||
"urn:ietf:params:jmap:sieve",
|
||||
"urn:ietf:params:jmap:filenode",
|
||||
"urn:inbuxa:jmap:registry",
|
||||
],
|
||||
"methodCalls": calls,
|
||||
|
||||
@@ -19,7 +19,7 @@ use integration::validate::{
|
||||
|
||||
use inbuxa_migrate::error::Error;
|
||||
use inbuxa_migrate::sync::import_dav;
|
||||
use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg};
|
||||
use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange};
|
||||
use rusqlite::Connection;
|
||||
use serde_json::Value;
|
||||
|
||||
@@ -39,7 +39,7 @@ fn dav_config(kind: DavKindArg, account: &Account, dav_root: &str) -> DavImportC
|
||||
allow_cleartext: true,
|
||||
dav_connections: 2,
|
||||
multiget_batch: 25,
|
||||
allow_source_change: false,
|
||||
source_change: SourceChange::Reject,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,7 +58,7 @@ fn dav_config_via_discovery(
|
||||
allow_cleartext: true,
|
||||
dav_connections: 2,
|
||||
multiget_batch: 25,
|
||||
allow_source_change: false,
|
||||
source_change: SourceChange::Reject,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -412,7 +412,7 @@ fn baikal_carddav_preserves_apple_item_labels() {
|
||||
allow_cleartext: true,
|
||||
dav_connections: 2,
|
||||
multiget_batch: 25,
|
||||
allow_source_change: false,
|
||||
source_change: SourceChange::Reject,
|
||||
};
|
||||
let summary = import_dav::run(common(&archive), cfg).expect("carddav import");
|
||||
assert!(
|
||||
|
||||
@@ -44,7 +44,7 @@ fn import(kind: DavKindArg, url: String, account: &Account, archive: &Path) -> S
|
||||
allow_cleartext: true,
|
||||
dav_connections: 2,
|
||||
multiget_batch: 25,
|
||||
allow_source_change: false,
|
||||
source_change: inbuxa_migrate::sync::import_dav::SourceChange::Reject,
|
||||
},
|
||||
)
|
||||
.expect("import");
|
||||
|
||||
@@ -367,3 +367,90 @@ fn the_guide_sets_up_the_accounts_previews_and_moves_them() {
|
||||
"{report}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires Docker"]
|
||||
fn files_go_into_inbuxa_in_its_own_shape_and_come_back() {
|
||||
let server = Inbuxa::start().expect("start inbuxa");
|
||||
let dir = TempDir::new().unwrap();
|
||||
let out = dir.path().join("out.sqlite");
|
||||
let back = dir.path().join("back.sqlite");
|
||||
{
|
||||
let conn = db::init::open(&out).unwrap();
|
||||
let blob = db::blobs::intern_blob(&conn, b"#!/bin/sh\necho hi\n").unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO file_nodes (id,parent_id,node_type,blob_id,name,media_type,created)
|
||||
VALUES (1,NULL,'directory',NULL,'docs',NULL,'2022-02-02T02:02:02Z'),
|
||||
(2,1,'file',?1,'run.sh','text/x-sh','2022-02-02T02:02:02Z')",
|
||||
[blob],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let run = |args: &[&str]| {
|
||||
let out = Command::new(BIN)
|
||||
.args(args)
|
||||
.env("INBUXA_MIGRATE_API_KEY", &server.token)
|
||||
.output()
|
||||
.unwrap();
|
||||
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
|
||||
assert!(out.status.success(), "{args:?}: {stderr}");
|
||||
};
|
||||
let target = [
|
||||
"--url",
|
||||
server.base_url.as_str(),
|
||||
"--auth-bearer",
|
||||
"--account-name",
|
||||
ADMIN_EMAIL,
|
||||
"--objects",
|
||||
"filenode",
|
||||
];
|
||||
run(&[&["export"][..], &target, &[out.to_str().unwrap()]].concat());
|
||||
|
||||
let account = server.account_of(ADMIN_EMAIL, ADMIN_PASSWORD).unwrap();
|
||||
let r = server
|
||||
.call(
|
||||
ADMIN_EMAIL,
|
||||
ADMIN_PASSWORD,
|
||||
json!([["FileNode/query", {"accountId": account}, "q"],
|
||||
["FileNode/get", {"accountId": account,
|
||||
"#ids": {"resultOf": "q", "name": "FileNode/query", "path": "/ids"},
|
||||
"properties": ["name", "nodeType", "type", "parentId"]}, "g"]]),
|
||||
)
|
||||
.unwrap();
|
||||
let list = r[1][1]["list"].as_array().unwrap_or_else(|| panic!("{r}"));
|
||||
let file = list
|
||||
.iter()
|
||||
.find(|n| n["name"] == "run.sh")
|
||||
.unwrap_or_else(|| panic!("no file: {r}"));
|
||||
assert_eq!(file["nodeType"], "file", "{file}");
|
||||
assert_eq!(file["type"], "text/x-sh", "{file}");
|
||||
assert!(
|
||||
list.iter()
|
||||
.any(|n| n["name"] == "docs" && n["nodeType"] == "directory"),
|
||||
"{r}"
|
||||
);
|
||||
|
||||
run(&[&["import", "jmap"][..], &target, &[back.to_str().unwrap()]].concat());
|
||||
let conn = db::init::open(&back).unwrap();
|
||||
let mut rows: Vec<(String, String, Option<String>)> = conn
|
||||
.prepare(
|
||||
"SELECT name, node_type, media_type FROM file_nodes WHERE name IN ('docs','run.sh')",
|
||||
)
|
||||
.unwrap()
|
||||
.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))
|
||||
.unwrap()
|
||||
.collect::<Result<_, _>>()
|
||||
.unwrap();
|
||||
rows.sort();
|
||||
assert_eq!(
|
||||
rows,
|
||||
[
|
||||
("docs".to_owned(), "directory".to_owned(), None),
|
||||
(
|
||||
"run.sh".to_owned(),
|
||||
"file".to_owned(),
|
||||
Some("text/x-sh".to_owned())
|
||||
)
|
||||
]
|
||||
);
|
||||
}
|
||||
@@ -19,7 +19,7 @@ use integration::{Account, Endpoint};
|
||||
|
||||
use inbuxa_migrate::error::Error;
|
||||
use inbuxa_migrate::sync::import_dav;
|
||||
use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg};
|
||||
use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange};
|
||||
use rusqlite::Connection;
|
||||
use serde_json::Value;
|
||||
|
||||
@@ -34,7 +34,7 @@ fn dav_config(kind: DavKindArg, account: &Account, http: &Endpoint) -> DavImport
|
||||
allow_cleartext: true,
|
||||
dav_connections: 2,
|
||||
multiget_batch: 25,
|
||||
allow_source_change: false,
|
||||
source_change: SourceChange::Reject,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ use integration::{Account, Endpoint};
|
||||
|
||||
use inbuxa_migrate::error::Error;
|
||||
use inbuxa_migrate::sync::import_dav;
|
||||
use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg};
|
||||
use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange};
|
||||
|
||||
fn webdav_config(account: &Account, http: &Endpoint) -> DavImportConfig {
|
||||
DavImportConfig {
|
||||
@@ -31,7 +31,7 @@ fn webdav_config(account: &Account, http: &Endpoint) -> DavImportConfig {
|
||||
allow_cleartext: true,
|
||||
dav_connections: 2,
|
||||
multiget_batch: 25,
|
||||
allow_source_change: false,
|
||||
source_change: SourceChange::Reject,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+312
-5
@@ -1340,7 +1340,9 @@ fn webdav_display_or_basename_test_via_parser_keeps_basename_semantics() {
|
||||
fn dry_run_writes_nothing_but_emits_per_collection_counts() {
|
||||
use inbuxa_migrate::logging::Logger;
|
||||
use inbuxa_migrate::sync::CommonConfig;
|
||||
use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, run};
|
||||
use inbuxa_migrate::sync::import_dav::{
|
||||
DavAuth, DavImportConfig, DavKindArg, SourceChange, run,
|
||||
};
|
||||
use std::path::PathBuf;
|
||||
|
||||
let mut server = mockito::Server::new();
|
||||
@@ -1409,7 +1411,7 @@ fn dry_run_writes_nothing_but_emits_per_collection_counts() {
|
||||
allow_cleartext: true,
|
||||
dav_connections: 1,
|
||||
multiget_batch: 50,
|
||||
allow_source_change: false,
|
||||
source_change: SourceChange::Reject,
|
||||
};
|
||||
let summary = run(common, config).expect("dry-run ok");
|
||||
|
||||
@@ -1453,7 +1455,9 @@ fn dav_source_change_protection_fires_across_users_on_same_root() {
|
||||
use encodify::base64::STANDARD;
|
||||
use inbuxa_migrate::logging::Logger;
|
||||
use inbuxa_migrate::sync::CommonConfig;
|
||||
use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, run};
|
||||
use inbuxa_migrate::sync::import_dav::{
|
||||
DavAuth, DavImportConfig, DavKindArg, SourceChange, run,
|
||||
};
|
||||
use std::path::PathBuf;
|
||||
|
||||
let mut server = mockito::Server::new();
|
||||
@@ -1568,7 +1572,7 @@ fn dav_source_change_protection_fires_across_users_on_same_root() {
|
||||
allow_cleartext: true,
|
||||
dav_connections: 1,
|
||||
multiget_batch: 50,
|
||||
allow_source_change: false,
|
||||
source_change: SourceChange::Reject,
|
||||
};
|
||||
|
||||
run(
|
||||
@@ -1698,7 +1702,7 @@ fn caldav_import_config(url: &str) -> inbuxa_migrate::sync::import_dav::DavImpor
|
||||
allow_cleartext: true,
|
||||
dav_connections: 1,
|
||||
multiget_batch: 8,
|
||||
allow_source_change: false,
|
||||
source_change: inbuxa_migrate::sync::import_dav::SourceChange::Reject,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1830,6 +1834,122 @@ END:VCALENDAR
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
// Kerio Connect sends calendar-data with bare CR line breaks, raw or as (issue #48).
|
||||
#[test]
|
||||
fn kerio_bare_cr_calendar_data_imports() {
|
||||
let mut server = mockito::Server::new();
|
||||
let url = server.url();
|
||||
let home = "/full-calendars/domain.ext/user/";
|
||||
let coll = "/full-calendars/domain.ext/user/ceb087f2-0604-4c31-a70e-9f68e90600c6/";
|
||||
|
||||
let home_set = format!(
|
||||
r#"<?xml version="1.0"?>
|
||||
<a:multistatus xmlns:a="DAV:" xmlns:f="urn:ietf:params:xml:ns:caldav">
|
||||
<a:response>
|
||||
<a:href>{coll}</a:href>
|
||||
<a:propstat>
|
||||
<a:prop>
|
||||
<a:resourcetype><a:collection/><f:calendar/></a:resourcetype>
|
||||
<a:displayname>Calendar</a:displayname>
|
||||
</a:prop>
|
||||
<a:status>HTTP/1.1 200 OK</a:status>
|
||||
</a:propstat>
|
||||
</a:response>
|
||||
</a:multistatus>"#
|
||||
);
|
||||
let _discovery = multistatus_response(&mut server, "PROPFIND", home, &home_set);
|
||||
|
||||
let item = |n: u32| format!("{coll}ceb087f2-0604-4c31-a70e-9f68e90600c6-0000000{n}.eml");
|
||||
let listing = format!(
|
||||
r#"<?xml version="1.0"?>
|
||||
<a:multistatus xmlns:a="DAV:">
|
||||
<a:response>
|
||||
<a:href>{}</a:href>
|
||||
<a:propstat><a:prop><a:resourcetype/><a:getetag>"k1"</a:getetag></a:prop>
|
||||
<a:status>HTTP/1.1 200 OK</a:status></a:propstat>
|
||||
</a:response>
|
||||
<a:response>
|
||||
<a:href>{}</a:href>
|
||||
<a:propstat><a:prop><a:resourcetype/><a:getetag>"k2"</a:getetag></a:prop>
|
||||
<a:status>HTTP/1.1 200 OK</a:status></a:propstat>
|
||||
</a:response>
|
||||
</a:multistatus>"#,
|
||||
item(1),
|
||||
item(2)
|
||||
);
|
||||
let _enumeration = multistatus_response(&mut server, "PROPFIND", coll, &listing);
|
||||
|
||||
let ical = |uid: &str, summary: &str, eol: &str| {
|
||||
[
|
||||
"BEGIN:VCALENDAR",
|
||||
"PRODID:-//Kerio Technologies//Kerio Connect//EN",
|
||||
"VERSION:2.0",
|
||||
"BEGIN:VEVENT",
|
||||
&format!("UID:{uid}"),
|
||||
"DTSTAMP:20260101T000000Z",
|
||||
"DTSTART:20260101T090000Z",
|
||||
"DTEND:20260101T100000Z",
|
||||
&format!("SUMMARY:{summary}"),
|
||||
"X-LABEL:0",
|
||||
"END:VEVENT",
|
||||
"END:VCALENDAR",
|
||||
"",
|
||||
]
|
||||
.join(eol)
|
||||
};
|
||||
let data = format!(
|
||||
r#"<?xml version="1.0"?>
|
||||
<a:multistatus xmlns:a="DAV:" xmlns:f="urn:ietf:params:xml:ns:caldav">
|
||||
<a:response>
|
||||
<a:href>{}</a:href>
|
||||
<a:propstat><a:prop><a:getetag>"k1"</a:getetag>
|
||||
<f:calendar-data><![CDATA[{}]]></f:calendar-data></a:prop>
|
||||
<a:status>HTTP/1.1 200 OK</a:status></a:propstat>
|
||||
</a:response>
|
||||
<a:response>
|
||||
<a:href>{}</a:href>
|
||||
<a:propstat><a:prop><a:getetag>"k2"</a:getetag>
|
||||
<f:calendar-data>{}</f:calendar-data></a:prop>
|
||||
<a:status>HTTP/1.1 200 OK</a:status></a:propstat>
|
||||
</a:response>
|
||||
</a:multistatus>"#,
|
||||
item(1),
|
||||
ical("k1@kerio", "Raw CR", "\r"),
|
||||
item(2),
|
||||
ical("k2@kerio", "Entity CR", " "),
|
||||
);
|
||||
let _multiget = multistatus_response(&mut server, "REPORT", coll, &data);
|
||||
|
||||
let archive = dav_archive();
|
||||
let summary = inbuxa_migrate::sync::import_dav::run(
|
||||
dav_common(&archive),
|
||||
caldav_import_config(&format!("{url}{home}")),
|
||||
)
|
||||
.expect("import");
|
||||
|
||||
let events = summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(t, _)| *t == "calendarevent")
|
||||
.map(|(_, c)| c.clone())
|
||||
.expect("calendarevent counts");
|
||||
assert_eq!(events.failed, 0, "{events:?}");
|
||||
assert_eq!(events.fetched, 2, "{events:?}");
|
||||
|
||||
let conn = rusqlite::Connection::open(&archive).expect("open archive");
|
||||
let mut titles: Vec<String> = conn
|
||||
.prepare("SELECT json_extract(data, '$.title') FROM calendar_events")
|
||||
.unwrap()
|
||||
.query_map([], |r| r.get(0))
|
||||
.unwrap()
|
||||
.map(Result::unwrap)
|
||||
.collect();
|
||||
titles.sort_unstable();
|
||||
assert_eq!(titles, ["Entity CR", "Raw CR"]);
|
||||
drop(conn);
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credentials_rejected_during_discovery_still_aborts_the_run() {
|
||||
let mut server = mockito::Server::new();
|
||||
@@ -1904,3 +2024,190 @@ fn dav_import_reports_both_phases_when_every_collection_fails() {
|
||||
);
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
}
|
||||
|
||||
fn webdav_share(server: &mut mockito::Server, prefix: &str, body: &str) -> Vec<mockito::Mock> {
|
||||
let url = server.url();
|
||||
let root = format!(
|
||||
r#"<?xml version="1.0"?>
|
||||
<d:multistatus xmlns:d="DAV:">
|
||||
<d:response>
|
||||
<d:href>{url}{prefix}</d:href>
|
||||
<d:propstat>
|
||||
<d:prop><d:resourcetype><d:collection/></d:resourcetype><d:displayname>share</d:displayname></d:prop>
|
||||
<d:status>HTTP/1.1 200 OK</d:status>
|
||||
</d:propstat>
|
||||
</d:response>
|
||||
<d:response>
|
||||
<d:href>{url}{prefix}docs/</d:href>
|
||||
<d:propstat>
|
||||
<d:prop><d:resourcetype><d:collection/></d:resourcetype><d:displayname>docs</d:displayname></d:prop>
|
||||
<d:status>HTTP/1.1 200 OK</d:status>
|
||||
</d:propstat>
|
||||
</d:response>
|
||||
</d:multistatus>"#
|
||||
);
|
||||
let docs = format!(
|
||||
r#"<?xml version="1.0"?>
|
||||
<d:multistatus xmlns:d="DAV:">
|
||||
<d:response>
|
||||
<d:href>{url}{prefix}docs/</d:href>
|
||||
<d:propstat>
|
||||
<d:prop><d:resourcetype><d:collection/></d:resourcetype><d:displayname>docs</d:displayname></d:prop>
|
||||
<d:status>HTTP/1.1 200 OK</d:status>
|
||||
</d:propstat>
|
||||
</d:response>
|
||||
<d:response>
|
||||
<d:href>{url}{prefix}docs/a.txt</d:href>
|
||||
<d:propstat>
|
||||
<d:prop><d:resourcetype/><d:getcontenttype>text/plain</d:getcontenttype><d:getetag>"a1"</d:getetag></d:prop>
|
||||
<d:status>HTTP/1.1 200 OK</d:status>
|
||||
</d:propstat>
|
||||
</d:response>
|
||||
</d:multistatus>"#
|
||||
);
|
||||
vec![
|
||||
multistatus_response(server, "PROPFIND", prefix, &root),
|
||||
multistatus_response(server, "PROPFIND", &format!("{prefix}docs/"), &docs),
|
||||
server
|
||||
.mock("GET", format!("{prefix}docs/a.txt").as_str())
|
||||
.with_status(200)
|
||||
.with_header("content-type", "text/plain")
|
||||
.with_body(body)
|
||||
.create(),
|
||||
]
|
||||
}
|
||||
|
||||
fn webdav_import_config(
|
||||
url: &str,
|
||||
source_change: inbuxa_migrate::sync::import_dav::SourceChange,
|
||||
) -> inbuxa_migrate::sync::import_dav::DavImportConfig {
|
||||
inbuxa_migrate::sync::import_dav::DavImportConfig {
|
||||
kind: inbuxa_migrate::sync::import_dav::DavKindArg::Webdav,
|
||||
source_change,
|
||||
..caldav_import_config(url)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allow_source_change_keeps_the_moved_share_old_tree() {
|
||||
use inbuxa_migrate::sync::import_dav::{SourceChange, run};
|
||||
|
||||
let mut server = mockito::Server::new();
|
||||
let url = server.url();
|
||||
let _old = webdav_share(&mut server, "/old/u/", "old\n");
|
||||
let _new = webdav_share(&mut server, "/new/u/", "new\n");
|
||||
let archive = dav_archive();
|
||||
|
||||
run(
|
||||
dav_common(&archive),
|
||||
webdav_import_config(&format!("{url}/old/u/"), SourceChange::Reject),
|
||||
)
|
||||
.expect("old share import");
|
||||
run(
|
||||
dav_common(&archive),
|
||||
webdav_import_config(&format!("{url}/new/u/"), SourceChange::Allow),
|
||||
)
|
||||
.expect("new share import");
|
||||
|
||||
let (nodes, sources) = (
|
||||
row_count(&archive, "file_nodes"),
|
||||
row_count(&archive, "sources"),
|
||||
);
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
assert_eq!(nodes, 4, "both trees stay side by side");
|
||||
assert_eq!(sources, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replace_source_drops_the_moved_share_old_tree() {
|
||||
use inbuxa_migrate::sync::import_dav::{SourceChange, run};
|
||||
|
||||
let mut server = mockito::Server::new();
|
||||
let url = server.url();
|
||||
let _old = webdav_share(&mut server, "/old/u/", "old\n");
|
||||
let _new = webdav_share(&mut server, "/new/u/", "new\n");
|
||||
let archive = dav_archive();
|
||||
|
||||
run(
|
||||
dav_common(&archive),
|
||||
webdav_import_config(&format!("{url}/old/u/"), SourceChange::Reject),
|
||||
)
|
||||
.expect("old share import");
|
||||
let summary = run(
|
||||
dav_common(&archive),
|
||||
webdav_import_config(&format!("{url}/new/u/"), SourceChange::Replace),
|
||||
)
|
||||
.expect("new share import");
|
||||
let rerun = run(
|
||||
dav_common(&archive),
|
||||
webdav_import_config(&format!("{url}/new/u/"), SourceChange::Reject),
|
||||
);
|
||||
|
||||
let conn = rusqlite::Connection::open(&archive).unwrap();
|
||||
let account: String = conn
|
||||
.query_row("SELECT account_id FROM sources", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
let orphans: i64 = conn
|
||||
.query_row(
|
||||
"SELECT count(*) FROM file_nodes
|
||||
WHERE id NOT IN (SELECT local_id FROM sync_id_dav)",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
drop(conn);
|
||||
let (nodes, sources, blobs) = (
|
||||
row_count(&archive, "file_nodes"),
|
||||
row_count(&archive, "sources"),
|
||||
row_count(&archive, "blobs"),
|
||||
);
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
|
||||
assert_eq!(nodes, 2, "only the new tree remains");
|
||||
assert_eq!(orphans, 0);
|
||||
assert_eq!(sources, 1);
|
||||
assert!(account.ends_with("/new/u/"), "kept source is {account}");
|
||||
assert_eq!(blobs, 1, "the old file's blob is collected");
|
||||
let filenode = summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(name, _)| *name == "filenode")
|
||||
.expect("filenode counts");
|
||||
assert_eq!(filenode.1.deleted, 2);
|
||||
assert!(
|
||||
rerun.is_ok(),
|
||||
"once replaced, the archive records a single source: {rerun:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dry_run_replace_source_keeps_the_old_tree() {
|
||||
use inbuxa_migrate::sync::import_dav::{SourceChange, run};
|
||||
|
||||
let mut server = mockito::Server::new();
|
||||
let url = server.url();
|
||||
let _old = webdav_share(&mut server, "/old/u/", "old\n");
|
||||
let _new = webdav_share(&mut server, "/new/u/", "new\n");
|
||||
let archive = dav_archive();
|
||||
|
||||
run(
|
||||
dav_common(&archive),
|
||||
webdav_import_config(&format!("{url}/old/u/"), SourceChange::Reject),
|
||||
)
|
||||
.expect("old share import");
|
||||
let mut common = dav_common(&archive);
|
||||
common.dry_run = true;
|
||||
run(
|
||||
common,
|
||||
webdav_import_config(&format!("{url}/new/u/"), SourceChange::Replace),
|
||||
)
|
||||
.expect("dry-run");
|
||||
|
||||
let (nodes, sources) = (
|
||||
row_count(&archive, "file_nodes"),
|
||||
row_count(&archive, "sources"),
|
||||
);
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
assert_eq!(nodes, 2);
|
||||
assert_eq!(sources, 1);
|
||||
}
|
||||
@@ -70,7 +70,7 @@ fn import(url: &str, archive: &Path) -> Summary {
|
||||
allow_cleartext: true,
|
||||
dav_connections: 1,
|
||||
multiget_batch: 25,
|
||||
allow_source_change: false,
|
||||
source_change: import_dav::SourceChange::Reject,
|
||||
},
|
||||
)
|
||||
.expect("import");
|
||||
|
||||
@@ -2394,3 +2394,248 @@ fn coordinator_signs_in_to_gmail_as_a_service_account_acting_for_the_user() {
|
||||
// Both connections signed in on the one token.
|
||||
_token.assert();
|
||||
}
|
||||
|
||||
const NOSELECT_LIST: &[&str] = &[
|
||||
"* LIST (\\HasChildren) \"/\" \"INBOX\"",
|
||||
"* LIST (\\Noselect \\HasChildren) \"/\" \"INBOX/Projects\"",
|
||||
"* LIST (\\HasNoChildren) \"/\" \"INBOX/Projects/Invoices\"",
|
||||
"* LIST (\\Noselect \\HasChildren) \"/\" \"A\"",
|
||||
"* LIST (\\HasNoChildren) \"/\" \"A/B/Reports\"",
|
||||
"* LIST (\\Noselect \\HasChildren) \"/\" \"Clients \"",
|
||||
"* LIST (\\HasNoChildren) \"/\" \"Clients /Reports\"",
|
||||
"* LIST (\\HasChildren) \"/\" \"Clients\"",
|
||||
"* LIST (\\HasNoChildren) \"/\" \"Clients/Reports\"",
|
||||
];
|
||||
|
||||
const NOSELECT_SELECTABLE: &[&str] = &[
|
||||
"INBOX",
|
||||
"INBOX/Projects/Invoices",
|
||||
"A/B/Reports",
|
||||
"Clients /Reports",
|
||||
"Clients",
|
||||
"Clients/Reports",
|
||||
];
|
||||
|
||||
fn noselect_control_script(selected: std::sync::Arc<Mutex<Vec<String>>>) -> Script {
|
||||
Box::new(move |conn: &mut MockConn| -> std::io::Result<()> {
|
||||
auth_preamble(conn, "IMAP4rev2 LITERAL+ AUTH=PLAIN")?;
|
||||
while let Ok((tag, cmd)) = conn.read_command() {
|
||||
if cmd.starts_with("LIST") {
|
||||
for line in NOSELECT_LIST {
|
||||
conn.write_line(line)?;
|
||||
}
|
||||
conn.write_line(&format!("{tag} OK LIST done"))?;
|
||||
} else if let Some(name) = cmd
|
||||
.strip_prefix("SELECT ")
|
||||
.or_else(|| cmd.strip_prefix("EXAMINE "))
|
||||
{
|
||||
selected
|
||||
.lock()
|
||||
.expect("selected")
|
||||
.push(name.trim_matches('"').to_owned());
|
||||
write_select(conn, &tag, 7, 1, 0)?;
|
||||
} else if cmd.starts_with("UID SEARCH") {
|
||||
conn.write_line("* SEARCH")?;
|
||||
conn.write_line(&format!("{tag} OK SEARCH done"))?;
|
||||
} else if cmd.starts_with("LOGOUT") {
|
||||
conn.write_line("* BYE bye")?;
|
||||
conn.write_line(&format!("{tag} OK"))?;
|
||||
break;
|
||||
} else {
|
||||
assert!(
|
||||
!cmd.starts_with("STATUS") && !cmd.contains("FETCH"),
|
||||
"unexpected {cmd}"
|
||||
);
|
||||
conn.write_line(&format!("{tag} OK"))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn mailbox_tree(conn: &Connection) -> Vec<(i64, String, Option<String>)> {
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT m.id, s.folder, p.name FROM mailboxes m
|
||||
JOIN sync_id_imap s ON s.local_id = m.id AND s.type_name = 'mailbox'
|
||||
LEFT JOIN mailboxes p ON p.id = m.parent_id
|
||||
ORDER BY s.folder",
|
||||
)
|
||||
.unwrap();
|
||||
stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))
|
||||
.unwrap()
|
||||
.collect::<Result<_, _>>()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn assert_noselect_tree(conn: &Connection) {
|
||||
let tree = mailbox_tree(conn);
|
||||
let pairs: Vec<(&str, Option<&str>)> = tree
|
||||
.iter()
|
||||
.map(|(_, f, p)| (f.as_str(), p.as_deref()))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
pairs,
|
||||
[
|
||||
("A", None),
|
||||
("A/B", Some("A")),
|
||||
("A/B/Reports", Some("B")),
|
||||
("Clients", None),
|
||||
("Clients ", None),
|
||||
("Clients /Reports", Some("Clients ")),
|
||||
("Clients/Reports", Some("Clients")),
|
||||
("INBOX", None),
|
||||
("INBOX/Projects", Some("INBOX")),
|
||||
("INBOX/Projects/Invoices", Some("Projects")),
|
||||
]
|
||||
);
|
||||
let roles: i64 = conn
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM mailboxes WHERE role IS NOT NULL AND name IN ('Projects', 'A', 'B', 'Clients ')",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(roles, 0);
|
||||
}
|
||||
|
||||
fn assert_only_selectable_selected(selected: &Mutex<Vec<String>>) {
|
||||
let mut got = selected.lock().expect("selected").clone();
|
||||
got.sort_unstable();
|
||||
let mut want: Vec<String> = NOSELECT_SELECTABLE
|
||||
.iter()
|
||||
.map(|s| (*s).to_owned())
|
||||
.collect();
|
||||
want.sort_unstable();
|
||||
assert_eq!(got, want);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coordinator_keeps_noselect_ancestors_and_never_selects_them() {
|
||||
let first = std::sync::Arc::new(Mutex::new(Vec::new()));
|
||||
let second = std::sync::Arc::new(Mutex::new(Vec::new()));
|
||||
let server = MockImap::start_scripts(vec![
|
||||
noselect_control_script(first.clone()),
|
||||
noselect_control_script(second.clone()),
|
||||
]);
|
||||
let archive = tempfile("noselect");
|
||||
let summary = run_import(&server, "alice", archive.clone(), |_| {}).expect("import");
|
||||
let mailbox = summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(k, _)| *k == "mailbox")
|
||||
.unwrap();
|
||||
assert_eq!(mailbox.1.created, 10);
|
||||
assert_eq!(mailbox.1.failed, 0);
|
||||
assert_only_selectable_selected(&first);
|
||||
let before = {
|
||||
let conn = Connection::open(&archive).unwrap();
|
||||
assert_noselect_tree(&conn);
|
||||
mailbox_tree(&conn)
|
||||
};
|
||||
|
||||
let summary = run_import(&server, "alice", archive.clone(), |_| {}).expect("reimport");
|
||||
let mailbox = summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(k, _)| *k == "mailbox")
|
||||
.unwrap();
|
||||
assert_eq!((mailbox.1.created, mailbox.1.deleted), (0, 0));
|
||||
assert_only_selectable_selected(&second);
|
||||
let conn = Connection::open(&archive).unwrap();
|
||||
assert_eq!(mailbox_tree(&conn), before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coordinator_heals_archive_missing_noselect_parents() {
|
||||
let first = std::sync::Arc::new(Mutex::new(Vec::new()));
|
||||
let second = std::sync::Arc::new(Mutex::new(Vec::new()));
|
||||
let server = MockImap::start_scripts(vec![
|
||||
noselect_control_script(first),
|
||||
noselect_control_script(second),
|
||||
]);
|
||||
let archive = tempfile("noselect_heal");
|
||||
run_import(&server, "alice", archive.clone(), |_| {}).expect("import");
|
||||
let child_id = {
|
||||
let conn = Connection::open(&archive).unwrap();
|
||||
let tree = mailbox_tree(&conn);
|
||||
let child = tree
|
||||
.iter()
|
||||
.find(|(_, f, _)| f == "INBOX/Projects/Invoices")
|
||||
.unwrap()
|
||||
.0;
|
||||
// simulate an archive written before structural parents were kept
|
||||
conn.execute(
|
||||
"UPDATE mailboxes SET parent_id = NULL WHERE id = ?1",
|
||||
[child],
|
||||
)
|
||||
.unwrap();
|
||||
let parent: i64 = conn
|
||||
.query_row(
|
||||
"SELECT local_id FROM sync_id_imap WHERE type_name = 'mailbox' AND folder = 'INBOX/Projects'",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute(
|
||||
"DELETE FROM sync_id_imap WHERE type_name = 'mailbox' AND folder = 'INBOX/Projects'",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
conn.execute("DELETE FROM mailboxes WHERE id = ?1", [parent])
|
||||
.unwrap();
|
||||
child
|
||||
};
|
||||
run_import(&server, "alice", archive.clone(), |_| {}).expect("reimport");
|
||||
let conn = Connection::open(&archive).unwrap();
|
||||
assert_noselect_tree(&conn);
|
||||
let healed = mailbox_tree(&conn)
|
||||
.into_iter()
|
||||
.find(|(_, f, _)| f == "INBOX/Projects/Invoices")
|
||||
.unwrap()
|
||||
.0;
|
||||
assert_eq!(healed, child_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dry_run_never_selects_noselect_ancestors() {
|
||||
let selected = std::sync::Arc::new(Mutex::new(Vec::new()));
|
||||
let server = MockImap::start_scripts(vec![noselect_control_script(selected.clone())]);
|
||||
let archive = tempfile("noselect_dry");
|
||||
let common = CommonConfig {
|
||||
archive: archive.clone(),
|
||||
threads: 1,
|
||||
dry_run: true,
|
||||
max_retries: 1,
|
||||
allow_invalid_certs: false,
|
||||
logger: Logger::from_flags(true, 0),
|
||||
};
|
||||
let config = ImapImportConfig {
|
||||
url: server.url(),
|
||||
auth: ImapAuth::Basic {
|
||||
user: "alice".to_owned(),
|
||||
password: "p@ss".to_owned(),
|
||||
},
|
||||
allow_cleartext: true,
|
||||
compress: false,
|
||||
include: Vec::new(),
|
||||
exclude: Vec::new(),
|
||||
exclude_special: Vec::new(),
|
||||
folder: Vec::new(),
|
||||
subscribed_only: false,
|
||||
automap: true,
|
||||
include_deleted: false,
|
||||
fetch_batch: 256,
|
||||
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
|
||||
imap_connections: 1,
|
||||
allow_source_change: false,
|
||||
};
|
||||
let summary = run(common, config).expect("dry run");
|
||||
let mailbox = summary
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(k, _)| *k == "mailbox")
|
||||
.unwrap();
|
||||
assert_eq!(mailbox.1.created, 10);
|
||||
assert_only_selectable_selected(&selected);
|
||||
}
|
||||
@@ -44,7 +44,7 @@ fn import(url: &str, archive: &Path) -> Summary {
|
||||
allow_cleartext: true,
|
||||
dav_connections: 1,
|
||||
multiget_batch: 25,
|
||||
allow_source_change: false,
|
||||
source_change: inbuxa_migrate::sync::import_dav::SourceChange::Reject,
|
||||
},
|
||||
)
|
||||
.expect("import");
|
||||
|
||||
+3
-2
@@ -333,8 +333,9 @@ fn principal_substring_prefilter_rejects_near_matches() {
|
||||
["Principal/get", { "list": [
|
||||
{ "id": "p1", "name": "alice2", "accounts": {} },
|
||||
{ "id": "p2", "name": "alice", "accounts": {
|
||||
"w": { "urn:ietf:params:jmap:principals:owner":
|
||||
{ "accountIdForPrincipal": "w" } } } }
|
||||
"w": { "name": "alice", "accountCapabilities": {
|
||||
"urn:ietf:params:jmap:principals:owner":
|
||||
{ "accountIdForPrincipal": "admin", "principalId": "p2" } } } } }
|
||||
] }, "g"]
|
||||
] })
|
||||
.to_string(),
|
||||
|
||||
+1002
File diff suppressed because it is too large.
Load diff
+3
-8
@@ -268,14 +268,9 @@ fn provision_seed_verify_and_teardown() {
|
||||
let owner = exact[0]
|
||||
.get("accounts")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|m| {
|
||||
m.iter().find_map(|(_, v)| {
|
||||
v.get("urn:ietf:params:jmap:principals:owner")
|
||||
.and_then(|o| o.get("accountIdForPrincipal"))
|
||||
.and_then(Value::as_str)
|
||||
})
|
||||
})
|
||||
.expect("accountIdForPrincipal");
|
||||
.and_then(|m| m.keys().next())
|
||||
.map(String::as_str)
|
||||
.expect("principal accounts key");
|
||||
assert_eq!(owner, test1.account_id, "resolved account id matches test1");
|
||||
|
||||
seeder::teardown(stalwart.base_url()).expect("teardown");
|
||||
|
||||
@@ -121,6 +121,32 @@ impl Admin {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn enable_directory_queries(&self) -> SeedResult<()> {
|
||||
let response = self.jmap.call(
|
||||
STALWART,
|
||||
"x:Sharing/set",
|
||||
&self.account_id,
|
||||
json!({ "update": { "singleton": { "allowDirectoryQueries": true } } }),
|
||||
)?;
|
||||
if response.pointer("/updated/singleton").is_none() {
|
||||
return Err(SeedError::Shape(format!(
|
||||
"x:Sharing/set not applied: {response}"
|
||||
)));
|
||||
}
|
||||
let response = self.jmap.call(
|
||||
STALWART,
|
||||
"x:Action/set",
|
||||
&self.account_id,
|
||||
json!({ "create": { "r": { "@type": "ReloadSettings" } } }),
|
||||
)?;
|
||||
if response.pointer("/created/r").is_none() {
|
||||
return Err(SeedError::Shape(format!(
|
||||
"ReloadSettings failed: {response}"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn teardown_domain(&self, name: &str) -> SeedResult<()> {
|
||||
let Some(domain_id) = self.domain_id(name)? else {
|
||||
return Ok(());
|
||||
|
||||
@@ -177,6 +177,7 @@ pub fn teardown(base_url: &str) -> SeedResult<()> {
|
||||
|
||||
pub fn provision(base_url: &str) -> SeedResult<Fixture> {
|
||||
let admin = Admin::connect(base_url, ADMIN_USER, ADMIN_PASSWORD)?;
|
||||
admin.enable_directory_queries()?;
|
||||
admin.teardown_domain(DOMAIN)?;
|
||||
admin.invalidate_caches()?;
|
||||
let domain_id = admin.ensure_domain(DOMAIN)?;
|
||||
|
||||
@@ -273,14 +273,14 @@ fn seed_filenodes(jmap: &Jmap, account_id: &str, tree: &[FileSpec]) -> SeedResul
|
||||
obj.insert("parentId".to_owned(), Value::String(format!("#{parent}")));
|
||||
}
|
||||
if spec.directory {
|
||||
obj.insert("nodeType".to_owned(), Value::String("directory".to_owned()));
|
||||
obj.insert("@type".to_owned(), Value::String("Directory".to_owned()));
|
||||
} else {
|
||||
let bytes = random_bytes(128 + i * 17)?;
|
||||
let blob_id = jmap.upload(account_id, "application/octet-stream", &bytes)?;
|
||||
obj.insert("nodeType".to_owned(), Value::String("file".to_owned()));
|
||||
obj.insert("@type".to_owned(), Value::String("File".to_owned()));
|
||||
obj.insert("blobId".to_owned(), Value::String(blob_id));
|
||||
obj.insert(
|
||||
"type".to_owned(),
|
||||
"mediaType".to_owned(),
|
||||
Value::String("application/octet-stream".to_owned()),
|
||||
);
|
||||
}
|
||||
@@ -400,6 +400,9 @@ fn seed_contacts_calendars(
|
||||
for immutable in ["method", "prodId"] {
|
||||
obj.remove(immutable);
|
||||
}
|
||||
if obj.contains_key("recurrenceRule") || obj.contains_key("recurrenceOverrides") {
|
||||
obj.remove("recurrenceId");
|
||||
}
|
||||
let calendar = &calendars[i % calendars.len()];
|
||||
obj.insert("calendarIds".to_owned(), json!({ calendar.as_str(): true }));
|
||||
jmap.set_create(
|
||||
|
||||
@@ -20,7 +20,7 @@ fn base_url() -> &'static str {
|
||||
use inbuxa_migrate::jmap::account::AccountSelector;
|
||||
use inbuxa_migrate::jmap::http::Auth;
|
||||
use inbuxa_migrate::logging::Logger;
|
||||
use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg};
|
||||
use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange};
|
||||
use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig};
|
||||
|
||||
fn tmp_archive(tag: &str) -> PathBuf {
|
||||
@@ -59,7 +59,7 @@ fn caldav_cfg(base_url: &str, domain: &str, localpart: &str) -> DavImportConfig
|
||||
allow_cleartext: false,
|
||||
dav_connections: 4,
|
||||
multiget_batch: 50,
|
||||
allow_source_change: false,
|
||||
source_change: SourceChange::Reject,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ use rusqlite::Connection;
|
||||
fn base_url() -> &'static str {
|
||||
shared_stalwart().base_url()
|
||||
}
|
||||
use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg};
|
||||
use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange};
|
||||
use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig};
|
||||
|
||||
fn tmp_archive(tag: &str) -> PathBuf {
|
||||
@@ -59,7 +59,7 @@ fn carddav_cfg(base_url: &str, domain: &str, localpart: &str) -> DavImportConfig
|
||||
allow_cleartext: false,
|
||||
dav_connections: 4,
|
||||
multiget_batch: 50,
|
||||
allow_source_change: false,
|
||||
source_change: SourceChange::Reject,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -377,16 +377,28 @@ fn graph_import_survives_a_full_export_to_stalwart() {
|
||||
.find(|c| c["name"]["full"] == "Graph Contact")
|
||||
.expect("the imported card reached the target");
|
||||
|
||||
let uri = card["media"]
|
||||
let photo = card["media"]
|
||||
.as_object()
|
||||
.and_then(|m| m.values().next())
|
||||
.and_then(|m| m["uri"].as_str())
|
||||
.expect("the contact photo survived as a Media resource");
|
||||
assert!(
|
||||
uri.starts_with("data:image/png;base64,"),
|
||||
let media_type = match photo["uri"].as_str() {
|
||||
Some(uri) => uri
|
||||
.strip_prefix("data:")
|
||||
.and_then(|rest| rest.split_once(';'))
|
||||
.map(|(mt, _)| mt),
|
||||
None => {
|
||||
assert!(
|
||||
photo["blobId"].is_string(),
|
||||
"the photo is either an inline data URI or an embedded blob: {photo}"
|
||||
);
|
||||
photo["mediaType"].as_str()
|
||||
}
|
||||
};
|
||||
assert_eq!(
|
||||
media_type,
|
||||
Some("image/png"),
|
||||
"Graph reports image/jpeg for every contact photo, so the media type must come \
|
||||
from the bytes; got {}",
|
||||
&uri[..uri.len().min(40)]
|
||||
from the bytes; got {photo}"
|
||||
);
|
||||
assert_eq!(card["keywords"]["work"], true, "categories became keywords");
|
||||
assert_eq!(
|
||||
@@ -409,16 +421,24 @@ fn graph_import_survives_a_full_export_to_stalwart() {
|
||||
.iter()
|
||||
.find(|e| e["title"] == "Event with an enclosure")
|
||||
.expect("the event with an attachment reached the target");
|
||||
let href = enclosure["links"]
|
||||
let link = enclosure["links"]
|
||||
.as_object()
|
||||
.and_then(|m| m.values().next())
|
||||
.and_then(|l| l["href"].as_str())
|
||||
.expect("the attachment survived as a Link");
|
||||
assert!(
|
||||
href.starts_with("data:text/plain;base64,"),
|
||||
"got {}",
|
||||
&href[..href.len().min(40)]
|
||||
);
|
||||
match link["href"].as_str() {
|
||||
Some(href) => assert!(
|
||||
href.starts_with("data:text/plain;base64,"),
|
||||
"got {}",
|
||||
&href[..href.len().min(40)]
|
||||
),
|
||||
None => {
|
||||
assert!(
|
||||
link["blobId"].is_string(),
|
||||
"the attachment is either an inline data URI or an embedded blob: {link}"
|
||||
);
|
||||
assert_eq!(link["contentType"], "text/plain", "got {link}");
|
||||
}
|
||||
}
|
||||
|
||||
let series = events
|
||||
.iter()
|
||||
@@ -449,7 +469,7 @@ fn graph_import_survives_a_full_export_to_stalwart() {
|
||||
assert!(
|
||||
files
|
||||
.iter()
|
||||
.any(|f| f["name"] == "report.txt" && f["blobId"].is_string()),
|
||||
.any(|f| f["name"] == "report.txt" && f["@type"] == "File" && f["blobId"].is_string()),
|
||||
"the drive file reached the target with a blob"
|
||||
);
|
||||
assert!(
|
||||
|
||||
+170
-1
@@ -8,11 +8,15 @@
|
||||
mod integration;
|
||||
mod seeder;
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use integration::Account;
|
||||
use integration::dovecot::Dovecot;
|
||||
use integration::stalwart::shared as shared_stalwart;
|
||||
use integration::validate::{emails_in_mailbox, mailbox_id_by_path, mailbox_path, open_archive};
|
||||
use rusqlite::Connection;
|
||||
use serde_json::json;
|
||||
|
||||
fn base_url() -> &'static str {
|
||||
shared_stalwart().base_url()
|
||||
@@ -77,6 +81,29 @@ fn imap_basic_config(localpart: &str) -> ImapImportConfig {
|
||||
}
|
||||
}
|
||||
|
||||
fn dovecot_config(account: &Account, imap: &integration::Endpoint) -> ImapImportConfig {
|
||||
ImapImportConfig {
|
||||
url: format!("imap://{}:{}", imap.host, imap.port),
|
||||
auth: ImapAuth::Basic {
|
||||
user: account.username.clone(),
|
||||
password: account.password.clone(),
|
||||
},
|
||||
allow_cleartext: false,
|
||||
compress: false,
|
||||
include: Vec::new(),
|
||||
exclude: Vec::new(),
|
||||
exclude_special: Vec::new(),
|
||||
folder: Vec::new(),
|
||||
subscribed_only: false,
|
||||
automap: true,
|
||||
include_deleted: false,
|
||||
fetch_batch: 64,
|
||||
fetch_batch_bytes: inbuxa_migrate::sync::batch::DEFAULT_BATCH_BYTES,
|
||||
imap_connections: 2,
|
||||
allow_source_change: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn count(conn: &Connection, table: &str) -> i64 {
|
||||
conn.query_row(&format!("SELECT count(*) FROM {table}"), [], |r| r.get(0))
|
||||
.unwrap()
|
||||
@@ -395,3 +422,145 @@ fn imap_imported_archive_exports_via_jmap() {
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
seeder::teardown(base_url()).expect("teardown");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires Docker"]
|
||||
fn dovecot_noselect_parents_keep_the_tree_through_export() {
|
||||
let d = Dovecot::start().expect("dovecot start");
|
||||
let account = d.accounts.first().expect("account").clone();
|
||||
let message = b"From: [email protected]\r\n\
|
||||
To: user1@inbuxa-migrate.test\r\n\
|
||||
Subject: Tree\r\n\
|
||||
Message-ID: <tree-1@inbuxa-migrate.test>\r\n\
|
||||
Date: Wed, 01 Jan 2025 12:00:00 +0000\r\n\
|
||||
\r\n\
|
||||
Structural parent probe.\r\n";
|
||||
let leaves = [
|
||||
"Projects/Invoices",
|
||||
"Clients/Acme/Invoices",
|
||||
"Archive",
|
||||
"Archive/2024",
|
||||
"Archive /2024",
|
||||
];
|
||||
let structural = ["Projects", "Clients", "Clients/Acme", "Archive "];
|
||||
let listed = d
|
||||
.create_leaves_without_parents(&account, &leaves, message)
|
||||
.expect("create hierarchy");
|
||||
for name in structural {
|
||||
let quoted = format!("\"{name}\"");
|
||||
let line = listed
|
||||
.iter()
|
||||
.find(|l| l.ends_with("ed) || l.ends_with(&format!(" {name}")))
|
||||
.unwrap_or_else(|| panic!("{name:?} missing from LIST: {listed:?}"));
|
||||
let lower = line.to_ascii_lowercase();
|
||||
assert!(
|
||||
lower.contains("\\noselect") || lower.contains("\\nonexistent"),
|
||||
"{name:?} should be non-selectable: {line}"
|
||||
);
|
||||
}
|
||||
|
||||
let archive = tmp_archive("dovecot-noselect");
|
||||
for pass in 0..2 {
|
||||
let summary = sync::import_imap::run(common(&archive), dovecot_config(&account, &d.imap))
|
||||
.expect("imap import");
|
||||
assert!(!summary.any_failed(), "pass {pass}: {summary:?}");
|
||||
}
|
||||
let conn = open_archive(&archive);
|
||||
for path in leaves.iter().chain(&structural) {
|
||||
let id = mailbox_id_by_path(&conn, path, '/')
|
||||
.unwrap_or_else(|| panic!("{path:?} not in the archive tree"));
|
||||
assert_eq!(mailbox_path(&conn, id, '/'), *path);
|
||||
}
|
||||
for path in structural {
|
||||
let id = mailbox_id_by_path(&conn, path, '/').expect("structural");
|
||||
assert_eq!(emails_in_mailbox(&conn, id), 0, "{path:?}");
|
||||
}
|
||||
for path in leaves {
|
||||
let id = mailbox_id_by_path(&conn, path, '/').expect("leaf");
|
||||
assert_eq!(emails_in_mailbox(&conn, id), 1, "{path:?}");
|
||||
}
|
||||
drop(conn);
|
||||
|
||||
let fx = seeder::provision(base_url()).expect("provision");
|
||||
let dst = fx.account(seeder::SYNC_OUT[0]).expect("sync-out target");
|
||||
let export = sync::export::run(
|
||||
CommonConfig {
|
||||
archive: archive.clone(),
|
||||
threads: 4,
|
||||
dry_run: false,
|
||||
max_retries: 5,
|
||||
allow_invalid_certs: true,
|
||||
logger: Logger::from_flags(false, 0),
|
||||
},
|
||||
ExportConfig {
|
||||
connect: ConnectConfig {
|
||||
url: fx.base_url.clone(),
|
||||
auth: Auth::Basic {
|
||||
user: dst.email.clone(),
|
||||
password: dst.password.clone(),
|
||||
},
|
||||
account: AccountSelector::Id(dst.account_id.clone()),
|
||||
},
|
||||
objects: None,
|
||||
prune: false,
|
||||
yes: true,
|
||||
},
|
||||
)
|
||||
.expect("export");
|
||||
assert!(!export.any_failed(), "export reported failures: {export:?}");
|
||||
|
||||
let jmap =
|
||||
seeder::jmap::Jmap::connect(&fx.base_url, &dst.email, &dst.password).expect("session");
|
||||
let resp = jmap
|
||||
.request(
|
||||
&["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"],
|
||||
json!([["Mailbox/get", {"accountId": dst.account_id, "ids": null}, "g"]]),
|
||||
)
|
||||
.expect("Mailbox/get");
|
||||
let list = resp["methodResponses"][0][1]["list"]
|
||||
.as_array()
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
let by_id: HashMap<&str, (&str, Option<&str>)> = list
|
||||
.iter()
|
||||
.filter_map(|m| {
|
||||
Some((
|
||||
m["id"].as_str()?,
|
||||
(m["name"].as_str()?, m["parentId"].as_str()),
|
||||
))
|
||||
})
|
||||
.collect();
|
||||
let paths: HashSet<String> = by_id
|
||||
.keys()
|
||||
.map(|id| {
|
||||
let mut parts = Vec::new();
|
||||
let mut cur = Some(*id);
|
||||
while let Some(c) = cur {
|
||||
let (name, parent) = by_id[c];
|
||||
parts.push(name);
|
||||
cur = parent;
|
||||
}
|
||||
parts.reverse();
|
||||
parts.join("/")
|
||||
})
|
||||
.collect();
|
||||
for path in leaves
|
||||
.iter()
|
||||
.chain(&structural)
|
||||
.filter(|p| !p.contains(' '))
|
||||
{
|
||||
assert!(
|
||||
paths.contains(*path),
|
||||
"{path:?} missing on target: {paths:?}"
|
||||
);
|
||||
}
|
||||
for orphan in ["Invoices", "Acme", "2024"] {
|
||||
assert!(
|
||||
!paths.contains(orphan),
|
||||
"{orphan:?} leaked to the top level"
|
||||
);
|
||||
}
|
||||
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
seeder::teardown(&fx.base_url).expect("teardown");
|
||||
}
|
||||
+123
-16
@@ -297,8 +297,8 @@ fn seed_deep_filenode_chain(jmap: &seeder::jmap::Jmap, account_id: &str, depth:
|
||||
serde_json::Value::String(format!("dir-{i}")),
|
||||
);
|
||||
obj.insert(
|
||||
"nodeType".to_owned(),
|
||||
serde_json::Value::String("directory".to_owned()),
|
||||
"@type".to_owned(),
|
||||
serde_json::Value::String("Directory".to_owned()),
|
||||
);
|
||||
if i > 0 {
|
||||
obj.insert(
|
||||
@@ -695,20 +695,44 @@ fn prune_non_tree_type_contact_card() {
|
||||
const ISSUE30_CARD_UID: &str = "inbuxa-migrate-issue30-card";
|
||||
const ISSUE30_EVENT_UID: &str = "inbuxa-migrate-issue30-event";
|
||||
|
||||
fn data_uri_bytes(resource: &Value, uri_key: &str, expect_media_type: &str) -> Vec<u8> {
|
||||
let uri = resource
|
||||
.get(uri_key)
|
||||
struct Target<'a> {
|
||||
client: &'a HttpClient,
|
||||
session: &'a Session,
|
||||
account: &'a str,
|
||||
}
|
||||
|
||||
fn resource_bytes(
|
||||
target: &Target<'_>,
|
||||
resource: &Value,
|
||||
uri_key: &str,
|
||||
type_key: &str,
|
||||
expect_media_type: &str,
|
||||
) -> Vec<u8> {
|
||||
if let Some(uri) = resource.get(uri_key).and_then(Value::as_str) {
|
||||
let prefix = format!("data:{expect_media_type};base64,");
|
||||
let payload = uri
|
||||
.strip_prefix(&prefix)
|
||||
.unwrap_or_else(|| panic!("{uri_key} is not a {prefix}... data URI: {resource}"));
|
||||
return STANDARD.decode(payload).expect("base64 payload");
|
||||
}
|
||||
let blob_id = resource
|
||||
.get("blobId")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_else(|| panic!("no {uri_key} on {resource}"));
|
||||
assert!(
|
||||
resource.get("blobId").is_none(),
|
||||
"target must not hold a blobId for {uri_key}: {resource}"
|
||||
.unwrap_or_else(|| panic!("neither {uri_key} nor blobId on {resource}"));
|
||||
assert_eq!(
|
||||
resource.get(type_key).and_then(Value::as_str),
|
||||
Some(expect_media_type),
|
||||
"embedded blob keeps its media type: {resource}"
|
||||
);
|
||||
let prefix = format!("data:{expect_media_type};base64,");
|
||||
let payload = uri
|
||||
.strip_prefix(&prefix)
|
||||
.unwrap_or_else(|| panic!("{uri_key} is not a {prefix}... data URI: {resource}"));
|
||||
STANDARD.decode(payload).expect("base64 payload")
|
||||
inbuxa_migrate::jmap::blobxfer::download_bytes(
|
||||
target.client,
|
||||
target.session,
|
||||
target.account,
|
||||
blob_id,
|
||||
expect_media_type,
|
||||
"blob",
|
||||
)
|
||||
.unwrap_or_else(|e| panic!("target blobId {blob_id} does not download: {e}"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -815,8 +839,13 @@ fn export_inlines_contact_and_event_blobs_instead_of_blob_ids() {
|
||||
let photo_resource = card
|
||||
.pointer("/media/photo")
|
||||
.unwrap_or_else(|| panic!("card has no media.photo: {card}"));
|
||||
let target = Target {
|
||||
client: &client,
|
||||
session: &session,
|
||||
account: &tgt.account_id,
|
||||
};
|
||||
assert_eq!(
|
||||
data_uri_bytes(photo_resource, "uri", "image/png"),
|
||||
resource_bytes(&target, photo_resource, "uri", "mediaType", "image/png"),
|
||||
photo,
|
||||
"contact photo bytes survived the export: {card}"
|
||||
);
|
||||
@@ -845,7 +874,7 @@ fn export_inlines_contact_and_event_blobs_instead_of_blob_ids() {
|
||||
.and_then(|m| m.values().next())
|
||||
.unwrap_or_else(|| panic!("event has no links: {event}"));
|
||||
assert_eq!(
|
||||
data_uri_bytes(link, "href", "text/plain"),
|
||||
resource_bytes(&target, link, "href", "contentType", "text/plain"),
|
||||
agenda,
|
||||
"calendar enclosure bytes survived the export: {event}"
|
||||
);
|
||||
@@ -933,6 +962,84 @@ fn export_round_trip_and_convergence() {
|
||||
seeder::teardown(base_url()).expect("teardown");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires Docker"]
|
||||
fn export_emails_in_multiple_import_batches() {
|
||||
// With maxSetObjects lowered to 2 the email export spans several Email/import batches and still converges.
|
||||
let fx = seeder::provision(base_url()).expect("provision");
|
||||
let src = fx.account("test1").expect("test1");
|
||||
let tgt = fx.account("test4").expect("test4");
|
||||
let archive = tmp_archive("batches");
|
||||
|
||||
let imp = ImportConfig {
|
||||
connect: ConnectConfig {
|
||||
url: base_url().to_owned(),
|
||||
auth: basic("test1"),
|
||||
account: AccountSelector::Id(src.account_id.clone()),
|
||||
},
|
||||
objects: None,
|
||||
allow_source_change: false,
|
||||
};
|
||||
let s = sync::import_jmap::run(common(&archive, false), imp).expect("import");
|
||||
assert!(!s.any_failed());
|
||||
let local_emails = {
|
||||
let conn = Connection::open(&archive).unwrap();
|
||||
count(&conn, "emails")
|
||||
};
|
||||
assert!(
|
||||
local_emails > 2,
|
||||
"need more than one batch of emails: {local_emails}"
|
||||
);
|
||||
|
||||
let mut small = Map::new();
|
||||
small.insert("maxSetObjects".to_owned(), json!(2));
|
||||
let _guard = JmapSettingsGuard::override_settings(small);
|
||||
|
||||
let probe = HttpClient::new(
|
||||
basic("test4"),
|
||||
RetryPolicy::new(5),
|
||||
CertOverride::for_url(true, base_url()),
|
||||
);
|
||||
let session = Session::discover(&probe, base_url()).expect("discover");
|
||||
let limits = session.core_limits().expect("core limits");
|
||||
assert_eq!(
|
||||
limits.max_objects_in_set, 2,
|
||||
"maxSetObjects override is advertised"
|
||||
);
|
||||
|
||||
let e1 = sync::export::run(
|
||||
common(&archive, false),
|
||||
export_cfg("test4", &tgt.account_id, false),
|
||||
)
|
||||
.expect("export 1");
|
||||
assert!(
|
||||
!e1.per_type.iter().any(|(_, c)| c.failed > 0),
|
||||
"export 1 had failures: {e1:?}"
|
||||
);
|
||||
let emails = e1
|
||||
.per_type
|
||||
.iter()
|
||||
.find(|(t, _)| *t == "Email")
|
||||
.map(|(_, c)| c.created)
|
||||
.unwrap_or(0);
|
||||
assert_eq!(
|
||||
emails as i64, local_emails,
|
||||
"every email created across batches"
|
||||
);
|
||||
|
||||
let e2 = sync::export::run(
|
||||
common(&archive, false),
|
||||
export_cfg("test4", &tgt.account_id, false),
|
||||
)
|
||||
.expect("export 2");
|
||||
let created_2: u64 = e2.per_type.iter().map(|(_, c)| c.created).sum();
|
||||
assert_eq!(created_2, 0, "second export is convergent: {e2:?}");
|
||||
|
||||
drop(_guard);
|
||||
let _ = std::fs::remove_file(&archive);
|
||||
seeder::teardown(base_url()).expect("teardown");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires Docker"]
|
||||
fn export_prune_destroys_unmatched() {
|
||||
|
||||
@@ -20,7 +20,7 @@ fn base_url() -> &'static str {
|
||||
use inbuxa_migrate::jmap::account::AccountSelector;
|
||||
use inbuxa_migrate::jmap::http::Auth;
|
||||
use inbuxa_migrate::logging::Logger;
|
||||
use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg};
|
||||
use inbuxa_migrate::sync::import_dav::{DavAuth, DavImportConfig, DavKindArg, SourceChange};
|
||||
use inbuxa_migrate::sync::{self, CommonConfig, ConnectConfig, ExportConfig, ImportConfig};
|
||||
|
||||
fn tmp_archive(tag: &str) -> PathBuf {
|
||||
@@ -59,7 +59,7 @@ fn webdav_cfg(root_url: String, domain: &str, localpart: &str) -> DavImportConfi
|
||||
allow_cleartext: false,
|
||||
dav_connections: 4,
|
||||
multiget_batch: 50,
|
||||
allow_source_change: false,
|
||||
source_change: SourceChange::Reject,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user