A message with two Message-ID headers is keyed by both in the archive, while a JMAP server's messageId reflects one, so export never paired it and imported it again on every run (shown live on inbuxa 27.1-rc.4). While target emails are left unpaired, a batch about to be imported is parsed by the target first (Email/parse) and each message it already holds under that key is matched, with its keywords and folders synced like any other match. A target that refuses Email/parse is not asked again; any other failure leaves that batch unchecked, imported as before. The parsed key also settles an import that ended without a clear answer. Upstream fixed the same case in v1.0.17 (#86); written here on our own code and tests.
515 lines
17 KiB
Rust
515 lines
17 KiB
Rust
/*
|
|
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC <[email protected]>
|
|
*
|
|
* SPDX-License-Identifier: Apache-2.0 OR MIT
|
|
*/
|
|
|
|
//! Exports into inbuxa itself, an image pinned by digest. Run before each
|
|
//! release: `cargo test --test integration_inbuxa -- --ignored`.
|
|
|
|
mod integration;
|
|
|
|
use std::fs;
|
|
use std::path::Path;
|
|
use std::process::Command;
|
|
|
|
use inbuxa_migrate::db;
|
|
use integration::guide::{post, settled, start};
|
|
use integration::inbuxa::{ADMIN_EMAIL, ADMIN_PASSWORD, Inbuxa};
|
|
use serde_json::{Value, json};
|
|
use tempfile::TempDir;
|
|
|
|
const BIN: &str = env!("CARGO_BIN_EXE_inbuxa-migrate");
|
|
|
|
fn message(folder: &Path, file: &str, id: &str, subject: &str) {
|
|
for sub in ["cur", "new", "tmp"] {
|
|
fs::create_dir_all(folder.join(sub)).unwrap();
|
|
}
|
|
fs::write(
|
|
folder.join("cur").join(file),
|
|
format!(
|
|
"From: [email protected]\r\nTo: [email protected]\r\nSubject: {subject}\r\n\
|
|
Message-ID: <{id}@example.org>\r\nDate: Mon, 4 Mar 2024 09:00:00 +0000\r\n\r\nHi\r\n"
|
|
),
|
|
)
|
|
.unwrap();
|
|
}
|
|
|
|
fn user(email: &str) -> Value {
|
|
let (name, domain) = email.split_once('@').unwrap();
|
|
json!({
|
|
"email": email, "name": name, "domain": domain, "description": null,
|
|
"roles": "user", "permissions": null, "aliases": [], "quotas": {},
|
|
"tenant": null, "locale": null, "timeZone": null, "externalId": null, "groups": [],
|
|
})
|
|
}
|
|
|
|
/// Each email in the account as (subject, number of mailboxes, keywords).
|
|
fn emails(server: &Inbuxa, email: &str, password: &str) -> Vec<(String, usize, Vec<String>)> {
|
|
let account = server.account_of(email, password).unwrap();
|
|
let r = server
|
|
.call(
|
|
email,
|
|
password,
|
|
json!([
|
|
["Email/query", {"accountId": account}, "q"],
|
|
["Email/get", {"accountId": account,
|
|
"#ids": {"resultOf": "q", "name": "Email/query", "path": "/ids"},
|
|
"properties": ["subject", "mailboxIds", "keywords"]}, "g"],
|
|
]),
|
|
)
|
|
.unwrap();
|
|
let mut out: Vec<_> = r[1][1]["list"]
|
|
.as_array()
|
|
.unwrap_or_else(|| panic!("{r}"))
|
|
.iter()
|
|
.map(|e| {
|
|
let mut keywords: Vec<String> = e["keywords"]
|
|
.as_object()
|
|
.map(|k| k.keys().cloned().collect())
|
|
.unwrap_or_default();
|
|
keywords.sort();
|
|
(
|
|
e["subject"].as_str().unwrap_or_default().to_owned(),
|
|
e["mailboxIds"].as_object().map_or(0, |m| m.len()),
|
|
keywords,
|
|
)
|
|
})
|
|
.collect();
|
|
out.sort();
|
|
out
|
|
}
|
|
|
|
fn report(work: &Path) -> Vec<String> {
|
|
fs::read_to_string(work.join("report.csv"))
|
|
.unwrap()
|
|
.lines()
|
|
.skip(1)
|
|
.map(|l| l.split(',').take(9).collect::<Vec<_>>().join(","))
|
|
.collect()
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires Docker"]
|
|
fn a_plan_sets_up_the_accounts_and_moves_each_with_one_admin_token() {
|
|
let server = Inbuxa::start().expect("start inbuxa");
|
|
let dir = TempDir::new().unwrap();
|
|
let root = dir.path();
|
|
|
|
fs::create_dir_all(root.join("directory")).unwrap();
|
|
let directory = json!({
|
|
"format": "inbuxa-migrate-directory", "formatVersion": 1,
|
|
"domains": [{"name": "other.test", "enabled": true, "description": null, "aliases": [],
|
|
"catchAllAddress": null, "subAddressing": "enabled", "tenant": null}],
|
|
"tenants": [], "roles": [], "groups": [], "lists": [],
|
|
"users": [user("[email protected]"), user("[email protected]")],
|
|
});
|
|
fs::write(root.join("directory/directory.json"), directory.to_string()).unwrap();
|
|
// Ann's message is in her inbox and in Archive: one email, two mailboxes.
|
|
message(&root.join("mail/ann"), "1:2,S", "ann-1", "For Ann");
|
|
message(&root.join("mail/ann/.Archive"), "1:2,S", "ann-1", "For Ann");
|
|
message(&root.join("mail/bob"), "1:2,", "bob-1", "For Bob");
|
|
|
|
let url = server.base_url.as_str();
|
|
let plan = json!({
|
|
"format": "inbuxa-migrate-plan", "formatVersion": 1,
|
|
"setup": [{"name": "directory",
|
|
"args": ["import-directory", "--url", url, "--auth-bearer",
|
|
"--passwords", "passwords.csv", "directory"],
|
|
"secrets": {"token": "ITEST_INBUXA_TOKEN"}}],
|
|
"accounts": {"directory": "directory/directory.json"},
|
|
"steps": [
|
|
{"name": "mail", "args": ["import", "maildir", "mail/{local}", "{archive}"]},
|
|
{"name": "inbuxa",
|
|
"args": ["export", "--url", url, "--auth-bearer", "--account-name", "{email}",
|
|
"--objects", "mailbox,email", "{archive}"],
|
|
"secrets": {"token": "ITEST_INBUXA_TOKEN"}},
|
|
],
|
|
});
|
|
fs::write(root.join("plan.json"), plan.to_string()).unwrap();
|
|
let work = root.join("work");
|
|
let migrate = |again: bool| {
|
|
let mut command = Command::new(BIN);
|
|
command
|
|
.args(["migrate", "-q", "--plan"])
|
|
.arg(root.join("plan.json"))
|
|
.arg(&work)
|
|
.env("ITEST_INBUXA_TOKEN", &server.token);
|
|
if again {
|
|
command.arg("--again");
|
|
}
|
|
let out = command.output().unwrap();
|
|
assert!(
|
|
out.status.success(),
|
|
"{}\n{}",
|
|
String::from_utf8_lossy(&out.stderr),
|
|
fs::read_to_string(work.join("report.csv")).unwrap_or_default()
|
|
);
|
|
};
|
|
|
|
migrate(false);
|
|
let rows = report(&work);
|
|
assert_eq!(rows[0], "(setup),directory,done,0,3,0,0,0,0", "{rows:?}");
|
|
let passwords = fs::read_to_string(root.join("passwords.csv")).unwrap();
|
|
let password = |email: &str| {
|
|
passwords
|
|
.lines()
|
|
.find_map(|l| l.strip_prefix(&format!("{email},")))
|
|
.unwrap_or_else(|| panic!("{email} not in {passwords}"))
|
|
.to_owned()
|
|
};
|
|
let (ann, bob) = (password("[email protected]"), password("[email protected]"));
|
|
let seen = vec!["$seen".to_owned()];
|
|
assert_eq!(
|
|
emails(&server, "[email protected]", &ann),
|
|
[("For Ann".to_owned(), 2, seen)]
|
|
);
|
|
assert_eq!(
|
|
emails(&server, "[email protected]", &bob),
|
|
[("For Bob".to_owned(), 1, Vec::new())]
|
|
);
|
|
// The administrator's own mailbox got nothing.
|
|
assert_eq!(emails(&server, ADMIN_EMAIL, ADMIN_PASSWORD), []);
|
|
|
|
// The final pass finds nothing new and sends nothing.
|
|
migrate(true);
|
|
let rows = report(&work);
|
|
assert!(
|
|
rows.iter()
|
|
.filter(|r| r.contains(",inbuxa,"))
|
|
.all(|r| r.contains(",inbuxa,done,0,0,0,0,")),
|
|
"{rows:?}"
|
|
);
|
|
assert_eq!(emails(&server, "[email protected]", &bob).len(), 1);
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires Docker"]
|
|
fn a_second_export_updates_flags_and_sieve_names_are_renamed() {
|
|
let server = Inbuxa::start().expect("start inbuxa");
|
|
let dir = TempDir::new().unwrap();
|
|
let maildir = dir.path().join("mail");
|
|
let archive = dir.path().join("archive.sqlite");
|
|
message(&maildir, "1:2,S", "m-1", "Flag me");
|
|
|
|
let run = |args: &[&str]| {
|
|
let out = Command::new(BIN)
|
|
.args(args)
|
|
.env("INBUXA_MIGRATE_API_KEY", &server.token)
|
|
.output()
|
|
.unwrap();
|
|
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
|
|
assert!(out.status.success(), "{args:?}: {stderr}");
|
|
(String::from_utf8_lossy(&out.stdout).into_owned(), stderr)
|
|
};
|
|
let import = || {
|
|
run(&[
|
|
"import",
|
|
"maildir",
|
|
maildir.to_str().unwrap(),
|
|
archive.to_str().unwrap(),
|
|
])
|
|
};
|
|
let export = || {
|
|
run(&[
|
|
"export",
|
|
"--url",
|
|
&server.base_url,
|
|
"--auth-bearer",
|
|
"--account-name",
|
|
ADMIN_EMAIL,
|
|
archive.to_str().unwrap(),
|
|
])
|
|
};
|
|
|
|
import();
|
|
{
|
|
let conn = db::init::open(&archive).unwrap();
|
|
let blob = db::blobs::intern_blob(
|
|
&conn,
|
|
b"require [\"fileinto\", \"vnd.stalwart.while\"];\nkeep;\n",
|
|
)
|
|
.unwrap();
|
|
conn.execute(
|
|
"INSERT INTO sieve_scripts (name, is_active, blob_id) VALUES ('main', 1, ?1)",
|
|
rusqlite::params![blob],
|
|
)
|
|
.unwrap();
|
|
}
|
|
let (_, stderr) = export();
|
|
assert!(
|
|
stderr.contains("renamed vnd.stalwart.while to vnd.inbuxa.while"),
|
|
"{stderr}"
|
|
);
|
|
assert_eq!(
|
|
emails(&server, ADMIN_EMAIL, ADMIN_PASSWORD),
|
|
[("Flag me".to_owned(), 1, vec!["$seen".to_owned()])]
|
|
);
|
|
let account = server.account_of(ADMIN_EMAIL, ADMIN_PASSWORD).unwrap();
|
|
let r = server
|
|
.call(
|
|
ADMIN_EMAIL,
|
|
ADMIN_PASSWORD,
|
|
json!([["SieveScript/get", {"accountId": account}, "s"]]),
|
|
)
|
|
.unwrap();
|
|
let script = r[0][1]["list"]
|
|
.as_array()
|
|
.and_then(|l| l.iter().find(|s| s["name"] == "main"))
|
|
.unwrap_or_else(|| panic!("no script: {r}"))
|
|
.clone();
|
|
assert_eq!(script["isActive"], true, "{script}");
|
|
let body = server
|
|
.download(
|
|
ADMIN_EMAIL,
|
|
ADMIN_PASSWORD,
|
|
&account,
|
|
script["blobId"].as_str().unwrap(),
|
|
)
|
|
.unwrap();
|
|
let body = String::from_utf8_lossy(&body);
|
|
assert!(
|
|
body.contains("vnd.inbuxa.while") && !body.contains("vnd.stalwart"),
|
|
"{body}"
|
|
);
|
|
|
|
// Flagged at the source: the next import and export carry it over.
|
|
fs::rename(maildir.join("cur/1:2,S"), maildir.join("cur/1:2,FS")).unwrap();
|
|
import();
|
|
let (stdout, _) = export();
|
|
assert!(
|
|
stdout.contains("Email: created=0 fetched=0 updated=1"),
|
|
"{stdout}"
|
|
);
|
|
assert_eq!(
|
|
emails(&server, ADMIN_EMAIL, ADMIN_PASSWORD),
|
|
[(
|
|
"Flag me".to_owned(),
|
|
1,
|
|
vec!["$flagged".to_owned(), "$seen".to_owned()]
|
|
)]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires Docker"]
|
|
fn the_guide_sets_up_the_accounts_previews_and_moves_them() {
|
|
let server = Inbuxa::start().expect("start inbuxa");
|
|
let dir = TempDir::new().unwrap();
|
|
let root = dir.path();
|
|
message(
|
|
&root.join("mail/example.org/cat"),
|
|
"1:2,S",
|
|
"cat-1",
|
|
"For Cat",
|
|
);
|
|
message(
|
|
&root.join("mail/other.test/dan"),
|
|
"1:2,",
|
|
"dan-1",
|
|
"For Dan",
|
|
);
|
|
let guide = start(BIN, root);
|
|
let form = json!({
|
|
"source": {"kind": "maildir", "path": "mail/{domain}/{local}"},
|
|
"accounts": ["[email protected]", "[email protected]"],
|
|
"create": true,
|
|
"target": {"url": server.base_url, "token": server.token},
|
|
"parallel": 2,
|
|
});
|
|
let (_, checked) = post(&guide, "api/check-target", &json!({"form": form}));
|
|
assert_eq!(checked["ok"], true, "{checked}");
|
|
let job = |kind: &str| {
|
|
let (status, body) = post(&guide, "api/job", &json!({"kind": kind, "form": form}));
|
|
assert_eq!(status, 200, "{body}");
|
|
let state = settled(&guide);
|
|
assert_eq!(state["job"]["exit"], 0, "{state}");
|
|
state["job"]["result"].clone()
|
|
};
|
|
|
|
// The preview creates nothing and reads each account's folder.
|
|
let preview = job("preview");
|
|
let report = preview["report"].as_str().unwrap();
|
|
assert!(report.contains("(setup),accounts,done,"), "{report}");
|
|
assert!(
|
|
report.contains("[email protected],import,done,2,2"),
|
|
"{report}"
|
|
);
|
|
assert!(server.account_of("[email protected]", "x").is_err());
|
|
|
|
let run = job("run");
|
|
let passwords =
|
|
fs::read_to_string(run["passwords"].as_str().expect("a passwords file")).unwrap();
|
|
let password = |email: &str| {
|
|
passwords
|
|
.lines()
|
|
.find_map(|l| l.strip_prefix(&format!("{email},")))
|
|
.unwrap_or_else(|| panic!("{email} not in {passwords}"))
|
|
.to_owned()
|
|
};
|
|
assert_eq!(
|
|
emails(&server, "[email protected]", &password("[email protected]")),
|
|
[("For Cat".to_owned(), 1, vec!["$seen".to_owned()])]
|
|
);
|
|
assert_eq!(
|
|
emails(&server, "[email protected]", &password("[email protected]")),
|
|
[("For Dan".to_owned(), 1, Vec::new())]
|
|
);
|
|
|
|
// The final pass finds nothing new.
|
|
let last = job("final");
|
|
let report = last["report"].as_str().unwrap();
|
|
assert!(
|
|
report
|
|
.lines()
|
|
.filter(|r| r.contains(",export,"))
|
|
.all(|r| r.contains(",export,done,0,0,0,0,")),
|
|
"{report}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires Docker"]
|
|
fn files_go_into_inbuxa_in_its_own_shape_and_come_back() {
|
|
let server = Inbuxa::start().expect("start inbuxa");
|
|
let dir = TempDir::new().unwrap();
|
|
let out = dir.path().join("out.sqlite");
|
|
let back = dir.path().join("back.sqlite");
|
|
{
|
|
let conn = db::init::open(&out).unwrap();
|
|
let blob = db::blobs::intern_blob(&conn, b"#!/bin/sh\necho hi\n").unwrap();
|
|
conn.execute(
|
|
"INSERT INTO file_nodes (id,parent_id,node_type,blob_id,name,media_type,created)
|
|
VALUES (1,NULL,'directory',NULL,'docs',NULL,'2022-02-02T02:02:02Z'),
|
|
(2,1,'file',?1,'run.sh','text/x-sh','2022-02-02T02:02:02Z')",
|
|
[blob],
|
|
)
|
|
.unwrap();
|
|
}
|
|
let run = |args: &[&str]| {
|
|
let out = Command::new(BIN)
|
|
.args(args)
|
|
.env("INBUXA_MIGRATE_API_KEY", &server.token)
|
|
.output()
|
|
.unwrap();
|
|
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
|
|
assert!(out.status.success(), "{args:?}: {stderr}");
|
|
};
|
|
let target = [
|
|
"--url",
|
|
server.base_url.as_str(),
|
|
"--auth-bearer",
|
|
"--account-name",
|
|
ADMIN_EMAIL,
|
|
"--objects",
|
|
"filenode",
|
|
];
|
|
run(&[&["export"][..], &target, &[out.to_str().unwrap()]].concat());
|
|
|
|
let account = server.account_of(ADMIN_EMAIL, ADMIN_PASSWORD).unwrap();
|
|
let r = server
|
|
.call(
|
|
ADMIN_EMAIL,
|
|
ADMIN_PASSWORD,
|
|
json!([["FileNode/query", {"accountId": account}, "q"],
|
|
["FileNode/get", {"accountId": account,
|
|
"#ids": {"resultOf": "q", "name": "FileNode/query", "path": "/ids"},
|
|
"properties": ["name", "nodeType", "type", "parentId"]}, "g"]]),
|
|
)
|
|
.unwrap();
|
|
let list = r[1][1]["list"].as_array().unwrap_or_else(|| panic!("{r}"));
|
|
let file = list
|
|
.iter()
|
|
.find(|n| n["name"] == "run.sh")
|
|
.unwrap_or_else(|| panic!("no file: {r}"));
|
|
assert_eq!(file["nodeType"], "file", "{file}");
|
|
assert_eq!(file["type"], "text/x-sh", "{file}");
|
|
assert!(
|
|
list.iter()
|
|
.any(|n| n["name"] == "docs" && n["nodeType"] == "directory"),
|
|
"{r}"
|
|
);
|
|
|
|
run(&[&["import", "jmap"][..], &target, &[back.to_str().unwrap()]].concat());
|
|
let conn = db::init::open(&back).unwrap();
|
|
let mut rows: Vec<(String, String, Option<String>)> = conn
|
|
.prepare(
|
|
"SELECT name, node_type, media_type FROM file_nodes WHERE name IN ('docs','run.sh')",
|
|
)
|
|
.unwrap()
|
|
.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))
|
|
.unwrap()
|
|
.collect::<Result<_, _>>()
|
|
.unwrap();
|
|
rows.sort();
|
|
assert_eq!(
|
|
rows,
|
|
[
|
|
("docs".to_owned(), "directory".to_owned(), None),
|
|
(
|
|
"run.sh".to_owned(),
|
|
"file".to_owned(),
|
|
Some("text/x-sh".to_owned())
|
|
)
|
|
]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires Docker"]
|
|
fn a_message_with_two_message_ids_is_not_exported_twice() {
|
|
let server = Inbuxa::start().expect("start inbuxa");
|
|
let dir = TempDir::new().unwrap();
|
|
let maildir = dir.path().join("mail");
|
|
let archive = dir.path().join("archive.sqlite");
|
|
for sub in ["cur", "new", "tmp"] {
|
|
fs::create_dir_all(maildir.join(sub)).unwrap();
|
|
}
|
|
// The archive keys it by both Message-IDs; inbuxa's messageId has one.
|
|
fs::write(
|
|
maildir.join("cur/1:2,S"),
|
|
"From: [email protected]\r\nTo: [email protected]\r\nSubject: Twice\r\n\
|
|
Message-ID: <[email protected]>\r\nMessage-ID: <[email protected]>\r\n\
|
|
Date: Mon, 4 Mar 2024 09:00:00 +0000\r\n\r\nHi\r\n",
|
|
)
|
|
.unwrap();
|
|
|
|
let run = |args: &[&str]| {
|
|
let out = Command::new(BIN)
|
|
.args(args)
|
|
.env("INBUXA_MIGRATE_API_KEY", &server.token)
|
|
.output()
|
|
.unwrap();
|
|
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
|
|
assert!(out.status.success(), "{args:?}: {stderr}");
|
|
String::from_utf8_lossy(&out.stdout).into_owned()
|
|
};
|
|
run(&[
|
|
"import",
|
|
"maildir",
|
|
maildir.to_str().unwrap(),
|
|
archive.to_str().unwrap(),
|
|
]);
|
|
let export = || {
|
|
run(&[
|
|
"export",
|
|
"--url",
|
|
&server.base_url,
|
|
"--auth-bearer",
|
|
"--account-name",
|
|
ADMIN_EMAIL,
|
|
"--objects",
|
|
"mailbox,email",
|
|
archive.to_str().unwrap(),
|
|
])
|
|
};
|
|
let first = export();
|
|
assert!(first.contains("Email: created=1"), "{first}");
|
|
let second = export();
|
|
assert!(second.contains("Email: created=0"), "{second}");
|
|
assert_eq!(
|
|
emails(&server, ADMIN_EMAIL, ADMIN_PASSWORD),
|
|
[("Twice".to_owned(), 1, vec!["$seen".to_owned()])]
|
|
);
|
|
}
|