Files
inbuxa-migrate/tests/integration_inbuxa.rs
jcoffey-dev 34981475f2
ci / test (pull_request) Skipped
github/ci (branch) GitHub Actions
ci / github (pull_request) Successful in 3m37s
ci / announce (pull_request) Skipped
Export: match a message the target holds under its own key
A message with two Message-ID headers is keyed by both in the archive,
while a JMAP server's messageId reflects one, so export never paired it
and imported it again on every run (shown live on inbuxa 27.1-rc.4).

While target emails are left unpaired, a batch about to be imported is
parsed by the target first (Email/parse) and each message it already
holds under that key is matched, with its keywords and folders synced
like any other match. A target that refuses Email/parse is not asked
again; any other failure leaves that batch unchecked, imported as before.
The parsed key also settles an import that ended without a clear answer.

Upstream fixed the same case in v1.0.17 (#86); written here on our own
code and tests.
2026-11-16 23:52:18 -07:00

515 lines
17 KiB
Rust

/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC <[email protected]>
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*/
//! Exports into inbuxa itself, an image pinned by digest. Run before each
//! release: `cargo test --test integration_inbuxa -- --ignored`.
mod integration;
use std::fs;
use std::path::Path;
use std::process::Command;
use inbuxa_migrate::db;
use integration::guide::{post, settled, start};
use integration::inbuxa::{ADMIN_EMAIL, ADMIN_PASSWORD, Inbuxa};
use serde_json::{Value, json};
use tempfile::TempDir;
const BIN: &str = env!("CARGO_BIN_EXE_inbuxa-migrate");
fn message(folder: &Path, file: &str, id: &str, subject: &str) {
for sub in ["cur", "new", "tmp"] {
fs::create_dir_all(folder.join(sub)).unwrap();
}
fs::write(
folder.join("cur").join(file),
format!(
"From: [email protected]\r\nTo: [email protected]\r\nSubject: {subject}\r\n\
Message-ID: <{id}@example.org>\r\nDate: Mon, 4 Mar 2024 09:00:00 +0000\r\n\r\nHi\r\n"
),
)
.unwrap();
}
fn user(email: &str) -> Value {
let (name, domain) = email.split_once('@').unwrap();
json!({
"email": email, "name": name, "domain": domain, "description": null,
"roles": "user", "permissions": null, "aliases": [], "quotas": {},
"tenant": null, "locale": null, "timeZone": null, "externalId": null, "groups": [],
})
}
/// Each email in the account as (subject, number of mailboxes, keywords).
fn emails(server: &Inbuxa, email: &str, password: &str) -> Vec<(String, usize, Vec<String>)> {
let account = server.account_of(email, password).unwrap();
let r = server
.call(
email,
password,
json!([
["Email/query", {"accountId": account}, "q"],
["Email/get", {"accountId": account,
"#ids": {"resultOf": "q", "name": "Email/query", "path": "/ids"},
"properties": ["subject", "mailboxIds", "keywords"]}, "g"],
]),
)
.unwrap();
let mut out: Vec<_> = r[1][1]["list"]
.as_array()
.unwrap_or_else(|| panic!("{r}"))
.iter()
.map(|e| {
let mut keywords: Vec<String> = e["keywords"]
.as_object()
.map(|k| k.keys().cloned().collect())
.unwrap_or_default();
keywords.sort();
(
e["subject"].as_str().unwrap_or_default().to_owned(),
e["mailboxIds"].as_object().map_or(0, |m| m.len()),
keywords,
)
})
.collect();
out.sort();
out
}
fn report(work: &Path) -> Vec<String> {
fs::read_to_string(work.join("report.csv"))
.unwrap()
.lines()
.skip(1)
.map(|l| l.split(',').take(9).collect::<Vec<_>>().join(","))
.collect()
}
#[test]
#[ignore = "requires Docker"]
fn a_plan_sets_up_the_accounts_and_moves_each_with_one_admin_token() {
let server = Inbuxa::start().expect("start inbuxa");
let dir = TempDir::new().unwrap();
let root = dir.path();
fs::create_dir_all(root.join("directory")).unwrap();
let directory = json!({
"format": "inbuxa-migrate-directory", "formatVersion": 1,
"domains": [{"name": "other.test", "enabled": true, "description": null, "aliases": [],
"catchAllAddress": null, "subAddressing": "enabled", "tenant": null}],
"tenants": [], "roles": [], "groups": [], "lists": [],
"users": [user("[email protected]"), user("[email protected]")],
});
fs::write(root.join("directory/directory.json"), directory.to_string()).unwrap();
// Ann's message is in her inbox and in Archive: one email, two mailboxes.
message(&root.join("mail/ann"), "1:2,S", "ann-1", "For Ann");
message(&root.join("mail/ann/.Archive"), "1:2,S", "ann-1", "For Ann");
message(&root.join("mail/bob"), "1:2,", "bob-1", "For Bob");
let url = server.base_url.as_str();
let plan = json!({
"format": "inbuxa-migrate-plan", "formatVersion": 1,
"setup": [{"name": "directory",
"args": ["import-directory", "--url", url, "--auth-bearer",
"--passwords", "passwords.csv", "directory"],
"secrets": {"token": "ITEST_INBUXA_TOKEN"}}],
"accounts": {"directory": "directory/directory.json"},
"steps": [
{"name": "mail", "args": ["import", "maildir", "mail/{local}", "{archive}"]},
{"name": "inbuxa",
"args": ["export", "--url", url, "--auth-bearer", "--account-name", "{email}",
"--objects", "mailbox,email", "{archive}"],
"secrets": {"token": "ITEST_INBUXA_TOKEN"}},
],
});
fs::write(root.join("plan.json"), plan.to_string()).unwrap();
let work = root.join("work");
let migrate = |again: bool| {
let mut command = Command::new(BIN);
command
.args(["migrate", "-q", "--plan"])
.arg(root.join("plan.json"))
.arg(&work)
.env("ITEST_INBUXA_TOKEN", &server.token);
if again {
command.arg("--again");
}
let out = command.output().unwrap();
assert!(
out.status.success(),
"{}\n{}",
String::from_utf8_lossy(&out.stderr),
fs::read_to_string(work.join("report.csv")).unwrap_or_default()
);
};
migrate(false);
let rows = report(&work);
assert_eq!(rows[0], "(setup),directory,done,0,3,0,0,0,0", "{rows:?}");
let passwords = fs::read_to_string(root.join("passwords.csv")).unwrap();
let password = |email: &str| {
passwords
.lines()
.find_map(|l| l.strip_prefix(&format!("{email},")))
.unwrap_or_else(|| panic!("{email} not in {passwords}"))
.to_owned()
};
let (ann, bob) = (password("[email protected]"), password("[email protected]"));
let seen = vec!["$seen".to_owned()];
assert_eq!(
emails(&server, "[email protected]", &ann),
[("For Ann".to_owned(), 2, seen)]
);
assert_eq!(
emails(&server, "[email protected]", &bob),
[("For Bob".to_owned(), 1, Vec::new())]
);
// The administrator's own mailbox got nothing.
assert_eq!(emails(&server, ADMIN_EMAIL, ADMIN_PASSWORD), []);
// The final pass finds nothing new and sends nothing.
migrate(true);
let rows = report(&work);
assert!(
rows.iter()
.filter(|r| r.contains(",inbuxa,"))
.all(|r| r.contains(",inbuxa,done,0,0,0,0,")),
"{rows:?}"
);
assert_eq!(emails(&server, "[email protected]", &bob).len(), 1);
}
#[test]
#[ignore = "requires Docker"]
fn a_second_export_updates_flags_and_sieve_names_are_renamed() {
let server = Inbuxa::start().expect("start inbuxa");
let dir = TempDir::new().unwrap();
let maildir = dir.path().join("mail");
let archive = dir.path().join("archive.sqlite");
message(&maildir, "1:2,S", "m-1", "Flag me");
let run = |args: &[&str]| {
let out = Command::new(BIN)
.args(args)
.env("INBUXA_MIGRATE_API_KEY", &server.token)
.output()
.unwrap();
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
assert!(out.status.success(), "{args:?}: {stderr}");
(String::from_utf8_lossy(&out.stdout).into_owned(), stderr)
};
let import = || {
run(&[
"import",
"maildir",
maildir.to_str().unwrap(),
archive.to_str().unwrap(),
])
};
let export = || {
run(&[
"export",
"--url",
&server.base_url,
"--auth-bearer",
"--account-name",
ADMIN_EMAIL,
archive.to_str().unwrap(),
])
};
import();
{
let conn = db::init::open(&archive).unwrap();
let blob = db::blobs::intern_blob(
&conn,
b"require [\"fileinto\", \"vnd.stalwart.while\"];\nkeep;\n",
)
.unwrap();
conn.execute(
"INSERT INTO sieve_scripts (name, is_active, blob_id) VALUES ('main', 1, ?1)",
rusqlite::params![blob],
)
.unwrap();
}
let (_, stderr) = export();
assert!(
stderr.contains("renamed vnd.stalwart.while to vnd.inbuxa.while"),
"{stderr}"
);
assert_eq!(
emails(&server, ADMIN_EMAIL, ADMIN_PASSWORD),
[("Flag me".to_owned(), 1, vec!["$seen".to_owned()])]
);
let account = server.account_of(ADMIN_EMAIL, ADMIN_PASSWORD).unwrap();
let r = server
.call(
ADMIN_EMAIL,
ADMIN_PASSWORD,
json!([["SieveScript/get", {"accountId": account}, "s"]]),
)
.unwrap();
let script = r[0][1]["list"]
.as_array()
.and_then(|l| l.iter().find(|s| s["name"] == "main"))
.unwrap_or_else(|| panic!("no script: {r}"))
.clone();
assert_eq!(script["isActive"], true, "{script}");
let body = server
.download(
ADMIN_EMAIL,
ADMIN_PASSWORD,
&account,
script["blobId"].as_str().unwrap(),
)
.unwrap();
let body = String::from_utf8_lossy(&body);
assert!(
body.contains("vnd.inbuxa.while") && !body.contains("vnd.stalwart"),
"{body}"
);
// Flagged at the source: the next import and export carry it over.
fs::rename(maildir.join("cur/1:2,S"), maildir.join("cur/1:2,FS")).unwrap();
import();
let (stdout, _) = export();
assert!(
stdout.contains("Email: created=0 fetched=0 updated=1"),
"{stdout}"
);
assert_eq!(
emails(&server, ADMIN_EMAIL, ADMIN_PASSWORD),
[(
"Flag me".to_owned(),
1,
vec!["$flagged".to_owned(), "$seen".to_owned()]
)]
);
}
#[test]
#[ignore = "requires Docker"]
fn the_guide_sets_up_the_accounts_previews_and_moves_them() {
let server = Inbuxa::start().expect("start inbuxa");
let dir = TempDir::new().unwrap();
let root = dir.path();
message(
&root.join("mail/example.org/cat"),
"1:2,S",
"cat-1",
"For Cat",
);
message(
&root.join("mail/other.test/dan"),
"1:2,",
"dan-1",
"For Dan",
);
let guide = start(BIN, root);
let form = json!({
"source": {"kind": "maildir", "path": "mail/{domain}/{local}"},
"accounts": ["[email protected]", "[email protected]"],
"create": true,
"target": {"url": server.base_url, "token": server.token},
"parallel": 2,
});
let (_, checked) = post(&guide, "api/check-target", &json!({"form": form}));
assert_eq!(checked["ok"], true, "{checked}");
let job = |kind: &str| {
let (status, body) = post(&guide, "api/job", &json!({"kind": kind, "form": form}));
assert_eq!(status, 200, "{body}");
let state = settled(&guide);
assert_eq!(state["job"]["exit"], 0, "{state}");
state["job"]["result"].clone()
};
// The preview creates nothing and reads each account's folder.
let preview = job("preview");
let report = preview["report"].as_str().unwrap();
assert!(report.contains("(setup),accounts,done,"), "{report}");
assert!(
report.contains("[email protected],import,done,2,2"),
"{report}"
);
assert!(server.account_of("[email protected]", "x").is_err());
let run = job("run");
let passwords =
fs::read_to_string(run["passwords"].as_str().expect("a passwords file")).unwrap();
let password = |email: &str| {
passwords
.lines()
.find_map(|l| l.strip_prefix(&format!("{email},")))
.unwrap_or_else(|| panic!("{email} not in {passwords}"))
.to_owned()
};
assert_eq!(
emails(&server, "[email protected]", &password("[email protected]")),
[("For Cat".to_owned(), 1, vec!["$seen".to_owned()])]
);
assert_eq!(
emails(&server, "[email protected]", &password("[email protected]")),
[("For Dan".to_owned(), 1, Vec::new())]
);
// The final pass finds nothing new.
let last = job("final");
let report = last["report"].as_str().unwrap();
assert!(
report
.lines()
.filter(|r| r.contains(",export,"))
.all(|r| r.contains(",export,done,0,0,0,0,")),
"{report}"
);
}
#[test]
#[ignore = "requires Docker"]
fn files_go_into_inbuxa_in_its_own_shape_and_come_back() {
let server = Inbuxa::start().expect("start inbuxa");
let dir = TempDir::new().unwrap();
let out = dir.path().join("out.sqlite");
let back = dir.path().join("back.sqlite");
{
let conn = db::init::open(&out).unwrap();
let blob = db::blobs::intern_blob(&conn, b"#!/bin/sh\necho hi\n").unwrap();
conn.execute(
"INSERT INTO file_nodes (id,parent_id,node_type,blob_id,name,media_type,created)
VALUES (1,NULL,'directory',NULL,'docs',NULL,'2022-02-02T02:02:02Z'),
(2,1,'file',?1,'run.sh','text/x-sh','2022-02-02T02:02:02Z')",
[blob],
)
.unwrap();
}
let run = |args: &[&str]| {
let out = Command::new(BIN)
.args(args)
.env("INBUXA_MIGRATE_API_KEY", &server.token)
.output()
.unwrap();
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
assert!(out.status.success(), "{args:?}: {stderr}");
};
let target = [
"--url",
server.base_url.as_str(),
"--auth-bearer",
"--account-name",
ADMIN_EMAIL,
"--objects",
"filenode",
];
run(&[&["export"][..], &target, &[out.to_str().unwrap()]].concat());
let account = server.account_of(ADMIN_EMAIL, ADMIN_PASSWORD).unwrap();
let r = server
.call(
ADMIN_EMAIL,
ADMIN_PASSWORD,
json!([["FileNode/query", {"accountId": account}, "q"],
["FileNode/get", {"accountId": account,
"#ids": {"resultOf": "q", "name": "FileNode/query", "path": "/ids"},
"properties": ["name", "nodeType", "type", "parentId"]}, "g"]]),
)
.unwrap();
let list = r[1][1]["list"].as_array().unwrap_or_else(|| panic!("{r}"));
let file = list
.iter()
.find(|n| n["name"] == "run.sh")
.unwrap_or_else(|| panic!("no file: {r}"));
assert_eq!(file["nodeType"], "file", "{file}");
assert_eq!(file["type"], "text/x-sh", "{file}");
assert!(
list.iter()
.any(|n| n["name"] == "docs" && n["nodeType"] == "directory"),
"{r}"
);
run(&[&["import", "jmap"][..], &target, &[back.to_str().unwrap()]].concat());
let conn = db::init::open(&back).unwrap();
let mut rows: Vec<(String, String, Option<String>)> = conn
.prepare(
"SELECT name, node_type, media_type FROM file_nodes WHERE name IN ('docs','run.sh')",
)
.unwrap()
.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))
.unwrap()
.collect::<Result<_, _>>()
.unwrap();
rows.sort();
assert_eq!(
rows,
[
("docs".to_owned(), "directory".to_owned(), None),
(
"run.sh".to_owned(),
"file".to_owned(),
Some("text/x-sh".to_owned())
)
]
);
}
#[test]
#[ignore = "requires Docker"]
fn a_message_with_two_message_ids_is_not_exported_twice() {
let server = Inbuxa::start().expect("start inbuxa");
let dir = TempDir::new().unwrap();
let maildir = dir.path().join("mail");
let archive = dir.path().join("archive.sqlite");
for sub in ["cur", "new", "tmp"] {
fs::create_dir_all(maildir.join(sub)).unwrap();
}
// The archive keys it by both Message-IDs; inbuxa's messageId has one.
fs::write(
maildir.join("cur/1:2,S"),
"From: [email protected]\r\nTo: [email protected]\r\nSubject: Twice\r\n\
Message-ID: <[email protected]>\r\nMessage-ID: <[email protected]>\r\n\
Date: Mon, 4 Mar 2024 09:00:00 +0000\r\n\r\nHi\r\n",
)
.unwrap();
let run = |args: &[&str]| {
let out = Command::new(BIN)
.args(args)
.env("INBUXA_MIGRATE_API_KEY", &server.token)
.output()
.unwrap();
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
assert!(out.status.success(), "{args:?}: {stderr}");
String::from_utf8_lossy(&out.stdout).into_owned()
};
run(&[
"import",
"maildir",
maildir.to_str().unwrap(),
archive.to_str().unwrap(),
]);
let export = || {
run(&[
"export",
"--url",
&server.base_url,
"--auth-bearer",
"--account-name",
ADMIN_EMAIL,
"--objects",
"mailbox,email",
archive.to_str().unwrap(),
])
};
let first = export();
assert!(first.contains("Email: created=1"), "{first}");
let second = export();
assert!(second.contains("Email: created=0"), "{second}");
assert_eq!(
emails(&server, ADMIN_EMAIL, ADMIN_PASSWORD),
[("Twice".to_owned(), 1, vec!["$seen".to_owned()])]
);
}