Files
jcoffey-dev 87b143b898
ci / test (pull_request) Skipped
github/ci (branch) GitHub Actions
ci / github (pull_request) Successful in 5m12s
ci / announce (pull_request) Skipped
Add import --estimate: preview counts, sizes and a rough time
Before committing to a migration, an operator wants to know how big it
is. --estimate connects to the source, reports how much is new to fetch
since the archive was last filled -- counts, bytes and a rough time --
and writes nothing. It implies --dry-run.

It reuses each source dry run's delta enumeration (new vs already in
the archive) and adds a size pass that asks only for sizes, never bodies:
over IMAP a UID FETCH RFC822.SIZE, on a Maildir the file lengths. Those
two sources report sizes today; the others fall back to a plain dry run
with a note, which is per-source follow-up work.

TypeCounts gains a bytes field; sync::estimate renders the report, with
a wide, clearly-rough time range since a first run has no measured
throughput to go on.
2026-11-12 08:29:47 -07:00

483 lines
16 KiB
Rust

/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC <[email protected]>
*
* SPDX-License-Identifier: Apache-2.0 OR MIT
*/
//! `export-files`: an archive written out as open formats, and those files
//! read back by `import maildir` and `import takeout` with nothing lost
//! that the formats can hold.
use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::path::{Path, PathBuf};
use inbuxa_migrate::db::{blobs, init};
use inbuxa_migrate::export_files::{ExportFilesConfig, run};
use inbuxa_migrate::logging::Logger;
use inbuxa_migrate::sync::CommonConfig;
use inbuxa_migrate::sync::import_dav::calcard;
use inbuxa_migrate::sync::import_maildir::{self, MaildirImportConfig};
use inbuxa_migrate::sync::import_takeout::{self, TakeoutImportConfig};
use rusqlite::{Connection, params};
use serde_json::Value;
use tempfile::TempDir;
const VCARD_ALICE: &str = "BEGIN:VCARD\r\nVERSION:4.0\r\nUID:card-alice\r\nFN:Alice Smith\r\n\
EMAIL:[email protected]\r\nEND:VCARD\r\n";
const VCARD_BOB: &str = "BEGIN:VCARD\r\nVERSION:4.0\r\nUID:card-bob\r\nFN:Bob Jones\r\n\
TEL:+1 555 0100\r\nEND:VCARD\r\n";
const EVENT: &str = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:x\r\nBEGIN:VEVENT\r\nUID:ev-standup\r\n\
DTSTAMP:20240101T000000Z\r\nDTSTART;TZID=Europe/Berlin:20240301T090000\r\nDURATION:PT15M\r\n\
RRULE:FREQ=WEEKLY;COUNT=4\r\nSUMMARY:Standup\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n";
fn message(subject: &str) -> Vec<u8> {
format!(
"From: [email protected]\r\nTo: [email protected]\r\nSubject: {subject}\r\n\
Message-ID: <{}@example.net>\r\nDate: Fri, 01 Mar 2024 12:00:00 +0000\r\n\r\n{subject}\r\n",
subject.replace(' ', "-")
)
.into_bytes()
}
/// An archive as an import would leave it: mail in nested and non-ASCII
/// folders, a message in two folders, standard and custom keywords, two
/// address books sharing a card, a calendar, Sieve scripts, files and a link.
fn seed(path: &Path) {
let conn = init::open(path).unwrap();
conn.execute(
"INSERT INTO sources (id, kind, session_url, account_id, account_name, username)
VALUES (1, 'jmap', 'https://mail.example.org/jmap/session', 'a1', 'alice', 'alice')",
[],
)
.unwrap();
for (id, name, parent, role) in [
(1, "Inbox", None, Some("inbox")),
(2, "Clients", Some(1), None),
(3, "Archive", None, Some("archive")),
(4, "2024", Some(3), None),
(5, "Équipe & co", None, None),
(6, "v1.0 notes", None, None),
] {
conn.execute(
"INSERT INTO mailboxes (id, name, parent_id, role) VALUES (?1, ?2, ?3, ?4)",
params![id, name, parent, role],
)
.unwrap();
}
for (subject, mailboxes, keywords) in [
("Seen and flagged", "[1]", r#"["$seen","$flagged"]"#),
("Junk in two folders", "[1,3]", r#"["$Junk"]"#),
("Client note", "[2]", r#"["$answered","project-x"]"#),
("Old report", "[4]", "[]"),
("Bonjour", "[5]", r#"["$seen"]"#),
("Release notes", "[6]", "[]"),
] {
let blob = blobs::intern_blob(&conn, &message(subject)).unwrap();
conn.execute(
"INSERT INTO emails (blob_id, received_at, mailbox_ids, keywords)
VALUES (?1, '2024-03-01T12:00:00Z', ?2, ?3)",
params![blob, mailboxes, keywords],
)
.unwrap();
}
conn.execute(
"INSERT INTO address_books (id, name, is_default) VALUES (1, 'Personal', 1), (2, 'Work', 0)",
[],
)
.unwrap();
for (vcard, books) in [(VCARD_ALICE, "[1,2]"), (VCARD_BOB, "[2]")] {
let card = calcard::vcard_to_jscontact(vcard, "/x").unwrap();
conn.execute(
"INSERT INTO contact_cards (uid, address_book_ids, data) VALUES (?1, ?2, ?3)",
params![card.uid, books, card.data.to_string()],
)
.unwrap();
}
conn.execute(
"INSERT INTO calendars (id, name, description, color, time_zone, is_default)
VALUES (1, 'Team, Berlin', 'Stand-ups', '#3366ff', 'Europe/Berlin', 1)",
[],
)
.unwrap();
let mut entry = calcard::ical_to_jscalendar_entries(EVENT)
.unwrap()
.remove(0);
calcard::strip_extracted_fields_from_event(&mut entry.data);
conn.execute(
"INSERT INTO calendar_events (calendar_ids, data) VALUES ('[1]', ?1)",
params![entry.data.to_string()],
)
.unwrap();
for (name, active, script) in [
(
"filters",
1,
"require \"fileinto\";\r\nif header :contains \"subject\" \"report\" { fileinto \"Archive\"; }\r\n",
),
("old", 0, "keep;\r\n"),
] {
let blob = blobs::intern_blob(&conn, script.as_bytes()).unwrap();
conn.execute(
"INSERT INTO sieve_scripts (name, is_active, blob_id) VALUES (?1, ?2, ?3)",
params![name, active, blob],
)
.unwrap();
}
let file_blob = blobs::intern_blob(&conn, b"hello\n").unwrap();
conn.execute(
"INSERT INTO file_nodes (id, parent_id, node_type, blob_id, target, name, media_type, created)
VALUES (1, NULL, 'directory', NULL, NULL, 'Docs', NULL, '2024-01-01T00:00:00Z'),
(2, 1, 'file', ?1, NULL, 'a.txt', 'text/plain', '2024-01-01T00:00:00Z'),
(3, NULL, 'symlink', NULL, '{\"path\":\"/etc/passwd\"}', 'link', NULL, '2024-01-01T00:00:00Z'),
(4, NULL, 'file', ?1, NULL, '../escape', NULL, '2024-01-01T00:00:00Z')",
params![file_blob],
)
.unwrap();
conn.execute(
"INSERT INTO identities (name, email) VALUES ('Alice Smith', '[email protected]')",
[],
)
.unwrap();
}
struct Fixture {
_td: TempDir,
root: PathBuf,
archive: PathBuf,
out: PathBuf,
}
fn exported() -> Fixture {
let td = tempfile::tempdir().unwrap();
let root = td.path().to_path_buf();
let archive = root.join("archive.sqlite");
let out = root.join("out");
seed(&archive);
let summary = run(ExportFilesConfig {
archive: archive.clone(),
dir: out.clone(),
objects: None,
dry_run: false,
logger: Logger::from_flags(true, 0),
})
.unwrap();
assert!(!summary.any_failed(), "{:?}", summary.per_type);
Fixture {
_td: td,
root,
archive,
out,
}
}
fn common(archive: &Path) -> CommonConfig {
CommonConfig {
archive: archive.to_path_buf(),
threads: 1,
dry_run: false,
estimate: false,
max_retries: 0,
allow_invalid_certs: false,
logger: Logger::from_flags(true, 0),
}
}
fn manifest(out: &Path) -> Value {
serde_json::from_str(&fs::read_to_string(out.join("manifest.json")).unwrap()).unwrap()
}
#[test]
fn writes_open_formats_and_a_manifest() {
let f = exported();
let mail = f.out.join("mail");
assert_eq!(fs::read_dir(mail.join("cur")).unwrap().count(), 2);
for dir in [
".INBOX.Clients",
".Archive",
".Archive.2024",
".&AMk-quipe &- co",
".v1_0 notes",
] {
assert!(mail.join(dir).join("cur").is_dir(), "{dir}");
assert!(mail.join(dir).join("maildirfolder").is_file(), "{dir}");
}
assert_eq!(
fs::read_to_string(mail.join("dovecot-keywords")).unwrap(),
"0 $Junk\n"
);
let personal = fs::read_to_string(f.out.join("contacts/Personal.vcf")).unwrap();
assert!(personal.contains("UID:card-alice") && !personal.contains("card-bob"));
let work = fs::read_to_string(f.out.join("contacts/Work.vcf")).unwrap();
assert_eq!(work.matches("BEGIN:VCARD").count(), 2);
let ics = fs::read_to_string(f.out.join("calendars/Team, Berlin.ics")).unwrap();
assert!(ics.contains("X-WR-CALNAME:Team\\, Berlin\r\n"), "{ics}");
assert!(ics.contains("UID:ev-standup"), "{ics}");
assert_eq!(ics.matches("BEGIN:VCALENDAR").count(), 1);
assert!(f.out.join("sieve/filters.sieve").is_file());
assert_eq!(
fs::read(f.out.join("files/Docs/a.txt")).unwrap(),
b"hello\n"
);
assert!(f.out.join("files/_.._escape").is_file());
assert!(!f.out.join("files/link").exists());
assert!(!f.root.join("escape").exists());
let m = manifest(&f.out);
assert_eq!(m["format"], "https://coffeylabs.org/specs/account-export/");
assert_eq!(m["formatVersion"], 1);
assert_eq!(m["sources"][0]["kind"], "jmap");
let folders = m["mail"]["folders"].as_array().unwrap();
let inbox = folders.iter().find(|x| x["role"] == "inbox").unwrap();
assert_eq!(inbox["dir"], "mail");
assert_eq!(inbox["name"], "Inbox");
let equipe = folders
.iter()
.find(|x| x["dir"] == "mail/.&AMk-quipe &- co")
.unwrap();
assert_eq!(equipe["name"], "Équipe & co");
let dotted = folders
.iter()
.find(|x| x["dir"] == "mail/.v1_0 notes")
.unwrap();
assert_eq!(dotted["name"], "v1.0 notes");
let sieve = m["sieve"].as_array().unwrap();
assert!(
sieve
.iter()
.any(|s| s["name"] == "filters" && s["active"] == true)
);
assert_eq!(m["files"]["links"][0]["path"], "files/link");
// Names that had to change on disk are kept with their real ones.
assert_eq!(m["files"]["names"]["files/_.._escape"], "../escape");
assert!(m["files"]["names"].get("files/Docs/a.txt").is_none());
assert_eq!(m["identities"][0]["email"], "[email protected]");
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mode = fs::metadata(&f.out).unwrap().permissions().mode();
assert_eq!(mode & 0o077, 0, "export readable by others: {mode:o}");
}
}
/// Each message as (folder path, subject) → keywords, from an archive.
fn mail_by_folder(conn: &Connection) -> BTreeMap<(String, String), BTreeSet<String>> {
let mut paths: BTreeMap<i64, String> = BTreeMap::new();
let mut stmt = conn
.prepare("SELECT id, name, parent_id, role FROM mailboxes")
.unwrap();
let rows: Vec<(i64, String, Option<i64>, Option<String>)> = stmt
.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))
.unwrap()
.map(Result::unwrap)
.collect();
let name_of = |id: i64| {
let (_, name, _, role) = rows.iter().find(|r| r.0 == id).unwrap();
if role.as_deref() == Some("inbox") {
"INBOX".to_owned()
} else {
name.clone()
}
};
for (id, _, _, _) in &rows {
let mut parts = vec![name_of(*id)];
let mut parent = rows.iter().find(|r| r.0 == *id).unwrap().2;
while let Some(p) = parent {
parts.insert(0, name_of(p));
parent = rows.iter().find(|r| r.0 == p).unwrap().2;
}
paths.insert(*id, parts.join("/"));
}
let mut out = BTreeMap::new();
let mut stmt = conn
.prepare("SELECT b.data, e.mailbox_ids, e.keywords FROM emails e JOIN blobs b ON b.id = e.blob_id")
.unwrap();
let rows = stmt
.query_map([], |r| {
Ok((
r.get::<_, Vec<u8>>(0)?,
r.get::<_, String>(1)?,
r.get::<_, String>(2)?,
))
})
.unwrap();
for row in rows {
let (data, mailboxes, keywords) = row.unwrap();
let text = String::from_utf8(data).unwrap();
let subject = text
.lines()
.find_map(|l| l.strip_prefix("Subject: "))
.unwrap()
.to_owned();
let keywords: BTreeSet<String> = serde_json::from_str::<Vec<String>>(&keywords)
.unwrap()
.into_iter()
.map(|k| k.to_lowercase())
.collect();
for mailbox in serde_json::from_str::<Vec<i64>>(&mailboxes).unwrap() {
out.insert((paths[&mailbox].clone(), subject.clone()), keywords.clone());
}
}
out
}
#[test]
fn mail_reads_back_with_import_maildir() {
let f = exported();
let back = f.root.join("back.sqlite");
import_maildir::run(
common(&back),
MaildirImportConfig {
maildir: f.out.join("mail"),
include: Vec::new(),
exclude: Vec::new(),
folder: Vec::new(),
automap: true,
include_deleted: false,
allow_source_change: false,
},
)
.unwrap();
let original = mail_by_folder(&Connection::open(&f.archive).unwrap());
let read_back = mail_by_folder(&Connection::open(&back).unwrap());
// Maildir++ has no `.` inside a folder name; the manifest keeps it.
let expected: BTreeMap<_, _> = original
.into_iter()
.map(|((path, subject), keywords)| ((path.replace("v1.0", "v1_0"), subject), keywords))
.collect();
assert_eq!(read_back, expected);
let conn = Connection::open(&back).unwrap();
let received: String = conn
.query_row("SELECT received_at FROM emails LIMIT 1", [], |r| r.get(0))
.unwrap();
assert_eq!(received, "2024-03-01T12:00:00Z");
}
#[test]
fn contacts_and_calendars_read_back_with_import_takeout() {
let f = exported();
let back = f.root.join("back.sqlite");
import_takeout::run(
common(&back),
TakeoutImportConfig {
takeout_root: f.out.clone(),
allow_source_change: false,
automap: true,
},
)
.unwrap();
let conn = Connection::open(&back).unwrap();
let uids: BTreeSet<String> = conn
.prepare("SELECT uid FROM contact_cards")
.unwrap()
.query_map([], |r| r.get(0))
.unwrap()
.map(Result::unwrap)
.collect();
assert_eq!(
uids,
BTreeSet::from(["card-alice".to_owned(), "card-bob".to_owned()])
);
let calendar: String = conn
.query_row("SELECT name FROM calendars", [], |r| r.get(0))
.unwrap();
assert_eq!(calendar, "Team, Berlin");
let event: String = conn
.query_row("SELECT data FROM calendar_events", [], |r| r.get(0))
.unwrap();
let event: Value = serde_json::from_str(&event).unwrap();
assert_eq!(event["uid"], "ev-standup");
assert_eq!(event["title"], "Standup");
assert_eq!(event["timeZone"], "Europe/Berlin");
assert_eq!(event["start"], "2024-03-01T09:00:00");
assert_eq!(event["recurrenceRule"]["frequency"], "weekly");
}
#[test]
fn refuses_a_directory_that_is_not_empty() {
let td = tempfile::tempdir().unwrap();
let archive = td.path().join("archive.sqlite");
seed(&archive);
let out = td.path().join("out");
fs::create_dir_all(&out).unwrap();
fs::write(out.join("keep.txt"), "mine").unwrap();
let err = run(ExportFilesConfig {
archive,
dir: out.clone(),
objects: None,
dry_run: false,
logger: Logger::from_flags(true, 0),
})
.unwrap_err();
assert_eq!(err.exit_code(), 1, "{err}");
assert_eq!(fs::read_dir(&out).unwrap().count(), 1);
}
#[test]
fn a_dry_run_counts_and_writes_nothing() {
let td = tempfile::tempdir().unwrap();
let archive = td.path().join("archive.sqlite");
seed(&archive);
let out = td.path().join("out");
let summary = run(ExportFilesConfig {
archive,
dir: out.clone(),
objects: None,
dry_run: true,
logger: Logger::from_flags(true, 0),
})
.unwrap();
assert!(!out.exists());
let email = summary
.per_type
.iter()
.find(|(t, _)| *t == "email")
.unwrap();
assert_eq!(email.1.created, 6);
}
#[test]
fn objects_limits_what_is_written() {
let td = tempfile::tempdir().unwrap();
let archive = td.path().join("archive.sqlite");
seed(&archive);
let out = td.path().join("out");
run(ExportFilesConfig {
archive,
dir: out.clone(),
objects: Some(vec![inbuxa_migrate::types::ObjectType::SieveScript]),
dry_run: false,
logger: Logger::from_flags(true, 0),
})
.unwrap();
assert!(out.join("sieve").is_dir());
assert!(!out.join("mail").exists());
assert!(manifest(&out).get("mail").is_none());
}
#[test]
fn a_missing_archive_is_a_usage_error_and_creates_nothing() {
let td = tempfile::tempdir().unwrap();
let archive = td.path().join("nope.sqlite");
let err = run(ExportFilesConfig {
archive: archive.clone(),
dir: td.path().join("out"),
objects: None,
dry_run: false,
logger: Logger::from_flags(true, 0),
})
.unwrap_err();
assert_eq!(err.exit_code(), 1);
assert!(!archive.exists());
}