From 6878fcc83f47d024aeb28c08ae4599967b68c5d6 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Sat, 31 Oct 2026 15:03:30 -0700 Subject: [PATCH] Keep the real names of files whose names changed on disk export-files lists the real name of every file, directory and link whose name had to change in the manifest's files.names, by its path on disk, and import files restores them. Entries that keep their names claim them first, so a real name already taken in its directory, or one that can't be a name, leaves the entry with its name on disk and a warning; a listed link whose name is taken is not made rather than replacing the entry there. The member is new and optional, so the layout stays at formatVersion 1. --- CHANGELOG.md | 8 +++ docs/usage.md | 6 +- src/export_files/mod.rs | 11 +++- src/sync/import_files/manifest.rs | 4 ++ src/sync/import_files/other.rs | 94 +++++++++++++++++++++++++++---- tests/export_files.rs | 3 + tests/import_files.rs | 82 ++++++++++++++++++++++++++- 7 files changed, 193 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14b76c3..ee67b75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a ## [Unreleased] +### Added +- `export-files` keeps the real name of every file, directory and link + whose name had to change on disk, in the manifest's new `files.names`, + and `import files` restores them. A real name that is taken in its + directory, or can't be a name, is left as it is on disk with a warning, + so a renamed entry never takes another's place; a listed link whose name + is taken is not made. + ## [27.1-rc.9] The ninth release candidate of the 27.1 series: an account export read diff --git a/docs/usage.md b/docs/usage.md index 47d2aef..e4349fa 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -232,7 +232,8 @@ manifest, so what those formats can't hold comes back too: - which address book and calendar were the default, and each calendar's description, color and time zone; - which Sieve script was active; -- the files' media types and links; +- the files' media types and links, and the real name of any file or + directory whose name had to change on disk; - the identities with their reply-to, bcc and signatures, and the calendar identities. @@ -541,7 +542,8 @@ DIR/ Every name comes from the source, so none is used as a path as it stands: separators and names Windows refuses become `_`, and names that clash, even -only by case, are numbered. +only by case, are numbered. The manifest keeps the real name of every file +and directory whose name changed, so a reader can restore it. `manifest.json` records the source account, every folder's real name, path, role and message count, each address book and calendar with its diff --git a/src/export_files/mod.rs b/src/export_files/mod.rs index 33cca6c..788c0ab 100644 --- a/src/export_files/mod.rs +++ b/src/export_files/mod.rs @@ -239,7 +239,8 @@ struct FileNode { /// The file tree as directories and files. Links are listed in the manifest /// and never made: the source chose their targets, and one could point -/// anywhere on this machine. +/// anywhere on this machine. A name that had to change on disk is listed with +/// its real one. fn write_files( conn: &Connection, root: &Path, @@ -281,6 +282,7 @@ fn write_files( counts: TypeCounts::default(), links: Vec::new(), media_types: serde_json::Map::new(), + names: serde_json::Map::new(), visited: std::collections::HashSet::new(), }; state.write_dir(None, root, "files")?; @@ -288,6 +290,7 @@ fn write_files( "path": "files", "mediaTypes": state.media_types, "links": state.links, + "names": state.names, }); Ok((state.counts, manifest)) } @@ -300,6 +303,8 @@ struct FilesState<'a> { counts: TypeCounts, links: Vec, media_types: serde_json::Map, + /// The real name of each entry whose name on disk had to change. + names: serde_json::Map, visited: std::collections::HashSet, } @@ -316,6 +321,10 @@ impl FilesState<'_> { let name = names.unique(&safe_component(&node.name), ""); let path = dir.join(&name); let rel_path = format!("{rel}/{name}"); + if name != node.name { + self.names + .insert(rel_path.clone(), Value::String(node.name.clone())); + } match node.node_type.as_str() { "directory" => { if !self.dry_run { diff --git a/src/sync/import_files/manifest.rs b/src/sync/import_files/manifest.rs index 2c9d01e..dcd2757 100644 --- a/src/sync/import_files/manifest.rs +++ b/src/sync/import_files/manifest.rs @@ -124,6 +124,10 @@ pub struct Files { pub media_types: BTreeMap, #[serde(default)] pub links: Vec, + /// The real name of each entry whose name on disk had to change, by its + /// path on disk. Writers before 27.1.0-rc.10 don't write it. + #[serde(default)] + pub names: BTreeMap, } #[derive(Debug, Deserialize)] diff --git a/src/sync/import_files/other.rs b/src/sync/import_files/other.rs index c3d1214..1d69942 100644 --- a/src/sync/import_files/other.rs +++ b/src/sync/import_files/other.rs @@ -8,7 +8,7 @@ //! media types and links, and the identities, which only the manifest //! holds. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::fs; use std::path::Path; @@ -150,6 +150,7 @@ pub fn import_files( source_id, files, dirs: HashMap::new(), + claimed: HashMap::new(), rerun: Rerun::default(), created, logger, @@ -183,6 +184,8 @@ struct FileTree<'a> { files: &'a Files, /// Each directory made, by its path in the export. dirs: HashMap, + /// The names given in each directory so far, so no two entries share one. + claimed: HashMap, HashSet>, rerun: Rerun, /// When the export was written: the time for what has none of its own. created: &'a str, @@ -200,7 +203,7 @@ struct Node<'a> { modified: Option, } -impl FileTree<'_> { +impl<'a> FileTree<'a> { fn walk( &mut self, dir: &Path, @@ -218,6 +221,7 @@ impl FileTree<'_> { } }; entries.sort_by_key(fs::DirEntry::file_name); + let mut found: Vec<(fs::DirEntry, String, fs::FileType)> = Vec::new(); for entry in entries { let Some(name) = entry.file_name().to_str().map(str::to_owned) else { self.logger.warn(&format!( @@ -242,6 +246,17 @@ impl FileTree<'_> { counts.skipped += 1; continue; } + if !file_type.is_dir() && !file_type.is_file() { + self.logger + .warn(&format!("{rel_path} is not a file or directory; skipped")); + counts.skipped += 1; + continue; + } + found.push((entry, name, file_type)); + } + let names = self.real_names(rel, parent, found.iter().map(|(_, name, _)| name.as_str())); + for ((entry, name, file_type), real) in found.into_iter().zip(names) { + let rel_path = format!("{rel}/{name}"); if file_type.is_dir() { // A writer sets the times of files, not of directories, so a // directory's time is only when the export was written. @@ -252,7 +267,7 @@ impl FileTree<'_> { node_type: "directory", blob: None, target: None, - name: &name, + name: &real, media_type: None, created: self.created.to_owned(), modified: None, @@ -261,7 +276,7 @@ impl FileTree<'_> { )?; self.dirs.insert(rel_path.clone(), id); self.walk(&entry.path(), &rel_path, Some(id), counts)?; - } else if file_type.is_file() { + } else { let modified = entry.metadata().ok().and_then(|m| mtime_rfc3339(&m)); let created = modified.clone().unwrap_or_else(|| self.created.to_owned()); let bytes = match fs::read(entry.path()) { @@ -281,22 +296,75 @@ impl FileTree<'_> { node_type: "file", blob: Some(blob), target: None, - name: &name, + name: &real, media_type, created, modified, }, counts, )?; - } else { - self.logger - .warn(&format!("{rel_path} is not a file or directory; skipped")); - counts.skipped += 1; } } Ok(()) } + /// The name each entry of one directory gets in the archive. Entries the + /// manifest doesn't rename keep their names on disk, and claim them + /// first, so a renamed entry never takes a name that is already there; one + /// whose real name is taken, or can't be a name, keeps its name on disk. + fn real_names<'n>( + &mut self, + rel: &str, + parent: Option, + on_disk: impl Iterator, + ) -> Vec { + let wanted: Vec<(&str, Option<&str>)> = on_disk + .map(|name| (name, self.renamed(&format!("{rel}/{name}"), name))) + .collect(); + let claimed = self.claimed.entry(parent).or_default(); + for (name, real) in &wanted { + if real.is_none() { + claimed.insert((*name).to_owned()); + } + } + let mut out = Vec::with_capacity(wanted.len()); + for (name, real) in wanted { + let Some(real) = real else { + out.push(name.to_owned()); + continue; + }; + if claimed.insert(real.to_owned()) { + out.push(real.to_owned()); + continue; + } + self.logger.warn(&format!( + "{rel}/{name}: its real name {real:?} is taken in the same directory; \ + it keeps the name it has on disk" + )); + claimed.insert(name.to_owned()); + out.push(name.to_owned()); + } + out + } + + /// The real name the manifest gives an entry, when it differs from the + /// name on disk and could be a name at all. + fn renamed(&self, rel_path: &str, on_disk: &str) -> Option<&'a str> { + let files: &'a Files = self.files; + let real = files.names.get(rel_path)?.as_str(); + if real == on_disk { + return None; + } + if real.is_empty() || real == "." || real == ".." || real.contains(['/', '\0']) { + self.logger.warn(&format!( + "{rel_path}: the manifest's name for it, {real:?}, can't be a name; \ + it keeps the name it has on disk" + )); + return None; + } + Some(real) + } + fn link( &mut self, part: &str, @@ -320,6 +388,12 @@ impl FileTree<'_> { None => return Err(format!("its directory {parent_rel} is not in the export")), } }; + let name = self.renamed(rel, name).unwrap_or(name).to_owned(); + if !self.claimed.entry(parent).or_default().insert(name.clone()) { + return Err(format!( + "its name {name:?} is taken by another entry in the same directory" + )); + } let target = (!target.is_null()).then(|| target.to_string()); self.node( rel, @@ -328,7 +402,7 @@ impl FileTree<'_> { node_type: "symlink", blob: None, target, - name, + name: &name, media_type: None, created: self.created.to_owned(), modified: None, diff --git a/tests/export_files.rs b/tests/export_files.rs index e6f2055..9a23918 100644 --- a/tests/export_files.rs +++ b/tests/export_files.rs @@ -254,6 +254,9 @@ fn writes_open_formats_and_a_manifest() { .any(|s| s["name"] == "filters" && s["active"] == true) ); assert_eq!(m["files"]["links"][0]["path"], "files/link"); + // Names that had to change on disk are kept with their real ones. + assert_eq!(m["files"]["names"]["files/_.._escape"], "../escape"); + assert!(m["files"]["names"].get("files/Docs/a.txt").is_none()); assert_eq!(m["identities"][0]["email"], "alice@example.org"); #[cfg(unix)] diff --git a/tests/import_files.rs b/tests/import_files.rs index 00fa61c..0cda843 100644 --- a/tests/import_files.rs +++ b/tests/import_files.rs @@ -145,7 +145,14 @@ fn seed(path: &Path) { (2, 1, 'file', ?1, NULL, 'a.txt', 'text/plain', '2024-01-01T00:00:00Z', '2024-02-01T10:00:00Z'), (3, 1, 'directory', NULL, NULL, 'Taxes', NULL, '2024-01-01T00:00:00Z', NULL), (4, 3, 'file', ?2, NULL, '2025.pdf', 'application/pdf', '2024-01-01T00:00:00Z', '2024-04-15T09:30:00Z'), - (5, 1, 'symlink', NULL, '{\"path\":\"Taxes/2025.pdf\"}', 'latest', NULL, '2024-01-01T00:00:00Z', NULL)", + (5, 1, 'symlink', NULL, '{\"path\":\"Taxes/2025.pdf\"}', 'latest', NULL, '2024-01-01T00:00:00Z', NULL), + (6, 1, 'file', ?1, NULL, '.profile', 'text/plain', '2024-01-01T00:00:00Z', '2024-02-01T10:00:00Z'), + (7, 1, 'file', ?1, NULL, 'Q3: plan?', NULL, '2024-01-01T00:00:00Z', '2024-03-01T08:00:00Z'), + (8, NULL, 'directory', NULL, NULL, 'Notes', NULL, '2024-01-01T00:00:00Z', NULL), + (9, NULL, 'file', ?1, NULL, 'notes', 'text/plain', '2024-01-01T00:00:00Z', '2024-05-01T12:00:00Z'), + (10, 8, 'symlink', NULL, '{\"path\":\"../Docs\"}', 'CON', NULL, '2024-01-01T00:00:00Z', NULL), + (11, 1, 'directory', NULL, NULL, 'Plans: 2026', NULL, '2024-01-01T00:00:00Z', NULL), + (12, 11, 'file', ?2, NULL, 'draft.pdf', 'application/pdf', '2024-01-01T00:00:00Z', '2024-06-01T09:00:00Z')", params![text, pdf], ) .unwrap(); @@ -432,6 +439,32 @@ fn an_exported_account_comes_back_whole() { "{mailboxes:?}" ); + // Files whose names had to change on disk kept their real names. + let names = &manifest(&f.out)["files"]["names"]; + assert_eq!( + names, + &json!({ + "files/Docs/Plans_ 2026": "Plans: 2026", + "files/Docs/Q3_ plan_": "Q3: plan?", + "files/Docs/_.profile": ".profile", + "files/Notes/CON_": "CON", + "files/notes (2)": "notes", + }) + ); + let files = &read_back["files"]; + for path in [ + "Docs/.profile", + "Docs/Q3: plan?", + "notes", + "Notes/CON", + "Docs/Plans: 2026/draft.pdf", + ] { + assert!( + files.iter().any(|f| f.starts_with(&format!("{path} "))), + "{path}: {files:?}" + ); + } + let conn = Connection::open(&back).unwrap(); let source: (String, String) = conn .query_row("SELECT kind, account_name FROM sources", [], |r| { @@ -656,7 +689,7 @@ fn a_dry_run_reads_the_export_and_writes_nothing() { assert_eq!(counts(&summary, "email").fetched, 8); assert_eq!(counts(&summary, "contactcard").fetched, 3); assert_eq!(counts(&summary, "calendarevent").fetched, 3); - assert_eq!(counts(&summary, "filenode").fetched, 5); + assert_eq!(counts(&summary, "filenode").fetched, 12); } #[test] @@ -756,6 +789,51 @@ fn nothing_outside_the_export_is_read() { assert_eq!(counts(&summary, "email").created, 7); } +#[test] +fn real_names_that_clash_or_cant_be_names_are_not_used() { + let f = exported(); + let mut m = manifest(&f.out); + let names = m["files"]["names"].as_object_mut().unwrap(); + // A file renamed onto its sibling directory, a name that climbs, an empty + // one, and a link renamed onto a file. + names.insert("files/Docs/a.txt".into(), json!("Taxes")); + names.insert("files/Docs/_.profile".into(), json!("../escape")); + names.insert("files/Docs/Q3_ plan_".into(), json!("")); + names.insert("files/Docs/latest".into(), json!("a.txt")); + fs::write(f.out.join("manifest.json"), m.to_string()).unwrap(); + + let back = f.root.join("back.sqlite"); + let summary = import(&f.out, &back).unwrap(); + // Only the link has nowhere to go. + assert_eq!( + counts(&summary, "filenode").failed, + 1, + "{:?}", + summary.per_type + ); + + let conn = Connection::open(&back).unwrap(); + let docs: BTreeSet = rows( + &conn, + "SELECT c.name || ' ' || c.node_type FROM file_nodes c + JOIN file_nodes p ON p.id = c.parent_id WHERE p.name = 'Docs'", + ) + .into_iter() + .map(|r| r[0].as_str().unwrap().to_owned()) + .collect(); + let expected: BTreeSet = [ + "Taxes directory", + "a.txt file", + "_.profile file", + "Q3_ plan_ file", + "Plans: 2026 directory", + ] + .into_iter() + .map(str::to_owned) + .collect(); + assert_eq!(docs, expected); +} + #[test] fn a_card_another_import_put_in_the_archive_keeps_its_books() { let f = exported(); -- 2.54.0