diff --git a/CHANGELOG.md b/CHANGELOG.md index 14b76c3..ee67b75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a ## [Unreleased] +### Added +- `export-files` keeps the real name of every file, directory and link + whose name had to change on disk, in the manifest's new `files.names`, + and `import files` restores them. A real name that is taken in its + directory, or can't be a name, is left as it is on disk with a warning, + so a renamed entry never takes another's place; a listed link whose name + is taken is not made. + ## [27.1-rc.9] The ninth release candidate of the 27.1 series: an account export read diff --git a/docs/usage.md b/docs/usage.md index 47d2aef..e4349fa 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -232,7 +232,8 @@ manifest, so what those formats can't hold comes back too: - which address book and calendar were the default, and each calendar's description, color and time zone; - which Sieve script was active; -- the files' media types and links; +- the files' media types and links, and the real name of any file or + directory whose name had to change on disk; - the identities with their reply-to, bcc and signatures, and the calendar identities. @@ -541,7 +542,8 @@ DIR/ Every name comes from the source, so none is used as a path as it stands: separators and names Windows refuses become `_`, and names that clash, even -only by case, are numbered. +only by case, are numbered. The manifest keeps the real name of every file +and directory whose name changed, so a reader can restore it. `manifest.json` records the source account, every folder's real name, path, role and message count, each address book and calendar with its diff --git a/src/export_files/mod.rs b/src/export_files/mod.rs index 33cca6c..788c0ab 100644 --- a/src/export_files/mod.rs +++ b/src/export_files/mod.rs @@ -239,7 +239,8 @@ struct FileNode { /// The file tree as directories and files. Links are listed in the manifest /// and never made: the source chose their targets, and one could point -/// anywhere on this machine. +/// anywhere on this machine. A name that had to change on disk is listed with +/// its real one. fn write_files( conn: &Connection, root: &Path, @@ -281,6 +282,7 @@ fn write_files( counts: TypeCounts::default(), links: Vec::new(), media_types: serde_json::Map::new(), + names: serde_json::Map::new(), visited: std::collections::HashSet::new(), }; state.write_dir(None, root, "files")?; @@ -288,6 +290,7 @@ fn write_files( "path": "files", "mediaTypes": state.media_types, "links": state.links, + "names": state.names, }); Ok((state.counts, manifest)) } @@ -300,6 +303,8 @@ struct FilesState<'a> { counts: TypeCounts, links: Vec, media_types: serde_json::Map, + /// The real name of each entry whose name on disk had to change. + names: serde_json::Map, visited: std::collections::HashSet, } @@ -316,6 +321,10 @@ impl FilesState<'_> { let name = names.unique(&safe_component(&node.name), ""); let path = dir.join(&name); let rel_path = format!("{rel}/{name}"); + if name != node.name { + self.names + .insert(rel_path.clone(), Value::String(node.name.clone())); + } match node.node_type.as_str() { "directory" => { if !self.dry_run { diff --git a/src/sync/import_files/manifest.rs b/src/sync/import_files/manifest.rs index 2c9d01e..dcd2757 100644 --- a/src/sync/import_files/manifest.rs +++ b/src/sync/import_files/manifest.rs @@ -124,6 +124,10 @@ pub struct Files { pub media_types: BTreeMap, #[serde(default)] pub links: Vec, + /// The real name of each entry whose name on disk had to change, by its + /// path on disk. Writers before 27.1.0-rc.10 don't write it. + #[serde(default)] + pub names: BTreeMap, } #[derive(Debug, Deserialize)] diff --git a/src/sync/import_files/other.rs b/src/sync/import_files/other.rs index c3d1214..1d69942 100644 --- a/src/sync/import_files/other.rs +++ b/src/sync/import_files/other.rs @@ -8,7 +8,7 @@ //! media types and links, and the identities, which only the manifest //! holds. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::fs; use std::path::Path; @@ -150,6 +150,7 @@ pub fn import_files( source_id, files, dirs: HashMap::new(), + claimed: HashMap::new(), rerun: Rerun::default(), created, logger, @@ -183,6 +184,8 @@ struct FileTree<'a> { files: &'a Files, /// Each directory made, by its path in the export. dirs: HashMap, + /// The names given in each directory so far, so no two entries share one. + claimed: HashMap, HashSet>, rerun: Rerun, /// When the export was written: the time for what has none of its own. created: &'a str, @@ -200,7 +203,7 @@ struct Node<'a> { modified: Option, } -impl FileTree<'_> { +impl<'a> FileTree<'a> { fn walk( &mut self, dir: &Path, @@ -218,6 +221,7 @@ impl FileTree<'_> { } }; entries.sort_by_key(fs::DirEntry::file_name); + let mut found: Vec<(fs::DirEntry, String, fs::FileType)> = Vec::new(); for entry in entries { let Some(name) = entry.file_name().to_str().map(str::to_owned) else { self.logger.warn(&format!( @@ -242,6 +246,17 @@ impl FileTree<'_> { counts.skipped += 1; continue; } + if !file_type.is_dir() && !file_type.is_file() { + self.logger + .warn(&format!("{rel_path} is not a file or directory; skipped")); + counts.skipped += 1; + continue; + } + found.push((entry, name, file_type)); + } + let names = self.real_names(rel, parent, found.iter().map(|(_, name, _)| name.as_str())); + for ((entry, name, file_type), real) in found.into_iter().zip(names) { + let rel_path = format!("{rel}/{name}"); if file_type.is_dir() { // A writer sets the times of files, not of directories, so a // directory's time is only when the export was written. @@ -252,7 +267,7 @@ impl FileTree<'_> { node_type: "directory", blob: None, target: None, - name: &name, + name: &real, media_type: None, created: self.created.to_owned(), modified: None, @@ -261,7 +276,7 @@ impl FileTree<'_> { )?; self.dirs.insert(rel_path.clone(), id); self.walk(&entry.path(), &rel_path, Some(id), counts)?; - } else if file_type.is_file() { + } else { let modified = entry.metadata().ok().and_then(|m| mtime_rfc3339(&m)); let created = modified.clone().unwrap_or_else(|| self.created.to_owned()); let bytes = match fs::read(entry.path()) { @@ -281,22 +296,75 @@ impl FileTree<'_> { node_type: "file", blob: Some(blob), target: None, - name: &name, + name: &real, media_type, created, modified, }, counts, )?; - } else { - self.logger - .warn(&format!("{rel_path} is not a file or directory; skipped")); - counts.skipped += 1; } } Ok(()) } + /// The name each entry of one directory gets in the archive. Entries the + /// manifest doesn't rename keep their names on disk, and claim them + /// first, so a renamed entry never takes a name that is already there; one + /// whose real name is taken, or can't be a name, keeps its name on disk. + fn real_names<'n>( + &mut self, + rel: &str, + parent: Option, + on_disk: impl Iterator, + ) -> Vec { + let wanted: Vec<(&str, Option<&str>)> = on_disk + .map(|name| (name, self.renamed(&format!("{rel}/{name}"), name))) + .collect(); + let claimed = self.claimed.entry(parent).or_default(); + for (name, real) in &wanted { + if real.is_none() { + claimed.insert((*name).to_owned()); + } + } + let mut out = Vec::with_capacity(wanted.len()); + for (name, real) in wanted { + let Some(real) = real else { + out.push(name.to_owned()); + continue; + }; + if claimed.insert(real.to_owned()) { + out.push(real.to_owned()); + continue; + } + self.logger.warn(&format!( + "{rel}/{name}: its real name {real:?} is taken in the same directory; \ + it keeps the name it has on disk" + )); + claimed.insert(name.to_owned()); + out.push(name.to_owned()); + } + out + } + + /// The real name the manifest gives an entry, when it differs from the + /// name on disk and could be a name at all. + fn renamed(&self, rel_path: &str, on_disk: &str) -> Option<&'a str> { + let files: &'a Files = self.files; + let real = files.names.get(rel_path)?.as_str(); + if real == on_disk { + return None; + } + if real.is_empty() || real == "." || real == ".." || real.contains(['/', '\0']) { + self.logger.warn(&format!( + "{rel_path}: the manifest's name for it, {real:?}, can't be a name; \ + it keeps the name it has on disk" + )); + return None; + } + Some(real) + } + fn link( &mut self, part: &str, @@ -320,6 +388,12 @@ impl FileTree<'_> { None => return Err(format!("its directory {parent_rel} is not in the export")), } }; + let name = self.renamed(rel, name).unwrap_or(name).to_owned(); + if !self.claimed.entry(parent).or_default().insert(name.clone()) { + return Err(format!( + "its name {name:?} is taken by another entry in the same directory" + )); + } let target = (!target.is_null()).then(|| target.to_string()); self.node( rel, @@ -328,7 +402,7 @@ impl FileTree<'_> { node_type: "symlink", blob: None, target, - name, + name: &name, media_type: None, created: self.created.to_owned(), modified: None, diff --git a/tests/export_files.rs b/tests/export_files.rs index e6f2055..9a23918 100644 --- a/tests/export_files.rs +++ b/tests/export_files.rs @@ -254,6 +254,9 @@ fn writes_open_formats_and_a_manifest() { .any(|s| s["name"] == "filters" && s["active"] == true) ); assert_eq!(m["files"]["links"][0]["path"], "files/link"); + // Names that had to change on disk are kept with their real ones. + assert_eq!(m["files"]["names"]["files/_.._escape"], "../escape"); + assert!(m["files"]["names"].get("files/Docs/a.txt").is_none()); assert_eq!(m["identities"][0]["email"], "alice@example.org"); #[cfg(unix)] diff --git a/tests/import_files.rs b/tests/import_files.rs index 00fa61c..0cda843 100644 --- a/tests/import_files.rs +++ b/tests/import_files.rs @@ -145,7 +145,14 @@ fn seed(path: &Path) { (2, 1, 'file', ?1, NULL, 'a.txt', 'text/plain', '2024-01-01T00:00:00Z', '2024-02-01T10:00:00Z'), (3, 1, 'directory', NULL, NULL, 'Taxes', NULL, '2024-01-01T00:00:00Z', NULL), (4, 3, 'file', ?2, NULL, '2025.pdf', 'application/pdf', '2024-01-01T00:00:00Z', '2024-04-15T09:30:00Z'), - (5, 1, 'symlink', NULL, '{\"path\":\"Taxes/2025.pdf\"}', 'latest', NULL, '2024-01-01T00:00:00Z', NULL)", + (5, 1, 'symlink', NULL, '{\"path\":\"Taxes/2025.pdf\"}', 'latest', NULL, '2024-01-01T00:00:00Z', NULL), + (6, 1, 'file', ?1, NULL, '.profile', 'text/plain', '2024-01-01T00:00:00Z', '2024-02-01T10:00:00Z'), + (7, 1, 'file', ?1, NULL, 'Q3: plan?', NULL, '2024-01-01T00:00:00Z', '2024-03-01T08:00:00Z'), + (8, NULL, 'directory', NULL, NULL, 'Notes', NULL, '2024-01-01T00:00:00Z', NULL), + (9, NULL, 'file', ?1, NULL, 'notes', 'text/plain', '2024-01-01T00:00:00Z', '2024-05-01T12:00:00Z'), + (10, 8, 'symlink', NULL, '{\"path\":\"../Docs\"}', 'CON', NULL, '2024-01-01T00:00:00Z', NULL), + (11, 1, 'directory', NULL, NULL, 'Plans: 2026', NULL, '2024-01-01T00:00:00Z', NULL), + (12, 11, 'file', ?2, NULL, 'draft.pdf', 'application/pdf', '2024-01-01T00:00:00Z', '2024-06-01T09:00:00Z')", params![text, pdf], ) .unwrap(); @@ -432,6 +439,32 @@ fn an_exported_account_comes_back_whole() { "{mailboxes:?}" ); + // Files whose names had to change on disk kept their real names. + let names = &manifest(&f.out)["files"]["names"]; + assert_eq!( + names, + &json!({ + "files/Docs/Plans_ 2026": "Plans: 2026", + "files/Docs/Q3_ plan_": "Q3: plan?", + "files/Docs/_.profile": ".profile", + "files/Notes/CON_": "CON", + "files/notes (2)": "notes", + }) + ); + let files = &read_back["files"]; + for path in [ + "Docs/.profile", + "Docs/Q3: plan?", + "notes", + "Notes/CON", + "Docs/Plans: 2026/draft.pdf", + ] { + assert!( + files.iter().any(|f| f.starts_with(&format!("{path} "))), + "{path}: {files:?}" + ); + } + let conn = Connection::open(&back).unwrap(); let source: (String, String) = conn .query_row("SELECT kind, account_name FROM sources", [], |r| { @@ -656,7 +689,7 @@ fn a_dry_run_reads_the_export_and_writes_nothing() { assert_eq!(counts(&summary, "email").fetched, 8); assert_eq!(counts(&summary, "contactcard").fetched, 3); assert_eq!(counts(&summary, "calendarevent").fetched, 3); - assert_eq!(counts(&summary, "filenode").fetched, 5); + assert_eq!(counts(&summary, "filenode").fetched, 12); } #[test] @@ -756,6 +789,51 @@ fn nothing_outside_the_export_is_read() { assert_eq!(counts(&summary, "email").created, 7); } +#[test] +fn real_names_that_clash_or_cant_be_names_are_not_used() { + let f = exported(); + let mut m = manifest(&f.out); + let names = m["files"]["names"].as_object_mut().unwrap(); + // A file renamed onto its sibling directory, a name that climbs, an empty + // one, and a link renamed onto a file. + names.insert("files/Docs/a.txt".into(), json!("Taxes")); + names.insert("files/Docs/_.profile".into(), json!("../escape")); + names.insert("files/Docs/Q3_ plan_".into(), json!("")); + names.insert("files/Docs/latest".into(), json!("a.txt")); + fs::write(f.out.join("manifest.json"), m.to_string()).unwrap(); + + let back = f.root.join("back.sqlite"); + let summary = import(&f.out, &back).unwrap(); + // Only the link has nowhere to go. + assert_eq!( + counts(&summary, "filenode").failed, + 1, + "{:?}", + summary.per_type + ); + + let conn = Connection::open(&back).unwrap(); + let docs: BTreeSet = rows( + &conn, + "SELECT c.name || ' ' || c.node_type FROM file_nodes c + JOIN file_nodes p ON p.id = c.parent_id WHERE p.name = 'Docs'", + ) + .into_iter() + .map(|r| r[0].as_str().unwrap().to_owned()) + .collect(); + let expected: BTreeSet = [ + "Taxes directory", + "a.txt file", + "_.profile file", + "Q3_ plan_ file", + "Plans: 2026 directory", + ] + .into_iter() + .map(str::to_owned) + .collect(); + assert_eq!(docs, expected); +} + #[test] fn a_card_another_import_put_in_the_archive_keeps_its_books() { let f = exported();