Keep the real names of files whose names changed on disk #56

Merged
jcoffey-dev merged 1 commits from feat/file-real-names into main 2026-10-31 22:08:02 +00:00
7 changed files with 193 additions and 15 deletions

No files matched your search

+8
View File
@@ -7,6 +7,14 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a
## [Unreleased]
### Added
- `export-files` keeps the real name of every file, directory and link
whose name had to change on disk, in the manifest's new `files.names`,
and `import files` restores them. A real name that is taken in its
directory, or can't be a name, is left as it is on disk with a warning,
so a renamed entry never takes another's place; a listed link whose name
is taken is not made.
## [27.1-rc.9]
The ninth release candidate of the 27.1 series: an account export read
+4 -2
View File
@@ -232,7 +232,8 @@ manifest, so what those formats can't hold comes back too:
- which address book and calendar were the default, and each calendar's
description, color and time zone;
- which Sieve script was active;
- the files' media types and links;
- the files' media types and links, and the real name of any file or
directory whose name had to change on disk;
- the identities with their reply-to, bcc and signatures, and the calendar
identities.
@@ -541,7 +542,8 @@ DIR/
Every name comes from the source, so none is used as a path as it stands:
separators and names Windows refuses become `_`, and names that clash, even
only by case, are numbered.
only by case, are numbered. The manifest keeps the real name of every file
and directory whose name changed, so a reader can restore it.
`manifest.json` records the source account, every folder's real name,
path, role and message count, each address book and calendar with its
+10 -1
View File
@@ -239,7 +239,8 @@ struct FileNode {
/// The file tree as directories and files. Links are listed in the manifest
/// and never made: the source chose their targets, and one could point
/// anywhere on this machine.
/// anywhere on this machine. A name that had to change on disk is listed with
/// its real one.
fn write_files(
conn: &Connection,
root: &Path,
@@ -281,6 +282,7 @@ fn write_files(
counts: TypeCounts::default(),
links: Vec::new(),
media_types: serde_json::Map::new(),
names: serde_json::Map::new(),
visited: std::collections::HashSet::new(),
};
state.write_dir(None, root, "files")?;
@@ -288,6 +290,7 @@ fn write_files(
"path": "files",
"mediaTypes": state.media_types,
"links": state.links,
"names": state.names,
});
Ok((state.counts, manifest))
}
@@ -300,6 +303,8 @@ struct FilesState<'a> {
counts: TypeCounts,
links: Vec<Value>,
media_types: serde_json::Map<String, Value>,
/// The real name of each entry whose name on disk had to change.
names: serde_json::Map<String, Value>,
visited: std::collections::HashSet<i64>,
}
@@ -316,6 +321,10 @@ impl FilesState<'_> {
let name = names.unique(&safe_component(&node.name), "");
let path = dir.join(&name);
let rel_path = format!("{rel}/{name}");
if name != node.name {
self.names
.insert(rel_path.clone(), Value::String(node.name.clone()));
}
match node.node_type.as_str() {
"directory" => {
if !self.dry_run {
+4
View File
@@ -124,6 +124,10 @@ pub struct Files {
pub media_types: BTreeMap<String, String>,
#[serde(default)]
pub links: Vec<Link>,
/// The real name of each entry whose name on disk had to change, by its
/// path on disk. Writers before 27.1.0-rc.10 don't write it.
#[serde(default)]
pub names: BTreeMap<String, String>,
}
#[derive(Debug, Deserialize)]
+84 -10
View File
@@ -8,7 +8,7 @@
//! media types and links, and the identities, which only the manifest
//! holds.
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::fs;
use std::path::Path;
@@ -150,6 +150,7 @@ pub fn import_files(
source_id,
files,
dirs: HashMap::new(),
claimed: HashMap::new(),
rerun: Rerun::default(),
created,
logger,
@@ -183,6 +184,8 @@ struct FileTree<'a> {
files: &'a Files,
/// Each directory made, by its path in the export.
dirs: HashMap<String, i64>,
/// The names given in each directory so far, so no two entries share one.
claimed: HashMap<Option<i64>, HashSet<String>>,
rerun: Rerun,
/// When the export was written: the time for what has none of its own.
created: &'a str,
@@ -200,7 +203,7 @@ struct Node<'a> {
modified: Option<String>,
}
impl FileTree<'_> {
impl<'a> FileTree<'a> {
fn walk(
&mut self,
dir: &Path,
@@ -218,6 +221,7 @@ impl FileTree<'_> {
}
};
entries.sort_by_key(fs::DirEntry::file_name);
let mut found: Vec<(fs::DirEntry, String, fs::FileType)> = Vec::new();
for entry in entries {
let Some(name) = entry.file_name().to_str().map(str::to_owned) else {
self.logger.warn(&format!(
@@ -242,6 +246,17 @@ impl FileTree<'_> {
counts.skipped += 1;
continue;
}
if !file_type.is_dir() && !file_type.is_file() {
self.logger
.warn(&format!("{rel_path} is not a file or directory; skipped"));
counts.skipped += 1;
continue;
}
found.push((entry, name, file_type));
}
let names = self.real_names(rel, parent, found.iter().map(|(_, name, _)| name.as_str()));
for ((entry, name, file_type), real) in found.into_iter().zip(names) {
let rel_path = format!("{rel}/{name}");
if file_type.is_dir() {
// A writer sets the times of files, not of directories, so a
// directory's time is only when the export was written.
@@ -252,7 +267,7 @@ impl FileTree<'_> {
node_type: "directory",
blob: None,
target: None,
name: &name,
name: &real,
media_type: None,
created: self.created.to_owned(),
modified: None,
@@ -261,7 +276,7 @@ impl FileTree<'_> {
)?;
self.dirs.insert(rel_path.clone(), id);
self.walk(&entry.path(), &rel_path, Some(id), counts)?;
} else if file_type.is_file() {
} else {
let modified = entry.metadata().ok().and_then(|m| mtime_rfc3339(&m));
let created = modified.clone().unwrap_or_else(|| self.created.to_owned());
let bytes = match fs::read(entry.path()) {
@@ -281,22 +296,75 @@ impl FileTree<'_> {
node_type: "file",
blob: Some(blob),
target: None,
name: &name,
name: &real,
media_type,
created,
modified,
},
counts,
)?;
} else {
self.logger
.warn(&format!("{rel_path} is not a file or directory; skipped"));
counts.skipped += 1;
}
}
Ok(())
}
/// The name each entry of one directory gets in the archive. Entries the
/// manifest doesn't rename keep their names on disk, and claim them
/// first, so a renamed entry never takes a name that is already there; one
/// whose real name is taken, or can't be a name, keeps its name on disk.
fn real_names<'n>(
&mut self,
rel: &str,
parent: Option<i64>,
on_disk: impl Iterator<Item = &'n str>,
) -> Vec<String> {
let wanted: Vec<(&str, Option<&str>)> = on_disk
.map(|name| (name, self.renamed(&format!("{rel}/{name}"), name)))
.collect();
let claimed = self.claimed.entry(parent).or_default();
for (name, real) in &wanted {
if real.is_none() {
claimed.insert((*name).to_owned());
}
}
let mut out = Vec::with_capacity(wanted.len());
for (name, real) in wanted {
let Some(real) = real else {
out.push(name.to_owned());
continue;
};
if claimed.insert(real.to_owned()) {
out.push(real.to_owned());
continue;
}
self.logger.warn(&format!(
"{rel}/{name}: its real name {real:?} is taken in the same directory; \
it keeps the name it has on disk"
));
claimed.insert(name.to_owned());
out.push(name.to_owned());
}
out
}
/// The real name the manifest gives an entry, when it differs from the
/// name on disk and could be a name at all.
fn renamed(&self, rel_path: &str, on_disk: &str) -> Option<&'a str> {
let files: &'a Files = self.files;
let real = files.names.get(rel_path)?.as_str();
if real == on_disk {
return None;
}
if real.is_empty() || real == "." || real == ".." || real.contains(['/', '\0']) {
self.logger.warn(&format!(
"{rel_path}: the manifest's name for it, {real:?}, can't be a name; \
it keeps the name it has on disk"
));
return None;
}
Some(real)
}
fn link(
&mut self,
part: &str,
@@ -320,6 +388,12 @@ impl FileTree<'_> {
None => return Err(format!("its directory {parent_rel} is not in the export")),
}
};
let name = self.renamed(rel, name).unwrap_or(name).to_owned();
if !self.claimed.entry(parent).or_default().insert(name.clone()) {
return Err(format!(
"its name {name:?} is taken by another entry in the same directory"
));
}
let target = (!target.is_null()).then(|| target.to_string());
self.node(
rel,
@@ -328,7 +402,7 @@ impl FileTree<'_> {
node_type: "symlink",
blob: None,
target,
name,
name: &name,
media_type: None,
created: self.created.to_owned(),
modified: None,
+3
View File
@@ -254,6 +254,9 @@ fn writes_open_formats_and_a_manifest() {
.any(|s| s["name"] == "filters" && s["active"] == true)
);
assert_eq!(m["files"]["links"][0]["path"], "files/link");
// Names that had to change on disk are kept with their real ones.
assert_eq!(m["files"]["names"]["files/_.._escape"], "../escape");
assert!(m["files"]["names"].get("files/Docs/a.txt").is_none());
assert_eq!(m["identities"][0]["email"], "[email protected]");
#[cfg(unix)]
+80 -2
View File
@@ -145,7 +145,14 @@ fn seed(path: &Path) {
(2, 1, 'file', ?1, NULL, 'a.txt', 'text/plain', '2024-01-01T00:00:00Z', '2024-02-01T10:00:00Z'),
(3, 1, 'directory', NULL, NULL, 'Taxes', NULL, '2024-01-01T00:00:00Z', NULL),
(4, 3, 'file', ?2, NULL, '2025.pdf', 'application/pdf', '2024-01-01T00:00:00Z', '2024-04-15T09:30:00Z'),
(5, 1, 'symlink', NULL, '{\"path\":\"Taxes/2025.pdf\"}', 'latest', NULL, '2024-01-01T00:00:00Z', NULL)",
(5, 1, 'symlink', NULL, '{\"path\":\"Taxes/2025.pdf\"}', 'latest', NULL, '2024-01-01T00:00:00Z', NULL),
(6, 1, 'file', ?1, NULL, '.profile', 'text/plain', '2024-01-01T00:00:00Z', '2024-02-01T10:00:00Z'),
(7, 1, 'file', ?1, NULL, 'Q3: plan?', NULL, '2024-01-01T00:00:00Z', '2024-03-01T08:00:00Z'),
(8, NULL, 'directory', NULL, NULL, 'Notes', NULL, '2024-01-01T00:00:00Z', NULL),
(9, NULL, 'file', ?1, NULL, 'notes', 'text/plain', '2024-01-01T00:00:00Z', '2024-05-01T12:00:00Z'),
(10, 8, 'symlink', NULL, '{\"path\":\"../Docs\"}', 'CON', NULL, '2024-01-01T00:00:00Z', NULL),
(11, 1, 'directory', NULL, NULL, 'Plans: 2026', NULL, '2024-01-01T00:00:00Z', NULL),
(12, 11, 'file', ?2, NULL, 'draft.pdf', 'application/pdf', '2024-01-01T00:00:00Z', '2024-06-01T09:00:00Z')",
params![text, pdf],
)
.unwrap();
@@ -432,6 +439,32 @@ fn an_exported_account_comes_back_whole() {
"{mailboxes:?}"
);
// Files whose names had to change on disk kept their real names.
let names = &manifest(&f.out)["files"]["names"];
assert_eq!(
names,
&json!({
"files/Docs/Plans_ 2026": "Plans: 2026",
"files/Docs/Q3_ plan_": "Q3: plan?",
"files/Docs/_.profile": ".profile",
"files/Notes/CON_": "CON",
"files/notes (2)": "notes",
})
);
let files = &read_back["files"];
for path in [
"Docs/.profile",
"Docs/Q3: plan?",
"notes",
"Notes/CON",
"Docs/Plans: 2026/draft.pdf",
] {
assert!(
files.iter().any(|f| f.starts_with(&format!("{path} "))),
"{path}: {files:?}"
);
}
let conn = Connection::open(&back).unwrap();
let source: (String, String) = conn
.query_row("SELECT kind, account_name FROM sources", [], |r| {
@@ -656,7 +689,7 @@ fn a_dry_run_reads_the_export_and_writes_nothing() {
assert_eq!(counts(&summary, "email").fetched, 8);
assert_eq!(counts(&summary, "contactcard").fetched, 3);
assert_eq!(counts(&summary, "calendarevent").fetched, 3);
assert_eq!(counts(&summary, "filenode").fetched, 5);
assert_eq!(counts(&summary, "filenode").fetched, 12);
}
#[test]
@@ -756,6 +789,51 @@ fn nothing_outside_the_export_is_read() {
assert_eq!(counts(&summary, "email").created, 7);
}
#[test]
fn real_names_that_clash_or_cant_be_names_are_not_used() {
let f = exported();
let mut m = manifest(&f.out);
let names = m["files"]["names"].as_object_mut().unwrap();
// A file renamed onto its sibling directory, a name that climbs, an empty
// one, and a link renamed onto a file.
names.insert("files/Docs/a.txt".into(), json!("Taxes"));
names.insert("files/Docs/_.profile".into(), json!("../escape"));
names.insert("files/Docs/Q3_ plan_".into(), json!(""));
names.insert("files/Docs/latest".into(), json!("a.txt"));
fs::write(f.out.join("manifest.json"), m.to_string()).unwrap();
let back = f.root.join("back.sqlite");
let summary = import(&f.out, &back).unwrap();
// Only the link has nowhere to go.
assert_eq!(
counts(&summary, "filenode").failed,
1,
"{:?}",
summary.per_type
);
let conn = Connection::open(&back).unwrap();
let docs: BTreeSet<String> = rows(
&conn,
"SELECT c.name || ' ' || c.node_type FROM file_nodes c
JOIN file_nodes p ON p.id = c.parent_id WHERE p.name = 'Docs'",
)
.into_iter()
.map(|r| r[0].as_str().unwrap().to_owned())
.collect();
let expected: BTreeSet<String> = [
"Taxes directory",
"a.txt file",
"_.profile file",
"Q3_ plan_ file",
"Plans: 2026 directory",
]
.into_iter()
.map(str::to_owned)
.collect();
assert_eq!(docs, expected);
}
#[test]
fn a_card_another_import_put_in_the_archive_keeps_its_books() {
let f = exported();