Keep the real names of files whose names changed on disk #56
No files matched your search
@@ -7,6 +7,14 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
- `export-files` keeps the real name of every file, directory and link
|
||||
whose name had to change on disk, in the manifest's new `files.names`,
|
||||
and `import files` restores them. A real name that is taken in its
|
||||
directory, or can't be a name, is left as it is on disk with a warning,
|
||||
so a renamed entry never takes another's place; a listed link whose name
|
||||
is taken is not made.
|
||||
|
||||
## [27.1-rc.9]
|
||||
|
||||
The ninth release candidate of the 27.1 series: an account export read
|
||||
|
||||
+4
-2
@@ -232,7 +232,8 @@ manifest, so what those formats can't hold comes back too:
|
||||
- which address book and calendar were the default, and each calendar's
|
||||
description, color and time zone;
|
||||
- which Sieve script was active;
|
||||
- the files' media types and links;
|
||||
- the files' media types and links, and the real name of any file or
|
||||
directory whose name had to change on disk;
|
||||
- the identities with their reply-to, bcc and signatures, and the calendar
|
||||
identities.
|
||||
|
||||
@@ -541,7 +542,8 @@ DIR/
|
||||
|
||||
Every name comes from the source, so none is used as a path as it stands:
|
||||
separators and names Windows refuses become `_`, and names that clash, even
|
||||
only by case, are numbered.
|
||||
only by case, are numbered. The manifest keeps the real name of every file
|
||||
and directory whose name changed, so a reader can restore it.
|
||||
|
||||
`manifest.json` records the source account, every folder's real name,
|
||||
path, role and message count, each address book and calendar with its
|
||||
|
||||
+10
-1
@@ -239,7 +239,8 @@ struct FileNode {
|
||||
|
||||
/// The file tree as directories and files. Links are listed in the manifest
|
||||
/// and never made: the source chose their targets, and one could point
|
||||
/// anywhere on this machine.
|
||||
/// anywhere on this machine. A name that had to change on disk is listed with
|
||||
/// its real one.
|
||||
fn write_files(
|
||||
conn: &Connection,
|
||||
root: &Path,
|
||||
@@ -281,6 +282,7 @@ fn write_files(
|
||||
counts: TypeCounts::default(),
|
||||
links: Vec::new(),
|
||||
media_types: serde_json::Map::new(),
|
||||
names: serde_json::Map::new(),
|
||||
visited: std::collections::HashSet::new(),
|
||||
};
|
||||
state.write_dir(None, root, "files")?;
|
||||
@@ -288,6 +290,7 @@ fn write_files(
|
||||
"path": "files",
|
||||
"mediaTypes": state.media_types,
|
||||
"links": state.links,
|
||||
"names": state.names,
|
||||
});
|
||||
Ok((state.counts, manifest))
|
||||
}
|
||||
@@ -300,6 +303,8 @@ struct FilesState<'a> {
|
||||
counts: TypeCounts,
|
||||
links: Vec<Value>,
|
||||
media_types: serde_json::Map<String, Value>,
|
||||
/// The real name of each entry whose name on disk had to change.
|
||||
names: serde_json::Map<String, Value>,
|
||||
visited: std::collections::HashSet<i64>,
|
||||
}
|
||||
|
||||
@@ -316,6 +321,10 @@ impl FilesState<'_> {
|
||||
let name = names.unique(&safe_component(&node.name), "");
|
||||
let path = dir.join(&name);
|
||||
let rel_path = format!("{rel}/{name}");
|
||||
if name != node.name {
|
||||
self.names
|
||||
.insert(rel_path.clone(), Value::String(node.name.clone()));
|
||||
}
|
||||
match node.node_type.as_str() {
|
||||
"directory" => {
|
||||
if !self.dry_run {
|
||||
|
||||
@@ -124,6 +124,10 @@ pub struct Files {
|
||||
pub media_types: BTreeMap<String, String>,
|
||||
#[serde(default)]
|
||||
pub links: Vec<Link>,
|
||||
/// The real name of each entry whose name on disk had to change, by its
|
||||
/// path on disk. Writers before 27.1.0-rc.10 don't write it.
|
||||
#[serde(default)]
|
||||
pub names: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
//! media types and links, and the identities, which only the manifest
|
||||
//! holds.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
@@ -150,6 +150,7 @@ pub fn import_files(
|
||||
source_id,
|
||||
files,
|
||||
dirs: HashMap::new(),
|
||||
claimed: HashMap::new(),
|
||||
rerun: Rerun::default(),
|
||||
created,
|
||||
logger,
|
||||
@@ -183,6 +184,8 @@ struct FileTree<'a> {
|
||||
files: &'a Files,
|
||||
/// Each directory made, by its path in the export.
|
||||
dirs: HashMap<String, i64>,
|
||||
/// The names given in each directory so far, so no two entries share one.
|
||||
claimed: HashMap<Option<i64>, HashSet<String>>,
|
||||
rerun: Rerun,
|
||||
/// When the export was written: the time for what has none of its own.
|
||||
created: &'a str,
|
||||
@@ -200,7 +203,7 @@ struct Node<'a> {
|
||||
modified: Option<String>,
|
||||
}
|
||||
|
||||
impl FileTree<'_> {
|
||||
impl<'a> FileTree<'a> {
|
||||
fn walk(
|
||||
&mut self,
|
||||
dir: &Path,
|
||||
@@ -218,6 +221,7 @@ impl FileTree<'_> {
|
||||
}
|
||||
};
|
||||
entries.sort_by_key(fs::DirEntry::file_name);
|
||||
let mut found: Vec<(fs::DirEntry, String, fs::FileType)> = Vec::new();
|
||||
for entry in entries {
|
||||
let Some(name) = entry.file_name().to_str().map(str::to_owned) else {
|
||||
self.logger.warn(&format!(
|
||||
@@ -242,6 +246,17 @@ impl FileTree<'_> {
|
||||
counts.skipped += 1;
|
||||
continue;
|
||||
}
|
||||
if !file_type.is_dir() && !file_type.is_file() {
|
||||
self.logger
|
||||
.warn(&format!("{rel_path} is not a file or directory; skipped"));
|
||||
counts.skipped += 1;
|
||||
continue;
|
||||
}
|
||||
found.push((entry, name, file_type));
|
||||
}
|
||||
let names = self.real_names(rel, parent, found.iter().map(|(_, name, _)| name.as_str()));
|
||||
for ((entry, name, file_type), real) in found.into_iter().zip(names) {
|
||||
let rel_path = format!("{rel}/{name}");
|
||||
if file_type.is_dir() {
|
||||
// A writer sets the times of files, not of directories, so a
|
||||
// directory's time is only when the export was written.
|
||||
@@ -252,7 +267,7 @@ impl FileTree<'_> {
|
||||
node_type: "directory",
|
||||
blob: None,
|
||||
target: None,
|
||||
name: &name,
|
||||
name: &real,
|
||||
media_type: None,
|
||||
created: self.created.to_owned(),
|
||||
modified: None,
|
||||
@@ -261,7 +276,7 @@ impl FileTree<'_> {
|
||||
)?;
|
||||
self.dirs.insert(rel_path.clone(), id);
|
||||
self.walk(&entry.path(), &rel_path, Some(id), counts)?;
|
||||
} else if file_type.is_file() {
|
||||
} else {
|
||||
let modified = entry.metadata().ok().and_then(|m| mtime_rfc3339(&m));
|
||||
let created = modified.clone().unwrap_or_else(|| self.created.to_owned());
|
||||
let bytes = match fs::read(entry.path()) {
|
||||
@@ -281,22 +296,75 @@ impl FileTree<'_> {
|
||||
node_type: "file",
|
||||
blob: Some(blob),
|
||||
target: None,
|
||||
name: &name,
|
||||
name: &real,
|
||||
media_type,
|
||||
created,
|
||||
modified,
|
||||
},
|
||||
counts,
|
||||
)?;
|
||||
} else {
|
||||
self.logger
|
||||
.warn(&format!("{rel_path} is not a file or directory; skipped"));
|
||||
counts.skipped += 1;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The name each entry of one directory gets in the archive. Entries the
|
||||
/// manifest doesn't rename keep their names on disk, and claim them
|
||||
/// first, so a renamed entry never takes a name that is already there; one
|
||||
/// whose real name is taken, or can't be a name, keeps its name on disk.
|
||||
fn real_names<'n>(
|
||||
&mut self,
|
||||
rel: &str,
|
||||
parent: Option<i64>,
|
||||
on_disk: impl Iterator<Item = &'n str>,
|
||||
) -> Vec<String> {
|
||||
let wanted: Vec<(&str, Option<&str>)> = on_disk
|
||||
.map(|name| (name, self.renamed(&format!("{rel}/{name}"), name)))
|
||||
.collect();
|
||||
let claimed = self.claimed.entry(parent).or_default();
|
||||
for (name, real) in &wanted {
|
||||
if real.is_none() {
|
||||
claimed.insert((*name).to_owned());
|
||||
}
|
||||
}
|
||||
let mut out = Vec::with_capacity(wanted.len());
|
||||
for (name, real) in wanted {
|
||||
let Some(real) = real else {
|
||||
out.push(name.to_owned());
|
||||
continue;
|
||||
};
|
||||
if claimed.insert(real.to_owned()) {
|
||||
out.push(real.to_owned());
|
||||
continue;
|
||||
}
|
||||
self.logger.warn(&format!(
|
||||
"{rel}/{name}: its real name {real:?} is taken in the same directory; \
|
||||
it keeps the name it has on disk"
|
||||
));
|
||||
claimed.insert(name.to_owned());
|
||||
out.push(name.to_owned());
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The real name the manifest gives an entry, when it differs from the
|
||||
/// name on disk and could be a name at all.
|
||||
fn renamed(&self, rel_path: &str, on_disk: &str) -> Option<&'a str> {
|
||||
let files: &'a Files = self.files;
|
||||
let real = files.names.get(rel_path)?.as_str();
|
||||
if real == on_disk {
|
||||
return None;
|
||||
}
|
||||
if real.is_empty() || real == "." || real == ".." || real.contains(['/', '\0']) {
|
||||
self.logger.warn(&format!(
|
||||
"{rel_path}: the manifest's name for it, {real:?}, can't be a name; \
|
||||
it keeps the name it has on disk"
|
||||
));
|
||||
return None;
|
||||
}
|
||||
Some(real)
|
||||
}
|
||||
|
||||
fn link(
|
||||
&mut self,
|
||||
part: &str,
|
||||
@@ -320,6 +388,12 @@ impl FileTree<'_> {
|
||||
None => return Err(format!("its directory {parent_rel} is not in the export")),
|
||||
}
|
||||
};
|
||||
let name = self.renamed(rel, name).unwrap_or(name).to_owned();
|
||||
if !self.claimed.entry(parent).or_default().insert(name.clone()) {
|
||||
return Err(format!(
|
||||
"its name {name:?} is taken by another entry in the same directory"
|
||||
));
|
||||
}
|
||||
let target = (!target.is_null()).then(|| target.to_string());
|
||||
self.node(
|
||||
rel,
|
||||
@@ -328,7 +402,7 @@ impl FileTree<'_> {
|
||||
node_type: "symlink",
|
||||
blob: None,
|
||||
target,
|
||||
name,
|
||||
name: &name,
|
||||
media_type: None,
|
||||
created: self.created.to_owned(),
|
||||
modified: None,
|
||||
|
||||
@@ -254,6 +254,9 @@ fn writes_open_formats_and_a_manifest() {
|
||||
.any(|s| s["name"] == "filters" && s["active"] == true)
|
||||
);
|
||||
assert_eq!(m["files"]["links"][0]["path"], "files/link");
|
||||
// Names that had to change on disk are kept with their real ones.
|
||||
assert_eq!(m["files"]["names"]["files/_.._escape"], "../escape");
|
||||
assert!(m["files"]["names"].get("files/Docs/a.txt").is_none());
|
||||
assert_eq!(m["identities"][0]["email"], "[email protected]");
|
||||
|
||||
#[cfg(unix)]
|
||||
|
||||
+80
-2
@@ -145,7 +145,14 @@ fn seed(path: &Path) {
|
||||
(2, 1, 'file', ?1, NULL, 'a.txt', 'text/plain', '2024-01-01T00:00:00Z', '2024-02-01T10:00:00Z'),
|
||||
(3, 1, 'directory', NULL, NULL, 'Taxes', NULL, '2024-01-01T00:00:00Z', NULL),
|
||||
(4, 3, 'file', ?2, NULL, '2025.pdf', 'application/pdf', '2024-01-01T00:00:00Z', '2024-04-15T09:30:00Z'),
|
||||
(5, 1, 'symlink', NULL, '{\"path\":\"Taxes/2025.pdf\"}', 'latest', NULL, '2024-01-01T00:00:00Z', NULL)",
|
||||
(5, 1, 'symlink', NULL, '{\"path\":\"Taxes/2025.pdf\"}', 'latest', NULL, '2024-01-01T00:00:00Z', NULL),
|
||||
(6, 1, 'file', ?1, NULL, '.profile', 'text/plain', '2024-01-01T00:00:00Z', '2024-02-01T10:00:00Z'),
|
||||
(7, 1, 'file', ?1, NULL, 'Q3: plan?', NULL, '2024-01-01T00:00:00Z', '2024-03-01T08:00:00Z'),
|
||||
(8, NULL, 'directory', NULL, NULL, 'Notes', NULL, '2024-01-01T00:00:00Z', NULL),
|
||||
(9, NULL, 'file', ?1, NULL, 'notes', 'text/plain', '2024-01-01T00:00:00Z', '2024-05-01T12:00:00Z'),
|
||||
(10, 8, 'symlink', NULL, '{\"path\":\"../Docs\"}', 'CON', NULL, '2024-01-01T00:00:00Z', NULL),
|
||||
(11, 1, 'directory', NULL, NULL, 'Plans: 2026', NULL, '2024-01-01T00:00:00Z', NULL),
|
||||
(12, 11, 'file', ?2, NULL, 'draft.pdf', 'application/pdf', '2024-01-01T00:00:00Z', '2024-06-01T09:00:00Z')",
|
||||
params![text, pdf],
|
||||
)
|
||||
.unwrap();
|
||||
@@ -432,6 +439,32 @@ fn an_exported_account_comes_back_whole() {
|
||||
"{mailboxes:?}"
|
||||
);
|
||||
|
||||
// Files whose names had to change on disk kept their real names.
|
||||
let names = &manifest(&f.out)["files"]["names"];
|
||||
assert_eq!(
|
||||
names,
|
||||
&json!({
|
||||
"files/Docs/Plans_ 2026": "Plans: 2026",
|
||||
"files/Docs/Q3_ plan_": "Q3: plan?",
|
||||
"files/Docs/_.profile": ".profile",
|
||||
"files/Notes/CON_": "CON",
|
||||
"files/notes (2)": "notes",
|
||||
})
|
||||
);
|
||||
let files = &read_back["files"];
|
||||
for path in [
|
||||
"Docs/.profile",
|
||||
"Docs/Q3: plan?",
|
||||
"notes",
|
||||
"Notes/CON",
|
||||
"Docs/Plans: 2026/draft.pdf",
|
||||
] {
|
||||
assert!(
|
||||
files.iter().any(|f| f.starts_with(&format!("{path} "))),
|
||||
"{path}: {files:?}"
|
||||
);
|
||||
}
|
||||
|
||||
let conn = Connection::open(&back).unwrap();
|
||||
let source: (String, String) = conn
|
||||
.query_row("SELECT kind, account_name FROM sources", [], |r| {
|
||||
@@ -656,7 +689,7 @@ fn a_dry_run_reads_the_export_and_writes_nothing() {
|
||||
assert_eq!(counts(&summary, "email").fetched, 8);
|
||||
assert_eq!(counts(&summary, "contactcard").fetched, 3);
|
||||
assert_eq!(counts(&summary, "calendarevent").fetched, 3);
|
||||
assert_eq!(counts(&summary, "filenode").fetched, 5);
|
||||
assert_eq!(counts(&summary, "filenode").fetched, 12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -756,6 +789,51 @@ fn nothing_outside_the_export_is_read() {
|
||||
assert_eq!(counts(&summary, "email").created, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn real_names_that_clash_or_cant_be_names_are_not_used() {
|
||||
let f = exported();
|
||||
let mut m = manifest(&f.out);
|
||||
let names = m["files"]["names"].as_object_mut().unwrap();
|
||||
// A file renamed onto its sibling directory, a name that climbs, an empty
|
||||
// one, and a link renamed onto a file.
|
||||
names.insert("files/Docs/a.txt".into(), json!("Taxes"));
|
||||
names.insert("files/Docs/_.profile".into(), json!("../escape"));
|
||||
names.insert("files/Docs/Q3_ plan_".into(), json!(""));
|
||||
names.insert("files/Docs/latest".into(), json!("a.txt"));
|
||||
fs::write(f.out.join("manifest.json"), m.to_string()).unwrap();
|
||||
|
||||
let back = f.root.join("back.sqlite");
|
||||
let summary = import(&f.out, &back).unwrap();
|
||||
// Only the link has nowhere to go.
|
||||
assert_eq!(
|
||||
counts(&summary, "filenode").failed,
|
||||
1,
|
||||
"{:?}",
|
||||
summary.per_type
|
||||
);
|
||||
|
||||
let conn = Connection::open(&back).unwrap();
|
||||
let docs: BTreeSet<String> = rows(
|
||||
&conn,
|
||||
"SELECT c.name || ' ' || c.node_type FROM file_nodes c
|
||||
JOIN file_nodes p ON p.id = c.parent_id WHERE p.name = 'Docs'",
|
||||
)
|
||||
.into_iter()
|
||||
.map(|r| r[0].as_str().unwrap().to_owned())
|
||||
.collect();
|
||||
let expected: BTreeSet<String> = [
|
||||
"Taxes directory",
|
||||
"a.txt file",
|
||||
"_.profile file",
|
||||
"Q3_ plan_ file",
|
||||
"Plans: 2026 directory",
|
||||
]
|
||||
.into_iter()
|
||||
.map(str::to_owned)
|
||||
.collect();
|
||||
assert_eq!(docs, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_card_another_import_put_in_the_archive_keeps_its_books() {
|
||||
let f = exported();
|
||||
|
||||
Reference in new issue
Block a user