diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a8b088..1ffffee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a ## [Unreleased] +### Changed +- Upstream v1.0.14 is merged in. Its HTTP timeouts are left out, since this + project already has its own, and so is its Dockerfile. + +### Fixed +- Google Takeout: Portuguese (Brazil) system labels now map to mailbox roles + and keywords, or are dropped, like their English counterparts. +- Google Takeout: a message labeled neither `Opened` nor `Unread` is now + imported as read. Only English and Portuguese (Brazil) labels are recognized, + so an unread label in another language becomes a folder and the message + imports as read. +- Google Takeout: a contact without a `UID` is stored once rather than once per + contact group file. Archives from an earlier version keep the old copies, so + re-import into a fresh archive. + ## [27.1-rc.10] The tenth release candidate of the 27.1 series: real file names kept diff --git a/Cargo.lock b/Cargo.lock index db276e0..ed79914 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2174,9 +2174,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", @@ -2901,9 +2901,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "3.4.2" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a7ac20be9b7726e0bbdbf974c059676d9acb1cd414961f570a4e8231cacd7fc" +checksum = "e7be76686bdf7198002062423ca0d8cf9b75a27c8e1de804513b97e97b555d9d" dependencies = [ "base64 0.23.1", "cookie_store", @@ -2921,9 +2921,9 @@ dependencies = [ [[package]] name = "ureq-proto" -version = "0.6.4" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f86fd172ccca569e458f61b6bdd6220965a9ef36e672a6852953b51a0e1583be" +checksum = "622dce33c252115466d83f63e9744a18f12b4406aae51a01c15ad8fcf1185eb3" dependencies = [ "base64 0.23.1", "http", diff --git a/Cargo.toml b/Cargo.toml index 8657741..ffc1f3e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ blake3 = "1.8.5" time = { version = "0.3.47", features = ["serde-well-known", "macros"] } url= "2.5.8" thiserror = "2.0.18" -ureq = { version = "3.3.0", default-features = false, features = ["json", "gzip", "rustls-no-provider", "platform-verifier"] } +ureq = { version = "3.5.0", default-features = false, features = ["json", "gzip", "rustls-no-provider", "platform-verifier"] } clap = { version = "4.6.1", features = ["derive"] } crossbeam-channel = "0.5.15" num_cpus = "1.17.0" diff --git a/scripts/upstream-import.py b/scripts/upstream-import.py index d2309e0..c683da1 100755 --- a/scripts/upstream-import.py +++ b/scripts/upstream-import.py @@ -50,8 +50,9 @@ REPO_URL = "https://git.coffeylabs.org/inbuxa/inbuxa-migrate" # The fork's own copies always win: upstream's versions of these are dropped # and main's are put in their place. OWNED = ["README.md", "CHANGELOG.md", ".gitignore", ".github", ".gitea"] -# Upstream's release tooling, which the fork replaced. -DROPPED = ["dist-workspace.toml", "wix", "assets/importer-exporter.jpg"] +# Upstream's release tooling, which the fork replaced, and its container build: +# the fork ships release binaries, checked by their published hashes. +DROPPED = ["dist-workspace.toml", "wix", "assets/importer-exporter.jpg", "Dockerfile", ".dockerignore"] # Order matters: the specific forms before the plain name. RENAMES = [ diff --git a/src/sync/import_takeout/contacts.rs b/src/sync/import_takeout/contacts.rs index 231eb71..64404a8 100644 --- a/src/sync/import_takeout/contacts.rs +++ b/src/sync/import_takeout/contacts.rs @@ -50,15 +50,7 @@ pub fn process_file( }; for (ordinal, chunk) in chunks.into_iter().enumerate() { - match insert_card( - &tx, - source_id, - book_local, - file_path, - ordinal, - &chunk, - card_counts, - ) { + match insert_card(&tx, source_id, book_local, &chunk, card_counts) { Ok(()) => {} Err(e) => { logger.warn(&format!("{file_path:?}#{ordinal}: vCard: {e}")); @@ -106,13 +98,10 @@ fn insert_card( tx: &Transaction<'_>, source_id: i64, book_local: i64, - file_path: &Path, - ordinal: usize, chunk: &str, counts: &mut TypeCounts, ) -> Result<(), Error> { - let synthetic_href = format!("{}#{ordinal}", file_path.to_string_lossy()); - let card = calcard::vcard_to_jscontact(chunk, &synthetic_href) + let card = calcard::vcard_to_jscontact(chunk, chunk) .map_err(|e| Error::Partial(format!("vCard parse: {e}")))?; let data_json = card.data.to_string(); let address_book_ids = format!("[{book_local}]"); diff --git a/src/sync/import_takeout/labels.rs b/src/sync/import_takeout/labels.rs index 90a6c57..646e8d3 100644 --- a/src/sync/import_takeout/labels.rs +++ b/src/sync/import_takeout/labels.rs @@ -69,6 +69,50 @@ fn decode_encoded_words(value: &str) -> Cow<'_, str> { } } +#[derive(Debug, Clone, Copy)] +enum SystemLabel { + Starred, + Important, + Opened, + Unread, + Category, + Mailbox(&'static str), +} + +impl SystemLabel { + fn parse(token: &str) -> Option { + hashify::map!( + token.as_bytes(), + SystemLabel, + "Starred" => SystemLabel::Starred, + "Important" => SystemLabel::Important, + "Opened" => SystemLabel::Opened, + "Unread" => SystemLabel::Unread, + "Archived" => SystemLabel::Mailbox("Archive"), + "Category Personal" => SystemLabel::Category, + "Category Promotions" => SystemLabel::Category, + "Category Social" => SystemLabel::Category, + "Category Updates" => SystemLabel::Category, + "Category Forums" => SystemLabel::Category, + "Com estrela" => SystemLabel::Starred, + "Importante" => SystemLabel::Important, + "Aberto" => SystemLabel::Opened, + "Não lidas" => SystemLabel::Unread, + "Caixa de entrada" => SystemLabel::Mailbox("Inbox"), + "Enviado" => SystemLabel::Mailbox("Sent"), + "Rascunhos" => SystemLabel::Mailbox("Drafts"), + "Lixeira" => SystemLabel::Mailbox("Trash"), + "Arquivada" => SystemLabel::Mailbox("Archive"), + "Categoria: pessoal" => SystemLabel::Category, + "Categoria: promoções" => SystemLabel::Category, + "Categoria: social" => SystemLabel::Category, + "Categoria: atualizações" => SystemLabel::Category, + "Categoria: fóruns" => SystemLabel::Category, + ) + .copied() + } +} + pub fn classify(tokens: &[String]) -> LabelClassification { let mut mailboxes: Vec = Vec::new(); let mut keywords: BTreeSet = BTreeSet::new(); @@ -76,32 +120,36 @@ pub fn classify(tokens: &[String]) -> LabelClassification { let mut had_unread = false; for tok in tokens { - if is_category_token(tok) { - continue; - } - match tok.as_str() { - "Starred" => { + let mailbox = match SystemLabel::parse(tok) { + Some(SystemLabel::Category) => continue, + Some(SystemLabel::Starred) => { keywords.insert("$flagged".to_owned()); + continue; } - "Important" => { + Some(SystemLabel::Important) => { keywords.insert("$important".to_owned()); + continue; } - "Opened" => { + Some(SystemLabel::Opened) => { had_opened = true; - keywords.insert("$seen".to_owned()); + continue; } - "Unread" => { + Some(SystemLabel::Unread) => { had_unread = true; + continue; } - other => { - let canonical = canonical_label(other); - if !mailboxes.iter().any(|m| m == &canonical) { - mailboxes.push(canonical); - } - } + Some(SystemLabel::Mailbox(name)) => name, + None => tok.as_str(), + }; + if !mailboxes.iter().any(|m| m == mailbox) { + mailboxes.push(mailbox.to_owned()); } } + if had_opened || (!had_unread && !tokens.is_empty()) { + keywords.insert("$seen".to_owned()); + } + let opened_won_over_unread = had_opened && had_unread; LabelClassification { mailboxes, @@ -144,24 +192,6 @@ pub fn assignments_for( .collect() } -fn is_category_token(t: &str) -> bool { - matches!( - t, - "Category Personal" - | "Category Promotions" - | "Category Social" - | "Category Updates" - | "Category Forums" - ) -} - -fn canonical_label(raw: &str) -> String { - match raw { - "Archived" => "Archive".to_owned(), - other => other.to_owned(), - } -} - #[cfg(test)] mod tests { use super::*; @@ -246,6 +276,68 @@ mod tests { assert!(c.had_unread_signal); } + #[test] + fn neither_opened_nor_unread_means_seen() { + let c = classify_str("Inbox,Important"); + assert!(c.keywords.contains("$seen")); + let c = classify_str("Sent"); + assert!(c.keywords.contains("$seen")); + } + + #[test] + fn no_labels_sets_no_keywords() { + assert!(classify(&[]).keywords.is_empty()); + } + + #[test] + fn pt_br_system_labels_map_like_english() { + let c = classify_str( + "Caixa de entrada,Enviado,Rascunhos,Lixeira,Arquivada,Importante,Com estrela,\ + Aberto,Categoria: pessoal,Categoria: atualizações,Categoria: promoções,Spam,Clientes", + ); + assert_eq!( + c.mailboxes, + [ + "Inbox", "Sent", "Drafts", "Trash", "Archive", "Spam", "Clientes" + ] + ); + assert_eq!(c.keywords_sorted(), ["$flagged", "$important", "$seen"]); + let roles: Vec<_> = assignments_for(&c, MappingOptions::default()) + .into_iter() + .map(|a| a.role) + .collect(); + assert_eq!( + roles, + [ + Some("inbox"), + Some("sent"), + Some("drafts"), + Some("trash"), + Some("archive"), + Some("junk"), + None + ] + ); + } + + #[test] + fn pt_br_unread_label_means_no_seen() { + let c = classify_str("Caixa de entrada,Não lidas"); + assert!(!c.keywords.contains("$seen")); + assert!(c.had_unread_signal); + } + + // Only English and Portuguese (Brazil) system labels are recognized. In any + // other language the unread label is taken for a folder and the message + // reads as seen. + #[test] + fn unread_label_in_another_language_is_not_recognized() { + let c = classify_str("Inbox,Ungelesen"); + assert!(c.keywords.contains("$seen")); + assert!(!c.had_unread_signal); + assert_eq!(c.mailboxes, ["Inbox", "Ungelesen"]); + } + #[test] fn opened_plus_unread_marks_collision_and_keeps_seen() { let c = classify_str("Inbox,Opened,Unread"); diff --git a/tests/sync_takeout.rs b/tests/sync_takeout.rs index ccda4ce..501b057 100644 --- a/tests/sync_takeout.rs +++ b/tests/sync_takeout.rs @@ -259,9 +259,9 @@ fn synthetic_takeout_imports_expected_row_counts() { assert_eq!(count(&conn, "address_books"), 1); assert_eq!( count(&conn, "contact_cards"), - 6, + 4, "2 UID-bearing cards dedup across the two .vcf files (=2); \ - 2 UID-less cards get distinct synthetic UIDs per file (=4); total 6" + 2 UID-less cards dedup by content across the two files (=2); total 4" ); let by = |t: &str| { @@ -456,9 +456,9 @@ fn contacts_dedup_by_uid_across_two_vcf_files() { |r| r.get(0), ) .unwrap(); - assert!( - synthetic_uids >= 2, - "expected synthetic UIDs for the 2 UID-less cards * 2 files" + assert_eq!( + synthetic_uids, 2, + "the 2 UID-less cards appear in both files but are stored once each" ); }