diff --git a/crates/common/src/hold.rs b/crates/common/src/hold.rs index 58be497..f92815e 100644 --- a/crates/common/src/hold.rs +++ b/crates/common/src/hold.rs @@ -10,7 +10,7 @@ //! there are few holds. use crate::Server; -use inbuxa_features::hold::{self, Hold, Member}; +use inbuxa_features::hold::{self, Hold, Keeping, Member}; impl Server { /// The active holds covering `account_id`, through its own name, its @@ -36,6 +36,15 @@ impl Server { hold::covering(self.store(), &member).await } + /// How `account_id`'s deleted items are kept: its holds' ranges and the + /// undelete period in force now (LH-4, UD-6a). + pub async fn keeping(&self, account_id: u32) -> trc::Result { + let retention = inbuxa_features::undelete::settings::retention(self.registry()) + .await? + .items; + Ok(Keeping::new(retention, &self.holds_on(account_id).await?)) + } + /// Whether any active hold covers `account_id` at all. pub async fn is_held(&self, account_id: u32) -> trc::Result { Ok(!self.holds_on(account_id).await?.is_empty()) diff --git a/crates/email/src/mailbox/destroy.rs b/crates/email/src/mailbox/destroy.rs index 91e89ef..1e095e8 100644 --- a/crates/email/src/mailbox/destroy.rs +++ b/crates/email/src/mailbox/destroy.rs @@ -92,10 +92,8 @@ impl MailboxDestroy for Server { let mut deleted_ids = RoaringBitmap::new(); let mut thread_ids = RoaringBitmap::new(); - // inbuxa: UD-1, UD-6a: the retention in force now - let retention = inbuxa_features::undelete::settings::retention(self.registry()) - .await? - .items; + // inbuxa: UD-1, UD-6a, LH-4: how this account's deletions are kept + let keeping = self.keeping(account_id).await?; self.archives( account_id, Collection::Email, @@ -125,10 +123,10 @@ impl MailboxDestroy for Server { deleted_ids.insert(message_id); thread_ids.insert(prev_message_data.inner.thread_id.to_native()); // inbuxa: UD-1, UD-4: a deleted message is noted for archiving - if let Some(retention) = retention { + if keeping.keeps_anything() { inbuxa_features::undelete::email::note( &mut batch, - retention, + &keeping, account_id, message_id, prev_message_data.inner.size.to_native() as u64, diff --git a/crates/email/src/message/delete.rs b/crates/email/src/message/delete.rs index cc781f0..256fb37 100644 --- a/crates/email/src/message/delete.rs +++ b/crates/email/src/message/delete.rs @@ -69,10 +69,8 @@ impl EmailDeletion for Server { batch .with_account_id(account_id) .with_collection(Collection::Email); - // inbuxa: UD-1, UD-6a: the retention in force now - let retention = inbuxa_features::undelete::settings::retention(self.registry()) - .await? - .items; + // inbuxa: UD-1, UD-6a, LH-4: how this account's deletions are kept + let keeping = self.keeping(account_id).await?; self.archives( account_id, Collection::Email, @@ -90,10 +88,10 @@ impl EmailDeletion for Server { } thread_ids.insert(metadata.inner.thread_id.to_native()); // inbuxa: UD-1, UD-4: a deleted message is noted for archiving - if let Some(retention) = retention { + if keeping.keeps_anything() { inbuxa_features::undelete::email::note( batch, - retention, + &keeping, account_id, document_id, metadata.inner.size.to_native() as u64, diff --git a/crates/email/src/sieve/delete.rs b/crates/email/src/sieve/delete.rs index c982ada..9ac94b6 100644 --- a/crates/email/src/sieve/delete.rs +++ b/crates/email/src/sieve/delete.rs @@ -44,12 +44,12 @@ impl SieveScriptDelete for Server { )) .await? { - // inbuxa: UD-1: a deleted script is kept, when archiving is on - if let Some(retention) = - inbuxa_features::undelete::settings::retention(self.registry()) - .await? - .items - { + // inbuxa: UD-1, LH-4: a deleted script is kept, when archiving + // is on or a hold covers the account (whole: scripts have no date) + let keeping = self.keeping(account_id).await?; + let now = store::write::now(); + if let Some(until) = keeping.until(now, keeping.is_held()) { + let retention = until.saturating_sub(now); let script = obj_ .deserialize::() .caused_by(trc::location!())?; diff --git a/crates/features/src/hold/mod.rs b/crates/features/src/hold/mod.rs index b940b47..4480dde 100644 --- a/crates/features/src/hold/mod.rs +++ b/crates/features/src/hold/mod.rs @@ -26,6 +26,85 @@ use store::{ }; use trc::AddContext; +/// The deadline a held archived item carries: the last second of 9999. It +/// never passes, so every expiry check keeps the item without knowing about +/// holds (LH-4, LH-5); releasing a hold gives it a real deadline (LH-10). +pub const HELD_UNTIL: u64 = 253_402_300_799; + +/// Whether an archived item's deadline marks it as held. Anything past the +/// year 9000 counts, so a deadline computed from a hold a moment earlier or +/// later still reads as held. +pub fn is_held_until(until: u64) -> bool { + until >= 221_845_392_000 +} + +/// A day, in seconds: the slack either side of a range for an event's start, +/// whose time zone isn't known here. +const DAY: u64 = 86_400; + +/// How an account's deleted items are kept: its holds' ranges, and the +/// undelete period for whatever no hold covers (LH-3, LH-4). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Keeping { + /// `archiveDeletedItemsFor`, in seconds, if undelete is on. + pub retention: Option, + /// Each active hold's range on this account; `(None, None)` is a whole + /// account. Empty when nothing holds it. + pub ranges: Vec<(Option, Option)>, +} + +impl Keeping { + pub fn new(retention: Option, holds: &[Hold]) -> Keeping { + Keeping { + retention, + ranges: holds.iter().map(|h| (h.from, h.to)).collect(), + } + } + + /// Whether any hold reaches the account at all. + pub fn is_held(&self) -> bool { + !self.ranges.is_empty() + } + + /// Whether deleted items need noting: something may keep them. + pub fn keeps_anything(&self) -> bool { + self.is_held() || self.retention.is_some() + } + + /// Whether a hold covers an item dated `date`. No date means the item is + /// held whole, whatever the range (LH-3). + pub fn covers(&self, date: Option) -> bool { + self.ranges.iter().any(|(from, to)| match date { + None => true, + Some(at) => { + from.is_none_or(|from| at >= from) && to.is_none_or(|to| at <= to) + } + }) + } + + /// Like `covers`, for an event's start: a day of slack either side, since + /// its time zone isn't known here. + pub fn covers_event(&self, start: Option) -> bool { + self.ranges.iter().any(|(from, to)| match start { + None => true, + Some(at) => { + from.is_none_or(|from| at + DAY >= from) + && to.is_none_or(|to| at <= to.saturating_add(DAY)) + } + }) + } + + /// Until when an item deleted at `now` is kept: held, the undelete + /// period, or not at all. + pub fn until(&self, now: u64, held: bool) -> Option { + if held { + Some(HELD_UNTIL) + } else { + self.retention.map(|retention| now + retention) + } + } +} + const FEATURE: u8 = b'H'; const KIND_HOLD: u8 = b'h'; @@ -511,6 +590,27 @@ mod tests { assert!(held.scope.covers(&member) && !held.scope.covers(&moved)); } + #[test] + fn keeping_deleted_items() { + let whole = Keeping::new(None, &[hold(accounts(&[2]), None, None)]); + assert!(whole.covers(Some(5)) && whole.covers(None)); + assert_eq!(whole.until(100, whole.covers(Some(5))), Some(HELD_UNTIL)); + assert!(is_held_until(whole.until(100, true).unwrap())); + + // LH-3: a range holds only what's inside it; outside, undelete's rules + let ranged = Keeping::new(Some(30), &[hold(accounts(&[2]), Some(1_000), Some(2_000))]); + assert!(ranged.covers(Some(1_500)) && !ranged.covers(Some(2_500))); + assert!(ranged.covers(None), "contacts, files and scripts are held whole"); + assert_eq!(ranged.until(100, ranged.covers(Some(2_500))), Some(130)); + assert!(ranged.covers_event(Some(2_000 + 3_600)), "a day of slack for an event"); + + // Neither held nor undelete: nothing is kept + let none = Keeping::new(None, &[]); + assert!(!none.keeps_anything()); + assert_eq!(none.until(100, false), None); + assert!(!is_held_until(100 + 30 * 365 * 86_400)); + } + #[test] fn stored_as_json() { let current = hold(accounts(&[2]), Some(100), None); diff --git a/crates/features/src/undelete/data.rs b/crates/features/src/undelete/data.rs index c398800..b170c98 100644 --- a/crates/features/src/undelete/data.rs +++ b/crates/features/src/undelete/data.rs @@ -123,6 +123,14 @@ pub struct EmailNote { pub size: u64, pub mailboxes: Vec, pub keywords: Vec, + /// LH-3: the ranges of the holds on the account when it was deleted. + /// Its received date is only known when it's archived, which decides + /// whether a hold keeps it after all. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub held_ranges: Vec<(Option, Option)>, + /// The undelete deadline for when no range covers it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub otherwise_until: Option, } /// What restore needs beyond the kept copy (UD-4, UD-8). diff --git a/crates/features/src/undelete/email.rs b/crates/features/src/undelete/email.rs index 83cc244..7d67380 100644 --- a/crates/features/src/undelete/email.rs +++ b/crates/features/src/undelete/email.rs @@ -12,9 +12,12 @@ //! is made if archiving is on, fixing the deadline then. When the data is //! finally removed, a noted message becomes an archived item. -use crate::undelete::{ - data::{self, EmailNote, Extra}, - records, +use crate::{ + hold::Keeping, + undelete::{ + data::{self, EmailNote, Extra}, + records, + }, }; use registry::{ schema::structs::{ArchivedEmail, ArchivedItem}, @@ -26,10 +29,12 @@ use store::{ }; use types::{blob::BlobId, blob_hash::BlobHash}; -/// Notes a deleted message, when archiving is on (`retention` seconds). +/// Notes a deleted message, when anything keeps it: undelete, or a legal +/// hold on the account (LH-4). A held note keeps it until it's archived, +/// when its received date says whether the hold's range covers it. pub fn note( batch: &mut BatchBuilder, - retention: u64, + keeping: &Keeping, account_id: u32, document_id: u32, size: u64, @@ -37,16 +42,23 @@ pub fn note( keywords: Vec, ) -> trc::Result<()> { let archived_at = now(); + // Held until the date is known; the undelete deadline otherwise + let otherwise_until = keeping.until(archived_at, false); + let Some(archived_until) = keeping.until(archived_at, keeping.is_held()) else { + return Ok(()); + }; data::note_email( batch, account_id, document_id, &EmailNote { archived_at, - archived_until: archived_at + retention, + archived_until, size, mailboxes, keywords, + held_ranges: keeping.ranges.clone(), + otherwise_until: if keeping.is_held() { otherwise_until } else { None }, }, ) } @@ -78,9 +90,27 @@ pub async fn archive( document_id: u32, summary: Summary<'_>, ) -> trc::Result { - let Some(note) = data::email_note(data, account_id, document_id).await? else { + let Some(mut note) = data::email_note(data, account_id, document_id).await? else { return Ok(false); }; + // LH-3: a held note's range decides now that the date is known; outside + // it, undelete's deadline, or nothing kept at all + if !note.held_ranges.is_empty() { + let keeping = Keeping { + retention: None, + ranges: std::mem::take(&mut note.held_ranges), + }; + if !keeping.covers(Some(summary.received_at)) { + match note.otherwise_until { + Some(until) => note.archived_until = until, + None => { + let mut batch = BatchBuilder::new(); + data::clear_email_note(&mut batch, account_id, document_id); + return data.write(batch.build_all()).await.map(|_| false); + } + } + } + } let item = ArchivedItem::Email(ArchivedEmail { from: summary.from.unwrap_or_default().to_string(), subject: summary.subject.unwrap_or_default().to_string(), diff --git a/crates/features/src/undelete/groupware.rs b/crates/features/src/undelete/groupware.rs index b94e7dc..a77eb0f 100644 --- a/crates/features/src/undelete/groupware.rs +++ b/crates/features/src/undelete/groupware.rs @@ -97,6 +97,39 @@ pub async fn take( Ok(Some(note)) } +/// A note, left in place: for a held account it's cleared only once its item +/// is archived, so a failure leaves it for the retry (LH-5). +pub async fn peek( + data: &Store, + kind: Kind, + account_id: u32, + document_id: u32, +) -> trc::Result> { + Ok(data + .get_value::>(ValueKey::from(note_class(kind, account_id, document_id))) + .await? + .map(|Json(note)| note)) +} + +/// Removes a note once its item is archived or needn't be. +pub async fn clear(data: &Store, kind: Kind, account_id: u32, document_id: u32) -> trc::Result<()> { + let mut batch = BatchBuilder::new(); + batch.clear(note_class(kind, account_id, document_id)); + data.write(batch.build_all()).await.map(|_| ()) +} + +/// An event's start, for a hold's range (LH-3). None for a recurring event, +/// which may have an occurrence anywhere, so a hold keeps it whole. +pub fn event_start(note: &Note) -> Option { + let text = note.content.as_deref()?; + if property(text, "RRULE").is_some() || property(text, "RDATE").is_some() { + return None; + } + property(text, "DTSTART") + .and_then(|v| ical_time(&v)) + .map(|t| t.max(0) as u64) +} + /// The value of the first line starting with `name` (as `NAME:` or /// `NAME;params:`) in iCalendar or vCard text, unfolded. fn property(text: &str, name: &str) -> Option { diff --git a/crates/imap/src/op/expunge.rs b/crates/imap/src/op/expunge.rs index 918f177..0a8aebb 100644 --- a/crates/imap/src/op/expunge.rs +++ b/crates/imap/src/op/expunge.rs @@ -243,10 +243,8 @@ impl SessionData { let mut fully_deleted = RoaringBitmap::new(); let mut thread_ids = RoaringBitmap::new(); - // inbuxa: UD-1, UD-6a: the retention in force now - let retention = inbuxa_features::undelete::settings::retention(self.server.registry()) - .await? - .items; + // inbuxa: UD-1, UD-6a, LH-4: how this account's deletions are kept + let keeping = self.server.keeping(account_id).await?; self.server .archives( account_id, @@ -270,10 +268,10 @@ impl SessionData { fully_deleted.insert(document_id); thread_ids.insert(metadata.inner.thread_id.to_native()); // inbuxa: UD-1, UD-4: a deleted message is noted for archiving - if let Some(retention) = retention { + if keeping.keeps_anything() { inbuxa_features::undelete::email::note( batch, - retention, + &keeping, account_id, document_id, metadata.inner.size.to_native() as u64, diff --git a/crates/services/src/task_manager/index.rs b/crates/services/src/task_manager/index.rs index 07ce970..53161e9 100644 --- a/crates/services/src/task_manager/index.rs +++ b/crates/services/src/task_manager/index.rs @@ -229,11 +229,19 @@ impl SearchIndexTask for Server { IndexDocumentType::Email => None, } && let Err(err) = archive_noted(self, kind, account_id, document_id).await { + // inbuxa: LH-5: the note stays, so the retry archives + // it; nothing a hold keeps is lost to a failure trc::error!( err.account_id(account_id) .document_id(document_id) .details("Failed to archive a deleted item") ); + results.push(IndexTaskResult { + task_type: TaskType::Delete, + index: task.document_type, + result: TaskResult::temporary("Failed to archive a deleted item"), + }); + continue; } document_deletions[idx] @@ -696,8 +704,9 @@ pub fn trace_search_document( document } -// inbuxa: UD-1, UD-4: archives a deleted file, event or contact noted at -// deletion, when archiving is on; otherwise its note is dropped +// inbuxa: UD-1, UD-4, LH-4: archives a deleted file, event or contact noted +// at deletion, when archiving is on or a hold covers it; otherwise its note is +// dropped. The note goes only once the item is archived. async fn archive_noted( server: &Server, kind: undelete::groupware::Kind, @@ -705,12 +714,22 @@ async fn archive_noted( document_id: u32, ) -> trc::Result<()> { let data = &server.core.storage.data; - let Some(note) = undelete::groupware::take(data, kind, account_id, document_id).await? else { + let Some(note) = undelete::groupware::peek(data, kind, account_id, document_id).await? else { return Ok(()); }; - let Some(retention) = undelete::settings::retention(server.registry()).await?.items else { - return Ok(()); + // LH-3: events by their start; contacts and files whole + let keeping = server.keeping(account_id).await?; + let held = match kind { + undelete::groupware::Kind::CalendarEvent => { + keeping.covers_event(undelete::groupware::event_start(¬e)) + } + _ => keeping.covers(None), }; + let now = store::write::now(); + let Some(until) = keeping.until(now, held) else { + return undelete::groupware::clear(data, kind, account_id, document_id).await; + }; + let retention = until.saturating_sub(now); let blob_hash = match (¬e.content, ¬e.blob_hash) { (Some(text), _) => { server @@ -723,11 +742,11 @@ async fn archive_noted( .into_err() .details("Invalid blob hash in undelete note") })?, - (None, None) => return Ok(()), + (None, None) => return undelete::groupware::clear(data, kind, account_id, document_id).await, }; undelete::groupware::archive(data, server.registry(), account_id, note, blob_hash, retention) - .await - .map(|_| ()) + .await?; + undelete::groupware::clear(data, kind, account_id, document_id).await } async fn delete_email_metadata( diff --git a/tests/src/system/legal_hold.rs b/tests/src/system/legal_hold.rs index f6f524e..12b139f 100644 --- a/tests/src/system/legal_hold.rs +++ b/tests/src/system/legal_hold.rs @@ -14,11 +14,22 @@ use crate::utils::{ }; use registry::schema::{ prelude::{ObjectType, Property}, - structs::{CertificateManagement, DkimManagement, DnsManagement, Domain, Tenant, UserRoles}, + structs::{ + CertificateManagement, DataRetention, DkimManagement, DnsManagement, Domain, Tenant, + UserRoles, + }, }; use serde_json::{Value, json}; +use types::id::Id; -const USING: &[&str] = &["urn:ietf:params:jmap:core", "urn:inbuxa:jmap"]; +const INBOX_ID: u32 = 0; + +const USING: &[&str] = &[ + "urn:ietf:params:jmap:core", + "urn:ietf:params:jmap:mail", + "urn:ietf:params:jmap:contacts", + "urn:inbuxa:jmap", +]; impl Account { async fn hold_call(&self, method: &str, mut arguments: Value) -> (String, Value) { @@ -38,6 +49,13 @@ impl Account { response } + async fn archived_items(&self) -> Vec { + let (_, response) = self + .hold_call("x:ArchivedItem/get", json!({"ids": null})) + .await; + response["list"].as_array().cloned().unwrap_or_default() + } + async fn hold_get(&self, id: &str) -> Value { let (name, response) = self .hold_call("inbuxa:LegalHold/get", json!({"ids": [id]})) @@ -266,6 +284,86 @@ pub async fn test(test: &mut TestServer) { "test 7, LH-2: the moved account escaped the hold" ); + // Test 6, LH-4: what a hold keeps, with undelete switched off, so only + // the hold can be keeping anything + admin + .registry_update_setting( + DataRetention { + archive_deleted_items_for: None, + ..Default::default() + }, + &[Property::ArchiveDeletedItemsFor], + ) + .await; + let held = admin + .create_user_account("held@example.com", "held-secret-4419", "Held", &[], vec![]) + .await; + let ranged = admin + .create_user_account("ranged@example.com", "ranged-secret-5530", "Ranged", &[], vec![]) + .await; + let response = admin + .hold_set(json!({"reason": "Preserve everything", "create": { + "w": {"name": "Matter 6001", "scope": {"accounts": [held.id_string()]}}, + "r": {"name": "Matter 6002", "from": "2020-01-01T00:00:00Z", "to": "2020-12-31T23:59:59Z", + "scope": {"accounts": [ranged.id_string()]}}}})) + .await; + assert!(response["created"]["w"]["id"].is_string(), "LH-1: {response}"); + assert!(response["created"]["r"]["id"].is_string(), "LH-1: {response}"); + + let held_client = held.jmap_client().await; + let ranged_client = ranged.jmap_client().await; + let whole = import(&held_client, "Held whole", None).await; + held_client.email_destroy(&whole).await.unwrap(); + // 2020-03-15: inside the range; now: outside it + let inside = import(&ranged_client, "Inside the range", Some(1_584_230_400)).await; + let outside = import(&ranged_client, "Outside the range", None).await; + ranged_client.email_destroy(&inside).await.unwrap(); + ranged_client.email_destroy(&outside).await.unwrap(); + + // LH-3: a contact is held whole, whatever the range + let (_, books) = ranged + .hold_call("AddressBook/get", json!({"ids": null})) + .await; + let book = books["list"][0]["id"] + .as_str() + .unwrap_or_else(|| panic!("no address book: {books}")) + .to_string(); + { + let (_, created) = ranged + .hold_call( + "ContactCard/set", + json!({"create": {"c": {"addressBookIds": {book: true}, + "name": {"full": "Kept Contact"}}}}), + ) + .await; + let card = created["created"]["c"]["id"].as_str().unwrap_or_default().to_string(); + let (_, destroyed) = ranged + .hold_call("ContactCard/set", json!({"destroy": [card]})) + .await; + assert!(destroyed["destroyed"][0].is_string(), "{destroyed}"); + } + test.wait_for_tasks().await; + + let is_held = |item: &Value| item["archivedUntil"].as_str().is_some_and(|u| u.starts_with("9999-")); + let kept = held.archived_items().await; + assert!( + kept.iter().any(|i| i["subject"] == "Held whole" && is_held(i)), + "test 6, LH-4: a held account's mail wasn't kept: {kept:?}" + ); + let kept = ranged.archived_items().await; + assert!( + kept.iter().any(|i| i["subject"] == "Inside the range" && is_held(i)), + "LH-3: mail inside the range wasn't kept: {kept:?}" + ); + assert!( + !kept.iter().any(|i| i["subject"] == "Outside the range"), + "LH-3: mail outside the range was kept, with undelete off: {kept:?}" + ); + assert!( + kept.iter().any(|i| i["name"] == "Kept Contact" && is_held(i)), + "LH-3: a contact wasn't kept whole: {kept:?}" + ); + // LH-10: release needs a reason, and a released hold stays, read-only let response = admin .hold_set(json!({"update": {hold_id.as_str(): {"released": true}}})) @@ -319,6 +417,23 @@ pub async fn test(test: &mut TestServer) { } } +async fn import( + client: &jmap_client::client::Client, + subject: &str, + received_at: Option, +) -> String { + client + .email_import( + format!("From: a@example.org\r\nSubject: {subject}\r\n\r\nBody.\r\n").into_bytes(), + [Id::from(INBOX_ID).to_string()], + None::>, + received_at, + ) + .await + .unwrap() + .take_id() +} + /// Runs these tests alone: `cargo test -p tests legal_hold_tests -- --ignored`. #[ignore] #[tokio::test(flavor = "multi_thread")]