Legal holds, step 3: deleted items in a held account are kept

Every way of deleting mail (JMAP, IMAP EXPUNGE, POP3, mailbox removal,
Trash emptying) and Sieve scripts, events, contacts and files now asks
how the account's deletions are kept: a hold keeps them with no expiry
(archivedUntil 9999-12-31), even with undelete off; otherwise undelete's
period applies as before (LH-4).

A hold's date range decides by the item's own date (LH-3). Mail is noted
as held at deletion and settled when it's archived, once its received
date is known; outside the range it gets undelete's deadline or isn't
kept. Events go by their start, with a day's slack for time zones;
recurring events, contacts, files and scripts are held whole.

A groupware item's note now stays until its archive succeeds, and a
failure retries the task instead of being logged and lost (LH-5).

crates/common/src/hold.rs

@@ -10,7 +10,7 @@
 //! there are few holds.
 
 use crate::Server;
-use inbuxa_features::hold::{self, Hold, Member};
+use inbuxa_features::hold::{self, Hold, Keeping, Member};
 
 impl Server {
     /// The active holds covering `account_id`, through its own name, its
@@ -36,6 +36,15 @@ impl Server {
         hold::covering(self.store(), &member).await
     }
 
+    /// How `account_id`'s deleted items are kept: its holds' ranges and the
+    /// undelete period in force now (LH-4, UD-6a).
+    pub async fn keeping(&self, account_id: u32) -> trc::Result<Keeping> {
+        let retention = inbuxa_features::undelete::settings::retention(self.registry())
+            .await?
+            .items;
+        Ok(Keeping::new(retention, &self.holds_on(account_id).await?))
+    }
+
     /// Whether any active hold covers `account_id` at all.
     pub async fn is_held(&self, account_id: u32) -> trc::Result<bool> {
         Ok(!self.holds_on(account_id).await?.is_empty())

crates/email/src/mailbox/destroy.rs

@@ -92,10 +92,8 @@ impl MailboxDestroy for Server {
 
                 let mut deleted_ids = RoaringBitmap::new();
                 let mut thread_ids = RoaringBitmap::new();
-                // inbuxa: UD-1, UD-6a: the retention in force now
-                let retention = inbuxa_features::undelete::settings::retention(self.registry())
-                    .await?
-                    .items;
+                // inbuxa: UD-1, UD-6a, LH-4: how this account's deletions are kept
+                let keeping = self.keeping(account_id).await?;
                 self.archives(
                     account_id,
                     Collection::Email,
@@ -125,10 +123,10 @@ impl MailboxDestroy for Server {
                             deleted_ids.insert(message_id);
                             thread_ids.insert(prev_message_data.inner.thread_id.to_native());
                             // inbuxa: UD-1, UD-4: a deleted message is noted for archiving
-                            if let Some(retention) = retention {
+                            if keeping.keeps_anything() {
                                 inbuxa_features::undelete::email::note(
                                     &mut batch,
-                                    retention,
+                                    &keeping,
                                     account_id,
                                     message_id,
                                     prev_message_data.inner.size.to_native() as u64,

crates/email/src/message/delete.rs

@@ -69,10 +69,8 @@ impl EmailDeletion for Server {
         batch
             .with_account_id(account_id)
             .with_collection(Collection::Email);
-        // inbuxa: UD-1, UD-6a: the retention in force now
-        let retention = inbuxa_features::undelete::settings::retention(self.registry())
-            .await?
-            .items;
+        // inbuxa: UD-1, UD-6a, LH-4: how this account's deletions are kept
+        let keeping = self.keeping(account_id).await?;
         self.archives(
             account_id,
             Collection::Email,
@@ -90,10 +88,10 @@ impl EmailDeletion for Server {
                 }
                 thread_ids.insert(metadata.inner.thread_id.to_native());
                 // inbuxa: UD-1, UD-4: a deleted message is noted for archiving
-                if let Some(retention) = retention {
+                if keeping.keeps_anything() {
                     inbuxa_features::undelete::email::note(
                         batch,
-                        retention,
+                        &keeping,
                         account_id,
                         document_id,
                         metadata.inner.size.to_native() as u64,

crates/email/src/sieve/delete.rs

@@ -44,12 +44,12 @@ impl SieveScriptDelete for Server {
             ))
             .await?
         {
-            // inbuxa: UD-1: a deleted script is kept, when archiving is on
-            if let Some(retention) =
-                inbuxa_features::undelete::settings::retention(self.registry())
-                    .await?
-                    .items
-            {
+            // inbuxa: UD-1, LH-4: a deleted script is kept, when archiving
+            // is on or a hold covers the account (whole: scripts have no date)
+            let keeping = self.keeping(account_id).await?;
+            let now = store::write::now();
+            if let Some(until) = keeping.until(now, keeping.is_held()) {
+                let retention = until.saturating_sub(now);
                 let script = obj_
                     .deserialize::<SieveScript>()
                     .caused_by(trc::location!())?;

crates/features/src/hold/mod.rs

@@ -26,6 +26,85 @@ use store::{
 };
 use trc::AddContext;
 
+/// The deadline a held archived item carries: the last second of 9999. It
+/// never passes, so every expiry check keeps the item without knowing about
+/// holds (LH-4, LH-5); releasing a hold gives it a real deadline (LH-10).
+pub const HELD_UNTIL: u64 = 253_402_300_799;
+
+/// Whether an archived item's deadline marks it as held. Anything past the
+/// year 9000 counts, so a deadline computed from a hold a moment earlier or
+/// later still reads as held.
+pub fn is_held_until(until: u64) -> bool {
+    until >= 221_845_392_000
+}
+
+/// A day, in seconds: the slack either side of a range for an event's start,
+/// whose time zone isn't known here.
+const DAY: u64 = 86_400;
+
+/// How an account's deleted items are kept: its holds' ranges, and the
+/// undelete period for whatever no hold covers (LH-3, LH-4).
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct Keeping {
+    /// `archiveDeletedItemsFor`, in seconds, if undelete is on.
+    pub retention: Option<u64>,
+    /// Each active hold's range on this account; `(None, None)` is a whole
+    /// account. Empty when nothing holds it.
+    pub ranges: Vec<(Option<u64>, Option<u64>)>,
+}
+
+impl Keeping {
+    pub fn new(retention: Option<u64>, holds: &[Hold]) -> Keeping {
+        Keeping {
+            retention,
+            ranges: holds.iter().map(|h| (h.from, h.to)).collect(),
+        }
+    }
+
+    /// Whether any hold reaches the account at all.
+    pub fn is_held(&self) -> bool {
+        !self.ranges.is_empty()
+    }
+
+    /// Whether deleted items need noting: something may keep them.
+    pub fn keeps_anything(&self) -> bool {
+        self.is_held() || self.retention.is_some()
+    }
+
+    /// Whether a hold covers an item dated `date`. No date means the item is
+    /// held whole, whatever the range (LH-3).
+    pub fn covers(&self, date: Option<u64>) -> bool {
+        self.ranges.iter().any(|(from, to)| match date {
+            None => true,
+            Some(at) => {
+                from.is_none_or(|from| at >= from) && to.is_none_or(|to| at <= to)
+            }
+        })
+    }
+
+    /// Like `covers`, for an event's start: a day of slack either side, since
+    /// its time zone isn't known here.
+    pub fn covers_event(&self, start: Option<u64>) -> bool {
+        self.ranges.iter().any(|(from, to)| match start {
+            None => true,
+            Some(at) => {
+                from.is_none_or(|from| at + DAY >= from)
+                    && to.is_none_or(|to| at <= to.saturating_add(DAY))
+            }
+        })
+    }
+
+    /// Until when an item deleted at `now` is kept: held, the undelete
+    /// period, or not at all.
+    pub fn until(&self, now: u64, held: bool) -> Option<u64> {
+        if held {
+            Some(HELD_UNTIL)
+        } else {
+            self.retention.map(|retention| now + retention)
+        }
+    }
+}
+
 const FEATURE: u8 = b'H';
 const KIND_HOLD: u8 = b'h';
 
@@ -511,6 +590,27 @@ mod tests {
         assert!(held.scope.covers(&member) && !held.scope.covers(&moved));
     }
 
+    #[test]
+    fn keeping_deleted_items() {
+        let whole = Keeping::new(None, &[hold(accounts(&[2]), None, None)]);
+        assert!(whole.covers(Some(5)) && whole.covers(None));
+        assert_eq!(whole.until(100, whole.covers(Some(5))), Some(HELD_UNTIL));
+        assert!(is_held_until(whole.until(100, true).unwrap()));
+
+        // LH-3: a range holds only what's inside it; outside, undelete's rules
+        let ranged = Keeping::new(Some(30), &[hold(accounts(&[2]), Some(1_000), Some(2_000))]);
+        assert!(ranged.covers(Some(1_500)) && !ranged.covers(Some(2_500)));
+        assert!(ranged.covers(None), "contacts, files and scripts are held whole");
+        assert_eq!(ranged.until(100, ranged.covers(Some(2_500))), Some(130));
+        assert!(ranged.covers_event(Some(2_000 + 3_600)), "a day of slack for an event");
+
+        // Neither held nor undelete: nothing is kept
+        let none = Keeping::new(None, &[]);
+        assert!(!none.keeps_anything());
+        assert_eq!(none.until(100, false), None);
+        assert!(!is_held_until(100 + 30 * 365 * 86_400));
+    }
+
     #[test]
     fn stored_as_json() {
         let current = hold(accounts(&[2]), Some(100), None);

crates/features/src/undelete/data.rs

@@ -123,6 +123,14 @@ pub struct EmailNote {
     pub size: u64,
     pub mailboxes: Vec<u32>,
     pub keywords: Vec<String>,
+    /// LH-3: the ranges of the holds on the account when it was deleted.
+    /// Its received date is only known when it's archived, which decides
+    /// whether a hold keeps it after all.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub held_ranges: Vec<(Option<u64>, Option<u64>)>,
+    /// The undelete deadline for when no range covers it.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub otherwise_until: Option<u64>,
 }
 
 /// What restore needs beyond the kept copy (UD-4, UD-8).

crates/features/src/undelete/email.rs

@@ -12,9 +12,12 @@
 //! is made if archiving is on, fixing the deadline then. When the data is
 //! finally removed, a noted message becomes an archived item.
 
-use crate::undelete::{
-    data::{self, EmailNote, Extra},
-    records,
+use crate::{
+    hold::Keeping,
+    undelete::{
+        data::{self, EmailNote, Extra},
+        records,
+    },
 };
 use registry::{
     schema::structs::{ArchivedEmail, ArchivedItem},
@@ -26,10 +29,12 @@ use store::{
 };
 use types::{blob::BlobId, blob_hash::BlobHash};
 
-/// Notes a deleted message, when archiving is on (`retention` seconds).
+/// Notes a deleted message, when anything keeps it: undelete, or a legal
+/// hold on the account (LH-4). A held note keeps it until it's archived,
+/// when its received date says whether the hold's range covers it.
 pub fn note(
     batch: &mut BatchBuilder,
-    retention: u64,
+    keeping: &Keeping,
     account_id: u32,
     document_id: u32,
     size: u64,
@@ -37,16 +42,23 @@ pub fn note(
     keywords: Vec<String>,
 ) -> trc::Result<()> {
     let archived_at = now();
+    // Held until the date is known; the undelete deadline otherwise
+    let otherwise_until = keeping.until(archived_at, false);
+    let Some(archived_until) = keeping.until(archived_at, keeping.is_held()) else {
+        return Ok(());
+    };
     data::note_email(
         batch,
         account_id,
         document_id,
         &EmailNote {
             archived_at,
-            archived_until: archived_at + retention,
+            archived_until,
             size,
             mailboxes,
             keywords,
+            held_ranges: keeping.ranges.clone(),
+            otherwise_until: if keeping.is_held() { otherwise_until } else { None },
         },
     )
 }
@@ -78,9 +90,27 @@ pub async fn archive(
     document_id: u32,
     summary: Summary<'_>,
 ) -> trc::Result<bool> {
-    let Some(note) = data::email_note(data, account_id, document_id).await? else {
+    let Some(mut note) = data::email_note(data, account_id, document_id).await? else {
         return Ok(false);
     };
+    // LH-3: a held note's range decides now that the date is known; outside
+    // it, undelete's deadline, or nothing kept at all
+    if !note.held_ranges.is_empty() {
+        let keeping = Keeping {
+            retention: None,
+            ranges: std::mem::take(&mut note.held_ranges),
+        };
+        if !keeping.covers(Some(summary.received_at)) {
+            match note.otherwise_until {
+                Some(until) => note.archived_until = until,
+                None => {
+                    let mut batch = BatchBuilder::new();
+                    data::clear_email_note(&mut batch, account_id, document_id);
+                    return data.write(batch.build_all()).await.map(|_| false);
+                }
+            }
+        }
+    }
     let item = ArchivedItem::Email(ArchivedEmail {
         from: summary.from.unwrap_or_default().to_string(),
         subject: summary.subject.unwrap_or_default().to_string(),

crates/features/src/undelete/groupware.rs

@@ -97,6 +97,39 @@ pub async fn take(
     Ok(Some(note))
 }
 
+/// A note, left in place: for a held account it's cleared only once its item
+/// is archived, so a failure leaves it for the retry (LH-5).
+pub async fn peek(
+    data: &Store,
+    kind: Kind,
+    account_id: u32,
+    document_id: u32,
+) -> trc::Result<Option<Note>> {
+    Ok(data
+        .get_value::<Json<Note>>(ValueKey::from(note_class(kind, account_id, document_id)))
+        .await?
+        .map(|Json(note)| note))
+}
+
+/// Removes a note once its item is archived or needn't be.
+pub async fn clear(data: &Store, kind: Kind, account_id: u32, document_id: u32) -> trc::Result<()> {
+    let mut batch = BatchBuilder::new();
+    batch.clear(note_class(kind, account_id, document_id));
+    data.write(batch.build_all()).await.map(|_| ())
+}
+
+/// An event's start, for a hold's range (LH-3). None for a recurring event,
+/// which may have an occurrence anywhere, so a hold keeps it whole.
+pub fn event_start(note: &Note) -> Option<u64> {
+    let text = note.content.as_deref()?;
+    if property(text, "RRULE").is_some() || property(text, "RDATE").is_some() {
+        return None;
+    }
+    property(text, "DTSTART")
+        .and_then(|v| ical_time(&v))
+        .map(|t| t.max(0) as u64)
+}
+
 /// The value of the first line starting with `name` (as `NAME:` or
 /// `NAME;params:`) in iCalendar or vCard text, unfolded.
 fn property(text: &str, name: &str) -> Option<String> {

crates/imap/src/op/expunge.rs

@@ -243,10 +243,8 @@ impl<T: SessionStream> SessionData<T> {
 
         let mut fully_deleted = RoaringBitmap::new();
         let mut thread_ids = RoaringBitmap::new();
-        // inbuxa: UD-1, UD-6a: the retention in force now
-        let retention = inbuxa_features::undelete::settings::retention(self.server.registry())
-            .await?
-            .items;
+        // inbuxa: UD-1, UD-6a, LH-4: how this account's deletions are kept
+        let keeping = self.server.keeping(account_id).await?;
         self.server
             .archives(
                 account_id,
@@ -270,10 +268,10 @@ impl<T: SessionStream> SessionData<T> {
                             fully_deleted.insert(document_id);
                             thread_ids.insert(metadata.inner.thread_id.to_native());
                             // inbuxa: UD-1, UD-4: a deleted message is noted for archiving
-                            if let Some(retention) = retention {
+                            if keeping.keeps_anything() {
                                 inbuxa_features::undelete::email::note(
                                     batch,
-                                    retention,
+                                    &keeping,
                                     account_id,
                                     document_id,
                                     metadata.inner.size.to_native() as u64,

crates/services/src/task_manager/index.rs

@@ -229,11 +229,19 @@ impl SearchIndexTask for Server {
                         IndexDocumentType::Email => None,
                     } && let Err(err) = archive_noted(self, kind, account_id, document_id).await
                     {
+                        // inbuxa: LH-5: the note stays, so the retry archives
+                        // it; nothing a hold keeps is lost to a failure
                         trc::error!(
                             err.account_id(account_id)
                                 .document_id(document_id)
                                 .details("Failed to archive a deleted item")
                         );
+                        results.push(IndexTaskResult {
+                            task_type: TaskType::Delete,
+                            index: task.document_type,
+                            result: TaskResult::temporary("Failed to archive a deleted item"),
+                        });
+                        continue;
                     }
 
                     document_deletions[idx]
@@ -696,8 +704,9 @@ pub fn trace_search_document(
     document
 }
 
-// inbuxa: UD-1, UD-4: archives a deleted file, event or contact noted at
-// deletion, when archiving is on; otherwise its note is dropped
+// inbuxa: UD-1, UD-4, LH-4: archives a deleted file, event or contact noted
+// at deletion, when archiving is on or a hold covers it; otherwise its note is
+// dropped. The note goes only once the item is archived.
 async fn archive_noted(
     server: &Server,
     kind: undelete::groupware::Kind,
@@ -705,12 +714,22 @@ async fn archive_noted(
     document_id: u32,
 ) -> trc::Result<()> {
     let data = &server.core.storage.data;
-    let Some(note) = undelete::groupware::take(data, kind, account_id, document_id).await? else {
+    let Some(note) = undelete::groupware::peek(data, kind, account_id, document_id).await? else {
         return Ok(());
     };
-    let Some(retention) = undelete::settings::retention(server.registry()).await?.items else {
-        return Ok(());
+    // LH-3: events by their start; contacts and files whole
+    let keeping = server.keeping(account_id).await?;
+    let held = match kind {
+        undelete::groupware::Kind::CalendarEvent => {
+            keeping.covers_event(undelete::groupware::event_start(&note))
+        }
+        _ => keeping.covers(None),
     };
+    let now = store::write::now();
+    let Some(until) = keeping.until(now, held) else {
+        return undelete::groupware::clear(data, kind, account_id, document_id).await;
+    };
+    let retention = until.saturating_sub(now);
     let blob_hash = match (&note.content, &note.blob_hash) {
         (Some(text), _) => {
             server
@@ -723,11 +742,11 @@ async fn archive_noted(
                 .into_err()
                 .details("Invalid blob hash in undelete note")
         })?,
-        (None, None) => return Ok(()),
+        (None, None) => return undelete::groupware::clear(data, kind, account_id, document_id).await,
     };
     undelete::groupware::archive(data, server.registry(), account_id, note, blob_hash, retention)
-        .await
-        .map(|_| ())
+        .await?;
+    undelete::groupware::clear(data, kind, account_id, document_id).await
 }
 
 async fn delete_email_metadata(