Legal holds, step 4: freezing, release, and the audit log

Placing or widening a hold freezes what's already archived in its scope
and range, its old deadline noted; releasing one gives each item no other
hold covers that deadline back, or release plus 30 days if later. One
pass over the archive does both and changes nothing twice (LH-6, LH-10,
LH-11). A held archived item can't be destroyed; restoring still can,
and the hold is named only to callers who may see holds (LH-7). Audit
records about a held account survive the purge (AU-7).

Fixes the daily clean-up of expired archived items (UD-13), which never
found any: the registry's unfiltered query reads an all-ids index that
archived items aren't in. Items are now walked account by account, kept
deleted accounts included. Expired items were still removed whenever
their account's archive was read.

crates/features/src/hold/mod.rs

@@ -107,6 +107,7 @@ impl Keeping {
 
 const FEATURE: u8 = b'H';
 const KIND_HOLD: u8 = b'h';
+const KIND_ORIGINAL: u8 = b'o';
 
 /// How many times creating a hold retries when another node took its id.
 const CREATE_ATTEMPTS: usize = 5;
@@ -367,6 +368,40 @@ fn key(id: u32) -> ValueKey<ValueClass> {
     ValueKey::from(class(id))
 }
 
+fn original_class(item_id: u64) -> ValueClass {
+    let mut key = Vec::with_capacity(10);
+    key.push(FEATURE);
+    key.push(KIND_ORIGINAL);
+    key.extend_from_slice(&item_id.to_be_bytes());
+    ValueClass::Any(AnyClass {
+        subspace: SUBSPACE_INBUXA,
+        key,
+    })
+}
+
+/// LH-10: an archived item's deadline from before a hold froze it, so a
+/// release can give it back (or a later one). None for an item held from
+/// its deletion, which never had one.
+pub async fn original_deadline(data: &Store, item_id: u64) -> trc::Result<Option<u64>> {
+    data.get_value::<u64>(ValueKey::from(original_class(item_id)))
+        .await
+        .caused_by(trc::location!())
+}
+
+/// Notes (`Some`) or forgets (`None`) an item's deadline from before it
+/// was frozen.
+pub async fn set_original_deadline(data: &Store, item_id: u64, until: Option<u64>) -> trc::Result<()> {
+    let mut batch = BatchBuilder::new();
+    match until {
+        Some(until) => batch.set(original_class(item_id), until.to_be_bytes().to_vec()),
+        None => batch.clear(original_class(item_id)),
+    };
+    data.write(batch.build_all())
+        .await
+        .caused_by(trc::location!())
+        .map(|_| ())
+}
+
 /// One hold, released or not.
 pub async fn get(data: &Store, id: u32) -> trc::Result<Option<Hold>> {
     Ok(data

crates/features/src/undelete/data.rs

@@ -470,8 +470,15 @@ mod tests {
             size: 3,
             mailboxes: vec![1],
             keywords: vec![],
+            held_ranges: vec![(Some(10), None)],
+            otherwise_until: Some(20),
         };
         let bytes = Json(&note).serialize().unwrap();
         assert_eq!(Json::<EmailNote>::deserialize(&bytes).unwrap().0, note);
+
+        // A note written before legal holds still reads, as not held
+        let old = br#"{"archived_at":1,"archived_until":2,"size":3,"mailboxes":[1],"keywords":[]}"#;
+        let read = Json::<EmailNote>::deserialize(old).unwrap().0;
+        assert!(read.held_ranges.is_empty() && read.otherwise_until.is_none());
     }
 }

crates/features/src/undelete/records.rs

@@ -89,6 +89,54 @@ pub async fn insert(
     Ok(id)
 }
 
+/// Moves an archived item's deadline, and its kept copy's with it: frozen
+/// by a hold (LH-6) or given a real one on release (LH-10). Returns the
+/// item as it now is.
+pub async fn set_deadline(
+    data: &Store,
+    registry: &RegistryStore,
+    id: Id,
+    item: &ArchivedItem,
+    until: u64,
+) -> trc::Result<ArchivedItem> {
+    let account_id = item.account_id().document_id();
+    let blob_hash = item.blob_id().hash.clone();
+    let before = item.archived_until().timestamp() as u64;
+    let mut updated = item.clone();
+    updated.set_archived_until(registry::types::datetime::UTCDateTime::from_timestamp(until as i64));
+
+    // The new link first, so the kept copy is never unlinked in between
+    let mut batch = BatchBuilder::new();
+    batch
+        .with_account_id(account_id)
+        .set(
+            BlobOp::Link {
+                hash: blob_hash.clone(),
+                to: BlobLink::Temporary { until },
+            },
+            vec![],
+        );
+    if before != until {
+        batch.clear(BlobOp::Link {
+            hash: blob_hash,
+            to: BlobLink::Temporary { until: before },
+        });
+    }
+    data::log_change(&mut batch, account_id, registry.assign_id(), id, Change::Updated);
+    data.write(batch.build_all())
+        .await
+        .caused_by(trc::location!())?;
+
+    let mut batch = BatchBuilder::new();
+    batch.set(item_class(id.id()), updated.to_pickled_vec());
+    registry
+        .store()
+        .write(batch.build_all())
+        .await
+        .caused_by(trc::location!())?;
+    Ok(updated)
+}
+
 /// Removes an archived item and releases its kept copy: on restore (UD-9),
 /// on destroy (UD-12) and past its deadline (UD-13).
 pub async fn remove(
@@ -184,15 +232,38 @@ pub async fn get(
     }
 }
 
+/// Every archived item on the server, account by account. Items are
+/// indexed by account only, so the registry's query without a filter,
+/// which reads its all-ids index, finds none of them.
+pub async fn all(data: &Store, registry: &RegistryStore) -> trc::Result<Vec<Id>> {
+    let mut accounts = registry
+        .query::<Vec<Id>>(RegistryQuery::new(ObjectType::Account))
+        .await
+        .caused_by(trc::location!())?
+        .into_iter()
+        .map(|id| id.document_id())
+        .collect::<Vec<_>>();
+    // Deleted accounts still kept have archived items too
+    accounts.extend(data::kept_accounts(data).await?.into_iter().map(|(id, _)| id));
+    accounts.sort_unstable();
+    accounts.dedup();
+    let mut items = Vec::new();
+    for account_id in accounts {
+        items.extend(
+            registry
+                .query::<Vec<Id>>(RegistryQuery::new(ObjectType::ArchivedItem).with_account(account_id))
+                .await
+                .caused_by(trc::location!())?,
+        );
+    }
+    Ok(items)
+}
+
 /// Removes every expired archived item on the server (UD-13), for the
 /// scheduled clean-up.
 pub async fn remove_expired(data: &Store, registry: &RegistryStore) -> trc::Result<usize> {
     let mut removed = 0;
-    for id in registry
-        .query::<Vec<Id>>(RegistryQuery::new(ObjectType::ArchivedItem))
-        .await
-        .caused_by(trc::location!())?
-    {
+    for id in all(data, registry).await? {
         if let Some(item) = registry.object::<ArchivedItem>(id).await?
             && is_expired(&item)
         {