From 5b963b06c66df00a42ec509cb7185fc5858a76d3 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 18 Sep 2026 21:07:27 -0700 Subject: [PATCH] Undelete: deleted files, calendar events, contacts and Sieve scripts are kept and restored where they were (UD-1, UD-8 to UD-11) Files, events and contacts are noted when deleted for good and archived by the unindex task when retention is on; Sieve scripts are archived at deletion. A restore goes back to its folder, calendar or address book if it still exists, takes a free " (restored)" name, comes back inactive for scripts, and is refused over quota with the item left archived. Acceptance tests 6, 8 and 12. --- Cargo.lock | 1 + crates/email/src/sieve/delete.rs | 27 ++ crates/features/src/undelete/data.rs | 2 + crates/features/src/undelete/groupware.rs | 282 ++++++++++++++++++ crates/features/src/undelete/mod.rs | 1 + crates/groupware/Cargo.toml | 1 + crates/groupware/src/calendar/storage.rs | 9 + crates/groupware/src/contact/storage.rs | 19 ++ crates/groupware/src/file/storage.rs | 18 ++ crates/groupware/src/inbuxa.rs | 108 +++++++ crates/groupware/src/lib.rs | 1 + .../src/task_manager/inbuxa_restore.rs | 272 +++++++++++++++++ crates/services/src/task_manager/index.rs | 49 +++ crates/services/src/task_manager/mod.rs | 1 + .../services/src/task_manager/restore_item.rs | 25 +- tests/src/system/undelete.rs | 131 ++++++++ 16 files changed, 946 insertions(+), 1 deletion(-) create mode 100644 crates/features/src/undelete/groupware.rs create mode 100644 crates/groupware/src/inbuxa.rs create mode 100644 crates/services/src/task_manager/inbuxa_restore.rs diff --git a/Cargo.lock b/Cargo.lock index 4b98a36..14a004a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3032,6 +3032,7 @@ dependencies = [ "icu_locale_core", "icu_plurals", "icu_provider", + "inbuxa-features", "indexmap 2.14.2", "nlp", "percent-encoding", diff --git a/crates/email/src/sieve/delete.rs b/crates/email/src/sieve/delete.rs index 730fb43..b030bb6 100644 --- a/crates/email/src/sieve/delete.rs +++ b/crates/email/src/sieve/delete.rs @@ -42,6 +42,33 @@ impl SieveScriptDelete for Server { )) .await? { + // inbuxa: UD-1: a deleted script is kept, when archiving is on + if let Some(retention) = + inbuxa_features::undelete::settings::retention(self.registry()) + .await? + .items + { + let script = obj_ + .deserialize::() + .caused_by(trc::location!())?; + let content = self + .blob_store() + .get_blob(script.blob_hash.as_slice(), 0..usize::MAX) + .await? + .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()) + .unwrap_or_default(); + inbuxa_features::undelete::groupware::archive_sieve( + &self.core.storage.data, + self.registry(), + account_id, + &script.name, + content, + script.blob_hash.clone(), + retention, + ) + .await?; + } + // Delete record batch .with_account_id(account_id) diff --git a/crates/features/src/undelete/data.rs b/crates/features/src/undelete/data.rs index 5ad9559..c69813d 100644 --- a/crates/features/src/undelete/data.rs +++ b/crates/features/src/undelete/data.rs @@ -137,6 +137,8 @@ pub enum Extra { parent_id: Option, name: String, media_type: Option, + #[serde(default)] + size: u32, }, CalendarEvent { calendar_ids: Vec, diff --git a/crates/features/src/undelete/groupware.rs b/crates/features/src/undelete/groupware.rs new file mode 100644 index 0000000..b94e7dc --- /dev/null +++ b/crates/features/src/undelete/groupware.rs @@ -0,0 +1,282 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! Deleted files, calendar events and contacts (UD-1, a deliberate +//! extension: upstream archives only email). +//! +//! The code that deletes them for good (over JMAP and WebDAV alike) builds +//! its changes without reaching the settings, so it notes every deletion: +//! the content (a calendar or contact's text, or a file's blob) and where +//! the item lived. The task that then removes the item from the search index +//! archives the note if archiving is on, or drops it. + +use crate::undelete::{ + data::{Extra, Json}, + records, +}; +use registry::{ + schema::structs::{ArchivedCalendarEvent, ArchivedContactCard, ArchivedFileNode, ArchivedItem}, + types::datetime::UTCDateTime, +}; +use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize}; +use store::{ + RegistryStore, Serialize, Store, ValueKey, + write::{AnyClass, BatchBuilder, ValueClass, now}, +}; +use types::{blob::BlobId, blob_hash::BlobHash, id::Id}; + +/// The kinds noted here, as stored in the note's key. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum Kind { + File = 0, + CalendarEvent = 1, + ContactCard = 2, +} + +/// A deleted item, noted at deletion. +#[derive(Debug, Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)] +pub struct Note { + pub deleted_at: u64, + pub created_at: i64, + /// A calendar event or contact's text. + pub content: Option, + /// A file's content, already a blob. + pub blob_hash: Option>, + pub extra: Extra, +} + +fn note_class(kind: Kind, account_id: u32, document_id: u32) -> ValueClass { + let mut key = Vec::with_capacity(11); + key.extend_from_slice(b"Ug"); + key.push(kind as u8); + key.extend_from_slice(&account_id.to_be_bytes()); + key.extend_from_slice(&document_id.to_be_bytes()); + ValueClass::Any(AnyClass { + subspace: store::SUBSPACE_INBUXA, + key, + }) +} + +/// Notes an item deleted for good (UD-1). +pub fn note( + batch: &mut BatchBuilder, + kind: Kind, + account_id: u32, + document_id: u32, + mut note: Note, +) -> trc::Result<()> { + note.deleted_at = now(); + batch.set( + note_class(kind, account_id, document_id), + Json(¬e).serialize()?, + ); + Ok(()) +} + +/// Takes an item's note, if it has one. +pub async fn take( + data: &Store, + kind: Kind, + account_id: u32, + document_id: u32, +) -> trc::Result> { + let class = note_class(kind, account_id, document_id); + let Some(Json(note)) = data + .get_value::>(ValueKey::from(class.clone())) + .await? + else { + return Ok(None); + }; + let mut batch = BatchBuilder::new(); + batch.clear(class); + data.write(batch.build_all()).await?; + Ok(Some(note)) +} + +/// The value of the first line starting with `name` (as `NAME:` or +/// `NAME;params:`) in iCalendar or vCard text, unfolded. +fn property(text: &str, name: &str) -> Option { + let mut lines = text.split("\r\n").flat_map(|l| l.split('\n')).peekable(); + while let Some(line) = lines.next() { + let Some(rest) = line + .get(..name.len()) + .filter(|head| head.eq_ignore_ascii_case(name)) + .map(|_| &line[name.len()..]) + else { + continue; + }; + if !(rest.starts_with(':') || rest.starts_with(';')) { + continue; + } + let mut value = rest.split_once(':')?.1.to_string(); + while let Some(next) = lines.peek() { + if let Some(continued) = next.strip_prefix(' ').or_else(|| next.strip_prefix('\t')) { + value.push_str(continued); + lines.next(); + } else { + break; + } + } + return Some(value.replace("\\,", ",").replace("\\;", ";").replace("\\n", " ")); + } + None +} + +/// An iCalendar date or date-time (`20260918`, `20260918T100000Z`) as a +/// Unix timestamp, read as UTC. +fn ical_time(value: &str) -> Option { + let digits = value.trim().trim_end_matches('Z'); + let (date, time) = digits.split_once('T').unwrap_or((digits, "000000")); + if date.len() != 8 || time.len() < 6 { + return None; + } + let n = |s: &str| s.parse::().ok(); + let (y, m, d) = (n(&date[..4])?, n(&date[4..6])?, n(&date[6..8])?); + let (hh, mm, ss) = (n(&time[..2])?, n(&time[2..4])?, n(&time[4..6])?); + // Days from the civil date (Howard Hinnant's algorithm) + let y = if m <= 2 { y - 1 } else { y }; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let doy = (153 * (if m > 2 { m - 3 } else { m + 9 }) + 2) / 5 + d - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + let days = era * 146097 + doe - 719468; + Some(days * 86_400 + hh * 3600 + mm * 60 + ss) +} + +/// Archives a noted item: holds its kept copy (`blob_hash`) until the +/// deadline and writes its record (UD-1, UD-4, UD-5). +pub async fn archive( + data: &Store, + registry: &RegistryStore, + account_id: u32, + note: Note, + blob_hash: BlobHash, + retention: u64, +) -> trc::Result { + let archived_at = now(); + let common = ( + types::id::Id::from(account_id), + UTCDateTime::from_timestamp(archived_at as i64), + UTCDateTime::from_timestamp((archived_at + retention) as i64), + BlobId::new(blob_hash, Default::default()), + UTCDateTime::from_timestamp(note.created_at), + ); + let text = note.content.as_deref().unwrap_or_default(); + let item = match ¬e.extra { + Extra::FileNode { name, .. } => ArchivedItem::FileNode(ArchivedFileNode { + name: name.clone(), + created_at: common.4, + account_id: common.0, + archived_at: common.1, + archived_until: common.2, + blob_id: common.3, + }), + Extra::CalendarEvent { .. } => ArchivedItem::CalendarEvent(ArchivedCalendarEvent { + title: property(text, "SUMMARY").unwrap_or_default(), + start_time: property(text, "DTSTART") + .and_then(|v| ical_time(&v)) + .map(UTCDateTime::from_timestamp), + created_at: common.4, + account_id: common.0, + archived_at: common.1, + archived_until: common.2, + blob_id: common.3, + }), + Extra::ContactCard { .. } => ArchivedItem::ContactCard(ArchivedContactCard { + name: property(text, "FN"), + created_at: common.4, + account_id: common.0, + archived_at: common.1, + archived_until: common.2, + blob_id: common.3, + }), + Extra::Email { .. } | Extra::SieveScript { .. } => { + return Err(trc::StoreEvent::UnexpectedError + .into_err() + .details("Not a groupware item")); + } + }; + records::insert(data, registry, &item, ¬e.extra).await +} + +/// The names a restored item tries, in order: the original, then with +/// ` (restored)`, then numbered (UD-8). +pub fn candidates(name: &str) -> impl Iterator + '_ { + let (stem, ext) = match name.rsplit_once('.') { + Some((stem, ext)) if !stem.is_empty() => (stem, format!(".{ext}")), + _ => (name, String::new()), + }; + std::iter::once(name.to_string()) + .chain(std::iter::once(format!("{stem} (restored){ext}"))) + .chain((2u32..).map(move |n| format!("{stem} (restored {n}){ext}"))) +} + +/// The first free name for a restored item (UD-8). +pub fn free_name(name: &str, is_taken: impl Fn(&str) -> bool) -> String { + candidates(name).find(|candidate| !is_taken(candidate)).unwrap() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_ical_and_vcard() { + let ics = "BEGIN:VCALENDAR\r\nBEGIN:VEVENT\r\nSUMMARY:Team\r\n lunch\r\nDTSTART;TZID=X:20260918T120000\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n"; + assert_eq!(property(ics, "SUMMARY").as_deref(), Some("Team lunch")); + assert_eq!( + property(ics, "DTSTART").and_then(|v| ical_time(&v)), + Some(1_789_732_800) + ); + let vcf = "BEGIN:VCARD\r\nVERSION:4.0\r\nFN:Jane Doe\r\nEND:VCARD\r\n"; + assert_eq!(property(vcf, "FN").as_deref(), Some("Jane Doe")); + assert_eq!(property(vcf, "N"), None); + } + + #[test] + fn restored_names() { + assert_eq!(free_name("a.txt", |_| false), "a.txt"); + assert_eq!(free_name("a.txt", |n| n == "a.txt"), "a (restored).txt"); + assert_eq!( + free_name("a.txt", |n| n == "a.txt" || n == "a (restored).txt"), + "a (restored 2).txt" + ); + assert_eq!(free_name("script", |n| n == "script"), "script (restored)"); + } +} + +/// Archives a Sieve script deleted for good (UD-1). Its content is already +/// a blob, held from now until the deadline. +pub async fn archive_sieve( + data: &Store, + registry: &RegistryStore, + account_id: u32, + name: &str, + content: String, + blob_hash: BlobHash, + retention: u64, +) -> trc::Result { + let archived_at = now(); + let item = ArchivedItem::SieveScript(registry::schema::structs::ArchivedSieveScript { + name: name.to_string(), + created_at: UTCDateTime::from_timestamp(archived_at as i64), + content, + account_id: types::id::Id::from(account_id), + archived_at: UTCDateTime::from_timestamp(archived_at as i64), + archived_until: UTCDateTime::from_timestamp((archived_at + retention) as i64), + blob_id: BlobId::new(blob_hash, Default::default()), + }); + records::insert( + data, + registry, + &item, + &Extra::SieveScript { + name: name.to_string(), + }, + ) + .await +} diff --git a/crates/features/src/undelete/mod.rs b/crates/features/src/undelete/mod.rs index 37189af..aea2b44 100644 --- a/crates/features/src/undelete/mod.rs +++ b/crates/features/src/undelete/mod.rs @@ -15,5 +15,6 @@ pub mod data; pub mod email; +pub mod groupware; pub mod records; pub mod settings; diff --git a/crates/groupware/Cargo.toml b/crates/groupware/Cargo.toml index 1d557f7..932f1f8 100644 --- a/crates/groupware/Cargo.toml +++ b/crates/groupware/Cargo.toml @@ -11,6 +11,7 @@ types = { path = "../types" } trc = { path = "../trc" } nlp = { path = "../nlp" } registry = { path = "../registry" } +inbuxa-features = { path = "../features" } calcard = { version = "0.3", features = ["rkyv"] } hashify = "0.2" rkyv = { version = "0.8.18", features = ["little_endian"] } diff --git a/crates/groupware/src/calendar/storage.rs b/crates/groupware/src/calendar/storage.rs index e566bc2..ef1f9db 100644 --- a/crates/groupware/src/calendar/storage.rs +++ b/crates/groupware/src/calendar/storage.rs @@ -462,6 +462,15 @@ impl DestroyArchive> { batch: &mut BatchBuilder, ) -> trc::Result<()> { let event = self.0; + // inbuxa: UD-1: noted for undelete + crate::inbuxa::note_event( + batch, + account_id, + document_id, + &event + .deserialize::() + .caused_by(trc::location!())?, + )?; // Delete event batch .with_account_id(account_id) diff --git a/crates/groupware/src/contact/storage.rs b/crates/groupware/src/contact/storage.rs index 01cfb69..50dda40 100644 --- a/crates/groupware/src/contact/storage.rs +++ b/crates/groupware/src/contact/storage.rs @@ -234,6 +234,15 @@ impl DestroyArchive> { ) .caused_by(trc::location!())?; } else { + // inbuxa: UD-1: noted for undelete + crate::inbuxa::note_card( + batch, + account_id, + document_id, + &card + .deserialize::() + .caused_by(trc::location!())?, + )?; // Delete card batch .with_document(document_id) @@ -262,6 +271,16 @@ impl DestroyArchive> { document_id: u32, batch: &mut BatchBuilder, ) -> trc::Result<()> { + // inbuxa: UD-1: noted for undelete + crate::inbuxa::note_card( + batch, + account_id, + document_id, + &self + .0 + .deserialize::() + .caused_by(trc::location!())?, + )?; batch .with_account_id(account_id) .with_collection(Collection::ContactCard) diff --git a/crates/groupware/src/file/storage.rs b/crates/groupware/src/file/storage.rs index a8f468d..3b55c68 100644 --- a/crates/groupware/src/file/storage.rs +++ b/crates/groupware/src/file/storage.rs @@ -82,6 +82,14 @@ impl DestroyArchive> { batch: &mut BatchBuilder, path: String, ) -> trc::Result<()> { + // inbuxa: UD-1: noted for undelete + crate::inbuxa::note_file( + batch, + account_id, + document_id, + &self.0.deserialize::().caused_by(trc::location!())?, + )?; + // Prepare write batch batch .with_account_id(account_id) @@ -143,6 +151,16 @@ impl DestroyArchive> { )) .await? { + // inbuxa: UD-1: noted for undelete + crate::inbuxa::note_file( + batch, + account_id, + document_id, + &node + .deserialize::() + .caused_by(trc::location!())?, + )?; + // Delete record batch .with_document(document_id) diff --git a/crates/groupware/src/inbuxa.rs b/crates/groupware/src/inbuxa.rs new file mode 100644 index 0000000..4ddc2f4 --- /dev/null +++ b/crates/groupware/src/inbuxa.rs @@ -0,0 +1,108 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! Undelete notes for files, calendar events and contacts deleted for good +//! (`docs/spec/features/undelete.md`, UD-1). The rules are in +//! `inbuxa_features::undelete`. + +use crate::{calendar::CalendarEvent, contact::ContactCard, file::FileNode}; +use inbuxa_features::undelete::{ + data::Extra, + groupware::{Kind, Note, note}, +}; +use registry::schema::{ + enums::IndexDocumentType, + structs::{Task, TaskIndexDocument, TaskStatus}, +}; +use store::write::BatchBuilder; + +/// A file (not a folder) deleted for good. +pub fn note_file( + batch: &mut BatchBuilder, + account_id: u32, + document_id: u32, + node: &FileNode, +) -> trc::Result<()> { + let Some(file) = &node.file else { + return Ok(()); + }; + // Files aren't search-indexed, so nothing else schedules the task that + // archives the note: schedule it here + batch.schedule_task(Task::UnindexDocument(TaskIndexDocument { + account_id: account_id.into(), + document_id: document_id.into(), + document_type: IndexDocumentType::File, + status: TaskStatus::now(), + })); + note( + batch, + Kind::File, + account_id, + document_id, + Note { + deleted_at: 0, + created_at: node.created, + content: None, + blob_hash: Some(file.blob_hash.as_slice().to_vec()), + extra: Extra::FileNode { + parent_id: Some(node.parent_id), + name: node.name.clone(), + media_type: file.media_type.clone(), + size: file.size, + }, + }, + ) +} + +/// A calendar event deleted for good. +pub fn note_event( + batch: &mut BatchBuilder, + account_id: u32, + document_id: u32, + event: &CalendarEvent, +) -> trc::Result<()> { + note( + batch, + Kind::CalendarEvent, + account_id, + document_id, + Note { + deleted_at: 0, + created_at: event.created, + content: Some(event.data.event.to_string()), + blob_hash: None, + extra: Extra::CalendarEvent { + calendar_ids: event.names.iter().map(|n| n.parent_id).collect(), + name: event.names.first().map(|n| n.name.clone()).unwrap_or_default(), + }, + }, + ) +} + +/// A contact deleted for good. +pub fn note_card( + batch: &mut BatchBuilder, + account_id: u32, + document_id: u32, + card: &ContactCard, +) -> trc::Result<()> { + note( + batch, + Kind::ContactCard, + account_id, + document_id, + Note { + deleted_at: 0, + created_at: card.created, + content: Some(card.card.to_string()), + blob_hash: None, + extra: Extra::ContactCard { + address_book_ids: card.names.iter().map(|n| n.parent_id).collect(), + name: card.names.first().map(|n| n.name.clone()).unwrap_or_default(), + }, + }, + ) +} diff --git a/crates/groupware/src/lib.rs b/crates/groupware/src/lib.rs index 70b26d3..2033549 100644 --- a/crates/groupware/src/lib.rs +++ b/crates/groupware/src/lib.rs @@ -16,6 +16,7 @@ pub mod cache; pub mod calendar; pub mod contact; pub mod file; +pub mod inbuxa; // inbuxa: undelete notes pub mod scheduling; #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/services/src/task_manager/inbuxa_restore.rs b/crates/services/src/task_manager/inbuxa_restore.rs new file mode 100644 index 0000000..da25547 --- /dev/null +++ b/crates/services/src/task_manager/inbuxa_restore.rs @@ -0,0 +1,272 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! Restoring archived files, calendar events, contacts and Sieve scripts +//! (`docs/spec/features/undelete.md`, UD-8 to UD-11). Email restores in +//! `restore_item.rs`. + +use crate::task_manager::TaskResult; +use calcard::{Entry, Parser, common::timezone::Tz}; +use common::{DavName, Server, auth::BuildAccessToken}; +use email::sieve::{SieveScript, ingest::SieveScriptIngest}; +use groupware::{ + cache::GroupwareCache, + calendar::{CalendarEvent, CalendarEventData}, + contact::ContactCard, + file::{FileNode, FileProperties}, +}; +use inbuxa_features::undelete::{self, data::Extra, groupware::candidates}; +use registry::schema::structs::{ArchivedItem, TaskRestoreArchivedItem}; +use store::write::BatchBuilder; +use trc::AddContext; +use types::{ + collection::{Collection, SyncCollection}, + id::Id, +}; + +/// The names already used under a parent in an account's resources. +fn names_under(resources: &common::DavResources, parent: Option) -> Vec { + resources + .paths + .iter() + .filter(|path| path.parent_id == parent) + .map(|path| { + path.path + .rsplit('/') + .next() + .unwrap_or(&path.path) + .to_string() + }) + .collect() +} + +fn free(name: &str, taken: &[String]) -> String { + let name = if name.is_empty() { "restored" } else { name }; + candidates(name) + .find(|candidate| !taken.iter().any(|t| t.eq_ignore_ascii_case(candidate))) + .unwrap() +} + +/// Restores a file, event, contact or script. `Ok(None)` once restored; +/// `Ok(Some(reason))` when it can't be, and stays archived. +pub(crate) async fn restore_other( + server: &Server, + task: &TaskRestoreArchivedItem, + item_id: Id, + item: &ArchivedItem, + extra: Option, +) -> trc::Result> { + let account_id = task.account_id.document_id(); + let access_token = server.access_token(account_id).await?.build(); + let account = server.account(account_id).await?; + let changed_by = access_token.account_tenant_ids(); + let hash = &task.blob_id.hash; + + let Some(bytes) = server + .blob_store() + .get_blob(hash.as_slice(), 0..usize::MAX) + .await? + else { + return Ok(Some("The kept copy is gone.".into())); + }; + + // UD-10: a restore counts against quota like anything new + if server + .has_available_quota(&account, bytes.len() as u64) + .await + .is_err() + { + return Ok(Some( + "Not restored: the account or its tenant is over quota.".into(), + )); + } + + let mut batch = BatchBuilder::new(); + match (item, extra) { + (ArchivedItem::FileNode(archived), extra) => { + let (parent_id, name, media_type) = match extra { + Some(Extra::FileNode { + parent_id, + name, + media_type, + .. + }) => (parent_id.unwrap_or(0), name, media_type), + _ => (0, archived.name.clone(), None), + }; + let resources = server + .fetch_dav_resources(account_id, account_id, SyncCollection::FileNode) + .await?; + // Back to its folder if that still exists, else the root + let parent_id = if parent_id > 0 + && resources.container_resource_by_id(parent_id - 1).is_some() + { + parent_id + } else { + 0 + }; + let parent = parent_id.checked_sub(1); + let name = free(&name, &names_under(&resources, parent)); + let document_id = server + .store() + .assign_document_ids(account_id, Collection::FileNode, 1) + .await?; + FileNode { + parent_id, + name, + file: Some(FileProperties { + blob_hash: hash.clone(), + size: bytes.len() as u32, + media_type, + executable: false, + }), + created: archived.created_at.timestamp(), + ..Default::default() + } + .insert(changed_by, account_id, document_id, false, true, &mut batch)?; + } + (ArchivedItem::CalendarEvent(_), extra) => { + let Entry::ICalendar(ical) = Parser::new(&String::from_utf8_lossy(&bytes)).entry() + else { + return Ok(Some("The kept event can't be read.".into())); + }; + let (calendar_ids, name) = match extra { + Some(Extra::CalendarEvent { calendar_ids, name }) => (calendar_ids, name), + _ => (vec![], String::new()), + }; + let resources = server + .fetch_dav_resources(account_id, account_id, SyncCollection::Calendar) + .await?; + // Back to its calendar if that still exists, else the default + let calendar_id = match calendar_ids + .into_iter() + .find(|id| resources.container_resource_by_id(*id).is_some()) + { + Some(id) => id, + None => match server + .get_or_create_default_calendar(account_id, account_id) + .await? + { + Some(id) => id, + None => return Ok(Some("The account has no calendar.".into())), + }, + }; + let name = free(&name, &names_under(&resources, Some(calendar_id))); + let mut next_alarm = None; + let event = CalendarEvent { + names: vec![DavName { + name, + parent_id: calendar_id, + }], + data: CalendarEventData::new( + ical, + Tz::Floating, + server.core.groupware.max_ical_instances, + &mut next_alarm, + ), + size: bytes.len() as u32, + ..Default::default() + }; + let document_id = server + .store() + .assign_document_ids(account_id, Collection::CalendarEvent, 1) + .await?; + event.insert(changed_by, account_id, document_id, next_alarm, &mut batch)?; + } + (ArchivedItem::ContactCard(_), extra) => { + let Entry::VCard(card) = Parser::new(&String::from_utf8_lossy(&bytes)).entry() else { + return Ok(Some("The kept contact can't be read.".into())); + }; + let (address_book_ids, name) = match extra { + Some(Extra::ContactCard { + address_book_ids, + name, + }) => (address_book_ids, name), + _ => (vec![], String::new()), + }; + let resources = server + .fetch_dav_resources(account_id, account_id, SyncCollection::AddressBook) + .await?; + // Back to its address book if that still exists, else the first + let book_id = match address_book_ids + .into_iter() + .find(|id| resources.container_resource_by_id(*id).is_some()) + { + Some(id) => id, + None => match resources.document_ids(true).next() { + Some(id) => id, + None => match server.create_default_addressbook(&account, &account).await? { + Some(id) => id, + None => return Ok(Some("The account has no address book.".into())), + }, + }, + }; + let name = free(&name, &names_under(&resources, Some(book_id))); + let document_id = server + .store() + .assign_document_ids(account_id, Collection::ContactCard, 1) + .await?; + ContactCard { + names: vec![DavName { + name, + parent_id: book_id, + }], + card, + size: bytes.len() as u32, + ..Default::default() + } + .insert(changed_by, account_id, document_id, &mut batch)?; + } + (ArchivedItem::SieveScript(archived), _) => { + // Back as an inactive script, never activated (UD-8) + let mut name = None; + for candidate in candidates(&archived.name).take(100) { + if server + .sieve_script_get_by_name(account_id, &candidate) + .await? + .is_none() + { + name = Some(candidate); + break; + } + } + let Some(name) = name else { + return Ok(Some("No free script name.".into())); + }; + let document_id = server + .store() + .assign_document_ids(account_id, Collection::SieveScript, 1) + .await?; + batch + .with_account_id(account_id) + .with_collection(Collection::SieveScript) + .with_document(document_id) + .custom( + common::storage::index::ObjectIndexBuilder::<(), _>::new() + .with_changes( + SieveScript::new(name, hash.clone()).with_size(bytes.len() as u32), + ) + .with_changed_by(changed_by), + ) + .caused_by(trc::location!())?; + } + (ArchivedItem::Email(_), _) => return Ok(Some("Not an email restore.".into())), + } + server.commit_batch(batch).await?; + + // UD-9: the record goes once the item is back + undelete::records::remove(&server.core.storage.data, server.registry(), item_id, item) + .await?; + Ok(None) +} + +/// The result of a restore that couldn't happen: the item stays archived, +/// and a new request can try again (UD-10). +pub(crate) async fn not_restored(server: &Server, item_id: Id, reason: String) -> trc::Result { + let mut batch = BatchBuilder::new(); + undelete::data::clear_restore_requested(&mut batch, item_id); + server.core.storage.data.write(batch.build_all()).await?; + Ok(TaskResult::permanent(reason)) +} diff --git a/crates/services/src/task_manager/index.rs b/crates/services/src/task_manager/index.rs index 420062b..492c822 100644 --- a/crates/services/src/task_manager/index.rs +++ b/crates/services/src/task_manager/index.rs @@ -10,6 +10,7 @@ use email::{ cache::MessageCacheFetch, message::metadata::{MESSAGE_RECEIVED_MASK, MessageMetadata}, }; +use inbuxa_features::undelete; use types::blob_hash::BlobHash; use groupware::{cache::GroupwareCache, calendar::CalendarEvent, contact::ContactCard}; use registry::{ @@ -210,6 +211,20 @@ impl SearchIndexTask for Server { IndexDocumentType::Contacts => 2, IndexDocumentType::File => 3, }; + // inbuxa: UD-1: a noted file, event or contact is archived + if let Some(kind) = match task.document_type { + IndexDocumentType::Calendar => Some(undelete::groupware::Kind::CalendarEvent), + IndexDocumentType::Contacts => Some(undelete::groupware::Kind::ContactCard), + IndexDocumentType::File => Some(undelete::groupware::Kind::File), + IndexDocumentType::Email => None, + } && let Err(err) = archive_noted(self, kind, account_id, document_id).await + { + trc::error!( + err.account_id(account_id) + .document_id(document_id) + .details("Failed to archive a deleted item") + ); + } document_deletions[idx] .entry(account_id) @@ -571,6 +586,40 @@ async fn build_tracing_span_document(_: &Server, _: u64) -> trc::Result trc::Result<()> { + let data = &server.core.storage.data; + let Some(note) = undelete::groupware::take(data, kind, account_id, document_id).await? else { + return Ok(()); + }; + let Some(retention) = undelete::settings::retention(server.registry()).await?.items else { + return Ok(()); + }; + let blob_hash = match (¬e.content, ¬e.blob_hash) { + (Some(text), _) => { + server + .put_temporary_blob(account_id, text.as_bytes(), 600) + .await? + .0 + } + (None, Some(hash)) => BlobHash::try_from_hash_slice(hash).map_err(|_| { + trc::StoreEvent::DataCorruption + .into_err() + .details("Invalid blob hash in undelete note") + })?, + (None, None) => return Ok(()), + }; + undelete::groupware::archive(data, server.registry(), account_id, note, blob_hash, retention) + .await + .map(|_| ()) +} + async fn delete_email_metadata( server: &Server, batch: &mut BatchBuilder, diff --git a/crates/services/src/task_manager/mod.rs b/crates/services/src/task_manager/mod.rs index b7d2a00..a950de4 100644 --- a/crates/services/src/task_manager/mod.rs +++ b/crates/services/src/task_manager/mod.rs @@ -21,6 +21,7 @@ pub mod destroy_account; pub mod dkim; pub mod dns; pub mod imip; +pub mod inbuxa_restore; // inbuxa: undelete pub mod index; pub mod lock; pub mod maintenance; diff --git a/crates/services/src/task_manager/restore_item.rs b/crates/services/src/task_manager/restore_item.rs index f10ac46..136a3f1 100644 --- a/crates/services/src/task_manager/restore_item.rs +++ b/crates/services/src/task_manager/restore_item.rs @@ -139,9 +139,32 @@ async fn restore_item(server: &Server, task: &TaskRestoreArchivedItem) -> trc::R Err(err) => Err(err.caused_by(trc::location!())), } } + // inbuxa: UD-1, UD-8: the other kinds the fork archives ArchivedItemType::FileNode | ArchivedItemType::CalendarEvent | ArchivedItemType::ContactCard - | ArchivedItemType::SieveScript => Ok(TaskResult::permanent("Not implemented")), + | ArchivedItemType::SieveScript => { + let Some((item_id, item, extra)) = undelete::records::for_restore( + &server.core.storage.data, + server.registry(), + task.account_id.document_id(), + task.blob_id.hash.as_slice(), + ) + .await? + else { + return Ok(TaskResult::Success(vec![])); + }; + match crate::task_manager::inbuxa_restore::restore_other( + server, task, item_id, &item, extra, + ) + .await? + { + None => Ok(TaskResult::Success(vec![])), + Some(reason) => { + crate::task_manager::inbuxa_restore::not_restored(server, item_id, reason) + .await + } + } + } } } diff --git a/tests/src/system/undelete.rs b/tests/src/system/undelete.rs index f46cdb3..fc6568b 100644 --- a/tests/src/system/undelete.rs +++ b/tests/src/system/undelete.rs @@ -9,6 +9,7 @@ use crate::utils::{ account::Account, + webdav::DummyWebDavClient, imap::{ImapConnection, Type}, jmap::JmapUtils, pop3::{self, Pop3Connection}, @@ -27,10 +28,13 @@ use registry::{ }, types::duration::Duration, }; +use hyper::StatusCode; use serde_json::{Value, json}; use types::id::Id; const SECRET: &str = "undelete test user passphrase"; +const EVENT: &str = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:test\r\nBEGIN:VEVENT\r\nUID:lunch-1\r\nDTSTAMP:20260918T100000Z\r\nDTSTART:20260918T120000Z\r\nDTEND:20260918T130000Z\r\nSUMMARY:Lunch\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n"; +const CARD: &str = "BEGIN:VCARD\r\nVERSION:4.0\r\nUID:jane-1\r\nFN:Jane Doe\r\nEND:VCARD\r\n"; const DAY: u64 = 86_400; pub async fn test(test: &mut TestServer) { @@ -267,6 +271,133 @@ pub async fn test(test: &mut TestServer) { "test 15" ); + // Acceptance test 6: a file, event, contact and script each come back to + // the right place (UD-1, UD-8) + let dav = DummyWebDavClient::new( + user.id().document_id(), + "undelete@example.org", + SECRET, + "undelete@example.org", + ); + let home = "undelete%40example.org"; + dav.mkcol("MKCOL", &format!("/dav/file/{home}/Folder/"), [], []) + .await + .with_status(StatusCode::CREATED); + let file = format!("/dav/file/{home}/Folder/notes.txt"); + let event = format!("/dav/cal/{home}/default/lunch.ics"); + let card = format!("/dav/card/{home}/default/jane.vcf"); + for (path, body) in [ + (&file, "Some notes.".to_string()), + (&event, EVENT.to_string()), + (&card, CARD.to_string()), + ] { + dav.request("PUT", path, body) + .await + .with_status(StatusCode::CREATED); + } + let script = client + .sieve_script_create("Filters", b"keep;".to_vec(), true) + .await + .unwrap() + .take_id(); + for path in [&file, &event, &card] { + dav.request("DELETE", path, "") + .await + .with_status(StatusCode::NO_CONTENT); + } + client.sieve_script_deactivate().await.unwrap(); + client.sieve_script_destroy(&script).await.unwrap(); + test.wait_for_tasks().await; + // A new file takes the old name, so the restored one is renamed + dav.request("PUT", &file, "Newer notes.") + .await + .with_status(StatusCode::CREATED); + for kind in ["FileNode", "CalendarEvent", "ContactCard", "SieveScript"] { + let item = user + .archived() + .await + .into_iter() + .find(|item| item["@type"] == kind) + .unwrap_or_else(|| panic!("test 6: no archived {kind}")); + user.request_restore(item.object_id()).await; + } + test.wait_for_tasks().await; + dav.request("GET", &format!("/dav/file/{home}/Folder/notes%20%28restored%29.txt"), "") + .await + .with_status(StatusCode::OK) + .with_body("Some notes."); + dav.request("GET", &event, "") + .await + .with_status(StatusCode::OK); + dav.request("GET", &card, "").await.with_status(StatusCode::OK); + let restored = client + .sieve_script_query( + jmap_client::sieve::query::Filter::name("Filters").into(), + None::>, + ) + .await + .unwrap(); + let restored = client + .sieve_script_get(restored.ids().first().expect("test 6: script"), None::>) + .await + .unwrap() + .unwrap(); + assert!(!restored.is_active(), "test 6: a restored script is inactive"); + assert!( + user.archived() + .await + .iter() + .all(|item| item["@type"] == "Email"), + "test 6: records removed" + ); + + // Acceptance test 8: a restore past the account's quota is refused, and + // the item stays archived (UD-10) + let used = test + .server + .get_used_quota_account(user.id().document_id()) + .await + .unwrap(); + admin + .registry_update_object( + ObjectType::Account, + user.id(), + json!({ Property::Quotas: {"maxDiskQuota": used} }), + ) + .await; + let item = user.archived_with_subject("Via POP3").await; + user.request_restore(item.object_id()).await; + test.wait_for_tasks_skip_failures().await; + assert_eq!(user.count_with_subject("Via POP3").await, 0, "test 8"); + assert_eq!( + user.archived_with_subject("Via POP3").await["status"], + "archived", + "test 8: still archived" + ); + // The refused restore stays in the queue as failed + admin.registry_destroy_all(ObjectType::Task).await; + admin + .registry_update_object( + ObjectType::Account, + user.id(), + json!({ Property::Quotas: {} }), + ) + .await; + + // Acceptance test 12: past its deadline an item isn't restorable + admin.set_retention(Some(1)).await; + let id = import(&client, "Short-lived", &[INBOX_ID], &[]).await; + client.email_destroy(&id).await.unwrap(); + test.wait_for_tasks().await; + tokio::time::sleep(std::time::Duration::from_millis(2100)).await; + assert!( + user.archived() + .await + .iter() + .all(|item| item["subject"] != "Short-lived"), + "test 12" + ); + // Clean up admin.set_retention(None).await; for item in user.archived().await {