From 6b65a50d0a31e3135d93998eb39ac89eb0ead2fa Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 20 Nov 2026 09:56:54 -0700 Subject: [PATCH] Count what's kept after deletion, and check what's really stored (#500) Part of the inbuxa/inbuxa-admin#257 remediation (Refs inbuxa/inbuxa-admin#257). Some stored data wasn't counted toward an account's or its tenant's disk quota, or was checked against less than was stored. - Undelete archive (UD-14 changed): an archived item now counts toward its account's and its tenant's usage while it's kept, and comes off when it goes: destroyed, expired, or restored (the restored item counts as itself from then on). Each counted item has a `q` + account + item key with its bytes, so it comes off exactly once (asserted) and items archived before this aren't taken off. Deleting never fails because of this; it just doesn't free space until the item goes. A restore is checked less what the kept copy already counts (UD-10), so restoring a counted item into a full account works; email ingest leaves a restore's check to the restore task. The per-account recount adds the counted archive. A deleted account's archive stays counted with the account and goes with it (records::discard leaves the counters). - Held spam-training samples (UD-14a): a sample's hold on a message's blob now ends with the account's own copy, when the message's data is removed with nothing archiving it or when its archived copy goes other than by restore. Holds are noted as `s` + account + hash + until, and ended notes are cleared by the scheduled clean-up. Counting held samples instead would count every trained message twice while it's still in the mailbox (the hold and the message share one blob). The classifier already drops a sample whose blob is gone from its replay pool, so the cost is a replay pool of mail that's still kept; what it learned stays. - Encrypted mail: ingest checks the stored size (the encrypted message plus the headers added on delivery) when it's more than was checked. - iTIP: a new invitation is checked against the event and the notification it stores (twice the message), an update against the event's growth plus the notification. - VacationResponse/set checks a bigger script against quota, like SieveScript/set, and answers overQuota. Left alone: parallel writes can each pass the check before any of them saves. A reservation would have to be released on every write path's success and failure, and a cheaper timed reservation would refuse mail for a moment after each write; refused SMTP deliveries become bounces. The overshoot is one burst of (concurrent writes) x (largest item), after which every check fails, and upstream behaves the same. Tests: undelete_tests (counted while kept, comes off on destroy, a counted item restores into a full account, a refused restore needs the account over its limit), vacation_response (a script past the quota is refused), samples key layout unit test. --- crates/email/src/message/ingest.rs | 28 +++- crates/features/src/undelete/accounts.rs | 3 +- crates/features/src/undelete/data.rs | 48 ++++++ crates/features/src/undelete/email.rs | 1 + crates/features/src/undelete/groupware.rs | 14 +- crates/features/src/undelete/mod.rs | 1 + crates/features/src/undelete/records.rs | 121 +++++++++++++- crates/features/src/undelete/samples.rs | 158 ++++++++++++++++++ crates/groupware/src/calendar/itip.rs | 10 +- crates/jmap/src/vacation/set.rs | 36 +++- .../src/task_manager/inbuxa_restore.rs | 11 +- crates/services/src/task_manager/index.rs | 15 +- .../services/src/task_manager/maintenance.rs | 8 + .../services/src/task_manager/restore_item.rs | 26 ++- docs/spec/features/undelete.md | 32 +++- tests/src/jmap/mail/vacation_response.rs | 39 +++++ tests/src/system/undelete.rs | 56 +++++-- 17 files changed, 562 insertions(+), 45 deletions(-) create mode 100644 crates/features/src/undelete/samples.rs diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index 81a738db..30a83d08 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -152,9 +152,14 @@ impl EmailIngest for Server { let tenant_id = params.access_token.tenant_id(); let mut raw_message_len = params.raw_message.len() as u64; let account = self.account(account_id).await.caused_by(trc::location!())?; - self.has_available_quota(&account, raw_message_len) - .await - .caused_by(trc::location!())?; + // inbuxa: UD-14: a restore checks for itself, crediting what its kept + // copy already counts + if params.source != IngestSource::Restore { + self.has_available_quota(&account, raw_message_len) + .await + .caused_by(trc::location!())?; + } + let checked_len = raw_message_len; // Parse message let mut raw_message = Cow::from(params.raw_message); @@ -562,6 +567,16 @@ impl EmailIngest for Server { false }; + // inbuxa: the size stored is the message after encryption, with the + // headers added on delivery; check that when it's more than was + // checked at the start + let stored_len = (message.raw_message.len() + extra_headers.len()) as u64; + if stored_len > checked_len && params.source != IngestSource::Restore { + self.has_available_quota(&account, stored_len) + .await + .caused_by(trc::location!())?; + } + // Store blob let (blob_hash, blob_hold) = if !is_encrypted && let Some(blob_hash) = params.blob_hash { (blob_hash.clone(), None) @@ -946,6 +961,13 @@ impl EmailIngest for Server { } .to_pickled_vec(); + // inbuxa: UD-14a: a held sample is noted, so its hold can end + // with the account's own copy of the message + if hold_sample { + let linked_to = batch.last_account_id().unwrap_or(account_id); + inbuxa_features::undelete::samples::note(batch, linked_to, &hash, until); + } + let object_id = ObjectType::SpamTrainingSample.to_id(); let item_id = SnowflakeIdGenerator::global_id().unwrap_or_default(); batch diff --git a/crates/features/src/undelete/accounts.rs b/crates/features/src/undelete/accounts.rs index 5ca278a9..ea852111 100644 --- a/crates/features/src/undelete/accounts.rs +++ b/crates/features/src/undelete/accounts.rs @@ -139,7 +139,8 @@ pub async fn forget(data: &Store, registry: &RegistryStore, account_id: u32) -> .caused_by(trc::location!())?; } for (id, item) in records::of_account(data, registry, account_id).await? { - records::remove(data, registry, id, &item).await?; + // Its usage goes with the account, so the counters stay as they are + records::discard(data, registry, id, &item).await?; } data::clear_account(data, account_id).await } diff --git a/crates/features/src/undelete/data.rs b/crates/features/src/undelete/data.rs index 26b608f2..49a9811b 100644 --- a/crates/features/src/undelete/data.rs +++ b/crates/features/src/undelete/data.rs @@ -22,6 +22,11 @@ //! - `a` + address: an address a kept account holds reserved (UD-16). //! - `g` + kind + account + document: a deleted calendar event, contact or //! file ([`super::groupware`]). +//! - `s` + account + blob hash + until: a spam-training sample's hold on a +//! message's blob ([`super::samples`], UD-14a). +//! - `q` + account + item id: the bytes an archived item counts toward its +//! account's and tenant's usage while it's kept (UD-14). Items archived +//! before counting began have none, and nothing is taken off for them. //! //! Values are JSON, so they read back across versions of the fork. @@ -40,6 +45,7 @@ const KIND_RESTORE: u8 = b'r'; const KIND_CHANGE: u8 = b'c'; const KIND_KEPT: u8 = b'k'; const KIND_RESERVED: u8 = b'a'; +const KIND_COUNTED: u8 = b'q'; fn class(kind: u8, rest: &[u8]) -> ValueClass { let mut key = Vec::with_capacity(1 + rest.len()); @@ -249,6 +255,46 @@ pub fn clear_blob_item(batch: &mut BatchBuilder, account_id: u32, blob_hash: &[u batch.clear(class(KIND_BLOB, &account_blob(account_id, blob_hash))); } +fn account_item(account_id: u32, item_id: Id) -> [u8; 12] { + let mut out = [0u8; 12]; + out[..4].copy_from_slice(&account_id.to_be_bytes()); + out[4..].copy_from_slice(&item_id.id().to_be_bytes()); + out +} + +/// UD-14: an archived item's counted bytes. +pub fn counted_class(account_id: u32, item_id: Id) -> ValueClass { + class(KIND_COUNTED, &account_item(account_id, item_id)) +} + +pub async fn counted(data: &Store, account_id: u32, item_id: Id) -> trc::Result> { + data.get_value::(key(KIND_COUNTED, &account_item(account_id, item_id))) + .await + .caused_by(trc::location!()) +} + +/// UD-14: the bytes all an account's archived items count, for recounting +/// its usage from scratch. +pub async fn counted_total(data: &Store, account_id: u32) -> trc::Result { + let mut total = 0u64; + data.iterate( + IterateParams::new( + key(KIND_COUNTED, &account_item(account_id, Id::new(0))), + key(KIND_COUNTED, &account_item(account_id, Id::new(u64::MAX))), + ) + .ascending(), + |_, value| { + if let Ok(bytes) = <[u8; 8]>::try_from(value) { + total = total.saturating_add(u64::from_be_bytes(bytes)); + } + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + Ok(total) +} + pub fn set_restore_requested(batch: &mut BatchBuilder, item_id: Id) { batch.set(class(KIND_RESTORE, &item_id.id().to_be_bytes()), vec![1]); } @@ -416,6 +462,8 @@ pub async fn clear_account(data: &Store, account_id: u32) -> trc::Result<()> { (vec![KIND_NOTE], vec![KIND_NOTE]), (vec![KIND_BLOB], vec![KIND_BLOB]), (vec![KIND_CHANGE], vec![KIND_CHANGE]), + (vec![KIND_COUNTED], vec![KIND_COUNTED]), + (vec![b's'], vec![b's']), ]; // The groupware notes, `g` + kind + account + document for kind in 0u8..3 { diff --git a/crates/features/src/undelete/email.rs b/crates/features/src/undelete/email.rs index db4cbff0..07619a63 100644 --- a/crates/features/src/undelete/email.rs +++ b/crates/features/src/undelete/email.rs @@ -133,6 +133,7 @@ pub async fn archive( mailboxes: note.mailboxes, keywords: note.keywords, }, + note.size, ) .await?; diff --git a/crates/features/src/undelete/groupware.rs b/crates/features/src/undelete/groupware.rs index 6a8ead02..45bbbf2a 100644 --- a/crates/features/src/undelete/groupware.rs +++ b/crates/features/src/undelete/groupware.rs @@ -238,11 +238,21 @@ pub async fn archive( .details("Not a groupware item")); } }; - records::insert(data, registry, &item, ¬e.extra).await + let size = kept_size(¬e); + records::insert(data, registry, &item, ¬e.extra, size).await } /// The names a restored item tries, in order: the original, then with /// ` (restored)`, then numbered (UD-8). +/// The bytes a kept file, event or contact counts while it's archived +/// (UD-14): a file's size, or the event's or contact's text. +fn kept_size(note: &Note) -> u64 { + match ¬e.extra { + Extra::FileNode { size, .. } if *size > 0 => *size as u64, + _ => note.content.as_ref().map_or(0, |text| text.len() as u64), + } +} + pub fn candidates(name: &str) -> impl Iterator + '_ { let (stem, ext) = match name.rsplit_once('.') { Some((stem, ext)) if !stem.is_empty() => (stem, format!(".{ext}")), @@ -301,6 +311,7 @@ pub async fn archive_sieve( retention: u64, ) -> trc::Result { let archived_at = now(); + let size = content.len() as u64; let item = ArchivedItem::SieveScript(registry::schema::structs::ArchivedSieveScript { name: name.to_string(), created_at: UTCDateTime::from_timestamp(archived_at as i64), @@ -317,6 +328,7 @@ pub async fn archive_sieve( &Extra::SieveScript { name: name.to_string(), }, + size, ) .await } diff --git a/crates/features/src/undelete/mod.rs b/crates/features/src/undelete/mod.rs index 0a51b4e3..35461e95 100644 --- a/crates/features/src/undelete/mod.rs +++ b/crates/features/src/undelete/mod.rs @@ -18,4 +18,5 @@ pub mod data; pub mod email; pub mod groupware; pub mod records; +pub mod samples; pub mod settings; diff --git a/crates/features/src/undelete/records.rs b/crates/features/src/undelete/records.rs index 798f8773..c241ccc4 100644 --- a/crates/features/src/undelete/records.rs +++ b/crates/features/src/undelete/records.rs @@ -8,17 +8,23 @@ //! its account index, with no link to the account, so deleting an account //! isn't refused while it has archived items. Upstream's account removal //! task clears exactly these two keys. +//! +//! UD-14: an item counts toward its account's and its tenant's storage while +//! it's kept, so deleting doesn't free space until the item goes. Without +//! this, a member could fill the account, delete, and fill it again, keeping +//! the limit's worth of data once more every time for the whole retention +//! period. use crate::undelete::data::{self, Change}; use registry::schema::prelude::Property; use registry::{ schema::{prelude::ObjectType, structs::ArchivedItem}, - types::{EnumImpl, ObjectImpl, index::IndexValue}, + types::{EnumImpl, ObjectImpl, id::ObjectId, index::IndexValue}, }; use store::{ RegistryStore, SerializeInfallible, Store, registry::RegistryQuery, - write::{BatchBuilder, BlobLink, BlobOp, RegistryClass, ValueClass, now}, + write::{BatchBuilder, BlobLink, BlobOp, RegistryClass, ValueClass, assert::AssertValue, now}, }; use trc::AddContext; use types::id::Id; @@ -39,13 +45,45 @@ fn item_class(item_id: u64) -> ValueClass { }) } +/// The tenant an account's usage counts toward: its own, or for a deleted +/// account still kept, the one it had (UD-15). +pub async fn tenant_of( + data: &Store, + registry: &RegistryStore, + account_id: u32, +) -> trc::Result> { + if let Some(account) = registry + .get(ObjectId::new(ObjectType::Account, Id::from(account_id))) + .await + .caused_by(trc::location!())? + { + return Ok(account.inner.member_tenant_id().map(|id| id.document_id())); + } + Ok(data::kept_account(data, account_id) + .await? + .and_then(|kept| kept.member_tenant_id) + .map(|id| Id::from(id).document_id())) +} + +/// Adds `bytes` to an account's usage, and to its tenant's (MT-20). +fn count(batch: &mut BatchBuilder, account_id: u32, tenant_id: Option, bytes: i64) { + batch + .with_account_id(account_id) + .add(ValueClass::Quota, bytes); + if let Some(tenant_id) = tenant_id { + batch.add(ValueClass::TenantQuota(tenant_id), bytes); + } +} + /// Writes a new archived item, holding its kept copy until `archivedUntil`, -/// with what restore needs beside it. Returns its id. +/// with what restore needs beside it, and counting its `size` toward the +/// account and its tenant (UD-14). Returns its id. pub async fn insert( data: &Store, registry: &RegistryStore, item: &ArchivedItem, extra: &data::Extra, + size: u64, ) -> trc::Result { let item_id = registry.assign_id(); let id = Id::new(item_id); @@ -65,6 +103,11 @@ pub async fn insert( ); data::set_extra(&mut batch, id, extra)?; data::set_blob_item(&mut batch, account_id, blob_hash.as_slice(), id); + if size > 0 { + let tenant_id = tenant_of(data, registry, account_id).await?; + count(&mut batch, account_id, tenant_id, size as i64); + batch.set(data::counted_class(account_id, id), size.serialize()); + } data::log_change( &mut batch, account_id, @@ -143,13 +186,57 @@ pub async fn set_deadline( Ok(updated) } -/// Removes an archived item and releases its kept copy: on restore (UD-9), -/// on destroy (UD-12) and past its deadline (UD-13). +/// Removes an archived item for good and releases its kept copy: on +/// destroy (UD-12) and past its deadline (UD-13). Its bytes come off the +/// account's and tenant's usage (UD-14), and a message's spam-training +/// holds end with it (UD-14a). pub async fn remove( data: &Store, registry: &RegistryStore, id: Id, item: &ArchivedItem, +) -> trc::Result<()> { + remove_(data, registry, id, item, true).await?; + if let ArchivedItem::Email(email) = item { + crate::undelete::samples::release( + data, + email.account_id.document_id(), + &email.blob_id.hash, + ) + .await?; + } + Ok(()) +} + +/// Removes an archived item that's been restored (UD-9): its bytes come +/// off, since the restored item counts as itself from now on. +pub async fn remove_restored( + data: &Store, + registry: &RegistryStore, + id: Id, + item: &ArchivedItem, +) -> trc::Result<()> { + remove_(data, registry, id, item, true).await +} + +/// Removes an archived item of an account being destroyed, leaving the +/// usage counters alone: the account's goes with it, and its tenant's is +/// settled from it. +pub async fn discard( + data: &Store, + registry: &RegistryStore, + id: Id, + item: &ArchivedItem, +) -> trc::Result<()> { + remove_(data, registry, id, item, false).await +} + +async fn remove_( + data: &Store, + registry: &RegistryStore, + id: Id, + item: &ArchivedItem, + uncount: bool, ) -> trc::Result<()> { let account_id = item.account_id().document_id(); let blob_hash = item.blob_id().hash.clone(); @@ -164,6 +251,27 @@ pub async fn remove( .await .caused_by(trc::location!())?; + // Its counted bytes come off once: the marker must still be there, or + // another removal already took them off + let counted = if uncount { + data::counted(data, account_id, id).await? + } else { + None + }; + if let Some(size) = counted { + let tenant_id = tenant_of(data, registry, account_id).await?; + let mut batch = BatchBuilder::new(); + batch + .assert_value(data::counted_class(account_id, id), AssertValue::Some) + .clear(data::counted_class(account_id, id)); + count(&mut batch, account_id, tenant_id, -(size as i64)); + match data.write(batch.build_all()).await { + Ok(_) => {} + Err(err) if err.is_assertion_failure() => {} + Err(err) => return Err(err.caused_by(trc::location!())), + } + } + let mut batch = BatchBuilder::new(); batch.with_account_id(account_id).clear(BlobOp::Link { hash: blob_hash.clone(), @@ -171,6 +279,9 @@ pub async fn remove( until: item.archived_until().timestamp() as u64, }, }); + if !uncount { + batch.clear(data::counted_class(account_id, id)); + } data::clear_extra(&mut batch, id); data::clear_blob_item(&mut batch, account_id, blob_hash.as_slice()); data::clear_restore_requested(&mut batch, id); diff --git a/crates/features/src/undelete/samples.rs b/crates/features/src/undelete/samples.rs new file mode 100644 index 00000000..307fd8cd --- /dev/null +++ b/crates/features/src/undelete/samples.rs @@ -0,0 +1,158 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! Spam-training samples held past their message (UD-14a). +//! +//! The classifier holds a trained message's blob for `holdSamplesFor` +//! (90 days by default) so it can replay it later. While the message is in +//! the account that costs nothing: the hold and the message share one blob, +//! counted once. Once the account's own copy is gone for good, though, the +//! held blob was stored with nobody counting it, so a member could fill the +//! account, train, delete and fill it again, past the account's and the +//! tenant's limit, for the whole hold period. +//! +//! So a hold now ends with the account's own copy: when the message's data +//! is removed and nothing archives it, or when its archived copy goes other +//! than by restore. The classifier already treats a blob that's gone as a +//! sample to drop from its replay pool, so this shrinks the pool to mail +//! that's still kept; what was learned from the sample stays learned. +//! +//! Each hold is noted as `s` + account + blob hash + until (empty value), so +//! it can be found from the message, with the link it set under that +//! account. + +use store::{ + IterateParams, SUBSPACE_UNDELETE, Store, ValueKey, + write::{AnyClass, BatchBuilder, BlobLink, BlobOp, ValueClass, now}, +}; +use trc::AddContext; +use types::blob_hash::{BLOB_HASH_LEN, BlobHash}; + +const KIND_SAMPLE: u8 = b's'; + +fn class(account_id: u32, hash: &[u8], until: u64) -> ValueClass { + let mut key = Vec::with_capacity(1 + 4 + hash.len() + 8); + key.push(KIND_SAMPLE); + key.extend_from_slice(&account_id.to_be_bytes()); + key.extend_from_slice(hash); + key.extend_from_slice(&until.to_be_bytes()); + ValueClass::Any(AnyClass { + subspace: SUBSPACE_UNDELETE, + key, + }) +} + +/// Notes a held sample: its blob, linked under `account_id` until `until`. +pub fn note(batch: &mut BatchBuilder, account_id: u32, hash: &BlobHash, until: u64) { + batch.set(class(account_id, hash.as_slice(), until), vec![]); +} + +/// The holds noted for a blob in an account, by their `until`. +async fn holds(data: &Store, account_id: u32, hash: &[u8]) -> trc::Result> { + let mut untils = Vec::new(); + data.iterate( + IterateParams::new( + ValueKey::from(class(account_id, hash, 0)), + ValueKey::from(class(account_id, hash, u64::MAX)), + ) + .ascending() + .no_values(), + |key, _| { + if key.len() >= 8 { + untils.push(u64::from_be_bytes(key[key.len() - 8..].try_into().unwrap())); + } + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + Ok(untils) +} + +/// The account's own copy of a message is gone for good: its samples' +/// holds on the blob end with it. +pub async fn release(data: &Store, account_id: u32, hash: &BlobHash) -> trc::Result<()> { + let untils = holds(data, account_id, hash.as_slice()).await?; + if untils.is_empty() { + return Ok(()); + } + let mut batch = BatchBuilder::new(); + batch.with_account_id(account_id); + for until in untils { + batch + .clear(BlobOp::Link { + hash: hash.clone(), + to: BlobLink::Temporary { until }, + }) + .clear(class(account_id, hash.as_slice(), until)); + } + data.write(batch.build_all()) + .await + .caused_by(trc::location!()) + .map(|_| ()) +} + +/// Clears notes whose hold has ended on its own, for the scheduled clean-up. +/// Returns how many went. +pub async fn purge_ended(data: &Store) -> trc::Result { + let now = now(); + let mut ended = Vec::new(); + data.iterate( + IterateParams::new( + ValueKey::from(ValueClass::Any(AnyClass { + subspace: SUBSPACE_UNDELETE, + key: vec![KIND_SAMPLE], + })), + ValueKey::from(ValueClass::Any(AnyClass { + subspace: SUBSPACE_UNDELETE, + key: vec![KIND_SAMPLE, u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX], + })), + ) + .ascending() + .no_values(), + |key, _| { + if key.len() == 1 + 4 + BLOB_HASH_LEN + 8 + && u64::from_be_bytes(key[key.len() - 8..].try_into().unwrap()) <= now + { + ended.push(key.to_vec()); + } + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + let count = ended.len(); + for chunk in ended.chunks(512) { + let mut batch = BatchBuilder::new(); + for key in chunk { + batch.clear(ValueClass::Any(AnyClass { + subspace: SUBSPACE_UNDELETE, + key: key.clone(), + })); + } + data.write(batch.build_all()) + .await + .caused_by(trc::location!())?; + } + Ok(count) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keys_sort_by_account_then_blob_then_until() { + let hash = BlobHash::generate(b"x"); + let key = |account, until| match class(account, hash.as_slice(), until) { + ValueClass::Any(any) => any.key, + _ => unreachable!(), + }; + assert_eq!(key(1, 5).len(), 1 + 4 + BLOB_HASH_LEN + 8); + assert!(key(1, 5) < key(1, 6)); + assert!(key(1, u64::MAX) < key(2, 0)); + } +} diff --git a/crates/groupware/src/calendar/itip.rs b/crates/groupware/src/calendar/itip.rs index 55c74e6b..8792cec9 100644 --- a/crates/groupware/src/calendar/itip.rs +++ b/crates/groupware/src/calendar/itip.rs @@ -263,11 +263,12 @@ impl ItipIngest for Server { let mut ical = itip.clone(); itip_import_message(&mut ical)?; - // Validate quota + // Validate quota: inbuxa: both the event and the notification + // beside it are stored, each the message's size if self .has_available_quota( self.account(account_id).await?.as_ref(), - itip_message.len() as u64, + 2 * itip_message.len() as u64, ) .await .is_err() @@ -782,7 +783,10 @@ async fn commit_itip_merge( } // Validate quota - let extra_bytes = (event.size as u64).saturating_sub(event_.inner.size.to_native() as u64); + // inbuxa: the notification stored beside the update counts too + let extra_bytes = (event.size as u64) + .saturating_sub(event_.inner.size.to_native() as u64) + .saturating_add(itip_size as u64); if extra_bytes > 0 && server .has_available_quota(server.account(account_id).await?.as_ref(), extra_bytes) diff --git a/crates/jmap/src/vacation/set.rs b/crates/jmap/src/vacation/set.rs index 44524906..f9eb3ad8 100644 --- a/crates/jmap/src/vacation/set.rs +++ b/crates/jmap/src/vacation/set.rs @@ -2,6 +2,8 @@ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ use super::get::VacationResponseGet; @@ -244,6 +246,7 @@ impl VacationResponseSet for Server { } } + let prev_size = prev_sieve.as_ref().map_or(0, |prev| prev.inner.size as u64); let mut obj = ObjectIndexBuilder::new() .with_current_opt(prev_sieve) .with_changes(sieve) @@ -265,14 +268,33 @@ impl VacationResponseSet for Server { // Create sieve script only if there are changes if build_script { + let script = self.build_script(obj.changes_mut().unwrap())?; + + // inbuxa: a bigger script counts against quota, as a Sieve + // script's does + let new_size = obj.changes_mut().unwrap().size as u64; + if new_size > prev_size { + let account = self.account(account_id).await?; + match self + .has_available_quota(&account, new_size - prev_size) + .await + { + Ok(()) => {} + Err(err) + if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) + || err.matches(trc::EventType::Limit( + trc::LimitEvent::TenantQuota, + )) => + { + return Ok(set_error(response, create_id, SetError::over_quota())); + } + Err(err) => return Err(err), + } + } + // Upload new blob - let (blob_hash, blob_hold) = self - .put_temporary_blob( - account_id, - &self.build_script(obj.changes_mut().unwrap())?, - 60, - ) - .await?; + let (blob_hash, blob_hold) = + self.put_temporary_blob(account_id, &script, 60).await?; obj.changes_mut().unwrap().blob_hash = blob_hash; batch.clear(blob_hold); }; diff --git a/crates/services/src/task_manager/inbuxa_restore.rs b/crates/services/src/task_manager/inbuxa_restore.rs index 914a22fb..9cb96706 100644 --- a/crates/services/src/task_manager/inbuxa_restore.rs +++ b/crates/services/src/task_manager/inbuxa_restore.rs @@ -73,9 +73,13 @@ pub(crate) async fn restore_other( return Ok(Some("The kept copy is gone.".into())); }; - // UD-10: a restore counts against quota like anything new + // UD-10, UD-14: a restore counts against quota like anything new, less + // what the kept copy already counts + let counted = undelete::data::counted(&server.core.storage.data, account_id, item_id) + .await? + .unwrap_or(0); if server - .has_available_quota(&account, bytes.len() as u64) + .has_available_quota(&account, (bytes.len() as u64).saturating_sub(counted)) .await .is_err() { @@ -259,7 +263,8 @@ pub(crate) async fn restore_other( server.commit_batch(batch).await?; // UD-9: the record goes once the item is back - undelete::records::remove(&server.core.storage.data, server.registry(), item_id, item).await?; + undelete::records::remove_restored(&server.core.storage.data, server.registry(), item_id, item) + .await?; Ok(None) } diff --git a/crates/services/src/task_manager/index.rs b/crates/services/src/task_manager/index.rs index 5fba7f7f..91b2c786 100644 --- a/crates/services/src/task_manager/index.rs +++ b/crates/services/src/task_manager/index.rs @@ -790,13 +790,14 @@ async fn delete_email_metadata( // inbuxa: UD-1, UD-4: a message noted at deletion is archived let root = metadata.contents.first().and_then(|c| c.parts.first()); - inbuxa_features::undelete::email::archive( + let blob_hash = BlobHash::from(&metadata.blob_hash); + let archived = inbuxa_features::undelete::email::archive( &server.core.storage.data, server.registry(), account_id, document_id, inbuxa_features::undelete::email::Summary { - blob_hash: BlobHash::from(&metadata.blob_hash), + blob_hash: blob_hash.clone(), from: root.and_then(|part| part.from()), subject: root.and_then(|part| part.subject()), received_at: metadata.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK, @@ -804,6 +805,16 @@ async fn delete_email_metadata( ) .await .caused_by(trc::location!())?; + // inbuxa: UD-14a: with no archived copy, its spam-sample holds end + if !archived { + inbuxa_features::undelete::samples::release( + &server.core.storage.data, + account_id, + &blob_hash, + ) + .await + .caused_by(trc::location!())?; + } } None => { trc::event!( diff --git a/crates/services/src/task_manager/maintenance.rs b/crates/services/src/task_manager/maintenance.rs index 32923310..5e6cfd98 100644 --- a/crates/services/src/task_manager/maintenance.rs +++ b/crates/services/src/task_manager/maintenance.rs @@ -236,6 +236,10 @@ async fn store_maintenance( ) .await .caused_by(trc::location!())?; + // inbuxa: UD-14a: and notes of spam-sample holds that have ended + inbuxa_features::undelete::samples::purge_ended(&server.core.storage.data) + .await + .caused_by(trc::location!())?; use common::telemetry::metrics::store::MetricsStore; // inbuxa: MON-17, MON-38: history past its retention goes; a @@ -551,6 +555,10 @@ async fn recalculate_quota(server: &Server, account_id: u32) -> trc::Result<()> .caused_by(trc::location!())?; } + // inbuxa: UD-14: kept deleted items count too + quota += inbuxa_features::undelete::data::counted_total(&server.core.storage.data, account_id) + .await? as i64; + let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) diff --git a/crates/services/src/task_manager/restore_item.rs b/crates/services/src/task_manager/restore_item.rs index 5cf50197..6e624aa0 100644 --- a/crates/services/src/task_manager/restore_item.rs +++ b/crates/services/src/task_manager/restore_item.rs @@ -95,6 +95,29 @@ async fn restore_item(server: &Server, task: &TaskRestoreArchivedItem) -> trc::R return Ok(TaskResult::permanent("Blob not found")); }; + // inbuxa: UD-10, UD-14: a restore counts against quota like + // anything new, less what the kept copy already counts. Ingest + // leaves a restore's check to this. + let counted = undelete::data::counted(data, account_id, item_id) + .await? + .unwrap_or(0); + let account = server + .account(account_id) + .await + .caused_by(trc::location!())?; + if server + .has_available_quota(&account, (bytes.len() as u64).saturating_sub(counted)) + .await + .is_err() + { + let mut batch = BatchBuilder::new(); + undelete::data::clear_restore_requested(&mut batch, item_id); + data.write(batch.build_all()).await?; + return Ok(TaskResult::permanent( + "Not restored: the account or its tenant is over quota.".to_string(), + )); + } + match server .email_ingest(IngestEmail { raw_message: &bytes, @@ -111,7 +134,8 @@ async fn restore_item(server: &Server, task: &TaskRestoreArchivedItem) -> trc::R { Ok(_) => { // inbuxa: UD-9: the archived record goes, and its copy is released - undelete::records::remove(data, server.registry(), item_id, &item).await?; + undelete::records::remove_restored(data, server.registry(), item_id, &item) + .await?; Ok(TaskResult::Success(vec![])) } // inbuxa: UD-10: over quota, the item stays archived and the task says why diff --git a/docs/spec/features/undelete.md b/docs/spec/features/undelete.md index 3f55ad27..dcf85a9f 100644 --- a/docs/spec/features/undelete.md +++ b/docs/spec/features/undelete.md @@ -146,8 +146,9 @@ Task type `RestoreArchivedItem`, carrying the `accountId`, the (observed 5). The restored item is a new item with a new id, and JMAP `/changes` reports it as created. - **UD-10.** A restore still counts against quota. It's refused if it would - take the account, or its tenant, past a limit. The item stays archived, and - the task records why. + take the account, or its tenant, past a limit, less what the archived copy + already counts (UD-14), so restoring a counted item is never refused just + for being counted. The item stays archived, and the task records why. - **UD-11.** Restoring is idempotent: asking twice, or asking while a restore task is already running, restores once. @@ -163,10 +164,25 @@ Task type `RestoreArchivedItem`, carrying the `accountId`, the ### Storage -- **UD-14.** Archived copies don't count toward the account's or tenant's - quota while archived, as upstream (observed 7). Counting them would make deleting mail to - free space fail. They do count in server storage reporting, and ihasmail's - admin dashboard shows the archive's total size per account. +- **UD-14.** Archived copies count toward the account's and its tenant's + quota while they're kept, and come off when they go: restored (the restored + item counts as itself from then on), destroyed or expired. This changed in + 27.1 (inbuxa-admin#257). Upstream doesn't count them (observed 7), and + neither did we at first, so that deleting would free space. But a member + could then fill the account, delete, and fill it again, storing the limit's + worth once more every cycle for the whole retention period, past both the + account's quota and the tenant's. Deleting never fails because of this: it + just doesn't free space until the item goes, and an account that's full + stops taking new mail. A user who needs the space back destroys the + archived items (UD-12). Items archived before 27.1 weren't counted, so + nothing is taken off when they go. A deleted account's archive stays + counted with the account (UD-15), and goes with it. +- **UD-14a.** A spam-training sample's hold on a message's blob ends with + the account's own copy: when the message's data is removed with nothing + archiving it, or when its archived copy goes other than by restore. Held + longer, the blob was stored with nobody counting it, for + `holdSamplesFor` (90 days by default). The classifier drops a sample whose + blob is gone from its replay pool; what it learned from the sample stays. ### Deleted accounts @@ -257,7 +273,9 @@ Stalwart-facing (SPEC.md ยง5). 11. Retention lowered from 30 to 7 days: existing deadlines don't move (UD-5). 12. Past `archivedUntil`: not restorable, and gone after clean-up (UD-13). -13. Archive size doesn't count toward quota (UD-14). +13. Archive size counts toward the account's and the tenant's quota while + kept, comes off when it goes, and a counted item restores in a full + account (UD-14, UD-10). 14. Deleted account: can't sign in, mail refused, name reserved, restorable by an admin with its data intact (UD-15 to UD-17). 15. A user can't see another user's archive. A tenant admin sees only its diff --git a/tests/src/jmap/mail/vacation_response.rs b/tests/src/jmap/mail/vacation_response.rs index d200f998..783b08ae 100644 --- a/tests/src/jmap/mail/vacation_response.rs +++ b/tests/src/jmap/mail/vacation_response.rs @@ -2,6 +2,8 @@ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ use crate::{ @@ -12,6 +14,8 @@ use crate::{ }; use chrono::{TimeDelta, Utc}; use jmap_client::core::response::VacationResponseSetResponse; +use registry::schema::prelude::{ObjectType, Property}; +use serde_json::json; use std::time::Instant; pub async fn test(test: &TestServer) { @@ -190,6 +194,41 @@ pub async fn test(test: &TestServer) { ) .await; + // inbuxa: a bigger vacation script counts against quota, as a Sieve + // script's does + let admin = test.account("admin@example.com"); + let used = server + .get_used_quota_account(account.id().document_id()) + .await + .unwrap(); + admin + .registry_update_object( + ObjectType::Account, + account.id(), + json!({ Property::Quotas: {"maxDiskQuota": used.max(1)} }), + ) + .await; + let mut request = client.build(); + request + .set_vacation_response() + .update("singleton") + .text_body(Some("x".repeat(4096))); + let mut response = request + .send_single::() + .await + .unwrap(); + assert!( + response.updated("singleton").is_err(), + "a vacation script past the quota is refused" + ); + admin + .registry_update_object( + ObjectType::Account, + account.id(), + json!({ Property::Quotas: {} }), + ) + .await; + // Remove test data client.vacation_response_disable().await.unwrap(); client.sieve_script_deactivate().await.unwrap(); diff --git a/tests/src/system/undelete.rs b/tests/src/system/undelete.rs index e9985a07..3bb6fd85 100644 --- a/tests/src/system/undelete.rs +++ b/tests/src/system/undelete.rs @@ -145,7 +145,8 @@ pub async fn test(test: &mut TestServer) { test.wait_for_tasks().await; assert_eq!(user.archived().await.len(), 4, "test 3"); - // Acceptance test 13: archived copies don't count toward quota + // Acceptance test 13: archived copies count toward quota while kept + // (UD-14) let used = test .server .get_used_quota_account(user.id().document_id()) @@ -164,17 +165,35 @@ pub async fn test(test: &mut TestServer) { .await; client.email_destroy(&id).await.unwrap(); test.wait_for_tasks().await; - assert_eq!( - test.server - .get_used_quota_account(user.id().document_id()) - .await - .unwrap(), - used, - "test 13" - ); let item = user.archived_with_subject("Two labels").await; + let size = item["size"].as_i64().expect("test 13: size"); + assert_eq!( + used_now(&test.server, user.id().document_id()).await, + used + size, + "test 13: counted while kept" + ); + // ... and a counted item restores even into a full account (UD-10) + admin + .registry_update_object( + ObjectType::Account, + user.id(), + json!({ Property::Quotas: {"maxDiskQuota": used + size} }), + ) + .await; user.request_restore(item.object_id()).await; test.wait_for_tasks().await; + assert_eq!( + used_now(&test.server, user.id().document_id()).await, + used + size, + "test 13: the restored message counts instead, once" + ); + admin + .registry_update_object( + ObjectType::Account, + user.id(), + json!({ Property::Quotas: {} }), + ) + .await; let restored = user.email_with_subject("Two labels").await; assert_eq!( restored["mailboxIds"], @@ -226,11 +245,18 @@ pub async fn test(test: &mut TestServer) { test.wait_for_tasks().await; assert_eq!(user.count_with_subject("Via JMAP").await, 1, "test 9"); - // Acceptance test 10: the user destroys an archived item for good (UD-12) + // Acceptance test 10: the user destroys an archived item for good (UD-12), + // and its space comes back (UD-14) let item = user.archived_with_subject("Via IMAP").await; + let before = used_now(&test.server, user.id().document_id()).await; user.registry_destroy(ObjectType::ArchivedItem, [item.object_id()]) .await .assert_destroyed(&[item.object_id()]); + assert_eq!( + used_now(&test.server, user.id().document_id()).await, + before - item["size"].as_i64().expect("test 10: size"), + "test 10: its bytes come off" + ); assert!( user.archived() .await @@ -425,7 +451,8 @@ pub async fn test(test: &mut TestServer) { ); // Acceptance test 8: a restore past the account's quota is refused, and - // the item stays archived (UD-10) + // the item stays archived (UD-10). The item's own bytes already count, so + // the account has to be over its limit for that let used = test .server .get_used_quota_account(user.id().document_id()) @@ -435,7 +462,7 @@ pub async fn test(test: &mut TestServer) { .registry_update_object( ObjectType::Account, user.id(), - json!({ Property::Quotas: {"maxDiskQuota": used} }), + json!({ Property::Quotas: {"maxDiskQuota": used - 1} }), ) .await; let item = user.archived_with_subject("Via POP3").await; @@ -1066,3 +1093,8 @@ impl Account { .await; } } + +// UD-14: what an account's usage counter says now +async fn used_now(server: &common::Server, account_id: u32) -> i64 { + server.get_used_quota_account(account_id).await.unwrap() +} -- 2.54.0