diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index 81a738db..30a83d08 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -152,9 +152,14 @@ impl EmailIngest for Server { let tenant_id = params.access_token.tenant_id(); let mut raw_message_len = params.raw_message.len() as u64; let account = self.account(account_id).await.caused_by(trc::location!())?; - self.has_available_quota(&account, raw_message_len) - .await - .caused_by(trc::location!())?; + // inbuxa: UD-14: a restore checks for itself, crediting what its kept + // copy already counts + if params.source != IngestSource::Restore { + self.has_available_quota(&account, raw_message_len) + .await + .caused_by(trc::location!())?; + } + let checked_len = raw_message_len; // Parse message let mut raw_message = Cow::from(params.raw_message); @@ -562,6 +567,16 @@ impl EmailIngest for Server { false }; + // inbuxa: the size stored is the message after encryption, with the + // headers added on delivery; check that when it's more than was + // checked at the start + let stored_len = (message.raw_message.len() + extra_headers.len()) as u64; + if stored_len > checked_len && params.source != IngestSource::Restore { + self.has_available_quota(&account, stored_len) + .await + .caused_by(trc::location!())?; + } + // Store blob let (blob_hash, blob_hold) = if !is_encrypted && let Some(blob_hash) = params.blob_hash { (blob_hash.clone(), None) @@ -946,6 +961,13 @@ impl EmailIngest for Server { } .to_pickled_vec(); + // inbuxa: UD-14a: a held sample is noted, so its hold can end + // with the account's own copy of the message + if hold_sample { + let linked_to = batch.last_account_id().unwrap_or(account_id); + inbuxa_features::undelete::samples::note(batch, linked_to, &hash, until); + } + let object_id = ObjectType::SpamTrainingSample.to_id(); let item_id = SnowflakeIdGenerator::global_id().unwrap_or_default(); batch diff --git a/crates/features/src/undelete/accounts.rs b/crates/features/src/undelete/accounts.rs index 5ca278a9..ea852111 100644 --- a/crates/features/src/undelete/accounts.rs +++ b/crates/features/src/undelete/accounts.rs @@ -139,7 +139,8 @@ pub async fn forget(data: &Store, registry: &RegistryStore, account_id: u32) -> .caused_by(trc::location!())?; } for (id, item) in records::of_account(data, registry, account_id).await? { - records::remove(data, registry, id, &item).await?; + // Its usage goes with the account, so the counters stay as they are + records::discard(data, registry, id, &item).await?; } data::clear_account(data, account_id).await } diff --git a/crates/features/src/undelete/data.rs b/crates/features/src/undelete/data.rs index 26b608f2..49a9811b 100644 --- a/crates/features/src/undelete/data.rs +++ b/crates/features/src/undelete/data.rs @@ -22,6 +22,11 @@ //! - `a` + address: an address a kept account holds reserved (UD-16). //! - `g` + kind + account + document: a deleted calendar event, contact or //! file ([`super::groupware`]). +//! - `s` + account + blob hash + until: a spam-training sample's hold on a +//! message's blob ([`super::samples`], UD-14a). +//! - `q` + account + item id: the bytes an archived item counts toward its +//! account's and tenant's usage while it's kept (UD-14). Items archived +//! before counting began have none, and nothing is taken off for them. //! //! Values are JSON, so they read back across versions of the fork. @@ -40,6 +45,7 @@ const KIND_RESTORE: u8 = b'r'; const KIND_CHANGE: u8 = b'c'; const KIND_KEPT: u8 = b'k'; const KIND_RESERVED: u8 = b'a'; +const KIND_COUNTED: u8 = b'q'; fn class(kind: u8, rest: &[u8]) -> ValueClass { let mut key = Vec::with_capacity(1 + rest.len()); @@ -249,6 +255,46 @@ pub fn clear_blob_item(batch: &mut BatchBuilder, account_id: u32, blob_hash: &[u batch.clear(class(KIND_BLOB, &account_blob(account_id, blob_hash))); } +fn account_item(account_id: u32, item_id: Id) -> [u8; 12] { + let mut out = [0u8; 12]; + out[..4].copy_from_slice(&account_id.to_be_bytes()); + out[4..].copy_from_slice(&item_id.id().to_be_bytes()); + out +} + +/// UD-14: an archived item's counted bytes. +pub fn counted_class(account_id: u32, item_id: Id) -> ValueClass { + class(KIND_COUNTED, &account_item(account_id, item_id)) +} + +pub async fn counted(data: &Store, account_id: u32, item_id: Id) -> trc::Result> { + data.get_value::(key(KIND_COUNTED, &account_item(account_id, item_id))) + .await + .caused_by(trc::location!()) +} + +/// UD-14: the bytes all an account's archived items count, for recounting +/// its usage from scratch. +pub async fn counted_total(data: &Store, account_id: u32) -> trc::Result { + let mut total = 0u64; + data.iterate( + IterateParams::new( + key(KIND_COUNTED, &account_item(account_id, Id::new(0))), + key(KIND_COUNTED, &account_item(account_id, Id::new(u64::MAX))), + ) + .ascending(), + |_, value| { + if let Ok(bytes) = <[u8; 8]>::try_from(value) { + total = total.saturating_add(u64::from_be_bytes(bytes)); + } + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + Ok(total) +} + pub fn set_restore_requested(batch: &mut BatchBuilder, item_id: Id) { batch.set(class(KIND_RESTORE, &item_id.id().to_be_bytes()), vec![1]); } @@ -416,6 +462,8 @@ pub async fn clear_account(data: &Store, account_id: u32) -> trc::Result<()> { (vec![KIND_NOTE], vec![KIND_NOTE]), (vec![KIND_BLOB], vec![KIND_BLOB]), (vec![KIND_CHANGE], vec![KIND_CHANGE]), + (vec![KIND_COUNTED], vec![KIND_COUNTED]), + (vec![b's'], vec![b's']), ]; // The groupware notes, `g` + kind + account + document for kind in 0u8..3 { diff --git a/crates/features/src/undelete/email.rs b/crates/features/src/undelete/email.rs index db4cbff0..07619a63 100644 --- a/crates/features/src/undelete/email.rs +++ b/crates/features/src/undelete/email.rs @@ -133,6 +133,7 @@ pub async fn archive( mailboxes: note.mailboxes, keywords: note.keywords, }, + note.size, ) .await?; diff --git a/crates/features/src/undelete/groupware.rs b/crates/features/src/undelete/groupware.rs index 6a8ead02..45bbbf2a 100644 --- a/crates/features/src/undelete/groupware.rs +++ b/crates/features/src/undelete/groupware.rs @@ -238,11 +238,21 @@ pub async fn archive( .details("Not a groupware item")); } }; - records::insert(data, registry, &item, ¬e.extra).await + let size = kept_size(¬e); + records::insert(data, registry, &item, ¬e.extra, size).await } /// The names a restored item tries, in order: the original, then with /// ` (restored)`, then numbered (UD-8). +/// The bytes a kept file, event or contact counts while it's archived +/// (UD-14): a file's size, or the event's or contact's text. +fn kept_size(note: &Note) -> u64 { + match ¬e.extra { + Extra::FileNode { size, .. } if *size > 0 => *size as u64, + _ => note.content.as_ref().map_or(0, |text| text.len() as u64), + } +} + pub fn candidates(name: &str) -> impl Iterator + '_ { let (stem, ext) = match name.rsplit_once('.') { Some((stem, ext)) if !stem.is_empty() => (stem, format!(".{ext}")), @@ -301,6 +311,7 @@ pub async fn archive_sieve( retention: u64, ) -> trc::Result { let archived_at = now(); + let size = content.len() as u64; let item = ArchivedItem::SieveScript(registry::schema::structs::ArchivedSieveScript { name: name.to_string(), created_at: UTCDateTime::from_timestamp(archived_at as i64), @@ -317,6 +328,7 @@ pub async fn archive_sieve( &Extra::SieveScript { name: name.to_string(), }, + size, ) .await } diff --git a/crates/features/src/undelete/mod.rs b/crates/features/src/undelete/mod.rs index 0a51b4e3..35461e95 100644 --- a/crates/features/src/undelete/mod.rs +++ b/crates/features/src/undelete/mod.rs @@ -18,4 +18,5 @@ pub mod data; pub mod email; pub mod groupware; pub mod records; +pub mod samples; pub mod settings; diff --git a/crates/features/src/undelete/records.rs b/crates/features/src/undelete/records.rs index 798f8773..c241ccc4 100644 --- a/crates/features/src/undelete/records.rs +++ b/crates/features/src/undelete/records.rs @@ -8,17 +8,23 @@ //! its account index, with no link to the account, so deleting an account //! isn't refused while it has archived items. Upstream's account removal //! task clears exactly these two keys. +//! +//! UD-14: an item counts toward its account's and its tenant's storage while +//! it's kept, so deleting doesn't free space until the item goes. Without +//! this, a member could fill the account, delete, and fill it again, keeping +//! the limit's worth of data once more every time for the whole retention +//! period. use crate::undelete::data::{self, Change}; use registry::schema::prelude::Property; use registry::{ schema::{prelude::ObjectType, structs::ArchivedItem}, - types::{EnumImpl, ObjectImpl, index::IndexValue}, + types::{EnumImpl, ObjectImpl, id::ObjectId, index::IndexValue}, }; use store::{ RegistryStore, SerializeInfallible, Store, registry::RegistryQuery, - write::{BatchBuilder, BlobLink, BlobOp, RegistryClass, ValueClass, now}, + write::{BatchBuilder, BlobLink, BlobOp, RegistryClass, ValueClass, assert::AssertValue, now}, }; use trc::AddContext; use types::id::Id; @@ -39,13 +45,45 @@ fn item_class(item_id: u64) -> ValueClass { }) } +/// The tenant an account's usage counts toward: its own, or for a deleted +/// account still kept, the one it had (UD-15). +pub async fn tenant_of( + data: &Store, + registry: &RegistryStore, + account_id: u32, +) -> trc::Result> { + if let Some(account) = registry + .get(ObjectId::new(ObjectType::Account, Id::from(account_id))) + .await + .caused_by(trc::location!())? + { + return Ok(account.inner.member_tenant_id().map(|id| id.document_id())); + } + Ok(data::kept_account(data, account_id) + .await? + .and_then(|kept| kept.member_tenant_id) + .map(|id| Id::from(id).document_id())) +} + +/// Adds `bytes` to an account's usage, and to its tenant's (MT-20). +fn count(batch: &mut BatchBuilder, account_id: u32, tenant_id: Option, bytes: i64) { + batch + .with_account_id(account_id) + .add(ValueClass::Quota, bytes); + if let Some(tenant_id) = tenant_id { + batch.add(ValueClass::TenantQuota(tenant_id), bytes); + } +} + /// Writes a new archived item, holding its kept copy until `archivedUntil`, -/// with what restore needs beside it. Returns its id. +/// with what restore needs beside it, and counting its `size` toward the +/// account and its tenant (UD-14). Returns its id. pub async fn insert( data: &Store, registry: &RegistryStore, item: &ArchivedItem, extra: &data::Extra, + size: u64, ) -> trc::Result { let item_id = registry.assign_id(); let id = Id::new(item_id); @@ -65,6 +103,11 @@ pub async fn insert( ); data::set_extra(&mut batch, id, extra)?; data::set_blob_item(&mut batch, account_id, blob_hash.as_slice(), id); + if size > 0 { + let tenant_id = tenant_of(data, registry, account_id).await?; + count(&mut batch, account_id, tenant_id, size as i64); + batch.set(data::counted_class(account_id, id), size.serialize()); + } data::log_change( &mut batch, account_id, @@ -143,13 +186,57 @@ pub async fn set_deadline( Ok(updated) } -/// Removes an archived item and releases its kept copy: on restore (UD-9), -/// on destroy (UD-12) and past its deadline (UD-13). +/// Removes an archived item for good and releases its kept copy: on +/// destroy (UD-12) and past its deadline (UD-13). Its bytes come off the +/// account's and tenant's usage (UD-14), and a message's spam-training +/// holds end with it (UD-14a). pub async fn remove( data: &Store, registry: &RegistryStore, id: Id, item: &ArchivedItem, +) -> trc::Result<()> { + remove_(data, registry, id, item, true).await?; + if let ArchivedItem::Email(email) = item { + crate::undelete::samples::release( + data, + email.account_id.document_id(), + &email.blob_id.hash, + ) + .await?; + } + Ok(()) +} + +/// Removes an archived item that's been restored (UD-9): its bytes come +/// off, since the restored item counts as itself from now on. +pub async fn remove_restored( + data: &Store, + registry: &RegistryStore, + id: Id, + item: &ArchivedItem, +) -> trc::Result<()> { + remove_(data, registry, id, item, true).await +} + +/// Removes an archived item of an account being destroyed, leaving the +/// usage counters alone: the account's goes with it, and its tenant's is +/// settled from it. +pub async fn discard( + data: &Store, + registry: &RegistryStore, + id: Id, + item: &ArchivedItem, +) -> trc::Result<()> { + remove_(data, registry, id, item, false).await +} + +async fn remove_( + data: &Store, + registry: &RegistryStore, + id: Id, + item: &ArchivedItem, + uncount: bool, ) -> trc::Result<()> { let account_id = item.account_id().document_id(); let blob_hash = item.blob_id().hash.clone(); @@ -164,6 +251,27 @@ pub async fn remove( .await .caused_by(trc::location!())?; + // Its counted bytes come off once: the marker must still be there, or + // another removal already took them off + let counted = if uncount { + data::counted(data, account_id, id).await? + } else { + None + }; + if let Some(size) = counted { + let tenant_id = tenant_of(data, registry, account_id).await?; + let mut batch = BatchBuilder::new(); + batch + .assert_value(data::counted_class(account_id, id), AssertValue::Some) + .clear(data::counted_class(account_id, id)); + count(&mut batch, account_id, tenant_id, -(size as i64)); + match data.write(batch.build_all()).await { + Ok(_) => {} + Err(err) if err.is_assertion_failure() => {} + Err(err) => return Err(err.caused_by(trc::location!())), + } + } + let mut batch = BatchBuilder::new(); batch.with_account_id(account_id).clear(BlobOp::Link { hash: blob_hash.clone(), @@ -171,6 +279,9 @@ pub async fn remove( until: item.archived_until().timestamp() as u64, }, }); + if !uncount { + batch.clear(data::counted_class(account_id, id)); + } data::clear_extra(&mut batch, id); data::clear_blob_item(&mut batch, account_id, blob_hash.as_slice()); data::clear_restore_requested(&mut batch, id); diff --git a/crates/features/src/undelete/samples.rs b/crates/features/src/undelete/samples.rs new file mode 100644 index 00000000..307fd8cd --- /dev/null +++ b/crates/features/src/undelete/samples.rs @@ -0,0 +1,158 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! Spam-training samples held past their message (UD-14a). +//! +//! The classifier holds a trained message's blob for `holdSamplesFor` +//! (90 days by default) so it can replay it later. While the message is in +//! the account that costs nothing: the hold and the message share one blob, +//! counted once. Once the account's own copy is gone for good, though, the +//! held blob was stored with nobody counting it, so a member could fill the +//! account, train, delete and fill it again, past the account's and the +//! tenant's limit, for the whole hold period. +//! +//! So a hold now ends with the account's own copy: when the message's data +//! is removed and nothing archives it, or when its archived copy goes other +//! than by restore. The classifier already treats a blob that's gone as a +//! sample to drop from its replay pool, so this shrinks the pool to mail +//! that's still kept; what was learned from the sample stays learned. +//! +//! Each hold is noted as `s` + account + blob hash + until (empty value), so +//! it can be found from the message, with the link it set under that +//! account. + +use store::{ + IterateParams, SUBSPACE_UNDELETE, Store, ValueKey, + write::{AnyClass, BatchBuilder, BlobLink, BlobOp, ValueClass, now}, +}; +use trc::AddContext; +use types::blob_hash::{BLOB_HASH_LEN, BlobHash}; + +const KIND_SAMPLE: u8 = b's'; + +fn class(account_id: u32, hash: &[u8], until: u64) -> ValueClass { + let mut key = Vec::with_capacity(1 + 4 + hash.len() + 8); + key.push(KIND_SAMPLE); + key.extend_from_slice(&account_id.to_be_bytes()); + key.extend_from_slice(hash); + key.extend_from_slice(&until.to_be_bytes()); + ValueClass::Any(AnyClass { + subspace: SUBSPACE_UNDELETE, + key, + }) +} + +/// Notes a held sample: its blob, linked under `account_id` until `until`. +pub fn note(batch: &mut BatchBuilder, account_id: u32, hash: &BlobHash, until: u64) { + batch.set(class(account_id, hash.as_slice(), until), vec![]); +} + +/// The holds noted for a blob in an account, by their `until`. +async fn holds(data: &Store, account_id: u32, hash: &[u8]) -> trc::Result> { + let mut untils = Vec::new(); + data.iterate( + IterateParams::new( + ValueKey::from(class(account_id, hash, 0)), + ValueKey::from(class(account_id, hash, u64::MAX)), + ) + .ascending() + .no_values(), + |key, _| { + if key.len() >= 8 { + untils.push(u64::from_be_bytes(key[key.len() - 8..].try_into().unwrap())); + } + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + Ok(untils) +} + +/// The account's own copy of a message is gone for good: its samples' +/// holds on the blob end with it. +pub async fn release(data: &Store, account_id: u32, hash: &BlobHash) -> trc::Result<()> { + let untils = holds(data, account_id, hash.as_slice()).await?; + if untils.is_empty() { + return Ok(()); + } + let mut batch = BatchBuilder::new(); + batch.with_account_id(account_id); + for until in untils { + batch + .clear(BlobOp::Link { + hash: hash.clone(), + to: BlobLink::Temporary { until }, + }) + .clear(class(account_id, hash.as_slice(), until)); + } + data.write(batch.build_all()) + .await + .caused_by(trc::location!()) + .map(|_| ()) +} + +/// Clears notes whose hold has ended on its own, for the scheduled clean-up. +/// Returns how many went. +pub async fn purge_ended(data: &Store) -> trc::Result { + let now = now(); + let mut ended = Vec::new(); + data.iterate( + IterateParams::new( + ValueKey::from(ValueClass::Any(AnyClass { + subspace: SUBSPACE_UNDELETE, + key: vec![KIND_SAMPLE], + })), + ValueKey::from(ValueClass::Any(AnyClass { + subspace: SUBSPACE_UNDELETE, + key: vec![KIND_SAMPLE, u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX], + })), + ) + .ascending() + .no_values(), + |key, _| { + if key.len() == 1 + 4 + BLOB_HASH_LEN + 8 + && u64::from_be_bytes(key[key.len() - 8..].try_into().unwrap()) <= now + { + ended.push(key.to_vec()); + } + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + let count = ended.len(); + for chunk in ended.chunks(512) { + let mut batch = BatchBuilder::new(); + for key in chunk { + batch.clear(ValueClass::Any(AnyClass { + subspace: SUBSPACE_UNDELETE, + key: key.clone(), + })); + } + data.write(batch.build_all()) + .await + .caused_by(trc::location!())?; + } + Ok(count) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keys_sort_by_account_then_blob_then_until() { + let hash = BlobHash::generate(b"x"); + let key = |account, until| match class(account, hash.as_slice(), until) { + ValueClass::Any(any) => any.key, + _ => unreachable!(), + }; + assert_eq!(key(1, 5).len(), 1 + 4 + BLOB_HASH_LEN + 8); + assert!(key(1, 5) < key(1, 6)); + assert!(key(1, u64::MAX) < key(2, 0)); + } +} diff --git a/crates/groupware/src/calendar/itip.rs b/crates/groupware/src/calendar/itip.rs index 55c74e6b..8792cec9 100644 --- a/crates/groupware/src/calendar/itip.rs +++ b/crates/groupware/src/calendar/itip.rs @@ -263,11 +263,12 @@ impl ItipIngest for Server { let mut ical = itip.clone(); itip_import_message(&mut ical)?; - // Validate quota + // Validate quota: inbuxa: both the event and the notification + // beside it are stored, each the message's size if self .has_available_quota( self.account(account_id).await?.as_ref(), - itip_message.len() as u64, + 2 * itip_message.len() as u64, ) .await .is_err() @@ -782,7 +783,10 @@ async fn commit_itip_merge( } // Validate quota - let extra_bytes = (event.size as u64).saturating_sub(event_.inner.size.to_native() as u64); + // inbuxa: the notification stored beside the update counts too + let extra_bytes = (event.size as u64) + .saturating_sub(event_.inner.size.to_native() as u64) + .saturating_add(itip_size as u64); if extra_bytes > 0 && server .has_available_quota(server.account(account_id).await?.as_ref(), extra_bytes) diff --git a/crates/jmap/src/vacation/set.rs b/crates/jmap/src/vacation/set.rs index 44524906..f9eb3ad8 100644 --- a/crates/jmap/src/vacation/set.rs +++ b/crates/jmap/src/vacation/set.rs @@ -2,6 +2,8 @@ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ use super::get::VacationResponseGet; @@ -244,6 +246,7 @@ impl VacationResponseSet for Server { } } + let prev_size = prev_sieve.as_ref().map_or(0, |prev| prev.inner.size as u64); let mut obj = ObjectIndexBuilder::new() .with_current_opt(prev_sieve) .with_changes(sieve) @@ -265,14 +268,33 @@ impl VacationResponseSet for Server { // Create sieve script only if there are changes if build_script { + let script = self.build_script(obj.changes_mut().unwrap())?; + + // inbuxa: a bigger script counts against quota, as a Sieve + // script's does + let new_size = obj.changes_mut().unwrap().size as u64; + if new_size > prev_size { + let account = self.account(account_id).await?; + match self + .has_available_quota(&account, new_size - prev_size) + .await + { + Ok(()) => {} + Err(err) + if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) + || err.matches(trc::EventType::Limit( + trc::LimitEvent::TenantQuota, + )) => + { + return Ok(set_error(response, create_id, SetError::over_quota())); + } + Err(err) => return Err(err), + } + } + // Upload new blob - let (blob_hash, blob_hold) = self - .put_temporary_blob( - account_id, - &self.build_script(obj.changes_mut().unwrap())?, - 60, - ) - .await?; + let (blob_hash, blob_hold) = + self.put_temporary_blob(account_id, &script, 60).await?; obj.changes_mut().unwrap().blob_hash = blob_hash; batch.clear(blob_hold); }; diff --git a/crates/services/src/task_manager/inbuxa_restore.rs b/crates/services/src/task_manager/inbuxa_restore.rs index 914a22fb..9cb96706 100644 --- a/crates/services/src/task_manager/inbuxa_restore.rs +++ b/crates/services/src/task_manager/inbuxa_restore.rs @@ -73,9 +73,13 @@ pub(crate) async fn restore_other( return Ok(Some("The kept copy is gone.".into())); }; - // UD-10: a restore counts against quota like anything new + // UD-10, UD-14: a restore counts against quota like anything new, less + // what the kept copy already counts + let counted = undelete::data::counted(&server.core.storage.data, account_id, item_id) + .await? + .unwrap_or(0); if server - .has_available_quota(&account, bytes.len() as u64) + .has_available_quota(&account, (bytes.len() as u64).saturating_sub(counted)) .await .is_err() { @@ -259,7 +263,8 @@ pub(crate) async fn restore_other( server.commit_batch(batch).await?; // UD-9: the record goes once the item is back - undelete::records::remove(&server.core.storage.data, server.registry(), item_id, item).await?; + undelete::records::remove_restored(&server.core.storage.data, server.registry(), item_id, item) + .await?; Ok(None) } diff --git a/crates/services/src/task_manager/index.rs b/crates/services/src/task_manager/index.rs index 5fba7f7f..91b2c786 100644 --- a/crates/services/src/task_manager/index.rs +++ b/crates/services/src/task_manager/index.rs @@ -790,13 +790,14 @@ async fn delete_email_metadata( // inbuxa: UD-1, UD-4: a message noted at deletion is archived let root = metadata.contents.first().and_then(|c| c.parts.first()); - inbuxa_features::undelete::email::archive( + let blob_hash = BlobHash::from(&metadata.blob_hash); + let archived = inbuxa_features::undelete::email::archive( &server.core.storage.data, server.registry(), account_id, document_id, inbuxa_features::undelete::email::Summary { - blob_hash: BlobHash::from(&metadata.blob_hash), + blob_hash: blob_hash.clone(), from: root.and_then(|part| part.from()), subject: root.and_then(|part| part.subject()), received_at: metadata.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK, @@ -804,6 +805,16 @@ async fn delete_email_metadata( ) .await .caused_by(trc::location!())?; + // inbuxa: UD-14a: with no archived copy, its spam-sample holds end + if !archived { + inbuxa_features::undelete::samples::release( + &server.core.storage.data, + account_id, + &blob_hash, + ) + .await + .caused_by(trc::location!())?; + } } None => { trc::event!( diff --git a/crates/services/src/task_manager/maintenance.rs b/crates/services/src/task_manager/maintenance.rs index 32923310..5e6cfd98 100644 --- a/crates/services/src/task_manager/maintenance.rs +++ b/crates/services/src/task_manager/maintenance.rs @@ -236,6 +236,10 @@ async fn store_maintenance( ) .await .caused_by(trc::location!())?; + // inbuxa: UD-14a: and notes of spam-sample holds that have ended + inbuxa_features::undelete::samples::purge_ended(&server.core.storage.data) + .await + .caused_by(trc::location!())?; use common::telemetry::metrics::store::MetricsStore; // inbuxa: MON-17, MON-38: history past its retention goes; a @@ -551,6 +555,10 @@ async fn recalculate_quota(server: &Server, account_id: u32) -> trc::Result<()> .caused_by(trc::location!())?; } + // inbuxa: UD-14: kept deleted items count too + quota += inbuxa_features::undelete::data::counted_total(&server.core.storage.data, account_id) + .await? as i64; + let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) diff --git a/crates/services/src/task_manager/restore_item.rs b/crates/services/src/task_manager/restore_item.rs index 5cf50197..6e624aa0 100644 --- a/crates/services/src/task_manager/restore_item.rs +++ b/crates/services/src/task_manager/restore_item.rs @@ -95,6 +95,29 @@ async fn restore_item(server: &Server, task: &TaskRestoreArchivedItem) -> trc::R return Ok(TaskResult::permanent("Blob not found")); }; + // inbuxa: UD-10, UD-14: a restore counts against quota like + // anything new, less what the kept copy already counts. Ingest + // leaves a restore's check to this. + let counted = undelete::data::counted(data, account_id, item_id) + .await? + .unwrap_or(0); + let account = server + .account(account_id) + .await + .caused_by(trc::location!())?; + if server + .has_available_quota(&account, (bytes.len() as u64).saturating_sub(counted)) + .await + .is_err() + { + let mut batch = BatchBuilder::new(); + undelete::data::clear_restore_requested(&mut batch, item_id); + data.write(batch.build_all()).await?; + return Ok(TaskResult::permanent( + "Not restored: the account or its tenant is over quota.".to_string(), + )); + } + match server .email_ingest(IngestEmail { raw_message: &bytes, @@ -111,7 +134,8 @@ async fn restore_item(server: &Server, task: &TaskRestoreArchivedItem) -> trc::R { Ok(_) => { // inbuxa: UD-9: the archived record goes, and its copy is released - undelete::records::remove(data, server.registry(), item_id, &item).await?; + undelete::records::remove_restored(data, server.registry(), item_id, &item) + .await?; Ok(TaskResult::Success(vec![])) } // inbuxa: UD-10: over quota, the item stays archived and the task says why diff --git a/docs/spec/features/undelete.md b/docs/spec/features/undelete.md index 3f55ad27..dcf85a9f 100644 --- a/docs/spec/features/undelete.md +++ b/docs/spec/features/undelete.md @@ -146,8 +146,9 @@ Task type `RestoreArchivedItem`, carrying the `accountId`, the (observed 5). The restored item is a new item with a new id, and JMAP `/changes` reports it as created. - **UD-10.** A restore still counts against quota. It's refused if it would - take the account, or its tenant, past a limit. The item stays archived, and - the task records why. + take the account, or its tenant, past a limit, less what the archived copy + already counts (UD-14), so restoring a counted item is never refused just + for being counted. The item stays archived, and the task records why. - **UD-11.** Restoring is idempotent: asking twice, or asking while a restore task is already running, restores once. @@ -163,10 +164,25 @@ Task type `RestoreArchivedItem`, carrying the `accountId`, the ### Storage -- **UD-14.** Archived copies don't count toward the account's or tenant's - quota while archived, as upstream (observed 7). Counting them would make deleting mail to - free space fail. They do count in server storage reporting, and ihasmail's - admin dashboard shows the archive's total size per account. +- **UD-14.** Archived copies count toward the account's and its tenant's + quota while they're kept, and come off when they go: restored (the restored + item counts as itself from then on), destroyed or expired. This changed in + 27.1 (inbuxa-admin#257). Upstream doesn't count them (observed 7), and + neither did we at first, so that deleting would free space. But a member + could then fill the account, delete, and fill it again, storing the limit's + worth once more every cycle for the whole retention period, past both the + account's quota and the tenant's. Deleting never fails because of this: it + just doesn't free space until the item goes, and an account that's full + stops taking new mail. A user who needs the space back destroys the + archived items (UD-12). Items archived before 27.1 weren't counted, so + nothing is taken off when they go. A deleted account's archive stays + counted with the account (UD-15), and goes with it. +- **UD-14a.** A spam-training sample's hold on a message's blob ends with + the account's own copy: when the message's data is removed with nothing + archiving it, or when its archived copy goes other than by restore. Held + longer, the blob was stored with nobody counting it, for + `holdSamplesFor` (90 days by default). The classifier drops a sample whose + blob is gone from its replay pool; what it learned from the sample stays. ### Deleted accounts @@ -257,7 +273,9 @@ Stalwart-facing (SPEC.md ยง5). 11. Retention lowered from 30 to 7 days: existing deadlines don't move (UD-5). 12. Past `archivedUntil`: not restorable, and gone after clean-up (UD-13). -13. Archive size doesn't count toward quota (UD-14). +13. Archive size counts toward the account's and the tenant's quota while + kept, comes off when it goes, and a counted item restores in a full + account (UD-14, UD-10). 14. Deleted account: can't sign in, mail refused, name reserved, restorable by an admin with its data intact (UD-15 to UD-17). 15. A user can't see another user's archive. A tenant admin sees only its diff --git a/tests/src/jmap/mail/vacation_response.rs b/tests/src/jmap/mail/vacation_response.rs index d200f998..783b08ae 100644 --- a/tests/src/jmap/mail/vacation_response.rs +++ b/tests/src/jmap/mail/vacation_response.rs @@ -2,6 +2,8 @@ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ use crate::{ @@ -12,6 +14,8 @@ use crate::{ }; use chrono::{TimeDelta, Utc}; use jmap_client::core::response::VacationResponseSetResponse; +use registry::schema::prelude::{ObjectType, Property}; +use serde_json::json; use std::time::Instant; pub async fn test(test: &TestServer) { @@ -190,6 +194,41 @@ pub async fn test(test: &TestServer) { ) .await; + // inbuxa: a bigger vacation script counts against quota, as a Sieve + // script's does + let admin = test.account("admin@example.com"); + let used = server + .get_used_quota_account(account.id().document_id()) + .await + .unwrap(); + admin + .registry_update_object( + ObjectType::Account, + account.id(), + json!({ Property::Quotas: {"maxDiskQuota": used.max(1)} }), + ) + .await; + let mut request = client.build(); + request + .set_vacation_response() + .update("singleton") + .text_body(Some("x".repeat(4096))); + let mut response = request + .send_single::() + .await + .unwrap(); + assert!( + response.updated("singleton").is_err(), + "a vacation script past the quota is refused" + ); + admin + .registry_update_object( + ObjectType::Account, + account.id(), + json!({ Property::Quotas: {} }), + ) + .await; + // Remove test data client.vacation_response_disable().await.unwrap(); client.sieve_script_deactivate().await.unwrap(); diff --git a/tests/src/system/undelete.rs b/tests/src/system/undelete.rs index e9985a07..3bb6fd85 100644 --- a/tests/src/system/undelete.rs +++ b/tests/src/system/undelete.rs @@ -145,7 +145,8 @@ pub async fn test(test: &mut TestServer) { test.wait_for_tasks().await; assert_eq!(user.archived().await.len(), 4, "test 3"); - // Acceptance test 13: archived copies don't count toward quota + // Acceptance test 13: archived copies count toward quota while kept + // (UD-14) let used = test .server .get_used_quota_account(user.id().document_id()) @@ -164,17 +165,35 @@ pub async fn test(test: &mut TestServer) { .await; client.email_destroy(&id).await.unwrap(); test.wait_for_tasks().await; - assert_eq!( - test.server - .get_used_quota_account(user.id().document_id()) - .await - .unwrap(), - used, - "test 13" - ); let item = user.archived_with_subject("Two labels").await; + let size = item["size"].as_i64().expect("test 13: size"); + assert_eq!( + used_now(&test.server, user.id().document_id()).await, + used + size, + "test 13: counted while kept" + ); + // ... and a counted item restores even into a full account (UD-10) + admin + .registry_update_object( + ObjectType::Account, + user.id(), + json!({ Property::Quotas: {"maxDiskQuota": used + size} }), + ) + .await; user.request_restore(item.object_id()).await; test.wait_for_tasks().await; + assert_eq!( + used_now(&test.server, user.id().document_id()).await, + used + size, + "test 13: the restored message counts instead, once" + ); + admin + .registry_update_object( + ObjectType::Account, + user.id(), + json!({ Property::Quotas: {} }), + ) + .await; let restored = user.email_with_subject("Two labels").await; assert_eq!( restored["mailboxIds"], @@ -226,11 +245,18 @@ pub async fn test(test: &mut TestServer) { test.wait_for_tasks().await; assert_eq!(user.count_with_subject("Via JMAP").await, 1, "test 9"); - // Acceptance test 10: the user destroys an archived item for good (UD-12) + // Acceptance test 10: the user destroys an archived item for good (UD-12), + // and its space comes back (UD-14) let item = user.archived_with_subject("Via IMAP").await; + let before = used_now(&test.server, user.id().document_id()).await; user.registry_destroy(ObjectType::ArchivedItem, [item.object_id()]) .await .assert_destroyed(&[item.object_id()]); + assert_eq!( + used_now(&test.server, user.id().document_id()).await, + before - item["size"].as_i64().expect("test 10: size"), + "test 10: its bytes come off" + ); assert!( user.archived() .await @@ -425,7 +451,8 @@ pub async fn test(test: &mut TestServer) { ); // Acceptance test 8: a restore past the account's quota is refused, and - // the item stays archived (UD-10) + // the item stays archived (UD-10). The item's own bytes already count, so + // the account has to be over its limit for that let used = test .server .get_used_quota_account(user.id().document_id()) @@ -435,7 +462,7 @@ pub async fn test(test: &mut TestServer) { .registry_update_object( ObjectType::Account, user.id(), - json!({ Property::Quotas: {"maxDiskQuota": used} }), + json!({ Property::Quotas: {"maxDiskQuota": used - 1} }), ) .await; let item = user.archived_with_subject("Via POP3").await; @@ -1066,3 +1093,8 @@ impl Account { .await; } } + +// UD-14: what an account's usage counter says now +async fn used_now(server: &common::Server, account_id: u32) -> i64 { + server.get_used_quota_account(account_id).await.unwrap() +}