Count what's kept after deletion, and check what's really stored (#500) #502

Merged
jcoffey-dev merged 1 commits from fix/tenant-uncounted-storage into main 2026-11-20 17:29:06 +00:00
17 changed files with 562 additions and 45 deletions

No files matched your search

+25 -3
View File
@@ -152,9 +152,14 @@ impl EmailIngest for Server {
let tenant_id = params.access_token.tenant_id();
let mut raw_message_len = params.raw_message.len() as u64;
let account = self.account(account_id).await.caused_by(trc::location!())?;
self.has_available_quota(&account, raw_message_len)
.await
.caused_by(trc::location!())?;
// inbuxa: UD-14: a restore checks for itself, crediting what its kept
// copy already counts
if params.source != IngestSource::Restore {
self.has_available_quota(&account, raw_message_len)
.await
.caused_by(trc::location!())?;
}
let checked_len = raw_message_len;
// Parse message
let mut raw_message = Cow::from(params.raw_message);
@@ -562,6 +567,16 @@ impl EmailIngest for Server {
false
};
// inbuxa: the size stored is the message after encryption, with the
// headers added on delivery; check that when it's more than was
// checked at the start
let stored_len = (message.raw_message.len() + extra_headers.len()) as u64;
if stored_len > checked_len && params.source != IngestSource::Restore {
self.has_available_quota(&account, stored_len)
.await
.caused_by(trc::location!())?;
}
// Store blob
let (blob_hash, blob_hold) = if !is_encrypted && let Some(blob_hash) = params.blob_hash {
(blob_hash.clone(), None)
@@ -946,6 +961,13 @@ impl EmailIngest for Server {
}
.to_pickled_vec();
// inbuxa: UD-14a: a held sample is noted, so its hold can end
// with the account's own copy of the message
if hold_sample {
let linked_to = batch.last_account_id().unwrap_or(account_id);
inbuxa_features::undelete::samples::note(batch, linked_to, &hash, until);
}
let object_id = ObjectType::SpamTrainingSample.to_id();
let item_id = SnowflakeIdGenerator::global_id().unwrap_or_default();
batch
+2 -1
View File
@@ -139,7 +139,8 @@ pub async fn forget(data: &Store, registry: &RegistryStore, account_id: u32) ->
.caused_by(trc::location!())?;
}
for (id, item) in records::of_account(data, registry, account_id).await? {
records::remove(data, registry, id, &item).await?;
// Its usage goes with the account, so the counters stay as they are
records::discard(data, registry, id, &item).await?;
}
data::clear_account(data, account_id).await
}
+48
View File
@@ -22,6 +22,11 @@
//! - `a` + address: an address a kept account holds reserved (UD-16).
//! - `g` + kind + account + document: a deleted calendar event, contact or
//! file ([`super::groupware`]).
//! - `s` + account + blob hash + until: a spam-training sample's hold on a
//! message's blob ([`super::samples`], UD-14a).
//! - `q` + account + item id: the bytes an archived item counts toward its
//! account's and tenant's usage while it's kept (UD-14). Items archived
//! before counting began have none, and nothing is taken off for them.
//!
//! Values are JSON, so they read back across versions of the fork.
@@ -40,6 +45,7 @@ const KIND_RESTORE: u8 = b'r';
const KIND_CHANGE: u8 = b'c';
const KIND_KEPT: u8 = b'k';
const KIND_RESERVED: u8 = b'a';
const KIND_COUNTED: u8 = b'q';
fn class(kind: u8, rest: &[u8]) -> ValueClass {
let mut key = Vec::with_capacity(1 + rest.len());
@@ -249,6 +255,46 @@ pub fn clear_blob_item(batch: &mut BatchBuilder, account_id: u32, blob_hash: &[u
batch.clear(class(KIND_BLOB, &account_blob(account_id, blob_hash)));
}
fn account_item(account_id: u32, item_id: Id) -> [u8; 12] {
let mut out = [0u8; 12];
out[..4].copy_from_slice(&account_id.to_be_bytes());
out[4..].copy_from_slice(&item_id.id().to_be_bytes());
out
}
/// UD-14: an archived item's counted bytes.
pub fn counted_class(account_id: u32, item_id: Id) -> ValueClass {
class(KIND_COUNTED, &account_item(account_id, item_id))
}
pub async fn counted(data: &Store, account_id: u32, item_id: Id) -> trc::Result<Option<u64>> {
data.get_value::<u64>(key(KIND_COUNTED, &account_item(account_id, item_id)))
.await
.caused_by(trc::location!())
}
/// UD-14: the bytes all an account's archived items count, for recounting
/// its usage from scratch.
pub async fn counted_total(data: &Store, account_id: u32) -> trc::Result<u64> {
let mut total = 0u64;
data.iterate(
IterateParams::new(
key(KIND_COUNTED, &account_item(account_id, Id::new(0))),
key(KIND_COUNTED, &account_item(account_id, Id::new(u64::MAX))),
)
.ascending(),
|_, value| {
if let Ok(bytes) = <[u8; 8]>::try_from(value) {
total = total.saturating_add(u64::from_be_bytes(bytes));
}
Ok(true)
},
)
.await
.caused_by(trc::location!())?;
Ok(total)
}
pub fn set_restore_requested(batch: &mut BatchBuilder, item_id: Id) {
batch.set(class(KIND_RESTORE, &item_id.id().to_be_bytes()), vec![1]);
}
@@ -416,6 +462,8 @@ pub async fn clear_account(data: &Store, account_id: u32) -> trc::Result<()> {
(vec![KIND_NOTE], vec![KIND_NOTE]),
(vec![KIND_BLOB], vec![KIND_BLOB]),
(vec![KIND_CHANGE], vec![KIND_CHANGE]),
(vec![KIND_COUNTED], vec![KIND_COUNTED]),
(vec![b's'], vec![b's']),
];
// The groupware notes, `g` + kind + account + document
for kind in 0u8..3 {
+1
View File
@@ -133,6 +133,7 @@ pub async fn archive(
mailboxes: note.mailboxes,
keywords: note.keywords,
},
note.size,
)
.await?;
+13 -1
View File
@@ -238,11 +238,21 @@ pub async fn archive(
.details("Not a groupware item"));
}
};
records::insert(data, registry, &item, &note.extra).await
let size = kept_size(&note);
records::insert(data, registry, &item, &note.extra, size).await
}
/// The names a restored item tries, in order: the original, then with
/// ` (restored)`, then numbered (UD-8).
/// The bytes a kept file, event or contact counts while it's archived
/// (UD-14): a file's size, or the event's or contact's text.
fn kept_size(note: &Note) -> u64 {
match &note.extra {
Extra::FileNode { size, .. } if *size > 0 => *size as u64,
_ => note.content.as_ref().map_or(0, |text| text.len() as u64),
}
}
pub fn candidates(name: &str) -> impl Iterator<Item = String> + '_ {
let (stem, ext) = match name.rsplit_once('.') {
Some((stem, ext)) if !stem.is_empty() => (stem, format!(".{ext}")),
@@ -301,6 +311,7 @@ pub async fn archive_sieve(
retention: u64,
) -> trc::Result<Id> {
let archived_at = now();
let size = content.len() as u64;
let item = ArchivedItem::SieveScript(registry::schema::structs::ArchivedSieveScript {
name: name.to_string(),
created_at: UTCDateTime::from_timestamp(archived_at as i64),
@@ -317,6 +328,7 @@ pub async fn archive_sieve(
&Extra::SieveScript {
name: name.to_string(),
},
size,
)
.await
}
+1
View File
@@ -18,4 +18,5 @@ pub mod data;
pub mod email;
pub mod groupware;
pub mod records;
pub mod samples;
pub mod settings;
+116 -5
View File
@@ -8,17 +8,23 @@
//! its account index, with no link to the account, so deleting an account
//! isn't refused while it has archived items. Upstream's account removal
//! task clears exactly these two keys.
//!
//! UD-14: an item counts toward its account's and its tenant's storage while
//! it's kept, so deleting doesn't free space until the item goes. Without
//! this, a member could fill the account, delete, and fill it again, keeping
//! the limit's worth of data once more every time for the whole retention
//! period.
use crate::undelete::data::{self, Change};
use registry::schema::prelude::Property;
use registry::{
schema::{prelude::ObjectType, structs::ArchivedItem},
types::{EnumImpl, ObjectImpl, index::IndexValue},
types::{EnumImpl, ObjectImpl, id::ObjectId, index::IndexValue},
};
use store::{
RegistryStore, SerializeInfallible, Store,
registry::RegistryQuery,
write::{BatchBuilder, BlobLink, BlobOp, RegistryClass, ValueClass, now},
write::{BatchBuilder, BlobLink, BlobOp, RegistryClass, ValueClass, assert::AssertValue, now},
};
use trc::AddContext;
use types::id::Id;
@@ -39,13 +45,45 @@ fn item_class(item_id: u64) -> ValueClass {
})
}
/// The tenant an account's usage counts toward: its own, or for a deleted
/// account still kept, the one it had (UD-15).
pub async fn tenant_of(
data: &Store,
registry: &RegistryStore,
account_id: u32,
) -> trc::Result<Option<u32>> {
if let Some(account) = registry
.get(ObjectId::new(ObjectType::Account, Id::from(account_id)))
.await
.caused_by(trc::location!())?
{
return Ok(account.inner.member_tenant_id().map(|id| id.document_id()));
}
Ok(data::kept_account(data, account_id)
.await?
.and_then(|kept| kept.member_tenant_id)
.map(|id| Id::from(id).document_id()))
}
/// Adds `bytes` to an account's usage, and to its tenant's (MT-20).
fn count(batch: &mut BatchBuilder, account_id: u32, tenant_id: Option<u32>, bytes: i64) {
batch
.with_account_id(account_id)
.add(ValueClass::Quota, bytes);
if let Some(tenant_id) = tenant_id {
batch.add(ValueClass::TenantQuota(tenant_id), bytes);
}
}
/// Writes a new archived item, holding its kept copy until `archivedUntil`,
/// with what restore needs beside it. Returns its id.
/// with what restore needs beside it, and counting its `size` toward the
/// account and its tenant (UD-14). Returns its id.
pub async fn insert(
data: &Store,
registry: &RegistryStore,
item: &ArchivedItem,
extra: &data::Extra,
size: u64,
) -> trc::Result<Id> {
let item_id = registry.assign_id();
let id = Id::new(item_id);
@@ -65,6 +103,11 @@ pub async fn insert(
);
data::set_extra(&mut batch, id, extra)?;
data::set_blob_item(&mut batch, account_id, blob_hash.as_slice(), id);
if size > 0 {
let tenant_id = tenant_of(data, registry, account_id).await?;
count(&mut batch, account_id, tenant_id, size as i64);
batch.set(data::counted_class(account_id, id), size.serialize());
}
data::log_change(
&mut batch,
account_id,
@@ -143,13 +186,57 @@ pub async fn set_deadline(
Ok(updated)
}
/// Removes an archived item and releases its kept copy: on restore (UD-9),
/// on destroy (UD-12) and past its deadline (UD-13).
/// Removes an archived item for good and releases its kept copy: on
/// destroy (UD-12) and past its deadline (UD-13). Its bytes come off the
/// account's and tenant's usage (UD-14), and a message's spam-training
/// holds end with it (UD-14a).
pub async fn remove(
data: &Store,
registry: &RegistryStore,
id: Id,
item: &ArchivedItem,
) -> trc::Result<()> {
remove_(data, registry, id, item, true).await?;
if let ArchivedItem::Email(email) = item {
crate::undelete::samples::release(
data,
email.account_id.document_id(),
&email.blob_id.hash,
)
.await?;
}
Ok(())
}
/// Removes an archived item that's been restored (UD-9): its bytes come
/// off, since the restored item counts as itself from now on.
pub async fn remove_restored(
data: &Store,
registry: &RegistryStore,
id: Id,
item: &ArchivedItem,
) -> trc::Result<()> {
remove_(data, registry, id, item, true).await
}
/// Removes an archived item of an account being destroyed, leaving the
/// usage counters alone: the account's goes with it, and its tenant's is
/// settled from it.
pub async fn discard(
data: &Store,
registry: &RegistryStore,
id: Id,
item: &ArchivedItem,
) -> trc::Result<()> {
remove_(data, registry, id, item, false).await
}
async fn remove_(
data: &Store,
registry: &RegistryStore,
id: Id,
item: &ArchivedItem,
uncount: bool,
) -> trc::Result<()> {
let account_id = item.account_id().document_id();
let blob_hash = item.blob_id().hash.clone();
@@ -164,6 +251,27 @@ pub async fn remove(
.await
.caused_by(trc::location!())?;
// Its counted bytes come off once: the marker must still be there, or
// another removal already took them off
let counted = if uncount {
data::counted(data, account_id, id).await?
} else {
None
};
if let Some(size) = counted {
let tenant_id = tenant_of(data, registry, account_id).await?;
let mut batch = BatchBuilder::new();
batch
.assert_value(data::counted_class(account_id, id), AssertValue::Some)
.clear(data::counted_class(account_id, id));
count(&mut batch, account_id, tenant_id, -(size as i64));
match data.write(batch.build_all()).await {
Ok(_) => {}
Err(err) if err.is_assertion_failure() => {}
Err(err) => return Err(err.caused_by(trc::location!())),
}
}
let mut batch = BatchBuilder::new();
batch.with_account_id(account_id).clear(BlobOp::Link {
hash: blob_hash.clone(),
@@ -171,6 +279,9 @@ pub async fn remove(
until: item.archived_until().timestamp() as u64,
},
});
if !uncount {
batch.clear(data::counted_class(account_id, id));
}
data::clear_extra(&mut batch, id);
data::clear_blob_item(&mut batch, account_id, blob_hash.as_slice());
data::clear_restore_requested(&mut batch, id);
+158
View File
@@ -0,0 +1,158 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! Spam-training samples held past their message (UD-14a).
//!
//! The classifier holds a trained message's blob for `holdSamplesFor`
//! (90 days by default) so it can replay it later. While the message is in
//! the account that costs nothing: the hold and the message share one blob,
//! counted once. Once the account's own copy is gone for good, though, the
//! held blob was stored with nobody counting it, so a member could fill the
//! account, train, delete and fill it again, past the account's and the
//! tenant's limit, for the whole hold period.
//!
//! So a hold now ends with the account's own copy: when the message's data
//! is removed and nothing archives it, or when its archived copy goes other
//! than by restore. The classifier already treats a blob that's gone as a
//! sample to drop from its replay pool, so this shrinks the pool to mail
//! that's still kept; what was learned from the sample stays learned.
//!
//! Each hold is noted as `s` + account + blob hash + until (empty value), so
//! it can be found from the message, with the link it set under that
//! account.
use store::{
IterateParams, SUBSPACE_UNDELETE, Store, ValueKey,
write::{AnyClass, BatchBuilder, BlobLink, BlobOp, ValueClass, now},
};
use trc::AddContext;
use types::blob_hash::{BLOB_HASH_LEN, BlobHash};
const KIND_SAMPLE: u8 = b's';
fn class(account_id: u32, hash: &[u8], until: u64) -> ValueClass {
let mut key = Vec::with_capacity(1 + 4 + hash.len() + 8);
key.push(KIND_SAMPLE);
key.extend_from_slice(&account_id.to_be_bytes());
key.extend_from_slice(hash);
key.extend_from_slice(&until.to_be_bytes());
ValueClass::Any(AnyClass {
subspace: SUBSPACE_UNDELETE,
key,
})
}
/// Notes a held sample: its blob, linked under `account_id` until `until`.
pub fn note(batch: &mut BatchBuilder, account_id: u32, hash: &BlobHash, until: u64) {
batch.set(class(account_id, hash.as_slice(), until), vec![]);
}
/// The holds noted for a blob in an account, by their `until`.
async fn holds(data: &Store, account_id: u32, hash: &[u8]) -> trc::Result<Vec<u64>> {
let mut untils = Vec::new();
data.iterate(
IterateParams::new(
ValueKey::from(class(account_id, hash, 0)),
ValueKey::from(class(account_id, hash, u64::MAX)),
)
.ascending()
.no_values(),
|key, _| {
if key.len() >= 8 {
untils.push(u64::from_be_bytes(key[key.len() - 8..].try_into().unwrap()));
}
Ok(true)
},
)
.await
.caused_by(trc::location!())?;
Ok(untils)
}
/// The account's own copy of a message is gone for good: its samples'
/// holds on the blob end with it.
pub async fn release(data: &Store, account_id: u32, hash: &BlobHash) -> trc::Result<()> {
let untils = holds(data, account_id, hash.as_slice()).await?;
if untils.is_empty() {
return Ok(());
}
let mut batch = BatchBuilder::new();
batch.with_account_id(account_id);
for until in untils {
batch
.clear(BlobOp::Link {
hash: hash.clone(),
to: BlobLink::Temporary { until },
})
.clear(class(account_id, hash.as_slice(), until));
}
data.write(batch.build_all())
.await
.caused_by(trc::location!())
.map(|_| ())
}
/// Clears notes whose hold has ended on its own, for the scheduled clean-up.
/// Returns how many went.
pub async fn purge_ended(data: &Store) -> trc::Result<usize> {
let now = now();
let mut ended = Vec::new();
data.iterate(
IterateParams::new(
ValueKey::from(ValueClass::Any(AnyClass {
subspace: SUBSPACE_UNDELETE,
key: vec![KIND_SAMPLE],
})),
ValueKey::from(ValueClass::Any(AnyClass {
subspace: SUBSPACE_UNDELETE,
key: vec![KIND_SAMPLE, u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX],
})),
)
.ascending()
.no_values(),
|key, _| {
if key.len() == 1 + 4 + BLOB_HASH_LEN + 8
&& u64::from_be_bytes(key[key.len() - 8..].try_into().unwrap()) <= now
{
ended.push(key.to_vec());
}
Ok(true)
},
)
.await
.caused_by(trc::location!())?;
let count = ended.len();
for chunk in ended.chunks(512) {
let mut batch = BatchBuilder::new();
for key in chunk {
batch.clear(ValueClass::Any(AnyClass {
subspace: SUBSPACE_UNDELETE,
key: key.clone(),
}));
}
data.write(batch.build_all())
.await
.caused_by(trc::location!())?;
}
Ok(count)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn keys_sort_by_account_then_blob_then_until() {
let hash = BlobHash::generate(b"x");
let key = |account, until| match class(account, hash.as_slice(), until) {
ValueClass::Any(any) => any.key,
_ => unreachable!(),
};
assert_eq!(key(1, 5).len(), 1 + 4 + BLOB_HASH_LEN + 8);
assert!(key(1, 5) < key(1, 6));
assert!(key(1, u64::MAX) < key(2, 0));
}
}
+7 -3
View File
@@ -263,11 +263,12 @@ impl ItipIngest for Server {
let mut ical = itip.clone();
itip_import_message(&mut ical)?;
// Validate quota
// Validate quota: inbuxa: both the event and the notification
// beside it are stored, each the message's size
if self
.has_available_quota(
self.account(account_id).await?.as_ref(),
itip_message.len() as u64,
2 * itip_message.len() as u64,
)
.await
.is_err()
@@ -782,7 +783,10 @@ async fn commit_itip_merge(
}
// Validate quota
let extra_bytes = (event.size as u64).saturating_sub(event_.inner.size.to_native() as u64);
// inbuxa: the notification stored beside the update counts too
let extra_bytes = (event.size as u64)
.saturating_sub(event_.inner.size.to_native() as u64)
.saturating_add(itip_size as u64);
if extra_bytes > 0
&& server
.has_available_quota(server.account(account_id).await?.as_ref(), extra_bytes)
+29 -7
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use super::get::VacationResponseGet;
@@ -244,6 +246,7 @@ impl VacationResponseSet for Server {
}
}
let prev_size = prev_sieve.as_ref().map_or(0, |prev| prev.inner.size as u64);
let mut obj = ObjectIndexBuilder::new()
.with_current_opt(prev_sieve)
.with_changes(sieve)
@@ -265,14 +268,33 @@ impl VacationResponseSet for Server {
// Create sieve script only if there are changes
if build_script {
let script = self.build_script(obj.changes_mut().unwrap())?;
// inbuxa: a bigger script counts against quota, as a Sieve
// script's does
let new_size = obj.changes_mut().unwrap().size as u64;
if new_size > prev_size {
let account = self.account(account_id).await?;
match self
.has_available_quota(&account, new_size - prev_size)
.await
{
Ok(()) => {}
Err(err)
if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota))
|| err.matches(trc::EventType::Limit(
trc::LimitEvent::TenantQuota,
)) =>
{
return Ok(set_error(response, create_id, SetError::over_quota()));
}
Err(err) => return Err(err),
}
}
// Upload new blob
let (blob_hash, blob_hold) = self
.put_temporary_blob(
account_id,
&self.build_script(obj.changes_mut().unwrap())?,
60,
)
.await?;
let (blob_hash, blob_hold) =
self.put_temporary_blob(account_id, &script, 60).await?;
obj.changes_mut().unwrap().blob_hash = blob_hash;
batch.clear(blob_hold);
};
@@ -73,9 +73,13 @@ pub(crate) async fn restore_other(
return Ok(Some("The kept copy is gone.".into()));
};
// UD-10: a restore counts against quota like anything new
// UD-10, UD-14: a restore counts against quota like anything new, less
// what the kept copy already counts
let counted = undelete::data::counted(&server.core.storage.data, account_id, item_id)
.await?
.unwrap_or(0);
if server
.has_available_quota(&account, bytes.len() as u64)
.has_available_quota(&account, (bytes.len() as u64).saturating_sub(counted))
.await
.is_err()
{
@@ -259,7 +263,8 @@ pub(crate) async fn restore_other(
server.commit_batch(batch).await?;
// UD-9: the record goes once the item is back
undelete::records::remove(&server.core.storage.data, server.registry(), item_id, item).await?;
undelete::records::remove_restored(&server.core.storage.data, server.registry(), item_id, item)
.await?;
Ok(None)
}
+13 -2
View File
@@ -790,13 +790,14 @@ async fn delete_email_metadata(
// inbuxa: UD-1, UD-4: a message noted at deletion is archived
let root = metadata.contents.first().and_then(|c| c.parts.first());
inbuxa_features::undelete::email::archive(
let blob_hash = BlobHash::from(&metadata.blob_hash);
let archived = inbuxa_features::undelete::email::archive(
&server.core.storage.data,
server.registry(),
account_id,
document_id,
inbuxa_features::undelete::email::Summary {
blob_hash: BlobHash::from(&metadata.blob_hash),
blob_hash: blob_hash.clone(),
from: root.and_then(|part| part.from()),
subject: root.and_then(|part| part.subject()),
received_at: metadata.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK,
@@ -804,6 +805,16 @@ async fn delete_email_metadata(
)
.await
.caused_by(trc::location!())?;
// inbuxa: UD-14a: with no archived copy, its spam-sample holds end
if !archived {
inbuxa_features::undelete::samples::release(
&server.core.storage.data,
account_id,
&blob_hash,
)
.await
.caused_by(trc::location!())?;
}
}
None => {
trc::event!(
@@ -236,6 +236,10 @@ async fn store_maintenance(
)
.await
.caused_by(trc::location!())?;
// inbuxa: UD-14a: and notes of spam-sample holds that have ended
inbuxa_features::undelete::samples::purge_ended(&server.core.storage.data)
.await
.caused_by(trc::location!())?;
use common::telemetry::metrics::store::MetricsStore;
// inbuxa: MON-17, MON-38: history past its retention goes; a
@@ -551,6 +555,10 @@ async fn recalculate_quota(server: &Server, account_id: u32) -> trc::Result<()>
.caused_by(trc::location!())?;
}
// inbuxa: UD-14: kept deleted items count too
quota += inbuxa_features::undelete::data::counted_total(&server.core.storage.data, account_id)
.await? as i64;
let mut batch = BatchBuilder::new();
batch
.with_account_id(account_id)
@@ -95,6 +95,29 @@ async fn restore_item(server: &Server, task: &TaskRestoreArchivedItem) -> trc::R
return Ok(TaskResult::permanent("Blob not found"));
};
// inbuxa: UD-10, UD-14: a restore counts against quota like
// anything new, less what the kept copy already counts. Ingest
// leaves a restore's check to this.
let counted = undelete::data::counted(data, account_id, item_id)
.await?
.unwrap_or(0);
let account = server
.account(account_id)
.await
.caused_by(trc::location!())?;
if server
.has_available_quota(&account, (bytes.len() as u64).saturating_sub(counted))
.await
.is_err()
{
let mut batch = BatchBuilder::new();
undelete::data::clear_restore_requested(&mut batch, item_id);
data.write(batch.build_all()).await?;
return Ok(TaskResult::permanent(
"Not restored: the account or its tenant is over quota.".to_string(),
));
}
match server
.email_ingest(IngestEmail {
raw_message: &bytes,
@@ -111,7 +134,8 @@ async fn restore_item(server: &Server, task: &TaskRestoreArchivedItem) -> trc::R
{
Ok(_) => {
// inbuxa: UD-9: the archived record goes, and its copy is released
undelete::records::remove(data, server.registry(), item_id, &item).await?;
undelete::records::remove_restored(data, server.registry(), item_id, &item)
.await?;
Ok(TaskResult::Success(vec![]))
}
// inbuxa: UD-10: over quota, the item stays archived and the task says why
+25 -7
View File
@@ -146,8 +146,9 @@ Task type `RestoreArchivedItem`, carrying the `accountId`, the
(observed 5). The restored item is a new item with a new id, and JMAP
`/changes` reports it as created.
- **UD-10.** A restore still counts against quota. It's refused if it would
take the account, or its tenant, past a limit. The item stays archived, and
the task records why.
take the account, or its tenant, past a limit, less what the archived copy
already counts (UD-14), so restoring a counted item is never refused just
for being counted. The item stays archived, and the task records why.
- **UD-11.** Restoring is idempotent: asking twice, or asking while a restore
task is already running, restores once.
@@ -163,10 +164,25 @@ Task type `RestoreArchivedItem`, carrying the `accountId`, the
### Storage
- **UD-14.** Archived copies don't count toward the account's or tenant's
quota while archived, as upstream (observed 7). Counting them would make deleting mail to
free space fail. They do count in server storage reporting, and ihasmail's
admin dashboard shows the archive's total size per account.
- **UD-14.** Archived copies count toward the account's and its tenant's
quota while they're kept, and come off when they go: restored (the restored
item counts as itself from then on), destroyed or expired. This changed in
27.1 (inbuxa-admin#257). Upstream doesn't count them (observed 7), and
neither did we at first, so that deleting would free space. But a member
could then fill the account, delete, and fill it again, storing the limit's
worth once more every cycle for the whole retention period, past both the
account's quota and the tenant's. Deleting never fails because of this: it
just doesn't free space until the item goes, and an account that's full
stops taking new mail. A user who needs the space back destroys the
archived items (UD-12). Items archived before 27.1 weren't counted, so
nothing is taken off when they go. A deleted account's archive stays
counted with the account (UD-15), and goes with it.
- **UD-14a.** A spam-training sample's hold on a message's blob ends with
the account's own copy: when the message's data is removed with nothing
archiving it, or when its archived copy goes other than by restore. Held
longer, the blob was stored with nobody counting it, for
`holdSamplesFor` (90 days by default). The classifier drops a sample whose
blob is gone from its replay pool; what it learned from the sample stays.
### Deleted accounts
@@ -257,7 +273,9 @@ Stalwart-facing (SPEC.md §5).
11. Retention lowered from 30 to 7 days: existing deadlines don't move
(UD-5).
12. Past `archivedUntil`: not restorable, and gone after clean-up (UD-13).
13. Archive size doesn't count toward quota (UD-14).
13. Archive size counts toward the account's and the tenant's quota while
kept, comes off when it goes, and a counted item restores in a full
account (UD-14, UD-10).
14. Deleted account: can't sign in, mail refused, name reserved, restorable
by an admin with its data intact (UD-15 to UD-17).
15. A user can't see another user's archive. A tenant admin sees only its
+39
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use crate::{
@@ -12,6 +14,8 @@ use crate::{
};
use chrono::{TimeDelta, Utc};
use jmap_client::core::response::VacationResponseSetResponse;
use registry::schema::prelude::{ObjectType, Property};
use serde_json::json;
use std::time::Instant;
pub async fn test(test: &TestServer) {
@@ -190,6 +194,41 @@ pub async fn test(test: &TestServer) {
)
.await;
// inbuxa: a bigger vacation script counts against quota, as a Sieve
// script's does
let admin = test.account("[email protected]");
let used = server
.get_used_quota_account(account.id().document_id())
.await
.unwrap();
admin
.registry_update_object(
ObjectType::Account,
account.id(),
json!({ Property::Quotas: {"maxDiskQuota": used.max(1)} }),
)
.await;
let mut request = client.build();
request
.set_vacation_response()
.update("singleton")
.text_body(Some("x".repeat(4096)));
let mut response = request
.send_single::<VacationResponseSetResponse>()
.await
.unwrap();
assert!(
response.updated("singleton").is_err(),
"a vacation script past the quota is refused"
);
admin
.registry_update_object(
ObjectType::Account,
account.id(),
json!({ Property::Quotas: {} }),
)
.await;
// Remove test data
client.vacation_response_disable().await.unwrap();
client.sieve_script_deactivate().await.unwrap();
+44 -12
View File
@@ -145,7 +145,8 @@ pub async fn test(test: &mut TestServer) {
test.wait_for_tasks().await;
assert_eq!(user.archived().await.len(), 4, "test 3");
// Acceptance test 13: archived copies don't count toward quota
// Acceptance test 13: archived copies count toward quota while kept
// (UD-14)
let used = test
.server
.get_used_quota_account(user.id().document_id())
@@ -164,17 +165,35 @@ pub async fn test(test: &mut TestServer) {
.await;
client.email_destroy(&id).await.unwrap();
test.wait_for_tasks().await;
assert_eq!(
test.server
.get_used_quota_account(user.id().document_id())
.await
.unwrap(),
used,
"test 13"
);
let item = user.archived_with_subject("Two labels").await;
let size = item["size"].as_i64().expect("test 13: size");
assert_eq!(
used_now(&test.server, user.id().document_id()).await,
used + size,
"test 13: counted while kept"
);
// ... and a counted item restores even into a full account (UD-10)
admin
.registry_update_object(
ObjectType::Account,
user.id(),
json!({ Property::Quotas: {"maxDiskQuota": used + size} }),
)
.await;
user.request_restore(item.object_id()).await;
test.wait_for_tasks().await;
assert_eq!(
used_now(&test.server, user.id().document_id()).await,
used + size,
"test 13: the restored message counts instead, once"
);
admin
.registry_update_object(
ObjectType::Account,
user.id(),
json!({ Property::Quotas: {} }),
)
.await;
let restored = user.email_with_subject("Two labels").await;
assert_eq!(
restored["mailboxIds"],
@@ -226,11 +245,18 @@ pub async fn test(test: &mut TestServer) {
test.wait_for_tasks().await;
assert_eq!(user.count_with_subject("Via JMAP").await, 1, "test 9");
// Acceptance test 10: the user destroys an archived item for good (UD-12)
// Acceptance test 10: the user destroys an archived item for good (UD-12),
// and its space comes back (UD-14)
let item = user.archived_with_subject("Via IMAP").await;
let before = used_now(&test.server, user.id().document_id()).await;
user.registry_destroy(ObjectType::ArchivedItem, [item.object_id()])
.await
.assert_destroyed(&[item.object_id()]);
assert_eq!(
used_now(&test.server, user.id().document_id()).await,
before - item["size"].as_i64().expect("test 10: size"),
"test 10: its bytes come off"
);
assert!(
user.archived()
.await
@@ -425,7 +451,8 @@ pub async fn test(test: &mut TestServer) {
);
// Acceptance test 8: a restore past the account's quota is refused, and
// the item stays archived (UD-10)
// the item stays archived (UD-10). The item's own bytes already count, so
// the account has to be over its limit for that
let used = test
.server
.get_used_quota_account(user.id().document_id())
@@ -435,7 +462,7 @@ pub async fn test(test: &mut TestServer) {
.registry_update_object(
ObjectType::Account,
user.id(),
json!({ Property::Quotas: {"maxDiskQuota": used} }),
json!({ Property::Quotas: {"maxDiskQuota": used - 1} }),
)
.await;
let item = user.archived_with_subject("Via POP3").await;
@@ -1066,3 +1093,8 @@ impl Account {
.await;
}
}
// UD-14: what an account's usage counter says now
async fn used_now(server: &common::Server, account_id: u32) -> i64 {
server.get_used_quota_account(account_id).await.unwrap()
}