Count what's kept after deletion, and check what's really stored (#500) #502
No files matched your search
@@ -152,9 +152,14 @@ impl EmailIngest for Server {
|
||||
let tenant_id = params.access_token.tenant_id();
|
||||
let mut raw_message_len = params.raw_message.len() as u64;
|
||||
let account = self.account(account_id).await.caused_by(trc::location!())?;
|
||||
self.has_available_quota(&account, raw_message_len)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
// inbuxa: UD-14: a restore checks for itself, crediting what its kept
|
||||
// copy already counts
|
||||
if params.source != IngestSource::Restore {
|
||||
self.has_available_quota(&account, raw_message_len)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
}
|
||||
let checked_len = raw_message_len;
|
||||
|
||||
// Parse message
|
||||
let mut raw_message = Cow::from(params.raw_message);
|
||||
@@ -562,6 +567,16 @@ impl EmailIngest for Server {
|
||||
false
|
||||
};
|
||||
|
||||
// inbuxa: the size stored is the message after encryption, with the
|
||||
// headers added on delivery; check that when it's more than was
|
||||
// checked at the start
|
||||
let stored_len = (message.raw_message.len() + extra_headers.len()) as u64;
|
||||
if stored_len > checked_len && params.source != IngestSource::Restore {
|
||||
self.has_available_quota(&account, stored_len)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
}
|
||||
|
||||
// Store blob
|
||||
let (blob_hash, blob_hold) = if !is_encrypted && let Some(blob_hash) = params.blob_hash {
|
||||
(blob_hash.clone(), None)
|
||||
@@ -946,6 +961,13 @@ impl EmailIngest for Server {
|
||||
}
|
||||
.to_pickled_vec();
|
||||
|
||||
// inbuxa: UD-14a: a held sample is noted, so its hold can end
|
||||
// with the account's own copy of the message
|
||||
if hold_sample {
|
||||
let linked_to = batch.last_account_id().unwrap_or(account_id);
|
||||
inbuxa_features::undelete::samples::note(batch, linked_to, &hash, until);
|
||||
}
|
||||
|
||||
let object_id = ObjectType::SpamTrainingSample.to_id();
|
||||
let item_id = SnowflakeIdGenerator::global_id().unwrap_or_default();
|
||||
batch
|
||||
|
||||
@@ -139,7 +139,8 @@ pub async fn forget(data: &Store, registry: &RegistryStore, account_id: u32) ->
|
||||
.caused_by(trc::location!())?;
|
||||
}
|
||||
for (id, item) in records::of_account(data, registry, account_id).await? {
|
||||
records::remove(data, registry, id, &item).await?;
|
||||
// Its usage goes with the account, so the counters stay as they are
|
||||
records::discard(data, registry, id, &item).await?;
|
||||
}
|
||||
data::clear_account(data, account_id).await
|
||||
}
|
||||
@@ -22,6 +22,11 @@
|
||||
//! - `a` + address: an address a kept account holds reserved (UD-16).
|
||||
//! - `g` + kind + account + document: a deleted calendar event, contact or
|
||||
//! file ([`super::groupware`]).
|
||||
//! - `s` + account + blob hash + until: a spam-training sample's hold on a
|
||||
//! message's blob ([`super::samples`], UD-14a).
|
||||
//! - `q` + account + item id: the bytes an archived item counts toward its
|
||||
//! account's and tenant's usage while it's kept (UD-14). Items archived
|
||||
//! before counting began have none, and nothing is taken off for them.
|
||||
//!
|
||||
//! Values are JSON, so they read back across versions of the fork.
|
||||
|
||||
@@ -40,6 +45,7 @@ const KIND_RESTORE: u8 = b'r';
|
||||
const KIND_CHANGE: u8 = b'c';
|
||||
const KIND_KEPT: u8 = b'k';
|
||||
const KIND_RESERVED: u8 = b'a';
|
||||
const KIND_COUNTED: u8 = b'q';
|
||||
|
||||
fn class(kind: u8, rest: &[u8]) -> ValueClass {
|
||||
let mut key = Vec::with_capacity(1 + rest.len());
|
||||
@@ -249,6 +255,46 @@ pub fn clear_blob_item(batch: &mut BatchBuilder, account_id: u32, blob_hash: &[u
|
||||
batch.clear(class(KIND_BLOB, &account_blob(account_id, blob_hash)));
|
||||
}
|
||||
|
||||
fn account_item(account_id: u32, item_id: Id) -> [u8; 12] {
|
||||
let mut out = [0u8; 12];
|
||||
out[..4].copy_from_slice(&account_id.to_be_bytes());
|
||||
out[4..].copy_from_slice(&item_id.id().to_be_bytes());
|
||||
out
|
||||
}
|
||||
|
||||
/// UD-14: an archived item's counted bytes.
|
||||
pub fn counted_class(account_id: u32, item_id: Id) -> ValueClass {
|
||||
class(KIND_COUNTED, &account_item(account_id, item_id))
|
||||
}
|
||||
|
||||
pub async fn counted(data: &Store, account_id: u32, item_id: Id) -> trc::Result<Option<u64>> {
|
||||
data.get_value::<u64>(key(KIND_COUNTED, &account_item(account_id, item_id)))
|
||||
.await
|
||||
.caused_by(trc::location!())
|
||||
}
|
||||
|
||||
/// UD-14: the bytes all an account's archived items count, for recounting
|
||||
/// its usage from scratch.
|
||||
pub async fn counted_total(data: &Store, account_id: u32) -> trc::Result<u64> {
|
||||
let mut total = 0u64;
|
||||
data.iterate(
|
||||
IterateParams::new(
|
||||
key(KIND_COUNTED, &account_item(account_id, Id::new(0))),
|
||||
key(KIND_COUNTED, &account_item(account_id, Id::new(u64::MAX))),
|
||||
)
|
||||
.ascending(),
|
||||
|_, value| {
|
||||
if let Ok(bytes) = <[u8; 8]>::try_from(value) {
|
||||
total = total.saturating_add(u64::from_be_bytes(bytes));
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
pub fn set_restore_requested(batch: &mut BatchBuilder, item_id: Id) {
|
||||
batch.set(class(KIND_RESTORE, &item_id.id().to_be_bytes()), vec![1]);
|
||||
}
|
||||
@@ -416,6 +462,8 @@ pub async fn clear_account(data: &Store, account_id: u32) -> trc::Result<()> {
|
||||
(vec![KIND_NOTE], vec![KIND_NOTE]),
|
||||
(vec![KIND_BLOB], vec![KIND_BLOB]),
|
||||
(vec![KIND_CHANGE], vec![KIND_CHANGE]),
|
||||
(vec![KIND_COUNTED], vec![KIND_COUNTED]),
|
||||
(vec![b's'], vec![b's']),
|
||||
];
|
||||
// The groupware notes, `g` + kind + account + document
|
||||
for kind in 0u8..3 {
|
||||
|
||||
@@ -133,6 +133,7 @@ pub async fn archive(
|
||||
mailboxes: note.mailboxes,
|
||||
keywords: note.keywords,
|
||||
},
|
||||
note.size,
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -238,11 +238,21 @@ pub async fn archive(
|
||||
.details("Not a groupware item"));
|
||||
}
|
||||
};
|
||||
records::insert(data, registry, &item, ¬e.extra).await
|
||||
let size = kept_size(¬e);
|
||||
records::insert(data, registry, &item, ¬e.extra, size).await
|
||||
}
|
||||
|
||||
/// The names a restored item tries, in order: the original, then with
|
||||
/// ` (restored)`, then numbered (UD-8).
|
||||
/// The bytes a kept file, event or contact counts while it's archived
|
||||
/// (UD-14): a file's size, or the event's or contact's text.
|
||||
fn kept_size(note: &Note) -> u64 {
|
||||
match ¬e.extra {
|
||||
Extra::FileNode { size, .. } if *size > 0 => *size as u64,
|
||||
_ => note.content.as_ref().map_or(0, |text| text.len() as u64),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn candidates(name: &str) -> impl Iterator<Item = String> + '_ {
|
||||
let (stem, ext) = match name.rsplit_once('.') {
|
||||
Some((stem, ext)) if !stem.is_empty() => (stem, format!(".{ext}")),
|
||||
@@ -301,6 +311,7 @@ pub async fn archive_sieve(
|
||||
retention: u64,
|
||||
) -> trc::Result<Id> {
|
||||
let archived_at = now();
|
||||
let size = content.len() as u64;
|
||||
let item = ArchivedItem::SieveScript(registry::schema::structs::ArchivedSieveScript {
|
||||
name: name.to_string(),
|
||||
created_at: UTCDateTime::from_timestamp(archived_at as i64),
|
||||
@@ -317,6 +328,7 @@ pub async fn archive_sieve(
|
||||
&Extra::SieveScript {
|
||||
name: name.to_string(),
|
||||
},
|
||||
size,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -18,4 +18,5 @@ pub mod data;
|
||||
pub mod email;
|
||||
pub mod groupware;
|
||||
pub mod records;
|
||||
pub mod samples;
|
||||
pub mod settings;
|
||||
@@ -8,17 +8,23 @@
|
||||
//! its account index, with no link to the account, so deleting an account
|
||||
//! isn't refused while it has archived items. Upstream's account removal
|
||||
//! task clears exactly these two keys.
|
||||
//!
|
||||
//! UD-14: an item counts toward its account's and its tenant's storage while
|
||||
//! it's kept, so deleting doesn't free space until the item goes. Without
|
||||
//! this, a member could fill the account, delete, and fill it again, keeping
|
||||
//! the limit's worth of data once more every time for the whole retention
|
||||
//! period.
|
||||
|
||||
use crate::undelete::data::{self, Change};
|
||||
use registry::schema::prelude::Property;
|
||||
use registry::{
|
||||
schema::{prelude::ObjectType, structs::ArchivedItem},
|
||||
types::{EnumImpl, ObjectImpl, index::IndexValue},
|
||||
types::{EnumImpl, ObjectImpl, id::ObjectId, index::IndexValue},
|
||||
};
|
||||
use store::{
|
||||
RegistryStore, SerializeInfallible, Store,
|
||||
registry::RegistryQuery,
|
||||
write::{BatchBuilder, BlobLink, BlobOp, RegistryClass, ValueClass, now},
|
||||
write::{BatchBuilder, BlobLink, BlobOp, RegistryClass, ValueClass, assert::AssertValue, now},
|
||||
};
|
||||
use trc::AddContext;
|
||||
use types::id::Id;
|
||||
@@ -39,13 +45,45 @@ fn item_class(item_id: u64) -> ValueClass {
|
||||
})
|
||||
}
|
||||
|
||||
/// The tenant an account's usage counts toward: its own, or for a deleted
|
||||
/// account still kept, the one it had (UD-15).
|
||||
pub async fn tenant_of(
|
||||
data: &Store,
|
||||
registry: &RegistryStore,
|
||||
account_id: u32,
|
||||
) -> trc::Result<Option<u32>> {
|
||||
if let Some(account) = registry
|
||||
.get(ObjectId::new(ObjectType::Account, Id::from(account_id)))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
{
|
||||
return Ok(account.inner.member_tenant_id().map(|id| id.document_id()));
|
||||
}
|
||||
Ok(data::kept_account(data, account_id)
|
||||
.await?
|
||||
.and_then(|kept| kept.member_tenant_id)
|
||||
.map(|id| Id::from(id).document_id()))
|
||||
}
|
||||
|
||||
/// Adds `bytes` to an account's usage, and to its tenant's (MT-20).
|
||||
fn count(batch: &mut BatchBuilder, account_id: u32, tenant_id: Option<u32>, bytes: i64) {
|
||||
batch
|
||||
.with_account_id(account_id)
|
||||
.add(ValueClass::Quota, bytes);
|
||||
if let Some(tenant_id) = tenant_id {
|
||||
batch.add(ValueClass::TenantQuota(tenant_id), bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes a new archived item, holding its kept copy until `archivedUntil`,
|
||||
/// with what restore needs beside it. Returns its id.
|
||||
/// with what restore needs beside it, and counting its `size` toward the
|
||||
/// account and its tenant (UD-14). Returns its id.
|
||||
pub async fn insert(
|
||||
data: &Store,
|
||||
registry: &RegistryStore,
|
||||
item: &ArchivedItem,
|
||||
extra: &data::Extra,
|
||||
size: u64,
|
||||
) -> trc::Result<Id> {
|
||||
let item_id = registry.assign_id();
|
||||
let id = Id::new(item_id);
|
||||
@@ -65,6 +103,11 @@ pub async fn insert(
|
||||
);
|
||||
data::set_extra(&mut batch, id, extra)?;
|
||||
data::set_blob_item(&mut batch, account_id, blob_hash.as_slice(), id);
|
||||
if size > 0 {
|
||||
let tenant_id = tenant_of(data, registry, account_id).await?;
|
||||
count(&mut batch, account_id, tenant_id, size as i64);
|
||||
batch.set(data::counted_class(account_id, id), size.serialize());
|
||||
}
|
||||
data::log_change(
|
||||
&mut batch,
|
||||
account_id,
|
||||
@@ -143,13 +186,57 @@ pub async fn set_deadline(
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
/// Removes an archived item and releases its kept copy: on restore (UD-9),
|
||||
/// on destroy (UD-12) and past its deadline (UD-13).
|
||||
/// Removes an archived item for good and releases its kept copy: on
|
||||
/// destroy (UD-12) and past its deadline (UD-13). Its bytes come off the
|
||||
/// account's and tenant's usage (UD-14), and a message's spam-training
|
||||
/// holds end with it (UD-14a).
|
||||
pub async fn remove(
|
||||
data: &Store,
|
||||
registry: &RegistryStore,
|
||||
id: Id,
|
||||
item: &ArchivedItem,
|
||||
) -> trc::Result<()> {
|
||||
remove_(data, registry, id, item, true).await?;
|
||||
if let ArchivedItem::Email(email) = item {
|
||||
crate::undelete::samples::release(
|
||||
data,
|
||||
email.account_id.document_id(),
|
||||
&email.blob_id.hash,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Removes an archived item that's been restored (UD-9): its bytes come
|
||||
/// off, since the restored item counts as itself from now on.
|
||||
pub async fn remove_restored(
|
||||
data: &Store,
|
||||
registry: &RegistryStore,
|
||||
id: Id,
|
||||
item: &ArchivedItem,
|
||||
) -> trc::Result<()> {
|
||||
remove_(data, registry, id, item, true).await
|
||||
}
|
||||
|
||||
/// Removes an archived item of an account being destroyed, leaving the
|
||||
/// usage counters alone: the account's goes with it, and its tenant's is
|
||||
/// settled from it.
|
||||
pub async fn discard(
|
||||
data: &Store,
|
||||
registry: &RegistryStore,
|
||||
id: Id,
|
||||
item: &ArchivedItem,
|
||||
) -> trc::Result<()> {
|
||||
remove_(data, registry, id, item, false).await
|
||||
}
|
||||
|
||||
async fn remove_(
|
||||
data: &Store,
|
||||
registry: &RegistryStore,
|
||||
id: Id,
|
||||
item: &ArchivedItem,
|
||||
uncount: bool,
|
||||
) -> trc::Result<()> {
|
||||
let account_id = item.account_id().document_id();
|
||||
let blob_hash = item.blob_id().hash.clone();
|
||||
@@ -164,6 +251,27 @@ pub async fn remove(
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
// Its counted bytes come off once: the marker must still be there, or
|
||||
// another removal already took them off
|
||||
let counted = if uncount {
|
||||
data::counted(data, account_id, id).await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(size) = counted {
|
||||
let tenant_id = tenant_of(data, registry, account_id).await?;
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch
|
||||
.assert_value(data::counted_class(account_id, id), AssertValue::Some)
|
||||
.clear(data::counted_class(account_id, id));
|
||||
count(&mut batch, account_id, tenant_id, -(size as i64));
|
||||
match data.write(batch.build_all()).await {
|
||||
Ok(_) => {}
|
||||
Err(err) if err.is_assertion_failure() => {}
|
||||
Err(err) => return Err(err.caused_by(trc::location!())),
|
||||
}
|
||||
}
|
||||
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.with_account_id(account_id).clear(BlobOp::Link {
|
||||
hash: blob_hash.clone(),
|
||||
@@ -171,6 +279,9 @@ pub async fn remove(
|
||||
until: item.archived_until().timestamp() as u64,
|
||||
},
|
||||
});
|
||||
if !uncount {
|
||||
batch.clear(data::counted_class(account_id, id));
|
||||
}
|
||||
data::clear_extra(&mut batch, id);
|
||||
data::clear_blob_item(&mut batch, account_id, blob_hash.as_slice());
|
||||
data::clear_restore_requested(&mut batch, id);
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Spam-training samples held past their message (UD-14a).
|
||||
//!
|
||||
//! The classifier holds a trained message's blob for `holdSamplesFor`
|
||||
//! (90 days by default) so it can replay it later. While the message is in
|
||||
//! the account that costs nothing: the hold and the message share one blob,
|
||||
//! counted once. Once the account's own copy is gone for good, though, the
|
||||
//! held blob was stored with nobody counting it, so a member could fill the
|
||||
//! account, train, delete and fill it again, past the account's and the
|
||||
//! tenant's limit, for the whole hold period.
|
||||
//!
|
||||
//! So a hold now ends with the account's own copy: when the message's data
|
||||
//! is removed and nothing archives it, or when its archived copy goes other
|
||||
//! than by restore. The classifier already treats a blob that's gone as a
|
||||
//! sample to drop from its replay pool, so this shrinks the pool to mail
|
||||
//! that's still kept; what was learned from the sample stays learned.
|
||||
//!
|
||||
//! Each hold is noted as `s` + account + blob hash + until (empty value), so
|
||||
//! it can be found from the message, with the link it set under that
|
||||
//! account.
|
||||
|
||||
use store::{
|
||||
IterateParams, SUBSPACE_UNDELETE, Store, ValueKey,
|
||||
write::{AnyClass, BatchBuilder, BlobLink, BlobOp, ValueClass, now},
|
||||
};
|
||||
use trc::AddContext;
|
||||
use types::blob_hash::{BLOB_HASH_LEN, BlobHash};
|
||||
|
||||
const KIND_SAMPLE: u8 = b's';
|
||||
|
||||
fn class(account_id: u32, hash: &[u8], until: u64) -> ValueClass {
|
||||
let mut key = Vec::with_capacity(1 + 4 + hash.len() + 8);
|
||||
key.push(KIND_SAMPLE);
|
||||
key.extend_from_slice(&account_id.to_be_bytes());
|
||||
key.extend_from_slice(hash);
|
||||
key.extend_from_slice(&until.to_be_bytes());
|
||||
ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_UNDELETE,
|
||||
key,
|
||||
})
|
||||
}
|
||||
|
||||
/// Notes a held sample: its blob, linked under `account_id` until `until`.
|
||||
pub fn note(batch: &mut BatchBuilder, account_id: u32, hash: &BlobHash, until: u64) {
|
||||
batch.set(class(account_id, hash.as_slice(), until), vec![]);
|
||||
}
|
||||
|
||||
/// The holds noted for a blob in an account, by their `until`.
|
||||
async fn holds(data: &Store, account_id: u32, hash: &[u8]) -> trc::Result<Vec<u64>> {
|
||||
let mut untils = Vec::new();
|
||||
data.iterate(
|
||||
IterateParams::new(
|
||||
ValueKey::from(class(account_id, hash, 0)),
|
||||
ValueKey::from(class(account_id, hash, u64::MAX)),
|
||||
)
|
||||
.ascending()
|
||||
.no_values(),
|
||||
|key, _| {
|
||||
if key.len() >= 8 {
|
||||
untils.push(u64::from_be_bytes(key[key.len() - 8..].try_into().unwrap()));
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
Ok(untils)
|
||||
}
|
||||
|
||||
/// The account's own copy of a message is gone for good: its samples'
|
||||
/// holds on the blob end with it.
|
||||
pub async fn release(data: &Store, account_id: u32, hash: &BlobHash) -> trc::Result<()> {
|
||||
let untils = holds(data, account_id, hash.as_slice()).await?;
|
||||
if untils.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.with_account_id(account_id);
|
||||
for until in untils {
|
||||
batch
|
||||
.clear(BlobOp::Link {
|
||||
hash: hash.clone(),
|
||||
to: BlobLink::Temporary { until },
|
||||
})
|
||||
.clear(class(account_id, hash.as_slice(), until));
|
||||
}
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
/// Clears notes whose hold has ended on its own, for the scheduled clean-up.
|
||||
/// Returns how many went.
|
||||
pub async fn purge_ended(data: &Store) -> trc::Result<usize> {
|
||||
let now = now();
|
||||
let mut ended = Vec::new();
|
||||
data.iterate(
|
||||
IterateParams::new(
|
||||
ValueKey::from(ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_UNDELETE,
|
||||
key: vec![KIND_SAMPLE],
|
||||
})),
|
||||
ValueKey::from(ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_UNDELETE,
|
||||
key: vec![KIND_SAMPLE, u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX],
|
||||
})),
|
||||
)
|
||||
.ascending()
|
||||
.no_values(),
|
||||
|key, _| {
|
||||
if key.len() == 1 + 4 + BLOB_HASH_LEN + 8
|
||||
&& u64::from_be_bytes(key[key.len() - 8..].try_into().unwrap()) <= now
|
||||
{
|
||||
ended.push(key.to_vec());
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
let count = ended.len();
|
||||
for chunk in ended.chunks(512) {
|
||||
let mut batch = BatchBuilder::new();
|
||||
for key in chunk {
|
||||
batch.clear(ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_UNDELETE,
|
||||
key: key.clone(),
|
||||
}));
|
||||
}
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
}
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn keys_sort_by_account_then_blob_then_until() {
|
||||
let hash = BlobHash::generate(b"x");
|
||||
let key = |account, until| match class(account, hash.as_slice(), until) {
|
||||
ValueClass::Any(any) => any.key,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
assert_eq!(key(1, 5).len(), 1 + 4 + BLOB_HASH_LEN + 8);
|
||||
assert!(key(1, 5) < key(1, 6));
|
||||
assert!(key(1, u64::MAX) < key(2, 0));
|
||||
}
|
||||
}
|
||||
@@ -263,11 +263,12 @@ impl ItipIngest for Server {
|
||||
let mut ical = itip.clone();
|
||||
itip_import_message(&mut ical)?;
|
||||
|
||||
// Validate quota
|
||||
// Validate quota: inbuxa: both the event and the notification
|
||||
// beside it are stored, each the message's size
|
||||
if self
|
||||
.has_available_quota(
|
||||
self.account(account_id).await?.as_ref(),
|
||||
itip_message.len() as u64,
|
||||
2 * itip_message.len() as u64,
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
@@ -782,7 +783,10 @@ async fn commit_itip_merge(
|
||||
}
|
||||
|
||||
// Validate quota
|
||||
let extra_bytes = (event.size as u64).saturating_sub(event_.inner.size.to_native() as u64);
|
||||
// inbuxa: the notification stored beside the update counts too
|
||||
let extra_bytes = (event.size as u64)
|
||||
.saturating_sub(event_.inner.size.to_native() as u64)
|
||||
.saturating_add(itip_size as u64);
|
||||
if extra_bytes > 0
|
||||
&& server
|
||||
.has_available_quota(server.account(account_id).await?.as_ref(), extra_bytes)
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use super::get::VacationResponseGet;
|
||||
@@ -244,6 +246,7 @@ impl VacationResponseSet for Server {
|
||||
}
|
||||
}
|
||||
|
||||
let prev_size = prev_sieve.as_ref().map_or(0, |prev| prev.inner.size as u64);
|
||||
let mut obj = ObjectIndexBuilder::new()
|
||||
.with_current_opt(prev_sieve)
|
||||
.with_changes(sieve)
|
||||
@@ -265,14 +268,33 @@ impl VacationResponseSet for Server {
|
||||
|
||||
// Create sieve script only if there are changes
|
||||
if build_script {
|
||||
let script = self.build_script(obj.changes_mut().unwrap())?;
|
||||
|
||||
// inbuxa: a bigger script counts against quota, as a Sieve
|
||||
// script's does
|
||||
let new_size = obj.changes_mut().unwrap().size as u64;
|
||||
if new_size > prev_size {
|
||||
let account = self.account(account_id).await?;
|
||||
match self
|
||||
.has_available_quota(&account, new_size - prev_size)
|
||||
.await
|
||||
{
|
||||
Ok(()) => {}
|
||||
Err(err)
|
||||
if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota))
|
||||
|| err.matches(trc::EventType::Limit(
|
||||
trc::LimitEvent::TenantQuota,
|
||||
)) =>
|
||||
{
|
||||
return Ok(set_error(response, create_id, SetError::over_quota()));
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
// Upload new blob
|
||||
let (blob_hash, blob_hold) = self
|
||||
.put_temporary_blob(
|
||||
account_id,
|
||||
&self.build_script(obj.changes_mut().unwrap())?,
|
||||
60,
|
||||
)
|
||||
.await?;
|
||||
let (blob_hash, blob_hold) =
|
||||
self.put_temporary_blob(account_id, &script, 60).await?;
|
||||
obj.changes_mut().unwrap().blob_hash = blob_hash;
|
||||
batch.clear(blob_hold);
|
||||
};
|
||||
|
||||
@@ -73,9 +73,13 @@ pub(crate) async fn restore_other(
|
||||
return Ok(Some("The kept copy is gone.".into()));
|
||||
};
|
||||
|
||||
// UD-10: a restore counts against quota like anything new
|
||||
// UD-10, UD-14: a restore counts against quota like anything new, less
|
||||
// what the kept copy already counts
|
||||
let counted = undelete::data::counted(&server.core.storage.data, account_id, item_id)
|
||||
.await?
|
||||
.unwrap_or(0);
|
||||
if server
|
||||
.has_available_quota(&account, bytes.len() as u64)
|
||||
.has_available_quota(&account, (bytes.len() as u64).saturating_sub(counted))
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
@@ -259,7 +263,8 @@ pub(crate) async fn restore_other(
|
||||
server.commit_batch(batch).await?;
|
||||
|
||||
// UD-9: the record goes once the item is back
|
||||
undelete::records::remove(&server.core.storage.data, server.registry(), item_id, item).await?;
|
||||
undelete::records::remove_restored(&server.core.storage.data, server.registry(), item_id, item)
|
||||
.await?;
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
|
||||
@@ -790,13 +790,14 @@ async fn delete_email_metadata(
|
||||
|
||||
// inbuxa: UD-1, UD-4: a message noted at deletion is archived
|
||||
let root = metadata.contents.first().and_then(|c| c.parts.first());
|
||||
inbuxa_features::undelete::email::archive(
|
||||
let blob_hash = BlobHash::from(&metadata.blob_hash);
|
||||
let archived = inbuxa_features::undelete::email::archive(
|
||||
&server.core.storage.data,
|
||||
server.registry(),
|
||||
account_id,
|
||||
document_id,
|
||||
inbuxa_features::undelete::email::Summary {
|
||||
blob_hash: BlobHash::from(&metadata.blob_hash),
|
||||
blob_hash: blob_hash.clone(),
|
||||
from: root.and_then(|part| part.from()),
|
||||
subject: root.and_then(|part| part.subject()),
|
||||
received_at: metadata.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK,
|
||||
@@ -804,6 +805,16 @@ async fn delete_email_metadata(
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
// inbuxa: UD-14a: with no archived copy, its spam-sample holds end
|
||||
if !archived {
|
||||
inbuxa_features::undelete::samples::release(
|
||||
&server.core.storage.data,
|
||||
account_id,
|
||||
&blob_hash,
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
}
|
||||
}
|
||||
None => {
|
||||
trc::event!(
|
||||
|
||||
@@ -236,6 +236,10 @@ async fn store_maintenance(
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
// inbuxa: UD-14a: and notes of spam-sample holds that have ended
|
||||
inbuxa_features::undelete::samples::purge_ended(&server.core.storage.data)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
use common::telemetry::metrics::store::MetricsStore;
|
||||
// inbuxa: MON-17, MON-38: history past its retention goes; a
|
||||
@@ -551,6 +555,10 @@ async fn recalculate_quota(server: &Server, account_id: u32) -> trc::Result<()>
|
||||
.caused_by(trc::location!())?;
|
||||
}
|
||||
|
||||
// inbuxa: UD-14: kept deleted items count too
|
||||
quota += inbuxa_features::undelete::data::counted_total(&server.core.storage.data, account_id)
|
||||
.await? as i64;
|
||||
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch
|
||||
.with_account_id(account_id)
|
||||
|
||||
@@ -95,6 +95,29 @@ async fn restore_item(server: &Server, task: &TaskRestoreArchivedItem) -> trc::R
|
||||
return Ok(TaskResult::permanent("Blob not found"));
|
||||
};
|
||||
|
||||
// inbuxa: UD-10, UD-14: a restore counts against quota like
|
||||
// anything new, less what the kept copy already counts. Ingest
|
||||
// leaves a restore's check to this.
|
||||
let counted = undelete::data::counted(data, account_id, item_id)
|
||||
.await?
|
||||
.unwrap_or(0);
|
||||
let account = server
|
||||
.account(account_id)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
if server
|
||||
.has_available_quota(&account, (bytes.len() as u64).saturating_sub(counted))
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
let mut batch = BatchBuilder::new();
|
||||
undelete::data::clear_restore_requested(&mut batch, item_id);
|
||||
data.write(batch.build_all()).await?;
|
||||
return Ok(TaskResult::permanent(
|
||||
"Not restored: the account or its tenant is over quota.".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
match server
|
||||
.email_ingest(IngestEmail {
|
||||
raw_message: &bytes,
|
||||
@@ -111,7 +134,8 @@ async fn restore_item(server: &Server, task: &TaskRestoreArchivedItem) -> trc::R
|
||||
{
|
||||
Ok(_) => {
|
||||
// inbuxa: UD-9: the archived record goes, and its copy is released
|
||||
undelete::records::remove(data, server.registry(), item_id, &item).await?;
|
||||
undelete::records::remove_restored(data, server.registry(), item_id, &item)
|
||||
.await?;
|
||||
Ok(TaskResult::Success(vec![]))
|
||||
}
|
||||
// inbuxa: UD-10: over quota, the item stays archived and the task says why
|
||||
|
||||
@@ -146,8 +146,9 @@ Task type `RestoreArchivedItem`, carrying the `accountId`, the
|
||||
(observed 5). The restored item is a new item with a new id, and JMAP
|
||||
`/changes` reports it as created.
|
||||
- **UD-10.** A restore still counts against quota. It's refused if it would
|
||||
take the account, or its tenant, past a limit. The item stays archived, and
|
||||
the task records why.
|
||||
take the account, or its tenant, past a limit, less what the archived copy
|
||||
already counts (UD-14), so restoring a counted item is never refused just
|
||||
for being counted. The item stays archived, and the task records why.
|
||||
- **UD-11.** Restoring is idempotent: asking twice, or asking while a restore
|
||||
task is already running, restores once.
|
||||
|
||||
@@ -163,10 +164,25 @@ Task type `RestoreArchivedItem`, carrying the `accountId`, the
|
||||
|
||||
### Storage
|
||||
|
||||
- **UD-14.** Archived copies don't count toward the account's or tenant's
|
||||
quota while archived, as upstream (observed 7). Counting them would make deleting mail to
|
||||
free space fail. They do count in server storage reporting, and ihasmail's
|
||||
admin dashboard shows the archive's total size per account.
|
||||
- **UD-14.** Archived copies count toward the account's and its tenant's
|
||||
quota while they're kept, and come off when they go: restored (the restored
|
||||
item counts as itself from then on), destroyed or expired. This changed in
|
||||
27.1 (inbuxa-admin#257). Upstream doesn't count them (observed 7), and
|
||||
neither did we at first, so that deleting would free space. But a member
|
||||
could then fill the account, delete, and fill it again, storing the limit's
|
||||
worth once more every cycle for the whole retention period, past both the
|
||||
account's quota and the tenant's. Deleting never fails because of this: it
|
||||
just doesn't free space until the item goes, and an account that's full
|
||||
stops taking new mail. A user who needs the space back destroys the
|
||||
archived items (UD-12). Items archived before 27.1 weren't counted, so
|
||||
nothing is taken off when they go. A deleted account's archive stays
|
||||
counted with the account (UD-15), and goes with it.
|
||||
- **UD-14a.** A spam-training sample's hold on a message's blob ends with
|
||||
the account's own copy: when the message's data is removed with nothing
|
||||
archiving it, or when its archived copy goes other than by restore. Held
|
||||
longer, the blob was stored with nobody counting it, for
|
||||
`holdSamplesFor` (90 days by default). The classifier drops a sample whose
|
||||
blob is gone from its replay pool; what it learned from the sample stays.
|
||||
|
||||
### Deleted accounts
|
||||
|
||||
@@ -257,7 +273,9 @@ Stalwart-facing (SPEC.md §5).
|
||||
11. Retention lowered from 30 to 7 days: existing deadlines don't move
|
||||
(UD-5).
|
||||
12. Past `archivedUntil`: not restorable, and gone after clean-up (UD-13).
|
||||
13. Archive size doesn't count toward quota (UD-14).
|
||||
13. Archive size counts toward the account's and the tenant's quota while
|
||||
kept, comes off when it goes, and a counted item restores in a full
|
||||
account (UD-14, UD-10).
|
||||
14. Deleted account: can't sign in, mail refused, name reserved, restorable
|
||||
by an admin with its data intact (UD-15 to UD-17).
|
||||
15. A user can't see another user's archive. A tenant admin sees only its
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use crate::{
|
||||
@@ -12,6 +14,8 @@ use crate::{
|
||||
};
|
||||
use chrono::{TimeDelta, Utc};
|
||||
use jmap_client::core::response::VacationResponseSetResponse;
|
||||
use registry::schema::prelude::{ObjectType, Property};
|
||||
use serde_json::json;
|
||||
use std::time::Instant;
|
||||
|
||||
pub async fn test(test: &TestServer) {
|
||||
@@ -190,6 +194,41 @@ pub async fn test(test: &TestServer) {
|
||||
)
|
||||
.await;
|
||||
|
||||
// inbuxa: a bigger vacation script counts against quota, as a Sieve
|
||||
// script's does
|
||||
let admin = test.account("[email protected]");
|
||||
let used = server
|
||||
.get_used_quota_account(account.id().document_id())
|
||||
.await
|
||||
.unwrap();
|
||||
admin
|
||||
.registry_update_object(
|
||||
ObjectType::Account,
|
||||
account.id(),
|
||||
json!({ Property::Quotas: {"maxDiskQuota": used.max(1)} }),
|
||||
)
|
||||
.await;
|
||||
let mut request = client.build();
|
||||
request
|
||||
.set_vacation_response()
|
||||
.update("singleton")
|
||||
.text_body(Some("x".repeat(4096)));
|
||||
let mut response = request
|
||||
.send_single::<VacationResponseSetResponse>()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
response.updated("singleton").is_err(),
|
||||
"a vacation script past the quota is refused"
|
||||
);
|
||||
admin
|
||||
.registry_update_object(
|
||||
ObjectType::Account,
|
||||
account.id(),
|
||||
json!({ Property::Quotas: {} }),
|
||||
)
|
||||
.await;
|
||||
|
||||
// Remove test data
|
||||
client.vacation_response_disable().await.unwrap();
|
||||
client.sieve_script_deactivate().await.unwrap();
|
||||
|
||||
@@ -145,7 +145,8 @@ pub async fn test(test: &mut TestServer) {
|
||||
test.wait_for_tasks().await;
|
||||
assert_eq!(user.archived().await.len(), 4, "test 3");
|
||||
|
||||
// Acceptance test 13: archived copies don't count toward quota
|
||||
// Acceptance test 13: archived copies count toward quota while kept
|
||||
// (UD-14)
|
||||
let used = test
|
||||
.server
|
||||
.get_used_quota_account(user.id().document_id())
|
||||
@@ -164,17 +165,35 @@ pub async fn test(test: &mut TestServer) {
|
||||
.await;
|
||||
client.email_destroy(&id).await.unwrap();
|
||||
test.wait_for_tasks().await;
|
||||
assert_eq!(
|
||||
test.server
|
||||
.get_used_quota_account(user.id().document_id())
|
||||
.await
|
||||
.unwrap(),
|
||||
used,
|
||||
"test 13"
|
||||
);
|
||||
let item = user.archived_with_subject("Two labels").await;
|
||||
let size = item["size"].as_i64().expect("test 13: size");
|
||||
assert_eq!(
|
||||
used_now(&test.server, user.id().document_id()).await,
|
||||
used + size,
|
||||
"test 13: counted while kept"
|
||||
);
|
||||
// ... and a counted item restores even into a full account (UD-10)
|
||||
admin
|
||||
.registry_update_object(
|
||||
ObjectType::Account,
|
||||
user.id(),
|
||||
json!({ Property::Quotas: {"maxDiskQuota": used + size} }),
|
||||
)
|
||||
.await;
|
||||
user.request_restore(item.object_id()).await;
|
||||
test.wait_for_tasks().await;
|
||||
assert_eq!(
|
||||
used_now(&test.server, user.id().document_id()).await,
|
||||
used + size,
|
||||
"test 13: the restored message counts instead, once"
|
||||
);
|
||||
admin
|
||||
.registry_update_object(
|
||||
ObjectType::Account,
|
||||
user.id(),
|
||||
json!({ Property::Quotas: {} }),
|
||||
)
|
||||
.await;
|
||||
let restored = user.email_with_subject("Two labels").await;
|
||||
assert_eq!(
|
||||
restored["mailboxIds"],
|
||||
@@ -226,11 +245,18 @@ pub async fn test(test: &mut TestServer) {
|
||||
test.wait_for_tasks().await;
|
||||
assert_eq!(user.count_with_subject("Via JMAP").await, 1, "test 9");
|
||||
|
||||
// Acceptance test 10: the user destroys an archived item for good (UD-12)
|
||||
// Acceptance test 10: the user destroys an archived item for good (UD-12),
|
||||
// and its space comes back (UD-14)
|
||||
let item = user.archived_with_subject("Via IMAP").await;
|
||||
let before = used_now(&test.server, user.id().document_id()).await;
|
||||
user.registry_destroy(ObjectType::ArchivedItem, [item.object_id()])
|
||||
.await
|
||||
.assert_destroyed(&[item.object_id()]);
|
||||
assert_eq!(
|
||||
used_now(&test.server, user.id().document_id()).await,
|
||||
before - item["size"].as_i64().expect("test 10: size"),
|
||||
"test 10: its bytes come off"
|
||||
);
|
||||
assert!(
|
||||
user.archived()
|
||||
.await
|
||||
@@ -425,7 +451,8 @@ pub async fn test(test: &mut TestServer) {
|
||||
);
|
||||
|
||||
// Acceptance test 8: a restore past the account's quota is refused, and
|
||||
// the item stays archived (UD-10)
|
||||
// the item stays archived (UD-10). The item's own bytes already count, so
|
||||
// the account has to be over its limit for that
|
||||
let used = test
|
||||
.server
|
||||
.get_used_quota_account(user.id().document_id())
|
||||
@@ -435,7 +462,7 @@ pub async fn test(test: &mut TestServer) {
|
||||
.registry_update_object(
|
||||
ObjectType::Account,
|
||||
user.id(),
|
||||
json!({ Property::Quotas: {"maxDiskQuota": used} }),
|
||||
json!({ Property::Quotas: {"maxDiskQuota": used - 1} }),
|
||||
)
|
||||
.await;
|
||||
let item = user.archived_with_subject("Via POP3").await;
|
||||
@@ -1066,3 +1093,8 @@ impl Account {
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
// UD-14: what an account's usage counter says now
|
||||
async fn used_now(server: &common::Server, account_id: u32) -> i64 {
|
||||
server.get_used_quota_account(account_id).await.unwrap()
|
||||
}
|
||||
Reference in new issue
Block a user