Tenant storage: count what's written where it's stored (#499) #507

Merged
jcoffey-dev merged 1 commits from fix/tenant-storage-accounting into main 2026-11-20 19:40:42 +00:00
12 changed files with 478 additions and 26 deletions

No files matched your search

+13 -7
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use crate::{auth::AccountTenantIds, sharing::notification::ShareNotification};
@@ -19,7 +21,7 @@ use store::{
Serialize, SerializeInfallible,
write::{
Archive, Archiver, BatchBuilder, BlobLink, BlobOp, IntoOperations, Params, SearchIndex,
ValueClass,
TENANT_OF_ACCOUNT, ValueClass,
},
};
use types::{
@@ -510,9 +512,11 @@ fn build_index(
batch.add(ValueClass::Quota, value);
if let Some(tenant_id) = tenant_id {
batch.add(ValueClass::TenantQuota(tenant_id), value);
}
// inbuxa: MT-20: the tenant is the data owner's, set at commit
batch.add(
ValueClass::TenantQuota(tenant_id.unwrap_or(TENANT_OF_ACCOUNT)),
value,
);
}
IndexValue::LogItem {
sync_collection,
@@ -731,9 +735,11 @@ fn merge_index(
let value = new_used as i64 - old_used as i64;
batch.add(ValueClass::Quota, value);
if let Some(tenant_id) = tenant_id {
batch.add(ValueClass::TenantQuota(tenant_id), value);
}
// inbuxa: MT-20: the tenant is the data owner's, set at commit
batch.add(
ValueClass::TenantQuota(tenant_id.unwrap_or(TENANT_OF_ACCOUNT)),
value,
);
}
(
IndexValue::LogItem {
+37 -1
View File
@@ -16,7 +16,10 @@ use registry::{
types::EnumImpl,
};
use std::sync::Arc;
use store::{ValueKey, write::ValueClass};
use store::{
ValueKey,
write::{BatchBuilder, ValueClass},
};
use trc::AddContext;
// inbuxa: the disk quota of the account a request writes into, with the bytes
@@ -52,6 +55,39 @@ impl Server {
.add_context(|err| err.caused_by(trc::location!()).account_id(account_id))
}
// inbuxa: MT-20: a batch's tenant storage changes go to the tenant of the
// account whose data changed, not the writer's: a sharee or delegate in
// another tenant, or a server administrator with none, would otherwise
// move the wrong counter. A deleted account still kept (UD-15) is its
// kept record's tenant's. Run before every write that carries them.
pub async fn resolve_tenant_quota(&self, batch: &mut BatchBuilder) -> trc::Result<()> {
let accounts = batch.tenant_quota_accounts();
if accounts.is_empty() {
return Ok(());
}
let mut tenants = Vec::with_capacity(accounts.len());
for account_id in accounts {
let tenant_id = match self.try_account(account_id).await? {
Some(account) => account.id_tenant,
None => inbuxa_features::undelete::data::kept_account(
&self.core.storage.data,
account_id,
)
.await?
.and_then(|kept| kept.member_tenant_id)
.map(|tenant_id| tenant_id as u32),
};
tenants.push((account_id, tenant_id));
}
batch.retarget_tenant_quota(|account_id| {
tenants
.iter()
.find(|(id, _)| *id == account_id)
.and_then(|(_, tenant_id)| *tenant_id)
});
Ok(())
}
// inbuxa: MT-20: storage used by all a tenant's members together
pub async fn get_used_quota_tenant(&self, tenant_id: u32) -> trc::Result<i64> {
inbuxa_features::tenancy::quota::used(&self.core.storage.data, tenant_id).await
+4
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use crate::{Server, ipc::PushNotification};
@@ -20,6 +22,8 @@ use utils::{map::bitmap::Bitmap, snowflake::SnowflakeIdGenerator};
impl Server {
pub async fn commit_batch(&self, mut builder: BatchBuilder) -> trc::Result<AssignedIds> {
let mut assigned_ids = AssignedIds::default();
// inbuxa: MT-20: tenant storage changes follow the data's owner
self.resolve_tenant_quota(&mut builder).await?;
let mut commit_points = builder.commit_points();
for commit_point in commit_points.iter() {
+17 -1
View File
@@ -559,6 +559,14 @@ async fn copy_event(
name: new_name.to_string(),
parent_id: to_calendar_id,
}];
// inbuxa: into another account, it counts against that account's
// quota and its tenant's, as a whole calendar or address book does
server
.has_available_quota(
server.account(to_account_id).await?.as_ref(),
new_event.size() as u64,
)
.await?;
let to_document_id = server
.store()
.assign_document_ids(to_account_id, Collection::CalendarEvent, 1)
@@ -750,6 +758,14 @@ async fn move_event(
)
.caused_by(trc::location!())?;
// inbuxa: into another account, it counts against that account's
// quota and its tenant's, as a whole calendar or address book does
server
.has_available_quota(
server.account(to_account_id).await?.as_ref(),
new_event.size() as u64,
)
.await?;
let to_document_id = server
.store()
.assign_document_ids(to_account_id, Collection::CalendarEvent, 1)
@@ -1086,7 +1102,7 @@ async fn copy_container(
.await
.caused_by(trc::location!())?;
new_event.names = vec![new_name];
required_space += new_event.size as u64;
required_space += new_event.size() as u64;
new_event
.insert(
access_token.account_tenant_ids(),
+17 -1
View File
@@ -530,6 +530,14 @@ async fn copy_card(
name: new_name.to_string(),
parent_id: to_addressbook_id,
}];
// inbuxa: into another account, it counts against that account's
// quota and its tenant's, as a whole calendar or address book does
server
.has_available_quota(
server.account(to_account_id).await?.as_ref(),
new_card.size() as u64,
)
.await?;
let to_document_id = server
.store()
.assign_document_ids(to_account_id, Collection::ContactCard, 1)
@@ -692,6 +700,14 @@ async fn move_card(
)
.caused_by(trc::location!())?;
// inbuxa: into another account, it counts against that account's
// quota and its tenant's, as a whole calendar or address book does
server
.has_available_quota(
server.account(to_account_id).await?.as_ref(),
new_card.size() as u64,
)
.await?;
let to_document_id = server
.store()
.assign_document_ids(to_account_id, Collection::ContactCard, 1)
@@ -1007,7 +1023,7 @@ async fn copy_container(
.await
.caused_by(trc::location!())?;
new_card.names = vec![new_name];
required_space += new_card.size as u64;
required_space += new_card.size() as u64;
new_card
.insert(
access_token.account_tenant_ids(),
+10 -3
View File
@@ -214,7 +214,8 @@ pub async fn used(data: &Store, tenant_id: u32) -> trc::Result<i64> {
}
/// Recomputes a tenant's storage usage from its members' own usage
/// (MT-21). Safe on a live server: the stored figure is corrected by the
/// (MT-21), deleted members still kept (UD-15) included: their data stays
/// until they're destroyed, which takes it off then. Safe on a live server: the stored figure is corrected by the
/// difference, so deliveries counted while this runs aren't lost. One that
/// lands between reading the members and reading the total can leave the
/// figure off by that message until the next run.
@@ -228,11 +229,17 @@ pub async fn recalculate(
.await
.caused_by(trc::location!())?;
let kept = crate::undelete::data::kept_accounts(data)
.await?
.into_iter()
.filter(|(_, kept)| kept.member_tenant_id == Some(tenant_id as u64))
.map(|(account_id, _)| account_id);
let mut total = 0i64;
for member in members {
for member in members.into_iter().map(|id| id.document_id()).chain(kept) {
total += data
.get_counter(ValueKey {
account_id: member.document_id(),
account_id: member,
collection: 0,
document_id: 0,
class: ValueClass::Quota,
@@ -57,12 +57,12 @@ async fn destroy_account(server: &Server, task: &TaskDestroyAccount) -> trc::Res
// inbuxa: LH-8, LH-10: a kept account waits for its time, and a held one
// for its release; "destroy now" clears the kept record first
if let Some(kept) =
inbuxa_features::undelete::data::kept_account(&server.core.storage.data, account_id).await?
{
let kept = inbuxa_features::undelete::data::kept_account(&server.core.storage.data, account_id)
.await?;
if let Some(kept) = &kept {
let now = store::write::now();
let held = inbuxa_features::hold::is_held_until(kept.kept_until)
|| server.is_kept_held(account_id, &kept).await?;
|| server.is_kept_held(account_id, kept).await?;
if held || kept.kept_until > now {
let retry = if held { now + 86_400 } else { kept.kept_until };
return Ok(TaskResult::deferred(
@@ -179,6 +179,21 @@ async fn destroy_account(server: &Server, task: &TaskDestroyAccount) -> trc::Res
// Unlink all accounts's blobs
destroy_account_blobs(server, account_id).await?;
// inbuxa: MT-20: what the account stored leaves its tenant's usage with
// it. Read before the data goes and taken off after, so a retry after a
// failed destroy (the counter still there) or a finished one (gone, so
// zero) takes it off once.
let tenant_id = match &kept {
Some(kept) => kept.member_tenant_id.map(|id| id as u32),
None => server
.registry()
.object::<registry::schema::structs::Domain>(task.account_domain_id)
.await?
.and_then(|domain| domain.member_tenant_id)
.map(|id| id.document_id()),
};
let used = server.get_used_quota_account(account_id).await?.max(0);
// Destroy account data
server
.store()
@@ -186,6 +201,20 @@ async fn destroy_account(server: &Server, task: &TaskDestroyAccount) -> trc::Res
.await
.caused_by(trc::location!())?;
if let Some(tenant_id) = tenant_id
&& used > 0
{
let mut batch = BatchBuilder::new();
batch.add(ValueClass::TenantQuota(tenant_id), -used);
server
.core
.storage
.data
.write(batch.build_all())
.await
.caused_by(trc::location!())?;
}
Ok(TaskResult::Success(vec![]))
}
+21 -3
View File
@@ -520,15 +520,33 @@ impl Store {
}
pub async fn danger_destroy_account(&self, account_id: u32) -> trc::Result<()> {
for subspace in [SUBSPACE_LOGS, SUBSPACE_INDEXES, SUBSPACE_COUNTER] {
// inbuxa: MT-20: a tenant's usage counter sits beside the account
// with the same number, [id][u8::MAX - 1], and accounts and tenants
// are numbered apart, so clearing the account's counters skips it
let tenant_quota = KeySerializer::new(U32_LEN + 1)
.write(account_id)
.write(u8::MAX - 1)
.finalize();
let after_tenant_quota = KeySerializer::new(U32_LEN + 1)
.write(account_id)
.write(u8::MAX)
.finalize();
let start = KeySerializer::new(U32_LEN).write(account_id).finalize();
let end = KeySerializer::new(U32_LEN).write(account_id + 1).finalize();
for (subspace, from, to) in [
(SUBSPACE_LOGS, &start, &end),
(SUBSPACE_INDEXES, &start, &end),
(SUBSPACE_COUNTER, &start, &tenant_quota),
(SUBSPACE_COUNTER, &after_tenant_quota, &end),
] {
self.delete_range(
AnyKey {
subspace,
key: KeySerializer::new(U32_LEN).write(account_id).finalize(),
key: from.as_slice(),
},
AnyKey {
subspace,
key: KeySerializer::new(U32_LEN).write(account_id + 1).finalize(),
key: to.as_slice(),
},
)
.await
+147 -2
View File
@@ -7,8 +7,8 @@
*/
use super::{
Batch, BatchBuilder, ChangedCollection, IntoOperations, Operation, ValueClass, ValueOp,
assert::ToAssertValue, log::VanishedItem,
Batch, BatchBuilder, ChangedCollection, IntoOperations, Operation, TENANT_OF_ACCOUNT,
ValueClass, ValueOp, assert::ToAssertValue, log::VanishedItem,
};
use crate::{
SerializeInfallible, U32_LEN,
@@ -26,6 +26,61 @@ use types::{
};
use utils::{map::vec_map::VecMap, snowflake::SnowflakeIdGenerator};
/// inbuxa: MT-20: sets each tenant storage change's tenant from the account
/// it's under; one with no tenant becomes a repeat of its account switch,
/// which writes nothing.
fn retarget_tenant_quota(ops: &mut [Operation], tenant_of: impl Fn(u32) -> Option<u32>) {
let mut current = None;
for op in ops.iter_mut() {
let account_id = match op {
Operation::AccountId { account_id } => {
current = Some(*account_id);
continue;
}
Operation::Value {
class: ValueClass::TenantQuota(_),
..
} => match current {
Some(account_id) => account_id,
None => continue,
},
_ => continue,
};
match tenant_of(account_id) {
Some(tenant_id) => {
if let Operation::Value {
class: ValueClass::TenantQuota(tenant),
..
} = op
{
*tenant = tenant_id;
}
}
None => *op = Operation::AccountId { account_id },
}
}
}
/// inbuxa: MT-20: a tenant storage change never resolved is dropped rather
/// than written under the placeholder.
fn drop_unresolved_tenant_quota(ops: &mut [Operation]) {
let mut current = None;
for op in ops.iter_mut() {
match op {
Operation::AccountId { account_id } => current = Some(*account_id),
Operation::Value {
class: ValueClass::TenantQuota(TENANT_OF_ACCOUNT),
..
} => {
if let Some(account_id) = current {
*op = Operation::AccountId { account_id };
}
}
_ => {}
}
}
}
impl BatchBuilder {
pub fn new() -> Self {
Self {
@@ -436,7 +491,42 @@ impl BatchBuilder {
}
}
/// inbuxa: MT-20: the accounts whose data this batch's tenant storage
/// changes are for, so their tenants can be looked up.
pub fn tenant_quota_accounts(&self) -> Vec<u32> {
let mut accounts = Vec::new();
let mut current = None;
for op in &self.ops {
match op {
Operation::AccountId { account_id } => current = Some(*account_id),
Operation::Value {
class: ValueClass::TenantQuota(_),
..
} => {
if let Some(account_id) = current
&& !accounts.contains(&account_id)
{
accounts.push(account_id);
}
}
_ => {}
}
}
accounts
}
/// inbuxa: MT-20: points each tenant storage change at the tenant of the
/// account whose data it is, whoever made the change, so an account's
/// usage and its tenant's always move together. A change for an account
/// with no tenant is dropped.
pub fn retarget_tenant_quota(&mut self, tenant_of: impl Fn(u32) -> Option<u32>) {
retarget_tenant_quota(&mut self.ops, tenant_of);
}
pub fn build_one(&mut self, commit_point: CommitPoint) -> Batch<'_> {
drop_unresolved_tenant_quota(
&mut self.ops[commit_point.offset_start..commit_point.offset_end],
);
Batch {
changes: &self.changed_collections,
ops: &mut self.ops[commit_point.offset_start..commit_point.offset_end],
@@ -445,6 +535,7 @@ impl BatchBuilder {
pub fn build_all(&mut self) -> Batch<'_> {
self.serialize_changes();
drop_unresolved_tenant_quota(&mut self.ops);
Batch {
changes: &self.changed_collections,
ops: self.ops.as_mut_slice(),
@@ -556,3 +647,57 @@ impl Default for BatchBuilder {
Self::new()
}
}
#[cfg(test)]
mod tenant_quota_tests {
use super::*;
fn tenant_changes(batch: &mut BatchBuilder) -> Vec<(u32, i64)> {
let mut out = Vec::new();
for op in batch.build_all().ops.iter() {
if let Operation::Value {
class: ValueClass::TenantQuota(tenant),
op: ValueOp::AtomicAdd(by),
} = op
{
out.push((*tenant, *by));
}
}
out
}
#[test]
fn tenant_changes_follow_the_owner() {
let mut batch = BatchBuilder::new();
// A writer in tenant 5 into account 1 (tenant 7); a writer with no
// tenant into account 2 (tenant 8); account 3 has none
batch
.with_account_id(1)
.add(ValueClass::Quota, 10)
.add(ValueClass::TenantQuota(5), 10)
.with_account_id(2)
.add(ValueClass::Quota, -4)
.add(ValueClass::TenantQuota(TENANT_OF_ACCOUNT), -4)
.with_account_id(3)
.add(ValueClass::Quota, 6)
.add(ValueClass::TenantQuota(5), 6);
assert_eq!(batch.tenant_quota_accounts(), vec![1, 2, 3]);
batch.retarget_tenant_quota(|account| match account {
1 => Some(7),
2 => Some(8),
_ => None,
});
assert_eq!(tenant_changes(&mut batch), vec![(7, 10), (8, -4)]);
}
#[test]
fn an_unresolved_change_is_dropped() {
let mut batch = BatchBuilder::new();
batch
.with_account_id(1)
.add(ValueClass::Quota, 10)
.add(ValueClass::TenantQuota(TENANT_OF_ACCOUNT), 10)
.add(ValueClass::TenantQuota(4), 10);
assert_eq!(tenant_changes(&mut batch), vec![(4, 10)]);
}
}
+6
View File
@@ -205,6 +205,12 @@ pub enum ValueClass {
NodeId(u16),
}
/// inbuxa: MT-20: a tenant storage change whose tenant isn't known yet. The
/// change belongs to the tenant of the account whose data it is, which
/// `BatchBuilder::retarget_tenant_quota` fills in before the write; one
/// still unresolved when the batch is built is dropped.
pub const TENANT_OF_ACCOUNT: u32 = u32::MAX;
#[derive(Debug, PartialEq, Clone, Eq, Hash)]
pub enum IndexPropertyClass {
Hash { property: u8, hash: CheekyHash },
+87 -3
View File
@@ -1264,13 +1264,95 @@ pub async fn test(test: &mut TestServer) {
.unwrap()
.clear();
// MT-21: a deleted member still kept holds its data, so a recount keeps
// counting it; destroying it for good takes its usage off (MT-20)
let d1 = d_users[0].id();
let d1_used = test
.server
.get_used_quota_account(d1.document_id())
.await
.unwrap();
assert!(d1_used > 0, "MT-20: d1 holds a message");
admin
.registry_update_setting(
structs::DataRetention {
archive_deleted_accounts_for: Some(
registry::types::duration::Duration::from_millis(86_400_000),
),
..Default::default()
},
&[Property::ArchiveDeletedAccountsFor],
)
.await;
admin
.registry_destroy(ObjectType::Account, [d1])
.await
.assert_destroyed(&[d1]);
test.wait_for_tasks_skip_not_due().await;
assert_eq!(
admin.tenant_usage(d_id).await,
two,
"MT-20: kept, still counted"
);
admin
.registry_create_object(Task::TenantMaintenance(TaskTenantMaintenance {
tenant_id: d_id,
maintenance_type: TaskTenantMaintenanceType::RecalculateQuota,
status: TaskStatus::now(),
}))
.await;
test.wait_for_tasks_skip_not_due().await;
assert_eq!(
admin.tenant_usage(d_id).await,
two,
"MT-21: a recount keeps a kept member"
);
let response = admin
.jmap_request(
&["urn:ietf:params:jmap:core", "urn:inbuxa:jmap"],
json!([["inbuxa:DeletedAccount/set", {
"accountId": admin.id_string(), "destroy": [d1.to_string()]
}, "0"]]),
)
.await
.to_string();
assert!(response.contains("destroyed"), "destroy now: {response}");
test.wait_for_tasks_skip_not_due().await;
assert_eq!(
admin.tenant_usage(d_id).await,
two - d1_used,
"MT-20: a destroyed member's usage leaves its tenant"
);
admin
.registry_update_setting(
structs::DataRetention::default(),
&[Property::ArchiveDeletedAccountsFor],
)
.await;
// Clean up, leaving the server as the other tests expect it
for account in [
let d_usage = admin.tenant_usage(d_id).await;
let cleanup = [
t_admin, t_user, t_group, u_admin, u_user, u_group, p_admin, p_user, s_user, d_admin,
] {
];
// MT-20: accounts and tenants are numbered apart, so one of these has
// tenant D's number, and destroying it leaves D's usage alone
assert!(
cleanup
.iter()
.any(|account| account.id().document_id() == d_id.document_id()),
"MT-20: no account shares tenant D's number"
);
for account in cleanup {
admin.destroy_account(account).await;
}
for name in ["d1", "d2", "d3"] {
test.wait_for_tasks().await;
assert_eq!(
admin.tenant_usage(d_id).await,
d_usage,
"MT-20: destroying another account changed tenant D's usage"
);
for name in ["d2", "d3"] {
let id = admin.account_id_by_name(name).await;
admin
.registry_destroy(ObjectType::Account, [id])
@@ -1282,6 +1364,8 @@ pub async fn test(test: &mut TestServer) {
.await
.assert_destroyed(&[q1, q2]);
test.wait_for_tasks().await;
// MT-20: with every member destroyed, nothing is left counted
assert_eq!(admin.tenant_usage(d_id).await, 0, "MT-20: all destroyed");
for domain in admin
.registry_query_ids(
ObjectType::Domain,
+86 -1
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use crate::utils::{
@@ -12,7 +14,15 @@ use ahash::AHashSet;
use dav_proto::Depth;
use groupware::DavResourceName;
use hyper::StatusCode;
use registry::schema::structs::Action;
use registry::{
schema::{
enums::StorageQuota,
prelude::{ObjectType, Property},
structs::Action,
},
types::EnumImpl,
};
use serde_json::json;
pub async fn test(test: &TestServer, assisted_discovery: bool) {
let admin = test.account("[email protected]");
@@ -796,6 +806,81 @@ pub async fn test(test: &TestServer, assisted_discovery: bool) {
.with_status(StatusCode::NO_CONTENT);
}
}
// inbuxa: Test 20: an event or card copied or moved one at a time into
// another account counts against that account's quota (and its
// tenant's), as a whole calendar or address book does
if resource_type != DavResourceName::File {
let support = test.account("[email protected]").id();
let used = test
.server
.get_used_quota_account(support.document_id())
.await
.unwrap()
.max(0);
admin
.registry_update_object(
ObjectType::Account,
support,
json!({
Property::Quotas: { StorageQuota::MaxDiskQuota.as_str(): used + 2048 }
}),
)
.await;
let src = format!("{user_base_path}/quota-src/");
let dst = format!("{group_base_path}/quota-dst/");
for path in [&src, &dst] {
client
.mkcol("MKCOL", path, [], [])
.await
.with_status(StatusCode::CREATED);
}
let (mut copied, mut refused) = (Vec::new(), false);
for i in 0..200 {
let item = format!("{src}item{i}");
client
.request_with_headers("PUT", &item, [], &resource_type.generate())
.await
.with_status(StatusCode::CREATED);
// An item moved away before it's indexed can leave its
// search entries behind; that race isn't what this checks
test.wait_for_tasks().await;
let method = if i % 2 == 0 { "COPY" } else { "MOVE" };
let to = format!("{dst}item{i}");
let response = client
.request_with_headers(method, &item, [("destination", to.as_str())], "")
.await;
match response.status {
StatusCode::CREATED => copied.push(to),
StatusCode::PRECONDITION_FAILED | StatusCode::INSUFFICIENT_STORAGE => {
refused = true;
break;
}
status => panic!("Test 20: {method} gave {status:?}"),
}
}
assert!(
refused && !copied.is_empty(),
"Test 20: {} copied into a 2 KiB quota and none refused",
copied.len()
);
// The copies are indexed in the background; let that finish
// before they go, or the index outlives them
test.wait_for_tasks().await;
for path in [&src, &dst] {
client
.request("DELETE", path, "")
.await
.with_status(StatusCode::NO_CONTENT);
}
admin
.registry_update_object(
ObjectType::Account,
support,
json!({ Property::Quotas: {} }),
)
.await;
}
}
client.delete_default_containers().await;