From 3b6b9b699fe289b60421b12fc16e8e2d48b74ee1 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 20 Nov 2026 07:27:51 -0700 Subject: [PATCH] Keep a tenant's accounts' disk quotas within its own, together (inbuxa-admin#257) #494 stopped any one account's disk quota going past its tenant's, but not several adding up to more: a 5 GB tenant could still give four accounts 5 GB each. BulTester spotted it on the issue. - x:Account/set (JMAP and SCIM) now refuses a disk quota that doesn't fit in what the tenant has left to give: its maxDiskQuota less the quotas of its other accounts, users and groups alike. The refusal names the limit, what's already given out and what's left. Accounts with no quota are always allowed and share what's left, which JMAP, IMAP and DAV already report. Only a write that grows the total is checked (a new quota, a raised one, an account joining the tenant), so lowering or clearing one always saves, even in a tenant already over a lowered limit (MT-18). The server administrator is held to it too. - Restoring a deleted account is checked as well. It skipped the check before, so a tenant could delete an account, give its space away and restore it. The refusal tells the restorer to lower other quotas first, since a kept account's quota can't be edited. - Moving a domain into a tenant counts the disk quotas of the accounts it brings (Refusal::DiskQuota, overQuota). - A write that could grow the total takes a per-tenant lock in the in-memory store (so across a cluster) before its checks and releases it once saved; one left behind expires after 30 seconds. Without it, two writes at the same moment could each fit the space left and together pass it. A write that can't get the lock in five seconds is refused with forbidden, "try again". Checked live against a debug build (5 GB tenant): four 5 GB accounts, one created; a batch of two 2 GB with 3 GB left, one created; raising past what's left refused, lowering saved; no-quota account created; the server admin refused too; 8 parallel 3 GB creates, 1 created; delete, give the space away, restore refused; a domain whose account has 2 GB moved into 1 GB left refused; the tenant admin can't raise its own tenant's limit. tenant_tests gains the sum and lowering cases; tenant, undelete, scim and legal_hold suites pass. --- crates/features/src/tenancy/domain_move.rs | 32 ++- crates/features/src/tenancy/quota.rs | 50 ++++- crates/features/src/tenancy/writes.rs | 242 +++++++++++++++++---- crates/jmap/src/inbuxa/deleted_account.rs | 42 +++- crates/jmap/src/inbuxa/mod.rs | 1 + crates/jmap/src/inbuxa/tenant_disk_lock.rs | 59 +++++ crates/jmap/src/registry/set.rs | 32 ++- tests/src/system/tenant.rs | 18 +- 8 files changed, 429 insertions(+), 47 deletions(-) create mode 100644 crates/jmap/src/inbuxa/tenant_disk_lock.rs diff --git a/crates/features/src/tenancy/domain_move.rs b/crates/features/src/tenancy/domain_move.rs index 3069962f..a1d99ffa 100644 --- a/crates/features/src/tenancy/domain_move.rs +++ b/crates/features/src/tenancy/domain_move.rs @@ -8,7 +8,8 @@ //! //! A domain moves into a tenant only from no tenant, and out of one only back //! to no tenant. Moving in, its principals (accounts, groups, mailing lists) -//! and DKIM keys move with it, after the tenant's limits are checked. Moving +//! and DKIM keys move with it, after the tenant's limits are checked, its +//! accounts' disk quotas included (inbuxa-admin#257). Moving //! out is refused while any principal on it is in the tenant; with none, its //! DKIM keys move out with it. A principal on it in a third tenant blocks //! either move, and so does any link the move would carry across a tenant @@ -44,6 +45,16 @@ pub enum Refusal { ForeignLink(ObjectId), /// The destination tenant's limit would be crossed. Limit(LimitReached), + /// The accounts moving in have more disk quota between them than the + /// destination has left to give (inbuxa-admin#257). + DiskQuota { + /// Already given to the tenant's accounts. + allocated: u64, + /// The moving accounts' quotas together. + adding: u64, + /// The tenant's maxDiskQuota. + limit: u64, + }, } /// A move that passed every check, ready to apply once the domain itself is @@ -88,6 +99,7 @@ pub async fn plan( // Sort out what links to the domain let mut carried = Vec::new(); let mut adding: AHashMap = AHashMap::new(); + let mut adding_disk = 0u64; let mut blocking = Vec::new(); let mut seen = AHashSet::new(); for referrer in registry.linked_objects(domain).await? { @@ -117,6 +129,9 @@ pub async fn plan( if let Some(quota) = quota::limit_for_id(referrer, kind) { *adding.entry(quota).or_default() += 1; } + if let Some(disk) = quota::account_disk_quota(&object.inner) { + adding_disk = adding_disk.saturating_add(disk); + } carried.push(referrer); } } else if moves_with_domain(object_type) && object_type != ObjectType::DkimSignature { @@ -168,6 +183,21 @@ pub async fn plan( return Ok(Err(Refusal::Limit(reached))); } } + + // inbuxa-admin#257: and the disk space its accounts were given + if adding_disk > 0 + && let Some(limit) = limit(TenantStorageQuota::MaxDiskQuota).filter(|limit| *limit > 0) + { + let allocated = + quota::allocated_disk(registry, tenant_id.document_id(), &AHashSet::new()).await?; + if allocated.saturating_add(adding_disk) > limit { + return Ok(Err(Refusal::DiskQuota { + allocated, + adding: adding_disk, + limit, + })); + } + } } Ok(Ok(Move { to, carried })) diff --git a/crates/features/src/tenancy/quota.rs b/crates/features/src/tenancy/quota.rs index 263bc43e..8c982143 100644 --- a/crates/features/src/tenancy/quota.rs +++ b/crates/features/src/tenancy/quota.rs @@ -5,11 +5,13 @@ */ //! Tenant limits: how many of each kind of object a tenant may hold -//! (MT-17, MT-18) and how much storage its members use (MT-20, MT-21). +//! (MT-17, MT-18), how much storage its members use (MT-20, MT-21), and how +//! much disk space its accounts are given (inbuxa-admin#257). +use ahash::AHashSet; use registry::{ schema::{ - enums::TenantStorageQuota, + enums::{StorageQuota, TenantStorageQuota}, prelude::{ObjectInner, ObjectType, Property}, structs::Account, }, @@ -98,6 +100,50 @@ pub async fn count( .caused_by(trc::location!()) } +/// An account's own disk quota, `None` when it has none (0 is none too). +/// Users and groups both have one. +pub fn account_disk_quota(object: &ObjectInner) -> Option { + let quotas = match object { + ObjectInner::Account(Account::User(obj)) => &obj.quotas, + ObjectInner::Account(Account::Group(obj)) => &obj.quotas, + _ => return None, + }; + quotas + .get(&StorageQuota::MaxDiskQuota) + .copied() + .filter(|quota| *quota > 0) +} + +/// The disk space a tenant has given its accounts: the sum of their own +/// disk quotas, leaving out `except` (inbuxa-admin#257). Accounts with no +/// quota share what's left and add nothing. +pub async fn allocated_disk( + registry: &RegistryStore, + tenant_id: u32, + except: &AHashSet, +) -> trc::Result { + let members = registry + .query::>(RegistryQuery::new(ObjectType::Account).with_tenant(Some(tenant_id))) + .await + .caused_by(trc::location!())?; + + let mut total = 0u64; + for member in members { + if except.contains(&member) { + continue; + } + if let Some(object) = registry + .get(ObjectId::new(ObjectType::Account, member)) + .await + .caused_by(trc::location!())? + && let Some(quota) = account_disk_quota(&object.inner) + { + total = total.saturating_add(quota); + } + } + Ok(total) +} + /// A count limit that adding objects would cross. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct LimitReached { diff --git a/crates/features/src/tenancy/writes.rs b/crates/features/src/tenancy/writes.rs index 4ecbafd9..a646a769 100644 --- a/crates/features/src/tenancy/writes.rs +++ b/crates/features/src/tenancy/writes.rs @@ -13,19 +13,22 @@ //! - MT-17: creating an object, or moving a domain in, stays within the //! tenant's count limits, and the server emits `limit.tenant-quota` when //! it doesn't. -//! - An account's disk quota fits within its tenant's (inbuxa-admin#257), -//! checked by `check_disk_quota` on JMAP and SCIM writes. +//! - A tenant's accounts' disk quotas add up to no more than the tenant's +//! (inbuxa-admin#257), checked by `check_disk_quota` on JMAP and SCIM +//! writes and on a restore, and by the domain-move plan. A write that +//! could grow the total holds the tenant's disk-quota lock +//! (`disk_quota_tenant`) from the check until it's saved. use crate::tenancy::{ domain_move::{self, Move, Refusal}, links, - quota::{self, LimitReached}, + quota::{self, LimitReached, account_disk_quota}, }; use ahash::AHashSet; use jmap_proto::error::set::{SetError, SetErrorType}; use registry::{ schema::{ - enums::{StorageQuota, TenantStorageQuota}, + enums::TenantStorageQuota, prelude::{Object, ObjectInner, Property}, structs::{Account, DkimManagement, Domain, Tenant}, }, @@ -65,19 +68,6 @@ pub async fn default_tenant(registry: &RegistryStore, object: &mut Object) -> tr Ok(()) } -/// An account's own disk quota, `None` when it has none (0 is none too). -fn account_disk_quota(object: &ObjectInner) -> Option { - let quotas = match object { - ObjectInner::Account(Account::User(obj)) => &obj.quotas, - ObjectInner::Account(Account::Group(obj)) => &obj.quotas, - _ => return None, - }; - quotas - .get(&StorageQuota::MaxDiskQuota) - .copied() - .filter(|quota| *quota > 0) -} - /// A tenant's count limit, `None` when it has none. async fn limits( registry: &RegistryStore, @@ -161,31 +151,64 @@ pub async fn check( Ok(Ok(after)) } -/// inbuxa-admin#257: an account's disk quota fits within its tenant's, so no -/// member gets more space than the whole tenant has. Run on a JMAP or SCIM -/// write, after `check`. Checked when the quota is set or changed, or the -/// account changes tenant; an account already over a lowered tenant limit -/// keeps its quota until someone changes it, like the count limits (MT-18). -/// No quota at all is fine: the account shares the tenant's space. Not run -/// for accounts a directory creates at sign-in or for a restored account, -/// which would otherwise be refused for a setting nobody can edit first. +/// The tenant whose allocated disk space this write could grow, if any +/// (inbuxa-admin#257): an account given a disk quota, or a bigger one, or +/// joining a tenant with one; or a domain moving into a tenant, which brings +/// its accounts. Such a write takes the tenant's disk-quota lock before its +/// checks and keeps it until it's saved, so two writes at once can't each +/// fit the space left and together pass it. Lowering or clearing a quota, +/// or any other change, needs no lock. +pub fn disk_quota_tenant(old: Option<&Object>, new: &Object) -> Option { + let tenant = new.inner.member_tenant_id()?; + let moved_in = old.is_none_or(|old| old.inner.member_tenant_id() != Some(tenant)); + match &new.inner { + ObjectInner::Account(_) => { + let quota = account_disk_quota(&new.inner)?; + let grows = moved_in + || old.is_some_and(|old| { + account_disk_quota(&old.inner).is_none_or(|before| quota > before) + }); + grows.then_some(tenant) + } + ObjectInner::Domain(_) if old.is_some() && moved_in => Some(tenant), + _ => None, + } +} + +/// inbuxa-admin#257: a tenant's accounts' disk quotas add up to no more than +/// the tenant's own, so it can't give out more space than it has, in one +/// account or across many. Run on a JMAP or SCIM write and on a restore, +/// after `check`, holding the lock `disk_quota_tenant` names. `id` is the +/// account on an update; `restoring` words the refusal for a restore, +/// whose quota can't be edited first. Only a write that grows the total is checked, so a +/// tenant already over a lowered limit can still edit its accounts and +/// lower their quotas, like the count limits (MT-18). No quota at all is +/// fine: the account shares what's left of the tenant's space. Not run for +/// accounts a directory creates at sign-in, which never carry a quota. pub async fn check_disk_quota( registry: &RegistryStore, + id: Option, old: Option<&Object>, new: &Object, + restoring: bool, ) -> trc::Result>> { - let tenant = new.inner.member_tenant_id(); - if let Some(tenant_id) = tenant + if let ObjectInner::Account(_) = &new.inner + && let Some(tenant_id) = disk_quota_tenant(old, new) && let Some(quota) = account_disk_quota(&new.inner) - && old.is_none_or(|old| { - account_disk_quota(&old.inner) != Some(quota) || old.inner.member_tenant_id() != tenant - }) && let Some(tenant_quota) = (limits(registry, tenant_id).await?)(TenantStorageQuota::MaxDiskQuota) .filter(|limit| *limit > 0) - && quota > tenant_quota { - return Ok(Err(over_tenant_disk_quota(quota, tenant_quota))); + let except = id.into_iter().collect::>(); + let others = quota::allocated_disk(registry, tenant_id.document_id(), &except).await?; + if others.saturating_add(quota) > tenant_quota { + return Ok(Err(over_tenant_disk_quota( + quota, + others, + tenant_quota, + restoring, + ))); + } } Ok(Ok(())) } @@ -240,17 +263,54 @@ fn over_quota(reached: LimitReached, tenant_id: Id) -> SetError { )) } -/// Refuses an account disk quota bigger than its tenant's (inbuxa-admin#257). -fn over_tenant_disk_quota(quota: u64, tenant_quota: u64) -> SetError { +/// Refuses an account disk quota that doesn't fit in what its tenant has +/// left to give (inbuxa-admin#257). +fn over_tenant_disk_quota( + quota: u64, + others: u64, + tenant_quota: u64, + restoring: bool, +) -> SetError { SetError::invalid_properties() .with_property(Property::Quotas) - .with_description(format!( - "The account's disk quota of {quota} bytes ({}) is more than its tenant's \ - maxDiskQuota of {tenant_quota} bytes ({}). Set it to {tenant_quota} or less, \ - or leave it empty to share the tenant's space.", + .with_description(disk_quota_message(quota, others, tenant_quota, restoring)) +} + +/// Why a disk quota doesn't fit: the tenant's limit, what its other +/// accounts already have, and what's left. +fn disk_quota_message(quota: u64, others: u64, tenant_quota: u64, restoring: bool) -> String { + let left = tenant_quota.saturating_sub(others); + let given = if others > 0 { + format!( + " Its other accounts already have {others} bytes ({}) of it, so {left} bytes ({}) \ + is left to give.", + readable_size(others), + readable_size(left), + ) + } else { + String::new() + }; + if restoring { + return format!( + "The account's disk quota of {quota} bytes ({}) doesn't fit in its tenant's \ + maxDiskQuota of {tenant_quota} bytes ({}).{given} It comes back with that quota, \ + so lower other accounts' quotas first, then restore it.", readable_size(quota), readable_size(tenant_quota), - )) + ); + } + let fix = if left > 0 { + format!("Set it to {left} or less, lower another account's,") + } else { + "Lower another account's quota,".to_string() + }; + format!( + "The account's disk quota of {quota} bytes ({}) doesn't fit in its tenant's \ + maxDiskQuota of {tenant_quota} bytes ({}).{given} {fix} or leave it empty to share \ + the tenant's space.", + readable_size(quota), + readable_size(tenant_quota), + ) } /// A byte count in binary units, for people reading an error: "50 GiB". @@ -272,6 +332,96 @@ fn readable_size(bytes: u64) -> String { #[cfg(test)] mod disk_quota_tests { use super::*; + use registry::schema::{ + enums::StorageQuota, + structs::{Domain, UserAccount}, + }; + + fn account(tenant: Option, quota: u64) -> Object { + Object::from(Account::User(UserAccount { + member_tenant_id: tenant.map(Id::from), + quotas: [(StorageQuota::MaxDiskQuota, quota)].into_iter().collect(), + ..Default::default() + })) + } + + fn domain(tenant: Option) -> Object { + Object::from(Domain { + member_tenant_id: tenant.map(Id::from), + ..Default::default() + }) + } + + #[test] + fn writes_that_grow_a_tenants_disk_space_take_its_lock() { + let tenant = Some(Id::from(7u64)); + // A new account with a quota, and one without + assert_eq!(disk_quota_tenant(None, &account(Some(7), 10)), tenant); + assert_eq!(disk_quota_tenant(None, &account(Some(7), 0)), None); + assert_eq!(disk_quota_tenant(None, &account(None, 10)), None); + // Raised, set for the first time, lowered, cleared, unchanged + let old = account(Some(7), 10); + assert_eq!(disk_quota_tenant(Some(&old), &account(Some(7), 11)), tenant); + assert_eq!(disk_quota_tenant(Some(&account(Some(7), 0)), &old), tenant); + assert_eq!(disk_quota_tenant(Some(&old), &account(Some(7), 9)), None); + assert_eq!(disk_quota_tenant(Some(&old), &account(Some(7), 0)), None); + assert_eq!(disk_quota_tenant(Some(&old), &old), None); + // Joining a tenant with a quota, even a smaller one + assert_eq!( + disk_quota_tenant(Some(&account(None, 10)), &account(Some(7), 5)), + tenant + ); + assert_eq!( + disk_quota_tenant(Some(&account(Some(8), 10)), &account(Some(7), 5)), + tenant + ); + // A domain moving in brings its accounts; a new or staying one doesn't + assert_eq!( + disk_quota_tenant(Some(&domain(None)), &domain(Some(7))), + tenant + ); + assert_eq!(disk_quota_tenant(None, &domain(Some(7))), None); + assert_eq!( + disk_quota_tenant(Some(&domain(Some(7))), &domain(Some(7))), + None + ); + assert_eq!( + disk_quota_tenant(Some(&domain(Some(7))), &domain(None)), + None + ); + } + + #[test] + fn the_refusal_says_what_is_left() { + const GIB: u64 = 1024 * 1024 * 1024; + let message = disk_quota_message(5 * GIB, 15 * GIB, 5 * GIB, false); + assert!( + message.contains("already have 16106127360 bytes (15 GiB)"), + "{message}" + ); + assert!( + message.contains("so 0 bytes (0 bytes) is left"), + "{message}" + ); + assert!( + message.contains(" Lower another account's quota, or leave"), + "{message}" + ); + let message = disk_quota_message(4 * GIB, GIB, 4 * GIB, false); + assert!( + message.contains("so 3221225472 bytes (3 GiB) is left"), + "{message}" + ); + let message = disk_quota_message(3 * GIB, 4 * GIB, 5 * GIB, true); + assert!(message.contains("then restore it"), "{message}"); + assert!(!message.contains("Set it to"), "{message}"); + let message = disk_quota_message(6 * GIB, 0, 5 * GIB, false); + assert!(!message.contains("already have"), "{message}"); + assert!( + message.contains("Set it to 5368709120 or less"), + "{message}" + ); + } #[test] fn readable_sizes() { @@ -283,7 +433,7 @@ mod disk_quota_tests { #[test] fn zero_quota_is_no_quota() { let user = |quota: u64| { - ObjectInner::Account(Account::User(registry::schema::structs::UserAccount { + ObjectInner::Account(Account::User(UserAccount { quotas: [(StorageQuota::MaxDiskQuota, quota)].into_iter().collect(), ..Default::default() })) @@ -308,5 +458,17 @@ fn refused_move(refusal: Refusal, tenant: Option) -> SetError { )), Refusal::ForeignLink(object_id) => foreign_key(object_id), Refusal::Limit(reached) => over_quota(reached, tenant.unwrap_or_default()), + Refusal::DiskQuota { + allocated, + adding, + limit, + } => SetError::new(SetErrorType::OverQuota).with_description(format!( + "The accounts on this domain have {adding} bytes ({}) of disk quota between them, \ + and the tenant has {} ({}) of its maxDiskQuota of {limit} bytes ({}) left to give.", + readable_size(adding), + limit.saturating_sub(allocated), + readable_size(limit.saturating_sub(allocated)), + readable_size(limit), + )), } } diff --git a/crates/jmap/src/inbuxa/deleted_account.rs b/crates/jmap/src/inbuxa/deleted_account.rs index 06ba0013..1a95d497 100644 --- a/crates/jmap/src/inbuxa/deleted_account.rs +++ b/crates/jmap/src/inbuxa/deleted_account.rs @@ -48,6 +48,8 @@ use trc::AddContext; use types::id::Id; use utils::snowflake::SnowflakeIdGenerator; +use crate::inbuxa::tenant_disk_lock; + type DValue = Value<'static, P, DeletedAccountValue>; const ALL: &[P] = &[ @@ -391,7 +393,6 @@ async fn restore_account( kept: KeptAccount, password: Option, ) -> trc::Result>> { - let account_id = id.document_id(); let Some(ObjectInner::Account(mut account)) = PickledStream::new(&kept.record) .and_then(|mut stream| ObjectInner::unpickle(ObjectType::Account, &mut stream)) else { @@ -447,11 +448,50 @@ async fn restore_account( inner: ObjectInner::Account(account), revision: 0, }; + + // inbuxa-admin#257: its disk quota must still fit what the tenant has + // left to give, checked and saved under the tenant's lock + let disk_lock = inbuxa_features::tenancy::writes::disk_quota_tenant(None, &object) + .map(|tenant_id| tenant_id.document_id()); + if let Some(tenant_id) = disk_lock + && !tenant_disk_lock::acquire(server, tenant_id).await? + { + return Ok(Err( + SetError::forbidden().with_description(tenant_disk_lock::BUSY) + )); + } + let restored = restore_checked(server, id, kept, object).await; + if let Some(tenant_id) = disk_lock { + tenant_disk_lock::release(server, tenant_id).await; + } + restored +} + +/// The rest of a restore, once the account is rebuilt: the tenancy checks, +/// then the save. +async fn restore_checked( + server: &Server, + id: Id, + kept: KeptAccount, + object: Object, +) -> trc::Result>> { + let account_id = id.document_id(); if let Err(err) = inbuxa_features::tenancy::writes::check(server.registry(), None, None, &object).await? { return Ok(Err(failed(err))); } + if let Err(err) = inbuxa_features::tenancy::writes::check_disk_quota( + server.registry(), + None, + None, + &object, + true, + ) + .await? + { + return Ok(Err(failed(err))); + } // Its addresses are free for it alone let mut batch = BatchBuilder::new(); diff --git a/crates/jmap/src/inbuxa/mod.rs b/crates/jmap/src/inbuxa/mod.rs index 6b58c2bb..1a3edd5a 100644 --- a/crates/jmap/src/inbuxa/mod.rs +++ b/crates/jmap/src/inbuxa/mod.rs @@ -44,6 +44,7 @@ pub mod sharing_policy; pub mod store_check; // inbuxa: checking the store's blobs pub mod store_snapshot; // inbuxa: a snapshot of the store pub mod telemetry; +pub mod tenant_disk_lock; // inbuxa: inbuxa-admin#257 pub mod tenant_protocol_policy; pub mod undelete; pub mod webhook_test; diff --git a/crates/jmap/src/inbuxa/tenant_disk_lock.rs b/crates/jmap/src/inbuxa/tenant_disk_lock.rs new file mode 100644 index 00000000..adea8448 --- /dev/null +++ b/crates/jmap/src/inbuxa/tenant_disk_lock.rs @@ -0,0 +1,59 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! inbuxa-admin#257: one write at a time may grow a tenant's allocated disk +//! space. The lock lives in the in-memory store, so it holds across a +//! cluster: without it, two accounts given quotas at the same moment could +//! each fit the space left and together pass the tenant's limit. Taken +//! before the checks and released once the write is saved; one left behind +//! by a failed request expires on its own. + +use common::{KV_LOCK_TASK, Server}; +use std::time::Duration; + +/// How long a lock outlives a request that never releases it. +const LOCK_SECONDS: u64 = 30; +/// How long to wait for another write to the same tenant, in all. +const WAIT_TRIES: u32 = 100; +const WAIT_STEP: Duration = Duration::from_millis(50); + +/// What a write that couldn't get the lock is told. +pub const BUSY: &str = "Another change to this tenant's disk quotas is being saved. Try again."; + +fn key(tenant_id: u32) -> Vec { + [b"tenant-disk-quota:".as_slice(), &tenant_id.to_be_bytes()].concat() +} + +/// Takes the tenant's lock, waiting up to five seconds for a write that +/// holds it. `false` when it's still held. +pub async fn acquire(server: &Server, tenant_id: u32) -> trc::Result { + let key = key(tenant_id); + for _ in 0..WAIT_TRIES { + if server + .in_memory_store() + .try_lock(KV_LOCK_TASK, &key, LOCK_SECONDS) + .await? + { + return Ok(true); + } + tokio::time::sleep(WAIT_STEP).await; + } + Ok(false) +} + +/// Releases the tenant's lock. A failure only leaves it to expire. +pub async fn release(server: &Server, tenant_id: u32) { + if let Err(err) = server + .in_memory_store() + .remove_lock(KV_LOCK_TASK, &key(tenant_id)) + .await + { + trc::error!( + err.details("Failed to release a tenant's disk-quota lock") + .ctx(trc::Key::Id, tenant_id) + ); + } +} diff --git a/crates/jmap/src/registry/set.rs b/crates/jmap/src/registry/set.rs index e2d53dd8..82ea134e 100644 --- a/crates/jmap/src/registry/set.rs +++ b/crates/jmap/src/registry/set.rs @@ -379,7 +379,13 @@ impl RegistrySet for Server { // Process modifications let mut cache_invalidator = CacheInvalidationBuilder::default(); + // inbuxa: inbuxa-admin#257: the tenant disk-quota lock the last + // write took, released once it's saved or refused + let mut disk_lock: Option = None; 'outer: for (modification, mut value, mut new_object) in modifications { + if let Some(tenant_id) = disk_lock.take() { + crate::inbuxa::tenant_disk_lock::release(self, tenant_id).await; + } // Initial validations let is_create = matches!(modification, Modification::Create { .. }); let mut unpatched_properties = VecMap::new(); @@ -655,6 +661,23 @@ impl RegistrySet for Server { Modification::Update { id, object } => (Some(*id), Some(object)), Modification::Create { .. } => (None, None), }; + + // inbuxa: inbuxa-admin#257: a write that could grow the + // tenant's allocated disk space holds its lock until saved + if let Some(tenant_id) = + inbuxa_features::tenancy::writes::disk_quota_tenant(stored, &new_object) + { + let tenant_id = tenant_id.document_id(); + if !crate::inbuxa::tenant_disk_lock::acquire(self, tenant_id).await? { + set.failed( + modification, + SetError::forbidden() + .with_description(crate::inbuxa::tenant_disk_lock::BUSY), + ); + continue 'outer; + } + disk_lock = Some(tenant_id); + } let after_save = match inbuxa_features::tenancy::writes::check( self.registry(), stored_id, @@ -670,12 +693,14 @@ impl RegistrySet for Server { } }; - // inbuxa: inbuxa-admin#257: an account's disk quota fits - // within its tenant's + // inbuxa: inbuxa-admin#257: the tenant's accounts' disk + // quotas add up to no more than its own if let Err(err) = inbuxa_features::tenancy::writes::check_disk_quota( self.registry(), + stored_id, stored, &new_object, + false, ) .await? { @@ -947,6 +972,9 @@ impl RegistrySet for Server { set.server.notify_task_queue(); } } + if let Some(tenant_id) = disk_lock.take() { + crate::inbuxa::tenant_disk_lock::release(self, tenant_id).await; + } // Process destroy for id in set.destroy.drain(..) { diff --git a/tests/src/system/tenant.rs b/tests/src/system/tenant.rs index 2d8e6342..d1b5a7b8 100644 --- a/tests/src/system/tenant.rs +++ b/tests/src/system/tenant.rs @@ -963,7 +963,7 @@ pub async fn test(test: &mut TestServer) { .await; // inbuxa-admin#257: an account's disk quota can't exceed its tenant's, - // whoever sets it + // whoever sets it, and neither can its accounts' quotas together let d_limit = (two + one / 2) as u64; let with_quota = |name: &str, quota: u64| { structs::Account::User(structs::UserAccount { @@ -983,6 +983,14 @@ pub async fn test(test: &mut TestServer) { let d4 = d_admin .registry_create_object(with_quota("d4", d_limit)) .await; + // ... nor can several add up to more than it + for writer in [&d_admin, &admin] { + writer + .registry_create_object_expect_err(with_quota("d5", 1)) + .await + .assert_type(SetErrorType::InvalidProperties) + .assert_description_contains("other accounts already have"); + } d_admin .registry_update_object_expect_err( ObjectType::Account, @@ -1003,6 +1011,14 @@ pub async fn test(test: &mut TestServer) { json!({ Property::Description: "still here" }), ) .await; + // ... and its quota can still be lowered, even while over + d_admin + .registry_update_object( + ObjectType::Account, + d4, + json!({ Property::Quotas: VecMap::from_iter([(StorageQuota::MaxDiskQuota, d_limit - 1)]) }), + ) + .await; admin .set_tenant_quota(d_id, TenantStorageQuota::MaxDiskQuota, d_limit) .await; -- 2.54.0