From 8f8d6965d5e4f0c5c92b79a603c409da88593399 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 20 Nov 2026 09:47:00 -0700 Subject: [PATCH] Hold tenant count limits under the tenant's write lock (#498) The count limits (MT-17) were checked by counting the tenant's objects and then saving, with nothing to stop another write in between. 16 account creates at once from four tenant administrators, with 4 left under maxAccounts 8, ended at 9 in 2 of 15 live rounds. Every count limit had the gap, and so did directory sign-in. - The disk-quota lock from #497 becomes the tenant's write lock, in common (Server::lock_tenant_writes / unlock_tenant_writes), one per tenant for counts and disk space alike, so a write never holds two. It keeps the old key so nodes on either version share it during a rolling upgrade. writes::locked_tenant names the tenant to lock: a counted object joining it, a domain adding DKIM keys, or a write that grows its allocated disk space. x:*/set (JMAP and SCIM), a deleted account's restore, and directory sign-in's user and group creation hold it from the tenancy checks to the save. - An object joining a tenant on an update (its tenant changes) now counts against the limit like a new one; before, only creates did. A domain's move is still counted by its plan (MT-8). - An update that adds DKIM algorithms to a domain, or switches it to automatic DKIM, counts the keys the DKIM task will generate against maxDkimKeys (MT-9); before, only a new domain's keys counted. Rotation, which replaces a key, isn't counted again. Checked live against a debug build: 15 rounds of 16 parallel creates into a tenant with 4 of 8 left, every round 8/8 (main's build passed the limit in 2 of 15). tenant_tests gains the DKIM-update and moved-role cases; tenant, undelete, scim and legal_hold suites pass. Part of the inbuxa/inbuxa-admin#257 remediation (Refs inbuxa/inbuxa-admin#257). --- crates/common/src/cache/directory.rs | 63 ++++++-- crates/common/src/lib.rs | 1 + crates/common/src/tenant_lock.rs | 65 ++++++++ crates/features/src/tenancy/writes.rs | 172 +++++++++++++++++---- crates/jmap/src/inbuxa/deleted_account.rs | 19 ++- crates/jmap/src/inbuxa/mod.rs | 1 - crates/jmap/src/inbuxa/tenant_disk_lock.rs | 59 ------- crates/jmap/src/registry/set.rs | 27 ++-- tests/src/system/tenant.rs | 42 +++++ 9 files changed, 320 insertions(+), 129 deletions(-) create mode 100644 crates/common/src/tenant_lock.rs delete mode 100644 crates/jmap/src/inbuxa/tenant_disk_lock.rs diff --git a/crates/common/src/cache/directory.rs b/crates/common/src/cache/directory.rs index 9445327f..d939ebfe 100644 --- a/crates/common/src/cache/directory.rs +++ b/crates/common/src/cache/directory.rs @@ -248,14 +248,20 @@ impl Server { ..Default::default() })); - // inbuxa: DIR-15 - self.check_tenant_limits(&account).await?; - match self - .registry() - .write(RegistryWrite::insert(&account)) - .await - .caused_by(trc::location!())? - { + // inbuxa: DIR-15, MT-17: checked and saved under the tenant's lock + let tenant_lock = self.lock_for_tenant_limits(&account).await?; + let written = match self.check_tenant_limits(&account).await { + Ok(()) => self + .registry() + .write(RegistryWrite::insert(&account)) + .await + .caused_by(trc::location!()), + Err(err) => Err(err), + }; + if let Some(tenant_id) = tenant_lock { + self.unlock_tenant_writes(tenant_id).await; + } + match written? { RegistryWriteResult::Success(id) => { let mut invalidator = CacheInvalidationBuilder::default(); invalidator.process_create(&account); @@ -420,14 +426,20 @@ impl Server { ..Default::default() })); - // inbuxa: DIR-15 - self.check_tenant_limits(&account).await?; - match self - .registry() - .write(RegistryWrite::insert(&account)) - .await - .caused_by(trc::location!())? - { + // inbuxa: DIR-15, MT-17: checked and saved under the tenant's lock + let tenant_lock = self.lock_for_tenant_limits(&account).await?; + let written = match self.check_tenant_limits(&account).await { + Ok(()) => self + .registry() + .write(RegistryWrite::insert(&account)) + .await + .caused_by(trc::location!()), + Err(err) => Err(err), + }; + if let Some(tenant_id) = tenant_lock { + self.unlock_tenant_writes(tenant_id).await; + } + match written? { RegistryWriteResult::Success(id) => { let mut invalidator = CacheInvalidationBuilder::default(); invalidator.process_create(&account); @@ -477,6 +489,25 @@ impl Server { /// inbuxa: DIR-15, MT-3, MT-17: an object created from a directory /// passes the same tenant checks as one created over JMAP. + /// MT-17: takes the lock of the tenant a new directory account joins, + /// so parallel sign-ins can't each find room under its limits. Returns + /// the tenant to unlock, `None` for an account in no tenant. + async fn lock_for_tenant_limits(&self, object: &Object) -> trc::Result> { + let Some(tenant_id) = inbuxa_features::tenancy::writes::locked_tenant(None, object) + .map(|tenant_id| tenant_id.document_id()) + else { + return Ok(None); + }; + if self.lock_tenant_writes(tenant_id).await? { + Ok(Some(tenant_id)) + } else { + Err(trc::AuthEvent::Error + .into_err() + .details(crate::tenant_lock::BUSY) + .reason("The directory's account can't be created")) + } + } + async fn check_tenant_limits(&self, object: &Object) -> trc::Result<()> { match inbuxa_features::tenancy::writes::check(self.registry(), None, None, object).await? { Ok(_) => Ok(()), diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 76edbb35..f85dc029 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -82,6 +82,7 @@ pub mod scripts; pub mod sharing; pub mod storage; pub mod telemetry; +pub mod tenant_lock; // inbuxa: MT-17, inbuxa-admin#257 pub use psl; diff --git a/crates/common/src/tenant_lock.rs b/crates/common/src/tenant_lock.rs new file mode 100644 index 00000000..f04ebb65 --- /dev/null +++ b/crates/common/src/tenant_lock.rs @@ -0,0 +1,65 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! One write at a time may grow a tenant's counted totals: its objects +//! against the count limits (MT-17) and the disk space its accounts are +//! given (inbuxa-admin#257). Without it, two writes at the same moment could +//! each find room for one more and together pass the limit. The lock lives +//! in the in-memory store, so it holds across a cluster. A write takes it +//! before its tenancy checks and releases it once saved; one left behind by +//! a failed request expires on its own. A write holds at most one tenant's +//! lock, so two can't wait on each other. + +use crate::{KV_LOCK_TASK, Server}; +use std::time::Duration; + +/// How long a lock outlives a request that never releases it. +const LOCK_SECONDS: u64 = 30; +/// How long to wait for another write to the same tenant, in all. +const WAIT_TRIES: u32 = 100; +const WAIT_STEP: Duration = Duration::from_millis(50); + +/// What a write that couldn't get the lock is told. +pub const BUSY: &str = "Another change to this tenant's limits is being saved. Try again."; + +// The key the disk-quota lock used, kept so nodes on either version share +// one lock during a rolling upgrade +fn key(tenant_id: u32) -> Vec { + [b"tenant-disk-quota:".as_slice(), &tenant_id.to_be_bytes()].concat() +} + +impl Server { + /// Takes the tenant's write lock, waiting up to five seconds for a write + /// that holds it. `false` when it's still held. + pub async fn lock_tenant_writes(&self, tenant_id: u32) -> trc::Result { + let key = key(tenant_id); + for _ in 0..WAIT_TRIES { + if self + .in_memory_store() + .try_lock(KV_LOCK_TASK, &key, LOCK_SECONDS) + .await? + { + return Ok(true); + } + tokio::time::sleep(WAIT_STEP).await; + } + Ok(false) + } + + /// Releases the tenant's write lock. A failure only leaves it to expire. + pub async fn unlock_tenant_writes(&self, tenant_id: u32) { + if let Err(err) = self + .in_memory_store() + .remove_lock(KV_LOCK_TASK, &key(tenant_id)) + .await + { + trc::error!( + err.details("Failed to release a tenant's write lock") + .ctx(trc::Key::Id, tenant_id) + ); + } + } +} diff --git a/crates/features/src/tenancy/writes.rs b/crates/features/src/tenancy/writes.rs index a646a769..a91fa616 100644 --- a/crates/features/src/tenancy/writes.rs +++ b/crates/features/src/tenancy/writes.rs @@ -10,14 +10,15 @@ //! - MT-7: a principal created without a tenant takes its domain's. //! - MT-8: a domain moves only through no tenant, taking its principals and //! keys in with it, and only when its people let it out. -//! - MT-17: creating an object, or moving a domain in, stays within the -//! tenant's count limits, and the server emits `limit.tenant-quota` when -//! it doesn't. +//! - MT-17: creating an object, moving one in, or moving a domain in stays +//! within the tenant's count limits, a domain's DKIM keys included (MT-9), +//! and the server emits `limit.tenant-quota` when it doesn't. //! - A tenant's accounts' disk quotas add up to no more than the tenant's //! (inbuxa-admin#257), checked by `check_disk_quota` on JMAP and SCIM -//! writes and on a restore, and by the domain-move plan. A write that -//! could grow the total holds the tenant's disk-quota lock -//! (`disk_quota_tenant`) from the check until it's saved. +//! writes and on a restore, and by the domain-move plan. +//! - A write that could grow a tenant's counted totals, its objects or its +//! allocated disk space, holds the tenant's write lock (`locked_tenant`) +//! from its checks until it's saved. use crate::tenancy::{ domain_move::{self, Move, Refusal}, @@ -120,10 +121,12 @@ pub async fn check( return Ok(Err(foreign_key(foreign))); } - // MT-17: count limits on a new object - if old.is_none() - && let Some(tenant_id) = tenant + // MT-17: count limits on an object joining the tenant: a new one, or one + // whose tenant changes. A domain's move is counted by its plan (MT-8). + if let Some(tenant_id) = tenant && let Some(quota) = quota::limit_for(&new.inner) + && joins_tenant(old, new) + && !matches!(new.inner, ObjectInner::Domain(_) if old.is_some()) { let limit = limits(registry, tenant_id).await?; if let Err(reached) = @@ -131,33 +134,85 @@ pub async fn check( { return Ok(Err(over_quota(reached, tenant_id))); } + } - // A new domain's generated DKIM keys, one per algorithm, count too (MT-9) - if let ObjectInner::Domain(Domain { - dkim_management: DkimManagement::Automatic(dkim), - .. - }) = &new.inner + // MT-9: a domain's generated DKIM keys, one per algorithm, count too: + // all of them on a new domain, and those it adds on an update, which the + // DKIM task then generates. Rotation, which replaces a key with a new one, + // isn't counted again. + if let Some(tenant_id) = tenant + && let Some(keys) = added_dkim_keys(old, new) + && keys > 0 + { + let limit = limits(registry, tenant_id).await?; + let quota = TenantStorageQuota::MaxDkimKeys; + if let Err(reached) = + quota::check(registry, tenant_id.document_id(), quota, limit(quota), keys).await? { - let quota = TenantStorageQuota::MaxDkimKeys; - let keys = dkim.algorithms.iter().count() as u64; - if let Err(reached) = - quota::check(registry, tenant_id.document_id(), quota, limit(quota), keys).await? - { - return Ok(Err(over_quota(reached, tenant_id))); - } + return Ok(Err(over_quota(reached, tenant_id))); } } Ok(Ok(after)) } +/// Whether `new` joins its tenant with this write: it's new, or its tenant +/// changes. +fn joins_tenant(old: Option<&Object>, new: &Object) -> bool { + old.is_none_or(|old| old.inner.member_tenant_id() != new.inner.member_tenant_id()) +} + +/// How many DKIM keys a write to a domain makes the DKIM task generate: one +/// per algorithm on a new automatic domain, or per algorithm an update adds +/// (an update into automatic mode adds them all). `None` for anything else. +fn added_dkim_keys(old: Option<&Object>, new: &Object) -> Option { + let ObjectInner::Domain(Domain { + dkim_management: DkimManagement::Automatic(dkim), + .. + }) = &new.inner + else { + return None; + }; + let before = match old.map(|old| &old.inner) { + Some(ObjectInner::Domain(Domain { + dkim_management: DkimManagement::Automatic(before), + .. + })) => Some(&before.algorithms), + _ => None, + }; + Some( + dkim.algorithms + .iter() + .filter(|algorithm| before.is_none_or(|before| !before.contains(algorithm))) + .count() as u64, + ) +} + +/// The tenant whose counted totals this write could grow, if any: an +/// object joining a tenant whose kind it limits (MT-17), a domain adding +/// DKIM keys (MT-9), or a write that could grow its allocated disk space +/// (`disk_quota_tenant`). Such a write takes the tenant's write lock +/// (common's `lock_tenant_writes`) before its tenancy checks and keeps it +/// until it's saved, so two writes at once can't each find room for one +/// more and together pass the limit. +pub fn locked_tenant(old: Option<&Object>, new: &Object) -> Option { + let tenant = new.inner.member_tenant_id()?; + let counted = quota::limit_for(&new.inner).is_some() && joins_tenant(old, new); + let dkim = added_dkim_keys(old, new).is_some_and(|keys| keys > 0); + if counted || dkim { + Some(tenant) + } else { + disk_quota_tenant(old, new) + } +} + /// The tenant whose allocated disk space this write could grow, if any /// (inbuxa-admin#257): an account given a disk quota, or a bigger one, or /// joining a tenant with one; or a domain moving into a tenant, which brings -/// its accounts. Such a write takes the tenant's disk-quota lock before its -/// checks and keeps it until it's saved, so two writes at once can't each -/// fit the space left and together pass it. Lowering or clearing a quota, -/// or any other change, needs no lock. +/// its accounts. Such a write takes the tenant's write lock (`locked_tenant`) +/// before its checks and keeps it until it's saved, so two writes at once +/// can't each fit the space left and together pass it. Lowering or clearing +/// a quota, or any other change, needs no lock. pub fn disk_quota_tenant(old: Option<&Object>, new: &Object) -> Option { let tenant = new.inner.member_tenant_id()?; let moved_in = old.is_none_or(|old| old.inner.member_tenant_id() != Some(tenant)); @@ -178,7 +233,7 @@ pub fn disk_quota_tenant(old: Option<&Object>, new: &Object) -> Option { /// inbuxa-admin#257: a tenant's accounts' disk quotas add up to no more than /// the tenant's own, so it can't give out more space than it has, in one /// account or across many. Run on a JMAP or SCIM write and on a restore, -/// after `check`, holding the lock `disk_quota_tenant` names. `id` is the +/// after `check`, holding the lock `locked_tenant` names. `id` is the /// account on an update; `restoring` words the refusal for a restore, /// whose quota can't be edited first. Only a write that grows the total is checked, so a /// tenant already over a lowered limit can still edit its accounts and @@ -332,9 +387,12 @@ fn readable_size(bytes: u64) -> String { #[cfg(test)] mod disk_quota_tests { use super::*; - use registry::schema::{ - enums::StorageQuota, - structs::{Domain, UserAccount}, + use registry::{ + schema::{ + enums::{DkimSignatureType, StorageQuota}, + structs::{DkimManagementProperties, Domain, UserAccount}, + }, + types::map::Map, }; fn account(tenant: Option, quota: u64) -> Object { @@ -391,6 +449,62 @@ mod disk_quota_tests { ); } + fn dkim_domain(tenant: Option, algorithms: &[DkimSignatureType]) -> Object { + Object::from(Domain { + member_tenant_id: tenant.map(Id::from), + dkim_management: DkimManagement::Automatic(DkimManagementProperties { + algorithms: Map::new(algorithms.to_vec()), + ..Default::default() + }), + ..Default::default() + }) + } + + #[test] + fn writes_that_grow_a_tenants_counts_take_its_lock() { + use DkimSignatureType::*; + let tenant = Some(Id::from(7u64)); + // A new counted object in a tenant, and one in none + assert_eq!(locked_tenant(None, &account(Some(7), 0)), tenant); + assert_eq!(locked_tenant(None, &account(None, 0)), None); + assert_eq!(locked_tenant(None, &domain(Some(7))), tenant); + // An edit that doesn't join or add keys takes none + let staying = account(Some(7), 0); + assert_eq!(locked_tenant(Some(&staying), &staying), None); + // Joining from no tenant or another + assert_eq!(locked_tenant(Some(&account(None, 0)), &staying), tenant); + assert_eq!( + locked_tenant(Some(&domain(Some(8))), &domain(Some(7))), + tenant + ); + // A domain adding DKIM algorithms, or not + let one = dkim_domain(Some(7), &[Dkim1RsaSha256]); + let two = dkim_domain(Some(7), &[Dkim1RsaSha256, Dkim1Ed25519Sha256]); + assert_eq!(locked_tenant(Some(&one), &two), tenant); + assert_eq!(locked_tenant(Some(&two), &one), None); + assert_eq!(locked_tenant(Some(&two), &two), None); + // Disk quota growth still locks + assert_eq!(locked_tenant(Some(&staying), &account(Some(7), 10)), tenant); + } + + #[test] + fn added_dkim_keys_counts_new_algorithms_only() { + use DkimSignatureType::*; + let one = dkim_domain(Some(7), &[Dkim1RsaSha256]); + let two = dkim_domain(Some(7), &[Dkim1RsaSha256, Dkim2Ed25519Sha256]); + assert_eq!(added_dkim_keys(None, &two), Some(2)); + assert_eq!(added_dkim_keys(Some(&one), &two), Some(1)); + assert_eq!(added_dkim_keys(Some(&two), &one), Some(0)); + // Manual to automatic adds them all; manual adds none + let manual = Object::from(Domain { + member_tenant_id: Some(Id::from(7u64)), + dkim_management: DkimManagement::Manual, + ..Default::default() + }); + assert_eq!(added_dkim_keys(Some(&manual), &two), Some(2)); + assert_eq!(added_dkim_keys(Some(&two), &manual), None); + } + #[test] fn the_refusal_says_what_is_left() { const GIB: u64 = 1024 * 1024 * 1024; diff --git a/crates/jmap/src/inbuxa/deleted_account.rs b/crates/jmap/src/inbuxa/deleted_account.rs index 1a95d497..945f6b31 100644 --- a/crates/jmap/src/inbuxa/deleted_account.rs +++ b/crates/jmap/src/inbuxa/deleted_account.rs @@ -48,8 +48,6 @@ use trc::AddContext; use types::id::Id; use utils::snowflake::SnowflakeIdGenerator; -use crate::inbuxa::tenant_disk_lock; - type DValue = Value<'static, P, DeletedAccountValue>; const ALL: &[P] = &[ @@ -449,20 +447,21 @@ async fn restore_account( revision: 0, }; - // inbuxa-admin#257: its disk quota must still fit what the tenant has - // left to give, checked and saved under the tenant's lock - let disk_lock = inbuxa_features::tenancy::writes::disk_quota_tenant(None, &object) + // MT-17, inbuxa-admin#257: it must still fit the tenant's count limits + // and the disk space it has left to give, checked and saved under the + // tenant's lock + let tenant_lock = inbuxa_features::tenancy::writes::locked_tenant(None, &object) .map(|tenant_id| tenant_id.document_id()); - if let Some(tenant_id) = disk_lock - && !tenant_disk_lock::acquire(server, tenant_id).await? + if let Some(tenant_id) = tenant_lock + && !server.lock_tenant_writes(tenant_id).await? { return Ok(Err( - SetError::forbidden().with_description(tenant_disk_lock::BUSY) + SetError::forbidden().with_description(common::tenant_lock::BUSY) )); } let restored = restore_checked(server, id, kept, object).await; - if let Some(tenant_id) = disk_lock { - tenant_disk_lock::release(server, tenant_id).await; + if let Some(tenant_id) = tenant_lock { + server.unlock_tenant_writes(tenant_id).await; } restored } diff --git a/crates/jmap/src/inbuxa/mod.rs b/crates/jmap/src/inbuxa/mod.rs index 1a3edd5a..6b58c2bb 100644 --- a/crates/jmap/src/inbuxa/mod.rs +++ b/crates/jmap/src/inbuxa/mod.rs @@ -44,7 +44,6 @@ pub mod sharing_policy; pub mod store_check; // inbuxa: checking the store's blobs pub mod store_snapshot; // inbuxa: a snapshot of the store pub mod telemetry; -pub mod tenant_disk_lock; // inbuxa: inbuxa-admin#257 pub mod tenant_protocol_policy; pub mod undelete; pub mod webhook_test; diff --git a/crates/jmap/src/inbuxa/tenant_disk_lock.rs b/crates/jmap/src/inbuxa/tenant_disk_lock.rs deleted file mode 100644 index adea8448..00000000 --- a/crates/jmap/src/inbuxa/tenant_disk_lock.rs +++ /dev/null @@ -1,59 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2026 Coffey Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only - */ - -//! inbuxa-admin#257: one write at a time may grow a tenant's allocated disk -//! space. The lock lives in the in-memory store, so it holds across a -//! cluster: without it, two accounts given quotas at the same moment could -//! each fit the space left and together pass the tenant's limit. Taken -//! before the checks and released once the write is saved; one left behind -//! by a failed request expires on its own. - -use common::{KV_LOCK_TASK, Server}; -use std::time::Duration; - -/// How long a lock outlives a request that never releases it. -const LOCK_SECONDS: u64 = 30; -/// How long to wait for another write to the same tenant, in all. -const WAIT_TRIES: u32 = 100; -const WAIT_STEP: Duration = Duration::from_millis(50); - -/// What a write that couldn't get the lock is told. -pub const BUSY: &str = "Another change to this tenant's disk quotas is being saved. Try again."; - -fn key(tenant_id: u32) -> Vec { - [b"tenant-disk-quota:".as_slice(), &tenant_id.to_be_bytes()].concat() -} - -/// Takes the tenant's lock, waiting up to five seconds for a write that -/// holds it. `false` when it's still held. -pub async fn acquire(server: &Server, tenant_id: u32) -> trc::Result { - let key = key(tenant_id); - for _ in 0..WAIT_TRIES { - if server - .in_memory_store() - .try_lock(KV_LOCK_TASK, &key, LOCK_SECONDS) - .await? - { - return Ok(true); - } - tokio::time::sleep(WAIT_STEP).await; - } - Ok(false) -} - -/// Releases the tenant's lock. A failure only leaves it to expire. -pub async fn release(server: &Server, tenant_id: u32) { - if let Err(err) = server - .in_memory_store() - .remove_lock(KV_LOCK_TASK, &key(tenant_id)) - .await - { - trc::error!( - err.details("Failed to release a tenant's disk-quota lock") - .ctx(trc::Key::Id, tenant_id) - ); - } -} diff --git a/crates/jmap/src/registry/set.rs b/crates/jmap/src/registry/set.rs index 82ea134e..d8bd6e7d 100644 --- a/crates/jmap/src/registry/set.rs +++ b/crates/jmap/src/registry/set.rs @@ -379,12 +379,12 @@ impl RegistrySet for Server { // Process modifications let mut cache_invalidator = CacheInvalidationBuilder::default(); - // inbuxa: inbuxa-admin#257: the tenant disk-quota lock the last - // write took, released once it's saved or refused - let mut disk_lock: Option = None; + // inbuxa: MT-17, inbuxa-admin#257: the tenant write lock the + // last write took, released once it's saved or refused + let mut tenant_lock: Option = None; 'outer: for (modification, mut value, mut new_object) in modifications { - if let Some(tenant_id) = disk_lock.take() { - crate::inbuxa::tenant_disk_lock::release(self, tenant_id).await; + if let Some(tenant_id) = tenant_lock.take() { + self.unlock_tenant_writes(tenant_id).await; } // Initial validations let is_create = matches!(modification, Modification::Create { .. }); @@ -662,21 +662,20 @@ impl RegistrySet for Server { Modification::Create { .. } => (None, None), }; - // inbuxa: inbuxa-admin#257: a write that could grow the - // tenant's allocated disk space holds its lock until saved + // inbuxa: MT-17, inbuxa-admin#257: a write that could grow the + // tenant's counted totals holds its lock until saved if let Some(tenant_id) = - inbuxa_features::tenancy::writes::disk_quota_tenant(stored, &new_object) + inbuxa_features::tenancy::writes::locked_tenant(stored, &new_object) { let tenant_id = tenant_id.document_id(); - if !crate::inbuxa::tenant_disk_lock::acquire(self, tenant_id).await? { + if !self.lock_tenant_writes(tenant_id).await? { set.failed( modification, - SetError::forbidden() - .with_description(crate::inbuxa::tenant_disk_lock::BUSY), + SetError::forbidden().with_description(common::tenant_lock::BUSY), ); continue 'outer; } - disk_lock = Some(tenant_id); + tenant_lock = Some(tenant_id); } let after_save = match inbuxa_features::tenancy::writes::check( self.registry(), @@ -972,8 +971,8 @@ impl RegistrySet for Server { set.server.notify_task_queue(); } } - if let Some(tenant_id) = disk_lock.take() { - crate::inbuxa::tenant_disk_lock::release(self, tenant_id).await; + if let Some(tenant_id) = tenant_lock.take() { + self.unlock_tenant_writes(tenant_id).await; } // Process destroy diff --git a/tests/src/system/tenant.rs b/tests/src/system/tenant.rs index d1b5a7b8..10af24a1 100644 --- a/tests/src/system/tenant.rs +++ b/tests/src/system/tenant.rs @@ -846,6 +846,48 @@ pub async fn test(test: &mut TestServer) { .await .assert_type(SetErrorType::OverQuota) .assert_description_contains("maxDkimKeys"); + // ... and so do the keys an update makes the DKIM task generate + admin + .registry_update_object_expect_err( + ObjectType::Domain, + q_domain, + json!({ Property::DkimManagement: Domain::default().dkim_management }), + ) + .await + .assert_type(SetErrorType::OverQuota) + .assert_description_contains("maxDkimKeys"); + // An object moving into the tenant counts like a new one, not only a + // domain's (MT-17) + let q_roles = admin + .registry_query_ids( + ObjectType::Role, + [(Property::MemberTenantId, q_id.to_string())], + Vec::<&str>::new(), + ) + .await + .len() as u64; + admin + .set_tenant_quota(q_id, TenantStorageQuota::MaxRoles, q_roles) + .await; + let loose_role = admin + .registry_create_object(structs::Role { + description: "joins tenant Q".to_string(), + ..Default::default() + }) + .await; + admin + .registry_update_object_expect_err( + ObjectType::Role, + loose_role, + json!({ Property::MemberTenantId: q_id }), + ) + .await + .assert_type(SetErrorType::OverQuota) + .assert_description_contains("maxRoles"); + admin + .registry_destroy(ObjectType::Role, [loose_role]) + .await + .assert_destroyed(&[loose_role]); // Acceptance test 13: the tenant's disk limit bounds all its members // together (MT-19, MT-20) -- 2.54.0