Hold tenant count limits under the tenant's write lock (#498) #501

Merged
jcoffey-dev merged 1 commits from fix/tenant-count-lock into main 2026-11-20 17:03:38 +00:00
9 changed files with 320 additions and 129 deletions

No files matched your search

+47 -16
View File
@@ -248,14 +248,20 @@ impl Server {
..Default::default()
}));
// inbuxa: DIR-15
self.check_tenant_limits(&account).await?;
match self
.registry()
.write(RegistryWrite::insert(&account))
.await
.caused_by(trc::location!())?
{
// inbuxa: DIR-15, MT-17: checked and saved under the tenant's lock
let tenant_lock = self.lock_for_tenant_limits(&account).await?;
let written = match self.check_tenant_limits(&account).await {
Ok(()) => self
.registry()
.write(RegistryWrite::insert(&account))
.await
.caused_by(trc::location!()),
Err(err) => Err(err),
};
if let Some(tenant_id) = tenant_lock {
self.unlock_tenant_writes(tenant_id).await;
}
match written? {
RegistryWriteResult::Success(id) => {
let mut invalidator = CacheInvalidationBuilder::default();
invalidator.process_create(&account);
@@ -420,14 +426,20 @@ impl Server {
..Default::default()
}));
// inbuxa: DIR-15
self.check_tenant_limits(&account).await?;
match self
.registry()
.write(RegistryWrite::insert(&account))
.await
.caused_by(trc::location!())?
{
// inbuxa: DIR-15, MT-17: checked and saved under the tenant's lock
let tenant_lock = self.lock_for_tenant_limits(&account).await?;
let written = match self.check_tenant_limits(&account).await {
Ok(()) => self
.registry()
.write(RegistryWrite::insert(&account))
.await
.caused_by(trc::location!()),
Err(err) => Err(err),
};
if let Some(tenant_id) = tenant_lock {
self.unlock_tenant_writes(tenant_id).await;
}
match written? {
RegistryWriteResult::Success(id) => {
let mut invalidator = CacheInvalidationBuilder::default();
invalidator.process_create(&account);
@@ -477,6 +489,25 @@ impl Server {
/// inbuxa: DIR-15, MT-3, MT-17: an object created from a directory
/// passes the same tenant checks as one created over JMAP.
/// MT-17: takes the lock of the tenant a new directory account joins,
/// so parallel sign-ins can't each find room under its limits. Returns
/// the tenant to unlock, `None` for an account in no tenant.
async fn lock_for_tenant_limits(&self, object: &Object) -> trc::Result<Option<u32>> {
let Some(tenant_id) = inbuxa_features::tenancy::writes::locked_tenant(None, object)
.map(|tenant_id| tenant_id.document_id())
else {
return Ok(None);
};
if self.lock_tenant_writes(tenant_id).await? {
Ok(Some(tenant_id))
} else {
Err(trc::AuthEvent::Error
.into_err()
.details(crate::tenant_lock::BUSY)
.reason("The directory's account can't be created"))
}
}
async fn check_tenant_limits(&self, object: &Object) -> trc::Result<()> {
match inbuxa_features::tenancy::writes::check(self.registry(), None, None, object).await? {
Ok(_) => Ok(()),
+1
View File
@@ -82,6 +82,7 @@ pub mod scripts;
pub mod sharing;
pub mod storage;
pub mod telemetry;
pub mod tenant_lock; // inbuxa: MT-17, inbuxa-admin#257
pub use psl;
+65
View File
@@ -0,0 +1,65 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! One write at a time may grow a tenant's counted totals: its objects
//! against the count limits (MT-17) and the disk space its accounts are
//! given (inbuxa-admin#257). Without it, two writes at the same moment could
//! each find room for one more and together pass the limit. The lock lives
//! in the in-memory store, so it holds across a cluster. A write takes it
//! before its tenancy checks and releases it once saved; one left behind by
//! a failed request expires on its own. A write holds at most one tenant's
//! lock, so two can't wait on each other.
use crate::{KV_LOCK_TASK, Server};
use std::time::Duration;
/// How long a lock outlives a request that never releases it.
const LOCK_SECONDS: u64 = 30;
/// How long to wait for another write to the same tenant, in all.
const WAIT_TRIES: u32 = 100;
const WAIT_STEP: Duration = Duration::from_millis(50);
/// What a write that couldn't get the lock is told.
pub const BUSY: &str = "Another change to this tenant's limits is being saved. Try again.";
// The key the disk-quota lock used, kept so nodes on either version share
// one lock during a rolling upgrade
fn key(tenant_id: u32) -> Vec<u8> {
[b"tenant-disk-quota:".as_slice(), &tenant_id.to_be_bytes()].concat()
}
impl Server {
/// Takes the tenant's write lock, waiting up to five seconds for a write
/// that holds it. `false` when it's still held.
pub async fn lock_tenant_writes(&self, tenant_id: u32) -> trc::Result<bool> {
let key = key(tenant_id);
for _ in 0..WAIT_TRIES {
if self
.in_memory_store()
.try_lock(KV_LOCK_TASK, &key, LOCK_SECONDS)
.await?
{
return Ok(true);
}
tokio::time::sleep(WAIT_STEP).await;
}
Ok(false)
}
/// Releases the tenant's write lock. A failure only leaves it to expire.
pub async fn unlock_tenant_writes(&self, tenant_id: u32) {
if let Err(err) = self
.in_memory_store()
.remove_lock(KV_LOCK_TASK, &key(tenant_id))
.await
{
trc::error!(
err.details("Failed to release a tenant's write lock")
.ctx(trc::Key::Id, tenant_id)
);
}
}
}
+143 -29
View File
@@ -10,14 +10,15 @@
//! - MT-7: a principal created without a tenant takes its domain's.
//! - MT-8: a domain moves only through no tenant, taking its principals and
//! keys in with it, and only when its people let it out.
//! - MT-17: creating an object, or moving a domain in, stays within the
//! tenant's count limits, and the server emits `limit.tenant-quota` when
//! it doesn't.
//! - MT-17: creating an object, moving one in, or moving a domain in stays
//! within the tenant's count limits, a domain's DKIM keys included (MT-9),
//! and the server emits `limit.tenant-quota` when it doesn't.
//! - A tenant's accounts' disk quotas add up to no more than the tenant's
//! (inbuxa-admin#257), checked by `check_disk_quota` on JMAP and SCIM
//! writes and on a restore, and by the domain-move plan. A write that
//! could grow the total holds the tenant's disk-quota lock
//! (`disk_quota_tenant`) from the check until it's saved.
//! writes and on a restore, and by the domain-move plan.
//! - A write that could grow a tenant's counted totals, its objects or its
//! allocated disk space, holds the tenant's write lock (`locked_tenant`)
//! from its checks until it's saved.
use crate::tenancy::{
domain_move::{self, Move, Refusal},
@@ -120,10 +121,12 @@ pub async fn check(
return Ok(Err(foreign_key(foreign)));
}
// MT-17: count limits on a new object
if old.is_none()
&& let Some(tenant_id) = tenant
// MT-17: count limits on an object joining the tenant: a new one, or one
// whose tenant changes. A domain's move is counted by its plan (MT-8).
if let Some(tenant_id) = tenant
&& let Some(quota) = quota::limit_for(&new.inner)
&& joins_tenant(old, new)
&& !matches!(new.inner, ObjectInner::Domain(_) if old.is_some())
{
let limit = limits(registry, tenant_id).await?;
if let Err(reached) =
@@ -131,33 +134,85 @@ pub async fn check(
{
return Ok(Err(over_quota(reached, tenant_id)));
}
}
// A new domain's generated DKIM keys, one per algorithm, count too (MT-9)
if let ObjectInner::Domain(Domain {
dkim_management: DkimManagement::Automatic(dkim),
..
}) = &new.inner
// MT-9: a domain's generated DKIM keys, one per algorithm, count too:
// all of them on a new domain, and those it adds on an update, which the
// DKIM task then generates. Rotation, which replaces a key with a new one,
// isn't counted again.
if let Some(tenant_id) = tenant
&& let Some(keys) = added_dkim_keys(old, new)
&& keys > 0
{
let limit = limits(registry, tenant_id).await?;
let quota = TenantStorageQuota::MaxDkimKeys;
if let Err(reached) =
quota::check(registry, tenant_id.document_id(), quota, limit(quota), keys).await?
{
let quota = TenantStorageQuota::MaxDkimKeys;
let keys = dkim.algorithms.iter().count() as u64;
if let Err(reached) =
quota::check(registry, tenant_id.document_id(), quota, limit(quota), keys).await?
{
return Ok(Err(over_quota(reached, tenant_id)));
}
return Ok(Err(over_quota(reached, tenant_id)));
}
}
Ok(Ok(after))
}
/// Whether `new` joins its tenant with this write: it's new, or its tenant
/// changes.
fn joins_tenant(old: Option<&Object>, new: &Object) -> bool {
old.is_none_or(|old| old.inner.member_tenant_id() != new.inner.member_tenant_id())
}
/// How many DKIM keys a write to a domain makes the DKIM task generate: one
/// per algorithm on a new automatic domain, or per algorithm an update adds
/// (an update into automatic mode adds them all). `None` for anything else.
fn added_dkim_keys(old: Option<&Object>, new: &Object) -> Option<u64> {
let ObjectInner::Domain(Domain {
dkim_management: DkimManagement::Automatic(dkim),
..
}) = &new.inner
else {
return None;
};
let before = match old.map(|old| &old.inner) {
Some(ObjectInner::Domain(Domain {
dkim_management: DkimManagement::Automatic(before),
..
})) => Some(&before.algorithms),
_ => None,
};
Some(
dkim.algorithms
.iter()
.filter(|algorithm| before.is_none_or(|before| !before.contains(algorithm)))
.count() as u64,
)
}
/// The tenant whose counted totals this write could grow, if any: an
/// object joining a tenant whose kind it limits (MT-17), a domain adding
/// DKIM keys (MT-9), or a write that could grow its allocated disk space
/// (`disk_quota_tenant`). Such a write takes the tenant's write lock
/// (common's `lock_tenant_writes`) before its tenancy checks and keeps it
/// until it's saved, so two writes at once can't each find room for one
/// more and together pass the limit.
pub fn locked_tenant(old: Option<&Object>, new: &Object) -> Option<Id> {
let tenant = new.inner.member_tenant_id()?;
let counted = quota::limit_for(&new.inner).is_some() && joins_tenant(old, new);
let dkim = added_dkim_keys(old, new).is_some_and(|keys| keys > 0);
if counted || dkim {
Some(tenant)
} else {
disk_quota_tenant(old, new)
}
}
/// The tenant whose allocated disk space this write could grow, if any
/// (inbuxa-admin#257): an account given a disk quota, or a bigger one, or
/// joining a tenant with one; or a domain moving into a tenant, which brings
/// its accounts. Such a write takes the tenant's disk-quota lock before its
/// checks and keeps it until it's saved, so two writes at once can't each
/// fit the space left and together pass it. Lowering or clearing a quota,
/// or any other change, needs no lock.
/// its accounts. Such a write takes the tenant's write lock (`locked_tenant`)
/// before its checks and keeps it until it's saved, so two writes at once
/// can't each fit the space left and together pass it. Lowering or clearing
/// a quota, or any other change, needs no lock.
pub fn disk_quota_tenant(old: Option<&Object>, new: &Object) -> Option<Id> {
let tenant = new.inner.member_tenant_id()?;
let moved_in = old.is_none_or(|old| old.inner.member_tenant_id() != Some(tenant));
@@ -178,7 +233,7 @@ pub fn disk_quota_tenant(old: Option<&Object>, new: &Object) -> Option<Id> {
/// inbuxa-admin#257: a tenant's accounts' disk quotas add up to no more than
/// the tenant's own, so it can't give out more space than it has, in one
/// account or across many. Run on a JMAP or SCIM write and on a restore,
/// after `check`, holding the lock `disk_quota_tenant` names. `id` is the
/// after `check`, holding the lock `locked_tenant` names. `id` is the
/// account on an update; `restoring` words the refusal for a restore,
/// whose quota can't be edited first. Only a write that grows the total is checked, so a
/// tenant already over a lowered limit can still edit its accounts and
@@ -332,9 +387,12 @@ fn readable_size(bytes: u64) -> String {
#[cfg(test)]
mod disk_quota_tests {
use super::*;
use registry::schema::{
enums::StorageQuota,
structs::{Domain, UserAccount},
use registry::{
schema::{
enums::{DkimSignatureType, StorageQuota},
structs::{DkimManagementProperties, Domain, UserAccount},
},
types::map::Map,
};
fn account(tenant: Option<u64>, quota: u64) -> Object {
@@ -391,6 +449,62 @@ mod disk_quota_tests {
);
}
fn dkim_domain(tenant: Option<u64>, algorithms: &[DkimSignatureType]) -> Object {
Object::from(Domain {
member_tenant_id: tenant.map(Id::from),
dkim_management: DkimManagement::Automatic(DkimManagementProperties {
algorithms: Map::new(algorithms.to_vec()),
..Default::default()
}),
..Default::default()
})
}
#[test]
fn writes_that_grow_a_tenants_counts_take_its_lock() {
use DkimSignatureType::*;
let tenant = Some(Id::from(7u64));
// A new counted object in a tenant, and one in none
assert_eq!(locked_tenant(None, &account(Some(7), 0)), tenant);
assert_eq!(locked_tenant(None, &account(None, 0)), None);
assert_eq!(locked_tenant(None, &domain(Some(7))), tenant);
// An edit that doesn't join or add keys takes none
let staying = account(Some(7), 0);
assert_eq!(locked_tenant(Some(&staying), &staying), None);
// Joining from no tenant or another
assert_eq!(locked_tenant(Some(&account(None, 0)), &staying), tenant);
assert_eq!(
locked_tenant(Some(&domain(Some(8))), &domain(Some(7))),
tenant
);
// A domain adding DKIM algorithms, or not
let one = dkim_domain(Some(7), &[Dkim1RsaSha256]);
let two = dkim_domain(Some(7), &[Dkim1RsaSha256, Dkim1Ed25519Sha256]);
assert_eq!(locked_tenant(Some(&one), &two), tenant);
assert_eq!(locked_tenant(Some(&two), &one), None);
assert_eq!(locked_tenant(Some(&two), &two), None);
// Disk quota growth still locks
assert_eq!(locked_tenant(Some(&staying), &account(Some(7), 10)), tenant);
}
#[test]
fn added_dkim_keys_counts_new_algorithms_only() {
use DkimSignatureType::*;
let one = dkim_domain(Some(7), &[Dkim1RsaSha256]);
let two = dkim_domain(Some(7), &[Dkim1RsaSha256, Dkim2Ed25519Sha256]);
assert_eq!(added_dkim_keys(None, &two), Some(2));
assert_eq!(added_dkim_keys(Some(&one), &two), Some(1));
assert_eq!(added_dkim_keys(Some(&two), &one), Some(0));
// Manual to automatic adds them all; manual adds none
let manual = Object::from(Domain {
member_tenant_id: Some(Id::from(7u64)),
dkim_management: DkimManagement::Manual,
..Default::default()
});
assert_eq!(added_dkim_keys(Some(&manual), &two), Some(2));
assert_eq!(added_dkim_keys(Some(&two), &manual), None);
}
#[test]
fn the_refusal_says_what_is_left() {
const GIB: u64 = 1024 * 1024 * 1024;
+9 -10
View File
@@ -48,8 +48,6 @@ use trc::AddContext;
use types::id::Id;
use utils::snowflake::SnowflakeIdGenerator;
use crate::inbuxa::tenant_disk_lock;
type DValue = Value<'static, P, DeletedAccountValue>;
const ALL: &[P] = &[
@@ -449,20 +447,21 @@ async fn restore_account(
revision: 0,
};
// inbuxa-admin#257: its disk quota must still fit what the tenant has
// left to give, checked and saved under the tenant's lock
let disk_lock = inbuxa_features::tenancy::writes::disk_quota_tenant(None, &object)
// MT-17, inbuxa-admin#257: it must still fit the tenant's count limits
// and the disk space it has left to give, checked and saved under the
// tenant's lock
let tenant_lock = inbuxa_features::tenancy::writes::locked_tenant(None, &object)
.map(|tenant_id| tenant_id.document_id());
if let Some(tenant_id) = disk_lock
&& !tenant_disk_lock::acquire(server, tenant_id).await?
if let Some(tenant_id) = tenant_lock
&& !server.lock_tenant_writes(tenant_id).await?
{
return Ok(Err(
SetError::forbidden().with_description(tenant_disk_lock::BUSY)
SetError::forbidden().with_description(common::tenant_lock::BUSY)
));
}
let restored = restore_checked(server, id, kept, object).await;
if let Some(tenant_id) = disk_lock {
tenant_disk_lock::release(server, tenant_id).await;
if let Some(tenant_id) = tenant_lock {
server.unlock_tenant_writes(tenant_id).await;
}
restored
}
-1
View File
@@ -44,7 +44,6 @@ pub mod sharing_policy;
pub mod store_check; // inbuxa: checking the store's blobs
pub mod store_snapshot; // inbuxa: a snapshot of the store
pub mod telemetry;
pub mod tenant_disk_lock; // inbuxa: inbuxa-admin#257
pub mod tenant_protocol_policy;
pub mod undelete;
pub mod webhook_test;
@@ -1,59 +0,0 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! inbuxa-admin#257: one write at a time may grow a tenant's allocated disk
//! space. The lock lives in the in-memory store, so it holds across a
//! cluster: without it, two accounts given quotas at the same moment could
//! each fit the space left and together pass the tenant's limit. Taken
//! before the checks and released once the write is saved; one left behind
//! by a failed request expires on its own.
use common::{KV_LOCK_TASK, Server};
use std::time::Duration;
/// How long a lock outlives a request that never releases it.
const LOCK_SECONDS: u64 = 30;
/// How long to wait for another write to the same tenant, in all.
const WAIT_TRIES: u32 = 100;
const WAIT_STEP: Duration = Duration::from_millis(50);
/// What a write that couldn't get the lock is told.
pub const BUSY: &str = "Another change to this tenant's disk quotas is being saved. Try again.";
fn key(tenant_id: u32) -> Vec<u8> {
[b"tenant-disk-quota:".as_slice(), &tenant_id.to_be_bytes()].concat()
}
/// Takes the tenant's lock, waiting up to five seconds for a write that
/// holds it. `false` when it's still held.
pub async fn acquire(server: &Server, tenant_id: u32) -> trc::Result<bool> {
let key = key(tenant_id);
for _ in 0..WAIT_TRIES {
if server
.in_memory_store()
.try_lock(KV_LOCK_TASK, &key, LOCK_SECONDS)
.await?
{
return Ok(true);
}
tokio::time::sleep(WAIT_STEP).await;
}
Ok(false)
}
/// Releases the tenant's lock. A failure only leaves it to expire.
pub async fn release(server: &Server, tenant_id: u32) {
if let Err(err) = server
.in_memory_store()
.remove_lock(KV_LOCK_TASK, &key(tenant_id))
.await
{
trc::error!(
err.details("Failed to release a tenant's disk-quota lock")
.ctx(trc::Key::Id, tenant_id)
);
}
}
+13 -14
View File
@@ -379,12 +379,12 @@ impl RegistrySet for Server {
// Process modifications
let mut cache_invalidator = CacheInvalidationBuilder::default();
// inbuxa: inbuxa-admin#257: the tenant disk-quota lock the last
// write took, released once it's saved or refused
let mut disk_lock: Option<u32> = None;
// inbuxa: MT-17, inbuxa-admin#257: the tenant write lock the
// last write took, released once it's saved or refused
let mut tenant_lock: Option<u32> = None;
'outer: for (modification, mut value, mut new_object) in modifications {
if let Some(tenant_id) = disk_lock.take() {
crate::inbuxa::tenant_disk_lock::release(self, tenant_id).await;
if let Some(tenant_id) = tenant_lock.take() {
self.unlock_tenant_writes(tenant_id).await;
}
// Initial validations
let is_create = matches!(modification, Modification::Create { .. });
@@ -662,21 +662,20 @@ impl RegistrySet for Server {
Modification::Create { .. } => (None, None),
};
// inbuxa: inbuxa-admin#257: a write that could grow the
// tenant's allocated disk space holds its lock until saved
// inbuxa: MT-17, inbuxa-admin#257: a write that could grow the
// tenant's counted totals holds its lock until saved
if let Some(tenant_id) =
inbuxa_features::tenancy::writes::disk_quota_tenant(stored, &new_object)
inbuxa_features::tenancy::writes::locked_tenant(stored, &new_object)
{
let tenant_id = tenant_id.document_id();
if !crate::inbuxa::tenant_disk_lock::acquire(self, tenant_id).await? {
if !self.lock_tenant_writes(tenant_id).await? {
set.failed(
modification,
SetError::forbidden()
.with_description(crate::inbuxa::tenant_disk_lock::BUSY),
SetError::forbidden().with_description(common::tenant_lock::BUSY),
);
continue 'outer;
}
disk_lock = Some(tenant_id);
tenant_lock = Some(tenant_id);
}
let after_save = match inbuxa_features::tenancy::writes::check(
self.registry(),
@@ -972,8 +971,8 @@ impl RegistrySet for Server {
set.server.notify_task_queue();
}
}
if let Some(tenant_id) = disk_lock.take() {
crate::inbuxa::tenant_disk_lock::release(self, tenant_id).await;
if let Some(tenant_id) = tenant_lock.take() {
self.unlock_tenant_writes(tenant_id).await;
}
// Process destroy
+42
View File
@@ -846,6 +846,48 @@ pub async fn test(test: &mut TestServer) {
.await
.assert_type(SetErrorType::OverQuota)
.assert_description_contains("maxDkimKeys");
// ... and so do the keys an update makes the DKIM task generate
admin
.registry_update_object_expect_err(
ObjectType::Domain,
q_domain,
json!({ Property::DkimManagement: Domain::default().dkim_management }),
)
.await
.assert_type(SetErrorType::OverQuota)
.assert_description_contains("maxDkimKeys");
// An object moving into the tenant counts like a new one, not only a
// domain's (MT-17)
let q_roles = admin
.registry_query_ids(
ObjectType::Role,
[(Property::MemberTenantId, q_id.to_string())],
Vec::<&str>::new(),
)
.await
.len() as u64;
admin
.set_tenant_quota(q_id, TenantStorageQuota::MaxRoles, q_roles)
.await;
let loose_role = admin
.registry_create_object(structs::Role {
description: "joins tenant Q".to_string(),
..Default::default()
})
.await;
admin
.registry_update_object_expect_err(
ObjectType::Role,
loose_role,
json!({ Property::MemberTenantId: q_id }),
)
.await
.assert_type(SetErrorType::OverQuota)
.assert_description_contains("maxRoles");
admin
.registry_destroy(ObjectType::Role, [loose_role])
.await
.assert_destroyed(&[loose_role]);
// Acceptance test 13: the tenant's disk limit bounds all its members
// together (MT-19, MT-20)