Hold tenant count limits under the tenant's write lock (#498) #501
No files matched your search
Vendored
+47
-16
@@ -248,14 +248,20 @@ impl Server {
|
||||
..Default::default()
|
||||
}));
|
||||
|
||||
// inbuxa: DIR-15
|
||||
self.check_tenant_limits(&account).await?;
|
||||
match self
|
||||
.registry()
|
||||
.write(RegistryWrite::insert(&account))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
{
|
||||
// inbuxa: DIR-15, MT-17: checked and saved under the tenant's lock
|
||||
let tenant_lock = self.lock_for_tenant_limits(&account).await?;
|
||||
let written = match self.check_tenant_limits(&account).await {
|
||||
Ok(()) => self
|
||||
.registry()
|
||||
.write(RegistryWrite::insert(&account))
|
||||
.await
|
||||
.caused_by(trc::location!()),
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
if let Some(tenant_id) = tenant_lock {
|
||||
self.unlock_tenant_writes(tenant_id).await;
|
||||
}
|
||||
match written? {
|
||||
RegistryWriteResult::Success(id) => {
|
||||
let mut invalidator = CacheInvalidationBuilder::default();
|
||||
invalidator.process_create(&account);
|
||||
@@ -420,14 +426,20 @@ impl Server {
|
||||
..Default::default()
|
||||
}));
|
||||
|
||||
// inbuxa: DIR-15
|
||||
self.check_tenant_limits(&account).await?;
|
||||
match self
|
||||
.registry()
|
||||
.write(RegistryWrite::insert(&account))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
{
|
||||
// inbuxa: DIR-15, MT-17: checked and saved under the tenant's lock
|
||||
let tenant_lock = self.lock_for_tenant_limits(&account).await?;
|
||||
let written = match self.check_tenant_limits(&account).await {
|
||||
Ok(()) => self
|
||||
.registry()
|
||||
.write(RegistryWrite::insert(&account))
|
||||
.await
|
||||
.caused_by(trc::location!()),
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
if let Some(tenant_id) = tenant_lock {
|
||||
self.unlock_tenant_writes(tenant_id).await;
|
||||
}
|
||||
match written? {
|
||||
RegistryWriteResult::Success(id) => {
|
||||
let mut invalidator = CacheInvalidationBuilder::default();
|
||||
invalidator.process_create(&account);
|
||||
@@ -477,6 +489,25 @@ impl Server {
|
||||
|
||||
/// inbuxa: DIR-15, MT-3, MT-17: an object created from a directory
|
||||
/// passes the same tenant checks as one created over JMAP.
|
||||
/// MT-17: takes the lock of the tenant a new directory account joins,
|
||||
/// so parallel sign-ins can't each find room under its limits. Returns
|
||||
/// the tenant to unlock, `None` for an account in no tenant.
|
||||
async fn lock_for_tenant_limits(&self, object: &Object) -> trc::Result<Option<u32>> {
|
||||
let Some(tenant_id) = inbuxa_features::tenancy::writes::locked_tenant(None, object)
|
||||
.map(|tenant_id| tenant_id.document_id())
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if self.lock_tenant_writes(tenant_id).await? {
|
||||
Ok(Some(tenant_id))
|
||||
} else {
|
||||
Err(trc::AuthEvent::Error
|
||||
.into_err()
|
||||
.details(crate::tenant_lock::BUSY)
|
||||
.reason("The directory's account can't be created"))
|
||||
}
|
||||
}
|
||||
|
||||
async fn check_tenant_limits(&self, object: &Object) -> trc::Result<()> {
|
||||
match inbuxa_features::tenancy::writes::check(self.registry(), None, None, object).await? {
|
||||
Ok(_) => Ok(()),
|
||||
|
||||
@@ -82,6 +82,7 @@ pub mod scripts;
|
||||
pub mod sharing;
|
||||
pub mod storage;
|
||||
pub mod telemetry;
|
||||
pub mod tenant_lock; // inbuxa: MT-17, inbuxa-admin#257
|
||||
|
||||
pub use psl;
|
||||
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! One write at a time may grow a tenant's counted totals: its objects
|
||||
//! against the count limits (MT-17) and the disk space its accounts are
|
||||
//! given (inbuxa-admin#257). Without it, two writes at the same moment could
|
||||
//! each find room for one more and together pass the limit. The lock lives
|
||||
//! in the in-memory store, so it holds across a cluster. A write takes it
|
||||
//! before its tenancy checks and releases it once saved; one left behind by
|
||||
//! a failed request expires on its own. A write holds at most one tenant's
|
||||
//! lock, so two can't wait on each other.
|
||||
|
||||
use crate::{KV_LOCK_TASK, Server};
|
||||
use std::time::Duration;
|
||||
|
||||
/// How long a lock outlives a request that never releases it.
|
||||
const LOCK_SECONDS: u64 = 30;
|
||||
/// How long to wait for another write to the same tenant, in all.
|
||||
const WAIT_TRIES: u32 = 100;
|
||||
const WAIT_STEP: Duration = Duration::from_millis(50);
|
||||
|
||||
/// What a write that couldn't get the lock is told.
|
||||
pub const BUSY: &str = "Another change to this tenant's limits is being saved. Try again.";
|
||||
|
||||
// The key the disk-quota lock used, kept so nodes on either version share
|
||||
// one lock during a rolling upgrade
|
||||
fn key(tenant_id: u32) -> Vec<u8> {
|
||||
[b"tenant-disk-quota:".as_slice(), &tenant_id.to_be_bytes()].concat()
|
||||
}
|
||||
|
||||
impl Server {
|
||||
/// Takes the tenant's write lock, waiting up to five seconds for a write
|
||||
/// that holds it. `false` when it's still held.
|
||||
pub async fn lock_tenant_writes(&self, tenant_id: u32) -> trc::Result<bool> {
|
||||
let key = key(tenant_id);
|
||||
for _ in 0..WAIT_TRIES {
|
||||
if self
|
||||
.in_memory_store()
|
||||
.try_lock(KV_LOCK_TASK, &key, LOCK_SECONDS)
|
||||
.await?
|
||||
{
|
||||
return Ok(true);
|
||||
}
|
||||
tokio::time::sleep(WAIT_STEP).await;
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Releases the tenant's write lock. A failure only leaves it to expire.
|
||||
pub async fn unlock_tenant_writes(&self, tenant_id: u32) {
|
||||
if let Err(err) = self
|
||||
.in_memory_store()
|
||||
.remove_lock(KV_LOCK_TASK, &key(tenant_id))
|
||||
.await
|
||||
{
|
||||
trc::error!(
|
||||
err.details("Failed to release a tenant's write lock")
|
||||
.ctx(trc::Key::Id, tenant_id)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,14 +10,15 @@
|
||||
//! - MT-7: a principal created without a tenant takes its domain's.
|
||||
//! - MT-8: a domain moves only through no tenant, taking its principals and
|
||||
//! keys in with it, and only when its people let it out.
|
||||
//! - MT-17: creating an object, or moving a domain in, stays within the
|
||||
//! tenant's count limits, and the server emits `limit.tenant-quota` when
|
||||
//! it doesn't.
|
||||
//! - MT-17: creating an object, moving one in, or moving a domain in stays
|
||||
//! within the tenant's count limits, a domain's DKIM keys included (MT-9),
|
||||
//! and the server emits `limit.tenant-quota` when it doesn't.
|
||||
//! - A tenant's accounts' disk quotas add up to no more than the tenant's
|
||||
//! (inbuxa-admin#257), checked by `check_disk_quota` on JMAP and SCIM
|
||||
//! writes and on a restore, and by the domain-move plan. A write that
|
||||
//! could grow the total holds the tenant's disk-quota lock
|
||||
//! (`disk_quota_tenant`) from the check until it's saved.
|
||||
//! writes and on a restore, and by the domain-move plan.
|
||||
//! - A write that could grow a tenant's counted totals, its objects or its
|
||||
//! allocated disk space, holds the tenant's write lock (`locked_tenant`)
|
||||
//! from its checks until it's saved.
|
||||
|
||||
use crate::tenancy::{
|
||||
domain_move::{self, Move, Refusal},
|
||||
@@ -120,10 +121,12 @@ pub async fn check(
|
||||
return Ok(Err(foreign_key(foreign)));
|
||||
}
|
||||
|
||||
// MT-17: count limits on a new object
|
||||
if old.is_none()
|
||||
&& let Some(tenant_id) = tenant
|
||||
// MT-17: count limits on an object joining the tenant: a new one, or one
|
||||
// whose tenant changes. A domain's move is counted by its plan (MT-8).
|
||||
if let Some(tenant_id) = tenant
|
||||
&& let Some(quota) = quota::limit_for(&new.inner)
|
||||
&& joins_tenant(old, new)
|
||||
&& !matches!(new.inner, ObjectInner::Domain(_) if old.is_some())
|
||||
{
|
||||
let limit = limits(registry, tenant_id).await?;
|
||||
if let Err(reached) =
|
||||
@@ -131,33 +134,85 @@ pub async fn check(
|
||||
{
|
||||
return Ok(Err(over_quota(reached, tenant_id)));
|
||||
}
|
||||
}
|
||||
|
||||
// A new domain's generated DKIM keys, one per algorithm, count too (MT-9)
|
||||
if let ObjectInner::Domain(Domain {
|
||||
dkim_management: DkimManagement::Automatic(dkim),
|
||||
..
|
||||
}) = &new.inner
|
||||
// MT-9: a domain's generated DKIM keys, one per algorithm, count too:
|
||||
// all of them on a new domain, and those it adds on an update, which the
|
||||
// DKIM task then generates. Rotation, which replaces a key with a new one,
|
||||
// isn't counted again.
|
||||
if let Some(tenant_id) = tenant
|
||||
&& let Some(keys) = added_dkim_keys(old, new)
|
||||
&& keys > 0
|
||||
{
|
||||
let limit = limits(registry, tenant_id).await?;
|
||||
let quota = TenantStorageQuota::MaxDkimKeys;
|
||||
if let Err(reached) =
|
||||
quota::check(registry, tenant_id.document_id(), quota, limit(quota), keys).await?
|
||||
{
|
||||
let quota = TenantStorageQuota::MaxDkimKeys;
|
||||
let keys = dkim.algorithms.iter().count() as u64;
|
||||
if let Err(reached) =
|
||||
quota::check(registry, tenant_id.document_id(), quota, limit(quota), keys).await?
|
||||
{
|
||||
return Ok(Err(over_quota(reached, tenant_id)));
|
||||
}
|
||||
return Ok(Err(over_quota(reached, tenant_id)));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Ok(after))
|
||||
}
|
||||
|
||||
/// Whether `new` joins its tenant with this write: it's new, or its tenant
|
||||
/// changes.
|
||||
fn joins_tenant(old: Option<&Object>, new: &Object) -> bool {
|
||||
old.is_none_or(|old| old.inner.member_tenant_id() != new.inner.member_tenant_id())
|
||||
}
|
||||
|
||||
/// How many DKIM keys a write to a domain makes the DKIM task generate: one
|
||||
/// per algorithm on a new automatic domain, or per algorithm an update adds
|
||||
/// (an update into automatic mode adds them all). `None` for anything else.
|
||||
fn added_dkim_keys(old: Option<&Object>, new: &Object) -> Option<u64> {
|
||||
let ObjectInner::Domain(Domain {
|
||||
dkim_management: DkimManagement::Automatic(dkim),
|
||||
..
|
||||
}) = &new.inner
|
||||
else {
|
||||
return None;
|
||||
};
|
||||
let before = match old.map(|old| &old.inner) {
|
||||
Some(ObjectInner::Domain(Domain {
|
||||
dkim_management: DkimManagement::Automatic(before),
|
||||
..
|
||||
})) => Some(&before.algorithms),
|
||||
_ => None,
|
||||
};
|
||||
Some(
|
||||
dkim.algorithms
|
||||
.iter()
|
||||
.filter(|algorithm| before.is_none_or(|before| !before.contains(algorithm)))
|
||||
.count() as u64,
|
||||
)
|
||||
}
|
||||
|
||||
/// The tenant whose counted totals this write could grow, if any: an
|
||||
/// object joining a tenant whose kind it limits (MT-17), a domain adding
|
||||
/// DKIM keys (MT-9), or a write that could grow its allocated disk space
|
||||
/// (`disk_quota_tenant`). Such a write takes the tenant's write lock
|
||||
/// (common's `lock_tenant_writes`) before its tenancy checks and keeps it
|
||||
/// until it's saved, so two writes at once can't each find room for one
|
||||
/// more and together pass the limit.
|
||||
pub fn locked_tenant(old: Option<&Object>, new: &Object) -> Option<Id> {
|
||||
let tenant = new.inner.member_tenant_id()?;
|
||||
let counted = quota::limit_for(&new.inner).is_some() && joins_tenant(old, new);
|
||||
let dkim = added_dkim_keys(old, new).is_some_and(|keys| keys > 0);
|
||||
if counted || dkim {
|
||||
Some(tenant)
|
||||
} else {
|
||||
disk_quota_tenant(old, new)
|
||||
}
|
||||
}
|
||||
|
||||
/// The tenant whose allocated disk space this write could grow, if any
|
||||
/// (inbuxa-admin#257): an account given a disk quota, or a bigger one, or
|
||||
/// joining a tenant with one; or a domain moving into a tenant, which brings
|
||||
/// its accounts. Such a write takes the tenant's disk-quota lock before its
|
||||
/// checks and keeps it until it's saved, so two writes at once can't each
|
||||
/// fit the space left and together pass it. Lowering or clearing a quota,
|
||||
/// or any other change, needs no lock.
|
||||
/// its accounts. Such a write takes the tenant's write lock (`locked_tenant`)
|
||||
/// before its checks and keeps it until it's saved, so two writes at once
|
||||
/// can't each fit the space left and together pass it. Lowering or clearing
|
||||
/// a quota, or any other change, needs no lock.
|
||||
pub fn disk_quota_tenant(old: Option<&Object>, new: &Object) -> Option<Id> {
|
||||
let tenant = new.inner.member_tenant_id()?;
|
||||
let moved_in = old.is_none_or(|old| old.inner.member_tenant_id() != Some(tenant));
|
||||
@@ -178,7 +233,7 @@ pub fn disk_quota_tenant(old: Option<&Object>, new: &Object) -> Option<Id> {
|
||||
/// inbuxa-admin#257: a tenant's accounts' disk quotas add up to no more than
|
||||
/// the tenant's own, so it can't give out more space than it has, in one
|
||||
/// account or across many. Run on a JMAP or SCIM write and on a restore,
|
||||
/// after `check`, holding the lock `disk_quota_tenant` names. `id` is the
|
||||
/// after `check`, holding the lock `locked_tenant` names. `id` is the
|
||||
/// account on an update; `restoring` words the refusal for a restore,
|
||||
/// whose quota can't be edited first. Only a write that grows the total is checked, so a
|
||||
/// tenant already over a lowered limit can still edit its accounts and
|
||||
@@ -332,9 +387,12 @@ fn readable_size(bytes: u64) -> String {
|
||||
#[cfg(test)]
|
||||
mod disk_quota_tests {
|
||||
use super::*;
|
||||
use registry::schema::{
|
||||
enums::StorageQuota,
|
||||
structs::{Domain, UserAccount},
|
||||
use registry::{
|
||||
schema::{
|
||||
enums::{DkimSignatureType, StorageQuota},
|
||||
structs::{DkimManagementProperties, Domain, UserAccount},
|
||||
},
|
||||
types::map::Map,
|
||||
};
|
||||
|
||||
fn account(tenant: Option<u64>, quota: u64) -> Object {
|
||||
@@ -391,6 +449,62 @@ mod disk_quota_tests {
|
||||
);
|
||||
}
|
||||
|
||||
fn dkim_domain(tenant: Option<u64>, algorithms: &[DkimSignatureType]) -> Object {
|
||||
Object::from(Domain {
|
||||
member_tenant_id: tenant.map(Id::from),
|
||||
dkim_management: DkimManagement::Automatic(DkimManagementProperties {
|
||||
algorithms: Map::new(algorithms.to_vec()),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn writes_that_grow_a_tenants_counts_take_its_lock() {
|
||||
use DkimSignatureType::*;
|
||||
let tenant = Some(Id::from(7u64));
|
||||
// A new counted object in a tenant, and one in none
|
||||
assert_eq!(locked_tenant(None, &account(Some(7), 0)), tenant);
|
||||
assert_eq!(locked_tenant(None, &account(None, 0)), None);
|
||||
assert_eq!(locked_tenant(None, &domain(Some(7))), tenant);
|
||||
// An edit that doesn't join or add keys takes none
|
||||
let staying = account(Some(7), 0);
|
||||
assert_eq!(locked_tenant(Some(&staying), &staying), None);
|
||||
// Joining from no tenant or another
|
||||
assert_eq!(locked_tenant(Some(&account(None, 0)), &staying), tenant);
|
||||
assert_eq!(
|
||||
locked_tenant(Some(&domain(Some(8))), &domain(Some(7))),
|
||||
tenant
|
||||
);
|
||||
// A domain adding DKIM algorithms, or not
|
||||
let one = dkim_domain(Some(7), &[Dkim1RsaSha256]);
|
||||
let two = dkim_domain(Some(7), &[Dkim1RsaSha256, Dkim1Ed25519Sha256]);
|
||||
assert_eq!(locked_tenant(Some(&one), &two), tenant);
|
||||
assert_eq!(locked_tenant(Some(&two), &one), None);
|
||||
assert_eq!(locked_tenant(Some(&two), &two), None);
|
||||
// Disk quota growth still locks
|
||||
assert_eq!(locked_tenant(Some(&staying), &account(Some(7), 10)), tenant);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn added_dkim_keys_counts_new_algorithms_only() {
|
||||
use DkimSignatureType::*;
|
||||
let one = dkim_domain(Some(7), &[Dkim1RsaSha256]);
|
||||
let two = dkim_domain(Some(7), &[Dkim1RsaSha256, Dkim2Ed25519Sha256]);
|
||||
assert_eq!(added_dkim_keys(None, &two), Some(2));
|
||||
assert_eq!(added_dkim_keys(Some(&one), &two), Some(1));
|
||||
assert_eq!(added_dkim_keys(Some(&two), &one), Some(0));
|
||||
// Manual to automatic adds them all; manual adds none
|
||||
let manual = Object::from(Domain {
|
||||
member_tenant_id: Some(Id::from(7u64)),
|
||||
dkim_management: DkimManagement::Manual,
|
||||
..Default::default()
|
||||
});
|
||||
assert_eq!(added_dkim_keys(Some(&manual), &two), Some(2));
|
||||
assert_eq!(added_dkim_keys(Some(&two), &manual), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_refusal_says_what_is_left() {
|
||||
const GIB: u64 = 1024 * 1024 * 1024;
|
||||
|
||||
@@ -48,8 +48,6 @@ use trc::AddContext;
|
||||
use types::id::Id;
|
||||
use utils::snowflake::SnowflakeIdGenerator;
|
||||
|
||||
use crate::inbuxa::tenant_disk_lock;
|
||||
|
||||
type DValue = Value<'static, P, DeletedAccountValue>;
|
||||
|
||||
const ALL: &[P] = &[
|
||||
@@ -449,20 +447,21 @@ async fn restore_account(
|
||||
revision: 0,
|
||||
};
|
||||
|
||||
// inbuxa-admin#257: its disk quota must still fit what the tenant has
|
||||
// left to give, checked and saved under the tenant's lock
|
||||
let disk_lock = inbuxa_features::tenancy::writes::disk_quota_tenant(None, &object)
|
||||
// MT-17, inbuxa-admin#257: it must still fit the tenant's count limits
|
||||
// and the disk space it has left to give, checked and saved under the
|
||||
// tenant's lock
|
||||
let tenant_lock = inbuxa_features::tenancy::writes::locked_tenant(None, &object)
|
||||
.map(|tenant_id| tenant_id.document_id());
|
||||
if let Some(tenant_id) = disk_lock
|
||||
&& !tenant_disk_lock::acquire(server, tenant_id).await?
|
||||
if let Some(tenant_id) = tenant_lock
|
||||
&& !server.lock_tenant_writes(tenant_id).await?
|
||||
{
|
||||
return Ok(Err(
|
||||
SetError::forbidden().with_description(tenant_disk_lock::BUSY)
|
||||
SetError::forbidden().with_description(common::tenant_lock::BUSY)
|
||||
));
|
||||
}
|
||||
let restored = restore_checked(server, id, kept, object).await;
|
||||
if let Some(tenant_id) = disk_lock {
|
||||
tenant_disk_lock::release(server, tenant_id).await;
|
||||
if let Some(tenant_id) = tenant_lock {
|
||||
server.unlock_tenant_writes(tenant_id).await;
|
||||
}
|
||||
restored
|
||||
}
|
||||
|
||||
@@ -44,7 +44,6 @@ pub mod sharing_policy;
|
||||
pub mod store_check; // inbuxa: checking the store's blobs
|
||||
pub mod store_snapshot; // inbuxa: a snapshot of the store
|
||||
pub mod telemetry;
|
||||
pub mod tenant_disk_lock; // inbuxa: inbuxa-admin#257
|
||||
pub mod tenant_protocol_policy;
|
||||
pub mod undelete;
|
||||
pub mod webhook_test;
|
||||
@@ -1,59 +0,0 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! inbuxa-admin#257: one write at a time may grow a tenant's allocated disk
|
||||
//! space. The lock lives in the in-memory store, so it holds across a
|
||||
//! cluster: without it, two accounts given quotas at the same moment could
|
||||
//! each fit the space left and together pass the tenant's limit. Taken
|
||||
//! before the checks and released once the write is saved; one left behind
|
||||
//! by a failed request expires on its own.
|
||||
|
||||
use common::{KV_LOCK_TASK, Server};
|
||||
use std::time::Duration;
|
||||
|
||||
/// How long a lock outlives a request that never releases it.
|
||||
const LOCK_SECONDS: u64 = 30;
|
||||
/// How long to wait for another write to the same tenant, in all.
|
||||
const WAIT_TRIES: u32 = 100;
|
||||
const WAIT_STEP: Duration = Duration::from_millis(50);
|
||||
|
||||
/// What a write that couldn't get the lock is told.
|
||||
pub const BUSY: &str = "Another change to this tenant's disk quotas is being saved. Try again.";
|
||||
|
||||
fn key(tenant_id: u32) -> Vec<u8> {
|
||||
[b"tenant-disk-quota:".as_slice(), &tenant_id.to_be_bytes()].concat()
|
||||
}
|
||||
|
||||
/// Takes the tenant's lock, waiting up to five seconds for a write that
|
||||
/// holds it. `false` when it's still held.
|
||||
pub async fn acquire(server: &Server, tenant_id: u32) -> trc::Result<bool> {
|
||||
let key = key(tenant_id);
|
||||
for _ in 0..WAIT_TRIES {
|
||||
if server
|
||||
.in_memory_store()
|
||||
.try_lock(KV_LOCK_TASK, &key, LOCK_SECONDS)
|
||||
.await?
|
||||
{
|
||||
return Ok(true);
|
||||
}
|
||||
tokio::time::sleep(WAIT_STEP).await;
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Releases the tenant's lock. A failure only leaves it to expire.
|
||||
pub async fn release(server: &Server, tenant_id: u32) {
|
||||
if let Err(err) = server
|
||||
.in_memory_store()
|
||||
.remove_lock(KV_LOCK_TASK, &key(tenant_id))
|
||||
.await
|
||||
{
|
||||
trc::error!(
|
||||
err.details("Failed to release a tenant's disk-quota lock")
|
||||
.ctx(trc::Key::Id, tenant_id)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -379,12 +379,12 @@ impl RegistrySet for Server {
|
||||
|
||||
// Process modifications
|
||||
let mut cache_invalidator = CacheInvalidationBuilder::default();
|
||||
// inbuxa: inbuxa-admin#257: the tenant disk-quota lock the last
|
||||
// write took, released once it's saved or refused
|
||||
let mut disk_lock: Option<u32> = None;
|
||||
// inbuxa: MT-17, inbuxa-admin#257: the tenant write lock the
|
||||
// last write took, released once it's saved or refused
|
||||
let mut tenant_lock: Option<u32> = None;
|
||||
'outer: for (modification, mut value, mut new_object) in modifications {
|
||||
if let Some(tenant_id) = disk_lock.take() {
|
||||
crate::inbuxa::tenant_disk_lock::release(self, tenant_id).await;
|
||||
if let Some(tenant_id) = tenant_lock.take() {
|
||||
self.unlock_tenant_writes(tenant_id).await;
|
||||
}
|
||||
// Initial validations
|
||||
let is_create = matches!(modification, Modification::Create { .. });
|
||||
@@ -662,21 +662,20 @@ impl RegistrySet for Server {
|
||||
Modification::Create { .. } => (None, None),
|
||||
};
|
||||
|
||||
// inbuxa: inbuxa-admin#257: a write that could grow the
|
||||
// tenant's allocated disk space holds its lock until saved
|
||||
// inbuxa: MT-17, inbuxa-admin#257: a write that could grow the
|
||||
// tenant's counted totals holds its lock until saved
|
||||
if let Some(tenant_id) =
|
||||
inbuxa_features::tenancy::writes::disk_quota_tenant(stored, &new_object)
|
||||
inbuxa_features::tenancy::writes::locked_tenant(stored, &new_object)
|
||||
{
|
||||
let tenant_id = tenant_id.document_id();
|
||||
if !crate::inbuxa::tenant_disk_lock::acquire(self, tenant_id).await? {
|
||||
if !self.lock_tenant_writes(tenant_id).await? {
|
||||
set.failed(
|
||||
modification,
|
||||
SetError::forbidden()
|
||||
.with_description(crate::inbuxa::tenant_disk_lock::BUSY),
|
||||
SetError::forbidden().with_description(common::tenant_lock::BUSY),
|
||||
);
|
||||
continue 'outer;
|
||||
}
|
||||
disk_lock = Some(tenant_id);
|
||||
tenant_lock = Some(tenant_id);
|
||||
}
|
||||
let after_save = match inbuxa_features::tenancy::writes::check(
|
||||
self.registry(),
|
||||
@@ -972,8 +971,8 @@ impl RegistrySet for Server {
|
||||
set.server.notify_task_queue();
|
||||
}
|
||||
}
|
||||
if let Some(tenant_id) = disk_lock.take() {
|
||||
crate::inbuxa::tenant_disk_lock::release(self, tenant_id).await;
|
||||
if let Some(tenant_id) = tenant_lock.take() {
|
||||
self.unlock_tenant_writes(tenant_id).await;
|
||||
}
|
||||
|
||||
// Process destroy
|
||||
|
||||
@@ -846,6 +846,48 @@ pub async fn test(test: &mut TestServer) {
|
||||
.await
|
||||
.assert_type(SetErrorType::OverQuota)
|
||||
.assert_description_contains("maxDkimKeys");
|
||||
// ... and so do the keys an update makes the DKIM task generate
|
||||
admin
|
||||
.registry_update_object_expect_err(
|
||||
ObjectType::Domain,
|
||||
q_domain,
|
||||
json!({ Property::DkimManagement: Domain::default().dkim_management }),
|
||||
)
|
||||
.await
|
||||
.assert_type(SetErrorType::OverQuota)
|
||||
.assert_description_contains("maxDkimKeys");
|
||||
// An object moving into the tenant counts like a new one, not only a
|
||||
// domain's (MT-17)
|
||||
let q_roles = admin
|
||||
.registry_query_ids(
|
||||
ObjectType::Role,
|
||||
[(Property::MemberTenantId, q_id.to_string())],
|
||||
Vec::<&str>::new(),
|
||||
)
|
||||
.await
|
||||
.len() as u64;
|
||||
admin
|
||||
.set_tenant_quota(q_id, TenantStorageQuota::MaxRoles, q_roles)
|
||||
.await;
|
||||
let loose_role = admin
|
||||
.registry_create_object(structs::Role {
|
||||
description: "joins tenant Q".to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
admin
|
||||
.registry_update_object_expect_err(
|
||||
ObjectType::Role,
|
||||
loose_role,
|
||||
json!({ Property::MemberTenantId: q_id }),
|
||||
)
|
||||
.await
|
||||
.assert_type(SetErrorType::OverQuota)
|
||||
.assert_description_contains("maxRoles");
|
||||
admin
|
||||
.registry_destroy(ObjectType::Role, [loose_role])
|
||||
.await
|
||||
.assert_destroyed(&[loose_role]);
|
||||
|
||||
// Acceptance test 13: the tenant's disk limit bounds all its members
|
||||
// together (MT-19, MT-20)
|
||||
|
||||
Reference in new issue
Block a user