Keep a tenant's accounts' disk quotas within its own, together (inbuxa-admin#257) #497
No files matched your search
@@ -8,7 +8,8 @@
|
||||
//!
|
||||
//! A domain moves into a tenant only from no tenant, and out of one only back
|
||||
//! to no tenant. Moving in, its principals (accounts, groups, mailing lists)
|
||||
//! and DKIM keys move with it, after the tenant's limits are checked. Moving
|
||||
//! and DKIM keys move with it, after the tenant's limits are checked, its
|
||||
//! accounts' disk quotas included (inbuxa-admin#257). Moving
|
||||
//! out is refused while any principal on it is in the tenant; with none, its
|
||||
//! DKIM keys move out with it. A principal on it in a third tenant blocks
|
||||
//! either move, and so does any link the move would carry across a tenant
|
||||
@@ -44,6 +45,16 @@ pub enum Refusal {
|
||||
ForeignLink(ObjectId),
|
||||
/// The destination tenant's limit would be crossed.
|
||||
Limit(LimitReached),
|
||||
/// The accounts moving in have more disk quota between them than the
|
||||
/// destination has left to give (inbuxa-admin#257).
|
||||
DiskQuota {
|
||||
/// Already given to the tenant's accounts.
|
||||
allocated: u64,
|
||||
/// The moving accounts' quotas together.
|
||||
adding: u64,
|
||||
/// The tenant's maxDiskQuota.
|
||||
limit: u64,
|
||||
},
|
||||
}
|
||||
|
||||
/// A move that passed every check, ready to apply once the domain itself is
|
||||
@@ -88,6 +99,7 @@ pub async fn plan(
|
||||
// Sort out what links to the domain
|
||||
let mut carried = Vec::new();
|
||||
let mut adding: AHashMap<TenantStorageQuota, u64> = AHashMap::new();
|
||||
let mut adding_disk = 0u64;
|
||||
let mut blocking = Vec::new();
|
||||
let mut seen = AHashSet::new();
|
||||
for referrer in registry.linked_objects(domain).await? {
|
||||
@@ -117,6 +129,9 @@ pub async fn plan(
|
||||
if let Some(quota) = quota::limit_for_id(referrer, kind) {
|
||||
*adding.entry(quota).or_default() += 1;
|
||||
}
|
||||
if let Some(disk) = quota::account_disk_quota(&object.inner) {
|
||||
adding_disk = adding_disk.saturating_add(disk);
|
||||
}
|
||||
carried.push(referrer);
|
||||
}
|
||||
} else if moves_with_domain(object_type) && object_type != ObjectType::DkimSignature {
|
||||
@@ -168,6 +183,21 @@ pub async fn plan(
|
||||
return Ok(Err(Refusal::Limit(reached)));
|
||||
}
|
||||
}
|
||||
|
||||
// inbuxa-admin#257: and the disk space its accounts were given
|
||||
if adding_disk > 0
|
||||
&& let Some(limit) = limit(TenantStorageQuota::MaxDiskQuota).filter(|limit| *limit > 0)
|
||||
{
|
||||
let allocated =
|
||||
quota::allocated_disk(registry, tenant_id.document_id(), &AHashSet::new()).await?;
|
||||
if allocated.saturating_add(adding_disk) > limit {
|
||||
return Ok(Err(Refusal::DiskQuota {
|
||||
allocated,
|
||||
adding: adding_disk,
|
||||
limit,
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Ok(Move { to, carried }))
|
||||
|
||||
@@ -5,11 +5,13 @@
|
||||
*/
|
||||
|
||||
//! Tenant limits: how many of each kind of object a tenant may hold
|
||||
//! (MT-17, MT-18) and how much storage its members use (MT-20, MT-21).
|
||||
//! (MT-17, MT-18), how much storage its members use (MT-20, MT-21), and how
|
||||
//! much disk space its accounts are given (inbuxa-admin#257).
|
||||
|
||||
use ahash::AHashSet;
|
||||
use registry::{
|
||||
schema::{
|
||||
enums::TenantStorageQuota,
|
||||
enums::{StorageQuota, TenantStorageQuota},
|
||||
prelude::{ObjectInner, ObjectType, Property},
|
||||
structs::Account,
|
||||
},
|
||||
@@ -98,6 +100,50 @@ pub async fn count(
|
||||
.caused_by(trc::location!())
|
||||
}
|
||||
|
||||
/// An account's own disk quota, `None` when it has none (0 is none too).
|
||||
/// Users and groups both have one.
|
||||
pub fn account_disk_quota(object: &ObjectInner) -> Option<u64> {
|
||||
let quotas = match object {
|
||||
ObjectInner::Account(Account::User(obj)) => &obj.quotas,
|
||||
ObjectInner::Account(Account::Group(obj)) => &obj.quotas,
|
||||
_ => return None,
|
||||
};
|
||||
quotas
|
||||
.get(&StorageQuota::MaxDiskQuota)
|
||||
.copied()
|
||||
.filter(|quota| *quota > 0)
|
||||
}
|
||||
|
||||
/// The disk space a tenant has given its accounts: the sum of their own
|
||||
/// disk quotas, leaving out `except` (inbuxa-admin#257). Accounts with no
|
||||
/// quota share what's left and add nothing.
|
||||
pub async fn allocated_disk(
|
||||
registry: &RegistryStore,
|
||||
tenant_id: u32,
|
||||
except: &AHashSet<Id>,
|
||||
) -> trc::Result<u64> {
|
||||
let members = registry
|
||||
.query::<Vec<Id>>(RegistryQuery::new(ObjectType::Account).with_tenant(Some(tenant_id)))
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
let mut total = 0u64;
|
||||
for member in members {
|
||||
if except.contains(&member) {
|
||||
continue;
|
||||
}
|
||||
if let Some(object) = registry
|
||||
.get(ObjectId::new(ObjectType::Account, member))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
&& let Some(quota) = account_disk_quota(&object.inner)
|
||||
{
|
||||
total = total.saturating_add(quota);
|
||||
}
|
||||
}
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
/// A count limit that adding objects would cross.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct LimitReached {
|
||||
|
||||
@@ -13,19 +13,22 @@
|
||||
//! - MT-17: creating an object, or moving a domain in, stays within the
|
||||
//! tenant's count limits, and the server emits `limit.tenant-quota` when
|
||||
//! it doesn't.
|
||||
//! - An account's disk quota fits within its tenant's (inbuxa-admin#257),
|
||||
//! checked by `check_disk_quota` on JMAP and SCIM writes.
|
||||
//! - A tenant's accounts' disk quotas add up to no more than the tenant's
|
||||
//! (inbuxa-admin#257), checked by `check_disk_quota` on JMAP and SCIM
|
||||
//! writes and on a restore, and by the domain-move plan. A write that
|
||||
//! could grow the total holds the tenant's disk-quota lock
|
||||
//! (`disk_quota_tenant`) from the check until it's saved.
|
||||
|
||||
use crate::tenancy::{
|
||||
domain_move::{self, Move, Refusal},
|
||||
links,
|
||||
quota::{self, LimitReached},
|
||||
quota::{self, LimitReached, account_disk_quota},
|
||||
};
|
||||
use ahash::AHashSet;
|
||||
use jmap_proto::error::set::{SetError, SetErrorType};
|
||||
use registry::{
|
||||
schema::{
|
||||
enums::{StorageQuota, TenantStorageQuota},
|
||||
enums::TenantStorageQuota,
|
||||
prelude::{Object, ObjectInner, Property},
|
||||
structs::{Account, DkimManagement, Domain, Tenant},
|
||||
},
|
||||
@@ -65,19 +68,6 @@ pub async fn default_tenant(registry: &RegistryStore, object: &mut Object) -> tr
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// An account's own disk quota, `None` when it has none (0 is none too).
|
||||
fn account_disk_quota(object: &ObjectInner) -> Option<u64> {
|
||||
let quotas = match object {
|
||||
ObjectInner::Account(Account::User(obj)) => &obj.quotas,
|
||||
ObjectInner::Account(Account::Group(obj)) => &obj.quotas,
|
||||
_ => return None,
|
||||
};
|
||||
quotas
|
||||
.get(&StorageQuota::MaxDiskQuota)
|
||||
.copied()
|
||||
.filter(|quota| *quota > 0)
|
||||
}
|
||||
|
||||
/// A tenant's count limit, `None` when it has none.
|
||||
async fn limits(
|
||||
registry: &RegistryStore,
|
||||
@@ -161,31 +151,64 @@ pub async fn check(
|
||||
Ok(Ok(after))
|
||||
}
|
||||
|
||||
/// inbuxa-admin#257: an account's disk quota fits within its tenant's, so no
|
||||
/// member gets more space than the whole tenant has. Run on a JMAP or SCIM
|
||||
/// write, after `check`. Checked when the quota is set or changed, or the
|
||||
/// account changes tenant; an account already over a lowered tenant limit
|
||||
/// keeps its quota until someone changes it, like the count limits (MT-18).
|
||||
/// No quota at all is fine: the account shares the tenant's space. Not run
|
||||
/// for accounts a directory creates at sign-in or for a restored account,
|
||||
/// which would otherwise be refused for a setting nobody can edit first.
|
||||
/// The tenant whose allocated disk space this write could grow, if any
|
||||
/// (inbuxa-admin#257): an account given a disk quota, or a bigger one, or
|
||||
/// joining a tenant with one; or a domain moving into a tenant, which brings
|
||||
/// its accounts. Such a write takes the tenant's disk-quota lock before its
|
||||
/// checks and keeps it until it's saved, so two writes at once can't each
|
||||
/// fit the space left and together pass it. Lowering or clearing a quota,
|
||||
/// or any other change, needs no lock.
|
||||
pub fn disk_quota_tenant(old: Option<&Object>, new: &Object) -> Option<Id> {
|
||||
let tenant = new.inner.member_tenant_id()?;
|
||||
let moved_in = old.is_none_or(|old| old.inner.member_tenant_id() != Some(tenant));
|
||||
match &new.inner {
|
||||
ObjectInner::Account(_) => {
|
||||
let quota = account_disk_quota(&new.inner)?;
|
||||
let grows = moved_in
|
||||
|| old.is_some_and(|old| {
|
||||
account_disk_quota(&old.inner).is_none_or(|before| quota > before)
|
||||
});
|
||||
grows.then_some(tenant)
|
||||
}
|
||||
ObjectInner::Domain(_) if old.is_some() && moved_in => Some(tenant),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// inbuxa-admin#257: a tenant's accounts' disk quotas add up to no more than
|
||||
/// the tenant's own, so it can't give out more space than it has, in one
|
||||
/// account or across many. Run on a JMAP or SCIM write and on a restore,
|
||||
/// after `check`, holding the lock `disk_quota_tenant` names. `id` is the
|
||||
/// account on an update; `restoring` words the refusal for a restore,
|
||||
/// whose quota can't be edited first. Only a write that grows the total is checked, so a
|
||||
/// tenant already over a lowered limit can still edit its accounts and
|
||||
/// lower their quotas, like the count limits (MT-18). No quota at all is
|
||||
/// fine: the account shares what's left of the tenant's space. Not run for
|
||||
/// accounts a directory creates at sign-in, which never carry a quota.
|
||||
pub async fn check_disk_quota(
|
||||
registry: &RegistryStore,
|
||||
id: Option<Id>,
|
||||
old: Option<&Object>,
|
||||
new: &Object,
|
||||
restoring: bool,
|
||||
) -> trc::Result<Result<(), SetError<Property>>> {
|
||||
let tenant = new.inner.member_tenant_id();
|
||||
if let Some(tenant_id) = tenant
|
||||
if let ObjectInner::Account(_) = &new.inner
|
||||
&& let Some(tenant_id) = disk_quota_tenant(old, new)
|
||||
&& let Some(quota) = account_disk_quota(&new.inner)
|
||||
&& old.is_none_or(|old| {
|
||||
account_disk_quota(&old.inner) != Some(quota) || old.inner.member_tenant_id() != tenant
|
||||
})
|
||||
&& let Some(tenant_quota) =
|
||||
(limits(registry, tenant_id).await?)(TenantStorageQuota::MaxDiskQuota)
|
||||
.filter(|limit| *limit > 0)
|
||||
&& quota > tenant_quota
|
||||
{
|
||||
return Ok(Err(over_tenant_disk_quota(quota, tenant_quota)));
|
||||
let except = id.into_iter().collect::<AHashSet<_>>();
|
||||
let others = quota::allocated_disk(registry, tenant_id.document_id(), &except).await?;
|
||||
if others.saturating_add(quota) > tenant_quota {
|
||||
return Ok(Err(over_tenant_disk_quota(
|
||||
quota,
|
||||
others,
|
||||
tenant_quota,
|
||||
restoring,
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(Ok(()))
|
||||
}
|
||||
@@ -240,17 +263,54 @@ fn over_quota(reached: LimitReached, tenant_id: Id) -> SetError<Property> {
|
||||
))
|
||||
}
|
||||
|
||||
/// Refuses an account disk quota bigger than its tenant's (inbuxa-admin#257).
|
||||
fn over_tenant_disk_quota(quota: u64, tenant_quota: u64) -> SetError<Property> {
|
||||
/// Refuses an account disk quota that doesn't fit in what its tenant has
|
||||
/// left to give (inbuxa-admin#257).
|
||||
fn over_tenant_disk_quota(
|
||||
quota: u64,
|
||||
others: u64,
|
||||
tenant_quota: u64,
|
||||
restoring: bool,
|
||||
) -> SetError<Property> {
|
||||
SetError::invalid_properties()
|
||||
.with_property(Property::Quotas)
|
||||
.with_description(format!(
|
||||
"The account's disk quota of {quota} bytes ({}) is more than its tenant's \
|
||||
maxDiskQuota of {tenant_quota} bytes ({}). Set it to {tenant_quota} or less, \
|
||||
or leave it empty to share the tenant's space.",
|
||||
.with_description(disk_quota_message(quota, others, tenant_quota, restoring))
|
||||
}
|
||||
|
||||
/// Why a disk quota doesn't fit: the tenant's limit, what its other
|
||||
/// accounts already have, and what's left.
|
||||
fn disk_quota_message(quota: u64, others: u64, tenant_quota: u64, restoring: bool) -> String {
|
||||
let left = tenant_quota.saturating_sub(others);
|
||||
let given = if others > 0 {
|
||||
format!(
|
||||
" Its other accounts already have {others} bytes ({}) of it, so {left} bytes ({}) \
|
||||
is left to give.",
|
||||
readable_size(others),
|
||||
readable_size(left),
|
||||
)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
if restoring {
|
||||
return format!(
|
||||
"The account's disk quota of {quota} bytes ({}) doesn't fit in its tenant's \
|
||||
maxDiskQuota of {tenant_quota} bytes ({}).{given} It comes back with that quota, \
|
||||
so lower other accounts' quotas first, then restore it.",
|
||||
readable_size(quota),
|
||||
readable_size(tenant_quota),
|
||||
))
|
||||
);
|
||||
}
|
||||
let fix = if left > 0 {
|
||||
format!("Set it to {left} or less, lower another account's,")
|
||||
} else {
|
||||
"Lower another account's quota,".to_string()
|
||||
};
|
||||
format!(
|
||||
"The account's disk quota of {quota} bytes ({}) doesn't fit in its tenant's \
|
||||
maxDiskQuota of {tenant_quota} bytes ({}).{given} {fix} or leave it empty to share \
|
||||
the tenant's space.",
|
||||
readable_size(quota),
|
||||
readable_size(tenant_quota),
|
||||
)
|
||||
}
|
||||
|
||||
/// A byte count in binary units, for people reading an error: "50 GiB".
|
||||
@@ -272,6 +332,96 @@ fn readable_size(bytes: u64) -> String {
|
||||
#[cfg(test)]
|
||||
mod disk_quota_tests {
|
||||
use super::*;
|
||||
use registry::schema::{
|
||||
enums::StorageQuota,
|
||||
structs::{Domain, UserAccount},
|
||||
};
|
||||
|
||||
fn account(tenant: Option<u64>, quota: u64) -> Object {
|
||||
Object::from(Account::User(UserAccount {
|
||||
member_tenant_id: tenant.map(Id::from),
|
||||
quotas: [(StorageQuota::MaxDiskQuota, quota)].into_iter().collect(),
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
|
||||
fn domain(tenant: Option<u64>) -> Object {
|
||||
Object::from(Domain {
|
||||
member_tenant_id: tenant.map(Id::from),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn writes_that_grow_a_tenants_disk_space_take_its_lock() {
|
||||
let tenant = Some(Id::from(7u64));
|
||||
// A new account with a quota, and one without
|
||||
assert_eq!(disk_quota_tenant(None, &account(Some(7), 10)), tenant);
|
||||
assert_eq!(disk_quota_tenant(None, &account(Some(7), 0)), None);
|
||||
assert_eq!(disk_quota_tenant(None, &account(None, 10)), None);
|
||||
// Raised, set for the first time, lowered, cleared, unchanged
|
||||
let old = account(Some(7), 10);
|
||||
assert_eq!(disk_quota_tenant(Some(&old), &account(Some(7), 11)), tenant);
|
||||
assert_eq!(disk_quota_tenant(Some(&account(Some(7), 0)), &old), tenant);
|
||||
assert_eq!(disk_quota_tenant(Some(&old), &account(Some(7), 9)), None);
|
||||
assert_eq!(disk_quota_tenant(Some(&old), &account(Some(7), 0)), None);
|
||||
assert_eq!(disk_quota_tenant(Some(&old), &old), None);
|
||||
// Joining a tenant with a quota, even a smaller one
|
||||
assert_eq!(
|
||||
disk_quota_tenant(Some(&account(None, 10)), &account(Some(7), 5)),
|
||||
tenant
|
||||
);
|
||||
assert_eq!(
|
||||
disk_quota_tenant(Some(&account(Some(8), 10)), &account(Some(7), 5)),
|
||||
tenant
|
||||
);
|
||||
// A domain moving in brings its accounts; a new or staying one doesn't
|
||||
assert_eq!(
|
||||
disk_quota_tenant(Some(&domain(None)), &domain(Some(7))),
|
||||
tenant
|
||||
);
|
||||
assert_eq!(disk_quota_tenant(None, &domain(Some(7))), None);
|
||||
assert_eq!(
|
||||
disk_quota_tenant(Some(&domain(Some(7))), &domain(Some(7))),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
disk_quota_tenant(Some(&domain(Some(7))), &domain(None)),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_refusal_says_what_is_left() {
|
||||
const GIB: u64 = 1024 * 1024 * 1024;
|
||||
let message = disk_quota_message(5 * GIB, 15 * GIB, 5 * GIB, false);
|
||||
assert!(
|
||||
message.contains("already have 16106127360 bytes (15 GiB)"),
|
||||
"{message}"
|
||||
);
|
||||
assert!(
|
||||
message.contains("so 0 bytes (0 bytes) is left"),
|
||||
"{message}"
|
||||
);
|
||||
assert!(
|
||||
message.contains(" Lower another account's quota, or leave"),
|
||||
"{message}"
|
||||
);
|
||||
let message = disk_quota_message(4 * GIB, GIB, 4 * GIB, false);
|
||||
assert!(
|
||||
message.contains("so 3221225472 bytes (3 GiB) is left"),
|
||||
"{message}"
|
||||
);
|
||||
let message = disk_quota_message(3 * GIB, 4 * GIB, 5 * GIB, true);
|
||||
assert!(message.contains("then restore it"), "{message}");
|
||||
assert!(!message.contains("Set it to"), "{message}");
|
||||
let message = disk_quota_message(6 * GIB, 0, 5 * GIB, false);
|
||||
assert!(!message.contains("already have"), "{message}");
|
||||
assert!(
|
||||
message.contains("Set it to 5368709120 or less"),
|
||||
"{message}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn readable_sizes() {
|
||||
@@ -283,7 +433,7 @@ mod disk_quota_tests {
|
||||
#[test]
|
||||
fn zero_quota_is_no_quota() {
|
||||
let user = |quota: u64| {
|
||||
ObjectInner::Account(Account::User(registry::schema::structs::UserAccount {
|
||||
ObjectInner::Account(Account::User(UserAccount {
|
||||
quotas: [(StorageQuota::MaxDiskQuota, quota)].into_iter().collect(),
|
||||
..Default::default()
|
||||
}))
|
||||
@@ -308,5 +458,17 @@ fn refused_move(refusal: Refusal, tenant: Option<Id>) -> SetError<Property> {
|
||||
)),
|
||||
Refusal::ForeignLink(object_id) => foreign_key(object_id),
|
||||
Refusal::Limit(reached) => over_quota(reached, tenant.unwrap_or_default()),
|
||||
Refusal::DiskQuota {
|
||||
allocated,
|
||||
adding,
|
||||
limit,
|
||||
} => SetError::new(SetErrorType::OverQuota).with_description(format!(
|
||||
"The accounts on this domain have {adding} bytes ({}) of disk quota between them, \
|
||||
and the tenant has {} ({}) of its maxDiskQuota of {limit} bytes ({}) left to give.",
|
||||
readable_size(adding),
|
||||
limit.saturating_sub(allocated),
|
||||
readable_size(limit.saturating_sub(allocated)),
|
||||
readable_size(limit),
|
||||
)),
|
||||
}
|
||||
}
|
||||
@@ -48,6 +48,8 @@ use trc::AddContext;
|
||||
use types::id::Id;
|
||||
use utils::snowflake::SnowflakeIdGenerator;
|
||||
|
||||
use crate::inbuxa::tenant_disk_lock;
|
||||
|
||||
type DValue = Value<'static, P, DeletedAccountValue>;
|
||||
|
||||
const ALL: &[P] = &[
|
||||
@@ -391,7 +393,6 @@ async fn restore_account(
|
||||
kept: KeptAccount,
|
||||
password: Option<String>,
|
||||
) -> trc::Result<Result<(), SetError<P>>> {
|
||||
let account_id = id.document_id();
|
||||
let Some(ObjectInner::Account(mut account)) = PickledStream::new(&kept.record)
|
||||
.and_then(|mut stream| ObjectInner::unpickle(ObjectType::Account, &mut stream))
|
||||
else {
|
||||
@@ -447,11 +448,50 @@ async fn restore_account(
|
||||
inner: ObjectInner::Account(account),
|
||||
revision: 0,
|
||||
};
|
||||
|
||||
// inbuxa-admin#257: its disk quota must still fit what the tenant has
|
||||
// left to give, checked and saved under the tenant's lock
|
||||
let disk_lock = inbuxa_features::tenancy::writes::disk_quota_tenant(None, &object)
|
||||
.map(|tenant_id| tenant_id.document_id());
|
||||
if let Some(tenant_id) = disk_lock
|
||||
&& !tenant_disk_lock::acquire(server, tenant_id).await?
|
||||
{
|
||||
return Ok(Err(
|
||||
SetError::forbidden().with_description(tenant_disk_lock::BUSY)
|
||||
));
|
||||
}
|
||||
let restored = restore_checked(server, id, kept, object).await;
|
||||
if let Some(tenant_id) = disk_lock {
|
||||
tenant_disk_lock::release(server, tenant_id).await;
|
||||
}
|
||||
restored
|
||||
}
|
||||
|
||||
/// The rest of a restore, once the account is rebuilt: the tenancy checks,
|
||||
/// then the save.
|
||||
async fn restore_checked(
|
||||
server: &Server,
|
||||
id: Id,
|
||||
kept: KeptAccount,
|
||||
object: Object,
|
||||
) -> trc::Result<Result<(), SetError<P>>> {
|
||||
let account_id = id.document_id();
|
||||
if let Err(err) =
|
||||
inbuxa_features::tenancy::writes::check(server.registry(), None, None, &object).await?
|
||||
{
|
||||
return Ok(Err(failed(err)));
|
||||
}
|
||||
if let Err(err) = inbuxa_features::tenancy::writes::check_disk_quota(
|
||||
server.registry(),
|
||||
None,
|
||||
None,
|
||||
&object,
|
||||
true,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
return Ok(Err(failed(err)));
|
||||
}
|
||||
|
||||
// Its addresses are free for it alone
|
||||
let mut batch = BatchBuilder::new();
|
||||
|
||||
@@ -44,6 +44,7 @@ pub mod sharing_policy;
|
||||
pub mod store_check; // inbuxa: checking the store's blobs
|
||||
pub mod store_snapshot; // inbuxa: a snapshot of the store
|
||||
pub mod telemetry;
|
||||
pub mod tenant_disk_lock; // inbuxa: inbuxa-admin#257
|
||||
pub mod tenant_protocol_policy;
|
||||
pub mod undelete;
|
||||
pub mod webhook_test;
|
||||
@@ -0,0 +1,59 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! inbuxa-admin#257: one write at a time may grow a tenant's allocated disk
|
||||
//! space. The lock lives in the in-memory store, so it holds across a
|
||||
//! cluster: without it, two accounts given quotas at the same moment could
|
||||
//! each fit the space left and together pass the tenant's limit. Taken
|
||||
//! before the checks and released once the write is saved; one left behind
|
||||
//! by a failed request expires on its own.
|
||||
|
||||
use common::{KV_LOCK_TASK, Server};
|
||||
use std::time::Duration;
|
||||
|
||||
/// How long a lock outlives a request that never releases it.
|
||||
const LOCK_SECONDS: u64 = 30;
|
||||
/// How long to wait for another write to the same tenant, in all.
|
||||
const WAIT_TRIES: u32 = 100;
|
||||
const WAIT_STEP: Duration = Duration::from_millis(50);
|
||||
|
||||
/// What a write that couldn't get the lock is told.
|
||||
pub const BUSY: &str = "Another change to this tenant's disk quotas is being saved. Try again.";
|
||||
|
||||
fn key(tenant_id: u32) -> Vec<u8> {
|
||||
[b"tenant-disk-quota:".as_slice(), &tenant_id.to_be_bytes()].concat()
|
||||
}
|
||||
|
||||
/// Takes the tenant's lock, waiting up to five seconds for a write that
|
||||
/// holds it. `false` when it's still held.
|
||||
pub async fn acquire(server: &Server, tenant_id: u32) -> trc::Result<bool> {
|
||||
let key = key(tenant_id);
|
||||
for _ in 0..WAIT_TRIES {
|
||||
if server
|
||||
.in_memory_store()
|
||||
.try_lock(KV_LOCK_TASK, &key, LOCK_SECONDS)
|
||||
.await?
|
||||
{
|
||||
return Ok(true);
|
||||
}
|
||||
tokio::time::sleep(WAIT_STEP).await;
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Releases the tenant's lock. A failure only leaves it to expire.
|
||||
pub async fn release(server: &Server, tenant_id: u32) {
|
||||
if let Err(err) = server
|
||||
.in_memory_store()
|
||||
.remove_lock(KV_LOCK_TASK, &key(tenant_id))
|
||||
.await
|
||||
{
|
||||
trc::error!(
|
||||
err.details("Failed to release a tenant's disk-quota lock")
|
||||
.ctx(trc::Key::Id, tenant_id)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -379,7 +379,13 @@ impl RegistrySet for Server {
|
||||
|
||||
// Process modifications
|
||||
let mut cache_invalidator = CacheInvalidationBuilder::default();
|
||||
// inbuxa: inbuxa-admin#257: the tenant disk-quota lock the last
|
||||
// write took, released once it's saved or refused
|
||||
let mut disk_lock: Option<u32> = None;
|
||||
'outer: for (modification, mut value, mut new_object) in modifications {
|
||||
if let Some(tenant_id) = disk_lock.take() {
|
||||
crate::inbuxa::tenant_disk_lock::release(self, tenant_id).await;
|
||||
}
|
||||
// Initial validations
|
||||
let is_create = matches!(modification, Modification::Create { .. });
|
||||
let mut unpatched_properties = VecMap::new();
|
||||
@@ -655,6 +661,23 @@ impl RegistrySet for Server {
|
||||
Modification::Update { id, object } => (Some(*id), Some(object)),
|
||||
Modification::Create { .. } => (None, None),
|
||||
};
|
||||
|
||||
// inbuxa: inbuxa-admin#257: a write that could grow the
|
||||
// tenant's allocated disk space holds its lock until saved
|
||||
if let Some(tenant_id) =
|
||||
inbuxa_features::tenancy::writes::disk_quota_tenant(stored, &new_object)
|
||||
{
|
||||
let tenant_id = tenant_id.document_id();
|
||||
if !crate::inbuxa::tenant_disk_lock::acquire(self, tenant_id).await? {
|
||||
set.failed(
|
||||
modification,
|
||||
SetError::forbidden()
|
||||
.with_description(crate::inbuxa::tenant_disk_lock::BUSY),
|
||||
);
|
||||
continue 'outer;
|
||||
}
|
||||
disk_lock = Some(tenant_id);
|
||||
}
|
||||
let after_save = match inbuxa_features::tenancy::writes::check(
|
||||
self.registry(),
|
||||
stored_id,
|
||||
@@ -670,12 +693,14 @@ impl RegistrySet for Server {
|
||||
}
|
||||
};
|
||||
|
||||
// inbuxa: inbuxa-admin#257: an account's disk quota fits
|
||||
// within its tenant's
|
||||
// inbuxa: inbuxa-admin#257: the tenant's accounts' disk
|
||||
// quotas add up to no more than its own
|
||||
if let Err(err) = inbuxa_features::tenancy::writes::check_disk_quota(
|
||||
self.registry(),
|
||||
stored_id,
|
||||
stored,
|
||||
&new_object,
|
||||
false,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
@@ -947,6 +972,9 @@ impl RegistrySet for Server {
|
||||
set.server.notify_task_queue();
|
||||
}
|
||||
}
|
||||
if let Some(tenant_id) = disk_lock.take() {
|
||||
crate::inbuxa::tenant_disk_lock::release(self, tenant_id).await;
|
||||
}
|
||||
|
||||
// Process destroy
|
||||
for id in set.destroy.drain(..) {
|
||||
|
||||
@@ -963,7 +963,7 @@ pub async fn test(test: &mut TestServer) {
|
||||
.await;
|
||||
|
||||
// inbuxa-admin#257: an account's disk quota can't exceed its tenant's,
|
||||
// whoever sets it
|
||||
// whoever sets it, and neither can its accounts' quotas together
|
||||
let d_limit = (two + one / 2) as u64;
|
||||
let with_quota = |name: &str, quota: u64| {
|
||||
structs::Account::User(structs::UserAccount {
|
||||
@@ -983,6 +983,14 @@ pub async fn test(test: &mut TestServer) {
|
||||
let d4 = d_admin
|
||||
.registry_create_object(with_quota("d4", d_limit))
|
||||
.await;
|
||||
// ... nor can several add up to more than it
|
||||
for writer in [&d_admin, &admin] {
|
||||
writer
|
||||
.registry_create_object_expect_err(with_quota("d5", 1))
|
||||
.await
|
||||
.assert_type(SetErrorType::InvalidProperties)
|
||||
.assert_description_contains("other accounts already have");
|
||||
}
|
||||
d_admin
|
||||
.registry_update_object_expect_err(
|
||||
ObjectType::Account,
|
||||
@@ -1003,6 +1011,14 @@ pub async fn test(test: &mut TestServer) {
|
||||
json!({ Property::Description: "still here" }),
|
||||
)
|
||||
.await;
|
||||
// ... and its quota can still be lowered, even while over
|
||||
d_admin
|
||||
.registry_update_object(
|
||||
ObjectType::Account,
|
||||
d4,
|
||||
json!({ Property::Quotas: VecMap::from_iter([(StorageQuota::MaxDiskQuota, d_limit - 1)]) }),
|
||||
)
|
||||
.await;
|
||||
admin
|
||||
.set_tenant_quota(d_id, TenantStorageQuota::MaxDiskQuota, d_limit)
|
||||
.await;
|
||||
|
||||
Reference in new issue
Block a user