Backport store check, store snapshot and blob store change (release/2026) #340

Merged
jcoffey-dev merged 1 commits from backport/store-tools-2026 into release/2026 2026-11-09 17:39:18 +00:00
57 changed files with 7852 additions and 53 deletions

No files matched your search

+20 -1
View File
@@ -287,7 +287,11 @@ impl Default for DefaultPermissions {
};
for permission_id in 0..Permission::COUNT {
let permission = Permission::from_id(permission_id as u16).unwrap();
// inbuxa: ids are shared with the 27.1 series, so this series
// has gaps where 27.1 has permissions it doesn't (690-694)
let Some(permission) = Permission::from_id(permission_id as u16) else {
continue;
};
match permission {
Permission::Authenticate
| Permission::AuthenticateWithAlias
@@ -366,6 +370,21 @@ impl Default for DefaultPermissions {
default.superuser.push(permission);
}
Permission::SysJournalSearch | Permission::SysJournalExport => {}
// inbuxa: store-check spec, SV-9: a check reads every
// account's data, so only the server's administrators run one
Permission::SysStoreCheck => {
default.superuser.push(permission);
}
// inbuxa: store-snapshot spec, SN-8: a snapshot holds every
// account's data, so only the server's administrators take one
Permission::SysStoreSnapshot => {
default.superuser.push(permission);
}
// inbuxa: blob-store-change spec, BM-9: a copy reads and
// writes every account's blobs
Permission::SysBlobStoreChange => {
default.superuser.push(permission);
}
// inbuxa: AL-12: tenant administrators lock and delegate
// within their tenant
Permission::SysAccountLockGet
+6 -1
View File
@@ -50,10 +50,15 @@ impl Storage {
bp.data_store.share_marks(&memory);
}
// inbuxa: blob-store-change spec, BM-2, BM-3: a blob store that
// moved keeps reading what it left behind
let blob = BlobStore::build(bp).await.unwrap_or_default();
let blob = crate::manager::blob_moves::at_start(bp, blob).await;
Storage {
registry: bp.registry.clone(),
data: bp.data_store.clone(),
blob: BlobStore::build(bp).await.unwrap_or_default(),
blob,
search,
coordinator,
memory,
+135
View File
@@ -0,0 +1,135 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! inbuxa: blob-store-change spec (docs/spec/features/blob-store-change.md),
//! at start-up: records where the blob store is and, when it moved, the
//! change (BM-2), then opens the old store of every unresolved change so
//! what the new store lacks is read from it (BM-3).
use inbuxa_features::blob_move::{self as model, Started};
use registry::schema::{prelude::ObjectType, structs};
use std::sync::Arc;
use store::{
BlobStore, Store,
build::blob::blob_store_location,
registry::bootstrap::Bootstrap,
write::{
blob_move::{MovingBlobStore, OldBlobStore},
now,
},
};
use types::id::Id;
/// The blob store to use: `blob` itself or, while changes are unresolved,
/// `blob` reading through to their old stores.
pub async fn at_start(bp: &mut Bootstrap, blob: BlobStore) -> BlobStore {
let data = bp.data_store.clone();
if matches!(data, Store::None) || matches!(blob, BlobStore::Store(Store::None)) {
return blob;
}
let settings = bp.setting_infallible::<structs::BlobStore>().await;
let location = blob_store_location(&settings);
let warn = |bp: &mut Bootstrap, message: String| {
bp.build_warning(ObjectType::BlobStore.singleton(), message);
};
let mut recorded = None;
match model::at_start(
&data,
location.clone(),
&settings,
bp.node_id() as u64,
now(),
)
.await
{
Ok(Started::Same) => {}
Ok(Started::Changed(id)) => {
recorded = Some(id);
if let Ok(Some(change)) = model::get(&data, id).await {
warn(
bp,
format!(
"The blob store moved from {} to {}. Blobs written before stay in {}; \
they're read from it and kept in the new store as they're needed. \
Copy the rest over with inbuxa:BlobStoreChange {}.",
change.from,
change.to,
change.from,
Id::from(id)
),
);
}
}
Err(err) => {
warn(
bp,
format!("Couldn't record where the blob store is: {err}"),
);
return blob;
}
}
let unresolved = match model::unresolved(&data).await {
Ok(changes) => changes,
Err(err) => {
warn(bp, format!("Couldn't read the blob store changes: {err}"));
return blob;
}
};
let mut old = Vec::with_capacity(unresolved.len());
for change in unresolved {
// BM-4 resolves a change whose old store is this one; one recorded
// before that rule could still say so
if change.from == location {
continue;
}
let Some(settings) = change.old_settings.clone() else {
continue;
};
let mut warnings = Vec::new();
match BlobStore::open(settings, &data, &mut warnings).await {
Ok(store) => {
old.push(OldBlobStore {
change_id: change.id,
store,
});
if recorded == Some(change.id) {
continue;
}
warn(
bp,
format!(
"The blob store changed from {} to {}, and blobs only {} has are \
read from it. Copy them over or dismiss the change \
(inbuxa:BlobStoreChange {}).",
change.from,
change.to,
change.from,
Id::from(change.id)
),
);
}
Err(err) => warn(
bp,
format!(
"The blob store used before the change to {} ({}) couldn't be opened, \
so blobs only it has can't be read: {err}",
change.to, change.from
),
),
}
for warning in warnings {
warn(bp, warning);
}
}
if old.is_empty() {
blob
} else {
BlobStore::Moving(Arc::new(MovingBlobStore::new(blob, old)))
}
}
@@ -33,7 +33,10 @@ use types::id::Id;
/// locks and legal holds (audit-hold-lock spec, AU-9, AL-12, LH-13), and
/// the data inventory (personal-data catalog spec), accepting security
/// to-do items (security to-do list spec), the deliverability check
/// (deliverability spec), and scheduled reports (scheduled-reports spec).
/// (deliverability spec), scheduled reports (scheduled-reports spec),
/// checking the store's blobs (store-check spec, SV-9), taking snapshots of
/// the store (store-snapshot spec, SN-8), and copying and dismissing changes
/// of blob store (blob-store-change spec, BM-9).
const ADMIN_GRANTS: &[Permission] = &[
Permission::SysAiExplain,
Permission::SysAuditGet,
@@ -62,6 +65,9 @@ const ADMIN_GRANTS: &[Permission] = &[
Permission::SysDeliverabilityCheck,
Permission::SysScheduledReportGet,
Permission::SysScheduledReportUpdate,
Permission::SysStoreCheck,
Permission::SysStoreSnapshot,
Permission::SysBlobStoreChange,
];
/// Granted to the server-level Compliance Officer role once it exists:
+1
View File
@@ -17,6 +17,7 @@ use utils::HttpLimitResponse;
pub mod application;
pub mod backup;
pub mod blob_moves; // inbuxa: blob-store-change spec, a blob store that moved
pub mod boot;
pub mod compliance_roles; // inbuxa: personal-data catalog, the compliance roles
pub mod console;
+840
View File
@@ -0,0 +1,840 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! Changing the blob store (docs/spec/features/blob-store-change.md): the
//! `inbuxa:BlobStoreChange` objects. A change is recorded when a node
//! starts with its blob store somewhere else than the store last said
//! (BM-2), and keeps the old store's settings so it can be read through
//! (BM-3) and copied from (BM-5). The copy itself runs in
//! `services::inbuxa_blob_move`; this module keeps what a change is.
//!
//! Kept in the fork's subspace (`store::SUBSPACE_INBUXA`), never in the
//! registry. Every key starts with `W` (where blobs are), then one byte for
//! the kind:
//!
//! - `l`: where the blob store is now, with its settings ([`Location`]);
//! - `c` + change id (u32): a change ([`BlobStoreChange`]), as JSON;
//! - `p` + change id (u32) + number (u32): one problem its latest copy
//! found ([`Problem`]);
//! - `r`: the id of the change being copied now (u32), so only one copy
//! runs at a time across the server (BM-5).
//!
//! Numbers are big-endian.
use registry::schema::structs;
use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize, de::DeserializeOwned};
use store::{
Deserialize, IterateParams, SUBSPACE_INBUXA, Serialize, SerializeInfallible, Store, U32_LEN,
ValueKey,
write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue},
};
use trc::AddContext;
const FEATURE: u8 = b'W';
const KIND_LOCATION: u8 = b'l';
const KIND_CHANGE: u8 = b'c';
const KIND_PROBLEM: u8 = b'p';
const KIND_RUNNING: u8 = b'r';
const WRITE_ATTEMPTS: usize = 5;
/// The changes kept; recording another forgets the oldest resolved one.
pub const MAX_CHANGES: usize = 20;
/// BM-6: the problems a copy keeps; the counts cover all of them.
pub const MAX_PROBLEMS: u64 = 1_000;
/// BM-5: the default limits, as a store check's.
pub const DEFAULT_BLOBS_PER_SECOND: u64 = 1_000;
pub const DEFAULT_BYTES_PER_SECOND: u64 = 20 * 1024 * 1024;
/// BM-6: a copy whose node has been quiet this long has stopped.
pub const STALE_AFTER: u64 = 5 * 60;
const MAX_TEXT: usize = 2_000;
/// Where the blob store is (BM-1), with the settings that open it.
#[derive(Debug, Clone, PartialEq, SerdeSerialize, SerdeDeserialize)]
pub struct Location {
pub location: String,
pub settings: structs::BlobStore,
}
/// BM-6, BM-4.
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
#[serde(rename_all = "lowercase")]
pub enum Status {
/// Recorded; nothing copied yet.
Pending,
Copying,
/// Everything is in the new store: resolved.
Copied,
/// A copy ended with blobs it couldn't copy.
Incomplete,
Stopped,
Failed,
Interrupted,
/// The blob store went back to where this change moved it from:
/// resolved (BM-4).
Reverted,
}
impl Status {
pub fn as_str(&self) -> &'static str {
match self {
Status::Pending => "pending",
Status::Copying => "copying",
Status::Copied => "copied",
Status::Incomplete => "incomplete",
Status::Stopped => "stopped",
Status::Failed => "failed",
Status::Interrupted => "interrupted",
Status::Reverted => "reverted",
}
}
/// BM-3: whether the old store is still read through.
pub fn is_unresolved(&self) -> bool {
!matches!(self, Status::Copied | Status::Reverted)
}
}
/// BM-5.
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
#[serde(rename_all = "lowercase")]
pub enum ProblemKind {
Missing,
Damaged,
Unreadable,
}
impl ProblemKind {
pub fn as_str(&self) -> &'static str {
match self {
ProblemKind::Missing => "missing",
ProblemKind::Damaged => "damaged",
ProblemKind::Unreadable => "unreadable",
}
}
}
/// BM-5: a blob a copy couldn't copy.
#[derive(Debug, Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
#[serde(rename_all = "camelCase")]
pub struct Problem {
pub kind: ProblemKind,
/// The blob's key: a blob id as hex, or a fixed name.
pub blob_id: String,
/// `email`, `file`, `queuedMessage`, `archivedItem`, `temporary` or
/// `fixed`: the first thing found holding it.
pub holder: String,
#[serde(default)]
pub account_id: Option<u32>,
#[serde(default)]
pub object_id: Option<u64>,
#[serde(default)]
pub detail: String,
}
/// BM-5: how far the latest copy got.
#[derive(Debug, Clone, Default, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct Progress {
/// Slices of the blob index done, of 256.
pub slices: u32,
pub blobs_checked: u64,
pub copied: u64,
pub already_there: u64,
pub bytes_copied: u64,
pub missing: u64,
pub damaged: u64,
pub unreadable: u64,
}
impl Progress {
pub fn problems(&self) -> u64 {
self.missing + self.damaged + self.unreadable
}
pub fn count(&mut self, kind: ProblemKind) {
match kind {
ProblemKind::Missing => self.missing += 1,
ProblemKind::Damaged => self.damaged += 1,
ProblemKind::Unreadable => self.unreadable += 1,
}
}
/// 0 to 1, by slices of the index.
pub fn share(&self) -> f64 {
self.slices.min(256) as f64 / 256.0
}
}
/// The latest copy of a change (BM-5, BM-6).
#[derive(Debug, Clone, PartialEq, SerdeSerialize, SerdeDeserialize)]
#[serde(rename_all = "camelCase")]
pub struct Copy {
pub started_at: u64,
pub started_by: String,
pub node_id: u64,
pub blobs_per_second: u64,
pub bytes_per_second: u64,
/// When its node last saved it.
pub heartbeat: u64,
#[serde(default)]
pub finished_at: Option<u64>,
#[serde(default)]
pub stop_requested: bool,
#[serde(default)]
pub progress: Progress,
#[serde(default)]
pub problems_kept: u64,
#[serde(default)]
pub error: Option<String>,
}
/// An `inbuxa:BlobStoreChange`.
#[derive(Debug, Clone, PartialEq, SerdeSerialize, SerdeDeserialize)]
#[serde(rename_all = "camelCase")]
pub struct BlobStoreChange {
pub id: u32,
pub from: String,
pub to: String,
pub changed_at: u64,
pub node_id: u64,
pub status: Status,
/// The old store's settings, to read it (BM-3) and copy from it (BM-5).
/// Dropped once the change is resolved; never shown (BM-9).
#[serde(default)]
pub old_settings: Option<structs::BlobStore>,
#[serde(default)]
pub copy: Option<Copy>,
}
impl BlobStoreChange {
/// As it reads now: a copy whose node went quiet has stopped with it
/// (BM-6), even before anything marked it so.
pub fn status_at(&self, now: u64) -> Status {
match &self.copy {
Some(copy) if self.status == Status::Copying && copy.heartbeat + STALE_AFTER < now => {
Status::Interrupted
}
_ => self.status,
}
}
pub fn set_error(&mut self, error: &str) {
if let Some(copy) = &mut self.copy {
copy.error = Some(error.trim().chars().take(MAX_TEXT).collect());
}
}
}
struct Json<T>(T);
impl<T: SerdeSerialize> Serialize for Json<&T> {
fn serialize(&self) -> trc::Result<Vec<u8>> {
serde_json::to_vec(self.0).map_err(|err| {
trc::StoreEvent::UnexpectedError
.into_err()
.details("Could not serialize a blob store change")
.reason(err)
})
}
}
impl<T: DeserializeOwned + Sync + Send> Deserialize for Json<T> {
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
serde_json::from_slice(bytes).map(Json).map_err(|err| {
trc::StoreEvent::DataCorruption
.into_err()
.details("Invalid blob store change")
.reason(err)
})
}
}
fn any(key: Vec<u8>) -> ValueClass {
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key,
})
}
fn location_class() -> ValueClass {
any(vec![FEATURE, KIND_LOCATION])
}
fn class(id: u32) -> ValueClass {
let mut key = Vec::with_capacity(2 + U32_LEN);
key.extend_from_slice(&[FEATURE, KIND_CHANGE]);
key.extend_from_slice(&id.to_be_bytes());
any(key)
}
fn problem_class(id: u32, number: u32) -> ValueClass {
let mut key = Vec::with_capacity(2 + U32_LEN * 2);
key.extend_from_slice(&[FEATURE, KIND_PROBLEM]);
key.extend_from_slice(&id.to_be_bytes());
key.extend_from_slice(&number.to_be_bytes());
any(key)
}
fn running_class() -> ValueClass {
any(vec![FEATURE, KIND_RUNNING])
}
fn is_assert_failure(err: &trc::Error) -> bool {
matches!(
err.as_ref(),
trc::EventType::Store(trc::StoreEvent::AssertValueFailed)
)
}
/// A value as stored, with a hash of its bytes for a compare-and-set.
struct Versioned<T>(T, u64);
impl<T: DeserializeOwned + Sync + Send> Deserialize for Versioned<T> {
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
let Json(value) = Json::<T>::deserialize(bytes)?;
Ok(Versioned(value, xxhash_rust::xxh3::xxh3_64(bytes)))
}
}
pub async fn get(data: &Store, id: u32) -> trc::Result<Option<BlobStoreChange>> {
Ok(data
.get_value::<Json<BlobStoreChange>>(ValueKey::from(class(id)))
.await
.caused_by(trc::location!())?
.map(|Json(change)| change))
}
/// Every change, oldest first.
pub async fn all(data: &Store) -> trc::Result<Vec<BlobStoreChange>> {
let mut changes = Vec::new();
data.iterate(
IterateParams::new(ValueKey::from(class(0)), ValueKey::from(class(u32::MAX))),
|_, value| {
if let Ok(Json(change)) = Json::<BlobStoreChange>::deserialize(value) {
changes.push(change);
}
Ok(true)
},
)
.await
.caused_by(trc::location!())?;
changes.sort_by_key(|c| c.id);
Ok(changes)
}
/// BM-3, BM-10: the changes still read through, newest first.
pub async fn unresolved(data: &Store) -> trc::Result<Vec<BlobStoreChange>> {
let mut changes = all(data).await?;
changes.retain(|c| c.status.is_unresolved() && c.old_settings.is_some());
changes.reverse();
Ok(changes)
}
/// Where the store last said the blob store was.
pub async fn location(data: &Store) -> trc::Result<Option<Location>> {
Ok(data
.get_value::<Json<Location>>(ValueKey::from(location_class()))
.await
.caused_by(trc::location!())?
.map(|Json(location)| location))
}
/// What [`at_start`] found.
#[derive(Debug, Clone, PartialEq)]
pub enum Started {
/// The blob store is where it was, or this is the first start that
/// records it.
Same,
/// This node recorded a change, the one with this id (BM-2).
Changed(u32),
}
/// BM-2, BM-4: at start-up, records where the blob store is and, when it
/// moved, a change from where it was. The first node to start with the new
/// setting records it: the location is written only if it's still what
/// this node read, so the others find the change already there.
pub async fn at_start(
data: &Store,
location: String,
settings: &structs::BlobStore,
node_id: u64,
now: u64,
) -> trc::Result<Started> {
let mut attempt = 0;
loop {
attempt += 1;
let last = data
.get_value::<Versioned<Location>>(ValueKey::from(location_class()))
.await
.caused_by(trc::location!())?;
let here = Location {
location: location.clone(),
settings: settings.clone(),
};
let mut batch = BatchBuilder::new();
let mut started = Started::Same;
let mut forgotten = Vec::new();
match &last {
None => {
batch.assert_value(location_class(), AssertValue::None);
}
Some(Versioned(last, version)) if last.location == location => {
if &last.settings == settings {
return Ok(Started::Same);
}
// The same place with other credentials or tuning: kept
// current, for when it's the old store of a later change
batch.assert_value(location_class(), AssertValue::Hash(*version));
}
Some(Versioned(last, version)) => {
batch.assert_value(location_class(), AssertValue::Hash(*version));
let changes = all(data).await?;
let id = changes.iter().map(|c| c.id).max().unwrap_or(0) + 1;
// BM-4: back where an unresolved change moved away from
for earlier in changes.iter().filter(|c| {
c.status.is_unresolved() && c.status != Status::Copying && c.from == location
}) {
let reverted = BlobStoreChange {
status: Status::Reverted,
old_settings: None,
..earlier.clone()
};
batch.set(class(earlier.id), Json(&reverted).serialize()?);
}
let change = BlobStoreChange {
id,
from: last.location.clone(),
to: location.clone(),
changed_at: now,
node_id,
status: Status::Pending,
old_settings: Some(last.settings.clone()),
copy: None,
};
batch.assert_value(class(id), AssertValue::None);
batch.set(class(id), Json(&change).serialize()?);
started = Started::Changed(id);
// The oldest resolved changes past MAX_CHANGES are forgotten
let over = (changes.len() + 1).saturating_sub(MAX_CHANGES);
for old in changes
.iter()
.filter(|c| !c.status.is_unresolved())
.take(over)
{
batch.clear(class(old.id));
forgotten.push(old.id);
}
}
}
batch.set(location_class(), Json(&here).serialize()?);
match data.write(batch.build_all()).await {
Ok(_) => {
for id in forgotten {
remove_problems(data, id).await?;
}
return Ok(started);
}
// Another node wrote first: read what it recorded
Err(err) if attempt < WRITE_ATTEMPTS && is_assert_failure(&err) => {}
Err(err) => return Err(err.caused_by(trc::location!())),
}
}
}
/// The id of the change being copied now, if any.
pub async fn running(data: &Store) -> trc::Result<Option<u32>> {
data.get_value::<u32>(ValueKey::from(running_class()))
.await
.caused_by(trc::location!())
}
/// BM-5: why a copy can't start.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Refusal {
/// No such change.
NotFound,
/// This change's copy runs now.
Running(u32),
/// The change is resolved, or has no old store to copy from.
Resolved,
}
/// BM-5: marks a change as being copied by `copy` and makes it the one
/// that runs. Refused while another copy runs: one whose node went quiet
/// is marked interrupted first.
pub async fn start_copy(
data: &Store,
id: u32,
copy: Copy,
now: u64,
) -> trc::Result<Result<BlobStoreChange, Refusal>> {
let mut attempt = 0;
loop {
attempt += 1;
if let Some(current) = running(data).await? {
match get(data, current).await? {
Some(c) if c.status_at(now) == Status::Copying => {
return Ok(Err(Refusal::Running(current)));
}
Some(c) => {
if c.status == Status::Copying {
update(data, current, |c| {
if c.status != Status::Copying {
return false;
}
c.status = Status::Interrupted;
if let Some(copy) = &mut c.copy {
copy.finished_at = Some(now);
}
true
})
.await?;
}
release(data, current).await?;
}
None => release(data, current).await?,
}
}
let Some(Versioned(change, version)) = data
.get_value::<Versioned<BlobStoreChange>>(ValueKey::from(class(id)))
.await
.caused_by(trc::location!())?
else {
return Ok(Err(Refusal::NotFound));
};
if !change.status.is_unresolved() || change.old_settings.is_none() {
return Ok(Err(Refusal::Resolved));
}
let change = BlobStoreChange {
status: Status::Copying,
copy: Some(copy.clone()),
..change
};
let mut batch = BatchBuilder::new();
batch.assert_value(class(id), AssertValue::Hash(version));
batch.assert_value(running_class(), AssertValue::None);
batch.set(class(id), Json(&change).serialize()?);
batch.set(running_class(), SerializeInfallible::serialize(&id));
match data.write(batch.build_all()).await {
Ok(_) => {
remove_problems(data, id).await?;
return Ok(Ok(change));
}
Err(err) if attempt < WRITE_ATTEMPTS && is_assert_failure(&err) => {}
Err(err) => return Err(err.caused_by(trc::location!())),
}
}
}
/// Lets another copy run, if `id` is the one that runs now.
pub async fn release(data: &Store, id: u32) -> trc::Result<()> {
let mut batch = BatchBuilder::new();
batch.assert_value(running_class(), AssertValue::U32(id));
batch.clear(running_class());
match data.write(batch.build_all()).await {
Ok(_) => Ok(()),
Err(err) if is_assert_failure(&err) => Ok(()),
Err(err) => Err(err.caused_by(trc::location!())),
}
}
/// Reads a change, changes it with `change`, and writes it back, again if
/// something else wrote it in between. `change` returns false to leave it
/// as it is. None if there's no such change.
pub async fn update<F>(data: &Store, id: u32, mut change: F) -> trc::Result<Option<BlobStoreChange>>
where
F: FnMut(&mut BlobStoreChange) -> bool,
{
loop {
let Some(Versioned(mut stored, version)) = data
.get_value::<Versioned<BlobStoreChange>>(ValueKey::from(class(id)))
.await
.caused_by(trc::location!())?
else {
return Ok(None);
};
if !change(&mut stored) {
return Ok(Some(stored));
}
let mut batch = BatchBuilder::new();
batch.assert_value(class(id), AssertValue::Hash(version));
batch.set(class(id), Json(&stored).serialize()?);
match data.write(batch.build_all()).await {
Ok(_) => return Ok(Some(stored)),
Err(err) if is_assert_failure(&err) => {}
Err(err) => return Err(err.caused_by(trc::location!())),
}
}
}
/// Keeps problems `first..` of a change's copy (BM-6): the caller numbers
/// them and stops at [`MAX_PROBLEMS`].
pub async fn add_problems(
data: &Store,
id: u32,
first: u64,
problems: &[Problem],
) -> trc::Result<()> {
let mut batch = BatchBuilder::new();
for (n, problem) in problems.iter().enumerate() {
let number = u32::try_from(first + n as u64).unwrap_or(u32::MAX);
batch.set(problem_class(id, number), Json(problem).serialize()?);
if batch.is_large_batch() {
data.write(batch.build_all())
.await
.caused_by(trc::location!())?;
batch = BatchBuilder::new();
}
}
if !batch.is_empty() {
data.write(batch.build_all())
.await
.caused_by(trc::location!())?;
}
Ok(())
}
/// The kept problems of a change's latest copy, in the order found.
pub async fn problems(data: &Store, id: u32) -> trc::Result<Vec<Problem>> {
let mut problems = Vec::new();
data.iterate(
IterateParams::new(
ValueKey::from(problem_class(id, 0)),
ValueKey::from(problem_class(id, u32::MAX)),
),
|_, value| {
if let Ok(Json(problem)) = Json::<Problem>::deserialize(value) {
problems.push(problem);
}
Ok(true)
},
)
.await
.caused_by(trc::location!())?;
Ok(problems)
}
async fn remove_problems(data: &Store, id: u32) -> trc::Result<()> {
data.delete_range(
ValueKey::from(problem_class(id, 0)),
ValueKey::from(problem_class(id, u32::MAX)),
)
.await
.caused_by(trc::location!())
}
/// BM-7: forgets a change, the old store's settings and its problems.
pub async fn remove(data: &Store, id: u32) -> trc::Result<()> {
remove_problems(data, id).await?;
let mut batch = BatchBuilder::new();
batch.clear(class(id));
data.write(batch.build_all())
.await
.caused_by(trc::location!())
.map(|_| ())
}
#[cfg(test)]
mod tests {
use super::*;
fn fs(path: &str) -> structs::BlobStore {
structs::BlobStore::FileSystem(structs::FileSystemStore {
path: path.into(),
depth: 2,
})
}
#[tokio::test]
async fn a_move_and_a_move_back() {
let data = store::backend::ephemeral::EphemeralStore::open();
let a = structs::BlobStore::Default;
let b = fs("/srv/blobs");
let (here_a, here_b) = ("the data store".to_string(), "dir".to_string());
// First start: only where it is
assert_eq!(
at_start(&data, here_a.clone(), &a, 1, 10).await.unwrap(),
Started::Same
);
assert_eq!(
at_start(&data, here_a.clone(), &a, 2, 11).await.unwrap(),
Started::Same
);
assert!(all(&data).await.unwrap().is_empty());
// A to B: one change, recorded once whichever node starts first
assert_eq!(
at_start(&data, here_b.clone(), &b, 1, 20).await.unwrap(),
Started::Changed(1)
);
assert_eq!(
at_start(&data, here_b.clone(), &b, 2, 21).await.unwrap(),
Started::Same
);
let first = get(&data, 1).await.unwrap().unwrap();
assert_eq!(
(first.from.as_str(), first.to.as_str()),
("the data store", "dir")
);
assert_eq!(first.old_settings, Some(a.clone()));
assert_eq!(first.status, Status::Pending);
// Back to A: the first change is reverted, the second is the one
// still read through
assert_eq!(
at_start(&data, here_a.clone(), &a, 1, 30).await.unwrap(),
Started::Changed(2)
);
let first = get(&data, 1).await.unwrap().unwrap();
assert_eq!(first.status, Status::Reverted);
assert_eq!(first.old_settings, None);
let open = unresolved(&data).await.unwrap();
assert_eq!(open.len(), 1);
assert_eq!(open[0].id, 2);
assert_eq!(open[0].old_settings, Some(b.clone()));
// A copy: one at a time, never of a resolved change
let copy = Copy {
started_at: 40,
started_by: "admin".into(),
node_id: 1,
blobs_per_second: 0,
bytes_per_second: 0,
heartbeat: 40,
finished_at: None,
stop_requested: false,
progress: Progress::default(),
problems_kept: 0,
error: None,
};
assert_eq!(
start_copy(&data, 1, copy.clone(), 40).await.unwrap(),
Err(Refusal::Resolved)
);
assert_eq!(
start_copy(&data, 9, copy.clone(), 40).await.unwrap(),
Err(Refusal::NotFound)
);
let started = start_copy(&data, 2, copy.clone(), 40)
.await
.unwrap()
.unwrap();
assert_eq!(started.status, Status::Copying);
assert_eq!(
start_copy(&data, 2, copy.clone(), 41).await.unwrap(),
Err(Refusal::Running(2))
);
// Its node went quiet: interrupted, and another may start
let again = start_copy(&data, 2, copy.clone(), 41 + STALE_AFTER)
.await
.unwrap()
.unwrap();
assert_eq!(again.status, Status::Copying);
// Finished: resolved, nothing read through any more
update(&data, 2, |c| {
c.status = Status::Copied;
c.old_settings = None;
true
})
.await
.unwrap();
release(&data, 2).await.unwrap();
assert!(unresolved(&data).await.unwrap().is_empty());
assert_eq!(running(&data).await.unwrap(), None);
// Where it was, again: nothing new
assert_eq!(
at_start(&data, here_a.clone(), &a, 1, 50).await.unwrap(),
Started::Same
);
remove(&data, 2).await.unwrap();
assert!(get(&data, 2).await.unwrap().is_none());
}
#[test]
fn keys_stay_in_their_own_letter() {
for class in [
location_class(),
class(1),
problem_class(1, 0),
running_class(),
] {
let ValueClass::Any(any) = class else {
unreachable!()
};
assert_eq!(any.key[0], b'W');
}
}
#[test]
fn resolved_is_copied_or_reverted() {
for status in [
Status::Pending,
Status::Copying,
Status::Incomplete,
Status::Stopped,
Status::Failed,
Status::Interrupted,
] {
assert!(status.is_unresolved(), "{status:?}");
}
assert!(!Status::Copied.is_unresolved());
assert!(!Status::Reverted.is_unresolved());
}
#[test]
fn a_quiet_copy_reads_as_interrupted() {
let change = BlobStoreChange {
id: 1,
from: "the data store".into(),
to: "the directory /srv/blobs, depth 2".into(),
changed_at: 1_000,
node_id: 1,
status: Status::Copying,
old_settings: Some(structs::BlobStore::Default),
copy: Some(Copy {
started_at: 1_000,
started_by: "admin".into(),
node_id: 1,
blobs_per_second: 0,
bytes_per_second: 0,
heartbeat: 1_000,
finished_at: None,
stop_requested: false,
progress: Progress::default(),
problems_kept: 0,
error: None,
}),
};
assert_eq!(change.status_at(1_000 + STALE_AFTER), Status::Copying);
assert_eq!(change.status_at(1_001 + STALE_AFTER), Status::Interrupted);
}
#[test]
fn a_change_keeps_the_old_settings_as_json() {
let change = BlobStoreChange {
id: 2,
from: "the directory /srv/blobs, depth 2".into(),
to: "the data store".into(),
changed_at: 5,
node_id: 1,
status: Status::Pending,
old_settings: Some(fs("/srv/blobs")),
copy: None,
};
let bytes = Json(&change).serialize().unwrap();
let Json(back) = Json::<BlobStoreChange>::deserialize(&bytes).unwrap();
assert_eq!(back, change);
}
}
+2
View File
@@ -20,6 +20,7 @@
pub mod ai;
pub mod audit;
pub mod blob_move; // inbuxa: changing the blob store (not a rebuild)
pub mod branding;
pub mod deliverability; // inbuxa: the deliverability check (not a rebuild)
pub mod hold;
@@ -30,5 +31,6 @@ pub mod masked_email;
pub mod privacy;
pub mod scheduled_reports; // inbuxa: scheduled reports and the weekly digest (not a rebuild)
pub mod security;
pub mod store_check; // inbuxa: checking the store's blobs (not a rebuild)
pub mod tenancy;
pub mod undelete;
+597
View File
@@ -0,0 +1,597 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! Checking the store's blobs (docs/spec/features/store-check.md): the
//! `inbuxa:StoreCheck` objects. A check reports which blobs the server's
//! data needs and doesn't have, and changes nothing. The check itself runs
//! in `services::inbuxa_store_check`; this module keeps what a check is and
//! what it found.
//!
//! Kept in the fork's subspace (`store::SUBSPACE_INBUXA`), never in the
//! registry (SV-1). Every key starts with `V` (verify), then one byte for
//! the kind:
//!
//! - `c` + check id (u32): a check ([`StoreCheck`]), as JSON;
//! - `p` + check id (u32) + number (u32): one of its problems ([`Problem`]),
//! as JSON, kept apart so a check stays small however much it found;
//! - `r`: the id of the check that runs now (u32), so only one runs at a
//! time across the server (SV-2).
//!
//! `Vs` is the store's version (`common::INBUXA_STORE_VERSION_KEY`), so no
//! kind here is `s`. Numbers are big-endian.
use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize, de::DeserializeOwned};
use store::{
Deserialize, IterateParams, SUBSPACE_INBUXA, Serialize, SerializeInfallible, Store, U32_LEN,
ValueKey,
write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue},
};
use trc::AddContext;
const FEATURE: u8 = b'V';
const KIND_CHECK: u8 = b'c';
const KIND_PROBLEM: u8 = b'p';
const KIND_RUNNING: u8 = b'r';
const CREATE_ATTEMPTS: usize = 5;
/// SV-1: the checks kept; starting another forgets the oldest.
pub const MAX_CHECKS: usize = 20;
/// SV-5: the problems a check keeps; the counts cover all of them.
pub const MAX_PROBLEMS: u64 = 1_000;
/// SV-6: the default limits.
pub const DEFAULT_BLOBS_PER_SECOND: u64 = 1_000;
pub const DEFAULT_BYTES_PER_SECOND: u64 = 20 * 1024 * 1024;
/// SV-8: a running check whose node has been quiet this long has stopped.
pub const STALE_AFTER: u64 = 5 * 60;
const MAX_TEXT: usize = 2_000;
/// SV-2.
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
#[serde(rename_all = "lowercase")]
pub enum Mode {
Quick,
Full,
}
impl Mode {
pub fn parse(value: &str) -> Option<Self> {
match value {
"quick" => Some(Mode::Quick),
"full" => Some(Mode::Full),
_ => None,
}
}
pub fn as_str(&self) -> &'static str {
match self {
Mode::Quick => "quick",
Mode::Full => "full",
}
}
}
/// SV-8.
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
#[serde(rename_all = "lowercase")]
pub enum Status {
Running,
Done,
Stopped,
Failed,
Interrupted,
}
impl Status {
pub fn as_str(&self) -> &'static str {
match self {
Status::Running => "running",
Status::Done => "done",
Status::Stopped => "stopped",
Status::Failed => "failed",
Status::Interrupted => "interrupted",
}
}
}
/// SV-4.
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
#[serde(rename_all = "lowercase")]
pub enum ProblemKind {
Missing,
Damaged,
Unlinked,
Unreadable,
}
impl ProblemKind {
pub fn as_str(&self) -> &'static str {
match self {
ProblemKind::Missing => "missing",
ProblemKind::Damaged => "damaged",
ProblemKind::Unlinked => "unlinked",
ProblemKind::Unreadable => "unreadable",
}
}
}
/// SV-4: one blob something needs and can't have, as it was found.
#[derive(Debug, Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
#[serde(rename_all = "camelCase")]
pub struct Problem {
pub kind: ProblemKind,
/// None for a queued message, which belongs to no account.
pub account_id: Option<u32>,
#[serde(default)]
pub account_name: String,
/// `email`, `file`, `queuedMessage`, ... (SV-4).
pub object: String,
/// The object's id: a document id, a queue id or an archived item's id.
pub object_id: u64,
/// The blob id, as hex.
pub blob_id: String,
/// Whether the store still has the blob's commit marker.
pub committed: bool,
#[serde(default)]
pub detail: String,
}
/// SV-7: how far a check got and what it found.
#[derive(Debug, Clone, Default, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct Progress {
/// Slices of the blob index done, of 256.
pub slices: u32,
pub blobs_checked: u64,
pub links_checked: u64,
pub bytes_read: u64,
pub messages_checked: u64,
/// Accounts whose mail metadata is checked, of how many (SV-3).
pub accounts_done: u64,
pub accounts_total: u64,
pub missing: u64,
pub damaged: u64,
pub unlinked: u64,
pub unreadable: u64,
}
impl Progress {
pub fn problems(&self) -> u64 {
self.missing + self.damaged + self.unlinked + self.unreadable
}
pub fn count(&mut self, kind: ProblemKind) {
match kind {
ProblemKind::Missing => self.missing += 1,
ProblemKind::Damaged => self.damaged += 1,
ProblemKind::Unlinked => self.unlinked += 1,
ProblemKind::Unreadable => self.unreadable += 1,
}
}
/// 0 to 1: the 256 slices are the first 90%, the mail metadata after
/// them the rest. A finished check is 1.
pub fn share(&self, status: Status) -> f64 {
if status == Status::Done {
return 1.0;
}
let slices = self.slices.min(256) as f64 / 256.0;
let accounts = if self.accounts_total == 0 {
0.0
} else {
self.accounts_done.min(self.accounts_total) as f64 / self.accounts_total as f64
};
slices * 0.9 + accounts * 0.1
}
}
/// An `inbuxa:StoreCheck` (SV-1).
#[derive(Debug, Clone, PartialEq, SerdeSerialize, SerdeDeserialize)]
#[serde(rename_all = "camelCase")]
pub struct StoreCheck {
pub id: u32,
pub mode: Mode,
pub blobs_per_second: u64,
pub bytes_per_second: u64,
pub status: Status,
pub created_at: u64,
pub created_by: String,
pub finished_at: Option<u64>,
/// The node it runs on.
pub node_id: u64,
/// When its node last saved it (SV-7, SV-8).
pub heartbeat: u64,
/// SV-8: an administrator asked it to stop; the running node sees it at
/// its next save.
#[serde(default)]
pub stop_requested: bool,
#[serde(default)]
pub progress: Progress,
/// Problems kept, at most [`MAX_PROBLEMS`].
#[serde(default)]
pub problems_kept: u64,
#[serde(default)]
pub error: Option<String>,
}
impl StoreCheck {
pub fn new(mode: Mode, created_by: String, node_id: u64, now: u64) -> Self {
StoreCheck {
id: 0,
mode,
blobs_per_second: DEFAULT_BLOBS_PER_SECOND,
bytes_per_second: DEFAULT_BYTES_PER_SECOND,
status: Status::Running,
created_at: now,
created_by,
finished_at: None,
node_id,
heartbeat: now,
stop_requested: false,
progress: Progress::default(),
problems_kept: 0,
error: None,
}
}
/// SV-8: as it reads now. A running check whose node went quiet has
/// stopped with it, even before anything marked it so.
pub fn status_at(&self, now: u64) -> Status {
if self.status == Status::Running && self.heartbeat + STALE_AFTER < now {
Status::Interrupted
} else {
self.status
}
}
pub fn set_error(&mut self, error: &str) {
self.error = Some(error.trim().chars().take(MAX_TEXT).collect());
}
}
struct Json<T>(T);
impl<T: SerdeSerialize> Serialize for Json<&T> {
fn serialize(&self) -> trc::Result<Vec<u8>> {
serde_json::to_vec(self.0).map_err(|err| {
trc::StoreEvent::UnexpectedError
.into_err()
.details("Could not serialize a store check")
.reason(err)
})
}
}
impl<T: DeserializeOwned + Sync + Send> Deserialize for Json<T> {
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
serde_json::from_slice(bytes).map(Json).map_err(|err| {
trc::StoreEvent::DataCorruption
.into_err()
.details("Invalid store check")
.reason(err)
})
}
}
fn any(key: Vec<u8>) -> ValueClass {
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key,
})
}
fn class(id: u32) -> ValueClass {
let mut key = Vec::with_capacity(2 + U32_LEN);
key.extend_from_slice(&[FEATURE, KIND_CHECK]);
key.extend_from_slice(&id.to_be_bytes());
any(key)
}
fn problem_class(id: u32, number: u32) -> ValueClass {
let mut key = Vec::with_capacity(2 + U32_LEN * 2);
key.extend_from_slice(&[FEATURE, KIND_PROBLEM]);
key.extend_from_slice(&id.to_be_bytes());
key.extend_from_slice(&number.to_be_bytes());
any(key)
}
fn running_class() -> ValueClass {
any(vec![FEATURE, KIND_RUNNING])
}
pub async fn get(data: &Store, id: u32) -> trc::Result<Option<StoreCheck>> {
Ok(data
.get_value::<Json<StoreCheck>>(ValueKey::from(class(id)))
.await
.caused_by(trc::location!())?
.map(|Json(check)| check))
}
/// Every check, oldest first.
pub async fn all(data: &Store) -> trc::Result<Vec<StoreCheck>> {
let mut checks = Vec::new();
data.iterate(
IterateParams::new(ValueKey::from(class(0)), ValueKey::from(class(u32::MAX))),
|_, value| {
if let Ok(Json(check)) = Json::<StoreCheck>::deserialize(value) {
checks.push(check);
}
Ok(true)
},
)
.await
.caused_by(trc::location!())?;
checks.sort_by_key(|c| c.id);
Ok(checks)
}
/// The id of the check that runs now, if any.
pub async fn running(data: &Store) -> trc::Result<Option<u32>> {
Ok(data
.get_value::<u32>(ValueKey::from(running_class()))
.await
.caused_by(trc::location!())?)
}
/// SV-2: why a check can't start.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Refusal {
/// This check runs now.
Running(u32),
}
/// Writes a new check under the next free id, which it returns, and makes
/// it the one that runs. Refused while another runs (SV-2): a running
/// check whose node went quiet is marked interrupted first (SV-8). The
/// oldest checks past [`MAX_CHECKS`] are forgotten (SV-1).
pub async fn create(
data: &Store,
check: &StoreCheck,
now: u64,
) -> trc::Result<Result<u32, Refusal>> {
let mut attempt = 0;
loop {
attempt += 1;
let checks = all(data).await?;
let current = running(data).await?;
if let Some(current) = current {
match checks.iter().find(|c| c.id == current) {
Some(c) if c.status_at(now) == Status::Running => {
return Ok(Err(Refusal::Running(current)));
}
_ => {
// Gone, finished or quiet: it no longer holds the place
if let Some(c) = checks.iter().find(|c| c.id == current)
&& c.status == Status::Running
{
update(data, current, |c| {
if c.status != Status::Running {
return false;
}
c.status = Status::Interrupted;
c.finished_at = Some(now);
true
})
.await?;
}
release(data, current).await?;
}
}
}
let id = checks.iter().map(|c| c.id).max().unwrap_or(0) + 1;
let stored = StoreCheck {
id,
..check.clone()
};
let mut batch = BatchBuilder::new();
batch.assert_value(class(id), AssertValue::None);
batch.assert_value(running_class(), AssertValue::None);
batch.set(class(id), Json(&stored).serialize()?);
batch.set(running_class(), SerializeInfallible::serialize(&id));
match data.write(batch.build_all()).await {
Ok(_) => {
// SV-1: the newest MAX_CHECKS stay
let mut checks = checks;
checks.push(stored);
let over = checks.len().saturating_sub(MAX_CHECKS);
for old in checks.iter().take(over) {
if old.status != Status::Running {
remove(data, old.id).await?;
}
}
return Ok(Ok(id));
}
Err(err)
if attempt < CREATE_ATTEMPTS
&& matches!(
err.as_ref(),
trc::EventType::Store(trc::StoreEvent::AssertValueFailed)
) => {}
Err(err) => return Err(err.caused_by(trc::location!())),
}
}
}
/// Lets another check run, if `id` is the one that runs now.
pub async fn release(data: &Store, id: u32) -> trc::Result<()> {
let mut batch = BatchBuilder::new();
batch.assert_value(running_class(), AssertValue::U32(id));
batch.clear(running_class());
match data.write(batch.build_all()).await {
Ok(_) => Ok(()),
Err(err)
if matches!(
err.as_ref(),
trc::EventType::Store(trc::StoreEvent::AssertValueFailed)
) =>
{
Ok(())
}
Err(err) => Err(err.caused_by(trc::location!())),
}
}
/// A check as stored, with the version [`replace`] checks: a hash of the
/// stored bytes.
struct Versioned(StoreCheck, u64);
impl Deserialize for Versioned {
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
let Json(check) = Json::<StoreCheck>::deserialize(bytes)?;
Ok(Versioned(check, xxhash_rust::xxh3::xxh3_64(bytes)))
}
}
async fn get_versioned(data: &Store, id: u32) -> trc::Result<Option<(StoreCheck, u64)>> {
Ok(data
.get_value::<Versioned>(ValueKey::from(class(id)))
.await
.caused_by(trc::location!())?
.map(|Versioned(check, version)| (check, version)))
}
/// Reads a check, changes it with `change`, and writes it back, again if
/// something else wrote it in between (the running node saving progress
/// while an administrator stops it). `change` returns false to leave it as
/// it is. None if there's no such check.
pub async fn update<F>(data: &Store, id: u32, mut change: F) -> trc::Result<Option<StoreCheck>>
where
F: FnMut(&mut StoreCheck) -> bool,
{
loop {
let Some((mut check, version)) = get_versioned(data, id).await? else {
return Ok(None);
};
if !change(&mut check) {
return Ok(Some(check));
}
let mut batch = BatchBuilder::new();
batch.assert_value(class(id), AssertValue::Hash(version));
batch.set(class(id), Json(&check).serialize()?);
match data.write(batch.build_all()).await {
Ok(_) => return Ok(Some(check)),
Err(err)
if matches!(
err.as_ref(),
trc::EventType::Store(trc::StoreEvent::AssertValueFailed)
) => {}
Err(err) => return Err(err.caused_by(trc::location!())),
}
}
}
/// Keeps problems `first..` of a check (SV-5): the caller numbers them and
/// stops at [`MAX_PROBLEMS`].
pub async fn add_problems(
data: &Store,
id: u32,
first: u64,
problems: &[Problem],
) -> trc::Result<()> {
let mut batch = BatchBuilder::new();
for (n, problem) in problems.iter().enumerate() {
let number = u32::try_from(first + n as u64).unwrap_or(u32::MAX);
batch.set(problem_class(id, number), Json(problem).serialize()?);
if batch.is_large_batch() {
data.write(batch.build_all())
.await
.caused_by(trc::location!())?;
batch = BatchBuilder::new();
}
}
if !batch.is_empty() {
data.write(batch.build_all())
.await
.caused_by(trc::location!())?;
}
Ok(())
}
/// A check's kept problems, in the order they were found.
pub async fn problems(data: &Store, id: u32) -> trc::Result<Vec<Problem>> {
let mut problems = Vec::new();
data.iterate(
IterateParams::new(
ValueKey::from(problem_class(id, 0)),
ValueKey::from(problem_class(id, u32::MAX)),
),
|_, value| {
if let Ok(Json(problem)) = Json::<Problem>::deserialize(value) {
problems.push(problem);
}
Ok(true)
},
)
.await
.caused_by(trc::location!())?;
Ok(problems)
}
/// Forgets a check and its problems (SV-1, SV-10).
pub async fn remove(data: &Store, id: u32) -> trc::Result<()> {
data.delete_range(
ValueKey::from(problem_class(id, 0)),
ValueKey::from(problem_class(id, u32::MAX)),
)
.await
.caused_by(trc::location!())?;
let mut batch = BatchBuilder::new();
batch.clear(class(id));
data.write(batch.build_all())
.await
.caused_by(trc::location!())
.map(|_| ())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn keys_stay_clear_of_the_store_version() {
for class in [class(1), problem_class(1, 0), running_class()] {
let ValueClass::Any(any) = class else {
unreachable!()
};
assert_eq!(any.key[0], b'V');
assert_ne!(any.key[1], b's');
}
}
#[test]
fn a_quiet_check_reads_as_interrupted() {
let check = StoreCheck::new(Mode::Quick, "admin".into(), 1, 1_000);
assert_eq!(check.status_at(1_000 + STALE_AFTER), Status::Running);
assert_eq!(check.status_at(1_001 + STALE_AFTER), Status::Interrupted);
let done = StoreCheck {
status: Status::Done,
..check
};
assert_eq!(done.status_at(u64::MAX / 2), Status::Done);
}
#[test]
fn problems_round_trip_as_json() {
let problem = Problem {
kind: ProblemKind::Missing,
account_id: Some(3),
account_name: "[email protected]".into(),
object: "email".into(),
object_id: 7,
blob_id: "ab".into(),
committed: true,
detail: String::new(),
};
let bytes = Json(&problem).serialize().unwrap();
let Json(back) = Json::<Problem>::deserialize(&bytes).unwrap();
assert_eq!(back, problem);
assert!(
String::from_utf8(bytes)
.unwrap()
.contains("\"accountId\":3")
);
}
}
@@ -0,0 +1,221 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! `inbuxa:BlobStoreChange/get` and `/set` under `urn:inbuxa:jmap`: a change
//! of blob store the server noticed at start-up (blob-store-change spec,
//! BM-2). The server records them; an update with `action: "copy"` copies
//! what the new store lacks from the old one, `action: "stop"` stops that,
//! and destroying one dismisses it (BM-5 to BM-7).
use crate::object::{AnyId, JmapObject, JmapObjectId};
use jmap_tools::{Element, Key, Property};
use std::{borrow::Cow, str::FromStr};
use types::id::Id;
#[derive(Debug, Clone, Default)]
pub struct BlobStoreChange;
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum BlobStoreChangeProperty {
Id,
From,
To,
ChangedAt,
Node,
Status,
CopyStartedAt,
CopyFinishedAt,
CopyStartedBy,
CopyNode,
BlobsPerSecond,
BytesPerSecond,
Progress,
BlobsChecked,
Copied,
AlreadyThere,
BytesCopied,
Missing,
Damaged,
Unreadable,
Problems,
Error,
Action,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum BlobStoreChangeValue {
Id(Id),
}
impl Property for BlobStoreChangeProperty {
fn try_parse(parent: Option<&Key<'_, Self>>, value: &str) -> Option<Self> {
match parent {
None => BlobStoreChangeProperty::parse(value),
Some(_) => None,
}
}
fn to_cow(&self) -> Cow<'static, str> {
match self {
BlobStoreChangeProperty::Id => "id",
BlobStoreChangeProperty::From => "from",
BlobStoreChangeProperty::To => "to",
BlobStoreChangeProperty::ChangedAt => "changedAt",
BlobStoreChangeProperty::Node => "node",
BlobStoreChangeProperty::Status => "status",
BlobStoreChangeProperty::CopyStartedAt => "copyStartedAt",
BlobStoreChangeProperty::CopyFinishedAt => "copyFinishedAt",
BlobStoreChangeProperty::CopyStartedBy => "copyStartedBy",
BlobStoreChangeProperty::CopyNode => "copyNode",
BlobStoreChangeProperty::BlobsPerSecond => "blobsPerSecond",
BlobStoreChangeProperty::BytesPerSecond => "bytesPerSecond",
BlobStoreChangeProperty::Progress => "progress",
BlobStoreChangeProperty::BlobsChecked => "blobsChecked",
BlobStoreChangeProperty::Copied => "copied",
BlobStoreChangeProperty::AlreadyThere => "alreadyThere",
BlobStoreChangeProperty::BytesCopied => "bytesCopied",
BlobStoreChangeProperty::Missing => "missing",
BlobStoreChangeProperty::Damaged => "damaged",
BlobStoreChangeProperty::Unreadable => "unreadable",
BlobStoreChangeProperty::Problems => "problems",
BlobStoreChangeProperty::Error => "error",
BlobStoreChangeProperty::Action => "action",
}
.into()
}
}
impl BlobStoreChangeProperty {
fn parse(value: &str) -> Option<Self> {
hashify::tiny_map!(value.as_bytes(),
b"id" => BlobStoreChangeProperty::Id,
b"from" => BlobStoreChangeProperty::From,
b"to" => BlobStoreChangeProperty::To,
b"changedAt" => BlobStoreChangeProperty::ChangedAt,
b"node" => BlobStoreChangeProperty::Node,
b"status" => BlobStoreChangeProperty::Status,
b"copyStartedAt" => BlobStoreChangeProperty::CopyStartedAt,
b"copyFinishedAt" => BlobStoreChangeProperty::CopyFinishedAt,
b"copyStartedBy" => BlobStoreChangeProperty::CopyStartedBy,
b"copyNode" => BlobStoreChangeProperty::CopyNode,
b"blobsPerSecond" => BlobStoreChangeProperty::BlobsPerSecond,
b"bytesPerSecond" => BlobStoreChangeProperty::BytesPerSecond,
b"progress" => BlobStoreChangeProperty::Progress,
b"blobsChecked" => BlobStoreChangeProperty::BlobsChecked,
b"copied" => BlobStoreChangeProperty::Copied,
b"alreadyThere" => BlobStoreChangeProperty::AlreadyThere,
b"bytesCopied" => BlobStoreChangeProperty::BytesCopied,
b"missing" => BlobStoreChangeProperty::Missing,
b"damaged" => BlobStoreChangeProperty::Damaged,
b"unreadable" => BlobStoreChangeProperty::Unreadable,
b"problems" => BlobStoreChangeProperty::Problems,
b"error" => BlobStoreChangeProperty::Error,
b"action" => BlobStoreChangeProperty::Action,
)
}
}
impl FromStr for BlobStoreChangeProperty {
type Err = ();
fn from_str(s: &str) -> Result<Self, Self::Err> {
BlobStoreChangeProperty::parse(s).ok_or(())
}
}
impl Element for BlobStoreChangeValue {
type Property = BlobStoreChangeProperty;
fn try_parse<P>(key: &Key<'_, Self::Property>, value: &str) -> Option<Self> {
match key {
Key::Property(BlobStoreChangeProperty::Id) => {
Id::from_str(value).ok().map(BlobStoreChangeValue::Id)
}
_ => None,
}
}
fn to_cow(&self) -> Cow<'static, str> {
match self {
BlobStoreChangeValue::Id(id) => id.to_string().into(),
}
}
}
impl JmapObject for BlobStoreChange {
type Property = BlobStoreChangeProperty;
type Element = BlobStoreChangeValue;
type Id = Id;
type Filter = ();
type Comparator = ();
type GetArguments = ();
type SetArguments<'de> = ();
type QueryArguments = ();
type CopyArguments = ();
type ParseArguments = ();
const ID_PROPERTY: Self::Property = BlobStoreChangeProperty::Id;
}
impl From<Id> for BlobStoreChangeValue {
fn from(id: Id) -> Self {
BlobStoreChangeValue::Id(id)
}
}
impl JmapObjectId for BlobStoreChangeValue {
fn as_id(&self) -> Option<Id> {
match self {
BlobStoreChangeValue::Id(id) => Some(*id),
}
}
fn as_any_id(&self) -> Option<AnyId> {
match self {
BlobStoreChangeValue::Id(id) => Some(AnyId::Id(*id)),
}
}
fn as_id_ref(&self) -> Option<&str> {
None
}
fn try_set_id(&mut self, new_id: AnyId) -> bool {
if let AnyId::Id(id) = new_id {
*self = BlobStoreChangeValue::Id(id);
true
} else {
false
}
}
}
impl JmapObjectId for BlobStoreChangeProperty {
fn as_id(&self) -> Option<Id> {
None
}
fn as_any_id(&self) -> Option<AnyId> {
None
}
fn as_id_ref(&self) -> Option<&str> {
None
}
fn try_set_id(&mut self, _: AnyId) -> bool {
false
}
}
@@ -0,0 +1,214 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! `inbuxa:StoreCheck/get` and `/set` under `urn:inbuxa:jmap`: checking that
//! the store has every blob its data needs (store-check spec, SV-1).
//! Creating one starts it; it runs in the background, and `get` says how
//! far it got and what it found. An update with `action: "stop"` stops it.
use crate::object::{AnyId, JmapObject, JmapObjectId};
use jmap_tools::{Element, Key, Property};
use std::{borrow::Cow, str::FromStr};
use types::id::Id;
#[derive(Debug, Clone, Default)]
pub struct StoreCheck;
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum StoreCheckProperty {
Id,
Mode,
BlobsPerSecond,
BytesPerSecond,
Status,
CreatedAt,
CreatedBy,
FinishedAt,
Node,
Progress,
BlobsChecked,
LinksChecked,
BytesRead,
MessagesChecked,
Missing,
Damaged,
Unlinked,
Unreadable,
Problems,
Error,
Action,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum StoreCheckValue {
Id(Id),
}
impl Property for StoreCheckProperty {
fn try_parse(parent: Option<&Key<'_, Self>>, value: &str) -> Option<Self> {
match parent {
None => StoreCheckProperty::parse(value),
Some(_) => None,
}
}
fn to_cow(&self) -> Cow<'static, str> {
match self {
StoreCheckProperty::Id => "id",
StoreCheckProperty::Mode => "mode",
StoreCheckProperty::BlobsPerSecond => "blobsPerSecond",
StoreCheckProperty::BytesPerSecond => "bytesPerSecond",
StoreCheckProperty::Status => "status",
StoreCheckProperty::CreatedAt => "createdAt",
StoreCheckProperty::CreatedBy => "createdBy",
StoreCheckProperty::FinishedAt => "finishedAt",
StoreCheckProperty::Node => "node",
StoreCheckProperty::Progress => "progress",
StoreCheckProperty::BlobsChecked => "blobsChecked",
StoreCheckProperty::LinksChecked => "linksChecked",
StoreCheckProperty::BytesRead => "bytesRead",
StoreCheckProperty::MessagesChecked => "messagesChecked",
StoreCheckProperty::Missing => "missing",
StoreCheckProperty::Damaged => "damaged",
StoreCheckProperty::Unlinked => "unlinked",
StoreCheckProperty::Unreadable => "unreadable",
StoreCheckProperty::Problems => "problems",
StoreCheckProperty::Error => "error",
StoreCheckProperty::Action => "action",
}
.into()
}
}
impl StoreCheckProperty {
fn parse(value: &str) -> Option<Self> {
hashify::tiny_map!(value.as_bytes(),
b"id" => StoreCheckProperty::Id,
b"mode" => StoreCheckProperty::Mode,
b"blobsPerSecond" => StoreCheckProperty::BlobsPerSecond,
b"bytesPerSecond" => StoreCheckProperty::BytesPerSecond,
b"status" => StoreCheckProperty::Status,
b"createdAt" => StoreCheckProperty::CreatedAt,
b"createdBy" => StoreCheckProperty::CreatedBy,
b"finishedAt" => StoreCheckProperty::FinishedAt,
b"node" => StoreCheckProperty::Node,
b"progress" => StoreCheckProperty::Progress,
b"blobsChecked" => StoreCheckProperty::BlobsChecked,
b"linksChecked" => StoreCheckProperty::LinksChecked,
b"bytesRead" => StoreCheckProperty::BytesRead,
b"messagesChecked" => StoreCheckProperty::MessagesChecked,
b"missing" => StoreCheckProperty::Missing,
b"damaged" => StoreCheckProperty::Damaged,
b"unlinked" => StoreCheckProperty::Unlinked,
b"unreadable" => StoreCheckProperty::Unreadable,
b"problems" => StoreCheckProperty::Problems,
b"error" => StoreCheckProperty::Error,
b"action" => StoreCheckProperty::Action,
)
}
}
impl FromStr for StoreCheckProperty {
type Err = ();
fn from_str(s: &str) -> Result<Self, Self::Err> {
StoreCheckProperty::parse(s).ok_or(())
}
}
impl Element for StoreCheckValue {
type Property = StoreCheckProperty;
fn try_parse<P>(key: &Key<'_, Self::Property>, value: &str) -> Option<Self> {
match key {
Key::Property(StoreCheckProperty::Id) => {
Id::from_str(value).ok().map(StoreCheckValue::Id)
}
_ => None,
}
}
fn to_cow(&self) -> Cow<'static, str> {
match self {
StoreCheckValue::Id(id) => id.to_string().into(),
}
}
}
impl JmapObject for StoreCheck {
type Property = StoreCheckProperty;
type Element = StoreCheckValue;
type Id = Id;
type Filter = ();
type Comparator = ();
type GetArguments = ();
type SetArguments<'de> = ();
type QueryArguments = ();
type CopyArguments = ();
type ParseArguments = ();
const ID_PROPERTY: Self::Property = StoreCheckProperty::Id;
}
impl From<Id> for StoreCheckValue {
fn from(id: Id) -> Self {
StoreCheckValue::Id(id)
}
}
impl JmapObjectId for StoreCheckValue {
fn as_id(&self) -> Option<Id> {
match self {
StoreCheckValue::Id(id) => Some(*id),
}
}
fn as_any_id(&self) -> Option<AnyId> {
match self {
StoreCheckValue::Id(id) => Some(AnyId::Id(*id)),
}
}
fn as_id_ref(&self) -> Option<&str> {
None
}
fn try_set_id(&mut self, new_id: AnyId) -> bool {
if let AnyId::Id(id) = new_id {
*self = StoreCheckValue::Id(id);
true
} else {
false
}
}
}
impl JmapObjectId for StoreCheckProperty {
fn as_id(&self) -> Option<Id> {
None
}
fn as_any_id(&self) -> Option<AnyId> {
None
}
fn as_id_ref(&self) -> Option<&str> {
None
}
fn try_set_id(&mut self, _: AnyId) -> bool {
false
}
}
@@ -0,0 +1,188 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! `inbuxa:StoreSnapshot/get` and `/set` under `urn:inbuxa:jmap`: a snapshot
//! of the store, taken without stopping the server (store-snapshot spec,
//! SN-1). Creating one takes it in the background; `get` lists the
//! snapshots on disk and says how the one being taken stands. Destroying one
//! deletes its directory.
use crate::object::{AnyId, JmapObject, JmapObjectId};
use jmap_tools::{Element, Key, Property};
use std::{borrow::Cow, str::FromStr};
use types::id::Id;
#[derive(Debug, Clone, Default)]
pub struct StoreSnapshot;
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum StoreSnapshotProperty {
Id,
Status,
CreatedAt,
CreatedBy,
FinishedAt,
Node,
Name,
Path,
Holds,
Size,
BlobFiles,
Error,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum StoreSnapshotValue {
Id(Id),
}
impl Property for StoreSnapshotProperty {
fn try_parse(parent: Option<&Key<'_, Self>>, value: &str) -> Option<Self> {
match parent {
None => StoreSnapshotProperty::parse(value),
Some(_) => None,
}
}
fn to_cow(&self) -> Cow<'static, str> {
match self {
StoreSnapshotProperty::Id => "id",
StoreSnapshotProperty::Status => "status",
StoreSnapshotProperty::CreatedAt => "createdAt",
StoreSnapshotProperty::CreatedBy => "createdBy",
StoreSnapshotProperty::FinishedAt => "finishedAt",
StoreSnapshotProperty::Node => "node",
StoreSnapshotProperty::Name => "name",
StoreSnapshotProperty::Path => "path",
StoreSnapshotProperty::Holds => "holds",
StoreSnapshotProperty::Size => "size",
StoreSnapshotProperty::BlobFiles => "blobFiles",
StoreSnapshotProperty::Error => "error",
}
.into()
}
}
impl StoreSnapshotProperty {
fn parse(value: &str) -> Option<Self> {
hashify::tiny_map!(value.as_bytes(),
b"id" => StoreSnapshotProperty::Id,
b"status" => StoreSnapshotProperty::Status,
b"createdAt" => StoreSnapshotProperty::CreatedAt,
b"createdBy" => StoreSnapshotProperty::CreatedBy,
b"finishedAt" => StoreSnapshotProperty::FinishedAt,
b"node" => StoreSnapshotProperty::Node,
b"name" => StoreSnapshotProperty::Name,
b"path" => StoreSnapshotProperty::Path,
b"holds" => StoreSnapshotProperty::Holds,
b"size" => StoreSnapshotProperty::Size,
b"blobFiles" => StoreSnapshotProperty::BlobFiles,
b"error" => StoreSnapshotProperty::Error,
)
}
}
impl FromStr for StoreSnapshotProperty {
type Err = ();
fn from_str(s: &str) -> Result<Self, Self::Err> {
StoreSnapshotProperty::parse(s).ok_or(())
}
}
impl Element for StoreSnapshotValue {
type Property = StoreSnapshotProperty;
fn try_parse<P>(key: &Key<'_, Self::Property>, value: &str) -> Option<Self> {
match key {
Key::Property(StoreSnapshotProperty::Id) => {
Id::from_str(value).ok().map(StoreSnapshotValue::Id)
}
_ => None,
}
}
fn to_cow(&self) -> Cow<'static, str> {
match self {
StoreSnapshotValue::Id(id) => id.to_string().into(),
}
}
}
impl JmapObject for StoreSnapshot {
type Property = StoreSnapshotProperty;
type Element = StoreSnapshotValue;
type Id = Id;
type Filter = ();
type Comparator = ();
type GetArguments = ();
type SetArguments<'de> = ();
type QueryArguments = ();
type CopyArguments = ();
type ParseArguments = ();
const ID_PROPERTY: Self::Property = StoreSnapshotProperty::Id;
}
impl From<Id> for StoreSnapshotValue {
fn from(id: Id) -> Self {
StoreSnapshotValue::Id(id)
}
}
impl JmapObjectId for StoreSnapshotValue {
fn as_id(&self) -> Option<Id> {
match self {
StoreSnapshotValue::Id(id) => Some(*id),
}
}
fn as_any_id(&self) -> Option<AnyId> {
match self {
StoreSnapshotValue::Id(id) => Some(AnyId::Id(*id)),
}
}
fn as_id_ref(&self) -> Option<&str> {
None
}
fn try_set_id(&mut self, new_id: AnyId) -> bool {
if let AnyId::Id(id) = new_id {
*self = StoreSnapshotValue::Id(id);
true
} else {
false
}
}
}
impl JmapObjectId for StoreSnapshotProperty {
fn as_id(&self) -> Option<Id> {
None
}
fn as_any_id(&self) -> Option<AnyId> {
None
}
fn as_id_ref(&self) -> Option<&str> {
None
}
fn try_set_id(&mut self, _: AnyId) -> bool {
false
}
}
+3
View File
@@ -26,6 +26,7 @@ pub mod identity;
pub mod inbuxa_account_lock; // inbuxa: account lock with delegation
pub mod inbuxa_ai_limits; // inbuxa: AI spam classification
pub mod inbuxa_audit; // inbuxa: the audit log
pub mod inbuxa_blob_store_change; // inbuxa: a change of blob store
pub mod inbuxa_data_inventory; // inbuxa: personal-data catalog
pub mod inbuxa_deleted_account; // inbuxa: undelete
pub mod inbuxa_deliverability_report; // inbuxa: the deliverability check
@@ -47,6 +48,8 @@ pub mod inbuxa_scheduled_report; // inbuxa: scheduled reports
pub mod inbuxa_scheduled_report_settings; // inbuxa: scheduled reports
pub mod inbuxa_security_acceptance; // inbuxa: accepted security to-do items
pub mod inbuxa_sharing_policy; // inbuxa: MA-C, who may share mail
pub mod inbuxa_store_check; // inbuxa: checking the store's blobs
pub mod inbuxa_store_snapshot; // inbuxa: a snapshot of the store
pub mod inbuxa_tenant_protocol_policy; // inbuxa: legacy protocols off, per tenant
pub mod mailbox;
pub mod participant_identity;
+9
View File
@@ -88,6 +88,15 @@ impl Response<'_> {
GetResponseMethod::MailRule(response) => {
response.eval_jptr(path, &mut results)
}
GetResponseMethod::StoreCheck(response) => {
response.eval_jptr(path, &mut results)
}
GetResponseMethod::StoreSnapshot(response) => {
response.eval_jptr(path, &mut results)
}
GetResponseMethod::BlobStoreChange(response) => {
response.eval_jptr(path, &mut results)
}
GetResponseMethod::SecurityAcceptance(response) => {
response.eval_jptr(path, &mut results)
}
@@ -55,6 +55,9 @@ impl Response<'_> {
GetRequestMethod::AccountLock(request) => request.resolve_references(self)?,
GetRequestMethod::LegalHold(request) => request.resolve_references(self)?,
GetRequestMethod::MailRule(request) => request.resolve_references(self)?,
GetRequestMethod::StoreCheck(request) => request.resolve_references(self)?,
GetRequestMethod::StoreSnapshot(request) => request.resolve_references(self)?,
GetRequestMethod::BlobStoreChange(request) => request.resolve_references(self)?,
GetRequestMethod::SecurityAcceptance(request) => {
request.resolve_references(self)?
}
@@ -149,6 +152,15 @@ impl Response<'_> {
SetRequestMethod::MailRule(request) => {
request.resolve_references(self, 1, false)?
}
SetRequestMethod::StoreCheck(request) => {
request.resolve_references(self, 1, false)?
}
SetRequestMethod::StoreSnapshot(request) => {
request.resolve_references(self, 1, false)?
}
SetRequestMethod::BlobStoreChange(request) => {
request.resolve_references(self, 1, false)?
}
SetRequestMethod::SecurityAcceptance(request) => {
request.resolve_references(self, 1, false)?
}
+24
View File
@@ -68,6 +68,12 @@ pub enum MethodObject {
ProtocolPolicy,
// inbuxa: DLP and mail flow rules
MailRule,
// inbuxa: checking the store's blobs
StoreCheck,
// inbuxa: a snapshot of the store
StoreSnapshot,
// inbuxa: a change of blob store
BlobStoreChange,
// inbuxa: accepted security to-do items
SecurityAcceptance,
// inbuxa: the deliverability check
@@ -124,6 +130,9 @@ impl MethodObject {
| MethodObject::LegalHold
| MethodObject::HoldExport
| MethodObject::MailRule
| MethodObject::StoreCheck
| MethodObject::StoreSnapshot
| MethodObject::BlobStoreChange
| MethodObject::SecurityAcceptance
| MethodObject::HeldMessage
| MethodObject::DeliverabilityReport
@@ -336,6 +345,12 @@ impl MethodName {
(MethodFunction::Set, MethodObject::LegalHold) => "inbuxa:LegalHold/set",
(MethodFunction::Get, MethodObject::MailRule) => "inbuxa:MailRule/get",
(MethodFunction::Set, MethodObject::MailRule) => "inbuxa:MailRule/set",
(MethodFunction::Get, MethodObject::StoreCheck) => "inbuxa:StoreCheck/get",
(MethodFunction::Set, MethodObject::StoreCheck) => "inbuxa:StoreCheck/set",
(MethodFunction::Get, MethodObject::StoreSnapshot) => "inbuxa:StoreSnapshot/get",
(MethodFunction::Set, MethodObject::StoreSnapshot) => "inbuxa:StoreSnapshot/set",
(MethodFunction::Get, MethodObject::BlobStoreChange) => "inbuxa:BlobStoreChange/get",
(MethodFunction::Set, MethodObject::BlobStoreChange) => "inbuxa:BlobStoreChange/set",
(MethodFunction::Get, MethodObject::SecurityAcceptance) => {
"inbuxa:SecurityAcceptance/get"
}
@@ -534,6 +549,12 @@ impl MethodName {
"inbuxa:LegalHold/set" => (MethodObject::LegalHold, MethodFunction::Set),
"inbuxa:MailRule/get" => (MethodObject::MailRule, MethodFunction::Get),
"inbuxa:MailRule/set" => (MethodObject::MailRule, MethodFunction::Set),
"inbuxa:StoreCheck/get" => (MethodObject::StoreCheck, MethodFunction::Get),
"inbuxa:StoreCheck/set" => (MethodObject::StoreCheck, MethodFunction::Set),
"inbuxa:StoreSnapshot/get" => (MethodObject::StoreSnapshot, MethodFunction::Get),
"inbuxa:StoreSnapshot/set" => (MethodObject::StoreSnapshot, MethodFunction::Set),
"inbuxa:BlobStoreChange/get" => (MethodObject::BlobStoreChange, MethodFunction::Get),
"inbuxa:BlobStoreChange/set" => (MethodObject::BlobStoreChange, MethodFunction::Set),
"inbuxa:SecurityAcceptance/get" => (MethodObject::SecurityAcceptance, MethodFunction::Get),
"inbuxa:SecurityAcceptance/set" => (MethodObject::SecurityAcceptance, MethodFunction::Set),
"inbuxa:DeliverabilityReport/get" => (MethodObject::DeliverabilityReport, MethodFunction::Get),
@@ -630,6 +651,9 @@ impl Display for MethodObject {
MethodObject::AccountLock => "inbuxa:AccountLock",
MethodObject::LegalHold => "inbuxa:LegalHold",
MethodObject::MailRule => "inbuxa:MailRule",
MethodObject::StoreCheck => "inbuxa:StoreCheck",
MethodObject::StoreSnapshot => "inbuxa:StoreSnapshot",
MethodObject::BlobStoreChange => "inbuxa:BlobStoreChange",
MethodObject::SecurityAcceptance => "inbuxa:SecurityAcceptance",
MethodObject::DeliverabilityReport => "inbuxa:DeliverabilityReport",
MethodObject::DeliverabilitySettings => "inbuxa:DeliverabilitySettings",
+6
View File
@@ -127,6 +127,9 @@ pub enum GetRequestMethod {
AccountLock(Box<GetRequest<crate::object::inbuxa_account_lock::AccountLock>>),
LegalHold(Box<GetRequest<crate::object::inbuxa_legal_hold::LegalHold>>),
MailRule(Box<GetRequest<crate::object::inbuxa_mail_rule::MailRule>>),
StoreCheck(Box<GetRequest<crate::object::inbuxa_store_check::StoreCheck>>),
StoreSnapshot(Box<GetRequest<crate::object::inbuxa_store_snapshot::StoreSnapshot>>),
BlobStoreChange(Box<GetRequest<crate::object::inbuxa_blob_store_change::BlobStoreChange>>),
SecurityAcceptance(
Box<GetRequest<crate::object::inbuxa_security_acceptance::SecurityAcceptance>>,
),
@@ -183,6 +186,9 @@ pub enum SetRequestMethod<'x> {
AccountLock(Box<SetRequest<'x, crate::object::inbuxa_account_lock::AccountLock>>),
LegalHold(Box<SetRequest<'x, crate::object::inbuxa_legal_hold::LegalHold>>),
MailRule(Box<SetRequest<'x, crate::object::inbuxa_mail_rule::MailRule>>),
StoreCheck(Box<SetRequest<'x, crate::object::inbuxa_store_check::StoreCheck>>),
StoreSnapshot(Box<SetRequest<'x, crate::object::inbuxa_store_snapshot::StoreSnapshot>>),
BlobStoreChange(Box<SetRequest<'x, crate::object::inbuxa_blob_store_change::BlobStoreChange>>),
SecurityAcceptance(
Box<SetRequest<'x, crate::object::inbuxa_security_acceptance::SecurityAcceptance>>,
),
+45
View File
@@ -689,6 +689,51 @@ impl<'de> Visitor<'de> for CallVisitor {
return Err(de::Error::invalid_length(1, &self));
}
},
// inbuxa: checking the store's blobs
(MethodFunction::Get, MethodObject::StoreCheck) => match seq.next_element() {
Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::StoreCheck(value)),
Err(err) => RequestMethod::invalid(err),
Ok(None) => {
return Err(de::Error::invalid_length(1, &self));
}
},
(MethodFunction::Set, MethodObject::StoreCheck) => match seq.next_element() {
Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::StoreCheck(value)),
Err(err) => RequestMethod::invalid(err),
Ok(None) => {
return Err(de::Error::invalid_length(1, &self));
}
},
// inbuxa: a snapshot of the store
(MethodFunction::Get, MethodObject::StoreSnapshot) => match seq.next_element() {
Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::StoreSnapshot(value)),
Err(err) => RequestMethod::invalid(err),
Ok(None) => {
return Err(de::Error::invalid_length(1, &self));
}
},
(MethodFunction::Set, MethodObject::StoreSnapshot) => match seq.next_element() {
Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::StoreSnapshot(value)),
Err(err) => RequestMethod::invalid(err),
Ok(None) => {
return Err(de::Error::invalid_length(1, &self));
}
},
// inbuxa: a change of blob store
(MethodFunction::Get, MethodObject::BlobStoreChange) => match seq.next_element() {
Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::BlobStoreChange(value)),
Err(err) => RequestMethod::invalid(err),
Ok(None) => {
return Err(de::Error::invalid_length(1, &self));
}
},
(MethodFunction::Set, MethodObject::BlobStoreChange) => match seq.next_element() {
Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::BlobStoreChange(value)),
Err(err) => RequestMethod::invalid(err),
Ok(None) => {
return Err(de::Error::invalid_length(1, &self));
}
},
// inbuxa: accepted security to-do items
(MethodFunction::Get, MethodObject::SecurityAcceptance) => match seq.next_element() {
Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::SecurityAcceptance(value)),
+53
View File
@@ -112,6 +112,9 @@ pub enum GetResponseMethod {
AccountLock(GetResponse<crate::object::inbuxa_account_lock::AccountLock>),
LegalHold(GetResponse<crate::object::inbuxa_legal_hold::LegalHold>),
MailRule(GetResponse<crate::object::inbuxa_mail_rule::MailRule>),
StoreCheck(GetResponse<crate::object::inbuxa_store_check::StoreCheck>),
StoreSnapshot(GetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>),
BlobStoreChange(GetResponse<crate::object::inbuxa_blob_store_change::BlobStoreChange>),
SecurityAcceptance(GetResponse<crate::object::inbuxa_security_acceptance::SecurityAcceptance>),
DeliverabilityReport(
GetResponse<crate::object::inbuxa_deliverability_report::DeliverabilityReport>,
@@ -166,6 +169,9 @@ pub enum SetResponseMethod {
AccountLock(Box<SetResponse<crate::object::inbuxa_account_lock::AccountLock>>),
LegalHold(Box<SetResponse<crate::object::inbuxa_legal_hold::LegalHold>>),
MailRule(Box<SetResponse<crate::object::inbuxa_mail_rule::MailRule>>),
StoreCheck(Box<SetResponse<crate::object::inbuxa_store_check::StoreCheck>>),
StoreSnapshot(Box<SetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>>),
BlobStoreChange(Box<SetResponse<crate::object::inbuxa_blob_store_change::BlobStoreChange>>),
SecurityAcceptance(
Box<SetResponse<crate::object::inbuxa_security_acceptance::SecurityAcceptance>>,
),
@@ -1033,6 +1039,53 @@ impl<'x> From<SetResponse<crate::object::inbuxa_mail_rule::MailRule>> for Respon
}
}
// inbuxa: checking the store's blobs
impl<'x> From<GetResponse<crate::object::inbuxa_store_check::StoreCheck>> for ResponseMethod<'x> {
fn from(value: GetResponse<crate::object::inbuxa_store_check::StoreCheck>) -> Self {
ResponseMethod::Get(GetResponseMethod::StoreCheck(value))
}
}
impl<'x> From<SetResponse<crate::object::inbuxa_store_check::StoreCheck>> for ResponseMethod<'x> {
fn from(value: SetResponse<crate::object::inbuxa_store_check::StoreCheck>) -> Self {
ResponseMethod::Set(SetResponseMethod::StoreCheck(Box::new(value)))
}
}
// inbuxa: a snapshot of the store
impl<'x> From<GetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>>
for ResponseMethod<'x>
{
fn from(value: GetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>) -> Self {
ResponseMethod::Get(GetResponseMethod::StoreSnapshot(value))
}
}
impl<'x> From<SetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>>
for ResponseMethod<'x>
{
fn from(value: SetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>) -> Self {
ResponseMethod::Set(SetResponseMethod::StoreSnapshot(Box::new(value)))
}
}
// inbuxa: a change of blob store
impl<'x> From<GetResponse<crate::object::inbuxa_blob_store_change::BlobStoreChange>>
for ResponseMethod<'x>
{
fn from(value: GetResponse<crate::object::inbuxa_blob_store_change::BlobStoreChange>) -> Self {
ResponseMethod::Get(GetResponseMethod::BlobStoreChange(value))
}
}
impl<'x> From<SetResponse<crate::object::inbuxa_blob_store_change::BlobStoreChange>>
for ResponseMethod<'x>
{
fn from(value: SetResponse<crate::object::inbuxa_blob_store_change::BlobStoreChange>) -> Self {
ResponseMethod::Set(SetResponseMethod::BlobStoreChange(Box::new(value)))
}
}
// inbuxa: journaling
impl<'x> From<GetResponse<crate::object::inbuxa_journal_entry::JournalEntry>>
for ResponseMethod<'x>
+33
View File
@@ -116,6 +116,12 @@ impl JmapAuthorization for AccessToken {
Permission::SysDlpPolicyGet
}
}
// inbuxa: store-check spec, SV-9
GetRequestMethod::StoreCheck(_) => Permission::SysStoreCheck,
// inbuxa: store-snapshot spec, SN-8
GetRequestMethod::StoreSnapshot(_) => Permission::SysStoreSnapshot,
// inbuxa: blob-store-change spec, BM-9
GetRequestMethod::BlobStoreChange(_) => Permission::SysBlobStoreChange,
// inbuxa: journaling (JR-18)
GetRequestMethod::Journal(_) => Permission::SysJournalGet,
GetRequestMethod::JournalEntry(_) => Permission::SysJournalSearch,
@@ -310,6 +316,30 @@ impl JmapAuthorization for AccessToken {
.details("You are not authorized to change mail rules"))
}
}
// inbuxa: store-check spec, SV-9
SetRequestMethod::StoreCheck(s) => validate_set(
s,
self,
Permission::SysStoreCheck,
Permission::SysStoreCheck,
Permission::SysStoreCheck,
),
// inbuxa: store-snapshot spec, SN-8
SetRequestMethod::StoreSnapshot(s) => validate_set(
s,
self,
Permission::SysStoreSnapshot,
Permission::SysStoreSnapshot,
Permission::SysStoreSnapshot,
),
// inbuxa: blob-store-change spec, BM-9
SetRequestMethod::BlobStoreChange(s) => validate_set(
s,
self,
Permission::SysBlobStoreChange,
Permission::SysBlobStoreChange,
Permission::SysBlobStoreChange,
),
// inbuxa: journaling (JR-18)
SetRequestMethod::Journal(s) => validate_set(
s,
@@ -566,6 +596,9 @@ impl JmapAuthorization for AccessToken {
| MethodObject::LegalHold
| MethodObject::HoldExport
| MethodObject::MailRule
| MethodObject::StoreCheck
| MethodObject::StoreSnapshot
| MethodObject::BlobStoreChange
| MethodObject::SecurityAcceptance
| MethodObject::DeliverabilityReport
| MethodObject::DeliverabilitySettings
+84
View File
@@ -290,6 +290,15 @@ impl RequestHandler for Server {
SetResponseMethod::MailRule(set_response) => {
set_response.update_created_ids(&mut response);
}
SetResponseMethod::StoreCheck(set_response) => {
set_response.update_created_ids(&mut response);
}
SetResponseMethod::StoreSnapshot(set_response) => {
set_response.update_created_ids(&mut response);
}
SetResponseMethod::BlobStoreChange(set_response) => {
set_response.update_created_ids(&mut response);
}
SetResponseMethod::SecurityAcceptance(set_response) => {
set_response.update_created_ids(&mut response);
}
@@ -556,6 +565,27 @@ impl RequestHandler for Server {
.await?
.into()
}
// inbuxa: checking the store's blobs
GetRequestMethod::StoreCheck(mut req) => {
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
crate::inbuxa::store_check::get(self, access_token, *req)
.await?
.into()
}
// inbuxa: a snapshot of the store
GetRequestMethod::StoreSnapshot(mut req) => {
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
crate::inbuxa::store_snapshot::get(self, access_token, *req)
.await?
.into()
}
// inbuxa: a change of blob store
GetRequestMethod::BlobStoreChange(mut req) => {
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
crate::inbuxa::blob_store_change::get(self, access_token, *req)
.await?
.into()
}
// inbuxa: accepted security to-do items
GetRequestMethod::SecurityAcceptance(mut req) => {
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
@@ -1112,6 +1142,60 @@ impl RequestHandler for Server {
.await?
.into()
}
// inbuxa: store-check spec, SV-9: starting and stopping are audited
SetRequestMethod::StoreCheck(mut req) => {
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
crate::inbuxa::audit::recorded(
self,
access_token,
session,
&method_name.obj.to_string(),
None,
None,
*req,
|req| Box::pin(crate::inbuxa::store_check::set(self, access_token, req)),
)
.await?
.into()
}
// inbuxa: store-snapshot spec, SN-8: taking and destroying are audited
SetRequestMethod::StoreSnapshot(mut req) => {
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
crate::inbuxa::audit::recorded(
self,
access_token,
session,
&method_name.obj.to_string(),
None,
None,
*req,
|req| Box::pin(crate::inbuxa::store_snapshot::set(self, access_token, req)),
)
.await?
.into()
}
// inbuxa: blob-store-change spec, BM-9: copying, stopping and dismissing are audited
SetRequestMethod::BlobStoreChange(mut req) => {
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
crate::inbuxa::audit::recorded(
self,
access_token,
session,
&method_name.obj.to_string(),
None,
None,
*req,
|req| {
Box::pin(crate::inbuxa::blob_store_change::set(
self,
access_token,
req,
))
},
)
.await?
.into()
}
// inbuxa: SS-26, every acceptance made or removed is in the
// audit log
SetRequestMethod::SecurityAcceptance(mut req) => {
+3
View File
@@ -431,6 +431,9 @@ impl IntermediateChangesResponse {
| MethodObject::LegalHold
| MethodObject::HoldExport
| MethodObject::MailRule
| MethodObject::StoreCheck
| MethodObject::StoreSnapshot
| MethodObject::BlobStoreChange
| MethodObject::SecurityAcceptance
| MethodObject::DeliverabilityReport
| MethodObject::DeliverabilitySettings
+45
View File
@@ -488,6 +488,51 @@ async fn fork_current(server: &Server, object: &str, id: &MaybeInvalid<Id>) -> O
}
MaybeInvalid::Invalid(_) => None,
},
// A store check: what kind, and how it stands (store-check spec, SV-9)
// A change of blob store: where from and to, and how it stands;
// never the old store's settings (blob-store-change spec, BM-9)
"inbuxa:BlobStoreChange" => match id {
MaybeInvalid::Value(id) => {
inbuxa_features::blob_move::get(data, u32::try_from(id.id()).ok()?)
.await
.ok()?
.map(|c| {
serde_json::json!({
"from": c.from,
"to": c.to,
"status": c.status.as_str(),
})
})
}
MaybeInvalid::Invalid(_) => None,
},
"inbuxa:StoreCheck" => match id {
MaybeInvalid::Value(id) => {
inbuxa_features::store_check::get(data, u32::try_from(id.id()).ok()?)
.await
.ok()?
.map(|c| {
serde_json::json!({
"mode": c.mode.as_str(),
"status": c.status.as_str(),
})
})
}
MaybeInvalid::Invalid(_) => None,
},
// A snapshot: its directory and how it stands (store-snapshot spec,
// SN-8)
"inbuxa:StoreSnapshot" => match id {
MaybeInvalid::Value(id) => services::inbuxa_store_snapshot::find(server, id.id())
.await
.map(|s| {
serde_json::json!({
"name": s.name,
"status": s.status.as_str(),
})
}),
MaybeInvalid::Invalid(_) => None,
},
_ => None,
}
}
+324
View File
@@ -0,0 +1,324 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! `inbuxa:BlobStoreChange` (blob-store-change spec): the changes of blob
//! store the server noticed at start-up, copying what the new store lacks
//! from the old one, stopping that, and dismissing a change. The copy is
//! `services::inbuxa_blob_move`. Copying, stopping and dismissing are
//! audited by the request layer (BM-9).
use super::json::json_to_value;
use common::{Server, auth::AccessToken};
use inbuxa_features::blob_move::{self as model, BlobStoreChange as Stored, Refusal, Status};
use jmap_proto::{
error::set::SetError,
method::{
get::{GetRequest, GetResponse},
set::{SetRequest, SetResponse},
},
object::inbuxa_blob_store_change::{
BlobStoreChange, BlobStoreChangeProperty as P, BlobStoreChangeValue,
},
request::IntoValid,
types::date::UTCDate,
};
use jmap_tools::{Key, Map, Value};
use serde_json::json;
use services::inbuxa_blob_move as copies;
use store::write::now;
use types::id::Id;
type CValue = Value<'static, P, BlobStoreChangeValue>;
const ALL: &[P] = &[
P::Id,
P::From,
P::To,
P::ChangedAt,
P::Node,
P::Status,
P::CopyStartedAt,
P::CopyFinishedAt,
P::CopyStartedBy,
P::CopyNode,
P::BlobsPerSecond,
P::BytesPerSecond,
P::Progress,
P::BlobsChecked,
P::Copied,
P::AlreadyThere,
P::BytesCopied,
P::Missing,
P::Damaged,
P::Unreadable,
P::Problems,
P::Error,
];
/// BM-9: a change is about the whole server's blobs, so only the server's
/// own administrators see and act on one.
fn server_level(access_token: &AccessToken) -> trc::Result<()> {
if access_token.tenant_id().is_some() {
Err(trc::JmapEvent::Forbidden
.into_err()
.details("A blob store change is about the whole server, not a tenant."))
} else {
Ok(())
}
}
fn date(seconds: u64) -> serde_json::Value {
json!(UTCDate::from_timestamp(seconds as i64).to_string())
}
async fn to_value(server: &Server, c: &Stored, properties: &[P], now: u64) -> trc::Result<CValue> {
let status = c.status_at(now);
let copy = c.copy.as_ref();
let p = copy.map(|copy| copy.progress.clone()).unwrap_or_default();
let mut out = Map::with_capacity(properties.len());
for property in properties {
let json = match property {
P::Id => {
out.insert_unchecked(
Key::Property(P::Id),
Value::Element(BlobStoreChangeValue::Id(Id::from(c.id))),
);
continue;
}
// BM-9: where the stores are, never their settings
P::From => json!(c.from),
P::To => json!(c.to),
P::ChangedAt => date(c.changed_at),
P::Node => json!(c.node_id.to_string()),
P::Status => json!(status.as_str()),
P::CopyStartedAt => copy.map_or(serde_json::Value::Null, |c| date(c.started_at)),
P::CopyFinishedAt => copy
.and_then(|c| c.finished_at)
.map_or(serde_json::Value::Null, date),
P::CopyStartedBy => copy.map_or(serde_json::Value::Null, |c| json!(c.started_by)),
P::CopyNode => copy.map_or(serde_json::Value::Null, |c| json!(c.node_id.to_string())),
P::BlobsPerSecond => {
copy.map_or(serde_json::Value::Null, |c| json!(c.blobs_per_second))
}
P::BytesPerSecond => {
copy.map_or(serde_json::Value::Null, |c| json!(c.bytes_per_second))
}
P::Progress => match status {
Status::Copied | Status::Reverted => json!(1.0),
_ => json!(p.share()),
},
P::BlobsChecked => json!(p.blobs_checked),
P::Copied => json!(p.copied),
P::AlreadyThere => json!(p.already_there),
P::BytesCopied => json!(p.bytes_copied),
P::Missing => json!(p.missing),
P::Damaged => json!(p.damaged),
P::Unreadable => json!(p.unreadable),
P::Problems => {
let problems = model::problems(server.store(), c.id).await?;
let mut list = serde_json::to_value(&problems).unwrap_or_default();
// The account as a JMAP id, like every other account id
if let Some(items) = list.as_array_mut() {
for item in items {
if let Some(account) = item.get("accountId").and_then(|v| v.as_u64()) {
item["accountId"] = json!(Id::from(account).to_string());
}
}
}
list
}
P::Error => json!(copy.and_then(|c| c.error.clone())),
// Write-only
P::Action => continue,
};
out.insert_unchecked(Key::Property(property.clone()), json_to_value(json));
}
Ok(Value::Object(out))
}
/// `inbuxa:BlobStoreChange/get`: every change kept, newest first.
pub async fn get(
server: &Server,
access_token: &AccessToken,
mut request: GetRequest<BlobStoreChange>,
) -> trc::Result<GetResponse<BlobStoreChange>> {
server_level(access_token)?;
let properties = request.unwrap_properties(ALL);
let (ids, not_found) = request.unwrap_ids(server.core.jmap.get_max_objects)?;
let mut response = GetResponse {
account_id: request.account_id.into(),
state: None,
list: Vec::new(),
not_found,
};
let mut changes = model::all(server.store()).await?;
changes.reverse();
let now = now();
match ids {
None => {
for c in &changes {
response
.list
.push(to_value(server, c, &properties, now).await?);
}
}
Some(ids) => {
for id in ids {
match changes.iter().find(|c| u64::from(c.id) == id.id()) {
Some(c) => response
.list
.push(to_value(server, c, &properties, now).await?),
None => response.push_not_found(id),
}
}
}
}
Ok(response)
}
fn invalid(property: P, why: impl Into<String>) -> SetError<P> {
SetError::invalid_properties()
.with_property(property)
.with_description(why.into())
}
/// A limit: a whole number, 0 for none (BM-5).
fn limit(property: P, value: &Value<'_, P, BlobStoreChangeValue>) -> Result<u64, SetError<P>> {
match value {
Value::Number(n) => n
.as_u64()
.ok_or_else(|| invalid(property, "A whole number of 0 or more; 0 is no limit.")),
_ => Err(invalid(
property,
"A whole number of 0 or more; 0 is no limit.",
)),
}
}
/// `inbuxa:BlobStoreChange/set`: the server records changes, so there's no
/// create. An update copies or stops a copy; destroy dismisses (BM-5 to
/// BM-7).
pub async fn set(
server: &Server,
access_token: &AccessToken,
mut request: SetRequest<'_, BlobStoreChange>,
) -> trc::Result<SetResponse<BlobStoreChange>> {
server_level(access_token)?;
let mut response = SetResponse::from_request(&request, server.core.jmap.set_max_objects)?;
let data = server.store();
let actor = server.audit_actor(access_token).await;
for (client_id, _) in request.unwrap_create() {
response.not_created.append(
client_id,
SetError::forbidden().with_description(
"The server records a change when it starts with the blob store moved.",
),
);
}
'update: for (id, value) in request.unwrap_update().into_valid() {
let Ok(change_id) = u32::try_from(id.id()) else {
response.not_updated.append(id, SetError::not_found());
continue;
};
let mut action = None;
let mut blobs_per_second = None;
let mut bytes_per_second = None;
for (key, value) in value.into_expanded_object() {
let outcome = match &key {
Key::Property(P::Action) => {
action = value.as_str().map(|s| s.to_string());
Ok(())
}
Key::Property(P::BlobsPerSecond) => {
limit(P::BlobsPerSecond, &value).map(|l| blobs_per_second = Some(l))
}
Key::Property(P::BytesPerSecond) => {
limit(P::BytesPerSecond, &value).map(|l| bytes_per_second = Some(l))
}
_ => Err(SetError::invalid_properties()
.with_property(key.clone().into_owned())
.with_description("A change can't be edited, only copied or stopped.")),
};
if let Err(error) = outcome {
response.not_updated.append(id, error);
continue 'update;
}
}
let Some(change) = model::get(data, change_id).await? else {
response.not_updated.append(id, SetError::not_found());
continue;
};
match action.as_deref() {
Some("copy") => {
match copies::start(
server,
change_id,
blobs_per_second,
bytes_per_second,
actor.name.clone(),
)
.await?
{
Ok(()) => response.updated.append(id, None),
Err(Refusal::Running(running)) => response.not_updated.append(
id,
SetError::forbidden().with_description(format!(
"A copy of change {} runs now; one copy runs at a time. \
Stop it or wait for it.",
Id::from(running)
)),
),
Err(Refusal::Resolved) => response.not_updated.append(
id,
SetError::forbidden()
.with_description("That change is resolved; there's nothing to copy."),
),
Err(Refusal::NotFound) => {
response.not_updated.append(id, SetError::not_found())
}
}
}
Some("stop") => {
if change.status_at(now()) != Status::Copying {
response.not_updated.append(
id,
SetError::forbidden().with_description("That change isn't being copied."),
);
continue;
}
copies::stop(server, change_id).await?;
response.updated.append(id, None);
}
_ => response
.not_updated
.append(id, invalid(P::Action, "The actions are copy and stop.")),
}
}
for id in request.unwrap_destroy().into_valid() {
let Ok(change_id) = u32::try_from(id.id()) else {
response.not_destroyed.append(id, SetError::not_found());
continue;
};
match model::get(data, change_id).await? {
None => response.not_destroyed.append(id, SetError::not_found()),
Some(c) if c.status_at(now()) == Status::Copying => response.not_destroyed.append(
id,
SetError::forbidden().with_description("It's being copied; stop the copy first."),
),
Some(_) => {
model::remove(data, change_id).await?;
// BM-7: this node stops reading through it at once; the
// others within a minute (BM-10)
copies::refresh(server).await;
response.destroyed.push(id);
}
}
}
Ok(response)
}
+43
View File
@@ -0,0 +1,43 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
// The 27.1 series keeps this in its relay module, which this series lacks;
// the store check, store snapshot and blob store change handlers share it.
use jmap_tools::{Key, Map, Property, Value};
use std::borrow::Cow;
/// A JSON value as a JMAP one, its object keys kept as plain keys.
pub(super) fn json_to_value<P, E>(json: serde_json::Value) -> Value<'static, P, E>
where
P: Property,
E: jmap_tools::Element<Property = P>,
{
match json {
serde_json::Value::Null => Value::Null,
serde_json::Value::Bool(b) => Value::Bool(b),
serde_json::Value::Number(n) => {
if let Some(n) = n.as_u64() {
Value::Number(n.into())
} else if let Some(n) = n.as_i64() {
Value::Number(n.into())
} else {
Value::Number(n.as_f64().unwrap_or_default().into())
}
}
serde_json::Value::String(s) => Value::Str(Cow::Owned(s)),
serde_json::Value::Array(items) => {
Value::Array(items.into_iter().map(json_to_value).collect())
}
serde_json::Value::Object(map) => {
let mut out = Map::with_capacity(map.len());
for (key, value) in map {
out.insert_unchecked(Key::Owned(key), json_to_value(value));
}
Value::Object(out)
}
}
}
+4
View File
@@ -12,6 +12,7 @@ pub mod account_lock;
pub mod ai_limits;
pub mod audit;
pub mod audit_log;
pub mod blob_store_change; // inbuxa: a change of blob store
pub mod data_inventory;
pub mod deleted_account;
pub mod deliverability; // inbuxa: the deliverability check
@@ -25,6 +26,7 @@ pub mod hold_export;
pub mod hold_export_api;
pub mod journal;
pub mod journal_entry;
pub mod json; // inbuxa: JSON values as JMAP ones
pub mod legal_hold;
pub mod log_settings;
pub mod mail_rule;
@@ -35,6 +37,8 @@ pub mod protocol_policy;
pub mod scheduled_reports; // inbuxa: scheduled reports and the weekly digest
pub mod security_acceptance;
pub mod sharing_policy;
pub mod store_check; // inbuxa: checking the store's blobs
pub mod store_snapshot; // inbuxa: a snapshot of the store
pub mod telemetry;
pub mod tenant_protocol_policy;
pub mod undelete;
+310
View File
@@ -0,0 +1,310 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! `inbuxa:StoreCheck` (store-check spec): starting a check of the store's
//! blobs, seeing how far it got and what it found, stopping it, and
//! forgetting it. The run is `services::inbuxa_store_check`. Starting and
//! stopping one are audited by the request layer (SV-9).
use super::json::json_to_value;
use common::{Server, auth::AccessToken};
use inbuxa_features::store_check::{self as model, Mode, Refusal, Status, StoreCheck as Stored};
use jmap_proto::{
error::set::SetError,
method::{
get::{GetRequest, GetResponse},
set::{SetRequest, SetResponse},
},
object::inbuxa_store_check::{StoreCheck, StoreCheckProperty as P, StoreCheckValue},
request::IntoValid,
types::date::UTCDate,
};
use jmap_tools::{Key, Map, Value};
use serde_json::json;
use services::inbuxa_store_check as runs;
use store::write::now;
use types::id::Id;
type CValue = Value<'static, P, StoreCheckValue>;
const ALL: &[P] = &[
P::Id,
P::Mode,
P::BlobsPerSecond,
P::BytesPerSecond,
P::Status,
P::CreatedAt,
P::CreatedBy,
P::FinishedAt,
P::Node,
P::Progress,
P::BlobsChecked,
P::LinksChecked,
P::BytesRead,
P::MessagesChecked,
P::Missing,
P::Damaged,
P::Unlinked,
P::Unreadable,
P::Problems,
P::Error,
];
/// SV-9: a check reads every account's data, so only the server's own
/// administrators run one.
fn server_level(access_token: &AccessToken) -> trc::Result<()> {
if access_token.tenant_id().is_some() {
Err(trc::JmapEvent::Forbidden
.into_err()
.details("A store check reads the whole server, not a tenant."))
} else {
Ok(())
}
}
fn date(seconds: u64) -> serde_json::Value {
json!(UTCDate::from_timestamp(seconds as i64).to_string())
}
async fn to_value(server: &Server, c: &Stored, properties: &[P], now: u64) -> trc::Result<CValue> {
let status = c.status_at(now);
let mut out = Map::with_capacity(properties.len());
for property in properties {
let p = &c.progress;
let json = match property {
P::Id => {
out.insert_unchecked(
Key::Property(P::Id),
Value::Element(StoreCheckValue::Id(Id::from(c.id))),
);
continue;
}
P::Mode => json!(c.mode.as_str()),
P::BlobsPerSecond => json!(c.blobs_per_second),
P::BytesPerSecond => json!(c.bytes_per_second),
P::Status => json!(status.as_str()),
P::CreatedAt => date(c.created_at),
P::CreatedBy => json!(c.created_by),
P::FinishedAt => c.finished_at.map_or(serde_json::Value::Null, date),
P::Node => json!(c.node_id.to_string()),
P::Progress => json!(p.share(status)),
P::BlobsChecked => json!(p.blobs_checked),
P::LinksChecked => json!(p.links_checked),
P::BytesRead => json!(p.bytes_read),
P::MessagesChecked => json!(p.messages_checked),
P::Missing => json!(p.missing),
P::Damaged => json!(p.damaged),
P::Unlinked => json!(p.unlinked),
P::Unreadable => json!(p.unreadable),
P::Problems => {
let problems = model::problems(server.store(), c.id).await?;
let mut list = serde_json::to_value(&problems).unwrap_or_default();
// The account as a JMAP id, like every other account id
if let Some(items) = list.as_array_mut() {
for item in items {
if let Some(account) = item.get("accountId").and_then(|v| v.as_u64()) {
item["accountId"] = json!(Id::from(account).to_string());
}
}
}
list
}
P::Error => json!(c.error),
// Write-only
P::Action => continue,
};
out.insert_unchecked(Key::Property(property.clone()), json_to_value(json));
}
Ok(Value::Object(out))
}
/// `inbuxa:StoreCheck/get`: every check kept, newest first.
pub async fn get(
server: &Server,
access_token: &AccessToken,
mut request: GetRequest<StoreCheck>,
) -> trc::Result<GetResponse<StoreCheck>> {
server_level(access_token)?;
let properties = request.unwrap_properties(ALL);
let (ids, not_found) = request.unwrap_ids(server.core.jmap.get_max_objects)?;
let mut response = GetResponse {
account_id: request.account_id.into(),
state: None,
list: Vec::new(),
not_found,
};
let mut checks = model::all(server.store()).await?;
checks.reverse();
let now = now();
match ids {
None => {
for c in &checks {
response
.list
.push(to_value(server, c, &properties, now).await?);
}
}
Some(ids) => {
for id in ids {
match checks.iter().find(|c| u64::from(c.id) == id.id()) {
Some(c) => response
.list
.push(to_value(server, c, &properties, now).await?),
None => response.push_not_found(id),
}
}
}
}
Ok(response)
}
fn invalid(property: P, why: impl Into<String>) -> SetError<P> {
SetError::invalid_properties()
.with_property(property)
.with_description(why.into())
}
/// A limit: a whole number, 0 for none (SV-6).
fn limit(property: P, value: &Value<'_, P, StoreCheckValue>) -> Result<u64, SetError<P>> {
match value {
Value::Number(n) => n
.as_u64()
.ok_or_else(|| invalid(property, "A whole number of 0 or more; 0 is no limit.")),
_ => Err(invalid(
property,
"A whole number of 0 or more; 0 is no limit.",
)),
}
}
/// `inbuxa:StoreCheck/set`: create starts a check; an update can only stop
/// one; destroy forgets a finished one (SV-2, SV-8, SV-10).
pub async fn set(
server: &Server,
access_token: &AccessToken,
mut request: SetRequest<'_, StoreCheck>,
) -> trc::Result<SetResponse<StoreCheck>> {
server_level(access_token)?;
let mut response = SetResponse::from_request(&request, server.core.jmap.set_max_objects)?;
let data = server.store();
let actor = server.audit_actor(access_token).await;
'create: for (client_id, value) in request.unwrap_create() {
let mut mode = Mode::Quick;
let mut blobs_per_second = None;
let mut bytes_per_second = None;
for (key, value) in value.into_expanded_object() {
let outcome = match &key {
Key::Property(P::Mode) => match value.as_str().and_then(|s| Mode::parse(&s)) {
Some(m) => {
mode = m;
Ok(())
}
None => Err(invalid(P::Mode, "quick or full.")),
},
Key::Property(P::BlobsPerSecond) => {
limit(P::BlobsPerSecond, &value).map(|l| blobs_per_second = Some(l))
}
Key::Property(P::BytesPerSecond) => {
limit(P::BytesPerSecond, &value).map(|l| bytes_per_second = Some(l))
}
_ => Err(SetError::invalid_properties().with_property(key.clone().into_owned())),
};
if let Err(error) = outcome {
response.not_created.append(client_id, error);
continue 'create;
}
}
match runs::start(
server,
mode,
blobs_per_second,
bytes_per_second,
actor.name.clone(),
)
.await?
{
Ok(id) => {
let mut out = Map::with_capacity(1);
out.insert_unchecked(
Key::Property(P::Id),
Value::Element(StoreCheckValue::Id(Id::from(id))),
);
response.created.insert(client_id, Value::Object(out));
}
Err(Refusal::Running(id)) => response.not_created.append(
client_id,
SetError::forbidden().with_description(format!(
"Check {} runs now; one check runs at a time. Stop it or wait for it.",
Id::from(id)
)),
),
}
}
for (id, value) in request.unwrap_update().into_valid() {
let Ok(check_id) = u32::try_from(id.id()) else {
response.not_updated.append(id, SetError::not_found());
continue;
};
let mut action = None;
let mut bad = None;
for (key, value) in value.into_expanded_object() {
match &key {
Key::Property(P::Action) => action = value.as_str().map(|s| s.to_string()),
_ => {
bad = Some(
SetError::invalid_properties()
.with_property(key.clone().into_owned())
.with_description("A check can't be changed, only stopped."),
)
}
}
}
if let Some(error) = bad {
response.not_updated.append(id, error);
continue;
}
if action.as_deref() != Some("stop") {
response
.not_updated
.append(id, invalid(P::Action, "The one action is stop."));
continue;
}
let Some(check) = model::get(data, check_id).await? else {
response.not_updated.append(id, SetError::not_found());
continue;
};
if check.status_at(now()) != Status::Running {
response.not_updated.append(
id,
SetError::forbidden().with_description("That check isn't running."),
);
continue;
}
runs::stop(server, check_id).await?;
response.updated.append(id, None);
}
for id in request.unwrap_destroy().into_valid() {
let Ok(check_id) = u32::try_from(id.id()) else {
response.not_destroyed.append(id, SetError::not_found());
continue;
};
match model::get(data, check_id).await? {
None => response.not_destroyed.append(id, SetError::not_found()),
Some(c) if c.status_at(now()) == Status::Running => response.not_destroyed.append(
id,
SetError::forbidden().with_description("It's running; stop it first."),
),
Some(_) => {
model::remove(data, check_id).await?;
response.destroyed.push(id);
}
}
}
Ok(response)
}
+185
View File
@@ -0,0 +1,185 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! `inbuxa:StoreSnapshot` (store-snapshot spec): taking a snapshot of the
//! store without stopping the server, listing the snapshots on disk, and
//! deleting one. The run is `services::inbuxa_store_snapshot`. Taking and
//! destroying one are audited by the request layer (SN-8).
use super::json::json_to_value;
use common::{Server, auth::AccessToken};
use jmap_proto::{
error::set::SetError,
method::{
get::{GetRequest, GetResponse},
set::{SetRequest, SetResponse},
},
object::inbuxa_store_snapshot::{
StoreSnapshot, StoreSnapshotProperty as P, StoreSnapshotValue,
},
request::IntoValid,
types::date::UTCDate,
};
use jmap_tools::{Key, Map, Value};
use serde_json::json;
use services::inbuxa_store_snapshot::{self as runs, Destroyed, Refusal, Snapshot};
use types::id::Id;
type SValue = Value<'static, P, StoreSnapshotValue>;
const ALL: &[P] = &[
P::Id,
P::Status,
P::CreatedAt,
P::CreatedBy,
P::FinishedAt,
P::Node,
P::Name,
P::Path,
P::Holds,
P::Size,
P::BlobFiles,
P::Error,
];
/// SN-8: a snapshot holds every account's data, so only the server's own
/// administrators take one.
fn server_level(access_token: &AccessToken) -> trc::Result<()> {
if access_token.tenant_id().is_some() {
Err(trc::JmapEvent::Forbidden
.into_err()
.details("A snapshot holds the whole server, not a tenant."))
} else {
Ok(())
}
}
fn date(seconds: u64) -> serde_json::Value {
json!(UTCDate::from_timestamp(seconds as i64).to_string())
}
fn to_value(s: &Snapshot, properties: &[P]) -> SValue {
let mut out = Map::with_capacity(properties.len());
for property in properties {
let json = match property {
P::Id => {
out.insert_unchecked(
Key::Property(P::Id),
Value::Element(StoreSnapshotValue::Id(Id::from(s.id))),
);
continue;
}
P::Status => json!(s.status.as_str()),
P::CreatedAt => date(s.created_at),
P::CreatedBy => json!(s.created_by),
P::FinishedAt => s.finished_at.map_or(serde_json::Value::Null, date),
P::Node => json!(s.node_id.to_string()),
P::Name => json!(s.name),
P::Path => json!(s.path.to_string_lossy()),
P::Holds => json!(s.holds),
P::Size => json!(s.size),
P::BlobFiles => json!(s.blob_files),
P::Error => json!(s.error),
};
out.insert_unchecked(Key::Property(property.clone()), json_to_value(json));
}
Value::Object(out)
}
/// `inbuxa:StoreSnapshot/get`: the snapshots on disk and the one being
/// taken, newest first (SN-7).
pub async fn get(
server: &Server,
access_token: &AccessToken,
mut request: GetRequest<StoreSnapshot>,
) -> trc::Result<GetResponse<StoreSnapshot>> {
server_level(access_token)?;
let properties = request.unwrap_properties(ALL);
let (ids, not_found) = request.unwrap_ids(server.core.jmap.get_max_objects)?;
let mut response = GetResponse {
account_id: request.account_id.into(),
state: None,
list: Vec::new(),
not_found,
};
let snapshots = runs::list(server).await;
match ids {
None => {
for s in &snapshots {
response.list.push(to_value(s, &properties));
}
}
Some(ids) => {
for id in ids {
match snapshots.iter().find(|s| s.id == id.id()) {
Some(s) => response.list.push(to_value(s, &properties)),
None => response.push_not_found(id),
}
}
}
}
Ok(response)
}
/// `inbuxa:StoreSnapshot/set`: create takes one; destroy deletes one; a
/// snapshot can't be changed (SN-1, SN-6, SN-9).
pub async fn set(
server: &Server,
access_token: &AccessToken,
mut request: SetRequest<'_, StoreSnapshot>,
) -> trc::Result<SetResponse<StoreSnapshot>> {
server_level(access_token)?;
let mut response = SetResponse::from_request(&request, server.core.jmap.set_max_objects)?;
let actor = server.audit_actor(access_token).await;
for (client_id, value) in request.unwrap_create() {
if let Some((key, _)) = value.into_expanded_object().next() {
response.not_created.append(
client_id,
SetError::invalid_properties()
.with_property(key.into_owned())
.with_description("A snapshot is taken as it is; it takes no properties."),
);
continue;
}
match runs::start(server, actor.name.clone()).await? {
Ok(snapshot) => {
response
.created
.insert(client_id, to_value(&snapshot, &[P::Id, P::Name, P::Path]));
}
Err(Refusal::Running(id)) => response.not_created.append(
client_id,
SetError::forbidden().with_description(format!(
"Snapshot {} is being taken; one is taken at a time. Wait for it.",
Id::from(id)
)),
),
Err(Refusal::Unsupported(why)) => response
.not_created
.append(client_id, SetError::forbidden().with_description(why)),
}
}
for (id, _) in request.unwrap_update().into_valid() {
response.not_updated.append(
id,
SetError::forbidden().with_description("A snapshot can't be changed."),
);
}
for id in request.unwrap_destroy().into_valid() {
match runs::destroy(server, id.id()).await? {
Destroyed::Gone => response.destroyed.push(id),
Destroyed::NotFound => response.not_destroyed.append(id, SetError::not_found()),
Destroyed::Running => response.not_destroyed.append(
id,
SetError::forbidden().with_description("It's being taken; wait for it to finish."),
),
}
}
Ok(response)
}
+6
View File
@@ -1780,6 +1780,12 @@ pub enum Permission {
// inbuxa: scheduled reports and the weekly digest
SysScheduledReportGet = 688,
SysScheduledReportUpdate = 689,
// inbuxa: checking the store's blobs
SysStoreCheck = 695,
// inbuxa: a snapshot of the store, without stopping the server
SysStoreSnapshot = 696,
// inbuxa: a change of blob store, copied and dismissed
SysBlobStoreChange = 697,
SysAccountGet = 219,
SysAccountCreate = 220,
SysAccountUpdate = 221,
+10 -1
View File
@@ -7160,6 +7160,9 @@ impl EnumImpl for Permission {
b"sysDeliverabilityCheck" => Permission::SysDeliverabilityCheck,
b"sysScheduledReportGet" => Permission::SysScheduledReportGet,
b"sysScheduledReportUpdate" => Permission::SysScheduledReportUpdate,
b"sysStoreCheck" => Permission::SysStoreCheck,
b"sysStoreSnapshot" => Permission::SysStoreSnapshot,
b"sysBlobStoreChange" => Permission::SysBlobStoreChange,
b"sysAccountGet" => Permission::SysAccountGet,
b"sysAccountCreate" => Permission::SysAccountCreate,
b"sysAccountUpdate" => Permission::SysAccountUpdate,
@@ -7866,6 +7869,9 @@ impl EnumImpl for Permission {
Permission::SysDeliverabilityCheck => "sysDeliverabilityCheck",
Permission::SysScheduledReportGet => "sysScheduledReportGet",
Permission::SysScheduledReportUpdate => "sysScheduledReportUpdate",
Permission::SysStoreCheck => "sysStoreCheck",
Permission::SysStoreSnapshot => "sysStoreSnapshot",
Permission::SysBlobStoreChange => "sysBlobStoreChange",
Permission::SysAccountGet => "sysAccountGet",
Permission::SysAccountCreate => "sysAccountCreate",
Permission::SysAccountUpdate => "sysAccountUpdate",
@@ -8565,6 +8571,9 @@ impl EnumImpl for Permission {
687 => Some(Permission::SysDeliverabilityCheck),
688 => Some(Permission::SysScheduledReportGet),
689 => Some(Permission::SysScheduledReportUpdate),
695 => Some(Permission::SysStoreCheck),
696 => Some(Permission::SysStoreSnapshot),
697 => Some(Permission::SysBlobStoreChange),
219 => Some(Permission::SysAccountGet),
220 => Some(Permission::SysAccountCreate),
221 => Some(Permission::SysAccountUpdate),
@@ -9009,7 +9018,7 @@ impl EnumImpl for Permission {
}
}
const COUNT: usize = 690;
const COUNT: usize = 698;
}
impl serde::Serialize for Permission {
+583
View File
@@ -0,0 +1,583 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! Changing the blob store (docs/spec/features/blob-store-change.md): the
//! copy behind an `inbuxa:BlobStoreChange`. It goes through the blob index
//! one slice at a time and copies each blob the current store lacks from
//! the change's old store, as the old store keeps it (BM-5). It runs on the
//! node the copy was started on. Also, on every node, it keeps reading
//! through only the changes still unresolved (BM-10).
use common::{
BuildServer, Inner, Server,
manager::{SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY},
};
use inbuxa_features::blob_move::{
self as model, BlobStoreChange, Copy, DEFAULT_BLOBS_PER_SECOND, DEFAULT_BYTES_PER_SECOND,
MAX_PROBLEMS, Problem, ProblemKind, Progress, Refusal, STALE_AFTER, Status,
};
use std::{
sync::Arc,
time::{Duration, Instant},
};
use store::{
BlobStore,
write::{
blob_check::{BlobHolder, LinkedBlob},
now,
},
};
use types::{blob_hash::BlobHash, collection::Collection};
/// How often a running copy saves its progress, which is also its
/// heartbeat and when it sees a stop (BM-6).
const SAVE_EVERY: Duration = Duration::from_secs(10);
/// BM-10: how often a node reads which changes are still unresolved.
const REFRESH_EVERY: Duration = Duration::from_secs(60);
fn node_id(server: &Server) -> u64 {
server.core.network.node_id
}
/// BM-5: starts copying a change on this node. Refused while another copy
/// runs, or when the change is resolved.
pub async fn start(
server: &Server,
id: u32,
blobs_per_second: Option<u64>,
bytes_per_second: Option<u64>,
started_by: String,
) -> trc::Result<Result<(), Refusal>> {
let now = now();
let copy = Copy {
started_at: now,
started_by,
node_id: node_id(server),
blobs_per_second: blobs_per_second.unwrap_or(DEFAULT_BLOBS_PER_SECOND),
bytes_per_second: bytes_per_second.unwrap_or(DEFAULT_BYTES_PER_SECOND),
heartbeat: now,
finished_at: None,
stop_requested: false,
progress: Progress::default(),
problems_kept: 0,
error: None,
};
let change = match model::start_copy(server.store(), id, copy, now).await? {
Ok(change) => change,
Err(refusal) => return Ok(Err(refusal)),
};
let server = server.clone();
tokio::spawn(async move {
run(&server, change).await;
});
Ok(Ok(()))
}
/// BM-6: asks a running copy to stop. Any node can: the one running it
/// sees the request at its next save. None if there's no such change.
pub async fn stop(server: &Server, id: u32) -> trc::Result<Option<BlobStoreChange>> {
model::update(server.store(), id, |c| match &mut c.copy {
Some(copy) if c.status == Status::Copying && !copy.stop_requested => {
copy.stop_requested = true;
true
}
_ => false,
})
.await
}
/// BM-6: at start-up, a copy this node ran stopped with it; now and then,
/// a copy on a node that went quiet is taken to have stopped too. BM-10:
/// and every minute, only unresolved changes are read through.
pub fn spawn_blob_moves(inner: Arc<Inner>) {
tokio::spawn(async move {
let server = inner.build_server();
let node = node_id(&server);
mark_interrupted(&server, |c| c.node_id == node).await;
loop {
tokio::time::sleep(REFRESH_EVERY).await;
let server = inner.build_server();
let cutoff = now().saturating_sub(STALE_AFTER);
mark_interrupted(&server, |c| c.node_id != node && c.heartbeat < cutoff).await;
refresh(&server).await;
}
});
}
/// BM-10: stops reading through changes resolved or dismissed anywhere.
pub async fn refresh(server: &Server) {
let Some(moving) = server.blob_store().moving() else {
return;
};
match model::unresolved(server.store()).await {
Ok(changes) => {
let ids = changes.iter().map(|c| c.id).collect::<Vec<_>>();
moving.retain_changes(&ids);
}
Err(err) => {
trc::error!(err.details("Failed to read the blob store changes"));
}
}
}
async fn mark_interrupted(server: &Server, which: impl Fn(&Copy) -> bool) {
let changes = match model::all(server.store()).await {
Ok(changes) => changes,
Err(err) => {
trc::error!(err.details("Failed to read the blob store changes"));
return;
}
};
for change in changes
.into_iter()
.filter(|c| c.status == Status::Copying && c.copy.as_ref().is_some_and(&which))
{
let result = model::update(server.store(), change.id, |c| {
if c.status != Status::Copying || !c.copy.as_ref().is_some_and(&which) {
return false;
}
c.status = Status::Interrupted;
if let Some(copy) = &mut c.copy {
copy.finished_at = Some(now());
}
true
})
.await;
if let Err(err) = result.and(model::release(server.store(), change.id).await) {
trc::error!(err.details("Failed to mark an interrupted blob copy"));
}
}
}
/// BM-5: keeps a rate, a second at a time.
struct Pace {
per_second: u64,
window: Instant,
used: u64,
}
impl Pace {
fn new(per_second: u64) -> Self {
Pace {
per_second,
window: Instant::now(),
used: 0,
}
}
async fn take(&mut self, amount: u64) {
if self.per_second == 0 {
return;
}
if self.window.elapsed() >= Duration::from_secs(1) {
self.window = Instant::now();
self.used = 0;
}
self.used += amount;
if self.used >= self.per_second {
let rest = Duration::from_secs(1).saturating_sub(self.window.elapsed());
tokio::time::sleep(rest).await;
self.window = Instant::now();
self.used = self.used.saturating_sub(self.per_second);
}
}
}
/// Who holds a blob, as a problem names it (BM-5).
struct Holder {
name: String,
account_id: Option<u32>,
object_id: Option<u64>,
/// Named by the hash of its content, so it can be checked.
content_named: bool,
}
impl Holder {
fn of(blob: &LinkedBlob) -> Holder {
match blob.holders.first() {
Some(BlobHolder::Document {
account_id,
collection,
document_id,
}) => Holder {
name: match Collection::from(*collection) {
Collection::Email => "email".into(),
Collection::FileNode => "file".into(),
Collection::SieveScript => "sieveScript".into(),
Collection::CalendarEvent => "calendarEvent".into(),
Collection::ContactCard => "contactCard".into(),
_ => format!("collection{collection}"),
},
account_id: Some(*account_id),
object_id: Some(*document_id as u64),
content_named: true,
},
Some(BlobHolder::Queue { id }) => Holder {
name: "queuedMessage".into(),
account_id: None,
object_id: Some(*id),
content_named: true,
},
Some(BlobHolder::Archived { account_id, item }) => Holder {
name: "archivedItem".into(),
account_id: Some(*account_id),
object_id: Some(item.id().id()),
content_named: true,
},
None => Holder {
name: "temporary".into(),
account_id: None,
object_id: None,
content_named: false,
},
}
}
fn fixed() -> Holder {
Holder {
name: "fixed".into(),
account_id: None,
object_id: None,
content_named: false,
}
}
}
struct Run<'x> {
server: &'x Server,
id: u32,
old: BlobStore,
target: BlobStore,
progress: Progress,
found: Vec<Problem>,
kept: u64,
blobs: Pace,
bytes: Pace,
saved: Instant,
}
/// How a run ended before it finished.
enum Halt {
Stopped,
Failed(trc::Error),
}
impl From<trc::Error> for Halt {
fn from(err: trc::Error) -> Self {
Halt::Failed(err)
}
}
fn key_name(key: &[u8]) -> String {
match BlobHash::try_from_hash_slice(key) {
Ok(hash) => hash.to_hex(),
Err(_) => String::from_utf8_lossy(key).into_owned(),
}
}
impl<'x> Run<'x> {
/// BM-6: counts a problem, logs it, and keeps the first ones.
fn problem(&mut self, kind: ProblemKind, key: &[u8], holder: &Holder, detail: String) {
self.progress.count(kind);
let blob_id = key_name(key);
trc::event!(
Store(trc::StoreEvent::DataCorruption),
Id = self.id,
AccountId = holder.account_id,
Collection = holder.name.clone(),
DocumentId = holder.object_id,
BlobId = blob_id.clone(),
Reason = kind.as_str(),
Details = detail.clone(),
);
if self.kept < MAX_PROBLEMS {
self.kept += 1;
self.found.push(Problem {
kind,
blob_id,
holder: holder.name.clone(),
account_id: holder.account_id,
object_id: holder.object_id,
detail,
});
}
}
/// Saves progress and the problems found since, now and then; a stop
/// asked for meanwhile ends the run.
async fn save(&mut self, force: bool) -> Result<(), Halt> {
if !force && self.saved.elapsed() < SAVE_EVERY {
return Ok(());
}
let data = self.server.store();
let first = self.kept - self.found.len() as u64;
model::add_problems(data, self.id, first, &self.found).await?;
self.found.clear();
let progress = self.progress.clone();
let kept = self.kept;
let saved = model::update(data, self.id, |c| {
if let Some(copy) = &mut c.copy {
copy.progress = progress.clone();
copy.problems_kept = kept;
copy.heartbeat = now();
}
true
})
.await?;
self.saved = Instant::now();
match saved {
Some(c) if c.copy.as_ref().is_some_and(|c| c.stop_requested) => Err(Halt::Stopped),
Some(_) => Ok(()),
// Dismissed under it: nothing left to copy for
None => Err(Halt::Stopped),
}
}
/// BM-5: one blob, copied unless the current store has it.
async fn one(&mut self, key: &[u8], holder: Holder) -> Result<(), Halt> {
self.blobs.take(1).await;
self.progress.blobs_checked += 1;
match self.target.stored_present(key).await {
Ok(true) => {
self.progress.already_there += 1;
return self.save(false).await;
}
Ok(false) => {}
Err(err) => {
self.problem(
ProblemKind::Unreadable,
key,
&holder,
format!("the current blob store couldn't say: {err}"),
);
return self.save(false).await;
}
}
let stored = match self.old.get_stored(key).await {
Ok(Some(stored)) => stored,
Ok(None) => {
// The spam files exist only once the classifier has trained
if holder.name != "fixed" {
self.problem(ProblemKind::Missing, key, &holder, String::new());
}
return self.save(false).await;
}
Err(err) => {
self.problem(
ProblemKind::Unreadable,
key,
&holder,
format!("the old blob store failed: {err}"),
);
return self.save(false).await;
}
};
let size = stored.len() as u64;
self.bytes.take(size).await;
if holder.content_named
&& let Ok(hash) = BlobHash::try_from_hash_slice(key)
{
match BlobStore::content_of_stored(key, stored.clone()) {
Ok(content) => {
let actual = BlobHash::generate(&content);
if actual != hash {
self.problem(
ProblemKind::Damaged,
key,
&holder,
format!(
"the old store's {} bytes hash to {}",
content.len(),
actual.to_hex()
),
);
return self.save(false).await;
}
}
Err(_) => {
self.problem(
ProblemKind::Damaged,
key,
&holder,
"it doesn't decompress".into(),
);
return self.save(false).await;
}
}
}
match self.target.put_stored(key, &stored).await {
Ok(()) => {
self.progress.copied += 1;
self.progress.bytes_copied += size;
}
Err(err) => self.problem(
ProblemKind::Unreadable,
key,
&holder,
format!("the current blob store couldn't write it: {err}"),
),
}
self.save(false).await
}
async fn go(&mut self) -> Result<(), Halt> {
for slice in 0u8..=255 {
let blobs = self.server.store().blobs_to_copy(slice).await?;
// A blob without its commit marker never finished being written
for blob in blobs.into_iter().filter(|b| b.committed) {
let holder = Holder::of(&blob);
self.one(blob.hash.as_slice(), holder).await?;
}
self.progress.slices = slice as u32 + 1;
self.save(false).await?;
}
// BM-5: the files kept under fixed names
for key in [SPAM_TRAINER_KEY, SPAM_CLASSIFIER_KEY] {
self.one(key, Holder::fixed()).await?;
}
Ok(())
}
}
/// The change's old store: the one already read through, or opened now.
async fn old_store(server: &Server, change: &BlobStoreChange) -> Result<BlobStore, String> {
if let Some(old) = server.blob_store().moving().and_then(|m| {
m.old_stores()
.iter()
.find(|o| o.change_id == change.id)
.map(|o| o.store.clone())
}) {
return Ok(old);
}
let Some(settings) = change.old_settings.clone() else {
return Err("The change keeps no settings for the old blob store".into());
};
let mut warnings = Vec::new();
BlobStore::open(settings, server.store(), &mut warnings).await
}
/// Runs a copy to its end and records how it ended (BM-6).
async fn run(server: &Server, change: BlobStoreChange) {
let id = change.id;
let (status, error, run) = match old_store(server, &change).await {
Err(err) => (
Status::Failed,
Some(format!("The old blob store couldn't be opened: {err}")),
None,
),
Ok(old) => {
let copy = change.copy.clone().unwrap_or_else(|| Copy {
started_at: now(),
started_by: String::new(),
node_id: node_id(server),
blobs_per_second: DEFAULT_BLOBS_PER_SECOND,
bytes_per_second: DEFAULT_BYTES_PER_SECOND,
heartbeat: now(),
finished_at: None,
stop_requested: false,
progress: Progress::default(),
problems_kept: 0,
error: None,
});
let mut run = Run {
server,
id,
old,
target: server.blob_store().current().clone(),
progress: Progress::default(),
found: Vec::new(),
kept: 0,
blobs: Pace::new(copy.blobs_per_second),
bytes: Pace::new(copy.bytes_per_second),
saved: Instant::now(),
};
let (status, error) = match run.go().await {
Ok(()) if run.progress.problems() == 0 => (Status::Copied, None),
Ok(()) => (Status::Incomplete, None),
Err(Halt::Stopped) => (Status::Stopped, None),
Err(Halt::Failed(err)) => {
let text = err
.value_as_str(trc::Key::Details)
.map(str::to_string)
.unwrap_or_else(|| err.to_string());
trc::error!(err.details("A blob copy failed"));
(Status::Failed, Some(text))
}
};
(status, error, Some(run))
}
};
let data = server.store();
let mut result = Ok(());
let (progress, kept) = match &run {
Some(run) => {
let first = run.kept - run.found.len() as u64;
result = model::add_problems(data, id, first, &run.found).await;
(run.progress.clone(), run.kept)
}
None => (Progress::default(), 0),
};
if result.is_ok() {
result = model::update(data, id, |c| {
c.status = status;
// BM-6: resolved, so the old store's settings go
if status == Status::Copied {
c.old_settings = None;
}
if let Some(copy) = &mut c.copy {
copy.progress = progress.clone();
copy.problems_kept = kept;
copy.finished_at = Some(now());
copy.heartbeat = now();
}
if let Some(error) = &error {
c.set_error(error);
}
true
})
.await
.map(|_| ());
}
if let Err(err) = result.and(model::release(data, id).await) {
trc::error!(err.details("Failed to save a blob copy's result"));
}
if status == Status::Copied {
refresh(server).await;
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn keys_are_named_by_hash_or_as_is() {
let hash = BlobHash::generate(b"hello");
assert_eq!(key_name(hash.as_slice()), hash.to_hex());
assert_eq!(key_name(SPAM_TRAINER_KEY), "INBUXA_SPAM_TRAIN_DATA.lz4");
}
#[test]
fn only_document_queue_and_kept_blobs_are_checked() {
let blob = |holders| LinkedBlob {
hash: BlobHash::generate(b"x"),
committed: true,
holders,
temporary: true,
};
assert!(!Holder::of(&blob(vec![])).content_named);
assert!(Holder::of(&blob(vec![BlobHolder::Queue { id: 4 }])).content_named);
let mail = Holder::of(&blob(vec![BlobHolder::Document {
account_id: 1,
collection: Collection::Email.into(),
document_id: 9,
}]));
assert_eq!(mail.name, "email");
assert_eq!((mail.account_id, mail.object_id), (Some(1), Some(9)));
assert!(!Holder::fixed().content_named);
}
}
+556
View File
@@ -0,0 +1,556 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! Checking the store's blobs (docs/spec/features/store-check.md): the run
//! behind an `inbuxa:StoreCheck`. It goes through the blob index one slice
//! at a time and asks the blob store for each blob something holds, then
//! through each account's mail metadata for the blob each message names
//! (SV-3). It reports and never repairs. It runs on the node the check was
//! started on, against the stores every node shares.
use common::{BuildServer, Inner, Server};
use email::message::metadata::MessageMetadata;
use inbuxa_features::store_check::{
self as model, MAX_PROBLEMS, Mode, Problem, ProblemKind, Progress, Refusal, STALE_AFTER,
Status, StoreCheck,
};
use registry::schema::prelude::ObjectType;
use std::{
collections::{BTreeSet, HashMap},
sync::Arc,
time::{Duration, Instant},
};
use store::{
Deserialize, IterateParams, U32_LEN, ValueKey,
registry::RegistryQuery,
roaring::RoaringBitmap,
write::{
AlignedBytes, Archive,
blob_check::{BlobHolder, LinkedBlob},
key::DeserializeBigEndian,
now,
},
};
use trc::AddContext;
use types::{blob::BlobClass, blob_hash::BlobHash, collection::Collection, field::EmailField};
/// SV-7: how often a running check saves its progress, which is also its
/// heartbeat and when it sees a stop.
const SAVE_EVERY: Duration = Duration::from_secs(10);
fn node_id(server: &Server) -> u64 {
server.core.network.node_id
}
/// SV-2: starts a check on this node. Refused while another runs.
pub async fn start(
server: &Server,
mode: Mode,
blobs_per_second: Option<u64>,
bytes_per_second: Option<u64>,
created_by: String,
) -> trc::Result<Result<u32, Refusal>> {
let now = now();
let mut check = StoreCheck::new(mode, created_by, node_id(server), now);
if let Some(limit) = blobs_per_second {
check.blobs_per_second = limit;
}
if let Some(limit) = bytes_per_second {
check.bytes_per_second = limit;
}
let id = match model::create(server.store(), &check, now).await? {
Ok(id) => id,
Err(refusal) => return Ok(Err(refusal)),
};
let check = StoreCheck { id, ..check };
let server = server.clone();
tokio::spawn(async move {
run(&server, check).await;
});
Ok(Ok(id))
}
/// SV-8: asks a running check to stop. Any node can: the one running it
/// sees the request at its next save. None if there's no such check.
pub async fn stop(server: &Server, id: u32) -> trc::Result<Option<StoreCheck>> {
model::update(server.store(), id, |c| {
if c.status != Status::Running || c.stop_requested {
return false;
}
c.stop_requested = true;
true
})
.await
}
/// SV-8: at start-up, a check this node ran stopped with it. Then, now and
/// then, a check on a node that went quiet is taken to have stopped too.
pub fn spawn_store_checks(inner: Arc<Inner>) {
tokio::spawn(async move {
let server = inner.build_server();
let node = node_id(&server);
mark_interrupted(&server, |c| c.node_id == node).await;
loop {
tokio::time::sleep(Duration::from_secs(60)).await;
let server = inner.build_server();
let cutoff = now().saturating_sub(STALE_AFTER);
mark_interrupted(&server, |c| c.node_id != node && c.heartbeat < cutoff).await;
}
});
}
async fn mark_interrupted(server: &Server, which: impl Fn(&StoreCheck) -> bool) {
let checks = match model::all(server.store()).await {
Ok(checks) => checks,
Err(err) => {
trc::error!(err.details("Failed to read the store checks"));
return;
}
};
for check in checks
.into_iter()
.filter(|c| c.status == Status::Running && which(c))
{
let result = model::update(server.store(), check.id, |c| {
if c.status != Status::Running || !which(c) {
return false;
}
c.status = Status::Interrupted;
c.finished_at = Some(now());
true
})
.await;
if let Err(err) = result.and(model::release(server.store(), check.id).await) {
trc::error!(err.details("Failed to mark an interrupted store check"));
}
}
}
/// SV-6: keeps a rate, a second at a time, so a slow stretch isn't made
/// up for with a burst.
struct Pace {
per_second: u64,
window: Instant,
used: u64,
}
impl Pace {
fn new(per_second: u64) -> Self {
Pace {
per_second,
window: Instant::now(),
used: 0,
}
}
async fn take(&mut self, amount: u64) {
if self.per_second == 0 {
return;
}
if self.window.elapsed() >= Duration::from_secs(1) {
self.window = Instant::now();
self.used = 0;
}
self.used += amount;
if self.used >= self.per_second {
let rest = Duration::from_secs(1).saturating_sub(self.window.elapsed());
tokio::time::sleep(rest).await;
self.window = Instant::now();
self.used = self.used.saturating_sub(self.per_second);
}
}
}
/// What a blob turned out to be (SV-4).
enum Found {
Fine,
Problem(ProblemKind, String),
}
/// The name a problem gives an object (SV-4).
fn object_name(collection: u8) -> String {
match Collection::from(collection) {
Collection::Email => "email".into(),
Collection::FileNode => "file".into(),
Collection::SieveScript => "sieveScript".into(),
Collection::CalendarEvent => "calendarEvent".into(),
Collection::ContactCard => "contactCard".into(),
_ => format!("collection{collection}"),
}
}
struct Run<'x> {
server: &'x Server,
check: StoreCheck,
progress: Progress,
found: Vec<Problem>,
kept: u64,
names: HashMap<u32, String>,
blobs: Pace,
bytes: Pace,
saved: Instant,
}
/// How a run ended before it finished.
enum Halt {
Stopped,
Failed(trc::Error),
}
impl From<trc::Error> for Halt {
fn from(err: trc::Error) -> Self {
Halt::Failed(err)
}
}
impl<'x> Run<'x> {
async fn account_name(&mut self, account_id: u32) -> String {
if let Some(name) = self.names.get(&account_id) {
return name.clone();
}
let name = self.server.audit_account_name(account_id).await;
self.names.insert(account_id, name.clone());
name
}
/// SV-4, SV-5: counts a problem, logs it, and keeps the first ones.
async fn problem(
&mut self,
kind: ProblemKind,
account_id: Option<u32>,
object: String,
object_id: u64,
hash: &BlobHash,
committed: bool,
detail: String,
) {
self.progress.count(kind);
let account_name = match account_id {
Some(id) => self.account_name(id).await,
None => String::new(),
};
let blob_id = hash.to_hex();
trc::event!(
Store(trc::StoreEvent::DataCorruption),
Id = self.check.id,
AccountId = account_id,
Collection = object.clone(),
DocumentId = object_id,
BlobId = blob_id.clone(),
Reason = kind.as_str(),
Details = detail.clone(),
);
if self.kept < MAX_PROBLEMS {
self.kept += 1;
self.found.push(Problem {
kind,
account_id,
account_name,
object,
object_id,
blob_id,
committed,
detail,
});
}
}
/// SV-7, SV-8: saves progress and the problems found since, now and
/// then; a stop asked for meanwhile ends the run.
async fn save(&mut self, force: bool) -> Result<(), Halt> {
if !force && self.saved.elapsed() < SAVE_EVERY {
return Ok(());
}
let data = self.server.store();
let first = self.kept - self.found.len() as u64;
model::add_problems(data, self.check.id, first, &self.found).await?;
self.found.clear();
let progress = self.progress.clone();
let kept = self.kept;
let saved = model::update(data, self.check.id, |c| {
c.progress = progress.clone();
c.problems_kept = kept;
c.heartbeat = now();
true
})
.await?;
self.saved = Instant::now();
match saved {
Some(c) if c.stop_requested => Err(Halt::Stopped),
Some(_) => Ok(()),
// Forgotten under it: nothing left to report to
None => Err(Halt::Stopped),
}
}
/// SV-2: whether the blob is there and, in a full check, intact.
async fn look_at(&mut self, hash: &BlobHash) -> Found {
let store = self.server.blob_store();
match self.check.mode {
Mode::Quick => match store.blob_present(hash.as_slice()).await {
Ok(true) => Found::Fine,
Ok(false) => Found::Problem(ProblemKind::Missing, String::new()),
Err(err) => Found::Problem(ProblemKind::Unreadable, err.to_string()),
},
Mode::Full => match store.get_blob(hash.as_slice(), 0..usize::MAX).await {
Ok(Some(content)) => {
let size = content.len() as u64;
self.progress.bytes_read += size;
self.bytes.take(size).await;
let actual = BlobHash::generate(&content);
if &actual == hash {
Found::Fine
} else {
Found::Problem(
ProblemKind::Damaged,
format!("its {size} bytes hash to {}", actual.to_hex()),
)
}
}
Ok(None) => Found::Problem(ProblemKind::Missing, String::new()),
Err(err)
if matches!(
err.as_ref(),
trc::EventType::Store(trc::StoreEvent::DecompressError)
) =>
{
Found::Problem(ProblemKind::Damaged, "it doesn't decompress".into())
}
Err(err) => Found::Problem(ProblemKind::Unreadable, err.to_string()),
},
}
}
/// SV-3, links: one slice of the blob index.
async fn slice(&mut self, slice: u8, mail: &mut BTreeSet<u32>) -> Result<(), Halt> {
let blobs: Vec<LinkedBlob> = self.server.store().linked_blobs(slice).await?;
for blob in blobs {
self.blobs.take(1).await;
self.progress.blobs_checked += 1;
self.progress.links_checked += blob.holders.len() as u64;
for holder in &blob.holders {
if let BlobHolder::Document {
account_id,
collection,
..
} = holder
&& Collection::from(*collection) == Collection::Email
{
mail.insert(*account_id);
}
}
if let Found::Problem(kind, detail) = self.look_at(&blob.hash).await {
// A shared blob is read once and reported for each holder
for holder in &blob.holders {
let (account_id, object, object_id) = match holder {
BlobHolder::Document {
account_id,
collection,
document_id,
} => (
Some(*account_id),
object_name(*collection),
*document_id as u64,
),
BlobHolder::Queue { id } => (None, "queuedMessage".to_string(), *id),
BlobHolder::Archived { account_id, item } => (
Some(*account_id),
"archivedItem".to_string(),
item.id().id(),
),
};
self.problem(
kind,
account_id,
object,
object_id,
&blob.hash,
blob.committed,
detail.clone(),
)
.await;
}
}
self.save(false).await?;
}
Ok(())
}
/// SV-3, mail metadata: each message names its blob; it must hold a
/// link to it, or nothing keeps the blob from being purged.
async fn mail(&mut self, account_id: u32) -> Result<(), Halt> {
let data = self.server.store();
let mut messages: Vec<(u32, BlobHash)> = Vec::new();
data.iterate(
IterateParams::new(
ValueKey::property(account_id, Collection::Email, 0, EmailField::Metadata),
ValueKey::property(
account_id,
Collection::Email,
u32::MAX,
EmailField::Metadata,
),
),
|key, value| {
let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?;
let archive = <Archive<AlignedBytes> as Deserialize>::deserialize(value)?;
let metadata = archive.unarchive::<MessageMetadata>()?;
messages.push((document_id, BlobHash::from(&metadata.blob_hash)));
Ok(true)
},
)
.await
.caused_by(trc::location!())?;
for (document_id, hash) in messages {
self.blobs.take(1).await;
self.progress.messages_checked += 1;
let linked = data
.blob_has_access(
&hash,
BlobClass::Linked {
account_id,
collection: Collection::Email.into(),
document_id,
},
)
.await?;
if !linked {
let committed = data.blob_exists(&hash).await?;
let detail = match self.server.blob_store().blob_present(hash.as_slice()).await {
Ok(true) => "the blob is still there".to_string(),
Ok(false) => "the blob is gone".to_string(),
Err(err) => format!("the blob store couldn't say: {err}"),
};
self.problem(
ProblemKind::Unlinked,
Some(account_id),
"email".into(),
document_id as u64,
&hash,
committed,
detail,
)
.await;
}
self.save(false).await?;
}
Ok(())
}
async fn go(&mut self) -> Result<(), Halt> {
let mut mail = BTreeSet::new();
for slice in 0u8..=255 {
self.slice(slice, &mut mail).await?;
self.progress.slices = slice as u32 + 1;
self.save(false).await?;
}
// Every account, and any whose mail is still linked without it
// being in the registry any more (kept after deletion)
let registry: RoaringBitmap = self
.server
.registry()
.query::<RoaringBitmap>(RegistryQuery::new(ObjectType::Account))
.await?;
mail.extend(registry.iter());
self.progress.accounts_total = mail.len() as u64;
for account_id in mail {
self.mail(account_id).await?;
self.progress.accounts_done += 1;
self.save(false).await?;
}
Ok(())
}
}
/// Runs a check to its end and records how it ended (SV-8).
async fn run(server: &Server, check: StoreCheck) {
let id = check.id;
let mut run = Run {
server,
blobs: Pace::new(check.blobs_per_second),
bytes: Pace::new(if check.mode == Mode::Full {
check.bytes_per_second
} else {
0
}),
check,
progress: Progress::default(),
found: Vec::new(),
kept: 0,
names: HashMap::new(),
saved: Instant::now(),
};
let ended = run.go().await;
let (status, error) = match ended {
Ok(()) => (Status::Done, None),
Err(Halt::Stopped) => (Status::Stopped, None),
Err(Halt::Failed(err)) => {
let text = err
.value_as_str(trc::Key::Details)
.map(str::to_string)
.unwrap_or_else(|| err.to_string());
trc::error!(err.details("A store check failed"));
(Status::Failed, Some(text))
}
};
let data = server.store();
let first = run.kept - run.found.len() as u64;
let mut result = model::add_problems(data, id, first, &run.found).await;
if result.is_ok() {
let progress = run.progress.clone();
let kept = run.kept;
result = model::update(data, id, |c| {
c.progress = progress.clone();
c.problems_kept = kept;
c.status = status;
c.finished_at = Some(now());
c.heartbeat = now();
if let Some(error) = &error {
c.set_error(error);
}
true
})
.await
.map(|_| ());
}
if let Err(err) = result.and(model::release(data, id).await) {
trc::error!(err.details("Failed to save a store check's result"));
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn pace_holds_a_rate() {
let mut pace = Pace::new(100);
let started = Instant::now();
for _ in 0..250 {
pace.take(1).await;
}
// 250 at 100 a second: two full seconds waited, a bit of a third
assert!(started.elapsed() >= Duration::from_millis(1_900));
let mut free = Pace::new(0);
let started = Instant::now();
for _ in 0..10_000 {
free.take(1).await;
}
assert!(started.elapsed() < Duration::from_millis(500));
}
#[test]
fn objects_are_named_by_collection() {
assert_eq!(object_name(Collection::Email.into()), "email");
assert_eq!(object_name(Collection::FileNode.into()), "file");
assert_eq!(object_name(Collection::SieveScript.into()), "sieveScript");
assert_eq!(object_name(Collection::Mailbox.into()), "collection1");
}
}
@@ -0,0 +1,502 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! Snapshots of the store (docs/spec/features/store-snapshot.md): the run
//! behind an `inbuxa:StoreSnapshot`. It takes a RocksDB checkpoint and, for
//! a filesystem blob store, a copy of the blob files with the blob purge
//! held, into a directory under the store's own (SN-2, SN-3). The snapshots
//! on disk are the record: each one's `snapshot.json` says what it is. Only
//! the one being taken, or the last one that failed, is kept in memory.
use common::{Inner, Server};
use mail_parser::DateTime;
use serde_json::{Value, json};
use std::{
path::{Path, PathBuf},
str::FromStr,
sync::{Arc, Mutex},
};
use store::write::{
now,
snapshot::{NotSnapshottable, SnapshotSource, copy_tree, hold_blob_purge, tree_size},
};
use types::id::Id;
/// SN-6: complete snapshots kept.
pub const KEEP: usize = 2;
const SNAPSHOTS_DIR: &str = "snapshots";
const MANIFEST: &str = "snapshot.json";
const LATEST: &str = "latest";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Status {
Running,
Done,
Failed,
}
impl Status {
pub fn as_str(&self) -> &'static str {
match self {
Status::Running => "running",
Status::Done => "done",
Status::Failed => "failed",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Snapshot {
/// When it was taken, in seconds: its name is that time.
pub id: u64,
pub name: String,
pub status: Status,
pub created_at: u64,
pub created_by: String,
pub finished_at: Option<u64>,
pub node_id: u64,
pub path: PathBuf,
pub holds: Vec<String>,
pub size: u64,
pub blob_files: u64,
pub error: Option<String>,
}
/// Why a snapshot isn't taken.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Refusal {
/// SN-6: this one is being taken.
Running(u64),
/// SN-1, SN-2: these stores can't have one.
Unsupported(String),
}
/// What `destroy` did.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Destroyed {
Gone,
NotFound,
Running,
}
/// SN-7: the one being taken, or the last one that failed, until restart.
static CURRENT: Mutex<Option<Snapshot>> = Mutex::new(None);
fn current() -> Option<Snapshot> {
CURRENT.lock().map(|c| c.clone()).unwrap_or(None)
}
fn set_current(snapshot: Option<Snapshot>) {
if let Ok(mut current) = CURRENT.lock() {
*current = snapshot;
}
}
/// The snapshots' directory, and what a snapshot would hold, or why there
/// can't be one here.
fn source(server: &Server) -> Result<(SnapshotSource, PathBuf), Refusal> {
match server.store().snapshot_source(server.blob_store()) {
Ok(source) => {
let root = match &source {
SnapshotSource::DataOnly { data_path }
| SnapshotSource::DataAndFiles { data_path, .. } => data_path.join(SNAPSHOTS_DIR),
};
Ok((source, root))
}
Err(NotSnapshottable::DataStore(kind)) => Err(Refusal::Unsupported(format!(
"A snapshot needs RocksDB as the data store, and this server's is {kind}. \
Use {kind}'s own backup."
))),
Err(NotSnapshottable::BlobStore(kind)) => Err(Refusal::Unsupported(format!(
"A snapshot copies blobs kept in the data store or in a filesystem blob store, \
and this server keeps them in {kind}."
))),
// blob-store-change spec, BM-8
Err(NotSnapshottable::BlobStoreChanging) => Err(Refusal::Unsupported(
"The blob store changed and some blobs may still be only in the old one. \
Copy them over or dismiss the change (inbuxa:BlobStoreChange) first."
.to_string(),
)),
}
}
/// `20261109T013000Z`: a snapshot's name, the UTC time it was taken.
pub fn name_for(seconds: u64) -> String {
let t = DateTime::from_timestamp(seconds as i64);
format!(
"{:04}{:02}{:02}T{:02}{:02}{:02}Z",
t.year, t.month, t.day, t.hour, t.minute, t.second
)
}
fn rfc3339(seconds: u64) -> String {
DateTime::from_timestamp(seconds as i64).to_rfc3339()
}
fn seconds_of(value: &Value) -> Option<u64> {
DateTime::parse_rfc3339(value.as_str()?).map(|t| t.to_timestamp() as u64)
}
fn io(err: std::io::Error) -> trc::Error {
trc::StoreEvent::FilesystemError.reason(err)
}
/// SN-1, SN-6: starts taking a snapshot on this node. Refused while one is
/// being taken, or when the stores can't have one.
pub async fn start(server: &Server, created_by: String) -> trc::Result<Result<Snapshot, Refusal>> {
let (source, root) = match source(server) {
Ok(found) => found,
Err(refusal) => return Ok(Err(refusal)),
};
let snapshot = {
let Ok(mut current) = CURRENT.lock() else {
return Err(trc::EventType::Server(trc::ServerEvent::ThreadError)
.into_err()
.details("The snapshot state is poisoned"));
};
if let Some(running) = current.as_ref().filter(|c| c.status == Status::Running) {
return Ok(Err(Refusal::Running(running.id)));
}
// Names are the time to the second: a second one within the same
// second takes the next
let mut at = now();
while root.join(name_for(at)).exists() {
at += 1;
}
let name = name_for(at);
let mut holds = vec!["data".to_string()];
if matches!(source, SnapshotSource::DataAndFiles { .. }) {
holds.push("blobs".to_string());
}
let snapshot = Snapshot {
id: at,
path: root.join(&name),
name,
status: Status::Running,
created_at: at,
created_by,
finished_at: None,
node_id: server.core.network.node_id,
holds,
size: 0,
blob_files: 0,
error: None,
};
*current = Some(snapshot.clone());
snapshot
};
let server = server.clone();
let running = snapshot.clone();
tokio::spawn(async move {
run(&server, source, root, running).await;
});
Ok(Ok(snapshot))
}
async fn run(server: &Server, source: SnapshotSource, root: PathBuf, mut snapshot: Snapshot) {
let partial = root.join(format!(".{}.partial", snapshot.name));
match take(server, &source, &root, &partial, &mut snapshot).await {
// Listed from disk from now on
Ok(()) => set_current(None),
Err(err) => {
let _ = tokio::fs::remove_dir_all(&partial).await;
snapshot.status = Status::Failed;
snapshot.finished_at = Some(now());
snapshot.error = Some(err.to_string());
trc::error!(err.details("Failed to take a store snapshot"));
set_current(Some(snapshot));
}
}
}
async fn take(
server: &Server,
source: &SnapshotSource,
root: &Path,
partial: &Path,
snapshot: &mut Snapshot,
) -> trc::Result<()> {
tokio::fs::create_dir_all(root).await.map_err(io)?;
let _ = tokio::fs::remove_dir_all(partial).await;
tokio::fs::create_dir(partial).await.map_err(io)?;
// SN-4: the purge is held before the checkpoint, so every blob the
// checkpoint links to is still there when it's copied
let hold = match source {
SnapshotSource::DataAndFiles { .. } => Some(hold_blob_purge().await),
SnapshotSource::DataOnly { .. } => None,
};
server.store().checkpoint(partial.join("data")).await?;
let blob_path = match source {
SnapshotSource::DataAndFiles { blob_path, .. } => {
let copy = copy_tree(
blob_path.clone(),
partial.join("blobs"),
data_path_of(source).to_path_buf(),
)
.await?;
snapshot.blob_files = copy.files;
Some(blob_path)
}
SnapshotSource::DataOnly { .. } => None,
};
drop(hold);
snapshot.size = tree_size(partial.to_path_buf()).await?;
snapshot.status = Status::Done;
snapshot.finished_at = Some(now());
let manifest = manifest(
snapshot,
data_path_of(source),
blob_path.map(|p| p.as_path()),
);
tokio::fs::write(
partial.join(MANIFEST),
serde_json::to_vec_pretty(&manifest).unwrap_or_default(),
)
.await
.map_err(io)?;
// SN-3: whole once it has its name
tokio::fs::rename(partial, &snapshot.path)
.await
.map_err(io)?;
if let Ok(dir) = tokio::fs::File::open(root).await {
let _ = dir.sync_all().await;
}
prune(root).await;
refresh_latest(root).await;
Ok(())
}
fn data_path_of(source: &SnapshotSource) -> &Path {
match source {
SnapshotSource::DataOnly { data_path } | SnapshotSource::DataAndFiles { data_path, .. } => {
data_path
}
}
}
/// SN-5.
fn manifest(snapshot: &Snapshot, data_path: &Path, blob_path: Option<&Path>) -> Value {
let mut restore = vec![
"Stop the server.".to_string(),
format!(
"Move the contents of {} aside, except snapshots/, and copy this snapshot's data/ into it.",
data_path.display()
),
];
if let Some(blob_path) = blob_path {
restore.push(format!(
"Move the contents of {} aside and copy this snapshot's blobs/ into it.",
blob_path.display()
));
}
restore.push("Start the server.".to_string());
json!({
"id": Id::from(snapshot.id).to_string(),
"name": snapshot.name,
"createdAt": rfc3339(snapshot.created_at),
"finishedAt": snapshot.finished_at.map(rfc3339),
"createdBy": snapshot.created_by,
"node": snapshot.node_id.to_string(),
"server": concat!(types::brand_server!(), " ", types::brand_version!()),
"holds": snapshot.holds,
"dataStore": {"type": "RocksDB", "path": data_path.to_string_lossy()},
"blobStore": match blob_path {
Some(path) => json!({"type": "filesystem", "path": path.to_string_lossy()}),
None => json!({"type": "in the data store"}),
},
"size": snapshot.size,
"blobFiles": snapshot.blob_files,
"restore": restore,
})
}
fn from_manifest(path: PathBuf, manifest: &Value) -> Option<Snapshot> {
let id = Id::from_str(manifest.get("id")?.as_str()?).ok()?.id();
Some(Snapshot {
id,
name: manifest.get("name")?.as_str()?.to_string(),
status: Status::Done,
created_at: seconds_of(manifest.get("createdAt")?)?,
created_by: manifest.get("createdBy")?.as_str()?.to_string(),
finished_at: manifest.get("finishedAt").and_then(seconds_of),
node_id: manifest.get("node")?.as_str()?.parse().ok()?,
path,
holds: manifest
.get("holds")?
.as_array()?
.iter()
.filter_map(|h| h.as_str().map(|s| s.to_string()))
.collect(),
size: manifest.get("size")?.as_u64()?,
blob_files: manifest.get("blobFiles")?.as_u64()?,
error: None,
})
}
/// The complete snapshots under `root`, oldest first. A directory without a
/// readable `snapshot.json` isn't one, and is left alone.
async fn on_disk(root: &Path) -> Vec<Snapshot> {
let mut found = Vec::new();
let Ok(mut entries) = tokio::fs::read_dir(root).await else {
return found;
};
while let Ok(Some(entry)) = entries.next_entry().await {
let name = entry.file_name().to_string_lossy().to_string();
if name.starts_with('.') || !entry.file_type().await.is_ok_and(|t| t.is_dir()) {
continue;
}
let path = entry.path();
let Ok(bytes) = tokio::fs::read(path.join(MANIFEST)).await else {
continue;
};
if let Some(snapshot) = serde_json::from_slice::<Value>(&bytes)
.ok()
.and_then(|manifest| from_manifest(path, &manifest))
.filter(|s| s.name == name)
{
found.push(snapshot);
}
}
found.sort_by_key(|s| s.id);
found
}
/// SN-6: the oldest beyond the latest [`KEEP`] go.
async fn prune(root: &Path) {
let snapshots = on_disk(root).await;
let surplus = snapshots.len().saturating_sub(KEEP);
for old in &snapshots[..surplus] {
if let Err(err) = tokio::fs::remove_dir_all(&old.path).await {
trc::error!(io(err).details("Failed to remove an old store snapshot"));
}
}
}
/// SN-3: `latest` names the newest complete snapshot, relative to the
/// snapshots' directory so it holds wherever that's mounted.
async fn refresh_latest(root: &Path) {
let latest = root.join(LATEST);
match on_disk(root).await.last() {
Some(newest) => {
let temp = root.join(format!(".{LATEST}.tmp"));
let _ = tokio::fs::remove_file(&temp).await;
let linked = async {
tokio::fs::symlink(&newest.name, &temp).await?;
tokio::fs::rename(&temp, &latest).await
}
.await;
if let Err(err) = linked {
trc::error!(io(err).details("Failed to point snapshots/latest at a snapshot"));
}
}
None => {
let _ = tokio::fs::remove_file(&latest).await;
}
}
}
/// Every snapshot, newest first: those on disk, and the one being taken or
/// the last one that failed (SN-7).
pub async fn list(server: &Server) -> Vec<Snapshot> {
let mut snapshots = match source(server) {
Ok((_, root)) => on_disk(&root).await,
Err(_) => Vec::new(),
};
if let Some(current) = current().filter(|c| !snapshots.iter().any(|s| s.id == c.id)) {
snapshots.push(current);
}
snapshots.sort_by_key(|s| std::cmp::Reverse(s.id));
snapshots
}
/// One snapshot, as [`list`] has it.
pub async fn find(server: &Server, id: u64) -> Option<Snapshot> {
list(server).await.into_iter().find(|s| s.id == id)
}
/// SN-9: deletes a snapshot's directory. One being taken can't be.
pub async fn destroy(server: &Server, id: u64) -> trc::Result<Destroyed> {
if let Some(current) = current().filter(|c| c.id == id) {
if current.status == Status::Running {
return Ok(Destroyed::Running);
}
set_current(None);
return Ok(Destroyed::Gone);
}
let Ok((_, root)) = source(server) else {
return Ok(Destroyed::NotFound);
};
let Some(snapshot) = on_disk(&root).await.into_iter().find(|s| s.id == id) else {
return Ok(Destroyed::NotFound);
};
tokio::fs::remove_dir_all(&snapshot.path)
.await
.map_err(io)?;
refresh_latest(&root).await;
Ok(Destroyed::Gone)
}
/// SN-7: a snapshot being taken when the server stopped left a partial
/// directory, removed when it starts.
pub fn spawn_store_snapshots(inner: Arc<Inner>) {
tokio::spawn(async move {
let server = common::BuildServer::build_server(&inner);
let Ok((_, root)) = source(&server) else {
return;
};
let Ok(mut entries) = tokio::fs::read_dir(&root).await else {
return;
};
while let Ok(Some(entry)) = entries.next_entry().await {
let name = entry.file_name().to_string_lossy().to_string();
if name.starts_with('.') && name.ends_with(".partial") {
let _ = tokio::fs::remove_dir_all(entry.path()).await;
}
}
});
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn names_are_the_time() {
assert_eq!(name_for(1_794_187_800), "20261109T013000Z");
assert_eq!(name_for(0), "19700101T000000Z");
}
#[test]
fn a_manifest_reads_back() {
let snapshot = Snapshot {
id: 1_794_187_800,
name: name_for(1_794_187_800),
status: Status::Done,
created_at: 1_794_187_800,
created_by: "[email protected]".into(),
finished_at: Some(1_794_187_802),
node_id: 3,
path: PathBuf::from("/var/lib/inbuxa/snapshots/20261109T013000Z"),
holds: vec!["data".into(), "blobs".into()],
size: 4096,
blob_files: 12,
error: None,
};
let manifest = manifest(
&snapshot,
Path::new("/var/lib/inbuxa/"),
Some(Path::new("/var/lib/inbuxa-blobs")),
);
assert_eq!(manifest["restore"].as_array().unwrap().len(), 4);
assert_eq!(
from_manifest(snapshot.path.clone(), &manifest).unwrap(),
snapshot
);
}
}
+13
View File
@@ -23,10 +23,13 @@ use crate::task_manager::{manager::spawn_task_manager, scheduler::spawn_task_sch
pub mod broadcast;
// inbuxa: AL-5, delegations end at their date
pub mod inbuxa_blob_move; // inbuxa: changing the blob store
pub mod inbuxa_deliverability; // inbuxa: the deliverability check
pub mod inbuxa_lock_expiry;
pub mod inbuxa_log_retention; // inbuxa: personal-data catalog, D1
pub mod inbuxa_scheduled_reports; // inbuxa: scheduled reports and the weekly digest
pub mod inbuxa_store_check; // inbuxa: checking the store's blobs
pub mod inbuxa_store_snapshot; // inbuxa: a snapshot of the store
pub mod state_manager;
pub mod task_manager;
@@ -78,6 +81,16 @@ impl SpawnServices for IpcReceivers {
// inbuxa: deliverability spec, DL-14: each node checks itself daily
inbuxa_deliverability::spawn_deliverability(inner.clone());
// inbuxa: store-check spec, SV-8: a check this node ran stopped with it
inbuxa_store_check::spawn_store_checks(inner.clone());
// inbuxa: blob-store-change spec, BM-6, BM-10: a copy this node
// ran stopped with it; resolved changes stop being read through
inbuxa_blob_move::spawn_blob_moves(inner.clone());
// inbuxa: store-snapshot spec, SN-7: a snapshot cut short by a
// stop leaves a partial directory
inbuxa_store_snapshot::spawn_store_snapshots(inner.clone());
// inbuxa: scheduled-reports spec, RP-16: every node looks for due reports
inbuxa_scheduled_reports::spawn_scheduled_reports(inner.clone());
@@ -310,6 +310,17 @@ async fn store_maintenance(
}
TaskStoreMaintenanceType::PurgeBlob => {
if let Some(shard_index) = task.shard_index {
// inbuxa: store-snapshot spec, SN-4: while a snapshot takes
// the blobs, the purge is put off by five minutes
let Some(_purging) = store::write::snapshot::blob_purge_may_run() else {
return Ok(TaskResult::Success(vec![Task::StoreMaintenance(
TaskStoreMaintenance {
maintenance_type: TaskStoreMaintenanceType::PurgeBlob,
shard_index: Some(shard_index),
status: TaskStatus::at(now() as i64 + 300),
},
)]));
};
server
.store()
.purge_blobs(server.blob_store().clone(), shard_index as u8)
+83 -3
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use crate::BlobStore;
@@ -77,9 +79,22 @@ impl FsStore {
fs::create_dir_all(blob_path.parent().unwrap())
.await
.map_err(into_error)?;
let mut blob_file = File::create(&blob_path).await.map_err(into_error)?;
blob_file.write_all(data).await.map_err(into_error)?;
blob_file.flush().await.map_err(into_error)?;
// inbuxa: store-snapshot spec, SN-10: written beside it and
// renamed into place, never rewritten in place. A snapshot holds
// blobs by hard link, and a rewrite would change its copy too; a
// stop mid-write would leave half a blob under its id.
let temp_path = temp_path_for(&blob_path);
let written = async {
let mut blob_file = File::create(&temp_path).await?;
blob_file.write_all(data).await?;
blob_file.flush().await?;
fs::rename(&temp_path, &blob_path).await
}
.await;
if let Err(err) = written {
let _ = fs::remove_file(&temp_path).await;
return Err(into_error(err));
}
}
Ok(())
@@ -95,6 +110,11 @@ impl FsStore {
}
}
/// inbuxa: the directory the blobs are under (store-snapshot spec).
pub fn root(&self) -> &std::path::Path {
&self.path
}
fn build_path(&self, key: &[u8]) -> PathBuf {
let mut path = self.path.clone();
@@ -106,6 +126,66 @@ impl FsStore {
}
}
/// inbuxa: SN-10. A blob's name is base32, so a name starting with a dot
/// is never a blob: the snapshot's copy and anything listing blobs skip it.
fn temp_path_for(blob_path: &std::path::Path) -> PathBuf {
static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let n = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let name = blob_path
.file_name()
.map(|name| name.to_string_lossy())
.unwrap_or_default();
blob_path.with_file_name(format!(".{name}.{}-{n}.tmp", std::process::id()))
}
fn into_error(err: std::io::Error) -> trc::Error {
trc::StoreEvent::FilesystemError.reason(err)
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn writes_go_through_a_temporary_file() {
let dir = std::env::temp_dir().join(format!("inbuxa-fs-sn10-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
let BlobStore::Fs(store) = FsStore::open(structs::FileSystemStore {
path: dir.to_string_lossy().to_string(),
depth: 2,
})
.await
.unwrap() else {
unreachable!()
};
let key = [7u8; 32];
store.put_blob(&key, b"first").await.unwrap();
let path = store.build_path(&key);
// A file held by a hard link, as a snapshot holds it, keeps what it
// had when the blob under its id is written again
let held = dir.join("held");
std::fs::hard_link(&path, &held).unwrap();
store.put_blob(&key, b"second, longer").await.unwrap();
assert_eq!(std::fs::read(&held).unwrap(), b"first");
assert_eq!(
store.get_blob(&key, 0..usize::MAX).await.unwrap().unwrap(),
b"second, longer"
);
// Nothing is left beside it
let names: Vec<_> = std::fs::read_dir(path.parent().unwrap())
.unwrap()
.map(|entry| entry.unwrap().file_name())
.collect();
assert_eq!(names.len(), 1, "{names:?}");
assert!(
temp_path_for(&path)
.file_name()
.unwrap()
.to_string_lossy()
.starts_with('.')
);
std::fs::remove_dir_all(&dir).unwrap();
}
}
@@ -0,0 +1,31 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! inbuxa: a checkpoint of the store (store-snapshot spec, SN-2): every
//! column family at one point in time, hard-linked where it can be.
use super::{RocksDbStore, into_error};
use rocksdb::checkpoint::Checkpoint;
use std::path::{Path, PathBuf};
impl RocksDbStore {
/// The store's directory.
pub fn path(&self) -> &Path {
self.db.path()
}
/// Writes a checkpoint to `dest`, which must not exist yet.
pub async fn checkpoint(&self, dest: PathBuf) -> trc::Result<()> {
let db = self.db.clone();
tokio::task::spawn_blocking(move || {
Checkpoint::new(&*db)
.and_then(|checkpoint| checkpoint.create_checkpoint(&dest))
.map_err(into_error)
})
.await
.map_err(|err| trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err))?
}
}
+3
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use std::sync::Arc;
@@ -11,6 +13,7 @@ use rocksdb::{BoundColumnFamily, MultiThreaded, OptimisticTransactionDB};
use crate::{SUBSPACE_BLOBS, SUBSPACE_INDEXES, SUBSPACE_LOGS};
pub mod blob;
pub mod checkpoint; // inbuxa: store-snapshot spec
pub mod main;
pub mod read;
pub mod write;
+115 -23
View File
@@ -6,14 +6,38 @@
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use crate::{BlobStore, backend::fs::FsStore, registry::bootstrap::Bootstrap};
use crate::{BlobStore, Store, backend::fs::FsStore, registry::bootstrap::Bootstrap};
use registry::schema::{prelude::ObjectType, structs};
#[allow(unreachable_patterns)]
impl BlobStore {
pub async fn build(bp: &mut Bootstrap) -> Option<Self> {
let result = match bp.setting_infallible::<structs::BlobStore>().await {
structs::BlobStore::Default => return Some(BlobStore::Store(bp.data_store.clone())),
let config = bp.setting_infallible::<structs::BlobStore>().await;
let mut warnings = Vec::new();
let result = BlobStore::open(config, &bp.data_store, &mut warnings).await;
for warning in warnings {
bp.build_warning(ObjectType::BlobStore.singleton(), warning);
}
match result {
Ok(store) => Some(store),
Err(err) => {
bp.build_error(ObjectType::BlobStore.singleton(), err);
None
}
}
}
/// inbuxa: the blob store a setting names. Split out of `build` so a
/// blob store the setting no longer names can be opened too, to read
/// what was left in it (blob-store-change spec, BM-2).
#[allow(unreachable_patterns)]
pub async fn open(
config: structs::BlobStore,
data_store: &Store,
warnings: &mut Vec<String>,
) -> Result<Self, String> {
match config {
structs::BlobStore::Default => Ok(BlobStore::Store(data_store.clone())),
#[cfg(feature = "foundation")]
structs::BlobStore::FoundationDb(foundation_db_store) => {
crate::backend::foundationdb::FdbStore::open(foundation_db_store)
@@ -43,27 +67,95 @@ impl BlobStore {
}
// inbuxa: ST-16 to ST-22
structs::BlobStore::Sharded(sharded) => {
let mut warnings = Vec::new();
let result = crate::backend::scaleout::ShardedBlobStore::open(
sharded,
&bp.data_store,
&mut warnings,
)
.await;
for warning in warnings {
bp.build_warning(ObjectType::BlobStore.singleton(), warning);
}
result
crate::backend::scaleout::ShardedBlobStore::open(sharded, data_store, warnings)
.await
}
_ => Err("Binary was not compiled with the selected blob store backend".to_string()),
};
match result {
Ok(store) => Some(store),
Err(err) => {
bp.build_error(ObjectType::BlobStore.singleton(), err);
None
}
}
}
}
/// inbuxa: blob-store-change spec, BM-1: where a blob store setting puts
/// blobs, without its credentials or tuning. Two settings with the same
/// place read the same blobs; a different place doesn't have them.
pub fn blob_store_location(config: &structs::BlobStore) -> String {
match config {
structs::BlobStore::Default => "the data store".to_string(),
// ST-16 to ST-22: the set moves blobs between its members itself
structs::BlobStore::Sharded(_) => "a sharded set".to_string(),
structs::BlobStore::S3(s3) => {
let region = match &s3.region {
structs::S3StoreRegion::Custom(custom) => {
format!("{} ({})", custom.custom_endpoint, custom.custom_region)
}
region => format!("{region:?}"),
};
with_prefix(
format!("S3 {region}, bucket {}", s3.bucket),
s3.key_prefix.as_deref(),
)
}
structs::BlobStore::Azure(azure) => with_prefix(
format!(
"Azure {}, container {}",
azure.storage_account, azure.container
),
azure.key_prefix.as_deref(),
),
structs::BlobStore::FileSystem(fs) => {
format!("the directory {}, depth {}", fs.path, fs.depth.min(5))
}
structs::BlobStore::FoundationDb(fdb) => format!(
"FoundationDB {}",
fdb.cluster_file
.as_deref()
.unwrap_or("(default cluster file)")
),
structs::BlobStore::PostgreSql(pg) => {
format!("PostgreSQL {}:{}/{}", pg.host, pg.port, pg.database)
}
structs::BlobStore::MySql(my) => {
format!("MySQL {}:{}/{}", my.host, my.port, my.database)
}
}
}
fn with_prefix(place: String, prefix: Option<&str>) -> String {
match prefix.filter(|p| !p.is_empty()) {
Some(prefix) => format!("{place}, prefix {prefix}"),
None => place,
}
}
#[cfg(test)]
mod tests {
use super::*;
fn fs(path: &str, depth: u64) -> structs::BlobStore {
structs::BlobStore::FileSystem(structs::FileSystemStore {
path: path.into(),
depth,
})
}
#[test]
fn a_place_is_where_blobs_go_not_how() {
assert_eq!(
blob_store_location(&structs::BlobStore::Default),
"the data store"
);
assert_eq!(
blob_store_location(&fs("/srv/blobs", 2)),
"the directory /srv/blobs, depth 2"
);
// The depth decides each blob's path, so it's part of the place
assert_ne!(
blob_store_location(&fs("/srv/blobs", 2)),
blob_store_location(&fs("/srv/blobs", 3))
);
assert_ne!(
blob_store_location(&fs("/srv/blobs", 2)),
blob_store_location(&fs("/srv/other", 2))
);
}
}
+47 -18
View File
@@ -15,6 +15,12 @@ const LZ4_MARKER: u8 = MAGIC_MARKER | 0x01;
//const ZSTD_MARKER: u8 = MAGIC_MARKER | 0x02;
const NONE_MARKER: u8 = 0x00;
/// What a stored blob decodes to: its content, or an empty value as is.
enum Decoded {
Content(Vec<u8>),
Bare(Vec<u8>),
}
impl BlobStore {
pub async fn get_blob(&self, key: &[u8], range: Range<usize>) -> trc::Result<Option<Vec<u8>>> {
let start_time = Instant::now();
@@ -27,11 +33,42 @@ impl BlobStore {
Size = result.as_ref().map_or(0, |data| data.len()),
);
let Some(mut data) = result else {
let Some(data) = result else {
return Ok(None);
};
let mut data = match data.last().copied() {
let mut data = match Self::decode_stored(key, data)? {
Decoded::Content(data) => data,
Decoded::Bare(data) => return Ok(Some(data)),
};
if range.start == 0 {
if range.end > data.len() {
Ok(Some(data))
} else {
data.truncate(range.end);
Ok(Some(data))
}
} else {
Ok(Some(
data.get(range.start..range.end)
.unwrap_or_default()
.to_vec(),
))
}
}
/// inbuxa: a blob's content from what the backend keeps, as
/// [`BlobStore::get_blob`] reads it (blob-store-change spec, BM-5: a copy
/// checks a blob's hash before writing it as it was stored).
pub fn content_of_stored(key: &[u8], data: Vec<u8>) -> trc::Result<Vec<u8>> {
Self::decode_stored(key, data).map(|decoded| match decoded {
Decoded::Content(data) | Decoded::Bare(data) => data,
})
}
fn decode_stored(key: &[u8], mut data: Vec<u8>) -> trc::Result<Decoded> {
let data = match data.last().copied() {
Some(LZ4_MARKER) => {
lz4_flex::decompress_size_prepended(data.get(..data.len() - 1).unwrap_or_default())
.map_err(|err| {
@@ -53,24 +90,10 @@ impl BlobStore {
data
}
None => {
return Ok(Some(data));
return Ok(Decoded::Bare(data));
}
};
if range.start == 0 {
if range.end > data.len() {
Ok(Some(data))
} else {
data.truncate(range.end);
Ok(Some(data))
}
} else {
Ok(Some(
data.get(range.start..range.end)
.unwrap_or_default()
.to_vec(),
))
}
Ok(Decoded::Content(data))
}
pub async fn put_blob(
@@ -183,6 +206,8 @@ impl BlobStore {
BlobStore::Azure(store) => store.get_blob(key, 0..usize::MAX).await,
// inbuxa: ST-17
BlobStore::Sharded(store) => store.get(key).await,
// inbuxa: BM-3
BlobStore::Moving(store) => store.raw_get(key).await,
}
}
@@ -212,6 +237,8 @@ impl BlobStore {
BlobStore::Azure(store) => store.put_blob(key, data).await,
// inbuxa: ST-18
BlobStore::Sharded(store) => store.put(key, data).await,
// inbuxa: BM-3: writes go to the current store only
BlobStore::Moving(store) => Box::pin(store.current.raw_put(key, data)).await,
}
}
@@ -241,6 +268,8 @@ impl BlobStore {
BlobStore::Azure(store) => store.delete_blob(key).await,
// inbuxa: ST-18
BlobStore::Sharded(store) => store.delete(key).await,
// inbuxa: BM-3: nothing is deleted from an old store
BlobStore::Moving(store) => Box::pin(store.current.raw_delete(key)).await,
}
}
}
+3
View File
@@ -185,6 +185,9 @@ pub enum BlobStore {
Azure(Arc<backend::azure::AzureStore>),
// inbuxa: ST-16 to ST-22
Sharded(Arc<backend::scaleout::ShardedBlobStore>),
// inbuxa: blob-store-change spec, BM-3: the current store, read through
// to the old ones while a change is unresolved
Moving(Arc<write::blob_move::MovingBlobStore>),
}
#[derive(Clone)]
+184
View File
@@ -0,0 +1,184 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! inbuxa: what the store-check spec (docs/spec/features/store-check.md)
//! reads from the store: each blob the data holds a link to, with who holds
//! it, one slice of the blob index at a time (SV-3), and whether a blob is
//! in the blob store without reading all of it (SV-2, quick).
use super::{BlobOp, ValueClass, key::DeserializeBigEndian, now};
use crate::{
BlobStore, Deserialize, IterateParams, Store, U16_LEN, U32_LEN, U64_LEN, ValueKey,
write::BlobLink,
};
use registry::types::id::ObjectId;
use trc::AddContext;
use types::blob_hash::{BLOB_HASH_LEN, BlobHash};
/// Who holds a link to a blob (SV-3).
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum BlobHolder {
/// A document: an email, a file, a Sieve script, ...
Document {
account_id: u32,
collection: u8,
document_id: u32,
},
/// A queued message.
Queue { id: u64 },
/// A deleted item kept to be restored, until its link expires.
Archived { account_id: u32, item: ObjectId },
}
/// A blob something holds, and whether the store has its commit marker.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LinkedBlob {
pub hash: BlobHash,
pub committed: bool,
pub holders: Vec<BlobHolder>,
/// inbuxa: blob-store-change spec, BM-5: a link that hasn't expired
/// and names nothing: an upload, a cached app bundle, a journal entry.
pub temporary: bool,
}
impl Store {
/// Every blob in one slice of the blob index (the first byte of its
/// hash) that something holds a link to. Uploads nothing holds yet and
/// expired links are left out: they're the purge's to clear.
pub async fn linked_blobs(&self, slice: u8) -> trc::Result<Vec<LinkedBlob>> {
let mut blobs = self.indexed_blobs(slice).await?;
blobs.retain(|b| !b.holders.is_empty());
Ok(blobs)
}
/// inbuxa: blob-store-change spec, BM-5: every blob in one slice of the
/// blob index that something holds, a plain link that hasn't expired
/// included. Expired links are left out.
pub async fn blobs_to_copy(&self, slice: u8) -> trc::Result<Vec<LinkedBlob>> {
let mut blobs = self.indexed_blobs(slice).await?;
blobs.retain(|b| !b.holders.is_empty() || b.temporary);
Ok(blobs)
}
async fn indexed_blobs(&self, slice: u8) -> trc::Result<Vec<LinkedBlob>> {
const TEMP_LINK: usize = BLOB_HASH_LEN + U32_LEN + U64_LEN;
const DOC_LINK: usize = BLOB_HASH_LEN + U32_LEN + 1 + U32_LEN;
const ID_LINK: usize = BLOB_HASH_LEN + U64_LEN;
let mut from_hash = BlobHash::default();
let mut to_hash = BlobHash::new_max();
from_hash.0[0] = slice;
to_hash.0[0] = slice;
let from_key = ValueKey {
account_id: 0,
collection: 0,
document_id: 0,
class: ValueClass::Blob(BlobOp::Commit { hash: from_hash }),
};
let to_key = ValueKey {
account_id: u32::MAX,
collection: u8::MAX,
document_id: u32::MAX,
class: ValueClass::Blob(BlobOp::Link {
hash: to_hash,
to: BlobLink::Document,
}),
};
let now = now();
let mut blobs: Vec<LinkedBlob> = Vec::new();
self.iterate(
IterateParams::new(from_key, to_key).ascending(),
|key, value| {
let hash = key
.get(0..BLOB_HASH_LEN)
.and_then(|h| BlobHash::try_from_hash_slice(h).ok())
.ok_or_else(|| {
trc::Error::corrupted_key(key, value.into(), trc::location!())
})?;
if blobs.last().is_none_or(|b| b.hash != hash) {
blobs.push(LinkedBlob {
hash,
committed: false,
holders: Vec::new(),
temporary: false,
});
}
let blob = blobs.last_mut().unwrap();
match key.len() {
BLOB_HASH_LEN => blob.committed = true,
DOC_LINK => blob.holders.push(BlobHolder::Document {
account_id: key.deserialize_be_u32(BLOB_HASH_LEN)?,
collection: key[BLOB_HASH_LEN + U32_LEN],
document_id: key.deserialize_be_u32(BLOB_HASH_LEN + U32_LEN + 1)?,
}),
ID_LINK => blob.holders.push(BlobHolder::Queue {
id: key.deserialize_be_u64(BLOB_HASH_LEN)?,
}),
TEMP_LINK => {
// Only a link that names a kept item: a plain one is
// an upload nothing holds yet
let until = key.deserialize_be_u64(BLOB_HASH_LEN + U32_LEN)?;
if until > now && value.len() == U16_LEN + U64_LEN {
blob.holders.push(BlobHolder::Archived {
account_id: key.deserialize_be_u32(BLOB_HASH_LEN)?,
item: ObjectId::deserialize(value)?,
});
} else if until > now {
blob.temporary = true;
}
}
_ => {
return Err(trc::Error::corrupted_key(
key,
value.into(),
trc::location!(),
));
}
}
Ok(true)
},
)
.await
.caused_by(trc::location!())?;
Ok(blobs)
}
}
impl BlobStore {
/// Whether the blob store has anything under `key`, reading as little
/// of it as each backend allows: a file's metadata, one byte of an
/// object, or, in a data store, the blob itself.
pub async fn blob_present(&self, key: &[u8]) -> trc::Result<bool> {
match self {
BlobStore::Fs(store) => store.get_blob(key, 0..1).await.map(|b| b.is_some()),
#[cfg(feature = "s3")]
BlobStore::S3(store) => match store.get_blob(key, 0..1).await {
Ok(found) => Ok(found.is_some()),
// An empty object has no first byte to read
Err(_) => self.raw_get(key).await.map(|b| b.is_some()),
},
#[cfg(feature = "azure")]
BlobStore::Azure(store) => match store.get_blob(key, 0..1).await {
Ok(found) => Ok(found.is_some()),
Err(_) => self.raw_get(key).await.map(|b| b.is_some()),
},
BlobStore::Sharded(store) => {
// Anywhere in the set: a misplaced blob is still there
for member in &store.members {
if Box::pin(member.blob_present(key)).await? {
return Ok(true);
}
}
Ok(false)
}
BlobStore::Store(_) => self.raw_get(key).await.map(|b| b.is_some()),
// inbuxa: blob-store-change spec, BM-3
BlobStore::Moving(store) => store.blob_present(key).await,
}
}
}
+234
View File
@@ -0,0 +1,234 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! inbuxa: what the blob-store-change spec
//! (docs/spec/features/blob-store-change.md) needs from the store: a blob
//! store that, while a change of blob store is unresolved, looks for what
//! it doesn't have in the stores the server used before, and keeps what it
//! finds there (BM-3). Writes and deletes go to the current store only.
use crate::BlobStore;
use arc_swap::ArcSwap;
use std::sync::Arc;
use trc::StoreEvent;
/// A blob store the server used before an unresolved change.
#[derive(Clone)]
pub struct OldBlobStore {
/// The `inbuxa:BlobStoreChange` that moved away from it.
pub change_id: u32,
pub store: BlobStore,
}
/// The current blob store, and the old ones read through while their
/// changes are unresolved, newest first (BM-3).
pub struct MovingBlobStore {
pub current: BlobStore,
old: ArcSwap<Vec<OldBlobStore>>,
}
impl MovingBlobStore {
pub fn new(current: BlobStore, old: Vec<OldBlobStore>) -> Self {
MovingBlobStore {
current,
old: ArcSwap::from_pointee(old),
}
}
/// The old stores still read through.
pub fn old_stores(&self) -> Arc<Vec<OldBlobStore>> {
self.old.load_full()
}
/// BM-10: keeps only the old stores of changes still unresolved.
pub fn retain_changes(&self, unresolved: &[u32]) {
let old = self.old.load();
if old.iter().all(|o| unresolved.contains(&o.change_id)) {
return;
}
let kept = old
.iter()
.filter(|o| unresolved.contains(&o.change_id))
.cloned()
.collect::<Vec<_>>();
self.old.store(Arc::new(kept));
}
/// BM-3: the blob as the current store has it or, failing that, as the
/// newest old store that has it does. A blob found in an old store is
/// written to the current one under the same key, as it was stored.
pub(crate) async fn raw_get(&self, key: &[u8]) -> trc::Result<Option<Vec<u8>>> {
if let Some(data) = Box::pin(self.current.raw_get(key)).await? {
return Ok(Some(data));
}
let old = self.old.load_full();
for old in old.iter() {
match Box::pin(old.store.raw_get(key)).await {
Ok(Some(data)) => {
if let Err(err) = Box::pin(self.current.raw_put(key, &data)).await {
trc::error!(
err.details("Failed to keep a blob read through an old blob store")
.ctx(trc::Key::Id, old.change_id)
.ctx(trc::Key::Key, key)
);
}
trc::event!(
Store(StoreEvent::BlobReadThrough),
Key = key,
Id = old.change_id,
Size = data.len(),
);
return Ok(Some(data));
}
Ok(None) => {}
Err(err) => {
// An old store that fails is skipped, not fatal: the
// current store answered, and it had nothing
trc::error!(
err.details("An old blob store failed to read")
.ctx(trc::Key::Id, old.change_id)
.ctx(trc::Key::Key, key)
.caused_by(trc::location!())
);
}
}
}
Ok(None)
}
/// Whether a read would find the blob: in the current store or in an
/// old one still read through.
pub(crate) async fn blob_present(&self, key: &[u8]) -> trc::Result<bool> {
if Box::pin(self.current.blob_present(key)).await? {
return Ok(true);
}
for old in self.old.load_full().iter() {
if Box::pin(old.store.blob_present(key)).await.unwrap_or(false) {
return Ok(true);
}
}
Ok(false)
}
}
impl BlobStore {
/// The store blobs are written to: past any old stores read through.
pub fn current(&self) -> &BlobStore {
match self {
BlobStore::Moving(moving) => moving.current.current(),
store => store,
}
}
/// The read-through wrapper, when there is one (BM-3).
pub fn moving(&self) -> Option<&Arc<MovingBlobStore>> {
match self {
BlobStore::Moving(moving) => Some(moving),
_ => None,
}
}
/// A blob exactly as the backend keeps it, compression marker and all,
/// without reading through (BM-5: a copy reads the old store itself).
pub async fn get_stored(&self, key: &[u8]) -> trc::Result<Option<Vec<u8>>> {
self.current().raw_get(key).await
}
/// Writes a blob exactly as [`BlobStore::get_stored`] gave it (BM-5).
pub async fn put_stored(&self, key: &[u8], data: &[u8]) -> trc::Result<()> {
self.current().raw_put(key, data).await
}
/// Whether the store itself has the blob, not reading through (BM-5).
pub async fn stored_present(&self, key: &[u8]) -> trc::Result<bool> {
self.current().blob_present(key).await
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{CompressionAlgo, backend::fs::FsStore};
use registry::schema::structs;
async fn fs_store(dir: &std::path::Path) -> BlobStore {
FsStore::open(structs::FileSystemStore {
path: dir.to_string_lossy().into_owned(),
depth: 2,
})
.await
.unwrap()
}
#[tokio::test]
async fn reads_through_and_keeps_what_it_found() {
let root = std::env::temp_dir().join(format!("inbuxa-blob-move-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
let old = fs_store(&root.join("old")).await;
let new = fs_store(&root.join("new")).await;
old.put_blob(b"left-behind", b"hello", CompressionAlgo::Lz4)
.await
.unwrap();
let moving = BlobStore::Moving(Arc::new(MovingBlobStore::new(
new.clone(),
vec![OldBlobStore {
change_id: 7,
store: old.clone(),
}],
)));
// Only the old store has it: read through, then kept in the new one
assert!(!new.blob_present(b"left-behind").await.unwrap());
assert!(moving.blob_present(b"left-behind").await.unwrap());
assert!(!moving.stored_present(b"left-behind").await.unwrap());
assert_eq!(
moving
.get_blob(b"left-behind", 0..usize::MAX)
.await
.unwrap()
.as_deref(),
Some(&b"hello"[..])
);
assert_eq!(
new.get_blob(b"left-behind", 0..usize::MAX)
.await
.unwrap()
.as_deref(),
Some(&b"hello"[..])
);
// In neither: absent
assert_eq!(
moving.get_blob(b"nowhere", 0..usize::MAX).await.unwrap(),
None
);
// Writes and deletes touch only the new store
moving
.put_blob(b"fresh", b"new mail", CompressionAlgo::None)
.await
.unwrap();
assert!(new.blob_present(b"fresh").await.unwrap());
assert!(!old.blob_present(b"fresh").await.unwrap());
moving.delete_blob(b"left-behind").await.unwrap();
assert!(!new.blob_present(b"left-behind").await.unwrap());
assert!(old.blob_present(b"left-behind").await.unwrap());
// Once the change is resolved, nothing more is read through
moving.moving().unwrap().retain_changes(&[]);
assert!(moving.moving().unwrap().old_stores().is_empty());
assert_eq!(
moving
.get_blob(b"left-behind", 0..usize::MAX)
.await
.unwrap(),
None
);
let _ = std::fs::remove_dir_all(&root);
}
}
+5
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use self::assert::AssertValue;
@@ -27,9 +29,12 @@ pub mod assert;
pub mod batch;
pub mod bitpack;
pub mod blob;
pub mod blob_check; // inbuxa: store-check spec
pub mod blob_move; // inbuxa: blob-store-change spec
pub mod key;
pub mod log;
pub mod serialize;
pub mod snapshot; // inbuxa: store-snapshot spec
pub(crate) const ARCHIVE_ALIGNMENT: usize = 16;
+248
View File
@@ -0,0 +1,248 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! inbuxa: what the store-snapshot spec (docs/spec/features/store-snapshot.md)
//! needs from the store: holding the blob purge (SN-4), a checkpoint of the
//! data store and a copy of a filesystem blob store's files (SN-2).
use crate::{BlobStore, Store};
use std::{
io,
path::{Path, PathBuf},
};
use tokio::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};
/// SN-4. A purge runs under the read side; a snapshot taking blobs holds
/// the write side. The data store is RocksDB, so one process has it open,
/// and a lock in that process is enough.
static BLOB_PURGE_GATE: RwLock<()> = RwLock::const_new(());
/// Whether a blob purge may run now: `None` while a snapshot takes blobs or
/// waits to. Keep the guard for as long as the purge runs.
pub fn blob_purge_may_run() -> Option<RwLockReadGuard<'static, ()>> {
BLOB_PURGE_GATE.try_read().ok()
}
/// Holds the blob purge until the guard is dropped. A purge already running
/// finishes first.
pub async fn hold_blob_purge() -> RwLockWriteGuard<'static, ()> {
BLOB_PURGE_GATE.write().await
}
/// What a snapshot can hold of this server's stores (SN-1, SN-2).
pub enum SnapshotSource {
/// Blobs are in the data store, so the checkpoint holds them.
DataOnly { data_path: PathBuf },
/// Blobs are files under `blob_path`, copied after the checkpoint.
DataAndFiles {
data_path: PathBuf,
blob_path: PathBuf,
},
}
/// Why a snapshot can't be taken of these stores.
pub enum NotSnapshottable {
DataStore(&'static str),
BlobStore(&'static str),
/// inbuxa: blob-store-change spec, BM-8: some blobs are still only in
/// the blob store the server used before.
BlobStoreChanging,
}
impl Store {
/// SN-2: a checkpoint of the data store into `dest`, which must not
/// exist yet. Only RocksDB has one.
#[cfg_attr(not(feature = "rocks"), allow(unused_variables))]
pub async fn checkpoint(&self, dest: PathBuf) -> trc::Result<()> {
match self {
#[cfg(feature = "rocks")]
Store::RocksDb(store) => store.checkpoint(dest).await,
_ => Err(trc::StoreEvent::NotSupported
.into_err()
.details("Only a RocksDB store has a checkpoint")),
}
}
/// What a snapshot of this data store and `blobs` would hold, or why
/// there can't be one.
#[cfg_attr(not(feature = "rocks"), allow(unused_variables, unreachable_code))]
pub fn snapshot_source(&self, blobs: &BlobStore) -> Result<SnapshotSource, NotSnapshottable> {
let data_path = match self {
#[cfg(feature = "rocks")]
Store::RocksDb(store) => store.path().to_path_buf(),
#[cfg(feature = "sqlite")]
Store::SQLite(_) => return Err(NotSnapshottable::DataStore("SQLite")),
#[cfg(feature = "foundation")]
Store::FoundationDb(_) => return Err(NotSnapshottable::DataStore("FoundationDB")),
#[cfg(feature = "postgres")]
Store::PostgreSQL(_) => return Err(NotSnapshottable::DataStore("PostgreSQL")),
#[cfg(feature = "mysql")]
Store::MySQL(_) => return Err(NotSnapshottable::DataStore("MySQL")),
Store::Replicated(_) => {
return Err(NotSnapshottable::DataStore("a replicated SQL database"));
}
Store::Ephemeral(_) | Store::None => {
return Err(NotSnapshottable::DataStore("an in-memory store"));
}
};
match blobs {
BlobStore::Store(store) if store.is_same(self) => {
Ok(SnapshotSource::DataOnly { data_path })
}
BlobStore::Fs(fs) => {
let blob_path = fs.root().to_path_buf();
// The blob files can't be told from the data store's own
// in one directory. A blob directory that holds the data
// store's is fine: the copy leaves that one out
if data_path == blob_path {
return Err(NotSnapshottable::BlobStore(
"the data store's own directory",
));
}
Ok(SnapshotSource::DataAndFiles {
data_path,
blob_path,
})
}
BlobStore::Store(_) => Err(NotSnapshottable::BlobStore("another database")),
#[cfg(feature = "s3")]
BlobStore::S3(_) => Err(NotSnapshottable::BlobStore("S3")),
#[cfg(feature = "azure")]
BlobStore::Azure(_) => Err(NotSnapshottable::BlobStore("Azure")),
BlobStore::Sharded(_) => Err(NotSnapshottable::BlobStore("a sharded set")),
// inbuxa: blob-store-change spec, BM-8: blobs only an old store
// has would be left out
BlobStore::Moving(moving) if !moving.old_stores().is_empty() => {
Err(NotSnapshottable::BlobStoreChanging)
}
BlobStore::Moving(moving) => self.snapshot_source(&moving.current),
}
}
}
/// What a copy of a tree did.
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub struct TreeCopy {
pub files: u64,
pub bytes: u64,
/// Files that couldn't be hard-linked and were copied.
pub copied: u64,
}
/// SN-2, SN-3: every file under `from` into `to`, hard-linked where it can
/// be and copied where it can't, leaving out `skip` (the data store's
/// directory, with the snapshots in it). Names starting with a dot are
/// skipped: they're blobs being written (SN-10).
pub async fn copy_tree(from: PathBuf, to: PathBuf, skip: PathBuf) -> trc::Result<TreeCopy> {
tokio::task::spawn_blocking(move || {
let mut done = TreeCopy::default();
copy_dir(&from, &to, &skip, &mut done).map(|_| done)
})
.await
.map_err(|err| trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err))?
.map_err(|err| trc::StoreEvent::FilesystemError.reason(err))
}
fn copy_dir(from: &Path, to: &Path, skip: &Path, done: &mut TreeCopy) -> io::Result<()> {
std::fs::create_dir_all(to)?;
for entry in std::fs::read_dir(from)? {
let entry = entry?;
let name = entry.file_name();
if name.as_encoded_bytes().first() == Some(&b'.') {
continue;
}
let (source, dest) = (entry.path(), to.join(&name));
if source == skip {
continue;
}
let kind = entry.file_type()?;
if kind.is_dir() {
copy_dir(&source, &dest, skip, done)?;
} else if kind.is_file() {
match std::fs::hard_link(&source, &dest) {
Ok(()) => {}
// Gone since it was listed: a purge ran before the hold, or
// it's an orphan nothing links to
Err(err) if err.kind() == io::ErrorKind::NotFound => continue,
Err(_) => {
match std::fs::copy(&source, &dest) {
Ok(_) => {}
Err(err) if err.kind() == io::ErrorKind::NotFound => continue,
Err(err) => return Err(err),
}
done.copied += 1;
}
}
done.files += 1;
done.bytes += std::fs::metadata(&dest)?.len();
}
}
Ok(())
}
/// Bytes in the files under `dir`.
pub async fn tree_size(dir: PathBuf) -> trc::Result<u64> {
fn size(dir: &Path) -> io::Result<u64> {
let mut total = 0;
for entry in std::fs::read_dir(dir)? {
let entry = entry?;
let kind = entry.file_type()?;
if kind.is_dir() {
total += size(&entry.path())?;
} else if kind.is_file() {
total += entry.metadata()?.len();
}
}
Ok(total)
}
tokio::task::spawn_blocking(move || size(&dir))
.await
.map_err(|err| trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err))?
.map_err(|err| trc::StoreEvent::FilesystemError.reason(err))
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn copies_a_tree_and_skips_what_is_being_written() {
let base = std::env::temp_dir().join(format!("inbuxa-sn-tree-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&base);
let from = base.join("from");
std::fs::create_dir_all(from.join("a/b")).unwrap();
std::fs::write(from.join("a/b/BLOB1"), b"one").unwrap();
std::fs::write(from.join("a/BLOB2"), b"two!").unwrap();
std::fs::write(from.join("a/b/.BLOB3.1-0.tmp"), b"half").unwrap();
std::fs::create_dir_all(from.join("rocks.db/snapshots")).unwrap();
std::fs::write(from.join("rocks.db/000001.sst"), b"data").unwrap();
let to = base.join("to");
let done = copy_tree(from.clone(), to.clone(), from.join("rocks.db"))
.await
.unwrap();
assert_eq!((done.files, done.bytes), (2, 7));
assert_eq!(std::fs::read(to.join("a/b/BLOB1")).unwrap(), b"one");
assert!(!to.join("a/b/.BLOB3.1-0.tmp").exists());
assert!(!to.join("rocks.db").exists());
assert_eq!(tree_size(to).await.unwrap(), 7);
std::fs::remove_dir_all(&base).unwrap();
}
#[test]
fn the_purge_waits_while_a_snapshot_holds_it() {
let runtime = tokio::runtime::Builder::new_current_thread()
.build()
.unwrap();
runtime.block_on(async {
assert!(blob_purge_may_run().is_some());
let hold = hold_blob_purge().await;
assert!(blob_purge_may_run().is_none());
drop(hold);
assert!(blob_purge_may_run().is_some());
});
}
}
+11 -3
View File
@@ -12,10 +12,14 @@
// auth.legacy-protocol-refused (legacy-protocols LP-6); 643 is
// security.legacy-protocols-changed (LP-8); 644 to 646 are the cluster
// coordinator's connection events; 647 and 648 are the audit log's
// (audit-hold-lock spec, AU-3, AU-8); 649 is cluster.node-id-reassigned.
// (audit-hold-lock spec, AU-3, AU-8); 649 is cluster.node-id-reassigned;
// 650 to 659 are inbuxa Relay's in the 27.1 series and unused here; 660 is
// store.blob-read-through (blob-store-change spec, BM-3). Ids are shared
// with 27.1 because stored settings name events by id.
// TOTAL_EVENT_COUNT sizes the collector's per-event tables, so it is one
// more than the highest id; the test below holds it to the variant list
pub const TOTAL_EVENT_COUNT: usize = 650;
// less those 10 unused ids
pub const TOTAL_EVENT_COUNT: usize = 661;
pub const TOTAL_METRIC_COUNT: usize = 369;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -853,6 +857,9 @@ pub enum StoreEvent {
CacheUpdate = 577,
CacheEntryTooLarge = 633,
BlobMissingMarker = 507,
/// inbuxa: blob-store-change spec, BM-3: a blob the blob store lacked
/// was found in the one it replaced, and kept.
BlobReadThrough = 660,
DataWrite = 513,
DataIterate = 512,
BlobRead = 508,
@@ -1397,7 +1404,8 @@ mod tests {
#[test]
fn event_ids_fit_the_collector_tables() {
let variants = EventType::variants();
assert_eq!(variants.len(), TOTAL_EVENT_COUNT);
// inbuxa: 650 to 659 are the 27.1 series' Relay events
assert_eq!(variants.len() + 10, TOTAL_EVENT_COUNT);
let mut ids = variants
.iter()
.map(|event| event.to_id())
+10
View File
@@ -609,6 +609,7 @@ impl EventType {
b"store.cache-update" => EventType::Store(StoreEvent::CacheUpdate),
b"store.cache-entry-too-large" => EventType::Store(StoreEvent::CacheEntryTooLarge),
b"store.blob-missing-marker" => EventType::Store(StoreEvent::BlobMissingMarker),
b"store.blob-read-through" => EventType::Store(StoreEvent::BlobReadThrough),
b"store.data-write" => EventType::Store(StoreEvent::DataWrite),
b"store.data-iterate" => EventType::Store(StoreEvent::DataIterate),
b"store.blob-read" => EventType::Store(StoreEvent::BlobRead),
@@ -1394,6 +1395,7 @@ impl EventType {
EventType::Store(StoreEvent::CacheUpdate) => "store.cache-update",
EventType::Store(StoreEvent::CacheEntryTooLarge) => "store.cache-entry-too-large",
EventType::Store(StoreEvent::BlobMissingMarker) => "store.blob-missing-marker",
EventType::Store(StoreEvent::BlobReadThrough) => "store.blob-read-through",
EventType::Store(StoreEvent::DataWrite) => "store.data-write",
EventType::Store(StoreEvent::DataIterate) => "store.data-iterate",
EventType::Store(StoreEvent::BlobRead) => "store.blob-read",
@@ -2072,6 +2074,7 @@ impl EventType {
EventType::Store(StoreEvent::CacheUpdate) => 577,
EventType::Store(StoreEvent::CacheEntryTooLarge) => 633,
EventType::Store(StoreEvent::BlobMissingMarker) => 507,
EventType::Store(StoreEvent::BlobReadThrough) => 660,
EventType::Store(StoreEvent::DataWrite) => 513,
EventType::Store(StoreEvent::DataIterate) => 512,
EventType::Store(StoreEvent::BlobRead) => 508,
@@ -2770,6 +2773,7 @@ impl EventType {
577 => Some(EventType::Store(StoreEvent::CacheUpdate)),
633 => Some(EventType::Store(StoreEvent::CacheEntryTooLarge)),
507 => Some(EventType::Store(StoreEvent::BlobMissingMarker)),
660 => Some(EventType::Store(StoreEvent::BlobReadThrough)),
513 => Some(EventType::Store(StoreEvent::DataWrite)),
512 => Some(EventType::Store(StoreEvent::DataIterate)),
508 => Some(EventType::Store(StoreEvent::BlobRead)),
@@ -3194,6 +3198,7 @@ impl EventType {
EventType::Store(StoreEvent::HttpStoreError) => Level::Warn,
EventType::Store(StoreEvent::CacheEntryTooLarge) => Level::Warn,
EventType::Store(StoreEvent::BlobMissingMarker) => Level::Warn,
EventType::Store(StoreEvent::BlobReadThrough) => Level::Info,
EventType::TaskManager(TaskManagerEvent::TaskFailed) => Level::Warn,
EventType::Telemetry(TelemetryEvent::AlertEvent) => Level::Warn,
EventType::Telemetry(TelemetryEvent::LogError) => Level::Warn,
@@ -3941,6 +3946,7 @@ impl EventType {
EventType::Store(StoreEvent::CacheUpdate) => "Cache update",
EventType::Store(StoreEvent::CacheEntryTooLarge) => "Cache entry too large",
EventType::Store(StoreEvent::BlobMissingMarker) => "Blob missing marker",
EventType::Store(StoreEvent::BlobReadThrough) => "Blob read through",
EventType::Store(StoreEvent::DataWrite) => "Write batch operation",
EventType::Store(StoreEvent::DataIterate) => "Data store iteration operation",
EventType::Store(StoreEvent::BlobRead) => "Blob read operation",
@@ -4302,6 +4308,9 @@ impl EventType {
EventType::Store(StoreEvent::CacheUpdate) => "Store error",
EventType::Store(StoreEvent::CacheEntryTooLarge) => "Cache entry too large to store",
EventType::Store(StoreEvent::BlobMissingMarker) => "Blob is missing marker",
EventType::Store(StoreEvent::BlobReadThrough) => {
"A blob was read from the blob store used before a change, and kept"
}
EventType::Store(StoreEvent::DataWrite) => "Store error",
EventType::Store(StoreEvent::DataIterate) => "Store error",
EventType::Store(StoreEvent::BlobRead) => "Store error",
@@ -4910,6 +4919,7 @@ impl EventType {
EventType::Store(StoreEvent::CacheUpdate),
EventType::Store(StoreEvent::CacheEntryTooLarge),
EventType::Store(StoreEvent::BlobMissingMarker),
EventType::Store(StoreEvent::BlobReadThrough),
EventType::Store(StoreEvent::DataWrite),
EventType::Store(StoreEvent::DataIterate),
EventType::Store(StoreEvent::BlobRead),
+6
View File
@@ -379,6 +379,12 @@ Not a rebuild: the **deliverability check** is INBUXA's own design (inbuxa-draft
Not a rebuild: **scheduled reports and the weekly digest** are INBUXA's own design (inbuxa-drafts `specs/scheduled-reports.md`). An administrator picks sections, a schedule in a time zone and recipients on this server; every node looks for due reports once a minute, one claims each run with the task lock, and the report is built from data the server already keeps and mailed DKIM-signed: `inbuxa:ScheduledReport`, `inbuxa:ScheduledReportSettings` and `inbuxa:ReportExport` (a download: a ZIP of a summary and CSVs) (`crates/jmap/src/inbuxa/scheduled_reports.rs`, `crates/features/src/scheduled_reports/`, `crates/services/src/inbuxa_scheduled_reports.rs`, `crates/smtp/src/reporting/inbuxa_send.rs`), and the `sysScheduledReportGet` and `sysScheduledReportUpdate` permissions. The weekly digest is a built-in report, on by default.
Not a rebuild: the **store check** is INBUXA's own design (`features/store-check.md`, inbuxa-server #322, building). An administrator starts a check of the store's blobs, quick (each one is there) or full (each one reads back intact), and gets a list of what's missing, damaged or no longer linked. One node runs it in the background, throttled, and keeps its progress: `inbuxa:StoreCheck` (`crates/jmap/src/inbuxa/store_check.rs`, `crates/features/src/store_check/`, `crates/services/src/inbuxa_store_check.rs`, `crates/store/src/write/blob_check.rs`), and the `sysStoreCheck` permission.
Not a rebuild: the **store snapshot** is INBUXA's own design (`features/store-snapshot.md`, inbuxa-server #323, building). An administrator has the server take a consistent copy of its RocksDB store, and of a filesystem blob store's files, into `snapshots/` in the store's directory while mail flows, for a backup tool to copy; the latest two are kept: `inbuxa:StoreSnapshot` (`crates/jmap/src/inbuxa/store_snapshot.rs`, `crates/services/src/inbuxa_store_snapshot.rs`, `crates/store/src/write/snapshot.rs`, `crates/store/src/backend/rocksdb/checkpoint.rs`), and the `sysStoreSnapshot` permission.
Not a rebuild: **changing the blob store** is INBUXA's own design (`features/blob-store-change.md`, inbuxa-server #333, building). When a node starts with the blob store setting pointing somewhere else, it records the change and keeps the old store's settings; until the change is resolved, a blob the new store lacks is read from the old one and kept, and an administrator has one node copy the rest over, checked by hash, with what couldn't be copied reported: `inbuxa:BlobStoreChange` (`crates/jmap/src/inbuxa/blob_store_change.rs`, `crates/features/src/blob_move/`, `crates/services/src/inbuxa_blob_move.rs`, `crates/common/src/manager/blob_moves.rs`, `crates/store/src/write/blob_move.rs`), the `store.blob-read-through` event, and the `sysBlobStoreChange` permission.
## 5. The web front ends
**Which ihasmail.** Public ihasmail stays Stalwart-facing: its code, docs,
+164
View File
@@ -0,0 +1,164 @@
# Feature spec: changing the blob store
Status: building, 2026-11-09 (inbuxa-server #333). Not a rebuild (SPEC.md
§4): inbuxa's own design.
## Provenance
| Source | Used for |
|---|---|
| Forum topic 238 | The need: a server moved its blobs to a filesystem store, took one message by SMTP, and went back to the data store a minute later. The message's metadata stayed; its blob was left in the filesystem store, and reading it failed from then on |
| This server's blob store build and dispatch (`crates/store/src/build/blob.rs`, `crates/store/src/dispatch/blob.rs`) | Where the blob store comes from, and that a change takes effect at start-up |
| The store check (`features/store-check.md`) and its blob index walk (`crates/store/src/write/blob_check.rs`) | Which blobs the data holds, how a blob proves its content, and how the object, permission and run are built |
Upstream's open code has nothing like it, and nothing here comes from
Enterprise code.
## What it is
The blob store setting (`x:BlobStore`) says where blobs go: in the data
store, in a directory, in a bucket. Changing it changes where the server
writes and reads them from the next start-up. It moves nothing. Before
this spec, every blob written under the old setting was left behind,
in either direction, and a message whose blob was left behind failed to
read with nothing to say why.
Now the server notices the change when it starts. It keeps reading what
it can't find from the old store and copies each blob it reads that way
into the new one. An administrator has it copy everything else, and the
change is resolved once the new store has every blob. Nothing is ever
deleted from the old store; that stays the administrator's to do.
## Requirements
- **BM-1.** Where a blob store is, without its credentials or tuning:
- the data store (`Default`);
- a filesystem store: its path and depth (the depth decides each blob's
path, so a new depth is a new place);
- S3: the region or endpoint, the bucket and the key prefix;
- Azure: the storage account, the container and the key prefix;
- PostgreSQL or MySQL: the host, port and database;
- FoundationDB: the cluster file;
- a sharded set is one place. The set moves blobs between its members
itself (scale-out spec, ST-16 to ST-22).
- **BM-2.** At start-up, a node compares where its blob store is (BM-1)
with where the store last said it was. When they differ, it records an
`inbuxa:BlobStoreChange` with both places, the time and the node, and
keeps the old store's settings in it so the old store can still be
read. It logs a warning that names both places. The first node to start
with the new setting records the change; the others find it recorded.
The first start of a server, or of a store from before this spec, only
records where the blob store is.
- **BM-3.** **Reading through.** While a change is unresolved (BM-5), a
blob the blob store doesn't have is looked for in the change's old
store. When it's there, it's returned and written to the current store
under the same key, with the event `store.blob-read-through` naming the
key and the change. Several unresolved changes are tried newest first.
Writes and deletes go to the current store only. An old store that
can't be opened at start-up is left out, with a warning, and the change
stays unresolved.
- **BM-4.** **Back where it was.** A change back to a store that an
earlier unresolved change moved away from (A to B, then back to A)
resolves that earlier change by itself: its old store is the current
one again. Its status is `reverted`. The new change (B to A) is the one
to copy from.
- **BM-5.** **Copying.** An update with `action: "copy"` starts a copy on
the node that takes it. One copy runs at a time across the server; a
second is refused while one runs. It goes through the blob index in its
256 slices and takes every committed blob that something holds: a
document, a queued message, a kept deleted item, or a link that hasn't
expired. It also takes the spam classifier's two files, which the
server keeps under fixed names. For each blob:
- already in the current store: counted as `alreadyThere`;
- in neither store: a `missing` problem;
- a blob a document, a queued message or a kept deleted item holds is
named by the hash of its content; when what the old store has doesn't
hash to its name, it isn't copied and is a `damaged` problem. Other
blobs (uploads, cached app bundles, the spam files) aren't named by
their content and are copied as they are;
- an error from either store: an `unreadable` problem, with the error;
- otherwise it's written to the current store as the old store has it,
compression and all: `copied`.
It's throttled like a store check (store-check spec, SV-6): by default
1,000 blobs and 20 MB a second, set with `blobsPerSecond` and
`bytesPerSecond` alongside the action; 0 is no limit.
- **BM-6.** A copy ends:
- `copied`: nothing was missing, damaged or unreadable. The change is
resolved: every node stops reading through it within a minute
(BM-10);
- `incomplete`: some blobs couldn't be copied. The first 1,000 problems
are kept, and every one is logged as `store.data-corruption`. The
change stays unresolved and can be copied again;
- `stopped`, by an update with `action: "stop"`; `failed`, with the
error; `interrupted`, when its node restarted or went quiet for five
minutes. Each leaves the change unresolved.
- **BM-7.** **Dismissing.** Destroying a change forgets it and the old
store's settings it kept, and reading through it stops. Nothing is
deleted from either store. A change whose copy runs can't be destroyed;
stop the copy first.
- **BM-8.** While a change is unresolved, a store snapshot is refused
(store-snapshot spec): it would leave out the blobs only the old store
has. The refusal says to copy or dismiss the change first.
- **BM-9.** Only the server's administrators see and act on changes:
`sysBlobStoreChange`, in the built-in Administrator role and granted
once to existing administrator roles. A tenant administrator is
refused. Copying, stopping and dismissing are audited; reading isn't.
`get` never returns the old store's settings, only where it is (BM-1).
- **BM-10.** On a cluster, every node reads the unresolved changes at
start-up and again every minute, so one that's resolved or dismissed
on any node stops being read through everywhere. During a rolling
restart the nodes still on the old setting write to the old store;
reading through covers that until the copy, which should start once
every node runs the new setting.
## The object
| Property | Type | |
|---|---|---|
| `id` | Id | |
| `from`, `to` | String | Where the old and the new blob store are (BM-1) |
| `changedAt` | UTCDate | When the first node started with the new setting |
| `node` | String | That node |
| `status` | String | `pending`, `copying`, `copied`, `incomplete`, `stopped`, `failed`, `interrupted`, `reverted` |
| `copyStartedAt`, `copyFinishedAt` | UTCDate | The latest copy |
| `copyStartedBy` | String | Who started it |
| `copyNode` | String | The node it runs on |
| `blobsPerSecond`, `bytesPerSecond` | UnsignedInt | Its limits |
| `progress` | Number | 0 to 1 |
| `blobsChecked`, `copied`, `alreadyThere`, `bytesCopied` | UnsignedInt | |
| `missing`, `damaged`, `unreadable` | UnsignedInt | Counts of each problem |
| `problems` | Object[] | The first 1,000 of the latest copy |
| `error` | String | Why a `failed` copy failed |
| `action` | String | Write-only: `copy` or `stop` |
A problem is `{kind, blobId, holder, accountId, objectId, detail}`:
`holder` is `email`, `file`, `queuedMessage`, `archivedItem`, `temporary`
or `fixed`, for the first thing found holding the blob.
## Tests
- Store level (`crates/store/src/write/blob_move.rs`): with two filesystem
stores, a blob only the old one has reads through and is then in the
new one; one in neither reads as absent; writes and deletes touch only
the new one; dropping the old store stops reading through.
- Where a store is (BM-1): credentials and tuning don't change it; a path,
depth, bucket or prefix does.
- Model (`crates/features/src/blob_move/`): recording a change, a change
back reverting the earlier one, and the copy's bookkeeping.
- `system::blob_store_change` (`tests/src/system/blob_store_change.rs`),
against a real server: a message delivered while a filesystem store is
set, the setting changed back and the server rebuilt; the message reads
through, the change is listed, a copy resolves it, and a damaged blob
in the old store is reported, not copied. Over JMAP: the refusals, stop,
the destroy rules, a tenant administrator's refusal.
## Not yet
- The console's page, and a warning on the blob store settings form
that existing blobs stay where they are.
- An `inbuxa-cli` command.
- Copying ahead of a change (the old store to a new one before the
setting changes). Reading through makes it unnecessary for the
server's own reads, but a long copy after the change means more
reads go to the old store meanwhile.
+133
View File
@@ -0,0 +1,133 @@
# Feature spec: checking the store's blobs
Status: building, 2026-11-08 (inbuxa-server #322). Not a rebuild (SPEC.md
§4): inbuxa's own design.
## Provenance
| Source | Used for |
|---|---|
| Forum topic 238 | The need: a mail whose metadata was there and whose blob wasn't, on a server that looked healthy, and a backup that copied it |
| This server's blob links (`crates/store/src/write/blob.rs`) and `BlobHash` (`crates/types/src/blob_hash.rs`) | What a document holds on to, and how a blob proves its content |
| This server's `inbuxa:HoldExport` and `inbuxa:Migration` | How a fork object, a background job and their permission are built |
Upstream's open code has nothing like it, and nothing here comes from
Enterprise code.
## What it is
An administrator asks the server to check that the blobs its data needs are
in the blob store, and, if asked, that each one is intact. The check
reports what it finds and changes nothing. It runs inside the server, on
the node the administrator reached, against the stores every node shares.
So it works while mail flows, on one machine or a cluster, whatever the
stores are. An offline command couldn't: RocksDB lets one process open the
store, and a cluster's data store and blob store may be on other machines.
## Requirements
- **SV-1.** A check is an `inbuxa:StoreCheck` under `urn:inbuxa:jmap`. It's
kept in the fork's subspace under `Vc` and its problems under `Vp`,
never in the registry. The latest 20 checks are kept; starting a 21st
forgets the oldest.
- **SV-2.** Creating one starts it. Its `mode` is:
- `quick`: every blob the check covers (SV-3) is in the blob store;
- `full`: also, each one reads back whole and its BLAKE3 hash is its id.
Blob ids are the hash of their content, so this proves the content,
not just that something is stored under the id.
One check runs at a time across the server; another create is refused
while one runs.
- **SV-3.** What it covers:
- **Links.** Every blob a document holds (`BlobLink::Document`: mail,
files, Sieve scripts and the rest, by collection), every blob a queued
message holds (`BlobLink::Id`), and every blob a kept deleted item
holds (a temporary link that names the archived item, not yet
expired). A blob several documents share is read once and reported
for each of them.
- **Mail metadata.** Each message's stored metadata names its blob. The
check finds the message's own link to it. A message that names a blob
without a link is a problem even when the blob is there, because
nothing stops the blob from being purged.
Not covered: uploads nothing holds yet (they expire on their own), and
files the server keeps under fixed names (spam model, app bundles).
- **SV-4.** A problem is one of:
- `missing`: the blob store has nothing under the id;
- `damaged` (full only): the content's hash isn't its id, or it doesn't
decompress;
- `unlinked`: a message's metadata names a blob it holds no link to,
with whether the blob is still there;
- `unreadable`: the blob store answered with an error, which is kept.
Each says the account (id and name), the object (`email`, `file`,
`sieveScript`, `calendarEvent`, `contactCard`, `queuedMessage`,
`archivedItem`, or the collection's number for any other), its id, the
blob id, and whether the store still has the blob's commit marker. A
missing blob with its marker was committed and lost afterwards; one
without it never finished being written.
- **SV-5.** A check keeps its first 1,000 problems. The counts cover all of
them. Every problem is also logged as an event (`store.data-corruption`),
with the account, collection, document and blob id, so problems past the
first 1,000 are in the log.
- **SV-6.** It's throttled, so a full check of a big store doesn't hold up
mail: by default at most 1,000 blobs a second and, in full mode, 20 MB a
second read. `blobsPerSecond` and `bytesPerSecond` set other limits at
create; 0 means no limit.
- **SV-7.** Progress: the share of the blob index done (it goes through it
in 256 slices), blobs and links checked, bytes read, problems by kind,
and accounts whose mail metadata was checked. It's saved at least every
10 seconds, which is also its heartbeat.
- **SV-8.** A check ends:
- `done`, problems or none;
- `stopped`, by an update with `action: "stop"`, which any node can
take: the running node sees it at its next save;
- `failed`, with the error, when the check itself couldn't go on;
- `interrupted`: its node restarted while it ran, or the node has been
quiet for five minutes.
- **SV-9.** Only administrators check the store: `sysStoreCheck`, in the
built-in Administrator role and granted once to existing administrator
roles. It reads every account's data, so a tenant administrator is
refused. Starting and stopping a check are audited; reading one isn't.
- **SV-10.** A finished check can be destroyed. A running one can't; stop
it first.
## The object
| Property | Type | |
|---|---|---|
| `id` | Id | |
| `mode` | String | `quick` or `full`, set at create |
| `blobsPerSecond`, `bytesPerSecond` | UnsignedInt | Limits, set at create (SV-6) |
| `status` | String | `running`, `done`, `stopped`, `failed`, `interrupted` |
| `createdAt`, `finishedAt` | UTCDate | |
| `createdBy` | String | Who started it |
| `node` | String | The node it runs on |
| `progress` | Number | 0 to 1 |
| `blobsChecked`, `linksChecked`, `bytesRead`, `messagesChecked` | UnsignedInt | |
| `missing`, `damaged`, `unlinked`, `unreadable` | UnsignedInt | Counts of each kind |
| `problems` | Object[] | The first 1,000 (SV-4, SV-5) |
| `error` | String | Why a `failed` check failed |
A problem is `{kind, accountId, accountName, object, objectId, blobId,
committed, detail}`.
## Tests
- `system::store_check` (`tests/src/system/store_check.rs`), against a real
server: a clean store reported clean; then blobs deleted from under
messages (missing, including a blob two messages share, reported once per
message), a blob overwritten with other content (damaged in full mode, not
in quick), and a message whose link was taken away (unlinked, with the
blob still there). Over JMAP: the refusal while one check runs, stop,
the destroy rules, bad input, the audit events, a user's refusal and a
tenant administrator's.
## Not yet
- The console's page and an `inbuxa-cli` command.
- A test with a queued message and a kept archived item holding a blob.
- Metadata for files and Sieve scripts, as for mail (SV-3): their links are
checked, but not that each one holds the blob its record names.
- Mail metadata of an account that holds no link at all and isn't in the
registry any more.
+151
View File
@@ -0,0 +1,151 @@
# Feature spec: a snapshot of the store, without stopping the server
Status: building, 2026-11-09 (inbuxa-server #323). Not a rebuild (SPEC.md
§4): inbuxa's own design.
## Provenance
| Source | Used for |
|---|---|
| Forum topic 238 | The need: a backup that stops the server, copies the store cold and starts it again, an outage each time |
| RocksDB's checkpoint (`rocksdb::checkpoint::Checkpoint`, rocksdb 0.25) | A consistent copy of every column family at once, by hard links on the same filesystem |
| This server's blob writes and purge (`crates/store/src/write/blob.rs`, `crates/store/src/backend/fs/mod.rs`) | Why a snapshot of the data store, then of the blobs, holds every blob its data links to |
| The store check (`features/store-check.md`) | How the object, its permission and its run are built |
Upstream's open code has nothing like it, and nothing here comes from
Enterprise code.
## What it is
An administrator asks the server for a snapshot of its store. The server
writes a consistent copy into a directory on its own machine, while mail
keeps flowing, and says when it's done. A backup tool then copies that
directory however it likes. Encryption, off-site copies and how long
backups are kept stay with the backup tool. The server keeps only its
latest two snapshots.
A snapshot is a cold copy that was taken hot. Restoring one is copying it
back while the server is stopped.
## Requirements
- **SN-1.** A snapshot is an `inbuxa:StoreSnapshot` under `urn:inbuxa:jmap`.
It needs RocksDB as the data store. With any other data store a create
is refused, and the refusal says to use the database's own backup:
PostgreSQL's or MySQL's, or FoundationDB's.
While a change of blob store is unresolved, a create is refused too:
some blobs may still be only in the old store, which the snapshot
leaves out (blob-store-change spec, BM-8).
- **SN-2.** What it holds:
- **The data store**: a RocksDB checkpoint of every column family, taken
at one point in time. Blobs kept in the data store (the default) are
in it too.
- **Blobs in a separate filesystem blob store**, copied after the
checkpoint. Blobs are written before anything links to them, and the
blob purge is held while the snapshot is taken (SN-4), so every blob
the checkpoint links to is there to copy. A blob written after the
checkpoint may be copied as well; nothing in the snapshot links to it.
A blob directory that holds the data store's directory is fine: the
copy leaves the data store's directory out. Blobs in the data store's
own directory are refused, since they can't be told from its files.
Any other blob store (S3, Azure, a sharded set, a SQL database) is
refused: a copy of the data store alone would point at blobs a later
purge could take away before the backup tool copied them.
Not in it: the search index when it's kept outside the data store
(Meilisearch, Elasticsearch, ...), rebuilt after a restore; and the
in-memory store (rate limits, locks), which a restarted server doesn't
need.
- **SN-3.** Where: under `snapshots/` in the RocksDB directory, one
directory per snapshot named for when it was taken (UTC), like
`snapshots/20261109T013000Z/`, with `data/`, `blobs/` when there are
separate blobs, and `snapshot.json`. Being on the store's own
filesystem, RocksDB and the blob copy hard-link their files, so a snapshot
is quick and at first takes almost no space. Where they can't link
(another filesystem), they copy. The location is fixed: a request can't
name a path.
A snapshot is written as `snapshots/.<name>.partial/` and renamed when
complete, so a directory with the plain name is always whole.
`snapshots/latest` is a link to the newest complete one.
- **SN-4.** While a snapshot takes blobs from a filesystem blob store, the
blob purge waits. A purge already running is let finish first. A purge
task that comes due meanwhile is put off by five minutes.
- **SN-5.** `snapshot.json` says when it was taken and finished, by whom,
the server version, what it holds (`data`, `blobs`), the data and blob
stores' paths, its size, and how to restore it.
- **SN-6.** One snapshot is made at a time; another create is refused
while one runs. When one completes, the oldest beyond the latest two are
deleted.
- **SN-7.** A snapshot is `running`, then `done`, or `failed` with the
error and its partial directory removed. One the server stopped in the
middle of leaves a partial directory, removed when the server next
starts, and isn't listed.
The complete snapshots on disk are the record: `get` lists those, read
from each one's `snapshot.json`, and, until the server restarts, the one
being taken or the last one that failed.
- **SN-8.** Only administrators take snapshots: `sysStoreSnapshot`, in the
built-in Administrator role and granted to existing administrator roles.
A snapshot holds every account's data, so a tenant administrator is
refused. Taking and destroying a snapshot are audited.
- **SN-9.** Destroying a snapshot deletes its directory. A running one
can't be destroyed.
- **SN-10.** A filesystem blob store writes each blob to a temporary file
in the same directory and renames it into place. Before, it rewrote a
file in place, which would also have changed the copy a snapshot holds
by hard link, and left a half-written blob behind if the server stopped
mid-write.
## The object
| Property | Type | |
|---|---|---|
| `id` | Id | |
| `status` | String | `running`, `done`, `failed` |
| `createdAt`, `finishedAt` | UTCDate | |
| `createdBy` | String | Who took it |
| `node` | String | The node that took it |
| `name` | String | Its directory under `snapshots/` |
| `path` | String | Its full path on the server's machine |
| `holds` | String[] | `data`, and `blobs` when they're separate |
| `size` | UnsignedInt | Bytes in its files |
| `blobFiles` | UnsignedInt | Blob files copied |
| `error` | String | Why a `failed` snapshot failed |
A create takes no properties.
## Restoring
1. Stop the server.
2. Move the RocksDB directory's contents aside, except `snapshots/`, and
copy the snapshot's `data/` into it.
3. With a separate filesystem blob store, do the same with its directory
and the snapshot's `blobs/`.
4. Start the server. A search index kept outside the data store isn't in
the snapshot; reindex the accounts once it's running.
A backup tool copying the store cold should leave out `snapshots/`, or
the store is copied twice.
## Tests
- `system::store_snapshot` (`tests/src/system/store_snapshot.rs`), against
a real server with RocksDB, run with blobs in the data store and again
with `BLOB_STORE=FileSystem`: a snapshot taken while mail is in the
store; the directory, its name, `latest` and the manifest; a RocksDB
store opened from its `data/` has the messages and their blobs (in it,
or in `blobs/`), and not a message that came after; no properties at
create, no updates; with separate blobs, one at a time and no destroy
while one is taken (the test holds the purge gate); only the latest two
kept; destroy; the audit events; a user's and a tenant administrator's
refusal.
- Unit tests: the filesystem blob store's writes (SN-10), the tree copy
and what it leaves out, the purge gate, snapshot names and the manifest.
## Not yet
- The console's page and an `inbuxa-cli` command a backup tool can call
and wait on.
- SQLite as the data store (`VACUUM INTO` would do it).
+59 -1
View File
@@ -20,7 +20,7 @@
[vocabulary]
categories = ["identifier", "contact", "network", "content", "metadata", "credential"]
whose = ["holder", "correspondent", "administrator"]
where = ["data-store", "blob-store", "search-store", "in-memory-store", "memory", "log-file", "external"]
where = ["data-store", "blob-store", "search-store", "in-memory-store", "memory", "log-file", "data-directory", "external"]
scope = ["tenant", "server"]
retention = ["unbounded", "object-life", "receiver"]
@@ -238,6 +238,38 @@ reason = ["content"]
createdBy = ["identifier"]
blobId = ["content", "identifier", "contact", "metadata"]
[object."inbuxa:StoreCheck"]
file = "inbuxa_store_check.rs"
default = "none"
whose = ["holder", "administrator"]
where = ["data-store"]
scope = "server"
retention = "object-life"
[object."inbuxa:StoreCheck".properties]
createdBy = ["identifier"]
problems = ["identifier", "metadata"]
[object."inbuxa:BlobStoreChange"]
file = "inbuxa_blob_store_change.rs"
default = "none"
whose = ["holder", "administrator"]
where = ["data-store"]
scope = "server"
retention = "object-life"
[object."inbuxa:BlobStoreChange".properties]
copyStartedBy = ["identifier"]
problems = ["identifier", "metadata"]
[object."inbuxa:StoreSnapshot"]
file = "inbuxa_store_snapshot.rs"
default = "none"
whose = ["administrator"]
where = ["data-directory"]
scope = "server"
retention = "object-life"
[object."inbuxa:StoreSnapshot".properties]
createdBy = ["identifier"]
[object."inbuxa:DeletedAccount"]
file = "inbuxa_deleted_account.rs"
default = "none"
@@ -351,6 +383,32 @@ retention = "receiver"
leaves_host = false
written_by = ["crates/services/src/inbuxa_scheduled_reports.rs"]
# A store snapshot: a copy of the whole data store, and of a filesystem
# blob store's files, under snapshots/ in the RocksDB directory. The latest
# two are kept; a backup tool takes them off the host if the administrator
# has one do so (store-snapshot spec, SN-3, SN-6).
[source."store-snapshots"]
categories = ["content", "identifier", "contact", "network", "metadata", "credential"]
whose = ["holder", "correspondent", "administrator"]
where = ["data-directory"]
scope = "server"
retention = "object-life"
leaves_host = false
written_by = ["crates/services/src/inbuxa_store_snapshot.rs"]
# A change of blob store keeps the old store's settings, credentials
# included, until it's resolved or dismissed, so the old store can be read
# and copied from. Never returned by get (blob-store-change spec, BM-2,
# BM-7, BM-9).
[source."blob-store-change-settings"]
categories = ["credential", "network"]
whose = ["administrator"]
where = ["data-store"]
scope = "server"
retention = "object-life"
leaves_host = false
written_by = ["crates/features/src/blob_move/mod.rs"]
[source."full-text-index"]
categories = ["content", "identifier", "contact", "metadata"]
whose = ["holder", "correspondent"]
Binary file not shown.
+1 -1
View File
@@ -1 +1 @@
sumxHhh5m_-OICu3KBWCIwgVgSpXxw-jcNAvXt63ATc
mF1m1A-76ThstOP9qsGUtzCS3tgqAF0rKBrla9jY5xI
+434
View File
@@ -0,0 +1,434 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! `inbuxa:BlobStoreChange` (docs/spec/features/blob-store-change.md).
//!
//! The forum topic 238 case: some mail went to a filesystem blob store,
//! then the setting went back to the data store. The test records that
//! change as a start-up would and gives the server the read-through store
//! it would build (a test server can't restart with another setting). One
//! message's blob is only in the old store, one is damaged there, one is
//! in neither. The first reads through and is kept; a copy copies the
//! rest it can, reports the damaged and the missing one, and leaves the
//! change unresolved. Put right, a second copy resolves it, and nothing
//! is read through any more. A slow copy stops when asked and can't be
//! dismissed while it runs. Nobody but an administrator reaches changes.
use crate::utils::{
account::Account,
server::{TestServer, TestServerBuilder},
};
use email::{mailbox::INBOX_ID, message::metadata::MessageMetadata};
use inbuxa_features::blob_move::{self as model, Started};
use registry::schema::{enums::CompressionAlgo, structs};
use serde_json::{Value, json};
use std::{str::FromStr, sync::Arc, time::Duration};
use store::{
BlobStore, ValueKey,
backend::fs::FsStore,
build::blob::blob_store_location,
write::{
AlignedBytes, Archive,
blob_move::{MovingBlobStore, OldBlobStore},
now,
},
};
use types::{blob_hash::BlobHash, collection::Collection, field::EmailField, id::Id};
const USING: &[&str] = &[
"urn:ietf:params:jmap:core",
"urn:inbuxa:jmap",
"urn:inbuxa:jmap:registry",
];
async fn call(account: &Account, method: &str, mut arguments: Value) -> (String, Value) {
if arguments.get("accountId").is_none() {
arguments["accountId"] = account.id_string().into();
}
let response = account
.jmap_request(USING, json!([[method, arguments, "0"]]))
.await;
let call = response
.0
.pointer("/methodResponses/0")
.cloned()
.unwrap_or_else(|| panic!("{method}: {}", response.0));
(
call[0].as_str().unwrap_or_default().to_string(),
call[1].clone(),
)
}
async fn get(admin: &Account, id: &str) -> Value {
let (_, response) = call(admin, "inbuxa:BlobStoreChange/get", json!({"ids": [id]})).await;
response["list"][0].clone()
}
async fn update(admin: &Account, id: &str, patch: Value) -> Value {
let (_, response) = call(
admin,
"inbuxa:BlobStoreChange/set",
json!({"update": {id: patch}}),
)
.await;
response
}
/// The change once its copy is over.
async fn settled(admin: &Account, id: &str) -> Value {
for _ in 0..900 {
let change = get(admin, id).await;
if change["status"] != "copying" {
return change;
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
panic!("the copy didn't finish: {}", get(admin, id).await);
}
async fn import(client: &jmap_client::client::Client, body: &str) -> u32 {
let id = client
.email_import(
format!("From: [email protected]\r\nSubject: {body}\r\n\r\n{body}\r\n").into_bytes(),
[Id::from(INBOX_ID).to_string()],
None::<Vec<&str>>,
None,
)
.await
.unwrap()
.take_id();
Id::from_str(&id).unwrap().document_id()
}
async fn blob_of(test: &TestServer, account_id: u32, document_id: u32) -> BlobHash {
let archive = test
.server
.store()
.get_value::<Archive<AlignedBytes>>(ValueKey::property(
account_id,
Collection::Email,
document_id,
EmailField::Metadata,
))
.await
.unwrap()
.expect("the message's metadata");
let metadata = archive.unarchive::<MessageMetadata>().unwrap();
BlobHash::from(&metadata.blob_hash)
}
fn count(change: &Value, kind: &str) -> u64 {
change[kind]
.as_u64()
.unwrap_or_else(|| panic!("no {kind}: {change}"))
}
/// Gives the server `blob` as its blob store, as a start-up would.
fn use_blob_store(test: &mut TestServer, blob: BlobStore) {
let mut core = (*test.server.inner.shared_core.load_full()).clone();
core.storage.blob = blob;
test.server.inner.shared_core.store(Arc::new(core));
test.reload_core();
}
/// Moves a message's blob from the current store to `old`, or puts
/// `content` there in its place.
async fn leave_behind(
test: &TestServer,
old: &BlobStore,
hash: &BlobHash,
content: Option<&[u8]>,
) -> Vec<u8> {
let current = test.server.blob_store().current().clone();
let original = current
.get_blob(hash.as_slice(), 0..usize::MAX)
.await
.unwrap()
.expect("the blob");
old.put_blob(
hash.as_slice(),
content.unwrap_or(&original),
CompressionAlgo::Lz4,
)
.await
.unwrap();
current.delete_blob(hash.as_slice()).await.unwrap();
original
}
pub async fn test(test: &mut TestServer) {
println!("Running blob store change tests...");
let data = test.server.store().clone();
// A start with blobs in a directory, then one with the setting back
let dir = format!("{}/old-blobs", test.tmp_dir());
let fs_settings = structs::FileSystemStore {
path: dir.clone(),
depth: 2,
};
let old_settings = structs::BlobStore::FileSystem(fs_settings.clone());
let old = FsStore::open(fs_settings).await.unwrap();
let here = structs::BlobStore::Default;
model::at_start(
&data,
blob_store_location(&old_settings),
&old_settings,
1,
now(),
)
.await
.unwrap();
let Started::Changed(change_id) =
model::at_start(&data, blob_store_location(&here), &here, 1, now())
.await
.unwrap()
else {
panic!("no change recorded");
};
let id = Id::from(change_id).to_string();
// The store a start-up would build: reading through to the old one
let current = test.server.blob_store().clone();
use_blob_store(
test,
BlobStore::Moving(Arc::new(MovingBlobStore::new(
current.clone(),
vec![OldBlobStore {
change_id,
store: old.clone(),
}],
))),
);
let admin = test.account("[email protected]");
let ann = test.account("[email protected]");
let ann_id = ann.id().document_id();
let client = ann.jmap_client().await;
let behind = import(&client, "Delivered while the filesystem store was set").await;
let damaged = import(&client, "Damaged in the old store").await;
let gone = import(&client, "In neither store").await;
let fine = import(&client, "Never moved").await;
let behind_hash = blob_of(test, ann_id, behind).await;
let damaged_hash = blob_of(test, ann_id, damaged).await;
let gone_hash = blob_of(test, ann_id, gone).await;
leave_behind(test, &old, &behind_hash, None).await;
let damaged_content =
leave_behind(test, &old, &damaged_hash, Some(b"Something else entirely")).await;
let gone_content = current
.get_blob(gone_hash.as_slice(), 0..usize::MAX)
.await
.unwrap()
.unwrap();
current.delete_blob(gone_hash.as_slice()).await.unwrap();
// Listed, with where from and to and never the old store's settings
let change = get(admin, &id).await;
assert_eq!(change["status"], "pending", "{change}");
assert_eq!(
change["from"],
blob_store_location(&old_settings),
"{change}"
);
assert_eq!(change["to"], "the data store", "{change}");
assert!(!change.to_string().contains("oldSettings"), "{change}");
assert!(!change.to_string().contains("FileSystem"), "{change}");
// BM-3: left behind, read through, and kept
assert!(!current.blob_present(behind_hash.as_slice()).await.unwrap());
let message = test.fetch_email(ann_id, behind).await;
assert!(
String::from_utf8_lossy(&message).contains("Delivered while the filesystem store was set"),
"read through the old store"
);
assert!(
current.blob_present(behind_hash.as_slice()).await.unwrap(),
"kept in the current store"
);
assert!(
old.blob_present(behind_hash.as_slice()).await.unwrap(),
"nothing deleted from the old store"
);
// Refusals
let (_, response) = call(
admin,
"inbuxa:BlobStoreChange/set",
json!({"create": {"x": {}}}),
)
.await;
assert_eq!(
response["notCreated"]["x"]["type"], "forbidden",
"{response}"
);
let response = update(admin, &id, json!({"action": "rename"})).await;
assert_eq!(
response["notUpdated"][&id]["type"], "invalidProperties",
"{response}"
);
let response = update(admin, &id, json!({"from": "elsewhere", "action": "copy"})).await;
assert_eq!(
response["notUpdated"][&id]["type"], "invalidProperties",
"{response}"
);
let response = update(admin, &id, json!({"action": "stop"})).await;
assert_eq!(
response["notUpdated"][&id]["type"], "forbidden",
"{response}"
);
let (name, response) = call(ann, "inbuxa:BlobStoreChange/get", json!({"ids": null})).await;
assert_eq!(name, "error", "a user reaches changes: {response}");
// A slow copy stops when asked, and can't be dismissed meanwhile
for n in 0..15 {
import(&client, &format!("Filler {n}")).await;
}
let response = update(admin, &id, json!({"action": "copy", "blobsPerSecond": 1})).await;
assert!(response["updated"].get(&id).is_some(), "{response}");
let response = update(admin, &id, json!({"action": "copy"})).await;
assert_eq!(
response["notUpdated"][&id]["type"], "forbidden",
"one copy at a time: {response}"
);
let (_, response) = call(
admin,
"inbuxa:BlobStoreChange/set",
json!({"destroy": [id]}),
)
.await;
assert_eq!(
response["notDestroyed"][&id]["type"], "forbidden",
"{response}"
);
let response = update(admin, &id, json!({"action": "stop"})).await;
assert!(response["updated"].get(&id).is_some(), "{response}");
let change = settled(admin, &id).await;
assert_eq!(change["status"], "stopped", "{change}");
assert_eq!(change["copyStartedBy"], "[email protected]", "{change}");
// A copy: the damaged and the missing are reported, not copied
let response = update(admin, &id, json!({"action": "copy", "blobsPerSecond": 0})).await;
assert!(response["updated"].get(&id).is_some(), "{response}");
let change = settled(admin, &id).await;
assert_eq!(change["status"], "incomplete", "{change}");
assert_eq!(count(&change, "missing"), 1, "{change}");
assert_eq!(count(&change, "damaged"), 1, "{change}");
assert_eq!(count(&change, "unreadable"), 0, "{change}");
assert!(count(&change, "alreadyThere") >= 2, "{change}");
let problems = change["problems"].as_array().unwrap().clone();
let of = |hash: &BlobHash| {
problems
.iter()
.find(|p| p["blobId"] == hash.to_hex())
.cloned()
.unwrap_or_else(|| panic!("no problem for {}: {change}", hash.to_hex()))
};
let missing = of(&gone_hash);
assert_eq!(missing["kind"], "missing", "{missing}");
assert_eq!(missing["holder"], "email", "{missing}");
assert_eq!(missing["accountId"], ann.id_string(), "{missing}");
assert_eq!(missing["objectId"], gone as u64, "{missing}");
assert_eq!(of(&damaged_hash)["kind"], "damaged");
assert!(
!current.blob_present(damaged_hash.as_slice()).await.unwrap(),
"a damaged blob isn't copied"
);
assert!(
test.server
.blob_store()
.moving()
.is_some_and(|m| !m.old_stores().is_empty()),
"still read through"
);
// Put right in the old store: a second copy resolves the change
old.put_blob(
damaged_hash.as_slice(),
&damaged_content,
CompressionAlgo::Lz4,
)
.await
.unwrap();
old.put_blob(gone_hash.as_slice(), &gone_content, CompressionAlgo::None)
.await
.unwrap();
let response = update(admin, &id, json!({"action": "copy", "blobsPerSecond": 0})).await;
assert!(response["updated"].get(&id).is_some(), "{response}");
let change = settled(admin, &id).await;
assert_eq!(change["status"], "copied", "{change}");
assert_eq!(count(&change, "copied"), 2, "{change}");
assert_eq!(change["progress"], 1.0, "{change}");
assert_eq!(change["problems"], json!([]), "{change}");
for hash in [&damaged_hash, &gone_hash] {
assert!(current.blob_present(hash.as_slice()).await.unwrap());
}
assert!(
test.server
.blob_store()
.moving()
.unwrap()
.old_stores()
.is_empty(),
"nothing read through once resolved"
);
assert!(String::from_utf8_lossy(&test.fetch_email(ann_id, fine).await).contains("Never moved"));
assert!(
model::get(&data, change_id)
.await
.unwrap()
.unwrap()
.old_settings
.is_none(),
"the old store's settings go once resolved"
);
let response = update(admin, &id, json!({"action": "copy"})).await;
assert_eq!(
response["notUpdated"][&id]["type"], "forbidden",
"nothing left to copy: {response}"
);
// Dismissed
let (_, response) = call(
admin,
"inbuxa:BlobStoreChange/set",
json!({"destroy": [id]}),
)
.await;
assert_eq!(response["destroyed"], json!([id]), "{response}");
let (_, response) = call(admin, "inbuxa:BlobStoreChange/get", json!({"ids": [id]})).await;
assert_eq!(response["notFound"], json!([id]), "{response}");
use_blob_store(test, current);
}
/// Runs these tests alone: `cargo test -p tests blob_store_change_tests -- --ignored`.
#[ignore]
#[tokio::test(flavor = "multi_thread")]
pub async fn blob_store_change_tests() {
let mut test = TestServerBuilder::new("blob_store_change_tests")
.await
.with_default_listeners()
.await
.build()
.await;
let admin = test.create_admin_account("[email protected]").await;
test.insert_account(admin);
let ann = test
.create_user_account(
"[email protected]",
"[email protected]",
"ann-secret-40917",
&[],
"Ann",
)
.await;
test.insert_account(ann);
self::test(&mut test).await;
if test.is_reset() {
test.temp_dir.delete();
}
}
+3
View File
@@ -15,6 +15,7 @@ pub mod audit; // inbuxa: the audit log
pub mod authentication;
pub mod authorization;
pub mod auto_reload; // inbuxa: registry writes apply at once
pub mod blob_store_change; // inbuxa: a change of blob store
pub mod branding;
pub mod compliance; // inbuxa: the compliance roles
pub mod crypto;
@@ -36,6 +37,8 @@ pub mod scheduled_reports; // inbuxa: scheduled reports and the weekly digest
pub mod security;
pub mod security_acceptances; // inbuxa: accepted security to-do items
pub mod sharing_policy; // inbuxa: MA-C, who may share mail
pub mod store_check; // inbuxa: checking the store's blobs
pub mod store_snapshot; // inbuxa: a snapshot of the store
pub mod task;
pub mod tenant;
pub mod tracer_reload; // inbuxa: tracers start over when their settings change
+421
View File
@@ -0,0 +1,421 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! `inbuxa:StoreCheck` (docs/spec/features/store-check.md).
//!
//! A clean store checks clean. Then blobs are taken from under messages:
//! one deleted with its commit marker left, one shared by two messages, one
//! overwritten with other content, and one message's link taken away while
//! its blob stays. A quick check finds the missing ones and the unlinked
//! one, each with the account and the message; a full check also finds the
//! damaged one. One check runs at a time, a running one stops when asked
//! and can't be destroyed, a finished one can, and nobody in a tenant or
//! without the permission reaches checks. Starting and stopping are
//! audited.
use crate::utils::{
account::Account,
server::{TestServer, TestServerBuilder},
};
use email::{mailbox::INBOX_ID, message::metadata::MessageMetadata};
use registry::schema::{
enums::CompressionAlgo,
prelude::{ObjectType, Property},
structs::{CertificateManagement, DkimManagement, DnsManagement, Domain, Tenant, UserRoles},
};
use serde_json::{Value, json};
use std::{str::FromStr, time::Duration};
use store::{
ValueKey,
write::{AlignedBytes, Archive, BatchBuilder, BlobLink, BlobOp},
};
use types::{blob_hash::BlobHash, collection::Collection, field::EmailField, id::Id};
const USING: &[&str] = &[
"urn:ietf:params:jmap:core",
"urn:inbuxa:jmap",
"urn:inbuxa:jmap:registry",
];
async fn call(account: &Account, method: &str, mut arguments: Value) -> (String, Value) {
if arguments.get("accountId").is_none() {
arguments["accountId"] = account.id_string().into();
}
let response = account
.jmap_request(USING, json!([[method, arguments, "0"]]))
.await;
let call = response
.0
.pointer("/methodResponses/0")
.cloned()
.unwrap_or_else(|| panic!("{method}: {}", response.0));
(
call[0].as_str().unwrap_or_default().to_string(),
call[1].clone(),
)
}
async fn start(admin: &Account, check: Value) -> String {
let (_, response) = call(
admin,
"inbuxa:StoreCheck/set",
json!({"create": {"c": check}}),
)
.await;
response["created"]["c"]["id"]
.as_str()
.unwrap_or_else(|| panic!("not started: {response}"))
.to_string()
}
async fn get(admin: &Account, id: &str) -> Value {
let (_, response) = call(admin, "inbuxa:StoreCheck/get", json!({"ids": [id]})).await;
response["list"][0].clone()
}
/// The check once it's over.
async fn settled(admin: &Account, id: &str) -> Value {
for _ in 0..600 {
let check = get(admin, id).await;
if check["status"] != "running" {
return check;
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
panic!("the check didn't finish: {}", get(admin, id).await);
}
async fn import(client: &jmap_client::client::Client, body: &str) -> u32 {
let id = client
.email_import(
format!("From: [email protected]\r\nSubject: {body}\r\n\r\n{body}\r\n").into_bytes(),
[Id::from(INBOX_ID).to_string()],
None::<Vec<&str>>,
None,
)
.await
.unwrap()
.take_id();
Id::from_str(&id).unwrap().document_id()
}
async fn blob_of(test: &TestServer, account_id: u32, document_id: u32) -> BlobHash {
let archive = test
.server
.store()
.get_value::<Archive<AlignedBytes>>(ValueKey::property(
account_id,
Collection::Email,
document_id,
EmailField::Metadata,
))
.await
.unwrap()
.expect("the message's metadata");
let metadata = archive.unarchive::<MessageMetadata>().unwrap();
BlobHash::from(&metadata.blob_hash)
}
fn count(check: &Value, kind: &str) -> u64 {
check[kind]
.as_u64()
.unwrap_or_else(|| panic!("no {kind}: {check}"))
}
/// The problems found for one message, as `kind` strings.
fn kinds_for(check: &Value, document_id: u32) -> Vec<String> {
check["problems"]
.as_array()
.unwrap_or_else(|| panic!("no problems list: {check}"))
.iter()
.filter(|p| p["objectId"].as_u64() == Some(document_id as u64))
.map(|p| p["kind"].as_str().unwrap_or_default().to_string())
.collect()
}
pub async fn test(test: &mut TestServer) {
println!("Running store check tests...");
let admin = test.account("[email protected]");
let ann = test.account("[email protected]");
let ann_id = ann.id().document_id();
let client = ann.jmap_client().await;
let missing = import(&client, "Missing blob").await;
let shared_a = import(&client, "Shared blob").await;
let shared_b = import(&client, "Shared blob").await;
let damaged = import(&client, "Damaged blob").await;
let unlinked = import(&client, "Unlinked message").await;
let fine = import(&client, "Nothing wrong").await;
assert_eq!(
blob_of(test, ann_id, shared_a).await,
blob_of(test, ann_id, shared_b).await,
"the same message twice shares its blob"
);
// A clean store checks clean
let id = start(admin, json!({"mode": "quick"})).await;
let check = settled(admin, &id).await;
assert_eq!(check["status"], "done", "{check}");
assert_eq!(check["progress"], 1.0, "{check}");
for kind in ["missing", "damaged", "unlinked", "unreadable"] {
assert_eq!(count(&check, kind), 0, "{kind} in a clean store: {check}");
}
assert!(count(&check, "blobsChecked") >= 5, "{check}");
assert!(count(&check, "messagesChecked") >= 6, "{check}");
assert_eq!(check["problems"], json!([]), "{check}");
assert_eq!(check["mode"], "quick", "{check}");
assert_eq!(check["createdBy"], "[email protected]", "{check}");
// Take blobs from under them
let blobs = test.server.blob_store();
blobs
.delete_blob(blob_of(test, ann_id, missing).await.as_slice())
.await
.unwrap();
blobs
.delete_blob(blob_of(test, ann_id, shared_a).await.as_slice())
.await
.unwrap();
blobs
.put_blob(
blob_of(test, ann_id, damaged).await.as_slice(),
b"Something else entirely",
CompressionAlgo::None,
)
.await
.unwrap();
let mut batch = BatchBuilder::new();
batch
.with_account_id(ann_id)
.with_collection(Collection::Email)
.with_document(unlinked)
.clear(BlobOp::Link {
hash: blob_of(test, ann_id, unlinked).await,
to: BlobLink::Document,
});
test.server.store().write(batch.build_all()).await.unwrap();
// Quick: what's gone, and the message without its link
let id = start(admin, json!({"mode": "quick"})).await;
let check = settled(admin, &id).await;
assert_eq!(check["status"], "done", "{check}");
assert_eq!(count(&check, "missing"), 3, "{check}");
assert_eq!(
count(&check, "damaged"),
0,
"quick doesn't read content: {check}"
);
assert_eq!(count(&check, "unlinked"), 1, "{check}");
assert_eq!(kinds_for(&check, missing), ["missing"], "{check}");
assert_eq!(kinds_for(&check, shared_a), ["missing"], "{check}");
assert_eq!(kinds_for(&check, shared_b), ["missing"], "{check}");
assert_eq!(kinds_for(&check, unlinked), ["unlinked"], "{check}");
assert!(kinds_for(&check, fine).is_empty(), "{check}");
let problem = check["problems"]
.as_array()
.unwrap()
.iter()
.find(|p| p["objectId"].as_u64() == Some(missing as u64))
.unwrap()
.clone();
assert_eq!(problem["object"], "email", "{problem}");
assert_eq!(problem["accountId"], ann.id_string(), "{problem}");
assert_eq!(problem["accountName"], "[email protected]", "{problem}");
assert_eq!(
problem["committed"], true,
"lost after it was written: {problem}"
);
assert_eq!(
problem["blobId"],
blob_of(test, ann_id, missing).await.to_hex(),
"{problem}"
);
let unlinked_problem = check["problems"]
.as_array()
.unwrap()
.iter()
.find(|p| p["objectId"].as_u64() == Some(unlinked as u64))
.unwrap()
.clone();
assert_eq!(
unlinked_problem["detail"], "the blob is still there",
"{unlinked_problem}"
);
// Full: the overwritten one too
let id = start(admin, json!({"mode": "full", "bytesPerSecond": 0})).await;
let check = settled(admin, &id).await;
assert_eq!(check["status"], "done", "{check}");
assert_eq!(count(&check, "missing"), 3, "{check}");
assert_eq!(count(&check, "damaged"), 1, "{check}");
assert_eq!(count(&check, "unlinked"), 1, "{check}");
assert_eq!(kinds_for(&check, damaged), ["damaged"], "{check}");
assert!(count(&check, "bytesRead") > 0, "{check}");
let full = id;
// One at a time; a running one stops when asked, and stays until then
let slow = start(admin, json!({"mode": "quick", "blobsPerSecond": 1})).await;
let (_, response) = call(
admin,
"inbuxa:StoreCheck/set",
json!({"create": {"again": {"mode": "quick"}}}),
)
.await;
assert!(
response["notCreated"]["again"]["description"]
.as_str()
.is_some_and(|d| d.contains("one check runs at a time")),
"a second check while one runs: {response}"
);
let (_, response) = call(admin, "inbuxa:StoreCheck/set", json!({"destroy": [&slow]})).await;
assert!(response["notDestroyed"].get(&slow).is_some(), "{response}");
let (_, response) = call(
admin,
"inbuxa:StoreCheck/set",
json!({"update": {&slow: {"mode": "full"}}}),
)
.await;
assert!(response["notUpdated"].get(&slow).is_some(), "{response}");
let (_, response) = call(
admin,
"inbuxa:StoreCheck/set",
json!({"update": {&slow: {"action": "stop"}}}),
)
.await;
assert!(response["updated"].get(&slow).is_some(), "{response}");
let check = settled(admin, &slow).await;
assert_eq!(check["status"], "stopped", "{check}");
assert!(check["progress"].as_f64().unwrap() < 1.0, "{check}");
let (_, response) = call(
admin,
"inbuxa:StoreCheck/set",
json!({"update": {&slow: {"action": "stop"}}}),
)
.await;
assert!(
response["notUpdated"].get(&slow).is_some(),
"stopping a stopped check: {response}"
);
// A finished one can be forgotten, with its problems
let (_, response) = call(admin, "inbuxa:StoreCheck/set", json!({"destroy": [&full]})).await;
assert_eq!(response["destroyed"], json!([full]), "{response}");
let (_, response) = call(admin, "inbuxa:StoreCheck/get", json!({"ids": [&full]})).await;
assert_eq!(response["notFound"], json!([full]), "{response}");
let id = Id::from_str(&full).unwrap().id() as u32;
assert!(
inbuxa_features::store_check::problems(test.server.store(), id)
.await
.unwrap()
.is_empty(),
"a forgotten check's problems are gone too"
);
// Bad input
for (create, why) in [
(json!({"mode": "thorough"}), "mode"),
(json!({"blobsPerSecond": -1}), "blobsPerSecond"),
(json!({"status": "done"}), "status"),
] {
let (_, response) = call(
admin,
"inbuxa:StoreCheck/set",
json!({"create": {"x": create}}),
)
.await;
assert!(
response["notCreated"].get("x").is_some(),
"{why}: {response}"
);
}
// Starting and stopping are audited
let (_, response) = call(
admin,
"inbuxa:AuditEvent/query",
json!({"filter": {"targetKind": "inbuxa:StoreCheck"}}),
)
.await;
assert!(
response["ids"].as_array().is_some_and(|ids| ids.len() >= 5),
"creates, the stop and the destroy audited: {response}"
);
// Nobody without the permission, or in a tenant, reaches checks
let (name, response) = call(ann, "inbuxa:StoreCheck/get", json!({"ids": null})).await;
assert_eq!(name, "error", "a user: {response}");
let tenant = admin
.registry_create_object(Tenant {
name: "Store check tenant".to_string(),
..Default::default()
})
.await;
admin
.registry_create_object(Domain {
name: "storecheck.example.org".to_string(),
is_enabled: true,
member_tenant_id: Some(tenant),
certificate_management: CertificateManagement::Manual,
dns_management: DnsManagement::Manual,
dkim_management: DkimManagement::Manual,
..Default::default()
})
.await;
let t_admin = admin
.create_user_account(
"[email protected]",
"tenant-admin-secret-4417",
"Tenant admin",
&[],
vec![],
)
.await;
admin
.registry_update_object(
ObjectType::Account,
t_admin.id(),
json!({Property::Roles: UserRoles::Admin}),
)
.await;
for (method, arguments) in [
("inbuxa:StoreCheck/get", json!({"ids": null})),
(
"inbuxa:StoreCheck/set",
json!({"create": {"x": {"mode": "quick"}}}),
),
] {
let (name, response) = call(&t_admin, method, arguments).await;
assert_eq!(name, "error", "{method} by a tenant: {response}");
}
}
/// Runs these tests alone: `cargo test -p tests store_check_tests -- --ignored`.
#[ignore]
#[tokio::test(flavor = "multi_thread")]
pub async fn store_check_tests() {
let mut test = TestServerBuilder::new("store_check_tests")
.await
.with_default_listeners()
.await
.build()
.await;
let admin = test.create_admin_account("[email protected]").await;
test.insert_account(admin);
let ann = test
.create_user_account(
"[email protected]",
"[email protected]",
"ann-secret-29871",
&[],
"Ann",
)
.await;
test.insert_account(ann);
self::test(&mut test).await;
if test.is_reset() {
test.temp_dir.delete();
}
}
+411
View File
@@ -0,0 +1,411 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! `inbuxa:StoreSnapshot` (docs/spec/features/store-snapshot.md).
//!
//! A snapshot taken while the server runs holds the mail that was in it and
//! not the mail that came after: a RocksDB store opened from its `data/`
//! has the messages, and their blobs are in it or, with a filesystem blob
//! store (`BLOB_STORE=FileSystem`), in its `blobs/`. The directory is
//! whole, named for its time, with `latest` and `snapshot.json`. Only the
//! latest two are kept; one can be destroyed, a snapshot can't be changed,
//! and one being taken can't be destroyed or joined by another. Nobody in
//! a tenant or without the permission reaches snapshots. Taking and
//! destroying are audited.
use crate::utils::{
account::Account,
server::{TestServer, TestServerBuilder},
};
use email::{mailbox::INBOX_ID, message::metadata::MessageMetadata};
use registry::schema::{
prelude::{ObjectType, Property},
structs::{
self, CertificateManagement, DkimManagement, DnsManagement, Domain, Tenant, UserRoles,
},
};
use serde_json::{Value, json};
use std::{path::PathBuf, str::FromStr, time::Duration};
use store::{
BlobStore, Store, ValueKey,
backend::{fs::FsStore, rocksdb::RocksDbStore},
write::{AlignedBytes, Archive, snapshot::hold_blob_purge},
};
use types::{blob_hash::BlobHash, collection::Collection, field::EmailField, id::Id};
const USING: &[&str] = &[
"urn:ietf:params:jmap:core",
"urn:inbuxa:jmap",
"urn:inbuxa:jmap:registry",
];
async fn call(account: &Account, method: &str, mut arguments: Value) -> (String, Value) {
if arguments.get("accountId").is_none() {
arguments["accountId"] = account.id_string().into();
}
let response = account
.jmap_request(USING, json!([[method, arguments, "0"]]))
.await;
let call = response
.0
.pointer("/methodResponses/0")
.cloned()
.unwrap_or_else(|| panic!("{method}: {}", response.0));
(
call[0].as_str().unwrap_or_default().to_string(),
call[1].clone(),
)
}
async fn take(admin: &Account) -> String {
let (_, response) = call(
admin,
"inbuxa:StoreSnapshot/set",
json!({"create": {"s": {}}}),
)
.await;
response["created"]["s"]["id"]
.as_str()
.unwrap_or_else(|| panic!("not taken: {response}"))
.to_string()
}
async fn get(admin: &Account, id: &str) -> Value {
let (_, response) = call(admin, "inbuxa:StoreSnapshot/get", json!({"ids": [id]})).await;
response["list"][0].clone()
}
async fn all(admin: &Account) -> Vec<Value> {
let (_, response) = call(admin, "inbuxa:StoreSnapshot/get", json!({"ids": null})).await;
response["list"]
.as_array()
.unwrap_or_else(|| panic!("no list: {response}"))
.clone()
}
/// The snapshot once it's taken.
async fn settled(admin: &Account, id: &str) -> Value {
for _ in 0..600 {
let snapshot = get(admin, id).await;
if snapshot["status"] != "running" {
return snapshot;
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
panic!("the snapshot wasn't taken: {}", get(admin, id).await);
}
async fn import(client: &jmap_client::client::Client, body: &str) -> u32 {
let id = client
.email_import(
format!("From: [email protected]\r\nSubject: {body}\r\n\r\n{body}\r\n").into_bytes(),
[Id::from(INBOX_ID).to_string()],
None::<Vec<&str>>,
None,
)
.await
.unwrap()
.take_id();
Id::from_str(&id).unwrap().document_id()
}
async fn blob_in(store: &Store, account_id: u32, document_id: u32) -> Option<BlobHash> {
let archive = store
.get_value::<Archive<AlignedBytes>>(ValueKey::property(
account_id,
Collection::Email,
document_id,
EmailField::Metadata,
))
.await
.unwrap()?;
let metadata = archive.unarchive::<MessageMetadata>().unwrap();
Some(BlobHash::from(&metadata.blob_hash))
}
fn path_of(snapshot: &Value) -> PathBuf {
PathBuf::from(
snapshot["path"]
.as_str()
.unwrap_or_else(|| panic!("no path: {snapshot}")),
)
}
pub async fn test(test: &mut TestServer) {
println!("Running store snapshot tests...");
let admin = test.account("[email protected]");
let ann = test.account("[email protected]");
let ann_id = ann.id().document_id();
let client = ann.jmap_client().await;
let separate_blobs = matches!(test.server.blob_store(), BlobStore::Fs(_));
println!(
"Blobs {}",
if separate_blobs {
"in a filesystem blob store"
} else {
"in the data store"
}
);
let before = [
import(&client, "Before the snapshot, one").await,
import(&client, "Before the snapshot, two").await,
];
assert!(all(admin).await.is_empty(), "no snapshots yet");
// A snapshot while the server runs
let id = take(admin).await;
let snapshot = settled(admin, &id).await;
assert_eq!(snapshot["status"], "done", "{snapshot}");
assert_eq!(snapshot["createdBy"], "[email protected]", "{snapshot}");
let holds = if separate_blobs {
json!(["data", "blobs"])
} else {
json!(["data"])
};
assert_eq!(snapshot["holds"], holds, "{snapshot}");
assert!(snapshot["size"].as_u64().unwrap() > 0, "{snapshot}");
let after = import(&client, "After the snapshot").await;
// The directory: whole, named for its time, `latest`, the manifest
let path = path_of(&snapshot);
let name = snapshot["name"].as_str().unwrap();
assert_eq!(path.file_name().unwrap().to_str().unwrap(), name);
assert!(name.len() == 16 && name.ends_with('Z'), "{name}");
let root = path.parent().unwrap().to_path_buf();
assert_eq!(root.file_name().unwrap(), "snapshots");
assert_eq!(
std::fs::read_link(root.join("latest")).unwrap(),
PathBuf::from(name)
);
assert!(path.join("data/CURRENT").exists(), "a RocksDB checkpoint");
assert_eq!(path.join("blobs").exists(), separate_blobs);
let manifest: Value =
serde_json::from_slice(&std::fs::read(path.join("snapshot.json")).unwrap()).unwrap();
assert_eq!(manifest["id"], json!(id), "{manifest}");
assert_eq!(manifest["holds"], holds, "{manifest}");
assert!(
manifest["restore"].as_array().unwrap().len() >= 3,
"{manifest}"
);
assert!(
std::fs::read_dir(&root).unwrap().all(|e| !e
.unwrap()
.file_name()
.to_string_lossy()
.ends_with(".partial")),
"nothing partial left"
);
// Restored, it has the mail that was there, and not what came after
let restored = RocksDbStore::open(structs::RocksDbStore {
path: path.join("data").to_string_lossy().to_string(),
..Default::default()
})
.await
.unwrap();
let blobs = if separate_blobs {
FsStore::open(structs::FileSystemStore {
path: path.join("blobs").to_string_lossy().to_string(),
..Default::default()
})
.await
.unwrap()
} else {
BlobStore::Store(restored.clone())
};
for (document_id, subject) in before.iter().zip(["one", "two"]) {
let hash = blob_in(&restored, ann_id, *document_id)
.await
.unwrap_or_else(|| panic!("message {document_id} isn't in the snapshot"));
let content = blobs
.get_blob(hash.as_slice(), 0..usize::MAX)
.await
.unwrap()
.unwrap_or_else(|| panic!("the blob of message {document_id} isn't in the snapshot"));
let content = String::from_utf8_lossy(&content);
assert!(content.contains(&format!("Before the snapshot, {subject}")));
}
assert!(
blob_in(&restored, ann_id, after).await.is_none(),
"mail that came after isn't in it"
);
drop(blobs);
drop(restored);
// A snapshot can't be changed, and takes no properties
let (_, response) = call(
admin,
"inbuxa:StoreSnapshot/set",
json!({"update": {&id: {"name": "x"}}}),
)
.await;
assert!(response["notUpdated"].get(&id).is_some(), "{response}");
let (_, response) = call(
admin,
"inbuxa:StoreSnapshot/set",
json!({"create": {"x": {"path": "/tmp/elsewhere"}}}),
)
.await;
assert_eq!(
response["notCreated"]["x"]["type"], "invalidProperties",
"{response}"
);
// One at a time, and one being taken can't be destroyed. With separate
// blobs a snapshot waits for the blob purge, so holding it here keeps
// one being taken for as long as this needs
if separate_blobs {
let hold = hold_blob_purge().await;
let waiting = take(admin).await;
assert_eq!(get(admin, &waiting).await["status"], "running");
let (_, response) = call(
admin,
"inbuxa:StoreSnapshot/set",
json!({"create": {"x": {}}}),
)
.await;
assert_eq!(
response["notCreated"]["x"]["type"], "forbidden",
"{response}"
);
let (_, response) = call(
admin,
"inbuxa:StoreSnapshot/set",
json!({"destroy": [&waiting]}),
)
.await;
assert_eq!(
response["notDestroyed"][&waiting]["type"], "forbidden",
"{response}"
);
drop(hold);
assert_eq!(settled(admin, &waiting).await["status"], "done");
} else {
let next = take(admin).await;
assert_eq!(settled(admin, &next).await["status"], "done");
}
// Only the latest two are kept
let third = take(admin).await;
let third_snapshot = settled(admin, &third).await;
let listed = all(admin).await;
assert_eq!(listed.len(), 2, "{listed:?}");
assert_eq!(listed[0]["id"], json!(third), "newest first: {listed:?}");
assert!(!path.exists(), "the oldest went");
let (_, response) = call(admin, "inbuxa:StoreSnapshot/get", json!({"ids": [&id]})).await;
assert_eq!(response["notFound"], json!([id]), "{response}");
assert_eq!(
std::fs::read_link(root.join("latest")).unwrap(),
PathBuf::from(third_snapshot["name"].as_str().unwrap())
);
// Destroying one deletes it; `latest` moves to the one left
let unknown = Id::from(1u64).to_string();
let (_, response) = call(
admin,
"inbuxa:StoreSnapshot/set",
json!({"destroy": [&third, &unknown]}),
)
.await;
assert_eq!(response["destroyed"], json!([third]), "{response}");
assert_eq!(
response["notDestroyed"][&unknown]["type"], "notFound",
"{response}"
);
assert!(!path_of(&third_snapshot).exists());
let left = all(admin).await;
assert_eq!(left.len(), 1, "{left:?}");
assert_eq!(
std::fs::read_link(root.join("latest")).unwrap(),
PathBuf::from(left[0]["name"].as_str().unwrap())
);
// Taking and destroying are audited
let (_, response) = call(
admin,
"inbuxa:AuditEvent/query",
json!({"filter": {"targetKind": "inbuxa:StoreSnapshot"}}),
)
.await;
assert!(
response["ids"].as_array().is_some_and(|ids| ids.len() >= 4),
"three taken and one destroyed, audited: {response}"
);
// Nobody without the permission, or in a tenant, reaches snapshots
let (name, response) = call(ann, "inbuxa:StoreSnapshot/get", json!({"ids": null})).await;
assert_eq!(name, "error", "a user: {response}");
let tenant = admin
.registry_create_object(Tenant {
name: "Store snapshot tenant".to_string(),
..Default::default()
})
.await;
admin
.registry_create_object(Domain {
name: "storesnapshot.example.org".to_string(),
member_tenant_id: Some(tenant),
certificate_management: CertificateManagement::Manual,
dns_management: DnsManagement::Manual,
dkim_management: DkimManagement::Manual,
..Default::default()
})
.await;
let t_admin = admin
.create_user_account(
"[email protected]",
"tenant-admin-secret-5521",
"Tenant admin",
&[],
vec![],
)
.await;
admin
.registry_update_object(
ObjectType::Account,
t_admin.id(),
json!({Property::Roles: UserRoles::Admin}),
)
.await;
for (method, arguments) in [
("inbuxa:StoreSnapshot/get", json!({"ids": null})),
("inbuxa:StoreSnapshot/set", json!({"create": {"x": {}}})),
] {
let (name, response) = call(&t_admin, method, arguments).await;
assert_eq!(name, "error", "{method} by a tenant: {response}");
}
}
/// Runs these tests alone, with RocksDB:
/// `STORE=RocksDb cargo test -p tests store_snapshot_tests -- --ignored`,
/// and again with `BLOB_STORE=FileSystem` for a separate blob store.
#[ignore]
#[tokio::test(flavor = "multi_thread")]
pub async fn store_snapshot_tests() {
let mut test = TestServerBuilder::new("store_snapshot_tests")
.await
.with_default_listeners()
.await
.build()
.await;
let admin = test.create_admin_account("[email protected]").await;
test.insert_account(admin);
let ann = test
.create_user_account(
"[email protected]",
"[email protected]",
"ann-secret-61734",
&[],
"Ann",
)
.await;
test.insert_account(ann);
self::test(&mut test).await;
if test.is_reset() {
test.temp_dir.delete();
}
}