Backport store check, store snapshot and blob store change (release/2026) #340
No files matched your search
@@ -287,7 +287,11 @@ impl Default for DefaultPermissions {
|
||||
};
|
||||
|
||||
for permission_id in 0..Permission::COUNT {
|
||||
let permission = Permission::from_id(permission_id as u16).unwrap();
|
||||
// inbuxa: ids are shared with the 27.1 series, so this series
|
||||
// has gaps where 27.1 has permissions it doesn't (690-694)
|
||||
let Some(permission) = Permission::from_id(permission_id as u16) else {
|
||||
continue;
|
||||
};
|
||||
match permission {
|
||||
Permission::Authenticate
|
||||
| Permission::AuthenticateWithAlias
|
||||
@@ -366,6 +370,21 @@ impl Default for DefaultPermissions {
|
||||
default.superuser.push(permission);
|
||||
}
|
||||
Permission::SysJournalSearch | Permission::SysJournalExport => {}
|
||||
// inbuxa: store-check spec, SV-9: a check reads every
|
||||
// account's data, so only the server's administrators run one
|
||||
Permission::SysStoreCheck => {
|
||||
default.superuser.push(permission);
|
||||
}
|
||||
// inbuxa: store-snapshot spec, SN-8: a snapshot holds every
|
||||
// account's data, so only the server's administrators take one
|
||||
Permission::SysStoreSnapshot => {
|
||||
default.superuser.push(permission);
|
||||
}
|
||||
// inbuxa: blob-store-change spec, BM-9: a copy reads and
|
||||
// writes every account's blobs
|
||||
Permission::SysBlobStoreChange => {
|
||||
default.superuser.push(permission);
|
||||
}
|
||||
// inbuxa: AL-12: tenant administrators lock and delegate
|
||||
// within their tenant
|
||||
Permission::SysAccountLockGet
|
||||
|
||||
@@ -50,10 +50,15 @@ impl Storage {
|
||||
bp.data_store.share_marks(&memory);
|
||||
}
|
||||
|
||||
// inbuxa: blob-store-change spec, BM-2, BM-3: a blob store that
|
||||
// moved keeps reading what it left behind
|
||||
let blob = BlobStore::build(bp).await.unwrap_or_default();
|
||||
let blob = crate::manager::blob_moves::at_start(bp, blob).await;
|
||||
|
||||
Storage {
|
||||
registry: bp.registry.clone(),
|
||||
data: bp.data_store.clone(),
|
||||
blob: BlobStore::build(bp).await.unwrap_or_default(),
|
||||
blob,
|
||||
search,
|
||||
coordinator,
|
||||
memory,
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! inbuxa: blob-store-change spec (docs/spec/features/blob-store-change.md),
|
||||
//! at start-up: records where the blob store is and, when it moved, the
|
||||
//! change (BM-2), then opens the old store of every unresolved change so
|
||||
//! what the new store lacks is read from it (BM-3).
|
||||
|
||||
use inbuxa_features::blob_move::{self as model, Started};
|
||||
use registry::schema::{prelude::ObjectType, structs};
|
||||
use std::sync::Arc;
|
||||
use store::{
|
||||
BlobStore, Store,
|
||||
build::blob::blob_store_location,
|
||||
registry::bootstrap::Bootstrap,
|
||||
write::{
|
||||
blob_move::{MovingBlobStore, OldBlobStore},
|
||||
now,
|
||||
},
|
||||
};
|
||||
use types::id::Id;
|
||||
|
||||
/// The blob store to use: `blob` itself or, while changes are unresolved,
|
||||
/// `blob` reading through to their old stores.
|
||||
pub async fn at_start(bp: &mut Bootstrap, blob: BlobStore) -> BlobStore {
|
||||
let data = bp.data_store.clone();
|
||||
if matches!(data, Store::None) || matches!(blob, BlobStore::Store(Store::None)) {
|
||||
return blob;
|
||||
}
|
||||
let settings = bp.setting_infallible::<structs::BlobStore>().await;
|
||||
let location = blob_store_location(&settings);
|
||||
let warn = |bp: &mut Bootstrap, message: String| {
|
||||
bp.build_warning(ObjectType::BlobStore.singleton(), message);
|
||||
};
|
||||
|
||||
let mut recorded = None;
|
||||
match model::at_start(
|
||||
&data,
|
||||
location.clone(),
|
||||
&settings,
|
||||
bp.node_id() as u64,
|
||||
now(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Started::Same) => {}
|
||||
Ok(Started::Changed(id)) => {
|
||||
recorded = Some(id);
|
||||
if let Ok(Some(change)) = model::get(&data, id).await {
|
||||
warn(
|
||||
bp,
|
||||
format!(
|
||||
"The blob store moved from {} to {}. Blobs written before stay in {}; \
|
||||
they're read from it and kept in the new store as they're needed. \
|
||||
Copy the rest over with inbuxa:BlobStoreChange {}.",
|
||||
change.from,
|
||||
change.to,
|
||||
change.from,
|
||||
Id::from(id)
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn(
|
||||
bp,
|
||||
format!("Couldn't record where the blob store is: {err}"),
|
||||
);
|
||||
return blob;
|
||||
}
|
||||
}
|
||||
|
||||
let unresolved = match model::unresolved(&data).await {
|
||||
Ok(changes) => changes,
|
||||
Err(err) => {
|
||||
warn(bp, format!("Couldn't read the blob store changes: {err}"));
|
||||
return blob;
|
||||
}
|
||||
};
|
||||
let mut old = Vec::with_capacity(unresolved.len());
|
||||
for change in unresolved {
|
||||
// BM-4 resolves a change whose old store is this one; one recorded
|
||||
// before that rule could still say so
|
||||
if change.from == location {
|
||||
continue;
|
||||
}
|
||||
let Some(settings) = change.old_settings.clone() else {
|
||||
continue;
|
||||
};
|
||||
let mut warnings = Vec::new();
|
||||
match BlobStore::open(settings, &data, &mut warnings).await {
|
||||
Ok(store) => {
|
||||
old.push(OldBlobStore {
|
||||
change_id: change.id,
|
||||
store,
|
||||
});
|
||||
if recorded == Some(change.id) {
|
||||
continue;
|
||||
}
|
||||
warn(
|
||||
bp,
|
||||
format!(
|
||||
"The blob store changed from {} to {}, and blobs only {} has are \
|
||||
read from it. Copy them over or dismiss the change \
|
||||
(inbuxa:BlobStoreChange {}).",
|
||||
change.from,
|
||||
change.to,
|
||||
change.from,
|
||||
Id::from(change.id)
|
||||
),
|
||||
);
|
||||
}
|
||||
Err(err) => warn(
|
||||
bp,
|
||||
format!(
|
||||
"The blob store used before the change to {} ({}) couldn't be opened, \
|
||||
so blobs only it has can't be read: {err}",
|
||||
change.to, change.from
|
||||
),
|
||||
),
|
||||
}
|
||||
for warning in warnings {
|
||||
warn(bp, warning);
|
||||
}
|
||||
}
|
||||
|
||||
if old.is_empty() {
|
||||
blob
|
||||
} else {
|
||||
BlobStore::Moving(Arc::new(MovingBlobStore::new(blob, old)))
|
||||
}
|
||||
}
|
||||
@@ -33,7 +33,10 @@ use types::id::Id;
|
||||
/// locks and legal holds (audit-hold-lock spec, AU-9, AL-12, LH-13), and
|
||||
/// the data inventory (personal-data catalog spec), accepting security
|
||||
/// to-do items (security to-do list spec), the deliverability check
|
||||
/// (deliverability spec), and scheduled reports (scheduled-reports spec).
|
||||
/// (deliverability spec), scheduled reports (scheduled-reports spec),
|
||||
/// checking the store's blobs (store-check spec, SV-9), taking snapshots of
|
||||
/// the store (store-snapshot spec, SN-8), and copying and dismissing changes
|
||||
/// of blob store (blob-store-change spec, BM-9).
|
||||
const ADMIN_GRANTS: &[Permission] = &[
|
||||
Permission::SysAiExplain,
|
||||
Permission::SysAuditGet,
|
||||
@@ -62,6 +65,9 @@ const ADMIN_GRANTS: &[Permission] = &[
|
||||
Permission::SysDeliverabilityCheck,
|
||||
Permission::SysScheduledReportGet,
|
||||
Permission::SysScheduledReportUpdate,
|
||||
Permission::SysStoreCheck,
|
||||
Permission::SysStoreSnapshot,
|
||||
Permission::SysBlobStoreChange,
|
||||
];
|
||||
|
||||
/// Granted to the server-level Compliance Officer role once it exists:
|
||||
|
||||
@@ -17,6 +17,7 @@ use utils::HttpLimitResponse;
|
||||
|
||||
pub mod application;
|
||||
pub mod backup;
|
||||
pub mod blob_moves; // inbuxa: blob-store-change spec, a blob store that moved
|
||||
pub mod boot;
|
||||
pub mod compliance_roles; // inbuxa: personal-data catalog, the compliance roles
|
||||
pub mod console;
|
||||
|
||||
@@ -0,0 +1,840 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Changing the blob store (docs/spec/features/blob-store-change.md): the
|
||||
//! `inbuxa:BlobStoreChange` objects. A change is recorded when a node
|
||||
//! starts with its blob store somewhere else than the store last said
|
||||
//! (BM-2), and keeps the old store's settings so it can be read through
|
||||
//! (BM-3) and copied from (BM-5). The copy itself runs in
|
||||
//! `services::inbuxa_blob_move`; this module keeps what a change is.
|
||||
//!
|
||||
//! Kept in the fork's subspace (`store::SUBSPACE_INBUXA`), never in the
|
||||
//! registry. Every key starts with `W` (where blobs are), then one byte for
|
||||
//! the kind:
|
||||
//!
|
||||
//! - `l`: where the blob store is now, with its settings ([`Location`]);
|
||||
//! - `c` + change id (u32): a change ([`BlobStoreChange`]), as JSON;
|
||||
//! - `p` + change id (u32) + number (u32): one problem its latest copy
|
||||
//! found ([`Problem`]);
|
||||
//! - `r`: the id of the change being copied now (u32), so only one copy
|
||||
//! runs at a time across the server (BM-5).
|
||||
//!
|
||||
//! Numbers are big-endian.
|
||||
|
||||
use registry::schema::structs;
|
||||
use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize, de::DeserializeOwned};
|
||||
use store::{
|
||||
Deserialize, IterateParams, SUBSPACE_INBUXA, Serialize, SerializeInfallible, Store, U32_LEN,
|
||||
ValueKey,
|
||||
write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue},
|
||||
};
|
||||
use trc::AddContext;
|
||||
|
||||
const FEATURE: u8 = b'W';
|
||||
const KIND_LOCATION: u8 = b'l';
|
||||
const KIND_CHANGE: u8 = b'c';
|
||||
const KIND_PROBLEM: u8 = b'p';
|
||||
const KIND_RUNNING: u8 = b'r';
|
||||
const WRITE_ATTEMPTS: usize = 5;
|
||||
|
||||
/// The changes kept; recording another forgets the oldest resolved one.
|
||||
pub const MAX_CHANGES: usize = 20;
|
||||
/// BM-6: the problems a copy keeps; the counts cover all of them.
|
||||
pub const MAX_PROBLEMS: u64 = 1_000;
|
||||
/// BM-5: the default limits, as a store check's.
|
||||
pub const DEFAULT_BLOBS_PER_SECOND: u64 = 1_000;
|
||||
pub const DEFAULT_BYTES_PER_SECOND: u64 = 20 * 1024 * 1024;
|
||||
/// BM-6: a copy whose node has been quiet this long has stopped.
|
||||
pub const STALE_AFTER: u64 = 5 * 60;
|
||||
const MAX_TEXT: usize = 2_000;
|
||||
|
||||
/// Where the blob store is (BM-1), with the settings that open it.
|
||||
#[derive(Debug, Clone, PartialEq, SerdeSerialize, SerdeDeserialize)]
|
||||
pub struct Location {
|
||||
pub location: String,
|
||||
pub settings: structs::BlobStore,
|
||||
}
|
||||
|
||||
/// BM-6, BM-4.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Status {
|
||||
/// Recorded; nothing copied yet.
|
||||
Pending,
|
||||
Copying,
|
||||
/// Everything is in the new store: resolved.
|
||||
Copied,
|
||||
/// A copy ended with blobs it couldn't copy.
|
||||
Incomplete,
|
||||
Stopped,
|
||||
Failed,
|
||||
Interrupted,
|
||||
/// The blob store went back to where this change moved it from:
|
||||
/// resolved (BM-4).
|
||||
Reverted,
|
||||
}
|
||||
|
||||
impl Status {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Status::Pending => "pending",
|
||||
Status::Copying => "copying",
|
||||
Status::Copied => "copied",
|
||||
Status::Incomplete => "incomplete",
|
||||
Status::Stopped => "stopped",
|
||||
Status::Failed => "failed",
|
||||
Status::Interrupted => "interrupted",
|
||||
Status::Reverted => "reverted",
|
||||
}
|
||||
}
|
||||
|
||||
/// BM-3: whether the old store is still read through.
|
||||
pub fn is_unresolved(&self) -> bool {
|
||||
!matches!(self, Status::Copied | Status::Reverted)
|
||||
}
|
||||
}
|
||||
|
||||
/// BM-5.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum ProblemKind {
|
||||
Missing,
|
||||
Damaged,
|
||||
Unreadable,
|
||||
}
|
||||
|
||||
impl ProblemKind {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
ProblemKind::Missing => "missing",
|
||||
ProblemKind::Damaged => "damaged",
|
||||
ProblemKind::Unreadable => "unreadable",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// BM-5: a blob a copy couldn't copy.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Problem {
|
||||
pub kind: ProblemKind,
|
||||
/// The blob's key: a blob id as hex, or a fixed name.
|
||||
pub blob_id: String,
|
||||
/// `email`, `file`, `queuedMessage`, `archivedItem`, `temporary` or
|
||||
/// `fixed`: the first thing found holding it.
|
||||
pub holder: String,
|
||||
#[serde(default)]
|
||||
pub account_id: Option<u32>,
|
||||
#[serde(default)]
|
||||
pub object_id: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub detail: String,
|
||||
}
|
||||
|
||||
/// BM-5: how far the latest copy got.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
pub struct Progress {
|
||||
/// Slices of the blob index done, of 256.
|
||||
pub slices: u32,
|
||||
pub blobs_checked: u64,
|
||||
pub copied: u64,
|
||||
pub already_there: u64,
|
||||
pub bytes_copied: u64,
|
||||
pub missing: u64,
|
||||
pub damaged: u64,
|
||||
pub unreadable: u64,
|
||||
}
|
||||
|
||||
impl Progress {
|
||||
pub fn problems(&self) -> u64 {
|
||||
self.missing + self.damaged + self.unreadable
|
||||
}
|
||||
|
||||
pub fn count(&mut self, kind: ProblemKind) {
|
||||
match kind {
|
||||
ProblemKind::Missing => self.missing += 1,
|
||||
ProblemKind::Damaged => self.damaged += 1,
|
||||
ProblemKind::Unreadable => self.unreadable += 1,
|
||||
}
|
||||
}
|
||||
|
||||
/// 0 to 1, by slices of the index.
|
||||
pub fn share(&self) -> f64 {
|
||||
self.slices.min(256) as f64 / 256.0
|
||||
}
|
||||
}
|
||||
|
||||
/// The latest copy of a change (BM-5, BM-6).
|
||||
#[derive(Debug, Clone, PartialEq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Copy {
|
||||
pub started_at: u64,
|
||||
pub started_by: String,
|
||||
pub node_id: u64,
|
||||
pub blobs_per_second: u64,
|
||||
pub bytes_per_second: u64,
|
||||
/// When its node last saved it.
|
||||
pub heartbeat: u64,
|
||||
#[serde(default)]
|
||||
pub finished_at: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub stop_requested: bool,
|
||||
#[serde(default)]
|
||||
pub progress: Progress,
|
||||
#[serde(default)]
|
||||
pub problems_kept: u64,
|
||||
#[serde(default)]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// An `inbuxa:BlobStoreChange`.
|
||||
#[derive(Debug, Clone, PartialEq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BlobStoreChange {
|
||||
pub id: u32,
|
||||
pub from: String,
|
||||
pub to: String,
|
||||
pub changed_at: u64,
|
||||
pub node_id: u64,
|
||||
pub status: Status,
|
||||
/// The old store's settings, to read it (BM-3) and copy from it (BM-5).
|
||||
/// Dropped once the change is resolved; never shown (BM-9).
|
||||
#[serde(default)]
|
||||
pub old_settings: Option<structs::BlobStore>,
|
||||
#[serde(default)]
|
||||
pub copy: Option<Copy>,
|
||||
}
|
||||
|
||||
impl BlobStoreChange {
|
||||
/// As it reads now: a copy whose node went quiet has stopped with it
|
||||
/// (BM-6), even before anything marked it so.
|
||||
pub fn status_at(&self, now: u64) -> Status {
|
||||
match &self.copy {
|
||||
Some(copy) if self.status == Status::Copying && copy.heartbeat + STALE_AFTER < now => {
|
||||
Status::Interrupted
|
||||
}
|
||||
_ => self.status,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_error(&mut self, error: &str) {
|
||||
if let Some(copy) = &mut self.copy {
|
||||
copy.error = Some(error.trim().chars().take(MAX_TEXT).collect());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Json<T>(T);
|
||||
|
||||
impl<T: SerdeSerialize> Serialize for Json<&T> {
|
||||
fn serialize(&self) -> trc::Result<Vec<u8>> {
|
||||
serde_json::to_vec(self.0).map_err(|err| {
|
||||
trc::StoreEvent::UnexpectedError
|
||||
.into_err()
|
||||
.details("Could not serialize a blob store change")
|
||||
.reason(err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: DeserializeOwned + Sync + Send> Deserialize for Json<T> {
|
||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
||||
serde_json::from_slice(bytes).map(Json).map_err(|err| {
|
||||
trc::StoreEvent::DataCorruption
|
||||
.into_err()
|
||||
.details("Invalid blob store change")
|
||||
.reason(err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn any(key: Vec<u8>) -> ValueClass {
|
||||
ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key,
|
||||
})
|
||||
}
|
||||
|
||||
fn location_class() -> ValueClass {
|
||||
any(vec![FEATURE, KIND_LOCATION])
|
||||
}
|
||||
|
||||
fn class(id: u32) -> ValueClass {
|
||||
let mut key = Vec::with_capacity(2 + U32_LEN);
|
||||
key.extend_from_slice(&[FEATURE, KIND_CHANGE]);
|
||||
key.extend_from_slice(&id.to_be_bytes());
|
||||
any(key)
|
||||
}
|
||||
|
||||
fn problem_class(id: u32, number: u32) -> ValueClass {
|
||||
let mut key = Vec::with_capacity(2 + U32_LEN * 2);
|
||||
key.extend_from_slice(&[FEATURE, KIND_PROBLEM]);
|
||||
key.extend_from_slice(&id.to_be_bytes());
|
||||
key.extend_from_slice(&number.to_be_bytes());
|
||||
any(key)
|
||||
}
|
||||
|
||||
fn running_class() -> ValueClass {
|
||||
any(vec![FEATURE, KIND_RUNNING])
|
||||
}
|
||||
|
||||
fn is_assert_failure(err: &trc::Error) -> bool {
|
||||
matches!(
|
||||
err.as_ref(),
|
||||
trc::EventType::Store(trc::StoreEvent::AssertValueFailed)
|
||||
)
|
||||
}
|
||||
|
||||
/// A value as stored, with a hash of its bytes for a compare-and-set.
|
||||
struct Versioned<T>(T, u64);
|
||||
|
||||
impl<T: DeserializeOwned + Sync + Send> Deserialize for Versioned<T> {
|
||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
||||
let Json(value) = Json::<T>::deserialize(bytes)?;
|
||||
Ok(Versioned(value, xxhash_rust::xxh3::xxh3_64(bytes)))
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get(data: &Store, id: u32) -> trc::Result<Option<BlobStoreChange>> {
|
||||
Ok(data
|
||||
.get_value::<Json<BlobStoreChange>>(ValueKey::from(class(id)))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.map(|Json(change)| change))
|
||||
}
|
||||
|
||||
/// Every change, oldest first.
|
||||
pub async fn all(data: &Store) -> trc::Result<Vec<BlobStoreChange>> {
|
||||
let mut changes = Vec::new();
|
||||
data.iterate(
|
||||
IterateParams::new(ValueKey::from(class(0)), ValueKey::from(class(u32::MAX))),
|
||||
|_, value| {
|
||||
if let Ok(Json(change)) = Json::<BlobStoreChange>::deserialize(value) {
|
||||
changes.push(change);
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
changes.sort_by_key(|c| c.id);
|
||||
Ok(changes)
|
||||
}
|
||||
|
||||
/// BM-3, BM-10: the changes still read through, newest first.
|
||||
pub async fn unresolved(data: &Store) -> trc::Result<Vec<BlobStoreChange>> {
|
||||
let mut changes = all(data).await?;
|
||||
changes.retain(|c| c.status.is_unresolved() && c.old_settings.is_some());
|
||||
changes.reverse();
|
||||
Ok(changes)
|
||||
}
|
||||
|
||||
/// Where the store last said the blob store was.
|
||||
pub async fn location(data: &Store) -> trc::Result<Option<Location>> {
|
||||
Ok(data
|
||||
.get_value::<Json<Location>>(ValueKey::from(location_class()))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.map(|Json(location)| location))
|
||||
}
|
||||
|
||||
/// What [`at_start`] found.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Started {
|
||||
/// The blob store is where it was, or this is the first start that
|
||||
/// records it.
|
||||
Same,
|
||||
/// This node recorded a change, the one with this id (BM-2).
|
||||
Changed(u32),
|
||||
}
|
||||
|
||||
/// BM-2, BM-4: at start-up, records where the blob store is and, when it
|
||||
/// moved, a change from where it was. The first node to start with the new
|
||||
/// setting records it: the location is written only if it's still what
|
||||
/// this node read, so the others find the change already there.
|
||||
pub async fn at_start(
|
||||
data: &Store,
|
||||
location: String,
|
||||
settings: &structs::BlobStore,
|
||||
node_id: u64,
|
||||
now: u64,
|
||||
) -> trc::Result<Started> {
|
||||
let mut attempt = 0;
|
||||
loop {
|
||||
attempt += 1;
|
||||
let last = data
|
||||
.get_value::<Versioned<Location>>(ValueKey::from(location_class()))
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
let here = Location {
|
||||
location: location.clone(),
|
||||
settings: settings.clone(),
|
||||
};
|
||||
|
||||
let mut batch = BatchBuilder::new();
|
||||
let mut started = Started::Same;
|
||||
let mut forgotten = Vec::new();
|
||||
match &last {
|
||||
None => {
|
||||
batch.assert_value(location_class(), AssertValue::None);
|
||||
}
|
||||
Some(Versioned(last, version)) if last.location == location => {
|
||||
if &last.settings == settings {
|
||||
return Ok(Started::Same);
|
||||
}
|
||||
// The same place with other credentials or tuning: kept
|
||||
// current, for when it's the old store of a later change
|
||||
batch.assert_value(location_class(), AssertValue::Hash(*version));
|
||||
}
|
||||
Some(Versioned(last, version)) => {
|
||||
batch.assert_value(location_class(), AssertValue::Hash(*version));
|
||||
let changes = all(data).await?;
|
||||
let id = changes.iter().map(|c| c.id).max().unwrap_or(0) + 1;
|
||||
|
||||
// BM-4: back where an unresolved change moved away from
|
||||
for earlier in changes.iter().filter(|c| {
|
||||
c.status.is_unresolved() && c.status != Status::Copying && c.from == location
|
||||
}) {
|
||||
let reverted = BlobStoreChange {
|
||||
status: Status::Reverted,
|
||||
old_settings: None,
|
||||
..earlier.clone()
|
||||
};
|
||||
batch.set(class(earlier.id), Json(&reverted).serialize()?);
|
||||
}
|
||||
|
||||
let change = BlobStoreChange {
|
||||
id,
|
||||
from: last.location.clone(),
|
||||
to: location.clone(),
|
||||
changed_at: now,
|
||||
node_id,
|
||||
status: Status::Pending,
|
||||
old_settings: Some(last.settings.clone()),
|
||||
copy: None,
|
||||
};
|
||||
batch.assert_value(class(id), AssertValue::None);
|
||||
batch.set(class(id), Json(&change).serialize()?);
|
||||
started = Started::Changed(id);
|
||||
|
||||
// The oldest resolved changes past MAX_CHANGES are forgotten
|
||||
let over = (changes.len() + 1).saturating_sub(MAX_CHANGES);
|
||||
for old in changes
|
||||
.iter()
|
||||
.filter(|c| !c.status.is_unresolved())
|
||||
.take(over)
|
||||
{
|
||||
batch.clear(class(old.id));
|
||||
forgotten.push(old.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
batch.set(location_class(), Json(&here).serialize()?);
|
||||
|
||||
match data.write(batch.build_all()).await {
|
||||
Ok(_) => {
|
||||
for id in forgotten {
|
||||
remove_problems(data, id).await?;
|
||||
}
|
||||
return Ok(started);
|
||||
}
|
||||
// Another node wrote first: read what it recorded
|
||||
Err(err) if attempt < WRITE_ATTEMPTS && is_assert_failure(&err) => {}
|
||||
Err(err) => return Err(err.caused_by(trc::location!())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The id of the change being copied now, if any.
|
||||
pub async fn running(data: &Store) -> trc::Result<Option<u32>> {
|
||||
data.get_value::<u32>(ValueKey::from(running_class()))
|
||||
.await
|
||||
.caused_by(trc::location!())
|
||||
}
|
||||
|
||||
/// BM-5: why a copy can't start.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Refusal {
|
||||
/// No such change.
|
||||
NotFound,
|
||||
/// This change's copy runs now.
|
||||
Running(u32),
|
||||
/// The change is resolved, or has no old store to copy from.
|
||||
Resolved,
|
||||
}
|
||||
|
||||
/// BM-5: marks a change as being copied by `copy` and makes it the one
|
||||
/// that runs. Refused while another copy runs: one whose node went quiet
|
||||
/// is marked interrupted first.
|
||||
pub async fn start_copy(
|
||||
data: &Store,
|
||||
id: u32,
|
||||
copy: Copy,
|
||||
now: u64,
|
||||
) -> trc::Result<Result<BlobStoreChange, Refusal>> {
|
||||
let mut attempt = 0;
|
||||
loop {
|
||||
attempt += 1;
|
||||
if let Some(current) = running(data).await? {
|
||||
match get(data, current).await? {
|
||||
Some(c) if c.status_at(now) == Status::Copying => {
|
||||
return Ok(Err(Refusal::Running(current)));
|
||||
}
|
||||
Some(c) => {
|
||||
if c.status == Status::Copying {
|
||||
update(data, current, |c| {
|
||||
if c.status != Status::Copying {
|
||||
return false;
|
||||
}
|
||||
c.status = Status::Interrupted;
|
||||
if let Some(copy) = &mut c.copy {
|
||||
copy.finished_at = Some(now);
|
||||
}
|
||||
true
|
||||
})
|
||||
.await?;
|
||||
}
|
||||
release(data, current).await?;
|
||||
}
|
||||
None => release(data, current).await?,
|
||||
}
|
||||
}
|
||||
|
||||
let Some(Versioned(change, version)) = data
|
||||
.get_value::<Versioned<BlobStoreChange>>(ValueKey::from(class(id)))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
else {
|
||||
return Ok(Err(Refusal::NotFound));
|
||||
};
|
||||
if !change.status.is_unresolved() || change.old_settings.is_none() {
|
||||
return Ok(Err(Refusal::Resolved));
|
||||
}
|
||||
let change = BlobStoreChange {
|
||||
status: Status::Copying,
|
||||
copy: Some(copy.clone()),
|
||||
..change
|
||||
};
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.assert_value(class(id), AssertValue::Hash(version));
|
||||
batch.assert_value(running_class(), AssertValue::None);
|
||||
batch.set(class(id), Json(&change).serialize()?);
|
||||
batch.set(running_class(), SerializeInfallible::serialize(&id));
|
||||
match data.write(batch.build_all()).await {
|
||||
Ok(_) => {
|
||||
remove_problems(data, id).await?;
|
||||
return Ok(Ok(change));
|
||||
}
|
||||
Err(err) if attempt < WRITE_ATTEMPTS && is_assert_failure(&err) => {}
|
||||
Err(err) => return Err(err.caused_by(trc::location!())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Lets another copy run, if `id` is the one that runs now.
|
||||
pub async fn release(data: &Store, id: u32) -> trc::Result<()> {
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.assert_value(running_class(), AssertValue::U32(id));
|
||||
batch.clear(running_class());
|
||||
match data.write(batch.build_all()).await {
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) if is_assert_failure(&err) => Ok(()),
|
||||
Err(err) => Err(err.caused_by(trc::location!())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads a change, changes it with `change`, and writes it back, again if
|
||||
/// something else wrote it in between. `change` returns false to leave it
|
||||
/// as it is. None if there's no such change.
|
||||
pub async fn update<F>(data: &Store, id: u32, mut change: F) -> trc::Result<Option<BlobStoreChange>>
|
||||
where
|
||||
F: FnMut(&mut BlobStoreChange) -> bool,
|
||||
{
|
||||
loop {
|
||||
let Some(Versioned(mut stored, version)) = data
|
||||
.get_value::<Versioned<BlobStoreChange>>(ValueKey::from(class(id)))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !change(&mut stored) {
|
||||
return Ok(Some(stored));
|
||||
}
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.assert_value(class(id), AssertValue::Hash(version));
|
||||
batch.set(class(id), Json(&stored).serialize()?);
|
||||
match data.write(batch.build_all()).await {
|
||||
Ok(_) => return Ok(Some(stored)),
|
||||
Err(err) if is_assert_failure(&err) => {}
|
||||
Err(err) => return Err(err.caused_by(trc::location!())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Keeps problems `first..` of a change's copy (BM-6): the caller numbers
|
||||
/// them and stops at [`MAX_PROBLEMS`].
|
||||
pub async fn add_problems(
|
||||
data: &Store,
|
||||
id: u32,
|
||||
first: u64,
|
||||
problems: &[Problem],
|
||||
) -> trc::Result<()> {
|
||||
let mut batch = BatchBuilder::new();
|
||||
for (n, problem) in problems.iter().enumerate() {
|
||||
let number = u32::try_from(first + n as u64).unwrap_or(u32::MAX);
|
||||
batch.set(problem_class(id, number), Json(problem).serialize()?);
|
||||
if batch.is_large_batch() {
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
batch = BatchBuilder::new();
|
||||
}
|
||||
}
|
||||
if !batch.is_empty() {
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The kept problems of a change's latest copy, in the order found.
|
||||
pub async fn problems(data: &Store, id: u32) -> trc::Result<Vec<Problem>> {
|
||||
let mut problems = Vec::new();
|
||||
data.iterate(
|
||||
IterateParams::new(
|
||||
ValueKey::from(problem_class(id, 0)),
|
||||
ValueKey::from(problem_class(id, u32::MAX)),
|
||||
),
|
||||
|_, value| {
|
||||
if let Ok(Json(problem)) = Json::<Problem>::deserialize(value) {
|
||||
problems.push(problem);
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
Ok(problems)
|
||||
}
|
||||
|
||||
async fn remove_problems(data: &Store, id: u32) -> trc::Result<()> {
|
||||
data.delete_range(
|
||||
ValueKey::from(problem_class(id, 0)),
|
||||
ValueKey::from(problem_class(id, u32::MAX)),
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())
|
||||
}
|
||||
|
||||
/// BM-7: forgets a change, the old store's settings and its problems.
|
||||
pub async fn remove(data: &Store, id: u32) -> trc::Result<()> {
|
||||
remove_problems(data, id).await?;
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.clear(class(id));
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn fs(path: &str) -> structs::BlobStore {
|
||||
structs::BlobStore::FileSystem(structs::FileSystemStore {
|
||||
path: path.into(),
|
||||
depth: 2,
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_move_and_a_move_back() {
|
||||
let data = store::backend::ephemeral::EphemeralStore::open();
|
||||
let a = structs::BlobStore::Default;
|
||||
let b = fs("/srv/blobs");
|
||||
let (here_a, here_b) = ("the data store".to_string(), "dir".to_string());
|
||||
|
||||
// First start: only where it is
|
||||
assert_eq!(
|
||||
at_start(&data, here_a.clone(), &a, 1, 10).await.unwrap(),
|
||||
Started::Same
|
||||
);
|
||||
assert_eq!(
|
||||
at_start(&data, here_a.clone(), &a, 2, 11).await.unwrap(),
|
||||
Started::Same
|
||||
);
|
||||
assert!(all(&data).await.unwrap().is_empty());
|
||||
|
||||
// A to B: one change, recorded once whichever node starts first
|
||||
assert_eq!(
|
||||
at_start(&data, here_b.clone(), &b, 1, 20).await.unwrap(),
|
||||
Started::Changed(1)
|
||||
);
|
||||
assert_eq!(
|
||||
at_start(&data, here_b.clone(), &b, 2, 21).await.unwrap(),
|
||||
Started::Same
|
||||
);
|
||||
let first = get(&data, 1).await.unwrap().unwrap();
|
||||
assert_eq!(
|
||||
(first.from.as_str(), first.to.as_str()),
|
||||
("the data store", "dir")
|
||||
);
|
||||
assert_eq!(first.old_settings, Some(a.clone()));
|
||||
assert_eq!(first.status, Status::Pending);
|
||||
|
||||
// Back to A: the first change is reverted, the second is the one
|
||||
// still read through
|
||||
assert_eq!(
|
||||
at_start(&data, here_a.clone(), &a, 1, 30).await.unwrap(),
|
||||
Started::Changed(2)
|
||||
);
|
||||
let first = get(&data, 1).await.unwrap().unwrap();
|
||||
assert_eq!(first.status, Status::Reverted);
|
||||
assert_eq!(first.old_settings, None);
|
||||
let open = unresolved(&data).await.unwrap();
|
||||
assert_eq!(open.len(), 1);
|
||||
assert_eq!(open[0].id, 2);
|
||||
assert_eq!(open[0].old_settings, Some(b.clone()));
|
||||
|
||||
// A copy: one at a time, never of a resolved change
|
||||
let copy = Copy {
|
||||
started_at: 40,
|
||||
started_by: "admin".into(),
|
||||
node_id: 1,
|
||||
blobs_per_second: 0,
|
||||
bytes_per_second: 0,
|
||||
heartbeat: 40,
|
||||
finished_at: None,
|
||||
stop_requested: false,
|
||||
progress: Progress::default(),
|
||||
problems_kept: 0,
|
||||
error: None,
|
||||
};
|
||||
assert_eq!(
|
||||
start_copy(&data, 1, copy.clone(), 40).await.unwrap(),
|
||||
Err(Refusal::Resolved)
|
||||
);
|
||||
assert_eq!(
|
||||
start_copy(&data, 9, copy.clone(), 40).await.unwrap(),
|
||||
Err(Refusal::NotFound)
|
||||
);
|
||||
let started = start_copy(&data, 2, copy.clone(), 40)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(started.status, Status::Copying);
|
||||
assert_eq!(
|
||||
start_copy(&data, 2, copy.clone(), 41).await.unwrap(),
|
||||
Err(Refusal::Running(2))
|
||||
);
|
||||
// Its node went quiet: interrupted, and another may start
|
||||
let again = start_copy(&data, 2, copy.clone(), 41 + STALE_AFTER)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(again.status, Status::Copying);
|
||||
|
||||
// Finished: resolved, nothing read through any more
|
||||
update(&data, 2, |c| {
|
||||
c.status = Status::Copied;
|
||||
c.old_settings = None;
|
||||
true
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
release(&data, 2).await.unwrap();
|
||||
assert!(unresolved(&data).await.unwrap().is_empty());
|
||||
assert_eq!(running(&data).await.unwrap(), None);
|
||||
|
||||
// Where it was, again: nothing new
|
||||
assert_eq!(
|
||||
at_start(&data, here_a.clone(), &a, 1, 50).await.unwrap(),
|
||||
Started::Same
|
||||
);
|
||||
remove(&data, 2).await.unwrap();
|
||||
assert!(get(&data, 2).await.unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keys_stay_in_their_own_letter() {
|
||||
for class in [
|
||||
location_class(),
|
||||
class(1),
|
||||
problem_class(1, 0),
|
||||
running_class(),
|
||||
] {
|
||||
let ValueClass::Any(any) = class else {
|
||||
unreachable!()
|
||||
};
|
||||
assert_eq!(any.key[0], b'W');
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolved_is_copied_or_reverted() {
|
||||
for status in [
|
||||
Status::Pending,
|
||||
Status::Copying,
|
||||
Status::Incomplete,
|
||||
Status::Stopped,
|
||||
Status::Failed,
|
||||
Status::Interrupted,
|
||||
] {
|
||||
assert!(status.is_unresolved(), "{status:?}");
|
||||
}
|
||||
assert!(!Status::Copied.is_unresolved());
|
||||
assert!(!Status::Reverted.is_unresolved());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_quiet_copy_reads_as_interrupted() {
|
||||
let change = BlobStoreChange {
|
||||
id: 1,
|
||||
from: "the data store".into(),
|
||||
to: "the directory /srv/blobs, depth 2".into(),
|
||||
changed_at: 1_000,
|
||||
node_id: 1,
|
||||
status: Status::Copying,
|
||||
old_settings: Some(structs::BlobStore::Default),
|
||||
copy: Some(Copy {
|
||||
started_at: 1_000,
|
||||
started_by: "admin".into(),
|
||||
node_id: 1,
|
||||
blobs_per_second: 0,
|
||||
bytes_per_second: 0,
|
||||
heartbeat: 1_000,
|
||||
finished_at: None,
|
||||
stop_requested: false,
|
||||
progress: Progress::default(),
|
||||
problems_kept: 0,
|
||||
error: None,
|
||||
}),
|
||||
};
|
||||
assert_eq!(change.status_at(1_000 + STALE_AFTER), Status::Copying);
|
||||
assert_eq!(change.status_at(1_001 + STALE_AFTER), Status::Interrupted);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_change_keeps_the_old_settings_as_json() {
|
||||
let change = BlobStoreChange {
|
||||
id: 2,
|
||||
from: "the directory /srv/blobs, depth 2".into(),
|
||||
to: "the data store".into(),
|
||||
changed_at: 5,
|
||||
node_id: 1,
|
||||
status: Status::Pending,
|
||||
old_settings: Some(fs("/srv/blobs")),
|
||||
copy: None,
|
||||
};
|
||||
let bytes = Json(&change).serialize().unwrap();
|
||||
let Json(back) = Json::<BlobStoreChange>::deserialize(&bytes).unwrap();
|
||||
assert_eq!(back, change);
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@
|
||||
|
||||
pub mod ai;
|
||||
pub mod audit;
|
||||
pub mod blob_move; // inbuxa: changing the blob store (not a rebuild)
|
||||
pub mod branding;
|
||||
pub mod deliverability; // inbuxa: the deliverability check (not a rebuild)
|
||||
pub mod hold;
|
||||
@@ -30,5 +31,6 @@ pub mod masked_email;
|
||||
pub mod privacy;
|
||||
pub mod scheduled_reports; // inbuxa: scheduled reports and the weekly digest (not a rebuild)
|
||||
pub mod security;
|
||||
pub mod store_check; // inbuxa: checking the store's blobs (not a rebuild)
|
||||
pub mod tenancy;
|
||||
pub mod undelete;
|
||||
@@ -0,0 +1,597 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Checking the store's blobs (docs/spec/features/store-check.md): the
|
||||
//! `inbuxa:StoreCheck` objects. A check reports which blobs the server's
|
||||
//! data needs and doesn't have, and changes nothing. The check itself runs
|
||||
//! in `services::inbuxa_store_check`; this module keeps what a check is and
|
||||
//! what it found.
|
||||
//!
|
||||
//! Kept in the fork's subspace (`store::SUBSPACE_INBUXA`), never in the
|
||||
//! registry (SV-1). Every key starts with `V` (verify), then one byte for
|
||||
//! the kind:
|
||||
//!
|
||||
//! - `c` + check id (u32): a check ([`StoreCheck`]), as JSON;
|
||||
//! - `p` + check id (u32) + number (u32): one of its problems ([`Problem`]),
|
||||
//! as JSON, kept apart so a check stays small however much it found;
|
||||
//! - `r`: the id of the check that runs now (u32), so only one runs at a
|
||||
//! time across the server (SV-2).
|
||||
//!
|
||||
//! `Vs` is the store's version (`common::INBUXA_STORE_VERSION_KEY`), so no
|
||||
//! kind here is `s`. Numbers are big-endian.
|
||||
|
||||
use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize, de::DeserializeOwned};
|
||||
use store::{
|
||||
Deserialize, IterateParams, SUBSPACE_INBUXA, Serialize, SerializeInfallible, Store, U32_LEN,
|
||||
ValueKey,
|
||||
write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue},
|
||||
};
|
||||
use trc::AddContext;
|
||||
|
||||
const FEATURE: u8 = b'V';
|
||||
const KIND_CHECK: u8 = b'c';
|
||||
const KIND_PROBLEM: u8 = b'p';
|
||||
const KIND_RUNNING: u8 = b'r';
|
||||
const CREATE_ATTEMPTS: usize = 5;
|
||||
|
||||
/// SV-1: the checks kept; starting another forgets the oldest.
|
||||
pub const MAX_CHECKS: usize = 20;
|
||||
/// SV-5: the problems a check keeps; the counts cover all of them.
|
||||
pub const MAX_PROBLEMS: u64 = 1_000;
|
||||
/// SV-6: the default limits.
|
||||
pub const DEFAULT_BLOBS_PER_SECOND: u64 = 1_000;
|
||||
pub const DEFAULT_BYTES_PER_SECOND: u64 = 20 * 1024 * 1024;
|
||||
/// SV-8: a running check whose node has been quiet this long has stopped.
|
||||
pub const STALE_AFTER: u64 = 5 * 60;
|
||||
const MAX_TEXT: usize = 2_000;
|
||||
|
||||
/// SV-2.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Mode {
|
||||
Quick,
|
||||
Full,
|
||||
}
|
||||
|
||||
impl Mode {
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"quick" => Some(Mode::Quick),
|
||||
"full" => Some(Mode::Full),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Mode::Quick => "quick",
|
||||
Mode::Full => "full",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// SV-8.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Status {
|
||||
Running,
|
||||
Done,
|
||||
Stopped,
|
||||
Failed,
|
||||
Interrupted,
|
||||
}
|
||||
|
||||
impl Status {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Status::Running => "running",
|
||||
Status::Done => "done",
|
||||
Status::Stopped => "stopped",
|
||||
Status::Failed => "failed",
|
||||
Status::Interrupted => "interrupted",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// SV-4.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum ProblemKind {
|
||||
Missing,
|
||||
Damaged,
|
||||
Unlinked,
|
||||
Unreadable,
|
||||
}
|
||||
|
||||
impl ProblemKind {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
ProblemKind::Missing => "missing",
|
||||
ProblemKind::Damaged => "damaged",
|
||||
ProblemKind::Unlinked => "unlinked",
|
||||
ProblemKind::Unreadable => "unreadable",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// SV-4: one blob something needs and can't have, as it was found.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Problem {
|
||||
pub kind: ProblemKind,
|
||||
/// None for a queued message, which belongs to no account.
|
||||
pub account_id: Option<u32>,
|
||||
#[serde(default)]
|
||||
pub account_name: String,
|
||||
/// `email`, `file`, `queuedMessage`, ... (SV-4).
|
||||
pub object: String,
|
||||
/// The object's id: a document id, a queue id or an archived item's id.
|
||||
pub object_id: u64,
|
||||
/// The blob id, as hex.
|
||||
pub blob_id: String,
|
||||
/// Whether the store still has the blob's commit marker.
|
||||
pub committed: bool,
|
||||
#[serde(default)]
|
||||
pub detail: String,
|
||||
}
|
||||
|
||||
/// SV-7: how far a check got and what it found.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
pub struct Progress {
|
||||
/// Slices of the blob index done, of 256.
|
||||
pub slices: u32,
|
||||
pub blobs_checked: u64,
|
||||
pub links_checked: u64,
|
||||
pub bytes_read: u64,
|
||||
pub messages_checked: u64,
|
||||
/// Accounts whose mail metadata is checked, of how many (SV-3).
|
||||
pub accounts_done: u64,
|
||||
pub accounts_total: u64,
|
||||
pub missing: u64,
|
||||
pub damaged: u64,
|
||||
pub unlinked: u64,
|
||||
pub unreadable: u64,
|
||||
}
|
||||
|
||||
impl Progress {
|
||||
pub fn problems(&self) -> u64 {
|
||||
self.missing + self.damaged + self.unlinked + self.unreadable
|
||||
}
|
||||
|
||||
pub fn count(&mut self, kind: ProblemKind) {
|
||||
match kind {
|
||||
ProblemKind::Missing => self.missing += 1,
|
||||
ProblemKind::Damaged => self.damaged += 1,
|
||||
ProblemKind::Unlinked => self.unlinked += 1,
|
||||
ProblemKind::Unreadable => self.unreadable += 1,
|
||||
}
|
||||
}
|
||||
|
||||
/// 0 to 1: the 256 slices are the first 90%, the mail metadata after
|
||||
/// them the rest. A finished check is 1.
|
||||
pub fn share(&self, status: Status) -> f64 {
|
||||
if status == Status::Done {
|
||||
return 1.0;
|
||||
}
|
||||
let slices = self.slices.min(256) as f64 / 256.0;
|
||||
let accounts = if self.accounts_total == 0 {
|
||||
0.0
|
||||
} else {
|
||||
self.accounts_done.min(self.accounts_total) as f64 / self.accounts_total as f64
|
||||
};
|
||||
slices * 0.9 + accounts * 0.1
|
||||
}
|
||||
}
|
||||
|
||||
/// An `inbuxa:StoreCheck` (SV-1).
|
||||
#[derive(Debug, Clone, PartialEq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct StoreCheck {
|
||||
pub id: u32,
|
||||
pub mode: Mode,
|
||||
pub blobs_per_second: u64,
|
||||
pub bytes_per_second: u64,
|
||||
pub status: Status,
|
||||
pub created_at: u64,
|
||||
pub created_by: String,
|
||||
pub finished_at: Option<u64>,
|
||||
/// The node it runs on.
|
||||
pub node_id: u64,
|
||||
/// When its node last saved it (SV-7, SV-8).
|
||||
pub heartbeat: u64,
|
||||
/// SV-8: an administrator asked it to stop; the running node sees it at
|
||||
/// its next save.
|
||||
#[serde(default)]
|
||||
pub stop_requested: bool,
|
||||
#[serde(default)]
|
||||
pub progress: Progress,
|
||||
/// Problems kept, at most [`MAX_PROBLEMS`].
|
||||
#[serde(default)]
|
||||
pub problems_kept: u64,
|
||||
#[serde(default)]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
impl StoreCheck {
|
||||
pub fn new(mode: Mode, created_by: String, node_id: u64, now: u64) -> Self {
|
||||
StoreCheck {
|
||||
id: 0,
|
||||
mode,
|
||||
blobs_per_second: DEFAULT_BLOBS_PER_SECOND,
|
||||
bytes_per_second: DEFAULT_BYTES_PER_SECOND,
|
||||
status: Status::Running,
|
||||
created_at: now,
|
||||
created_by,
|
||||
finished_at: None,
|
||||
node_id,
|
||||
heartbeat: now,
|
||||
stop_requested: false,
|
||||
progress: Progress::default(),
|
||||
problems_kept: 0,
|
||||
error: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// SV-8: as it reads now. A running check whose node went quiet has
|
||||
/// stopped with it, even before anything marked it so.
|
||||
pub fn status_at(&self, now: u64) -> Status {
|
||||
if self.status == Status::Running && self.heartbeat + STALE_AFTER < now {
|
||||
Status::Interrupted
|
||||
} else {
|
||||
self.status
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_error(&mut self, error: &str) {
|
||||
self.error = Some(error.trim().chars().take(MAX_TEXT).collect());
|
||||
}
|
||||
}
|
||||
|
||||
struct Json<T>(T);
|
||||
|
||||
impl<T: SerdeSerialize> Serialize for Json<&T> {
|
||||
fn serialize(&self) -> trc::Result<Vec<u8>> {
|
||||
serde_json::to_vec(self.0).map_err(|err| {
|
||||
trc::StoreEvent::UnexpectedError
|
||||
.into_err()
|
||||
.details("Could not serialize a store check")
|
||||
.reason(err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: DeserializeOwned + Sync + Send> Deserialize for Json<T> {
|
||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
||||
serde_json::from_slice(bytes).map(Json).map_err(|err| {
|
||||
trc::StoreEvent::DataCorruption
|
||||
.into_err()
|
||||
.details("Invalid store check")
|
||||
.reason(err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn any(key: Vec<u8>) -> ValueClass {
|
||||
ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key,
|
||||
})
|
||||
}
|
||||
|
||||
fn class(id: u32) -> ValueClass {
|
||||
let mut key = Vec::with_capacity(2 + U32_LEN);
|
||||
key.extend_from_slice(&[FEATURE, KIND_CHECK]);
|
||||
key.extend_from_slice(&id.to_be_bytes());
|
||||
any(key)
|
||||
}
|
||||
|
||||
fn problem_class(id: u32, number: u32) -> ValueClass {
|
||||
let mut key = Vec::with_capacity(2 + U32_LEN * 2);
|
||||
key.extend_from_slice(&[FEATURE, KIND_PROBLEM]);
|
||||
key.extend_from_slice(&id.to_be_bytes());
|
||||
key.extend_from_slice(&number.to_be_bytes());
|
||||
any(key)
|
||||
}
|
||||
|
||||
fn running_class() -> ValueClass {
|
||||
any(vec![FEATURE, KIND_RUNNING])
|
||||
}
|
||||
|
||||
pub async fn get(data: &Store, id: u32) -> trc::Result<Option<StoreCheck>> {
|
||||
Ok(data
|
||||
.get_value::<Json<StoreCheck>>(ValueKey::from(class(id)))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.map(|Json(check)| check))
|
||||
}
|
||||
|
||||
/// Every check, oldest first.
|
||||
pub async fn all(data: &Store) -> trc::Result<Vec<StoreCheck>> {
|
||||
let mut checks = Vec::new();
|
||||
data.iterate(
|
||||
IterateParams::new(ValueKey::from(class(0)), ValueKey::from(class(u32::MAX))),
|
||||
|_, value| {
|
||||
if let Ok(Json(check)) = Json::<StoreCheck>::deserialize(value) {
|
||||
checks.push(check);
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
checks.sort_by_key(|c| c.id);
|
||||
Ok(checks)
|
||||
}
|
||||
|
||||
/// The id of the check that runs now, if any.
|
||||
pub async fn running(data: &Store) -> trc::Result<Option<u32>> {
|
||||
Ok(data
|
||||
.get_value::<u32>(ValueKey::from(running_class()))
|
||||
.await
|
||||
.caused_by(trc::location!())?)
|
||||
}
|
||||
|
||||
/// SV-2: why a check can't start.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Refusal {
|
||||
/// This check runs now.
|
||||
Running(u32),
|
||||
}
|
||||
|
||||
/// Writes a new check under the next free id, which it returns, and makes
|
||||
/// it the one that runs. Refused while another runs (SV-2): a running
|
||||
/// check whose node went quiet is marked interrupted first (SV-8). The
|
||||
/// oldest checks past [`MAX_CHECKS`] are forgotten (SV-1).
|
||||
pub async fn create(
|
||||
data: &Store,
|
||||
check: &StoreCheck,
|
||||
now: u64,
|
||||
) -> trc::Result<Result<u32, Refusal>> {
|
||||
let mut attempt = 0;
|
||||
loop {
|
||||
attempt += 1;
|
||||
let checks = all(data).await?;
|
||||
let current = running(data).await?;
|
||||
if let Some(current) = current {
|
||||
match checks.iter().find(|c| c.id == current) {
|
||||
Some(c) if c.status_at(now) == Status::Running => {
|
||||
return Ok(Err(Refusal::Running(current)));
|
||||
}
|
||||
_ => {
|
||||
// Gone, finished or quiet: it no longer holds the place
|
||||
if let Some(c) = checks.iter().find(|c| c.id == current)
|
||||
&& c.status == Status::Running
|
||||
{
|
||||
update(data, current, |c| {
|
||||
if c.status != Status::Running {
|
||||
return false;
|
||||
}
|
||||
c.status = Status::Interrupted;
|
||||
c.finished_at = Some(now);
|
||||
true
|
||||
})
|
||||
.await?;
|
||||
}
|
||||
release(data, current).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let id = checks.iter().map(|c| c.id).max().unwrap_or(0) + 1;
|
||||
let stored = StoreCheck {
|
||||
id,
|
||||
..check.clone()
|
||||
};
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.assert_value(class(id), AssertValue::None);
|
||||
batch.assert_value(running_class(), AssertValue::None);
|
||||
batch.set(class(id), Json(&stored).serialize()?);
|
||||
batch.set(running_class(), SerializeInfallible::serialize(&id));
|
||||
match data.write(batch.build_all()).await {
|
||||
Ok(_) => {
|
||||
// SV-1: the newest MAX_CHECKS stay
|
||||
let mut checks = checks;
|
||||
checks.push(stored);
|
||||
let over = checks.len().saturating_sub(MAX_CHECKS);
|
||||
for old in checks.iter().take(over) {
|
||||
if old.status != Status::Running {
|
||||
remove(data, old.id).await?;
|
||||
}
|
||||
}
|
||||
return Ok(Ok(id));
|
||||
}
|
||||
Err(err)
|
||||
if attempt < CREATE_ATTEMPTS
|
||||
&& matches!(
|
||||
err.as_ref(),
|
||||
trc::EventType::Store(trc::StoreEvent::AssertValueFailed)
|
||||
) => {}
|
||||
Err(err) => return Err(err.caused_by(trc::location!())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Lets another check run, if `id` is the one that runs now.
|
||||
pub async fn release(data: &Store, id: u32) -> trc::Result<()> {
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.assert_value(running_class(), AssertValue::U32(id));
|
||||
batch.clear(running_class());
|
||||
match data.write(batch.build_all()).await {
|
||||
Ok(_) => Ok(()),
|
||||
Err(err)
|
||||
if matches!(
|
||||
err.as_ref(),
|
||||
trc::EventType::Store(trc::StoreEvent::AssertValueFailed)
|
||||
) =>
|
||||
{
|
||||
Ok(())
|
||||
}
|
||||
Err(err) => Err(err.caused_by(trc::location!())),
|
||||
}
|
||||
}
|
||||
|
||||
/// A check as stored, with the version [`replace`] checks: a hash of the
|
||||
/// stored bytes.
|
||||
struct Versioned(StoreCheck, u64);
|
||||
|
||||
impl Deserialize for Versioned {
|
||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
||||
let Json(check) = Json::<StoreCheck>::deserialize(bytes)?;
|
||||
Ok(Versioned(check, xxhash_rust::xxh3::xxh3_64(bytes)))
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_versioned(data: &Store, id: u32) -> trc::Result<Option<(StoreCheck, u64)>> {
|
||||
Ok(data
|
||||
.get_value::<Versioned>(ValueKey::from(class(id)))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.map(|Versioned(check, version)| (check, version)))
|
||||
}
|
||||
|
||||
/// Reads a check, changes it with `change`, and writes it back, again if
|
||||
/// something else wrote it in between (the running node saving progress
|
||||
/// while an administrator stops it). `change` returns false to leave it as
|
||||
/// it is. None if there's no such check.
|
||||
pub async fn update<F>(data: &Store, id: u32, mut change: F) -> trc::Result<Option<StoreCheck>>
|
||||
where
|
||||
F: FnMut(&mut StoreCheck) -> bool,
|
||||
{
|
||||
loop {
|
||||
let Some((mut check, version)) = get_versioned(data, id).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !change(&mut check) {
|
||||
return Ok(Some(check));
|
||||
}
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.assert_value(class(id), AssertValue::Hash(version));
|
||||
batch.set(class(id), Json(&check).serialize()?);
|
||||
match data.write(batch.build_all()).await {
|
||||
Ok(_) => return Ok(Some(check)),
|
||||
Err(err)
|
||||
if matches!(
|
||||
err.as_ref(),
|
||||
trc::EventType::Store(trc::StoreEvent::AssertValueFailed)
|
||||
) => {}
|
||||
Err(err) => return Err(err.caused_by(trc::location!())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Keeps problems `first..` of a check (SV-5): the caller numbers them and
|
||||
/// stops at [`MAX_PROBLEMS`].
|
||||
pub async fn add_problems(
|
||||
data: &Store,
|
||||
id: u32,
|
||||
first: u64,
|
||||
problems: &[Problem],
|
||||
) -> trc::Result<()> {
|
||||
let mut batch = BatchBuilder::new();
|
||||
for (n, problem) in problems.iter().enumerate() {
|
||||
let number = u32::try_from(first + n as u64).unwrap_or(u32::MAX);
|
||||
batch.set(problem_class(id, number), Json(problem).serialize()?);
|
||||
if batch.is_large_batch() {
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
batch = BatchBuilder::new();
|
||||
}
|
||||
}
|
||||
if !batch.is_empty() {
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A check's kept problems, in the order they were found.
|
||||
pub async fn problems(data: &Store, id: u32) -> trc::Result<Vec<Problem>> {
|
||||
let mut problems = Vec::new();
|
||||
data.iterate(
|
||||
IterateParams::new(
|
||||
ValueKey::from(problem_class(id, 0)),
|
||||
ValueKey::from(problem_class(id, u32::MAX)),
|
||||
),
|
||||
|_, value| {
|
||||
if let Ok(Json(problem)) = Json::<Problem>::deserialize(value) {
|
||||
problems.push(problem);
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
Ok(problems)
|
||||
}
|
||||
|
||||
/// Forgets a check and its problems (SV-1, SV-10).
|
||||
pub async fn remove(data: &Store, id: u32) -> trc::Result<()> {
|
||||
data.delete_range(
|
||||
ValueKey::from(problem_class(id, 0)),
|
||||
ValueKey::from(problem_class(id, u32::MAX)),
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.clear(class(id));
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn keys_stay_clear_of_the_store_version() {
|
||||
for class in [class(1), problem_class(1, 0), running_class()] {
|
||||
let ValueClass::Any(any) = class else {
|
||||
unreachable!()
|
||||
};
|
||||
assert_eq!(any.key[0], b'V');
|
||||
assert_ne!(any.key[1], b's');
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_quiet_check_reads_as_interrupted() {
|
||||
let check = StoreCheck::new(Mode::Quick, "admin".into(), 1, 1_000);
|
||||
assert_eq!(check.status_at(1_000 + STALE_AFTER), Status::Running);
|
||||
assert_eq!(check.status_at(1_001 + STALE_AFTER), Status::Interrupted);
|
||||
let done = StoreCheck {
|
||||
status: Status::Done,
|
||||
..check
|
||||
};
|
||||
assert_eq!(done.status_at(u64::MAX / 2), Status::Done);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn problems_round_trip_as_json() {
|
||||
let problem = Problem {
|
||||
kind: ProblemKind::Missing,
|
||||
account_id: Some(3),
|
||||
account_name: "[email protected]".into(),
|
||||
object: "email".into(),
|
||||
object_id: 7,
|
||||
blob_id: "ab".into(),
|
||||
committed: true,
|
||||
detail: String::new(),
|
||||
};
|
||||
let bytes = Json(&problem).serialize().unwrap();
|
||||
let Json(back) = Json::<Problem>::deserialize(&bytes).unwrap();
|
||||
assert_eq!(back, problem);
|
||||
assert!(
|
||||
String::from_utf8(bytes)
|
||||
.unwrap()
|
||||
.contains("\"accountId\":3")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:BlobStoreChange/get` and `/set` under `urn:inbuxa:jmap`: a change
|
||||
//! of blob store the server noticed at start-up (blob-store-change spec,
|
||||
//! BM-2). The server records them; an update with `action: "copy"` copies
|
||||
//! what the new store lacks from the old one, `action: "stop"` stops that,
|
||||
//! and destroying one dismisses it (BM-5 to BM-7).
|
||||
|
||||
use crate::object::{AnyId, JmapObject, JmapObjectId};
|
||||
use jmap_tools::{Element, Key, Property};
|
||||
use std::{borrow::Cow, str::FromStr};
|
||||
use types::id::Id;
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct BlobStoreChange;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum BlobStoreChangeProperty {
|
||||
Id,
|
||||
From,
|
||||
To,
|
||||
ChangedAt,
|
||||
Node,
|
||||
Status,
|
||||
CopyStartedAt,
|
||||
CopyFinishedAt,
|
||||
CopyStartedBy,
|
||||
CopyNode,
|
||||
BlobsPerSecond,
|
||||
BytesPerSecond,
|
||||
Progress,
|
||||
BlobsChecked,
|
||||
Copied,
|
||||
AlreadyThere,
|
||||
BytesCopied,
|
||||
Missing,
|
||||
Damaged,
|
||||
Unreadable,
|
||||
Problems,
|
||||
Error,
|
||||
Action,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum BlobStoreChangeValue {
|
||||
Id(Id),
|
||||
}
|
||||
|
||||
impl Property for BlobStoreChangeProperty {
|
||||
fn try_parse(parent: Option<&Key<'_, Self>>, value: &str) -> Option<Self> {
|
||||
match parent {
|
||||
None => BlobStoreChangeProperty::parse(value),
|
||||
Some(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_cow(&self) -> Cow<'static, str> {
|
||||
match self {
|
||||
BlobStoreChangeProperty::Id => "id",
|
||||
BlobStoreChangeProperty::From => "from",
|
||||
BlobStoreChangeProperty::To => "to",
|
||||
BlobStoreChangeProperty::ChangedAt => "changedAt",
|
||||
BlobStoreChangeProperty::Node => "node",
|
||||
BlobStoreChangeProperty::Status => "status",
|
||||
BlobStoreChangeProperty::CopyStartedAt => "copyStartedAt",
|
||||
BlobStoreChangeProperty::CopyFinishedAt => "copyFinishedAt",
|
||||
BlobStoreChangeProperty::CopyStartedBy => "copyStartedBy",
|
||||
BlobStoreChangeProperty::CopyNode => "copyNode",
|
||||
BlobStoreChangeProperty::BlobsPerSecond => "blobsPerSecond",
|
||||
BlobStoreChangeProperty::BytesPerSecond => "bytesPerSecond",
|
||||
BlobStoreChangeProperty::Progress => "progress",
|
||||
BlobStoreChangeProperty::BlobsChecked => "blobsChecked",
|
||||
BlobStoreChangeProperty::Copied => "copied",
|
||||
BlobStoreChangeProperty::AlreadyThere => "alreadyThere",
|
||||
BlobStoreChangeProperty::BytesCopied => "bytesCopied",
|
||||
BlobStoreChangeProperty::Missing => "missing",
|
||||
BlobStoreChangeProperty::Damaged => "damaged",
|
||||
BlobStoreChangeProperty::Unreadable => "unreadable",
|
||||
BlobStoreChangeProperty::Problems => "problems",
|
||||
BlobStoreChangeProperty::Error => "error",
|
||||
BlobStoreChangeProperty::Action => "action",
|
||||
}
|
||||
.into()
|
||||
}
|
||||
}
|
||||
|
||||
impl BlobStoreChangeProperty {
|
||||
fn parse(value: &str) -> Option<Self> {
|
||||
hashify::tiny_map!(value.as_bytes(),
|
||||
b"id" => BlobStoreChangeProperty::Id,
|
||||
b"from" => BlobStoreChangeProperty::From,
|
||||
b"to" => BlobStoreChangeProperty::To,
|
||||
b"changedAt" => BlobStoreChangeProperty::ChangedAt,
|
||||
b"node" => BlobStoreChangeProperty::Node,
|
||||
b"status" => BlobStoreChangeProperty::Status,
|
||||
b"copyStartedAt" => BlobStoreChangeProperty::CopyStartedAt,
|
||||
b"copyFinishedAt" => BlobStoreChangeProperty::CopyFinishedAt,
|
||||
b"copyStartedBy" => BlobStoreChangeProperty::CopyStartedBy,
|
||||
b"copyNode" => BlobStoreChangeProperty::CopyNode,
|
||||
b"blobsPerSecond" => BlobStoreChangeProperty::BlobsPerSecond,
|
||||
b"bytesPerSecond" => BlobStoreChangeProperty::BytesPerSecond,
|
||||
b"progress" => BlobStoreChangeProperty::Progress,
|
||||
b"blobsChecked" => BlobStoreChangeProperty::BlobsChecked,
|
||||
b"copied" => BlobStoreChangeProperty::Copied,
|
||||
b"alreadyThere" => BlobStoreChangeProperty::AlreadyThere,
|
||||
b"bytesCopied" => BlobStoreChangeProperty::BytesCopied,
|
||||
b"missing" => BlobStoreChangeProperty::Missing,
|
||||
b"damaged" => BlobStoreChangeProperty::Damaged,
|
||||
b"unreadable" => BlobStoreChangeProperty::Unreadable,
|
||||
b"problems" => BlobStoreChangeProperty::Problems,
|
||||
b"error" => BlobStoreChangeProperty::Error,
|
||||
b"action" => BlobStoreChangeProperty::Action,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for BlobStoreChangeProperty {
|
||||
type Err = ();
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
BlobStoreChangeProperty::parse(s).ok_or(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Element for BlobStoreChangeValue {
|
||||
type Property = BlobStoreChangeProperty;
|
||||
|
||||
fn try_parse<P>(key: &Key<'_, Self::Property>, value: &str) -> Option<Self> {
|
||||
match key {
|
||||
Key::Property(BlobStoreChangeProperty::Id) => {
|
||||
Id::from_str(value).ok().map(BlobStoreChangeValue::Id)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_cow(&self) -> Cow<'static, str> {
|
||||
match self {
|
||||
BlobStoreChangeValue::Id(id) => id.to_string().into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObject for BlobStoreChange {
|
||||
type Property = BlobStoreChangeProperty;
|
||||
|
||||
type Element = BlobStoreChangeValue;
|
||||
|
||||
type Id = Id;
|
||||
|
||||
type Filter = ();
|
||||
|
||||
type Comparator = ();
|
||||
|
||||
type GetArguments = ();
|
||||
|
||||
type SetArguments<'de> = ();
|
||||
|
||||
type QueryArguments = ();
|
||||
|
||||
type CopyArguments = ();
|
||||
|
||||
type ParseArguments = ();
|
||||
|
||||
const ID_PROPERTY: Self::Property = BlobStoreChangeProperty::Id;
|
||||
}
|
||||
|
||||
impl From<Id> for BlobStoreChangeValue {
|
||||
fn from(id: Id) -> Self {
|
||||
BlobStoreChangeValue::Id(id)
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObjectId for BlobStoreChangeValue {
|
||||
fn as_id(&self) -> Option<Id> {
|
||||
match self {
|
||||
BlobStoreChangeValue::Id(id) => Some(*id),
|
||||
}
|
||||
}
|
||||
|
||||
fn as_any_id(&self) -> Option<AnyId> {
|
||||
match self {
|
||||
BlobStoreChangeValue::Id(id) => Some(AnyId::Id(*id)),
|
||||
}
|
||||
}
|
||||
|
||||
fn as_id_ref(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
|
||||
fn try_set_id(&mut self, new_id: AnyId) -> bool {
|
||||
if let AnyId::Id(id) = new_id {
|
||||
*self = BlobStoreChangeValue::Id(id);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObjectId for BlobStoreChangeProperty {
|
||||
fn as_id(&self) -> Option<Id> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_any_id(&self) -> Option<AnyId> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_id_ref(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
|
||||
fn try_set_id(&mut self, _: AnyId) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:StoreCheck/get` and `/set` under `urn:inbuxa:jmap`: checking that
|
||||
//! the store has every blob its data needs (store-check spec, SV-1).
|
||||
//! Creating one starts it; it runs in the background, and `get` says how
|
||||
//! far it got and what it found. An update with `action: "stop"` stops it.
|
||||
|
||||
use crate::object::{AnyId, JmapObject, JmapObjectId};
|
||||
use jmap_tools::{Element, Key, Property};
|
||||
use std::{borrow::Cow, str::FromStr};
|
||||
use types::id::Id;
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct StoreCheck;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum StoreCheckProperty {
|
||||
Id,
|
||||
Mode,
|
||||
BlobsPerSecond,
|
||||
BytesPerSecond,
|
||||
Status,
|
||||
CreatedAt,
|
||||
CreatedBy,
|
||||
FinishedAt,
|
||||
Node,
|
||||
Progress,
|
||||
BlobsChecked,
|
||||
LinksChecked,
|
||||
BytesRead,
|
||||
MessagesChecked,
|
||||
Missing,
|
||||
Damaged,
|
||||
Unlinked,
|
||||
Unreadable,
|
||||
Problems,
|
||||
Error,
|
||||
Action,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum StoreCheckValue {
|
||||
Id(Id),
|
||||
}
|
||||
|
||||
impl Property for StoreCheckProperty {
|
||||
fn try_parse(parent: Option<&Key<'_, Self>>, value: &str) -> Option<Self> {
|
||||
match parent {
|
||||
None => StoreCheckProperty::parse(value),
|
||||
Some(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_cow(&self) -> Cow<'static, str> {
|
||||
match self {
|
||||
StoreCheckProperty::Id => "id",
|
||||
StoreCheckProperty::Mode => "mode",
|
||||
StoreCheckProperty::BlobsPerSecond => "blobsPerSecond",
|
||||
StoreCheckProperty::BytesPerSecond => "bytesPerSecond",
|
||||
StoreCheckProperty::Status => "status",
|
||||
StoreCheckProperty::CreatedAt => "createdAt",
|
||||
StoreCheckProperty::CreatedBy => "createdBy",
|
||||
StoreCheckProperty::FinishedAt => "finishedAt",
|
||||
StoreCheckProperty::Node => "node",
|
||||
StoreCheckProperty::Progress => "progress",
|
||||
StoreCheckProperty::BlobsChecked => "blobsChecked",
|
||||
StoreCheckProperty::LinksChecked => "linksChecked",
|
||||
StoreCheckProperty::BytesRead => "bytesRead",
|
||||
StoreCheckProperty::MessagesChecked => "messagesChecked",
|
||||
StoreCheckProperty::Missing => "missing",
|
||||
StoreCheckProperty::Damaged => "damaged",
|
||||
StoreCheckProperty::Unlinked => "unlinked",
|
||||
StoreCheckProperty::Unreadable => "unreadable",
|
||||
StoreCheckProperty::Problems => "problems",
|
||||
StoreCheckProperty::Error => "error",
|
||||
StoreCheckProperty::Action => "action",
|
||||
}
|
||||
.into()
|
||||
}
|
||||
}
|
||||
|
||||
impl StoreCheckProperty {
|
||||
fn parse(value: &str) -> Option<Self> {
|
||||
hashify::tiny_map!(value.as_bytes(),
|
||||
b"id" => StoreCheckProperty::Id,
|
||||
b"mode" => StoreCheckProperty::Mode,
|
||||
b"blobsPerSecond" => StoreCheckProperty::BlobsPerSecond,
|
||||
b"bytesPerSecond" => StoreCheckProperty::BytesPerSecond,
|
||||
b"status" => StoreCheckProperty::Status,
|
||||
b"createdAt" => StoreCheckProperty::CreatedAt,
|
||||
b"createdBy" => StoreCheckProperty::CreatedBy,
|
||||
b"finishedAt" => StoreCheckProperty::FinishedAt,
|
||||
b"node" => StoreCheckProperty::Node,
|
||||
b"progress" => StoreCheckProperty::Progress,
|
||||
b"blobsChecked" => StoreCheckProperty::BlobsChecked,
|
||||
b"linksChecked" => StoreCheckProperty::LinksChecked,
|
||||
b"bytesRead" => StoreCheckProperty::BytesRead,
|
||||
b"messagesChecked" => StoreCheckProperty::MessagesChecked,
|
||||
b"missing" => StoreCheckProperty::Missing,
|
||||
b"damaged" => StoreCheckProperty::Damaged,
|
||||
b"unlinked" => StoreCheckProperty::Unlinked,
|
||||
b"unreadable" => StoreCheckProperty::Unreadable,
|
||||
b"problems" => StoreCheckProperty::Problems,
|
||||
b"error" => StoreCheckProperty::Error,
|
||||
b"action" => StoreCheckProperty::Action,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for StoreCheckProperty {
|
||||
type Err = ();
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
StoreCheckProperty::parse(s).ok_or(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Element for StoreCheckValue {
|
||||
type Property = StoreCheckProperty;
|
||||
|
||||
fn try_parse<P>(key: &Key<'_, Self::Property>, value: &str) -> Option<Self> {
|
||||
match key {
|
||||
Key::Property(StoreCheckProperty::Id) => {
|
||||
Id::from_str(value).ok().map(StoreCheckValue::Id)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_cow(&self) -> Cow<'static, str> {
|
||||
match self {
|
||||
StoreCheckValue::Id(id) => id.to_string().into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObject for StoreCheck {
|
||||
type Property = StoreCheckProperty;
|
||||
|
||||
type Element = StoreCheckValue;
|
||||
|
||||
type Id = Id;
|
||||
|
||||
type Filter = ();
|
||||
|
||||
type Comparator = ();
|
||||
|
||||
type GetArguments = ();
|
||||
|
||||
type SetArguments<'de> = ();
|
||||
|
||||
type QueryArguments = ();
|
||||
|
||||
type CopyArguments = ();
|
||||
|
||||
type ParseArguments = ();
|
||||
|
||||
const ID_PROPERTY: Self::Property = StoreCheckProperty::Id;
|
||||
}
|
||||
|
||||
impl From<Id> for StoreCheckValue {
|
||||
fn from(id: Id) -> Self {
|
||||
StoreCheckValue::Id(id)
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObjectId for StoreCheckValue {
|
||||
fn as_id(&self) -> Option<Id> {
|
||||
match self {
|
||||
StoreCheckValue::Id(id) => Some(*id),
|
||||
}
|
||||
}
|
||||
|
||||
fn as_any_id(&self) -> Option<AnyId> {
|
||||
match self {
|
||||
StoreCheckValue::Id(id) => Some(AnyId::Id(*id)),
|
||||
}
|
||||
}
|
||||
|
||||
fn as_id_ref(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
|
||||
fn try_set_id(&mut self, new_id: AnyId) -> bool {
|
||||
if let AnyId::Id(id) = new_id {
|
||||
*self = StoreCheckValue::Id(id);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObjectId for StoreCheckProperty {
|
||||
fn as_id(&self) -> Option<Id> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_any_id(&self) -> Option<AnyId> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_id_ref(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
|
||||
fn try_set_id(&mut self, _: AnyId) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:StoreSnapshot/get` and `/set` under `urn:inbuxa:jmap`: a snapshot
|
||||
//! of the store, taken without stopping the server (store-snapshot spec,
|
||||
//! SN-1). Creating one takes it in the background; `get` lists the
|
||||
//! snapshots on disk and says how the one being taken stands. Destroying one
|
||||
//! deletes its directory.
|
||||
|
||||
use crate::object::{AnyId, JmapObject, JmapObjectId};
|
||||
use jmap_tools::{Element, Key, Property};
|
||||
use std::{borrow::Cow, str::FromStr};
|
||||
use types::id::Id;
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct StoreSnapshot;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum StoreSnapshotProperty {
|
||||
Id,
|
||||
Status,
|
||||
CreatedAt,
|
||||
CreatedBy,
|
||||
FinishedAt,
|
||||
Node,
|
||||
Name,
|
||||
Path,
|
||||
Holds,
|
||||
Size,
|
||||
BlobFiles,
|
||||
Error,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum StoreSnapshotValue {
|
||||
Id(Id),
|
||||
}
|
||||
|
||||
impl Property for StoreSnapshotProperty {
|
||||
fn try_parse(parent: Option<&Key<'_, Self>>, value: &str) -> Option<Self> {
|
||||
match parent {
|
||||
None => StoreSnapshotProperty::parse(value),
|
||||
Some(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_cow(&self) -> Cow<'static, str> {
|
||||
match self {
|
||||
StoreSnapshotProperty::Id => "id",
|
||||
StoreSnapshotProperty::Status => "status",
|
||||
StoreSnapshotProperty::CreatedAt => "createdAt",
|
||||
StoreSnapshotProperty::CreatedBy => "createdBy",
|
||||
StoreSnapshotProperty::FinishedAt => "finishedAt",
|
||||
StoreSnapshotProperty::Node => "node",
|
||||
StoreSnapshotProperty::Name => "name",
|
||||
StoreSnapshotProperty::Path => "path",
|
||||
StoreSnapshotProperty::Holds => "holds",
|
||||
StoreSnapshotProperty::Size => "size",
|
||||
StoreSnapshotProperty::BlobFiles => "blobFiles",
|
||||
StoreSnapshotProperty::Error => "error",
|
||||
}
|
||||
.into()
|
||||
}
|
||||
}
|
||||
|
||||
impl StoreSnapshotProperty {
|
||||
fn parse(value: &str) -> Option<Self> {
|
||||
hashify::tiny_map!(value.as_bytes(),
|
||||
b"id" => StoreSnapshotProperty::Id,
|
||||
b"status" => StoreSnapshotProperty::Status,
|
||||
b"createdAt" => StoreSnapshotProperty::CreatedAt,
|
||||
b"createdBy" => StoreSnapshotProperty::CreatedBy,
|
||||
b"finishedAt" => StoreSnapshotProperty::FinishedAt,
|
||||
b"node" => StoreSnapshotProperty::Node,
|
||||
b"name" => StoreSnapshotProperty::Name,
|
||||
b"path" => StoreSnapshotProperty::Path,
|
||||
b"holds" => StoreSnapshotProperty::Holds,
|
||||
b"size" => StoreSnapshotProperty::Size,
|
||||
b"blobFiles" => StoreSnapshotProperty::BlobFiles,
|
||||
b"error" => StoreSnapshotProperty::Error,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for StoreSnapshotProperty {
|
||||
type Err = ();
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
StoreSnapshotProperty::parse(s).ok_or(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Element for StoreSnapshotValue {
|
||||
type Property = StoreSnapshotProperty;
|
||||
|
||||
fn try_parse<P>(key: &Key<'_, Self::Property>, value: &str) -> Option<Self> {
|
||||
match key {
|
||||
Key::Property(StoreSnapshotProperty::Id) => {
|
||||
Id::from_str(value).ok().map(StoreSnapshotValue::Id)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_cow(&self) -> Cow<'static, str> {
|
||||
match self {
|
||||
StoreSnapshotValue::Id(id) => id.to_string().into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObject for StoreSnapshot {
|
||||
type Property = StoreSnapshotProperty;
|
||||
|
||||
type Element = StoreSnapshotValue;
|
||||
|
||||
type Id = Id;
|
||||
|
||||
type Filter = ();
|
||||
|
||||
type Comparator = ();
|
||||
|
||||
type GetArguments = ();
|
||||
|
||||
type SetArguments<'de> = ();
|
||||
|
||||
type QueryArguments = ();
|
||||
|
||||
type CopyArguments = ();
|
||||
|
||||
type ParseArguments = ();
|
||||
|
||||
const ID_PROPERTY: Self::Property = StoreSnapshotProperty::Id;
|
||||
}
|
||||
|
||||
impl From<Id> for StoreSnapshotValue {
|
||||
fn from(id: Id) -> Self {
|
||||
StoreSnapshotValue::Id(id)
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObjectId for StoreSnapshotValue {
|
||||
fn as_id(&self) -> Option<Id> {
|
||||
match self {
|
||||
StoreSnapshotValue::Id(id) => Some(*id),
|
||||
}
|
||||
}
|
||||
|
||||
fn as_any_id(&self) -> Option<AnyId> {
|
||||
match self {
|
||||
StoreSnapshotValue::Id(id) => Some(AnyId::Id(*id)),
|
||||
}
|
||||
}
|
||||
|
||||
fn as_id_ref(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
|
||||
fn try_set_id(&mut self, new_id: AnyId) -> bool {
|
||||
if let AnyId::Id(id) = new_id {
|
||||
*self = StoreSnapshotValue::Id(id);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObjectId for StoreSnapshotProperty {
|
||||
fn as_id(&self) -> Option<Id> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_any_id(&self) -> Option<AnyId> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_id_ref(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
|
||||
fn try_set_id(&mut self, _: AnyId) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,7 @@ pub mod identity;
|
||||
pub mod inbuxa_account_lock; // inbuxa: account lock with delegation
|
||||
pub mod inbuxa_ai_limits; // inbuxa: AI spam classification
|
||||
pub mod inbuxa_audit; // inbuxa: the audit log
|
||||
pub mod inbuxa_blob_store_change; // inbuxa: a change of blob store
|
||||
pub mod inbuxa_data_inventory; // inbuxa: personal-data catalog
|
||||
pub mod inbuxa_deleted_account; // inbuxa: undelete
|
||||
pub mod inbuxa_deliverability_report; // inbuxa: the deliverability check
|
||||
@@ -47,6 +48,8 @@ pub mod inbuxa_scheduled_report; // inbuxa: scheduled reports
|
||||
pub mod inbuxa_scheduled_report_settings; // inbuxa: scheduled reports
|
||||
pub mod inbuxa_security_acceptance; // inbuxa: accepted security to-do items
|
||||
pub mod inbuxa_sharing_policy; // inbuxa: MA-C, who may share mail
|
||||
pub mod inbuxa_store_check; // inbuxa: checking the store's blobs
|
||||
pub mod inbuxa_store_snapshot; // inbuxa: a snapshot of the store
|
||||
pub mod inbuxa_tenant_protocol_policy; // inbuxa: legacy protocols off, per tenant
|
||||
pub mod mailbox;
|
||||
pub mod participant_identity;
|
||||
|
||||
@@ -88,6 +88,15 @@ impl Response<'_> {
|
||||
GetResponseMethod::MailRule(response) => {
|
||||
response.eval_jptr(path, &mut results)
|
||||
}
|
||||
GetResponseMethod::StoreCheck(response) => {
|
||||
response.eval_jptr(path, &mut results)
|
||||
}
|
||||
GetResponseMethod::StoreSnapshot(response) => {
|
||||
response.eval_jptr(path, &mut results)
|
||||
}
|
||||
GetResponseMethod::BlobStoreChange(response) => {
|
||||
response.eval_jptr(path, &mut results)
|
||||
}
|
||||
GetResponseMethod::SecurityAcceptance(response) => {
|
||||
response.eval_jptr(path, &mut results)
|
||||
}
|
||||
|
||||
@@ -55,6 +55,9 @@ impl Response<'_> {
|
||||
GetRequestMethod::AccountLock(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::LegalHold(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::MailRule(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::StoreCheck(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::StoreSnapshot(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::BlobStoreChange(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::SecurityAcceptance(request) => {
|
||||
request.resolve_references(self)?
|
||||
}
|
||||
@@ -149,6 +152,15 @@ impl Response<'_> {
|
||||
SetRequestMethod::MailRule(request) => {
|
||||
request.resolve_references(self, 1, false)?
|
||||
}
|
||||
SetRequestMethod::StoreCheck(request) => {
|
||||
request.resolve_references(self, 1, false)?
|
||||
}
|
||||
SetRequestMethod::StoreSnapshot(request) => {
|
||||
request.resolve_references(self, 1, false)?
|
||||
}
|
||||
SetRequestMethod::BlobStoreChange(request) => {
|
||||
request.resolve_references(self, 1, false)?
|
||||
}
|
||||
SetRequestMethod::SecurityAcceptance(request) => {
|
||||
request.resolve_references(self, 1, false)?
|
||||
}
|
||||
|
||||
@@ -68,6 +68,12 @@ pub enum MethodObject {
|
||||
ProtocolPolicy,
|
||||
// inbuxa: DLP and mail flow rules
|
||||
MailRule,
|
||||
// inbuxa: checking the store's blobs
|
||||
StoreCheck,
|
||||
// inbuxa: a snapshot of the store
|
||||
StoreSnapshot,
|
||||
// inbuxa: a change of blob store
|
||||
BlobStoreChange,
|
||||
// inbuxa: accepted security to-do items
|
||||
SecurityAcceptance,
|
||||
// inbuxa: the deliverability check
|
||||
@@ -124,6 +130,9 @@ impl MethodObject {
|
||||
| MethodObject::LegalHold
|
||||
| MethodObject::HoldExport
|
||||
| MethodObject::MailRule
|
||||
| MethodObject::StoreCheck
|
||||
| MethodObject::StoreSnapshot
|
||||
| MethodObject::BlobStoreChange
|
||||
| MethodObject::SecurityAcceptance
|
||||
| MethodObject::HeldMessage
|
||||
| MethodObject::DeliverabilityReport
|
||||
@@ -336,6 +345,12 @@ impl MethodName {
|
||||
(MethodFunction::Set, MethodObject::LegalHold) => "inbuxa:LegalHold/set",
|
||||
(MethodFunction::Get, MethodObject::MailRule) => "inbuxa:MailRule/get",
|
||||
(MethodFunction::Set, MethodObject::MailRule) => "inbuxa:MailRule/set",
|
||||
(MethodFunction::Get, MethodObject::StoreCheck) => "inbuxa:StoreCheck/get",
|
||||
(MethodFunction::Set, MethodObject::StoreCheck) => "inbuxa:StoreCheck/set",
|
||||
(MethodFunction::Get, MethodObject::StoreSnapshot) => "inbuxa:StoreSnapshot/get",
|
||||
(MethodFunction::Set, MethodObject::StoreSnapshot) => "inbuxa:StoreSnapshot/set",
|
||||
(MethodFunction::Get, MethodObject::BlobStoreChange) => "inbuxa:BlobStoreChange/get",
|
||||
(MethodFunction::Set, MethodObject::BlobStoreChange) => "inbuxa:BlobStoreChange/set",
|
||||
(MethodFunction::Get, MethodObject::SecurityAcceptance) => {
|
||||
"inbuxa:SecurityAcceptance/get"
|
||||
}
|
||||
@@ -534,6 +549,12 @@ impl MethodName {
|
||||
"inbuxa:LegalHold/set" => (MethodObject::LegalHold, MethodFunction::Set),
|
||||
"inbuxa:MailRule/get" => (MethodObject::MailRule, MethodFunction::Get),
|
||||
"inbuxa:MailRule/set" => (MethodObject::MailRule, MethodFunction::Set),
|
||||
"inbuxa:StoreCheck/get" => (MethodObject::StoreCheck, MethodFunction::Get),
|
||||
"inbuxa:StoreCheck/set" => (MethodObject::StoreCheck, MethodFunction::Set),
|
||||
"inbuxa:StoreSnapshot/get" => (MethodObject::StoreSnapshot, MethodFunction::Get),
|
||||
"inbuxa:StoreSnapshot/set" => (MethodObject::StoreSnapshot, MethodFunction::Set),
|
||||
"inbuxa:BlobStoreChange/get" => (MethodObject::BlobStoreChange, MethodFunction::Get),
|
||||
"inbuxa:BlobStoreChange/set" => (MethodObject::BlobStoreChange, MethodFunction::Set),
|
||||
"inbuxa:SecurityAcceptance/get" => (MethodObject::SecurityAcceptance, MethodFunction::Get),
|
||||
"inbuxa:SecurityAcceptance/set" => (MethodObject::SecurityAcceptance, MethodFunction::Set),
|
||||
"inbuxa:DeliverabilityReport/get" => (MethodObject::DeliverabilityReport, MethodFunction::Get),
|
||||
@@ -630,6 +651,9 @@ impl Display for MethodObject {
|
||||
MethodObject::AccountLock => "inbuxa:AccountLock",
|
||||
MethodObject::LegalHold => "inbuxa:LegalHold",
|
||||
MethodObject::MailRule => "inbuxa:MailRule",
|
||||
MethodObject::StoreCheck => "inbuxa:StoreCheck",
|
||||
MethodObject::StoreSnapshot => "inbuxa:StoreSnapshot",
|
||||
MethodObject::BlobStoreChange => "inbuxa:BlobStoreChange",
|
||||
MethodObject::SecurityAcceptance => "inbuxa:SecurityAcceptance",
|
||||
MethodObject::DeliverabilityReport => "inbuxa:DeliverabilityReport",
|
||||
MethodObject::DeliverabilitySettings => "inbuxa:DeliverabilitySettings",
|
||||
|
||||
@@ -127,6 +127,9 @@ pub enum GetRequestMethod {
|
||||
AccountLock(Box<GetRequest<crate::object::inbuxa_account_lock::AccountLock>>),
|
||||
LegalHold(Box<GetRequest<crate::object::inbuxa_legal_hold::LegalHold>>),
|
||||
MailRule(Box<GetRequest<crate::object::inbuxa_mail_rule::MailRule>>),
|
||||
StoreCheck(Box<GetRequest<crate::object::inbuxa_store_check::StoreCheck>>),
|
||||
StoreSnapshot(Box<GetRequest<crate::object::inbuxa_store_snapshot::StoreSnapshot>>),
|
||||
BlobStoreChange(Box<GetRequest<crate::object::inbuxa_blob_store_change::BlobStoreChange>>),
|
||||
SecurityAcceptance(
|
||||
Box<GetRequest<crate::object::inbuxa_security_acceptance::SecurityAcceptance>>,
|
||||
),
|
||||
@@ -183,6 +186,9 @@ pub enum SetRequestMethod<'x> {
|
||||
AccountLock(Box<SetRequest<'x, crate::object::inbuxa_account_lock::AccountLock>>),
|
||||
LegalHold(Box<SetRequest<'x, crate::object::inbuxa_legal_hold::LegalHold>>),
|
||||
MailRule(Box<SetRequest<'x, crate::object::inbuxa_mail_rule::MailRule>>),
|
||||
StoreCheck(Box<SetRequest<'x, crate::object::inbuxa_store_check::StoreCheck>>),
|
||||
StoreSnapshot(Box<SetRequest<'x, crate::object::inbuxa_store_snapshot::StoreSnapshot>>),
|
||||
BlobStoreChange(Box<SetRequest<'x, crate::object::inbuxa_blob_store_change::BlobStoreChange>>),
|
||||
SecurityAcceptance(
|
||||
Box<SetRequest<'x, crate::object::inbuxa_security_acceptance::SecurityAcceptance>>,
|
||||
),
|
||||
|
||||
@@ -689,6 +689,51 @@ impl<'de> Visitor<'de> for CallVisitor {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
// inbuxa: checking the store's blobs
|
||||
(MethodFunction::Get, MethodObject::StoreCheck) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::StoreCheck(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
Ok(None) => {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
(MethodFunction::Set, MethodObject::StoreCheck) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::StoreCheck(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
Ok(None) => {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
// inbuxa: a snapshot of the store
|
||||
(MethodFunction::Get, MethodObject::StoreSnapshot) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::StoreSnapshot(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
Ok(None) => {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
(MethodFunction::Set, MethodObject::StoreSnapshot) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::StoreSnapshot(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
Ok(None) => {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
// inbuxa: a change of blob store
|
||||
(MethodFunction::Get, MethodObject::BlobStoreChange) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::BlobStoreChange(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
Ok(None) => {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
(MethodFunction::Set, MethodObject::BlobStoreChange) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::BlobStoreChange(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
Ok(None) => {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
// inbuxa: accepted security to-do items
|
||||
(MethodFunction::Get, MethodObject::SecurityAcceptance) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::SecurityAcceptance(value)),
|
||||
|
||||
@@ -112,6 +112,9 @@ pub enum GetResponseMethod {
|
||||
AccountLock(GetResponse<crate::object::inbuxa_account_lock::AccountLock>),
|
||||
LegalHold(GetResponse<crate::object::inbuxa_legal_hold::LegalHold>),
|
||||
MailRule(GetResponse<crate::object::inbuxa_mail_rule::MailRule>),
|
||||
StoreCheck(GetResponse<crate::object::inbuxa_store_check::StoreCheck>),
|
||||
StoreSnapshot(GetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>),
|
||||
BlobStoreChange(GetResponse<crate::object::inbuxa_blob_store_change::BlobStoreChange>),
|
||||
SecurityAcceptance(GetResponse<crate::object::inbuxa_security_acceptance::SecurityAcceptance>),
|
||||
DeliverabilityReport(
|
||||
GetResponse<crate::object::inbuxa_deliverability_report::DeliverabilityReport>,
|
||||
@@ -166,6 +169,9 @@ pub enum SetResponseMethod {
|
||||
AccountLock(Box<SetResponse<crate::object::inbuxa_account_lock::AccountLock>>),
|
||||
LegalHold(Box<SetResponse<crate::object::inbuxa_legal_hold::LegalHold>>),
|
||||
MailRule(Box<SetResponse<crate::object::inbuxa_mail_rule::MailRule>>),
|
||||
StoreCheck(Box<SetResponse<crate::object::inbuxa_store_check::StoreCheck>>),
|
||||
StoreSnapshot(Box<SetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>>),
|
||||
BlobStoreChange(Box<SetResponse<crate::object::inbuxa_blob_store_change::BlobStoreChange>>),
|
||||
SecurityAcceptance(
|
||||
Box<SetResponse<crate::object::inbuxa_security_acceptance::SecurityAcceptance>>,
|
||||
),
|
||||
@@ -1033,6 +1039,53 @@ impl<'x> From<SetResponse<crate::object::inbuxa_mail_rule::MailRule>> for Respon
|
||||
}
|
||||
}
|
||||
|
||||
// inbuxa: checking the store's blobs
|
||||
impl<'x> From<GetResponse<crate::object::inbuxa_store_check::StoreCheck>> for ResponseMethod<'x> {
|
||||
fn from(value: GetResponse<crate::object::inbuxa_store_check::StoreCheck>) -> Self {
|
||||
ResponseMethod::Get(GetResponseMethod::StoreCheck(value))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> From<SetResponse<crate::object::inbuxa_store_check::StoreCheck>> for ResponseMethod<'x> {
|
||||
fn from(value: SetResponse<crate::object::inbuxa_store_check::StoreCheck>) -> Self {
|
||||
ResponseMethod::Set(SetResponseMethod::StoreCheck(Box::new(value)))
|
||||
}
|
||||
}
|
||||
|
||||
// inbuxa: a snapshot of the store
|
||||
impl<'x> From<GetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>>
|
||||
for ResponseMethod<'x>
|
||||
{
|
||||
fn from(value: GetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>) -> Self {
|
||||
ResponseMethod::Get(GetResponseMethod::StoreSnapshot(value))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> From<SetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>>
|
||||
for ResponseMethod<'x>
|
||||
{
|
||||
fn from(value: SetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>) -> Self {
|
||||
ResponseMethod::Set(SetResponseMethod::StoreSnapshot(Box::new(value)))
|
||||
}
|
||||
}
|
||||
|
||||
// inbuxa: a change of blob store
|
||||
impl<'x> From<GetResponse<crate::object::inbuxa_blob_store_change::BlobStoreChange>>
|
||||
for ResponseMethod<'x>
|
||||
{
|
||||
fn from(value: GetResponse<crate::object::inbuxa_blob_store_change::BlobStoreChange>) -> Self {
|
||||
ResponseMethod::Get(GetResponseMethod::BlobStoreChange(value))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> From<SetResponse<crate::object::inbuxa_blob_store_change::BlobStoreChange>>
|
||||
for ResponseMethod<'x>
|
||||
{
|
||||
fn from(value: SetResponse<crate::object::inbuxa_blob_store_change::BlobStoreChange>) -> Self {
|
||||
ResponseMethod::Set(SetResponseMethod::BlobStoreChange(Box::new(value)))
|
||||
}
|
||||
}
|
||||
|
||||
// inbuxa: journaling
|
||||
impl<'x> From<GetResponse<crate::object::inbuxa_journal_entry::JournalEntry>>
|
||||
for ResponseMethod<'x>
|
||||
|
||||
@@ -116,6 +116,12 @@ impl JmapAuthorization for AccessToken {
|
||||
Permission::SysDlpPolicyGet
|
||||
}
|
||||
}
|
||||
// inbuxa: store-check spec, SV-9
|
||||
GetRequestMethod::StoreCheck(_) => Permission::SysStoreCheck,
|
||||
// inbuxa: store-snapshot spec, SN-8
|
||||
GetRequestMethod::StoreSnapshot(_) => Permission::SysStoreSnapshot,
|
||||
// inbuxa: blob-store-change spec, BM-9
|
||||
GetRequestMethod::BlobStoreChange(_) => Permission::SysBlobStoreChange,
|
||||
// inbuxa: journaling (JR-18)
|
||||
GetRequestMethod::Journal(_) => Permission::SysJournalGet,
|
||||
GetRequestMethod::JournalEntry(_) => Permission::SysJournalSearch,
|
||||
@@ -310,6 +316,30 @@ impl JmapAuthorization for AccessToken {
|
||||
.details("You are not authorized to change mail rules"))
|
||||
}
|
||||
}
|
||||
// inbuxa: store-check spec, SV-9
|
||||
SetRequestMethod::StoreCheck(s) => validate_set(
|
||||
s,
|
||||
self,
|
||||
Permission::SysStoreCheck,
|
||||
Permission::SysStoreCheck,
|
||||
Permission::SysStoreCheck,
|
||||
),
|
||||
// inbuxa: store-snapshot spec, SN-8
|
||||
SetRequestMethod::StoreSnapshot(s) => validate_set(
|
||||
s,
|
||||
self,
|
||||
Permission::SysStoreSnapshot,
|
||||
Permission::SysStoreSnapshot,
|
||||
Permission::SysStoreSnapshot,
|
||||
),
|
||||
// inbuxa: blob-store-change spec, BM-9
|
||||
SetRequestMethod::BlobStoreChange(s) => validate_set(
|
||||
s,
|
||||
self,
|
||||
Permission::SysBlobStoreChange,
|
||||
Permission::SysBlobStoreChange,
|
||||
Permission::SysBlobStoreChange,
|
||||
),
|
||||
// inbuxa: journaling (JR-18)
|
||||
SetRequestMethod::Journal(s) => validate_set(
|
||||
s,
|
||||
@@ -566,6 +596,9 @@ impl JmapAuthorization for AccessToken {
|
||||
| MethodObject::LegalHold
|
||||
| MethodObject::HoldExport
|
||||
| MethodObject::MailRule
|
||||
| MethodObject::StoreCheck
|
||||
| MethodObject::StoreSnapshot
|
||||
| MethodObject::BlobStoreChange
|
||||
| MethodObject::SecurityAcceptance
|
||||
| MethodObject::DeliverabilityReport
|
||||
| MethodObject::DeliverabilitySettings
|
||||
|
||||
@@ -290,6 +290,15 @@ impl RequestHandler for Server {
|
||||
SetResponseMethod::MailRule(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
SetResponseMethod::StoreCheck(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
SetResponseMethod::StoreSnapshot(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
SetResponseMethod::BlobStoreChange(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
SetResponseMethod::SecurityAcceptance(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
@@ -556,6 +565,27 @@ impl RequestHandler for Server {
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: checking the store's blobs
|
||||
GetRequestMethod::StoreCheck(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::store_check::get(self, access_token, *req)
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: a snapshot of the store
|
||||
GetRequestMethod::StoreSnapshot(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::store_snapshot::get(self, access_token, *req)
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: a change of blob store
|
||||
GetRequestMethod::BlobStoreChange(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::blob_store_change::get(self, access_token, *req)
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: accepted security to-do items
|
||||
GetRequestMethod::SecurityAcceptance(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
@@ -1112,6 +1142,60 @@ impl RequestHandler for Server {
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: store-check spec, SV-9: starting and stopping are audited
|
||||
SetRequestMethod::StoreCheck(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::audit::recorded(
|
||||
self,
|
||||
access_token,
|
||||
session,
|
||||
&method_name.obj.to_string(),
|
||||
None,
|
||||
None,
|
||||
*req,
|
||||
|req| Box::pin(crate::inbuxa::store_check::set(self, access_token, req)),
|
||||
)
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: store-snapshot spec, SN-8: taking and destroying are audited
|
||||
SetRequestMethod::StoreSnapshot(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::audit::recorded(
|
||||
self,
|
||||
access_token,
|
||||
session,
|
||||
&method_name.obj.to_string(),
|
||||
None,
|
||||
None,
|
||||
*req,
|
||||
|req| Box::pin(crate::inbuxa::store_snapshot::set(self, access_token, req)),
|
||||
)
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: blob-store-change spec, BM-9: copying, stopping and dismissing are audited
|
||||
SetRequestMethod::BlobStoreChange(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::audit::recorded(
|
||||
self,
|
||||
access_token,
|
||||
session,
|
||||
&method_name.obj.to_string(),
|
||||
None,
|
||||
None,
|
||||
*req,
|
||||
|req| {
|
||||
Box::pin(crate::inbuxa::blob_store_change::set(
|
||||
self,
|
||||
access_token,
|
||||
req,
|
||||
))
|
||||
},
|
||||
)
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: SS-26, every acceptance made or removed is in the
|
||||
// audit log
|
||||
SetRequestMethod::SecurityAcceptance(mut req) => {
|
||||
|
||||
@@ -431,6 +431,9 @@ impl IntermediateChangesResponse {
|
||||
| MethodObject::LegalHold
|
||||
| MethodObject::HoldExport
|
||||
| MethodObject::MailRule
|
||||
| MethodObject::StoreCheck
|
||||
| MethodObject::StoreSnapshot
|
||||
| MethodObject::BlobStoreChange
|
||||
| MethodObject::SecurityAcceptance
|
||||
| MethodObject::DeliverabilityReport
|
||||
| MethodObject::DeliverabilitySettings
|
||||
|
||||
@@ -488,6 +488,51 @@ async fn fork_current(server: &Server, object: &str, id: &MaybeInvalid<Id>) -> O
|
||||
}
|
||||
MaybeInvalid::Invalid(_) => None,
|
||||
},
|
||||
// A store check: what kind, and how it stands (store-check spec, SV-9)
|
||||
// A change of blob store: where from and to, and how it stands;
|
||||
// never the old store's settings (blob-store-change spec, BM-9)
|
||||
"inbuxa:BlobStoreChange" => match id {
|
||||
MaybeInvalid::Value(id) => {
|
||||
inbuxa_features::blob_move::get(data, u32::try_from(id.id()).ok()?)
|
||||
.await
|
||||
.ok()?
|
||||
.map(|c| {
|
||||
serde_json::json!({
|
||||
"from": c.from,
|
||||
"to": c.to,
|
||||
"status": c.status.as_str(),
|
||||
})
|
||||
})
|
||||
}
|
||||
MaybeInvalid::Invalid(_) => None,
|
||||
},
|
||||
"inbuxa:StoreCheck" => match id {
|
||||
MaybeInvalid::Value(id) => {
|
||||
inbuxa_features::store_check::get(data, u32::try_from(id.id()).ok()?)
|
||||
.await
|
||||
.ok()?
|
||||
.map(|c| {
|
||||
serde_json::json!({
|
||||
"mode": c.mode.as_str(),
|
||||
"status": c.status.as_str(),
|
||||
})
|
||||
})
|
||||
}
|
||||
MaybeInvalid::Invalid(_) => None,
|
||||
},
|
||||
// A snapshot: its directory and how it stands (store-snapshot spec,
|
||||
// SN-8)
|
||||
"inbuxa:StoreSnapshot" => match id {
|
||||
MaybeInvalid::Value(id) => services::inbuxa_store_snapshot::find(server, id.id())
|
||||
.await
|
||||
.map(|s| {
|
||||
serde_json::json!({
|
||||
"name": s.name,
|
||||
"status": s.status.as_str(),
|
||||
})
|
||||
}),
|
||||
MaybeInvalid::Invalid(_) => None,
|
||||
},
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,324 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:BlobStoreChange` (blob-store-change spec): the changes of blob
|
||||
//! store the server noticed at start-up, copying what the new store lacks
|
||||
//! from the old one, stopping that, and dismissing a change. The copy is
|
||||
//! `services::inbuxa_blob_move`. Copying, stopping and dismissing are
|
||||
//! audited by the request layer (BM-9).
|
||||
|
||||
use super::json::json_to_value;
|
||||
use common::{Server, auth::AccessToken};
|
||||
use inbuxa_features::blob_move::{self as model, BlobStoreChange as Stored, Refusal, Status};
|
||||
use jmap_proto::{
|
||||
error::set::SetError,
|
||||
method::{
|
||||
get::{GetRequest, GetResponse},
|
||||
set::{SetRequest, SetResponse},
|
||||
},
|
||||
object::inbuxa_blob_store_change::{
|
||||
BlobStoreChange, BlobStoreChangeProperty as P, BlobStoreChangeValue,
|
||||
},
|
||||
request::IntoValid,
|
||||
types::date::UTCDate,
|
||||
};
|
||||
use jmap_tools::{Key, Map, Value};
|
||||
use serde_json::json;
|
||||
use services::inbuxa_blob_move as copies;
|
||||
use store::write::now;
|
||||
use types::id::Id;
|
||||
|
||||
type CValue = Value<'static, P, BlobStoreChangeValue>;
|
||||
|
||||
const ALL: &[P] = &[
|
||||
P::Id,
|
||||
P::From,
|
||||
P::To,
|
||||
P::ChangedAt,
|
||||
P::Node,
|
||||
P::Status,
|
||||
P::CopyStartedAt,
|
||||
P::CopyFinishedAt,
|
||||
P::CopyStartedBy,
|
||||
P::CopyNode,
|
||||
P::BlobsPerSecond,
|
||||
P::BytesPerSecond,
|
||||
P::Progress,
|
||||
P::BlobsChecked,
|
||||
P::Copied,
|
||||
P::AlreadyThere,
|
||||
P::BytesCopied,
|
||||
P::Missing,
|
||||
P::Damaged,
|
||||
P::Unreadable,
|
||||
P::Problems,
|
||||
P::Error,
|
||||
];
|
||||
|
||||
/// BM-9: a change is about the whole server's blobs, so only the server's
|
||||
/// own administrators see and act on one.
|
||||
fn server_level(access_token: &AccessToken) -> trc::Result<()> {
|
||||
if access_token.tenant_id().is_some() {
|
||||
Err(trc::JmapEvent::Forbidden
|
||||
.into_err()
|
||||
.details("A blob store change is about the whole server, not a tenant."))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn date(seconds: u64) -> serde_json::Value {
|
||||
json!(UTCDate::from_timestamp(seconds as i64).to_string())
|
||||
}
|
||||
|
||||
async fn to_value(server: &Server, c: &Stored, properties: &[P], now: u64) -> trc::Result<CValue> {
|
||||
let status = c.status_at(now);
|
||||
let copy = c.copy.as_ref();
|
||||
let p = copy.map(|copy| copy.progress.clone()).unwrap_or_default();
|
||||
let mut out = Map::with_capacity(properties.len());
|
||||
for property in properties {
|
||||
let json = match property {
|
||||
P::Id => {
|
||||
out.insert_unchecked(
|
||||
Key::Property(P::Id),
|
||||
Value::Element(BlobStoreChangeValue::Id(Id::from(c.id))),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
// BM-9: where the stores are, never their settings
|
||||
P::From => json!(c.from),
|
||||
P::To => json!(c.to),
|
||||
P::ChangedAt => date(c.changed_at),
|
||||
P::Node => json!(c.node_id.to_string()),
|
||||
P::Status => json!(status.as_str()),
|
||||
P::CopyStartedAt => copy.map_or(serde_json::Value::Null, |c| date(c.started_at)),
|
||||
P::CopyFinishedAt => copy
|
||||
.and_then(|c| c.finished_at)
|
||||
.map_or(serde_json::Value::Null, date),
|
||||
P::CopyStartedBy => copy.map_or(serde_json::Value::Null, |c| json!(c.started_by)),
|
||||
P::CopyNode => copy.map_or(serde_json::Value::Null, |c| json!(c.node_id.to_string())),
|
||||
P::BlobsPerSecond => {
|
||||
copy.map_or(serde_json::Value::Null, |c| json!(c.blobs_per_second))
|
||||
}
|
||||
P::BytesPerSecond => {
|
||||
copy.map_or(serde_json::Value::Null, |c| json!(c.bytes_per_second))
|
||||
}
|
||||
P::Progress => match status {
|
||||
Status::Copied | Status::Reverted => json!(1.0),
|
||||
_ => json!(p.share()),
|
||||
},
|
||||
P::BlobsChecked => json!(p.blobs_checked),
|
||||
P::Copied => json!(p.copied),
|
||||
P::AlreadyThere => json!(p.already_there),
|
||||
P::BytesCopied => json!(p.bytes_copied),
|
||||
P::Missing => json!(p.missing),
|
||||
P::Damaged => json!(p.damaged),
|
||||
P::Unreadable => json!(p.unreadable),
|
||||
P::Problems => {
|
||||
let problems = model::problems(server.store(), c.id).await?;
|
||||
let mut list = serde_json::to_value(&problems).unwrap_or_default();
|
||||
// The account as a JMAP id, like every other account id
|
||||
if let Some(items) = list.as_array_mut() {
|
||||
for item in items {
|
||||
if let Some(account) = item.get("accountId").and_then(|v| v.as_u64()) {
|
||||
item["accountId"] = json!(Id::from(account).to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
list
|
||||
}
|
||||
P::Error => json!(copy.and_then(|c| c.error.clone())),
|
||||
// Write-only
|
||||
P::Action => continue,
|
||||
};
|
||||
out.insert_unchecked(Key::Property(property.clone()), json_to_value(json));
|
||||
}
|
||||
Ok(Value::Object(out))
|
||||
}
|
||||
|
||||
/// `inbuxa:BlobStoreChange/get`: every change kept, newest first.
|
||||
pub async fn get(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: GetRequest<BlobStoreChange>,
|
||||
) -> trc::Result<GetResponse<BlobStoreChange>> {
|
||||
server_level(access_token)?;
|
||||
let properties = request.unwrap_properties(ALL);
|
||||
let (ids, not_found) = request.unwrap_ids(server.core.jmap.get_max_objects)?;
|
||||
let mut response = GetResponse {
|
||||
account_id: request.account_id.into(),
|
||||
state: None,
|
||||
list: Vec::new(),
|
||||
not_found,
|
||||
};
|
||||
let mut changes = model::all(server.store()).await?;
|
||||
changes.reverse();
|
||||
let now = now();
|
||||
match ids {
|
||||
None => {
|
||||
for c in &changes {
|
||||
response
|
||||
.list
|
||||
.push(to_value(server, c, &properties, now).await?);
|
||||
}
|
||||
}
|
||||
Some(ids) => {
|
||||
for id in ids {
|
||||
match changes.iter().find(|c| u64::from(c.id) == id.id()) {
|
||||
Some(c) => response
|
||||
.list
|
||||
.push(to_value(server, c, &properties, now).await?),
|
||||
None => response.push_not_found(id),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn invalid(property: P, why: impl Into<String>) -> SetError<P> {
|
||||
SetError::invalid_properties()
|
||||
.with_property(property)
|
||||
.with_description(why.into())
|
||||
}
|
||||
|
||||
/// A limit: a whole number, 0 for none (BM-5).
|
||||
fn limit(property: P, value: &Value<'_, P, BlobStoreChangeValue>) -> Result<u64, SetError<P>> {
|
||||
match value {
|
||||
Value::Number(n) => n
|
||||
.as_u64()
|
||||
.ok_or_else(|| invalid(property, "A whole number of 0 or more; 0 is no limit.")),
|
||||
_ => Err(invalid(
|
||||
property,
|
||||
"A whole number of 0 or more; 0 is no limit.",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// `inbuxa:BlobStoreChange/set`: the server records changes, so there's no
|
||||
/// create. An update copies or stops a copy; destroy dismisses (BM-5 to
|
||||
/// BM-7).
|
||||
pub async fn set(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: SetRequest<'_, BlobStoreChange>,
|
||||
) -> trc::Result<SetResponse<BlobStoreChange>> {
|
||||
server_level(access_token)?;
|
||||
let mut response = SetResponse::from_request(&request, server.core.jmap.set_max_objects)?;
|
||||
let data = server.store();
|
||||
let actor = server.audit_actor(access_token).await;
|
||||
|
||||
for (client_id, _) in request.unwrap_create() {
|
||||
response.not_created.append(
|
||||
client_id,
|
||||
SetError::forbidden().with_description(
|
||||
"The server records a change when it starts with the blob store moved.",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
'update: for (id, value) in request.unwrap_update().into_valid() {
|
||||
let Ok(change_id) = u32::try_from(id.id()) else {
|
||||
response.not_updated.append(id, SetError::not_found());
|
||||
continue;
|
||||
};
|
||||
let mut action = None;
|
||||
let mut blobs_per_second = None;
|
||||
let mut bytes_per_second = None;
|
||||
for (key, value) in value.into_expanded_object() {
|
||||
let outcome = match &key {
|
||||
Key::Property(P::Action) => {
|
||||
action = value.as_str().map(|s| s.to_string());
|
||||
Ok(())
|
||||
}
|
||||
Key::Property(P::BlobsPerSecond) => {
|
||||
limit(P::BlobsPerSecond, &value).map(|l| blobs_per_second = Some(l))
|
||||
}
|
||||
Key::Property(P::BytesPerSecond) => {
|
||||
limit(P::BytesPerSecond, &value).map(|l| bytes_per_second = Some(l))
|
||||
}
|
||||
_ => Err(SetError::invalid_properties()
|
||||
.with_property(key.clone().into_owned())
|
||||
.with_description("A change can't be edited, only copied or stopped.")),
|
||||
};
|
||||
if let Err(error) = outcome {
|
||||
response.not_updated.append(id, error);
|
||||
continue 'update;
|
||||
}
|
||||
}
|
||||
let Some(change) = model::get(data, change_id).await? else {
|
||||
response.not_updated.append(id, SetError::not_found());
|
||||
continue;
|
||||
};
|
||||
match action.as_deref() {
|
||||
Some("copy") => {
|
||||
match copies::start(
|
||||
server,
|
||||
change_id,
|
||||
blobs_per_second,
|
||||
bytes_per_second,
|
||||
actor.name.clone(),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
Ok(()) => response.updated.append(id, None),
|
||||
Err(Refusal::Running(running)) => response.not_updated.append(
|
||||
id,
|
||||
SetError::forbidden().with_description(format!(
|
||||
"A copy of change {} runs now; one copy runs at a time. \
|
||||
Stop it or wait for it.",
|
||||
Id::from(running)
|
||||
)),
|
||||
),
|
||||
Err(Refusal::Resolved) => response.not_updated.append(
|
||||
id,
|
||||
SetError::forbidden()
|
||||
.with_description("That change is resolved; there's nothing to copy."),
|
||||
),
|
||||
Err(Refusal::NotFound) => {
|
||||
response.not_updated.append(id, SetError::not_found())
|
||||
}
|
||||
}
|
||||
}
|
||||
Some("stop") => {
|
||||
if change.status_at(now()) != Status::Copying {
|
||||
response.not_updated.append(
|
||||
id,
|
||||
SetError::forbidden().with_description("That change isn't being copied."),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
copies::stop(server, change_id).await?;
|
||||
response.updated.append(id, None);
|
||||
}
|
||||
_ => response
|
||||
.not_updated
|
||||
.append(id, invalid(P::Action, "The actions are copy and stop.")),
|
||||
}
|
||||
}
|
||||
|
||||
for id in request.unwrap_destroy().into_valid() {
|
||||
let Ok(change_id) = u32::try_from(id.id()) else {
|
||||
response.not_destroyed.append(id, SetError::not_found());
|
||||
continue;
|
||||
};
|
||||
match model::get(data, change_id).await? {
|
||||
None => response.not_destroyed.append(id, SetError::not_found()),
|
||||
Some(c) if c.status_at(now()) == Status::Copying => response.not_destroyed.append(
|
||||
id,
|
||||
SetError::forbidden().with_description("It's being copied; stop the copy first."),
|
||||
),
|
||||
Some(_) => {
|
||||
model::remove(data, change_id).await?;
|
||||
// BM-7: this node stops reading through it at once; the
|
||||
// others within a minute (BM-10)
|
||||
copies::refresh(server).await;
|
||||
response.destroyed.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
// The 27.1 series keeps this in its relay module, which this series lacks;
|
||||
// the store check, store snapshot and blob store change handlers share it.
|
||||
|
||||
use jmap_tools::{Key, Map, Property, Value};
|
||||
use std::borrow::Cow;
|
||||
|
||||
/// A JSON value as a JMAP one, its object keys kept as plain keys.
|
||||
pub(super) fn json_to_value<P, E>(json: serde_json::Value) -> Value<'static, P, E>
|
||||
where
|
||||
P: Property,
|
||||
E: jmap_tools::Element<Property = P>,
|
||||
{
|
||||
match json {
|
||||
serde_json::Value::Null => Value::Null,
|
||||
serde_json::Value::Bool(b) => Value::Bool(b),
|
||||
serde_json::Value::Number(n) => {
|
||||
if let Some(n) = n.as_u64() {
|
||||
Value::Number(n.into())
|
||||
} else if let Some(n) = n.as_i64() {
|
||||
Value::Number(n.into())
|
||||
} else {
|
||||
Value::Number(n.as_f64().unwrap_or_default().into())
|
||||
}
|
||||
}
|
||||
serde_json::Value::String(s) => Value::Str(Cow::Owned(s)),
|
||||
serde_json::Value::Array(items) => {
|
||||
Value::Array(items.into_iter().map(json_to_value).collect())
|
||||
}
|
||||
serde_json::Value::Object(map) => {
|
||||
let mut out = Map::with_capacity(map.len());
|
||||
for (key, value) in map {
|
||||
out.insert_unchecked(Key::Owned(key), json_to_value(value));
|
||||
}
|
||||
Value::Object(out)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ pub mod account_lock;
|
||||
pub mod ai_limits;
|
||||
pub mod audit;
|
||||
pub mod audit_log;
|
||||
pub mod blob_store_change; // inbuxa: a change of blob store
|
||||
pub mod data_inventory;
|
||||
pub mod deleted_account;
|
||||
pub mod deliverability; // inbuxa: the deliverability check
|
||||
@@ -25,6 +26,7 @@ pub mod hold_export;
|
||||
pub mod hold_export_api;
|
||||
pub mod journal;
|
||||
pub mod journal_entry;
|
||||
pub mod json; // inbuxa: JSON values as JMAP ones
|
||||
pub mod legal_hold;
|
||||
pub mod log_settings;
|
||||
pub mod mail_rule;
|
||||
@@ -35,6 +37,8 @@ pub mod protocol_policy;
|
||||
pub mod scheduled_reports; // inbuxa: scheduled reports and the weekly digest
|
||||
pub mod security_acceptance;
|
||||
pub mod sharing_policy;
|
||||
pub mod store_check; // inbuxa: checking the store's blobs
|
||||
pub mod store_snapshot; // inbuxa: a snapshot of the store
|
||||
pub mod telemetry;
|
||||
pub mod tenant_protocol_policy;
|
||||
pub mod undelete;
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:StoreCheck` (store-check spec): starting a check of the store's
|
||||
//! blobs, seeing how far it got and what it found, stopping it, and
|
||||
//! forgetting it. The run is `services::inbuxa_store_check`. Starting and
|
||||
//! stopping one are audited by the request layer (SV-9).
|
||||
|
||||
use super::json::json_to_value;
|
||||
use common::{Server, auth::AccessToken};
|
||||
use inbuxa_features::store_check::{self as model, Mode, Refusal, Status, StoreCheck as Stored};
|
||||
use jmap_proto::{
|
||||
error::set::SetError,
|
||||
method::{
|
||||
get::{GetRequest, GetResponse},
|
||||
set::{SetRequest, SetResponse},
|
||||
},
|
||||
object::inbuxa_store_check::{StoreCheck, StoreCheckProperty as P, StoreCheckValue},
|
||||
request::IntoValid,
|
||||
types::date::UTCDate,
|
||||
};
|
||||
use jmap_tools::{Key, Map, Value};
|
||||
use serde_json::json;
|
||||
use services::inbuxa_store_check as runs;
|
||||
use store::write::now;
|
||||
use types::id::Id;
|
||||
|
||||
type CValue = Value<'static, P, StoreCheckValue>;
|
||||
|
||||
const ALL: &[P] = &[
|
||||
P::Id,
|
||||
P::Mode,
|
||||
P::BlobsPerSecond,
|
||||
P::BytesPerSecond,
|
||||
P::Status,
|
||||
P::CreatedAt,
|
||||
P::CreatedBy,
|
||||
P::FinishedAt,
|
||||
P::Node,
|
||||
P::Progress,
|
||||
P::BlobsChecked,
|
||||
P::LinksChecked,
|
||||
P::BytesRead,
|
||||
P::MessagesChecked,
|
||||
P::Missing,
|
||||
P::Damaged,
|
||||
P::Unlinked,
|
||||
P::Unreadable,
|
||||
P::Problems,
|
||||
P::Error,
|
||||
];
|
||||
|
||||
/// SV-9: a check reads every account's data, so only the server's own
|
||||
/// administrators run one.
|
||||
fn server_level(access_token: &AccessToken) -> trc::Result<()> {
|
||||
if access_token.tenant_id().is_some() {
|
||||
Err(trc::JmapEvent::Forbidden
|
||||
.into_err()
|
||||
.details("A store check reads the whole server, not a tenant."))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn date(seconds: u64) -> serde_json::Value {
|
||||
json!(UTCDate::from_timestamp(seconds as i64).to_string())
|
||||
}
|
||||
|
||||
async fn to_value(server: &Server, c: &Stored, properties: &[P], now: u64) -> trc::Result<CValue> {
|
||||
let status = c.status_at(now);
|
||||
let mut out = Map::with_capacity(properties.len());
|
||||
for property in properties {
|
||||
let p = &c.progress;
|
||||
let json = match property {
|
||||
P::Id => {
|
||||
out.insert_unchecked(
|
||||
Key::Property(P::Id),
|
||||
Value::Element(StoreCheckValue::Id(Id::from(c.id))),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
P::Mode => json!(c.mode.as_str()),
|
||||
P::BlobsPerSecond => json!(c.blobs_per_second),
|
||||
P::BytesPerSecond => json!(c.bytes_per_second),
|
||||
P::Status => json!(status.as_str()),
|
||||
P::CreatedAt => date(c.created_at),
|
||||
P::CreatedBy => json!(c.created_by),
|
||||
P::FinishedAt => c.finished_at.map_or(serde_json::Value::Null, date),
|
||||
P::Node => json!(c.node_id.to_string()),
|
||||
P::Progress => json!(p.share(status)),
|
||||
P::BlobsChecked => json!(p.blobs_checked),
|
||||
P::LinksChecked => json!(p.links_checked),
|
||||
P::BytesRead => json!(p.bytes_read),
|
||||
P::MessagesChecked => json!(p.messages_checked),
|
||||
P::Missing => json!(p.missing),
|
||||
P::Damaged => json!(p.damaged),
|
||||
P::Unlinked => json!(p.unlinked),
|
||||
P::Unreadable => json!(p.unreadable),
|
||||
P::Problems => {
|
||||
let problems = model::problems(server.store(), c.id).await?;
|
||||
let mut list = serde_json::to_value(&problems).unwrap_or_default();
|
||||
// The account as a JMAP id, like every other account id
|
||||
if let Some(items) = list.as_array_mut() {
|
||||
for item in items {
|
||||
if let Some(account) = item.get("accountId").and_then(|v| v.as_u64()) {
|
||||
item["accountId"] = json!(Id::from(account).to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
list
|
||||
}
|
||||
P::Error => json!(c.error),
|
||||
// Write-only
|
||||
P::Action => continue,
|
||||
};
|
||||
out.insert_unchecked(Key::Property(property.clone()), json_to_value(json));
|
||||
}
|
||||
Ok(Value::Object(out))
|
||||
}
|
||||
|
||||
/// `inbuxa:StoreCheck/get`: every check kept, newest first.
|
||||
pub async fn get(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: GetRequest<StoreCheck>,
|
||||
) -> trc::Result<GetResponse<StoreCheck>> {
|
||||
server_level(access_token)?;
|
||||
let properties = request.unwrap_properties(ALL);
|
||||
let (ids, not_found) = request.unwrap_ids(server.core.jmap.get_max_objects)?;
|
||||
let mut response = GetResponse {
|
||||
account_id: request.account_id.into(),
|
||||
state: None,
|
||||
list: Vec::new(),
|
||||
not_found,
|
||||
};
|
||||
let mut checks = model::all(server.store()).await?;
|
||||
checks.reverse();
|
||||
let now = now();
|
||||
match ids {
|
||||
None => {
|
||||
for c in &checks {
|
||||
response
|
||||
.list
|
||||
.push(to_value(server, c, &properties, now).await?);
|
||||
}
|
||||
}
|
||||
Some(ids) => {
|
||||
for id in ids {
|
||||
match checks.iter().find(|c| u64::from(c.id) == id.id()) {
|
||||
Some(c) => response
|
||||
.list
|
||||
.push(to_value(server, c, &properties, now).await?),
|
||||
None => response.push_not_found(id),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn invalid(property: P, why: impl Into<String>) -> SetError<P> {
|
||||
SetError::invalid_properties()
|
||||
.with_property(property)
|
||||
.with_description(why.into())
|
||||
}
|
||||
|
||||
/// A limit: a whole number, 0 for none (SV-6).
|
||||
fn limit(property: P, value: &Value<'_, P, StoreCheckValue>) -> Result<u64, SetError<P>> {
|
||||
match value {
|
||||
Value::Number(n) => n
|
||||
.as_u64()
|
||||
.ok_or_else(|| invalid(property, "A whole number of 0 or more; 0 is no limit.")),
|
||||
_ => Err(invalid(
|
||||
property,
|
||||
"A whole number of 0 or more; 0 is no limit.",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// `inbuxa:StoreCheck/set`: create starts a check; an update can only stop
|
||||
/// one; destroy forgets a finished one (SV-2, SV-8, SV-10).
|
||||
pub async fn set(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: SetRequest<'_, StoreCheck>,
|
||||
) -> trc::Result<SetResponse<StoreCheck>> {
|
||||
server_level(access_token)?;
|
||||
let mut response = SetResponse::from_request(&request, server.core.jmap.set_max_objects)?;
|
||||
let data = server.store();
|
||||
let actor = server.audit_actor(access_token).await;
|
||||
|
||||
'create: for (client_id, value) in request.unwrap_create() {
|
||||
let mut mode = Mode::Quick;
|
||||
let mut blobs_per_second = None;
|
||||
let mut bytes_per_second = None;
|
||||
for (key, value) in value.into_expanded_object() {
|
||||
let outcome = match &key {
|
||||
Key::Property(P::Mode) => match value.as_str().and_then(|s| Mode::parse(&s)) {
|
||||
Some(m) => {
|
||||
mode = m;
|
||||
Ok(())
|
||||
}
|
||||
None => Err(invalid(P::Mode, "quick or full.")),
|
||||
},
|
||||
Key::Property(P::BlobsPerSecond) => {
|
||||
limit(P::BlobsPerSecond, &value).map(|l| blobs_per_second = Some(l))
|
||||
}
|
||||
Key::Property(P::BytesPerSecond) => {
|
||||
limit(P::BytesPerSecond, &value).map(|l| bytes_per_second = Some(l))
|
||||
}
|
||||
_ => Err(SetError::invalid_properties().with_property(key.clone().into_owned())),
|
||||
};
|
||||
if let Err(error) = outcome {
|
||||
response.not_created.append(client_id, error);
|
||||
continue 'create;
|
||||
}
|
||||
}
|
||||
match runs::start(
|
||||
server,
|
||||
mode,
|
||||
blobs_per_second,
|
||||
bytes_per_second,
|
||||
actor.name.clone(),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
Ok(id) => {
|
||||
let mut out = Map::with_capacity(1);
|
||||
out.insert_unchecked(
|
||||
Key::Property(P::Id),
|
||||
Value::Element(StoreCheckValue::Id(Id::from(id))),
|
||||
);
|
||||
response.created.insert(client_id, Value::Object(out));
|
||||
}
|
||||
Err(Refusal::Running(id)) => response.not_created.append(
|
||||
client_id,
|
||||
SetError::forbidden().with_description(format!(
|
||||
"Check {} runs now; one check runs at a time. Stop it or wait for it.",
|
||||
Id::from(id)
|
||||
)),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
for (id, value) in request.unwrap_update().into_valid() {
|
||||
let Ok(check_id) = u32::try_from(id.id()) else {
|
||||
response.not_updated.append(id, SetError::not_found());
|
||||
continue;
|
||||
};
|
||||
let mut action = None;
|
||||
let mut bad = None;
|
||||
for (key, value) in value.into_expanded_object() {
|
||||
match &key {
|
||||
Key::Property(P::Action) => action = value.as_str().map(|s| s.to_string()),
|
||||
_ => {
|
||||
bad = Some(
|
||||
SetError::invalid_properties()
|
||||
.with_property(key.clone().into_owned())
|
||||
.with_description("A check can't be changed, only stopped."),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(error) = bad {
|
||||
response.not_updated.append(id, error);
|
||||
continue;
|
||||
}
|
||||
if action.as_deref() != Some("stop") {
|
||||
response
|
||||
.not_updated
|
||||
.append(id, invalid(P::Action, "The one action is stop."));
|
||||
continue;
|
||||
}
|
||||
let Some(check) = model::get(data, check_id).await? else {
|
||||
response.not_updated.append(id, SetError::not_found());
|
||||
continue;
|
||||
};
|
||||
if check.status_at(now()) != Status::Running {
|
||||
response.not_updated.append(
|
||||
id,
|
||||
SetError::forbidden().with_description("That check isn't running."),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
runs::stop(server, check_id).await?;
|
||||
response.updated.append(id, None);
|
||||
}
|
||||
|
||||
for id in request.unwrap_destroy().into_valid() {
|
||||
let Ok(check_id) = u32::try_from(id.id()) else {
|
||||
response.not_destroyed.append(id, SetError::not_found());
|
||||
continue;
|
||||
};
|
||||
match model::get(data, check_id).await? {
|
||||
None => response.not_destroyed.append(id, SetError::not_found()),
|
||||
Some(c) if c.status_at(now()) == Status::Running => response.not_destroyed.append(
|
||||
id,
|
||||
SetError::forbidden().with_description("It's running; stop it first."),
|
||||
),
|
||||
Some(_) => {
|
||||
model::remove(data, check_id).await?;
|
||||
response.destroyed.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:StoreSnapshot` (store-snapshot spec): taking a snapshot of the
|
||||
//! store without stopping the server, listing the snapshots on disk, and
|
||||
//! deleting one. The run is `services::inbuxa_store_snapshot`. Taking and
|
||||
//! destroying one are audited by the request layer (SN-8).
|
||||
|
||||
use super::json::json_to_value;
|
||||
use common::{Server, auth::AccessToken};
|
||||
use jmap_proto::{
|
||||
error::set::SetError,
|
||||
method::{
|
||||
get::{GetRequest, GetResponse},
|
||||
set::{SetRequest, SetResponse},
|
||||
},
|
||||
object::inbuxa_store_snapshot::{
|
||||
StoreSnapshot, StoreSnapshotProperty as P, StoreSnapshotValue,
|
||||
},
|
||||
request::IntoValid,
|
||||
types::date::UTCDate,
|
||||
};
|
||||
use jmap_tools::{Key, Map, Value};
|
||||
use serde_json::json;
|
||||
use services::inbuxa_store_snapshot::{self as runs, Destroyed, Refusal, Snapshot};
|
||||
use types::id::Id;
|
||||
|
||||
type SValue = Value<'static, P, StoreSnapshotValue>;
|
||||
|
||||
const ALL: &[P] = &[
|
||||
P::Id,
|
||||
P::Status,
|
||||
P::CreatedAt,
|
||||
P::CreatedBy,
|
||||
P::FinishedAt,
|
||||
P::Node,
|
||||
P::Name,
|
||||
P::Path,
|
||||
P::Holds,
|
||||
P::Size,
|
||||
P::BlobFiles,
|
||||
P::Error,
|
||||
];
|
||||
|
||||
/// SN-8: a snapshot holds every account's data, so only the server's own
|
||||
/// administrators take one.
|
||||
fn server_level(access_token: &AccessToken) -> trc::Result<()> {
|
||||
if access_token.tenant_id().is_some() {
|
||||
Err(trc::JmapEvent::Forbidden
|
||||
.into_err()
|
||||
.details("A snapshot holds the whole server, not a tenant."))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn date(seconds: u64) -> serde_json::Value {
|
||||
json!(UTCDate::from_timestamp(seconds as i64).to_string())
|
||||
}
|
||||
|
||||
fn to_value(s: &Snapshot, properties: &[P]) -> SValue {
|
||||
let mut out = Map::with_capacity(properties.len());
|
||||
for property in properties {
|
||||
let json = match property {
|
||||
P::Id => {
|
||||
out.insert_unchecked(
|
||||
Key::Property(P::Id),
|
||||
Value::Element(StoreSnapshotValue::Id(Id::from(s.id))),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
P::Status => json!(s.status.as_str()),
|
||||
P::CreatedAt => date(s.created_at),
|
||||
P::CreatedBy => json!(s.created_by),
|
||||
P::FinishedAt => s.finished_at.map_or(serde_json::Value::Null, date),
|
||||
P::Node => json!(s.node_id.to_string()),
|
||||
P::Name => json!(s.name),
|
||||
P::Path => json!(s.path.to_string_lossy()),
|
||||
P::Holds => json!(s.holds),
|
||||
P::Size => json!(s.size),
|
||||
P::BlobFiles => json!(s.blob_files),
|
||||
P::Error => json!(s.error),
|
||||
};
|
||||
out.insert_unchecked(Key::Property(property.clone()), json_to_value(json));
|
||||
}
|
||||
Value::Object(out)
|
||||
}
|
||||
|
||||
/// `inbuxa:StoreSnapshot/get`: the snapshots on disk and the one being
|
||||
/// taken, newest first (SN-7).
|
||||
pub async fn get(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: GetRequest<StoreSnapshot>,
|
||||
) -> trc::Result<GetResponse<StoreSnapshot>> {
|
||||
server_level(access_token)?;
|
||||
let properties = request.unwrap_properties(ALL);
|
||||
let (ids, not_found) = request.unwrap_ids(server.core.jmap.get_max_objects)?;
|
||||
let mut response = GetResponse {
|
||||
account_id: request.account_id.into(),
|
||||
state: None,
|
||||
list: Vec::new(),
|
||||
not_found,
|
||||
};
|
||||
let snapshots = runs::list(server).await;
|
||||
match ids {
|
||||
None => {
|
||||
for s in &snapshots {
|
||||
response.list.push(to_value(s, &properties));
|
||||
}
|
||||
}
|
||||
Some(ids) => {
|
||||
for id in ids {
|
||||
match snapshots.iter().find(|s| s.id == id.id()) {
|
||||
Some(s) => response.list.push(to_value(s, &properties)),
|
||||
None => response.push_not_found(id),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
/// `inbuxa:StoreSnapshot/set`: create takes one; destroy deletes one; a
|
||||
/// snapshot can't be changed (SN-1, SN-6, SN-9).
|
||||
pub async fn set(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: SetRequest<'_, StoreSnapshot>,
|
||||
) -> trc::Result<SetResponse<StoreSnapshot>> {
|
||||
server_level(access_token)?;
|
||||
let mut response = SetResponse::from_request(&request, server.core.jmap.set_max_objects)?;
|
||||
let actor = server.audit_actor(access_token).await;
|
||||
|
||||
for (client_id, value) in request.unwrap_create() {
|
||||
if let Some((key, _)) = value.into_expanded_object().next() {
|
||||
response.not_created.append(
|
||||
client_id,
|
||||
SetError::invalid_properties()
|
||||
.with_property(key.into_owned())
|
||||
.with_description("A snapshot is taken as it is; it takes no properties."),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
match runs::start(server, actor.name.clone()).await? {
|
||||
Ok(snapshot) => {
|
||||
response
|
||||
.created
|
||||
.insert(client_id, to_value(&snapshot, &[P::Id, P::Name, P::Path]));
|
||||
}
|
||||
Err(Refusal::Running(id)) => response.not_created.append(
|
||||
client_id,
|
||||
SetError::forbidden().with_description(format!(
|
||||
"Snapshot {} is being taken; one is taken at a time. Wait for it.",
|
||||
Id::from(id)
|
||||
)),
|
||||
),
|
||||
Err(Refusal::Unsupported(why)) => response
|
||||
.not_created
|
||||
.append(client_id, SetError::forbidden().with_description(why)),
|
||||
}
|
||||
}
|
||||
|
||||
for (id, _) in request.unwrap_update().into_valid() {
|
||||
response.not_updated.append(
|
||||
id,
|
||||
SetError::forbidden().with_description("A snapshot can't be changed."),
|
||||
);
|
||||
}
|
||||
|
||||
for id in request.unwrap_destroy().into_valid() {
|
||||
match runs::destroy(server, id.id()).await? {
|
||||
Destroyed::Gone => response.destroyed.push(id),
|
||||
Destroyed::NotFound => response.not_destroyed.append(id, SetError::not_found()),
|
||||
Destroyed::Running => response.not_destroyed.append(
|
||||
id,
|
||||
SetError::forbidden().with_description("It's being taken; wait for it to finish."),
|
||||
),
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
@@ -1780,6 +1780,12 @@ pub enum Permission {
|
||||
// inbuxa: scheduled reports and the weekly digest
|
||||
SysScheduledReportGet = 688,
|
||||
SysScheduledReportUpdate = 689,
|
||||
// inbuxa: checking the store's blobs
|
||||
SysStoreCheck = 695,
|
||||
// inbuxa: a snapshot of the store, without stopping the server
|
||||
SysStoreSnapshot = 696,
|
||||
// inbuxa: a change of blob store, copied and dismissed
|
||||
SysBlobStoreChange = 697,
|
||||
SysAccountGet = 219,
|
||||
SysAccountCreate = 220,
|
||||
SysAccountUpdate = 221,
|
||||
|
||||
@@ -7160,6 +7160,9 @@ impl EnumImpl for Permission {
|
||||
b"sysDeliverabilityCheck" => Permission::SysDeliverabilityCheck,
|
||||
b"sysScheduledReportGet" => Permission::SysScheduledReportGet,
|
||||
b"sysScheduledReportUpdate" => Permission::SysScheduledReportUpdate,
|
||||
b"sysStoreCheck" => Permission::SysStoreCheck,
|
||||
b"sysStoreSnapshot" => Permission::SysStoreSnapshot,
|
||||
b"sysBlobStoreChange" => Permission::SysBlobStoreChange,
|
||||
b"sysAccountGet" => Permission::SysAccountGet,
|
||||
b"sysAccountCreate" => Permission::SysAccountCreate,
|
||||
b"sysAccountUpdate" => Permission::SysAccountUpdate,
|
||||
@@ -7866,6 +7869,9 @@ impl EnumImpl for Permission {
|
||||
Permission::SysDeliverabilityCheck => "sysDeliverabilityCheck",
|
||||
Permission::SysScheduledReportGet => "sysScheduledReportGet",
|
||||
Permission::SysScheduledReportUpdate => "sysScheduledReportUpdate",
|
||||
Permission::SysStoreCheck => "sysStoreCheck",
|
||||
Permission::SysStoreSnapshot => "sysStoreSnapshot",
|
||||
Permission::SysBlobStoreChange => "sysBlobStoreChange",
|
||||
Permission::SysAccountGet => "sysAccountGet",
|
||||
Permission::SysAccountCreate => "sysAccountCreate",
|
||||
Permission::SysAccountUpdate => "sysAccountUpdate",
|
||||
@@ -8565,6 +8571,9 @@ impl EnumImpl for Permission {
|
||||
687 => Some(Permission::SysDeliverabilityCheck),
|
||||
688 => Some(Permission::SysScheduledReportGet),
|
||||
689 => Some(Permission::SysScheduledReportUpdate),
|
||||
695 => Some(Permission::SysStoreCheck),
|
||||
696 => Some(Permission::SysStoreSnapshot),
|
||||
697 => Some(Permission::SysBlobStoreChange),
|
||||
219 => Some(Permission::SysAccountGet),
|
||||
220 => Some(Permission::SysAccountCreate),
|
||||
221 => Some(Permission::SysAccountUpdate),
|
||||
@@ -9009,7 +9018,7 @@ impl EnumImpl for Permission {
|
||||
}
|
||||
}
|
||||
|
||||
const COUNT: usize = 690;
|
||||
const COUNT: usize = 698;
|
||||
}
|
||||
|
||||
impl serde::Serialize for Permission {
|
||||
|
||||
@@ -0,0 +1,583 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Changing the blob store (docs/spec/features/blob-store-change.md): the
|
||||
//! copy behind an `inbuxa:BlobStoreChange`. It goes through the blob index
|
||||
//! one slice at a time and copies each blob the current store lacks from
|
||||
//! the change's old store, as the old store keeps it (BM-5). It runs on the
|
||||
//! node the copy was started on. Also, on every node, it keeps reading
|
||||
//! through only the changes still unresolved (BM-10).
|
||||
|
||||
use common::{
|
||||
BuildServer, Inner, Server,
|
||||
manager::{SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY},
|
||||
};
|
||||
use inbuxa_features::blob_move::{
|
||||
self as model, BlobStoreChange, Copy, DEFAULT_BLOBS_PER_SECOND, DEFAULT_BYTES_PER_SECOND,
|
||||
MAX_PROBLEMS, Problem, ProblemKind, Progress, Refusal, STALE_AFTER, Status,
|
||||
};
|
||||
use std::{
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use store::{
|
||||
BlobStore,
|
||||
write::{
|
||||
blob_check::{BlobHolder, LinkedBlob},
|
||||
now,
|
||||
},
|
||||
};
|
||||
use types::{blob_hash::BlobHash, collection::Collection};
|
||||
|
||||
/// How often a running copy saves its progress, which is also its
|
||||
/// heartbeat and when it sees a stop (BM-6).
|
||||
const SAVE_EVERY: Duration = Duration::from_secs(10);
|
||||
/// BM-10: how often a node reads which changes are still unresolved.
|
||||
const REFRESH_EVERY: Duration = Duration::from_secs(60);
|
||||
|
||||
fn node_id(server: &Server) -> u64 {
|
||||
server.core.network.node_id
|
||||
}
|
||||
|
||||
/// BM-5: starts copying a change on this node. Refused while another copy
|
||||
/// runs, or when the change is resolved.
|
||||
pub async fn start(
|
||||
server: &Server,
|
||||
id: u32,
|
||||
blobs_per_second: Option<u64>,
|
||||
bytes_per_second: Option<u64>,
|
||||
started_by: String,
|
||||
) -> trc::Result<Result<(), Refusal>> {
|
||||
let now = now();
|
||||
let copy = Copy {
|
||||
started_at: now,
|
||||
started_by,
|
||||
node_id: node_id(server),
|
||||
blobs_per_second: blobs_per_second.unwrap_or(DEFAULT_BLOBS_PER_SECOND),
|
||||
bytes_per_second: bytes_per_second.unwrap_or(DEFAULT_BYTES_PER_SECOND),
|
||||
heartbeat: now,
|
||||
finished_at: None,
|
||||
stop_requested: false,
|
||||
progress: Progress::default(),
|
||||
problems_kept: 0,
|
||||
error: None,
|
||||
};
|
||||
let change = match model::start_copy(server.store(), id, copy, now).await? {
|
||||
Ok(change) => change,
|
||||
Err(refusal) => return Ok(Err(refusal)),
|
||||
};
|
||||
let server = server.clone();
|
||||
tokio::spawn(async move {
|
||||
run(&server, change).await;
|
||||
});
|
||||
Ok(Ok(()))
|
||||
}
|
||||
|
||||
/// BM-6: asks a running copy to stop. Any node can: the one running it
|
||||
/// sees the request at its next save. None if there's no such change.
|
||||
pub async fn stop(server: &Server, id: u32) -> trc::Result<Option<BlobStoreChange>> {
|
||||
model::update(server.store(), id, |c| match &mut c.copy {
|
||||
Some(copy) if c.status == Status::Copying && !copy.stop_requested => {
|
||||
copy.stop_requested = true;
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// BM-6: at start-up, a copy this node ran stopped with it; now and then,
|
||||
/// a copy on a node that went quiet is taken to have stopped too. BM-10:
|
||||
/// and every minute, only unresolved changes are read through.
|
||||
pub fn spawn_blob_moves(inner: Arc<Inner>) {
|
||||
tokio::spawn(async move {
|
||||
let server = inner.build_server();
|
||||
let node = node_id(&server);
|
||||
mark_interrupted(&server, |c| c.node_id == node).await;
|
||||
loop {
|
||||
tokio::time::sleep(REFRESH_EVERY).await;
|
||||
let server = inner.build_server();
|
||||
let cutoff = now().saturating_sub(STALE_AFTER);
|
||||
mark_interrupted(&server, |c| c.node_id != node && c.heartbeat < cutoff).await;
|
||||
refresh(&server).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// BM-10: stops reading through changes resolved or dismissed anywhere.
|
||||
pub async fn refresh(server: &Server) {
|
||||
let Some(moving) = server.blob_store().moving() else {
|
||||
return;
|
||||
};
|
||||
match model::unresolved(server.store()).await {
|
||||
Ok(changes) => {
|
||||
let ids = changes.iter().map(|c| c.id).collect::<Vec<_>>();
|
||||
moving.retain_changes(&ids);
|
||||
}
|
||||
Err(err) => {
|
||||
trc::error!(err.details("Failed to read the blob store changes"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn mark_interrupted(server: &Server, which: impl Fn(&Copy) -> bool) {
|
||||
let changes = match model::all(server.store()).await {
|
||||
Ok(changes) => changes,
|
||||
Err(err) => {
|
||||
trc::error!(err.details("Failed to read the blob store changes"));
|
||||
return;
|
||||
}
|
||||
};
|
||||
for change in changes
|
||||
.into_iter()
|
||||
.filter(|c| c.status == Status::Copying && c.copy.as_ref().is_some_and(&which))
|
||||
{
|
||||
let result = model::update(server.store(), change.id, |c| {
|
||||
if c.status != Status::Copying || !c.copy.as_ref().is_some_and(&which) {
|
||||
return false;
|
||||
}
|
||||
c.status = Status::Interrupted;
|
||||
if let Some(copy) = &mut c.copy {
|
||||
copy.finished_at = Some(now());
|
||||
}
|
||||
true
|
||||
})
|
||||
.await;
|
||||
if let Err(err) = result.and(model::release(server.store(), change.id).await) {
|
||||
trc::error!(err.details("Failed to mark an interrupted blob copy"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// BM-5: keeps a rate, a second at a time.
|
||||
struct Pace {
|
||||
per_second: u64,
|
||||
window: Instant,
|
||||
used: u64,
|
||||
}
|
||||
|
||||
impl Pace {
|
||||
fn new(per_second: u64) -> Self {
|
||||
Pace {
|
||||
per_second,
|
||||
window: Instant::now(),
|
||||
used: 0,
|
||||
}
|
||||
}
|
||||
|
||||
async fn take(&mut self, amount: u64) {
|
||||
if self.per_second == 0 {
|
||||
return;
|
||||
}
|
||||
if self.window.elapsed() >= Duration::from_secs(1) {
|
||||
self.window = Instant::now();
|
||||
self.used = 0;
|
||||
}
|
||||
self.used += amount;
|
||||
if self.used >= self.per_second {
|
||||
let rest = Duration::from_secs(1).saturating_sub(self.window.elapsed());
|
||||
tokio::time::sleep(rest).await;
|
||||
self.window = Instant::now();
|
||||
self.used = self.used.saturating_sub(self.per_second);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Who holds a blob, as a problem names it (BM-5).
|
||||
struct Holder {
|
||||
name: String,
|
||||
account_id: Option<u32>,
|
||||
object_id: Option<u64>,
|
||||
/// Named by the hash of its content, so it can be checked.
|
||||
content_named: bool,
|
||||
}
|
||||
|
||||
impl Holder {
|
||||
fn of(blob: &LinkedBlob) -> Holder {
|
||||
match blob.holders.first() {
|
||||
Some(BlobHolder::Document {
|
||||
account_id,
|
||||
collection,
|
||||
document_id,
|
||||
}) => Holder {
|
||||
name: match Collection::from(*collection) {
|
||||
Collection::Email => "email".into(),
|
||||
Collection::FileNode => "file".into(),
|
||||
Collection::SieveScript => "sieveScript".into(),
|
||||
Collection::CalendarEvent => "calendarEvent".into(),
|
||||
Collection::ContactCard => "contactCard".into(),
|
||||
_ => format!("collection{collection}"),
|
||||
},
|
||||
account_id: Some(*account_id),
|
||||
object_id: Some(*document_id as u64),
|
||||
content_named: true,
|
||||
},
|
||||
Some(BlobHolder::Queue { id }) => Holder {
|
||||
name: "queuedMessage".into(),
|
||||
account_id: None,
|
||||
object_id: Some(*id),
|
||||
content_named: true,
|
||||
},
|
||||
Some(BlobHolder::Archived { account_id, item }) => Holder {
|
||||
name: "archivedItem".into(),
|
||||
account_id: Some(*account_id),
|
||||
object_id: Some(item.id().id()),
|
||||
content_named: true,
|
||||
},
|
||||
None => Holder {
|
||||
name: "temporary".into(),
|
||||
account_id: None,
|
||||
object_id: None,
|
||||
content_named: false,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn fixed() -> Holder {
|
||||
Holder {
|
||||
name: "fixed".into(),
|
||||
account_id: None,
|
||||
object_id: None,
|
||||
content_named: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Run<'x> {
|
||||
server: &'x Server,
|
||||
id: u32,
|
||||
old: BlobStore,
|
||||
target: BlobStore,
|
||||
progress: Progress,
|
||||
found: Vec<Problem>,
|
||||
kept: u64,
|
||||
blobs: Pace,
|
||||
bytes: Pace,
|
||||
saved: Instant,
|
||||
}
|
||||
|
||||
/// How a run ended before it finished.
|
||||
enum Halt {
|
||||
Stopped,
|
||||
Failed(trc::Error),
|
||||
}
|
||||
|
||||
impl From<trc::Error> for Halt {
|
||||
fn from(err: trc::Error) -> Self {
|
||||
Halt::Failed(err)
|
||||
}
|
||||
}
|
||||
|
||||
fn key_name(key: &[u8]) -> String {
|
||||
match BlobHash::try_from_hash_slice(key) {
|
||||
Ok(hash) => hash.to_hex(),
|
||||
Err(_) => String::from_utf8_lossy(key).into_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> Run<'x> {
|
||||
/// BM-6: counts a problem, logs it, and keeps the first ones.
|
||||
fn problem(&mut self, kind: ProblemKind, key: &[u8], holder: &Holder, detail: String) {
|
||||
self.progress.count(kind);
|
||||
let blob_id = key_name(key);
|
||||
trc::event!(
|
||||
Store(trc::StoreEvent::DataCorruption),
|
||||
Id = self.id,
|
||||
AccountId = holder.account_id,
|
||||
Collection = holder.name.clone(),
|
||||
DocumentId = holder.object_id,
|
||||
BlobId = blob_id.clone(),
|
||||
Reason = kind.as_str(),
|
||||
Details = detail.clone(),
|
||||
);
|
||||
if self.kept < MAX_PROBLEMS {
|
||||
self.kept += 1;
|
||||
self.found.push(Problem {
|
||||
kind,
|
||||
blob_id,
|
||||
holder: holder.name.clone(),
|
||||
account_id: holder.account_id,
|
||||
object_id: holder.object_id,
|
||||
detail,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Saves progress and the problems found since, now and then; a stop
|
||||
/// asked for meanwhile ends the run.
|
||||
async fn save(&mut self, force: bool) -> Result<(), Halt> {
|
||||
if !force && self.saved.elapsed() < SAVE_EVERY {
|
||||
return Ok(());
|
||||
}
|
||||
let data = self.server.store();
|
||||
let first = self.kept - self.found.len() as u64;
|
||||
model::add_problems(data, self.id, first, &self.found).await?;
|
||||
self.found.clear();
|
||||
let progress = self.progress.clone();
|
||||
let kept = self.kept;
|
||||
let saved = model::update(data, self.id, |c| {
|
||||
if let Some(copy) = &mut c.copy {
|
||||
copy.progress = progress.clone();
|
||||
copy.problems_kept = kept;
|
||||
copy.heartbeat = now();
|
||||
}
|
||||
true
|
||||
})
|
||||
.await?;
|
||||
self.saved = Instant::now();
|
||||
match saved {
|
||||
Some(c) if c.copy.as_ref().is_some_and(|c| c.stop_requested) => Err(Halt::Stopped),
|
||||
Some(_) => Ok(()),
|
||||
// Dismissed under it: nothing left to copy for
|
||||
None => Err(Halt::Stopped),
|
||||
}
|
||||
}
|
||||
|
||||
/// BM-5: one blob, copied unless the current store has it.
|
||||
async fn one(&mut self, key: &[u8], holder: Holder) -> Result<(), Halt> {
|
||||
self.blobs.take(1).await;
|
||||
self.progress.blobs_checked += 1;
|
||||
match self.target.stored_present(key).await {
|
||||
Ok(true) => {
|
||||
self.progress.already_there += 1;
|
||||
return self.save(false).await;
|
||||
}
|
||||
Ok(false) => {}
|
||||
Err(err) => {
|
||||
self.problem(
|
||||
ProblemKind::Unreadable,
|
||||
key,
|
||||
&holder,
|
||||
format!("the current blob store couldn't say: {err}"),
|
||||
);
|
||||
return self.save(false).await;
|
||||
}
|
||||
}
|
||||
let stored = match self.old.get_stored(key).await {
|
||||
Ok(Some(stored)) => stored,
|
||||
Ok(None) => {
|
||||
// The spam files exist only once the classifier has trained
|
||||
if holder.name != "fixed" {
|
||||
self.problem(ProblemKind::Missing, key, &holder, String::new());
|
||||
}
|
||||
return self.save(false).await;
|
||||
}
|
||||
Err(err) => {
|
||||
self.problem(
|
||||
ProblemKind::Unreadable,
|
||||
key,
|
||||
&holder,
|
||||
format!("the old blob store failed: {err}"),
|
||||
);
|
||||
return self.save(false).await;
|
||||
}
|
||||
};
|
||||
let size = stored.len() as u64;
|
||||
self.bytes.take(size).await;
|
||||
if holder.content_named
|
||||
&& let Ok(hash) = BlobHash::try_from_hash_slice(key)
|
||||
{
|
||||
match BlobStore::content_of_stored(key, stored.clone()) {
|
||||
Ok(content) => {
|
||||
let actual = BlobHash::generate(&content);
|
||||
if actual != hash {
|
||||
self.problem(
|
||||
ProblemKind::Damaged,
|
||||
key,
|
||||
&holder,
|
||||
format!(
|
||||
"the old store's {} bytes hash to {}",
|
||||
content.len(),
|
||||
actual.to_hex()
|
||||
),
|
||||
);
|
||||
return self.save(false).await;
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
self.problem(
|
||||
ProblemKind::Damaged,
|
||||
key,
|
||||
&holder,
|
||||
"it doesn't decompress".into(),
|
||||
);
|
||||
return self.save(false).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
match self.target.put_stored(key, &stored).await {
|
||||
Ok(()) => {
|
||||
self.progress.copied += 1;
|
||||
self.progress.bytes_copied += size;
|
||||
}
|
||||
Err(err) => self.problem(
|
||||
ProblemKind::Unreadable,
|
||||
key,
|
||||
&holder,
|
||||
format!("the current blob store couldn't write it: {err}"),
|
||||
),
|
||||
}
|
||||
self.save(false).await
|
||||
}
|
||||
|
||||
async fn go(&mut self) -> Result<(), Halt> {
|
||||
for slice in 0u8..=255 {
|
||||
let blobs = self.server.store().blobs_to_copy(slice).await?;
|
||||
// A blob without its commit marker never finished being written
|
||||
for blob in blobs.into_iter().filter(|b| b.committed) {
|
||||
let holder = Holder::of(&blob);
|
||||
self.one(blob.hash.as_slice(), holder).await?;
|
||||
}
|
||||
self.progress.slices = slice as u32 + 1;
|
||||
self.save(false).await?;
|
||||
}
|
||||
// BM-5: the files kept under fixed names
|
||||
for key in [SPAM_TRAINER_KEY, SPAM_CLASSIFIER_KEY] {
|
||||
self.one(key, Holder::fixed()).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// The change's old store: the one already read through, or opened now.
|
||||
async fn old_store(server: &Server, change: &BlobStoreChange) -> Result<BlobStore, String> {
|
||||
if let Some(old) = server.blob_store().moving().and_then(|m| {
|
||||
m.old_stores()
|
||||
.iter()
|
||||
.find(|o| o.change_id == change.id)
|
||||
.map(|o| o.store.clone())
|
||||
}) {
|
||||
return Ok(old);
|
||||
}
|
||||
let Some(settings) = change.old_settings.clone() else {
|
||||
return Err("The change keeps no settings for the old blob store".into());
|
||||
};
|
||||
let mut warnings = Vec::new();
|
||||
BlobStore::open(settings, server.store(), &mut warnings).await
|
||||
}
|
||||
|
||||
/// Runs a copy to its end and records how it ended (BM-6).
|
||||
async fn run(server: &Server, change: BlobStoreChange) {
|
||||
let id = change.id;
|
||||
let (status, error, run) = match old_store(server, &change).await {
|
||||
Err(err) => (
|
||||
Status::Failed,
|
||||
Some(format!("The old blob store couldn't be opened: {err}")),
|
||||
None,
|
||||
),
|
||||
Ok(old) => {
|
||||
let copy = change.copy.clone().unwrap_or_else(|| Copy {
|
||||
started_at: now(),
|
||||
started_by: String::new(),
|
||||
node_id: node_id(server),
|
||||
blobs_per_second: DEFAULT_BLOBS_PER_SECOND,
|
||||
bytes_per_second: DEFAULT_BYTES_PER_SECOND,
|
||||
heartbeat: now(),
|
||||
finished_at: None,
|
||||
stop_requested: false,
|
||||
progress: Progress::default(),
|
||||
problems_kept: 0,
|
||||
error: None,
|
||||
});
|
||||
let mut run = Run {
|
||||
server,
|
||||
id,
|
||||
old,
|
||||
target: server.blob_store().current().clone(),
|
||||
progress: Progress::default(),
|
||||
found: Vec::new(),
|
||||
kept: 0,
|
||||
blobs: Pace::new(copy.blobs_per_second),
|
||||
bytes: Pace::new(copy.bytes_per_second),
|
||||
saved: Instant::now(),
|
||||
};
|
||||
let (status, error) = match run.go().await {
|
||||
Ok(()) if run.progress.problems() == 0 => (Status::Copied, None),
|
||||
Ok(()) => (Status::Incomplete, None),
|
||||
Err(Halt::Stopped) => (Status::Stopped, None),
|
||||
Err(Halt::Failed(err)) => {
|
||||
let text = err
|
||||
.value_as_str(trc::Key::Details)
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| err.to_string());
|
||||
trc::error!(err.details("A blob copy failed"));
|
||||
(Status::Failed, Some(text))
|
||||
}
|
||||
};
|
||||
(status, error, Some(run))
|
||||
}
|
||||
};
|
||||
|
||||
let data = server.store();
|
||||
let mut result = Ok(());
|
||||
let (progress, kept) = match &run {
|
||||
Some(run) => {
|
||||
let first = run.kept - run.found.len() as u64;
|
||||
result = model::add_problems(data, id, first, &run.found).await;
|
||||
(run.progress.clone(), run.kept)
|
||||
}
|
||||
None => (Progress::default(), 0),
|
||||
};
|
||||
if result.is_ok() {
|
||||
result = model::update(data, id, |c| {
|
||||
c.status = status;
|
||||
// BM-6: resolved, so the old store's settings go
|
||||
if status == Status::Copied {
|
||||
c.old_settings = None;
|
||||
}
|
||||
if let Some(copy) = &mut c.copy {
|
||||
copy.progress = progress.clone();
|
||||
copy.problems_kept = kept;
|
||||
copy.finished_at = Some(now());
|
||||
copy.heartbeat = now();
|
||||
}
|
||||
if let Some(error) = &error {
|
||||
c.set_error(error);
|
||||
}
|
||||
true
|
||||
})
|
||||
.await
|
||||
.map(|_| ());
|
||||
}
|
||||
if let Err(err) = result.and(model::release(data, id).await) {
|
||||
trc::error!(err.details("Failed to save a blob copy's result"));
|
||||
}
|
||||
if status == Status::Copied {
|
||||
refresh(server).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn keys_are_named_by_hash_or_as_is() {
|
||||
let hash = BlobHash::generate(b"hello");
|
||||
assert_eq!(key_name(hash.as_slice()), hash.to_hex());
|
||||
assert_eq!(key_name(SPAM_TRAINER_KEY), "INBUXA_SPAM_TRAIN_DATA.lz4");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_document_queue_and_kept_blobs_are_checked() {
|
||||
let blob = |holders| LinkedBlob {
|
||||
hash: BlobHash::generate(b"x"),
|
||||
committed: true,
|
||||
holders,
|
||||
temporary: true,
|
||||
};
|
||||
assert!(!Holder::of(&blob(vec![])).content_named);
|
||||
assert!(Holder::of(&blob(vec![BlobHolder::Queue { id: 4 }])).content_named);
|
||||
let mail = Holder::of(&blob(vec![BlobHolder::Document {
|
||||
account_id: 1,
|
||||
collection: Collection::Email.into(),
|
||||
document_id: 9,
|
||||
}]));
|
||||
assert_eq!(mail.name, "email");
|
||||
assert_eq!((mail.account_id, mail.object_id), (Some(1), Some(9)));
|
||||
assert!(!Holder::fixed().content_named);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,556 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Checking the store's blobs (docs/spec/features/store-check.md): the run
|
||||
//! behind an `inbuxa:StoreCheck`. It goes through the blob index one slice
|
||||
//! at a time and asks the blob store for each blob something holds, then
|
||||
//! through each account's mail metadata for the blob each message names
|
||||
//! (SV-3). It reports and never repairs. It runs on the node the check was
|
||||
//! started on, against the stores every node shares.
|
||||
|
||||
use common::{BuildServer, Inner, Server};
|
||||
use email::message::metadata::MessageMetadata;
|
||||
use inbuxa_features::store_check::{
|
||||
self as model, MAX_PROBLEMS, Mode, Problem, ProblemKind, Progress, Refusal, STALE_AFTER,
|
||||
Status, StoreCheck,
|
||||
};
|
||||
use registry::schema::prelude::ObjectType;
|
||||
|
||||
use std::{
|
||||
collections::{BTreeSet, HashMap},
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use store::{
|
||||
Deserialize, IterateParams, U32_LEN, ValueKey,
|
||||
registry::RegistryQuery,
|
||||
roaring::RoaringBitmap,
|
||||
write::{
|
||||
AlignedBytes, Archive,
|
||||
blob_check::{BlobHolder, LinkedBlob},
|
||||
key::DeserializeBigEndian,
|
||||
now,
|
||||
},
|
||||
};
|
||||
use trc::AddContext;
|
||||
use types::{blob::BlobClass, blob_hash::BlobHash, collection::Collection, field::EmailField};
|
||||
|
||||
/// SV-7: how often a running check saves its progress, which is also its
|
||||
/// heartbeat and when it sees a stop.
|
||||
const SAVE_EVERY: Duration = Duration::from_secs(10);
|
||||
|
||||
fn node_id(server: &Server) -> u64 {
|
||||
server.core.network.node_id
|
||||
}
|
||||
|
||||
/// SV-2: starts a check on this node. Refused while another runs.
|
||||
pub async fn start(
|
||||
server: &Server,
|
||||
mode: Mode,
|
||||
blobs_per_second: Option<u64>,
|
||||
bytes_per_second: Option<u64>,
|
||||
created_by: String,
|
||||
) -> trc::Result<Result<u32, Refusal>> {
|
||||
let now = now();
|
||||
let mut check = StoreCheck::new(mode, created_by, node_id(server), now);
|
||||
if let Some(limit) = blobs_per_second {
|
||||
check.blobs_per_second = limit;
|
||||
}
|
||||
if let Some(limit) = bytes_per_second {
|
||||
check.bytes_per_second = limit;
|
||||
}
|
||||
let id = match model::create(server.store(), &check, now).await? {
|
||||
Ok(id) => id,
|
||||
Err(refusal) => return Ok(Err(refusal)),
|
||||
};
|
||||
let check = StoreCheck { id, ..check };
|
||||
let server = server.clone();
|
||||
tokio::spawn(async move {
|
||||
run(&server, check).await;
|
||||
});
|
||||
Ok(Ok(id))
|
||||
}
|
||||
|
||||
/// SV-8: asks a running check to stop. Any node can: the one running it
|
||||
/// sees the request at its next save. None if there's no such check.
|
||||
pub async fn stop(server: &Server, id: u32) -> trc::Result<Option<StoreCheck>> {
|
||||
model::update(server.store(), id, |c| {
|
||||
if c.status != Status::Running || c.stop_requested {
|
||||
return false;
|
||||
}
|
||||
c.stop_requested = true;
|
||||
true
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// SV-8: at start-up, a check this node ran stopped with it. Then, now and
|
||||
/// then, a check on a node that went quiet is taken to have stopped too.
|
||||
pub fn spawn_store_checks(inner: Arc<Inner>) {
|
||||
tokio::spawn(async move {
|
||||
let server = inner.build_server();
|
||||
let node = node_id(&server);
|
||||
mark_interrupted(&server, |c| c.node_id == node).await;
|
||||
loop {
|
||||
tokio::time::sleep(Duration::from_secs(60)).await;
|
||||
let server = inner.build_server();
|
||||
let cutoff = now().saturating_sub(STALE_AFTER);
|
||||
mark_interrupted(&server, |c| c.node_id != node && c.heartbeat < cutoff).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async fn mark_interrupted(server: &Server, which: impl Fn(&StoreCheck) -> bool) {
|
||||
let checks = match model::all(server.store()).await {
|
||||
Ok(checks) => checks,
|
||||
Err(err) => {
|
||||
trc::error!(err.details("Failed to read the store checks"));
|
||||
return;
|
||||
}
|
||||
};
|
||||
for check in checks
|
||||
.into_iter()
|
||||
.filter(|c| c.status == Status::Running && which(c))
|
||||
{
|
||||
let result = model::update(server.store(), check.id, |c| {
|
||||
if c.status != Status::Running || !which(c) {
|
||||
return false;
|
||||
}
|
||||
c.status = Status::Interrupted;
|
||||
c.finished_at = Some(now());
|
||||
true
|
||||
})
|
||||
.await;
|
||||
if let Err(err) = result.and(model::release(server.store(), check.id).await) {
|
||||
trc::error!(err.details("Failed to mark an interrupted store check"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// SV-6: keeps a rate, a second at a time, so a slow stretch isn't made
|
||||
/// up for with a burst.
|
||||
struct Pace {
|
||||
per_second: u64,
|
||||
window: Instant,
|
||||
used: u64,
|
||||
}
|
||||
|
||||
impl Pace {
|
||||
fn new(per_second: u64) -> Self {
|
||||
Pace {
|
||||
per_second,
|
||||
window: Instant::now(),
|
||||
used: 0,
|
||||
}
|
||||
}
|
||||
|
||||
async fn take(&mut self, amount: u64) {
|
||||
if self.per_second == 0 {
|
||||
return;
|
||||
}
|
||||
if self.window.elapsed() >= Duration::from_secs(1) {
|
||||
self.window = Instant::now();
|
||||
self.used = 0;
|
||||
}
|
||||
self.used += amount;
|
||||
if self.used >= self.per_second {
|
||||
let rest = Duration::from_secs(1).saturating_sub(self.window.elapsed());
|
||||
tokio::time::sleep(rest).await;
|
||||
self.window = Instant::now();
|
||||
self.used = self.used.saturating_sub(self.per_second);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What a blob turned out to be (SV-4).
|
||||
enum Found {
|
||||
Fine,
|
||||
Problem(ProblemKind, String),
|
||||
}
|
||||
|
||||
/// The name a problem gives an object (SV-4).
|
||||
fn object_name(collection: u8) -> String {
|
||||
match Collection::from(collection) {
|
||||
Collection::Email => "email".into(),
|
||||
Collection::FileNode => "file".into(),
|
||||
Collection::SieveScript => "sieveScript".into(),
|
||||
Collection::CalendarEvent => "calendarEvent".into(),
|
||||
Collection::ContactCard => "contactCard".into(),
|
||||
_ => format!("collection{collection}"),
|
||||
}
|
||||
}
|
||||
|
||||
struct Run<'x> {
|
||||
server: &'x Server,
|
||||
check: StoreCheck,
|
||||
progress: Progress,
|
||||
found: Vec<Problem>,
|
||||
kept: u64,
|
||||
names: HashMap<u32, String>,
|
||||
blobs: Pace,
|
||||
bytes: Pace,
|
||||
saved: Instant,
|
||||
}
|
||||
|
||||
/// How a run ended before it finished.
|
||||
enum Halt {
|
||||
Stopped,
|
||||
Failed(trc::Error),
|
||||
}
|
||||
|
||||
impl From<trc::Error> for Halt {
|
||||
fn from(err: trc::Error) -> Self {
|
||||
Halt::Failed(err)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> Run<'x> {
|
||||
async fn account_name(&mut self, account_id: u32) -> String {
|
||||
if let Some(name) = self.names.get(&account_id) {
|
||||
return name.clone();
|
||||
}
|
||||
let name = self.server.audit_account_name(account_id).await;
|
||||
self.names.insert(account_id, name.clone());
|
||||
name
|
||||
}
|
||||
|
||||
/// SV-4, SV-5: counts a problem, logs it, and keeps the first ones.
|
||||
async fn problem(
|
||||
&mut self,
|
||||
kind: ProblemKind,
|
||||
account_id: Option<u32>,
|
||||
object: String,
|
||||
object_id: u64,
|
||||
hash: &BlobHash,
|
||||
committed: bool,
|
||||
detail: String,
|
||||
) {
|
||||
self.progress.count(kind);
|
||||
let account_name = match account_id {
|
||||
Some(id) => self.account_name(id).await,
|
||||
None => String::new(),
|
||||
};
|
||||
let blob_id = hash.to_hex();
|
||||
trc::event!(
|
||||
Store(trc::StoreEvent::DataCorruption),
|
||||
Id = self.check.id,
|
||||
AccountId = account_id,
|
||||
Collection = object.clone(),
|
||||
DocumentId = object_id,
|
||||
BlobId = blob_id.clone(),
|
||||
Reason = kind.as_str(),
|
||||
Details = detail.clone(),
|
||||
);
|
||||
if self.kept < MAX_PROBLEMS {
|
||||
self.kept += 1;
|
||||
self.found.push(Problem {
|
||||
kind,
|
||||
account_id,
|
||||
account_name,
|
||||
object,
|
||||
object_id,
|
||||
blob_id,
|
||||
committed,
|
||||
detail,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// SV-7, SV-8: saves progress and the problems found since, now and
|
||||
/// then; a stop asked for meanwhile ends the run.
|
||||
async fn save(&mut self, force: bool) -> Result<(), Halt> {
|
||||
if !force && self.saved.elapsed() < SAVE_EVERY {
|
||||
return Ok(());
|
||||
}
|
||||
let data = self.server.store();
|
||||
let first = self.kept - self.found.len() as u64;
|
||||
model::add_problems(data, self.check.id, first, &self.found).await?;
|
||||
self.found.clear();
|
||||
let progress = self.progress.clone();
|
||||
let kept = self.kept;
|
||||
let saved = model::update(data, self.check.id, |c| {
|
||||
c.progress = progress.clone();
|
||||
c.problems_kept = kept;
|
||||
c.heartbeat = now();
|
||||
true
|
||||
})
|
||||
.await?;
|
||||
self.saved = Instant::now();
|
||||
match saved {
|
||||
Some(c) if c.stop_requested => Err(Halt::Stopped),
|
||||
Some(_) => Ok(()),
|
||||
// Forgotten under it: nothing left to report to
|
||||
None => Err(Halt::Stopped),
|
||||
}
|
||||
}
|
||||
|
||||
/// SV-2: whether the blob is there and, in a full check, intact.
|
||||
async fn look_at(&mut self, hash: &BlobHash) -> Found {
|
||||
let store = self.server.blob_store();
|
||||
match self.check.mode {
|
||||
Mode::Quick => match store.blob_present(hash.as_slice()).await {
|
||||
Ok(true) => Found::Fine,
|
||||
Ok(false) => Found::Problem(ProblemKind::Missing, String::new()),
|
||||
Err(err) => Found::Problem(ProblemKind::Unreadable, err.to_string()),
|
||||
},
|
||||
Mode::Full => match store.get_blob(hash.as_slice(), 0..usize::MAX).await {
|
||||
Ok(Some(content)) => {
|
||||
let size = content.len() as u64;
|
||||
self.progress.bytes_read += size;
|
||||
self.bytes.take(size).await;
|
||||
let actual = BlobHash::generate(&content);
|
||||
if &actual == hash {
|
||||
Found::Fine
|
||||
} else {
|
||||
Found::Problem(
|
||||
ProblemKind::Damaged,
|
||||
format!("its {size} bytes hash to {}", actual.to_hex()),
|
||||
)
|
||||
}
|
||||
}
|
||||
Ok(None) => Found::Problem(ProblemKind::Missing, String::new()),
|
||||
Err(err)
|
||||
if matches!(
|
||||
err.as_ref(),
|
||||
trc::EventType::Store(trc::StoreEvent::DecompressError)
|
||||
) =>
|
||||
{
|
||||
Found::Problem(ProblemKind::Damaged, "it doesn't decompress".into())
|
||||
}
|
||||
Err(err) => Found::Problem(ProblemKind::Unreadable, err.to_string()),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// SV-3, links: one slice of the blob index.
|
||||
async fn slice(&mut self, slice: u8, mail: &mut BTreeSet<u32>) -> Result<(), Halt> {
|
||||
let blobs: Vec<LinkedBlob> = self.server.store().linked_blobs(slice).await?;
|
||||
for blob in blobs {
|
||||
self.blobs.take(1).await;
|
||||
self.progress.blobs_checked += 1;
|
||||
self.progress.links_checked += blob.holders.len() as u64;
|
||||
for holder in &blob.holders {
|
||||
if let BlobHolder::Document {
|
||||
account_id,
|
||||
collection,
|
||||
..
|
||||
} = holder
|
||||
&& Collection::from(*collection) == Collection::Email
|
||||
{
|
||||
mail.insert(*account_id);
|
||||
}
|
||||
}
|
||||
if let Found::Problem(kind, detail) = self.look_at(&blob.hash).await {
|
||||
// A shared blob is read once and reported for each holder
|
||||
for holder in &blob.holders {
|
||||
let (account_id, object, object_id) = match holder {
|
||||
BlobHolder::Document {
|
||||
account_id,
|
||||
collection,
|
||||
document_id,
|
||||
} => (
|
||||
Some(*account_id),
|
||||
object_name(*collection),
|
||||
*document_id as u64,
|
||||
),
|
||||
BlobHolder::Queue { id } => (None, "queuedMessage".to_string(), *id),
|
||||
BlobHolder::Archived { account_id, item } => (
|
||||
Some(*account_id),
|
||||
"archivedItem".to_string(),
|
||||
item.id().id(),
|
||||
),
|
||||
};
|
||||
self.problem(
|
||||
kind,
|
||||
account_id,
|
||||
object,
|
||||
object_id,
|
||||
&blob.hash,
|
||||
blob.committed,
|
||||
detail.clone(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
self.save(false).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// SV-3, mail metadata: each message names its blob; it must hold a
|
||||
/// link to it, or nothing keeps the blob from being purged.
|
||||
async fn mail(&mut self, account_id: u32) -> Result<(), Halt> {
|
||||
let data = self.server.store();
|
||||
let mut messages: Vec<(u32, BlobHash)> = Vec::new();
|
||||
data.iterate(
|
||||
IterateParams::new(
|
||||
ValueKey::property(account_id, Collection::Email, 0, EmailField::Metadata),
|
||||
ValueKey::property(
|
||||
account_id,
|
||||
Collection::Email,
|
||||
u32::MAX,
|
||||
EmailField::Metadata,
|
||||
),
|
||||
),
|
||||
|key, value| {
|
||||
let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?;
|
||||
let archive = <Archive<AlignedBytes> as Deserialize>::deserialize(value)?;
|
||||
let metadata = archive.unarchive::<MessageMetadata>()?;
|
||||
messages.push((document_id, BlobHash::from(&metadata.blob_hash)));
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
for (document_id, hash) in messages {
|
||||
self.blobs.take(1).await;
|
||||
self.progress.messages_checked += 1;
|
||||
let linked = data
|
||||
.blob_has_access(
|
||||
&hash,
|
||||
BlobClass::Linked {
|
||||
account_id,
|
||||
collection: Collection::Email.into(),
|
||||
document_id,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if !linked {
|
||||
let committed = data.blob_exists(&hash).await?;
|
||||
let detail = match self.server.blob_store().blob_present(hash.as_slice()).await {
|
||||
Ok(true) => "the blob is still there".to_string(),
|
||||
Ok(false) => "the blob is gone".to_string(),
|
||||
Err(err) => format!("the blob store couldn't say: {err}"),
|
||||
};
|
||||
self.problem(
|
||||
ProblemKind::Unlinked,
|
||||
Some(account_id),
|
||||
"email".into(),
|
||||
document_id as u64,
|
||||
&hash,
|
||||
committed,
|
||||
detail,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.save(false).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn go(&mut self) -> Result<(), Halt> {
|
||||
let mut mail = BTreeSet::new();
|
||||
for slice in 0u8..=255 {
|
||||
self.slice(slice, &mut mail).await?;
|
||||
self.progress.slices = slice as u32 + 1;
|
||||
self.save(false).await?;
|
||||
}
|
||||
|
||||
// Every account, and any whose mail is still linked without it
|
||||
// being in the registry any more (kept after deletion)
|
||||
let registry: RoaringBitmap = self
|
||||
.server
|
||||
.registry()
|
||||
.query::<RoaringBitmap>(RegistryQuery::new(ObjectType::Account))
|
||||
.await?;
|
||||
mail.extend(registry.iter());
|
||||
self.progress.accounts_total = mail.len() as u64;
|
||||
for account_id in mail {
|
||||
self.mail(account_id).await?;
|
||||
self.progress.accounts_done += 1;
|
||||
self.save(false).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs a check to its end and records how it ended (SV-8).
|
||||
async fn run(server: &Server, check: StoreCheck) {
|
||||
let id = check.id;
|
||||
let mut run = Run {
|
||||
server,
|
||||
blobs: Pace::new(check.blobs_per_second),
|
||||
bytes: Pace::new(if check.mode == Mode::Full {
|
||||
check.bytes_per_second
|
||||
} else {
|
||||
0
|
||||
}),
|
||||
check,
|
||||
progress: Progress::default(),
|
||||
found: Vec::new(),
|
||||
kept: 0,
|
||||
names: HashMap::new(),
|
||||
saved: Instant::now(),
|
||||
};
|
||||
let ended = run.go().await;
|
||||
let (status, error) = match ended {
|
||||
Ok(()) => (Status::Done, None),
|
||||
Err(Halt::Stopped) => (Status::Stopped, None),
|
||||
Err(Halt::Failed(err)) => {
|
||||
let text = err
|
||||
.value_as_str(trc::Key::Details)
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| err.to_string());
|
||||
trc::error!(err.details("A store check failed"));
|
||||
(Status::Failed, Some(text))
|
||||
}
|
||||
};
|
||||
|
||||
let data = server.store();
|
||||
let first = run.kept - run.found.len() as u64;
|
||||
let mut result = model::add_problems(data, id, first, &run.found).await;
|
||||
if result.is_ok() {
|
||||
let progress = run.progress.clone();
|
||||
let kept = run.kept;
|
||||
result = model::update(data, id, |c| {
|
||||
c.progress = progress.clone();
|
||||
c.problems_kept = kept;
|
||||
c.status = status;
|
||||
c.finished_at = Some(now());
|
||||
c.heartbeat = now();
|
||||
if let Some(error) = &error {
|
||||
c.set_error(error);
|
||||
}
|
||||
true
|
||||
})
|
||||
.await
|
||||
.map(|_| ());
|
||||
}
|
||||
if let Err(err) = result.and(model::release(data, id).await) {
|
||||
trc::error!(err.details("Failed to save a store check's result"));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn pace_holds_a_rate() {
|
||||
let mut pace = Pace::new(100);
|
||||
let started = Instant::now();
|
||||
for _ in 0..250 {
|
||||
pace.take(1).await;
|
||||
}
|
||||
// 250 at 100 a second: two full seconds waited, a bit of a third
|
||||
assert!(started.elapsed() >= Duration::from_millis(1_900));
|
||||
let mut free = Pace::new(0);
|
||||
let started = Instant::now();
|
||||
for _ in 0..10_000 {
|
||||
free.take(1).await;
|
||||
}
|
||||
assert!(started.elapsed() < Duration::from_millis(500));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn objects_are_named_by_collection() {
|
||||
assert_eq!(object_name(Collection::Email.into()), "email");
|
||||
assert_eq!(object_name(Collection::FileNode.into()), "file");
|
||||
assert_eq!(object_name(Collection::SieveScript.into()), "sieveScript");
|
||||
assert_eq!(object_name(Collection::Mailbox.into()), "collection1");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,502 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Snapshots of the store (docs/spec/features/store-snapshot.md): the run
|
||||
//! behind an `inbuxa:StoreSnapshot`. It takes a RocksDB checkpoint and, for
|
||||
//! a filesystem blob store, a copy of the blob files with the blob purge
|
||||
//! held, into a directory under the store's own (SN-2, SN-3). The snapshots
|
||||
//! on disk are the record: each one's `snapshot.json` says what it is. Only
|
||||
//! the one being taken, or the last one that failed, is kept in memory.
|
||||
|
||||
use common::{Inner, Server};
|
||||
use mail_parser::DateTime;
|
||||
use serde_json::{Value, json};
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
str::FromStr,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
use store::write::{
|
||||
now,
|
||||
snapshot::{NotSnapshottable, SnapshotSource, copy_tree, hold_blob_purge, tree_size},
|
||||
};
|
||||
use types::id::Id;
|
||||
|
||||
/// SN-6: complete snapshots kept.
|
||||
pub const KEEP: usize = 2;
|
||||
const SNAPSHOTS_DIR: &str = "snapshots";
|
||||
const MANIFEST: &str = "snapshot.json";
|
||||
const LATEST: &str = "latest";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Status {
|
||||
Running,
|
||||
Done,
|
||||
Failed,
|
||||
}
|
||||
|
||||
impl Status {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Status::Running => "running",
|
||||
Status::Done => "done",
|
||||
Status::Failed => "failed",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Snapshot {
|
||||
/// When it was taken, in seconds: its name is that time.
|
||||
pub id: u64,
|
||||
pub name: String,
|
||||
pub status: Status,
|
||||
pub created_at: u64,
|
||||
pub created_by: String,
|
||||
pub finished_at: Option<u64>,
|
||||
pub node_id: u64,
|
||||
pub path: PathBuf,
|
||||
pub holds: Vec<String>,
|
||||
pub size: u64,
|
||||
pub blob_files: u64,
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Why a snapshot isn't taken.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Refusal {
|
||||
/// SN-6: this one is being taken.
|
||||
Running(u64),
|
||||
/// SN-1, SN-2: these stores can't have one.
|
||||
Unsupported(String),
|
||||
}
|
||||
|
||||
/// What `destroy` did.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Destroyed {
|
||||
Gone,
|
||||
NotFound,
|
||||
Running,
|
||||
}
|
||||
|
||||
/// SN-7: the one being taken, or the last one that failed, until restart.
|
||||
static CURRENT: Mutex<Option<Snapshot>> = Mutex::new(None);
|
||||
|
||||
fn current() -> Option<Snapshot> {
|
||||
CURRENT.lock().map(|c| c.clone()).unwrap_or(None)
|
||||
}
|
||||
|
||||
fn set_current(snapshot: Option<Snapshot>) {
|
||||
if let Ok(mut current) = CURRENT.lock() {
|
||||
*current = snapshot;
|
||||
}
|
||||
}
|
||||
|
||||
/// The snapshots' directory, and what a snapshot would hold, or why there
|
||||
/// can't be one here.
|
||||
fn source(server: &Server) -> Result<(SnapshotSource, PathBuf), Refusal> {
|
||||
match server.store().snapshot_source(server.blob_store()) {
|
||||
Ok(source) => {
|
||||
let root = match &source {
|
||||
SnapshotSource::DataOnly { data_path }
|
||||
| SnapshotSource::DataAndFiles { data_path, .. } => data_path.join(SNAPSHOTS_DIR),
|
||||
};
|
||||
Ok((source, root))
|
||||
}
|
||||
Err(NotSnapshottable::DataStore(kind)) => Err(Refusal::Unsupported(format!(
|
||||
"A snapshot needs RocksDB as the data store, and this server's is {kind}. \
|
||||
Use {kind}'s own backup."
|
||||
))),
|
||||
Err(NotSnapshottable::BlobStore(kind)) => Err(Refusal::Unsupported(format!(
|
||||
"A snapshot copies blobs kept in the data store or in a filesystem blob store, \
|
||||
and this server keeps them in {kind}."
|
||||
))),
|
||||
// blob-store-change spec, BM-8
|
||||
Err(NotSnapshottable::BlobStoreChanging) => Err(Refusal::Unsupported(
|
||||
"The blob store changed and some blobs may still be only in the old one. \
|
||||
Copy them over or dismiss the change (inbuxa:BlobStoreChange) first."
|
||||
.to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// `20261109T013000Z`: a snapshot's name, the UTC time it was taken.
|
||||
pub fn name_for(seconds: u64) -> String {
|
||||
let t = DateTime::from_timestamp(seconds as i64);
|
||||
format!(
|
||||
"{:04}{:02}{:02}T{:02}{:02}{:02}Z",
|
||||
t.year, t.month, t.day, t.hour, t.minute, t.second
|
||||
)
|
||||
}
|
||||
|
||||
fn rfc3339(seconds: u64) -> String {
|
||||
DateTime::from_timestamp(seconds as i64).to_rfc3339()
|
||||
}
|
||||
|
||||
fn seconds_of(value: &Value) -> Option<u64> {
|
||||
DateTime::parse_rfc3339(value.as_str()?).map(|t| t.to_timestamp() as u64)
|
||||
}
|
||||
|
||||
fn io(err: std::io::Error) -> trc::Error {
|
||||
trc::StoreEvent::FilesystemError.reason(err)
|
||||
}
|
||||
|
||||
/// SN-1, SN-6: starts taking a snapshot on this node. Refused while one is
|
||||
/// being taken, or when the stores can't have one.
|
||||
pub async fn start(server: &Server, created_by: String) -> trc::Result<Result<Snapshot, Refusal>> {
|
||||
let (source, root) = match source(server) {
|
||||
Ok(found) => found,
|
||||
Err(refusal) => return Ok(Err(refusal)),
|
||||
};
|
||||
let snapshot = {
|
||||
let Ok(mut current) = CURRENT.lock() else {
|
||||
return Err(trc::EventType::Server(trc::ServerEvent::ThreadError)
|
||||
.into_err()
|
||||
.details("The snapshot state is poisoned"));
|
||||
};
|
||||
if let Some(running) = current.as_ref().filter(|c| c.status == Status::Running) {
|
||||
return Ok(Err(Refusal::Running(running.id)));
|
||||
}
|
||||
// Names are the time to the second: a second one within the same
|
||||
// second takes the next
|
||||
let mut at = now();
|
||||
while root.join(name_for(at)).exists() {
|
||||
at += 1;
|
||||
}
|
||||
let name = name_for(at);
|
||||
let mut holds = vec!["data".to_string()];
|
||||
if matches!(source, SnapshotSource::DataAndFiles { .. }) {
|
||||
holds.push("blobs".to_string());
|
||||
}
|
||||
let snapshot = Snapshot {
|
||||
id: at,
|
||||
path: root.join(&name),
|
||||
name,
|
||||
status: Status::Running,
|
||||
created_at: at,
|
||||
created_by,
|
||||
finished_at: None,
|
||||
node_id: server.core.network.node_id,
|
||||
holds,
|
||||
size: 0,
|
||||
blob_files: 0,
|
||||
error: None,
|
||||
};
|
||||
*current = Some(snapshot.clone());
|
||||
snapshot
|
||||
};
|
||||
let server = server.clone();
|
||||
let running = snapshot.clone();
|
||||
tokio::spawn(async move {
|
||||
run(&server, source, root, running).await;
|
||||
});
|
||||
Ok(Ok(snapshot))
|
||||
}
|
||||
|
||||
async fn run(server: &Server, source: SnapshotSource, root: PathBuf, mut snapshot: Snapshot) {
|
||||
let partial = root.join(format!(".{}.partial", snapshot.name));
|
||||
match take(server, &source, &root, &partial, &mut snapshot).await {
|
||||
// Listed from disk from now on
|
||||
Ok(()) => set_current(None),
|
||||
Err(err) => {
|
||||
let _ = tokio::fs::remove_dir_all(&partial).await;
|
||||
snapshot.status = Status::Failed;
|
||||
snapshot.finished_at = Some(now());
|
||||
snapshot.error = Some(err.to_string());
|
||||
trc::error!(err.details("Failed to take a store snapshot"));
|
||||
set_current(Some(snapshot));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn take(
|
||||
server: &Server,
|
||||
source: &SnapshotSource,
|
||||
root: &Path,
|
||||
partial: &Path,
|
||||
snapshot: &mut Snapshot,
|
||||
) -> trc::Result<()> {
|
||||
tokio::fs::create_dir_all(root).await.map_err(io)?;
|
||||
let _ = tokio::fs::remove_dir_all(partial).await;
|
||||
tokio::fs::create_dir(partial).await.map_err(io)?;
|
||||
|
||||
// SN-4: the purge is held before the checkpoint, so every blob the
|
||||
// checkpoint links to is still there when it's copied
|
||||
let hold = match source {
|
||||
SnapshotSource::DataAndFiles { .. } => Some(hold_blob_purge().await),
|
||||
SnapshotSource::DataOnly { .. } => None,
|
||||
};
|
||||
server.store().checkpoint(partial.join("data")).await?;
|
||||
let blob_path = match source {
|
||||
SnapshotSource::DataAndFiles { blob_path, .. } => {
|
||||
let copy = copy_tree(
|
||||
blob_path.clone(),
|
||||
partial.join("blobs"),
|
||||
data_path_of(source).to_path_buf(),
|
||||
)
|
||||
.await?;
|
||||
snapshot.blob_files = copy.files;
|
||||
Some(blob_path)
|
||||
}
|
||||
SnapshotSource::DataOnly { .. } => None,
|
||||
};
|
||||
drop(hold);
|
||||
|
||||
snapshot.size = tree_size(partial.to_path_buf()).await?;
|
||||
snapshot.status = Status::Done;
|
||||
snapshot.finished_at = Some(now());
|
||||
let manifest = manifest(
|
||||
snapshot,
|
||||
data_path_of(source),
|
||||
blob_path.map(|p| p.as_path()),
|
||||
);
|
||||
tokio::fs::write(
|
||||
partial.join(MANIFEST),
|
||||
serde_json::to_vec_pretty(&manifest).unwrap_or_default(),
|
||||
)
|
||||
.await
|
||||
.map_err(io)?;
|
||||
|
||||
// SN-3: whole once it has its name
|
||||
tokio::fs::rename(partial, &snapshot.path)
|
||||
.await
|
||||
.map_err(io)?;
|
||||
if let Ok(dir) = tokio::fs::File::open(root).await {
|
||||
let _ = dir.sync_all().await;
|
||||
}
|
||||
prune(root).await;
|
||||
refresh_latest(root).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn data_path_of(source: &SnapshotSource) -> &Path {
|
||||
match source {
|
||||
SnapshotSource::DataOnly { data_path } | SnapshotSource::DataAndFiles { data_path, .. } => {
|
||||
data_path
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// SN-5.
|
||||
fn manifest(snapshot: &Snapshot, data_path: &Path, blob_path: Option<&Path>) -> Value {
|
||||
let mut restore = vec![
|
||||
"Stop the server.".to_string(),
|
||||
format!(
|
||||
"Move the contents of {} aside, except snapshots/, and copy this snapshot's data/ into it.",
|
||||
data_path.display()
|
||||
),
|
||||
];
|
||||
if let Some(blob_path) = blob_path {
|
||||
restore.push(format!(
|
||||
"Move the contents of {} aside and copy this snapshot's blobs/ into it.",
|
||||
blob_path.display()
|
||||
));
|
||||
}
|
||||
restore.push("Start the server.".to_string());
|
||||
json!({
|
||||
"id": Id::from(snapshot.id).to_string(),
|
||||
"name": snapshot.name,
|
||||
"createdAt": rfc3339(snapshot.created_at),
|
||||
"finishedAt": snapshot.finished_at.map(rfc3339),
|
||||
"createdBy": snapshot.created_by,
|
||||
"node": snapshot.node_id.to_string(),
|
||||
"server": concat!(types::brand_server!(), " ", types::brand_version!()),
|
||||
"holds": snapshot.holds,
|
||||
"dataStore": {"type": "RocksDB", "path": data_path.to_string_lossy()},
|
||||
"blobStore": match blob_path {
|
||||
Some(path) => json!({"type": "filesystem", "path": path.to_string_lossy()}),
|
||||
None => json!({"type": "in the data store"}),
|
||||
},
|
||||
"size": snapshot.size,
|
||||
"blobFiles": snapshot.blob_files,
|
||||
"restore": restore,
|
||||
})
|
||||
}
|
||||
|
||||
fn from_manifest(path: PathBuf, manifest: &Value) -> Option<Snapshot> {
|
||||
let id = Id::from_str(manifest.get("id")?.as_str()?).ok()?.id();
|
||||
Some(Snapshot {
|
||||
id,
|
||||
name: manifest.get("name")?.as_str()?.to_string(),
|
||||
status: Status::Done,
|
||||
created_at: seconds_of(manifest.get("createdAt")?)?,
|
||||
created_by: manifest.get("createdBy")?.as_str()?.to_string(),
|
||||
finished_at: manifest.get("finishedAt").and_then(seconds_of),
|
||||
node_id: manifest.get("node")?.as_str()?.parse().ok()?,
|
||||
path,
|
||||
holds: manifest
|
||||
.get("holds")?
|
||||
.as_array()?
|
||||
.iter()
|
||||
.filter_map(|h| h.as_str().map(|s| s.to_string()))
|
||||
.collect(),
|
||||
size: manifest.get("size")?.as_u64()?,
|
||||
blob_files: manifest.get("blobFiles")?.as_u64()?,
|
||||
error: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// The complete snapshots under `root`, oldest first. A directory without a
|
||||
/// readable `snapshot.json` isn't one, and is left alone.
|
||||
async fn on_disk(root: &Path) -> Vec<Snapshot> {
|
||||
let mut found = Vec::new();
|
||||
let Ok(mut entries) = tokio::fs::read_dir(root).await else {
|
||||
return found;
|
||||
};
|
||||
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
if name.starts_with('.') || !entry.file_type().await.is_ok_and(|t| t.is_dir()) {
|
||||
continue;
|
||||
}
|
||||
let path = entry.path();
|
||||
let Ok(bytes) = tokio::fs::read(path.join(MANIFEST)).await else {
|
||||
continue;
|
||||
};
|
||||
if let Some(snapshot) = serde_json::from_slice::<Value>(&bytes)
|
||||
.ok()
|
||||
.and_then(|manifest| from_manifest(path, &manifest))
|
||||
.filter(|s| s.name == name)
|
||||
{
|
||||
found.push(snapshot);
|
||||
}
|
||||
}
|
||||
found.sort_by_key(|s| s.id);
|
||||
found
|
||||
}
|
||||
|
||||
/// SN-6: the oldest beyond the latest [`KEEP`] go.
|
||||
async fn prune(root: &Path) {
|
||||
let snapshots = on_disk(root).await;
|
||||
let surplus = snapshots.len().saturating_sub(KEEP);
|
||||
for old in &snapshots[..surplus] {
|
||||
if let Err(err) = tokio::fs::remove_dir_all(&old.path).await {
|
||||
trc::error!(io(err).details("Failed to remove an old store snapshot"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// SN-3: `latest` names the newest complete snapshot, relative to the
|
||||
/// snapshots' directory so it holds wherever that's mounted.
|
||||
async fn refresh_latest(root: &Path) {
|
||||
let latest = root.join(LATEST);
|
||||
match on_disk(root).await.last() {
|
||||
Some(newest) => {
|
||||
let temp = root.join(format!(".{LATEST}.tmp"));
|
||||
let _ = tokio::fs::remove_file(&temp).await;
|
||||
let linked = async {
|
||||
tokio::fs::symlink(&newest.name, &temp).await?;
|
||||
tokio::fs::rename(&temp, &latest).await
|
||||
}
|
||||
.await;
|
||||
if let Err(err) = linked {
|
||||
trc::error!(io(err).details("Failed to point snapshots/latest at a snapshot"));
|
||||
}
|
||||
}
|
||||
None => {
|
||||
let _ = tokio::fs::remove_file(&latest).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Every snapshot, newest first: those on disk, and the one being taken or
|
||||
/// the last one that failed (SN-7).
|
||||
pub async fn list(server: &Server) -> Vec<Snapshot> {
|
||||
let mut snapshots = match source(server) {
|
||||
Ok((_, root)) => on_disk(&root).await,
|
||||
Err(_) => Vec::new(),
|
||||
};
|
||||
if let Some(current) = current().filter(|c| !snapshots.iter().any(|s| s.id == c.id)) {
|
||||
snapshots.push(current);
|
||||
}
|
||||
snapshots.sort_by_key(|s| std::cmp::Reverse(s.id));
|
||||
snapshots
|
||||
}
|
||||
|
||||
/// One snapshot, as [`list`] has it.
|
||||
pub async fn find(server: &Server, id: u64) -> Option<Snapshot> {
|
||||
list(server).await.into_iter().find(|s| s.id == id)
|
||||
}
|
||||
|
||||
/// SN-9: deletes a snapshot's directory. One being taken can't be.
|
||||
pub async fn destroy(server: &Server, id: u64) -> trc::Result<Destroyed> {
|
||||
if let Some(current) = current().filter(|c| c.id == id) {
|
||||
if current.status == Status::Running {
|
||||
return Ok(Destroyed::Running);
|
||||
}
|
||||
set_current(None);
|
||||
return Ok(Destroyed::Gone);
|
||||
}
|
||||
let Ok((_, root)) = source(server) else {
|
||||
return Ok(Destroyed::NotFound);
|
||||
};
|
||||
let Some(snapshot) = on_disk(&root).await.into_iter().find(|s| s.id == id) else {
|
||||
return Ok(Destroyed::NotFound);
|
||||
};
|
||||
tokio::fs::remove_dir_all(&snapshot.path)
|
||||
.await
|
||||
.map_err(io)?;
|
||||
refresh_latest(&root).await;
|
||||
Ok(Destroyed::Gone)
|
||||
}
|
||||
|
||||
/// SN-7: a snapshot being taken when the server stopped left a partial
|
||||
/// directory, removed when it starts.
|
||||
pub fn spawn_store_snapshots(inner: Arc<Inner>) {
|
||||
tokio::spawn(async move {
|
||||
let server = common::BuildServer::build_server(&inner);
|
||||
let Ok((_, root)) = source(&server) else {
|
||||
return;
|
||||
};
|
||||
let Ok(mut entries) = tokio::fs::read_dir(&root).await else {
|
||||
return;
|
||||
};
|
||||
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
if name.starts_with('.') && name.ends_with(".partial") {
|
||||
let _ = tokio::fs::remove_dir_all(entry.path()).await;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn names_are_the_time() {
|
||||
assert_eq!(name_for(1_794_187_800), "20261109T013000Z");
|
||||
assert_eq!(name_for(0), "19700101T000000Z");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_manifest_reads_back() {
|
||||
let snapshot = Snapshot {
|
||||
id: 1_794_187_800,
|
||||
name: name_for(1_794_187_800),
|
||||
status: Status::Done,
|
||||
created_at: 1_794_187_800,
|
||||
created_by: "[email protected]".into(),
|
||||
finished_at: Some(1_794_187_802),
|
||||
node_id: 3,
|
||||
path: PathBuf::from("/var/lib/inbuxa/snapshots/20261109T013000Z"),
|
||||
holds: vec!["data".into(), "blobs".into()],
|
||||
size: 4096,
|
||||
blob_files: 12,
|
||||
error: None,
|
||||
};
|
||||
let manifest = manifest(
|
||||
&snapshot,
|
||||
Path::new("/var/lib/inbuxa/"),
|
||||
Some(Path::new("/var/lib/inbuxa-blobs")),
|
||||
);
|
||||
assert_eq!(manifest["restore"].as_array().unwrap().len(), 4);
|
||||
assert_eq!(
|
||||
from_manifest(snapshot.path.clone(), &manifest).unwrap(),
|
||||
snapshot
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -23,10 +23,13 @@ use crate::task_manager::{manager::spawn_task_manager, scheduler::spawn_task_sch
|
||||
|
||||
pub mod broadcast;
|
||||
// inbuxa: AL-5, delegations end at their date
|
||||
pub mod inbuxa_blob_move; // inbuxa: changing the blob store
|
||||
pub mod inbuxa_deliverability; // inbuxa: the deliverability check
|
||||
pub mod inbuxa_lock_expiry;
|
||||
pub mod inbuxa_log_retention; // inbuxa: personal-data catalog, D1
|
||||
pub mod inbuxa_scheduled_reports; // inbuxa: scheduled reports and the weekly digest
|
||||
pub mod inbuxa_store_check; // inbuxa: checking the store's blobs
|
||||
pub mod inbuxa_store_snapshot; // inbuxa: a snapshot of the store
|
||||
pub mod state_manager;
|
||||
pub mod task_manager;
|
||||
|
||||
@@ -78,6 +81,16 @@ impl SpawnServices for IpcReceivers {
|
||||
// inbuxa: deliverability spec, DL-14: each node checks itself daily
|
||||
inbuxa_deliverability::spawn_deliverability(inner.clone());
|
||||
|
||||
// inbuxa: store-check spec, SV-8: a check this node ran stopped with it
|
||||
inbuxa_store_check::spawn_store_checks(inner.clone());
|
||||
|
||||
// inbuxa: blob-store-change spec, BM-6, BM-10: a copy this node
|
||||
// ran stopped with it; resolved changes stop being read through
|
||||
inbuxa_blob_move::spawn_blob_moves(inner.clone());
|
||||
|
||||
// inbuxa: store-snapshot spec, SN-7: a snapshot cut short by a
|
||||
// stop leaves a partial directory
|
||||
inbuxa_store_snapshot::spawn_store_snapshots(inner.clone());
|
||||
// inbuxa: scheduled-reports spec, RP-16: every node looks for due reports
|
||||
inbuxa_scheduled_reports::spawn_scheduled_reports(inner.clone());
|
||||
|
||||
|
||||
@@ -310,6 +310,17 @@ async fn store_maintenance(
|
||||
}
|
||||
TaskStoreMaintenanceType::PurgeBlob => {
|
||||
if let Some(shard_index) = task.shard_index {
|
||||
// inbuxa: store-snapshot spec, SN-4: while a snapshot takes
|
||||
// the blobs, the purge is put off by five minutes
|
||||
let Some(_purging) = store::write::snapshot::blob_purge_may_run() else {
|
||||
return Ok(TaskResult::Success(vec![Task::StoreMaintenance(
|
||||
TaskStoreMaintenance {
|
||||
maintenance_type: TaskStoreMaintenanceType::PurgeBlob,
|
||||
shard_index: Some(shard_index),
|
||||
status: TaskStatus::at(now() as i64 + 300),
|
||||
},
|
||||
)]));
|
||||
};
|
||||
server
|
||||
.store()
|
||||
.purge_blobs(server.blob_store().clone(), shard_index as u8)
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use crate::BlobStore;
|
||||
@@ -77,9 +79,22 @@ impl FsStore {
|
||||
fs::create_dir_all(blob_path.parent().unwrap())
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
let mut blob_file = File::create(&blob_path).await.map_err(into_error)?;
|
||||
blob_file.write_all(data).await.map_err(into_error)?;
|
||||
blob_file.flush().await.map_err(into_error)?;
|
||||
// inbuxa: store-snapshot spec, SN-10: written beside it and
|
||||
// renamed into place, never rewritten in place. A snapshot holds
|
||||
// blobs by hard link, and a rewrite would change its copy too; a
|
||||
// stop mid-write would leave half a blob under its id.
|
||||
let temp_path = temp_path_for(&blob_path);
|
||||
let written = async {
|
||||
let mut blob_file = File::create(&temp_path).await?;
|
||||
blob_file.write_all(data).await?;
|
||||
blob_file.flush().await?;
|
||||
fs::rename(&temp_path, &blob_path).await
|
||||
}
|
||||
.await;
|
||||
if let Err(err) = written {
|
||||
let _ = fs::remove_file(&temp_path).await;
|
||||
return Err(into_error(err));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -95,6 +110,11 @@ impl FsStore {
|
||||
}
|
||||
}
|
||||
|
||||
/// inbuxa: the directory the blobs are under (store-snapshot spec).
|
||||
pub fn root(&self) -> &std::path::Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
fn build_path(&self, key: &[u8]) -> PathBuf {
|
||||
let mut path = self.path.clone();
|
||||
|
||||
@@ -106,6 +126,66 @@ impl FsStore {
|
||||
}
|
||||
}
|
||||
|
||||
/// inbuxa: SN-10. A blob's name is base32, so a name starting with a dot
|
||||
/// is never a blob: the snapshot's copy and anything listing blobs skip it.
|
||||
fn temp_path_for(blob_path: &std::path::Path) -> PathBuf {
|
||||
static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
|
||||
let n = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let name = blob_path
|
||||
.file_name()
|
||||
.map(|name| name.to_string_lossy())
|
||||
.unwrap_or_default();
|
||||
blob_path.with_file_name(format!(".{name}.{}-{n}.tmp", std::process::id()))
|
||||
}
|
||||
|
||||
fn into_error(err: std::io::Error) -> trc::Error {
|
||||
trc::StoreEvent::FilesystemError.reason(err)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn writes_go_through_a_temporary_file() {
|
||||
let dir = std::env::temp_dir().join(format!("inbuxa-fs-sn10-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
let BlobStore::Fs(store) = FsStore::open(structs::FileSystemStore {
|
||||
path: dir.to_string_lossy().to_string(),
|
||||
depth: 2,
|
||||
})
|
||||
.await
|
||||
.unwrap() else {
|
||||
unreachable!()
|
||||
};
|
||||
let key = [7u8; 32];
|
||||
store.put_blob(&key, b"first").await.unwrap();
|
||||
let path = store.build_path(&key);
|
||||
|
||||
// A file held by a hard link, as a snapshot holds it, keeps what it
|
||||
// had when the blob under its id is written again
|
||||
let held = dir.join("held");
|
||||
std::fs::hard_link(&path, &held).unwrap();
|
||||
store.put_blob(&key, b"second, longer").await.unwrap();
|
||||
assert_eq!(std::fs::read(&held).unwrap(), b"first");
|
||||
assert_eq!(
|
||||
store.get_blob(&key, 0..usize::MAX).await.unwrap().unwrap(),
|
||||
b"second, longer"
|
||||
);
|
||||
|
||||
// Nothing is left beside it
|
||||
let names: Vec<_> = std::fs::read_dir(path.parent().unwrap())
|
||||
.unwrap()
|
||||
.map(|entry| entry.unwrap().file_name())
|
||||
.collect();
|
||||
assert_eq!(names.len(), 1, "{names:?}");
|
||||
assert!(
|
||||
temp_path_for(&path)
|
||||
.file_name()
|
||||
.unwrap()
|
||||
.to_string_lossy()
|
||||
.starts_with('.')
|
||||
);
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! inbuxa: a checkpoint of the store (store-snapshot spec, SN-2): every
|
||||
//! column family at one point in time, hard-linked where it can be.
|
||||
|
||||
use super::{RocksDbStore, into_error};
|
||||
use rocksdb::checkpoint::Checkpoint;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
impl RocksDbStore {
|
||||
/// The store's directory.
|
||||
pub fn path(&self) -> &Path {
|
||||
self.db.path()
|
||||
}
|
||||
|
||||
/// Writes a checkpoint to `dest`, which must not exist yet.
|
||||
pub async fn checkpoint(&self, dest: PathBuf) -> trc::Result<()> {
|
||||
let db = self.db.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
Checkpoint::new(&*db)
|
||||
.and_then(|checkpoint| checkpoint.create_checkpoint(&dest))
|
||||
.map_err(into_error)
|
||||
})
|
||||
.await
|
||||
.map_err(|err| trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err))?
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use std::sync::Arc;
|
||||
@@ -11,6 +13,7 @@ use rocksdb::{BoundColumnFamily, MultiThreaded, OptimisticTransactionDB};
|
||||
use crate::{SUBSPACE_BLOBS, SUBSPACE_INDEXES, SUBSPACE_LOGS};
|
||||
|
||||
pub mod blob;
|
||||
pub mod checkpoint; // inbuxa: store-snapshot spec
|
||||
pub mod main;
|
||||
pub mod read;
|
||||
pub mod write;
|
||||
|
||||
+115
-23
@@ -6,14 +6,38 @@
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use crate::{BlobStore, backend::fs::FsStore, registry::bootstrap::Bootstrap};
|
||||
use crate::{BlobStore, Store, backend::fs::FsStore, registry::bootstrap::Bootstrap};
|
||||
use registry::schema::{prelude::ObjectType, structs};
|
||||
|
||||
#[allow(unreachable_patterns)]
|
||||
impl BlobStore {
|
||||
pub async fn build(bp: &mut Bootstrap) -> Option<Self> {
|
||||
let result = match bp.setting_infallible::<structs::BlobStore>().await {
|
||||
structs::BlobStore::Default => return Some(BlobStore::Store(bp.data_store.clone())),
|
||||
let config = bp.setting_infallible::<structs::BlobStore>().await;
|
||||
let mut warnings = Vec::new();
|
||||
let result = BlobStore::open(config, &bp.data_store, &mut warnings).await;
|
||||
for warning in warnings {
|
||||
bp.build_warning(ObjectType::BlobStore.singleton(), warning);
|
||||
}
|
||||
|
||||
match result {
|
||||
Ok(store) => Some(store),
|
||||
Err(err) => {
|
||||
bp.build_error(ObjectType::BlobStore.singleton(), err);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// inbuxa: the blob store a setting names. Split out of `build` so a
|
||||
/// blob store the setting no longer names can be opened too, to read
|
||||
/// what was left in it (blob-store-change spec, BM-2).
|
||||
#[allow(unreachable_patterns)]
|
||||
pub async fn open(
|
||||
config: structs::BlobStore,
|
||||
data_store: &Store,
|
||||
warnings: &mut Vec<String>,
|
||||
) -> Result<Self, String> {
|
||||
match config {
|
||||
structs::BlobStore::Default => Ok(BlobStore::Store(data_store.clone())),
|
||||
#[cfg(feature = "foundation")]
|
||||
structs::BlobStore::FoundationDb(foundation_db_store) => {
|
||||
crate::backend::foundationdb::FdbStore::open(foundation_db_store)
|
||||
@@ -43,27 +67,95 @@ impl BlobStore {
|
||||
}
|
||||
// inbuxa: ST-16 to ST-22
|
||||
structs::BlobStore::Sharded(sharded) => {
|
||||
let mut warnings = Vec::new();
|
||||
let result = crate::backend::scaleout::ShardedBlobStore::open(
|
||||
sharded,
|
||||
&bp.data_store,
|
||||
&mut warnings,
|
||||
)
|
||||
.await;
|
||||
for warning in warnings {
|
||||
bp.build_warning(ObjectType::BlobStore.singleton(), warning);
|
||||
}
|
||||
result
|
||||
crate::backend::scaleout::ShardedBlobStore::open(sharded, data_store, warnings)
|
||||
.await
|
||||
}
|
||||
_ => Err("Binary was not compiled with the selected blob store backend".to_string()),
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(store) => Some(store),
|
||||
Err(err) => {
|
||||
bp.build_error(ObjectType::BlobStore.singleton(), err);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// inbuxa: blob-store-change spec, BM-1: where a blob store setting puts
|
||||
/// blobs, without its credentials or tuning. Two settings with the same
|
||||
/// place read the same blobs; a different place doesn't have them.
|
||||
pub fn blob_store_location(config: &structs::BlobStore) -> String {
|
||||
match config {
|
||||
structs::BlobStore::Default => "the data store".to_string(),
|
||||
// ST-16 to ST-22: the set moves blobs between its members itself
|
||||
structs::BlobStore::Sharded(_) => "a sharded set".to_string(),
|
||||
structs::BlobStore::S3(s3) => {
|
||||
let region = match &s3.region {
|
||||
structs::S3StoreRegion::Custom(custom) => {
|
||||
format!("{} ({})", custom.custom_endpoint, custom.custom_region)
|
||||
}
|
||||
region => format!("{region:?}"),
|
||||
};
|
||||
with_prefix(
|
||||
format!("S3 {region}, bucket {}", s3.bucket),
|
||||
s3.key_prefix.as_deref(),
|
||||
)
|
||||
}
|
||||
structs::BlobStore::Azure(azure) => with_prefix(
|
||||
format!(
|
||||
"Azure {}, container {}",
|
||||
azure.storage_account, azure.container
|
||||
),
|
||||
azure.key_prefix.as_deref(),
|
||||
),
|
||||
structs::BlobStore::FileSystem(fs) => {
|
||||
format!("the directory {}, depth {}", fs.path, fs.depth.min(5))
|
||||
}
|
||||
structs::BlobStore::FoundationDb(fdb) => format!(
|
||||
"FoundationDB {}",
|
||||
fdb.cluster_file
|
||||
.as_deref()
|
||||
.unwrap_or("(default cluster file)")
|
||||
),
|
||||
structs::BlobStore::PostgreSql(pg) => {
|
||||
format!("PostgreSQL {}:{}/{}", pg.host, pg.port, pg.database)
|
||||
}
|
||||
structs::BlobStore::MySql(my) => {
|
||||
format!("MySQL {}:{}/{}", my.host, my.port, my.database)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn with_prefix(place: String, prefix: Option<&str>) -> String {
|
||||
match prefix.filter(|p| !p.is_empty()) {
|
||||
Some(prefix) => format!("{place}, prefix {prefix}"),
|
||||
None => place,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn fs(path: &str, depth: u64) -> structs::BlobStore {
|
||||
structs::BlobStore::FileSystem(structs::FileSystemStore {
|
||||
path: path.into(),
|
||||
depth,
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_place_is_where_blobs_go_not_how() {
|
||||
assert_eq!(
|
||||
blob_store_location(&structs::BlobStore::Default),
|
||||
"the data store"
|
||||
);
|
||||
assert_eq!(
|
||||
blob_store_location(&fs("/srv/blobs", 2)),
|
||||
"the directory /srv/blobs, depth 2"
|
||||
);
|
||||
// The depth decides each blob's path, so it's part of the place
|
||||
assert_ne!(
|
||||
blob_store_location(&fs("/srv/blobs", 2)),
|
||||
blob_store_location(&fs("/srv/blobs", 3))
|
||||
);
|
||||
assert_ne!(
|
||||
blob_store_location(&fs("/srv/blobs", 2)),
|
||||
blob_store_location(&fs("/srv/other", 2))
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,12 @@ const LZ4_MARKER: u8 = MAGIC_MARKER | 0x01;
|
||||
//const ZSTD_MARKER: u8 = MAGIC_MARKER | 0x02;
|
||||
const NONE_MARKER: u8 = 0x00;
|
||||
|
||||
/// What a stored blob decodes to: its content, or an empty value as is.
|
||||
enum Decoded {
|
||||
Content(Vec<u8>),
|
||||
Bare(Vec<u8>),
|
||||
}
|
||||
|
||||
impl BlobStore {
|
||||
pub async fn get_blob(&self, key: &[u8], range: Range<usize>) -> trc::Result<Option<Vec<u8>>> {
|
||||
let start_time = Instant::now();
|
||||
@@ -27,11 +33,42 @@ impl BlobStore {
|
||||
Size = result.as_ref().map_or(0, |data| data.len()),
|
||||
);
|
||||
|
||||
let Some(mut data) = result else {
|
||||
let Some(data) = result else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let mut data = match data.last().copied() {
|
||||
let mut data = match Self::decode_stored(key, data)? {
|
||||
Decoded::Content(data) => data,
|
||||
Decoded::Bare(data) => return Ok(Some(data)),
|
||||
};
|
||||
|
||||
if range.start == 0 {
|
||||
if range.end > data.len() {
|
||||
Ok(Some(data))
|
||||
} else {
|
||||
data.truncate(range.end);
|
||||
Ok(Some(data))
|
||||
}
|
||||
} else {
|
||||
Ok(Some(
|
||||
data.get(range.start..range.end)
|
||||
.unwrap_or_default()
|
||||
.to_vec(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// inbuxa: a blob's content from what the backend keeps, as
|
||||
/// [`BlobStore::get_blob`] reads it (blob-store-change spec, BM-5: a copy
|
||||
/// checks a blob's hash before writing it as it was stored).
|
||||
pub fn content_of_stored(key: &[u8], data: Vec<u8>) -> trc::Result<Vec<u8>> {
|
||||
Self::decode_stored(key, data).map(|decoded| match decoded {
|
||||
Decoded::Content(data) | Decoded::Bare(data) => data,
|
||||
})
|
||||
}
|
||||
|
||||
fn decode_stored(key: &[u8], mut data: Vec<u8>) -> trc::Result<Decoded> {
|
||||
let data = match data.last().copied() {
|
||||
Some(LZ4_MARKER) => {
|
||||
lz4_flex::decompress_size_prepended(data.get(..data.len() - 1).unwrap_or_default())
|
||||
.map_err(|err| {
|
||||
@@ -53,24 +90,10 @@ impl BlobStore {
|
||||
data
|
||||
}
|
||||
None => {
|
||||
return Ok(Some(data));
|
||||
return Ok(Decoded::Bare(data));
|
||||
}
|
||||
};
|
||||
|
||||
if range.start == 0 {
|
||||
if range.end > data.len() {
|
||||
Ok(Some(data))
|
||||
} else {
|
||||
data.truncate(range.end);
|
||||
Ok(Some(data))
|
||||
}
|
||||
} else {
|
||||
Ok(Some(
|
||||
data.get(range.start..range.end)
|
||||
.unwrap_or_default()
|
||||
.to_vec(),
|
||||
))
|
||||
}
|
||||
Ok(Decoded::Content(data))
|
||||
}
|
||||
|
||||
pub async fn put_blob(
|
||||
@@ -183,6 +206,8 @@ impl BlobStore {
|
||||
BlobStore::Azure(store) => store.get_blob(key, 0..usize::MAX).await,
|
||||
// inbuxa: ST-17
|
||||
BlobStore::Sharded(store) => store.get(key).await,
|
||||
// inbuxa: BM-3
|
||||
BlobStore::Moving(store) => store.raw_get(key).await,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,6 +237,8 @@ impl BlobStore {
|
||||
BlobStore::Azure(store) => store.put_blob(key, data).await,
|
||||
// inbuxa: ST-18
|
||||
BlobStore::Sharded(store) => store.put(key, data).await,
|
||||
// inbuxa: BM-3: writes go to the current store only
|
||||
BlobStore::Moving(store) => Box::pin(store.current.raw_put(key, data)).await,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -241,6 +268,8 @@ impl BlobStore {
|
||||
BlobStore::Azure(store) => store.delete_blob(key).await,
|
||||
// inbuxa: ST-18
|
||||
BlobStore::Sharded(store) => store.delete(key).await,
|
||||
// inbuxa: BM-3: nothing is deleted from an old store
|
||||
BlobStore::Moving(store) => Box::pin(store.current.raw_delete(key)).await,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -185,6 +185,9 @@ pub enum BlobStore {
|
||||
Azure(Arc<backend::azure::AzureStore>),
|
||||
// inbuxa: ST-16 to ST-22
|
||||
Sharded(Arc<backend::scaleout::ShardedBlobStore>),
|
||||
// inbuxa: blob-store-change spec, BM-3: the current store, read through
|
||||
// to the old ones while a change is unresolved
|
||||
Moving(Arc<write::blob_move::MovingBlobStore>),
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! inbuxa: what the store-check spec (docs/spec/features/store-check.md)
|
||||
//! reads from the store: each blob the data holds a link to, with who holds
|
||||
//! it, one slice of the blob index at a time (SV-3), and whether a blob is
|
||||
//! in the blob store without reading all of it (SV-2, quick).
|
||||
|
||||
use super::{BlobOp, ValueClass, key::DeserializeBigEndian, now};
|
||||
use crate::{
|
||||
BlobStore, Deserialize, IterateParams, Store, U16_LEN, U32_LEN, U64_LEN, ValueKey,
|
||||
write::BlobLink,
|
||||
};
|
||||
use registry::types::id::ObjectId;
|
||||
use trc::AddContext;
|
||||
use types::blob_hash::{BLOB_HASH_LEN, BlobHash};
|
||||
|
||||
/// Who holds a link to a blob (SV-3).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum BlobHolder {
|
||||
/// A document: an email, a file, a Sieve script, ...
|
||||
Document {
|
||||
account_id: u32,
|
||||
collection: u8,
|
||||
document_id: u32,
|
||||
},
|
||||
/// A queued message.
|
||||
Queue { id: u64 },
|
||||
/// A deleted item kept to be restored, until its link expires.
|
||||
Archived { account_id: u32, item: ObjectId },
|
||||
}
|
||||
|
||||
/// A blob something holds, and whether the store has its commit marker.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct LinkedBlob {
|
||||
pub hash: BlobHash,
|
||||
pub committed: bool,
|
||||
pub holders: Vec<BlobHolder>,
|
||||
/// inbuxa: blob-store-change spec, BM-5: a link that hasn't expired
|
||||
/// and names nothing: an upload, a cached app bundle, a journal entry.
|
||||
pub temporary: bool,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
/// Every blob in one slice of the blob index (the first byte of its
|
||||
/// hash) that something holds a link to. Uploads nothing holds yet and
|
||||
/// expired links are left out: they're the purge's to clear.
|
||||
pub async fn linked_blobs(&self, slice: u8) -> trc::Result<Vec<LinkedBlob>> {
|
||||
let mut blobs = self.indexed_blobs(slice).await?;
|
||||
blobs.retain(|b| !b.holders.is_empty());
|
||||
Ok(blobs)
|
||||
}
|
||||
|
||||
/// inbuxa: blob-store-change spec, BM-5: every blob in one slice of the
|
||||
/// blob index that something holds, a plain link that hasn't expired
|
||||
/// included. Expired links are left out.
|
||||
pub async fn blobs_to_copy(&self, slice: u8) -> trc::Result<Vec<LinkedBlob>> {
|
||||
let mut blobs = self.indexed_blobs(slice).await?;
|
||||
blobs.retain(|b| !b.holders.is_empty() || b.temporary);
|
||||
Ok(blobs)
|
||||
}
|
||||
|
||||
async fn indexed_blobs(&self, slice: u8) -> trc::Result<Vec<LinkedBlob>> {
|
||||
const TEMP_LINK: usize = BLOB_HASH_LEN + U32_LEN + U64_LEN;
|
||||
const DOC_LINK: usize = BLOB_HASH_LEN + U32_LEN + 1 + U32_LEN;
|
||||
const ID_LINK: usize = BLOB_HASH_LEN + U64_LEN;
|
||||
|
||||
let mut from_hash = BlobHash::default();
|
||||
let mut to_hash = BlobHash::new_max();
|
||||
from_hash.0[0] = slice;
|
||||
to_hash.0[0] = slice;
|
||||
let from_key = ValueKey {
|
||||
account_id: 0,
|
||||
collection: 0,
|
||||
document_id: 0,
|
||||
class: ValueClass::Blob(BlobOp::Commit { hash: from_hash }),
|
||||
};
|
||||
let to_key = ValueKey {
|
||||
account_id: u32::MAX,
|
||||
collection: u8::MAX,
|
||||
document_id: u32::MAX,
|
||||
class: ValueClass::Blob(BlobOp::Link {
|
||||
hash: to_hash,
|
||||
to: BlobLink::Document,
|
||||
}),
|
||||
};
|
||||
|
||||
let now = now();
|
||||
let mut blobs: Vec<LinkedBlob> = Vec::new();
|
||||
self.iterate(
|
||||
IterateParams::new(from_key, to_key).ascending(),
|
||||
|key, value| {
|
||||
let hash = key
|
||||
.get(0..BLOB_HASH_LEN)
|
||||
.and_then(|h| BlobHash::try_from_hash_slice(h).ok())
|
||||
.ok_or_else(|| {
|
||||
trc::Error::corrupted_key(key, value.into(), trc::location!())
|
||||
})?;
|
||||
if blobs.last().is_none_or(|b| b.hash != hash) {
|
||||
blobs.push(LinkedBlob {
|
||||
hash,
|
||||
committed: false,
|
||||
holders: Vec::new(),
|
||||
temporary: false,
|
||||
});
|
||||
}
|
||||
let blob = blobs.last_mut().unwrap();
|
||||
match key.len() {
|
||||
BLOB_HASH_LEN => blob.committed = true,
|
||||
DOC_LINK => blob.holders.push(BlobHolder::Document {
|
||||
account_id: key.deserialize_be_u32(BLOB_HASH_LEN)?,
|
||||
collection: key[BLOB_HASH_LEN + U32_LEN],
|
||||
document_id: key.deserialize_be_u32(BLOB_HASH_LEN + U32_LEN + 1)?,
|
||||
}),
|
||||
ID_LINK => blob.holders.push(BlobHolder::Queue {
|
||||
id: key.deserialize_be_u64(BLOB_HASH_LEN)?,
|
||||
}),
|
||||
TEMP_LINK => {
|
||||
// Only a link that names a kept item: a plain one is
|
||||
// an upload nothing holds yet
|
||||
let until = key.deserialize_be_u64(BLOB_HASH_LEN + U32_LEN)?;
|
||||
if until > now && value.len() == U16_LEN + U64_LEN {
|
||||
blob.holders.push(BlobHolder::Archived {
|
||||
account_id: key.deserialize_be_u32(BLOB_HASH_LEN)?,
|
||||
item: ObjectId::deserialize(value)?,
|
||||
});
|
||||
} else if until > now {
|
||||
blob.temporary = true;
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
return Err(trc::Error::corrupted_key(
|
||||
key,
|
||||
value.into(),
|
||||
trc::location!(),
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
Ok(blobs)
|
||||
}
|
||||
}
|
||||
|
||||
impl BlobStore {
|
||||
/// Whether the blob store has anything under `key`, reading as little
|
||||
/// of it as each backend allows: a file's metadata, one byte of an
|
||||
/// object, or, in a data store, the blob itself.
|
||||
pub async fn blob_present(&self, key: &[u8]) -> trc::Result<bool> {
|
||||
match self {
|
||||
BlobStore::Fs(store) => store.get_blob(key, 0..1).await.map(|b| b.is_some()),
|
||||
#[cfg(feature = "s3")]
|
||||
BlobStore::S3(store) => match store.get_blob(key, 0..1).await {
|
||||
Ok(found) => Ok(found.is_some()),
|
||||
// An empty object has no first byte to read
|
||||
Err(_) => self.raw_get(key).await.map(|b| b.is_some()),
|
||||
},
|
||||
#[cfg(feature = "azure")]
|
||||
BlobStore::Azure(store) => match store.get_blob(key, 0..1).await {
|
||||
Ok(found) => Ok(found.is_some()),
|
||||
Err(_) => self.raw_get(key).await.map(|b| b.is_some()),
|
||||
},
|
||||
BlobStore::Sharded(store) => {
|
||||
// Anywhere in the set: a misplaced blob is still there
|
||||
for member in &store.members {
|
||||
if Box::pin(member.blob_present(key)).await? {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
BlobStore::Store(_) => self.raw_get(key).await.map(|b| b.is_some()),
|
||||
// inbuxa: blob-store-change spec, BM-3
|
||||
BlobStore::Moving(store) => store.blob_present(key).await,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! inbuxa: what the blob-store-change spec
|
||||
//! (docs/spec/features/blob-store-change.md) needs from the store: a blob
|
||||
//! store that, while a change of blob store is unresolved, looks for what
|
||||
//! it doesn't have in the stores the server used before, and keeps what it
|
||||
//! finds there (BM-3). Writes and deletes go to the current store only.
|
||||
|
||||
use crate::BlobStore;
|
||||
use arc_swap::ArcSwap;
|
||||
use std::sync::Arc;
|
||||
use trc::StoreEvent;
|
||||
|
||||
/// A blob store the server used before an unresolved change.
|
||||
#[derive(Clone)]
|
||||
pub struct OldBlobStore {
|
||||
/// The `inbuxa:BlobStoreChange` that moved away from it.
|
||||
pub change_id: u32,
|
||||
pub store: BlobStore,
|
||||
}
|
||||
|
||||
/// The current blob store, and the old ones read through while their
|
||||
/// changes are unresolved, newest first (BM-3).
|
||||
pub struct MovingBlobStore {
|
||||
pub current: BlobStore,
|
||||
old: ArcSwap<Vec<OldBlobStore>>,
|
||||
}
|
||||
|
||||
impl MovingBlobStore {
|
||||
pub fn new(current: BlobStore, old: Vec<OldBlobStore>) -> Self {
|
||||
MovingBlobStore {
|
||||
current,
|
||||
old: ArcSwap::from_pointee(old),
|
||||
}
|
||||
}
|
||||
|
||||
/// The old stores still read through.
|
||||
pub fn old_stores(&self) -> Arc<Vec<OldBlobStore>> {
|
||||
self.old.load_full()
|
||||
}
|
||||
|
||||
/// BM-10: keeps only the old stores of changes still unresolved.
|
||||
pub fn retain_changes(&self, unresolved: &[u32]) {
|
||||
let old = self.old.load();
|
||||
if old.iter().all(|o| unresolved.contains(&o.change_id)) {
|
||||
return;
|
||||
}
|
||||
let kept = old
|
||||
.iter()
|
||||
.filter(|o| unresolved.contains(&o.change_id))
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
self.old.store(Arc::new(kept));
|
||||
}
|
||||
|
||||
/// BM-3: the blob as the current store has it or, failing that, as the
|
||||
/// newest old store that has it does. A blob found in an old store is
|
||||
/// written to the current one under the same key, as it was stored.
|
||||
pub(crate) async fn raw_get(&self, key: &[u8]) -> trc::Result<Option<Vec<u8>>> {
|
||||
if let Some(data) = Box::pin(self.current.raw_get(key)).await? {
|
||||
return Ok(Some(data));
|
||||
}
|
||||
let old = self.old.load_full();
|
||||
for old in old.iter() {
|
||||
match Box::pin(old.store.raw_get(key)).await {
|
||||
Ok(Some(data)) => {
|
||||
if let Err(err) = Box::pin(self.current.raw_put(key, &data)).await {
|
||||
trc::error!(
|
||||
err.details("Failed to keep a blob read through an old blob store")
|
||||
.ctx(trc::Key::Id, old.change_id)
|
||||
.ctx(trc::Key::Key, key)
|
||||
);
|
||||
}
|
||||
trc::event!(
|
||||
Store(StoreEvent::BlobReadThrough),
|
||||
Key = key,
|
||||
Id = old.change_id,
|
||||
Size = data.len(),
|
||||
);
|
||||
return Ok(Some(data));
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(err) => {
|
||||
// An old store that fails is skipped, not fatal: the
|
||||
// current store answered, and it had nothing
|
||||
trc::error!(
|
||||
err.details("An old blob store failed to read")
|
||||
.ctx(trc::Key::Id, old.change_id)
|
||||
.ctx(trc::Key::Key, key)
|
||||
.caused_by(trc::location!())
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Whether a read would find the blob: in the current store or in an
|
||||
/// old one still read through.
|
||||
pub(crate) async fn blob_present(&self, key: &[u8]) -> trc::Result<bool> {
|
||||
if Box::pin(self.current.blob_present(key)).await? {
|
||||
return Ok(true);
|
||||
}
|
||||
for old in self.old.load_full().iter() {
|
||||
if Box::pin(old.store.blob_present(key)).await.unwrap_or(false) {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
impl BlobStore {
|
||||
/// The store blobs are written to: past any old stores read through.
|
||||
pub fn current(&self) -> &BlobStore {
|
||||
match self {
|
||||
BlobStore::Moving(moving) => moving.current.current(),
|
||||
store => store,
|
||||
}
|
||||
}
|
||||
|
||||
/// The read-through wrapper, when there is one (BM-3).
|
||||
pub fn moving(&self) -> Option<&Arc<MovingBlobStore>> {
|
||||
match self {
|
||||
BlobStore::Moving(moving) => Some(moving),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// A blob exactly as the backend keeps it, compression marker and all,
|
||||
/// without reading through (BM-5: a copy reads the old store itself).
|
||||
pub async fn get_stored(&self, key: &[u8]) -> trc::Result<Option<Vec<u8>>> {
|
||||
self.current().raw_get(key).await
|
||||
}
|
||||
|
||||
/// Writes a blob exactly as [`BlobStore::get_stored`] gave it (BM-5).
|
||||
pub async fn put_stored(&self, key: &[u8], data: &[u8]) -> trc::Result<()> {
|
||||
self.current().raw_put(key, data).await
|
||||
}
|
||||
|
||||
/// Whether the store itself has the blob, not reading through (BM-5).
|
||||
pub async fn stored_present(&self, key: &[u8]) -> trc::Result<bool> {
|
||||
self.current().blob_present(key).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{CompressionAlgo, backend::fs::FsStore};
|
||||
use registry::schema::structs;
|
||||
|
||||
async fn fs_store(dir: &std::path::Path) -> BlobStore {
|
||||
FsStore::open(structs::FileSystemStore {
|
||||
path: dir.to_string_lossy().into_owned(),
|
||||
depth: 2,
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reads_through_and_keeps_what_it_found() {
|
||||
let root = std::env::temp_dir().join(format!("inbuxa-blob-move-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
let old = fs_store(&root.join("old")).await;
|
||||
let new = fs_store(&root.join("new")).await;
|
||||
old.put_blob(b"left-behind", b"hello", CompressionAlgo::Lz4)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let moving = BlobStore::Moving(Arc::new(MovingBlobStore::new(
|
||||
new.clone(),
|
||||
vec![OldBlobStore {
|
||||
change_id: 7,
|
||||
store: old.clone(),
|
||||
}],
|
||||
)));
|
||||
|
||||
// Only the old store has it: read through, then kept in the new one
|
||||
assert!(!new.blob_present(b"left-behind").await.unwrap());
|
||||
assert!(moving.blob_present(b"left-behind").await.unwrap());
|
||||
assert!(!moving.stored_present(b"left-behind").await.unwrap());
|
||||
assert_eq!(
|
||||
moving
|
||||
.get_blob(b"left-behind", 0..usize::MAX)
|
||||
.await
|
||||
.unwrap()
|
||||
.as_deref(),
|
||||
Some(&b"hello"[..])
|
||||
);
|
||||
assert_eq!(
|
||||
new.get_blob(b"left-behind", 0..usize::MAX)
|
||||
.await
|
||||
.unwrap()
|
||||
.as_deref(),
|
||||
Some(&b"hello"[..])
|
||||
);
|
||||
|
||||
// In neither: absent
|
||||
assert_eq!(
|
||||
moving.get_blob(b"nowhere", 0..usize::MAX).await.unwrap(),
|
||||
None
|
||||
);
|
||||
|
||||
// Writes and deletes touch only the new store
|
||||
moving
|
||||
.put_blob(b"fresh", b"new mail", CompressionAlgo::None)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(new.blob_present(b"fresh").await.unwrap());
|
||||
assert!(!old.blob_present(b"fresh").await.unwrap());
|
||||
moving.delete_blob(b"left-behind").await.unwrap();
|
||||
assert!(!new.blob_present(b"left-behind").await.unwrap());
|
||||
assert!(old.blob_present(b"left-behind").await.unwrap());
|
||||
|
||||
// Once the change is resolved, nothing more is read through
|
||||
moving.moving().unwrap().retain_changes(&[]);
|
||||
assert!(moving.moving().unwrap().old_stores().is_empty());
|
||||
assert_eq!(
|
||||
moving
|
||||
.get_blob(b"left-behind", 0..usize::MAX)
|
||||
.await
|
||||
.unwrap(),
|
||||
None
|
||||
);
|
||||
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use self::assert::AssertValue;
|
||||
@@ -27,9 +29,12 @@ pub mod assert;
|
||||
pub mod batch;
|
||||
pub mod bitpack;
|
||||
pub mod blob;
|
||||
pub mod blob_check; // inbuxa: store-check spec
|
||||
pub mod blob_move; // inbuxa: blob-store-change spec
|
||||
pub mod key;
|
||||
pub mod log;
|
||||
pub mod serialize;
|
||||
pub mod snapshot; // inbuxa: store-snapshot spec
|
||||
|
||||
pub(crate) const ARCHIVE_ALIGNMENT: usize = 16;
|
||||
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! inbuxa: what the store-snapshot spec (docs/spec/features/store-snapshot.md)
|
||||
//! needs from the store: holding the blob purge (SN-4), a checkpoint of the
|
||||
//! data store and a copy of a filesystem blob store's files (SN-2).
|
||||
|
||||
use crate::{BlobStore, Store};
|
||||
use std::{
|
||||
io,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
use tokio::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};
|
||||
|
||||
/// SN-4. A purge runs under the read side; a snapshot taking blobs holds
|
||||
/// the write side. The data store is RocksDB, so one process has it open,
|
||||
/// and a lock in that process is enough.
|
||||
static BLOB_PURGE_GATE: RwLock<()> = RwLock::const_new(());
|
||||
|
||||
/// Whether a blob purge may run now: `None` while a snapshot takes blobs or
|
||||
/// waits to. Keep the guard for as long as the purge runs.
|
||||
pub fn blob_purge_may_run() -> Option<RwLockReadGuard<'static, ()>> {
|
||||
BLOB_PURGE_GATE.try_read().ok()
|
||||
}
|
||||
|
||||
/// Holds the blob purge until the guard is dropped. A purge already running
|
||||
/// finishes first.
|
||||
pub async fn hold_blob_purge() -> RwLockWriteGuard<'static, ()> {
|
||||
BLOB_PURGE_GATE.write().await
|
||||
}
|
||||
|
||||
/// What a snapshot can hold of this server's stores (SN-1, SN-2).
|
||||
pub enum SnapshotSource {
|
||||
/// Blobs are in the data store, so the checkpoint holds them.
|
||||
DataOnly { data_path: PathBuf },
|
||||
/// Blobs are files under `blob_path`, copied after the checkpoint.
|
||||
DataAndFiles {
|
||||
data_path: PathBuf,
|
||||
blob_path: PathBuf,
|
||||
},
|
||||
}
|
||||
|
||||
/// Why a snapshot can't be taken of these stores.
|
||||
pub enum NotSnapshottable {
|
||||
DataStore(&'static str),
|
||||
BlobStore(&'static str),
|
||||
/// inbuxa: blob-store-change spec, BM-8: some blobs are still only in
|
||||
/// the blob store the server used before.
|
||||
BlobStoreChanging,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
/// SN-2: a checkpoint of the data store into `dest`, which must not
|
||||
/// exist yet. Only RocksDB has one.
|
||||
#[cfg_attr(not(feature = "rocks"), allow(unused_variables))]
|
||||
pub async fn checkpoint(&self, dest: PathBuf) -> trc::Result<()> {
|
||||
match self {
|
||||
#[cfg(feature = "rocks")]
|
||||
Store::RocksDb(store) => store.checkpoint(dest).await,
|
||||
_ => Err(trc::StoreEvent::NotSupported
|
||||
.into_err()
|
||||
.details("Only a RocksDB store has a checkpoint")),
|
||||
}
|
||||
}
|
||||
|
||||
/// What a snapshot of this data store and `blobs` would hold, or why
|
||||
/// there can't be one.
|
||||
#[cfg_attr(not(feature = "rocks"), allow(unused_variables, unreachable_code))]
|
||||
pub fn snapshot_source(&self, blobs: &BlobStore) -> Result<SnapshotSource, NotSnapshottable> {
|
||||
let data_path = match self {
|
||||
#[cfg(feature = "rocks")]
|
||||
Store::RocksDb(store) => store.path().to_path_buf(),
|
||||
#[cfg(feature = "sqlite")]
|
||||
Store::SQLite(_) => return Err(NotSnapshottable::DataStore("SQLite")),
|
||||
#[cfg(feature = "foundation")]
|
||||
Store::FoundationDb(_) => return Err(NotSnapshottable::DataStore("FoundationDB")),
|
||||
#[cfg(feature = "postgres")]
|
||||
Store::PostgreSQL(_) => return Err(NotSnapshottable::DataStore("PostgreSQL")),
|
||||
#[cfg(feature = "mysql")]
|
||||
Store::MySQL(_) => return Err(NotSnapshottable::DataStore("MySQL")),
|
||||
Store::Replicated(_) => {
|
||||
return Err(NotSnapshottable::DataStore("a replicated SQL database"));
|
||||
}
|
||||
Store::Ephemeral(_) | Store::None => {
|
||||
return Err(NotSnapshottable::DataStore("an in-memory store"));
|
||||
}
|
||||
};
|
||||
match blobs {
|
||||
BlobStore::Store(store) if store.is_same(self) => {
|
||||
Ok(SnapshotSource::DataOnly { data_path })
|
||||
}
|
||||
BlobStore::Fs(fs) => {
|
||||
let blob_path = fs.root().to_path_buf();
|
||||
// The blob files can't be told from the data store's own
|
||||
// in one directory. A blob directory that holds the data
|
||||
// store's is fine: the copy leaves that one out
|
||||
if data_path == blob_path {
|
||||
return Err(NotSnapshottable::BlobStore(
|
||||
"the data store's own directory",
|
||||
));
|
||||
}
|
||||
Ok(SnapshotSource::DataAndFiles {
|
||||
data_path,
|
||||
blob_path,
|
||||
})
|
||||
}
|
||||
BlobStore::Store(_) => Err(NotSnapshottable::BlobStore("another database")),
|
||||
#[cfg(feature = "s3")]
|
||||
BlobStore::S3(_) => Err(NotSnapshottable::BlobStore("S3")),
|
||||
#[cfg(feature = "azure")]
|
||||
BlobStore::Azure(_) => Err(NotSnapshottable::BlobStore("Azure")),
|
||||
BlobStore::Sharded(_) => Err(NotSnapshottable::BlobStore("a sharded set")),
|
||||
// inbuxa: blob-store-change spec, BM-8: blobs only an old store
|
||||
// has would be left out
|
||||
BlobStore::Moving(moving) if !moving.old_stores().is_empty() => {
|
||||
Err(NotSnapshottable::BlobStoreChanging)
|
||||
}
|
||||
BlobStore::Moving(moving) => self.snapshot_source(&moving.current),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What a copy of a tree did.
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct TreeCopy {
|
||||
pub files: u64,
|
||||
pub bytes: u64,
|
||||
/// Files that couldn't be hard-linked and were copied.
|
||||
pub copied: u64,
|
||||
}
|
||||
|
||||
/// SN-2, SN-3: every file under `from` into `to`, hard-linked where it can
|
||||
/// be and copied where it can't, leaving out `skip` (the data store's
|
||||
/// directory, with the snapshots in it). Names starting with a dot are
|
||||
/// skipped: they're blobs being written (SN-10).
|
||||
pub async fn copy_tree(from: PathBuf, to: PathBuf, skip: PathBuf) -> trc::Result<TreeCopy> {
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let mut done = TreeCopy::default();
|
||||
copy_dir(&from, &to, &skip, &mut done).map(|_| done)
|
||||
})
|
||||
.await
|
||||
.map_err(|err| trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err))?
|
||||
.map_err(|err| trc::StoreEvent::FilesystemError.reason(err))
|
||||
}
|
||||
|
||||
fn copy_dir(from: &Path, to: &Path, skip: &Path, done: &mut TreeCopy) -> io::Result<()> {
|
||||
std::fs::create_dir_all(to)?;
|
||||
for entry in std::fs::read_dir(from)? {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
if name.as_encoded_bytes().first() == Some(&b'.') {
|
||||
continue;
|
||||
}
|
||||
let (source, dest) = (entry.path(), to.join(&name));
|
||||
if source == skip {
|
||||
continue;
|
||||
}
|
||||
let kind = entry.file_type()?;
|
||||
if kind.is_dir() {
|
||||
copy_dir(&source, &dest, skip, done)?;
|
||||
} else if kind.is_file() {
|
||||
match std::fs::hard_link(&source, &dest) {
|
||||
Ok(()) => {}
|
||||
// Gone since it was listed: a purge ran before the hold, or
|
||||
// it's an orphan nothing links to
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => continue,
|
||||
Err(_) => {
|
||||
match std::fs::copy(&source, &dest) {
|
||||
Ok(_) => {}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => continue,
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
done.copied += 1;
|
||||
}
|
||||
}
|
||||
done.files += 1;
|
||||
done.bytes += std::fs::metadata(&dest)?.len();
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Bytes in the files under `dir`.
|
||||
pub async fn tree_size(dir: PathBuf) -> trc::Result<u64> {
|
||||
fn size(dir: &Path) -> io::Result<u64> {
|
||||
let mut total = 0;
|
||||
for entry in std::fs::read_dir(dir)? {
|
||||
let entry = entry?;
|
||||
let kind = entry.file_type()?;
|
||||
if kind.is_dir() {
|
||||
total += size(&entry.path())?;
|
||||
} else if kind.is_file() {
|
||||
total += entry.metadata()?.len();
|
||||
}
|
||||
}
|
||||
Ok(total)
|
||||
}
|
||||
tokio::task::spawn_blocking(move || size(&dir))
|
||||
.await
|
||||
.map_err(|err| trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err))?
|
||||
.map_err(|err| trc::StoreEvent::FilesystemError.reason(err))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn copies_a_tree_and_skips_what_is_being_written() {
|
||||
let base = std::env::temp_dir().join(format!("inbuxa-sn-tree-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&base);
|
||||
let from = base.join("from");
|
||||
std::fs::create_dir_all(from.join("a/b")).unwrap();
|
||||
std::fs::write(from.join("a/b/BLOB1"), b"one").unwrap();
|
||||
std::fs::write(from.join("a/BLOB2"), b"two!").unwrap();
|
||||
std::fs::write(from.join("a/b/.BLOB3.1-0.tmp"), b"half").unwrap();
|
||||
std::fs::create_dir_all(from.join("rocks.db/snapshots")).unwrap();
|
||||
std::fs::write(from.join("rocks.db/000001.sst"), b"data").unwrap();
|
||||
|
||||
let to = base.join("to");
|
||||
let done = copy_tree(from.clone(), to.clone(), from.join("rocks.db"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!((done.files, done.bytes), (2, 7));
|
||||
assert_eq!(std::fs::read(to.join("a/b/BLOB1")).unwrap(), b"one");
|
||||
assert!(!to.join("a/b/.BLOB3.1-0.tmp").exists());
|
||||
assert!(!to.join("rocks.db").exists());
|
||||
assert_eq!(tree_size(to).await.unwrap(), 7);
|
||||
std::fs::remove_dir_all(&base).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_purge_waits_while_a_snapshot_holds_it() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.build()
|
||||
.unwrap();
|
||||
runtime.block_on(async {
|
||||
assert!(blob_purge_may_run().is_some());
|
||||
let hold = hold_blob_purge().await;
|
||||
assert!(blob_purge_may_run().is_none());
|
||||
drop(hold);
|
||||
assert!(blob_purge_may_run().is_some());
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -12,10 +12,14 @@
|
||||
// auth.legacy-protocol-refused (legacy-protocols LP-6); 643 is
|
||||
// security.legacy-protocols-changed (LP-8); 644 to 646 are the cluster
|
||||
// coordinator's connection events; 647 and 648 are the audit log's
|
||||
// (audit-hold-lock spec, AU-3, AU-8); 649 is cluster.node-id-reassigned.
|
||||
// (audit-hold-lock spec, AU-3, AU-8); 649 is cluster.node-id-reassigned;
|
||||
// 650 to 659 are inbuxa Relay's in the 27.1 series and unused here; 660 is
|
||||
// store.blob-read-through (blob-store-change spec, BM-3). Ids are shared
|
||||
// with 27.1 because stored settings name events by id.
|
||||
// TOTAL_EVENT_COUNT sizes the collector's per-event tables, so it is one
|
||||
// more than the highest id; the test below holds it to the variant list
|
||||
pub const TOTAL_EVENT_COUNT: usize = 650;
|
||||
// less those 10 unused ids
|
||||
pub const TOTAL_EVENT_COUNT: usize = 661;
|
||||
pub const TOTAL_METRIC_COUNT: usize = 369;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
@@ -853,6 +857,9 @@ pub enum StoreEvent {
|
||||
CacheUpdate = 577,
|
||||
CacheEntryTooLarge = 633,
|
||||
BlobMissingMarker = 507,
|
||||
/// inbuxa: blob-store-change spec, BM-3: a blob the blob store lacked
|
||||
/// was found in the one it replaced, and kept.
|
||||
BlobReadThrough = 660,
|
||||
DataWrite = 513,
|
||||
DataIterate = 512,
|
||||
BlobRead = 508,
|
||||
@@ -1397,7 +1404,8 @@ mod tests {
|
||||
#[test]
|
||||
fn event_ids_fit_the_collector_tables() {
|
||||
let variants = EventType::variants();
|
||||
assert_eq!(variants.len(), TOTAL_EVENT_COUNT);
|
||||
// inbuxa: 650 to 659 are the 27.1 series' Relay events
|
||||
assert_eq!(variants.len() + 10, TOTAL_EVENT_COUNT);
|
||||
let mut ids = variants
|
||||
.iter()
|
||||
.map(|event| event.to_id())
|
||||
|
||||
@@ -609,6 +609,7 @@ impl EventType {
|
||||
b"store.cache-update" => EventType::Store(StoreEvent::CacheUpdate),
|
||||
b"store.cache-entry-too-large" => EventType::Store(StoreEvent::CacheEntryTooLarge),
|
||||
b"store.blob-missing-marker" => EventType::Store(StoreEvent::BlobMissingMarker),
|
||||
b"store.blob-read-through" => EventType::Store(StoreEvent::BlobReadThrough),
|
||||
b"store.data-write" => EventType::Store(StoreEvent::DataWrite),
|
||||
b"store.data-iterate" => EventType::Store(StoreEvent::DataIterate),
|
||||
b"store.blob-read" => EventType::Store(StoreEvent::BlobRead),
|
||||
@@ -1394,6 +1395,7 @@ impl EventType {
|
||||
EventType::Store(StoreEvent::CacheUpdate) => "store.cache-update",
|
||||
EventType::Store(StoreEvent::CacheEntryTooLarge) => "store.cache-entry-too-large",
|
||||
EventType::Store(StoreEvent::BlobMissingMarker) => "store.blob-missing-marker",
|
||||
EventType::Store(StoreEvent::BlobReadThrough) => "store.blob-read-through",
|
||||
EventType::Store(StoreEvent::DataWrite) => "store.data-write",
|
||||
EventType::Store(StoreEvent::DataIterate) => "store.data-iterate",
|
||||
EventType::Store(StoreEvent::BlobRead) => "store.blob-read",
|
||||
@@ -2072,6 +2074,7 @@ impl EventType {
|
||||
EventType::Store(StoreEvent::CacheUpdate) => 577,
|
||||
EventType::Store(StoreEvent::CacheEntryTooLarge) => 633,
|
||||
EventType::Store(StoreEvent::BlobMissingMarker) => 507,
|
||||
EventType::Store(StoreEvent::BlobReadThrough) => 660,
|
||||
EventType::Store(StoreEvent::DataWrite) => 513,
|
||||
EventType::Store(StoreEvent::DataIterate) => 512,
|
||||
EventType::Store(StoreEvent::BlobRead) => 508,
|
||||
@@ -2770,6 +2773,7 @@ impl EventType {
|
||||
577 => Some(EventType::Store(StoreEvent::CacheUpdate)),
|
||||
633 => Some(EventType::Store(StoreEvent::CacheEntryTooLarge)),
|
||||
507 => Some(EventType::Store(StoreEvent::BlobMissingMarker)),
|
||||
660 => Some(EventType::Store(StoreEvent::BlobReadThrough)),
|
||||
513 => Some(EventType::Store(StoreEvent::DataWrite)),
|
||||
512 => Some(EventType::Store(StoreEvent::DataIterate)),
|
||||
508 => Some(EventType::Store(StoreEvent::BlobRead)),
|
||||
@@ -3194,6 +3198,7 @@ impl EventType {
|
||||
EventType::Store(StoreEvent::HttpStoreError) => Level::Warn,
|
||||
EventType::Store(StoreEvent::CacheEntryTooLarge) => Level::Warn,
|
||||
EventType::Store(StoreEvent::BlobMissingMarker) => Level::Warn,
|
||||
EventType::Store(StoreEvent::BlobReadThrough) => Level::Info,
|
||||
EventType::TaskManager(TaskManagerEvent::TaskFailed) => Level::Warn,
|
||||
EventType::Telemetry(TelemetryEvent::AlertEvent) => Level::Warn,
|
||||
EventType::Telemetry(TelemetryEvent::LogError) => Level::Warn,
|
||||
@@ -3941,6 +3946,7 @@ impl EventType {
|
||||
EventType::Store(StoreEvent::CacheUpdate) => "Cache update",
|
||||
EventType::Store(StoreEvent::CacheEntryTooLarge) => "Cache entry too large",
|
||||
EventType::Store(StoreEvent::BlobMissingMarker) => "Blob missing marker",
|
||||
EventType::Store(StoreEvent::BlobReadThrough) => "Blob read through",
|
||||
EventType::Store(StoreEvent::DataWrite) => "Write batch operation",
|
||||
EventType::Store(StoreEvent::DataIterate) => "Data store iteration operation",
|
||||
EventType::Store(StoreEvent::BlobRead) => "Blob read operation",
|
||||
@@ -4302,6 +4308,9 @@ impl EventType {
|
||||
EventType::Store(StoreEvent::CacheUpdate) => "Store error",
|
||||
EventType::Store(StoreEvent::CacheEntryTooLarge) => "Cache entry too large to store",
|
||||
EventType::Store(StoreEvent::BlobMissingMarker) => "Blob is missing marker",
|
||||
EventType::Store(StoreEvent::BlobReadThrough) => {
|
||||
"A blob was read from the blob store used before a change, and kept"
|
||||
}
|
||||
EventType::Store(StoreEvent::DataWrite) => "Store error",
|
||||
EventType::Store(StoreEvent::DataIterate) => "Store error",
|
||||
EventType::Store(StoreEvent::BlobRead) => "Store error",
|
||||
@@ -4910,6 +4919,7 @@ impl EventType {
|
||||
EventType::Store(StoreEvent::CacheUpdate),
|
||||
EventType::Store(StoreEvent::CacheEntryTooLarge),
|
||||
EventType::Store(StoreEvent::BlobMissingMarker),
|
||||
EventType::Store(StoreEvent::BlobReadThrough),
|
||||
EventType::Store(StoreEvent::DataWrite),
|
||||
EventType::Store(StoreEvent::DataIterate),
|
||||
EventType::Store(StoreEvent::BlobRead),
|
||||
|
||||
@@ -379,6 +379,12 @@ Not a rebuild: the **deliverability check** is INBUXA's own design (inbuxa-draft
|
||||
|
||||
Not a rebuild: **scheduled reports and the weekly digest** are INBUXA's own design (inbuxa-drafts `specs/scheduled-reports.md`). An administrator picks sections, a schedule in a time zone and recipients on this server; every node looks for due reports once a minute, one claims each run with the task lock, and the report is built from data the server already keeps and mailed DKIM-signed: `inbuxa:ScheduledReport`, `inbuxa:ScheduledReportSettings` and `inbuxa:ReportExport` (a download: a ZIP of a summary and CSVs) (`crates/jmap/src/inbuxa/scheduled_reports.rs`, `crates/features/src/scheduled_reports/`, `crates/services/src/inbuxa_scheduled_reports.rs`, `crates/smtp/src/reporting/inbuxa_send.rs`), and the `sysScheduledReportGet` and `sysScheduledReportUpdate` permissions. The weekly digest is a built-in report, on by default.
|
||||
|
||||
Not a rebuild: the **store check** is INBUXA's own design (`features/store-check.md`, inbuxa-server #322, building). An administrator starts a check of the store's blobs, quick (each one is there) or full (each one reads back intact), and gets a list of what's missing, damaged or no longer linked. One node runs it in the background, throttled, and keeps its progress: `inbuxa:StoreCheck` (`crates/jmap/src/inbuxa/store_check.rs`, `crates/features/src/store_check/`, `crates/services/src/inbuxa_store_check.rs`, `crates/store/src/write/blob_check.rs`), and the `sysStoreCheck` permission.
|
||||
|
||||
Not a rebuild: the **store snapshot** is INBUXA's own design (`features/store-snapshot.md`, inbuxa-server #323, building). An administrator has the server take a consistent copy of its RocksDB store, and of a filesystem blob store's files, into `snapshots/` in the store's directory while mail flows, for a backup tool to copy; the latest two are kept: `inbuxa:StoreSnapshot` (`crates/jmap/src/inbuxa/store_snapshot.rs`, `crates/services/src/inbuxa_store_snapshot.rs`, `crates/store/src/write/snapshot.rs`, `crates/store/src/backend/rocksdb/checkpoint.rs`), and the `sysStoreSnapshot` permission.
|
||||
|
||||
Not a rebuild: **changing the blob store** is INBUXA's own design (`features/blob-store-change.md`, inbuxa-server #333, building). When a node starts with the blob store setting pointing somewhere else, it records the change and keeps the old store's settings; until the change is resolved, a blob the new store lacks is read from the old one and kept, and an administrator has one node copy the rest over, checked by hash, with what couldn't be copied reported: `inbuxa:BlobStoreChange` (`crates/jmap/src/inbuxa/blob_store_change.rs`, `crates/features/src/blob_move/`, `crates/services/src/inbuxa_blob_move.rs`, `crates/common/src/manager/blob_moves.rs`, `crates/store/src/write/blob_move.rs`), the `store.blob-read-through` event, and the `sysBlobStoreChange` permission.
|
||||
|
||||
## 5. The web front ends
|
||||
|
||||
**Which ihasmail.** Public ihasmail stays Stalwart-facing: its code, docs,
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
# Feature spec: changing the blob store
|
||||
|
||||
Status: building, 2026-11-09 (inbuxa-server #333). Not a rebuild (SPEC.md
|
||||
§4): inbuxa's own design.
|
||||
|
||||
## Provenance
|
||||
|
||||
| Source | Used for |
|
||||
|---|---|
|
||||
| Forum topic 238 | The need: a server moved its blobs to a filesystem store, took one message by SMTP, and went back to the data store a minute later. The message's metadata stayed; its blob was left in the filesystem store, and reading it failed from then on |
|
||||
| This server's blob store build and dispatch (`crates/store/src/build/blob.rs`, `crates/store/src/dispatch/blob.rs`) | Where the blob store comes from, and that a change takes effect at start-up |
|
||||
| The store check (`features/store-check.md`) and its blob index walk (`crates/store/src/write/blob_check.rs`) | Which blobs the data holds, how a blob proves its content, and how the object, permission and run are built |
|
||||
|
||||
Upstream's open code has nothing like it, and nothing here comes from
|
||||
Enterprise code.
|
||||
|
||||
## What it is
|
||||
|
||||
The blob store setting (`x:BlobStore`) says where blobs go: in the data
|
||||
store, in a directory, in a bucket. Changing it changes where the server
|
||||
writes and reads them from the next start-up. It moves nothing. Before
|
||||
this spec, every blob written under the old setting was left behind,
|
||||
in either direction, and a message whose blob was left behind failed to
|
||||
read with nothing to say why.
|
||||
|
||||
Now the server notices the change when it starts. It keeps reading what
|
||||
it can't find from the old store and copies each blob it reads that way
|
||||
into the new one. An administrator has it copy everything else, and the
|
||||
change is resolved once the new store has every blob. Nothing is ever
|
||||
deleted from the old store; that stays the administrator's to do.
|
||||
|
||||
## Requirements
|
||||
|
||||
- **BM-1.** Where a blob store is, without its credentials or tuning:
|
||||
- the data store (`Default`);
|
||||
- a filesystem store: its path and depth (the depth decides each blob's
|
||||
path, so a new depth is a new place);
|
||||
- S3: the region or endpoint, the bucket and the key prefix;
|
||||
- Azure: the storage account, the container and the key prefix;
|
||||
- PostgreSQL or MySQL: the host, port and database;
|
||||
- FoundationDB: the cluster file;
|
||||
- a sharded set is one place. The set moves blobs between its members
|
||||
itself (scale-out spec, ST-16 to ST-22).
|
||||
- **BM-2.** At start-up, a node compares where its blob store is (BM-1)
|
||||
with where the store last said it was. When they differ, it records an
|
||||
`inbuxa:BlobStoreChange` with both places, the time and the node, and
|
||||
keeps the old store's settings in it so the old store can still be
|
||||
read. It logs a warning that names both places. The first node to start
|
||||
with the new setting records the change; the others find it recorded.
|
||||
The first start of a server, or of a store from before this spec, only
|
||||
records where the blob store is.
|
||||
- **BM-3.** **Reading through.** While a change is unresolved (BM-5), a
|
||||
blob the blob store doesn't have is looked for in the change's old
|
||||
store. When it's there, it's returned and written to the current store
|
||||
under the same key, with the event `store.blob-read-through` naming the
|
||||
key and the change. Several unresolved changes are tried newest first.
|
||||
Writes and deletes go to the current store only. An old store that
|
||||
can't be opened at start-up is left out, with a warning, and the change
|
||||
stays unresolved.
|
||||
- **BM-4.** **Back where it was.** A change back to a store that an
|
||||
earlier unresolved change moved away from (A to B, then back to A)
|
||||
resolves that earlier change by itself: its old store is the current
|
||||
one again. Its status is `reverted`. The new change (B to A) is the one
|
||||
to copy from.
|
||||
- **BM-5.** **Copying.** An update with `action: "copy"` starts a copy on
|
||||
the node that takes it. One copy runs at a time across the server; a
|
||||
second is refused while one runs. It goes through the blob index in its
|
||||
256 slices and takes every committed blob that something holds: a
|
||||
document, a queued message, a kept deleted item, or a link that hasn't
|
||||
expired. It also takes the spam classifier's two files, which the
|
||||
server keeps under fixed names. For each blob:
|
||||
- already in the current store: counted as `alreadyThere`;
|
||||
- in neither store: a `missing` problem;
|
||||
- a blob a document, a queued message or a kept deleted item holds is
|
||||
named by the hash of its content; when what the old store has doesn't
|
||||
hash to its name, it isn't copied and is a `damaged` problem. Other
|
||||
blobs (uploads, cached app bundles, the spam files) aren't named by
|
||||
their content and are copied as they are;
|
||||
- an error from either store: an `unreadable` problem, with the error;
|
||||
- otherwise it's written to the current store as the old store has it,
|
||||
compression and all: `copied`.
|
||||
|
||||
It's throttled like a store check (store-check spec, SV-6): by default
|
||||
1,000 blobs and 20 MB a second, set with `blobsPerSecond` and
|
||||
`bytesPerSecond` alongside the action; 0 is no limit.
|
||||
- **BM-6.** A copy ends:
|
||||
- `copied`: nothing was missing, damaged or unreadable. The change is
|
||||
resolved: every node stops reading through it within a minute
|
||||
(BM-10);
|
||||
- `incomplete`: some blobs couldn't be copied. The first 1,000 problems
|
||||
are kept, and every one is logged as `store.data-corruption`. The
|
||||
change stays unresolved and can be copied again;
|
||||
- `stopped`, by an update with `action: "stop"`; `failed`, with the
|
||||
error; `interrupted`, when its node restarted or went quiet for five
|
||||
minutes. Each leaves the change unresolved.
|
||||
- **BM-7.** **Dismissing.** Destroying a change forgets it and the old
|
||||
store's settings it kept, and reading through it stops. Nothing is
|
||||
deleted from either store. A change whose copy runs can't be destroyed;
|
||||
stop the copy first.
|
||||
- **BM-8.** While a change is unresolved, a store snapshot is refused
|
||||
(store-snapshot spec): it would leave out the blobs only the old store
|
||||
has. The refusal says to copy or dismiss the change first.
|
||||
- **BM-9.** Only the server's administrators see and act on changes:
|
||||
`sysBlobStoreChange`, in the built-in Administrator role and granted
|
||||
once to existing administrator roles. A tenant administrator is
|
||||
refused. Copying, stopping and dismissing are audited; reading isn't.
|
||||
`get` never returns the old store's settings, only where it is (BM-1).
|
||||
- **BM-10.** On a cluster, every node reads the unresolved changes at
|
||||
start-up and again every minute, so one that's resolved or dismissed
|
||||
on any node stops being read through everywhere. During a rolling
|
||||
restart the nodes still on the old setting write to the old store;
|
||||
reading through covers that until the copy, which should start once
|
||||
every node runs the new setting.
|
||||
|
||||
## The object
|
||||
|
||||
| Property | Type | |
|
||||
|---|---|---|
|
||||
| `id` | Id | |
|
||||
| `from`, `to` | String | Where the old and the new blob store are (BM-1) |
|
||||
| `changedAt` | UTCDate | When the first node started with the new setting |
|
||||
| `node` | String | That node |
|
||||
| `status` | String | `pending`, `copying`, `copied`, `incomplete`, `stopped`, `failed`, `interrupted`, `reverted` |
|
||||
| `copyStartedAt`, `copyFinishedAt` | UTCDate | The latest copy |
|
||||
| `copyStartedBy` | String | Who started it |
|
||||
| `copyNode` | String | The node it runs on |
|
||||
| `blobsPerSecond`, `bytesPerSecond` | UnsignedInt | Its limits |
|
||||
| `progress` | Number | 0 to 1 |
|
||||
| `blobsChecked`, `copied`, `alreadyThere`, `bytesCopied` | UnsignedInt | |
|
||||
| `missing`, `damaged`, `unreadable` | UnsignedInt | Counts of each problem |
|
||||
| `problems` | Object[] | The first 1,000 of the latest copy |
|
||||
| `error` | String | Why a `failed` copy failed |
|
||||
| `action` | String | Write-only: `copy` or `stop` |
|
||||
|
||||
A problem is `{kind, blobId, holder, accountId, objectId, detail}`:
|
||||
`holder` is `email`, `file`, `queuedMessage`, `archivedItem`, `temporary`
|
||||
or `fixed`, for the first thing found holding the blob.
|
||||
|
||||
## Tests
|
||||
|
||||
- Store level (`crates/store/src/write/blob_move.rs`): with two filesystem
|
||||
stores, a blob only the old one has reads through and is then in the
|
||||
new one; one in neither reads as absent; writes and deletes touch only
|
||||
the new one; dropping the old store stops reading through.
|
||||
- Where a store is (BM-1): credentials and tuning don't change it; a path,
|
||||
depth, bucket or prefix does.
|
||||
- Model (`crates/features/src/blob_move/`): recording a change, a change
|
||||
back reverting the earlier one, and the copy's bookkeeping.
|
||||
- `system::blob_store_change` (`tests/src/system/blob_store_change.rs`),
|
||||
against a real server: a message delivered while a filesystem store is
|
||||
set, the setting changed back and the server rebuilt; the message reads
|
||||
through, the change is listed, a copy resolves it, and a damaged blob
|
||||
in the old store is reported, not copied. Over JMAP: the refusals, stop,
|
||||
the destroy rules, a tenant administrator's refusal.
|
||||
|
||||
## Not yet
|
||||
|
||||
- The console's page, and a warning on the blob store settings form
|
||||
that existing blobs stay where they are.
|
||||
- An `inbuxa-cli` command.
|
||||
- Copying ahead of a change (the old store to a new one before the
|
||||
setting changes). Reading through makes it unnecessary for the
|
||||
server's own reads, but a long copy after the change means more
|
||||
reads go to the old store meanwhile.
|
||||
@@ -0,0 +1,133 @@
|
||||
# Feature spec: checking the store's blobs
|
||||
|
||||
Status: building, 2026-11-08 (inbuxa-server #322). Not a rebuild (SPEC.md
|
||||
§4): inbuxa's own design.
|
||||
|
||||
## Provenance
|
||||
|
||||
| Source | Used for |
|
||||
|---|---|
|
||||
| Forum topic 238 | The need: a mail whose metadata was there and whose blob wasn't, on a server that looked healthy, and a backup that copied it |
|
||||
| This server's blob links (`crates/store/src/write/blob.rs`) and `BlobHash` (`crates/types/src/blob_hash.rs`) | What a document holds on to, and how a blob proves its content |
|
||||
| This server's `inbuxa:HoldExport` and `inbuxa:Migration` | How a fork object, a background job and their permission are built |
|
||||
|
||||
Upstream's open code has nothing like it, and nothing here comes from
|
||||
Enterprise code.
|
||||
|
||||
## What it is
|
||||
|
||||
An administrator asks the server to check that the blobs its data needs are
|
||||
in the blob store, and, if asked, that each one is intact. The check
|
||||
reports what it finds and changes nothing. It runs inside the server, on
|
||||
the node the administrator reached, against the stores every node shares.
|
||||
So it works while mail flows, on one machine or a cluster, whatever the
|
||||
stores are. An offline command couldn't: RocksDB lets one process open the
|
||||
store, and a cluster's data store and blob store may be on other machines.
|
||||
|
||||
## Requirements
|
||||
|
||||
- **SV-1.** A check is an `inbuxa:StoreCheck` under `urn:inbuxa:jmap`. It's
|
||||
kept in the fork's subspace under `Vc` and its problems under `Vp`,
|
||||
never in the registry. The latest 20 checks are kept; starting a 21st
|
||||
forgets the oldest.
|
||||
- **SV-2.** Creating one starts it. Its `mode` is:
|
||||
- `quick`: every blob the check covers (SV-3) is in the blob store;
|
||||
- `full`: also, each one reads back whole and its BLAKE3 hash is its id.
|
||||
Blob ids are the hash of their content, so this proves the content,
|
||||
not just that something is stored under the id.
|
||||
|
||||
One check runs at a time across the server; another create is refused
|
||||
while one runs.
|
||||
- **SV-3.** What it covers:
|
||||
- **Links.** Every blob a document holds (`BlobLink::Document`: mail,
|
||||
files, Sieve scripts and the rest, by collection), every blob a queued
|
||||
message holds (`BlobLink::Id`), and every blob a kept deleted item
|
||||
holds (a temporary link that names the archived item, not yet
|
||||
expired). A blob several documents share is read once and reported
|
||||
for each of them.
|
||||
- **Mail metadata.** Each message's stored metadata names its blob. The
|
||||
check finds the message's own link to it. A message that names a blob
|
||||
without a link is a problem even when the blob is there, because
|
||||
nothing stops the blob from being purged.
|
||||
|
||||
Not covered: uploads nothing holds yet (they expire on their own), and
|
||||
files the server keeps under fixed names (spam model, app bundles).
|
||||
- **SV-4.** A problem is one of:
|
||||
- `missing`: the blob store has nothing under the id;
|
||||
- `damaged` (full only): the content's hash isn't its id, or it doesn't
|
||||
decompress;
|
||||
- `unlinked`: a message's metadata names a blob it holds no link to,
|
||||
with whether the blob is still there;
|
||||
- `unreadable`: the blob store answered with an error, which is kept.
|
||||
|
||||
Each says the account (id and name), the object (`email`, `file`,
|
||||
`sieveScript`, `calendarEvent`, `contactCard`, `queuedMessage`,
|
||||
`archivedItem`, or the collection's number for any other), its id, the
|
||||
blob id, and whether the store still has the blob's commit marker. A
|
||||
missing blob with its marker was committed and lost afterwards; one
|
||||
without it never finished being written.
|
||||
- **SV-5.** A check keeps its first 1,000 problems. The counts cover all of
|
||||
them. Every problem is also logged as an event (`store.data-corruption`),
|
||||
with the account, collection, document and blob id, so problems past the
|
||||
first 1,000 are in the log.
|
||||
- **SV-6.** It's throttled, so a full check of a big store doesn't hold up
|
||||
mail: by default at most 1,000 blobs a second and, in full mode, 20 MB a
|
||||
second read. `blobsPerSecond` and `bytesPerSecond` set other limits at
|
||||
create; 0 means no limit.
|
||||
- **SV-7.** Progress: the share of the blob index done (it goes through it
|
||||
in 256 slices), blobs and links checked, bytes read, problems by kind,
|
||||
and accounts whose mail metadata was checked. It's saved at least every
|
||||
10 seconds, which is also its heartbeat.
|
||||
- **SV-8.** A check ends:
|
||||
- `done`, problems or none;
|
||||
- `stopped`, by an update with `action: "stop"`, which any node can
|
||||
take: the running node sees it at its next save;
|
||||
- `failed`, with the error, when the check itself couldn't go on;
|
||||
- `interrupted`: its node restarted while it ran, or the node has been
|
||||
quiet for five minutes.
|
||||
- **SV-9.** Only administrators check the store: `sysStoreCheck`, in the
|
||||
built-in Administrator role and granted once to existing administrator
|
||||
roles. It reads every account's data, so a tenant administrator is
|
||||
refused. Starting and stopping a check are audited; reading one isn't.
|
||||
- **SV-10.** A finished check can be destroyed. A running one can't; stop
|
||||
it first.
|
||||
|
||||
## The object
|
||||
|
||||
| Property | Type | |
|
||||
|---|---|---|
|
||||
| `id` | Id | |
|
||||
| `mode` | String | `quick` or `full`, set at create |
|
||||
| `blobsPerSecond`, `bytesPerSecond` | UnsignedInt | Limits, set at create (SV-6) |
|
||||
| `status` | String | `running`, `done`, `stopped`, `failed`, `interrupted` |
|
||||
| `createdAt`, `finishedAt` | UTCDate | |
|
||||
| `createdBy` | String | Who started it |
|
||||
| `node` | String | The node it runs on |
|
||||
| `progress` | Number | 0 to 1 |
|
||||
| `blobsChecked`, `linksChecked`, `bytesRead`, `messagesChecked` | UnsignedInt | |
|
||||
| `missing`, `damaged`, `unlinked`, `unreadable` | UnsignedInt | Counts of each kind |
|
||||
| `problems` | Object[] | The first 1,000 (SV-4, SV-5) |
|
||||
| `error` | String | Why a `failed` check failed |
|
||||
|
||||
A problem is `{kind, accountId, accountName, object, objectId, blobId,
|
||||
committed, detail}`.
|
||||
|
||||
## Tests
|
||||
|
||||
- `system::store_check` (`tests/src/system/store_check.rs`), against a real
|
||||
server: a clean store reported clean; then blobs deleted from under
|
||||
messages (missing, including a blob two messages share, reported once per
|
||||
message), a blob overwritten with other content (damaged in full mode, not
|
||||
in quick), and a message whose link was taken away (unlinked, with the
|
||||
blob still there). Over JMAP: the refusal while one check runs, stop,
|
||||
the destroy rules, bad input, the audit events, a user's refusal and a
|
||||
tenant administrator's.
|
||||
|
||||
## Not yet
|
||||
|
||||
- The console's page and an `inbuxa-cli` command.
|
||||
- A test with a queued message and a kept archived item holding a blob.
|
||||
- Metadata for files and Sieve scripts, as for mail (SV-3): their links are
|
||||
checked, but not that each one holds the blob its record names.
|
||||
- Mail metadata of an account that holds no link at all and isn't in the
|
||||
registry any more.
|
||||
@@ -0,0 +1,151 @@
|
||||
# Feature spec: a snapshot of the store, without stopping the server
|
||||
|
||||
Status: building, 2026-11-09 (inbuxa-server #323). Not a rebuild (SPEC.md
|
||||
§4): inbuxa's own design.
|
||||
|
||||
## Provenance
|
||||
|
||||
| Source | Used for |
|
||||
|---|---|
|
||||
| Forum topic 238 | The need: a backup that stops the server, copies the store cold and starts it again, an outage each time |
|
||||
| RocksDB's checkpoint (`rocksdb::checkpoint::Checkpoint`, rocksdb 0.25) | A consistent copy of every column family at once, by hard links on the same filesystem |
|
||||
| This server's blob writes and purge (`crates/store/src/write/blob.rs`, `crates/store/src/backend/fs/mod.rs`) | Why a snapshot of the data store, then of the blobs, holds every blob its data links to |
|
||||
| The store check (`features/store-check.md`) | How the object, its permission and its run are built |
|
||||
|
||||
Upstream's open code has nothing like it, and nothing here comes from
|
||||
Enterprise code.
|
||||
|
||||
## What it is
|
||||
|
||||
An administrator asks the server for a snapshot of its store. The server
|
||||
writes a consistent copy into a directory on its own machine, while mail
|
||||
keeps flowing, and says when it's done. A backup tool then copies that
|
||||
directory however it likes. Encryption, off-site copies and how long
|
||||
backups are kept stay with the backup tool. The server keeps only its
|
||||
latest two snapshots.
|
||||
|
||||
A snapshot is a cold copy that was taken hot. Restoring one is copying it
|
||||
back while the server is stopped.
|
||||
|
||||
## Requirements
|
||||
|
||||
- **SN-1.** A snapshot is an `inbuxa:StoreSnapshot` under `urn:inbuxa:jmap`.
|
||||
It needs RocksDB as the data store. With any other data store a create
|
||||
is refused, and the refusal says to use the database's own backup:
|
||||
PostgreSQL's or MySQL's, or FoundationDB's.
|
||||
While a change of blob store is unresolved, a create is refused too:
|
||||
some blobs may still be only in the old store, which the snapshot
|
||||
leaves out (blob-store-change spec, BM-8).
|
||||
- **SN-2.** What it holds:
|
||||
- **The data store**: a RocksDB checkpoint of every column family, taken
|
||||
at one point in time. Blobs kept in the data store (the default) are
|
||||
in it too.
|
||||
- **Blobs in a separate filesystem blob store**, copied after the
|
||||
checkpoint. Blobs are written before anything links to them, and the
|
||||
blob purge is held while the snapshot is taken (SN-4), so every blob
|
||||
the checkpoint links to is there to copy. A blob written after the
|
||||
checkpoint may be copied as well; nothing in the snapshot links to it.
|
||||
|
||||
A blob directory that holds the data store's directory is fine: the
|
||||
copy leaves the data store's directory out. Blobs in the data store's
|
||||
own directory are refused, since they can't be told from its files.
|
||||
Any other blob store (S3, Azure, a sharded set, a SQL database) is
|
||||
refused: a copy of the data store alone would point at blobs a later
|
||||
purge could take away before the backup tool copied them.
|
||||
|
||||
Not in it: the search index when it's kept outside the data store
|
||||
(Meilisearch, Elasticsearch, ...), rebuilt after a restore; and the
|
||||
in-memory store (rate limits, locks), which a restarted server doesn't
|
||||
need.
|
||||
- **SN-3.** Where: under `snapshots/` in the RocksDB directory, one
|
||||
directory per snapshot named for when it was taken (UTC), like
|
||||
`snapshots/20261109T013000Z/`, with `data/`, `blobs/` when there are
|
||||
separate blobs, and `snapshot.json`. Being on the store's own
|
||||
filesystem, RocksDB and the blob copy hard-link their files, so a snapshot
|
||||
is quick and at first takes almost no space. Where they can't link
|
||||
(another filesystem), they copy. The location is fixed: a request can't
|
||||
name a path.
|
||||
|
||||
A snapshot is written as `snapshots/.<name>.partial/` and renamed when
|
||||
complete, so a directory with the plain name is always whole.
|
||||
`snapshots/latest` is a link to the newest complete one.
|
||||
- **SN-4.** While a snapshot takes blobs from a filesystem blob store, the
|
||||
blob purge waits. A purge already running is let finish first. A purge
|
||||
task that comes due meanwhile is put off by five minutes.
|
||||
- **SN-5.** `snapshot.json` says when it was taken and finished, by whom,
|
||||
the server version, what it holds (`data`, `blobs`), the data and blob
|
||||
stores' paths, its size, and how to restore it.
|
||||
- **SN-6.** One snapshot is made at a time; another create is refused
|
||||
while one runs. When one completes, the oldest beyond the latest two are
|
||||
deleted.
|
||||
- **SN-7.** A snapshot is `running`, then `done`, or `failed` with the
|
||||
error and its partial directory removed. One the server stopped in the
|
||||
middle of leaves a partial directory, removed when the server next
|
||||
starts, and isn't listed.
|
||||
|
||||
The complete snapshots on disk are the record: `get` lists those, read
|
||||
from each one's `snapshot.json`, and, until the server restarts, the one
|
||||
being taken or the last one that failed.
|
||||
- **SN-8.** Only administrators take snapshots: `sysStoreSnapshot`, in the
|
||||
built-in Administrator role and granted to existing administrator roles.
|
||||
A snapshot holds every account's data, so a tenant administrator is
|
||||
refused. Taking and destroying a snapshot are audited.
|
||||
- **SN-9.** Destroying a snapshot deletes its directory. A running one
|
||||
can't be destroyed.
|
||||
- **SN-10.** A filesystem blob store writes each blob to a temporary file
|
||||
in the same directory and renames it into place. Before, it rewrote a
|
||||
file in place, which would also have changed the copy a snapshot holds
|
||||
by hard link, and left a half-written blob behind if the server stopped
|
||||
mid-write.
|
||||
|
||||
## The object
|
||||
|
||||
| Property | Type | |
|
||||
|---|---|---|
|
||||
| `id` | Id | |
|
||||
| `status` | String | `running`, `done`, `failed` |
|
||||
| `createdAt`, `finishedAt` | UTCDate | |
|
||||
| `createdBy` | String | Who took it |
|
||||
| `node` | String | The node that took it |
|
||||
| `name` | String | Its directory under `snapshots/` |
|
||||
| `path` | String | Its full path on the server's machine |
|
||||
| `holds` | String[] | `data`, and `blobs` when they're separate |
|
||||
| `size` | UnsignedInt | Bytes in its files |
|
||||
| `blobFiles` | UnsignedInt | Blob files copied |
|
||||
| `error` | String | Why a `failed` snapshot failed |
|
||||
|
||||
A create takes no properties.
|
||||
|
||||
## Restoring
|
||||
|
||||
1. Stop the server.
|
||||
2. Move the RocksDB directory's contents aside, except `snapshots/`, and
|
||||
copy the snapshot's `data/` into it.
|
||||
3. With a separate filesystem blob store, do the same with its directory
|
||||
and the snapshot's `blobs/`.
|
||||
4. Start the server. A search index kept outside the data store isn't in
|
||||
the snapshot; reindex the accounts once it's running.
|
||||
|
||||
A backup tool copying the store cold should leave out `snapshots/`, or
|
||||
the store is copied twice.
|
||||
|
||||
## Tests
|
||||
|
||||
- `system::store_snapshot` (`tests/src/system/store_snapshot.rs`), against
|
||||
a real server with RocksDB, run with blobs in the data store and again
|
||||
with `BLOB_STORE=FileSystem`: a snapshot taken while mail is in the
|
||||
store; the directory, its name, `latest` and the manifest; a RocksDB
|
||||
store opened from its `data/` has the messages and their blobs (in it,
|
||||
or in `blobs/`), and not a message that came after; no properties at
|
||||
create, no updates; with separate blobs, one at a time and no destroy
|
||||
while one is taken (the test holds the purge gate); only the latest two
|
||||
kept; destroy; the audit events; a user's and a tenant administrator's
|
||||
refusal.
|
||||
- Unit tests: the filesystem blob store's writes (SN-10), the tree copy
|
||||
and what it leaves out, the purge gate, snapshot names and the manifest.
|
||||
|
||||
## Not yet
|
||||
|
||||
- The console's page and an `inbuxa-cli` command a backup tool can call
|
||||
and wait on.
|
||||
- SQLite as the data store (`VACUUM INTO` would do it).
|
||||
@@ -20,7 +20,7 @@
|
||||
[vocabulary]
|
||||
categories = ["identifier", "contact", "network", "content", "metadata", "credential"]
|
||||
whose = ["holder", "correspondent", "administrator"]
|
||||
where = ["data-store", "blob-store", "search-store", "in-memory-store", "memory", "log-file", "external"]
|
||||
where = ["data-store", "blob-store", "search-store", "in-memory-store", "memory", "log-file", "data-directory", "external"]
|
||||
scope = ["tenant", "server"]
|
||||
retention = ["unbounded", "object-life", "receiver"]
|
||||
|
||||
@@ -238,6 +238,38 @@ reason = ["content"]
|
||||
createdBy = ["identifier"]
|
||||
blobId = ["content", "identifier", "contact", "metadata"]
|
||||
|
||||
[object."inbuxa:StoreCheck"]
|
||||
file = "inbuxa_store_check.rs"
|
||||
default = "none"
|
||||
whose = ["holder", "administrator"]
|
||||
where = ["data-store"]
|
||||
scope = "server"
|
||||
retention = "object-life"
|
||||
[object."inbuxa:StoreCheck".properties]
|
||||
createdBy = ["identifier"]
|
||||
problems = ["identifier", "metadata"]
|
||||
|
||||
[object."inbuxa:BlobStoreChange"]
|
||||
file = "inbuxa_blob_store_change.rs"
|
||||
default = "none"
|
||||
whose = ["holder", "administrator"]
|
||||
where = ["data-store"]
|
||||
scope = "server"
|
||||
retention = "object-life"
|
||||
[object."inbuxa:BlobStoreChange".properties]
|
||||
copyStartedBy = ["identifier"]
|
||||
problems = ["identifier", "metadata"]
|
||||
|
||||
[object."inbuxa:StoreSnapshot"]
|
||||
file = "inbuxa_store_snapshot.rs"
|
||||
default = "none"
|
||||
whose = ["administrator"]
|
||||
where = ["data-directory"]
|
||||
scope = "server"
|
||||
retention = "object-life"
|
||||
[object."inbuxa:StoreSnapshot".properties]
|
||||
createdBy = ["identifier"]
|
||||
|
||||
[object."inbuxa:DeletedAccount"]
|
||||
file = "inbuxa_deleted_account.rs"
|
||||
default = "none"
|
||||
@@ -351,6 +383,32 @@ retention = "receiver"
|
||||
leaves_host = false
|
||||
written_by = ["crates/services/src/inbuxa_scheduled_reports.rs"]
|
||||
|
||||
# A store snapshot: a copy of the whole data store, and of a filesystem
|
||||
# blob store's files, under snapshots/ in the RocksDB directory. The latest
|
||||
# two are kept; a backup tool takes them off the host if the administrator
|
||||
# has one do so (store-snapshot spec, SN-3, SN-6).
|
||||
[source."store-snapshots"]
|
||||
categories = ["content", "identifier", "contact", "network", "metadata", "credential"]
|
||||
whose = ["holder", "correspondent", "administrator"]
|
||||
where = ["data-directory"]
|
||||
scope = "server"
|
||||
retention = "object-life"
|
||||
leaves_host = false
|
||||
written_by = ["crates/services/src/inbuxa_store_snapshot.rs"]
|
||||
|
||||
# A change of blob store keeps the old store's settings, credentials
|
||||
# included, until it's resolved or dismissed, so the old store can be read
|
||||
# and copied from. Never returned by get (blob-store-change spec, BM-2,
|
||||
# BM-7, BM-9).
|
||||
[source."blob-store-change-settings"]
|
||||
categories = ["credential", "network"]
|
||||
whose = ["administrator"]
|
||||
where = ["data-store"]
|
||||
scope = "server"
|
||||
retention = "object-life"
|
||||
leaves_host = false
|
||||
written_by = ["crates/features/src/blob_move/mod.rs"]
|
||||
|
||||
[source."full-text-index"]
|
||||
categories = ["content", "identifier", "contact", "metadata"]
|
||||
whose = ["holder", "correspondent"]
|
||||
|
||||
Binary file not shown.
@@ -1 +1 @@
|
||||
sumxHhh5m_-OICu3KBWCIwgVgSpXxw-jcNAvXt63ATc
|
||||
mF1m1A-76ThstOP9qsGUtzCS3tgqAF0rKBrla9jY5xI
|
||||
@@ -0,0 +1,434 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:BlobStoreChange` (docs/spec/features/blob-store-change.md).
|
||||
//!
|
||||
//! The forum topic 238 case: some mail went to a filesystem blob store,
|
||||
//! then the setting went back to the data store. The test records that
|
||||
//! change as a start-up would and gives the server the read-through store
|
||||
//! it would build (a test server can't restart with another setting). One
|
||||
//! message's blob is only in the old store, one is damaged there, one is
|
||||
//! in neither. The first reads through and is kept; a copy copies the
|
||||
//! rest it can, reports the damaged and the missing one, and leaves the
|
||||
//! change unresolved. Put right, a second copy resolves it, and nothing
|
||||
//! is read through any more. A slow copy stops when asked and can't be
|
||||
//! dismissed while it runs. Nobody but an administrator reaches changes.
|
||||
|
||||
use crate::utils::{
|
||||
account::Account,
|
||||
server::{TestServer, TestServerBuilder},
|
||||
};
|
||||
use email::{mailbox::INBOX_ID, message::metadata::MessageMetadata};
|
||||
use inbuxa_features::blob_move::{self as model, Started};
|
||||
use registry::schema::{enums::CompressionAlgo, structs};
|
||||
use serde_json::{Value, json};
|
||||
use std::{str::FromStr, sync::Arc, time::Duration};
|
||||
use store::{
|
||||
BlobStore, ValueKey,
|
||||
backend::fs::FsStore,
|
||||
build::blob::blob_store_location,
|
||||
write::{
|
||||
AlignedBytes, Archive,
|
||||
blob_move::{MovingBlobStore, OldBlobStore},
|
||||
now,
|
||||
},
|
||||
};
|
||||
use types::{blob_hash::BlobHash, collection::Collection, field::EmailField, id::Id};
|
||||
|
||||
const USING: &[&str] = &[
|
||||
"urn:ietf:params:jmap:core",
|
||||
"urn:inbuxa:jmap",
|
||||
"urn:inbuxa:jmap:registry",
|
||||
];
|
||||
|
||||
async fn call(account: &Account, method: &str, mut arguments: Value) -> (String, Value) {
|
||||
if arguments.get("accountId").is_none() {
|
||||
arguments["accountId"] = account.id_string().into();
|
||||
}
|
||||
let response = account
|
||||
.jmap_request(USING, json!([[method, arguments, "0"]]))
|
||||
.await;
|
||||
let call = response
|
||||
.0
|
||||
.pointer("/methodResponses/0")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| panic!("{method}: {}", response.0));
|
||||
(
|
||||
call[0].as_str().unwrap_or_default().to_string(),
|
||||
call[1].clone(),
|
||||
)
|
||||
}
|
||||
|
||||
async fn get(admin: &Account, id: &str) -> Value {
|
||||
let (_, response) = call(admin, "inbuxa:BlobStoreChange/get", json!({"ids": [id]})).await;
|
||||
response["list"][0].clone()
|
||||
}
|
||||
|
||||
async fn update(admin: &Account, id: &str, patch: Value) -> Value {
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:BlobStoreChange/set",
|
||||
json!({"update": {id: patch}}),
|
||||
)
|
||||
.await;
|
||||
response
|
||||
}
|
||||
|
||||
/// The change once its copy is over.
|
||||
async fn settled(admin: &Account, id: &str) -> Value {
|
||||
for _ in 0..900 {
|
||||
let change = get(admin, id).await;
|
||||
if change["status"] != "copying" {
|
||||
return change;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
panic!("the copy didn't finish: {}", get(admin, id).await);
|
||||
}
|
||||
|
||||
async fn import(client: &jmap_client::client::Client, body: &str) -> u32 {
|
||||
let id = client
|
||||
.email_import(
|
||||
format!("From: [email protected]\r\nSubject: {body}\r\n\r\n{body}\r\n").into_bytes(),
|
||||
[Id::from(INBOX_ID).to_string()],
|
||||
None::<Vec<&str>>,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.take_id();
|
||||
Id::from_str(&id).unwrap().document_id()
|
||||
}
|
||||
|
||||
async fn blob_of(test: &TestServer, account_id: u32, document_id: u32) -> BlobHash {
|
||||
let archive = test
|
||||
.server
|
||||
.store()
|
||||
.get_value::<Archive<AlignedBytes>>(ValueKey::property(
|
||||
account_id,
|
||||
Collection::Email,
|
||||
document_id,
|
||||
EmailField::Metadata,
|
||||
))
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("the message's metadata");
|
||||
let metadata = archive.unarchive::<MessageMetadata>().unwrap();
|
||||
BlobHash::from(&metadata.blob_hash)
|
||||
}
|
||||
|
||||
fn count(change: &Value, kind: &str) -> u64 {
|
||||
change[kind]
|
||||
.as_u64()
|
||||
.unwrap_or_else(|| panic!("no {kind}: {change}"))
|
||||
}
|
||||
|
||||
/// Gives the server `blob` as its blob store, as a start-up would.
|
||||
fn use_blob_store(test: &mut TestServer, blob: BlobStore) {
|
||||
let mut core = (*test.server.inner.shared_core.load_full()).clone();
|
||||
core.storage.blob = blob;
|
||||
test.server.inner.shared_core.store(Arc::new(core));
|
||||
test.reload_core();
|
||||
}
|
||||
|
||||
/// Moves a message's blob from the current store to `old`, or puts
|
||||
/// `content` there in its place.
|
||||
async fn leave_behind(
|
||||
test: &TestServer,
|
||||
old: &BlobStore,
|
||||
hash: &BlobHash,
|
||||
content: Option<&[u8]>,
|
||||
) -> Vec<u8> {
|
||||
let current = test.server.blob_store().current().clone();
|
||||
let original = current
|
||||
.get_blob(hash.as_slice(), 0..usize::MAX)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("the blob");
|
||||
old.put_blob(
|
||||
hash.as_slice(),
|
||||
content.unwrap_or(&original),
|
||||
CompressionAlgo::Lz4,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
current.delete_blob(hash.as_slice()).await.unwrap();
|
||||
original
|
||||
}
|
||||
|
||||
pub async fn test(test: &mut TestServer) {
|
||||
println!("Running blob store change tests...");
|
||||
let data = test.server.store().clone();
|
||||
|
||||
// A start with blobs in a directory, then one with the setting back
|
||||
let dir = format!("{}/old-blobs", test.tmp_dir());
|
||||
let fs_settings = structs::FileSystemStore {
|
||||
path: dir.clone(),
|
||||
depth: 2,
|
||||
};
|
||||
let old_settings = structs::BlobStore::FileSystem(fs_settings.clone());
|
||||
let old = FsStore::open(fs_settings).await.unwrap();
|
||||
let here = structs::BlobStore::Default;
|
||||
model::at_start(
|
||||
&data,
|
||||
blob_store_location(&old_settings),
|
||||
&old_settings,
|
||||
1,
|
||||
now(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let Started::Changed(change_id) =
|
||||
model::at_start(&data, blob_store_location(&here), &here, 1, now())
|
||||
.await
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("no change recorded");
|
||||
};
|
||||
let id = Id::from(change_id).to_string();
|
||||
|
||||
// The store a start-up would build: reading through to the old one
|
||||
let current = test.server.blob_store().clone();
|
||||
use_blob_store(
|
||||
test,
|
||||
BlobStore::Moving(Arc::new(MovingBlobStore::new(
|
||||
current.clone(),
|
||||
vec![OldBlobStore {
|
||||
change_id,
|
||||
store: old.clone(),
|
||||
}],
|
||||
))),
|
||||
);
|
||||
|
||||
let admin = test.account("[email protected]");
|
||||
let ann = test.account("[email protected]");
|
||||
let ann_id = ann.id().document_id();
|
||||
let client = ann.jmap_client().await;
|
||||
|
||||
let behind = import(&client, "Delivered while the filesystem store was set").await;
|
||||
let damaged = import(&client, "Damaged in the old store").await;
|
||||
let gone = import(&client, "In neither store").await;
|
||||
let fine = import(&client, "Never moved").await;
|
||||
|
||||
let behind_hash = blob_of(test, ann_id, behind).await;
|
||||
let damaged_hash = blob_of(test, ann_id, damaged).await;
|
||||
let gone_hash = blob_of(test, ann_id, gone).await;
|
||||
leave_behind(test, &old, &behind_hash, None).await;
|
||||
let damaged_content =
|
||||
leave_behind(test, &old, &damaged_hash, Some(b"Something else entirely")).await;
|
||||
let gone_content = current
|
||||
.get_blob(gone_hash.as_slice(), 0..usize::MAX)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
current.delete_blob(gone_hash.as_slice()).await.unwrap();
|
||||
|
||||
// Listed, with where from and to and never the old store's settings
|
||||
let change = get(admin, &id).await;
|
||||
assert_eq!(change["status"], "pending", "{change}");
|
||||
assert_eq!(
|
||||
change["from"],
|
||||
blob_store_location(&old_settings),
|
||||
"{change}"
|
||||
);
|
||||
assert_eq!(change["to"], "the data store", "{change}");
|
||||
assert!(!change.to_string().contains("oldSettings"), "{change}");
|
||||
assert!(!change.to_string().contains("FileSystem"), "{change}");
|
||||
|
||||
// BM-3: left behind, read through, and kept
|
||||
assert!(!current.blob_present(behind_hash.as_slice()).await.unwrap());
|
||||
let message = test.fetch_email(ann_id, behind).await;
|
||||
assert!(
|
||||
String::from_utf8_lossy(&message).contains("Delivered while the filesystem store was set"),
|
||||
"read through the old store"
|
||||
);
|
||||
assert!(
|
||||
current.blob_present(behind_hash.as_slice()).await.unwrap(),
|
||||
"kept in the current store"
|
||||
);
|
||||
assert!(
|
||||
old.blob_present(behind_hash.as_slice()).await.unwrap(),
|
||||
"nothing deleted from the old store"
|
||||
);
|
||||
|
||||
// Refusals
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:BlobStoreChange/set",
|
||||
json!({"create": {"x": {}}}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
response["notCreated"]["x"]["type"], "forbidden",
|
||||
"{response}"
|
||||
);
|
||||
let response = update(admin, &id, json!({"action": "rename"})).await;
|
||||
assert_eq!(
|
||||
response["notUpdated"][&id]["type"], "invalidProperties",
|
||||
"{response}"
|
||||
);
|
||||
let response = update(admin, &id, json!({"from": "elsewhere", "action": "copy"})).await;
|
||||
assert_eq!(
|
||||
response["notUpdated"][&id]["type"], "invalidProperties",
|
||||
"{response}"
|
||||
);
|
||||
let response = update(admin, &id, json!({"action": "stop"})).await;
|
||||
assert_eq!(
|
||||
response["notUpdated"][&id]["type"], "forbidden",
|
||||
"{response}"
|
||||
);
|
||||
let (name, response) = call(ann, "inbuxa:BlobStoreChange/get", json!({"ids": null})).await;
|
||||
assert_eq!(name, "error", "a user reaches changes: {response}");
|
||||
|
||||
// A slow copy stops when asked, and can't be dismissed meanwhile
|
||||
for n in 0..15 {
|
||||
import(&client, &format!("Filler {n}")).await;
|
||||
}
|
||||
let response = update(admin, &id, json!({"action": "copy", "blobsPerSecond": 1})).await;
|
||||
assert!(response["updated"].get(&id).is_some(), "{response}");
|
||||
let response = update(admin, &id, json!({"action": "copy"})).await;
|
||||
assert_eq!(
|
||||
response["notUpdated"][&id]["type"], "forbidden",
|
||||
"one copy at a time: {response}"
|
||||
);
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:BlobStoreChange/set",
|
||||
json!({"destroy": [id]}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
response["notDestroyed"][&id]["type"], "forbidden",
|
||||
"{response}"
|
||||
);
|
||||
let response = update(admin, &id, json!({"action": "stop"})).await;
|
||||
assert!(response["updated"].get(&id).is_some(), "{response}");
|
||||
let change = settled(admin, &id).await;
|
||||
assert_eq!(change["status"], "stopped", "{change}");
|
||||
assert_eq!(change["copyStartedBy"], "[email protected]", "{change}");
|
||||
|
||||
// A copy: the damaged and the missing are reported, not copied
|
||||
let response = update(admin, &id, json!({"action": "copy", "blobsPerSecond": 0})).await;
|
||||
assert!(response["updated"].get(&id).is_some(), "{response}");
|
||||
let change = settled(admin, &id).await;
|
||||
assert_eq!(change["status"], "incomplete", "{change}");
|
||||
assert_eq!(count(&change, "missing"), 1, "{change}");
|
||||
assert_eq!(count(&change, "damaged"), 1, "{change}");
|
||||
assert_eq!(count(&change, "unreadable"), 0, "{change}");
|
||||
assert!(count(&change, "alreadyThere") >= 2, "{change}");
|
||||
let problems = change["problems"].as_array().unwrap().clone();
|
||||
let of = |hash: &BlobHash| {
|
||||
problems
|
||||
.iter()
|
||||
.find(|p| p["blobId"] == hash.to_hex())
|
||||
.cloned()
|
||||
.unwrap_or_else(|| panic!("no problem for {}: {change}", hash.to_hex()))
|
||||
};
|
||||
let missing = of(&gone_hash);
|
||||
assert_eq!(missing["kind"], "missing", "{missing}");
|
||||
assert_eq!(missing["holder"], "email", "{missing}");
|
||||
assert_eq!(missing["accountId"], ann.id_string(), "{missing}");
|
||||
assert_eq!(missing["objectId"], gone as u64, "{missing}");
|
||||
assert_eq!(of(&damaged_hash)["kind"], "damaged");
|
||||
assert!(
|
||||
!current.blob_present(damaged_hash.as_slice()).await.unwrap(),
|
||||
"a damaged blob isn't copied"
|
||||
);
|
||||
assert!(
|
||||
test.server
|
||||
.blob_store()
|
||||
.moving()
|
||||
.is_some_and(|m| !m.old_stores().is_empty()),
|
||||
"still read through"
|
||||
);
|
||||
|
||||
// Put right in the old store: a second copy resolves the change
|
||||
old.put_blob(
|
||||
damaged_hash.as_slice(),
|
||||
&damaged_content,
|
||||
CompressionAlgo::Lz4,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
old.put_blob(gone_hash.as_slice(), &gone_content, CompressionAlgo::None)
|
||||
.await
|
||||
.unwrap();
|
||||
let response = update(admin, &id, json!({"action": "copy", "blobsPerSecond": 0})).await;
|
||||
assert!(response["updated"].get(&id).is_some(), "{response}");
|
||||
let change = settled(admin, &id).await;
|
||||
assert_eq!(change["status"], "copied", "{change}");
|
||||
assert_eq!(count(&change, "copied"), 2, "{change}");
|
||||
assert_eq!(change["progress"], 1.0, "{change}");
|
||||
assert_eq!(change["problems"], json!([]), "{change}");
|
||||
for hash in [&damaged_hash, &gone_hash] {
|
||||
assert!(current.blob_present(hash.as_slice()).await.unwrap());
|
||||
}
|
||||
assert!(
|
||||
test.server
|
||||
.blob_store()
|
||||
.moving()
|
||||
.unwrap()
|
||||
.old_stores()
|
||||
.is_empty(),
|
||||
"nothing read through once resolved"
|
||||
);
|
||||
assert!(String::from_utf8_lossy(&test.fetch_email(ann_id, fine).await).contains("Never moved"));
|
||||
assert!(
|
||||
model::get(&data, change_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.old_settings
|
||||
.is_none(),
|
||||
"the old store's settings go once resolved"
|
||||
);
|
||||
let response = update(admin, &id, json!({"action": "copy"})).await;
|
||||
assert_eq!(
|
||||
response["notUpdated"][&id]["type"], "forbidden",
|
||||
"nothing left to copy: {response}"
|
||||
);
|
||||
|
||||
// Dismissed
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:BlobStoreChange/set",
|
||||
json!({"destroy": [id]}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response["destroyed"], json!([id]), "{response}");
|
||||
let (_, response) = call(admin, "inbuxa:BlobStoreChange/get", json!({"ids": [id]})).await;
|
||||
assert_eq!(response["notFound"], json!([id]), "{response}");
|
||||
|
||||
use_blob_store(test, current);
|
||||
}
|
||||
|
||||
/// Runs these tests alone: `cargo test -p tests blob_store_change_tests -- --ignored`.
|
||||
#[ignore]
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
pub async fn blob_store_change_tests() {
|
||||
let mut test = TestServerBuilder::new("blob_store_change_tests")
|
||||
.await
|
||||
.with_default_listeners()
|
||||
.await
|
||||
.build()
|
||||
.await;
|
||||
let admin = test.create_admin_account("[email protected]").await;
|
||||
test.insert_account(admin);
|
||||
let ann = test
|
||||
.create_user_account(
|
||||
"[email protected]",
|
||||
"[email protected]",
|
||||
"ann-secret-40917",
|
||||
&[],
|
||||
"Ann",
|
||||
)
|
||||
.await;
|
||||
test.insert_account(ann);
|
||||
self::test(&mut test).await;
|
||||
if test.is_reset() {
|
||||
test.temp_dir.delete();
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ pub mod audit; // inbuxa: the audit log
|
||||
pub mod authentication;
|
||||
pub mod authorization;
|
||||
pub mod auto_reload; // inbuxa: registry writes apply at once
|
||||
pub mod blob_store_change; // inbuxa: a change of blob store
|
||||
pub mod branding;
|
||||
pub mod compliance; // inbuxa: the compliance roles
|
||||
pub mod crypto;
|
||||
@@ -36,6 +37,8 @@ pub mod scheduled_reports; // inbuxa: scheduled reports and the weekly digest
|
||||
pub mod security;
|
||||
pub mod security_acceptances; // inbuxa: accepted security to-do items
|
||||
pub mod sharing_policy; // inbuxa: MA-C, who may share mail
|
||||
pub mod store_check; // inbuxa: checking the store's blobs
|
||||
pub mod store_snapshot; // inbuxa: a snapshot of the store
|
||||
pub mod task;
|
||||
pub mod tenant;
|
||||
pub mod tracer_reload; // inbuxa: tracers start over when their settings change
|
||||
|
||||
@@ -0,0 +1,421 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:StoreCheck` (docs/spec/features/store-check.md).
|
||||
//!
|
||||
//! A clean store checks clean. Then blobs are taken from under messages:
|
||||
//! one deleted with its commit marker left, one shared by two messages, one
|
||||
//! overwritten with other content, and one message's link taken away while
|
||||
//! its blob stays. A quick check finds the missing ones and the unlinked
|
||||
//! one, each with the account and the message; a full check also finds the
|
||||
//! damaged one. One check runs at a time, a running one stops when asked
|
||||
//! and can't be destroyed, a finished one can, and nobody in a tenant or
|
||||
//! without the permission reaches checks. Starting and stopping are
|
||||
//! audited.
|
||||
|
||||
use crate::utils::{
|
||||
account::Account,
|
||||
server::{TestServer, TestServerBuilder},
|
||||
};
|
||||
use email::{mailbox::INBOX_ID, message::metadata::MessageMetadata};
|
||||
use registry::schema::{
|
||||
enums::CompressionAlgo,
|
||||
prelude::{ObjectType, Property},
|
||||
structs::{CertificateManagement, DkimManagement, DnsManagement, Domain, Tenant, UserRoles},
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use std::{str::FromStr, time::Duration};
|
||||
use store::{
|
||||
ValueKey,
|
||||
write::{AlignedBytes, Archive, BatchBuilder, BlobLink, BlobOp},
|
||||
};
|
||||
use types::{blob_hash::BlobHash, collection::Collection, field::EmailField, id::Id};
|
||||
|
||||
const USING: &[&str] = &[
|
||||
"urn:ietf:params:jmap:core",
|
||||
"urn:inbuxa:jmap",
|
||||
"urn:inbuxa:jmap:registry",
|
||||
];
|
||||
|
||||
async fn call(account: &Account, method: &str, mut arguments: Value) -> (String, Value) {
|
||||
if arguments.get("accountId").is_none() {
|
||||
arguments["accountId"] = account.id_string().into();
|
||||
}
|
||||
let response = account
|
||||
.jmap_request(USING, json!([[method, arguments, "0"]]))
|
||||
.await;
|
||||
let call = response
|
||||
.0
|
||||
.pointer("/methodResponses/0")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| panic!("{method}: {}", response.0));
|
||||
(
|
||||
call[0].as_str().unwrap_or_default().to_string(),
|
||||
call[1].clone(),
|
||||
)
|
||||
}
|
||||
|
||||
async fn start(admin: &Account, check: Value) -> String {
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:StoreCheck/set",
|
||||
json!({"create": {"c": check}}),
|
||||
)
|
||||
.await;
|
||||
response["created"]["c"]["id"]
|
||||
.as_str()
|
||||
.unwrap_or_else(|| panic!("not started: {response}"))
|
||||
.to_string()
|
||||
}
|
||||
|
||||
async fn get(admin: &Account, id: &str) -> Value {
|
||||
let (_, response) = call(admin, "inbuxa:StoreCheck/get", json!({"ids": [id]})).await;
|
||||
response["list"][0].clone()
|
||||
}
|
||||
|
||||
/// The check once it's over.
|
||||
async fn settled(admin: &Account, id: &str) -> Value {
|
||||
for _ in 0..600 {
|
||||
let check = get(admin, id).await;
|
||||
if check["status"] != "running" {
|
||||
return check;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
panic!("the check didn't finish: {}", get(admin, id).await);
|
||||
}
|
||||
|
||||
async fn import(client: &jmap_client::client::Client, body: &str) -> u32 {
|
||||
let id = client
|
||||
.email_import(
|
||||
format!("From: [email protected]\r\nSubject: {body}\r\n\r\n{body}\r\n").into_bytes(),
|
||||
[Id::from(INBOX_ID).to_string()],
|
||||
None::<Vec<&str>>,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.take_id();
|
||||
Id::from_str(&id).unwrap().document_id()
|
||||
}
|
||||
|
||||
async fn blob_of(test: &TestServer, account_id: u32, document_id: u32) -> BlobHash {
|
||||
let archive = test
|
||||
.server
|
||||
.store()
|
||||
.get_value::<Archive<AlignedBytes>>(ValueKey::property(
|
||||
account_id,
|
||||
Collection::Email,
|
||||
document_id,
|
||||
EmailField::Metadata,
|
||||
))
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("the message's metadata");
|
||||
let metadata = archive.unarchive::<MessageMetadata>().unwrap();
|
||||
BlobHash::from(&metadata.blob_hash)
|
||||
}
|
||||
|
||||
fn count(check: &Value, kind: &str) -> u64 {
|
||||
check[kind]
|
||||
.as_u64()
|
||||
.unwrap_or_else(|| panic!("no {kind}: {check}"))
|
||||
}
|
||||
|
||||
/// The problems found for one message, as `kind` strings.
|
||||
fn kinds_for(check: &Value, document_id: u32) -> Vec<String> {
|
||||
check["problems"]
|
||||
.as_array()
|
||||
.unwrap_or_else(|| panic!("no problems list: {check}"))
|
||||
.iter()
|
||||
.filter(|p| p["objectId"].as_u64() == Some(document_id as u64))
|
||||
.map(|p| p["kind"].as_str().unwrap_or_default().to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub async fn test(test: &mut TestServer) {
|
||||
println!("Running store check tests...");
|
||||
let admin = test.account("[email protected]");
|
||||
let ann = test.account("[email protected]");
|
||||
let ann_id = ann.id().document_id();
|
||||
let client = ann.jmap_client().await;
|
||||
|
||||
let missing = import(&client, "Missing blob").await;
|
||||
let shared_a = import(&client, "Shared blob").await;
|
||||
let shared_b = import(&client, "Shared blob").await;
|
||||
let damaged = import(&client, "Damaged blob").await;
|
||||
let unlinked = import(&client, "Unlinked message").await;
|
||||
let fine = import(&client, "Nothing wrong").await;
|
||||
assert_eq!(
|
||||
blob_of(test, ann_id, shared_a).await,
|
||||
blob_of(test, ann_id, shared_b).await,
|
||||
"the same message twice shares its blob"
|
||||
);
|
||||
|
||||
// A clean store checks clean
|
||||
let id = start(admin, json!({"mode": "quick"})).await;
|
||||
let check = settled(admin, &id).await;
|
||||
assert_eq!(check["status"], "done", "{check}");
|
||||
assert_eq!(check["progress"], 1.0, "{check}");
|
||||
for kind in ["missing", "damaged", "unlinked", "unreadable"] {
|
||||
assert_eq!(count(&check, kind), 0, "{kind} in a clean store: {check}");
|
||||
}
|
||||
assert!(count(&check, "blobsChecked") >= 5, "{check}");
|
||||
assert!(count(&check, "messagesChecked") >= 6, "{check}");
|
||||
assert_eq!(check["problems"], json!([]), "{check}");
|
||||
assert_eq!(check["mode"], "quick", "{check}");
|
||||
assert_eq!(check["createdBy"], "[email protected]", "{check}");
|
||||
|
||||
// Take blobs from under them
|
||||
let blobs = test.server.blob_store();
|
||||
blobs
|
||||
.delete_blob(blob_of(test, ann_id, missing).await.as_slice())
|
||||
.await
|
||||
.unwrap();
|
||||
blobs
|
||||
.delete_blob(blob_of(test, ann_id, shared_a).await.as_slice())
|
||||
.await
|
||||
.unwrap();
|
||||
blobs
|
||||
.put_blob(
|
||||
blob_of(test, ann_id, damaged).await.as_slice(),
|
||||
b"Something else entirely",
|
||||
CompressionAlgo::None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch
|
||||
.with_account_id(ann_id)
|
||||
.with_collection(Collection::Email)
|
||||
.with_document(unlinked)
|
||||
.clear(BlobOp::Link {
|
||||
hash: blob_of(test, ann_id, unlinked).await,
|
||||
to: BlobLink::Document,
|
||||
});
|
||||
test.server.store().write(batch.build_all()).await.unwrap();
|
||||
|
||||
// Quick: what's gone, and the message without its link
|
||||
let id = start(admin, json!({"mode": "quick"})).await;
|
||||
let check = settled(admin, &id).await;
|
||||
assert_eq!(check["status"], "done", "{check}");
|
||||
assert_eq!(count(&check, "missing"), 3, "{check}");
|
||||
assert_eq!(
|
||||
count(&check, "damaged"),
|
||||
0,
|
||||
"quick doesn't read content: {check}"
|
||||
);
|
||||
assert_eq!(count(&check, "unlinked"), 1, "{check}");
|
||||
assert_eq!(kinds_for(&check, missing), ["missing"], "{check}");
|
||||
assert_eq!(kinds_for(&check, shared_a), ["missing"], "{check}");
|
||||
assert_eq!(kinds_for(&check, shared_b), ["missing"], "{check}");
|
||||
assert_eq!(kinds_for(&check, unlinked), ["unlinked"], "{check}");
|
||||
assert!(kinds_for(&check, fine).is_empty(), "{check}");
|
||||
let problem = check["problems"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|p| p["objectId"].as_u64() == Some(missing as u64))
|
||||
.unwrap()
|
||||
.clone();
|
||||
assert_eq!(problem["object"], "email", "{problem}");
|
||||
assert_eq!(problem["accountId"], ann.id_string(), "{problem}");
|
||||
assert_eq!(problem["accountName"], "[email protected]", "{problem}");
|
||||
assert_eq!(
|
||||
problem["committed"], true,
|
||||
"lost after it was written: {problem}"
|
||||
);
|
||||
assert_eq!(
|
||||
problem["blobId"],
|
||||
blob_of(test, ann_id, missing).await.to_hex(),
|
||||
"{problem}"
|
||||
);
|
||||
let unlinked_problem = check["problems"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|p| p["objectId"].as_u64() == Some(unlinked as u64))
|
||||
.unwrap()
|
||||
.clone();
|
||||
assert_eq!(
|
||||
unlinked_problem["detail"], "the blob is still there",
|
||||
"{unlinked_problem}"
|
||||
);
|
||||
|
||||
// Full: the overwritten one too
|
||||
let id = start(admin, json!({"mode": "full", "bytesPerSecond": 0})).await;
|
||||
let check = settled(admin, &id).await;
|
||||
assert_eq!(check["status"], "done", "{check}");
|
||||
assert_eq!(count(&check, "missing"), 3, "{check}");
|
||||
assert_eq!(count(&check, "damaged"), 1, "{check}");
|
||||
assert_eq!(count(&check, "unlinked"), 1, "{check}");
|
||||
assert_eq!(kinds_for(&check, damaged), ["damaged"], "{check}");
|
||||
assert!(count(&check, "bytesRead") > 0, "{check}");
|
||||
let full = id;
|
||||
|
||||
// One at a time; a running one stops when asked, and stays until then
|
||||
let slow = start(admin, json!({"mode": "quick", "blobsPerSecond": 1})).await;
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:StoreCheck/set",
|
||||
json!({"create": {"again": {"mode": "quick"}}}),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
response["notCreated"]["again"]["description"]
|
||||
.as_str()
|
||||
.is_some_and(|d| d.contains("one check runs at a time")),
|
||||
"a second check while one runs: {response}"
|
||||
);
|
||||
let (_, response) = call(admin, "inbuxa:StoreCheck/set", json!({"destroy": [&slow]})).await;
|
||||
assert!(response["notDestroyed"].get(&slow).is_some(), "{response}");
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:StoreCheck/set",
|
||||
json!({"update": {&slow: {"mode": "full"}}}),
|
||||
)
|
||||
.await;
|
||||
assert!(response["notUpdated"].get(&slow).is_some(), "{response}");
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:StoreCheck/set",
|
||||
json!({"update": {&slow: {"action": "stop"}}}),
|
||||
)
|
||||
.await;
|
||||
assert!(response["updated"].get(&slow).is_some(), "{response}");
|
||||
let check = settled(admin, &slow).await;
|
||||
assert_eq!(check["status"], "stopped", "{check}");
|
||||
assert!(check["progress"].as_f64().unwrap() < 1.0, "{check}");
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:StoreCheck/set",
|
||||
json!({"update": {&slow: {"action": "stop"}}}),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
response["notUpdated"].get(&slow).is_some(),
|
||||
"stopping a stopped check: {response}"
|
||||
);
|
||||
|
||||
// A finished one can be forgotten, with its problems
|
||||
let (_, response) = call(admin, "inbuxa:StoreCheck/set", json!({"destroy": [&full]})).await;
|
||||
assert_eq!(response["destroyed"], json!([full]), "{response}");
|
||||
let (_, response) = call(admin, "inbuxa:StoreCheck/get", json!({"ids": [&full]})).await;
|
||||
assert_eq!(response["notFound"], json!([full]), "{response}");
|
||||
let id = Id::from_str(&full).unwrap().id() as u32;
|
||||
assert!(
|
||||
inbuxa_features::store_check::problems(test.server.store(), id)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty(),
|
||||
"a forgotten check's problems are gone too"
|
||||
);
|
||||
|
||||
// Bad input
|
||||
for (create, why) in [
|
||||
(json!({"mode": "thorough"}), "mode"),
|
||||
(json!({"blobsPerSecond": -1}), "blobsPerSecond"),
|
||||
(json!({"status": "done"}), "status"),
|
||||
] {
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:StoreCheck/set",
|
||||
json!({"create": {"x": create}}),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
response["notCreated"].get("x").is_some(),
|
||||
"{why}: {response}"
|
||||
);
|
||||
}
|
||||
|
||||
// Starting and stopping are audited
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:AuditEvent/query",
|
||||
json!({"filter": {"targetKind": "inbuxa:StoreCheck"}}),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
response["ids"].as_array().is_some_and(|ids| ids.len() >= 5),
|
||||
"creates, the stop and the destroy audited: {response}"
|
||||
);
|
||||
|
||||
// Nobody without the permission, or in a tenant, reaches checks
|
||||
let (name, response) = call(ann, "inbuxa:StoreCheck/get", json!({"ids": null})).await;
|
||||
assert_eq!(name, "error", "a user: {response}");
|
||||
let tenant = admin
|
||||
.registry_create_object(Tenant {
|
||||
name: "Store check tenant".to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
admin
|
||||
.registry_create_object(Domain {
|
||||
name: "storecheck.example.org".to_string(),
|
||||
is_enabled: true,
|
||||
member_tenant_id: Some(tenant),
|
||||
certificate_management: CertificateManagement::Manual,
|
||||
dns_management: DnsManagement::Manual,
|
||||
dkim_management: DkimManagement::Manual,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
let t_admin = admin
|
||||
.create_user_account(
|
||||
"[email protected]",
|
||||
"tenant-admin-secret-4417",
|
||||
"Tenant admin",
|
||||
&[],
|
||||
vec![],
|
||||
)
|
||||
.await;
|
||||
admin
|
||||
.registry_update_object(
|
||||
ObjectType::Account,
|
||||
t_admin.id(),
|
||||
json!({Property::Roles: UserRoles::Admin}),
|
||||
)
|
||||
.await;
|
||||
for (method, arguments) in [
|
||||
("inbuxa:StoreCheck/get", json!({"ids": null})),
|
||||
(
|
||||
"inbuxa:StoreCheck/set",
|
||||
json!({"create": {"x": {"mode": "quick"}}}),
|
||||
),
|
||||
] {
|
||||
let (name, response) = call(&t_admin, method, arguments).await;
|
||||
assert_eq!(name, "error", "{method} by a tenant: {response}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs these tests alone: `cargo test -p tests store_check_tests -- --ignored`.
|
||||
#[ignore]
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
pub async fn store_check_tests() {
|
||||
let mut test = TestServerBuilder::new("store_check_tests")
|
||||
.await
|
||||
.with_default_listeners()
|
||||
.await
|
||||
.build()
|
||||
.await;
|
||||
let admin = test.create_admin_account("[email protected]").await;
|
||||
test.insert_account(admin);
|
||||
let ann = test
|
||||
.create_user_account(
|
||||
"[email protected]",
|
||||
"[email protected]",
|
||||
"ann-secret-29871",
|
||||
&[],
|
||||
"Ann",
|
||||
)
|
||||
.await;
|
||||
test.insert_account(ann);
|
||||
self::test(&mut test).await;
|
||||
if test.is_reset() {
|
||||
test.temp_dir.delete();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,411 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:StoreSnapshot` (docs/spec/features/store-snapshot.md).
|
||||
//!
|
||||
//! A snapshot taken while the server runs holds the mail that was in it and
|
||||
//! not the mail that came after: a RocksDB store opened from its `data/`
|
||||
//! has the messages, and their blobs are in it or, with a filesystem blob
|
||||
//! store (`BLOB_STORE=FileSystem`), in its `blobs/`. The directory is
|
||||
//! whole, named for its time, with `latest` and `snapshot.json`. Only the
|
||||
//! latest two are kept; one can be destroyed, a snapshot can't be changed,
|
||||
//! and one being taken can't be destroyed or joined by another. Nobody in
|
||||
//! a tenant or without the permission reaches snapshots. Taking and
|
||||
//! destroying are audited.
|
||||
|
||||
use crate::utils::{
|
||||
account::Account,
|
||||
server::{TestServer, TestServerBuilder},
|
||||
};
|
||||
use email::{mailbox::INBOX_ID, message::metadata::MessageMetadata};
|
||||
use registry::schema::{
|
||||
prelude::{ObjectType, Property},
|
||||
structs::{
|
||||
self, CertificateManagement, DkimManagement, DnsManagement, Domain, Tenant, UserRoles,
|
||||
},
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use std::{path::PathBuf, str::FromStr, time::Duration};
|
||||
use store::{
|
||||
BlobStore, Store, ValueKey,
|
||||
backend::{fs::FsStore, rocksdb::RocksDbStore},
|
||||
write::{AlignedBytes, Archive, snapshot::hold_blob_purge},
|
||||
};
|
||||
use types::{blob_hash::BlobHash, collection::Collection, field::EmailField, id::Id};
|
||||
|
||||
const USING: &[&str] = &[
|
||||
"urn:ietf:params:jmap:core",
|
||||
"urn:inbuxa:jmap",
|
||||
"urn:inbuxa:jmap:registry",
|
||||
];
|
||||
|
||||
async fn call(account: &Account, method: &str, mut arguments: Value) -> (String, Value) {
|
||||
if arguments.get("accountId").is_none() {
|
||||
arguments["accountId"] = account.id_string().into();
|
||||
}
|
||||
let response = account
|
||||
.jmap_request(USING, json!([[method, arguments, "0"]]))
|
||||
.await;
|
||||
let call = response
|
||||
.0
|
||||
.pointer("/methodResponses/0")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| panic!("{method}: {}", response.0));
|
||||
(
|
||||
call[0].as_str().unwrap_or_default().to_string(),
|
||||
call[1].clone(),
|
||||
)
|
||||
}
|
||||
|
||||
async fn take(admin: &Account) -> String {
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:StoreSnapshot/set",
|
||||
json!({"create": {"s": {}}}),
|
||||
)
|
||||
.await;
|
||||
response["created"]["s"]["id"]
|
||||
.as_str()
|
||||
.unwrap_or_else(|| panic!("not taken: {response}"))
|
||||
.to_string()
|
||||
}
|
||||
|
||||
async fn get(admin: &Account, id: &str) -> Value {
|
||||
let (_, response) = call(admin, "inbuxa:StoreSnapshot/get", json!({"ids": [id]})).await;
|
||||
response["list"][0].clone()
|
||||
}
|
||||
|
||||
async fn all(admin: &Account) -> Vec<Value> {
|
||||
let (_, response) = call(admin, "inbuxa:StoreSnapshot/get", json!({"ids": null})).await;
|
||||
response["list"]
|
||||
.as_array()
|
||||
.unwrap_or_else(|| panic!("no list: {response}"))
|
||||
.clone()
|
||||
}
|
||||
|
||||
/// The snapshot once it's taken.
|
||||
async fn settled(admin: &Account, id: &str) -> Value {
|
||||
for _ in 0..600 {
|
||||
let snapshot = get(admin, id).await;
|
||||
if snapshot["status"] != "running" {
|
||||
return snapshot;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
panic!("the snapshot wasn't taken: {}", get(admin, id).await);
|
||||
}
|
||||
|
||||
async fn import(client: &jmap_client::client::Client, body: &str) -> u32 {
|
||||
let id = client
|
||||
.email_import(
|
||||
format!("From: [email protected]\r\nSubject: {body}\r\n\r\n{body}\r\n").into_bytes(),
|
||||
[Id::from(INBOX_ID).to_string()],
|
||||
None::<Vec<&str>>,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.take_id();
|
||||
Id::from_str(&id).unwrap().document_id()
|
||||
}
|
||||
|
||||
async fn blob_in(store: &Store, account_id: u32, document_id: u32) -> Option<BlobHash> {
|
||||
let archive = store
|
||||
.get_value::<Archive<AlignedBytes>>(ValueKey::property(
|
||||
account_id,
|
||||
Collection::Email,
|
||||
document_id,
|
||||
EmailField::Metadata,
|
||||
))
|
||||
.await
|
||||
.unwrap()?;
|
||||
let metadata = archive.unarchive::<MessageMetadata>().unwrap();
|
||||
Some(BlobHash::from(&metadata.blob_hash))
|
||||
}
|
||||
|
||||
fn path_of(snapshot: &Value) -> PathBuf {
|
||||
PathBuf::from(
|
||||
snapshot["path"]
|
||||
.as_str()
|
||||
.unwrap_or_else(|| panic!("no path: {snapshot}")),
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn test(test: &mut TestServer) {
|
||||
println!("Running store snapshot tests...");
|
||||
let admin = test.account("[email protected]");
|
||||
let ann = test.account("[email protected]");
|
||||
let ann_id = ann.id().document_id();
|
||||
let client = ann.jmap_client().await;
|
||||
let separate_blobs = matches!(test.server.blob_store(), BlobStore::Fs(_));
|
||||
println!(
|
||||
"Blobs {}",
|
||||
if separate_blobs {
|
||||
"in a filesystem blob store"
|
||||
} else {
|
||||
"in the data store"
|
||||
}
|
||||
);
|
||||
|
||||
let before = [
|
||||
import(&client, "Before the snapshot, one").await,
|
||||
import(&client, "Before the snapshot, two").await,
|
||||
];
|
||||
assert!(all(admin).await.is_empty(), "no snapshots yet");
|
||||
|
||||
// A snapshot while the server runs
|
||||
let id = take(admin).await;
|
||||
let snapshot = settled(admin, &id).await;
|
||||
assert_eq!(snapshot["status"], "done", "{snapshot}");
|
||||
assert_eq!(snapshot["createdBy"], "[email protected]", "{snapshot}");
|
||||
let holds = if separate_blobs {
|
||||
json!(["data", "blobs"])
|
||||
} else {
|
||||
json!(["data"])
|
||||
};
|
||||
assert_eq!(snapshot["holds"], holds, "{snapshot}");
|
||||
assert!(snapshot["size"].as_u64().unwrap() > 0, "{snapshot}");
|
||||
let after = import(&client, "After the snapshot").await;
|
||||
|
||||
// The directory: whole, named for its time, `latest`, the manifest
|
||||
let path = path_of(&snapshot);
|
||||
let name = snapshot["name"].as_str().unwrap();
|
||||
assert_eq!(path.file_name().unwrap().to_str().unwrap(), name);
|
||||
assert!(name.len() == 16 && name.ends_with('Z'), "{name}");
|
||||
let root = path.parent().unwrap().to_path_buf();
|
||||
assert_eq!(root.file_name().unwrap(), "snapshots");
|
||||
assert_eq!(
|
||||
std::fs::read_link(root.join("latest")).unwrap(),
|
||||
PathBuf::from(name)
|
||||
);
|
||||
assert!(path.join("data/CURRENT").exists(), "a RocksDB checkpoint");
|
||||
assert_eq!(path.join("blobs").exists(), separate_blobs);
|
||||
let manifest: Value =
|
||||
serde_json::from_slice(&std::fs::read(path.join("snapshot.json")).unwrap()).unwrap();
|
||||
assert_eq!(manifest["id"], json!(id), "{manifest}");
|
||||
assert_eq!(manifest["holds"], holds, "{manifest}");
|
||||
assert!(
|
||||
manifest["restore"].as_array().unwrap().len() >= 3,
|
||||
"{manifest}"
|
||||
);
|
||||
assert!(
|
||||
std::fs::read_dir(&root).unwrap().all(|e| !e
|
||||
.unwrap()
|
||||
.file_name()
|
||||
.to_string_lossy()
|
||||
.ends_with(".partial")),
|
||||
"nothing partial left"
|
||||
);
|
||||
|
||||
// Restored, it has the mail that was there, and not what came after
|
||||
let restored = RocksDbStore::open(structs::RocksDbStore {
|
||||
path: path.join("data").to_string_lossy().to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let blobs = if separate_blobs {
|
||||
FsStore::open(structs::FileSystemStore {
|
||||
path: path.join("blobs").to_string_lossy().to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
} else {
|
||||
BlobStore::Store(restored.clone())
|
||||
};
|
||||
for (document_id, subject) in before.iter().zip(["one", "two"]) {
|
||||
let hash = blob_in(&restored, ann_id, *document_id)
|
||||
.await
|
||||
.unwrap_or_else(|| panic!("message {document_id} isn't in the snapshot"));
|
||||
let content = blobs
|
||||
.get_blob(hash.as_slice(), 0..usize::MAX)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap_or_else(|| panic!("the blob of message {document_id} isn't in the snapshot"));
|
||||
let content = String::from_utf8_lossy(&content);
|
||||
assert!(content.contains(&format!("Before the snapshot, {subject}")));
|
||||
}
|
||||
assert!(
|
||||
blob_in(&restored, ann_id, after).await.is_none(),
|
||||
"mail that came after isn't in it"
|
||||
);
|
||||
drop(blobs);
|
||||
drop(restored);
|
||||
|
||||
// A snapshot can't be changed, and takes no properties
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:StoreSnapshot/set",
|
||||
json!({"update": {&id: {"name": "x"}}}),
|
||||
)
|
||||
.await;
|
||||
assert!(response["notUpdated"].get(&id).is_some(), "{response}");
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:StoreSnapshot/set",
|
||||
json!({"create": {"x": {"path": "/tmp/elsewhere"}}}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
response["notCreated"]["x"]["type"], "invalidProperties",
|
||||
"{response}"
|
||||
);
|
||||
|
||||
// One at a time, and one being taken can't be destroyed. With separate
|
||||
// blobs a snapshot waits for the blob purge, so holding it here keeps
|
||||
// one being taken for as long as this needs
|
||||
if separate_blobs {
|
||||
let hold = hold_blob_purge().await;
|
||||
let waiting = take(admin).await;
|
||||
assert_eq!(get(admin, &waiting).await["status"], "running");
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:StoreSnapshot/set",
|
||||
json!({"create": {"x": {}}}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
response["notCreated"]["x"]["type"], "forbidden",
|
||||
"{response}"
|
||||
);
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:StoreSnapshot/set",
|
||||
json!({"destroy": [&waiting]}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
response["notDestroyed"][&waiting]["type"], "forbidden",
|
||||
"{response}"
|
||||
);
|
||||
drop(hold);
|
||||
assert_eq!(settled(admin, &waiting).await["status"], "done");
|
||||
} else {
|
||||
let next = take(admin).await;
|
||||
assert_eq!(settled(admin, &next).await["status"], "done");
|
||||
}
|
||||
|
||||
// Only the latest two are kept
|
||||
let third = take(admin).await;
|
||||
let third_snapshot = settled(admin, &third).await;
|
||||
let listed = all(admin).await;
|
||||
assert_eq!(listed.len(), 2, "{listed:?}");
|
||||
assert_eq!(listed[0]["id"], json!(third), "newest first: {listed:?}");
|
||||
assert!(!path.exists(), "the oldest went");
|
||||
let (_, response) = call(admin, "inbuxa:StoreSnapshot/get", json!({"ids": [&id]})).await;
|
||||
assert_eq!(response["notFound"], json!([id]), "{response}");
|
||||
assert_eq!(
|
||||
std::fs::read_link(root.join("latest")).unwrap(),
|
||||
PathBuf::from(third_snapshot["name"].as_str().unwrap())
|
||||
);
|
||||
|
||||
// Destroying one deletes it; `latest` moves to the one left
|
||||
let unknown = Id::from(1u64).to_string();
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:StoreSnapshot/set",
|
||||
json!({"destroy": [&third, &unknown]}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response["destroyed"], json!([third]), "{response}");
|
||||
assert_eq!(
|
||||
response["notDestroyed"][&unknown]["type"], "notFound",
|
||||
"{response}"
|
||||
);
|
||||
assert!(!path_of(&third_snapshot).exists());
|
||||
let left = all(admin).await;
|
||||
assert_eq!(left.len(), 1, "{left:?}");
|
||||
assert_eq!(
|
||||
std::fs::read_link(root.join("latest")).unwrap(),
|
||||
PathBuf::from(left[0]["name"].as_str().unwrap())
|
||||
);
|
||||
|
||||
// Taking and destroying are audited
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:AuditEvent/query",
|
||||
json!({"filter": {"targetKind": "inbuxa:StoreSnapshot"}}),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
response["ids"].as_array().is_some_and(|ids| ids.len() >= 4),
|
||||
"three taken and one destroyed, audited: {response}"
|
||||
);
|
||||
|
||||
// Nobody without the permission, or in a tenant, reaches snapshots
|
||||
let (name, response) = call(ann, "inbuxa:StoreSnapshot/get", json!({"ids": null})).await;
|
||||
assert_eq!(name, "error", "a user: {response}");
|
||||
let tenant = admin
|
||||
.registry_create_object(Tenant {
|
||||
name: "Store snapshot tenant".to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
admin
|
||||
.registry_create_object(Domain {
|
||||
name: "storesnapshot.example.org".to_string(),
|
||||
member_tenant_id: Some(tenant),
|
||||
certificate_management: CertificateManagement::Manual,
|
||||
dns_management: DnsManagement::Manual,
|
||||
dkim_management: DkimManagement::Manual,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
let t_admin = admin
|
||||
.create_user_account(
|
||||
"[email protected]",
|
||||
"tenant-admin-secret-5521",
|
||||
"Tenant admin",
|
||||
&[],
|
||||
vec![],
|
||||
)
|
||||
.await;
|
||||
admin
|
||||
.registry_update_object(
|
||||
ObjectType::Account,
|
||||
t_admin.id(),
|
||||
json!({Property::Roles: UserRoles::Admin}),
|
||||
)
|
||||
.await;
|
||||
for (method, arguments) in [
|
||||
("inbuxa:StoreSnapshot/get", json!({"ids": null})),
|
||||
("inbuxa:StoreSnapshot/set", json!({"create": {"x": {}}})),
|
||||
] {
|
||||
let (name, response) = call(&t_admin, method, arguments).await;
|
||||
assert_eq!(name, "error", "{method} by a tenant: {response}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs these tests alone, with RocksDB:
|
||||
/// `STORE=RocksDb cargo test -p tests store_snapshot_tests -- --ignored`,
|
||||
/// and again with `BLOB_STORE=FileSystem` for a separate blob store.
|
||||
#[ignore]
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
pub async fn store_snapshot_tests() {
|
||||
let mut test = TestServerBuilder::new("store_snapshot_tests")
|
||||
.await
|
||||
.with_default_listeners()
|
||||
.await
|
||||
.build()
|
||||
.await;
|
||||
let admin = test.create_admin_account("[email protected]").await;
|
||||
test.insert_account(admin);
|
||||
let ann = test
|
||||
.create_user_account(
|
||||
"[email protected]",
|
||||
"[email protected]",
|
||||
"ann-secret-61734",
|
||||
&[],
|
||||
"Ann",
|
||||
)
|
||||
.await;
|
||||
test.insert_account(ann);
|
||||
self::test(&mut test).await;
|
||||
if test.is_reset() {
|
||||
test.temp_dir.delete();
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user