Changing the blob store: read through and copy what it left behind (#333) #335
No files matched your search
@@ -398,6 +398,11 @@ impl Default for DefaultPermissions {
|
||||
Permission::SysStoreSnapshot => {
|
||||
default.superuser.push(permission);
|
||||
}
|
||||
// inbuxa: blob-store-change spec, BM-9: a copy reads and
|
||||
// writes every account's blobs
|
||||
Permission::SysBlobStoreChange => {
|
||||
default.superuser.push(permission);
|
||||
}
|
||||
// inbuxa: AL-12: tenant administrators lock and delegate
|
||||
// within their tenant
|
||||
Permission::SysAccountLockGet
|
||||
|
||||
@@ -50,10 +50,15 @@ impl Storage {
|
||||
bp.data_store.share_marks(&memory);
|
||||
}
|
||||
|
||||
// inbuxa: blob-store-change spec, BM-2, BM-3: a blob store that
|
||||
// moved keeps reading what it left behind
|
||||
let blob = BlobStore::build(bp).await.unwrap_or_default();
|
||||
let blob = crate::manager::blob_moves::at_start(bp, blob).await;
|
||||
|
||||
Storage {
|
||||
registry: bp.registry.clone(),
|
||||
data: bp.data_store.clone(),
|
||||
blob: BlobStore::build(bp).await.unwrap_or_default(),
|
||||
blob,
|
||||
search,
|
||||
coordinator,
|
||||
memory,
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! inbuxa: blob-store-change spec (docs/spec/features/blob-store-change.md),
|
||||
//! at start-up: records where the blob store is and, when it moved, the
|
||||
//! change (BM-2), then opens the old store of every unresolved change so
|
||||
//! what the new store lacks is read from it (BM-3).
|
||||
|
||||
use inbuxa_features::blob_move::{self as model, Started};
|
||||
use registry::schema::{prelude::ObjectType, structs};
|
||||
use std::sync::Arc;
|
||||
use store::{
|
||||
BlobStore, Store,
|
||||
build::blob::blob_store_location,
|
||||
registry::bootstrap::Bootstrap,
|
||||
write::{
|
||||
blob_move::{MovingBlobStore, OldBlobStore},
|
||||
now,
|
||||
},
|
||||
};
|
||||
use types::id::Id;
|
||||
|
||||
/// The blob store to use: `blob` itself or, while changes are unresolved,
|
||||
/// `blob` reading through to their old stores.
|
||||
pub async fn at_start(bp: &mut Bootstrap, blob: BlobStore) -> BlobStore {
|
||||
let data = bp.data_store.clone();
|
||||
if matches!(data, Store::None) || matches!(blob, BlobStore::Store(Store::None)) {
|
||||
return blob;
|
||||
}
|
||||
let settings = bp.setting_infallible::<structs::BlobStore>().await;
|
||||
let location = blob_store_location(&settings);
|
||||
let warn = |bp: &mut Bootstrap, message: String| {
|
||||
bp.build_warning(ObjectType::BlobStore.singleton(), message);
|
||||
};
|
||||
|
||||
let mut recorded = None;
|
||||
match model::at_start(
|
||||
&data,
|
||||
location.clone(),
|
||||
&settings,
|
||||
bp.node_id() as u64,
|
||||
now(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Started::Same) => {}
|
||||
Ok(Started::Changed(id)) => {
|
||||
recorded = Some(id);
|
||||
if let Ok(Some(change)) = model::get(&data, id).await {
|
||||
warn(
|
||||
bp,
|
||||
format!(
|
||||
"The blob store moved from {} to {}. Blobs written before stay in {}; \
|
||||
they're read from it and kept in the new store as they're needed. \
|
||||
Copy the rest over with inbuxa:BlobStoreChange {}.",
|
||||
change.from,
|
||||
change.to,
|
||||
change.from,
|
||||
Id::from(id)
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn(
|
||||
bp,
|
||||
format!("Couldn't record where the blob store is: {err}"),
|
||||
);
|
||||
return blob;
|
||||
}
|
||||
}
|
||||
|
||||
let unresolved = match model::unresolved(&data).await {
|
||||
Ok(changes) => changes,
|
||||
Err(err) => {
|
||||
warn(bp, format!("Couldn't read the blob store changes: {err}"));
|
||||
return blob;
|
||||
}
|
||||
};
|
||||
let mut old = Vec::with_capacity(unresolved.len());
|
||||
for change in unresolved {
|
||||
// BM-4 resolves a change whose old store is this one; one recorded
|
||||
// before that rule could still say so
|
||||
if change.from == location {
|
||||
continue;
|
||||
}
|
||||
let Some(settings) = change.old_settings.clone() else {
|
||||
continue;
|
||||
};
|
||||
let mut warnings = Vec::new();
|
||||
match BlobStore::open(settings, &data, &mut warnings).await {
|
||||
Ok(store) => {
|
||||
old.push(OldBlobStore {
|
||||
change_id: change.id,
|
||||
store,
|
||||
});
|
||||
if recorded == Some(change.id) {
|
||||
continue;
|
||||
}
|
||||
warn(
|
||||
bp,
|
||||
format!(
|
||||
"The blob store changed from {} to {}, and blobs only {} has are \
|
||||
read from it. Copy them over or dismiss the change \
|
||||
(inbuxa:BlobStoreChange {}).",
|
||||
change.from,
|
||||
change.to,
|
||||
change.from,
|
||||
Id::from(change.id)
|
||||
),
|
||||
);
|
||||
}
|
||||
Err(err) => warn(
|
||||
bp,
|
||||
format!(
|
||||
"The blob store used before the change to {} ({}) couldn't be opened, \
|
||||
so blobs only it has can't be read: {err}",
|
||||
change.to, change.from
|
||||
),
|
||||
),
|
||||
}
|
||||
for warning in warnings {
|
||||
warn(bp, warning);
|
||||
}
|
||||
}
|
||||
|
||||
if old.is_empty() {
|
||||
blob
|
||||
} else {
|
||||
BlobStore::Moving(Arc::new(MovingBlobStore::new(blob, old)))
|
||||
}
|
||||
}
|
||||
@@ -35,8 +35,9 @@ use types::id::Id;
|
||||
/// to-do items (security to-do list spec), the deliverability check
|
||||
/// (deliverability spec), scheduled reports (scheduled-reports spec),
|
||||
/// inbuxa Relays (relay spec), running migrations (migrations spec,
|
||||
/// MG-10), checking the store's blobs (store-check spec, SV-9), and taking
|
||||
/// snapshots of the store (store-snapshot spec, SN-8).
|
||||
/// MG-10), checking the store's blobs (store-check spec, SV-9), taking
|
||||
/// snapshots of the store (store-snapshot spec, SN-8), and copying and
|
||||
/// dismissing changes of blob store (blob-store-change spec, BM-9).
|
||||
const ADMIN_GRANTS: &[Permission] = &[
|
||||
Permission::SysAiExplain,
|
||||
Permission::SysAuditGet,
|
||||
@@ -72,6 +73,7 @@ const ADMIN_GRANTS: &[Permission] = &[
|
||||
Permission::SysMigrationRun,
|
||||
Permission::SysStoreCheck,
|
||||
Permission::SysStoreSnapshot,
|
||||
Permission::SysBlobStoreChange,
|
||||
];
|
||||
|
||||
/// Granted to the server-level Compliance Officer role once it exists:
|
||||
|
||||
@@ -17,6 +17,7 @@ use utils::HttpLimitResponse;
|
||||
|
||||
pub mod application;
|
||||
pub mod backup;
|
||||
pub mod blob_moves; // inbuxa: blob-store-change spec, a blob store that moved
|
||||
pub mod boot;
|
||||
pub mod compliance_roles; // inbuxa: personal-data catalog, the compliance roles
|
||||
pub mod console;
|
||||
|
||||
@@ -0,0 +1,840 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Changing the blob store (docs/spec/features/blob-store-change.md): the
|
||||
//! `inbuxa:BlobStoreChange` objects. A change is recorded when a node
|
||||
//! starts with its blob store somewhere else than the store last said
|
||||
//! (BM-2), and keeps the old store's settings so it can be read through
|
||||
//! (BM-3) and copied from (BM-5). The copy itself runs in
|
||||
//! `services::inbuxa_blob_move`; this module keeps what a change is.
|
||||
//!
|
||||
//! Kept in the fork's subspace (`store::SUBSPACE_INBUXA`), never in the
|
||||
//! registry. Every key starts with `W` (where blobs are), then one byte for
|
||||
//! the kind:
|
||||
//!
|
||||
//! - `l`: where the blob store is now, with its settings ([`Location`]);
|
||||
//! - `c` + change id (u32): a change ([`BlobStoreChange`]), as JSON;
|
||||
//! - `p` + change id (u32) + number (u32): one problem its latest copy
|
||||
//! found ([`Problem`]);
|
||||
//! - `r`: the id of the change being copied now (u32), so only one copy
|
||||
//! runs at a time across the server (BM-5).
|
||||
//!
|
||||
//! Numbers are big-endian.
|
||||
|
||||
use registry::schema::structs;
|
||||
use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize, de::DeserializeOwned};
|
||||
use store::{
|
||||
Deserialize, IterateParams, SUBSPACE_INBUXA, Serialize, SerializeInfallible, Store, U32_LEN,
|
||||
ValueKey,
|
||||
write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue},
|
||||
};
|
||||
use trc::AddContext;
|
||||
|
||||
const FEATURE: u8 = b'W';
|
||||
const KIND_LOCATION: u8 = b'l';
|
||||
const KIND_CHANGE: u8 = b'c';
|
||||
const KIND_PROBLEM: u8 = b'p';
|
||||
const KIND_RUNNING: u8 = b'r';
|
||||
const WRITE_ATTEMPTS: usize = 5;
|
||||
|
||||
/// The changes kept; recording another forgets the oldest resolved one.
|
||||
pub const MAX_CHANGES: usize = 20;
|
||||
/// BM-6: the problems a copy keeps; the counts cover all of them.
|
||||
pub const MAX_PROBLEMS: u64 = 1_000;
|
||||
/// BM-5: the default limits, as a store check's.
|
||||
pub const DEFAULT_BLOBS_PER_SECOND: u64 = 1_000;
|
||||
pub const DEFAULT_BYTES_PER_SECOND: u64 = 20 * 1024 * 1024;
|
||||
/// BM-6: a copy whose node has been quiet this long has stopped.
|
||||
pub const STALE_AFTER: u64 = 5 * 60;
|
||||
const MAX_TEXT: usize = 2_000;
|
||||
|
||||
/// Where the blob store is (BM-1), with the settings that open it.
|
||||
#[derive(Debug, Clone, PartialEq, SerdeSerialize, SerdeDeserialize)]
|
||||
pub struct Location {
|
||||
pub location: String,
|
||||
pub settings: structs::BlobStore,
|
||||
}
|
||||
|
||||
/// BM-6, BM-4.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Status {
|
||||
/// Recorded; nothing copied yet.
|
||||
Pending,
|
||||
Copying,
|
||||
/// Everything is in the new store: resolved.
|
||||
Copied,
|
||||
/// A copy ended with blobs it couldn't copy.
|
||||
Incomplete,
|
||||
Stopped,
|
||||
Failed,
|
||||
Interrupted,
|
||||
/// The blob store went back to where this change moved it from:
|
||||
/// resolved (BM-4).
|
||||
Reverted,
|
||||
}
|
||||
|
||||
impl Status {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Status::Pending => "pending",
|
||||
Status::Copying => "copying",
|
||||
Status::Copied => "copied",
|
||||
Status::Incomplete => "incomplete",
|
||||
Status::Stopped => "stopped",
|
||||
Status::Failed => "failed",
|
||||
Status::Interrupted => "interrupted",
|
||||
Status::Reverted => "reverted",
|
||||
}
|
||||
}
|
||||
|
||||
/// BM-3: whether the old store is still read through.
|
||||
pub fn is_unresolved(&self) -> bool {
|
||||
!matches!(self, Status::Copied | Status::Reverted)
|
||||
}
|
||||
}
|
||||
|
||||
/// BM-5.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum ProblemKind {
|
||||
Missing,
|
||||
Damaged,
|
||||
Unreadable,
|
||||
}
|
||||
|
||||
impl ProblemKind {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
ProblemKind::Missing => "missing",
|
||||
ProblemKind::Damaged => "damaged",
|
||||
ProblemKind::Unreadable => "unreadable",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// BM-5: a blob a copy couldn't copy.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Problem {
|
||||
pub kind: ProblemKind,
|
||||
/// The blob's key: a blob id as hex, or a fixed name.
|
||||
pub blob_id: String,
|
||||
/// `email`, `file`, `queuedMessage`, `archivedItem`, `temporary` or
|
||||
/// `fixed`: the first thing found holding it.
|
||||
pub holder: String,
|
||||
#[serde(default)]
|
||||
pub account_id: Option<u32>,
|
||||
#[serde(default)]
|
||||
pub object_id: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub detail: String,
|
||||
}
|
||||
|
||||
/// BM-5: how far the latest copy got.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
pub struct Progress {
|
||||
/// Slices of the blob index done, of 256.
|
||||
pub slices: u32,
|
||||
pub blobs_checked: u64,
|
||||
pub copied: u64,
|
||||
pub already_there: u64,
|
||||
pub bytes_copied: u64,
|
||||
pub missing: u64,
|
||||
pub damaged: u64,
|
||||
pub unreadable: u64,
|
||||
}
|
||||
|
||||
impl Progress {
|
||||
pub fn problems(&self) -> u64 {
|
||||
self.missing + self.damaged + self.unreadable
|
||||
}
|
||||
|
||||
pub fn count(&mut self, kind: ProblemKind) {
|
||||
match kind {
|
||||
ProblemKind::Missing => self.missing += 1,
|
||||
ProblemKind::Damaged => self.damaged += 1,
|
||||
ProblemKind::Unreadable => self.unreadable += 1,
|
||||
}
|
||||
}
|
||||
|
||||
/// 0 to 1, by slices of the index.
|
||||
pub fn share(&self) -> f64 {
|
||||
self.slices.min(256) as f64 / 256.0
|
||||
}
|
||||
}
|
||||
|
||||
/// The latest copy of a change (BM-5, BM-6).
|
||||
#[derive(Debug, Clone, PartialEq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Copy {
|
||||
pub started_at: u64,
|
||||
pub started_by: String,
|
||||
pub node_id: u64,
|
||||
pub blobs_per_second: u64,
|
||||
pub bytes_per_second: u64,
|
||||
/// When its node last saved it.
|
||||
pub heartbeat: u64,
|
||||
#[serde(default)]
|
||||
pub finished_at: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub stop_requested: bool,
|
||||
#[serde(default)]
|
||||
pub progress: Progress,
|
||||
#[serde(default)]
|
||||
pub problems_kept: u64,
|
||||
#[serde(default)]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// An `inbuxa:BlobStoreChange`.
|
||||
#[derive(Debug, Clone, PartialEq, SerdeSerialize, SerdeDeserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BlobStoreChange {
|
||||
pub id: u32,
|
||||
pub from: String,
|
||||
pub to: String,
|
||||
pub changed_at: u64,
|
||||
pub node_id: u64,
|
||||
pub status: Status,
|
||||
/// The old store's settings, to read it (BM-3) and copy from it (BM-5).
|
||||
/// Dropped once the change is resolved; never shown (BM-9).
|
||||
#[serde(default)]
|
||||
pub old_settings: Option<structs::BlobStore>,
|
||||
#[serde(default)]
|
||||
pub copy: Option<Copy>,
|
||||
}
|
||||
|
||||
impl BlobStoreChange {
|
||||
/// As it reads now: a copy whose node went quiet has stopped with it
|
||||
/// (BM-6), even before anything marked it so.
|
||||
pub fn status_at(&self, now: u64) -> Status {
|
||||
match &self.copy {
|
||||
Some(copy) if self.status == Status::Copying && copy.heartbeat + STALE_AFTER < now => {
|
||||
Status::Interrupted
|
||||
}
|
||||
_ => self.status,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_error(&mut self, error: &str) {
|
||||
if let Some(copy) = &mut self.copy {
|
||||
copy.error = Some(error.trim().chars().take(MAX_TEXT).collect());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Json<T>(T);
|
||||
|
||||
impl<T: SerdeSerialize> Serialize for Json<&T> {
|
||||
fn serialize(&self) -> trc::Result<Vec<u8>> {
|
||||
serde_json::to_vec(self.0).map_err(|err| {
|
||||
trc::StoreEvent::UnexpectedError
|
||||
.into_err()
|
||||
.details("Could not serialize a blob store change")
|
||||
.reason(err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: DeserializeOwned + Sync + Send> Deserialize for Json<T> {
|
||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
||||
serde_json::from_slice(bytes).map(Json).map_err(|err| {
|
||||
trc::StoreEvent::DataCorruption
|
||||
.into_err()
|
||||
.details("Invalid blob store change")
|
||||
.reason(err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn any(key: Vec<u8>) -> ValueClass {
|
||||
ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key,
|
||||
})
|
||||
}
|
||||
|
||||
fn location_class() -> ValueClass {
|
||||
any(vec![FEATURE, KIND_LOCATION])
|
||||
}
|
||||
|
||||
fn class(id: u32) -> ValueClass {
|
||||
let mut key = Vec::with_capacity(2 + U32_LEN);
|
||||
key.extend_from_slice(&[FEATURE, KIND_CHANGE]);
|
||||
key.extend_from_slice(&id.to_be_bytes());
|
||||
any(key)
|
||||
}
|
||||
|
||||
fn problem_class(id: u32, number: u32) -> ValueClass {
|
||||
let mut key = Vec::with_capacity(2 + U32_LEN * 2);
|
||||
key.extend_from_slice(&[FEATURE, KIND_PROBLEM]);
|
||||
key.extend_from_slice(&id.to_be_bytes());
|
||||
key.extend_from_slice(&number.to_be_bytes());
|
||||
any(key)
|
||||
}
|
||||
|
||||
fn running_class() -> ValueClass {
|
||||
any(vec![FEATURE, KIND_RUNNING])
|
||||
}
|
||||
|
||||
fn is_assert_failure(err: &trc::Error) -> bool {
|
||||
matches!(
|
||||
err.as_ref(),
|
||||
trc::EventType::Store(trc::StoreEvent::AssertValueFailed)
|
||||
)
|
||||
}
|
||||
|
||||
/// A value as stored, with a hash of its bytes for a compare-and-set.
|
||||
struct Versioned<T>(T, u64);
|
||||
|
||||
impl<T: DeserializeOwned + Sync + Send> Deserialize for Versioned<T> {
|
||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
||||
let Json(value) = Json::<T>::deserialize(bytes)?;
|
||||
Ok(Versioned(value, xxhash_rust::xxh3::xxh3_64(bytes)))
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get(data: &Store, id: u32) -> trc::Result<Option<BlobStoreChange>> {
|
||||
Ok(data
|
||||
.get_value::<Json<BlobStoreChange>>(ValueKey::from(class(id)))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.map(|Json(change)| change))
|
||||
}
|
||||
|
||||
/// Every change, oldest first.
|
||||
pub async fn all(data: &Store) -> trc::Result<Vec<BlobStoreChange>> {
|
||||
let mut changes = Vec::new();
|
||||
data.iterate(
|
||||
IterateParams::new(ValueKey::from(class(0)), ValueKey::from(class(u32::MAX))),
|
||||
|_, value| {
|
||||
if let Ok(Json(change)) = Json::<BlobStoreChange>::deserialize(value) {
|
||||
changes.push(change);
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
changes.sort_by_key(|c| c.id);
|
||||
Ok(changes)
|
||||
}
|
||||
|
||||
/// BM-3, BM-10: the changes still read through, newest first.
|
||||
pub async fn unresolved(data: &Store) -> trc::Result<Vec<BlobStoreChange>> {
|
||||
let mut changes = all(data).await?;
|
||||
changes.retain(|c| c.status.is_unresolved() && c.old_settings.is_some());
|
||||
changes.reverse();
|
||||
Ok(changes)
|
||||
}
|
||||
|
||||
/// Where the store last said the blob store was.
|
||||
pub async fn location(data: &Store) -> trc::Result<Option<Location>> {
|
||||
Ok(data
|
||||
.get_value::<Json<Location>>(ValueKey::from(location_class()))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.map(|Json(location)| location))
|
||||
}
|
||||
|
||||
/// What [`at_start`] found.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Started {
|
||||
/// The blob store is where it was, or this is the first start that
|
||||
/// records it.
|
||||
Same,
|
||||
/// This node recorded a change, the one with this id (BM-2).
|
||||
Changed(u32),
|
||||
}
|
||||
|
||||
/// BM-2, BM-4: at start-up, records where the blob store is and, when it
|
||||
/// moved, a change from where it was. The first node to start with the new
|
||||
/// setting records it: the location is written only if it's still what
|
||||
/// this node read, so the others find the change already there.
|
||||
pub async fn at_start(
|
||||
data: &Store,
|
||||
location: String,
|
||||
settings: &structs::BlobStore,
|
||||
node_id: u64,
|
||||
now: u64,
|
||||
) -> trc::Result<Started> {
|
||||
let mut attempt = 0;
|
||||
loop {
|
||||
attempt += 1;
|
||||
let last = data
|
||||
.get_value::<Versioned<Location>>(ValueKey::from(location_class()))
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
let here = Location {
|
||||
location: location.clone(),
|
||||
settings: settings.clone(),
|
||||
};
|
||||
|
||||
let mut batch = BatchBuilder::new();
|
||||
let mut started = Started::Same;
|
||||
let mut forgotten = Vec::new();
|
||||
match &last {
|
||||
None => {
|
||||
batch.assert_value(location_class(), AssertValue::None);
|
||||
}
|
||||
Some(Versioned(last, version)) if last.location == location => {
|
||||
if &last.settings == settings {
|
||||
return Ok(Started::Same);
|
||||
}
|
||||
// The same place with other credentials or tuning: kept
|
||||
// current, for when it's the old store of a later change
|
||||
batch.assert_value(location_class(), AssertValue::Hash(*version));
|
||||
}
|
||||
Some(Versioned(last, version)) => {
|
||||
batch.assert_value(location_class(), AssertValue::Hash(*version));
|
||||
let changes = all(data).await?;
|
||||
let id = changes.iter().map(|c| c.id).max().unwrap_or(0) + 1;
|
||||
|
||||
// BM-4: back where an unresolved change moved away from
|
||||
for earlier in changes.iter().filter(|c| {
|
||||
c.status.is_unresolved() && c.status != Status::Copying && c.from == location
|
||||
}) {
|
||||
let reverted = BlobStoreChange {
|
||||
status: Status::Reverted,
|
||||
old_settings: None,
|
||||
..earlier.clone()
|
||||
};
|
||||
batch.set(class(earlier.id), Json(&reverted).serialize()?);
|
||||
}
|
||||
|
||||
let change = BlobStoreChange {
|
||||
id,
|
||||
from: last.location.clone(),
|
||||
to: location.clone(),
|
||||
changed_at: now,
|
||||
node_id,
|
||||
status: Status::Pending,
|
||||
old_settings: Some(last.settings.clone()),
|
||||
copy: None,
|
||||
};
|
||||
batch.assert_value(class(id), AssertValue::None);
|
||||
batch.set(class(id), Json(&change).serialize()?);
|
||||
started = Started::Changed(id);
|
||||
|
||||
// The oldest resolved changes past MAX_CHANGES are forgotten
|
||||
let over = (changes.len() + 1).saturating_sub(MAX_CHANGES);
|
||||
for old in changes
|
||||
.iter()
|
||||
.filter(|c| !c.status.is_unresolved())
|
||||
.take(over)
|
||||
{
|
||||
batch.clear(class(old.id));
|
||||
forgotten.push(old.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
batch.set(location_class(), Json(&here).serialize()?);
|
||||
|
||||
match data.write(batch.build_all()).await {
|
||||
Ok(_) => {
|
||||
for id in forgotten {
|
||||
remove_problems(data, id).await?;
|
||||
}
|
||||
return Ok(started);
|
||||
}
|
||||
// Another node wrote first: read what it recorded
|
||||
Err(err) if attempt < WRITE_ATTEMPTS && is_assert_failure(&err) => {}
|
||||
Err(err) => return Err(err.caused_by(trc::location!())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The id of the change being copied now, if any.
|
||||
pub async fn running(data: &Store) -> trc::Result<Option<u32>> {
|
||||
data.get_value::<u32>(ValueKey::from(running_class()))
|
||||
.await
|
||||
.caused_by(trc::location!())
|
||||
}
|
||||
|
||||
/// BM-5: why a copy can't start.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Refusal {
|
||||
/// No such change.
|
||||
NotFound,
|
||||
/// This change's copy runs now.
|
||||
Running(u32),
|
||||
/// The change is resolved, or has no old store to copy from.
|
||||
Resolved,
|
||||
}
|
||||
|
||||
/// BM-5: marks a change as being copied by `copy` and makes it the one
|
||||
/// that runs. Refused while another copy runs: one whose node went quiet
|
||||
/// is marked interrupted first.
|
||||
pub async fn start_copy(
|
||||
data: &Store,
|
||||
id: u32,
|
||||
copy: Copy,
|
||||
now: u64,
|
||||
) -> trc::Result<Result<BlobStoreChange, Refusal>> {
|
||||
let mut attempt = 0;
|
||||
loop {
|
||||
attempt += 1;
|
||||
if let Some(current) = running(data).await? {
|
||||
match get(data, current).await? {
|
||||
Some(c) if c.status_at(now) == Status::Copying => {
|
||||
return Ok(Err(Refusal::Running(current)));
|
||||
}
|
||||
Some(c) => {
|
||||
if c.status == Status::Copying {
|
||||
update(data, current, |c| {
|
||||
if c.status != Status::Copying {
|
||||
return false;
|
||||
}
|
||||
c.status = Status::Interrupted;
|
||||
if let Some(copy) = &mut c.copy {
|
||||
copy.finished_at = Some(now);
|
||||
}
|
||||
true
|
||||
})
|
||||
.await?;
|
||||
}
|
||||
release(data, current).await?;
|
||||
}
|
||||
None => release(data, current).await?,
|
||||
}
|
||||
}
|
||||
|
||||
let Some(Versioned(change, version)) = data
|
||||
.get_value::<Versioned<BlobStoreChange>>(ValueKey::from(class(id)))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
else {
|
||||
return Ok(Err(Refusal::NotFound));
|
||||
};
|
||||
if !change.status.is_unresolved() || change.old_settings.is_none() {
|
||||
return Ok(Err(Refusal::Resolved));
|
||||
}
|
||||
let change = BlobStoreChange {
|
||||
status: Status::Copying,
|
||||
copy: Some(copy.clone()),
|
||||
..change
|
||||
};
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.assert_value(class(id), AssertValue::Hash(version));
|
||||
batch.assert_value(running_class(), AssertValue::None);
|
||||
batch.set(class(id), Json(&change).serialize()?);
|
||||
batch.set(running_class(), SerializeInfallible::serialize(&id));
|
||||
match data.write(batch.build_all()).await {
|
||||
Ok(_) => {
|
||||
remove_problems(data, id).await?;
|
||||
return Ok(Ok(change));
|
||||
}
|
||||
Err(err) if attempt < WRITE_ATTEMPTS && is_assert_failure(&err) => {}
|
||||
Err(err) => return Err(err.caused_by(trc::location!())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Lets another copy run, if `id` is the one that runs now.
|
||||
pub async fn release(data: &Store, id: u32) -> trc::Result<()> {
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.assert_value(running_class(), AssertValue::U32(id));
|
||||
batch.clear(running_class());
|
||||
match data.write(batch.build_all()).await {
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) if is_assert_failure(&err) => Ok(()),
|
||||
Err(err) => Err(err.caused_by(trc::location!())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads a change, changes it with `change`, and writes it back, again if
|
||||
/// something else wrote it in between. `change` returns false to leave it
|
||||
/// as it is. None if there's no such change.
|
||||
pub async fn update<F>(data: &Store, id: u32, mut change: F) -> trc::Result<Option<BlobStoreChange>>
|
||||
where
|
||||
F: FnMut(&mut BlobStoreChange) -> bool,
|
||||
{
|
||||
loop {
|
||||
let Some(Versioned(mut stored, version)) = data
|
||||
.get_value::<Versioned<BlobStoreChange>>(ValueKey::from(class(id)))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !change(&mut stored) {
|
||||
return Ok(Some(stored));
|
||||
}
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.assert_value(class(id), AssertValue::Hash(version));
|
||||
batch.set(class(id), Json(&stored).serialize()?);
|
||||
match data.write(batch.build_all()).await {
|
||||
Ok(_) => return Ok(Some(stored)),
|
||||
Err(err) if is_assert_failure(&err) => {}
|
||||
Err(err) => return Err(err.caused_by(trc::location!())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Keeps problems `first..` of a change's copy (BM-6): the caller numbers
|
||||
/// them and stops at [`MAX_PROBLEMS`].
|
||||
pub async fn add_problems(
|
||||
data: &Store,
|
||||
id: u32,
|
||||
first: u64,
|
||||
problems: &[Problem],
|
||||
) -> trc::Result<()> {
|
||||
let mut batch = BatchBuilder::new();
|
||||
for (n, problem) in problems.iter().enumerate() {
|
||||
let number = u32::try_from(first + n as u64).unwrap_or(u32::MAX);
|
||||
batch.set(problem_class(id, number), Json(problem).serialize()?);
|
||||
if batch.is_large_batch() {
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
batch = BatchBuilder::new();
|
||||
}
|
||||
}
|
||||
if !batch.is_empty() {
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The kept problems of a change's latest copy, in the order found.
|
||||
pub async fn problems(data: &Store, id: u32) -> trc::Result<Vec<Problem>> {
|
||||
let mut problems = Vec::new();
|
||||
data.iterate(
|
||||
IterateParams::new(
|
||||
ValueKey::from(problem_class(id, 0)),
|
||||
ValueKey::from(problem_class(id, u32::MAX)),
|
||||
),
|
||||
|_, value| {
|
||||
if let Ok(Json(problem)) = Json::<Problem>::deserialize(value) {
|
||||
problems.push(problem);
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
Ok(problems)
|
||||
}
|
||||
|
||||
async fn remove_problems(data: &Store, id: u32) -> trc::Result<()> {
|
||||
data.delete_range(
|
||||
ValueKey::from(problem_class(id, 0)),
|
||||
ValueKey::from(problem_class(id, u32::MAX)),
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())
|
||||
}
|
||||
|
||||
/// BM-7: forgets a change, the old store's settings and its problems.
|
||||
pub async fn remove(data: &Store, id: u32) -> trc::Result<()> {
|
||||
remove_problems(data, id).await?;
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.clear(class(id));
|
||||
data.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn fs(path: &str) -> structs::BlobStore {
|
||||
structs::BlobStore::FileSystem(structs::FileSystemStore {
|
||||
path: path.into(),
|
||||
depth: 2,
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_move_and_a_move_back() {
|
||||
let data = store::backend::ephemeral::EphemeralStore::open();
|
||||
let a = structs::BlobStore::Default;
|
||||
let b = fs("/srv/blobs");
|
||||
let (here_a, here_b) = ("the data store".to_string(), "dir".to_string());
|
||||
|
||||
// First start: only where it is
|
||||
assert_eq!(
|
||||
at_start(&data, here_a.clone(), &a, 1, 10).await.unwrap(),
|
||||
Started::Same
|
||||
);
|
||||
assert_eq!(
|
||||
at_start(&data, here_a.clone(), &a, 2, 11).await.unwrap(),
|
||||
Started::Same
|
||||
);
|
||||
assert!(all(&data).await.unwrap().is_empty());
|
||||
|
||||
// A to B: one change, recorded once whichever node starts first
|
||||
assert_eq!(
|
||||
at_start(&data, here_b.clone(), &b, 1, 20).await.unwrap(),
|
||||
Started::Changed(1)
|
||||
);
|
||||
assert_eq!(
|
||||
at_start(&data, here_b.clone(), &b, 2, 21).await.unwrap(),
|
||||
Started::Same
|
||||
);
|
||||
let first = get(&data, 1).await.unwrap().unwrap();
|
||||
assert_eq!(
|
||||
(first.from.as_str(), first.to.as_str()),
|
||||
("the data store", "dir")
|
||||
);
|
||||
assert_eq!(first.old_settings, Some(a.clone()));
|
||||
assert_eq!(first.status, Status::Pending);
|
||||
|
||||
// Back to A: the first change is reverted, the second is the one
|
||||
// still read through
|
||||
assert_eq!(
|
||||
at_start(&data, here_a.clone(), &a, 1, 30).await.unwrap(),
|
||||
Started::Changed(2)
|
||||
);
|
||||
let first = get(&data, 1).await.unwrap().unwrap();
|
||||
assert_eq!(first.status, Status::Reverted);
|
||||
assert_eq!(first.old_settings, None);
|
||||
let open = unresolved(&data).await.unwrap();
|
||||
assert_eq!(open.len(), 1);
|
||||
assert_eq!(open[0].id, 2);
|
||||
assert_eq!(open[0].old_settings, Some(b.clone()));
|
||||
|
||||
// A copy: one at a time, never of a resolved change
|
||||
let copy = Copy {
|
||||
started_at: 40,
|
||||
started_by: "admin".into(),
|
||||
node_id: 1,
|
||||
blobs_per_second: 0,
|
||||
bytes_per_second: 0,
|
||||
heartbeat: 40,
|
||||
finished_at: None,
|
||||
stop_requested: false,
|
||||
progress: Progress::default(),
|
||||
problems_kept: 0,
|
||||
error: None,
|
||||
};
|
||||
assert_eq!(
|
||||
start_copy(&data, 1, copy.clone(), 40).await.unwrap(),
|
||||
Err(Refusal::Resolved)
|
||||
);
|
||||
assert_eq!(
|
||||
start_copy(&data, 9, copy.clone(), 40).await.unwrap(),
|
||||
Err(Refusal::NotFound)
|
||||
);
|
||||
let started = start_copy(&data, 2, copy.clone(), 40)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(started.status, Status::Copying);
|
||||
assert_eq!(
|
||||
start_copy(&data, 2, copy.clone(), 41).await.unwrap(),
|
||||
Err(Refusal::Running(2))
|
||||
);
|
||||
// Its node went quiet: interrupted, and another may start
|
||||
let again = start_copy(&data, 2, copy.clone(), 41 + STALE_AFTER)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(again.status, Status::Copying);
|
||||
|
||||
// Finished: resolved, nothing read through any more
|
||||
update(&data, 2, |c| {
|
||||
c.status = Status::Copied;
|
||||
c.old_settings = None;
|
||||
true
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
release(&data, 2).await.unwrap();
|
||||
assert!(unresolved(&data).await.unwrap().is_empty());
|
||||
assert_eq!(running(&data).await.unwrap(), None);
|
||||
|
||||
// Where it was, again: nothing new
|
||||
assert_eq!(
|
||||
at_start(&data, here_a.clone(), &a, 1, 50).await.unwrap(),
|
||||
Started::Same
|
||||
);
|
||||
remove(&data, 2).await.unwrap();
|
||||
assert!(get(&data, 2).await.unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keys_stay_in_their_own_letter() {
|
||||
for class in [
|
||||
location_class(),
|
||||
class(1),
|
||||
problem_class(1, 0),
|
||||
running_class(),
|
||||
] {
|
||||
let ValueClass::Any(any) = class else {
|
||||
unreachable!()
|
||||
};
|
||||
assert_eq!(any.key[0], b'W');
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolved_is_copied_or_reverted() {
|
||||
for status in [
|
||||
Status::Pending,
|
||||
Status::Copying,
|
||||
Status::Incomplete,
|
||||
Status::Stopped,
|
||||
Status::Failed,
|
||||
Status::Interrupted,
|
||||
] {
|
||||
assert!(status.is_unresolved(), "{status:?}");
|
||||
}
|
||||
assert!(!Status::Copied.is_unresolved());
|
||||
assert!(!Status::Reverted.is_unresolved());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_quiet_copy_reads_as_interrupted() {
|
||||
let change = BlobStoreChange {
|
||||
id: 1,
|
||||
from: "the data store".into(),
|
||||
to: "the directory /srv/blobs, depth 2".into(),
|
||||
changed_at: 1_000,
|
||||
node_id: 1,
|
||||
status: Status::Copying,
|
||||
old_settings: Some(structs::BlobStore::Default),
|
||||
copy: Some(Copy {
|
||||
started_at: 1_000,
|
||||
started_by: "admin".into(),
|
||||
node_id: 1,
|
||||
blobs_per_second: 0,
|
||||
bytes_per_second: 0,
|
||||
heartbeat: 1_000,
|
||||
finished_at: None,
|
||||
stop_requested: false,
|
||||
progress: Progress::default(),
|
||||
problems_kept: 0,
|
||||
error: None,
|
||||
}),
|
||||
};
|
||||
assert_eq!(change.status_at(1_000 + STALE_AFTER), Status::Copying);
|
||||
assert_eq!(change.status_at(1_001 + STALE_AFTER), Status::Interrupted);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_change_keeps_the_old_settings_as_json() {
|
||||
let change = BlobStoreChange {
|
||||
id: 2,
|
||||
from: "the directory /srv/blobs, depth 2".into(),
|
||||
to: "the data store".into(),
|
||||
changed_at: 5,
|
||||
node_id: 1,
|
||||
status: Status::Pending,
|
||||
old_settings: Some(fs("/srv/blobs")),
|
||||
copy: None,
|
||||
};
|
||||
let bytes = Json(&change).serialize().unwrap();
|
||||
let Json(back) = Json::<BlobStoreChange>::deserialize(&bytes).unwrap();
|
||||
assert_eq!(back, change);
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@
|
||||
|
||||
pub mod ai;
|
||||
pub mod audit;
|
||||
pub mod blob_move; // inbuxa: changing the blob store (not a rebuild)
|
||||
pub mod branding;
|
||||
pub mod deliverability; // inbuxa: the deliverability check (not a rebuild)
|
||||
pub mod hold;
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:BlobStoreChange/get` and `/set` under `urn:inbuxa:jmap`: a change
|
||||
//! of blob store the server noticed at start-up (blob-store-change spec,
|
||||
//! BM-2). The server records them; an update with `action: "copy"` copies
|
||||
//! what the new store lacks from the old one, `action: "stop"` stops that,
|
||||
//! and destroying one dismisses it (BM-5 to BM-7).
|
||||
|
||||
use crate::object::{AnyId, JmapObject, JmapObjectId};
|
||||
use jmap_tools::{Element, Key, Property};
|
||||
use std::{borrow::Cow, str::FromStr};
|
||||
use types::id::Id;
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct BlobStoreChange;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum BlobStoreChangeProperty {
|
||||
Id,
|
||||
From,
|
||||
To,
|
||||
ChangedAt,
|
||||
Node,
|
||||
Status,
|
||||
CopyStartedAt,
|
||||
CopyFinishedAt,
|
||||
CopyStartedBy,
|
||||
CopyNode,
|
||||
BlobsPerSecond,
|
||||
BytesPerSecond,
|
||||
Progress,
|
||||
BlobsChecked,
|
||||
Copied,
|
||||
AlreadyThere,
|
||||
BytesCopied,
|
||||
Missing,
|
||||
Damaged,
|
||||
Unreadable,
|
||||
Problems,
|
||||
Error,
|
||||
Action,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum BlobStoreChangeValue {
|
||||
Id(Id),
|
||||
}
|
||||
|
||||
impl Property for BlobStoreChangeProperty {
|
||||
fn try_parse(parent: Option<&Key<'_, Self>>, value: &str) -> Option<Self> {
|
||||
match parent {
|
||||
None => BlobStoreChangeProperty::parse(value),
|
||||
Some(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_cow(&self) -> Cow<'static, str> {
|
||||
match self {
|
||||
BlobStoreChangeProperty::Id => "id",
|
||||
BlobStoreChangeProperty::From => "from",
|
||||
BlobStoreChangeProperty::To => "to",
|
||||
BlobStoreChangeProperty::ChangedAt => "changedAt",
|
||||
BlobStoreChangeProperty::Node => "node",
|
||||
BlobStoreChangeProperty::Status => "status",
|
||||
BlobStoreChangeProperty::CopyStartedAt => "copyStartedAt",
|
||||
BlobStoreChangeProperty::CopyFinishedAt => "copyFinishedAt",
|
||||
BlobStoreChangeProperty::CopyStartedBy => "copyStartedBy",
|
||||
BlobStoreChangeProperty::CopyNode => "copyNode",
|
||||
BlobStoreChangeProperty::BlobsPerSecond => "blobsPerSecond",
|
||||
BlobStoreChangeProperty::BytesPerSecond => "bytesPerSecond",
|
||||
BlobStoreChangeProperty::Progress => "progress",
|
||||
BlobStoreChangeProperty::BlobsChecked => "blobsChecked",
|
||||
BlobStoreChangeProperty::Copied => "copied",
|
||||
BlobStoreChangeProperty::AlreadyThere => "alreadyThere",
|
||||
BlobStoreChangeProperty::BytesCopied => "bytesCopied",
|
||||
BlobStoreChangeProperty::Missing => "missing",
|
||||
BlobStoreChangeProperty::Damaged => "damaged",
|
||||
BlobStoreChangeProperty::Unreadable => "unreadable",
|
||||
BlobStoreChangeProperty::Problems => "problems",
|
||||
BlobStoreChangeProperty::Error => "error",
|
||||
BlobStoreChangeProperty::Action => "action",
|
||||
}
|
||||
.into()
|
||||
}
|
||||
}
|
||||
|
||||
impl BlobStoreChangeProperty {
|
||||
fn parse(value: &str) -> Option<Self> {
|
||||
hashify::tiny_map!(value.as_bytes(),
|
||||
b"id" => BlobStoreChangeProperty::Id,
|
||||
b"from" => BlobStoreChangeProperty::From,
|
||||
b"to" => BlobStoreChangeProperty::To,
|
||||
b"changedAt" => BlobStoreChangeProperty::ChangedAt,
|
||||
b"node" => BlobStoreChangeProperty::Node,
|
||||
b"status" => BlobStoreChangeProperty::Status,
|
||||
b"copyStartedAt" => BlobStoreChangeProperty::CopyStartedAt,
|
||||
b"copyFinishedAt" => BlobStoreChangeProperty::CopyFinishedAt,
|
||||
b"copyStartedBy" => BlobStoreChangeProperty::CopyStartedBy,
|
||||
b"copyNode" => BlobStoreChangeProperty::CopyNode,
|
||||
b"blobsPerSecond" => BlobStoreChangeProperty::BlobsPerSecond,
|
||||
b"bytesPerSecond" => BlobStoreChangeProperty::BytesPerSecond,
|
||||
b"progress" => BlobStoreChangeProperty::Progress,
|
||||
b"blobsChecked" => BlobStoreChangeProperty::BlobsChecked,
|
||||
b"copied" => BlobStoreChangeProperty::Copied,
|
||||
b"alreadyThere" => BlobStoreChangeProperty::AlreadyThere,
|
||||
b"bytesCopied" => BlobStoreChangeProperty::BytesCopied,
|
||||
b"missing" => BlobStoreChangeProperty::Missing,
|
||||
b"damaged" => BlobStoreChangeProperty::Damaged,
|
||||
b"unreadable" => BlobStoreChangeProperty::Unreadable,
|
||||
b"problems" => BlobStoreChangeProperty::Problems,
|
||||
b"error" => BlobStoreChangeProperty::Error,
|
||||
b"action" => BlobStoreChangeProperty::Action,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for BlobStoreChangeProperty {
|
||||
type Err = ();
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
BlobStoreChangeProperty::parse(s).ok_or(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Element for BlobStoreChangeValue {
|
||||
type Property = BlobStoreChangeProperty;
|
||||
|
||||
fn try_parse<P>(key: &Key<'_, Self::Property>, value: &str) -> Option<Self> {
|
||||
match key {
|
||||
Key::Property(BlobStoreChangeProperty::Id) => {
|
||||
Id::from_str(value).ok().map(BlobStoreChangeValue::Id)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_cow(&self) -> Cow<'static, str> {
|
||||
match self {
|
||||
BlobStoreChangeValue::Id(id) => id.to_string().into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObject for BlobStoreChange {
|
||||
type Property = BlobStoreChangeProperty;
|
||||
|
||||
type Element = BlobStoreChangeValue;
|
||||
|
||||
type Id = Id;
|
||||
|
||||
type Filter = ();
|
||||
|
||||
type Comparator = ();
|
||||
|
||||
type GetArguments = ();
|
||||
|
||||
type SetArguments<'de> = ();
|
||||
|
||||
type QueryArguments = ();
|
||||
|
||||
type CopyArguments = ();
|
||||
|
||||
type ParseArguments = ();
|
||||
|
||||
const ID_PROPERTY: Self::Property = BlobStoreChangeProperty::Id;
|
||||
}
|
||||
|
||||
impl From<Id> for BlobStoreChangeValue {
|
||||
fn from(id: Id) -> Self {
|
||||
BlobStoreChangeValue::Id(id)
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObjectId for BlobStoreChangeValue {
|
||||
fn as_id(&self) -> Option<Id> {
|
||||
match self {
|
||||
BlobStoreChangeValue::Id(id) => Some(*id),
|
||||
}
|
||||
}
|
||||
|
||||
fn as_any_id(&self) -> Option<AnyId> {
|
||||
match self {
|
||||
BlobStoreChangeValue::Id(id) => Some(AnyId::Id(*id)),
|
||||
}
|
||||
}
|
||||
|
||||
fn as_id_ref(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
|
||||
fn try_set_id(&mut self, new_id: AnyId) -> bool {
|
||||
if let AnyId::Id(id) = new_id {
|
||||
*self = BlobStoreChangeValue::Id(id);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl JmapObjectId for BlobStoreChangeProperty {
|
||||
fn as_id(&self) -> Option<Id> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_any_id(&self) -> Option<AnyId> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_id_ref(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
|
||||
fn try_set_id(&mut self, _: AnyId) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,7 @@ pub mod identity;
|
||||
pub mod inbuxa_account_lock; // inbuxa: account lock with delegation
|
||||
pub mod inbuxa_ai_limits; // inbuxa: AI spam classification
|
||||
pub mod inbuxa_audit; // inbuxa: the audit log
|
||||
pub mod inbuxa_blob_store_change; // inbuxa: a change of blob store
|
||||
pub mod inbuxa_data_inventory; // inbuxa: personal-data catalog
|
||||
pub mod inbuxa_deleted_account; // inbuxa: undelete
|
||||
pub mod inbuxa_deliverability_report; // inbuxa: the deliverability check
|
||||
|
||||
@@ -97,6 +97,9 @@ impl Response<'_> {
|
||||
GetResponseMethod::StoreSnapshot(response) => {
|
||||
response.eval_jptr(path, &mut results)
|
||||
}
|
||||
GetResponseMethod::BlobStoreChange(response) => {
|
||||
response.eval_jptr(path, &mut results)
|
||||
}
|
||||
GetResponseMethod::Relay(response) => {
|
||||
response.eval_jptr(path, &mut results)
|
||||
}
|
||||
|
||||
@@ -58,6 +58,7 @@ impl Response<'_> {
|
||||
GetRequestMethod::Migration(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::StoreCheck(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::StoreSnapshot(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::BlobStoreChange(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::Relay(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::RelayMessage(request) => request.resolve_references(self)?,
|
||||
GetRequestMethod::SecurityAcceptance(request) => {
|
||||
@@ -163,6 +164,9 @@ impl Response<'_> {
|
||||
SetRequestMethod::StoreSnapshot(request) => {
|
||||
request.resolve_references(self, 1, false)?
|
||||
}
|
||||
SetRequestMethod::BlobStoreChange(request) => {
|
||||
request.resolve_references(self, 1, false)?
|
||||
}
|
||||
SetRequestMethod::Relay(request) => request.resolve_references(self, 1, false)?,
|
||||
SetRequestMethod::RelayMessage(request) => {
|
||||
request.resolve_references(self, 1, false)?
|
||||
|
||||
@@ -74,6 +74,8 @@ pub enum MethodObject {
|
||||
StoreCheck,
|
||||
// inbuxa: a snapshot of the store
|
||||
StoreSnapshot,
|
||||
// inbuxa: a change of blob store
|
||||
BlobStoreChange,
|
||||
Relay,
|
||||
RelayMessage,
|
||||
// inbuxa: accepted security to-do items
|
||||
@@ -135,6 +137,7 @@ impl MethodObject {
|
||||
| MethodObject::Migration
|
||||
| MethodObject::StoreCheck
|
||||
| MethodObject::StoreSnapshot
|
||||
| MethodObject::BlobStoreChange
|
||||
| MethodObject::Relay
|
||||
| MethodObject::RelayMessage
|
||||
| MethodObject::SecurityAcceptance
|
||||
@@ -355,6 +358,8 @@ impl MethodName {
|
||||
(MethodFunction::Set, MethodObject::StoreCheck) => "inbuxa:StoreCheck/set",
|
||||
(MethodFunction::Get, MethodObject::StoreSnapshot) => "inbuxa:StoreSnapshot/get",
|
||||
(MethodFunction::Set, MethodObject::StoreSnapshot) => "inbuxa:StoreSnapshot/set",
|
||||
(MethodFunction::Get, MethodObject::BlobStoreChange) => "inbuxa:BlobStoreChange/get",
|
||||
(MethodFunction::Set, MethodObject::BlobStoreChange) => "inbuxa:BlobStoreChange/set",
|
||||
(MethodFunction::Get, MethodObject::Relay) => "inbuxa:Relay/get",
|
||||
(MethodFunction::Set, MethodObject::Relay) => "inbuxa:Relay/set",
|
||||
(MethodFunction::Get, MethodObject::RelayMessage) => "inbuxa:RelayMessage/get",
|
||||
@@ -564,6 +569,8 @@ impl MethodName {
|
||||
"inbuxa:StoreCheck/set" => (MethodObject::StoreCheck, MethodFunction::Set),
|
||||
"inbuxa:StoreSnapshot/get" => (MethodObject::StoreSnapshot, MethodFunction::Get),
|
||||
"inbuxa:StoreSnapshot/set" => (MethodObject::StoreSnapshot, MethodFunction::Set),
|
||||
"inbuxa:BlobStoreChange/get" => (MethodObject::BlobStoreChange, MethodFunction::Get),
|
||||
"inbuxa:BlobStoreChange/set" => (MethodObject::BlobStoreChange, MethodFunction::Set),
|
||||
"inbuxa:Relay/get" => (MethodObject::Relay, MethodFunction::Get),
|
||||
"inbuxa:Relay/set" => (MethodObject::Relay, MethodFunction::Set),
|
||||
"inbuxa:RelayMessage/get" => (MethodObject::RelayMessage, MethodFunction::Get),
|
||||
@@ -668,6 +675,7 @@ impl Display for MethodObject {
|
||||
MethodObject::Migration => "inbuxa:Migration",
|
||||
MethodObject::StoreCheck => "inbuxa:StoreCheck",
|
||||
MethodObject::StoreSnapshot => "inbuxa:StoreSnapshot",
|
||||
MethodObject::BlobStoreChange => "inbuxa:BlobStoreChange",
|
||||
MethodObject::Relay => "inbuxa:Relay",
|
||||
MethodObject::RelayMessage => "inbuxa:RelayMessage",
|
||||
MethodObject::SecurityAcceptance => "inbuxa:SecurityAcceptance",
|
||||
|
||||
@@ -130,6 +130,7 @@ pub enum GetRequestMethod {
|
||||
Migration(Box<GetRequest<crate::object::inbuxa_migration::Migration>>),
|
||||
StoreCheck(Box<GetRequest<crate::object::inbuxa_store_check::StoreCheck>>),
|
||||
StoreSnapshot(Box<GetRequest<crate::object::inbuxa_store_snapshot::StoreSnapshot>>),
|
||||
BlobStoreChange(Box<GetRequest<crate::object::inbuxa_blob_store_change::BlobStoreChange>>),
|
||||
Relay(Box<GetRequest<crate::object::inbuxa_relay::Relay>>),
|
||||
RelayMessage(Box<GetRequest<crate::object::inbuxa_relay_message::RelayMessage>>),
|
||||
SecurityAcceptance(
|
||||
@@ -191,6 +192,7 @@ pub enum SetRequestMethod<'x> {
|
||||
Migration(Box<SetRequest<'x, crate::object::inbuxa_migration::Migration>>),
|
||||
StoreCheck(Box<SetRequest<'x, crate::object::inbuxa_store_check::StoreCheck>>),
|
||||
StoreSnapshot(Box<SetRequest<'x, crate::object::inbuxa_store_snapshot::StoreSnapshot>>),
|
||||
BlobStoreChange(Box<SetRequest<'x, crate::object::inbuxa_blob_store_change::BlobStoreChange>>),
|
||||
Relay(Box<SetRequest<'x, crate::object::inbuxa_relay::Relay>>),
|
||||
RelayMessage(Box<SetRequest<'x, crate::object::inbuxa_relay_message::RelayMessage>>),
|
||||
SecurityAcceptance(
|
||||
|
||||
@@ -734,6 +734,21 @@ impl<'de> Visitor<'de> for CallVisitor {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
// inbuxa: a change of blob store
|
||||
(MethodFunction::Get, MethodObject::BlobStoreChange) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::BlobStoreChange(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
Ok(None) => {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
(MethodFunction::Set, MethodObject::BlobStoreChange) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::BlobStoreChange(value)),
|
||||
Err(err) => RequestMethod::invalid(err),
|
||||
Ok(None) => {
|
||||
return Err(de::Error::invalid_length(1, &self));
|
||||
}
|
||||
},
|
||||
// inbuxa: inbuxa Relay
|
||||
(MethodFunction::Get, MethodObject::Relay) => match seq.next_element() {
|
||||
Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Relay(value)),
|
||||
|
||||
@@ -115,6 +115,7 @@ pub enum GetResponseMethod {
|
||||
Migration(GetResponse<crate::object::inbuxa_migration::Migration>),
|
||||
StoreCheck(GetResponse<crate::object::inbuxa_store_check::StoreCheck>),
|
||||
StoreSnapshot(GetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>),
|
||||
BlobStoreChange(GetResponse<crate::object::inbuxa_blob_store_change::BlobStoreChange>),
|
||||
Relay(GetResponse<crate::object::inbuxa_relay::Relay>),
|
||||
RelayMessage(GetResponse<crate::object::inbuxa_relay_message::RelayMessage>),
|
||||
SecurityAcceptance(GetResponse<crate::object::inbuxa_security_acceptance::SecurityAcceptance>),
|
||||
@@ -174,6 +175,7 @@ pub enum SetResponseMethod {
|
||||
Migration(Box<SetResponse<crate::object::inbuxa_migration::Migration>>),
|
||||
StoreCheck(Box<SetResponse<crate::object::inbuxa_store_check::StoreCheck>>),
|
||||
StoreSnapshot(Box<SetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>>),
|
||||
BlobStoreChange(Box<SetResponse<crate::object::inbuxa_blob_store_change::BlobStoreChange>>),
|
||||
Relay(Box<SetResponse<crate::object::inbuxa_relay::Relay>>),
|
||||
RelayMessage(Box<SetResponse<crate::object::inbuxa_relay_message::RelayMessage>>),
|
||||
SecurityAcceptance(
|
||||
@@ -1086,6 +1088,23 @@ impl<'x> From<SetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>>
|
||||
}
|
||||
}
|
||||
|
||||
// inbuxa: a change of blob store
|
||||
impl<'x> From<GetResponse<crate::object::inbuxa_blob_store_change::BlobStoreChange>>
|
||||
for ResponseMethod<'x>
|
||||
{
|
||||
fn from(value: GetResponse<crate::object::inbuxa_blob_store_change::BlobStoreChange>) -> Self {
|
||||
ResponseMethod::Get(GetResponseMethod::BlobStoreChange(value))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> From<SetResponse<crate::object::inbuxa_blob_store_change::BlobStoreChange>>
|
||||
for ResponseMethod<'x>
|
||||
{
|
||||
fn from(value: SetResponse<crate::object::inbuxa_blob_store_change::BlobStoreChange>) -> Self {
|
||||
ResponseMethod::Set(SetResponseMethod::BlobStoreChange(Box::new(value)))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> From<GetResponse<crate::object::inbuxa_relay::Relay>> for ResponseMethod<'x> {
|
||||
fn from(value: GetResponse<crate::object::inbuxa_relay::Relay>) -> Self {
|
||||
ResponseMethod::Get(GetResponseMethod::Relay(value))
|
||||
|
||||
@@ -122,6 +122,8 @@ impl JmapAuthorization for AccessToken {
|
||||
GetRequestMethod::StoreCheck(_) => Permission::SysStoreCheck,
|
||||
// inbuxa: store-snapshot spec, SN-8
|
||||
GetRequestMethod::StoreSnapshot(_) => Permission::SysStoreSnapshot,
|
||||
// inbuxa: blob-store-change spec, BM-9
|
||||
GetRequestMethod::BlobStoreChange(_) => Permission::SysBlobStoreChange,
|
||||
// inbuxa: inbuxa Relay
|
||||
GetRequestMethod::Relay(_) => Permission::SysRelayGet,
|
||||
GetRequestMethod::RelayMessage(_) => Permission::SysRelayQueueGet,
|
||||
@@ -343,6 +345,14 @@ impl JmapAuthorization for AccessToken {
|
||||
Permission::SysStoreSnapshot,
|
||||
Permission::SysStoreSnapshot,
|
||||
),
|
||||
// inbuxa: blob-store-change spec, BM-9
|
||||
SetRequestMethod::BlobStoreChange(s) => validate_set(
|
||||
s,
|
||||
self,
|
||||
Permission::SysBlobStoreChange,
|
||||
Permission::SysBlobStoreChange,
|
||||
Permission::SysBlobStoreChange,
|
||||
),
|
||||
// inbuxa: inbuxa Relay; a new join token is an update
|
||||
SetRequestMethod::Relay(s) => validate_set(
|
||||
s,
|
||||
@@ -618,6 +628,7 @@ impl JmapAuthorization for AccessToken {
|
||||
| MethodObject::Migration
|
||||
| MethodObject::StoreCheck
|
||||
| MethodObject::StoreSnapshot
|
||||
| MethodObject::BlobStoreChange
|
||||
| MethodObject::Relay
|
||||
| MethodObject::RelayMessage
|
||||
| MethodObject::SecurityAcceptance
|
||||
|
||||
@@ -299,6 +299,9 @@ impl RequestHandler for Server {
|
||||
SetResponseMethod::StoreSnapshot(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
SetResponseMethod::BlobStoreChange(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
SetResponseMethod::Relay(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
@@ -592,6 +595,13 @@ impl RequestHandler for Server {
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: a change of blob store
|
||||
GetRequestMethod::BlobStoreChange(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::blob_store_change::get(self, access_token, *req)
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: inbuxa Relay
|
||||
GetRequestMethod::Relay(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
@@ -1217,6 +1227,28 @@ impl RequestHandler for Server {
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: blob-store-change spec, BM-9: copying, stopping and dismissing are audited
|
||||
SetRequestMethod::BlobStoreChange(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::audit::recorded(
|
||||
self,
|
||||
access_token,
|
||||
session,
|
||||
&method_name.obj.to_string(),
|
||||
None,
|
||||
None,
|
||||
*req,
|
||||
|req| {
|
||||
Box::pin(crate::inbuxa::blob_store_change::set(
|
||||
self,
|
||||
access_token,
|
||||
req,
|
||||
))
|
||||
},
|
||||
)
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
SetRequestMethod::Relay(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
let reason = req.arguments.reason.clone();
|
||||
|
||||
@@ -434,6 +434,7 @@ impl IntermediateChangesResponse {
|
||||
| MethodObject::Migration
|
||||
| MethodObject::StoreCheck
|
||||
| MethodObject::StoreSnapshot
|
||||
| MethodObject::BlobStoreChange
|
||||
| MethodObject::Relay
|
||||
| MethodObject::RelayMessage
|
||||
| MethodObject::SecurityAcceptance
|
||||
|
||||
@@ -512,6 +512,23 @@ async fn fork_current(server: &Server, object: &str, id: &MaybeInvalid<Id>) -> O
|
||||
MaybeInvalid::Invalid(_) => None,
|
||||
},
|
||||
// A store check: what kind, and how it stands (store-check spec, SV-9)
|
||||
// A change of blob store: where from and to, and how it stands;
|
||||
// never the old store's settings (blob-store-change spec, BM-9)
|
||||
"inbuxa:BlobStoreChange" => match id {
|
||||
MaybeInvalid::Value(id) => {
|
||||
inbuxa_features::blob_move::get(data, u32::try_from(id.id()).ok()?)
|
||||
.await
|
||||
.ok()?
|
||||
.map(|c| {
|
||||
serde_json::json!({
|
||||
"from": c.from,
|
||||
"to": c.to,
|
||||
"status": c.status.as_str(),
|
||||
})
|
||||
})
|
||||
}
|
||||
MaybeInvalid::Invalid(_) => None,
|
||||
},
|
||||
"inbuxa:StoreCheck" => match id {
|
||||
MaybeInvalid::Value(id) => {
|
||||
inbuxa_features::store_check::get(data, u32::try_from(id.id()).ok()?)
|
||||
|
||||
@@ -0,0 +1,324 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:BlobStoreChange` (blob-store-change spec): the changes of blob
|
||||
//! store the server noticed at start-up, copying what the new store lacks
|
||||
//! from the old one, stopping that, and dismissing a change. The copy is
|
||||
//! `services::inbuxa_blob_move`. Copying, stopping and dismissing are
|
||||
//! audited by the request layer (BM-9).
|
||||
|
||||
use super::relay::json_to_value;
|
||||
use common::{Server, auth::AccessToken};
|
||||
use inbuxa_features::blob_move::{self as model, BlobStoreChange as Stored, Refusal, Status};
|
||||
use jmap_proto::{
|
||||
error::set::SetError,
|
||||
method::{
|
||||
get::{GetRequest, GetResponse},
|
||||
set::{SetRequest, SetResponse},
|
||||
},
|
||||
object::inbuxa_blob_store_change::{
|
||||
BlobStoreChange, BlobStoreChangeProperty as P, BlobStoreChangeValue,
|
||||
},
|
||||
request::IntoValid,
|
||||
types::date::UTCDate,
|
||||
};
|
||||
use jmap_tools::{Key, Map, Value};
|
||||
use serde_json::json;
|
||||
use services::inbuxa_blob_move as copies;
|
||||
use store::write::now;
|
||||
use types::id::Id;
|
||||
|
||||
type CValue = Value<'static, P, BlobStoreChangeValue>;
|
||||
|
||||
const ALL: &[P] = &[
|
||||
P::Id,
|
||||
P::From,
|
||||
P::To,
|
||||
P::ChangedAt,
|
||||
P::Node,
|
||||
P::Status,
|
||||
P::CopyStartedAt,
|
||||
P::CopyFinishedAt,
|
||||
P::CopyStartedBy,
|
||||
P::CopyNode,
|
||||
P::BlobsPerSecond,
|
||||
P::BytesPerSecond,
|
||||
P::Progress,
|
||||
P::BlobsChecked,
|
||||
P::Copied,
|
||||
P::AlreadyThere,
|
||||
P::BytesCopied,
|
||||
P::Missing,
|
||||
P::Damaged,
|
||||
P::Unreadable,
|
||||
P::Problems,
|
||||
P::Error,
|
||||
];
|
||||
|
||||
/// BM-9: a change is about the whole server's blobs, so only the server's
|
||||
/// own administrators see and act on one.
|
||||
fn server_level(access_token: &AccessToken) -> trc::Result<()> {
|
||||
if access_token.tenant_id().is_some() {
|
||||
Err(trc::JmapEvent::Forbidden
|
||||
.into_err()
|
||||
.details("A blob store change is about the whole server, not a tenant."))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn date(seconds: u64) -> serde_json::Value {
|
||||
json!(UTCDate::from_timestamp(seconds as i64).to_string())
|
||||
}
|
||||
|
||||
async fn to_value(server: &Server, c: &Stored, properties: &[P], now: u64) -> trc::Result<CValue> {
|
||||
let status = c.status_at(now);
|
||||
let copy = c.copy.as_ref();
|
||||
let p = copy.map(|copy| copy.progress.clone()).unwrap_or_default();
|
||||
let mut out = Map::with_capacity(properties.len());
|
||||
for property in properties {
|
||||
let json = match property {
|
||||
P::Id => {
|
||||
out.insert_unchecked(
|
||||
Key::Property(P::Id),
|
||||
Value::Element(BlobStoreChangeValue::Id(Id::from(c.id))),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
// BM-9: where the stores are, never their settings
|
||||
P::From => json!(c.from),
|
||||
P::To => json!(c.to),
|
||||
P::ChangedAt => date(c.changed_at),
|
||||
P::Node => json!(c.node_id.to_string()),
|
||||
P::Status => json!(status.as_str()),
|
||||
P::CopyStartedAt => copy.map_or(serde_json::Value::Null, |c| date(c.started_at)),
|
||||
P::CopyFinishedAt => copy
|
||||
.and_then(|c| c.finished_at)
|
||||
.map_or(serde_json::Value::Null, date),
|
||||
P::CopyStartedBy => copy.map_or(serde_json::Value::Null, |c| json!(c.started_by)),
|
||||
P::CopyNode => copy.map_or(serde_json::Value::Null, |c| json!(c.node_id.to_string())),
|
||||
P::BlobsPerSecond => {
|
||||
copy.map_or(serde_json::Value::Null, |c| json!(c.blobs_per_second))
|
||||
}
|
||||
P::BytesPerSecond => {
|
||||
copy.map_or(serde_json::Value::Null, |c| json!(c.bytes_per_second))
|
||||
}
|
||||
P::Progress => match status {
|
||||
Status::Copied | Status::Reverted => json!(1.0),
|
||||
_ => json!(p.share()),
|
||||
},
|
||||
P::BlobsChecked => json!(p.blobs_checked),
|
||||
P::Copied => json!(p.copied),
|
||||
P::AlreadyThere => json!(p.already_there),
|
||||
P::BytesCopied => json!(p.bytes_copied),
|
||||
P::Missing => json!(p.missing),
|
||||
P::Damaged => json!(p.damaged),
|
||||
P::Unreadable => json!(p.unreadable),
|
||||
P::Problems => {
|
||||
let problems = model::problems(server.store(), c.id).await?;
|
||||
let mut list = serde_json::to_value(&problems).unwrap_or_default();
|
||||
// The account as a JMAP id, like every other account id
|
||||
if let Some(items) = list.as_array_mut() {
|
||||
for item in items {
|
||||
if let Some(account) = item.get("accountId").and_then(|v| v.as_u64()) {
|
||||
item["accountId"] = json!(Id::from(account).to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
list
|
||||
}
|
||||
P::Error => json!(copy.and_then(|c| c.error.clone())),
|
||||
// Write-only
|
||||
P::Action => continue,
|
||||
};
|
||||
out.insert_unchecked(Key::Property(property.clone()), json_to_value(json));
|
||||
}
|
||||
Ok(Value::Object(out))
|
||||
}
|
||||
|
||||
/// `inbuxa:BlobStoreChange/get`: every change kept, newest first.
|
||||
pub async fn get(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: GetRequest<BlobStoreChange>,
|
||||
) -> trc::Result<GetResponse<BlobStoreChange>> {
|
||||
server_level(access_token)?;
|
||||
let properties = request.unwrap_properties(ALL);
|
||||
let (ids, not_found) = request.unwrap_ids(server.core.jmap.get_max_objects)?;
|
||||
let mut response = GetResponse {
|
||||
account_id: request.account_id.into(),
|
||||
state: None,
|
||||
list: Vec::new(),
|
||||
not_found,
|
||||
};
|
||||
let mut changes = model::all(server.store()).await?;
|
||||
changes.reverse();
|
||||
let now = now();
|
||||
match ids {
|
||||
None => {
|
||||
for c in &changes {
|
||||
response
|
||||
.list
|
||||
.push(to_value(server, c, &properties, now).await?);
|
||||
}
|
||||
}
|
||||
Some(ids) => {
|
||||
for id in ids {
|
||||
match changes.iter().find(|c| u64::from(c.id) == id.id()) {
|
||||
Some(c) => response
|
||||
.list
|
||||
.push(to_value(server, c, &properties, now).await?),
|
||||
None => response.push_not_found(id),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn invalid(property: P, why: impl Into<String>) -> SetError<P> {
|
||||
SetError::invalid_properties()
|
||||
.with_property(property)
|
||||
.with_description(why.into())
|
||||
}
|
||||
|
||||
/// A limit: a whole number, 0 for none (BM-5).
|
||||
fn limit(property: P, value: &Value<'_, P, BlobStoreChangeValue>) -> Result<u64, SetError<P>> {
|
||||
match value {
|
||||
Value::Number(n) => n
|
||||
.as_u64()
|
||||
.ok_or_else(|| invalid(property, "A whole number of 0 or more; 0 is no limit.")),
|
||||
_ => Err(invalid(
|
||||
property,
|
||||
"A whole number of 0 or more; 0 is no limit.",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// `inbuxa:BlobStoreChange/set`: the server records changes, so there's no
|
||||
/// create. An update copies or stops a copy; destroy dismisses (BM-5 to
|
||||
/// BM-7).
|
||||
pub async fn set(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: SetRequest<'_, BlobStoreChange>,
|
||||
) -> trc::Result<SetResponse<BlobStoreChange>> {
|
||||
server_level(access_token)?;
|
||||
let mut response = SetResponse::from_request(&request, server.core.jmap.set_max_objects)?;
|
||||
let data = server.store();
|
||||
let actor = server.audit_actor(access_token).await;
|
||||
|
||||
for (client_id, _) in request.unwrap_create() {
|
||||
response.not_created.append(
|
||||
client_id,
|
||||
SetError::forbidden().with_description(
|
||||
"The server records a change when it starts with the blob store moved.",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
'update: for (id, value) in request.unwrap_update().into_valid() {
|
||||
let Ok(change_id) = u32::try_from(id.id()) else {
|
||||
response.not_updated.append(id, SetError::not_found());
|
||||
continue;
|
||||
};
|
||||
let mut action = None;
|
||||
let mut blobs_per_second = None;
|
||||
let mut bytes_per_second = None;
|
||||
for (key, value) in value.into_expanded_object() {
|
||||
let outcome = match &key {
|
||||
Key::Property(P::Action) => {
|
||||
action = value.as_str().map(|s| s.to_string());
|
||||
Ok(())
|
||||
}
|
||||
Key::Property(P::BlobsPerSecond) => {
|
||||
limit(P::BlobsPerSecond, &value).map(|l| blobs_per_second = Some(l))
|
||||
}
|
||||
Key::Property(P::BytesPerSecond) => {
|
||||
limit(P::BytesPerSecond, &value).map(|l| bytes_per_second = Some(l))
|
||||
}
|
||||
_ => Err(SetError::invalid_properties()
|
||||
.with_property(key.clone().into_owned())
|
||||
.with_description("A change can't be edited, only copied or stopped.")),
|
||||
};
|
||||
if let Err(error) = outcome {
|
||||
response.not_updated.append(id, error);
|
||||
continue 'update;
|
||||
}
|
||||
}
|
||||
let Some(change) = model::get(data, change_id).await? else {
|
||||
response.not_updated.append(id, SetError::not_found());
|
||||
continue;
|
||||
};
|
||||
match action.as_deref() {
|
||||
Some("copy") => {
|
||||
match copies::start(
|
||||
server,
|
||||
change_id,
|
||||
blobs_per_second,
|
||||
bytes_per_second,
|
||||
actor.name.clone(),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
Ok(()) => response.updated.append(id, None),
|
||||
Err(Refusal::Running(running)) => response.not_updated.append(
|
||||
id,
|
||||
SetError::forbidden().with_description(format!(
|
||||
"A copy of change {} runs now; one copy runs at a time. \
|
||||
Stop it or wait for it.",
|
||||
Id::from(running)
|
||||
)),
|
||||
),
|
||||
Err(Refusal::Resolved) => response.not_updated.append(
|
||||
id,
|
||||
SetError::forbidden()
|
||||
.with_description("That change is resolved; there's nothing to copy."),
|
||||
),
|
||||
Err(Refusal::NotFound) => {
|
||||
response.not_updated.append(id, SetError::not_found())
|
||||
}
|
||||
}
|
||||
}
|
||||
Some("stop") => {
|
||||
if change.status_at(now()) != Status::Copying {
|
||||
response.not_updated.append(
|
||||
id,
|
||||
SetError::forbidden().with_description("That change isn't being copied."),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
copies::stop(server, change_id).await?;
|
||||
response.updated.append(id, None);
|
||||
}
|
||||
_ => response
|
||||
.not_updated
|
||||
.append(id, invalid(P::Action, "The actions are copy and stop.")),
|
||||
}
|
||||
}
|
||||
|
||||
for id in request.unwrap_destroy().into_valid() {
|
||||
let Ok(change_id) = u32::try_from(id.id()) else {
|
||||
response.not_destroyed.append(id, SetError::not_found());
|
||||
continue;
|
||||
};
|
||||
match model::get(data, change_id).await? {
|
||||
None => response.not_destroyed.append(id, SetError::not_found()),
|
||||
Some(c) if c.status_at(now()) == Status::Copying => response.not_destroyed.append(
|
||||
id,
|
||||
SetError::forbidden().with_description("It's being copied; stop the copy first."),
|
||||
),
|
||||
Some(_) => {
|
||||
model::remove(data, change_id).await?;
|
||||
// BM-7: this node stops reading through it at once; the
|
||||
// others within a minute (BM-10)
|
||||
copies::refresh(server).await;
|
||||
response.destroyed.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
@@ -12,6 +12,7 @@ pub mod account_lock;
|
||||
pub mod ai_limits;
|
||||
pub mod audit;
|
||||
pub mod audit_log;
|
||||
pub mod blob_store_change; // inbuxa: a change of blob store
|
||||
pub mod data_inventory;
|
||||
pub mod deleted_account;
|
||||
pub mod deliverability; // inbuxa: the deliverability check
|
||||
|
||||
@@ -1795,6 +1795,8 @@ pub enum Permission {
|
||||
SysStoreCheck = 695,
|
||||
// inbuxa: a snapshot of the store, without stopping the server
|
||||
SysStoreSnapshot = 696,
|
||||
// inbuxa: a change of blob store, copied and dismissed
|
||||
SysBlobStoreChange = 697,
|
||||
SysAccountGet = 219,
|
||||
SysAccountCreate = 220,
|
||||
SysAccountUpdate = 221,
|
||||
|
||||
@@ -7173,6 +7173,7 @@ impl EnumImpl for Permission {
|
||||
b"sysMigrationRun" => Permission::SysMigrationRun,
|
||||
b"sysStoreCheck" => Permission::SysStoreCheck,
|
||||
b"sysStoreSnapshot" => Permission::SysStoreSnapshot,
|
||||
b"sysBlobStoreChange" => Permission::SysBlobStoreChange,
|
||||
b"sysAccountGet" => Permission::SysAccountGet,
|
||||
b"sysAccountCreate" => Permission::SysAccountCreate,
|
||||
b"sysAccountUpdate" => Permission::SysAccountUpdate,
|
||||
@@ -7886,6 +7887,7 @@ impl EnumImpl for Permission {
|
||||
Permission::SysMigrationRun => "sysMigrationRun",
|
||||
Permission::SysStoreCheck => "sysStoreCheck",
|
||||
Permission::SysStoreSnapshot => "sysStoreSnapshot",
|
||||
Permission::SysBlobStoreChange => "sysBlobStoreChange",
|
||||
Permission::SysAccountGet => "sysAccountGet",
|
||||
Permission::SysAccountCreate => "sysAccountCreate",
|
||||
Permission::SysAccountUpdate => "sysAccountUpdate",
|
||||
@@ -8592,6 +8594,7 @@ impl EnumImpl for Permission {
|
||||
694 => Some(Permission::SysMigrationRun),
|
||||
695 => Some(Permission::SysStoreCheck),
|
||||
696 => Some(Permission::SysStoreSnapshot),
|
||||
697 => Some(Permission::SysBlobStoreChange),
|
||||
219 => Some(Permission::SysAccountGet),
|
||||
220 => Some(Permission::SysAccountCreate),
|
||||
221 => Some(Permission::SysAccountUpdate),
|
||||
@@ -9036,7 +9039,7 @@ impl EnumImpl for Permission {
|
||||
}
|
||||
}
|
||||
|
||||
const COUNT: usize = 697;
|
||||
const COUNT: usize = 698;
|
||||
}
|
||||
|
||||
impl serde::Serialize for Permission {
|
||||
|
||||
@@ -0,0 +1,583 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Changing the blob store (docs/spec/features/blob-store-change.md): the
|
||||
//! copy behind an `inbuxa:BlobStoreChange`. It goes through the blob index
|
||||
//! one slice at a time and copies each blob the current store lacks from
|
||||
//! the change's old store, as the old store keeps it (BM-5). It runs on the
|
||||
//! node the copy was started on. Also, on every node, it keeps reading
|
||||
//! through only the changes still unresolved (BM-10).
|
||||
|
||||
use common::{
|
||||
BuildServer, Inner, Server,
|
||||
manager::{SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY},
|
||||
};
|
||||
use inbuxa_features::blob_move::{
|
||||
self as model, BlobStoreChange, Copy, DEFAULT_BLOBS_PER_SECOND, DEFAULT_BYTES_PER_SECOND,
|
||||
MAX_PROBLEMS, Problem, ProblemKind, Progress, Refusal, STALE_AFTER, Status,
|
||||
};
|
||||
use std::{
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use store::{
|
||||
BlobStore,
|
||||
write::{
|
||||
blob_check::{BlobHolder, LinkedBlob},
|
||||
now,
|
||||
},
|
||||
};
|
||||
use types::{blob_hash::BlobHash, collection::Collection};
|
||||
|
||||
/// How often a running copy saves its progress, which is also its
|
||||
/// heartbeat and when it sees a stop (BM-6).
|
||||
const SAVE_EVERY: Duration = Duration::from_secs(10);
|
||||
/// BM-10: how often a node reads which changes are still unresolved.
|
||||
const REFRESH_EVERY: Duration = Duration::from_secs(60);
|
||||
|
||||
fn node_id(server: &Server) -> u64 {
|
||||
server.core.network.node_id
|
||||
}
|
||||
|
||||
/// BM-5: starts copying a change on this node. Refused while another copy
|
||||
/// runs, or when the change is resolved.
|
||||
pub async fn start(
|
||||
server: &Server,
|
||||
id: u32,
|
||||
blobs_per_second: Option<u64>,
|
||||
bytes_per_second: Option<u64>,
|
||||
started_by: String,
|
||||
) -> trc::Result<Result<(), Refusal>> {
|
||||
let now = now();
|
||||
let copy = Copy {
|
||||
started_at: now,
|
||||
started_by,
|
||||
node_id: node_id(server),
|
||||
blobs_per_second: blobs_per_second.unwrap_or(DEFAULT_BLOBS_PER_SECOND),
|
||||
bytes_per_second: bytes_per_second.unwrap_or(DEFAULT_BYTES_PER_SECOND),
|
||||
heartbeat: now,
|
||||
finished_at: None,
|
||||
stop_requested: false,
|
||||
progress: Progress::default(),
|
||||
problems_kept: 0,
|
||||
error: None,
|
||||
};
|
||||
let change = match model::start_copy(server.store(), id, copy, now).await? {
|
||||
Ok(change) => change,
|
||||
Err(refusal) => return Ok(Err(refusal)),
|
||||
};
|
||||
let server = server.clone();
|
||||
tokio::spawn(async move {
|
||||
run(&server, change).await;
|
||||
});
|
||||
Ok(Ok(()))
|
||||
}
|
||||
|
||||
/// BM-6: asks a running copy to stop. Any node can: the one running it
|
||||
/// sees the request at its next save. None if there's no such change.
|
||||
pub async fn stop(server: &Server, id: u32) -> trc::Result<Option<BlobStoreChange>> {
|
||||
model::update(server.store(), id, |c| match &mut c.copy {
|
||||
Some(copy) if c.status == Status::Copying && !copy.stop_requested => {
|
||||
copy.stop_requested = true;
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// BM-6: at start-up, a copy this node ran stopped with it; now and then,
|
||||
/// a copy on a node that went quiet is taken to have stopped too. BM-10:
|
||||
/// and every minute, only unresolved changes are read through.
|
||||
pub fn spawn_blob_moves(inner: Arc<Inner>) {
|
||||
tokio::spawn(async move {
|
||||
let server = inner.build_server();
|
||||
let node = node_id(&server);
|
||||
mark_interrupted(&server, |c| c.node_id == node).await;
|
||||
loop {
|
||||
tokio::time::sleep(REFRESH_EVERY).await;
|
||||
let server = inner.build_server();
|
||||
let cutoff = now().saturating_sub(STALE_AFTER);
|
||||
mark_interrupted(&server, |c| c.node_id != node && c.heartbeat < cutoff).await;
|
||||
refresh(&server).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// BM-10: stops reading through changes resolved or dismissed anywhere.
|
||||
pub async fn refresh(server: &Server) {
|
||||
let Some(moving) = server.blob_store().moving() else {
|
||||
return;
|
||||
};
|
||||
match model::unresolved(server.store()).await {
|
||||
Ok(changes) => {
|
||||
let ids = changes.iter().map(|c| c.id).collect::<Vec<_>>();
|
||||
moving.retain_changes(&ids);
|
||||
}
|
||||
Err(err) => {
|
||||
trc::error!(err.details("Failed to read the blob store changes"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn mark_interrupted(server: &Server, which: impl Fn(&Copy) -> bool) {
|
||||
let changes = match model::all(server.store()).await {
|
||||
Ok(changes) => changes,
|
||||
Err(err) => {
|
||||
trc::error!(err.details("Failed to read the blob store changes"));
|
||||
return;
|
||||
}
|
||||
};
|
||||
for change in changes
|
||||
.into_iter()
|
||||
.filter(|c| c.status == Status::Copying && c.copy.as_ref().is_some_and(&which))
|
||||
{
|
||||
let result = model::update(server.store(), change.id, |c| {
|
||||
if c.status != Status::Copying || !c.copy.as_ref().is_some_and(&which) {
|
||||
return false;
|
||||
}
|
||||
c.status = Status::Interrupted;
|
||||
if let Some(copy) = &mut c.copy {
|
||||
copy.finished_at = Some(now());
|
||||
}
|
||||
true
|
||||
})
|
||||
.await;
|
||||
if let Err(err) = result.and(model::release(server.store(), change.id).await) {
|
||||
trc::error!(err.details("Failed to mark an interrupted blob copy"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// BM-5: keeps a rate, a second at a time.
|
||||
struct Pace {
|
||||
per_second: u64,
|
||||
window: Instant,
|
||||
used: u64,
|
||||
}
|
||||
|
||||
impl Pace {
|
||||
fn new(per_second: u64) -> Self {
|
||||
Pace {
|
||||
per_second,
|
||||
window: Instant::now(),
|
||||
used: 0,
|
||||
}
|
||||
}
|
||||
|
||||
async fn take(&mut self, amount: u64) {
|
||||
if self.per_second == 0 {
|
||||
return;
|
||||
}
|
||||
if self.window.elapsed() >= Duration::from_secs(1) {
|
||||
self.window = Instant::now();
|
||||
self.used = 0;
|
||||
}
|
||||
self.used += amount;
|
||||
if self.used >= self.per_second {
|
||||
let rest = Duration::from_secs(1).saturating_sub(self.window.elapsed());
|
||||
tokio::time::sleep(rest).await;
|
||||
self.window = Instant::now();
|
||||
self.used = self.used.saturating_sub(self.per_second);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Who holds a blob, as a problem names it (BM-5).
|
||||
struct Holder {
|
||||
name: String,
|
||||
account_id: Option<u32>,
|
||||
object_id: Option<u64>,
|
||||
/// Named by the hash of its content, so it can be checked.
|
||||
content_named: bool,
|
||||
}
|
||||
|
||||
impl Holder {
|
||||
fn of(blob: &LinkedBlob) -> Holder {
|
||||
match blob.holders.first() {
|
||||
Some(BlobHolder::Document {
|
||||
account_id,
|
||||
collection,
|
||||
document_id,
|
||||
}) => Holder {
|
||||
name: match Collection::from(*collection) {
|
||||
Collection::Email => "email".into(),
|
||||
Collection::FileNode => "file".into(),
|
||||
Collection::SieveScript => "sieveScript".into(),
|
||||
Collection::CalendarEvent => "calendarEvent".into(),
|
||||
Collection::ContactCard => "contactCard".into(),
|
||||
_ => format!("collection{collection}"),
|
||||
},
|
||||
account_id: Some(*account_id),
|
||||
object_id: Some(*document_id as u64),
|
||||
content_named: true,
|
||||
},
|
||||
Some(BlobHolder::Queue { id }) => Holder {
|
||||
name: "queuedMessage".into(),
|
||||
account_id: None,
|
||||
object_id: Some(*id),
|
||||
content_named: true,
|
||||
},
|
||||
Some(BlobHolder::Archived { account_id, item }) => Holder {
|
||||
name: "archivedItem".into(),
|
||||
account_id: Some(*account_id),
|
||||
object_id: Some(item.id().id()),
|
||||
content_named: true,
|
||||
},
|
||||
None => Holder {
|
||||
name: "temporary".into(),
|
||||
account_id: None,
|
||||
object_id: None,
|
||||
content_named: false,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn fixed() -> Holder {
|
||||
Holder {
|
||||
name: "fixed".into(),
|
||||
account_id: None,
|
||||
object_id: None,
|
||||
content_named: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Run<'x> {
|
||||
server: &'x Server,
|
||||
id: u32,
|
||||
old: BlobStore,
|
||||
target: BlobStore,
|
||||
progress: Progress,
|
||||
found: Vec<Problem>,
|
||||
kept: u64,
|
||||
blobs: Pace,
|
||||
bytes: Pace,
|
||||
saved: Instant,
|
||||
}
|
||||
|
||||
/// How a run ended before it finished.
|
||||
enum Halt {
|
||||
Stopped,
|
||||
Failed(trc::Error),
|
||||
}
|
||||
|
||||
impl From<trc::Error> for Halt {
|
||||
fn from(err: trc::Error) -> Self {
|
||||
Halt::Failed(err)
|
||||
}
|
||||
}
|
||||
|
||||
fn key_name(key: &[u8]) -> String {
|
||||
match BlobHash::try_from_hash_slice(key) {
|
||||
Ok(hash) => hash.to_hex(),
|
||||
Err(_) => String::from_utf8_lossy(key).into_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> Run<'x> {
|
||||
/// BM-6: counts a problem, logs it, and keeps the first ones.
|
||||
fn problem(&mut self, kind: ProblemKind, key: &[u8], holder: &Holder, detail: String) {
|
||||
self.progress.count(kind);
|
||||
let blob_id = key_name(key);
|
||||
trc::event!(
|
||||
Store(trc::StoreEvent::DataCorruption),
|
||||
Id = self.id,
|
||||
AccountId = holder.account_id,
|
||||
Collection = holder.name.clone(),
|
||||
DocumentId = holder.object_id,
|
||||
BlobId = blob_id.clone(),
|
||||
Reason = kind.as_str(),
|
||||
Details = detail.clone(),
|
||||
);
|
||||
if self.kept < MAX_PROBLEMS {
|
||||
self.kept += 1;
|
||||
self.found.push(Problem {
|
||||
kind,
|
||||
blob_id,
|
||||
holder: holder.name.clone(),
|
||||
account_id: holder.account_id,
|
||||
object_id: holder.object_id,
|
||||
detail,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Saves progress and the problems found since, now and then; a stop
|
||||
/// asked for meanwhile ends the run.
|
||||
async fn save(&mut self, force: bool) -> Result<(), Halt> {
|
||||
if !force && self.saved.elapsed() < SAVE_EVERY {
|
||||
return Ok(());
|
||||
}
|
||||
let data = self.server.store();
|
||||
let first = self.kept - self.found.len() as u64;
|
||||
model::add_problems(data, self.id, first, &self.found).await?;
|
||||
self.found.clear();
|
||||
let progress = self.progress.clone();
|
||||
let kept = self.kept;
|
||||
let saved = model::update(data, self.id, |c| {
|
||||
if let Some(copy) = &mut c.copy {
|
||||
copy.progress = progress.clone();
|
||||
copy.problems_kept = kept;
|
||||
copy.heartbeat = now();
|
||||
}
|
||||
true
|
||||
})
|
||||
.await?;
|
||||
self.saved = Instant::now();
|
||||
match saved {
|
||||
Some(c) if c.copy.as_ref().is_some_and(|c| c.stop_requested) => Err(Halt::Stopped),
|
||||
Some(_) => Ok(()),
|
||||
// Dismissed under it: nothing left to copy for
|
||||
None => Err(Halt::Stopped),
|
||||
}
|
||||
}
|
||||
|
||||
/// BM-5: one blob, copied unless the current store has it.
|
||||
async fn one(&mut self, key: &[u8], holder: Holder) -> Result<(), Halt> {
|
||||
self.blobs.take(1).await;
|
||||
self.progress.blobs_checked += 1;
|
||||
match self.target.stored_present(key).await {
|
||||
Ok(true) => {
|
||||
self.progress.already_there += 1;
|
||||
return self.save(false).await;
|
||||
}
|
||||
Ok(false) => {}
|
||||
Err(err) => {
|
||||
self.problem(
|
||||
ProblemKind::Unreadable,
|
||||
key,
|
||||
&holder,
|
||||
format!("the current blob store couldn't say: {err}"),
|
||||
);
|
||||
return self.save(false).await;
|
||||
}
|
||||
}
|
||||
let stored = match self.old.get_stored(key).await {
|
||||
Ok(Some(stored)) => stored,
|
||||
Ok(None) => {
|
||||
// The spam files exist only once the classifier has trained
|
||||
if holder.name != "fixed" {
|
||||
self.problem(ProblemKind::Missing, key, &holder, String::new());
|
||||
}
|
||||
return self.save(false).await;
|
||||
}
|
||||
Err(err) => {
|
||||
self.problem(
|
||||
ProblemKind::Unreadable,
|
||||
key,
|
||||
&holder,
|
||||
format!("the old blob store failed: {err}"),
|
||||
);
|
||||
return self.save(false).await;
|
||||
}
|
||||
};
|
||||
let size = stored.len() as u64;
|
||||
self.bytes.take(size).await;
|
||||
if holder.content_named
|
||||
&& let Ok(hash) = BlobHash::try_from_hash_slice(key)
|
||||
{
|
||||
match BlobStore::content_of_stored(key, stored.clone()) {
|
||||
Ok(content) => {
|
||||
let actual = BlobHash::generate(&content);
|
||||
if actual != hash {
|
||||
self.problem(
|
||||
ProblemKind::Damaged,
|
||||
key,
|
||||
&holder,
|
||||
format!(
|
||||
"the old store's {} bytes hash to {}",
|
||||
content.len(),
|
||||
actual.to_hex()
|
||||
),
|
||||
);
|
||||
return self.save(false).await;
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
self.problem(
|
||||
ProblemKind::Damaged,
|
||||
key,
|
||||
&holder,
|
||||
"it doesn't decompress".into(),
|
||||
);
|
||||
return self.save(false).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
match self.target.put_stored(key, &stored).await {
|
||||
Ok(()) => {
|
||||
self.progress.copied += 1;
|
||||
self.progress.bytes_copied += size;
|
||||
}
|
||||
Err(err) => self.problem(
|
||||
ProblemKind::Unreadable,
|
||||
key,
|
||||
&holder,
|
||||
format!("the current blob store couldn't write it: {err}"),
|
||||
),
|
||||
}
|
||||
self.save(false).await
|
||||
}
|
||||
|
||||
async fn go(&mut self) -> Result<(), Halt> {
|
||||
for slice in 0u8..=255 {
|
||||
let blobs = self.server.store().blobs_to_copy(slice).await?;
|
||||
// A blob without its commit marker never finished being written
|
||||
for blob in blobs.into_iter().filter(|b| b.committed) {
|
||||
let holder = Holder::of(&blob);
|
||||
self.one(blob.hash.as_slice(), holder).await?;
|
||||
}
|
||||
self.progress.slices = slice as u32 + 1;
|
||||
self.save(false).await?;
|
||||
}
|
||||
// BM-5: the files kept under fixed names
|
||||
for key in [SPAM_TRAINER_KEY, SPAM_CLASSIFIER_KEY] {
|
||||
self.one(key, Holder::fixed()).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// The change's old store: the one already read through, or opened now.
|
||||
async fn old_store(server: &Server, change: &BlobStoreChange) -> Result<BlobStore, String> {
|
||||
if let Some(old) = server.blob_store().moving().and_then(|m| {
|
||||
m.old_stores()
|
||||
.iter()
|
||||
.find(|o| o.change_id == change.id)
|
||||
.map(|o| o.store.clone())
|
||||
}) {
|
||||
return Ok(old);
|
||||
}
|
||||
let Some(settings) = change.old_settings.clone() else {
|
||||
return Err("The change keeps no settings for the old blob store".into());
|
||||
};
|
||||
let mut warnings = Vec::new();
|
||||
BlobStore::open(settings, server.store(), &mut warnings).await
|
||||
}
|
||||
|
||||
/// Runs a copy to its end and records how it ended (BM-6).
|
||||
async fn run(server: &Server, change: BlobStoreChange) {
|
||||
let id = change.id;
|
||||
let (status, error, run) = match old_store(server, &change).await {
|
||||
Err(err) => (
|
||||
Status::Failed,
|
||||
Some(format!("The old blob store couldn't be opened: {err}")),
|
||||
None,
|
||||
),
|
||||
Ok(old) => {
|
||||
let copy = change.copy.clone().unwrap_or_else(|| Copy {
|
||||
started_at: now(),
|
||||
started_by: String::new(),
|
||||
node_id: node_id(server),
|
||||
blobs_per_second: DEFAULT_BLOBS_PER_SECOND,
|
||||
bytes_per_second: DEFAULT_BYTES_PER_SECOND,
|
||||
heartbeat: now(),
|
||||
finished_at: None,
|
||||
stop_requested: false,
|
||||
progress: Progress::default(),
|
||||
problems_kept: 0,
|
||||
error: None,
|
||||
});
|
||||
let mut run = Run {
|
||||
server,
|
||||
id,
|
||||
old,
|
||||
target: server.blob_store().current().clone(),
|
||||
progress: Progress::default(),
|
||||
found: Vec::new(),
|
||||
kept: 0,
|
||||
blobs: Pace::new(copy.blobs_per_second),
|
||||
bytes: Pace::new(copy.bytes_per_second),
|
||||
saved: Instant::now(),
|
||||
};
|
||||
let (status, error) = match run.go().await {
|
||||
Ok(()) if run.progress.problems() == 0 => (Status::Copied, None),
|
||||
Ok(()) => (Status::Incomplete, None),
|
||||
Err(Halt::Stopped) => (Status::Stopped, None),
|
||||
Err(Halt::Failed(err)) => {
|
||||
let text = err
|
||||
.value_as_str(trc::Key::Details)
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| err.to_string());
|
||||
trc::error!(err.details("A blob copy failed"));
|
||||
(Status::Failed, Some(text))
|
||||
}
|
||||
};
|
||||
(status, error, Some(run))
|
||||
}
|
||||
};
|
||||
|
||||
let data = server.store();
|
||||
let mut result = Ok(());
|
||||
let (progress, kept) = match &run {
|
||||
Some(run) => {
|
||||
let first = run.kept - run.found.len() as u64;
|
||||
result = model::add_problems(data, id, first, &run.found).await;
|
||||
(run.progress.clone(), run.kept)
|
||||
}
|
||||
None => (Progress::default(), 0),
|
||||
};
|
||||
if result.is_ok() {
|
||||
result = model::update(data, id, |c| {
|
||||
c.status = status;
|
||||
// BM-6: resolved, so the old store's settings go
|
||||
if status == Status::Copied {
|
||||
c.old_settings = None;
|
||||
}
|
||||
if let Some(copy) = &mut c.copy {
|
||||
copy.progress = progress.clone();
|
||||
copy.problems_kept = kept;
|
||||
copy.finished_at = Some(now());
|
||||
copy.heartbeat = now();
|
||||
}
|
||||
if let Some(error) = &error {
|
||||
c.set_error(error);
|
||||
}
|
||||
true
|
||||
})
|
||||
.await
|
||||
.map(|_| ());
|
||||
}
|
||||
if let Err(err) = result.and(model::release(data, id).await) {
|
||||
trc::error!(err.details("Failed to save a blob copy's result"));
|
||||
}
|
||||
if status == Status::Copied {
|
||||
refresh(server).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn keys_are_named_by_hash_or_as_is() {
|
||||
let hash = BlobHash::generate(b"hello");
|
||||
assert_eq!(key_name(hash.as_slice()), hash.to_hex());
|
||||
assert_eq!(key_name(SPAM_TRAINER_KEY), "INBUXA_SPAM_TRAIN_DATA.lz4");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_document_queue_and_kept_blobs_are_checked() {
|
||||
let blob = |holders| LinkedBlob {
|
||||
hash: BlobHash::generate(b"x"),
|
||||
committed: true,
|
||||
holders,
|
||||
temporary: true,
|
||||
};
|
||||
assert!(!Holder::of(&blob(vec![])).content_named);
|
||||
assert!(Holder::of(&blob(vec![BlobHolder::Queue { id: 4 }])).content_named);
|
||||
let mail = Holder::of(&blob(vec![BlobHolder::Document {
|
||||
account_id: 1,
|
||||
collection: Collection::Email.into(),
|
||||
document_id: 9,
|
||||
}]));
|
||||
assert_eq!(mail.name, "email");
|
||||
assert_eq!((mail.account_id, mail.object_id), (Some(1), Some(9)));
|
||||
assert!(!Holder::fixed().content_named);
|
||||
}
|
||||
}
|
||||
@@ -114,6 +114,12 @@ fn source(server: &Server) -> Result<(SnapshotSource, PathBuf), Refusal> {
|
||||
"A snapshot copies blobs kept in the data store or in a filesystem blob store, \
|
||||
and this server keeps them in {kind}."
|
||||
))),
|
||||
// blob-store-change spec, BM-8
|
||||
Err(NotSnapshottable::BlobStoreChanging) => Err(Refusal::Unsupported(
|
||||
"The blob store changed and some blobs may still be only in the old one. \
|
||||
Copy them over or dismiss the change (inbuxa:BlobStoreChange) first."
|
||||
.to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ use crate::task_manager::{manager::spawn_task_manager, scheduler::spawn_task_sch
|
||||
|
||||
pub mod broadcast;
|
||||
// inbuxa: AL-5, delegations end at their date
|
||||
pub mod inbuxa_blob_move; // inbuxa: changing the blob store
|
||||
pub mod inbuxa_deliverability; // inbuxa: the deliverability check
|
||||
pub mod inbuxa_lock_expiry;
|
||||
pub mod inbuxa_log_retention; // inbuxa: personal-data catalog, D1
|
||||
@@ -89,6 +90,10 @@ impl SpawnServices for IpcReceivers {
|
||||
// inbuxa: store-check spec, SV-8: a check this node ran stopped with it
|
||||
inbuxa_store_check::spawn_store_checks(inner.clone());
|
||||
|
||||
// inbuxa: blob-store-change spec, BM-6, BM-10: a copy this node
|
||||
// ran stopped with it; resolved changes stop being read through
|
||||
inbuxa_blob_move::spawn_blob_moves(inner.clone());
|
||||
|
||||
// inbuxa: store-snapshot spec, SN-7: a snapshot cut short by a
|
||||
// stop leaves a partial directory
|
||||
inbuxa_store_snapshot::spawn_store_snapshots(inner.clone());
|
||||
|
||||
+115
-23
@@ -6,14 +6,38 @@
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use crate::{BlobStore, backend::fs::FsStore, registry::bootstrap::Bootstrap};
|
||||
use crate::{BlobStore, Store, backend::fs::FsStore, registry::bootstrap::Bootstrap};
|
||||
use registry::schema::{prelude::ObjectType, structs};
|
||||
|
||||
#[allow(unreachable_patterns)]
|
||||
impl BlobStore {
|
||||
pub async fn build(bp: &mut Bootstrap) -> Option<Self> {
|
||||
let result = match bp.setting_infallible::<structs::BlobStore>().await {
|
||||
structs::BlobStore::Default => return Some(BlobStore::Store(bp.data_store.clone())),
|
||||
let config = bp.setting_infallible::<structs::BlobStore>().await;
|
||||
let mut warnings = Vec::new();
|
||||
let result = BlobStore::open(config, &bp.data_store, &mut warnings).await;
|
||||
for warning in warnings {
|
||||
bp.build_warning(ObjectType::BlobStore.singleton(), warning);
|
||||
}
|
||||
|
||||
match result {
|
||||
Ok(store) => Some(store),
|
||||
Err(err) => {
|
||||
bp.build_error(ObjectType::BlobStore.singleton(), err);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// inbuxa: the blob store a setting names. Split out of `build` so a
|
||||
/// blob store the setting no longer names can be opened too, to read
|
||||
/// what was left in it (blob-store-change spec, BM-2).
|
||||
#[allow(unreachable_patterns)]
|
||||
pub async fn open(
|
||||
config: structs::BlobStore,
|
||||
data_store: &Store,
|
||||
warnings: &mut Vec<String>,
|
||||
) -> Result<Self, String> {
|
||||
match config {
|
||||
structs::BlobStore::Default => Ok(BlobStore::Store(data_store.clone())),
|
||||
#[cfg(feature = "foundation")]
|
||||
structs::BlobStore::FoundationDb(foundation_db_store) => {
|
||||
crate::backend::foundationdb::FdbStore::open(foundation_db_store)
|
||||
@@ -43,27 +67,95 @@ impl BlobStore {
|
||||
}
|
||||
// inbuxa: ST-16 to ST-22
|
||||
structs::BlobStore::Sharded(sharded) => {
|
||||
let mut warnings = Vec::new();
|
||||
let result = crate::backend::scaleout::ShardedBlobStore::open(
|
||||
sharded,
|
||||
&bp.data_store,
|
||||
&mut warnings,
|
||||
)
|
||||
.await;
|
||||
for warning in warnings {
|
||||
bp.build_warning(ObjectType::BlobStore.singleton(), warning);
|
||||
}
|
||||
result
|
||||
crate::backend::scaleout::ShardedBlobStore::open(sharded, data_store, warnings)
|
||||
.await
|
||||
}
|
||||
_ => Err("Binary was not compiled with the selected blob store backend".to_string()),
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(store) => Some(store),
|
||||
Err(err) => {
|
||||
bp.build_error(ObjectType::BlobStore.singleton(), err);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// inbuxa: blob-store-change spec, BM-1: where a blob store setting puts
|
||||
/// blobs, without its credentials or tuning. Two settings with the same
|
||||
/// place read the same blobs; a different place doesn't have them.
|
||||
pub fn blob_store_location(config: &structs::BlobStore) -> String {
|
||||
match config {
|
||||
structs::BlobStore::Default => "the data store".to_string(),
|
||||
// ST-16 to ST-22: the set moves blobs between its members itself
|
||||
structs::BlobStore::Sharded(_) => "a sharded set".to_string(),
|
||||
structs::BlobStore::S3(s3) => {
|
||||
let region = match &s3.region {
|
||||
structs::S3StoreRegion::Custom(custom) => {
|
||||
format!("{} ({})", custom.custom_endpoint, custom.custom_region)
|
||||
}
|
||||
region => format!("{region:?}"),
|
||||
};
|
||||
with_prefix(
|
||||
format!("S3 {region}, bucket {}", s3.bucket),
|
||||
s3.key_prefix.as_deref(),
|
||||
)
|
||||
}
|
||||
structs::BlobStore::Azure(azure) => with_prefix(
|
||||
format!(
|
||||
"Azure {}, container {}",
|
||||
azure.storage_account, azure.container
|
||||
),
|
||||
azure.key_prefix.as_deref(),
|
||||
),
|
||||
structs::BlobStore::FileSystem(fs) => {
|
||||
format!("the directory {}, depth {}", fs.path, fs.depth.min(5))
|
||||
}
|
||||
structs::BlobStore::FoundationDb(fdb) => format!(
|
||||
"FoundationDB {}",
|
||||
fdb.cluster_file
|
||||
.as_deref()
|
||||
.unwrap_or("(default cluster file)")
|
||||
),
|
||||
structs::BlobStore::PostgreSql(pg) => {
|
||||
format!("PostgreSQL {}:{}/{}", pg.host, pg.port, pg.database)
|
||||
}
|
||||
structs::BlobStore::MySql(my) => {
|
||||
format!("MySQL {}:{}/{}", my.host, my.port, my.database)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn with_prefix(place: String, prefix: Option<&str>) -> String {
|
||||
match prefix.filter(|p| !p.is_empty()) {
|
||||
Some(prefix) => format!("{place}, prefix {prefix}"),
|
||||
None => place,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn fs(path: &str, depth: u64) -> structs::BlobStore {
|
||||
structs::BlobStore::FileSystem(structs::FileSystemStore {
|
||||
path: path.into(),
|
||||
depth,
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_place_is_where_blobs_go_not_how() {
|
||||
assert_eq!(
|
||||
blob_store_location(&structs::BlobStore::Default),
|
||||
"the data store"
|
||||
);
|
||||
assert_eq!(
|
||||
blob_store_location(&fs("/srv/blobs", 2)),
|
||||
"the directory /srv/blobs, depth 2"
|
||||
);
|
||||
// The depth decides each blob's path, so it's part of the place
|
||||
assert_ne!(
|
||||
blob_store_location(&fs("/srv/blobs", 2)),
|
||||
blob_store_location(&fs("/srv/blobs", 3))
|
||||
);
|
||||
assert_ne!(
|
||||
blob_store_location(&fs("/srv/blobs", 2)),
|
||||
blob_store_location(&fs("/srv/other", 2))
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,12 @@ const LZ4_MARKER: u8 = MAGIC_MARKER | 0x01;
|
||||
//const ZSTD_MARKER: u8 = MAGIC_MARKER | 0x02;
|
||||
const NONE_MARKER: u8 = 0x00;
|
||||
|
||||
/// What a stored blob decodes to: its content, or an empty value as is.
|
||||
enum Decoded {
|
||||
Content(Vec<u8>),
|
||||
Bare(Vec<u8>),
|
||||
}
|
||||
|
||||
impl BlobStore {
|
||||
pub async fn get_blob(&self, key: &[u8], range: Range<usize>) -> trc::Result<Option<Vec<u8>>> {
|
||||
let start_time = Instant::now();
|
||||
@@ -27,11 +33,42 @@ impl BlobStore {
|
||||
Size = result.as_ref().map_or(0, |data| data.len()),
|
||||
);
|
||||
|
||||
let Some(mut data) = result else {
|
||||
let Some(data) = result else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let mut data = match data.last().copied() {
|
||||
let mut data = match Self::decode_stored(key, data)? {
|
||||
Decoded::Content(data) => data,
|
||||
Decoded::Bare(data) => return Ok(Some(data)),
|
||||
};
|
||||
|
||||
if range.start == 0 {
|
||||
if range.end > data.len() {
|
||||
Ok(Some(data))
|
||||
} else {
|
||||
data.truncate(range.end);
|
||||
Ok(Some(data))
|
||||
}
|
||||
} else {
|
||||
Ok(Some(
|
||||
data.get(range.start..range.end)
|
||||
.unwrap_or_default()
|
||||
.to_vec(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// inbuxa: a blob's content from what the backend keeps, as
|
||||
/// [`BlobStore::get_blob`] reads it (blob-store-change spec, BM-5: a copy
|
||||
/// checks a blob's hash before writing it as it was stored).
|
||||
pub fn content_of_stored(key: &[u8], data: Vec<u8>) -> trc::Result<Vec<u8>> {
|
||||
Self::decode_stored(key, data).map(|decoded| match decoded {
|
||||
Decoded::Content(data) | Decoded::Bare(data) => data,
|
||||
})
|
||||
}
|
||||
|
||||
fn decode_stored(key: &[u8], mut data: Vec<u8>) -> trc::Result<Decoded> {
|
||||
let data = match data.last().copied() {
|
||||
Some(LZ4_MARKER) => {
|
||||
lz4_flex::decompress_size_prepended(data.get(..data.len() - 1).unwrap_or_default())
|
||||
.map_err(|err| {
|
||||
@@ -53,24 +90,10 @@ impl BlobStore {
|
||||
data
|
||||
}
|
||||
None => {
|
||||
return Ok(Some(data));
|
||||
return Ok(Decoded::Bare(data));
|
||||
}
|
||||
};
|
||||
|
||||
if range.start == 0 {
|
||||
if range.end > data.len() {
|
||||
Ok(Some(data))
|
||||
} else {
|
||||
data.truncate(range.end);
|
||||
Ok(Some(data))
|
||||
}
|
||||
} else {
|
||||
Ok(Some(
|
||||
data.get(range.start..range.end)
|
||||
.unwrap_or_default()
|
||||
.to_vec(),
|
||||
))
|
||||
}
|
||||
Ok(Decoded::Content(data))
|
||||
}
|
||||
|
||||
pub async fn put_blob(
|
||||
@@ -183,6 +206,8 @@ impl BlobStore {
|
||||
BlobStore::Azure(store) => store.get_blob(key, 0..usize::MAX).await,
|
||||
// inbuxa: ST-17
|
||||
BlobStore::Sharded(store) => store.get(key).await,
|
||||
// inbuxa: BM-3
|
||||
BlobStore::Moving(store) => store.raw_get(key).await,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,6 +237,8 @@ impl BlobStore {
|
||||
BlobStore::Azure(store) => store.put_blob(key, data).await,
|
||||
// inbuxa: ST-18
|
||||
BlobStore::Sharded(store) => store.put(key, data).await,
|
||||
// inbuxa: BM-3: writes go to the current store only
|
||||
BlobStore::Moving(store) => Box::pin(store.current.raw_put(key, data)).await,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -241,6 +268,8 @@ impl BlobStore {
|
||||
BlobStore::Azure(store) => store.delete_blob(key).await,
|
||||
// inbuxa: ST-18
|
||||
BlobStore::Sharded(store) => store.delete(key).await,
|
||||
// inbuxa: BM-3: nothing is deleted from an old store
|
||||
BlobStore::Moving(store) => Box::pin(store.current.raw_delete(key)).await,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -185,6 +185,9 @@ pub enum BlobStore {
|
||||
Azure(Arc<backend::azure::AzureStore>),
|
||||
// inbuxa: ST-16 to ST-22
|
||||
Sharded(Arc<backend::scaleout::ShardedBlobStore>),
|
||||
// inbuxa: blob-store-change spec, BM-3: the current store, read through
|
||||
// to the old ones while a change is unresolved
|
||||
Moving(Arc<write::blob_move::MovingBlobStore>),
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
|
||||
@@ -39,6 +39,9 @@ pub struct LinkedBlob {
|
||||
pub hash: BlobHash,
|
||||
pub committed: bool,
|
||||
pub holders: Vec<BlobHolder>,
|
||||
/// inbuxa: blob-store-change spec, BM-5: a link that hasn't expired
|
||||
/// and names nothing: an upload, a cached app bundle, a journal entry.
|
||||
pub temporary: bool,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
@@ -46,6 +49,21 @@ impl Store {
|
||||
/// hash) that something holds a link to. Uploads nothing holds yet and
|
||||
/// expired links are left out: they're the purge's to clear.
|
||||
pub async fn linked_blobs(&self, slice: u8) -> trc::Result<Vec<LinkedBlob>> {
|
||||
let mut blobs = self.indexed_blobs(slice).await?;
|
||||
blobs.retain(|b| !b.holders.is_empty());
|
||||
Ok(blobs)
|
||||
}
|
||||
|
||||
/// inbuxa: blob-store-change spec, BM-5: every blob in one slice of the
|
||||
/// blob index that something holds, a plain link that hasn't expired
|
||||
/// included. Expired links are left out.
|
||||
pub async fn blobs_to_copy(&self, slice: u8) -> trc::Result<Vec<LinkedBlob>> {
|
||||
let mut blobs = self.indexed_blobs(slice).await?;
|
||||
blobs.retain(|b| !b.holders.is_empty() || b.temporary);
|
||||
Ok(blobs)
|
||||
}
|
||||
|
||||
async fn indexed_blobs(&self, slice: u8) -> trc::Result<Vec<LinkedBlob>> {
|
||||
const TEMP_LINK: usize = BLOB_HASH_LEN + U32_LEN + U64_LEN;
|
||||
const DOC_LINK: usize = BLOB_HASH_LEN + U32_LEN + 1 + U32_LEN;
|
||||
const ID_LINK: usize = BLOB_HASH_LEN + U64_LEN;
|
||||
@@ -86,6 +104,7 @@ impl Store {
|
||||
hash,
|
||||
committed: false,
|
||||
holders: Vec::new(),
|
||||
temporary: false,
|
||||
});
|
||||
}
|
||||
let blob = blobs.last_mut().unwrap();
|
||||
@@ -108,6 +127,8 @@ impl Store {
|
||||
account_id: key.deserialize_be_u32(BLOB_HASH_LEN)?,
|
||||
item: ObjectId::deserialize(value)?,
|
||||
});
|
||||
} else if until > now {
|
||||
blob.temporary = true;
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
@@ -124,7 +145,6 @@ impl Store {
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
blobs.retain(|b| !b.holders.is_empty());
|
||||
Ok(blobs)
|
||||
}
|
||||
}
|
||||
@@ -157,6 +177,8 @@ impl BlobStore {
|
||||
Ok(false)
|
||||
}
|
||||
BlobStore::Store(_) => self.raw_get(key).await.map(|b| b.is_some()),
|
||||
// inbuxa: blob-store-change spec, BM-3
|
||||
BlobStore::Moving(store) => store.blob_present(key).await,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! inbuxa: what the blob-store-change spec
|
||||
//! (docs/spec/features/blob-store-change.md) needs from the store: a blob
|
||||
//! store that, while a change of blob store is unresolved, looks for what
|
||||
//! it doesn't have in the stores the server used before, and keeps what it
|
||||
//! finds there (BM-3). Writes and deletes go to the current store only.
|
||||
|
||||
use crate::BlobStore;
|
||||
use arc_swap::ArcSwap;
|
||||
use std::sync::Arc;
|
||||
use trc::StoreEvent;
|
||||
|
||||
/// A blob store the server used before an unresolved change.
|
||||
#[derive(Clone)]
|
||||
pub struct OldBlobStore {
|
||||
/// The `inbuxa:BlobStoreChange` that moved away from it.
|
||||
pub change_id: u32,
|
||||
pub store: BlobStore,
|
||||
}
|
||||
|
||||
/// The current blob store, and the old ones read through while their
|
||||
/// changes are unresolved, newest first (BM-3).
|
||||
pub struct MovingBlobStore {
|
||||
pub current: BlobStore,
|
||||
old: ArcSwap<Vec<OldBlobStore>>,
|
||||
}
|
||||
|
||||
impl MovingBlobStore {
|
||||
pub fn new(current: BlobStore, old: Vec<OldBlobStore>) -> Self {
|
||||
MovingBlobStore {
|
||||
current,
|
||||
old: ArcSwap::from_pointee(old),
|
||||
}
|
||||
}
|
||||
|
||||
/// The old stores still read through.
|
||||
pub fn old_stores(&self) -> Arc<Vec<OldBlobStore>> {
|
||||
self.old.load_full()
|
||||
}
|
||||
|
||||
/// BM-10: keeps only the old stores of changes still unresolved.
|
||||
pub fn retain_changes(&self, unresolved: &[u32]) {
|
||||
let old = self.old.load();
|
||||
if old.iter().all(|o| unresolved.contains(&o.change_id)) {
|
||||
return;
|
||||
}
|
||||
let kept = old
|
||||
.iter()
|
||||
.filter(|o| unresolved.contains(&o.change_id))
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
self.old.store(Arc::new(kept));
|
||||
}
|
||||
|
||||
/// BM-3: the blob as the current store has it or, failing that, as the
|
||||
/// newest old store that has it does. A blob found in an old store is
|
||||
/// written to the current one under the same key, as it was stored.
|
||||
pub(crate) async fn raw_get(&self, key: &[u8]) -> trc::Result<Option<Vec<u8>>> {
|
||||
if let Some(data) = Box::pin(self.current.raw_get(key)).await? {
|
||||
return Ok(Some(data));
|
||||
}
|
||||
let old = self.old.load_full();
|
||||
for old in old.iter() {
|
||||
match Box::pin(old.store.raw_get(key)).await {
|
||||
Ok(Some(data)) => {
|
||||
if let Err(err) = Box::pin(self.current.raw_put(key, &data)).await {
|
||||
trc::error!(
|
||||
err.details("Failed to keep a blob read through an old blob store")
|
||||
.ctx(trc::Key::Id, old.change_id)
|
||||
.ctx(trc::Key::Key, key)
|
||||
);
|
||||
}
|
||||
trc::event!(
|
||||
Store(StoreEvent::BlobReadThrough),
|
||||
Key = key,
|
||||
Id = old.change_id,
|
||||
Size = data.len(),
|
||||
);
|
||||
return Ok(Some(data));
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(err) => {
|
||||
// An old store that fails is skipped, not fatal: the
|
||||
// current store answered, and it had nothing
|
||||
trc::error!(
|
||||
err.details("An old blob store failed to read")
|
||||
.ctx(trc::Key::Id, old.change_id)
|
||||
.ctx(trc::Key::Key, key)
|
||||
.caused_by(trc::location!())
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Whether a read would find the blob: in the current store or in an
|
||||
/// old one still read through.
|
||||
pub(crate) async fn blob_present(&self, key: &[u8]) -> trc::Result<bool> {
|
||||
if Box::pin(self.current.blob_present(key)).await? {
|
||||
return Ok(true);
|
||||
}
|
||||
for old in self.old.load_full().iter() {
|
||||
if Box::pin(old.store.blob_present(key)).await.unwrap_or(false) {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
impl BlobStore {
|
||||
/// The store blobs are written to: past any old stores read through.
|
||||
pub fn current(&self) -> &BlobStore {
|
||||
match self {
|
||||
BlobStore::Moving(moving) => moving.current.current(),
|
||||
store => store,
|
||||
}
|
||||
}
|
||||
|
||||
/// The read-through wrapper, when there is one (BM-3).
|
||||
pub fn moving(&self) -> Option<&Arc<MovingBlobStore>> {
|
||||
match self {
|
||||
BlobStore::Moving(moving) => Some(moving),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// A blob exactly as the backend keeps it, compression marker and all,
|
||||
/// without reading through (BM-5: a copy reads the old store itself).
|
||||
pub async fn get_stored(&self, key: &[u8]) -> trc::Result<Option<Vec<u8>>> {
|
||||
self.current().raw_get(key).await
|
||||
}
|
||||
|
||||
/// Writes a blob exactly as [`BlobStore::get_stored`] gave it (BM-5).
|
||||
pub async fn put_stored(&self, key: &[u8], data: &[u8]) -> trc::Result<()> {
|
||||
self.current().raw_put(key, data).await
|
||||
}
|
||||
|
||||
/// Whether the store itself has the blob, not reading through (BM-5).
|
||||
pub async fn stored_present(&self, key: &[u8]) -> trc::Result<bool> {
|
||||
self.current().blob_present(key).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{CompressionAlgo, backend::fs::FsStore};
|
||||
use registry::schema::structs;
|
||||
|
||||
async fn fs_store(dir: &std::path::Path) -> BlobStore {
|
||||
FsStore::open(structs::FileSystemStore {
|
||||
path: dir.to_string_lossy().into_owned(),
|
||||
depth: 2,
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reads_through_and_keeps_what_it_found() {
|
||||
let root = std::env::temp_dir().join(format!("inbuxa-blob-move-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
let old = fs_store(&root.join("old")).await;
|
||||
let new = fs_store(&root.join("new")).await;
|
||||
old.put_blob(b"left-behind", b"hello", CompressionAlgo::Lz4)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let moving = BlobStore::Moving(Arc::new(MovingBlobStore::new(
|
||||
new.clone(),
|
||||
vec![OldBlobStore {
|
||||
change_id: 7,
|
||||
store: old.clone(),
|
||||
}],
|
||||
)));
|
||||
|
||||
// Only the old store has it: read through, then kept in the new one
|
||||
assert!(!new.blob_present(b"left-behind").await.unwrap());
|
||||
assert!(moving.blob_present(b"left-behind").await.unwrap());
|
||||
assert!(!moving.stored_present(b"left-behind").await.unwrap());
|
||||
assert_eq!(
|
||||
moving
|
||||
.get_blob(b"left-behind", 0..usize::MAX)
|
||||
.await
|
||||
.unwrap()
|
||||
.as_deref(),
|
||||
Some(&b"hello"[..])
|
||||
);
|
||||
assert_eq!(
|
||||
new.get_blob(b"left-behind", 0..usize::MAX)
|
||||
.await
|
||||
.unwrap()
|
||||
.as_deref(),
|
||||
Some(&b"hello"[..])
|
||||
);
|
||||
|
||||
// In neither: absent
|
||||
assert_eq!(
|
||||
moving.get_blob(b"nowhere", 0..usize::MAX).await.unwrap(),
|
||||
None
|
||||
);
|
||||
|
||||
// Writes and deletes touch only the new store
|
||||
moving
|
||||
.put_blob(b"fresh", b"new mail", CompressionAlgo::None)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(new.blob_present(b"fresh").await.unwrap());
|
||||
assert!(!old.blob_present(b"fresh").await.unwrap());
|
||||
moving.delete_blob(b"left-behind").await.unwrap();
|
||||
assert!(!new.blob_present(b"left-behind").await.unwrap());
|
||||
assert!(old.blob_present(b"left-behind").await.unwrap());
|
||||
|
||||
// Once the change is resolved, nothing more is read through
|
||||
moving.moving().unwrap().retain_changes(&[]);
|
||||
assert!(moving.moving().unwrap().old_stores().is_empty());
|
||||
assert_eq!(
|
||||
moving
|
||||
.get_blob(b"left-behind", 0..usize::MAX)
|
||||
.await
|
||||
.unwrap(),
|
||||
None
|
||||
);
|
||||
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
}
|
||||
@@ -30,6 +30,7 @@ pub mod batch;
|
||||
pub mod bitpack;
|
||||
pub mod blob;
|
||||
pub mod blob_check; // inbuxa: store-check spec
|
||||
pub mod blob_move; // inbuxa: blob-store-change spec
|
||||
pub mod key;
|
||||
pub mod log;
|
||||
pub mod serialize;
|
||||
|
||||
@@ -47,6 +47,9 @@ pub enum SnapshotSource {
|
||||
pub enum NotSnapshottable {
|
||||
DataStore(&'static str),
|
||||
BlobStore(&'static str),
|
||||
/// inbuxa: blob-store-change spec, BM-8: some blobs are still only in
|
||||
/// the blob store the server used before.
|
||||
BlobStoreChanging,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
@@ -110,6 +113,12 @@ impl Store {
|
||||
#[cfg(feature = "azure")]
|
||||
BlobStore::Azure(_) => Err(NotSnapshottable::BlobStore("Azure")),
|
||||
BlobStore::Sharded(_) => Err(NotSnapshottable::BlobStore("a sharded set")),
|
||||
// inbuxa: blob-store-change spec, BM-8: blobs only an old store
|
||||
// has would be left out
|
||||
BlobStore::Moving(moving) if !moving.old_stores().is_empty() => {
|
||||
Err(NotSnapshottable::BlobStoreChanging)
|
||||
}
|
||||
BlobStore::Moving(moving) => self.snapshot_source(&moving.current),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
// 650 to 659 are inbuxa Relay's (relay spec, RL-23, RL-T23).
|
||||
// TOTAL_EVENT_COUNT sizes the collector's per-event tables, so it is one
|
||||
// more than the highest id; the test below holds it to the variant list
|
||||
pub const TOTAL_EVENT_COUNT: usize = 660;
|
||||
pub const TOTAL_EVENT_COUNT: usize = 661;
|
||||
pub const TOTAL_METRIC_COUNT: usize = 369;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
@@ -874,6 +874,9 @@ pub enum StoreEvent {
|
||||
CacheUpdate = 577,
|
||||
CacheEntryTooLarge = 633,
|
||||
BlobMissingMarker = 507,
|
||||
/// inbuxa: blob-store-change spec, BM-3: a blob the blob store lacked
|
||||
/// was found in the one it replaced, and kept.
|
||||
BlobReadThrough = 660,
|
||||
DataWrite = 513,
|
||||
DataIterate = 512,
|
||||
BlobRead = 508,
|
||||
|
||||
@@ -620,6 +620,7 @@ impl EventType {
|
||||
b"store.cache-update" => EventType::Store(StoreEvent::CacheUpdate),
|
||||
b"store.cache-entry-too-large" => EventType::Store(StoreEvent::CacheEntryTooLarge),
|
||||
b"store.blob-missing-marker" => EventType::Store(StoreEvent::BlobMissingMarker),
|
||||
b"store.blob-read-through" => EventType::Store(StoreEvent::BlobReadThrough),
|
||||
b"store.data-write" => EventType::Store(StoreEvent::DataWrite),
|
||||
b"store.data-iterate" => EventType::Store(StoreEvent::DataIterate),
|
||||
b"store.blob-read" => EventType::Store(StoreEvent::BlobRead),
|
||||
@@ -1416,6 +1417,7 @@ impl EventType {
|
||||
EventType::Store(StoreEvent::CacheUpdate) => "store.cache-update",
|
||||
EventType::Store(StoreEvent::CacheEntryTooLarge) => "store.cache-entry-too-large",
|
||||
EventType::Store(StoreEvent::BlobMissingMarker) => "store.blob-missing-marker",
|
||||
EventType::Store(StoreEvent::BlobReadThrough) => "store.blob-read-through",
|
||||
EventType::Store(StoreEvent::DataWrite) => "store.data-write",
|
||||
EventType::Store(StoreEvent::DataIterate) => "store.data-iterate",
|
||||
EventType::Store(StoreEvent::BlobRead) => "store.blob-read",
|
||||
@@ -2105,6 +2107,7 @@ impl EventType {
|
||||
EventType::Store(StoreEvent::CacheUpdate) => 577,
|
||||
EventType::Store(StoreEvent::CacheEntryTooLarge) => 633,
|
||||
EventType::Store(StoreEvent::BlobMissingMarker) => 507,
|
||||
EventType::Store(StoreEvent::BlobReadThrough) => 660,
|
||||
EventType::Store(StoreEvent::DataWrite) => 513,
|
||||
EventType::Store(StoreEvent::DataIterate) => 512,
|
||||
EventType::Store(StoreEvent::BlobRead) => 508,
|
||||
@@ -2814,6 +2817,7 @@ impl EventType {
|
||||
577 => Some(EventType::Store(StoreEvent::CacheUpdate)),
|
||||
633 => Some(EventType::Store(StoreEvent::CacheEntryTooLarge)),
|
||||
507 => Some(EventType::Store(StoreEvent::BlobMissingMarker)),
|
||||
660 => Some(EventType::Store(StoreEvent::BlobReadThrough)),
|
||||
513 => Some(EventType::Store(StoreEvent::DataWrite)),
|
||||
512 => Some(EventType::Store(StoreEvent::DataIterate)),
|
||||
508 => Some(EventType::Store(StoreEvent::BlobRead)),
|
||||
@@ -3249,6 +3253,7 @@ impl EventType {
|
||||
EventType::Store(StoreEvent::HttpStoreError) => Level::Warn,
|
||||
EventType::Store(StoreEvent::CacheEntryTooLarge) => Level::Warn,
|
||||
EventType::Store(StoreEvent::BlobMissingMarker) => Level::Warn,
|
||||
EventType::Store(StoreEvent::BlobReadThrough) => Level::Info,
|
||||
EventType::TaskManager(TaskManagerEvent::TaskFailed) => Level::Warn,
|
||||
EventType::Telemetry(TelemetryEvent::AlertEvent) => Level::Warn,
|
||||
EventType::Telemetry(TelemetryEvent::LogError) => Level::Warn,
|
||||
@@ -4007,6 +4012,7 @@ impl EventType {
|
||||
EventType::Store(StoreEvent::CacheUpdate) => "Cache update",
|
||||
EventType::Store(StoreEvent::CacheEntryTooLarge) => "Cache entry too large",
|
||||
EventType::Store(StoreEvent::BlobMissingMarker) => "Blob missing marker",
|
||||
EventType::Store(StoreEvent::BlobReadThrough) => "Blob read through",
|
||||
EventType::Store(StoreEvent::DataWrite) => "Write batch operation",
|
||||
EventType::Store(StoreEvent::DataIterate) => "Data store iteration operation",
|
||||
EventType::Store(StoreEvent::BlobRead) => "Blob read operation",
|
||||
@@ -4368,6 +4374,9 @@ impl EventType {
|
||||
EventType::Store(StoreEvent::CacheUpdate) => "Store error",
|
||||
EventType::Store(StoreEvent::CacheEntryTooLarge) => "Cache entry too large to store",
|
||||
EventType::Store(StoreEvent::BlobMissingMarker) => "Blob is missing marker",
|
||||
EventType::Store(StoreEvent::BlobReadThrough) => {
|
||||
"A blob was read from the blob store used before a change, and kept"
|
||||
}
|
||||
EventType::Store(StoreEvent::DataWrite) => "Store error",
|
||||
EventType::Store(StoreEvent::DataIterate) => "Store error",
|
||||
EventType::Store(StoreEvent::BlobRead) => "Store error",
|
||||
@@ -4987,6 +4996,7 @@ impl EventType {
|
||||
EventType::Store(StoreEvent::CacheUpdate),
|
||||
EventType::Store(StoreEvent::CacheEntryTooLarge),
|
||||
EventType::Store(StoreEvent::BlobMissingMarker),
|
||||
EventType::Store(StoreEvent::BlobReadThrough),
|
||||
EventType::Store(StoreEvent::DataWrite),
|
||||
EventType::Store(StoreEvent::DataIterate),
|
||||
EventType::Store(StoreEvent::BlobRead),
|
||||
|
||||
@@ -390,6 +390,8 @@ Not a rebuild: the **store check** is INBUXA's own design (`features/store-check
|
||||
|
||||
Not a rebuild: the **store snapshot** is INBUXA's own design (`features/store-snapshot.md`, inbuxa-server #323, building). An administrator has the server take a consistent copy of its RocksDB store, and of a filesystem blob store's files, into `snapshots/` in the store's directory while mail flows, for a backup tool to copy; the latest two are kept: `inbuxa:StoreSnapshot` (`crates/jmap/src/inbuxa/store_snapshot.rs`, `crates/services/src/inbuxa_store_snapshot.rs`, `crates/store/src/write/snapshot.rs`, `crates/store/src/backend/rocksdb/checkpoint.rs`), and the `sysStoreSnapshot` permission.
|
||||
|
||||
Not a rebuild: **changing the blob store** is INBUXA's own design (`features/blob-store-change.md`, inbuxa-server #333, building). When a node starts with the blob store setting pointing somewhere else, it records the change and keeps the old store's settings; until the change is resolved, a blob the new store lacks is read from the old one and kept, and an administrator has one node copy the rest over, checked by hash, with what couldn't be copied reported: `inbuxa:BlobStoreChange` (`crates/jmap/src/inbuxa/blob_store_change.rs`, `crates/features/src/blob_move/`, `crates/services/src/inbuxa_blob_move.rs`, `crates/common/src/manager/blob_moves.rs`, `crates/store/src/write/blob_move.rs`), the `store.blob-read-through` event, and the `sysBlobStoreChange` permission.
|
||||
|
||||
Not a rebuild: **the store layout, changed in steps** is INBUXA's own design (`features/store-layout.md`, approved 2026-11-08). A store carries its layout version (`Vs` in `_`); at boot one node, holding a lease, runs each step from the stored version up and marks the store after each. Step 1 → 2 gives SQL rate-limit counters their own expiry column; step 2 → 3 moves the audit log, journal, holds and undelete notes into subspaces of their own. A step stops rollback, so it ships at a series.
|
||||
|
||||
Not a rebuild: **inbuxa-relay** is INBUXA's own design (`features/relay.md`, settled 2026-10-29, in progress). It's a small edge appliance that takes mail in on port 25 for one downstream server or cluster, and sends that downstream's outgoing mail on. The downstream dials out to it over JMAP (`urn:inbuxa:jmap:relay`), so the protected network exposes no mail port, and it is administered only from the downstream's inbuxa-admin.
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
# Feature spec: changing the blob store
|
||||
|
||||
Status: building, 2026-11-09 (inbuxa-server #333). Not a rebuild (SPEC.md
|
||||
§4): inbuxa's own design.
|
||||
|
||||
## Provenance
|
||||
|
||||
| Source | Used for |
|
||||
|---|---|
|
||||
| Forum topic 238 | The need: a server moved its blobs to a filesystem store, took one message by SMTP, and went back to the data store a minute later. The message's metadata stayed; its blob was left in the filesystem store, and reading it failed from then on |
|
||||
| This server's blob store build and dispatch (`crates/store/src/build/blob.rs`, `crates/store/src/dispatch/blob.rs`) | Where the blob store comes from, and that a change takes effect at start-up |
|
||||
| The store check (`features/store-check.md`) and its blob index walk (`crates/store/src/write/blob_check.rs`) | Which blobs the data holds, how a blob proves its content, and how the object, permission and run are built |
|
||||
|
||||
Upstream's open code has nothing like it, and nothing here comes from
|
||||
Enterprise code.
|
||||
|
||||
## What it is
|
||||
|
||||
The blob store setting (`x:BlobStore`) says where blobs go: in the data
|
||||
store, in a directory, in a bucket. Changing it changes where the server
|
||||
writes and reads them from the next start-up. It moves nothing. Before
|
||||
this spec, every blob written under the old setting was left behind,
|
||||
in either direction, and a message whose blob was left behind failed to
|
||||
read with nothing to say why.
|
||||
|
||||
Now the server notices the change when it starts. It keeps reading what
|
||||
it can't find from the old store and copies each blob it reads that way
|
||||
into the new one. An administrator has it copy everything else, and the
|
||||
change is resolved once the new store has every blob. Nothing is ever
|
||||
deleted from the old store; that stays the administrator's to do.
|
||||
|
||||
## Requirements
|
||||
|
||||
- **BM-1.** Where a blob store is, without its credentials or tuning:
|
||||
- the data store (`Default`);
|
||||
- a filesystem store: its path and depth (the depth decides each blob's
|
||||
path, so a new depth is a new place);
|
||||
- S3: the region or endpoint, the bucket and the key prefix;
|
||||
- Azure: the storage account, the container and the key prefix;
|
||||
- PostgreSQL or MySQL: the host, port and database;
|
||||
- FoundationDB: the cluster file;
|
||||
- a sharded set is one place. The set moves blobs between its members
|
||||
itself (scale-out spec, ST-16 to ST-22).
|
||||
- **BM-2.** At start-up, a node compares where its blob store is (BM-1)
|
||||
with where the store last said it was. When they differ, it records an
|
||||
`inbuxa:BlobStoreChange` with both places, the time and the node, and
|
||||
keeps the old store's settings in it so the old store can still be
|
||||
read. It logs a warning that names both places. The first node to start
|
||||
with the new setting records the change; the others find it recorded.
|
||||
The first start of a server, or of a store from before this spec, only
|
||||
records where the blob store is.
|
||||
- **BM-3.** **Reading through.** While a change is unresolved (BM-5), a
|
||||
blob the blob store doesn't have is looked for in the change's old
|
||||
store. When it's there, it's returned and written to the current store
|
||||
under the same key, with the event `store.blob-read-through` naming the
|
||||
key and the change. Several unresolved changes are tried newest first.
|
||||
Writes and deletes go to the current store only. An old store that
|
||||
can't be opened at start-up is left out, with a warning, and the change
|
||||
stays unresolved.
|
||||
- **BM-4.** **Back where it was.** A change back to a store that an
|
||||
earlier unresolved change moved away from (A to B, then back to A)
|
||||
resolves that earlier change by itself: its old store is the current
|
||||
one again. Its status is `reverted`. The new change (B to A) is the one
|
||||
to copy from.
|
||||
- **BM-5.** **Copying.** An update with `action: "copy"` starts a copy on
|
||||
the node that takes it. One copy runs at a time across the server; a
|
||||
second is refused while one runs. It goes through the blob index in its
|
||||
256 slices and takes every committed blob that something holds: a
|
||||
document, a queued message, a kept deleted item, or a link that hasn't
|
||||
expired. It also takes the spam classifier's two files, which the
|
||||
server keeps under fixed names. For each blob:
|
||||
- already in the current store: counted as `alreadyThere`;
|
||||
- in neither store: a `missing` problem;
|
||||
- a blob a document, a queued message or a kept deleted item holds is
|
||||
named by the hash of its content; when what the old store has doesn't
|
||||
hash to its name, it isn't copied and is a `damaged` problem. Other
|
||||
blobs (uploads, cached app bundles, the spam files) aren't named by
|
||||
their content and are copied as they are;
|
||||
- an error from either store: an `unreadable` problem, with the error;
|
||||
- otherwise it's written to the current store as the old store has it,
|
||||
compression and all: `copied`.
|
||||
|
||||
It's throttled like a store check (store-check spec, SV-6): by default
|
||||
1,000 blobs and 20 MB a second, set with `blobsPerSecond` and
|
||||
`bytesPerSecond` alongside the action; 0 is no limit.
|
||||
- **BM-6.** A copy ends:
|
||||
- `copied`: nothing was missing, damaged or unreadable. The change is
|
||||
resolved: every node stops reading through it within a minute
|
||||
(BM-10);
|
||||
- `incomplete`: some blobs couldn't be copied. The first 1,000 problems
|
||||
are kept, and every one is logged as `store.data-corruption`. The
|
||||
change stays unresolved and can be copied again;
|
||||
- `stopped`, by an update with `action: "stop"`; `failed`, with the
|
||||
error; `interrupted`, when its node restarted or went quiet for five
|
||||
minutes. Each leaves the change unresolved.
|
||||
- **BM-7.** **Dismissing.** Destroying a change forgets it and the old
|
||||
store's settings it kept, and reading through it stops. Nothing is
|
||||
deleted from either store. A change whose copy runs can't be destroyed;
|
||||
stop the copy first.
|
||||
- **BM-8.** While a change is unresolved, a store snapshot is refused
|
||||
(store-snapshot spec): it would leave out the blobs only the old store
|
||||
has. The refusal says to copy or dismiss the change first.
|
||||
- **BM-9.** Only the server's administrators see and act on changes:
|
||||
`sysBlobStoreChange`, in the built-in Administrator role and granted
|
||||
once to existing administrator roles. A tenant administrator is
|
||||
refused. Copying, stopping and dismissing are audited; reading isn't.
|
||||
`get` never returns the old store's settings, only where it is (BM-1).
|
||||
- **BM-10.** On a cluster, every node reads the unresolved changes at
|
||||
start-up and again every minute, so one that's resolved or dismissed
|
||||
on any node stops being read through everywhere. During a rolling
|
||||
restart the nodes still on the old setting write to the old store;
|
||||
reading through covers that until the copy, which should start once
|
||||
every node runs the new setting.
|
||||
|
||||
## The object
|
||||
|
||||
| Property | Type | |
|
||||
|---|---|---|
|
||||
| `id` | Id | |
|
||||
| `from`, `to` | String | Where the old and the new blob store are (BM-1) |
|
||||
| `changedAt` | UTCDate | When the first node started with the new setting |
|
||||
| `node` | String | That node |
|
||||
| `status` | String | `pending`, `copying`, `copied`, `incomplete`, `stopped`, `failed`, `interrupted`, `reverted` |
|
||||
| `copyStartedAt`, `copyFinishedAt` | UTCDate | The latest copy |
|
||||
| `copyStartedBy` | String | Who started it |
|
||||
| `copyNode` | String | The node it runs on |
|
||||
| `blobsPerSecond`, `bytesPerSecond` | UnsignedInt | Its limits |
|
||||
| `progress` | Number | 0 to 1 |
|
||||
| `blobsChecked`, `copied`, `alreadyThere`, `bytesCopied` | UnsignedInt | |
|
||||
| `missing`, `damaged`, `unreadable` | UnsignedInt | Counts of each problem |
|
||||
| `problems` | Object[] | The first 1,000 of the latest copy |
|
||||
| `error` | String | Why a `failed` copy failed |
|
||||
| `action` | String | Write-only: `copy` or `stop` |
|
||||
|
||||
A problem is `{kind, blobId, holder, accountId, objectId, detail}`:
|
||||
`holder` is `email`, `file`, `queuedMessage`, `archivedItem`, `temporary`
|
||||
or `fixed`, for the first thing found holding the blob.
|
||||
|
||||
## Tests
|
||||
|
||||
- Store level (`crates/store/src/write/blob_move.rs`): with two filesystem
|
||||
stores, a blob only the old one has reads through and is then in the
|
||||
new one; one in neither reads as absent; writes and deletes touch only
|
||||
the new one; dropping the old store stops reading through.
|
||||
- Where a store is (BM-1): credentials and tuning don't change it; a path,
|
||||
depth, bucket or prefix does.
|
||||
- Model (`crates/features/src/blob_move/`): recording a change, a change
|
||||
back reverting the earlier one, and the copy's bookkeeping.
|
||||
- `system::blob_store_change` (`tests/src/system/blob_store_change.rs`),
|
||||
against a real server: a message delivered while a filesystem store is
|
||||
set, the setting changed back and the server rebuilt; the message reads
|
||||
through, the change is listed, a copy resolves it, and a damaged blob
|
||||
in the old store is reported, not copied. Over JMAP: the refusals, stop,
|
||||
the destroy rules, a tenant administrator's refusal.
|
||||
|
||||
## Not yet
|
||||
|
||||
- The console's page, and a warning on the blob store settings form
|
||||
that existing blobs stay where they are.
|
||||
- An `inbuxa-cli` command.
|
||||
- Copying ahead of a change (the old store to a new one before the
|
||||
setting changes). Reading through makes it unnecessary for the
|
||||
server's own reads, but a long copy after the change means more
|
||||
reads go to the old store meanwhile.
|
||||
@@ -33,6 +33,9 @@ back while the server is stopped.
|
||||
It needs RocksDB as the data store. With any other data store a create
|
||||
is refused, and the refusal says to use the database's own backup:
|
||||
PostgreSQL's or MySQL's, or FoundationDB's.
|
||||
While a change of blob store is unresolved, a create is refused too:
|
||||
some blobs may still be only in the old store, which the snapshot
|
||||
leaves out (blob-store-change spec, BM-8).
|
||||
- **SN-2.** What it holds:
|
||||
- **The data store**: a RocksDB checkpoint of every column family, taken
|
||||
at one point in time. Blobs kept in the data store (the default) are
|
||||
|
||||
@@ -281,6 +281,17 @@ retention = "object-life"
|
||||
createdBy = ["identifier"]
|
||||
problems = ["identifier", "metadata"]
|
||||
|
||||
[object."inbuxa:BlobStoreChange"]
|
||||
file = "inbuxa_blob_store_change.rs"
|
||||
default = "none"
|
||||
whose = ["holder", "administrator"]
|
||||
where = ["data-store"]
|
||||
scope = "server"
|
||||
retention = "object-life"
|
||||
[object."inbuxa:BlobStoreChange".properties]
|
||||
copyStartedBy = ["identifier"]
|
||||
problems = ["identifier", "metadata"]
|
||||
|
||||
[object."inbuxa:StoreSnapshot"]
|
||||
file = "inbuxa_store_snapshot.rs"
|
||||
default = "none"
|
||||
@@ -446,6 +457,19 @@ retention = "object-life"
|
||||
leaves_host = false
|
||||
written_by = ["crates/services/src/inbuxa_store_snapshot.rs"]
|
||||
|
||||
# A change of blob store keeps the old store's settings, credentials
|
||||
# included, until it's resolved or dismissed, so the old store can be read
|
||||
# and copied from. Never returned by get (blob-store-change spec, BM-2,
|
||||
# BM-7, BM-9).
|
||||
[source."blob-store-change-settings"]
|
||||
categories = ["credential", "network"]
|
||||
whose = ["administrator"]
|
||||
where = ["data-store"]
|
||||
scope = "server"
|
||||
retention = "object-life"
|
||||
leaves_host = false
|
||||
written_by = ["crates/features/src/blob_move/mod.rs"]
|
||||
|
||||
[source."full-text-index"]
|
||||
categories = ["content", "identifier", "contact", "metadata"]
|
||||
whose = ["holder", "correspondent"]
|
||||
|
||||
Binary file not shown.
@@ -1 +1 @@
|
||||
6ZXhO67yQsLEItXqRvBehskGGCr53WdaWL9Nh6i7Nm0
|
||||
4CKuuG4Ui14qDOzQkXcTRcBtVmGCzEOGriRedFWwJE8
|
||||
@@ -0,0 +1,434 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:BlobStoreChange` (docs/spec/features/blob-store-change.md).
|
||||
//!
|
||||
//! The forum topic 238 case: some mail went to a filesystem blob store,
|
||||
//! then the setting went back to the data store. The test records that
|
||||
//! change as a start-up would and gives the server the read-through store
|
||||
//! it would build (a test server can't restart with another setting). One
|
||||
//! message's blob is only in the old store, one is damaged there, one is
|
||||
//! in neither. The first reads through and is kept; a copy copies the
|
||||
//! rest it can, reports the damaged and the missing one, and leaves the
|
||||
//! change unresolved. Put right, a second copy resolves it, and nothing
|
||||
//! is read through any more. A slow copy stops when asked and can't be
|
||||
//! dismissed while it runs. Nobody but an administrator reaches changes.
|
||||
|
||||
use crate::utils::{
|
||||
account::Account,
|
||||
server::{TestServer, TestServerBuilder},
|
||||
};
|
||||
use email::{mailbox::INBOX_ID, message::metadata::MessageMetadata};
|
||||
use inbuxa_features::blob_move::{self as model, Started};
|
||||
use registry::schema::{enums::CompressionAlgo, structs};
|
||||
use serde_json::{Value, json};
|
||||
use std::{str::FromStr, sync::Arc, time::Duration};
|
||||
use store::{
|
||||
BlobStore, ValueKey,
|
||||
backend::fs::FsStore,
|
||||
build::blob::blob_store_location,
|
||||
write::{
|
||||
AlignedBytes, Archive,
|
||||
blob_move::{MovingBlobStore, OldBlobStore},
|
||||
now,
|
||||
},
|
||||
};
|
||||
use types::{blob_hash::BlobHash, collection::Collection, field::EmailField, id::Id};
|
||||
|
||||
const USING: &[&str] = &[
|
||||
"urn:ietf:params:jmap:core",
|
||||
"urn:inbuxa:jmap",
|
||||
"urn:inbuxa:jmap:registry",
|
||||
];
|
||||
|
||||
async fn call(account: &Account, method: &str, mut arguments: Value) -> (String, Value) {
|
||||
if arguments.get("accountId").is_none() {
|
||||
arguments["accountId"] = account.id_string().into();
|
||||
}
|
||||
let response = account
|
||||
.jmap_request(USING, json!([[method, arguments, "0"]]))
|
||||
.await;
|
||||
let call = response
|
||||
.0
|
||||
.pointer("/methodResponses/0")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| panic!("{method}: {}", response.0));
|
||||
(
|
||||
call[0].as_str().unwrap_or_default().to_string(),
|
||||
call[1].clone(),
|
||||
)
|
||||
}
|
||||
|
||||
async fn get(admin: &Account, id: &str) -> Value {
|
||||
let (_, response) = call(admin, "inbuxa:BlobStoreChange/get", json!({"ids": [id]})).await;
|
||||
response["list"][0].clone()
|
||||
}
|
||||
|
||||
async fn update(admin: &Account, id: &str, patch: Value) -> Value {
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:BlobStoreChange/set",
|
||||
json!({"update": {id: patch}}),
|
||||
)
|
||||
.await;
|
||||
response
|
||||
}
|
||||
|
||||
/// The change once its copy is over.
|
||||
async fn settled(admin: &Account, id: &str) -> Value {
|
||||
for _ in 0..900 {
|
||||
let change = get(admin, id).await;
|
||||
if change["status"] != "copying" {
|
||||
return change;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
panic!("the copy didn't finish: {}", get(admin, id).await);
|
||||
}
|
||||
|
||||
async fn import(client: &jmap_client::client::Client, body: &str) -> u32 {
|
||||
let id = client
|
||||
.email_import(
|
||||
format!("From: [email protected]\r\nSubject: {body}\r\n\r\n{body}\r\n").into_bytes(),
|
||||
[Id::from(INBOX_ID).to_string()],
|
||||
None::<Vec<&str>>,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.take_id();
|
||||
Id::from_str(&id).unwrap().document_id()
|
||||
}
|
||||
|
||||
async fn blob_of(test: &TestServer, account_id: u32, document_id: u32) -> BlobHash {
|
||||
let archive = test
|
||||
.server
|
||||
.store()
|
||||
.get_value::<Archive<AlignedBytes>>(ValueKey::property(
|
||||
account_id,
|
||||
Collection::Email,
|
||||
document_id,
|
||||
EmailField::Metadata,
|
||||
))
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("the message's metadata");
|
||||
let metadata = archive.unarchive::<MessageMetadata>().unwrap();
|
||||
BlobHash::from(&metadata.blob_hash)
|
||||
}
|
||||
|
||||
fn count(change: &Value, kind: &str) -> u64 {
|
||||
change[kind]
|
||||
.as_u64()
|
||||
.unwrap_or_else(|| panic!("no {kind}: {change}"))
|
||||
}
|
||||
|
||||
/// Gives the server `blob` as its blob store, as a start-up would.
|
||||
fn use_blob_store(test: &mut TestServer, blob: BlobStore) {
|
||||
let mut core = (*test.server.inner.shared_core.load_full()).clone();
|
||||
core.storage.blob = blob;
|
||||
test.server.inner.shared_core.store(Arc::new(core));
|
||||
test.reload_core();
|
||||
}
|
||||
|
||||
/// Moves a message's blob from the current store to `old`, or puts
|
||||
/// `content` there in its place.
|
||||
async fn leave_behind(
|
||||
test: &TestServer,
|
||||
old: &BlobStore,
|
||||
hash: &BlobHash,
|
||||
content: Option<&[u8]>,
|
||||
) -> Vec<u8> {
|
||||
let current = test.server.blob_store().current().clone();
|
||||
let original = current
|
||||
.get_blob(hash.as_slice(), 0..usize::MAX)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("the blob");
|
||||
old.put_blob(
|
||||
hash.as_slice(),
|
||||
content.unwrap_or(&original),
|
||||
CompressionAlgo::Lz4,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
current.delete_blob(hash.as_slice()).await.unwrap();
|
||||
original
|
||||
}
|
||||
|
||||
pub async fn test(test: &mut TestServer) {
|
||||
println!("Running blob store change tests...");
|
||||
let data = test.server.store().clone();
|
||||
|
||||
// A start with blobs in a directory, then one with the setting back
|
||||
let dir = format!("{}/old-blobs", test.tmp_dir());
|
||||
let fs_settings = structs::FileSystemStore {
|
||||
path: dir.clone(),
|
||||
depth: 2,
|
||||
};
|
||||
let old_settings = structs::BlobStore::FileSystem(fs_settings.clone());
|
||||
let old = FsStore::open(fs_settings).await.unwrap();
|
||||
let here = structs::BlobStore::Default;
|
||||
model::at_start(
|
||||
&data,
|
||||
blob_store_location(&old_settings),
|
||||
&old_settings,
|
||||
1,
|
||||
now(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let Started::Changed(change_id) =
|
||||
model::at_start(&data, blob_store_location(&here), &here, 1, now())
|
||||
.await
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("no change recorded");
|
||||
};
|
||||
let id = Id::from(change_id).to_string();
|
||||
|
||||
// The store a start-up would build: reading through to the old one
|
||||
let current = test.server.blob_store().clone();
|
||||
use_blob_store(
|
||||
test,
|
||||
BlobStore::Moving(Arc::new(MovingBlobStore::new(
|
||||
current.clone(),
|
||||
vec![OldBlobStore {
|
||||
change_id,
|
||||
store: old.clone(),
|
||||
}],
|
||||
))),
|
||||
);
|
||||
|
||||
let admin = test.account("[email protected]");
|
||||
let ann = test.account("[email protected]");
|
||||
let ann_id = ann.id().document_id();
|
||||
let client = ann.jmap_client().await;
|
||||
|
||||
let behind = import(&client, "Delivered while the filesystem store was set").await;
|
||||
let damaged = import(&client, "Damaged in the old store").await;
|
||||
let gone = import(&client, "In neither store").await;
|
||||
let fine = import(&client, "Never moved").await;
|
||||
|
||||
let behind_hash = blob_of(test, ann_id, behind).await;
|
||||
let damaged_hash = blob_of(test, ann_id, damaged).await;
|
||||
let gone_hash = blob_of(test, ann_id, gone).await;
|
||||
leave_behind(test, &old, &behind_hash, None).await;
|
||||
let damaged_content =
|
||||
leave_behind(test, &old, &damaged_hash, Some(b"Something else entirely")).await;
|
||||
let gone_content = current
|
||||
.get_blob(gone_hash.as_slice(), 0..usize::MAX)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
current.delete_blob(gone_hash.as_slice()).await.unwrap();
|
||||
|
||||
// Listed, with where from and to and never the old store's settings
|
||||
let change = get(admin, &id).await;
|
||||
assert_eq!(change["status"], "pending", "{change}");
|
||||
assert_eq!(
|
||||
change["from"],
|
||||
blob_store_location(&old_settings),
|
||||
"{change}"
|
||||
);
|
||||
assert_eq!(change["to"], "the data store", "{change}");
|
||||
assert!(!change.to_string().contains("oldSettings"), "{change}");
|
||||
assert!(!change.to_string().contains("FileSystem"), "{change}");
|
||||
|
||||
// BM-3: left behind, read through, and kept
|
||||
assert!(!current.blob_present(behind_hash.as_slice()).await.unwrap());
|
||||
let message = test.fetch_email(ann_id, behind).await;
|
||||
assert!(
|
||||
String::from_utf8_lossy(&message).contains("Delivered while the filesystem store was set"),
|
||||
"read through the old store"
|
||||
);
|
||||
assert!(
|
||||
current.blob_present(behind_hash.as_slice()).await.unwrap(),
|
||||
"kept in the current store"
|
||||
);
|
||||
assert!(
|
||||
old.blob_present(behind_hash.as_slice()).await.unwrap(),
|
||||
"nothing deleted from the old store"
|
||||
);
|
||||
|
||||
// Refusals
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:BlobStoreChange/set",
|
||||
json!({"create": {"x": {}}}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
response["notCreated"]["x"]["type"], "forbidden",
|
||||
"{response}"
|
||||
);
|
||||
let response = update(admin, &id, json!({"action": "rename"})).await;
|
||||
assert_eq!(
|
||||
response["notUpdated"][&id]["type"], "invalidProperties",
|
||||
"{response}"
|
||||
);
|
||||
let response = update(admin, &id, json!({"from": "elsewhere", "action": "copy"})).await;
|
||||
assert_eq!(
|
||||
response["notUpdated"][&id]["type"], "invalidProperties",
|
||||
"{response}"
|
||||
);
|
||||
let response = update(admin, &id, json!({"action": "stop"})).await;
|
||||
assert_eq!(
|
||||
response["notUpdated"][&id]["type"], "forbidden",
|
||||
"{response}"
|
||||
);
|
||||
let (name, response) = call(ann, "inbuxa:BlobStoreChange/get", json!({"ids": null})).await;
|
||||
assert_eq!(name, "error", "a user reaches changes: {response}");
|
||||
|
||||
// A slow copy stops when asked, and can't be dismissed meanwhile
|
||||
for n in 0..15 {
|
||||
import(&client, &format!("Filler {n}")).await;
|
||||
}
|
||||
let response = update(admin, &id, json!({"action": "copy", "blobsPerSecond": 1})).await;
|
||||
assert!(response["updated"].get(&id).is_some(), "{response}");
|
||||
let response = update(admin, &id, json!({"action": "copy"})).await;
|
||||
assert_eq!(
|
||||
response["notUpdated"][&id]["type"], "forbidden",
|
||||
"one copy at a time: {response}"
|
||||
);
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:BlobStoreChange/set",
|
||||
json!({"destroy": [id]}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
response["notDestroyed"][&id]["type"], "forbidden",
|
||||
"{response}"
|
||||
);
|
||||
let response = update(admin, &id, json!({"action": "stop"})).await;
|
||||
assert!(response["updated"].get(&id).is_some(), "{response}");
|
||||
let change = settled(admin, &id).await;
|
||||
assert_eq!(change["status"], "stopped", "{change}");
|
||||
assert_eq!(change["copyStartedBy"], "[email protected]", "{change}");
|
||||
|
||||
// A copy: the damaged and the missing are reported, not copied
|
||||
let response = update(admin, &id, json!({"action": "copy", "blobsPerSecond": 0})).await;
|
||||
assert!(response["updated"].get(&id).is_some(), "{response}");
|
||||
let change = settled(admin, &id).await;
|
||||
assert_eq!(change["status"], "incomplete", "{change}");
|
||||
assert_eq!(count(&change, "missing"), 1, "{change}");
|
||||
assert_eq!(count(&change, "damaged"), 1, "{change}");
|
||||
assert_eq!(count(&change, "unreadable"), 0, "{change}");
|
||||
assert!(count(&change, "alreadyThere") >= 2, "{change}");
|
||||
let problems = change["problems"].as_array().unwrap().clone();
|
||||
let of = |hash: &BlobHash| {
|
||||
problems
|
||||
.iter()
|
||||
.find(|p| p["blobId"] == hash.to_hex())
|
||||
.cloned()
|
||||
.unwrap_or_else(|| panic!("no problem for {}: {change}", hash.to_hex()))
|
||||
};
|
||||
let missing = of(&gone_hash);
|
||||
assert_eq!(missing["kind"], "missing", "{missing}");
|
||||
assert_eq!(missing["holder"], "email", "{missing}");
|
||||
assert_eq!(missing["accountId"], ann.id_string(), "{missing}");
|
||||
assert_eq!(missing["objectId"], gone as u64, "{missing}");
|
||||
assert_eq!(of(&damaged_hash)["kind"], "damaged");
|
||||
assert!(
|
||||
!current.blob_present(damaged_hash.as_slice()).await.unwrap(),
|
||||
"a damaged blob isn't copied"
|
||||
);
|
||||
assert!(
|
||||
test.server
|
||||
.blob_store()
|
||||
.moving()
|
||||
.is_some_and(|m| !m.old_stores().is_empty()),
|
||||
"still read through"
|
||||
);
|
||||
|
||||
// Put right in the old store: a second copy resolves the change
|
||||
old.put_blob(
|
||||
damaged_hash.as_slice(),
|
||||
&damaged_content,
|
||||
CompressionAlgo::Lz4,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
old.put_blob(gone_hash.as_slice(), &gone_content, CompressionAlgo::None)
|
||||
.await
|
||||
.unwrap();
|
||||
let response = update(admin, &id, json!({"action": "copy", "blobsPerSecond": 0})).await;
|
||||
assert!(response["updated"].get(&id).is_some(), "{response}");
|
||||
let change = settled(admin, &id).await;
|
||||
assert_eq!(change["status"], "copied", "{change}");
|
||||
assert_eq!(count(&change, "copied"), 2, "{change}");
|
||||
assert_eq!(change["progress"], 1.0, "{change}");
|
||||
assert_eq!(change["problems"], json!([]), "{change}");
|
||||
for hash in [&damaged_hash, &gone_hash] {
|
||||
assert!(current.blob_present(hash.as_slice()).await.unwrap());
|
||||
}
|
||||
assert!(
|
||||
test.server
|
||||
.blob_store()
|
||||
.moving()
|
||||
.unwrap()
|
||||
.old_stores()
|
||||
.is_empty(),
|
||||
"nothing read through once resolved"
|
||||
);
|
||||
assert!(String::from_utf8_lossy(&test.fetch_email(ann_id, fine).await).contains("Never moved"));
|
||||
assert!(
|
||||
model::get(&data, change_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.old_settings
|
||||
.is_none(),
|
||||
"the old store's settings go once resolved"
|
||||
);
|
||||
let response = update(admin, &id, json!({"action": "copy"})).await;
|
||||
assert_eq!(
|
||||
response["notUpdated"][&id]["type"], "forbidden",
|
||||
"nothing left to copy: {response}"
|
||||
);
|
||||
|
||||
// Dismissed
|
||||
let (_, response) = call(
|
||||
admin,
|
||||
"inbuxa:BlobStoreChange/set",
|
||||
json!({"destroy": [id]}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response["destroyed"], json!([id]), "{response}");
|
||||
let (_, response) = call(admin, "inbuxa:BlobStoreChange/get", json!({"ids": [id]})).await;
|
||||
assert_eq!(response["notFound"], json!([id]), "{response}");
|
||||
|
||||
use_blob_store(test, current);
|
||||
}
|
||||
|
||||
/// Runs these tests alone: `cargo test -p tests blob_store_change_tests -- --ignored`.
|
||||
#[ignore]
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
pub async fn blob_store_change_tests() {
|
||||
let mut test = TestServerBuilder::new("blob_store_change_tests")
|
||||
.await
|
||||
.with_default_listeners()
|
||||
.await
|
||||
.build()
|
||||
.await;
|
||||
let admin = test.create_admin_account("[email protected]").await;
|
||||
test.insert_account(admin);
|
||||
let ann = test
|
||||
.create_user_account(
|
||||
"[email protected]",
|
||||
"[email protected]",
|
||||
"ann-secret-40917",
|
||||
&[],
|
||||
"Ann",
|
||||
)
|
||||
.await;
|
||||
test.insert_account(ann);
|
||||
self::test(&mut test).await;
|
||||
if test.is_reset() {
|
||||
test.temp_dir.delete();
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ pub mod audit; // inbuxa: the audit log
|
||||
pub mod authentication;
|
||||
pub mod authorization;
|
||||
pub mod auto_reload; // inbuxa: registry writes apply at once
|
||||
pub mod blob_store_change; // inbuxa: a change of blob store
|
||||
pub mod branding;
|
||||
pub mod compliance; // inbuxa: the compliance roles
|
||||
pub mod crypto;
|
||||
|
||||
Reference in new issue
Block a user