Blob store changes: clean up the old store once resolved #358

Merged
jcoffey-dev merged 1 commits from feat/old-store-cleanup into main 2026-11-10 18:11:22 +00:00
13 changed files with 1720 additions and 129 deletions

No files matched your search

+19 -4
View File
@@ -9,7 +9,8 @@
//! change (BM-2), then opens the old store of every unresolved change so
//! what the new store lacks is read from it (BM-3), and the store of a
//! planned change so deletes follow to it (blob-store-copy-ahead spec,
//! CA-4).
//! CA-4). It also notes where this node's blob store is, for a cleanup of
//! an old store to check (blob-store-cleanup spec, OC-2).
use inbuxa_features::blob_move::{self as model, Started};
use registry::schema::{prelude::ObjectType, structs};
@@ -72,7 +73,7 @@ pub async fn at_start(bp: &mut Bootstrap, blob: BlobStore) -> BlobStore {
bp,
format!("Couldn't record where the blob store is: {err}"),
);
return blob.wrapped(Vec::new());
return located(blob.wrapped(Vec::new()), &location);
}
}
@@ -80,7 +81,7 @@ pub async fn at_start(bp: &mut Bootstrap, blob: BlobStore) -> BlobStore {
Ok(changes) => changes,
Err(err) => {
warn(bp, format!("Couldn't read the blob store changes: {err}"));
return blob.wrapped(Vec::new());
return located(blob.wrapped(Vec::new()), &location);
}
};
let mut old = Vec::with_capacity(unresolved.len());
@@ -130,7 +131,13 @@ pub async fn at_start(bp: &mut Bootstrap, blob: BlobStore) -> BlobStore {
}
}
let wrapped = blob.wrapped(old);
let wrapped = located(blob.wrapped(old), &location);
if let Err(err) = model::note_node(&data, bp.node_id() as u64, &location, now()).await {
warn(
bp,
format!("Couldn't note where this node's blob store is: {err}"),
);
}
match model::planned(&data).await {
Ok(Some(plan)) => {
let mut warnings = Vec::new();
@@ -163,3 +170,11 @@ pub async fn at_start(bp: &mut Bootstrap, blob: BlobStore) -> BlobStore {
}
wrapped
}
/// OC-2: the blob store, knowing where it is.
fn located(store: BlobStore, location: &str) -> BlobStore {
if let Some(moving) = store.moving() {
moving.set_location(location.to_string());
}
store
}
+552 -47
View File
@@ -10,8 +10,10 @@
//! (BM-2), and keeps the old store's settings so it can be read through
//! (BM-3) and copied from (BM-5). A change can also be planned before the
//! setting moves, with the new store's settings, and copied ahead
//! (docs/spec/features/blob-store-copy-ahead.md). The copy itself runs in
//! `services::inbuxa_blob_move`; this module keeps what a change is.
//! (docs/spec/features/blob-store-copy-ahead.md). Once resolved, its old
//! store's settings are kept so the blobs left there can be cleaned up
//! (docs/spec/features/blob-store-cleanup.md). The copy and the cleanup run
//! in `services::inbuxa_blob_move`; this module keeps what a change is.
//!
//! Kept in the fork's subspace (`store::SUBSPACE_INBUXA`), never in the
//! registry. Every key starts with `W` (where blobs are), then one byte for
@@ -21,8 +23,10 @@
//! - `c` + change id (u32): a change ([`BlobStoreChange`]), as JSON;
//! - `p` + change id (u32) + number (u32): one problem its latest copy
//! found ([`Problem`]);
//! - `r`: the id of the change being copied now (u32), so only one copy
//! runs at a time across the server (BM-5).
//! - `r`: the id of the change being copied or cleaned up now (u32), so
//! only one runs at a time across the server (BM-5, OC-3);
//! - `n` + node id (u64): where that node's blob store is, and when it last
//! said so ([`NodePlace`], OC-2).
//!
//! Numbers are big-endian.
@@ -30,7 +34,7 @@ use registry::schema::structs;
use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize, de::DeserializeOwned};
use store::{
Deserialize, IterateParams, SUBSPACE_INBUXA, Serialize, SerializeInfallible, Store, U32_LEN,
ValueKey,
U64_LEN, ValueKey,
write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue},
};
use trc::AddContext;
@@ -40,6 +44,7 @@ const KIND_LOCATION: u8 = b'l';
const KIND_CHANGE: u8 = b'c';
const KIND_PROBLEM: u8 = b'p';
const KIND_RUNNING: u8 = b'r';
const KIND_NODE: u8 = b'n';
const WRITE_ATTEMPTS: usize = 5;
/// The changes kept; recording another forgets the oldest resolved one.
@@ -51,6 +56,8 @@ pub const DEFAULT_BLOBS_PER_SECOND: u64 = 1_000;
pub const DEFAULT_BYTES_PER_SECOND: u64 = 20 * 1024 * 1024;
/// BM-6: a copy whose node has been quiet this long has stopped.
pub const STALE_AFTER: u64 = 5 * 60;
/// OC-2: a node not heard from for this long is left out.
pub const NODE_QUIET_AFTER: u64 = 10 * 60;
const MAX_TEXT: usize = 2_000;
/// Where the blob store is (BM-1), with the settings that open it.
@@ -106,6 +113,9 @@ pub enum ProblemKind {
Missing,
Damaged,
Unreadable,
/// OC-4: a blob not named by its content whose copy in the new store
/// isn't the old one's.
Different,
}
impl ProblemKind {
@@ -114,6 +124,7 @@ impl ProblemKind {
ProblemKind::Missing => "missing",
ProblemKind::Damaged => "damaged",
ProblemKind::Unreadable => "unreadable",
ProblemKind::Different => "different",
}
}
}
@@ -197,6 +208,8 @@ impl Progress {
ProblemKind::Missing => self.missing += 1,
ProblemKind::Damaged => self.damaged += 1,
ProblemKind::Unreadable => self.unreadable += 1,
// A copy compares nothing byte for byte, so never finds one
ProblemKind::Different => self.damaged += 1,
}
}
@@ -229,6 +242,120 @@ pub struct Copy {
pub error: Option<String>,
}
/// OC-6: how a cleanup of the old store stands.
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
#[serde(rename_all = "lowercase")]
pub enum CleanupStatus {
Running,
/// Nothing kept: every blob the server holds is out of the old store.
Done,
/// Some blobs were kept in the old store (OC-4).
Incomplete,
Stopped,
Failed,
Interrupted,
}
impl CleanupStatus {
pub fn as_str(&self) -> &'static str {
match self {
CleanupStatus::Running => "running",
CleanupStatus::Done => "done",
CleanupStatus::Incomplete => "incomplete",
CleanupStatus::Stopped => "stopped",
CleanupStatus::Failed => "failed",
CleanupStatus::Interrupted => "interrupted",
}
}
}
/// OC-4: how far the latest cleanup got.
#[derive(Debug, Clone, Default, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct CleanupProgress {
/// Slices of the blob index done, of 256.
pub slices: u32,
pub blobs_checked: u64,
/// Deleted from the old store, or for a preview, that would be.
pub deleted: u64,
pub bytes_freed: u64,
/// Not in the old store.
pub not_there: u64,
pub missing: u64,
pub damaged: u64,
pub different: u64,
pub unreadable: u64,
}
impl CleanupProgress {
pub fn problems(&self) -> u64 {
self.missing + self.damaged + self.different + self.unreadable
}
pub fn count(&mut self, kind: ProblemKind) {
match kind {
ProblemKind::Missing => self.missing += 1,
ProblemKind::Damaged => self.damaged += 1,
ProblemKind::Different => self.different += 1,
ProblemKind::Unreadable => self.unreadable += 1,
}
}
pub fn share(&self) -> f64 {
self.slices.min(256) as f64 / 256.0
}
}
/// The latest cleanup of a resolved change's old store, or a preview of
/// one (OC-3 to OC-6).
#[derive(Debug, Clone, PartialEq, SerdeSerialize, SerdeDeserialize)]
#[serde(rename_all = "camelCase")]
pub struct Cleanup {
pub status: CleanupStatus,
/// OC-5: a preview, which deletes nothing.
#[serde(default)]
pub dry_run: bool,
pub started_at: u64,
pub started_by: String,
pub node_id: u64,
pub blobs_per_second: u64,
pub bytes_per_second: u64,
/// When its node last saved it.
pub heartbeat: u64,
#[serde(default)]
pub finished_at: Option<u64>,
#[serde(default)]
pub stop_requested: bool,
#[serde(default)]
pub progress: CleanupProgress,
#[serde(default)]
pub problems_kept: u64,
/// OC-9: snapshots on disk taken before the change.
#[serde(default)]
pub snapshots_before: u64,
#[serde(default)]
pub error: Option<String>,
}
impl Cleanup {
/// As it reads now: one whose node went quiet has stopped with it.
pub fn status_at(&self, now: u64) -> CleanupStatus {
if self.status == CleanupStatus::Running && self.heartbeat + STALE_AFTER < now {
CleanupStatus::Interrupted
} else {
self.status
}
}
}
/// OC-2: where a node's blob store is, and when it last said so.
#[derive(Debug, Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
#[serde(rename_all = "camelCase")]
pub struct NodePlace {
pub location: String,
pub at: u64,
}
/// An `inbuxa:BlobStoreChange`.
#[derive(Debug, Clone, PartialEq, SerdeSerialize, SerdeDeserialize)]
#[serde(rename_all = "camelCase")]
@@ -239,8 +366,9 @@ pub struct BlobStoreChange {
pub changed_at: u64,
pub node_id: u64,
pub status: Status,
/// The old store's settings, to read it (BM-3) and copy from it (BM-5).
/// Dropped once the change is resolved; never shown (BM-9).
/// The old store's settings, to read it (BM-3), copy from it (BM-5) and,
/// once resolved, clean it up (OC-1). Dropped when a cleanup is done or
/// the change is dismissed; never shown (BM-9).
#[serde(default)]
pub old_settings: Option<structs::BlobStore>,
#[serde(default)]
@@ -261,6 +389,9 @@ pub struct BlobStoreChange {
pub applied_at: Option<u64>,
#[serde(default)]
pub applied_by: Option<String>,
/// OC-3 to OC-6: the latest cleanup of the old store.
#[serde(default)]
pub cleanup: Option<Cleanup>,
}
impl BlobStoreChange {
@@ -295,6 +426,53 @@ impl BlobStoreChange {
copy.error = Some(error.trim().chars().take(MAX_TEXT).collect());
}
}
/// OC-6: whether its cleanup runs, as it reads now.
pub fn cleaning_up(&self, now: u64) -> bool {
self.cleanup
.as_ref()
.is_some_and(|c| c.status_at(now) == CleanupStatus::Running)
}
/// BM-5, OC-3: whether a copy or a cleanup of it runs, as it reads now.
pub fn busy(&self, now: u64) -> bool {
self.status_at(now) == Status::Copying || self.cleaning_up(now)
}
/// Marks a copy or cleanup whose node went quiet as interrupted. False
/// when there was none.
pub fn mark_interrupted(&mut self, now: u64) -> bool {
let mut marked = false;
if self.status == Status::Copying && self.status_at(now) == Status::Interrupted {
self.status = Status::Interrupted;
if let Some(copy) = &mut self.copy {
copy.finished_at = Some(now);
}
marked = true;
}
if let Some(cleanup) = &mut self.cleanup
&& cleanup.status == CleanupStatus::Running
&& cleanup.status_at(now) == CleanupStatus::Interrupted
{
cleanup.status = CleanupStatus::Interrupted;
cleanup.finished_at = Some(now);
marked = true;
}
marked
}
}
/// The resolved changes to forget so no more than [`MAX_CHANGES`] are kept
/// once `adding` more are: oldest first, those with no old store kept for a
/// cleanup before those with one, and never one whose cleanup runs (OC-11).
fn to_forget(changes: &[BlobStoreChange], adding: usize, now: u64) -> Vec<u32> {
let over = (changes.len() + adding).saturating_sub(MAX_CHANGES);
let mut resolved = changes
.iter()
.filter(|c| c.stage == Stage::Changed && !c.status.is_unresolved() && !c.cleaning_up(now))
.collect::<Vec<_>>();
resolved.sort_by_key(|c| (c.old_settings.is_some(), c.id));
resolved.into_iter().take(over).map(|c| c.id).collect()
}
struct Json<T>(T);
@@ -524,6 +702,7 @@ pub async fn at_start(
copied_ahead: None,
applied_at: None,
applied_by: None,
cleanup: None,
}
}
};
@@ -531,15 +710,9 @@ pub async fn at_start(
started = Started::Changed(change.id);
// The oldest resolved changes past MAX_CHANGES are forgotten
let over =
(changes.len() + usize::from(plan.is_none())).saturating_sub(MAX_CHANGES);
for old in changes
.iter()
.filter(|c| c.stage == Stage::Changed && !c.status.is_unresolved())
.take(over)
{
batch.clear(class(old.id));
forgotten.push(old.id);
for old in to_forget(&changes, usize::from(plan.is_none()), now) {
batch.clear(class(old));
forgotten.push(old);
}
}
}
@@ -650,6 +823,7 @@ pub async fn plan(
copied_ahead: None,
applied_at: None,
applied_by: None,
cleanup: None,
};
let mut batch = BatchBuilder::new();
@@ -657,15 +831,10 @@ pub async fn plan(
batch.assert_value(location_class(), AssertValue::Hash(version));
batch.assert_value(class(id), AssertValue::None);
batch.set(class(id), Json(&change).serialize()?);
let over = (changes.len() + 1).saturating_sub(MAX_CHANGES);
let mut forgotten = Vec::new();
for old in changes
.iter()
.filter(|c| c.stage == Stage::Changed && !c.status.is_unresolved())
.take(over)
{
batch.clear(class(old.id));
forgotten.push(old.id);
for old in to_forget(&changes, 1, store::write::now()) {
batch.clear(class(old));
forgotten.push(old);
}
match data.write(batch.build_all()).await {
Ok(_) => {
@@ -713,29 +882,8 @@ pub async fn start_copy(
let mut attempt = 0;
loop {
attempt += 1;
if let Some(current) = running(data).await? {
match get(data, current).await? {
Some(c) if c.status_at(now) == Status::Copying => {
return Ok(Err(Refusal::Running(current)));
}
Some(c) => {
if c.status == Status::Copying {
update(data, current, |c| {
if c.status != Status::Copying {
return false;
}
c.status = Status::Interrupted;
if let Some(copy) = &mut c.copy {
copy.finished_at = Some(now);
}
true
})
.await?;
}
release(data, current).await?;
}
None => release(data, current).await?,
}
if let Some(current) = busy_with(data, now).await? {
return Ok(Err(Refusal::Running(current)));
}
let Some(Versioned(change, version)) = data
@@ -780,6 +928,174 @@ pub async fn start_copy(
}
}
/// BM-5, OC-3: the change a copy or cleanup runs for now, if one runs. One
/// whose node went quiet is marked interrupted and lets another run.
async fn busy_with(data: &Store, now: u64) -> trc::Result<Option<u32>> {
let Some(current) = running(data).await? else {
return Ok(None);
};
match get(data, current).await? {
Some(c) if c.busy(now) => return Ok(Some(current)),
Some(_) => {
update(data, current, |c| c.mark_interrupted(now)).await?;
}
None => {}
}
release(data, current).await?;
Ok(None)
}
/// OC-3: why a cleanup can't start. The checks of the stores themselves
/// (the old one opens, the two don't overlap) are the caller's.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CleanupRefusal {
NotFound,
/// A copy or cleanup of this change runs now.
Running(u32),
/// The change isn't resolved by a copy yet.
NotResolved,
/// The blob store went back to the change's old store (BM-4).
Reverted,
/// Its old store's settings weren't kept (OC-1), or a cleanup is done.
NoOldStore,
/// The change's new store isn't the blob store now, or its old store is.
NotCurrent,
/// This planned change copies ahead to the old store.
PlannedThere(u32),
/// This node has its blob store at this other place.
NodeElsewhere(u64, String),
}
/// OC-3: marks a change as being cleaned up by `cleanup` and makes it the
/// one that runs.
pub async fn start_cleanup(
data: &Store,
id: u32,
cleanup: Cleanup,
now: u64,
) -> trc::Result<Result<BlobStoreChange, CleanupRefusal>> {
let mut attempt = 0;
loop {
attempt += 1;
if let Some(current) = busy_with(data, now).await? {
return Ok(Err(CleanupRefusal::Running(current)));
}
let Some(Versioned(change, version)) = data
.get_value::<Versioned<BlobStoreChange>>(ValueKey::from(class(id)))
.await
.caused_by(trc::location!())?
else {
return Ok(Err(CleanupRefusal::NotFound));
};
if let Some(refusal) = cleanup_refusal(data, &change, now).await? {
return Ok(Err(refusal));
}
let change = BlobStoreChange {
cleanup: Some(cleanup.clone()),
..change
};
let mut batch = BatchBuilder::new();
batch.assert_value(class(id), AssertValue::Hash(version));
batch.assert_value(running_class(), AssertValue::None);
batch.set(class(id), Json(&change).serialize()?);
batch.set(running_class(), SerializeInfallible::serialize(&id));
match data.write(batch.build_all()).await {
Ok(_) => {
remove_problems(data, id).await?;
return Ok(Ok(change));
}
Err(err) if attempt < WRITE_ATTEMPTS && is_assert_failure(&err) => {}
Err(err) => return Err(err.caused_by(trc::location!())),
}
}
}
/// OC-3: what stops a change's old store being cleaned up, if anything.
pub async fn cleanup_refusal(
data: &Store,
change: &BlobStoreChange,
now: u64,
) -> trc::Result<Option<CleanupRefusal>> {
if change.stage != Stage::Changed {
return Ok(Some(CleanupRefusal::NotResolved));
}
match change.status {
Status::Copied => {}
Status::Reverted => return Ok(Some(CleanupRefusal::Reverted)),
_ => return Ok(Some(CleanupRefusal::NotResolved)),
}
if change.old_settings.is_none() {
return Ok(Some(CleanupRefusal::NoOldStore));
}
let here = location(data).await?;
if here
.as_ref()
.is_none_or(|here| here.location != change.to || here.location == change.from)
{
return Ok(Some(CleanupRefusal::NotCurrent));
}
if let Some(plan) = all(data)
.await?
.into_iter()
.find(|c| c.stage == Stage::Planned && c.to == change.from)
{
return Ok(Some(CleanupRefusal::PlannedThere(plan.id)));
}
if let Some((node, place)) = node_places(data)
.await?
.into_iter()
.find(|(_, p)| p.at + NODE_QUIET_AFTER >= now && p.location != change.to)
{
return Ok(Some(CleanupRefusal::NodeElsewhere(node, place.location)));
}
Ok(None)
}
fn node_class(node_id: u64) -> ValueClass {
let mut key = Vec::with_capacity(2 + U64_LEN);
key.extend_from_slice(&[FEATURE, KIND_NODE]);
key.extend_from_slice(&node_id.to_be_bytes());
any(key)
}
/// OC-2: notes where this node's blob store is, now.
pub async fn note_node(data: &Store, node_id: u64, location: &str, now: u64) -> trc::Result<()> {
let place = NodePlace {
location: location.to_string(),
at: now,
};
let mut batch = BatchBuilder::new();
batch.set(node_class(node_id), Json(&place).serialize()?);
data.write(batch.build_all())
.await
.caused_by(trc::location!())
.map(|_| ())
}
/// OC-2: where each node last said its blob store is, by node id.
pub async fn node_places(data: &Store) -> trc::Result<Vec<(u64, NodePlace)>> {
let mut places = Vec::new();
data.iterate(
IterateParams::new(
ValueKey::from(node_class(0)),
ValueKey::from(node_class(u64::MAX)),
),
|key, value| {
let node = key
.get(key.len().saturating_sub(U64_LEN)..)
.and_then(|b| b.try_into().ok())
.map(u64::from_be_bytes);
if let (Some(node), Ok(Json(place))) = (node, Json::<NodePlace>::deserialize(value)) {
places.push((node, place));
}
Ok(true)
},
)
.await
.caused_by(trc::location!())?;
Ok(places)
}
/// Lets another copy run, if `id` is the one that runs now.
pub async fn release(data: &Store, id: u32) -> trc::Result<()> {
let mut batch = BatchBuilder::new();
@@ -1210,6 +1526,7 @@ mod tests {
class(1),
problem_class(1, 0),
running_class(),
node_class(7),
] {
let ValueClass::Any(any) = class else {
unreachable!()
@@ -1262,6 +1579,7 @@ mod tests {
copied_ahead: None,
applied_at: None,
applied_by: None,
cleanup: None,
};
assert_eq!(change.status_at(1_000 + STALE_AFTER), Status::Copying);
assert_eq!(change.status_at(1_001 + STALE_AFTER), Status::Interrupted);
@@ -1283,9 +1601,196 @@ mod tests {
copied_ahead: None,
applied_at: None,
applied_by: None,
cleanup: None,
};
let bytes = Json(&change).serialize().unwrap();
let Json(back) = Json::<BlobStoreChange>::deserialize(&bytes).unwrap();
assert_eq!(back, change);
}
fn a_cleanup(at: u64) -> Cleanup {
Cleanup {
status: CleanupStatus::Running,
dry_run: false,
started_at: at,
started_by: "admin".into(),
node_id: 1,
blobs_per_second: 0,
bytes_per_second: 0,
heartbeat: at,
finished_at: None,
stop_requested: false,
progress: CleanupProgress::default(),
problems_kept: 0,
snapshots_before: 0,
error: None,
}
}
#[tokio::test]
async fn a_resolved_change_is_cleaned_up_once() {
let data = store::backend::ephemeral::EphemeralStore::open();
let a = fs("/srv/old");
let b = structs::BlobStore::Default;
let (here_a, here_b) = ("old".to_string(), "the data store".to_string());
at_start(&data, here_a.clone(), &a, 1, 10).await.unwrap();
assert_eq!(
at_start(&data, here_b.clone(), &b, 1, 20).await.unwrap(),
Started::Changed(1)
);
// OC-3: not before it's resolved
assert_eq!(
start_cleanup(&data, 1, a_cleanup(30), 30).await.unwrap(),
Err(CleanupRefusal::NotResolved)
);
assert_eq!(
start_cleanup(&data, 9, a_cleanup(30), 30).await.unwrap(),
Err(CleanupRefusal::NotFound)
);
// OC-1: resolved, with the old store's settings kept; nothing read
// through
update(&data, 1, |c| {
c.status = Status::Copied;
true
})
.await
.unwrap();
assert!(unresolved(&data).await.unwrap().is_empty());
assert_eq!(
get(&data, 1).await.unwrap().unwrap().old_settings,
Some(a.clone())
);
// OC-2, OC-3: a node still on the old store refuses it; one quiet
// for long enough doesn't
note_node(&data, 2, &here_a, 40).await.unwrap();
note_node(&data, 1, &here_b, 40).await.unwrap();
assert_eq!(
start_cleanup(&data, 1, a_cleanup(40), 40).await.unwrap(),
Err(CleanupRefusal::NodeElsewhere(2, here_a.clone()))
);
let later = 41 + NODE_QUIET_AFTER;
note_node(&data, 1, &here_b, later).await.unwrap();
assert_eq!(node_places(&data).await.unwrap().len(), 2);
// OC-3: not while a planned change copies ahead to the old store
plan(&data, here_a.clone(), a.clone())
.await
.unwrap()
.unwrap();
assert_eq!(
start_cleanup(&data, 1, a_cleanup(later), later)
.await
.unwrap(),
Err(CleanupRefusal::PlannedThere(2))
);
remove(&data, 2).await.unwrap();
// Started: one at a time, and it can't be copied meanwhile
let started = start_cleanup(&data, 1, a_cleanup(later), later)
.await
.unwrap()
.unwrap();
assert!(started.cleaning_up(later));
assert_eq!(running(&data).await.unwrap(), Some(1));
assert_eq!(
start_cleanup(&data, 1, a_cleanup(later), later + 1)
.await
.unwrap(),
Err(CleanupRefusal::Running(1))
);
// Its node went quiet: interrupted, and it can run again
let quiet = later + STALE_AFTER + 1;
let again = start_cleanup(&data, 1, a_cleanup(quiet), quiet)
.await
.unwrap()
.unwrap();
assert!(again.cleaning_up(quiet));
// Done: the old store's settings go, and there's nothing to clean
update(&data, 1, |c| {
if let Some(cleanup) = &mut c.cleanup {
cleanup.status = CleanupStatus::Done;
}
c.old_settings = None;
true
})
.await
.unwrap();
release(&data, 1).await.unwrap();
assert_eq!(
start_cleanup(&data, 1, a_cleanup(quiet), quiet + 1)
.await
.unwrap(),
Err(CleanupRefusal::NoOldStore)
);
}
#[tokio::test]
async fn not_once_the_blob_store_is_back_in_the_old_store() {
let data = store::backend::ephemeral::EphemeralStore::open();
let a = fs("/srv/old");
let b = structs::BlobStore::Default;
at_start(&data, "old".into(), &a, 1, 10).await.unwrap();
at_start(&data, "the data store".into(), &b, 1, 20)
.await
.unwrap();
update(&data, 1, |c| {
c.status = Status::Copied;
true
})
.await
.unwrap();
// B back to A: the copied change's old store is the blob store again
at_start(&data, "old".into(), &a, 1, 30).await.unwrap();
assert_eq!(
start_cleanup(&data, 1, a_cleanup(40), 40).await.unwrap(),
Err(CleanupRefusal::NotCurrent)
);
}
#[test]
fn forgetting_keeps_old_stores_kept_for_a_cleanup_last() {
let change = |id: u32, kept: bool, cleanup: Option<Cleanup>| BlobStoreChange {
id,
from: "a".into(),
to: "b".into(),
changed_at: 1,
node_id: 1,
status: Status::Copied,
old_settings: kept.then(|| structs::BlobStore::Default),
copy: None,
stage: Stage::Changed,
target_settings: None,
copied_ahead: None,
applied_at: None,
applied_by: None,
cleanup,
};
let mut changes = vec![
change(1, true, None),
change(2, false, None),
change(3, true, Some(a_cleanup(100))),
];
changes.extend((4..=MAX_CHANGES as u32).map(|id| change(id, false, None)));
// One more to add: the oldest without a kept store goes first
assert_eq!(to_forget(&changes, 1, 100), [2]);
assert_eq!(
to_forget(&changes, 1 + MAX_CHANGES - 2, 100).last(),
Some(&1)
);
// One being cleaned up is never forgotten
assert!(!to_forget(&changes, MAX_CHANGES, 100).contains(&3));
}
#[test]
fn a_change_from_before_cleanups_reads_without_one() {
let json = br#"{"id":1,"from":"a","to":"b","changedAt":1,"nodeId":1,"status":"copied"}"#;
let Json(change) = Json::<BlobStoreChange>::deserialize(json).unwrap();
assert_eq!(change.cleanup, None);
assert!(!change.busy(10));
}
}
@@ -9,7 +9,9 @@
//! BM-2). The server records them; an update with `action: "copy"` copies
//! what the new store lacks from the old one, `action: "stop"` stops that,
//! and destroying one dismisses it (BM-5 to BM-7). A create with a `target`
//! plans one, to copy ahead and `apply` (blob-store-copy-ahead spec).
//! plans one, to copy ahead and `apply` (blob-store-copy-ahead spec). On a
//! resolved change, `action: "cleanUp"` deletes from the old store what the
//! current one has (blob-store-cleanup spec).
use crate::object::{AnyId, JmapObject, JmapObjectId};
use jmap_tools::{Element, Key, Property};
@@ -49,6 +51,10 @@ pub enum BlobStoreChangeProperty {
CopiedAhead,
AppliedAt,
AppliedBy,
OldStoreKept,
Cleanup,
DryRun,
Confirm,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
@@ -94,6 +100,10 @@ impl Property for BlobStoreChangeProperty {
BlobStoreChangeProperty::CopiedAhead => "copiedAhead",
BlobStoreChangeProperty::AppliedAt => "appliedAt",
BlobStoreChangeProperty::AppliedBy => "appliedBy",
BlobStoreChangeProperty::OldStoreKept => "oldStoreKept",
BlobStoreChangeProperty::Cleanup => "cleanup",
BlobStoreChangeProperty::DryRun => "dryRun",
BlobStoreChangeProperty::Confirm => "confirm",
}
.into()
}
@@ -130,6 +140,10 @@ impl BlobStoreChangeProperty {
b"copiedAhead" => BlobStoreChangeProperty::CopiedAhead,
b"appliedAt" => BlobStoreChangeProperty::AppliedAt,
b"appliedBy" => BlobStoreChangeProperty::AppliedBy,
b"oldStoreKept" => BlobStoreChangeProperty::OldStoreKept,
b"cleanup" => BlobStoreChangeProperty::Cleanup,
b"dryRun" => BlobStoreChangeProperty::DryRun,
b"confirm" => BlobStoreChangeProperty::Confirm,
)
}
}
@@ -147,7 +161,7 @@ impl Element for BlobStoreChangeValue {
fn try_parse<P>(key: &Key<'_, Self::Property>, value: &str) -> Option<Self> {
match key {
Key::Property(BlobStoreChangeProperty::Id) => {
Key::Property(BlobStoreChangeProperty::Id | BlobStoreChangeProperty::Confirm) => {
Id::from_str(value).ok().map(BlobStoreChangeValue::Id)
}
_ => None,
+172 -24
View File
@@ -8,14 +8,16 @@
//! store the server noticed at start-up, copying what the new store lacks
//! from the old one, stopping that, and dismissing a change. A create plans
//! one, which is copied ahead and applied to the setting
//! (blob-store-copy-ahead spec). The copy is `services::inbuxa_blob_move`.
//! Planning, copying, applying, stopping and dismissing are audited by the
//! request layer (BM-9, CA-9).
//! (blob-store-copy-ahead spec). A resolved change's old store is cleaned
//! up, or a cleanup previewed (blob-store-cleanup spec). The copy is
//! `services::inbuxa_blob_move`, the cleanup `services::inbuxa_blob_cleanup`.
//! Planning, copying, applying, cleaning up, stopping and dismissing are
//! audited by the request layer (BM-9, CA-9, OC-12).
use super::relay::json_to_value;
use common::{Server, auth::AccessToken};
use inbuxa_features::blob_move::{
self as model, BlobStoreChange as Stored, PlanRefusal, Refusal, Stage, Status,
self as model, BlobStoreChange as Stored, CleanupRefusal, PlanRefusal, Refusal, Stage, Status,
};
use jmap_proto::{
error::set::SetError,
@@ -35,7 +37,11 @@ use registry::schema::{
structs,
};
use serde_json::json;
use services::inbuxa_blob_move::{self as copies, NotPlanned};
use services::{
inbuxa_blob_cleanup::{self as cleanups, NotCleaned},
inbuxa_blob_move::{self as copies, NotPlanned},
};
use std::str::FromStr;
use store::{
registry::write::{RegistryWrite, RegistryWriteResult},
write::now,
@@ -71,6 +77,8 @@ const ALL: &[P] = &[
P::CopiedAhead,
P::AppliedAt,
P::AppliedBy,
P::OldStoreKept,
P::Cleanup,
];
/// BM-9: a change is about the whole server's blobs, so only the server's
@@ -124,6 +132,40 @@ async fn to_value(server: &Server, c: &Stored, properties: &[P], now: u64) -> tr
None => serde_json::Value::Null,
},
P::AppliedAt => c.applied_at.map_or(serde_json::Value::Null, date),
// OC-1: whether there's an old store to clean up, never its settings
P::OldStoreKept => json!(c.stage == Stage::Changed && c.old_settings.is_some()),
P::Cleanup => match &c.cleanup {
Some(cleanup) => {
let p = &cleanup.progress;
let status = cleanup.status_at(now);
json!({
"status": status.as_str(),
"dryRun": cleanup.dry_run,
"startedAt": date(cleanup.started_at),
"startedBy": cleanup.started_by,
"finishedAt": cleanup.finished_at.map_or(serde_json::Value::Null, date),
"node": cleanup.node_id.to_string(),
"blobsPerSecond": cleanup.blobs_per_second,
"bytesPerSecond": cleanup.bytes_per_second,
"progress": match status {
model::CleanupStatus::Done => 1.0,
_ => p.share(),
},
"blobsChecked": p.blobs_checked,
"deleted": p.deleted,
"bytesFreed": p.bytes_freed,
"notThere": p.not_there,
"missing": p.missing,
"damaged": p.damaged,
"different": p.different,
"unreadable": p.unreadable,
"problems": problems(server, c.id).await?,
"snapshotsBefore": cleanup.snapshots_before,
"error": cleanup.error,
})
}
None => serde_json::Value::Null,
},
P::AppliedBy => json!(c.applied_by),
P::Status => json!(status.as_str()),
P::CopyStartedAt => copy.map_or(serde_json::Value::Null, |c| date(c.started_at)),
@@ -149,28 +191,66 @@ async fn to_value(server: &Server, c: &Stored, properties: &[P], now: u64) -> tr
P::Missing => json!(p.missing),
P::Damaged => json!(p.damaged),
P::Unreadable => json!(p.unreadable),
P::Problems => {
let problems = model::problems(server.store(), c.id).await?;
let mut list = serde_json::to_value(&problems).unwrap_or_default();
// The account as a JMAP id, like every other account id
if let Some(items) = list.as_array_mut() {
for item in items {
if let Some(account) = item.get("accountId").and_then(|v| v.as_u64()) {
item["accountId"] = json!(Id::from(account).to_string());
}
}
}
list
}
// A cleanup's problems are its own, under cleanup (OC-6)
P::Problems if c.cleanup.is_some() => json!([]),
P::Problems => problems(server, c.id).await?,
P::Error => json!(copy.and_then(|c| c.error.clone())),
// Write-only; a planned store's settings are never shown (CA-1)
P::Action | P::Target => continue,
P::Action | P::Target | P::DryRun | P::Confirm => continue,
};
out.insert_unchecked(Key::Property(property.clone()), json_to_value(json));
}
Ok(Value::Object(out))
}
/// The kept problems of a change's latest copy or cleanup.
async fn problems(server: &Server, id: u32) -> trc::Result<serde_json::Value> {
let problems = model::problems(server.store(), id).await?;
let mut list = serde_json::to_value(&problems).unwrap_or_default();
// The account as a JMAP id, like every other account id
if let Some(items) = list.as_array_mut() {
for item in items {
if let Some(account) = item.get("accountId").and_then(|v| v.as_u64()) {
item["accountId"] = json!(Id::from(account).to_string());
}
}
}
Ok(list)
}
/// OC-3: why a cleanup was refused, for the administrator.
fn cleanup_refused(refusal: CleanupRefusal) -> SetError<P> {
let why = match refusal {
CleanupRefusal::NotFound => return SetError::not_found(),
CleanupRefusal::Running(id) => format!(
"A copy or cleanup of change {} runs now; one runs at a time. Stop it or wait for it.",
Id::from(id)
),
CleanupRefusal::NotResolved => {
"Only a change whose copy ended copied can be cleaned up.".into()
}
CleanupRefusal::Reverted => {
"The blob store went back to this change's old store; there's nothing to clean up."
.into()
}
CleanupRefusal::NoOldStore => "The server doesn't keep this change's old store any \
more: its cleanup is done, or the change was resolved before old stores were kept."
.into(),
CleanupRefusal::NotCurrent => "This change's new store isn't the blob store now, so \
the old store may still be needed."
.into(),
CleanupRefusal::PlannedThere(id) => format!(
"Change {} is planned to copy ahead to this store. Dismiss it first.",
Id::from(id)
),
CleanupRefusal::NodeElsewhere(node, place) => format!(
"Node {node} still has its blob store at {place}. Restart it with the new setting \
first."
),
};
SetError::forbidden().with_description(why)
}
/// `inbuxa:BlobStoreChange/get`: every change kept, newest first.
pub async fn get(
server: &Server,
@@ -379,6 +459,8 @@ pub async fn set(
let mut action = None;
let mut blobs_per_second = None;
let mut bytes_per_second = None;
let mut dry_run = false;
let mut confirm = None;
for (key, value) in value.into_expanded_object() {
let outcome = match &key {
Key::Property(P::Action) => {
@@ -391,9 +473,29 @@ pub async fn set(
Key::Property(P::BytesPerSecond) => {
limit(P::BytesPerSecond, &value).map(|l| bytes_per_second = Some(l))
}
Key::Property(P::DryRun) => match value {
Value::Bool(b) => {
dry_run = b;
Ok(())
}
_ => Err(invalid(P::DryRun, "true or false.")),
},
Key::Property(P::Confirm) => match value {
Value::Element(BlobStoreChangeValue::Id(confirmed)) => {
confirm = Some(confirmed);
Ok(())
}
Value::Str(text) => {
confirm = Id::from_str(&text).ok();
Ok(())
}
_ => Err(invalid(P::Confirm, "The change's own id.")),
},
_ => Err(SetError::invalid_properties()
.with_property(key.clone().into_owned())
.with_description("A change can't be edited, only copied or stopped.")),
.with_description(
"A change can't be edited, only copied, cleaned up or stopped.",
)),
};
if let Err(error) = outcome {
response.not_updated.append(id, error);
@@ -480,11 +582,52 @@ pub async fn set(
),
}
}
Some("stop") => {
if change.status_at(now()) != Status::Copying {
Some("cleanUp") => {
// OC-8: backups taken before the change still need the old store
if !dry_run && confirm != Some(id) {
response.not_updated.append(
id,
SetError::forbidden().with_description("That change isn't being copied."),
SetError::forbidden().with_description(
"Backups of the data store taken before this change was resolved \
need the old store's blobs to restore in full. Keep the old store \
until those backups have aged out, then set confirm to this \
change's id. A preview (dryRun) needs no confirmation.",
),
);
continue;
}
match cleanups::start(
server,
change_id,
dry_run,
blobs_per_second,
bytes_per_second,
actor.name.clone(),
)
.await?
{
Ok(()) => response.updated.append(id, None),
Err(NotCleaned::Refused(refusal)) => {
response.not_updated.append(id, cleanup_refused(refusal))
}
Err(NotCleaned::Unusable(why)) => response
.not_updated
.append(id, SetError::forbidden().with_description(why)),
Err(NotCleaned::Overlap) => response.not_updated.append(
id,
SetError::forbidden().with_description(
"The old store and the blob store share a place, so deleting from \
one would delete from the other. Nothing was deleted.",
),
),
}
}
Some("stop") => {
if !change.busy(now()) {
response.not_updated.append(
id,
SetError::forbidden()
.with_description("That change isn't being copied or cleaned up."),
);
continue;
}
@@ -493,7 +636,7 @@ pub async fn set(
}
_ => response.not_updated.append(
id,
invalid(P::Action, "The actions are copy, stop and apply."),
invalid(P::Action, "The actions are copy, stop, apply and cleanUp."),
),
}
}
@@ -509,6 +652,11 @@ pub async fn set(
id,
SetError::forbidden().with_description("It's being copied; stop the copy first."),
),
Some(c) if c.cleaning_up(now()) => response.not_destroyed.append(
id,
SetError::forbidden()
.with_description("Its old store is being cleaned up; stop that first."),
),
Some(_) => {
model::remove(data, change_id).await?;
// BM-7: this node stops reading through it at once, or
+531
View File
@@ -0,0 +1,531 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! Cleaning up the old store after a blob store change
//! (docs/spec/features/blob-store-cleanup.md). Once a change is resolved,
//! this goes through the blob index as the copy does and deletes from the
//! old store each blob it has read back from the current one and checked
//! (OC-4). A blob that fails the check stays where it is and is reported.
//! A preview does the same and deletes nothing (OC-5). It runs on the node
//! it was started on, one run at a time across the server with the copies.
use crate::{
inbuxa_blob_move::{Holder, Pace, key_name, node_id, old_store},
inbuxa_store_snapshot,
};
use common::{
Server,
manager::{SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY},
};
use inbuxa_features::{
audit::{Action, Actor, Outcome, Record, Target},
blob_move::{
self as model, BlobStoreChange, Cleanup, CleanupProgress, CleanupRefusal, CleanupStatus,
DEFAULT_BLOBS_PER_SECOND, DEFAULT_BYTES_PER_SECOND, MAX_PROBLEMS, Problem, ProblemKind,
},
};
use std::time::{Duration, Instant};
use store::{BlobStore, build::blob::blob_store_places, write::now};
use types::{blob_hash::BlobHash, id::Id};
/// How often a running cleanup saves its progress, which is also its
/// heartbeat and when it sees a stop.
const SAVE_EVERY: Duration = Duration::from_secs(10);
/// OC-3: written to the current store and looked for in the old one, to
/// tell two settings that reach the same place.
const OVERLAP_PROBE_KEY: &[u8] = b"INBUXA_BLOB_STORE_OVERLAP_PROBE";
/// OC-3: why a cleanup didn't start.
pub enum NotCleaned {
Refused(CleanupRefusal),
/// The old store can't be opened: the reason.
Unusable(String),
/// Deleting from the old store would delete from the current one.
Overlap,
}
/// OC-3, OC-5: starts a cleanup, or with `dry_run` a preview, of a change's
/// old store on this node. The confirmation (OC-8) is the caller's.
pub async fn start(
server: &Server,
id: u32,
dry_run: bool,
blobs_per_second: Option<u64>,
bytes_per_second: Option<u64>,
started_by: String,
) -> trc::Result<Result<(), NotCleaned>> {
let data = server.store();
let now = now();
let Some(change) = model::get(data, id).await? else {
return Ok(Err(NotCleaned::Refused(CleanupRefusal::NotFound)));
};
if let Some(refusal) = model::cleanup_refusal(data, &change, now).await? {
return Ok(Err(NotCleaned::Refused(refusal)));
}
let old = match old_store(server, &change).await {
Ok(old) => old,
Err(err) => {
return Ok(Err(NotCleaned::Unusable(format!(
"The old blob store couldn't be opened: {err}"
))));
}
};
let current = server.blob_store().current().clone();
match overlaps(data, &change, &old, &current).await {
Ok(false) => {}
Ok(true) => return Ok(Err(NotCleaned::Overlap)),
Err(why) => return Ok(Err(NotCleaned::Unusable(why))),
}
let snapshots_before = inbuxa_store_snapshot::list(server)
.await
.iter()
.filter(|s| {
s.status == inbuxa_store_snapshot::Status::Done && s.created_at < change.changed_at
})
.count() as u64;
let cleanup = Cleanup {
status: CleanupStatus::Running,
dry_run,
started_at: now,
started_by,
node_id: node_id(server),
blobs_per_second: blobs_per_second.unwrap_or(DEFAULT_BLOBS_PER_SECOND),
bytes_per_second: bytes_per_second.unwrap_or(DEFAULT_BYTES_PER_SECOND),
heartbeat: now,
finished_at: None,
stop_requested: false,
progress: CleanupProgress::default(),
problems_kept: 0,
snapshots_before,
error: None,
};
let change = match model::start_cleanup(data, id, cleanup, now).await? {
Ok(change) => change,
Err(refusal) => return Ok(Err(NotCleaned::Refused(refusal))),
};
let server = server.clone();
tokio::spawn(async move {
run(&server, change, old, current).await;
});
Ok(Ok(()))
}
/// OC-3: whether the two stores share a place: one a sharded set with the
/// other among its members, or a probe written to the current store found
/// in the old one.
async fn overlaps(
data: &store::Store,
change: &BlobStoreChange,
old: &BlobStore,
current: &BlobStore,
) -> Result<bool, String> {
let here = model::location(data)
.await
.map_err(|err| format!("Couldn't read where the blob store is: {err}"))?;
if let (Some(old_settings), Some(here)) = (&change.old_settings, &here) {
let old_places = blob_store_places(old_settings);
if blob_store_places(&here.settings)
.iter()
.any(|place| old_places.contains(place))
{
return Ok(true);
}
}
probe_overlap(old, current).await
}
/// OC-3: writes a probe to `current` and looks for it in `old`.
pub(crate) async fn probe_overlap(old: &BlobStore, current: &BlobStore) -> Result<bool, String> {
let written = format!("inbuxa {}", now()).into_bytes();
current
.put_stored(OVERLAP_PROBE_KEY, &written)
.await
.map_err(|err| format!("A test blob couldn't be written to the blob store: {err}"))?;
let seen = old.get_stored(OVERLAP_PROBE_KEY).await;
let deleted = current.delete_blob(OVERLAP_PROBE_KEY).await;
let seen = seen
.map_err(|err| format!("The old blob store couldn't be read: {err}"))?
.is_some_and(|read| read == written);
deleted.map_err(|err| format!("A test blob couldn't be deleted from the blob store: {err}"))?;
Ok(seen)
}
/// How a run ended before it finished.
enum Halt {
Stopped,
Failed(trc::Error),
}
impl From<trc::Error> for Halt {
fn from(err: trc::Error) -> Self {
Halt::Failed(err)
}
}
struct Run<'x> {
server: &'x Server,
id: u32,
dry_run: bool,
old: BlobStore,
current: BlobStore,
progress: CleanupProgress,
found: Vec<Problem>,
kept: u64,
blobs: Pace,
bytes: Pace,
saved: Instant,
}
impl Run<'_> {
/// OC-4: counts a problem, logs it, and keeps the first ones. The blob
/// stays in the old store.
fn problem(&mut self, kind: ProblemKind, key: &[u8], holder: &Holder, detail: String) {
self.progress.count(kind);
let blob_id = key_name(key);
trc::event!(
Store(trc::StoreEvent::DataCorruption),
Id = self.id,
AccountId = holder.account_id,
Collection = holder.name.clone(),
DocumentId = holder.object_id,
BlobId = blob_id.clone(),
Reason = kind.as_str(),
Details = detail.clone(),
);
if self.kept < MAX_PROBLEMS {
self.kept += 1;
self.found.push(Problem {
kind,
blob_id,
holder: holder.name.clone(),
account_id: holder.account_id,
object_id: holder.object_id,
detail,
});
}
}
/// Saves progress and the problems found since, now and then; a stop
/// asked for meanwhile ends the run.
async fn save(&mut self, force: bool) -> Result<(), Halt> {
if !force && self.saved.elapsed() < SAVE_EVERY {
return Ok(());
}
let data = self.server.store();
let first = self.kept - self.found.len() as u64;
model::add_problems(data, self.id, first, &self.found).await?;
self.found.clear();
let progress = self.progress.clone();
let kept = self.kept;
let saved = model::update(data, self.id, |c| {
if let Some(cleanup) = &mut c.cleanup {
cleanup.progress = progress.clone();
cleanup.problems_kept = kept;
cleanup.heartbeat = now();
}
true
})
.await?;
self.saved = Instant::now();
match saved {
Some(c) if c.cleanup.as_ref().is_some_and(|c| c.stop_requested) => Err(Halt::Stopped),
Some(_) => Ok(()),
None => Err(Halt::Stopped),
}
}
/// OC-4: one blob, deleted from the old store once the current one is
/// checked to have it.
async fn one(&mut self, key: &[u8], holder: Holder) -> Result<(), Halt> {
self.blobs.take(1).await;
self.progress.blobs_checked += 1;
match self.old.stored_present(key).await {
Ok(true) => {}
Ok(false) => {
self.progress.not_there += 1;
return self.save(false).await;
}
Err(err) => {
self.problem(
ProblemKind::Unreadable,
key,
&holder,
format!("the old blob store couldn't say: {err}"),
);
return self.save(false).await;
}
}
let kept = match self.current.get_stored(key).await {
Ok(Some(kept)) => kept,
Ok(None) => {
self.problem(
ProblemKind::Missing,
key,
&holder,
"the current blob store doesn't have it".into(),
);
return self.save(false).await;
}
Err(err) => {
self.problem(
ProblemKind::Unreadable,
key,
&holder,
format!("the current blob store failed: {err}"),
);
return self.save(false).await;
}
};
let size = kept.len() as u64;
self.bytes.take(size).await;
if holder.content_named
&& let Ok(hash) = BlobHash::try_from_hash_slice(key)
{
match BlobStore::content_of_stored(key, kept) {
Ok(content) => {
let actual = BlobHash::generate(&content);
if actual != hash {
self.problem(
ProblemKind::Damaged,
key,
&holder,
format!(
"the current blob store's {} bytes hash to {}",
content.len(),
actual.to_hex()
),
);
return self.save(false).await;
}
}
Err(_) => {
self.problem(
ProblemKind::Damaged,
key,
&holder,
"the current blob store's copy doesn't decompress".into(),
);
return self.save(false).await;
}
}
} else {
match self.old.get_stored(key).await {
Ok(Some(old)) => {
self.bytes.take(old.len() as u64).await;
if old != kept {
self.problem(
ProblemKind::Different,
key,
&holder,
format!(
"the current blob store has {} bytes, the old one {}",
kept.len(),
old.len()
),
);
return self.save(false).await;
}
}
Ok(None) => {
self.progress.not_there += 1;
return self.save(false).await;
}
Err(err) => {
self.problem(
ProblemKind::Unreadable,
key,
&holder,
format!("the old blob store failed: {err}"),
);
return self.save(false).await;
}
}
}
if self.dry_run {
self.progress.deleted += 1;
self.progress.bytes_freed += size;
return self.save(false).await;
}
// Only the blob's data, from the old store itself: the blob index
// is the data store's and stays as it is
match self.old.delete_blob(key).await {
Ok(_) => {
self.progress.deleted += 1;
self.progress.bytes_freed += size;
}
Err(err) => self.problem(
ProblemKind::Unreadable,
key,
&holder,
format!("the old blob store couldn't delete it: {err}"),
),
}
self.save(false).await
}
async fn go(&mut self) -> Result<(), Halt> {
for slice in 0u8..=255 {
let blobs = self.server.store().blobs_to_copy(slice).await?;
for blob in blobs.into_iter().filter(|b| b.committed) {
let holder = Holder::of(&blob);
self.one(blob.hash.as_slice(), holder).await?;
}
self.progress.slices = slice as u32 + 1;
self.save(false).await?;
}
for key in [SPAM_TRAINER_KEY, SPAM_CLASSIFIER_KEY] {
self.one(key, Holder::fixed()).await?;
}
Ok(())
}
}
/// Runs a cleanup to its end and records how it ended (OC-6, OC-12).
async fn run(server: &Server, change: BlobStoreChange, old: BlobStore, current: BlobStore) {
let id = change.id;
let cleanup = change.cleanup.clone();
let dry_run = cleanup.as_ref().is_some_and(|c| c.dry_run);
let mut run = Run {
server,
id,
dry_run,
old,
current,
progress: CleanupProgress::default(),
found: Vec::new(),
kept: 0,
blobs: Pace::new(
cleanup
.as_ref()
.map_or(DEFAULT_BLOBS_PER_SECOND, |c| c.blobs_per_second),
),
bytes: Pace::new(
cleanup
.as_ref()
.map_or(DEFAULT_BYTES_PER_SECOND, |c| c.bytes_per_second),
),
saved: Instant::now(),
};
let (status, error) = match run.go().await {
Ok(()) if run.progress.problems() == 0 => (CleanupStatus::Done, None),
Ok(()) => (CleanupStatus::Incomplete, None),
Err(Halt::Stopped) => (CleanupStatus::Stopped, None),
Err(Halt::Failed(err)) => {
let text = err
.value_as_str(trc::Key::Details)
.map(str::to_string)
.unwrap_or_else(|| err.to_string());
trc::error!(err.details("A blob store cleanup failed"));
(CleanupStatus::Failed, Some(text))
}
};
let data = server.store();
let first = run.kept - run.found.len() as u64;
let progress = run.progress.clone();
let kept = run.kept;
let mut result = model::add_problems(data, id, first, &run.found).await;
if result.is_ok() {
result = model::update(data, id, |c| {
if let Some(cleanup) = &mut c.cleanup {
cleanup.status = status;
cleanup.progress = progress.clone();
cleanup.problems_kept = kept;
cleanup.finished_at = Some(now());
cleanup.heartbeat = now();
cleanup.error = error
.as_ref()
.map(|e| e.trim().chars().take(2_000).collect());
}
// OC-1, OC-6: done, so the old store's settings go
if status == CleanupStatus::Done && !dry_run {
c.old_settings = None;
}
true
})
.await
.map(|_| ());
}
if let Err(err) = result.and(model::release(data, id).await) {
trc::error!(err.details("Failed to save a blob store cleanup's result"));
}
// OC-12: each run's end, with its counts
let p = &run.progress;
let details = format!(
"{} of the old blob store ({}): {}, {} blobs ({} bytes) {}, {} kept, {} not there",
if dry_run {
"A preview of a cleanup"
} else {
"A cleanup"
},
change.from,
status.as_str(),
p.deleted,
p.bytes_freed,
if dry_run {
"would be deleted"
} else {
"deleted"
},
p.problems(),
p.not_there,
);
server
.audit_note(Record {
at: now() * 1000,
actor: Actor::system("BlobStoreCleanup"),
via: None,
remote_ip: None,
action: if dry_run {
Action::Verify
} else {
Action::Destroy
},
target: Target {
kind: "inbuxa:BlobStoreChange".into(),
id: Some(Id::from(id).to_string()),
name: None,
account_id: None,
tenant_id: None,
},
changes: vec![],
details: Some(details),
reason: None,
outcome: if status == CleanupStatus::Failed {
Outcome::refused("failed", error)
} else {
Outcome::success()
},
})
.await;
}
#[cfg(test)]
mod tests {
use super::*;
use registry::schema::structs;
use store::backend::fs::FsStore;
#[tokio::test]
async fn the_probe_finds_one_directory_under_two_names() {
let dir = std::env::temp_dir().join(format!("inbuxa-overlap-{}", std::process::id()));
let path = dir.to_string_lossy().to_string();
let open = |path: String| async move {
FsStore::open(structs::FileSystemStore { path, depth: 2 })
.await
.unwrap()
};
let one = open(path.clone()).await;
let same = open(format!("{path}/.")).await;
let other = open(format!("{path}-other")).await;
assert!(probe_overlap(&same, &one).await.unwrap());
assert!(!probe_overlap(&other, &one).await.unwrap());
assert!(!one.blob_present(OVERLAP_PROBE_KEY).await.unwrap());
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(format!("{path}-other"));
}
}
+82 -43
View File
@@ -11,16 +11,19 @@
//! change is copied the other way round: from the current store to the
//! planned one (blob-store-copy-ahead spec, CA-2). It runs on the node the
//! copy was started on. Also, on every node, it keeps reading through only
//! the changes still unresolved (BM-10), and deletes following to the
//! planned store, if there is one (CA-4).
//! the changes still unresolved (BM-10), deletes following to the planned
//! store, if there is one (CA-4), and notes where its blob store is
//! (blob-store-cleanup spec, OC-2). The cleanup of a resolved change's old
//! store is `inbuxa_blob_cleanup`.
use common::{
BuildServer, Inner, Server,
manager::{SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY},
};
use inbuxa_features::blob_move::{
self as model, BlobStoreChange, Copy, DEFAULT_BLOBS_PER_SECOND, DEFAULT_BYTES_PER_SECOND,
MAX_PROBLEMS, PlanRefusal, Problem, ProblemKind, Progress, Refusal, STALE_AFTER, Stage, Status,
self as model, BlobStoreChange, CleanupStatus, Copy, DEFAULT_BLOBS_PER_SECOND,
DEFAULT_BYTES_PER_SECOND, MAX_PROBLEMS, PlanRefusal, Problem, ProblemKind, Progress, Refusal,
STALE_AFTER, Stage, Status,
};
use registry::schema::structs;
use std::{
@@ -47,7 +50,7 @@ const REFRESH_EVERY: Duration = Duration::from_secs(60);
/// before anything relies on it.
const PROBE_KEY: &[u8] = b"INBUXA_BLOB_STORE_PROBE";
fn node_id(server: &Server) -> u64 {
pub(crate) fn node_id(server: &Server) -> u64 {
server.core.network.node_id
}
@@ -145,15 +148,25 @@ async fn probe(store: &BlobStore) -> Result<(), String> {
Ok(())
}
/// BM-6: asks a running copy to stop. Any node can: the one running it
/// sees the request at its next save. None if there's no such change.
/// BM-6, OC-6: asks a running copy or cleanup to stop. Any node can: the
/// one running it sees the request at its next save. None if there's no
/// such change.
pub async fn stop(server: &Server, id: u32) -> trc::Result<Option<BlobStoreChange>> {
model::update(server.store(), id, |c| match &mut c.copy {
Some(copy) if c.status == Status::Copying && !copy.stop_requested => {
copy.stop_requested = true;
true
model::update(server.store(), id, |c| {
if let Some(cleanup) = &mut c.cleanup
&& cleanup.status == CleanupStatus::Running
&& !cleanup.stop_requested
{
cleanup.stop_requested = true;
return true;
}
match &mut c.copy {
Some(copy) if c.status == Status::Copying && !copy.stop_requested => {
copy.stop_requested = true;
true
}
_ => false,
}
_ => false,
})
.await
}
@@ -165,12 +178,13 @@ pub fn spawn_blob_moves(inner: Arc<Inner>) {
tokio::spawn(async move {
let server = inner.build_server();
let node = node_id(&server);
mark_interrupted(&server, |c| c.node_id == node).await;
mark_interrupted(&server, |run, _| run == node).await;
refresh(&server).await;
loop {
tokio::time::sleep(REFRESH_EVERY).await;
let server = inner.build_server();
let cutoff = now().saturating_sub(STALE_AFTER);
mark_interrupted(&server, |c| c.node_id != node && c.heartbeat < cutoff).await;
mark_interrupted(&server, |run, heartbeat| run != node && heartbeat < cutoff).await;
refresh(&server).await;
}
});
@@ -183,6 +197,12 @@ pub async fn refresh(server: &Server) {
let Some(moving) = server.blob_store().moving() else {
return;
};
// OC-2: where this node's blob store is, for a cleanup to check
if let Some(location) = moving.location()
&& let Err(err) = model::note_node(server.store(), node_id(server), &location, now()).await
{
trc::error!(err.details("Failed to note where this node's blob store is"));
}
match model::unresolved(server.store()).await {
Ok(changes) => {
let ids = changes.iter().map(|c| c.id).collect::<Vec<_>>();
@@ -230,7 +250,9 @@ pub async fn refresh(server: &Server) {
}
}
async fn mark_interrupted(server: &Server, which: impl Fn(&Copy) -> bool) {
/// BM-6, OC-6: marks the copies and cleanups `which` picks, by their node
/// and heartbeat, as interrupted.
async fn mark_interrupted(server: &Server, which: impl Fn(u64, u64) -> bool) {
let changes = match model::all(server.store()).await {
Ok(changes) => changes,
Err(err) => {
@@ -238,36 +260,52 @@ async fn mark_interrupted(server: &Server, which: impl Fn(&Copy) -> bool) {
return;
}
};
for change in changes
.into_iter()
.filter(|c| c.status == Status::Copying && c.copy.as_ref().is_some_and(&which))
{
let copying = |c: &BlobStoreChange| {
c.status == Status::Copying
&& c.copy
.as_ref()
.is_some_and(|copy| which(copy.node_id, copy.heartbeat))
};
let cleaning = |c: &BlobStoreChange| {
c.cleanup.as_ref().is_some_and(|cleanup| {
cleanup.status == CleanupStatus::Running && which(cleanup.node_id, cleanup.heartbeat)
})
};
for change in changes.into_iter().filter(|c| copying(c) || cleaning(c)) {
let result = model::update(server.store(), change.id, |c| {
if c.status != Status::Copying || !c.copy.as_ref().is_some_and(&which) {
return false;
let mut marked = false;
if copying(c) {
c.status = Status::Interrupted;
if let Some(copy) = &mut c.copy {
copy.finished_at = Some(now());
}
marked = true;
}
c.status = Status::Interrupted;
if let Some(copy) = &mut c.copy {
copy.finished_at = Some(now());
if cleaning(c)
&& let Some(cleanup) = &mut c.cleanup
{
cleanup.status = CleanupStatus::Interrupted;
cleanup.finished_at = Some(now());
marked = true;
}
true
marked
})
.await;
if let Err(err) = result.and(model::release(server.store(), change.id).await) {
trc::error!(err.details("Failed to mark an interrupted blob copy"));
trc::error!(err.details("Failed to mark an interrupted blob copy or cleanup"));
}
}
}
/// BM-5: keeps a rate, a second at a time.
struct Pace {
pub(crate) struct Pace {
per_second: u64,
window: Instant,
used: u64,
}
impl Pace {
fn new(per_second: u64) -> Self {
pub(crate) fn new(per_second: u64) -> Self {
Pace {
per_second,
window: Instant::now(),
@@ -275,7 +313,7 @@ impl Pace {
}
}
async fn take(&mut self, amount: u64) {
pub(crate) async fn take(&mut self, amount: u64) {
if self.per_second == 0 {
return;
}
@@ -294,16 +332,16 @@ impl Pace {
}
/// Who holds a blob, as a problem names it (BM-5).
struct Holder {
name: String,
account_id: Option<u32>,
object_id: Option<u64>,
pub(crate) struct Holder {
pub(crate) name: String,
pub(crate) account_id: Option<u32>,
pub(crate) object_id: Option<u64>,
/// Named by the hash of its content, so it can be checked.
content_named: bool,
pub(crate) content_named: bool,
}
impl Holder {
fn of(blob: &LinkedBlob) -> Holder {
pub(crate) fn of(blob: &LinkedBlob) -> Holder {
match blob.holders.first() {
Some(BlobHolder::Document {
account_id,
@@ -343,7 +381,7 @@ impl Holder {
}
}
fn fixed() -> Holder {
pub(crate) fn fixed() -> Holder {
Holder {
name: "fixed".into(),
account_id: None,
@@ -380,7 +418,7 @@ impl From<trc::Error> for Halt {
}
}
fn key_name(key: &[u8]) -> String {
pub(crate) fn key_name(key: &[u8]) -> String {
match BlobHash::try_from_hash_slice(key) {
Ok(hash) => hash.to_hex(),
Err(_) => String::from_utf8_lossy(key).into_owned(),
@@ -600,7 +638,10 @@ async fn stores(
}
/// The change's old store: the one already read through, or opened now.
async fn old_store(server: &Server, change: &BlobStoreChange) -> Result<BlobStore, String> {
pub(crate) async fn old_store(
server: &Server,
change: &BlobStoreChange,
) -> Result<BlobStore, String> {
if let Some(old) = server.blob_store().moving().and_then(|m| {
m.old_stores()
.iter()
@@ -682,12 +723,10 @@ async fn run(server: &Server, change: BlobStoreChange) {
if ahead && c.stage != Stage::Planned {
return false;
}
// BM-6: resolved, and nothing is read through any more. The old
// store's settings stay for a cleanup (blob-store-cleanup spec,
// OC-1). A copy ahead resolves nothing (CA-2)
c.status = status;
// BM-6: resolved, so the old store's settings go. A copy ahead
// resolves nothing (CA-2)
if status == Status::Copied && !ahead {
c.old_settings = None;
}
if let Some(copy) = &mut c.copy {
copy.progress = progress.clone();
copy.problems_kept = kept;
+1
View File
@@ -23,6 +23,7 @@ use crate::task_manager::{manager::spawn_task_manager, scheduler::spawn_task_sch
pub mod broadcast;
// inbuxa: AL-5, delegations end at their date
pub mod inbuxa_blob_cleanup; // inbuxa: cleaning up after a blob store change
pub mod inbuxa_blob_move; // inbuxa: changing the blob store
pub mod inbuxa_deliverability; // inbuxa: the deliverability check
pub mod inbuxa_lock_expiry;
+48
View File
@@ -120,6 +120,29 @@ pub fn blob_store_location(config: &structs::BlobStore) -> String {
}
}
/// Where each store behind a blob store setting is (BM-1): the setting's
/// own place, or for a sharded set each member's, so two settings that
/// share a store can be told (blob-store-cleanup spec, OC-3).
pub fn blob_store_places(config: &structs::BlobStore) -> Vec<String> {
match config {
structs::BlobStore::Sharded(set) => set
.stores
.iter()
.map(|member| {
blob_store_location(&match member.clone() {
structs::BlobStoreBase::S3(s) => structs::BlobStore::S3(s),
structs::BlobStoreBase::Azure(s) => structs::BlobStore::Azure(s),
structs::BlobStoreBase::FileSystem(s) => structs::BlobStore::FileSystem(s),
structs::BlobStoreBase::FoundationDb(s) => structs::BlobStore::FoundationDb(s),
structs::BlobStoreBase::PostgreSql(s) => structs::BlobStore::PostgreSql(s),
structs::BlobStoreBase::MySql(s) => structs::BlobStore::MySql(s),
})
})
.collect(),
config => vec![blob_store_location(config)],
}
}
fn with_prefix(place: String, prefix: Option<&str>) -> String {
match prefix.filter(|p| !p.is_empty()) {
Some(prefix) => format!("{place}, prefix {prefix}"),
@@ -158,4 +181,29 @@ mod tests {
blob_store_location(&fs("/srv/other", 2))
);
}
#[test]
fn a_sharded_set_is_its_members_places() {
let mut stores = registry::types::list::List::with_capacity(2);
for path in ["/srv/a", "/srv/b"] {
stores.push(structs::BlobStoreBase::FileSystem(
structs::FileSystemStore {
path: path.into(),
depth: 2,
},
));
}
let set = structs::BlobStore::Sharded(structs::ShardedBlobStore { stores });
assert_eq!(
blob_store_places(&set),
[
blob_store_location(&fs("/srv/a", 2)),
blob_store_location(&fs("/srv/b", 2))
]
);
assert_eq!(
blob_store_places(&fs("/srv/a", 2)),
[blob_store_location(&fs("/srv/a", 2))]
);
}
}
+13
View File
@@ -48,6 +48,9 @@ pub struct MovingBlobStore {
/// The planned change whose store couldn't be opened, so it's reported
/// once rather than at every refresh.
planned_failed: AtomicU32,
/// Where this node's blob store is (BM-1), as the node notes it with the
/// changes it reads (blob-store-cleanup spec, OC-2).
location: ArcSwapOption<String>,
}
impl MovingBlobStore {
@@ -57,9 +60,19 @@ impl MovingBlobStore {
old: ArcSwap::from_pointee(old),
planned: ArcSwapOption::empty(),
planned_failed: AtomicU32::new(0),
location: ArcSwapOption::empty(),
}
}
/// OC-2: where this node's blob store is, once start-up has said.
pub fn location(&self) -> Option<Arc<String>> {
self.location.load_full()
}
pub fn set_location(&self, location: String) {
self.location.store(Some(Arc::new(location)));
}
/// CA-4: the planned store deletes go to, if a change is planned.
pub fn planned(&self) -> Option<Arc<PlannedBlobStore>> {
self.planned.load_full()
+1 -1
View File
@@ -390,7 +390,7 @@ Not a rebuild: the **store check** is INBUXA's own design (`features/store-check
Not a rebuild: the **store snapshot** is INBUXA's own design (`features/store-snapshot.md`, inbuxa-server #323, building). An administrator has the server take a consistent copy of its RocksDB store, and of a filesystem blob store's files, into `snapshots/` in the store's directory while mail flows, for a backup tool to copy; the latest two are kept: `inbuxa:StoreSnapshot` (`crates/jmap/src/inbuxa/store_snapshot.rs`, `crates/services/src/inbuxa_store_snapshot.rs`, `crates/store/src/write/snapshot.rs`, `crates/store/src/backend/rocksdb/checkpoint.rs`), and the `sysStoreSnapshot` permission.
Not a rebuild: **changing the blob store** is INBUXA's own design (`features/blob-store-change.md`, inbuxa-server #333, building). When a node starts with the blob store setting pointing somewhere else, it records the change and keeps the old store's settings; until the change is resolved, a blob the new store lacks is read from the old one and kept, and an administrator has one node copy the rest over, checked by hash, with what couldn't be copied reported: `inbuxa:BlobStoreChange` (`crates/jmap/src/inbuxa/blob_store_change.rs`, `crates/features/src/blob_move/`, `crates/services/src/inbuxa_blob_move.rs`, `crates/common/src/manager/blob_moves.rs`, `crates/store/src/write/blob_move.rs`), the `store.blob-read-through` event, and the `sysBlobStoreChange` permission. Planning a change and copying ahead of it (`features/blob-store-copy-ahead.md`, inbuxa-server #341, building): `inbuxa:BlobStoreChange/set` creates a planned change with the new store's settings, copied ahead and applied to `x:BlobStore`; deletes follow to it until the switch. Cleaning up the old store once a change is resolved (`features/blob-store-cleanup.md`, inbuxa-server #342, approved): `action: "cleanUp"` deletes from the old store each blob checked in the new one, after a preview and a confirmation.
Not a rebuild: **changing the blob store** is INBUXA's own design (`features/blob-store-change.md`, inbuxa-server #333, building). When a node starts with the blob store setting pointing somewhere else, it records the change and keeps the old store's settings; until the change is resolved, a blob the new store lacks is read from the old one and kept, and an administrator has one node copy the rest over, checked by hash, with what couldn't be copied reported: `inbuxa:BlobStoreChange` (`crates/jmap/src/inbuxa/blob_store_change.rs`, `crates/features/src/blob_move/`, `crates/services/src/inbuxa_blob_move.rs`, `crates/common/src/manager/blob_moves.rs`, `crates/store/src/write/blob_move.rs`), the `store.blob-read-through` event, and the `sysBlobStoreChange` permission. Planning a change and copying ahead of it (`features/blob-store-copy-ahead.md`, inbuxa-server #341, building): `inbuxa:BlobStoreChange/set` creates a planned change with the new store's settings, copied ahead and applied to `x:BlobStore`; deletes follow to it until the switch. Cleaning up the old store once a change is resolved (`features/blob-store-cleanup.md`, inbuxa-server #342, building): `action: "cleanUp"` deletes from the old store each blob checked in the new one, after a preview and a confirmation.
Not a rebuild: **the store layout, changed in steps** is INBUXA's own design (`features/store-layout.md`, approved 2026-11-08). A store carries its layout version (`Vs` in `_`); at boot one node, holding a lease, runs each step from the stored version up and marks the store after each. Step 1 → 2 gives SQL rate-limit counters their own expiry column; step 2 → 3 moves the audit log, journal, holds and undelete notes into subspaces of their own. A step stops rollback, so it ships at a series.
+4 -3
View File
@@ -26,8 +26,9 @@ read with nothing to say why.
Now the server notices the change when it starts. It keeps reading what
it can't find from the old store and copies each blob it reads that way
into the new one. An administrator has it copy everything else, and the
change is resolved once the new store has every blob. Nothing is ever
deleted from the old store; that stays the administrator's to do.
change is resolved once the new store has every blob. Nothing is
deleted from the old store unless an administrator asks for a cleanup
afterwards (`features/blob-store-cleanup.md`).
## Requirements
@@ -161,4 +162,4 @@ or `fixed`, for the first thing found holding the blob.
- Copying ahead of a change (the old store to a new one before the
setting changes): `features/blob-store-copy-ahead.md` (approved, #341).
- Deleting what a resolved change left in the old store:
`features/blob-store-cleanup.md` (approved, #342).
`features/blob-store-cleanup.md` (building, #342).
+4 -3
View File
@@ -1,7 +1,7 @@
# Feature spec: cleaning up the old store after a blob store change
Status: approved, 2026-11-10 (inbuxa-server #342, PR #357), with the
answers under [Settled](#settled). Not a rebuild (SPEC.md
Status: building, 2026-11-10 (inbuxa-server #342). Approved 2026-11-10
(PR #357), with the answers under [Settled](#settled). Not a rebuild (SPEC.md
§4): inbuxa's own design. It extends `features/blob-store-change.md`
(BM-1 to BM-10) and `features/blob-store-copy-ahead.md` (CA-1 to CA-9),
which it doesn't change except where it says so.
@@ -163,7 +163,8 @@ blobsPerSecond, bytesPerSecond, progress, blobsChecked, deleted,
bytesFreed, notThere, missing, damaged, different, unreadable, problems,
snapshotsBefore, error}`, with `status` one of `running`, `done`,
`incomplete`, `stopped`, `failed`, `interrupted`. A problem is as BM-5's,
with `different` as a kind.
with `different` as a kind. Once a change has a cleanup, its problems are the
cleanup's, under `cleanup.problems`, and the top-level `problems` is empty.
## Tests
+277 -2
View File
@@ -23,6 +23,13 @@
//! store meanwhile. Mail delivered after the copy ahead is read through once
//! the server switches, and the last pass copies only what the copy ahead
//! didn't have.
//!
//! Then cleaning up (docs/spec/features/blob-store-cleanup.md): a resolved
//! change's old store is previewed, refused without a confirmation and
//! while a node is on the old store, and cleaned up. A blob missing from
//! the current store and one damaged there stay in the old store and are
//! reported; put right, a second run is done and the old store's settings
//! go.
use crate::utils::{
account::Account,
@@ -394,8 +401,8 @@ pub async fn test(test: &mut TestServer) {
.unwrap()
.unwrap()
.old_settings
.is_none(),
"the old store's settings go once resolved"
.is_some(),
"the old store's settings are kept for a cleanup (OC-1)"
);
let response = update(admin, &id, json!({"action": "copy"})).await;
assert_eq!(
@@ -416,6 +423,7 @@ pub async fn test(test: &mut TestServer) {
use_blob_store(test, current);
copy_ahead(test).await;
clean_up(test).await;
}
async fn plan(admin: &Account, target: Value) -> Value {
@@ -706,6 +714,273 @@ async fn copy_ahead(test: &mut TestServer) {
);
}
/// The cleanup once its run is over.
async fn cleaned(admin: &Account, id: &str) -> Value {
for _ in 0..900 {
let change = get(admin, id).await;
if change["cleanup"]["status"] != "running" {
return change;
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
panic!("the cleanup didn't finish: {}", get(admin, id).await);
}
/// The blob-store-cleanup spec, OC-1 to OC-11.
async fn clean_up(test: &mut TestServer) {
println!("Running blob store cleanup tests...");
let data = test.server.store().clone();
let admin = test.account("[email protected]");
let ann = test.account("[email protected]");
let ann_id = ann.id().document_id();
let client = ann.jmap_client().await;
let current = test.server.blob_store().current().clone();
for change in model::all(&data).await.unwrap() {
model::remove(&data, change.id).await.unwrap();
}
// Mail in both stores, as a copy leaves it
let dir = format!("{}/cleanup-old-blobs", test.tmp_dir());
let fs_settings = structs::FileSystemStore {
path: dir.clone(),
depth: 2,
};
let old_settings = structs::BlobStore::FileSystem(fs_settings.clone());
let old = FsStore::open(fs_settings).await.unwrap();
let here = structs::BlobStore::Default;
let kept = import(&client, "In both stores").await;
let altered = import(&client, "Damaged in the current store").await;
let removed = import(&client, "Gone from the current store").await;
let kept_hash = blob_of(test, ann_id, kept).await;
let altered_hash = blob_of(test, ann_id, altered).await;
let removed_hash = blob_of(test, ann_id, removed).await;
for hash in [&kept_hash, &altered_hash, &removed_hash] {
let stored = current.get_stored(hash.as_slice()).await.unwrap().unwrap();
old.put_stored(hash.as_slice(), &stored).await.unwrap();
}
model::at_start(
&data,
blob_store_location(&old_settings),
&old_settings,
1,
now(),
)
.await
.unwrap();
let Started::Changed(change_id) =
model::at_start(&data, blob_store_location(&here), &here, 1, now())
.await
.unwrap()
else {
panic!("no change recorded");
};
let id = Id::from(change_id).to_string();
// OC-3: not before it's resolved
let response = update(admin, &id, json!({"action": "cleanUp", "dryRun": true})).await;
assert_eq!(
response["notUpdated"][&id]["type"], "forbidden",
"{response}"
);
// Resolved by a copy; OC-1: the old store is kept
let response = update(admin, &id, json!({"action": "copy", "blobsPerSecond": 0})).await;
assert!(response["updated"].get(&id).is_some(), "{response}");
let change = settled(admin, &id).await;
assert_eq!(change["status"], "copied", "{change}");
assert_eq!(change["oldStoreKept"], true, "{change}");
assert_eq!(change["cleanup"], Value::Null, "{change}");
// OC-5: a preview deletes nothing
let response = update(
admin,
&id,
json!({"action": "cleanUp", "dryRun": true, "blobsPerSecond": 0}),
)
.await;
assert!(response["updated"].get(&id).is_some(), "{response}");
let change = cleaned(admin, &id).await;
let cleanup = &change["cleanup"];
assert_eq!(cleanup["status"], "done", "{change}");
assert_eq!(cleanup["dryRun"], true, "{change}");
assert_eq!(cleanup["startedBy"], "[email protected]", "{change}");
assert_eq!(count(cleanup, "deleted"), 3, "{change}");
assert!(count(cleanup, "bytesFreed") > 0, "{change}");
assert!(count(cleanup, "notThere") >= 1, "{change}");
assert!(old.blob_present(kept_hash.as_slice()).await.unwrap());
assert_eq!(change["oldStoreKept"], true, "{change}");
// OC-8: deleting needs the change's own id as the confirmation
let response = update(admin, &id, json!({"action": "cleanUp"})).await;
assert_eq!(
response["notUpdated"][&id]["type"], "forbidden",
"{response}"
);
let other = Id::from(change_id + 100).to_string();
let response = update(admin, &id, json!({"action": "cleanUp", "confirm": other})).await;
assert_eq!(
response["notUpdated"][&id]["type"], "forbidden",
"{response}"
);
// OC-2, OC-3: a node still on the old store refuses it
let old_place = blob_store_location(&old_settings);
model::note_node(&data, 99, &old_place, now())
.await
.unwrap();
let response = update(admin, &id, json!({"action": "cleanUp", "confirm": id})).await;
assert_eq!(
response["notUpdated"][&id]["type"], "forbidden",
"{response}"
);
assert!(
response["notUpdated"][&id]["description"]
.as_str()
.unwrap_or_default()
.contains("Node 99"),
"{response}"
);
model::note_node(&data, 99, &old_place, now() - 3_600)
.await
.unwrap();
// A slow one stops when asked, and can't be dismissed meanwhile
let response = update(
admin,
&id,
json!({"action": "cleanUp", "dryRun": true, "blobsPerSecond": 1}),
)
.await;
assert!(response["updated"].get(&id).is_some(), "{response}");
let response = update(admin, &id, json!({"action": "copy"})).await;
assert_eq!(
response["notUpdated"][&id]["type"], "forbidden",
"{response}"
);
let (_, response) = call(
admin,
"inbuxa:BlobStoreChange/set",
json!({"destroy": [id]}),
)
.await;
assert_eq!(
response["notDestroyed"][&id]["type"], "forbidden",
"{response}"
);
let response = update(admin, &id, json!({"action": "stop"})).await;
assert!(response["updated"].get(&id).is_some(), "{response}");
let change = cleaned(admin, &id).await;
assert_eq!(change["cleanup"]["status"], "stopped", "{change}");
// OC-4: what the current store lacks or holds damaged stays in the old one
let altered_content = current
.get_stored(altered_hash.as_slice())
.await
.unwrap()
.unwrap();
current
.put_blob(
altered_hash.as_slice(),
b"Something else entirely",
CompressionAlgo::None,
)
.await
.unwrap();
let removed_content = current
.get_stored(removed_hash.as_slice())
.await
.unwrap()
.unwrap();
current.delete_blob(removed_hash.as_slice()).await.unwrap();
let response = update(
admin,
&id,
json!({"action": "cleanUp", "confirm": id, "blobsPerSecond": 0}),
)
.await;
assert!(response["updated"].get(&id).is_some(), "{response}");
let change = cleaned(admin, &id).await;
let cleanup = &change["cleanup"];
assert_eq!(cleanup["status"], "incomplete", "{change}");
assert_eq!(cleanup["dryRun"], false, "{change}");
assert_eq!(count(cleanup, "deleted"), 1, "{change}");
assert_eq!(count(cleanup, "missing"), 1, "{change}");
assert_eq!(count(cleanup, "damaged"), 1, "{change}");
assert_eq!(change["problems"], json!([]), "{change}");
let problems = cleanup["problems"].as_array().unwrap();
let kind_of = |hash: &BlobHash| {
problems
.iter()
.find(|p| p["blobId"] == hash.to_hex())
.map(|p| p["kind"].clone())
.unwrap_or_else(|| panic!("no problem for {}: {change}", hash.to_hex()))
};
assert_eq!(kind_of(&removed_hash), "missing");
assert_eq!(kind_of(&altered_hash), "damaged");
assert!(!old.blob_present(kept_hash.as_slice()).await.unwrap());
for hash in [&altered_hash, &removed_hash] {
assert!(old.blob_present(hash.as_slice()).await.unwrap(), "kept");
}
assert!(
String::from_utf8_lossy(&test.fetch_email(ann_id, kept).await).contains("In both stores"),
"the message reads from the current store"
);
assert_eq!(get(admin, &id).await["oldStoreKept"], true);
// Put right: a second run is done, and the old store's settings go
current
.put_stored(altered_hash.as_slice(), &altered_content)
.await
.unwrap();
current
.put_stored(removed_hash.as_slice(), &removed_content)
.await
.unwrap();
let response = update(
admin,
&id,
json!({"action": "cleanUp", "confirm": id, "blobsPerSecond": 0}),
)
.await;
assert!(response["updated"].get(&id).is_some(), "{response}");
let change = cleaned(admin, &id).await;
let cleanup = &change["cleanup"];
assert_eq!(cleanup["status"], "done", "{change}");
assert_eq!(count(cleanup, "deleted"), 2, "{change}");
assert_eq!(change["oldStoreKept"], false, "{change}");
for hash in [&kept_hash, &altered_hash, &removed_hash] {
assert!(!old.blob_present(hash.as_slice()).await.unwrap());
}
for (message, text) in [
(altered, "Damaged in the current store"),
(removed, "Gone from the current store"),
] {
assert!(String::from_utf8_lossy(&test.fetch_email(ann_id, message).await).contains(text));
}
assert!(
model::get(&data, change_id)
.await
.unwrap()
.unwrap()
.old_settings
.is_none()
);
let response = update(admin, &id, json!({"action": "cleanUp", "confirm": id})).await;
assert_eq!(
response["notUpdated"][&id]["type"], "forbidden",
"nothing left to clean up: {response}"
);
// Dismissed
let (_, response) = call(
admin,
"inbuxa:BlobStoreChange/set",
json!({"destroy": [id]}),
)
.await;
assert_eq!(response["destroyed"], json!([id]), "{response}");
}
/// Runs these tests alone: `cargo test -p tests blob_store_change_tests -- --ignored`.
#[ignore]
#[tokio::test(flavor = "multi_thread")]