From 64d7f484c44f24f21bb253ed88c17d853a2efe30 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Mon, 9 Nov 2026 23:43:09 -0700 Subject: [PATCH] Blob store changes: plan one ahead, copy ahead, apply A create of inbuxa:BlobStoreChange with a target (an x:BlobStore value) plans a change from where the blob store is now. The target is opened and probed first; planning is refused for the same place, a second plan, an unsettled change, or a store that can't be written. Copying a planned change copies from the current store to the planned one, with the same walk, checks and throttle as the copy after a change. While it's planned, deletes on the current store also go to the planned store. Nothing reads from it and a snapshot isn't refused. apply writes the target to the blob store setting once the copy ahead ended copied. When a node starts with that setting, the planned change becomes the change, with the copy ahead kept as copiedAhead, so the last pass copies only what was written in between. The server now keeps its blob store in the read-through wrapper from start-up, so a plan takes effect without a restart. A planned store's settings are never returned and are redacted in the audit log. Spec: docs/spec/features/blob-store-copy-ahead.md (#341). --- crates/common/src/manager/blob_moves.rs | 52 +- crates/features/src/audit/diff.rs | 28 +- crates/features/src/blob_move/mod.rs | 487 +++++++++++++++++- .../src/object/inbuxa_blob_store_change.rs | 18 +- crates/jmap/src/inbuxa/audit.rs | 1 + crates/jmap/src/inbuxa/blob_store_change.rs | 243 ++++++++- crates/services/src/inbuxa_blob_move.rs | 200 ++++++- crates/store/src/dispatch/blob.rs | 5 +- crates/store/src/write/blob_move.rs | 137 ++++- docs/spec/SPEC.md | 2 +- docs/spec/features/blob-store-copy-ahead.md | 4 +- resources/privacy/catalog.toml | 1 + tests/src/store/scaleout.rs | 2 +- tests/src/system/blob_store_change.rs | 303 ++++++++++- tests/src/system/store_snapshot.rs | 2 +- 15 files changed, 1399 insertions(+), 86 deletions(-) diff --git a/crates/common/src/manager/blob_moves.rs b/crates/common/src/manager/blob_moves.rs index 60f25b43..e45b0470 100644 --- a/crates/common/src/manager/blob_moves.rs +++ b/crates/common/src/manager/blob_moves.rs @@ -7,24 +7,27 @@ //! inbuxa: blob-store-change spec (docs/spec/features/blob-store-change.md), //! at start-up: records where the blob store is and, when it moved, the //! change (BM-2), then opens the old store of every unresolved change so -//! what the new store lacks is read from it (BM-3). +//! what the new store lacks is read from it (BM-3), and the store of a +//! planned change so deletes follow to it (blob-store-copy-ahead spec, +//! CA-4). use inbuxa_features::blob_move::{self as model, Started}; use registry::schema::{prelude::ObjectType, structs}; -use std::sync::Arc; use store::{ BlobStore, Store, build::blob::blob_store_location, registry::bootstrap::Bootstrap, write::{ - blob_move::{MovingBlobStore, OldBlobStore}, + blob_move::{OldBlobStore, PlannedBlobStore}, now, }, }; use types::id::Id; -/// The blob store to use: `blob` itself or, while changes are unresolved, -/// `blob` reading through to their old stores. +/// The blob store to use: `blob`, reading through to the old stores of +/// unresolved changes, and with deletes following to a planned store. It's +/// wrapped even when there are none, so a change planned later takes effect +/// without a restart (CA-3). pub async fn at_start(bp: &mut Bootstrap, blob: BlobStore) -> BlobStore { let data = bp.data_store.clone(); if matches!(data, Store::None) || matches!(blob, BlobStore::Store(Store::None)) { @@ -69,7 +72,7 @@ pub async fn at_start(bp: &mut Bootstrap, blob: BlobStore) -> BlobStore { bp, format!("Couldn't record where the blob store is: {err}"), ); - return blob; + return blob.wrapped(Vec::new()); } } @@ -77,7 +80,7 @@ pub async fn at_start(bp: &mut Bootstrap, blob: BlobStore) -> BlobStore { Ok(changes) => changes, Err(err) => { warn(bp, format!("Couldn't read the blob store changes: {err}")); - return blob; + return blob.wrapped(Vec::new()); } }; let mut old = Vec::with_capacity(unresolved.len()); @@ -127,9 +130,36 @@ pub async fn at_start(bp: &mut Bootstrap, blob: BlobStore) -> BlobStore { } } - if old.is_empty() { - blob - } else { - BlobStore::Moving(Arc::new(MovingBlobStore::new(blob, old))) + let wrapped = blob.wrapped(old); + match model::planned(&data).await { + Ok(Some(plan)) => { + let mut warnings = Vec::new(); + let settings = plan.target_settings.clone().unwrap_or_default(); + match BlobStore::open(settings, &data, &mut warnings).await { + Ok(store) => { + if let Some(moving) = wrapped.moving() { + moving.set_planned(Some(PlannedBlobStore { + change_id: plan.id, + store, + })); + } + } + Err(err) => warn( + bp, + format!( + "The blob store planned for {} couldn't be opened, so deletes don't \ + follow to it until it can (inbuxa:BlobStoreChange {}): {err}", + plan.to, + Id::from(plan.id) + ), + ), + } + for warning in warnings { + warn(bp, warning); + } + } + Ok(None) => {} + Err(err) => warn(bp, format!("Couldn't read the planned blob store: {err}")), } + wrapped } diff --git a/crates/features/src/audit/diff.rs b/crates/features/src/audit/diff.rs index 55239829..2d8a7938 100644 --- a/crates/features/src/audit/diff.rs +++ b/crates/features/src/audit/diff.rs @@ -26,12 +26,22 @@ const ALWAYS_SECRET: &[&str] = &[ "otpAuth", ]; +/// Properties of the fork's own objects that hold a whole set of settings, +/// credentials and all, under names the rules above don't catch. +const HOLDS_SECRETS: &[(&str, &str)] = &[ + // blob-store-copy-ahead spec, CA-1: the planned store's settings + ("inbuxa:BlobStoreChange", "target"), +]; + /// Whether `property` of `object` (`x:AiModel`, `apiKey`) holds a secret. pub fn is_secret(object: &str, property: &str) -> bool { let lower = property.to_ascii_lowercase(); - ALWAYS_SECRET + HOLDS_SECRETS .iter() - .any(|name| lower == name.to_ascii_lowercase()) + .any(|(o, p)| *o == object && *p == property) + || ALWAYS_SECRET + .iter() + .any(|name| lower == name.to_ascii_lowercase()) || lower.ends_with("secret") || lower.ends_with("password") || lower.ends_with("privatekey") @@ -171,6 +181,20 @@ mod tests { assert!(is_secret("inbuxa:Relay", "smtpPrivateKey")); } + #[test] + fn a_planned_blob_store_is_never_kept() { + let create = json!({ + "target": {"@type": "S3", "bucket": "mail", "secretKey": "s3-secret", + "sessionToken": "s3-session"}, + }); + let changes = patch("inbuxa:BlobStoreChange", None, create.as_object().unwrap()); + assert_eq!(changes, vec![Change::redacted("target")]); + let text = serde_json::to_string(&changes).unwrap(); + assert!(!text.contains("s3-secret") && !text.contains("s3-session")); + // Only there: another object's `target` is an ordinary property + assert!(!is_secret("x:Thing", "target")); + } + #[test] fn secrets_the_schema_knows() { // x:AiModel's httpAuth holds a secret inside one of its variants diff --git a/crates/features/src/blob_move/mod.rs b/crates/features/src/blob_move/mod.rs index 305a0760..e4b5af2b 100644 --- a/crates/features/src/blob_move/mod.rs +++ b/crates/features/src/blob_move/mod.rs @@ -8,7 +8,9 @@ //! `inbuxa:BlobStoreChange` objects. A change is recorded when a node //! starts with its blob store somewhere else than the store last said //! (BM-2), and keeps the old store's settings so it can be read through -//! (BM-3) and copied from (BM-5). The copy itself runs in +//! (BM-3) and copied from (BM-5). A change can also be planned before the +//! setting moves, with the new store's settings, and copied ahead +//! (docs/spec/features/blob-store-copy-ahead.md). The copy itself runs in //! `services::inbuxa_blob_move`; this module keeps what a change is. //! //! Kept in the fork's subspace (`store::SUBSPACE_INBUXA`), never in the @@ -116,6 +118,42 @@ impl ProblemKind { } } +/// CA-1, CA-5: whether the setting has moved yet. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)] +#[serde(rename_all = "lowercase")] +pub enum Stage { + /// Planned: the new store's settings are kept, and a copy goes from the + /// current store to it. Nothing reads through it. + Planned, + /// The setting moved (BM-2). Every change recorded before planning + /// existed is one. + #[default] + Changed, +} + +impl Stage { + pub fn as_str(&self) -> &'static str { + match self { + Stage::Planned => "planned", + Stage::Changed => "changed", + } + } +} + +/// CA-5: the copy ahead of a planned change, kept once the setting moved. +#[derive(Debug, Clone, PartialEq, SerdeSerialize, SerdeDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct CopiedAhead { + pub started_at: u64, + #[serde(default)] + pub finished_at: Option, + pub status: Status, + pub copied: u64, + pub already_there: u64, + pub bytes_copied: u64, + pub problems: u64, +} + /// BM-5: a blob a copy couldn't copy. #[derive(Debug, Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)] #[serde(rename_all = "camelCase")] @@ -207,9 +245,40 @@ pub struct BlobStoreChange { pub old_settings: Option, #[serde(default)] pub copy: Option, + /// CA-1. + #[serde(default)] + pub stage: Stage, + /// CA-1: a planned change's new store, to copy to (CA-2), follow deletes + /// to (CA-4) and write to the setting (CA-7). Dropped at the switch; + /// never shown. + #[serde(default)] + pub target_settings: Option, + /// CA-5. + #[serde(default)] + pub copied_ahead: Option, + /// CA-7: when and by whom the setting was written. + #[serde(default)] + pub applied_at: Option, + #[serde(default)] + pub applied_by: Option, } impl BlobStoreChange { + /// BM-3: whether the old store is still read through. A planned change + /// never is (CA-3). + pub fn is_unresolved(&self) -> bool { + self.stage == Stage::Changed && self.status.is_unresolved() + } + + /// What a copy of it would copy from: the old store, or for a planned + /// change the current one (CA-2). + pub fn can_copy(&self) -> bool { + match self.stage { + Stage::Changed => self.is_unresolved() && self.old_settings.is_some(), + Stage::Planned => self.target_settings.is_some(), + } + } + /// As it reads now: a copy whose node went quiet has stopped with it /// (BM-6), even before anything marked it so. pub fn status_at(&self, now: u64) -> Status { @@ -328,11 +397,20 @@ pub async fn all(data: &Store) -> trc::Result> { /// BM-3, BM-10: the changes still read through, newest first. pub async fn unresolved(data: &Store) -> trc::Result> { let mut changes = all(data).await?; - changes.retain(|c| c.status.is_unresolved() && c.old_settings.is_some()); + changes.retain(|c| c.is_unresolved() && c.old_settings.is_some()); changes.reverse(); Ok(changes) } +/// CA-1, CA-3: the planned change, if there is one. +pub async fn planned(data: &Store) -> trc::Result> { + Ok(all(data) + .await? + .into_iter() + .rev() + .find(|c| c.stage == Stage::Planned && c.target_settings.is_some())) +} + /// Where the store last said the blob store was. pub async fn location(data: &Store) -> trc::Result> { Ok(data @@ -397,7 +475,7 @@ pub async fn at_start( // BM-4: back where an unresolved change moved away from for earlier in changes.iter().filter(|c| { - c.status.is_unresolved() && c.status != Status::Copying && c.from == location + c.is_unresolved() && c.status != Status::Copying && c.from == location }) { let reverted = BlobStoreChange { status: Status::Reverted, @@ -407,25 +485,57 @@ pub async fn at_start( batch.set(class(earlier.id), Json(&reverted).serialize()?); } - let change = BlobStoreChange { - id, - from: last.location.clone(), - to: location.clone(), - changed_at: now, - node_id, - status: Status::Pending, - old_settings: Some(last.settings.clone()), - copy: None, + // CA-5: the planned change for this move becomes the change + let plan = changes.iter().find(|c| { + c.stage == Stage::Planned && c.from == last.location && c.to == location + }); + let change = match plan { + Some(plan) => { + let stored = data + .get_value::>(ValueKey::from(class(plan.id))) + .await + .caused_by(trc::location!())?; + let Some(Versioned(plan, plan_version)) = stored else { + continue; + }; + batch.assert_value(class(plan.id), AssertValue::Hash(plan_version)); + if running(data).await? == Some(plan.id) { + // Its copy ahead is cut short; its node sees the + // switch at its next save + batch.assert_value(running_class(), AssertValue::U32(plan.id)); + batch.clear(running_class()); + } + forgotten.push(plan.id); + switched(plan, last.settings.clone(), node_id, now) + } + None => { + batch.assert_value(class(id), AssertValue::None); + BlobStoreChange { + id, + from: last.location.clone(), + to: location.clone(), + changed_at: now, + node_id, + status: Status::Pending, + old_settings: Some(last.settings.clone()), + copy: None, + stage: Stage::Changed, + target_settings: None, + copied_ahead: None, + applied_at: None, + applied_by: None, + } + } }; - batch.assert_value(class(id), AssertValue::None); - batch.set(class(id), Json(&change).serialize()?); - started = Started::Changed(id); + batch.set(class(change.id), Json(&change).serialize()?); + started = Started::Changed(change.id); // The oldest resolved changes past MAX_CHANGES are forgotten - let over = (changes.len() + 1).saturating_sub(MAX_CHANGES); + let over = + (changes.len() + usize::from(plan.is_none())).saturating_sub(MAX_CHANGES); for old in changes .iter() - .filter(|c| !c.status.is_unresolved()) + .filter(|c| c.stage == Stage::Changed && !c.status.is_unresolved()) .take(over) { batch.clear(class(old.id)); @@ -437,6 +547,8 @@ pub async fn at_start( match data.write(batch.build_all()).await { Ok(_) => { + // Forgotten changes' problems, and a switched plan's from its + // copy ahead: its count is kept in `copiedAhead` (CA-5) for id in forgotten { remove_problems(data, id).await?; } @@ -449,6 +561,125 @@ pub async fn at_start( } } +/// CA-5: a planned change, as the change it becomes when the setting moves +/// to its store: read through from `old` until copied, its copy ahead kept. +fn switched( + plan: BlobStoreChange, + old: structs::BlobStore, + node_id: u64, + now: u64, +) -> BlobStoreChange { + let copied_ahead = plan.copy.as_ref().map(|copy| CopiedAhead { + started_at: copy.started_at, + finished_at: copy.finished_at.or(Some(now)), + status: match plan.status { + Status::Copying => Status::Interrupted, + status => status, + }, + copied: copy.progress.copied, + already_there: copy.progress.already_there, + bytes_copied: copy.progress.bytes_copied, + problems: copy.progress.problems(), + }); + BlobStoreChange { + changed_at: now, + node_id, + status: Status::Pending, + old_settings: Some(old), + copy: None, + stage: Stage::Changed, + target_settings: None, + copied_ahead, + ..plan + } +} + +/// CA-1: why a change can't be planned. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PlanRefusal { + /// The new store is where the blob store is now. + SamePlace, + /// This change is planned already. + AlreadyPlanned(u32), + /// This change is still read through. + Unresolved(u32), + /// The server hasn't recorded where its blob store is yet. + NoLocation, +} + +/// CA-1: records a planned change from where the blob store is now to +/// `to`, keeping `target` to open it. The checks of the store itself (it +/// opens, a probe reads back) are the caller's. +pub async fn plan( + data: &Store, + to: String, + target: structs::BlobStore, +) -> trc::Result> { + let mut attempt = 0; + loop { + attempt += 1; + let Some(Versioned(here, version)) = data + .get_value::>(ValueKey::from(location_class())) + .await + .caused_by(trc::location!())? + else { + return Ok(Err(PlanRefusal::NoLocation)); + }; + if here.location == to { + return Ok(Err(PlanRefusal::SamePlace)); + } + let changes = all(data).await?; + if let Some(planned) = changes.iter().find(|c| c.stage == Stage::Planned) { + return Ok(Err(PlanRefusal::AlreadyPlanned(planned.id))); + } + if let Some(open) = changes.iter().rev().find(|c| c.is_unresolved()) { + return Ok(Err(PlanRefusal::Unresolved(open.id))); + } + let id = changes.iter().map(|c| c.id).max().unwrap_or(0) + 1; + let change = BlobStoreChange { + id, + from: here.location.clone(), + to: to.clone(), + changed_at: 0, + node_id: 0, + status: Status::Pending, + old_settings: None, + copy: None, + stage: Stage::Planned, + target_settings: Some(target.clone()), + copied_ahead: None, + applied_at: None, + applied_by: None, + }; + + let mut batch = BatchBuilder::new(); + // Planned from where the blob store is: refused if it moved meanwhile + batch.assert_value(location_class(), AssertValue::Hash(version)); + batch.assert_value(class(id), AssertValue::None); + batch.set(class(id), Json(&change).serialize()?); + let over = (changes.len() + 1).saturating_sub(MAX_CHANGES); + let mut forgotten = Vec::new(); + for old in changes + .iter() + .filter(|c| c.stage == Stage::Changed && !c.status.is_unresolved()) + .take(over) + { + batch.clear(class(old.id)); + forgotten.push(old.id); + } + match data.write(batch.build_all()).await { + Ok(_) => { + for id in forgotten { + remove_problems(data, id).await?; + } + return Ok(Ok(change)); + } + Err(err) if attempt < WRITE_ATTEMPTS && is_assert_failure(&err) => {} + Err(err) => return Err(err.caused_by(trc::location!())), + } + } +} + /// The id of the change being copied now, if any. pub async fn running(data: &Store) -> trc::Result> { data.get_value::(ValueKey::from(running_class())) @@ -465,6 +696,9 @@ pub enum Refusal { Running(u32), /// The change is resolved, or has no old store to copy from. Resolved, + /// CA-2: the blob store moved since the change was planned, so copying + /// ahead from it would fill the planned store from the wrong place. + Moved, } /// BM-5: marks a change as being copied by `copy` and makes it the one @@ -511,9 +745,20 @@ pub async fn start_copy( else { return Ok(Err(Refusal::NotFound)); }; - if !change.status.is_unresolved() || change.old_settings.is_none() { + if !change.can_copy() { return Ok(Err(Refusal::Resolved)); } + let location = data + .get_value::>(ValueKey::from(location_class())) + .await + .caused_by(trc::location!())?; + if change.stage == Stage::Planned + && location + .as_ref() + .is_none_or(|Versioned(here, _)| here.location != change.from) + { + return Ok(Err(Refusal::Moved)); + } let change = BlobStoreChange { status: Status::Copying, copy: Some(copy.clone()), @@ -762,6 +1007,202 @@ mod tests { assert!(get(&data, 2).await.unwrap().is_none()); } + fn a_copy(at: u64) -> Copy { + Copy { + started_at: at, + started_by: "admin".into(), + node_id: 1, + blobs_per_second: 0, + bytes_per_second: 0, + heartbeat: at, + finished_at: None, + stop_requested: false, + progress: Progress::default(), + problems_kept: 0, + error: None, + } + } + + #[tokio::test] + async fn a_planned_change_becomes_the_change() { + let data = store::backend::ephemeral::EphemeralStore::open(); + let a = structs::BlobStore::Default; + let b = fs("/srv/blobs"); + let (here_a, here_b) = ("the data store".to_string(), "dir".to_string()); + + // CA-1: nothing to plan from before the server says where it is + assert_eq!( + plan(&data, here_b.clone(), b.clone()).await.unwrap(), + Err(PlanRefusal::NoLocation) + ); + at_start(&data, here_a.clone(), &a, 1, 10).await.unwrap(); + assert_eq!( + plan(&data, here_a.clone(), a.clone()).await.unwrap(), + Err(PlanRefusal::SamePlace) + ); + let planned = plan(&data, here_b.clone(), b.clone()) + .await + .unwrap() + .unwrap(); + assert_eq!(planned.stage, Stage::Planned); + assert_eq!(planned.target_settings, Some(b.clone())); + assert_eq!( + (planned.from.as_str(), planned.to.as_str()), + ("the data store", "dir") + ); + assert_eq!( + plan(&data, "elsewhere".into(), fs("/srv/other")) + .await + .unwrap(), + Err(PlanRefusal::AlreadyPlanned(planned.id)) + ); + + // CA-3: not read through, so not unresolved + assert!(!planned.is_unresolved()); + assert!(unresolved(&data).await.unwrap().is_empty()); + assert_eq!( + super::planned(&data).await.unwrap().map(|c| c.id), + Some(planned.id) + ); + + // CA-2: copied ahead; `copied` resolves nothing + start_copy(&data, planned.id, a_copy(20), 20) + .await + .unwrap() + .unwrap(); + update(&data, planned.id, |c| { + c.status = Status::Copied; + if let Some(copy) = &mut c.copy { + copy.progress.copied = 7; + copy.progress.bytes_copied = 700; + copy.finished_at = Some(25); + } + true + }) + .await + .unwrap(); + release(&data, planned.id).await.unwrap(); + let ahead = get(&data, planned.id).await.unwrap().unwrap(); + assert_eq!(ahead.stage, Stage::Planned); + assert!(ahead.target_settings.is_some()); + + // CA-5: the switch makes it the change, no new one + assert_eq!( + at_start(&data, here_b.clone(), &b, 2, 30).await.unwrap(), + Started::Changed(planned.id) + ); + assert_eq!(all(&data).await.unwrap().len(), 1); + let change = get(&data, planned.id).await.unwrap().unwrap(); + assert_eq!(change.stage, Stage::Changed); + assert_eq!(change.status, Status::Pending); + assert_eq!((change.changed_at, change.node_id), (30, 2)); + assert_eq!(change.old_settings, Some(a.clone())); + assert_eq!(change.target_settings, None); + assert_eq!(change.copy, None); + let kept = change.copied_ahead.clone().unwrap(); + assert_eq!( + ( + kept.status, + kept.copied, + kept.bytes_copied, + kept.finished_at + ), + (Status::Copied, 7, 700, Some(25)) + ); + assert!(change.is_unresolved()); + assert_eq!(super::planned(&data).await.unwrap(), None); + + // CA-1: nothing more can be planned while it's read through + assert_eq!( + plan(&data, "elsewhere".into(), fs("/srv/other")) + .await + .unwrap(), + Err(PlanRefusal::Unresolved(planned.id)) + ); + } + + #[tokio::test] + async fn a_plan_for_somewhere_else_stays_planned() { + let data = store::backend::ephemeral::EphemeralStore::open(); + let a = structs::BlobStore::Default; + at_start(&data, "the data store".into(), &a, 1, 10) + .await + .unwrap(); + let planned = plan(&data, "dir".into(), fs("/srv/blobs")) + .await + .unwrap() + .unwrap(); + start_copy(&data, planned.id, a_copy(20), 20) + .await + .unwrap() + .unwrap(); + + // The setting moved somewhere else: a change of its own; the plan is + // still planned, and its copy can't start again from here (CA-2) + let other = fs("/srv/other"); + let Started::Changed(id) = at_start(&data, "other".into(), &other, 1, 30) + .await + .unwrap() + else { + panic!("no change recorded"); + }; + assert_ne!(id, planned.id); + assert_eq!( + get(&data, planned.id).await.unwrap().unwrap().stage, + Stage::Planned + ); + update(&data, planned.id, |c| { + c.status = Status::Stopped; + true + }) + .await + .unwrap(); + release(&data, planned.id).await.unwrap(); + assert_eq!( + start_copy(&data, planned.id, a_copy(40), 40).await.unwrap(), + Err(Refusal::Moved) + ); + } + + #[tokio::test] + async fn the_switch_cuts_a_running_copy_ahead_short() { + let data = store::backend::ephemeral::EphemeralStore::open(); + let a = structs::BlobStore::Default; + let b = fs("/srv/blobs"); + at_start(&data, "the data store".into(), &a, 1, 10) + .await + .unwrap(); + let planned = plan(&data, "dir".into(), b.clone()).await.unwrap().unwrap(); + start_copy(&data, planned.id, a_copy(20), 20) + .await + .unwrap() + .unwrap(); + assert_eq!(running(&data).await.unwrap(), Some(planned.id)); + + at_start(&data, "dir".into(), &b, 2, 30).await.unwrap(); + let change = get(&data, planned.id).await.unwrap().unwrap(); + assert_eq!(change.stage, Stage::Changed); + assert_eq!( + change.copied_ahead.map(|c| c.status), + Some(Status::Interrupted) + ); + // The last pass can start + assert_eq!(running(&data).await.unwrap(), None); + start_copy(&data, planned.id, a_copy(40), 40) + .await + .unwrap() + .unwrap(); + } + + #[test] + fn a_change_from_before_planning_reads_as_changed() { + let old = br#"{"id":1,"from":"a","to":"b","changedAt":5,"nodeId":1,"status":"pending","oldSettings":{"@type":"Default"}}"#; + let Json(change) = Json::::deserialize(old).unwrap(); + assert_eq!(change.stage, Stage::Changed); + assert!(change.is_unresolved()); + assert!(change.can_copy()); + } + #[test] fn keys_stay_in_their_own_letter() { for class in [ @@ -816,6 +1257,11 @@ mod tests { problems_kept: 0, error: None, }), + stage: Stage::Changed, + target_settings: None, + copied_ahead: None, + applied_at: None, + applied_by: None, }; assert_eq!(change.status_at(1_000 + STALE_AFTER), Status::Copying); assert_eq!(change.status_at(1_001 + STALE_AFTER), Status::Interrupted); @@ -832,6 +1278,11 @@ mod tests { status: Status::Pending, old_settings: Some(fs("/srv/blobs")), copy: None, + stage: Stage::Changed, + target_settings: None, + copied_ahead: None, + applied_at: None, + applied_by: None, }; let bytes = Json(&change).serialize().unwrap(); let Json(back) = Json::::deserialize(&bytes).unwrap(); diff --git a/crates/jmap-proto/src/object/inbuxa_blob_store_change.rs b/crates/jmap-proto/src/object/inbuxa_blob_store_change.rs index de107440..c4146f2c 100644 --- a/crates/jmap-proto/src/object/inbuxa_blob_store_change.rs +++ b/crates/jmap-proto/src/object/inbuxa_blob_store_change.rs @@ -8,7 +8,8 @@ //! of blob store the server noticed at start-up (blob-store-change spec, //! BM-2). The server records them; an update with `action: "copy"` copies //! what the new store lacks from the old one, `action: "stop"` stops that, -//! and destroying one dismisses it (BM-5 to BM-7). +//! and destroying one dismisses it (BM-5 to BM-7). A create with a `target` +//! plans one, to copy ahead and `apply` (blob-store-copy-ahead spec). use crate::object::{AnyId, JmapObject, JmapObjectId}; use jmap_tools::{Element, Key, Property}; @@ -43,6 +44,11 @@ pub enum BlobStoreChangeProperty { Problems, Error, Action, + Stage, + Target, + CopiedAhead, + AppliedAt, + AppliedBy, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -83,6 +89,11 @@ impl Property for BlobStoreChangeProperty { BlobStoreChangeProperty::Problems => "problems", BlobStoreChangeProperty::Error => "error", BlobStoreChangeProperty::Action => "action", + BlobStoreChangeProperty::Stage => "stage", + BlobStoreChangeProperty::Target => "target", + BlobStoreChangeProperty::CopiedAhead => "copiedAhead", + BlobStoreChangeProperty::AppliedAt => "appliedAt", + BlobStoreChangeProperty::AppliedBy => "appliedBy", } .into() } @@ -114,6 +125,11 @@ impl BlobStoreChangeProperty { b"problems" => BlobStoreChangeProperty::Problems, b"error" => BlobStoreChangeProperty::Error, b"action" => BlobStoreChangeProperty::Action, + b"stage" => BlobStoreChangeProperty::Stage, + b"target" => BlobStoreChangeProperty::Target, + b"copiedAhead" => BlobStoreChangeProperty::CopiedAhead, + b"appliedAt" => BlobStoreChangeProperty::AppliedAt, + b"appliedBy" => BlobStoreChangeProperty::AppliedBy, ) } } diff --git a/crates/jmap/src/inbuxa/audit.rs b/crates/jmap/src/inbuxa/audit.rs index f58b159d..6ba14154 100644 --- a/crates/jmap/src/inbuxa/audit.rs +++ b/crates/jmap/src/inbuxa/audit.rs @@ -523,6 +523,7 @@ async fn fork_current(server: &Server, object: &str, id: &MaybeInvalid) -> O serde_json::json!({ "from": c.from, "to": c.to, + "stage": c.stage.as_str(), "status": c.status.as_str(), }) }) diff --git a/crates/jmap/src/inbuxa/blob_store_change.rs b/crates/jmap/src/inbuxa/blob_store_change.rs index d7b6bab0..60455944 100644 --- a/crates/jmap/src/inbuxa/blob_store_change.rs +++ b/crates/jmap/src/inbuxa/blob_store_change.rs @@ -6,13 +6,17 @@ //! `inbuxa:BlobStoreChange` (blob-store-change spec): the changes of blob //! store the server noticed at start-up, copying what the new store lacks -//! from the old one, stopping that, and dismissing a change. The copy is -//! `services::inbuxa_blob_move`. Copying, stopping and dismissing are -//! audited by the request layer (BM-9). +//! from the old one, stopping that, and dismissing a change. A create plans +//! one, which is copied ahead and applied to the setting +//! (blob-store-copy-ahead spec). The copy is `services::inbuxa_blob_move`. +//! Planning, copying, applying, stopping and dismissing are audited by the +//! request layer (BM-9, CA-9). use super::relay::json_to_value; use common::{Server, auth::AccessToken}; -use inbuxa_features::blob_move::{self as model, BlobStoreChange as Stored, Refusal, Status}; +use inbuxa_features::blob_move::{ + self as model, BlobStoreChange as Stored, PlanRefusal, Refusal, Stage, Status, +}; use jmap_proto::{ error::set::SetError, method::{ @@ -26,9 +30,16 @@ use jmap_proto::{ types::date::UTCDate, }; use jmap_tools::{Key, Map, Value}; +use registry::schema::{ + prelude::{Object, ObjectImpl, ObjectType}, + structs, +}; use serde_json::json; -use services::inbuxa_blob_move as copies; -use store::write::now; +use services::inbuxa_blob_move::{self as copies, NotPlanned}; +use store::{ + registry::write::{RegistryWrite, RegistryWriteResult}, + write::now, +}; use types::id::Id; type CValue = Value<'static, P, BlobStoreChangeValue>; @@ -56,6 +67,10 @@ const ALL: &[P] = &[ P::Unreadable, P::Problems, P::Error, + P::Stage, + P::CopiedAhead, + P::AppliedAt, + P::AppliedBy, ]; /// BM-9: a change is about the whole server's blobs, so only the server's @@ -91,8 +106,25 @@ async fn to_value(server: &Server, c: &Stored, properties: &[P], now: u64) -> tr // BM-9: where the stores are, never their settings P::From => json!(c.from), P::To => json!(c.to), + // CA-1: a planned change hasn't happened yet + P::ChangedAt | P::Node if c.stage == Stage::Planned => serde_json::Value::Null, P::ChangedAt => date(c.changed_at), P::Node => json!(c.node_id.to_string()), + P::Stage => json!(c.stage.as_str()), + P::CopiedAhead => match &c.copied_ahead { + Some(a) => json!({ + "startedAt": date(a.started_at), + "finishedAt": a.finished_at.map_or(serde_json::Value::Null, date), + "status": a.status.as_str(), + "copied": a.copied, + "alreadyThere": a.already_there, + "bytesCopied": a.bytes_copied, + "problems": a.problems, + }), + None => serde_json::Value::Null, + }, + P::AppliedAt => c.applied_at.map_or(serde_json::Value::Null, date), + P::AppliedBy => json!(c.applied_by), P::Status => json!(status.as_str()), P::CopyStartedAt => copy.map_or(serde_json::Value::Null, |c| date(c.started_at)), P::CopyFinishedAt => copy @@ -131,8 +163,8 @@ async fn to_value(server: &Server, c: &Stored, properties: &[P], now: u64) -> tr list } P::Error => json!(copy.and_then(|c| c.error.clone())), - // Write-only - P::Action => continue, + // Write-only; a planned store's settings are never shown (CA-1) + P::Action | P::Target => continue, }; out.insert_unchecked(Key::Property(property.clone()), json_to_value(json)); } @@ -198,9 +230,66 @@ fn limit(property: P, value: &Value<'_, P, BlobStoreChangeValue>) -> Result trc::Result> { + let registry = server.registry(); + let result = match registry.get(ObjectType::BlobStore.singleton()).await? { + Some(old) if old.revision != 0 => { + let mut new = old.clone(); + new.inner = Object::from(target).inner; + registry + .write(RegistryWrite::update(Id::singleton(), &new, &old)) + .await? + } + _ => { + registry + .write(RegistryWrite::insert(&Object::from(target))) + .await? + } + }; + Ok(match result { + RegistryWriteResult::Success(_) => Ok(()), + RegistryWriteResult::ValidationError { errors } => Err(errors + .iter() + .map(|e| e.to_string()) + .collect::>() + .join("; ")), + other => Err(format!("{other:?}")), + }) +} + +/// CA-1: the planned store's settings, as the blob store setting takes them. +fn target_settings( + value: &Value<'_, P, BlobStoreChangeValue>, +) -> Result> { + let json = serde_json::to_value(value).unwrap_or_default(); + let settings: structs::BlobStore = serde_json::from_value(json).map_err(|err| { + invalid( + P::Target, + format!("Not a blob store setting, as x:BlobStore takes it: {err}"), + ) + })?; + let mut errors = Vec::new(); + if !settings.validate(&mut errors) { + return Err(invalid( + P::Target, + errors + .iter() + .map(|e| e.to_string()) + .collect::>() + .join("; "), + )); + } + Ok(settings) +} + +/// `inbuxa:BlobStoreChange/set`: the server records changes; a create plans +/// one (CA-1). An update copies or stops a copy, or applies a planned one; +/// destroy dismisses (BM-5 to BM-7, CA-2 to CA-8). pub async fn set( server: &Server, access_token: &AccessToken, @@ -211,13 +300,75 @@ pub async fn set( let data = server.store(); let actor = server.audit_actor(access_token).await; - for (client_id, _) in request.unwrap_create() { - response.not_created.append( - client_id, - SetError::forbidden().with_description( - "The server records a change when it starts with the blob store moved.", + 'create: for (client_id, value) in request.unwrap_create() { + let mut target = None; + for (key, value) in value.into_expanded_object() { + match &key { + Key::Property(P::Target) => match target_settings(&value) { + Ok(settings) => target = Some(settings), + Err(error) => { + response.not_created.append(client_id, error); + continue 'create; + } + }, + _ => { + response.not_created.append( + client_id, + SetError::invalid_properties() + .with_property(key.clone().into_owned()) + .with_description( + "Planning a change takes only the new store's settings, \ + as target.", + ), + ); + continue 'create; + } + } + } + let Some(target) = target else { + response.not_created.append( + client_id, + SetError::forbidden().with_description( + "The server records a change when it starts with the blob store moved. \ + To plan one ahead, give the new store's settings as target.", + ), + ); + continue; + }; + let refused = |why: String| SetError::forbidden().with_description(why); + match copies::plan(server, target).await? { + Ok(change) => { + let mut out = Map::with_capacity(1); + out.insert_unchecked( + Key::Property(P::Id), + Value::Element(BlobStoreChangeValue::Id(Id::from(change.id))), + ); + response.created.insert(client_id, Value::Object(out)); + } + Err(NotPlanned::Unusable(why)) => response + .not_created + .append(client_id, invalid(P::Target, why)), + Err(NotPlanned::Refused(refusal)) => response.not_created.append( + client_id, + refused(match refusal { + PlanRefusal::SamePlace => { + "That's where the blob store is now; there's nothing to move.".into() + } + PlanRefusal::AlreadyPlanned(id) => format!( + "Change {} is planned already; one at a time. Dismiss it first.", + Id::from(id) + ), + PlanRefusal::Unresolved(id) => format!( + "Change {} isn't settled: blobs only its old store has wouldn't be \ + copied ahead. Copy or dismiss it first.", + Id::from(id) + ), + PlanRefusal::NoLocation => { + "The server hasn't recorded where its blob store is yet.".into() + } + }), ), - ); + } } 'update: for (id, value) in request.unwrap_update().into_valid() { @@ -278,11 +429,57 @@ pub async fn set( SetError::forbidden() .with_description("That change is resolved; there's nothing to copy."), ), + Err(Refusal::Moved) => response.not_updated.append( + id, + SetError::forbidden().with_description( + "The blob store moved since this change was planned. Dismiss it \ + and plan again from where the blob store is now.", + ), + ), Err(Refusal::NotFound) => { response.not_updated.append(id, SetError::not_found()) } } } + Some("apply") => { + let why = if change.stage != Stage::Planned { + Some("Only a planned change can be applied.") + } else if change.applied_at.is_some() { + Some("It's applied already; the setting takes effect when the nodes restart.") + } else if change.status_at(now()) != Status::Copied { + Some("Copy it ahead first: it can be applied once that copy ends copied.") + } else { + None + }; + if let Some(why) = why { + response + .not_updated + .append(id, SetError::forbidden().with_description(why)); + continue; + } + let Some(target) = change.target_settings.clone() else { + response.not_updated.append(id, SetError::not_found()); + continue; + }; + match write_setting(server, target).await? { + Ok(()) => { + let by = actor.name.clone(); + model::update(data, change_id, |c| { + c.applied_at = Some(now()); + c.applied_by = Some(by.clone()); + true + }) + .await?; + response.updated.append(id, None); + } + Err(why) => response.not_updated.append( + id, + SetError::forbidden().with_description(format!( + "The blob store setting wasn't written: {why}" + )), + ), + } + } Some("stop") => { if change.status_at(now()) != Status::Copying { response.not_updated.append( @@ -294,9 +491,10 @@ pub async fn set( copies::stop(server, change_id).await?; response.updated.append(id, None); } - _ => response - .not_updated - .append(id, invalid(P::Action, "The actions are copy and stop.")), + _ => response.not_updated.append( + id, + invalid(P::Action, "The actions are copy, stop and apply."), + ), } } @@ -313,8 +511,9 @@ pub async fn set( ), Some(_) => { model::remove(data, change_id).await?; - // BM-7: this node stops reading through it at once; the - // others within a minute (BM-10) + // BM-7: this node stops reading through it at once, or + // following deletes to it (CA-8); the others within a minute + // (BM-10) copies::refresh(server).await; response.destroyed.push(id); } diff --git a/crates/services/src/inbuxa_blob_move.rs b/crates/services/src/inbuxa_blob_move.rs index 181cc49a..9daf5708 100644 --- a/crates/services/src/inbuxa_blob_move.rs +++ b/crates/services/src/inbuxa_blob_move.rs @@ -7,9 +7,12 @@ //! Changing the blob store (docs/spec/features/blob-store-change.md): the //! copy behind an `inbuxa:BlobStoreChange`. It goes through the blob index //! one slice at a time and copies each blob the current store lacks from -//! the change's old store, as the old store keeps it (BM-5). It runs on the -//! node the copy was started on. Also, on every node, it keeps reading -//! through only the changes still unresolved (BM-10). +//! the change's old store, as the old store keeps it (BM-5). A planned +//! change is copied the other way round: from the current store to the +//! planned one (blob-store-copy-ahead spec, CA-2). It runs on the node the +//! copy was started on. Also, on every node, it keeps reading through only +//! the changes still unresolved (BM-10), and deletes following to the +//! planned store, if there is one (CA-4). use common::{ BuildServer, Inner, Server, @@ -17,16 +20,19 @@ use common::{ }; use inbuxa_features::blob_move::{ self as model, BlobStoreChange, Copy, DEFAULT_BLOBS_PER_SECOND, DEFAULT_BYTES_PER_SECOND, - MAX_PROBLEMS, Problem, ProblemKind, Progress, Refusal, STALE_AFTER, Status, + MAX_PROBLEMS, PlanRefusal, Problem, ProblemKind, Progress, Refusal, STALE_AFTER, Stage, Status, }; +use registry::schema::structs; use std::{ sync::Arc, time::{Duration, Instant}, }; use store::{ BlobStore, + build::blob::blob_store_location, write::{ blob_check::{BlobHolder, LinkedBlob}, + blob_move::PlannedBlobStore, now, }, }; @@ -37,6 +43,9 @@ use types::{blob_hash::BlobHash, collection::Collection}; const SAVE_EVERY: Duration = Duration::from_secs(10); /// BM-10: how often a node reads which changes are still unresolved. const REFRESH_EVERY: Duration = Duration::from_secs(60); +/// CA-1: written to a planned store, read back and deleted, to check it +/// before anything relies on it. +const PROBE_KEY: &[u8] = b"INBUXA_BLOB_STORE_PROBE"; fn node_id(server: &Server) -> u64 { server.core.network.node_id @@ -76,6 +85,66 @@ pub async fn start( Ok(Ok(())) } +/// CA-1: why a change wasn't planned. +pub enum NotPlanned { + Refused(PlanRefusal), + /// The planned store can't be opened, or the probe failed: the reason. + Unusable(String), +} + +/// CA-1: plans a change to the store `target` names, once it opens and a +/// probe written to it reads back and deletes. +pub async fn plan( + server: &Server, + target: structs::BlobStore, +) -> trc::Result> { + let to = blob_store_location(&target); + let mut warnings = Vec::new(); + let store = match BlobStore::open(target.clone(), server.store(), &mut warnings).await { + Ok(store) => store, + Err(err) => { + return Ok(Err(NotPlanned::Unusable(format!( + "The blob store couldn't be opened: {err}" + )))); + } + }; + if let Err(why) = probe(&store).await { + return Ok(Err(NotPlanned::Unusable(why))); + } + match model::plan(server.store(), to, target).await? { + Ok(change) => { + if let Some(moving) = server.blob_store().moving() { + moving.set_planned(Some(PlannedBlobStore { + change_id: change.id, + store, + })); + } + Ok(Ok(change)) + } + Err(refusal) => Ok(Err(NotPlanned::Refused(refusal))), + } +} + +async fn probe(store: &BlobStore) -> Result<(), String> { + let written = format!("inbuxa {}", now()).into_bytes(); + store + .put_stored(PROBE_KEY, &written) + .await + .map_err(|err| format!("A test blob couldn't be written to it: {err}"))?; + let read = store + .get_stored(PROBE_KEY) + .await + .map_err(|err| format!("A test blob written to it couldn't be read back: {err}"))?; + if read.as_deref() != Some(&written[..]) { + return Err("A test blob written to it didn't read back the same.".into()); + } + store + .delete_blob(PROBE_KEY) + .await + .map_err(|err| format!("A test blob written to it couldn't be deleted: {err}"))?; + Ok(()) +} + /// BM-6: asks a running copy to stop. Any node can: the one running it /// sees the request at its next save. None if there's no such change. pub async fn stop(server: &Server, id: u32) -> trc::Result> { @@ -108,6 +177,8 @@ pub fn spawn_blob_moves(inner: Arc) { } /// BM-10: stops reading through changes resolved or dismissed anywhere. +/// CA-3, CA-4: and follows deletes to a change planned anywhere, until it's +/// switched to or dismissed. pub async fn refresh(server: &Server) { let Some(moving) = server.blob_store().moving() else { return; @@ -121,6 +192,42 @@ pub async fn refresh(server: &Server) { trc::error!(err.details("Failed to read the blob store changes")); } } + let plan = match model::planned(server.store()).await { + Ok(plan) => plan, + Err(err) => { + trc::error!(err.details("Failed to read the planned blob store")); + return; + } + }; + match plan { + None => moving.set_planned(None), + Some(plan) if moving.planned().is_some_and(|p| p.change_id == plan.id) => {} + Some(plan) => { + let settings = plan.target_settings.clone().unwrap_or_default(); + let mut warnings = Vec::new(); + match BlobStore::open(settings, server.store(), &mut warnings).await { + Ok(store) => { + moving.planned_open_failed(plan.id, false); + moving.set_planned(Some(PlannedBlobStore { + change_id: plan.id, + store, + })); + } + Err(err) => { + moving.set_planned(None); + if !moving.planned_open_failed(plan.id, true) { + trc::error!( + trc::StoreEvent::UnexpectedError + .into_err() + .details("The planned blob store couldn't be opened") + .ctx(trc::Key::Id, plan.id) + .reason(err) + ); + } + } + } + } + } } async fn mark_interrupted(server: &Server, which: impl Fn(&Copy) -> bool) { @@ -249,6 +356,8 @@ impl Holder { struct Run<'x> { server: &'x Server, id: u32, + /// CA-2: copying ahead, from the current store to the planned one. + ahead: bool, old: BlobStore, target: BlobStore, progress: Progress, @@ -330,14 +439,26 @@ impl<'x> Run<'x> { self.saved = Instant::now(); match saved { Some(c) if c.copy.as_ref().is_some_and(|c| c.stop_requested) => Err(Halt::Stopped), + // CA-5: switched to while copying ahead; the last pass is next + Some(c) if self.ahead && c.stage != Stage::Planned => Err(Halt::Stopped), Some(_) => Ok(()), // Dismissed under it: nothing left to copy for None => Err(Halt::Stopped), } } - /// BM-5: one blob, copied unless the current store has it. + /// What a problem calls the store copied from and the one copied to. + fn names(&self) -> (&'static str, &'static str) { + if self.ahead { + ("the current blob store", "the planned blob store") + } else { + ("the old blob store", "the current blob store") + } + } + + /// BM-5: one blob, copied unless the store copied to has it. async fn one(&mut self, key: &[u8], holder: Holder) -> Result<(), Halt> { + let (from, to) = self.names(); self.blobs.take(1).await; self.progress.blobs_checked += 1; match self.target.stored_present(key).await { @@ -351,7 +472,7 @@ impl<'x> Run<'x> { ProblemKind::Unreadable, key, &holder, - format!("the current blob store couldn't say: {err}"), + format!("{to} couldn't say: {err}"), ); return self.save(false).await; } @@ -370,7 +491,7 @@ impl<'x> Run<'x> { ProblemKind::Unreadable, key, &holder, - format!("the old blob store failed: {err}"), + format!("{from} failed: {err}"), ); return self.save(false).await; } @@ -389,7 +510,7 @@ impl<'x> Run<'x> { key, &holder, format!( - "the old store's {} bytes hash to {}", + "{from}'s {} bytes hash to {}", content.len(), actual.to_hex() ), @@ -417,7 +538,7 @@ impl<'x> Run<'x> { ProblemKind::Unreadable, key, &holder, - format!("the current blob store couldn't write it: {err}"), + format!("{to} couldn't write it: {err}"), ), } self.save(false).await @@ -442,6 +563,42 @@ impl<'x> Run<'x> { } } +/// CA-2: the planned store: the one deletes follow to, or opened now. +async fn planned_store(server: &Server, change: &BlobStoreChange) -> Result { + if let Some(planned) = server + .blob_store() + .moving() + .and_then(|m| m.planned()) + .filter(|p| p.change_id == change.id) + { + return Ok(planned.store.clone()); + } + let Some(settings) = change.target_settings.clone() else { + return Err("The change keeps no settings for the planned blob store".into()); + }; + let mut warnings = Vec::new(); + BlobStore::open(settings, server.store(), &mut warnings).await +} + +/// What a copy reads from and writes to: the old store and the current +/// one, or for a planned change the current one and the planned one. +async fn stores( + server: &Server, + change: &BlobStoreChange, +) -> Result<(BlobStore, BlobStore), String> { + let current = server.blob_store().current().clone(); + match change.stage { + Stage::Changed => old_store(server, change) + .await + .map(|old| (old, current)) + .map_err(|err| format!("The old blob store couldn't be opened: {err}")), + Stage::Planned => planned_store(server, change) + .await + .map(|planned| (current, planned)) + .map_err(|err| format!("The planned blob store couldn't be opened: {err}")), + } +} + /// The change's old store: the one already read through, or opened now. async fn old_store(server: &Server, change: &BlobStoreChange) -> Result { if let Some(old) = server.blob_store().moving().and_then(|m| { @@ -462,13 +619,10 @@ async fn old_store(server: &Server, change: &BlobStoreChange) -> Result ( - Status::Failed, - Some(format!("The old blob store couldn't be opened: {err}")), - None, - ), - Ok(old) => { + let ahead = change.stage == Stage::Planned; + let (status, error, run) = match stores(server, &change).await { + Err(err) => (Status::Failed, Some(err), None), + Ok((old, target)) => { let copy = change.copy.clone().unwrap_or_else(|| Copy { started_at: now(), started_by: String::new(), @@ -485,8 +639,9 @@ async fn run(server: &Server, change: BlobStoreChange) { let mut run = Run { server, id, + ahead, old, - target: server.blob_store().current().clone(), + target, progress: Progress::default(), found: Vec::new(), kept: 0, @@ -523,9 +678,14 @@ async fn run(server: &Server, change: BlobStoreChange) { }; if result.is_ok() { result = model::update(data, id, |c| { + // CA-5: switched to under a copy ahead: that change is its own now + if ahead && c.stage != Stage::Planned { + return false; + } c.status = status; - // BM-6: resolved, so the old store's settings go - if status == Status::Copied { + // BM-6: resolved, so the old store's settings go. A copy ahead + // resolves nothing (CA-2) + if status == Status::Copied && !ahead { c.old_settings = None; } if let Some(copy) = &mut c.copy { @@ -545,7 +705,7 @@ async fn run(server: &Server, change: BlobStoreChange) { if let Err(err) = result.and(model::release(data, id).await) { trc::error!(err.details("Failed to save a blob copy's result")); } - if status == Status::Copied { + if status == Status::Copied && !ahead { refresh(server).await; } } diff --git a/crates/store/src/dispatch/blob.rs b/crates/store/src/dispatch/blob.rs index e15c88b9..97a58e63 100644 --- a/crates/store/src/dispatch/blob.rs +++ b/crates/store/src/dispatch/blob.rs @@ -268,8 +268,9 @@ impl BlobStore { BlobStore::Azure(store) => store.delete_blob(key).await, // inbuxa: ST-18 BlobStore::Sharded(store) => store.delete(key).await, - // inbuxa: BM-3: nothing is deleted from an old store - BlobStore::Moving(store) => Box::pin(store.current.raw_delete(key)).await, + // inbuxa: BM-3: nothing is deleted from an old store; CA-4: a + // planned store follows + BlobStore::Moving(store) => Box::pin(store.raw_delete(key)).await, } } } diff --git a/crates/store/src/write/blob_move.rs b/crates/store/src/write/blob_move.rs index 6b0efd42..e6a321a8 100644 --- a/crates/store/src/write/blob_move.rs +++ b/crates/store/src/write/blob_move.rs @@ -8,11 +8,16 @@ //! (docs/spec/features/blob-store-change.md) needs from the store: a blob //! store that, while a change of blob store is unresolved, looks for what //! it doesn't have in the stores the server used before, and keeps what it -//! finds there (BM-3). Writes and deletes go to the current store only. +//! finds there (BM-3). Writes and deletes go to the current store only, +//! except that while a change is planned, a delete also goes to the planned +//! store (blob-store-copy-ahead spec, CA-4). use crate::BlobStore; -use arc_swap::ArcSwap; -use std::sync::Arc; +use arc_swap::{ArcSwap, ArcSwapOption}; +use std::sync::{ + Arc, + atomic::{AtomicU32, Ordering}, +}; use trc::StoreEvent; /// A blob store the server used before an unresolved change. @@ -23,11 +28,26 @@ pub struct OldBlobStore { pub store: BlobStore, } +/// The blob store a planned change will move to (CA-1), which deletes +/// follow until the switch (CA-4). +#[derive(Clone)] +pub struct PlannedBlobStore { + /// The planned `inbuxa:BlobStoreChange`. + pub change_id: u32, + pub store: BlobStore, +} + /// The current blob store, and the old ones read through while their -/// changes are unresolved, newest first (BM-3). +/// changes are unresolved, newest first (BM-3). The server keeps its blob +/// store in one of these from start-up, with no old stores at all most of +/// the time, so a planned store can be set while it runs (CA-3). pub struct MovingBlobStore { pub current: BlobStore, old: ArcSwap>, + planned: ArcSwapOption, + /// The planned change whose store couldn't be opened, so it's reported + /// once rather than at every refresh. + planned_failed: AtomicU32, } impl MovingBlobStore { @@ -35,9 +55,52 @@ impl MovingBlobStore { MovingBlobStore { current, old: ArcSwap::from_pointee(old), + planned: ArcSwapOption::empty(), + planned_failed: AtomicU32::new(0), } } + /// CA-4: the planned store deletes go to, if a change is planned. + pub fn planned(&self) -> Option> { + self.planned.load_full() + } + + /// CA-3, CA-5: the planned store, or none once the change is switched to + /// or dismissed. + pub fn set_planned(&self, planned: Option) { + self.planned.store(planned.map(Arc::new)); + } + + /// Whether this planned change's store failed to open before; marks it + /// so when `failed`. + pub fn planned_open_failed(&self, change_id: u32, failed: bool) -> bool { + if failed { + self.planned_failed.swap(change_id, Ordering::Relaxed) == change_id + } else { + self.planned_failed + .compare_exchange(change_id, 0, Ordering::Relaxed, Ordering::Relaxed) + .ok(); + false + } + } + + /// CA-4: deletes from the current store, then from the planned one. An + /// error from the planned store is logged, not returned. + pub(crate) async fn raw_delete(&self, key: &[u8]) -> trc::Result { + let deleted = Box::pin(self.current.raw_delete(key)).await?; + if let Some(planned) = self.planned.load_full() + && let Err(err) = Box::pin(planned.store.raw_delete(key)).await + { + trc::error!( + err.details("Failed to delete a blob from the planned blob store") + .ctx(trc::Key::Id, planned.change_id) + .ctx(trc::Key::Key, key) + .caused_by(trc::location!()) + ); + } + Ok(deleted) + } + /// The old stores still read through. pub fn old_stores(&self) -> Arc> { self.old.load_full() @@ -146,6 +209,12 @@ impl BlobStore { pub async fn stored_present(&self, key: &[u8]) -> trc::Result { self.current().blob_present(key).await } + + /// A blob store as the server holds it: in a [`MovingBlobStore`], so old + /// and planned stores can be added while it runs (BM-3, CA-3). + pub fn wrapped(self, old: Vec) -> BlobStore { + BlobStore::Moving(Arc::new(MovingBlobStore::new(self, old))) + } } #[cfg(test)] @@ -231,4 +300,64 @@ mod tests { let _ = std::fs::remove_dir_all(&root); } + + #[tokio::test] + async fn deletes_follow_to_a_planned_store() { + let root = std::env::temp_dir().join(format!("inbuxa-blob-plan-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let current = fs_store(&root.join("current")).await; + let planned = fs_store(&root.join("planned")).await; + let moving = current.clone().wrapped(Vec::new()); + for key in [&b"copied-ahead"[..], b"kept"] { + current + .put_blob(key, b"mail", CompressionAlgo::None) + .await + .unwrap(); + planned + .put_blob(key, b"mail", CompressionAlgo::None) + .await + .unwrap(); + } + + // Nothing planned: a delete touches the current store only + moving.delete_blob(b"kept").await.unwrap(); + assert!(!current.blob_present(b"kept").await.unwrap()); + assert!(planned.blob_present(b"kept").await.unwrap()); + + // Planned: it follows + moving.moving().unwrap().set_planned(Some(PlannedBlobStore { + change_id: 3, + store: planned.clone(), + })); + moving.delete_blob(b"copied-ahead").await.unwrap(); + assert!(!current.blob_present(b"copied-ahead").await.unwrap()); + assert!(!planned.blob_present(b"copied-ahead").await.unwrap()); + + // Writes and reads still go to the current store only (CA-3) + moving + .put_blob(b"fresh", b"new mail", CompressionAlgo::None) + .await + .unwrap(); + assert!(current.blob_present(b"fresh").await.unwrap()); + assert!(!planned.blob_present(b"fresh").await.unwrap()); + + // A planned store that fails doesn't fail the delete + let _ = std::fs::remove_dir_all(root.join("planned")); + std::fs::write(root.join("planned"), b"not a directory").unwrap(); + assert!(moving.delete_blob(b"fresh").await.is_ok()); + assert!(!current.blob_present(b"fresh").await.unwrap()); + + // Switched to or dismissed: deletes stop following + moving.moving().unwrap().set_planned(None); + assert!(moving.moving().unwrap().planned().is_none()); + + // An open failure is reported once per change + let m = moving.moving().unwrap(); + assert!(!m.planned_open_failed(4, true)); + assert!(m.planned_open_failed(4, true)); + assert!(!m.planned_open_failed(4, false)); + assert!(!m.planned_open_failed(4, true)); + + let _ = std::fs::remove_dir_all(&root); + } } diff --git a/docs/spec/SPEC.md b/docs/spec/SPEC.md index cb23dc68..c316b857 100644 --- a/docs/spec/SPEC.md +++ b/docs/spec/SPEC.md @@ -390,7 +390,7 @@ Not a rebuild: the **store check** is INBUXA's own design (`features/store-check Not a rebuild: the **store snapshot** is INBUXA's own design (`features/store-snapshot.md`, inbuxa-server #323, building). An administrator has the server take a consistent copy of its RocksDB store, and of a filesystem blob store's files, into `snapshots/` in the store's directory while mail flows, for a backup tool to copy; the latest two are kept: `inbuxa:StoreSnapshot` (`crates/jmap/src/inbuxa/store_snapshot.rs`, `crates/services/src/inbuxa_store_snapshot.rs`, `crates/store/src/write/snapshot.rs`, `crates/store/src/backend/rocksdb/checkpoint.rs`), and the `sysStoreSnapshot` permission. -Not a rebuild: **changing the blob store** is INBUXA's own design (`features/blob-store-change.md`, inbuxa-server #333, building). When a node starts with the blob store setting pointing somewhere else, it records the change and keeps the old store's settings; until the change is resolved, a blob the new store lacks is read from the old one and kept, and an administrator has one node copy the rest over, checked by hash, with what couldn't be copied reported: `inbuxa:BlobStoreChange` (`crates/jmap/src/inbuxa/blob_store_change.rs`, `crates/features/src/blob_move/`, `crates/services/src/inbuxa_blob_move.rs`, `crates/common/src/manager/blob_moves.rs`, `crates/store/src/write/blob_move.rs`), the `store.blob-read-through` event, and the `sysBlobStoreChange` permission. Planning a change and copying ahead of it (`features/blob-store-copy-ahead.md`, inbuxa-server #341, approved 2026-11-10). +Not a rebuild: **changing the blob store** is INBUXA's own design (`features/blob-store-change.md`, inbuxa-server #333, building). When a node starts with the blob store setting pointing somewhere else, it records the change and keeps the old store's settings; until the change is resolved, a blob the new store lacks is read from the old one and kept, and an administrator has one node copy the rest over, checked by hash, with what couldn't be copied reported: `inbuxa:BlobStoreChange` (`crates/jmap/src/inbuxa/blob_store_change.rs`, `crates/features/src/blob_move/`, `crates/services/src/inbuxa_blob_move.rs`, `crates/common/src/manager/blob_moves.rs`, `crates/store/src/write/blob_move.rs`), the `store.blob-read-through` event, and the `sysBlobStoreChange` permission. Planning a change and copying ahead of it (`features/blob-store-copy-ahead.md`, inbuxa-server #341, building): `inbuxa:BlobStoreChange/set` creates a planned change with the new store's settings, copied ahead and applied to `x:BlobStore`; deletes follow to it until the switch. Not a rebuild: **the store layout, changed in steps** is INBUXA's own design (`features/store-layout.md`, approved 2026-11-08). A store carries its layout version (`Vs` in `_`); at boot one node, holding a lease, runs each step from the stored version up and marks the store after each. Step 1 → 2 gives SQL rate-limit counters their own expiry column; step 2 → 3 moves the audit log, journal, holds and undelete notes into subspaces of their own. A step stops rollback, so it ships at a series. diff --git a/docs/spec/features/blob-store-copy-ahead.md b/docs/spec/features/blob-store-copy-ahead.md index fb42ae01..d950b546 100644 --- a/docs/spec/features/blob-store-copy-ahead.md +++ b/docs/spec/features/blob-store-copy-ahead.md @@ -1,7 +1,7 @@ # Feature spec: copying blobs ahead of a blob store change -Status: **approved 2026-11-10** (inbuxa-server #341, PR #354), with the -answers under [Settled](#settled). Not a rebuild (SPEC.md +Status: building, 2026-11-10 (inbuxa-server #341). Approved 2026-11-10 +(PR #354), with the answers under [Settled](#settled). Not a rebuild (SPEC.md §4): inbuxa's own design. It extends `features/blob-store-change.md` (BM-1 to BM-10), which it doesn't change except where it says so. diff --git a/resources/privacy/catalog.toml b/resources/privacy/catalog.toml index 405d734a..1b052612 100644 --- a/resources/privacy/catalog.toml +++ b/resources/privacy/catalog.toml @@ -290,6 +290,7 @@ scope = "server" retention = "object-life" [object."inbuxa:BlobStoreChange".properties] copyStartedBy = ["identifier"] +appliedBy = ["identifier"] problems = ["identifier", "metadata"] [object."inbuxa:StoreSnapshot"] diff --git a/tests/src/store/scaleout.rs b/tests/src/store/scaleout.rs index 9821f565..e2b73e65 100644 --- a/tests/src/store/scaleout.rs +++ b/tests/src/store/scaleout.rs @@ -236,7 +236,7 @@ pub async fn scaleout_blob_tests() { // ST-1: nothing changes for a server without one assert!( - matches!(test.server.core.storage.blob, BlobStore::Store(_)), + matches!(test.server.core.storage.blob.current(), BlobStore::Store(_)), "the default blob store is unchanged (ST-1)" ); diff --git a/tests/src/system/blob_store_change.rs b/tests/src/system/blob_store_change.rs index 62dd9dbe..f726951d 100644 --- a/tests/src/system/blob_store_change.rs +++ b/tests/src/system/blob_store_change.rs @@ -16,6 +16,13 @@ //! change unresolved. Put right, a second copy resolves it, and nothing //! is read through any more. A slow copy stops when asked and can't be //! dismissed while it runs. Nobody but an administrator reaches changes. +//! +//! Then copying ahead (docs/spec/features/blob-store-copy-ahead.md): a +//! change to a filesystem store is planned, refused where it should be, +//! copied ahead and applied to the setting; deletes follow to the planned +//! store meanwhile. Mail delivered after the copy ahead is read through once +//! the server switches, and the last pass copies only what the copy ahead +//! didn't have. use crate::utils::{ account::Account, @@ -23,13 +30,18 @@ use crate::utils::{ }; use email::{mailbox::INBOX_ID, message::metadata::MessageMetadata}; use inbuxa_features::blob_move::{self as model, Started}; -use registry::schema::{enums::CompressionAlgo, structs}; +use registry::schema::{ + enums::CompressionAlgo, + prelude::{Object, ObjectType}, + structs, +}; use serde_json::{Value, json}; use std::{str::FromStr, sync::Arc, time::Duration}; use store::{ BlobStore, ValueKey, backend::fs::FsStore, build::blob::blob_store_location, + registry::write::{RegistryWrite, RegistryWriteResult}, write::{ AlignedBytes, Archive, blob_move::{MovingBlobStore, OldBlobStore}, @@ -403,6 +415,295 @@ pub async fn test(test: &mut TestServer) { assert_eq!(response["notFound"], json!([id]), "{response}"); use_blob_store(test, current); + copy_ahead(test).await; +} + +async fn plan(admin: &Account, target: Value) -> Value { + let (_, response) = call( + admin, + "inbuxa:BlobStoreChange/set", + json!({"create": {"p": {"target": target}}}), + ) + .await; + response +} + +async fn setting(test: &TestServer) -> structs::BlobStore { + test.server + .registry() + .object::(Id::singleton()) + .await + .unwrap() + .unwrap_or_default() +} + +async fn put_setting(test: &TestServer, settings: structs::BlobStore) { + let registry = test.server.registry(); + let old = registry + .get(ObjectType::BlobStore.singleton()) + .await + .unwrap() + .expect("the setting apply wrote"); + let mut new = old.clone(); + new.inner = Object::from(settings).inner; + assert!(matches!( + registry + .write(RegistryWrite::update(Id::singleton(), &new, &old)) + .await + .unwrap(), + RegistryWriteResult::Success(_) + )); +} + +/// The blob-store-copy-ahead spec, CA-1 to CA-8. +async fn copy_ahead(test: &mut TestServer) { + println!("Running blob store copy ahead tests..."); + let data = test.server.store().clone(); + let admin = test.account("admin@example.com"); + let ann = test.account("ann@example.com"); + let ann_id = ann.id().document_id(); + let client = ann.jmap_client().await; + let current = test.server.blob_store().clone(); + // Where the server last said its blob store is: the data store, as the + // test above left it, whatever the setting is + let here = structs::BlobStore::Default; + let original = setting(test).await; + + // From a server with no change left: with blobs in a directory, the + // test above leaves the move from it to its old store behind + for change in model::all(&data).await.unwrap() { + model::remove(&data, change.id).await.unwrap(); + } + + let dir = format!("{}/planned-blobs", test.tmp_dir()); + let fs_settings = structs::FileSystemStore { + path: dir.clone(), + depth: 2, + }; + let target = structs::BlobStore::FileSystem(fs_settings.clone()); + let target_json = serde_json::to_value(&target).unwrap(); + let planned_store = FsStore::open(fs_settings).await.unwrap(); + + let before = import(&client, "Delivered before the copy ahead").await; + let before_hash = blob_of(test, ann_id, before).await; + + // CA-1: refusals + let response = plan(admin, json!({"@type": "Bogus"})).await; + assert_eq!( + response["notCreated"]["p"]["type"], "invalidProperties", + "{response}" + ); + let response = plan(admin, serde_json::to_value(&here).unwrap()).await; + assert_eq!( + response["notCreated"]["p"]["type"], "forbidden", + "the same place: {response}" + ); + let response = plan( + admin, + json!({"@type": "FileSystem", "path": "/dev/null/blobs", "depth": 2}), + ) + .await; + assert_eq!( + response["notCreated"]["p"]["type"], "invalidProperties", + "a store that can't be written: {response}" + ); + let (_, response) = call( + admin, + "inbuxa:BlobStoreChange/set", + json!({"create": {"p": {"target": target_json, "from": "x"}}}), + ) + .await; + assert_eq!( + response["notCreated"]["p"]["type"], "invalidProperties", + "{response}" + ); + + // Planned + let response = plan(admin, target_json.clone()).await; + let id = response["created"]["p"]["id"] + .as_str() + .unwrap_or_else(|| panic!("not planned: {response}")) + .to_string(); + let change_id = Id::from_str(&id).unwrap().document_id(); + let change = get(admin, &id).await; + assert_eq!(change["stage"], "planned", "{change}"); + assert_eq!(change["status"], "pending", "{change}"); + assert_eq!(change["from"], "the data store", "{change}"); + assert_eq!(change["to"], blob_store_location(&target), "{change}"); + assert_eq!(change["changedAt"], Value::Null, "{change}"); + assert!(!change.to_string().contains("FileSystem"), "{change}"); + let response = plan( + admin, + json!({"@type": "FileSystem", "path": format!("{dir}-2"), "depth": 2}), + ) + .await; + assert_eq!( + response["notCreated"]["p"]["type"], "forbidden", + "one planned at a time: {response}" + ); + let response = update(admin, &id, json!({"action": "apply"})).await; + assert_eq!( + response["notUpdated"][&id]["type"], "forbidden", + "applied before it's copied ahead: {response}" + ); + // CA-3: not read through, and a snapshot isn't refused for it + assert!(model::unresolved(&data).await.unwrap().is_empty()); + + // CA-4: deletes follow + let planned = test + .server + .blob_store() + .moving() + .and_then(|m| m.planned()) + .expect("deletes follow to the planned store"); + assert_eq!(planned.change_id, change_id); + for store in [current.current(), &planned_store] { + store + .put_blob(b"deleted-meanwhile", b"x", CompressionAlgo::None) + .await + .unwrap(); + } + test.server + .blob_store() + .delete_blob(b"deleted-meanwhile") + .await + .unwrap(); + assert!( + !planned_store + .blob_present(b"deleted-meanwhile") + .await + .unwrap(), + "a delete followed to the planned store" + ); + + // CA-2: copied ahead; nothing resolved, the setting untouched + let response = update(admin, &id, json!({"action": "copy", "blobsPerSecond": 0})).await; + assert!(response["updated"].get(&id).is_some(), "{response}"); + let change = settled(admin, &id).await; + assert_eq!(change["status"], "copied", "{change}"); + assert_eq!(change["stage"], "planned", "{change}"); + let copied_ahead = count(&change, "copied"); + assert!(copied_ahead >= 1, "{change}"); + assert!( + planned_store + .blob_present(before_hash.as_slice()) + .await + .unwrap(), + "copied ahead" + ); + assert_eq!(setting(test).await, original); + + // Written after the copy ahead: only the current store has them + let late = import(&client, "Delivered after the copy ahead").await; + let later = import(&client, "Also after the copy ahead").await; + let late_hash = blob_of(test, ann_id, late).await; + let later_hash = blob_of(test, ann_id, later).await; + assert!( + !planned_store + .blob_present(late_hash.as_slice()) + .await + .unwrap() + ); + + // CA-7: applied, once + let response = update(admin, &id, json!({"action": "apply"})).await; + assert!(response["updated"].get(&id).is_some(), "{response}"); + assert_eq!(setting(test).await, target); + let change = get(admin, &id).await; + assert_eq!(change["appliedBy"], "admin@example.com", "{change}"); + assert!(change["appliedAt"].is_string(), "{change}"); + let response = update(admin, &id, json!({"action": "apply"})).await; + assert_eq!( + response["notUpdated"][&id]["type"], "forbidden", + "{response}" + ); + + // CA-5: the switch, as a start with the new setting would make it + let Started::Changed(switched) = + model::at_start(&data, blob_store_location(&target), &target, 1, now()) + .await + .unwrap() + else { + panic!("no change at the switch"); + }; + assert_eq!(switched, change_id, "the planned change became the change"); + use_blob_store( + test, + BlobStore::Moving(Arc::new(MovingBlobStore::new( + planned_store.clone(), + vec![OldBlobStore { + change_id, + store: current.current().clone(), + }], + ))), + ); + let admin = test.account("admin@example.com"); + let change = get(admin, &id).await; + assert_eq!(change["stage"], "changed", "{change}"); + assert_eq!(change["status"], "pending", "{change}"); + assert!(change["changedAt"].is_string(), "{change}"); + assert_eq!(change["copiedAhead"]["status"], "copied", "{change}"); + assert_eq!(change["copiedAhead"]["copied"], copied_ahead, "{change}"); + assert!( + model::get(&data, change_id) + .await + .unwrap() + .unwrap() + .target_settings + .is_none(), + "the planned store's settings go at the switch" + ); + + // BM-3: what came after the copy ahead reads through + let message = test.fetch_email(ann_id, late).await; + assert!(String::from_utf8_lossy(&message).contains("Delivered after the copy ahead")); + assert!( + planned_store + .blob_present(late_hash.as_slice()) + .await + .unwrap() + ); + + // CA-6: the last pass copies only what the copy ahead didn't have + let response = update(admin, &id, json!({"action": "copy", "blobsPerSecond": 0})).await; + assert!(response["updated"].get(&id).is_some(), "{response}"); + let change = settled(admin, &id).await; + assert_eq!(change["status"], "copied", "{change}"); + let last_pass = count(&change, "copied"); + assert!(last_pass >= 1 && last_pass < copied_ahead, "{change}"); + assert!(count(&change, "alreadyThere") >= copied_ahead, "{change}"); + assert!( + planned_store + .blob_present(later_hash.as_slice()) + .await + .unwrap() + ); + + // Put back as it was + let (_, response) = call( + admin, + "inbuxa:BlobStoreChange/set", + json!({"destroy": [id]}), + ) + .await; + assert_eq!(response["destroyed"], json!([id]), "{response}"); + use_blob_store(test, current.clone()); + put_setting(test, original.clone()).await; + if let Started::Changed(back) = + model::at_start(&data, blob_store_location(&here), &here, 1, now()) + .await + .unwrap() + { + model::remove(&data, back).await.unwrap(); + } + // The planned store is gone with the change + services::inbuxa_blob_move::refresh(&test.server).await; + assert!( + test.server + .blob_store() + .moving() + .is_none_or(|m| m.planned().is_none()) + ); } /// Runs these tests alone: `cargo test -p tests blob_store_change_tests -- --ignored`. diff --git a/tests/src/system/store_snapshot.rs b/tests/src/system/store_snapshot.rs index 4dd85f7d..91f5cc94 100644 --- a/tests/src/system/store_snapshot.rs +++ b/tests/src/system/store_snapshot.rs @@ -154,7 +154,7 @@ pub async fn test(test: &mut TestServer) { let ann = test.account("ann@example.com"); let ann_id = ann.id().document_id(); let client = ann.jmap_client().await; - let separate_blobs = matches!(test.server.blob_store(), BlobStore::Fs(_)); + let separate_blobs = matches!(test.server.blob_store().current(), BlobStore::Fs(_)); println!( "Blobs {}", if separate_blobs { -- 2.54.0