From 9511782313692af5b7f96065041612becf700b9c Mon Sep 17 00:00:00 2001 From: John Coffey Date: Mon, 9 Nov 2026 21:01:55 -0700 Subject: [PATCH] Snapshots: listed and destroyed also while a new one is refused While a blob store change isn't settled, a new snapshot is refused (BM-8). The snapshots already on disk went with it: get listed none, destroy said notFound, and the start-up cleanup skipped partial directories, because all three found the snapshots' directory through the same check that refuses a new one. Found with inbuxa Admin's new Snapshots page. The directory now comes from the RocksDB data store alone. The system test takes the blob store through an unsettled change and checks the refusal, the snapshot still listed, and its destroy. --- crates/services/src/inbuxa_store_snapshot.rs | 21 +++++-- crates/store/src/write/snapshot.rs | 11 ++++ docs/spec/features/store-snapshot.md | 8 ++- tests/src/system/store_snapshot.rs | 66 ++++++++++++++++++-- 4 files changed, 94 insertions(+), 12 deletions(-) diff --git a/crates/services/src/inbuxa_store_snapshot.rs b/crates/services/src/inbuxa_store_snapshot.rs index e5ece3f0..2b300fab 100644 --- a/crates/services/src/inbuxa_store_snapshot.rs +++ b/crates/services/src/inbuxa_store_snapshot.rs @@ -123,6 +123,17 @@ fn source(server: &Server) -> Result<(SnapshotSource, PathBuf), Refusal> { } } +/// The snapshots' directory with a RocksDB data store, whether or not a +/// snapshot can be taken now. A blob store change that isn't settled +/// refuses a new one (BM-8), but the ones on disk are still listed, +/// destroyed and cleaned up (SN-7, SN-9). +fn root(server: &Server) -> Option { + server + .store() + .snapshot_data_path() + .map(|path| path.join(SNAPSHOTS_DIR)) +} + /// `20261109T013000Z`: a snapshot's name, the UTC time it was taken. pub fn name_for(seconds: u64) -> String { let t = DateTime::from_timestamp(seconds as i64); @@ -404,9 +415,9 @@ async fn refresh_latest(root: &Path) { /// Every snapshot, newest first: those on disk, and the one being taken or /// the last one that failed (SN-7). pub async fn list(server: &Server) -> Vec { - let mut snapshots = match source(server) { - Ok((_, root)) => on_disk(&root).await, - Err(_) => Vec::new(), + let mut snapshots = match root(server) { + Some(root) => on_disk(&root).await, + None => Vec::new(), }; if let Some(current) = current().filter(|c| !snapshots.iter().any(|s| s.id == c.id)) { snapshots.push(current); @@ -429,7 +440,7 @@ pub async fn destroy(server: &Server, id: u64) -> trc::Result { set_current(None); return Ok(Destroyed::Gone); } - let Ok((_, root)) = source(server) else { + let Some(root) = root(server) else { return Ok(Destroyed::NotFound); }; let Some(snapshot) = on_disk(&root).await.into_iter().find(|s| s.id == id) else { @@ -447,7 +458,7 @@ pub async fn destroy(server: &Server, id: u64) -> trc::Result { pub fn spawn_store_snapshots(inner: Arc) { tokio::spawn(async move { let server = common::BuildServer::build_server(&inner); - let Ok((_, root)) = source(&server) else { + let Some(root) = root(&server) else { return; }; let Ok(mut entries) = tokio::fs::read_dir(&root).await else { diff --git a/crates/store/src/write/snapshot.rs b/crates/store/src/write/snapshot.rs index 5c41e77b..b55ad930 100644 --- a/crates/store/src/write/snapshot.rs +++ b/crates/store/src/write/snapshot.rs @@ -69,6 +69,17 @@ impl Store { /// What a snapshot of this data store and `blobs` would hold, or why /// there can't be one. #[cfg_attr(not(feature = "rocks"), allow(unused_variables, unreachable_code))] + /// The RocksDB directory snapshots are kept under (SN-3), whether or not + /// one can be taken now: those already there are still listed, destroyed + /// and cleaned up while a new one would be refused (SN-7, BM-8). + pub fn snapshot_data_path(&self) -> Option { + match self { + #[cfg(feature = "rocks")] + Store::RocksDb(store) => Some(store.path().to_path_buf()), + _ => None, + } + } + pub fn snapshot_source(&self, blobs: &BlobStore) -> Result { let data_path = match self { #[cfg(feature = "rocks")] diff --git a/docs/spec/features/store-snapshot.md b/docs/spec/features/store-snapshot.md index 1d9fd2d7..3337f775 100644 --- a/docs/spec/features/store-snapshot.md +++ b/docs/spec/features/store-snapshot.md @@ -85,7 +85,8 @@ back while the server is stopped. The complete snapshots on disk are the record: `get` lists those, read from each one's `snapshot.json`, and, until the server restarts, the one - being taken or the last one that failed. + being taken or the last one that failed. They're listed, and can be + destroyed, also while a new snapshot would be refused (SN-1). - **SN-8.** Only administrators take snapshots: `sysStoreSnapshot`, in the built-in Administrator role and granted to existing administrator roles. A snapshot holds every account's data, so a tenant administrator is @@ -139,8 +140,9 @@ the store is copied twice. or in `blobs/`), and not a message that came after; no properties at create, no updates; with separate blobs, one at a time and no destroy while one is taken (the test holds the purge gate); only the latest two - kept; destroy; the audit events; a user's and a tenant administrator's - refusal. + kept; destroy; while a blob store change isn't settled, the refusal, + and the snapshot on disk still listed and destroyed; the audit events; a + user's and a tenant administrator's refusal. - Unit tests: the filesystem blob store's writes (SN-10), the tree copy and what it leaves out, the purge gate, snapshot names and the manifest. diff --git a/tests/src/system/store_snapshot.rs b/tests/src/system/store_snapshot.rs index 6f25d35f..4dd85f7d 100644 --- a/tests/src/system/store_snapshot.rs +++ b/tests/src/system/store_snapshot.rs @@ -13,8 +13,9 @@ //! whole, named for its time, with `latest` and `snapshot.json`. Only the //! latest two are kept; one can be destroyed, a snapshot can't be changed, //! and one being taken can't be destroyed or joined by another. Nobody in -//! a tenant or without the permission reaches snapshots. Taking and -//! destroying are audited. +//! a tenant or without the permission reaches snapshots. While a blob store +//! change isn't settled, a new one is refused and those on disk are still +//! listed and destroyed. Taking and destroying are audited. use crate::utils::{ account::Account, @@ -28,11 +29,15 @@ use registry::schema::{ }, }; use serde_json::{Value, json}; -use std::{path::PathBuf, str::FromStr, time::Duration}; +use std::{path::PathBuf, str::FromStr, sync::Arc, time::Duration}; use store::{ BlobStore, Store, ValueKey, backend::{fs::FsStore, rocksdb::RocksDbStore}, - write::{AlignedBytes, Archive, snapshot::hold_blob_purge}, + write::{ + AlignedBytes, Archive, + blob_move::{MovingBlobStore, OldBlobStore}, + snapshot::hold_blob_purge, + }, }; use types::{blob_hash::BlobHash, collection::Collection, field::EmailField, id::Id}; @@ -134,6 +139,15 @@ fn path_of(snapshot: &Value) -> PathBuf { ) } +/// Gives the server `blob` as its blob store, as a start-up would. Requests +/// build their server from the shared core, so the test's own handle is +/// left as it is. +fn use_blob_store(test: &TestServer, blob: BlobStore) { + let mut core = (*test.server.inner.shared_core.load_full()).clone(); + core.storage.blob = blob; + test.server.inner.shared_core.store(Arc::new(core)); +} + pub async fn test(test: &mut TestServer) { println!("Running store snapshot tests..."); let admin = test.account("admin@example.com"); @@ -324,6 +338,50 @@ pub async fn test(test: &mut TestServer) { PathBuf::from(left[0]["name"].as_str().unwrap()) ); + // BM-8: while a blob store change isn't settled, a new snapshot is + // refused, and the ones on disk are still listed and can be destroyed + let blobs = test.server.blob_store().clone(); + let old = FsStore::open(structs::FileSystemStore { + path: format!("{}/old-blobs", test.tmp_dir()), + depth: 2, + }) + .await + .unwrap(); + use_blob_store( + test, + BlobStore::Moving(Arc::new(MovingBlobStore::new( + blobs.clone(), + vec![OldBlobStore { + change_id: 1, + store: old, + }], + ))), + ); + let (_, response) = call( + admin, + "inbuxa:StoreSnapshot/set", + json!({"create": {"x": {}}}), + ) + .await; + assert!( + response["notCreated"]["x"]["description"] + .as_str() + .is_some_and(|d| d.contains("blob store changed")), + "{response}" + ); + let kept = all(admin).await; + assert_eq!(kept.len(), 1, "still listed while refused: {kept:?}"); + let kept_id = kept[0]["id"].as_str().unwrap().to_string(); + let (_, response) = call( + admin, + "inbuxa:StoreSnapshot/set", + json!({"destroy": [&kept_id]}), + ) + .await; + assert_eq!(response["destroyed"], json!([kept_id]), "{response}"); + assert!(!path_of(&kept[0]).exists()); + use_blob_store(test, blobs); + // Taking and destroying are audited let (_, response) = call( admin, -- 2.54.0