diff --git a/crates/common/src/auth/permissions.rs b/crates/common/src/auth/permissions.rs index c2cbe544..da5c456f 100644 --- a/crates/common/src/auth/permissions.rs +++ b/crates/common/src/auth/permissions.rs @@ -393,6 +393,11 @@ impl Default for DefaultPermissions { Permission::SysStoreCheck => { default.superuser.push(permission); } + // inbuxa: store-snapshot spec, SN-8: a snapshot holds every + // account's data, so only the server's administrators take one + Permission::SysStoreSnapshot => { + default.superuser.push(permission); + } // inbuxa: AL-12: tenant administrators lock and delegate // within their tenant Permission::SysAccountLockGet diff --git a/crates/common/src/manager/granted_permissions.rs b/crates/common/src/manager/granted_permissions.rs index b8ce6f29..2ea04b5e 100644 --- a/crates/common/src/manager/granted_permissions.rs +++ b/crates/common/src/manager/granted_permissions.rs @@ -35,7 +35,8 @@ use types::id::Id; /// to-do items (security to-do list spec), the deliverability check /// (deliverability spec), scheduled reports (scheduled-reports spec), /// inbuxa Relays (relay spec), running migrations (migrations spec, -/// MG-10), and checking the store's blobs (store-check spec, SV-9). +/// MG-10), checking the store's blobs (store-check spec, SV-9), and taking +/// snapshots of the store (store-snapshot spec, SN-8). const ADMIN_GRANTS: &[Permission] = &[ Permission::SysAiExplain, Permission::SysAuditGet, @@ -70,6 +71,7 @@ const ADMIN_GRANTS: &[Permission] = &[ Permission::SysRelayQueueUpdate, Permission::SysMigrationRun, Permission::SysStoreCheck, + Permission::SysStoreSnapshot, ]; /// Granted to the server-level Compliance Officer role once it exists: diff --git a/crates/jmap-proto/src/object/inbuxa_store_snapshot.rs b/crates/jmap-proto/src/object/inbuxa_store_snapshot.rs new file mode 100644 index 00000000..e991dec5 --- /dev/null +++ b/crates/jmap-proto/src/object/inbuxa_store_snapshot.rs @@ -0,0 +1,188 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! `inbuxa:StoreSnapshot/get` and `/set` under `urn:inbuxa:jmap`: a snapshot +//! of the store, taken without stopping the server (store-snapshot spec, +//! SN-1). Creating one takes it in the background; `get` lists the +//! snapshots on disk and says how the one being taken stands. Destroying one +//! deletes its directory. + +use crate::object::{AnyId, JmapObject, JmapObjectId}; +use jmap_tools::{Element, Key, Property}; +use std::{borrow::Cow, str::FromStr}; +use types::id::Id; + +#[derive(Debug, Clone, Default)] +pub struct StoreSnapshot; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum StoreSnapshotProperty { + Id, + Status, + CreatedAt, + CreatedBy, + FinishedAt, + Node, + Name, + Path, + Holds, + Size, + BlobFiles, + Error, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum StoreSnapshotValue { + Id(Id), +} + +impl Property for StoreSnapshotProperty { + fn try_parse(parent: Option<&Key<'_, Self>>, value: &str) -> Option { + match parent { + None => StoreSnapshotProperty::parse(value), + Some(_) => None, + } + } + + fn to_cow(&self) -> Cow<'static, str> { + match self { + StoreSnapshotProperty::Id => "id", + StoreSnapshotProperty::Status => "status", + StoreSnapshotProperty::CreatedAt => "createdAt", + StoreSnapshotProperty::CreatedBy => "createdBy", + StoreSnapshotProperty::FinishedAt => "finishedAt", + StoreSnapshotProperty::Node => "node", + StoreSnapshotProperty::Name => "name", + StoreSnapshotProperty::Path => "path", + StoreSnapshotProperty::Holds => "holds", + StoreSnapshotProperty::Size => "size", + StoreSnapshotProperty::BlobFiles => "blobFiles", + StoreSnapshotProperty::Error => "error", + } + .into() + } +} + +impl StoreSnapshotProperty { + fn parse(value: &str) -> Option { + hashify::tiny_map!(value.as_bytes(), + b"id" => StoreSnapshotProperty::Id, + b"status" => StoreSnapshotProperty::Status, + b"createdAt" => StoreSnapshotProperty::CreatedAt, + b"createdBy" => StoreSnapshotProperty::CreatedBy, + b"finishedAt" => StoreSnapshotProperty::FinishedAt, + b"node" => StoreSnapshotProperty::Node, + b"name" => StoreSnapshotProperty::Name, + b"path" => StoreSnapshotProperty::Path, + b"holds" => StoreSnapshotProperty::Holds, + b"size" => StoreSnapshotProperty::Size, + b"blobFiles" => StoreSnapshotProperty::BlobFiles, + b"error" => StoreSnapshotProperty::Error, + ) + } +} + +impl FromStr for StoreSnapshotProperty { + type Err = (); + + fn from_str(s: &str) -> Result { + StoreSnapshotProperty::parse(s).ok_or(()) + } +} + +impl Element for StoreSnapshotValue { + type Property = StoreSnapshotProperty; + + fn try_parse

(key: &Key<'_, Self::Property>, value: &str) -> Option { + match key { + Key::Property(StoreSnapshotProperty::Id) => { + Id::from_str(value).ok().map(StoreSnapshotValue::Id) + } + _ => None, + } + } + + fn to_cow(&self) -> Cow<'static, str> { + match self { + StoreSnapshotValue::Id(id) => id.to_string().into(), + } + } +} + +impl JmapObject for StoreSnapshot { + type Property = StoreSnapshotProperty; + + type Element = StoreSnapshotValue; + + type Id = Id; + + type Filter = (); + + type Comparator = (); + + type GetArguments = (); + + type SetArguments<'de> = (); + + type QueryArguments = (); + + type CopyArguments = (); + + type ParseArguments = (); + + const ID_PROPERTY: Self::Property = StoreSnapshotProperty::Id; +} + +impl From for StoreSnapshotValue { + fn from(id: Id) -> Self { + StoreSnapshotValue::Id(id) + } +} + +impl JmapObjectId for StoreSnapshotValue { + fn as_id(&self) -> Option { + match self { + StoreSnapshotValue::Id(id) => Some(*id), + } + } + + fn as_any_id(&self) -> Option { + match self { + StoreSnapshotValue::Id(id) => Some(AnyId::Id(*id)), + } + } + + fn as_id_ref(&self) -> Option<&str> { + None + } + + fn try_set_id(&mut self, new_id: AnyId) -> bool { + if let AnyId::Id(id) = new_id { + *self = StoreSnapshotValue::Id(id); + true + } else { + false + } + } +} + +impl JmapObjectId for StoreSnapshotProperty { + fn as_id(&self) -> Option { + None + } + + fn as_any_id(&self) -> Option { + None + } + + fn as_id_ref(&self) -> Option<&str> { + None + } + + fn try_set_id(&mut self, _: AnyId) -> bool { + false + } +} diff --git a/crates/jmap-proto/src/object/mod.rs b/crates/jmap-proto/src/object/mod.rs index 14e67093..3604dd87 100644 --- a/crates/jmap-proto/src/object/mod.rs +++ b/crates/jmap-proto/src/object/mod.rs @@ -51,6 +51,7 @@ pub mod inbuxa_scheduled_report_settings; // inbuxa: scheduled reports pub mod inbuxa_security_acceptance; // inbuxa: accepted security to-do items pub mod inbuxa_sharing_policy; // inbuxa: MA-C, who may share mail pub mod inbuxa_store_check; // inbuxa: checking the store's blobs +pub mod inbuxa_store_snapshot; // inbuxa: a snapshot of the store pub mod inbuxa_tenant_protocol_policy; // inbuxa: legacy protocols off, per tenant pub mod mailbox; pub mod participant_identity; diff --git a/crates/jmap-proto/src/references/eval.rs b/crates/jmap-proto/src/references/eval.rs index d0b35558..ef9ba8c8 100644 --- a/crates/jmap-proto/src/references/eval.rs +++ b/crates/jmap-proto/src/references/eval.rs @@ -94,6 +94,9 @@ impl Response<'_> { GetResponseMethod::StoreCheck(response) => { response.eval_jptr(path, &mut results) } + GetResponseMethod::StoreSnapshot(response) => { + response.eval_jptr(path, &mut results) + } GetResponseMethod::Relay(response) => { response.eval_jptr(path, &mut results) } diff --git a/crates/jmap-proto/src/references/resolve.rs b/crates/jmap-proto/src/references/resolve.rs index 4ce30f4c..e9a36ab4 100644 --- a/crates/jmap-proto/src/references/resolve.rs +++ b/crates/jmap-proto/src/references/resolve.rs @@ -57,6 +57,7 @@ impl Response<'_> { GetRequestMethod::MailRule(request) => request.resolve_references(self)?, GetRequestMethod::Migration(request) => request.resolve_references(self)?, GetRequestMethod::StoreCheck(request) => request.resolve_references(self)?, + GetRequestMethod::StoreSnapshot(request) => request.resolve_references(self)?, GetRequestMethod::Relay(request) => request.resolve_references(self)?, GetRequestMethod::RelayMessage(request) => request.resolve_references(self)?, GetRequestMethod::SecurityAcceptance(request) => { @@ -159,6 +160,9 @@ impl Response<'_> { SetRequestMethod::StoreCheck(request) => { request.resolve_references(self, 1, false)? } + SetRequestMethod::StoreSnapshot(request) => { + request.resolve_references(self, 1, false)? + } SetRequestMethod::Relay(request) => request.resolve_references(self, 1, false)?, SetRequestMethod::RelayMessage(request) => { request.resolve_references(self, 1, false)? diff --git a/crates/jmap-proto/src/request/method.rs b/crates/jmap-proto/src/request/method.rs index e80cd643..40fb4525 100644 --- a/crates/jmap-proto/src/request/method.rs +++ b/crates/jmap-proto/src/request/method.rs @@ -72,6 +72,8 @@ pub enum MethodObject { Migration, // inbuxa: checking the store's blobs StoreCheck, + // inbuxa: a snapshot of the store + StoreSnapshot, Relay, RelayMessage, // inbuxa: accepted security to-do items @@ -132,6 +134,7 @@ impl MethodObject { | MethodObject::MailRule | MethodObject::Migration | MethodObject::StoreCheck + | MethodObject::StoreSnapshot | MethodObject::Relay | MethodObject::RelayMessage | MethodObject::SecurityAcceptance @@ -350,6 +353,8 @@ impl MethodName { (MethodFunction::Set, MethodObject::Migration) => "inbuxa:Migration/set", (MethodFunction::Get, MethodObject::StoreCheck) => "inbuxa:StoreCheck/get", (MethodFunction::Set, MethodObject::StoreCheck) => "inbuxa:StoreCheck/set", + (MethodFunction::Get, MethodObject::StoreSnapshot) => "inbuxa:StoreSnapshot/get", + (MethodFunction::Set, MethodObject::StoreSnapshot) => "inbuxa:StoreSnapshot/set", (MethodFunction::Get, MethodObject::Relay) => "inbuxa:Relay/get", (MethodFunction::Set, MethodObject::Relay) => "inbuxa:Relay/set", (MethodFunction::Get, MethodObject::RelayMessage) => "inbuxa:RelayMessage/get", @@ -557,6 +562,8 @@ impl MethodName { "inbuxa:Migration/set" => (MethodObject::Migration, MethodFunction::Set), "inbuxa:StoreCheck/get" => (MethodObject::StoreCheck, MethodFunction::Get), "inbuxa:StoreCheck/set" => (MethodObject::StoreCheck, MethodFunction::Set), + "inbuxa:StoreSnapshot/get" => (MethodObject::StoreSnapshot, MethodFunction::Get), + "inbuxa:StoreSnapshot/set" => (MethodObject::StoreSnapshot, MethodFunction::Set), "inbuxa:Relay/get" => (MethodObject::Relay, MethodFunction::Get), "inbuxa:Relay/set" => (MethodObject::Relay, MethodFunction::Set), "inbuxa:RelayMessage/get" => (MethodObject::RelayMessage, MethodFunction::Get), @@ -660,6 +667,7 @@ impl Display for MethodObject { MethodObject::MailRule => "inbuxa:MailRule", MethodObject::Migration => "inbuxa:Migration", MethodObject::StoreCheck => "inbuxa:StoreCheck", + MethodObject::StoreSnapshot => "inbuxa:StoreSnapshot", MethodObject::Relay => "inbuxa:Relay", MethodObject::RelayMessage => "inbuxa:RelayMessage", MethodObject::SecurityAcceptance => "inbuxa:SecurityAcceptance", diff --git a/crates/jmap-proto/src/request/mod.rs b/crates/jmap-proto/src/request/mod.rs index 5ecf6eeb..5aebe268 100644 --- a/crates/jmap-proto/src/request/mod.rs +++ b/crates/jmap-proto/src/request/mod.rs @@ -129,6 +129,7 @@ pub enum GetRequestMethod { MailRule(Box>), Migration(Box>), StoreCheck(Box>), + StoreSnapshot(Box>), Relay(Box>), RelayMessage(Box>), SecurityAcceptance( @@ -189,6 +190,7 @@ pub enum SetRequestMethod<'x> { MailRule(Box>), Migration(Box>), StoreCheck(Box>), + StoreSnapshot(Box>), Relay(Box>), RelayMessage(Box>), SecurityAcceptance( diff --git a/crates/jmap-proto/src/request/parser.rs b/crates/jmap-proto/src/request/parser.rs index c5591fea..dd83f977 100644 --- a/crates/jmap-proto/src/request/parser.rs +++ b/crates/jmap-proto/src/request/parser.rs @@ -719,6 +719,21 @@ impl<'de> Visitor<'de> for CallVisitor { return Err(de::Error::invalid_length(1, &self)); } }, + // inbuxa: a snapshot of the store + (MethodFunction::Get, MethodObject::StoreSnapshot) => match seq.next_element() { + Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::StoreSnapshot(value)), + Err(err) => RequestMethod::invalid(err), + Ok(None) => { + return Err(de::Error::invalid_length(1, &self)); + } + }, + (MethodFunction::Set, MethodObject::StoreSnapshot) => match seq.next_element() { + Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::StoreSnapshot(value)), + Err(err) => RequestMethod::invalid(err), + Ok(None) => { + return Err(de::Error::invalid_length(1, &self)); + } + }, // inbuxa: inbuxa Relay (MethodFunction::Get, MethodObject::Relay) => match seq.next_element() { Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Relay(value)), diff --git a/crates/jmap-proto/src/response/mod.rs b/crates/jmap-proto/src/response/mod.rs index 77c41731..a11fc39c 100644 --- a/crates/jmap-proto/src/response/mod.rs +++ b/crates/jmap-proto/src/response/mod.rs @@ -114,6 +114,7 @@ pub enum GetResponseMethod { MailRule(GetResponse), Migration(GetResponse), StoreCheck(GetResponse), + StoreSnapshot(GetResponse), Relay(GetResponse), RelayMessage(GetResponse), SecurityAcceptance(GetResponse), @@ -172,6 +173,7 @@ pub enum SetResponseMethod { MailRule(Box>), Migration(Box>), StoreCheck(Box>), + StoreSnapshot(Box>), Relay(Box>), RelayMessage(Box>), SecurityAcceptance( @@ -1067,6 +1069,23 @@ impl<'x> From> for Re } } +// inbuxa: a snapshot of the store +impl<'x> From> + for ResponseMethod<'x> +{ + fn from(value: GetResponse) -> Self { + ResponseMethod::Get(GetResponseMethod::StoreSnapshot(value)) + } +} + +impl<'x> From> + for ResponseMethod<'x> +{ + fn from(value: SetResponse) -> Self { + ResponseMethod::Set(SetResponseMethod::StoreSnapshot(Box::new(value))) + } +} + impl<'x> From> for ResponseMethod<'x> { fn from(value: GetResponse) -> Self { ResponseMethod::Get(GetResponseMethod::Relay(value)) diff --git a/crates/jmap/src/api/auth.rs b/crates/jmap/src/api/auth.rs index 43cab232..bfa2d0b5 100644 --- a/crates/jmap/src/api/auth.rs +++ b/crates/jmap/src/api/auth.rs @@ -120,6 +120,8 @@ impl JmapAuthorization for AccessToken { GetRequestMethod::Migration(_) => Permission::SysMigrationRun, // inbuxa: store-check spec, SV-9 GetRequestMethod::StoreCheck(_) => Permission::SysStoreCheck, + // inbuxa: store-snapshot spec, SN-8 + GetRequestMethod::StoreSnapshot(_) => Permission::SysStoreSnapshot, // inbuxa: inbuxa Relay GetRequestMethod::Relay(_) => Permission::SysRelayGet, GetRequestMethod::RelayMessage(_) => Permission::SysRelayQueueGet, @@ -333,6 +335,14 @@ impl JmapAuthorization for AccessToken { Permission::SysStoreCheck, Permission::SysStoreCheck, ), + // inbuxa: store-snapshot spec, SN-8 + SetRequestMethod::StoreSnapshot(s) => validate_set( + s, + self, + Permission::SysStoreSnapshot, + Permission::SysStoreSnapshot, + Permission::SysStoreSnapshot, + ), // inbuxa: inbuxa Relay; a new join token is an update SetRequestMethod::Relay(s) => validate_set( s, @@ -607,6 +617,7 @@ impl JmapAuthorization for AccessToken { | MethodObject::MailRule | MethodObject::Migration | MethodObject::StoreCheck + | MethodObject::StoreSnapshot | MethodObject::Relay | MethodObject::RelayMessage | MethodObject::SecurityAcceptance diff --git a/crates/jmap/src/api/request.rs b/crates/jmap/src/api/request.rs index 49943280..b18a95c5 100644 --- a/crates/jmap/src/api/request.rs +++ b/crates/jmap/src/api/request.rs @@ -296,6 +296,9 @@ impl RequestHandler for Server { SetResponseMethod::StoreCheck(set_response) => { set_response.update_created_ids(&mut response); } + SetResponseMethod::StoreSnapshot(set_response) => { + set_response.update_created_ids(&mut response); + } SetResponseMethod::Relay(set_response) => { set_response.update_created_ids(&mut response); } @@ -582,6 +585,13 @@ impl RequestHandler for Server { .await? .into() } + // inbuxa: a snapshot of the store + GetRequestMethod::StoreSnapshot(mut req) => { + resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; + crate::inbuxa::store_snapshot::get(self, access_token, *req) + .await? + .into() + } // inbuxa: inbuxa Relay GetRequestMethod::Relay(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; @@ -1191,6 +1201,22 @@ impl RequestHandler for Server { .await? .into() } + // inbuxa: store-snapshot spec, SN-8: taking and destroying are audited + SetRequestMethod::StoreSnapshot(mut req) => { + resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; + crate::inbuxa::audit::recorded( + self, + access_token, + session, + &method_name.obj.to_string(), + None, + None, + *req, + |req| Box::pin(crate::inbuxa::store_snapshot::set(self, access_token, req)), + ) + .await? + .into() + } SetRequestMethod::Relay(mut req) => { resolve_account_id(&mut req.account_id, method_name.obj, access_token)?; let reason = req.arguments.reason.clone(); diff --git a/crates/jmap/src/changes/get.rs b/crates/jmap/src/changes/get.rs index 948fedda..77020782 100644 --- a/crates/jmap/src/changes/get.rs +++ b/crates/jmap/src/changes/get.rs @@ -433,6 +433,7 @@ impl IntermediateChangesResponse { | MethodObject::MailRule | MethodObject::Migration | MethodObject::StoreCheck + | MethodObject::StoreSnapshot | MethodObject::Relay | MethodObject::RelayMessage | MethodObject::SecurityAcceptance diff --git a/crates/jmap/src/inbuxa/audit.rs b/crates/jmap/src/inbuxa/audit.rs index 63a96e29..f4dc54f9 100644 --- a/crates/jmap/src/inbuxa/audit.rs +++ b/crates/jmap/src/inbuxa/audit.rs @@ -526,6 +526,19 @@ async fn fork_current(server: &Server, object: &str, id: &MaybeInvalid) -> O } MaybeInvalid::Invalid(_) => None, }, + // A snapshot: its directory and how it stands (store-snapshot spec, + // SN-8) + "inbuxa:StoreSnapshot" => match id { + MaybeInvalid::Value(id) => services::inbuxa_store_snapshot::find(server, id.id()) + .await + .map(|s| { + serde_json::json!({ + "name": s.name, + "status": s.status.as_str(), + }) + }), + MaybeInvalid::Invalid(_) => None, + }, // A relay as an administrator sets it: never its keys or token "inbuxa:Relay" => match id { MaybeInvalid::Value(id) => { diff --git a/crates/jmap/src/inbuxa/mod.rs b/crates/jmap/src/inbuxa/mod.rs index 425154ff..eaa36168 100644 --- a/crates/jmap/src/inbuxa/mod.rs +++ b/crates/jmap/src/inbuxa/mod.rs @@ -39,6 +39,7 @@ pub mod scheduled_reports; // inbuxa: scheduled reports and the weekly digest pub mod security_acceptance; pub mod sharing_policy; pub mod store_check; // inbuxa: checking the store's blobs +pub mod store_snapshot; // inbuxa: a snapshot of the store pub mod telemetry; pub mod tenant_protocol_policy; pub mod undelete; diff --git a/crates/jmap/src/inbuxa/store_snapshot.rs b/crates/jmap/src/inbuxa/store_snapshot.rs new file mode 100644 index 00000000..52b274fe --- /dev/null +++ b/crates/jmap/src/inbuxa/store_snapshot.rs @@ -0,0 +1,185 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! `inbuxa:StoreSnapshot` (store-snapshot spec): taking a snapshot of the +//! store without stopping the server, listing the snapshots on disk, and +//! deleting one. The run is `services::inbuxa_store_snapshot`. Taking and +//! destroying one are audited by the request layer (SN-8). + +use super::relay::json_to_value; +use common::{Server, auth::AccessToken}; +use jmap_proto::{ + error::set::SetError, + method::{ + get::{GetRequest, GetResponse}, + set::{SetRequest, SetResponse}, + }, + object::inbuxa_store_snapshot::{ + StoreSnapshot, StoreSnapshotProperty as P, StoreSnapshotValue, + }, + request::IntoValid, + types::date::UTCDate, +}; +use jmap_tools::{Key, Map, Value}; +use serde_json::json; +use services::inbuxa_store_snapshot::{self as runs, Destroyed, Refusal, Snapshot}; +use types::id::Id; + +type SValue = Value<'static, P, StoreSnapshotValue>; + +const ALL: &[P] = &[ + P::Id, + P::Status, + P::CreatedAt, + P::CreatedBy, + P::FinishedAt, + P::Node, + P::Name, + P::Path, + P::Holds, + P::Size, + P::BlobFiles, + P::Error, +]; + +/// SN-8: a snapshot holds every account's data, so only the server's own +/// administrators take one. +fn server_level(access_token: &AccessToken) -> trc::Result<()> { + if access_token.tenant_id().is_some() { + Err(trc::JmapEvent::Forbidden + .into_err() + .details("A snapshot holds the whole server, not a tenant.")) + } else { + Ok(()) + } +} + +fn date(seconds: u64) -> serde_json::Value { + json!(UTCDate::from_timestamp(seconds as i64).to_string()) +} + +fn to_value(s: &Snapshot, properties: &[P]) -> SValue { + let mut out = Map::with_capacity(properties.len()); + for property in properties { + let json = match property { + P::Id => { + out.insert_unchecked( + Key::Property(P::Id), + Value::Element(StoreSnapshotValue::Id(Id::from(s.id))), + ); + continue; + } + P::Status => json!(s.status.as_str()), + P::CreatedAt => date(s.created_at), + P::CreatedBy => json!(s.created_by), + P::FinishedAt => s.finished_at.map_or(serde_json::Value::Null, date), + P::Node => json!(s.node_id.to_string()), + P::Name => json!(s.name), + P::Path => json!(s.path.to_string_lossy()), + P::Holds => json!(s.holds), + P::Size => json!(s.size), + P::BlobFiles => json!(s.blob_files), + P::Error => json!(s.error), + }; + out.insert_unchecked(Key::Property(property.clone()), json_to_value(json)); + } + Value::Object(out) +} + +/// `inbuxa:StoreSnapshot/get`: the snapshots on disk and the one being +/// taken, newest first (SN-7). +pub async fn get( + server: &Server, + access_token: &AccessToken, + mut request: GetRequest, +) -> trc::Result> { + server_level(access_token)?; + let properties = request.unwrap_properties(ALL); + let (ids, not_found) = request.unwrap_ids(server.core.jmap.get_max_objects)?; + let mut response = GetResponse { + account_id: request.account_id.into(), + state: None, + list: Vec::new(), + not_found, + }; + let snapshots = runs::list(server).await; + match ids { + None => { + for s in &snapshots { + response.list.push(to_value(s, &properties)); + } + } + Some(ids) => { + for id in ids { + match snapshots.iter().find(|s| s.id == id.id()) { + Some(s) => response.list.push(to_value(s, &properties)), + None => response.push_not_found(id), + } + } + } + } + Ok(response) +} + +/// `inbuxa:StoreSnapshot/set`: create takes one; destroy deletes one; a +/// snapshot can't be changed (SN-1, SN-6, SN-9). +pub async fn set( + server: &Server, + access_token: &AccessToken, + mut request: SetRequest<'_, StoreSnapshot>, +) -> trc::Result> { + server_level(access_token)?; + let mut response = SetResponse::from_request(&request, server.core.jmap.set_max_objects)?; + let actor = server.audit_actor(access_token).await; + + for (client_id, value) in request.unwrap_create() { + if let Some((key, _)) = value.into_expanded_object().next() { + response.not_created.append( + client_id, + SetError::invalid_properties() + .with_property(key.into_owned()) + .with_description("A snapshot is taken as it is; it takes no properties."), + ); + continue; + } + match runs::start(server, actor.name.clone()).await? { + Ok(snapshot) => { + response + .created + .insert(client_id, to_value(&snapshot, &[P::Id, P::Name, P::Path])); + } + Err(Refusal::Running(id)) => response.not_created.append( + client_id, + SetError::forbidden().with_description(format!( + "Snapshot {} is being taken; one is taken at a time. Wait for it.", + Id::from(id) + )), + ), + Err(Refusal::Unsupported(why)) => response + .not_created + .append(client_id, SetError::forbidden().with_description(why)), + } + } + + for (id, _) in request.unwrap_update().into_valid() { + response.not_updated.append( + id, + SetError::forbidden().with_description("A snapshot can't be changed."), + ); + } + + for id in request.unwrap_destroy().into_valid() { + match runs::destroy(server, id.id()).await? { + Destroyed::Gone => response.destroyed.push(id), + Destroyed::NotFound => response.not_destroyed.append(id, SetError::not_found()), + Destroyed::Running => response.not_destroyed.append( + id, + SetError::forbidden().with_description("It's being taken; wait for it to finish."), + ), + } + } + Ok(response) +} diff --git a/crates/registry/src/schema/enums.rs b/crates/registry/src/schema/enums.rs index 240a66b7..f649c721 100644 --- a/crates/registry/src/schema/enums.rs +++ b/crates/registry/src/schema/enums.rs @@ -1793,6 +1793,8 @@ pub enum Permission { SysMigrationRun = 694, // inbuxa: checking the store's blobs SysStoreCheck = 695, + // inbuxa: a snapshot of the store, without stopping the server + SysStoreSnapshot = 696, SysAccountGet = 219, SysAccountCreate = 220, SysAccountUpdate = 221, diff --git a/crates/registry/src/schema/enums_impl.rs b/crates/registry/src/schema/enums_impl.rs index e5ff1f9b..fae79c4c 100644 --- a/crates/registry/src/schema/enums_impl.rs +++ b/crates/registry/src/schema/enums_impl.rs @@ -7172,6 +7172,7 @@ impl EnumImpl for Permission { b"sysRelayQueueUpdate" => Permission::SysRelayQueueUpdate, b"sysMigrationRun" => Permission::SysMigrationRun, b"sysStoreCheck" => Permission::SysStoreCheck, + b"sysStoreSnapshot" => Permission::SysStoreSnapshot, b"sysAccountGet" => Permission::SysAccountGet, b"sysAccountCreate" => Permission::SysAccountCreate, b"sysAccountUpdate" => Permission::SysAccountUpdate, @@ -7884,6 +7885,7 @@ impl EnumImpl for Permission { Permission::SysRelayQueueUpdate => "sysRelayQueueUpdate", Permission::SysMigrationRun => "sysMigrationRun", Permission::SysStoreCheck => "sysStoreCheck", + Permission::SysStoreSnapshot => "sysStoreSnapshot", Permission::SysAccountGet => "sysAccountGet", Permission::SysAccountCreate => "sysAccountCreate", Permission::SysAccountUpdate => "sysAccountUpdate", @@ -8589,6 +8591,7 @@ impl EnumImpl for Permission { 693 => Some(Permission::SysRelayQueueUpdate), 694 => Some(Permission::SysMigrationRun), 695 => Some(Permission::SysStoreCheck), + 696 => Some(Permission::SysStoreSnapshot), 219 => Some(Permission::SysAccountGet), 220 => Some(Permission::SysAccountCreate), 221 => Some(Permission::SysAccountUpdate), @@ -9033,7 +9036,7 @@ impl EnumImpl for Permission { } } - const COUNT: usize = 696; + const COUNT: usize = 697; } impl serde::Serialize for Permission { diff --git a/crates/services/src/inbuxa_store_snapshot.rs b/crates/services/src/inbuxa_store_snapshot.rs new file mode 100644 index 00000000..5eb9bc9d --- /dev/null +++ b/crates/services/src/inbuxa_store_snapshot.rs @@ -0,0 +1,496 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! Snapshots of the store (docs/spec/features/store-snapshot.md): the run +//! behind an `inbuxa:StoreSnapshot`. It takes a RocksDB checkpoint and, for +//! a filesystem blob store, a copy of the blob files with the blob purge +//! held, into a directory under the store's own (SN-2, SN-3). The snapshots +//! on disk are the record: each one's `snapshot.json` says what it is. Only +//! the one being taken, or the last one that failed, is kept in memory. + +use common::{Inner, Server}; +use mail_parser::DateTime; +use serde_json::{Value, json}; +use std::{ + path::{Path, PathBuf}, + str::FromStr, + sync::{Arc, Mutex}, +}; +use store::write::{ + now, + snapshot::{NotSnapshottable, SnapshotSource, copy_tree, hold_blob_purge, tree_size}, +}; +use types::id::Id; + +/// SN-6: complete snapshots kept. +pub const KEEP: usize = 2; +const SNAPSHOTS_DIR: &str = "snapshots"; +const MANIFEST: &str = "snapshot.json"; +const LATEST: &str = "latest"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Status { + Running, + Done, + Failed, +} + +impl Status { + pub fn as_str(&self) -> &'static str { + match self { + Status::Running => "running", + Status::Done => "done", + Status::Failed => "failed", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Snapshot { + /// When it was taken, in seconds: its name is that time. + pub id: u64, + pub name: String, + pub status: Status, + pub created_at: u64, + pub created_by: String, + pub finished_at: Option, + pub node_id: u64, + pub path: PathBuf, + pub holds: Vec, + pub size: u64, + pub blob_files: u64, + pub error: Option, +} + +/// Why a snapshot isn't taken. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Refusal { + /// SN-6: this one is being taken. + Running(u64), + /// SN-1, SN-2: these stores can't have one. + Unsupported(String), +} + +/// What `destroy` did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Destroyed { + Gone, + NotFound, + Running, +} + +/// SN-7: the one being taken, or the last one that failed, until restart. +static CURRENT: Mutex> = Mutex::new(None); + +fn current() -> Option { + CURRENT.lock().map(|c| c.clone()).unwrap_or(None) +} + +fn set_current(snapshot: Option) { + if let Ok(mut current) = CURRENT.lock() { + *current = snapshot; + } +} + +/// The snapshots' directory, and what a snapshot would hold, or why there +/// can't be one here. +fn source(server: &Server) -> Result<(SnapshotSource, PathBuf), Refusal> { + match server.store().snapshot_source(server.blob_store()) { + Ok(source) => { + let root = match &source { + SnapshotSource::DataOnly { data_path } + | SnapshotSource::DataAndFiles { data_path, .. } => data_path.join(SNAPSHOTS_DIR), + }; + Ok((source, root)) + } + Err(NotSnapshottable::DataStore(kind)) => Err(Refusal::Unsupported(format!( + "A snapshot needs RocksDB as the data store, and this server's is {kind}. \ + Use {kind}'s own backup." + ))), + Err(NotSnapshottable::BlobStore(kind)) => Err(Refusal::Unsupported(format!( + "A snapshot copies blobs kept in the data store or in a filesystem blob store, \ + and this server keeps them in {kind}." + ))), + } +} + +/// `20261109T013000Z`: a snapshot's name, the UTC time it was taken. +pub fn name_for(seconds: u64) -> String { + let t = DateTime::from_timestamp(seconds as i64); + format!( + "{:04}{:02}{:02}T{:02}{:02}{:02}Z", + t.year, t.month, t.day, t.hour, t.minute, t.second + ) +} + +fn rfc3339(seconds: u64) -> String { + DateTime::from_timestamp(seconds as i64).to_rfc3339() +} + +fn seconds_of(value: &Value) -> Option { + DateTime::parse_rfc3339(value.as_str()?).map(|t| t.to_timestamp() as u64) +} + +fn io(err: std::io::Error) -> trc::Error { + trc::StoreEvent::FilesystemError.reason(err) +} + +/// SN-1, SN-6: starts taking a snapshot on this node. Refused while one is +/// being taken, or when the stores can't have one. +pub async fn start(server: &Server, created_by: String) -> trc::Result> { + let (source, root) = match source(server) { + Ok(found) => found, + Err(refusal) => return Ok(Err(refusal)), + }; + let snapshot = { + let Ok(mut current) = CURRENT.lock() else { + return Err(trc::EventType::Server(trc::ServerEvent::ThreadError) + .into_err() + .details("The snapshot state is poisoned")); + }; + if let Some(running) = current.as_ref().filter(|c| c.status == Status::Running) { + return Ok(Err(Refusal::Running(running.id))); + } + // Names are the time to the second: a second one within the same + // second takes the next + let mut at = now(); + while root.join(name_for(at)).exists() { + at += 1; + } + let name = name_for(at); + let mut holds = vec!["data".to_string()]; + if matches!(source, SnapshotSource::DataAndFiles { .. }) { + holds.push("blobs".to_string()); + } + let snapshot = Snapshot { + id: at, + path: root.join(&name), + name, + status: Status::Running, + created_at: at, + created_by, + finished_at: None, + node_id: server.core.network.node_id, + holds, + size: 0, + blob_files: 0, + error: None, + }; + *current = Some(snapshot.clone()); + snapshot + }; + let server = server.clone(); + let running = snapshot.clone(); + tokio::spawn(async move { + run(&server, source, root, running).await; + }); + Ok(Ok(snapshot)) +} + +async fn run(server: &Server, source: SnapshotSource, root: PathBuf, mut snapshot: Snapshot) { + let partial = root.join(format!(".{}.partial", snapshot.name)); + match take(server, &source, &root, &partial, &mut snapshot).await { + // Listed from disk from now on + Ok(()) => set_current(None), + Err(err) => { + let _ = tokio::fs::remove_dir_all(&partial).await; + snapshot.status = Status::Failed; + snapshot.finished_at = Some(now()); + snapshot.error = Some(err.to_string()); + trc::error!(err.details("Failed to take a store snapshot")); + set_current(Some(snapshot)); + } + } +} + +async fn take( + server: &Server, + source: &SnapshotSource, + root: &Path, + partial: &Path, + snapshot: &mut Snapshot, +) -> trc::Result<()> { + tokio::fs::create_dir_all(root).await.map_err(io)?; + let _ = tokio::fs::remove_dir_all(partial).await; + tokio::fs::create_dir(partial).await.map_err(io)?; + + // SN-4: the purge is held before the checkpoint, so every blob the + // checkpoint links to is still there when it's copied + let hold = match source { + SnapshotSource::DataAndFiles { .. } => Some(hold_blob_purge().await), + SnapshotSource::DataOnly { .. } => None, + }; + server.store().checkpoint(partial.join("data")).await?; + let blob_path = match source { + SnapshotSource::DataAndFiles { blob_path, .. } => { + let copy = copy_tree( + blob_path.clone(), + partial.join("blobs"), + data_path_of(source).to_path_buf(), + ) + .await?; + snapshot.blob_files = copy.files; + Some(blob_path) + } + SnapshotSource::DataOnly { .. } => None, + }; + drop(hold); + + snapshot.size = tree_size(partial.to_path_buf()).await?; + snapshot.status = Status::Done; + snapshot.finished_at = Some(now()); + let manifest = manifest( + snapshot, + data_path_of(source), + blob_path.map(|p| p.as_path()), + ); + tokio::fs::write( + partial.join(MANIFEST), + serde_json::to_vec_pretty(&manifest).unwrap_or_default(), + ) + .await + .map_err(io)?; + + // SN-3: whole once it has its name + tokio::fs::rename(partial, &snapshot.path) + .await + .map_err(io)?; + if let Ok(dir) = tokio::fs::File::open(root).await { + let _ = dir.sync_all().await; + } + prune(root).await; + refresh_latest(root).await; + Ok(()) +} + +fn data_path_of(source: &SnapshotSource) -> &Path { + match source { + SnapshotSource::DataOnly { data_path } | SnapshotSource::DataAndFiles { data_path, .. } => { + data_path + } + } +} + +/// SN-5. +fn manifest(snapshot: &Snapshot, data_path: &Path, blob_path: Option<&Path>) -> Value { + let mut restore = vec![ + "Stop the server.".to_string(), + format!( + "Move the contents of {} aside, except snapshots/, and copy this snapshot's data/ into it.", + data_path.display() + ), + ]; + if let Some(blob_path) = blob_path { + restore.push(format!( + "Move the contents of {} aside and copy this snapshot's blobs/ into it.", + blob_path.display() + )); + } + restore.push("Start the server.".to_string()); + json!({ + "id": Id::from(snapshot.id).to_string(), + "name": snapshot.name, + "createdAt": rfc3339(snapshot.created_at), + "finishedAt": snapshot.finished_at.map(rfc3339), + "createdBy": snapshot.created_by, + "node": snapshot.node_id.to_string(), + "server": concat!(types::brand_server!(), " ", types::brand_version!()), + "holds": snapshot.holds, + "dataStore": {"type": "RocksDB", "path": data_path.to_string_lossy()}, + "blobStore": match blob_path { + Some(path) => json!({"type": "filesystem", "path": path.to_string_lossy()}), + None => json!({"type": "in the data store"}), + }, + "size": snapshot.size, + "blobFiles": snapshot.blob_files, + "restore": restore, + }) +} + +fn from_manifest(path: PathBuf, manifest: &Value) -> Option { + let id = Id::from_str(manifest.get("id")?.as_str()?).ok()?.id(); + Some(Snapshot { + id, + name: manifest.get("name")?.as_str()?.to_string(), + status: Status::Done, + created_at: seconds_of(manifest.get("createdAt")?)?, + created_by: manifest.get("createdBy")?.as_str()?.to_string(), + finished_at: manifest.get("finishedAt").and_then(seconds_of), + node_id: manifest.get("node")?.as_str()?.parse().ok()?, + path, + holds: manifest + .get("holds")? + .as_array()? + .iter() + .filter_map(|h| h.as_str().map(|s| s.to_string())) + .collect(), + size: manifest.get("size")?.as_u64()?, + blob_files: manifest.get("blobFiles")?.as_u64()?, + error: None, + }) +} + +/// The complete snapshots under `root`, oldest first. A directory without a +/// readable `snapshot.json` isn't one, and is left alone. +async fn on_disk(root: &Path) -> Vec { + let mut found = Vec::new(); + let Ok(mut entries) = tokio::fs::read_dir(root).await else { + return found; + }; + while let Ok(Some(entry)) = entries.next_entry().await { + let name = entry.file_name().to_string_lossy().to_string(); + if name.starts_with('.') || !entry.file_type().await.is_ok_and(|t| t.is_dir()) { + continue; + } + let path = entry.path(); + let Ok(bytes) = tokio::fs::read(path.join(MANIFEST)).await else { + continue; + }; + if let Some(snapshot) = serde_json::from_slice::(&bytes) + .ok() + .and_then(|manifest| from_manifest(path, &manifest)) + .filter(|s| s.name == name) + { + found.push(snapshot); + } + } + found.sort_by_key(|s| s.id); + found +} + +/// SN-6: the oldest beyond the latest [`KEEP`] go. +async fn prune(root: &Path) { + let snapshots = on_disk(root).await; + let surplus = snapshots.len().saturating_sub(KEEP); + for old in &snapshots[..surplus] { + if let Err(err) = tokio::fs::remove_dir_all(&old.path).await { + trc::error!(io(err).details("Failed to remove an old store snapshot")); + } + } +} + +/// SN-3: `latest` names the newest complete snapshot, relative to the +/// snapshots' directory so it holds wherever that's mounted. +async fn refresh_latest(root: &Path) { + let latest = root.join(LATEST); + match on_disk(root).await.last() { + Some(newest) => { + let temp = root.join(format!(".{LATEST}.tmp")); + let _ = tokio::fs::remove_file(&temp).await; + let linked = async { + tokio::fs::symlink(&newest.name, &temp).await?; + tokio::fs::rename(&temp, &latest).await + } + .await; + if let Err(err) = linked { + trc::error!(io(err).details("Failed to point snapshots/latest at a snapshot")); + } + } + None => { + let _ = tokio::fs::remove_file(&latest).await; + } + } +} + +/// Every snapshot, newest first: those on disk, and the one being taken or +/// the last one that failed (SN-7). +pub async fn list(server: &Server) -> Vec { + let mut snapshots = match source(server) { + Ok((_, root)) => on_disk(&root).await, + Err(_) => Vec::new(), + }; + if let Some(current) = current().filter(|c| !snapshots.iter().any(|s| s.id == c.id)) { + snapshots.push(current); + } + snapshots.sort_by_key(|s| std::cmp::Reverse(s.id)); + snapshots +} + +/// One snapshot, as [`list`] has it. +pub async fn find(server: &Server, id: u64) -> Option { + list(server).await.into_iter().find(|s| s.id == id) +} + +/// SN-9: deletes a snapshot's directory. One being taken can't be. +pub async fn destroy(server: &Server, id: u64) -> trc::Result { + if let Some(current) = current().filter(|c| c.id == id) { + if current.status == Status::Running { + return Ok(Destroyed::Running); + } + set_current(None); + return Ok(Destroyed::Gone); + } + let Ok((_, root)) = source(server) else { + return Ok(Destroyed::NotFound); + }; + let Some(snapshot) = on_disk(&root).await.into_iter().find(|s| s.id == id) else { + return Ok(Destroyed::NotFound); + }; + tokio::fs::remove_dir_all(&snapshot.path) + .await + .map_err(io)?; + refresh_latest(&root).await; + Ok(Destroyed::Gone) +} + +/// SN-7: a snapshot being taken when the server stopped left a partial +/// directory, removed when it starts. +pub fn spawn_store_snapshots(inner: Arc) { + tokio::spawn(async move { + let server = common::BuildServer::build_server(&inner); + let Ok((_, root)) = source(&server) else { + return; + }; + let Ok(mut entries) = tokio::fs::read_dir(&root).await else { + return; + }; + while let Ok(Some(entry)) = entries.next_entry().await { + let name = entry.file_name().to_string_lossy().to_string(); + if name.starts_with('.') && name.ends_with(".partial") { + let _ = tokio::fs::remove_dir_all(entry.path()).await; + } + } + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_are_the_time() { + assert_eq!(name_for(1_794_187_800), "20261109T013000Z"); + assert_eq!(name_for(0), "19700101T000000Z"); + } + + #[test] + fn a_manifest_reads_back() { + let snapshot = Snapshot { + id: 1_794_187_800, + name: name_for(1_794_187_800), + status: Status::Done, + created_at: 1_794_187_800, + created_by: "admin@example.com".into(), + finished_at: Some(1_794_187_802), + node_id: 3, + path: PathBuf::from("/var/lib/inbuxa/snapshots/20261109T013000Z"), + holds: vec!["data".into(), "blobs".into()], + size: 4096, + blob_files: 12, + error: None, + }; + let manifest = manifest( + &snapshot, + Path::new("/var/lib/inbuxa/"), + Some(Path::new("/var/lib/inbuxa-blobs")), + ); + assert_eq!(manifest["restore"].as_array().unwrap().len(), 4); + assert_eq!( + from_manifest(snapshot.path.clone(), &manifest).unwrap(), + snapshot + ); + } +} diff --git a/crates/services/src/lib.rs b/crates/services/src/lib.rs index ddb4a75f..86ad461c 100644 --- a/crates/services/src/lib.rs +++ b/crates/services/src/lib.rs @@ -31,6 +31,7 @@ pub mod inbuxa_relay_certificates; // inbuxa: inbuxa Relays' certificates pub mod inbuxa_relay_link; // inbuxa: links to inbuxa Relays pub mod inbuxa_scheduled_reports; // inbuxa: scheduled reports and the weekly digest pub mod inbuxa_store_check; // inbuxa: checking the store's blobs +pub mod inbuxa_store_snapshot; // inbuxa: a snapshot of the store pub mod state_manager; pub mod task_manager; @@ -88,6 +89,10 @@ impl SpawnServices for IpcReceivers { // inbuxa: store-check spec, SV-8: a check this node ran stopped with it inbuxa_store_check::spawn_store_checks(inner.clone()); + // inbuxa: store-snapshot spec, SN-7: a snapshot cut short by a + // stop leaves a partial directory + inbuxa_store_snapshot::spawn_store_snapshots(inner.clone()); + // inbuxa: relay spec, "The link service": every node links to every relay inbuxa_relay_link::spawn_relay_links(inner.clone()); diff --git a/crates/services/src/task_manager/maintenance.rs b/crates/services/src/task_manager/maintenance.rs index 186ca4e1..844b352b 100644 --- a/crates/services/src/task_manager/maintenance.rs +++ b/crates/services/src/task_manager/maintenance.rs @@ -310,6 +310,17 @@ async fn store_maintenance( } TaskStoreMaintenanceType::PurgeBlob => { if let Some(shard_index) = task.shard_index { + // inbuxa: store-snapshot spec, SN-4: while a snapshot takes + // the blobs, the purge is put off by five minutes + let Some(_purging) = store::write::snapshot::blob_purge_may_run() else { + return Ok(TaskResult::Success(vec![Task::StoreMaintenance( + TaskStoreMaintenance { + maintenance_type: TaskStoreMaintenanceType::PurgeBlob, + shard_index: Some(shard_index), + status: TaskStatus::at(now() as i64 + 300), + }, + )])); + }; server .store() .purge_blobs(server.blob_store().clone(), shard_index as u8) diff --git a/crates/store/src/backend/fs/mod.rs b/crates/store/src/backend/fs/mod.rs index d9a4b5d7..c4ce609a 100644 --- a/crates/store/src/backend/fs/mod.rs +++ b/crates/store/src/backend/fs/mod.rs @@ -2,6 +2,8 @@ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ use crate::BlobStore; @@ -77,9 +79,22 @@ impl FsStore { fs::create_dir_all(blob_path.parent().unwrap()) .await .map_err(into_error)?; - let mut blob_file = File::create(&blob_path).await.map_err(into_error)?; - blob_file.write_all(data).await.map_err(into_error)?; - blob_file.flush().await.map_err(into_error)?; + // inbuxa: store-snapshot spec, SN-10: written beside it and + // renamed into place, never rewritten in place. A snapshot holds + // blobs by hard link, and a rewrite would change its copy too; a + // stop mid-write would leave half a blob under its id. + let temp_path = temp_path_for(&blob_path); + let written = async { + let mut blob_file = File::create(&temp_path).await?; + blob_file.write_all(data).await?; + blob_file.flush().await?; + fs::rename(&temp_path, &blob_path).await + } + .await; + if let Err(err) = written { + let _ = fs::remove_file(&temp_path).await; + return Err(into_error(err)); + } } Ok(()) @@ -95,6 +110,11 @@ impl FsStore { } } + /// inbuxa: the directory the blobs are under (store-snapshot spec). + pub fn root(&self) -> &std::path::Path { + &self.path + } + fn build_path(&self, key: &[u8]) -> PathBuf { let mut path = self.path.clone(); @@ -106,6 +126,66 @@ impl FsStore { } } +/// inbuxa: SN-10. A blob's name is base32, so a name starting with a dot +/// is never a blob: the snapshot's copy and anything listing blobs skip it. +fn temp_path_for(blob_path: &std::path::Path) -> PathBuf { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let n = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let name = blob_path + .file_name() + .map(|name| name.to_string_lossy()) + .unwrap_or_default(); + blob_path.with_file_name(format!(".{name}.{}-{n}.tmp", std::process::id())) +} + fn into_error(err: std::io::Error) -> trc::Error { trc::StoreEvent::FilesystemError.reason(err) } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn writes_go_through_a_temporary_file() { + let dir = std::env::temp_dir().join(format!("inbuxa-fs-sn10-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let BlobStore::Fs(store) = FsStore::open(structs::FileSystemStore { + path: dir.to_string_lossy().to_string(), + depth: 2, + }) + .await + .unwrap() else { + unreachable!() + }; + let key = [7u8; 32]; + store.put_blob(&key, b"first").await.unwrap(); + let path = store.build_path(&key); + + // A file held by a hard link, as a snapshot holds it, keeps what it + // had when the blob under its id is written again + let held = dir.join("held"); + std::fs::hard_link(&path, &held).unwrap(); + store.put_blob(&key, b"second, longer").await.unwrap(); + assert_eq!(std::fs::read(&held).unwrap(), b"first"); + assert_eq!( + store.get_blob(&key, 0..usize::MAX).await.unwrap().unwrap(), + b"second, longer" + ); + + // Nothing is left beside it + let names: Vec<_> = std::fs::read_dir(path.parent().unwrap()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert_eq!(names.len(), 1, "{names:?}"); + assert!( + temp_path_for(&path) + .file_name() + .unwrap() + .to_string_lossy() + .starts_with('.') + ); + std::fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/crates/store/src/backend/rocksdb/checkpoint.rs b/crates/store/src/backend/rocksdb/checkpoint.rs new file mode 100644 index 00000000..91720fe0 --- /dev/null +++ b/crates/store/src/backend/rocksdb/checkpoint.rs @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! inbuxa: a checkpoint of the store (store-snapshot spec, SN-2): every +//! column family at one point in time, hard-linked where it can be. + +use super::{RocksDbStore, into_error}; +use rocksdb::checkpoint::Checkpoint; +use std::path::{Path, PathBuf}; + +impl RocksDbStore { + /// The store's directory. + pub fn path(&self) -> &Path { + self.db.path() + } + + /// Writes a checkpoint to `dest`, which must not exist yet. + pub async fn checkpoint(&self, dest: PathBuf) -> trc::Result<()> { + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + Checkpoint::new(&*db) + .and_then(|checkpoint| checkpoint.create_checkpoint(&dest)) + .map_err(into_error) + }) + .await + .map_err(|err| trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err))? + } +} diff --git a/crates/store/src/backend/rocksdb/mod.rs b/crates/store/src/backend/rocksdb/mod.rs index ed409815..3840f3c8 100644 --- a/crates/store/src/backend/rocksdb/mod.rs +++ b/crates/store/src/backend/rocksdb/mod.rs @@ -2,6 +2,8 @@ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ use std::sync::Arc; @@ -11,6 +13,7 @@ use rocksdb::{BoundColumnFamily, MultiThreaded, OptimisticTransactionDB}; use crate::{SUBSPACE_BLOBS, SUBSPACE_INDEXES, SUBSPACE_LOGS}; pub mod blob; +pub mod checkpoint; // inbuxa: store-snapshot spec pub mod main; pub mod read; pub mod write; diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index bfb3e808..c8ab6e80 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -33,6 +33,7 @@ pub mod blob_check; // inbuxa: store-check spec pub mod key; pub mod log; pub mod serialize; +pub mod snapshot; // inbuxa: store-snapshot spec pub(crate) const ARCHIVE_ALIGNMENT: usize = 16; diff --git a/crates/store/src/write/snapshot.rs b/crates/store/src/write/snapshot.rs new file mode 100644 index 00000000..aff8a543 --- /dev/null +++ b/crates/store/src/write/snapshot.rs @@ -0,0 +1,239 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! inbuxa: what the store-snapshot spec (docs/spec/features/store-snapshot.md) +//! needs from the store: holding the blob purge (SN-4), a checkpoint of the +//! data store and a copy of a filesystem blob store's files (SN-2). + +use crate::{BlobStore, Store}; +use std::{ + io, + path::{Path, PathBuf}, +}; +use tokio::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard}; + +/// SN-4. A purge runs under the read side; a snapshot taking blobs holds +/// the write side. The data store is RocksDB, so one process has it open, +/// and a lock in that process is enough. +static BLOB_PURGE_GATE: RwLock<()> = RwLock::const_new(()); + +/// Whether a blob purge may run now: `None` while a snapshot takes blobs or +/// waits to. Keep the guard for as long as the purge runs. +pub fn blob_purge_may_run() -> Option> { + BLOB_PURGE_GATE.try_read().ok() +} + +/// Holds the blob purge until the guard is dropped. A purge already running +/// finishes first. +pub async fn hold_blob_purge() -> RwLockWriteGuard<'static, ()> { + BLOB_PURGE_GATE.write().await +} + +/// What a snapshot can hold of this server's stores (SN-1, SN-2). +pub enum SnapshotSource { + /// Blobs are in the data store, so the checkpoint holds them. + DataOnly { data_path: PathBuf }, + /// Blobs are files under `blob_path`, copied after the checkpoint. + DataAndFiles { + data_path: PathBuf, + blob_path: PathBuf, + }, +} + +/// Why a snapshot can't be taken of these stores. +pub enum NotSnapshottable { + DataStore(&'static str), + BlobStore(&'static str), +} + +impl Store { + /// SN-2: a checkpoint of the data store into `dest`, which must not + /// exist yet. Only RocksDB has one. + #[cfg_attr(not(feature = "rocks"), allow(unused_variables))] + pub async fn checkpoint(&self, dest: PathBuf) -> trc::Result<()> { + match self { + #[cfg(feature = "rocks")] + Store::RocksDb(store) => store.checkpoint(dest).await, + _ => Err(trc::StoreEvent::NotSupported + .into_err() + .details("Only a RocksDB store has a checkpoint")), + } + } + + /// What a snapshot of this data store and `blobs` would hold, or why + /// there can't be one. + #[cfg_attr(not(feature = "rocks"), allow(unused_variables, unreachable_code))] + pub fn snapshot_source(&self, blobs: &BlobStore) -> Result { + let data_path = match self { + #[cfg(feature = "rocks")] + Store::RocksDb(store) => store.path().to_path_buf(), + #[cfg(feature = "sqlite")] + Store::SQLite(_) => return Err(NotSnapshottable::DataStore("SQLite")), + #[cfg(feature = "foundation")] + Store::FoundationDb(_) => return Err(NotSnapshottable::DataStore("FoundationDB")), + #[cfg(feature = "postgres")] + Store::PostgreSQL(_) => return Err(NotSnapshottable::DataStore("PostgreSQL")), + #[cfg(feature = "mysql")] + Store::MySQL(_) => return Err(NotSnapshottable::DataStore("MySQL")), + Store::Replicated(_) => { + return Err(NotSnapshottable::DataStore("a replicated SQL database")); + } + Store::Ephemeral(_) | Store::None => { + return Err(NotSnapshottable::DataStore("an in-memory store")); + } + }; + match blobs { + BlobStore::Store(store) if store.is_same(self) => { + Ok(SnapshotSource::DataOnly { data_path }) + } + BlobStore::Fs(fs) => { + let blob_path = fs.root().to_path_buf(); + // The blob files can't be told from the data store's own + // in one directory. A blob directory that holds the data + // store's is fine: the copy leaves that one out + if data_path == blob_path { + return Err(NotSnapshottable::BlobStore( + "the data store's own directory", + )); + } + Ok(SnapshotSource::DataAndFiles { + data_path, + blob_path, + }) + } + BlobStore::Store(_) => Err(NotSnapshottable::BlobStore("another database")), + #[cfg(feature = "s3")] + BlobStore::S3(_) => Err(NotSnapshottable::BlobStore("S3")), + #[cfg(feature = "azure")] + BlobStore::Azure(_) => Err(NotSnapshottable::BlobStore("Azure")), + BlobStore::Sharded(_) => Err(NotSnapshottable::BlobStore("a sharded set")), + } + } +} + +/// What a copy of a tree did. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct TreeCopy { + pub files: u64, + pub bytes: u64, + /// Files that couldn't be hard-linked and were copied. + pub copied: u64, +} + +/// SN-2, SN-3: every file under `from` into `to`, hard-linked where it can +/// be and copied where it can't, leaving out `skip` (the data store's +/// directory, with the snapshots in it). Names starting with a dot are +/// skipped: they're blobs being written (SN-10). +pub async fn copy_tree(from: PathBuf, to: PathBuf, skip: PathBuf) -> trc::Result { + tokio::task::spawn_blocking(move || { + let mut done = TreeCopy::default(); + copy_dir(&from, &to, &skip, &mut done).map(|_| done) + }) + .await + .map_err(|err| trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err))? + .map_err(|err| trc::StoreEvent::FilesystemError.reason(err)) +} + +fn copy_dir(from: &Path, to: &Path, skip: &Path, done: &mut TreeCopy) -> io::Result<()> { + std::fs::create_dir_all(to)?; + for entry in std::fs::read_dir(from)? { + let entry = entry?; + let name = entry.file_name(); + if name.as_encoded_bytes().first() == Some(&b'.') { + continue; + } + let (source, dest) = (entry.path(), to.join(&name)); + if source == skip { + continue; + } + let kind = entry.file_type()?; + if kind.is_dir() { + copy_dir(&source, &dest, skip, done)?; + } else if kind.is_file() { + match std::fs::hard_link(&source, &dest) { + Ok(()) => {} + // Gone since it was listed: a purge ran before the hold, or + // it's an orphan nothing links to + Err(err) if err.kind() == io::ErrorKind::NotFound => continue, + Err(_) => { + match std::fs::copy(&source, &dest) { + Ok(_) => {} + Err(err) if err.kind() == io::ErrorKind::NotFound => continue, + Err(err) => return Err(err), + } + done.copied += 1; + } + } + done.files += 1; + done.bytes += std::fs::metadata(&dest)?.len(); + } + } + Ok(()) +} + +/// Bytes in the files under `dir`. +pub async fn tree_size(dir: PathBuf) -> trc::Result { + fn size(dir: &Path) -> io::Result { + let mut total = 0; + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let kind = entry.file_type()?; + if kind.is_dir() { + total += size(&entry.path())?; + } else if kind.is_file() { + total += entry.metadata()?.len(); + } + } + Ok(total) + } + tokio::task::spawn_blocking(move || size(&dir)) + .await + .map_err(|err| trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err))? + .map_err(|err| trc::StoreEvent::FilesystemError.reason(err)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn copies_a_tree_and_skips_what_is_being_written() { + let base = std::env::temp_dir().join(format!("inbuxa-sn-tree-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&base); + let from = base.join("from"); + std::fs::create_dir_all(from.join("a/b")).unwrap(); + std::fs::write(from.join("a/b/BLOB1"), b"one").unwrap(); + std::fs::write(from.join("a/BLOB2"), b"two!").unwrap(); + std::fs::write(from.join("a/b/.BLOB3.1-0.tmp"), b"half").unwrap(); + std::fs::create_dir_all(from.join("rocks.db/snapshots")).unwrap(); + std::fs::write(from.join("rocks.db/000001.sst"), b"data").unwrap(); + + let to = base.join("to"); + let done = copy_tree(from.clone(), to.clone(), from.join("rocks.db")) + .await + .unwrap(); + assert_eq!((done.files, done.bytes), (2, 7)); + assert_eq!(std::fs::read(to.join("a/b/BLOB1")).unwrap(), b"one"); + assert!(!to.join("a/b/.BLOB3.1-0.tmp").exists()); + assert!(!to.join("rocks.db").exists()); + assert_eq!(tree_size(to).await.unwrap(), 7); + std::fs::remove_dir_all(&base).unwrap(); + } + + #[test] + fn the_purge_waits_while_a_snapshot_holds_it() { + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + runtime.block_on(async { + assert!(blob_purge_may_run().is_some()); + let hold = hold_blob_purge().await; + assert!(blob_purge_may_run().is_none()); + drop(hold); + assert!(blob_purge_may_run().is_some()); + }); + } +} diff --git a/docs/spec/SPEC.md b/docs/spec/SPEC.md index 7c2b5578..e9e45953 100644 --- a/docs/spec/SPEC.md +++ b/docs/spec/SPEC.md @@ -388,6 +388,8 @@ Not a rebuild: **migrations into this server** are INBUXA's own design (`feature Not a rebuild: the **store check** is INBUXA's own design (`features/store-check.md`, inbuxa-server #322, building). An administrator starts a check of the store's blobs, quick (each one is there) or full (each one reads back intact), and gets a list of what's missing, damaged or no longer linked. One node runs it in the background, throttled, and keeps its progress: `inbuxa:StoreCheck` (`crates/jmap/src/inbuxa/store_check.rs`, `crates/features/src/store_check/`, `crates/services/src/inbuxa_store_check.rs`, `crates/store/src/write/blob_check.rs`), and the `sysStoreCheck` permission. +Not a rebuild: the **store snapshot** is INBUXA's own design (`features/store-snapshot.md`, inbuxa-server #323, building). An administrator has the server take a consistent copy of its RocksDB store, and of a filesystem blob store's files, into `snapshots/` in the store's directory while mail flows, for a backup tool to copy; the latest two are kept: `inbuxa:StoreSnapshot` (`crates/jmap/src/inbuxa/store_snapshot.rs`, `crates/services/src/inbuxa_store_snapshot.rs`, `crates/store/src/write/snapshot.rs`, `crates/store/src/backend/rocksdb/checkpoint.rs`), and the `sysStoreSnapshot` permission. + Not a rebuild: **the store layout, changed in steps** is INBUXA's own design (`features/store-layout.md`, approved 2026-11-08). A store carries its layout version (`Vs` in `_`); at boot one node, holding a lease, runs each step from the stored version up and marks the store after each. Step 1 → 2 gives SQL rate-limit counters their own expiry column; step 2 → 3 moves the audit log, journal, holds and undelete notes into subspaces of their own. A step stops rollback, so it ships at a series. Not a rebuild: **inbuxa-relay** is INBUXA's own design (`features/relay.md`, settled 2026-10-29, in progress). It's a small edge appliance that takes mail in on port 25 for one downstream server or cluster, and sends that downstream's outgoing mail on. The downstream dials out to it over JMAP (`urn:inbuxa:jmap:relay`), so the protected network exposes no mail port, and it is administered only from the downstream's inbuxa-admin. diff --git a/docs/spec/features/store-snapshot.md b/docs/spec/features/store-snapshot.md new file mode 100644 index 00000000..e8a99b5e --- /dev/null +++ b/docs/spec/features/store-snapshot.md @@ -0,0 +1,148 @@ +# Feature spec: a snapshot of the store, without stopping the server + +Status: building, 2026-11-09 (inbuxa-server #323). Not a rebuild (SPEC.md +§4): inbuxa's own design. + +## Provenance + +| Source | Used for | +|---|---| +| Forum topic 238 | The need: a backup that stops the server, copies the store cold and starts it again, an outage each time | +| RocksDB's checkpoint (`rocksdb::checkpoint::Checkpoint`, rocksdb 0.25) | A consistent copy of every column family at once, by hard links on the same filesystem | +| This server's blob writes and purge (`crates/store/src/write/blob.rs`, `crates/store/src/backend/fs/mod.rs`) | Why a snapshot of the data store, then of the blobs, holds every blob its data links to | +| The store check (`features/store-check.md`) | How the object, its permission and its run are built | + +Upstream's open code has nothing like it, and nothing here comes from +Enterprise code. + +## What it is + +An administrator asks the server for a snapshot of its store. The server +writes a consistent copy into a directory on its own machine, while mail +keeps flowing, and says when it's done. A backup tool then copies that +directory however it likes. Encryption, off-site copies and how long +backups are kept stay with the backup tool. The server keeps only its +latest two snapshots. + +A snapshot is a cold copy that was taken hot. Restoring one is copying it +back while the server is stopped. + +## Requirements + +- **SN-1.** A snapshot is an `inbuxa:StoreSnapshot` under `urn:inbuxa:jmap`. + It needs RocksDB as the data store. With any other data store a create + is refused, and the refusal says to use the database's own backup: + PostgreSQL's or MySQL's, or FoundationDB's. +- **SN-2.** What it holds: + - **The data store**: a RocksDB checkpoint of every column family, taken + at one point in time. Blobs kept in the data store (the default) are + in it too. + - **Blobs in a separate filesystem blob store**, copied after the + checkpoint. Blobs are written before anything links to them, and the + blob purge is held while the snapshot is taken (SN-4), so every blob + the checkpoint links to is there to copy. A blob written after the + checkpoint may be copied as well; nothing in the snapshot links to it. + + A blob directory that holds the data store's directory is fine: the + copy leaves the data store's directory out. Blobs in the data store's + own directory are refused, since they can't be told from its files. + Any other blob store (S3, Azure, a sharded set, a SQL database) is + refused: a copy of the data store alone would point at blobs a later + purge could take away before the backup tool copied them. + + Not in it: the search index when it's kept outside the data store + (Meilisearch, Elasticsearch, ...), rebuilt after a restore; and the + in-memory store (rate limits, locks), which a restarted server doesn't + need. +- **SN-3.** Where: under `snapshots/` in the RocksDB directory, one + directory per snapshot named for when it was taken (UTC), like + `snapshots/20261109T013000Z/`, with `data/`, `blobs/` when there are + separate blobs, and `snapshot.json`. Being on the store's own + filesystem, RocksDB and the blob copy hard-link their files, so a snapshot + is quick and at first takes almost no space. Where they can't link + (another filesystem), they copy. The location is fixed: a request can't + name a path. + + A snapshot is written as `snapshots/..partial/` and renamed when + complete, so a directory with the plain name is always whole. + `snapshots/latest` is a link to the newest complete one. +- **SN-4.** While a snapshot takes blobs from a filesystem blob store, the + blob purge waits. A purge already running is let finish first. A purge + task that comes due meanwhile is put off by five minutes. +- **SN-5.** `snapshot.json` says when it was taken and finished, by whom, + the server version, what it holds (`data`, `blobs`), the data and blob + stores' paths, its size, and how to restore it. +- **SN-6.** One snapshot is made at a time; another create is refused + while one runs. When one completes, the oldest beyond the latest two are + deleted. +- **SN-7.** A snapshot is `running`, then `done`, or `failed` with the + error and its partial directory removed. One the server stopped in the + middle of leaves a partial directory, removed when the server next + starts, and isn't listed. + + The complete snapshots on disk are the record: `get` lists those, read + from each one's `snapshot.json`, and, until the server restarts, the one + being taken or the last one that failed. +- **SN-8.** Only administrators take snapshots: `sysStoreSnapshot`, in the + built-in Administrator role and granted to existing administrator roles. + A snapshot holds every account's data, so a tenant administrator is + refused. Taking and destroying a snapshot are audited. +- **SN-9.** Destroying a snapshot deletes its directory. A running one + can't be destroyed. +- **SN-10.** A filesystem blob store writes each blob to a temporary file + in the same directory and renames it into place. Before, it rewrote a + file in place, which would also have changed the copy a snapshot holds + by hard link, and left a half-written blob behind if the server stopped + mid-write. + +## The object + +| Property | Type | | +|---|---|---| +| `id` | Id | | +| `status` | String | `running`, `done`, `failed` | +| `createdAt`, `finishedAt` | UTCDate | | +| `createdBy` | String | Who took it | +| `node` | String | The node that took it | +| `name` | String | Its directory under `snapshots/` | +| `path` | String | Its full path on the server's machine | +| `holds` | String[] | `data`, and `blobs` when they're separate | +| `size` | UnsignedInt | Bytes in its files | +| `blobFiles` | UnsignedInt | Blob files copied | +| `error` | String | Why a `failed` snapshot failed | + +A create takes no properties. + +## Restoring + +1. Stop the server. +2. Move the RocksDB directory's contents aside, except `snapshots/`, and + copy the snapshot's `data/` into it. +3. With a separate filesystem blob store, do the same with its directory + and the snapshot's `blobs/`. +4. Start the server. A search index kept outside the data store isn't in + the snapshot; reindex the accounts once it's running. + +A backup tool copying the store cold should leave out `snapshots/`, or +the store is copied twice. + +## Tests + +- `system::store_snapshot` (`tests/src/system/store_snapshot.rs`), against + a real server with RocksDB, run with blobs in the data store and again + with `BLOB_STORE=FileSystem`: a snapshot taken while mail is in the + store; the directory, its name, `latest` and the manifest; a RocksDB + store opened from its `data/` has the messages and their blobs (in it, + or in `blobs/`), and not a message that came after; no properties at + create, no updates; with separate blobs, one at a time and no destroy + while one is taken (the test holds the purge gate); only the latest two + kept; destroy; the audit events; a user's and a tenant administrator's + refusal. +- Unit tests: the filesystem blob store's writes (SN-10), the tree copy + and what it leaves out, the purge gate, snapshot names and the manifest. + +## Not yet + +- The console's page and an `inbuxa-cli` command a backup tool can call + and wait on. +- SQLite as the data store (`VACUUM INTO` would do it). diff --git a/resources/privacy/catalog.toml b/resources/privacy/catalog.toml index 1d5e121f..af34a9e6 100644 --- a/resources/privacy/catalog.toml +++ b/resources/privacy/catalog.toml @@ -281,6 +281,16 @@ retention = "object-life" createdBy = ["identifier"] problems = ["identifier", "metadata"] +[object."inbuxa:StoreSnapshot"] +file = "inbuxa_store_snapshot.rs" +default = "none" +whose = ["administrator"] +where = ["data-directory"] +scope = "server" +retention = "object-life" +[object."inbuxa:StoreSnapshot".properties] +createdBy = ["identifier"] + [object."inbuxa:Migration"] file = "inbuxa_migration.rs" default = "none" @@ -423,6 +433,19 @@ retention = "object-life" leaves_host = false written_by = ["crates/services/src/inbuxa_migrations.rs"] +# A store snapshot: a copy of the whole data store, and of a filesystem +# blob store's files, under snapshots/ in the RocksDB directory. The latest +# two are kept; a backup tool takes them off the host if the administrator +# has one do so (store-snapshot spec, SN-3, SN-6). +[source."store-snapshots"] +categories = ["content", "identifier", "contact", "network", "metadata", "credential"] +whose = ["holder", "correspondent", "administrator"] +where = ["data-directory"] +scope = "server" +retention = "object-life" +leaves_host = false +written_by = ["crates/services/src/inbuxa_store_snapshot.rs"] + [source."full-text-index"] categories = ["content", "identifier", "contact", "metadata"] whose = ["holder", "correspondent"] diff --git a/resources/schema/schema.json.gz b/resources/schema/schema.json.gz index 3889c2d8..2d5a4fd5 100644 Binary files a/resources/schema/schema.json.gz and b/resources/schema/schema.json.gz differ diff --git a/resources/schema/schema.json.sha256 b/resources/schema/schema.json.sha256 index ae467ecc..4aa9d99d 100644 --- a/resources/schema/schema.json.sha256 +++ b/resources/schema/schema.json.sha256 @@ -1 +1 @@ -Z7m4tA0knFnM6t30zrTSVKWBuNKOEGEMouWPHB3_ED4 \ No newline at end of file +6ZXhO67yQsLEItXqRvBehskGGCr53WdaWL9Nh6i7Nm0 \ No newline at end of file diff --git a/tests/src/system/mod.rs b/tests/src/system/mod.rs index 69731127..bffd2c6c 100644 --- a/tests/src/system/mod.rs +++ b/tests/src/system/mod.rs @@ -39,6 +39,7 @@ pub mod security; pub mod security_acceptances; // inbuxa: accepted security to-do items pub mod sharing_policy; // inbuxa: MA-C, who may share mail pub mod store_check; // inbuxa: checking the store's blobs +pub mod store_snapshot; // inbuxa: a snapshot of the store pub mod task; pub mod tenant; pub mod tracer_reload; // inbuxa: tracers start over when their settings change diff --git a/tests/src/system/store_snapshot.rs b/tests/src/system/store_snapshot.rs new file mode 100644 index 00000000..6f25d35f --- /dev/null +++ b/tests/src/system/store_snapshot.rs @@ -0,0 +1,411 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! `inbuxa:StoreSnapshot` (docs/spec/features/store-snapshot.md). +//! +//! A snapshot taken while the server runs holds the mail that was in it and +//! not the mail that came after: a RocksDB store opened from its `data/` +//! has the messages, and their blobs are in it or, with a filesystem blob +//! store (`BLOB_STORE=FileSystem`), in its `blobs/`. The directory is +//! whole, named for its time, with `latest` and `snapshot.json`. Only the +//! latest two are kept; one can be destroyed, a snapshot can't be changed, +//! and one being taken can't be destroyed or joined by another. Nobody in +//! a tenant or without the permission reaches snapshots. Taking and +//! destroying are audited. + +use crate::utils::{ + account::Account, + server::{TestServer, TestServerBuilder}, +}; +use email::{mailbox::INBOX_ID, message::metadata::MessageMetadata}; +use registry::schema::{ + prelude::{ObjectType, Property}, + structs::{ + self, CertificateManagement, DkimManagement, DnsManagement, Domain, Tenant, UserRoles, + }, +}; +use serde_json::{Value, json}; +use std::{path::PathBuf, str::FromStr, time::Duration}; +use store::{ + BlobStore, Store, ValueKey, + backend::{fs::FsStore, rocksdb::RocksDbStore}, + write::{AlignedBytes, Archive, snapshot::hold_blob_purge}, +}; +use types::{blob_hash::BlobHash, collection::Collection, field::EmailField, id::Id}; + +const USING: &[&str] = &[ + "urn:ietf:params:jmap:core", + "urn:inbuxa:jmap", + "urn:inbuxa:jmap:registry", +]; + +async fn call(account: &Account, method: &str, mut arguments: Value) -> (String, Value) { + if arguments.get("accountId").is_none() { + arguments["accountId"] = account.id_string().into(); + } + let response = account + .jmap_request(USING, json!([[method, arguments, "0"]])) + .await; + let call = response + .0 + .pointer("/methodResponses/0") + .cloned() + .unwrap_or_else(|| panic!("{method}: {}", response.0)); + ( + call[0].as_str().unwrap_or_default().to_string(), + call[1].clone(), + ) +} + +async fn take(admin: &Account) -> String { + let (_, response) = call( + admin, + "inbuxa:StoreSnapshot/set", + json!({"create": {"s": {}}}), + ) + .await; + response["created"]["s"]["id"] + .as_str() + .unwrap_or_else(|| panic!("not taken: {response}")) + .to_string() +} + +async fn get(admin: &Account, id: &str) -> Value { + let (_, response) = call(admin, "inbuxa:StoreSnapshot/get", json!({"ids": [id]})).await; + response["list"][0].clone() +} + +async fn all(admin: &Account) -> Vec { + let (_, response) = call(admin, "inbuxa:StoreSnapshot/get", json!({"ids": null})).await; + response["list"] + .as_array() + .unwrap_or_else(|| panic!("no list: {response}")) + .clone() +} + +/// The snapshot once it's taken. +async fn settled(admin: &Account, id: &str) -> Value { + for _ in 0..600 { + let snapshot = get(admin, id).await; + if snapshot["status"] != "running" { + return snapshot; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!("the snapshot wasn't taken: {}", get(admin, id).await); +} + +async fn import(client: &jmap_client::client::Client, body: &str) -> u32 { + let id = client + .email_import( + format!("From: a@example.org\r\nSubject: {body}\r\n\r\n{body}\r\n").into_bytes(), + [Id::from(INBOX_ID).to_string()], + None::>, + None, + ) + .await + .unwrap() + .take_id(); + Id::from_str(&id).unwrap().document_id() +} + +async fn blob_in(store: &Store, account_id: u32, document_id: u32) -> Option { + let archive = store + .get_value::>(ValueKey::property( + account_id, + Collection::Email, + document_id, + EmailField::Metadata, + )) + .await + .unwrap()?; + let metadata = archive.unarchive::().unwrap(); + Some(BlobHash::from(&metadata.blob_hash)) +} + +fn path_of(snapshot: &Value) -> PathBuf { + PathBuf::from( + snapshot["path"] + .as_str() + .unwrap_or_else(|| panic!("no path: {snapshot}")), + ) +} + +pub async fn test(test: &mut TestServer) { + println!("Running store snapshot tests..."); + let admin = test.account("admin@example.com"); + let ann = test.account("ann@example.com"); + let ann_id = ann.id().document_id(); + let client = ann.jmap_client().await; + let separate_blobs = matches!(test.server.blob_store(), BlobStore::Fs(_)); + println!( + "Blobs {}", + if separate_blobs { + "in a filesystem blob store" + } else { + "in the data store" + } + ); + + let before = [ + import(&client, "Before the snapshot, one").await, + import(&client, "Before the snapshot, two").await, + ]; + assert!(all(admin).await.is_empty(), "no snapshots yet"); + + // A snapshot while the server runs + let id = take(admin).await; + let snapshot = settled(admin, &id).await; + assert_eq!(snapshot["status"], "done", "{snapshot}"); + assert_eq!(snapshot["createdBy"], "admin@example.com", "{snapshot}"); + let holds = if separate_blobs { + json!(["data", "blobs"]) + } else { + json!(["data"]) + }; + assert_eq!(snapshot["holds"], holds, "{snapshot}"); + assert!(snapshot["size"].as_u64().unwrap() > 0, "{snapshot}"); + let after = import(&client, "After the snapshot").await; + + // The directory: whole, named for its time, `latest`, the manifest + let path = path_of(&snapshot); + let name = snapshot["name"].as_str().unwrap(); + assert_eq!(path.file_name().unwrap().to_str().unwrap(), name); + assert!(name.len() == 16 && name.ends_with('Z'), "{name}"); + let root = path.parent().unwrap().to_path_buf(); + assert_eq!(root.file_name().unwrap(), "snapshots"); + assert_eq!( + std::fs::read_link(root.join("latest")).unwrap(), + PathBuf::from(name) + ); + assert!(path.join("data/CURRENT").exists(), "a RocksDB checkpoint"); + assert_eq!(path.join("blobs").exists(), separate_blobs); + let manifest: Value = + serde_json::from_slice(&std::fs::read(path.join("snapshot.json")).unwrap()).unwrap(); + assert_eq!(manifest["id"], json!(id), "{manifest}"); + assert_eq!(manifest["holds"], holds, "{manifest}"); + assert!( + manifest["restore"].as_array().unwrap().len() >= 3, + "{manifest}" + ); + assert!( + std::fs::read_dir(&root).unwrap().all(|e| !e + .unwrap() + .file_name() + .to_string_lossy() + .ends_with(".partial")), + "nothing partial left" + ); + + // Restored, it has the mail that was there, and not what came after + let restored = RocksDbStore::open(structs::RocksDbStore { + path: path.join("data").to_string_lossy().to_string(), + ..Default::default() + }) + .await + .unwrap(); + let blobs = if separate_blobs { + FsStore::open(structs::FileSystemStore { + path: path.join("blobs").to_string_lossy().to_string(), + ..Default::default() + }) + .await + .unwrap() + } else { + BlobStore::Store(restored.clone()) + }; + for (document_id, subject) in before.iter().zip(["one", "two"]) { + let hash = blob_in(&restored, ann_id, *document_id) + .await + .unwrap_or_else(|| panic!("message {document_id} isn't in the snapshot")); + let content = blobs + .get_blob(hash.as_slice(), 0..usize::MAX) + .await + .unwrap() + .unwrap_or_else(|| panic!("the blob of message {document_id} isn't in the snapshot")); + let content = String::from_utf8_lossy(&content); + assert!(content.contains(&format!("Before the snapshot, {subject}"))); + } + assert!( + blob_in(&restored, ann_id, after).await.is_none(), + "mail that came after isn't in it" + ); + drop(blobs); + drop(restored); + + // A snapshot can't be changed, and takes no properties + let (_, response) = call( + admin, + "inbuxa:StoreSnapshot/set", + json!({"update": {&id: {"name": "x"}}}), + ) + .await; + assert!(response["notUpdated"].get(&id).is_some(), "{response}"); + let (_, response) = call( + admin, + "inbuxa:StoreSnapshot/set", + json!({"create": {"x": {"path": "/tmp/elsewhere"}}}), + ) + .await; + assert_eq!( + response["notCreated"]["x"]["type"], "invalidProperties", + "{response}" + ); + + // One at a time, and one being taken can't be destroyed. With separate + // blobs a snapshot waits for the blob purge, so holding it here keeps + // one being taken for as long as this needs + if separate_blobs { + let hold = hold_blob_purge().await; + let waiting = take(admin).await; + assert_eq!(get(admin, &waiting).await["status"], "running"); + let (_, response) = call( + admin, + "inbuxa:StoreSnapshot/set", + json!({"create": {"x": {}}}), + ) + .await; + assert_eq!( + response["notCreated"]["x"]["type"], "forbidden", + "{response}" + ); + let (_, response) = call( + admin, + "inbuxa:StoreSnapshot/set", + json!({"destroy": [&waiting]}), + ) + .await; + assert_eq!( + response["notDestroyed"][&waiting]["type"], "forbidden", + "{response}" + ); + drop(hold); + assert_eq!(settled(admin, &waiting).await["status"], "done"); + } else { + let next = take(admin).await; + assert_eq!(settled(admin, &next).await["status"], "done"); + } + + // Only the latest two are kept + let third = take(admin).await; + let third_snapshot = settled(admin, &third).await; + let listed = all(admin).await; + assert_eq!(listed.len(), 2, "{listed:?}"); + assert_eq!(listed[0]["id"], json!(third), "newest first: {listed:?}"); + assert!(!path.exists(), "the oldest went"); + let (_, response) = call(admin, "inbuxa:StoreSnapshot/get", json!({"ids": [&id]})).await; + assert_eq!(response["notFound"], json!([id]), "{response}"); + assert_eq!( + std::fs::read_link(root.join("latest")).unwrap(), + PathBuf::from(third_snapshot["name"].as_str().unwrap()) + ); + + // Destroying one deletes it; `latest` moves to the one left + let unknown = Id::from(1u64).to_string(); + let (_, response) = call( + admin, + "inbuxa:StoreSnapshot/set", + json!({"destroy": [&third, &unknown]}), + ) + .await; + assert_eq!(response["destroyed"], json!([third]), "{response}"); + assert_eq!( + response["notDestroyed"][&unknown]["type"], "notFound", + "{response}" + ); + assert!(!path_of(&third_snapshot).exists()); + let left = all(admin).await; + assert_eq!(left.len(), 1, "{left:?}"); + assert_eq!( + std::fs::read_link(root.join("latest")).unwrap(), + PathBuf::from(left[0]["name"].as_str().unwrap()) + ); + + // Taking and destroying are audited + let (_, response) = call( + admin, + "inbuxa:AuditEvent/query", + json!({"filter": {"targetKind": "inbuxa:StoreSnapshot"}}), + ) + .await; + assert!( + response["ids"].as_array().is_some_and(|ids| ids.len() >= 4), + "three taken and one destroyed, audited: {response}" + ); + + // Nobody without the permission, or in a tenant, reaches snapshots + let (name, response) = call(ann, "inbuxa:StoreSnapshot/get", json!({"ids": null})).await; + assert_eq!(name, "error", "a user: {response}"); + let tenant = admin + .registry_create_object(Tenant { + name: "Store snapshot tenant".to_string(), + ..Default::default() + }) + .await; + admin + .registry_create_object(Domain { + name: "storesnapshot.example.org".to_string(), + member_tenant_id: Some(tenant), + certificate_management: CertificateManagement::Manual, + dns_management: DnsManagement::Manual, + dkim_management: DkimManagement::Manual, + ..Default::default() + }) + .await; + let t_admin = admin + .create_user_account( + "tadmin@storesnapshot.example.org", + "tenant-admin-secret-5521", + "Tenant admin", + &[], + vec![], + ) + .await; + admin + .registry_update_object( + ObjectType::Account, + t_admin.id(), + json!({Property::Roles: UserRoles::Admin}), + ) + .await; + for (method, arguments) in [ + ("inbuxa:StoreSnapshot/get", json!({"ids": null})), + ("inbuxa:StoreSnapshot/set", json!({"create": {"x": {}}})), + ] { + let (name, response) = call(&t_admin, method, arguments).await; + assert_eq!(name, "error", "{method} by a tenant: {response}"); + } +} + +/// Runs these tests alone, with RocksDB: +/// `STORE=RocksDb cargo test -p tests store_snapshot_tests -- --ignored`, +/// and again with `BLOB_STORE=FileSystem` for a separate blob store. +#[ignore] +#[tokio::test(flavor = "multi_thread")] +pub async fn store_snapshot_tests() { + let mut test = TestServerBuilder::new("store_snapshot_tests") + .await + .with_default_listeners() + .await + .build() + .await; + let admin = test.create_admin_account("admin@example.com").await; + test.insert_account(admin); + let ann = test + .create_user_account( + "admin@example.com", + "ann@example.com", + "ann-secret-61734", + &[], + "Ann", + ) + .await; + test.insert_account(ann); + self::test(&mut test).await; + if test.is_reset() { + test.temp_dir.delete(); + } +}