Store snapshot: a consistent copy of the store without stopping the server (#323) #328

Merged
jcoffey-dev merged 2 commits from feat/323-store-snapshot into main 2026-11-09 06:13:57 +00:00
33 changed files with 1946 additions and 6 deletions

No files matched your search

+5
View File
@@ -393,6 +393,11 @@ impl Default for DefaultPermissions {
Permission::SysStoreCheck => {
default.superuser.push(permission);
}
// inbuxa: store-snapshot spec, SN-8: a snapshot holds every
// account's data, so only the server's administrators take one
Permission::SysStoreSnapshot => {
default.superuser.push(permission);
}
// inbuxa: AL-12: tenant administrators lock and delegate
// within their tenant
Permission::SysAccountLockGet
@@ -35,7 +35,8 @@ use types::id::Id;
/// to-do items (security to-do list spec), the deliverability check
/// (deliverability spec), scheduled reports (scheduled-reports spec),
/// inbuxa Relays (relay spec), running migrations (migrations spec,
/// MG-10), and checking the store's blobs (store-check spec, SV-9).
/// MG-10), checking the store's blobs (store-check spec, SV-9), and taking
/// snapshots of the store (store-snapshot spec, SN-8).
const ADMIN_GRANTS: &[Permission] = &[
Permission::SysAiExplain,
Permission::SysAuditGet,
@@ -70,6 +71,7 @@ const ADMIN_GRANTS: &[Permission] = &[
Permission::SysRelayQueueUpdate,
Permission::SysMigrationRun,
Permission::SysStoreCheck,
Permission::SysStoreSnapshot,
];
/// Granted to the server-level Compliance Officer role once it exists:
@@ -0,0 +1,188 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! `inbuxa:StoreSnapshot/get` and `/set` under `urn:inbuxa:jmap`: a snapshot
//! of the store, taken without stopping the server (store-snapshot spec,
//! SN-1). Creating one takes it in the background; `get` lists the
//! snapshots on disk and says how the one being taken stands. Destroying one
//! deletes its directory.
use crate::object::{AnyId, JmapObject, JmapObjectId};
use jmap_tools::{Element, Key, Property};
use std::{borrow::Cow, str::FromStr};
use types::id::Id;
#[derive(Debug, Clone, Default)]
pub struct StoreSnapshot;
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum StoreSnapshotProperty {
Id,
Status,
CreatedAt,
CreatedBy,
FinishedAt,
Node,
Name,
Path,
Holds,
Size,
BlobFiles,
Error,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum StoreSnapshotValue {
Id(Id),
}
impl Property for StoreSnapshotProperty {
fn try_parse(parent: Option<&Key<'_, Self>>, value: &str) -> Option<Self> {
match parent {
None => StoreSnapshotProperty::parse(value),
Some(_) => None,
}
}
fn to_cow(&self) -> Cow<'static, str> {
match self {
StoreSnapshotProperty::Id => "id",
StoreSnapshotProperty::Status => "status",
StoreSnapshotProperty::CreatedAt => "createdAt",
StoreSnapshotProperty::CreatedBy => "createdBy",
StoreSnapshotProperty::FinishedAt => "finishedAt",
StoreSnapshotProperty::Node => "node",
StoreSnapshotProperty::Name => "name",
StoreSnapshotProperty::Path => "path",
StoreSnapshotProperty::Holds => "holds",
StoreSnapshotProperty::Size => "size",
StoreSnapshotProperty::BlobFiles => "blobFiles",
StoreSnapshotProperty::Error => "error",
}
.into()
}
}
impl StoreSnapshotProperty {
fn parse(value: &str) -> Option<Self> {
hashify::tiny_map!(value.as_bytes(),
b"id" => StoreSnapshotProperty::Id,
b"status" => StoreSnapshotProperty::Status,
b"createdAt" => StoreSnapshotProperty::CreatedAt,
b"createdBy" => StoreSnapshotProperty::CreatedBy,
b"finishedAt" => StoreSnapshotProperty::FinishedAt,
b"node" => StoreSnapshotProperty::Node,
b"name" => StoreSnapshotProperty::Name,
b"path" => StoreSnapshotProperty::Path,
b"holds" => StoreSnapshotProperty::Holds,
b"size" => StoreSnapshotProperty::Size,
b"blobFiles" => StoreSnapshotProperty::BlobFiles,
b"error" => StoreSnapshotProperty::Error,
)
}
}
impl FromStr for StoreSnapshotProperty {
type Err = ();
fn from_str(s: &str) -> Result<Self, Self::Err> {
StoreSnapshotProperty::parse(s).ok_or(())
}
}
impl Element for StoreSnapshotValue {
type Property = StoreSnapshotProperty;
fn try_parse<P>(key: &Key<'_, Self::Property>, value: &str) -> Option<Self> {
match key {
Key::Property(StoreSnapshotProperty::Id) => {
Id::from_str(value).ok().map(StoreSnapshotValue::Id)
}
_ => None,
}
}
fn to_cow(&self) -> Cow<'static, str> {
match self {
StoreSnapshotValue::Id(id) => id.to_string().into(),
}
}
}
impl JmapObject for StoreSnapshot {
type Property = StoreSnapshotProperty;
type Element = StoreSnapshotValue;
type Id = Id;
type Filter = ();
type Comparator = ();
type GetArguments = ();
type SetArguments<'de> = ();
type QueryArguments = ();
type CopyArguments = ();
type ParseArguments = ();
const ID_PROPERTY: Self::Property = StoreSnapshotProperty::Id;
}
impl From<Id> for StoreSnapshotValue {
fn from(id: Id) -> Self {
StoreSnapshotValue::Id(id)
}
}
impl JmapObjectId for StoreSnapshotValue {
fn as_id(&self) -> Option<Id> {
match self {
StoreSnapshotValue::Id(id) => Some(*id),
}
}
fn as_any_id(&self) -> Option<AnyId> {
match self {
StoreSnapshotValue::Id(id) => Some(AnyId::Id(*id)),
}
}
fn as_id_ref(&self) -> Option<&str> {
None
}
fn try_set_id(&mut self, new_id: AnyId) -> bool {
if let AnyId::Id(id) = new_id {
*self = StoreSnapshotValue::Id(id);
true
} else {
false
}
}
}
impl JmapObjectId for StoreSnapshotProperty {
fn as_id(&self) -> Option<Id> {
None
}
fn as_any_id(&self) -> Option<AnyId> {
None
}
fn as_id_ref(&self) -> Option<&str> {
None
}
fn try_set_id(&mut self, _: AnyId) -> bool {
false
}
}
+1
View File
@@ -51,6 +51,7 @@ pub mod inbuxa_scheduled_report_settings; // inbuxa: scheduled reports
pub mod inbuxa_security_acceptance; // inbuxa: accepted security to-do items
pub mod inbuxa_sharing_policy; // inbuxa: MA-C, who may share mail
pub mod inbuxa_store_check; // inbuxa: checking the store's blobs
pub mod inbuxa_store_snapshot; // inbuxa: a snapshot of the store
pub mod inbuxa_tenant_protocol_policy; // inbuxa: legacy protocols off, per tenant
pub mod mailbox;
pub mod participant_identity;
+3
View File
@@ -94,6 +94,9 @@ impl Response<'_> {
GetResponseMethod::StoreCheck(response) => {
response.eval_jptr(path, &mut results)
}
GetResponseMethod::StoreSnapshot(response) => {
response.eval_jptr(path, &mut results)
}
GetResponseMethod::Relay(response) => {
response.eval_jptr(path, &mut results)
}
@@ -57,6 +57,7 @@ impl Response<'_> {
GetRequestMethod::MailRule(request) => request.resolve_references(self)?,
GetRequestMethod::Migration(request) => request.resolve_references(self)?,
GetRequestMethod::StoreCheck(request) => request.resolve_references(self)?,
GetRequestMethod::StoreSnapshot(request) => request.resolve_references(self)?,
GetRequestMethod::Relay(request) => request.resolve_references(self)?,
GetRequestMethod::RelayMessage(request) => request.resolve_references(self)?,
GetRequestMethod::SecurityAcceptance(request) => {
@@ -159,6 +160,9 @@ impl Response<'_> {
SetRequestMethod::StoreCheck(request) => {
request.resolve_references(self, 1, false)?
}
SetRequestMethod::StoreSnapshot(request) => {
request.resolve_references(self, 1, false)?
}
SetRequestMethod::Relay(request) => request.resolve_references(self, 1, false)?,
SetRequestMethod::RelayMessage(request) => {
request.resolve_references(self, 1, false)?
+8
View File
@@ -72,6 +72,8 @@ pub enum MethodObject {
Migration,
// inbuxa: checking the store's blobs
StoreCheck,
// inbuxa: a snapshot of the store
StoreSnapshot,
Relay,
RelayMessage,
// inbuxa: accepted security to-do items
@@ -132,6 +134,7 @@ impl MethodObject {
| MethodObject::MailRule
| MethodObject::Migration
| MethodObject::StoreCheck
| MethodObject::StoreSnapshot
| MethodObject::Relay
| MethodObject::RelayMessage
| MethodObject::SecurityAcceptance
@@ -350,6 +353,8 @@ impl MethodName {
(MethodFunction::Set, MethodObject::Migration) => "inbuxa:Migration/set",
(MethodFunction::Get, MethodObject::StoreCheck) => "inbuxa:StoreCheck/get",
(MethodFunction::Set, MethodObject::StoreCheck) => "inbuxa:StoreCheck/set",
(MethodFunction::Get, MethodObject::StoreSnapshot) => "inbuxa:StoreSnapshot/get",
(MethodFunction::Set, MethodObject::StoreSnapshot) => "inbuxa:StoreSnapshot/set",
(MethodFunction::Get, MethodObject::Relay) => "inbuxa:Relay/get",
(MethodFunction::Set, MethodObject::Relay) => "inbuxa:Relay/set",
(MethodFunction::Get, MethodObject::RelayMessage) => "inbuxa:RelayMessage/get",
@@ -557,6 +562,8 @@ impl MethodName {
"inbuxa:Migration/set" => (MethodObject::Migration, MethodFunction::Set),
"inbuxa:StoreCheck/get" => (MethodObject::StoreCheck, MethodFunction::Get),
"inbuxa:StoreCheck/set" => (MethodObject::StoreCheck, MethodFunction::Set),
"inbuxa:StoreSnapshot/get" => (MethodObject::StoreSnapshot, MethodFunction::Get),
"inbuxa:StoreSnapshot/set" => (MethodObject::StoreSnapshot, MethodFunction::Set),
"inbuxa:Relay/get" => (MethodObject::Relay, MethodFunction::Get),
"inbuxa:Relay/set" => (MethodObject::Relay, MethodFunction::Set),
"inbuxa:RelayMessage/get" => (MethodObject::RelayMessage, MethodFunction::Get),
@@ -660,6 +667,7 @@ impl Display for MethodObject {
MethodObject::MailRule => "inbuxa:MailRule",
MethodObject::Migration => "inbuxa:Migration",
MethodObject::StoreCheck => "inbuxa:StoreCheck",
MethodObject::StoreSnapshot => "inbuxa:StoreSnapshot",
MethodObject::Relay => "inbuxa:Relay",
MethodObject::RelayMessage => "inbuxa:RelayMessage",
MethodObject::SecurityAcceptance => "inbuxa:SecurityAcceptance",
+2
View File
@@ -129,6 +129,7 @@ pub enum GetRequestMethod {
MailRule(Box<GetRequest<crate::object::inbuxa_mail_rule::MailRule>>),
Migration(Box<GetRequest<crate::object::inbuxa_migration::Migration>>),
StoreCheck(Box<GetRequest<crate::object::inbuxa_store_check::StoreCheck>>),
StoreSnapshot(Box<GetRequest<crate::object::inbuxa_store_snapshot::StoreSnapshot>>),
Relay(Box<GetRequest<crate::object::inbuxa_relay::Relay>>),
RelayMessage(Box<GetRequest<crate::object::inbuxa_relay_message::RelayMessage>>),
SecurityAcceptance(
@@ -189,6 +190,7 @@ pub enum SetRequestMethod<'x> {
MailRule(Box<SetRequest<'x, crate::object::inbuxa_mail_rule::MailRule>>),
Migration(Box<SetRequest<'x, crate::object::inbuxa_migration::Migration>>),
StoreCheck(Box<SetRequest<'x, crate::object::inbuxa_store_check::StoreCheck>>),
StoreSnapshot(Box<SetRequest<'x, crate::object::inbuxa_store_snapshot::StoreSnapshot>>),
Relay(Box<SetRequest<'x, crate::object::inbuxa_relay::Relay>>),
RelayMessage(Box<SetRequest<'x, crate::object::inbuxa_relay_message::RelayMessage>>),
SecurityAcceptance(
+15
View File
@@ -719,6 +719,21 @@ impl<'de> Visitor<'de> for CallVisitor {
return Err(de::Error::invalid_length(1, &self));
}
},
// inbuxa: a snapshot of the store
(MethodFunction::Get, MethodObject::StoreSnapshot) => match seq.next_element() {
Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::StoreSnapshot(value)),
Err(err) => RequestMethod::invalid(err),
Ok(None) => {
return Err(de::Error::invalid_length(1, &self));
}
},
(MethodFunction::Set, MethodObject::StoreSnapshot) => match seq.next_element() {
Ok(Some(value)) => RequestMethod::Set(SetRequestMethod::StoreSnapshot(value)),
Err(err) => RequestMethod::invalid(err),
Ok(None) => {
return Err(de::Error::invalid_length(1, &self));
}
},
// inbuxa: inbuxa Relay
(MethodFunction::Get, MethodObject::Relay) => match seq.next_element() {
Ok(Some(value)) => RequestMethod::Get(GetRequestMethod::Relay(value)),
+19
View File
@@ -114,6 +114,7 @@ pub enum GetResponseMethod {
MailRule(GetResponse<crate::object::inbuxa_mail_rule::MailRule>),
Migration(GetResponse<crate::object::inbuxa_migration::Migration>),
StoreCheck(GetResponse<crate::object::inbuxa_store_check::StoreCheck>),
StoreSnapshot(GetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>),
Relay(GetResponse<crate::object::inbuxa_relay::Relay>),
RelayMessage(GetResponse<crate::object::inbuxa_relay_message::RelayMessage>),
SecurityAcceptance(GetResponse<crate::object::inbuxa_security_acceptance::SecurityAcceptance>),
@@ -172,6 +173,7 @@ pub enum SetResponseMethod {
MailRule(Box<SetResponse<crate::object::inbuxa_mail_rule::MailRule>>),
Migration(Box<SetResponse<crate::object::inbuxa_migration::Migration>>),
StoreCheck(Box<SetResponse<crate::object::inbuxa_store_check::StoreCheck>>),
StoreSnapshot(Box<SetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>>),
Relay(Box<SetResponse<crate::object::inbuxa_relay::Relay>>),
RelayMessage(Box<SetResponse<crate::object::inbuxa_relay_message::RelayMessage>>),
SecurityAcceptance(
@@ -1067,6 +1069,23 @@ impl<'x> From<SetResponse<crate::object::inbuxa_store_check::StoreCheck>> for Re
}
}
// inbuxa: a snapshot of the store
impl<'x> From<GetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>>
for ResponseMethod<'x>
{
fn from(value: GetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>) -> Self {
ResponseMethod::Get(GetResponseMethod::StoreSnapshot(value))
}
}
impl<'x> From<SetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>>
for ResponseMethod<'x>
{
fn from(value: SetResponse<crate::object::inbuxa_store_snapshot::StoreSnapshot>) -> Self {
ResponseMethod::Set(SetResponseMethod::StoreSnapshot(Box::new(value)))
}
}
impl<'x> From<GetResponse<crate::object::inbuxa_relay::Relay>> for ResponseMethod<'x> {
fn from(value: GetResponse<crate::object::inbuxa_relay::Relay>) -> Self {
ResponseMethod::Get(GetResponseMethod::Relay(value))
+11
View File
@@ -120,6 +120,8 @@ impl JmapAuthorization for AccessToken {
GetRequestMethod::Migration(_) => Permission::SysMigrationRun,
// inbuxa: store-check spec, SV-9
GetRequestMethod::StoreCheck(_) => Permission::SysStoreCheck,
// inbuxa: store-snapshot spec, SN-8
GetRequestMethod::StoreSnapshot(_) => Permission::SysStoreSnapshot,
// inbuxa: inbuxa Relay
GetRequestMethod::Relay(_) => Permission::SysRelayGet,
GetRequestMethod::RelayMessage(_) => Permission::SysRelayQueueGet,
@@ -333,6 +335,14 @@ impl JmapAuthorization for AccessToken {
Permission::SysStoreCheck,
Permission::SysStoreCheck,
),
// inbuxa: store-snapshot spec, SN-8
SetRequestMethod::StoreSnapshot(s) => validate_set(
s,
self,
Permission::SysStoreSnapshot,
Permission::SysStoreSnapshot,
Permission::SysStoreSnapshot,
),
// inbuxa: inbuxa Relay; a new join token is an update
SetRequestMethod::Relay(s) => validate_set(
s,
@@ -607,6 +617,7 @@ impl JmapAuthorization for AccessToken {
| MethodObject::MailRule
| MethodObject::Migration
| MethodObject::StoreCheck
| MethodObject::StoreSnapshot
| MethodObject::Relay
| MethodObject::RelayMessage
| MethodObject::SecurityAcceptance
+26
View File
@@ -296,6 +296,9 @@ impl RequestHandler for Server {
SetResponseMethod::StoreCheck(set_response) => {
set_response.update_created_ids(&mut response);
}
SetResponseMethod::StoreSnapshot(set_response) => {
set_response.update_created_ids(&mut response);
}
SetResponseMethod::Relay(set_response) => {
set_response.update_created_ids(&mut response);
}
@@ -582,6 +585,13 @@ impl RequestHandler for Server {
.await?
.into()
}
// inbuxa: a snapshot of the store
GetRequestMethod::StoreSnapshot(mut req) => {
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
crate::inbuxa::store_snapshot::get(self, access_token, *req)
.await?
.into()
}
// inbuxa: inbuxa Relay
GetRequestMethod::Relay(mut req) => {
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
@@ -1191,6 +1201,22 @@ impl RequestHandler for Server {
.await?
.into()
}
// inbuxa: store-snapshot spec, SN-8: taking and destroying are audited
SetRequestMethod::StoreSnapshot(mut req) => {
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
crate::inbuxa::audit::recorded(
self,
access_token,
session,
&method_name.obj.to_string(),
None,
None,
*req,
|req| Box::pin(crate::inbuxa::store_snapshot::set(self, access_token, req)),
)
.await?
.into()
}
SetRequestMethod::Relay(mut req) => {
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
let reason = req.arguments.reason.clone();
+1
View File
@@ -433,6 +433,7 @@ impl IntermediateChangesResponse {
| MethodObject::MailRule
| MethodObject::Migration
| MethodObject::StoreCheck
| MethodObject::StoreSnapshot
| MethodObject::Relay
| MethodObject::RelayMessage
| MethodObject::SecurityAcceptance
+13
View File
@@ -526,6 +526,19 @@ async fn fork_current(server: &Server, object: &str, id: &MaybeInvalid<Id>) -> O
}
MaybeInvalid::Invalid(_) => None,
},
// A snapshot: its directory and how it stands (store-snapshot spec,
// SN-8)
"inbuxa:StoreSnapshot" => match id {
MaybeInvalid::Value(id) => services::inbuxa_store_snapshot::find(server, id.id())
.await
.map(|s| {
serde_json::json!({
"name": s.name,
"status": s.status.as_str(),
})
}),
MaybeInvalid::Invalid(_) => None,
},
// A relay as an administrator sets it: never its keys or token
"inbuxa:Relay" => match id {
MaybeInvalid::Value(id) => {
+1
View File
@@ -39,6 +39,7 @@ pub mod scheduled_reports; // inbuxa: scheduled reports and the weekly digest
pub mod security_acceptance;
pub mod sharing_policy;
pub mod store_check; // inbuxa: checking the store's blobs
pub mod store_snapshot; // inbuxa: a snapshot of the store
pub mod telemetry;
pub mod tenant_protocol_policy;
pub mod undelete;
+185
View File
@@ -0,0 +1,185 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! `inbuxa:StoreSnapshot` (store-snapshot spec): taking a snapshot of the
//! store without stopping the server, listing the snapshots on disk, and
//! deleting one. The run is `services::inbuxa_store_snapshot`. Taking and
//! destroying one are audited by the request layer (SN-8).
use super::relay::json_to_value;
use common::{Server, auth::AccessToken};
use jmap_proto::{
error::set::SetError,
method::{
get::{GetRequest, GetResponse},
set::{SetRequest, SetResponse},
},
object::inbuxa_store_snapshot::{
StoreSnapshot, StoreSnapshotProperty as P, StoreSnapshotValue,
},
request::IntoValid,
types::date::UTCDate,
};
use jmap_tools::{Key, Map, Value};
use serde_json::json;
use services::inbuxa_store_snapshot::{self as runs, Destroyed, Refusal, Snapshot};
use types::id::Id;
type SValue = Value<'static, P, StoreSnapshotValue>;
const ALL: &[P] = &[
P::Id,
P::Status,
P::CreatedAt,
P::CreatedBy,
P::FinishedAt,
P::Node,
P::Name,
P::Path,
P::Holds,
P::Size,
P::BlobFiles,
P::Error,
];
/// SN-8: a snapshot holds every account's data, so only the server's own
/// administrators take one.
fn server_level(access_token: &AccessToken) -> trc::Result<()> {
if access_token.tenant_id().is_some() {
Err(trc::JmapEvent::Forbidden
.into_err()
.details("A snapshot holds the whole server, not a tenant."))
} else {
Ok(())
}
}
fn date(seconds: u64) -> serde_json::Value {
json!(UTCDate::from_timestamp(seconds as i64).to_string())
}
fn to_value(s: &Snapshot, properties: &[P]) -> SValue {
let mut out = Map::with_capacity(properties.len());
for property in properties {
let json = match property {
P::Id => {
out.insert_unchecked(
Key::Property(P::Id),
Value::Element(StoreSnapshotValue::Id(Id::from(s.id))),
);
continue;
}
P::Status => json!(s.status.as_str()),
P::CreatedAt => date(s.created_at),
P::CreatedBy => json!(s.created_by),
P::FinishedAt => s.finished_at.map_or(serde_json::Value::Null, date),
P::Node => json!(s.node_id.to_string()),
P::Name => json!(s.name),
P::Path => json!(s.path.to_string_lossy()),
P::Holds => json!(s.holds),
P::Size => json!(s.size),
P::BlobFiles => json!(s.blob_files),
P::Error => json!(s.error),
};
out.insert_unchecked(Key::Property(property.clone()), json_to_value(json));
}
Value::Object(out)
}
/// `inbuxa:StoreSnapshot/get`: the snapshots on disk and the one being
/// taken, newest first (SN-7).
pub async fn get(
server: &Server,
access_token: &AccessToken,
mut request: GetRequest<StoreSnapshot>,
) -> trc::Result<GetResponse<StoreSnapshot>> {
server_level(access_token)?;
let properties = request.unwrap_properties(ALL);
let (ids, not_found) = request.unwrap_ids(server.core.jmap.get_max_objects)?;
let mut response = GetResponse {
account_id: request.account_id.into(),
state: None,
list: Vec::new(),
not_found,
};
let snapshots = runs::list(server).await;
match ids {
None => {
for s in &snapshots {
response.list.push(to_value(s, &properties));
}
}
Some(ids) => {
for id in ids {
match snapshots.iter().find(|s| s.id == id.id()) {
Some(s) => response.list.push(to_value(s, &properties)),
None => response.push_not_found(id),
}
}
}
}
Ok(response)
}
/// `inbuxa:StoreSnapshot/set`: create takes one; destroy deletes one; a
/// snapshot can't be changed (SN-1, SN-6, SN-9).
pub async fn set(
server: &Server,
access_token: &AccessToken,
mut request: SetRequest<'_, StoreSnapshot>,
) -> trc::Result<SetResponse<StoreSnapshot>> {
server_level(access_token)?;
let mut response = SetResponse::from_request(&request, server.core.jmap.set_max_objects)?;
let actor = server.audit_actor(access_token).await;
for (client_id, value) in request.unwrap_create() {
if let Some((key, _)) = value.into_expanded_object().next() {
response.not_created.append(
client_id,
SetError::invalid_properties()
.with_property(key.into_owned())
.with_description("A snapshot is taken as it is; it takes no properties."),
);
continue;
}
match runs::start(server, actor.name.clone()).await? {
Ok(snapshot) => {
response
.created
.insert(client_id, to_value(&snapshot, &[P::Id, P::Name, P::Path]));
}
Err(Refusal::Running(id)) => response.not_created.append(
client_id,
SetError::forbidden().with_description(format!(
"Snapshot {} is being taken; one is taken at a time. Wait for it.",
Id::from(id)
)),
),
Err(Refusal::Unsupported(why)) => response
.not_created
.append(client_id, SetError::forbidden().with_description(why)),
}
}
for (id, _) in request.unwrap_update().into_valid() {
response.not_updated.append(
id,
SetError::forbidden().with_description("A snapshot can't be changed."),
);
}
for id in request.unwrap_destroy().into_valid() {
match runs::destroy(server, id.id()).await? {
Destroyed::Gone => response.destroyed.push(id),
Destroyed::NotFound => response.not_destroyed.append(id, SetError::not_found()),
Destroyed::Running => response.not_destroyed.append(
id,
SetError::forbidden().with_description("It's being taken; wait for it to finish."),
),
}
}
Ok(response)
}
+2
View File
@@ -1793,6 +1793,8 @@ pub enum Permission {
SysMigrationRun = 694,
// inbuxa: checking the store's blobs
SysStoreCheck = 695,
// inbuxa: a snapshot of the store, without stopping the server
SysStoreSnapshot = 696,
SysAccountGet = 219,
SysAccountCreate = 220,
SysAccountUpdate = 221,
+4 -1
View File
@@ -7172,6 +7172,7 @@ impl EnumImpl for Permission {
b"sysRelayQueueUpdate" => Permission::SysRelayQueueUpdate,
b"sysMigrationRun" => Permission::SysMigrationRun,
b"sysStoreCheck" => Permission::SysStoreCheck,
b"sysStoreSnapshot" => Permission::SysStoreSnapshot,
b"sysAccountGet" => Permission::SysAccountGet,
b"sysAccountCreate" => Permission::SysAccountCreate,
b"sysAccountUpdate" => Permission::SysAccountUpdate,
@@ -7884,6 +7885,7 @@ impl EnumImpl for Permission {
Permission::SysRelayQueueUpdate => "sysRelayQueueUpdate",
Permission::SysMigrationRun => "sysMigrationRun",
Permission::SysStoreCheck => "sysStoreCheck",
Permission::SysStoreSnapshot => "sysStoreSnapshot",
Permission::SysAccountGet => "sysAccountGet",
Permission::SysAccountCreate => "sysAccountCreate",
Permission::SysAccountUpdate => "sysAccountUpdate",
@@ -8589,6 +8591,7 @@ impl EnumImpl for Permission {
693 => Some(Permission::SysRelayQueueUpdate),
694 => Some(Permission::SysMigrationRun),
695 => Some(Permission::SysStoreCheck),
696 => Some(Permission::SysStoreSnapshot),
219 => Some(Permission::SysAccountGet),
220 => Some(Permission::SysAccountCreate),
221 => Some(Permission::SysAccountUpdate),
@@ -9033,7 +9036,7 @@ impl EnumImpl for Permission {
}
}
const COUNT: usize = 696;
const COUNT: usize = 697;
}
impl serde::Serialize for Permission {
@@ -0,0 +1,496 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! Snapshots of the store (docs/spec/features/store-snapshot.md): the run
//! behind an `inbuxa:StoreSnapshot`. It takes a RocksDB checkpoint and, for
//! a filesystem blob store, a copy of the blob files with the blob purge
//! held, into a directory under the store's own (SN-2, SN-3). The snapshots
//! on disk are the record: each one's `snapshot.json` says what it is. Only
//! the one being taken, or the last one that failed, is kept in memory.
use common::{Inner, Server};
use mail_parser::DateTime;
use serde_json::{Value, json};
use std::{
path::{Path, PathBuf},
str::FromStr,
sync::{Arc, Mutex},
};
use store::write::{
now,
snapshot::{NotSnapshottable, SnapshotSource, copy_tree, hold_blob_purge, tree_size},
};
use types::id::Id;
/// SN-6: complete snapshots kept.
pub const KEEP: usize = 2;
const SNAPSHOTS_DIR: &str = "snapshots";
const MANIFEST: &str = "snapshot.json";
const LATEST: &str = "latest";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Status {
Running,
Done,
Failed,
}
impl Status {
pub fn as_str(&self) -> &'static str {
match self {
Status::Running => "running",
Status::Done => "done",
Status::Failed => "failed",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Snapshot {
/// When it was taken, in seconds: its name is that time.
pub id: u64,
pub name: String,
pub status: Status,
pub created_at: u64,
pub created_by: String,
pub finished_at: Option<u64>,
pub node_id: u64,
pub path: PathBuf,
pub holds: Vec<String>,
pub size: u64,
pub blob_files: u64,
pub error: Option<String>,
}
/// Why a snapshot isn't taken.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Refusal {
/// SN-6: this one is being taken.
Running(u64),
/// SN-1, SN-2: these stores can't have one.
Unsupported(String),
}
/// What `destroy` did.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Destroyed {
Gone,
NotFound,
Running,
}
/// SN-7: the one being taken, or the last one that failed, until restart.
static CURRENT: Mutex<Option<Snapshot>> = Mutex::new(None);
fn current() -> Option<Snapshot> {
CURRENT.lock().map(|c| c.clone()).unwrap_or(None)
}
fn set_current(snapshot: Option<Snapshot>) {
if let Ok(mut current) = CURRENT.lock() {
*current = snapshot;
}
}
/// The snapshots' directory, and what a snapshot would hold, or why there
/// can't be one here.
fn source(server: &Server) -> Result<(SnapshotSource, PathBuf), Refusal> {
match server.store().snapshot_source(server.blob_store()) {
Ok(source) => {
let root = match &source {
SnapshotSource::DataOnly { data_path }
| SnapshotSource::DataAndFiles { data_path, .. } => data_path.join(SNAPSHOTS_DIR),
};
Ok((source, root))
}
Err(NotSnapshottable::DataStore(kind)) => Err(Refusal::Unsupported(format!(
"A snapshot needs RocksDB as the data store, and this server's is {kind}. \
Use {kind}'s own backup."
))),
Err(NotSnapshottable::BlobStore(kind)) => Err(Refusal::Unsupported(format!(
"A snapshot copies blobs kept in the data store or in a filesystem blob store, \
and this server keeps them in {kind}."
))),
}
}
/// `20261109T013000Z`: a snapshot's name, the UTC time it was taken.
pub fn name_for(seconds: u64) -> String {
let t = DateTime::from_timestamp(seconds as i64);
format!(
"{:04}{:02}{:02}T{:02}{:02}{:02}Z",
t.year, t.month, t.day, t.hour, t.minute, t.second
)
}
fn rfc3339(seconds: u64) -> String {
DateTime::from_timestamp(seconds as i64).to_rfc3339()
}
fn seconds_of(value: &Value) -> Option<u64> {
DateTime::parse_rfc3339(value.as_str()?).map(|t| t.to_timestamp() as u64)
}
fn io(err: std::io::Error) -> trc::Error {
trc::StoreEvent::FilesystemError.reason(err)
}
/// SN-1, SN-6: starts taking a snapshot on this node. Refused while one is
/// being taken, or when the stores can't have one.
pub async fn start(server: &Server, created_by: String) -> trc::Result<Result<Snapshot, Refusal>> {
let (source, root) = match source(server) {
Ok(found) => found,
Err(refusal) => return Ok(Err(refusal)),
};
let snapshot = {
let Ok(mut current) = CURRENT.lock() else {
return Err(trc::EventType::Server(trc::ServerEvent::ThreadError)
.into_err()
.details("The snapshot state is poisoned"));
};
if let Some(running) = current.as_ref().filter(|c| c.status == Status::Running) {
return Ok(Err(Refusal::Running(running.id)));
}
// Names are the time to the second: a second one within the same
// second takes the next
let mut at = now();
while root.join(name_for(at)).exists() {
at += 1;
}
let name = name_for(at);
let mut holds = vec!["data".to_string()];
if matches!(source, SnapshotSource::DataAndFiles { .. }) {
holds.push("blobs".to_string());
}
let snapshot = Snapshot {
id: at,
path: root.join(&name),
name,
status: Status::Running,
created_at: at,
created_by,
finished_at: None,
node_id: server.core.network.node_id,
holds,
size: 0,
blob_files: 0,
error: None,
};
*current = Some(snapshot.clone());
snapshot
};
let server = server.clone();
let running = snapshot.clone();
tokio::spawn(async move {
run(&server, source, root, running).await;
});
Ok(Ok(snapshot))
}
async fn run(server: &Server, source: SnapshotSource, root: PathBuf, mut snapshot: Snapshot) {
let partial = root.join(format!(".{}.partial", snapshot.name));
match take(server, &source, &root, &partial, &mut snapshot).await {
// Listed from disk from now on
Ok(()) => set_current(None),
Err(err) => {
let _ = tokio::fs::remove_dir_all(&partial).await;
snapshot.status = Status::Failed;
snapshot.finished_at = Some(now());
snapshot.error = Some(err.to_string());
trc::error!(err.details("Failed to take a store snapshot"));
set_current(Some(snapshot));
}
}
}
async fn take(
server: &Server,
source: &SnapshotSource,
root: &Path,
partial: &Path,
snapshot: &mut Snapshot,
) -> trc::Result<()> {
tokio::fs::create_dir_all(root).await.map_err(io)?;
let _ = tokio::fs::remove_dir_all(partial).await;
tokio::fs::create_dir(partial).await.map_err(io)?;
// SN-4: the purge is held before the checkpoint, so every blob the
// checkpoint links to is still there when it's copied
let hold = match source {
SnapshotSource::DataAndFiles { .. } => Some(hold_blob_purge().await),
SnapshotSource::DataOnly { .. } => None,
};
server.store().checkpoint(partial.join("data")).await?;
let blob_path = match source {
SnapshotSource::DataAndFiles { blob_path, .. } => {
let copy = copy_tree(
blob_path.clone(),
partial.join("blobs"),
data_path_of(source).to_path_buf(),
)
.await?;
snapshot.blob_files = copy.files;
Some(blob_path)
}
SnapshotSource::DataOnly { .. } => None,
};
drop(hold);
snapshot.size = tree_size(partial.to_path_buf()).await?;
snapshot.status = Status::Done;
snapshot.finished_at = Some(now());
let manifest = manifest(
snapshot,
data_path_of(source),
blob_path.map(|p| p.as_path()),
);
tokio::fs::write(
partial.join(MANIFEST),
serde_json::to_vec_pretty(&manifest).unwrap_or_default(),
)
.await
.map_err(io)?;
// SN-3: whole once it has its name
tokio::fs::rename(partial, &snapshot.path)
.await
.map_err(io)?;
if let Ok(dir) = tokio::fs::File::open(root).await {
let _ = dir.sync_all().await;
}
prune(root).await;
refresh_latest(root).await;
Ok(())
}
fn data_path_of(source: &SnapshotSource) -> &Path {
match source {
SnapshotSource::DataOnly { data_path } | SnapshotSource::DataAndFiles { data_path, .. } => {
data_path
}
}
}
/// SN-5.
fn manifest(snapshot: &Snapshot, data_path: &Path, blob_path: Option<&Path>) -> Value {
let mut restore = vec![
"Stop the server.".to_string(),
format!(
"Move the contents of {} aside, except snapshots/, and copy this snapshot's data/ into it.",
data_path.display()
),
];
if let Some(blob_path) = blob_path {
restore.push(format!(
"Move the contents of {} aside and copy this snapshot's blobs/ into it.",
blob_path.display()
));
}
restore.push("Start the server.".to_string());
json!({
"id": Id::from(snapshot.id).to_string(),
"name": snapshot.name,
"createdAt": rfc3339(snapshot.created_at),
"finishedAt": snapshot.finished_at.map(rfc3339),
"createdBy": snapshot.created_by,
"node": snapshot.node_id.to_string(),
"server": concat!(types::brand_server!(), " ", types::brand_version!()),
"holds": snapshot.holds,
"dataStore": {"type": "RocksDB", "path": data_path.to_string_lossy()},
"blobStore": match blob_path {
Some(path) => json!({"type": "filesystem", "path": path.to_string_lossy()}),
None => json!({"type": "in the data store"}),
},
"size": snapshot.size,
"blobFiles": snapshot.blob_files,
"restore": restore,
})
}
fn from_manifest(path: PathBuf, manifest: &Value) -> Option<Snapshot> {
let id = Id::from_str(manifest.get("id")?.as_str()?).ok()?.id();
Some(Snapshot {
id,
name: manifest.get("name")?.as_str()?.to_string(),
status: Status::Done,
created_at: seconds_of(manifest.get("createdAt")?)?,
created_by: manifest.get("createdBy")?.as_str()?.to_string(),
finished_at: manifest.get("finishedAt").and_then(seconds_of),
node_id: manifest.get("node")?.as_str()?.parse().ok()?,
path,
holds: manifest
.get("holds")?
.as_array()?
.iter()
.filter_map(|h| h.as_str().map(|s| s.to_string()))
.collect(),
size: manifest.get("size")?.as_u64()?,
blob_files: manifest.get("blobFiles")?.as_u64()?,
error: None,
})
}
/// The complete snapshots under `root`, oldest first. A directory without a
/// readable `snapshot.json` isn't one, and is left alone.
async fn on_disk(root: &Path) -> Vec<Snapshot> {
let mut found = Vec::new();
let Ok(mut entries) = tokio::fs::read_dir(root).await else {
return found;
};
while let Ok(Some(entry)) = entries.next_entry().await {
let name = entry.file_name().to_string_lossy().to_string();
if name.starts_with('.') || !entry.file_type().await.is_ok_and(|t| t.is_dir()) {
continue;
}
let path = entry.path();
let Ok(bytes) = tokio::fs::read(path.join(MANIFEST)).await else {
continue;
};
if let Some(snapshot) = serde_json::from_slice::<Value>(&bytes)
.ok()
.and_then(|manifest| from_manifest(path, &manifest))
.filter(|s| s.name == name)
{
found.push(snapshot);
}
}
found.sort_by_key(|s| s.id);
found
}
/// SN-6: the oldest beyond the latest [`KEEP`] go.
async fn prune(root: &Path) {
let snapshots = on_disk(root).await;
let surplus = snapshots.len().saturating_sub(KEEP);
for old in &snapshots[..surplus] {
if let Err(err) = tokio::fs::remove_dir_all(&old.path).await {
trc::error!(io(err).details("Failed to remove an old store snapshot"));
}
}
}
/// SN-3: `latest` names the newest complete snapshot, relative to the
/// snapshots' directory so it holds wherever that's mounted.
async fn refresh_latest(root: &Path) {
let latest = root.join(LATEST);
match on_disk(root).await.last() {
Some(newest) => {
let temp = root.join(format!(".{LATEST}.tmp"));
let _ = tokio::fs::remove_file(&temp).await;
let linked = async {
tokio::fs::symlink(&newest.name, &temp).await?;
tokio::fs::rename(&temp, &latest).await
}
.await;
if let Err(err) = linked {
trc::error!(io(err).details("Failed to point snapshots/latest at a snapshot"));
}
}
None => {
let _ = tokio::fs::remove_file(&latest).await;
}
}
}
/// Every snapshot, newest first: those on disk, and the one being taken or
/// the last one that failed (SN-7).
pub async fn list(server: &Server) -> Vec<Snapshot> {
let mut snapshots = match source(server) {
Ok((_, root)) => on_disk(&root).await,
Err(_) => Vec::new(),
};
if let Some(current) = current().filter(|c| !snapshots.iter().any(|s| s.id == c.id)) {
snapshots.push(current);
}
snapshots.sort_by_key(|s| std::cmp::Reverse(s.id));
snapshots
}
/// One snapshot, as [`list`] has it.
pub async fn find(server: &Server, id: u64) -> Option<Snapshot> {
list(server).await.into_iter().find(|s| s.id == id)
}
/// SN-9: deletes a snapshot's directory. One being taken can't be.
pub async fn destroy(server: &Server, id: u64) -> trc::Result<Destroyed> {
if let Some(current) = current().filter(|c| c.id == id) {
if current.status == Status::Running {
return Ok(Destroyed::Running);
}
set_current(None);
return Ok(Destroyed::Gone);
}
let Ok((_, root)) = source(server) else {
return Ok(Destroyed::NotFound);
};
let Some(snapshot) = on_disk(&root).await.into_iter().find(|s| s.id == id) else {
return Ok(Destroyed::NotFound);
};
tokio::fs::remove_dir_all(&snapshot.path)
.await
.map_err(io)?;
refresh_latest(&root).await;
Ok(Destroyed::Gone)
}
/// SN-7: a snapshot being taken when the server stopped left a partial
/// directory, removed when it starts.
pub fn spawn_store_snapshots(inner: Arc<Inner>) {
tokio::spawn(async move {
let server = common::BuildServer::build_server(&inner);
let Ok((_, root)) = source(&server) else {
return;
};
let Ok(mut entries) = tokio::fs::read_dir(&root).await else {
return;
};
while let Ok(Some(entry)) = entries.next_entry().await {
let name = entry.file_name().to_string_lossy().to_string();
if name.starts_with('.') && name.ends_with(".partial") {
let _ = tokio::fs::remove_dir_all(entry.path()).await;
}
}
});
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn names_are_the_time() {
assert_eq!(name_for(1_794_187_800), "20261109T013000Z");
assert_eq!(name_for(0), "19700101T000000Z");
}
#[test]
fn a_manifest_reads_back() {
let snapshot = Snapshot {
id: 1_794_187_800,
name: name_for(1_794_187_800),
status: Status::Done,
created_at: 1_794_187_800,
created_by: "[email protected]".into(),
finished_at: Some(1_794_187_802),
node_id: 3,
path: PathBuf::from("/var/lib/inbuxa/snapshots/20261109T013000Z"),
holds: vec!["data".into(), "blobs".into()],
size: 4096,
blob_files: 12,
error: None,
};
let manifest = manifest(
&snapshot,
Path::new("/var/lib/inbuxa/"),
Some(Path::new("/var/lib/inbuxa-blobs")),
);
assert_eq!(manifest["restore"].as_array().unwrap().len(), 4);
assert_eq!(
from_manifest(snapshot.path.clone(), &manifest).unwrap(),
snapshot
);
}
}
+5
View File
@@ -31,6 +31,7 @@ pub mod inbuxa_relay_certificates; // inbuxa: inbuxa Relays' certificates
pub mod inbuxa_relay_link; // inbuxa: links to inbuxa Relays
pub mod inbuxa_scheduled_reports; // inbuxa: scheduled reports and the weekly digest
pub mod inbuxa_store_check; // inbuxa: checking the store's blobs
pub mod inbuxa_store_snapshot; // inbuxa: a snapshot of the store
pub mod state_manager;
pub mod task_manager;
@@ -88,6 +89,10 @@ impl SpawnServices for IpcReceivers {
// inbuxa: store-check spec, SV-8: a check this node ran stopped with it
inbuxa_store_check::spawn_store_checks(inner.clone());
// inbuxa: store-snapshot spec, SN-7: a snapshot cut short by a
// stop leaves a partial directory
inbuxa_store_snapshot::spawn_store_snapshots(inner.clone());
// inbuxa: relay spec, "The link service": every node links to every relay
inbuxa_relay_link::spawn_relay_links(inner.clone());
@@ -310,6 +310,17 @@ async fn store_maintenance(
}
TaskStoreMaintenanceType::PurgeBlob => {
if let Some(shard_index) = task.shard_index {
// inbuxa: store-snapshot spec, SN-4: while a snapshot takes
// the blobs, the purge is put off by five minutes
let Some(_purging) = store::write::snapshot::blob_purge_may_run() else {
return Ok(TaskResult::Success(vec![Task::StoreMaintenance(
TaskStoreMaintenance {
maintenance_type: TaskStoreMaintenanceType::PurgeBlob,
shard_index: Some(shard_index),
status: TaskStatus::at(now() as i64 + 300),
},
)]));
};
server
.store()
.purge_blobs(server.blob_store().clone(), shard_index as u8)
+83 -3
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use crate::BlobStore;
@@ -77,9 +79,22 @@ impl FsStore {
fs::create_dir_all(blob_path.parent().unwrap())
.await
.map_err(into_error)?;
let mut blob_file = File::create(&blob_path).await.map_err(into_error)?;
blob_file.write_all(data).await.map_err(into_error)?;
blob_file.flush().await.map_err(into_error)?;
// inbuxa: store-snapshot spec, SN-10: written beside it and
// renamed into place, never rewritten in place. A snapshot holds
// blobs by hard link, and a rewrite would change its copy too; a
// stop mid-write would leave half a blob under its id.
let temp_path = temp_path_for(&blob_path);
let written = async {
let mut blob_file = File::create(&temp_path).await?;
blob_file.write_all(data).await?;
blob_file.flush().await?;
fs::rename(&temp_path, &blob_path).await
}
.await;
if let Err(err) = written {
let _ = fs::remove_file(&temp_path).await;
return Err(into_error(err));
}
}
Ok(())
@@ -95,6 +110,11 @@ impl FsStore {
}
}
/// inbuxa: the directory the blobs are under (store-snapshot spec).
pub fn root(&self) -> &std::path::Path {
&self.path
}
fn build_path(&self, key: &[u8]) -> PathBuf {
let mut path = self.path.clone();
@@ -106,6 +126,66 @@ impl FsStore {
}
}
/// inbuxa: SN-10. A blob's name is base32, so a name starting with a dot
/// is never a blob: the snapshot's copy and anything listing blobs skip it.
fn temp_path_for(blob_path: &std::path::Path) -> PathBuf {
static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let n = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let name = blob_path
.file_name()
.map(|name| name.to_string_lossy())
.unwrap_or_default();
blob_path.with_file_name(format!(".{name}.{}-{n}.tmp", std::process::id()))
}
fn into_error(err: std::io::Error) -> trc::Error {
trc::StoreEvent::FilesystemError.reason(err)
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn writes_go_through_a_temporary_file() {
let dir = std::env::temp_dir().join(format!("inbuxa-fs-sn10-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
let BlobStore::Fs(store) = FsStore::open(structs::FileSystemStore {
path: dir.to_string_lossy().to_string(),
depth: 2,
})
.await
.unwrap() else {
unreachable!()
};
let key = [7u8; 32];
store.put_blob(&key, b"first").await.unwrap();
let path = store.build_path(&key);
// A file held by a hard link, as a snapshot holds it, keeps what it
// had when the blob under its id is written again
let held = dir.join("held");
std::fs::hard_link(&path, &held).unwrap();
store.put_blob(&key, b"second, longer").await.unwrap();
assert_eq!(std::fs::read(&held).unwrap(), b"first");
assert_eq!(
store.get_blob(&key, 0..usize::MAX).await.unwrap().unwrap(),
b"second, longer"
);
// Nothing is left beside it
let names: Vec<_> = std::fs::read_dir(path.parent().unwrap())
.unwrap()
.map(|entry| entry.unwrap().file_name())
.collect();
assert_eq!(names.len(), 1, "{names:?}");
assert!(
temp_path_for(&path)
.file_name()
.unwrap()
.to_string_lossy()
.starts_with('.')
);
std::fs::remove_dir_all(&dir).unwrap();
}
}
@@ -0,0 +1,31 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! inbuxa: a checkpoint of the store (store-snapshot spec, SN-2): every
//! column family at one point in time, hard-linked where it can be.
use super::{RocksDbStore, into_error};
use rocksdb::checkpoint::Checkpoint;
use std::path::{Path, PathBuf};
impl RocksDbStore {
/// The store's directory.
pub fn path(&self) -> &Path {
self.db.path()
}
/// Writes a checkpoint to `dest`, which must not exist yet.
pub async fn checkpoint(&self, dest: PathBuf) -> trc::Result<()> {
let db = self.db.clone();
tokio::task::spawn_blocking(move || {
Checkpoint::new(&*db)
.and_then(|checkpoint| checkpoint.create_checkpoint(&dest))
.map_err(into_error)
})
.await
.map_err(|err| trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err))?
}
}
+3
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use std::sync::Arc;
@@ -11,6 +13,7 @@ use rocksdb::{BoundColumnFamily, MultiThreaded, OptimisticTransactionDB};
use crate::{SUBSPACE_BLOBS, SUBSPACE_INDEXES, SUBSPACE_LOGS};
pub mod blob;
pub mod checkpoint; // inbuxa: store-snapshot spec
pub mod main;
pub mod read;
pub mod write;
+1
View File
@@ -33,6 +33,7 @@ pub mod blob_check; // inbuxa: store-check spec
pub mod key;
pub mod log;
pub mod serialize;
pub mod snapshot; // inbuxa: store-snapshot spec
pub(crate) const ARCHIVE_ALIGNMENT: usize = 16;
+239
View File
@@ -0,0 +1,239 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! inbuxa: what the store-snapshot spec (docs/spec/features/store-snapshot.md)
//! needs from the store: holding the blob purge (SN-4), a checkpoint of the
//! data store and a copy of a filesystem blob store's files (SN-2).
use crate::{BlobStore, Store};
use std::{
io,
path::{Path, PathBuf},
};
use tokio::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};
/// SN-4. A purge runs under the read side; a snapshot taking blobs holds
/// the write side. The data store is RocksDB, so one process has it open,
/// and a lock in that process is enough.
static BLOB_PURGE_GATE: RwLock<()> = RwLock::const_new(());
/// Whether a blob purge may run now: `None` while a snapshot takes blobs or
/// waits to. Keep the guard for as long as the purge runs.
pub fn blob_purge_may_run() -> Option<RwLockReadGuard<'static, ()>> {
BLOB_PURGE_GATE.try_read().ok()
}
/// Holds the blob purge until the guard is dropped. A purge already running
/// finishes first.
pub async fn hold_blob_purge() -> RwLockWriteGuard<'static, ()> {
BLOB_PURGE_GATE.write().await
}
/// What a snapshot can hold of this server's stores (SN-1, SN-2).
pub enum SnapshotSource {
/// Blobs are in the data store, so the checkpoint holds them.
DataOnly { data_path: PathBuf },
/// Blobs are files under `blob_path`, copied after the checkpoint.
DataAndFiles {
data_path: PathBuf,
blob_path: PathBuf,
},
}
/// Why a snapshot can't be taken of these stores.
pub enum NotSnapshottable {
DataStore(&'static str),
BlobStore(&'static str),
}
impl Store {
/// SN-2: a checkpoint of the data store into `dest`, which must not
/// exist yet. Only RocksDB has one.
#[cfg_attr(not(feature = "rocks"), allow(unused_variables))]
pub async fn checkpoint(&self, dest: PathBuf) -> trc::Result<()> {
match self {
#[cfg(feature = "rocks")]
Store::RocksDb(store) => store.checkpoint(dest).await,
_ => Err(trc::StoreEvent::NotSupported
.into_err()
.details("Only a RocksDB store has a checkpoint")),
}
}
/// What a snapshot of this data store and `blobs` would hold, or why
/// there can't be one.
#[cfg_attr(not(feature = "rocks"), allow(unused_variables, unreachable_code))]
pub fn snapshot_source(&self, blobs: &BlobStore) -> Result<SnapshotSource, NotSnapshottable> {
let data_path = match self {
#[cfg(feature = "rocks")]
Store::RocksDb(store) => store.path().to_path_buf(),
#[cfg(feature = "sqlite")]
Store::SQLite(_) => return Err(NotSnapshottable::DataStore("SQLite")),
#[cfg(feature = "foundation")]
Store::FoundationDb(_) => return Err(NotSnapshottable::DataStore("FoundationDB")),
#[cfg(feature = "postgres")]
Store::PostgreSQL(_) => return Err(NotSnapshottable::DataStore("PostgreSQL")),
#[cfg(feature = "mysql")]
Store::MySQL(_) => return Err(NotSnapshottable::DataStore("MySQL")),
Store::Replicated(_) => {
return Err(NotSnapshottable::DataStore("a replicated SQL database"));
}
Store::Ephemeral(_) | Store::None => {
return Err(NotSnapshottable::DataStore("an in-memory store"));
}
};
match blobs {
BlobStore::Store(store) if store.is_same(self) => {
Ok(SnapshotSource::DataOnly { data_path })
}
BlobStore::Fs(fs) => {
let blob_path = fs.root().to_path_buf();
// The blob files can't be told from the data store's own
// in one directory. A blob directory that holds the data
// store's is fine: the copy leaves that one out
if data_path == blob_path {
return Err(NotSnapshottable::BlobStore(
"the data store's own directory",
));
}
Ok(SnapshotSource::DataAndFiles {
data_path,
blob_path,
})
}
BlobStore::Store(_) => Err(NotSnapshottable::BlobStore("another database")),
#[cfg(feature = "s3")]
BlobStore::S3(_) => Err(NotSnapshottable::BlobStore("S3")),
#[cfg(feature = "azure")]
BlobStore::Azure(_) => Err(NotSnapshottable::BlobStore("Azure")),
BlobStore::Sharded(_) => Err(NotSnapshottable::BlobStore("a sharded set")),
}
}
}
/// What a copy of a tree did.
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub struct TreeCopy {
pub files: u64,
pub bytes: u64,
/// Files that couldn't be hard-linked and were copied.
pub copied: u64,
}
/// SN-2, SN-3: every file under `from` into `to`, hard-linked where it can
/// be and copied where it can't, leaving out `skip` (the data store's
/// directory, with the snapshots in it). Names starting with a dot are
/// skipped: they're blobs being written (SN-10).
pub async fn copy_tree(from: PathBuf, to: PathBuf, skip: PathBuf) -> trc::Result<TreeCopy> {
tokio::task::spawn_blocking(move || {
let mut done = TreeCopy::default();
copy_dir(&from, &to, &skip, &mut done).map(|_| done)
})
.await
.map_err(|err| trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err))?
.map_err(|err| trc::StoreEvent::FilesystemError.reason(err))
}
fn copy_dir(from: &Path, to: &Path, skip: &Path, done: &mut TreeCopy) -> io::Result<()> {
std::fs::create_dir_all(to)?;
for entry in std::fs::read_dir(from)? {
let entry = entry?;
let name = entry.file_name();
if name.as_encoded_bytes().first() == Some(&b'.') {
continue;
}
let (source, dest) = (entry.path(), to.join(&name));
if source == skip {
continue;
}
let kind = entry.file_type()?;
if kind.is_dir() {
copy_dir(&source, &dest, skip, done)?;
} else if kind.is_file() {
match std::fs::hard_link(&source, &dest) {
Ok(()) => {}
// Gone since it was listed: a purge ran before the hold, or
// it's an orphan nothing links to
Err(err) if err.kind() == io::ErrorKind::NotFound => continue,
Err(_) => {
match std::fs::copy(&source, &dest) {
Ok(_) => {}
Err(err) if err.kind() == io::ErrorKind::NotFound => continue,
Err(err) => return Err(err),
}
done.copied += 1;
}
}
done.files += 1;
done.bytes += std::fs::metadata(&dest)?.len();
}
}
Ok(())
}
/// Bytes in the files under `dir`.
pub async fn tree_size(dir: PathBuf) -> trc::Result<u64> {
fn size(dir: &Path) -> io::Result<u64> {
let mut total = 0;
for entry in std::fs::read_dir(dir)? {
let entry = entry?;
let kind = entry.file_type()?;
if kind.is_dir() {
total += size(&entry.path())?;
} else if kind.is_file() {
total += entry.metadata()?.len();
}
}
Ok(total)
}
tokio::task::spawn_blocking(move || size(&dir))
.await
.map_err(|err| trc::EventType::Server(trc::ServerEvent::ThreadError).reason(err))?
.map_err(|err| trc::StoreEvent::FilesystemError.reason(err))
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn copies_a_tree_and_skips_what_is_being_written() {
let base = std::env::temp_dir().join(format!("inbuxa-sn-tree-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&base);
let from = base.join("from");
std::fs::create_dir_all(from.join("a/b")).unwrap();
std::fs::write(from.join("a/b/BLOB1"), b"one").unwrap();
std::fs::write(from.join("a/BLOB2"), b"two!").unwrap();
std::fs::write(from.join("a/b/.BLOB3.1-0.tmp"), b"half").unwrap();
std::fs::create_dir_all(from.join("rocks.db/snapshots")).unwrap();
std::fs::write(from.join("rocks.db/000001.sst"), b"data").unwrap();
let to = base.join("to");
let done = copy_tree(from.clone(), to.clone(), from.join("rocks.db"))
.await
.unwrap();
assert_eq!((done.files, done.bytes), (2, 7));
assert_eq!(std::fs::read(to.join("a/b/BLOB1")).unwrap(), b"one");
assert!(!to.join("a/b/.BLOB3.1-0.tmp").exists());
assert!(!to.join("rocks.db").exists());
assert_eq!(tree_size(to).await.unwrap(), 7);
std::fs::remove_dir_all(&base).unwrap();
}
#[test]
fn the_purge_waits_while_a_snapshot_holds_it() {
let runtime = tokio::runtime::Builder::new_current_thread()
.build()
.unwrap();
runtime.block_on(async {
assert!(blob_purge_may_run().is_some());
let hold = hold_blob_purge().await;
assert!(blob_purge_may_run().is_none());
drop(hold);
assert!(blob_purge_may_run().is_some());
});
}
}
+2
View File
@@ -388,6 +388,8 @@ Not a rebuild: **migrations into this server** are INBUXA's own design (`feature
Not a rebuild: the **store check** is INBUXA's own design (`features/store-check.md`, inbuxa-server #322, building). An administrator starts a check of the store's blobs, quick (each one is there) or full (each one reads back intact), and gets a list of what's missing, damaged or no longer linked. One node runs it in the background, throttled, and keeps its progress: `inbuxa:StoreCheck` (`crates/jmap/src/inbuxa/store_check.rs`, `crates/features/src/store_check/`, `crates/services/src/inbuxa_store_check.rs`, `crates/store/src/write/blob_check.rs`), and the `sysStoreCheck` permission.
Not a rebuild: the **store snapshot** is INBUXA's own design (`features/store-snapshot.md`, inbuxa-server #323, building). An administrator has the server take a consistent copy of its RocksDB store, and of a filesystem blob store's files, into `snapshots/` in the store's directory while mail flows, for a backup tool to copy; the latest two are kept: `inbuxa:StoreSnapshot` (`crates/jmap/src/inbuxa/store_snapshot.rs`, `crates/services/src/inbuxa_store_snapshot.rs`, `crates/store/src/write/snapshot.rs`, `crates/store/src/backend/rocksdb/checkpoint.rs`), and the `sysStoreSnapshot` permission.
Not a rebuild: **the store layout, changed in steps** is INBUXA's own design (`features/store-layout.md`, approved 2026-11-08). A store carries its layout version (`Vs` in `_`); at boot one node, holding a lease, runs each step from the stored version up and marks the store after each. Step 1 → 2 gives SQL rate-limit counters their own expiry column; step 2 → 3 moves the audit log, journal, holds and undelete notes into subspaces of their own. A step stops rollback, so it ships at a series.
Not a rebuild: **inbuxa-relay** is INBUXA's own design (`features/relay.md`, settled 2026-10-29, in progress). It's a small edge appliance that takes mail in on port 25 for one downstream server or cluster, and sends that downstream's outgoing mail on. The downstream dials out to it over JMAP (`urn:inbuxa:jmap:relay`), so the protected network exposes no mail port, and it is administered only from the downstream's inbuxa-admin.
+148
View File
@@ -0,0 +1,148 @@
# Feature spec: a snapshot of the store, without stopping the server
Status: building, 2026-11-09 (inbuxa-server #323). Not a rebuild (SPEC.md
§4): inbuxa's own design.
## Provenance
| Source | Used for |
|---|---|
| Forum topic 238 | The need: a backup that stops the server, copies the store cold and starts it again, an outage each time |
| RocksDB's checkpoint (`rocksdb::checkpoint::Checkpoint`, rocksdb 0.25) | A consistent copy of every column family at once, by hard links on the same filesystem |
| This server's blob writes and purge (`crates/store/src/write/blob.rs`, `crates/store/src/backend/fs/mod.rs`) | Why a snapshot of the data store, then of the blobs, holds every blob its data links to |
| The store check (`features/store-check.md`) | How the object, its permission and its run are built |
Upstream's open code has nothing like it, and nothing here comes from
Enterprise code.
## What it is
An administrator asks the server for a snapshot of its store. The server
writes a consistent copy into a directory on its own machine, while mail
keeps flowing, and says when it's done. A backup tool then copies that
directory however it likes. Encryption, off-site copies and how long
backups are kept stay with the backup tool. The server keeps only its
latest two snapshots.
A snapshot is a cold copy that was taken hot. Restoring one is copying it
back while the server is stopped.
## Requirements
- **SN-1.** A snapshot is an `inbuxa:StoreSnapshot` under `urn:inbuxa:jmap`.
It needs RocksDB as the data store. With any other data store a create
is refused, and the refusal says to use the database's own backup:
PostgreSQL's or MySQL's, or FoundationDB's.
- **SN-2.** What it holds:
- **The data store**: a RocksDB checkpoint of every column family, taken
at one point in time. Blobs kept in the data store (the default) are
in it too.
- **Blobs in a separate filesystem blob store**, copied after the
checkpoint. Blobs are written before anything links to them, and the
blob purge is held while the snapshot is taken (SN-4), so every blob
the checkpoint links to is there to copy. A blob written after the
checkpoint may be copied as well; nothing in the snapshot links to it.
A blob directory that holds the data store's directory is fine: the
copy leaves the data store's directory out. Blobs in the data store's
own directory are refused, since they can't be told from its files.
Any other blob store (S3, Azure, a sharded set, a SQL database) is
refused: a copy of the data store alone would point at blobs a later
purge could take away before the backup tool copied them.
Not in it: the search index when it's kept outside the data store
(Meilisearch, Elasticsearch, ...), rebuilt after a restore; and the
in-memory store (rate limits, locks), which a restarted server doesn't
need.
- **SN-3.** Where: under `snapshots/` in the RocksDB directory, one
directory per snapshot named for when it was taken (UTC), like
`snapshots/20261109T013000Z/`, with `data/`, `blobs/` when there are
separate blobs, and `snapshot.json`. Being on the store's own
filesystem, RocksDB and the blob copy hard-link their files, so a snapshot
is quick and at first takes almost no space. Where they can't link
(another filesystem), they copy. The location is fixed: a request can't
name a path.
A snapshot is written as `snapshots/.<name>.partial/` and renamed when
complete, so a directory with the plain name is always whole.
`snapshots/latest` is a link to the newest complete one.
- **SN-4.** While a snapshot takes blobs from a filesystem blob store, the
blob purge waits. A purge already running is let finish first. A purge
task that comes due meanwhile is put off by five minutes.
- **SN-5.** `snapshot.json` says when it was taken and finished, by whom,
the server version, what it holds (`data`, `blobs`), the data and blob
stores' paths, its size, and how to restore it.
- **SN-6.** One snapshot is made at a time; another create is refused
while one runs. When one completes, the oldest beyond the latest two are
deleted.
- **SN-7.** A snapshot is `running`, then `done`, or `failed` with the
error and its partial directory removed. One the server stopped in the
middle of leaves a partial directory, removed when the server next
starts, and isn't listed.
The complete snapshots on disk are the record: `get` lists those, read
from each one's `snapshot.json`, and, until the server restarts, the one
being taken or the last one that failed.
- **SN-8.** Only administrators take snapshots: `sysStoreSnapshot`, in the
built-in Administrator role and granted to existing administrator roles.
A snapshot holds every account's data, so a tenant administrator is
refused. Taking and destroying a snapshot are audited.
- **SN-9.** Destroying a snapshot deletes its directory. A running one
can't be destroyed.
- **SN-10.** A filesystem blob store writes each blob to a temporary file
in the same directory and renames it into place. Before, it rewrote a
file in place, which would also have changed the copy a snapshot holds
by hard link, and left a half-written blob behind if the server stopped
mid-write.
## The object
| Property | Type | |
|---|---|---|
| `id` | Id | |
| `status` | String | `running`, `done`, `failed` |
| `createdAt`, `finishedAt` | UTCDate | |
| `createdBy` | String | Who took it |
| `node` | String | The node that took it |
| `name` | String | Its directory under `snapshots/` |
| `path` | String | Its full path on the server's machine |
| `holds` | String[] | `data`, and `blobs` when they're separate |
| `size` | UnsignedInt | Bytes in its files |
| `blobFiles` | UnsignedInt | Blob files copied |
| `error` | String | Why a `failed` snapshot failed |
A create takes no properties.
## Restoring
1. Stop the server.
2. Move the RocksDB directory's contents aside, except `snapshots/`, and
copy the snapshot's `data/` into it.
3. With a separate filesystem blob store, do the same with its directory
and the snapshot's `blobs/`.
4. Start the server. A search index kept outside the data store isn't in
the snapshot; reindex the accounts once it's running.
A backup tool copying the store cold should leave out `snapshots/`, or
the store is copied twice.
## Tests
- `system::store_snapshot` (`tests/src/system/store_snapshot.rs`), against
a real server with RocksDB, run with blobs in the data store and again
with `BLOB_STORE=FileSystem`: a snapshot taken while mail is in the
store; the directory, its name, `latest` and the manifest; a RocksDB
store opened from its `data/` has the messages and their blobs (in it,
or in `blobs/`), and not a message that came after; no properties at
create, no updates; with separate blobs, one at a time and no destroy
while one is taken (the test holds the purge gate); only the latest two
kept; destroy; the audit events; a user's and a tenant administrator's
refusal.
- Unit tests: the filesystem blob store's writes (SN-10), the tree copy
and what it leaves out, the purge gate, snapshot names and the manifest.
## Not yet
- The console's page and an `inbuxa-cli` command a backup tool can call
and wait on.
- SQLite as the data store (`VACUUM INTO` would do it).
+23
View File
@@ -281,6 +281,16 @@ retention = "object-life"
createdBy = ["identifier"]
problems = ["identifier", "metadata"]
[object."inbuxa:StoreSnapshot"]
file = "inbuxa_store_snapshot.rs"
default = "none"
whose = ["administrator"]
where = ["data-directory"]
scope = "server"
retention = "object-life"
[object."inbuxa:StoreSnapshot".properties]
createdBy = ["identifier"]
[object."inbuxa:Migration"]
file = "inbuxa_migration.rs"
default = "none"
@@ -423,6 +433,19 @@ retention = "object-life"
leaves_host = false
written_by = ["crates/services/src/inbuxa_migrations.rs"]
# A store snapshot: a copy of the whole data store, and of a filesystem
# blob store's files, under snapshots/ in the RocksDB directory. The latest
# two are kept; a backup tool takes them off the host if the administrator
# has one do so (store-snapshot spec, SN-3, SN-6).
[source."store-snapshots"]
categories = ["content", "identifier", "contact", "network", "metadata", "credential"]
whose = ["holder", "correspondent", "administrator"]
where = ["data-directory"]
scope = "server"
retention = "object-life"
leaves_host = false
written_by = ["crates/services/src/inbuxa_store_snapshot.rs"]
[source."full-text-index"]
categories = ["content", "identifier", "contact", "metadata"]
whose = ["holder", "correspondent"]
Binary file not shown.
+1 -1
View File
@@ -1 +1 @@
Z7m4tA0knFnM6t30zrTSVKWBuNKOEGEMouWPHB3_ED4
6ZXhO67yQsLEItXqRvBehskGGCr53WdaWL9Nh6i7Nm0
+1
View File
@@ -39,6 +39,7 @@ pub mod security;
pub mod security_acceptances; // inbuxa: accepted security to-do items
pub mod sharing_policy; // inbuxa: MA-C, who may share mail
pub mod store_check; // inbuxa: checking the store's blobs
pub mod store_snapshot; // inbuxa: a snapshot of the store
pub mod task;
pub mod tenant;
pub mod tracer_reload; // inbuxa: tracers start over when their settings change
+411
View File
@@ -0,0 +1,411 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! `inbuxa:StoreSnapshot` (docs/spec/features/store-snapshot.md).
//!
//! A snapshot taken while the server runs holds the mail that was in it and
//! not the mail that came after: a RocksDB store opened from its `data/`
//! has the messages, and their blobs are in it or, with a filesystem blob
//! store (`BLOB_STORE=FileSystem`), in its `blobs/`. The directory is
//! whole, named for its time, with `latest` and `snapshot.json`. Only the
//! latest two are kept; one can be destroyed, a snapshot can't be changed,
//! and one being taken can't be destroyed or joined by another. Nobody in
//! a tenant or without the permission reaches snapshots. Taking and
//! destroying are audited.
use crate::utils::{
account::Account,
server::{TestServer, TestServerBuilder},
};
use email::{mailbox::INBOX_ID, message::metadata::MessageMetadata};
use registry::schema::{
prelude::{ObjectType, Property},
structs::{
self, CertificateManagement, DkimManagement, DnsManagement, Domain, Tenant, UserRoles,
},
};
use serde_json::{Value, json};
use std::{path::PathBuf, str::FromStr, time::Duration};
use store::{
BlobStore, Store, ValueKey,
backend::{fs::FsStore, rocksdb::RocksDbStore},
write::{AlignedBytes, Archive, snapshot::hold_blob_purge},
};
use types::{blob_hash::BlobHash, collection::Collection, field::EmailField, id::Id};
const USING: &[&str] = &[
"urn:ietf:params:jmap:core",
"urn:inbuxa:jmap",
"urn:inbuxa:jmap:registry",
];
async fn call(account: &Account, method: &str, mut arguments: Value) -> (String, Value) {
if arguments.get("accountId").is_none() {
arguments["accountId"] = account.id_string().into();
}
let response = account
.jmap_request(USING, json!([[method, arguments, "0"]]))
.await;
let call = response
.0
.pointer("/methodResponses/0")
.cloned()
.unwrap_or_else(|| panic!("{method}: {}", response.0));
(
call[0].as_str().unwrap_or_default().to_string(),
call[1].clone(),
)
}
async fn take(admin: &Account) -> String {
let (_, response) = call(
admin,
"inbuxa:StoreSnapshot/set",
json!({"create": {"s": {}}}),
)
.await;
response["created"]["s"]["id"]
.as_str()
.unwrap_or_else(|| panic!("not taken: {response}"))
.to_string()
}
async fn get(admin: &Account, id: &str) -> Value {
let (_, response) = call(admin, "inbuxa:StoreSnapshot/get", json!({"ids": [id]})).await;
response["list"][0].clone()
}
async fn all(admin: &Account) -> Vec<Value> {
let (_, response) = call(admin, "inbuxa:StoreSnapshot/get", json!({"ids": null})).await;
response["list"]
.as_array()
.unwrap_or_else(|| panic!("no list: {response}"))
.clone()
}
/// The snapshot once it's taken.
async fn settled(admin: &Account, id: &str) -> Value {
for _ in 0..600 {
let snapshot = get(admin, id).await;
if snapshot["status"] != "running" {
return snapshot;
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
panic!("the snapshot wasn't taken: {}", get(admin, id).await);
}
async fn import(client: &jmap_client::client::Client, body: &str) -> u32 {
let id = client
.email_import(
format!("From: [email protected]\r\nSubject: {body}\r\n\r\n{body}\r\n").into_bytes(),
[Id::from(INBOX_ID).to_string()],
None::<Vec<&str>>,
None,
)
.await
.unwrap()
.take_id();
Id::from_str(&id).unwrap().document_id()
}
async fn blob_in(store: &Store, account_id: u32, document_id: u32) -> Option<BlobHash> {
let archive = store
.get_value::<Archive<AlignedBytes>>(ValueKey::property(
account_id,
Collection::Email,
document_id,
EmailField::Metadata,
))
.await
.unwrap()?;
let metadata = archive.unarchive::<MessageMetadata>().unwrap();
Some(BlobHash::from(&metadata.blob_hash))
}
fn path_of(snapshot: &Value) -> PathBuf {
PathBuf::from(
snapshot["path"]
.as_str()
.unwrap_or_else(|| panic!("no path: {snapshot}")),
)
}
pub async fn test(test: &mut TestServer) {
println!("Running store snapshot tests...");
let admin = test.account("[email protected]");
let ann = test.account("[email protected]");
let ann_id = ann.id().document_id();
let client = ann.jmap_client().await;
let separate_blobs = matches!(test.server.blob_store(), BlobStore::Fs(_));
println!(
"Blobs {}",
if separate_blobs {
"in a filesystem blob store"
} else {
"in the data store"
}
);
let before = [
import(&client, "Before the snapshot, one").await,
import(&client, "Before the snapshot, two").await,
];
assert!(all(admin).await.is_empty(), "no snapshots yet");
// A snapshot while the server runs
let id = take(admin).await;
let snapshot = settled(admin, &id).await;
assert_eq!(snapshot["status"], "done", "{snapshot}");
assert_eq!(snapshot["createdBy"], "[email protected]", "{snapshot}");
let holds = if separate_blobs {
json!(["data", "blobs"])
} else {
json!(["data"])
};
assert_eq!(snapshot["holds"], holds, "{snapshot}");
assert!(snapshot["size"].as_u64().unwrap() > 0, "{snapshot}");
let after = import(&client, "After the snapshot").await;
// The directory: whole, named for its time, `latest`, the manifest
let path = path_of(&snapshot);
let name = snapshot["name"].as_str().unwrap();
assert_eq!(path.file_name().unwrap().to_str().unwrap(), name);
assert!(name.len() == 16 && name.ends_with('Z'), "{name}");
let root = path.parent().unwrap().to_path_buf();
assert_eq!(root.file_name().unwrap(), "snapshots");
assert_eq!(
std::fs::read_link(root.join("latest")).unwrap(),
PathBuf::from(name)
);
assert!(path.join("data/CURRENT").exists(), "a RocksDB checkpoint");
assert_eq!(path.join("blobs").exists(), separate_blobs);
let manifest: Value =
serde_json::from_slice(&std::fs::read(path.join("snapshot.json")).unwrap()).unwrap();
assert_eq!(manifest["id"], json!(id), "{manifest}");
assert_eq!(manifest["holds"], holds, "{manifest}");
assert!(
manifest["restore"].as_array().unwrap().len() >= 3,
"{manifest}"
);
assert!(
std::fs::read_dir(&root).unwrap().all(|e| !e
.unwrap()
.file_name()
.to_string_lossy()
.ends_with(".partial")),
"nothing partial left"
);
// Restored, it has the mail that was there, and not what came after
let restored = RocksDbStore::open(structs::RocksDbStore {
path: path.join("data").to_string_lossy().to_string(),
..Default::default()
})
.await
.unwrap();
let blobs = if separate_blobs {
FsStore::open(structs::FileSystemStore {
path: path.join("blobs").to_string_lossy().to_string(),
..Default::default()
})
.await
.unwrap()
} else {
BlobStore::Store(restored.clone())
};
for (document_id, subject) in before.iter().zip(["one", "two"]) {
let hash = blob_in(&restored, ann_id, *document_id)
.await
.unwrap_or_else(|| panic!("message {document_id} isn't in the snapshot"));
let content = blobs
.get_blob(hash.as_slice(), 0..usize::MAX)
.await
.unwrap()
.unwrap_or_else(|| panic!("the blob of message {document_id} isn't in the snapshot"));
let content = String::from_utf8_lossy(&content);
assert!(content.contains(&format!("Before the snapshot, {subject}")));
}
assert!(
blob_in(&restored, ann_id, after).await.is_none(),
"mail that came after isn't in it"
);
drop(blobs);
drop(restored);
// A snapshot can't be changed, and takes no properties
let (_, response) = call(
admin,
"inbuxa:StoreSnapshot/set",
json!({"update": {&id: {"name": "x"}}}),
)
.await;
assert!(response["notUpdated"].get(&id).is_some(), "{response}");
let (_, response) = call(
admin,
"inbuxa:StoreSnapshot/set",
json!({"create": {"x": {"path": "/tmp/elsewhere"}}}),
)
.await;
assert_eq!(
response["notCreated"]["x"]["type"], "invalidProperties",
"{response}"
);
// One at a time, and one being taken can't be destroyed. With separate
// blobs a snapshot waits for the blob purge, so holding it here keeps
// one being taken for as long as this needs
if separate_blobs {
let hold = hold_blob_purge().await;
let waiting = take(admin).await;
assert_eq!(get(admin, &waiting).await["status"], "running");
let (_, response) = call(
admin,
"inbuxa:StoreSnapshot/set",
json!({"create": {"x": {}}}),
)
.await;
assert_eq!(
response["notCreated"]["x"]["type"], "forbidden",
"{response}"
);
let (_, response) = call(
admin,
"inbuxa:StoreSnapshot/set",
json!({"destroy": [&waiting]}),
)
.await;
assert_eq!(
response["notDestroyed"][&waiting]["type"], "forbidden",
"{response}"
);
drop(hold);
assert_eq!(settled(admin, &waiting).await["status"], "done");
} else {
let next = take(admin).await;
assert_eq!(settled(admin, &next).await["status"], "done");
}
// Only the latest two are kept
let third = take(admin).await;
let third_snapshot = settled(admin, &third).await;
let listed = all(admin).await;
assert_eq!(listed.len(), 2, "{listed:?}");
assert_eq!(listed[0]["id"], json!(third), "newest first: {listed:?}");
assert!(!path.exists(), "the oldest went");
let (_, response) = call(admin, "inbuxa:StoreSnapshot/get", json!({"ids": [&id]})).await;
assert_eq!(response["notFound"], json!([id]), "{response}");
assert_eq!(
std::fs::read_link(root.join("latest")).unwrap(),
PathBuf::from(third_snapshot["name"].as_str().unwrap())
);
// Destroying one deletes it; `latest` moves to the one left
let unknown = Id::from(1u64).to_string();
let (_, response) = call(
admin,
"inbuxa:StoreSnapshot/set",
json!({"destroy": [&third, &unknown]}),
)
.await;
assert_eq!(response["destroyed"], json!([third]), "{response}");
assert_eq!(
response["notDestroyed"][&unknown]["type"], "notFound",
"{response}"
);
assert!(!path_of(&third_snapshot).exists());
let left = all(admin).await;
assert_eq!(left.len(), 1, "{left:?}");
assert_eq!(
std::fs::read_link(root.join("latest")).unwrap(),
PathBuf::from(left[0]["name"].as_str().unwrap())
);
// Taking and destroying are audited
let (_, response) = call(
admin,
"inbuxa:AuditEvent/query",
json!({"filter": {"targetKind": "inbuxa:StoreSnapshot"}}),
)
.await;
assert!(
response["ids"].as_array().is_some_and(|ids| ids.len() >= 4),
"three taken and one destroyed, audited: {response}"
);
// Nobody without the permission, or in a tenant, reaches snapshots
let (name, response) = call(ann, "inbuxa:StoreSnapshot/get", json!({"ids": null})).await;
assert_eq!(name, "error", "a user: {response}");
let tenant = admin
.registry_create_object(Tenant {
name: "Store snapshot tenant".to_string(),
..Default::default()
})
.await;
admin
.registry_create_object(Domain {
name: "storesnapshot.example.org".to_string(),
member_tenant_id: Some(tenant),
certificate_management: CertificateManagement::Manual,
dns_management: DnsManagement::Manual,
dkim_management: DkimManagement::Manual,
..Default::default()
})
.await;
let t_admin = admin
.create_user_account(
"[email protected]",
"tenant-admin-secret-5521",
"Tenant admin",
&[],
vec![],
)
.await;
admin
.registry_update_object(
ObjectType::Account,
t_admin.id(),
json!({Property::Roles: UserRoles::Admin}),
)
.await;
for (method, arguments) in [
("inbuxa:StoreSnapshot/get", json!({"ids": null})),
("inbuxa:StoreSnapshot/set", json!({"create": {"x": {}}})),
] {
let (name, response) = call(&t_admin, method, arguments).await;
assert_eq!(name, "error", "{method} by a tenant: {response}");
}
}
/// Runs these tests alone, with RocksDB:
/// `STORE=RocksDb cargo test -p tests store_snapshot_tests -- --ignored`,
/// and again with `BLOB_STORE=FileSystem` for a separate blob store.
#[ignore]
#[tokio::test(flavor = "multi_thread")]
pub async fn store_snapshot_tests() {
let mut test = TestServerBuilder::new("store_snapshot_tests")
.await
.with_default_listeners()
.await
.build()
.await;
let admin = test.create_admin_account("[email protected]").await;
test.insert_account(admin);
let ann = test
.create_user_account(
"[email protected]",
"[email protected]",
"ann-secret-61734",
&[],
"Ann",
)
.await;
test.insert_account(ann);
self::test(&mut test).await;
if test.is_reset() {
test.temp_dir.delete();
}
}