Store layout step 2 → 3: the audit log, journal, holds and undelete notes get subspaces of their own #368

Merged
jcoffey-dev merged 2 commits from feat/store-layout-step-3 into main 2026-11-11 06:44:15 +00:00
32 changed files with 1525 additions and 176 deletions

No files matched your search

+3 -1
View File
@@ -116,7 +116,9 @@ pub const DATABASE_SCHEMA_VERSION: u32 = 6;
// 1 - the 0.16 layout plus the fork's own subspace (2026.10.x, 27.1-rc.16)
// 2 - in-memory counters on SQL keep their own expiry (store-layout spec
// SL-10); the steps are in crates/migration/src/layout.rs
pub const INBUXA_STORE_VERSION: u32 = 2;
// 3 - the audit log, journal, holds and undelete notes have subspaces of
// their own (SL-16), and RocksDB's legacy `v` and `c` are gone (SL-20)
pub const INBUXA_STORE_VERSION: u32 = 3;
pub const INBUXA_STORE_VERSION_KEY: &[u8] = b"Vs";
pub const LONG_1D_SLUMBER: Duration = Duration::from_secs(60 * 60 * 24);
+15 -3
View File
@@ -40,6 +40,8 @@ pub(super) enum Family {
Report = 6,
Telemetry = 7,
Tasks = 8,
// inbuxa: SL-19: the fork's large data (store layout version 3)
Compliance = 9,
}
type TaskHandle = (tokio::task::JoinHandle<()>, std::thread::JoinHandle<()>);
@@ -81,6 +83,7 @@ impl Core {
Family::Report,
Family::Telemetry,
Family::Tasks,
Family::Compliance,
]
.into_iter()
.collect();
@@ -110,7 +113,7 @@ impl Core {
let store = self.storage.data.clone();
let blob_store = self.storage.blob.clone();
let (handle, writer) = spawn_writer(
dest.join(format!("subspace_{}", char::from(subspace))),
dest.join(format!("subspace_{}", store::subspace_name(subspace))),
subspace,
schema_version,
);
@@ -175,7 +178,7 @@ impl Core {
fn backup_subspace(&self, dest: &Path, subspace: u8, schema_version: u32) -> TaskHandle {
let store = self.storage.data.clone();
let (handle, writer) = spawn_writer(
dest.join(format!("subspace_{}", char::from(subspace))),
dest.join(format!("subspace_{}", store::subspace_name(subspace))),
subspace,
schema_version,
);
@@ -342,13 +345,21 @@ impl Family {
// travel in this family and in `data`, so they ride along.
SUBSPACE_DELETED_ITEMS,
SUBSPACE_SPAM_SAMPLES,
store::SUBSPACE_INBUXA, // inbuxa: the fork's own data (masked email, undelete, policies)
store::SUBSPACE_INBUXA, // inbuxa: the fork's own settings and small records (masked email, policies)
],
Family::Changelog => &[SUBSPACE_LOGS],
Family::Queue => &[SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUEUE_EVENT],
Family::Report => &[SUBSPACE_REPORT_OUT, SUBSPACE_REPORT_IN],
Family::Telemetry => &[SUBSPACE_TELEMETRY_SPAN, SUBSPACE_TELEMETRY_METRIC],
Family::Tasks => &[SUBSPACE_TASK_QUEUE],
// inbuxa: SL-19: in a family of its own, so a configuration
// export (`registry`) can leave out the audit log and journal
Family::Compliance => &[
store::SUBSPACE_AUDIT,
store::SUBSPACE_JOURNAL,
store::SUBSPACE_HOLDS,
store::SUBSPACE_UNDELETE,
],
}
}
@@ -362,6 +373,7 @@ impl Family {
"report" => Ok(Family::Report),
"telemetry" => Ok(Family::Telemetry),
"tasks" => Ok(Family::Tasks),
"compliance" => Ok(Family::Compliance),
_ => Err(format!("Unknown family {}", family)),
}
}
+18 -15
View File
@@ -4,9 +4,9 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! The audit log's storage (AU-2, AU-3, AU-6, AU-7), in the fork's own
//! subspace (`store::SUBSPACE_INBUXA`). Every key starts with `L`, then one
//! byte for the kind:
//! The audit log's storage (AU-2, AU-3, AU-6, AU-7), in a subspace of its
//! own (`store::SUBSPACE_AUDIT`, store layout version 3). Every key starts
//! with one byte for the kind:
//!
//! - `e` + node + seq: one entry of that node's chain, as JSON. An entry is
//! an event, or the outcome of an event written before its change was
@@ -18,7 +18,8 @@
//! both add the same seq.
//! - `f` + node: where the chain starts after purging, and the hash the
//! first kept entry names.
//! - `s`: the settings (`keepFor`).
//! - `s`: the settings (`keepFor`). These stay in the fork's `_` subspace,
//! as `L` + `s`, so a configuration export carries them.
//!
//! Numbers are big-endian, so keys sort in time and chain order. Each node
//! writes only its own chain, so nodes never contend for a key; nothing about
@@ -31,7 +32,7 @@ use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize};
use sha2::{Digest, Sha256};
use std::{fmt, net::IpAddr, str::FromStr};
use store::{
Deserialize, IterateParams, SUBSPACE_INBUXA, Serialize, Store, ValueKey,
Deserialize, IterateParams, SUBSPACE_AUDIT, SUBSPACE_INBUXA, Serialize, Store, ValueKey,
write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue},
};
use tokio::sync::Mutex;
@@ -229,16 +230,17 @@ impl Deserialize for U64 {
}
fn class(kind: u8, parts: &[u64]) -> ValueClass {
let mut key = Vec::with_capacity(2 + parts.len() * 8);
key.push(FEATURE);
// SL-16: the settings stay in `_`; the log has a subspace of its own
let (subspace, mut key) = if kind == KIND_SETTINGS {
(SUBSPACE_INBUXA, vec![FEATURE])
} else {
(SUBSPACE_AUDIT, Vec::with_capacity(1 + parts.len() * 8))
};
key.push(kind);
for part in parts {
key.extend_from_slice(&part.to_be_bytes());
}
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key,
})
ValueClass::Any(AnyClass { subspace, key })
}
fn key(kind: u8, parts: &[u64]) -> ValueKey<ValueClass> {
@@ -259,11 +261,11 @@ pub fn head_key(node: u64) -> ValueKey<ValueClass> {
/// The numbers after the kind byte, read from the key's tail: the iterator
/// may or may not hand back the subspace byte.
fn parse_key(key: &[u8], kind: u8, parts: usize) -> Option<Vec<u64>> {
let len = 2 + parts * 8;
let len = 1 + parts * 8;
let tail = key.get(key.len().checked_sub(len)?..)?;
(tail[0] == FEATURE && tail[1] == kind).then_some(())?;
(tail[0] == kind).then_some(())?;
Some(
tail[2..]
tail[1..]
.chunks_exact(8)
.map(|chunk| u64::from_be_bytes(chunk.try_into().unwrap()))
.collect(),
@@ -875,7 +877,8 @@ mod tests {
panic!()
};
assert_eq!(parse_key(&any.key, KIND_TIME, 3), Some(vec![5, 3, 9]));
let mut with_subspace = vec![SUBSPACE_INBUXA];
assert_eq!(any.subspace, SUBSPACE_AUDIT);
let mut with_subspace = vec![SUBSPACE_AUDIT];
with_subspace.extend_from_slice(&any.key);
assert_eq!(parse_key(&with_subspace, KIND_TIME, 3), Some(vec![5, 3, 9]));
assert_eq!(parse_key(&any.key, KIND_ENTRY, 3), None);
+10 -13
View File
@@ -11,8 +11,9 @@
//! While any active hold covers an item, nothing may destroy it. A hold is
//! never deleted: releasing it keeps it, read-only, for the audit trail.
//!
//! Kept in the fork's subspace (`store::SUBSPACE_INBUXA`). Every key starts
//! with `H`, then one byte for the kind:
//! Kept in a subspace of their own (`store::SUBSPACE_HOLDS`, store layout
//! version 3; before it, in `_` under `H`). Every key starts with one byte
//! for the kind:
//!
//! - `h` + hold id (u32): the hold, as JSON.
//!
@@ -21,7 +22,7 @@
use registry::schema::{prelude::ObjectInner, structs::Account};
use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize};
use store::{
Deserialize, IterateParams, SUBSPACE_INBUXA, Serialize, Store, ValueKey,
Deserialize, IterateParams, SUBSPACE_HOLDS, Serialize, Store, ValueKey,
write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue},
};
use trc::AddContext;
@@ -103,7 +104,6 @@ impl Keeping {
}
}
const FEATURE: u8 = b'H';
const KIND_HOLD: u8 = b'h';
const KIND_ORIGINAL: u8 = b'o';
const KIND_EXPORT: u8 = b'e';
@@ -401,12 +401,11 @@ impl<T: serde::de::DeserializeOwned + Sync + Send> Deserialize for Json<T> {
}
fn class(id: u32) -> ValueClass {
let mut key = Vec::with_capacity(6);
key.push(FEATURE);
let mut key = Vec::with_capacity(5);
key.push(KIND_HOLD);
key.extend_from_slice(&id.to_be_bytes());
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
subspace: SUBSPACE_HOLDS,
key,
})
}
@@ -416,12 +415,11 @@ fn key(id: u32) -> ValueKey<ValueClass> {
}
fn original_class(item_id: u64) -> ValueClass {
let mut key = Vec::with_capacity(10);
key.push(FEATURE);
let mut key = Vec::with_capacity(9);
key.push(KIND_ORIGINAL);
key.extend_from_slice(&item_id.to_be_bytes());
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
subspace: SUBSPACE_HOLDS,
key,
})
}
@@ -454,12 +452,11 @@ pub async fn set_original_deadline(
}
fn export_class(id: u32) -> ValueClass {
let mut key = Vec::with_capacity(6);
key.push(FEATURE);
let mut key = Vec::with_capacity(5);
key.push(KIND_EXPORT);
key.extend_from_slice(&id.to_be_bytes());
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
subspace: SUBSPACE_HOLDS,
key,
})
}
+6 -6
View File
@@ -4,7 +4,8 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! Reports on their way to an outside archive (JR-7). Keys, after `J`:
//! Reports on their way to an outside archive (JR-7), in the journal's
//! subspace (`store::SUBSPACE_JOURNAL`). Keys:
//!
//! - `o` + the report's queue id: what goes into the built-in journal if
//! the archive never takes the report, as JSON. Cleared once it's
@@ -12,10 +13,10 @@
//! - `w` + journal id (u32): how often that journal's archive didn't take a
//! report, and the last time and reason, for the console's warning.
use super::{FEATURE, Json, entries::Entry};
use super::{Json, entries::Entry};
use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize};
use store::{
SUBSPACE_INBUXA, Serialize, Store, ValueKey,
SUBSPACE_JOURNAL, Serialize, Store, ValueKey,
write::{AnyClass, BatchBuilder, ValueClass},
};
use trc::AddContext;
@@ -44,12 +45,11 @@ pub struct Failures {
}
fn class(kind: u8, id: &[u8]) -> ValueClass {
let mut key = Vec::with_capacity(2 + id.len());
key.push(FEATURE);
let mut key = Vec::with_capacity(1 + id.len());
key.push(kind);
key.extend_from_slice(id);
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
subspace: SUBSPACE_JOURNAL,
key,
})
}
+11 -10
View File
@@ -4,7 +4,8 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! The built-in journal (JR-5, JR-6, JR-13). Keys, after `J`:
//! The built-in journal (JR-5, JR-6, JR-13), in a subspace of its own
//! (`store::SUBSPACE_JOURNAL`, store layout version 3). Keys:
//!
//! - `e` + node + seq: a chain link: its seq, the hash of the link before
//! it, and the SHA-256 of its entry. One chain per node, as the audit log
@@ -26,13 +27,13 @@
//! its entry is purged. Nothing here changes or removes an entry before
//! its time; nothing in JMAP can.
use super::{Direction, FEATURE, Json};
use super::{Direction, Json};
use crate::hold::HELD_UNTIL;
use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize};
use sha2::{Digest, Sha256};
use std::fmt;
use store::{
BlobStore, Deserialize, IterateParams, SUBSPACE_INBUXA, Serialize, Store, ValueKey,
BlobStore, Deserialize, IterateParams, SUBSPACE_JOURNAL, Serialize, Store, ValueKey,
write::{AnyClass, BatchBuilder, BlobLink, BlobOp, ValueClass, assert::AssertValue},
};
use tokio::sync::Mutex;
@@ -164,14 +165,13 @@ impl Deserialize for Raw {
}
fn class(kind: u8, parts: &[u64]) -> ValueClass {
let mut key = Vec::with_capacity(2 + parts.len() * 8);
key.push(FEATURE);
let mut key = Vec::with_capacity(1 + parts.len() * 8);
key.push(kind);
for part in parts {
key.extend_from_slice(&part.to_be_bytes());
}
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
subspace: SUBSPACE_JOURNAL,
key,
})
}
@@ -187,11 +187,11 @@ pub fn content_key(id: EntryId) -> ValueKey<ValueClass> {
/// The numbers after the kind byte, from the key's tail.
fn parse_key(key: &[u8], kind: u8, parts: usize) -> Option<Vec<u64>> {
let len = 2 + parts * 8;
let len = 1 + parts * 8;
let tail = key.get(key.len().checked_sub(len)?..)?;
(tail[0] == FEATURE && tail[1] == kind).then_some(())?;
(tail[0] == kind).then_some(())?;
Some(
tail[2..]
tail[1..]
.chunks_exact(8)
.map(|chunk| u64::from_be_bytes(chunk.try_into().unwrap()))
.collect(),
@@ -777,7 +777,8 @@ mod tests {
panic!()
};
assert_eq!(parse_key(&any.key, KIND_EXPIRY, 3), Some(vec![5, 3, 9]));
let mut with_subspace = vec![SUBSPACE_INBUXA];
assert_eq!(any.subspace, SUBSPACE_JOURNAL);
let mut with_subspace = vec![SUBSPACE_JOURNAL];
with_subspace.extend_from_slice(&any.key);
assert_eq!(
parse_key(&with_subspace, KIND_EXPIRY, 3),
+5 -3
View File
@@ -12,9 +12,11 @@
//! - [`report`]: the journal report around the untouched message (JR-3);
//! - [`entries`]: the built-in journal and its chain (JR-5, JR-6, JR-13).
//!
//! Kept in the fork's subspace (`store::SUBSPACE_INBUXA`). Every key starts
//! with `J`; journals are `j` + id (u32), as JSON. There are few, so they're
//! read whole.
//! Journals are settings, so they're kept in the fork's subspace
//! (`store::SUBSPACE_INBUXA`), where a configuration export carries them:
//! `J` + `j` + id (u32), as JSON. There are few, so they're read whole. What
//! they journal has a subspace of its own (`store::SUBSPACE_JOURNAL`; see
//! [`entries`] and [`archive`]).
pub mod archive;
pub mod entries;
+12 -14
View File
@@ -4,8 +4,9 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! What undelete keeps in the fork's own subspace (`store::SUBSPACE_INBUXA`).
//! Every key starts with `U`, then one byte for the kind:
//! What undelete keeps, in a subspace of its own (`store::SUBSPACE_UNDELETE`,
//! store layout version 3; before it, in `_` under `U`). Every key starts
//! with one byte for the kind:
//!
//! - `n` + account + document: a deleted email waiting for its archive
//! record, with what only the deletion knows (mailboxes, keywords, size)
@@ -19,18 +20,19 @@
//! `/changes`.
//! - `k` + account id: a deleted account kept for its period (UD-15a).
//! - `a` + address: an address a kept account holds reserved (UD-16).
//! - `g` + kind + account + document: a deleted calendar event, contact or
//! file ([`super::groupware`]).
//!
//! Values are JSON, so they read back across versions of the fork.
use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize, de::DeserializeOwned};
use store::{
Deserialize, IterateParams, SUBSPACE_INBUXA, Serialize, Store, ValueKey,
Deserialize, IterateParams, SUBSPACE_UNDELETE, Serialize, Store, ValueKey,
write::{AnyClass, BatchBuilder, ValueClass},
};
use trc::AddContext;
use types::id::Id;
const FEATURE: u8 = b'U';
const KIND_NOTE: u8 = b'n';
const KIND_EXTRA: u8 = b'x';
const KIND_BLOB: u8 = b'b';
@@ -40,12 +42,11 @@ const KIND_KEPT: u8 = b'k';
const KIND_RESERVED: u8 = b'a';
fn class(kind: u8, rest: &[u8]) -> ValueClass {
let mut key = Vec::with_capacity(2 + rest.len());
key.push(FEATURE);
let mut key = Vec::with_capacity(1 + rest.len());
key.push(kind);
key.extend_from_slice(rest);
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
subspace: SUBSPACE_UNDELETE,
key,
})
}
@@ -416,7 +417,7 @@ pub async fn clear_account(data: &Store, account_id: u32) -> trc::Result<()> {
(vec![KIND_BLOB], vec![KIND_BLOB]),
(vec![KIND_CHANGE], vec![KIND_CHANGE]),
];
// The groupware notes, `Ug` + kind + account + document
// The groupware notes, `g` + kind + account + document
for kind in 0u8..3 {
ranges.push((vec![b'g', kind], vec![b'g', kind]));
}
@@ -433,14 +434,11 @@ pub async fn clear_account(data: &Store, account_id: u32) -> trc::Result<()> {
Ok(())
}
/// A key under `U` spelled out in full (the groupware notes, `Ug`).
/// A key spelled out in full (the groupware notes, `g`).
fn class_raw(rest: &[u8]) -> ValueClass {
let mut key = Vec::with_capacity(1 + rest.len());
key.push(FEATURE);
key.extend_from_slice(rest);
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key,
subspace: SUBSPACE_UNDELETE,
key: rest.to_vec(),
})
}
+3 -3
View File
@@ -50,13 +50,13 @@ pub struct Note {
}
fn note_class(kind: Kind, account_id: u32, document_id: u32) -> ValueClass {
let mut key = Vec::with_capacity(11);
key.extend_from_slice(b"Ug");
let mut key = Vec::with_capacity(10);
key.push(b'g');
key.push(kind as u8);
key.extend_from_slice(&account_id.to_be_bytes());
key.extend_from_slice(&document_id.to_be_bytes());
ValueClass::Any(AnyClass {
subspace: store::SUBSPACE_INBUXA,
subspace: store::SUBSPACE_UNDELETE,
key,
})
}
+547
View File
@@ -0,0 +1,547 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! inbuxa: store layout step 2 -> 3 (store-layout spec SL-16, SL-17, SL-20).
//! The audit log, the journal, holds and undelete notes move out of the
//! fork's `_` subspace into subspaces of their own, each key without the
//! letter that named its feature. Their settings stay in `_`, where a
//! configuration export carries them. RocksDB's legacy `v` and `c` go.
//!
//! Every key in `_` belongs to a feature, by its first byte, listed in
//! [`OWNERS`]. A key nobody claims stops the step before it changes
//! anything: it would mean a feature this step doesn't know about.
use store::{
IterateParams, SUBSPACE_AUDIT, SUBSPACE_HOLDS, SUBSPACE_INBUXA, SUBSPACE_JOURNAL,
SUBSPACE_UNDELETE, Store, ValueKey,
write::{AnyClass, AnyKey, BatchBuilder, ValueClass},
};
use trc::AddContext;
/// Set once every copy has been checked, before the first original is
/// deleted, so a run that stops while deleting doesn't check again against
/// originals it already deleted. Cleared at the end.
pub const CHECKED_KEY: &[u8] = b"Vt";
/// Rows copied at a time where the backend can't copy on the server.
const COPY_CHUNK: usize = 10_000;
/// What happens to the keys under one first byte of `_`.
pub enum Owner {
/// They stay in `_`.
Stays(&'static str),
/// They move to `to`, without the first byte. `kinds` are the second
/// bytes that move and `stays` those that stay; any other stops the step.
Moves {
what: &'static str,
to: u8,
kinds: &'static [u8],
stays: &'static [u8],
/// Kinds whose values are compared row for row after the copy, not
/// only counted: the chain heads and floors.
compared: &'static [u8],
},
}
/// SL-16: every first byte in use in `_`, and its owner.
pub const OWNERS: &[(u8, Owner)] = &[
(b'A', Owner::Stays("AI limits")),
(b'B', Owner::Stays("first boot defaults")),
(b'C', Owner::Stays("account snapshots")),
(b'D', Owner::Stays("deliverability reports")),
(b'E', Owner::Stays("inbuxa relay")),
(b'F', Owner::Stays("front ends")),
(b'G', Owner::Stays("scheduled reports")),
(
b'H',
Owner::Moves {
what: "legal holds",
to: SUBSPACE_HOLDS,
kinds: b"hoe",
stays: b"",
compared: b"h",
},
),
(b'I', Owner::Stays("migrations")),
(
b'J',
Owner::Moves {
what: "journal",
to: SUBSPACE_JOURNAL,
kinds: b"ecptxhfow",
stays: b"j",
compared: b"hf",
},
),
(b'K', Owner::Stays("account locks")),
(
b'L',
Owner::Moves {
what: "audit log",
to: SUBSPACE_AUDIT,
kinds: b"etohf",
stays: b"s",
compared: b"hf",
},
),
(b'M', Owner::Stays("masked email")),
(b'P', Owner::Stays("permissions and protocol policies")),
(b'Q', Owner::Stays("acceptance records")),
(b'R', Owner::Stays("mail flow rules")),
(
b'S',
Owner::Stays("spam rules, scale-out storage and earlier scheduled reports"),
),
(b'T', Owner::Stays("log files")),
(
b'U',
Owner::Moves {
what: "undelete notes",
to: SUBSPACE_UNDELETE,
kinds: b"nxbrckag",
stays: b"",
compared: b"",
},
),
(b'V', Owner::Stays("store checks and the store layout")),
(
b'W',
Owner::Stays("blob store changes and the sharing policy"),
),
];
/// Step 2 -> 3. Returns how many rows moved.
pub async fn fork_data_own_subspaces(store: &Store) -> trc::Result<u64> {
let moving = moving_prefixes(store).await?;
// Always copy, even after a run that stopped while deleting: whatever
// is left of the originals is copied again before it goes
for prefix in &moving {
copy(store, prefix).await?;
}
let checked = store
.key_exists(inbuxa_key(CHECKED_KEY))
.await
.caused_by(trc::location!())?;
let mut moved = 0;
if !checked {
for prefix in &moving {
moved += check(store, prefix).await?;
}
let mut batch = BatchBuilder::new();
batch.set(inbuxa_class(CHECKED_KEY), vec![1]);
store
.write(batch.build_all())
.await
.caused_by(trc::location!())?;
}
for prefix in &moving {
store
.delete_range(
AnyKey {
subspace: SUBSPACE_INBUXA,
key: prefix.from.clone(),
},
AnyKey {
subspace: SUBSPACE_INBUXA,
key: prefix.to.clone(),
},
)
.await
.caused_by(trc::location!())?;
}
let dropped = store
.drop_legacy_subspaces()
.await
.caused_by(trc::location!())?;
if dropped > 0 {
trc::event!(
Server(trc::ServerEvent::Startup),
Details = "Dropped the legacy v and c column families",
Total = dropped,
);
}
let mut batch = BatchBuilder::new();
batch.clear(inbuxa_class(CHECKED_KEY));
store
.write(batch.build_all())
.await
.caused_by(trc::location!())?;
Ok(moved)
}
/// One second byte under a moving feature: its range in `_` and where it
/// goes.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Prefix {
what: &'static str,
/// The two bytes in `_`.
from: Vec<u8>,
/// The first key after them in `_`.
to: Vec<u8>,
dest: u8,
compared: bool,
}
/// SL-16: lists the two-byte prefixes in `_` by skipping from one to the
/// next, so it reads one key per prefix, not every key. Stops on any key
/// nobody claims, before anything is written.
pub async fn moving_prefixes(store: &Store) -> trc::Result<Vec<Prefix>> {
let mut begin = Vec::new();
let mut unclaimed = Vec::new();
let mut moving = Vec::new();
loop {
let mut first = None;
store
.iterate(
IterateParams::new(inbuxa_key(&begin), inbuxa_key(&[u8::MAX; 64]))
.no_values()
.only_first(),
|key, _| {
first = Some(key.to_vec());
Ok(false)
},
)
.await
.caused_by(trc::location!())?;
let Some(key) = first else {
break;
};
let prefix = &key[..key.len().min(2)];
match claim(prefix) {
Claim::Stays => {}
Claim::Moves(found) => moving.push(found),
Claim::Nobody => unclaimed.push(printable(prefix)),
}
// A one-byte key is followed by the longer keys under it
begin = if key.len() == 1 {
vec![key[0], 0]
} else {
match next_prefix(prefix) {
Some(next) => next,
None => break,
}
};
}
if unclaimed.is_empty() {
Ok(moving)
} else {
Err(trc::StoreEvent::UnexpectedError
.into_err()
.details(format!(
"Store layout step 2 to 3 found keys in the _ subspace that no feature claims ({}). Nothing was changed",
unclaimed.join(", ")
))
.caused_by(trc::location!()))
}
}
enum Claim {
Stays,
Moves(Prefix),
Nobody,
}
fn claim(prefix: &[u8]) -> Claim {
let Some((_, owner)) = OWNERS.iter().find(|(byte, _)| Some(byte) == prefix.first()) else {
return Claim::Nobody;
};
match owner {
Owner::Stays(_) => Claim::Stays,
Owner::Moves {
what,
to,
kinds,
stays,
compared,
} => match prefix.get(1) {
Some(kind) if stays.contains(kind) => Claim::Stays,
Some(kind) if kinds.contains(kind) => Claim::Moves(Prefix {
what,
from: prefix.to_vec(),
to: next_prefix(prefix).unwrap_or_else(|| vec![u8::MAX; 64]),
dest: *to,
compared: compared.contains(kind),
}),
_ => Claim::Nobody,
},
}
}
/// The first byte string after every key that starts with `prefix`.
fn next_prefix(prefix: &[u8]) -> Option<Vec<u8>> {
let mut next = prefix.to_vec();
while let Some(last) = next.pop() {
if last < u8::MAX {
next.push(last + 1);
return Some(next);
}
}
None
}
fn printable(prefix: &[u8]) -> String {
prefix
.iter()
.map(|&byte| {
if byte.is_ascii_graphic() {
(byte as char).to_string()
} else {
format!("\\x{byte:02x}")
}
})
.collect()
}
/// SL-17: on SQL in one statement on the server, elsewhere a row at a
/// time. A row already copied is written again, unchanged.
async fn copy(store: &Store, prefix: &Prefix) -> trc::Result<()> {
let copied = store
.copy_range(
AnyKey {
subspace: SUBSPACE_INBUXA,
key: prefix.from.clone(),
},
AnyKey {
subspace: SUBSPACE_INBUXA,
key: prefix.to.clone(),
},
prefix.dest,
1,
)
.await
.caused_by(trc::location!())?;
if copied.is_some() {
return Ok(());
}
// A chunk at a time, so a large journal isn't held in memory
let mut begin = prefix.from.clone();
loop {
let mut rows = Vec::new();
store
.iterate(
IterateParams::new(inbuxa_key(&begin), inbuxa_key(&prefix.to)),
|key, value| {
if key.starts_with(&prefix.from) {
rows.push((key.to_vec(), value.to_vec()));
}
Ok(rows.len() < COPY_CHUNK)
},
)
.await
.caused_by(trc::location!())?;
let Some((last, _)) = rows.last() else {
return Ok(());
};
let full = rows.len() == COPY_CHUNK;
// The next chunk starts right after the last key of this one
begin = last.clone();
begin.push(0);
let mut batch = BatchBuilder::new();
for (key, value) in rows {
batch.set(
ValueClass::Any(AnyClass {
subspace: prefix.dest,
key: key[1..].to_vec(),
}),
value,
);
if batch.is_large_batch() {
store
.write(batch.build_all())
.await
.caused_by(trc::location!())?;
batch = BatchBuilder::new();
}
}
if !batch.is_empty() {
store
.write(batch.build_all())
.await
.caused_by(trc::location!())?;
}
if !full {
return Ok(());
}
}
}
/// SL-17: the copy has as many rows as the originals, and the chain heads
/// and floors are the same, byte for byte. Returns the rows checked.
async fn check(store: &Store, prefix: &Prefix) -> trc::Result<u64> {
let dest_from = prefix.from[1..].to_vec();
let dest_to = next_prefix(&dest_from).unwrap_or_else(|| vec![u8::MAX; 64]);
let originals = rows(
store,
SUBSPACE_INBUXA,
&prefix.from,
&prefix.to,
prefix.compared,
)
.await?;
let copies = rows(store, prefix.dest, &dest_from, &dest_to, prefix.compared).await?;
let matches = if prefix.compared {
originals.values == copies.values
} else {
originals.count == copies.count
};
if !matches {
return Err(trc::StoreEvent::UnexpectedError
.into_err()
.details(format!(
"Store layout step 2 to 3: the copy of the {} ({}) doesn't match its originals: {} rows were copied of {}. Nothing was deleted",
prefix.what,
printable(&prefix.from),
copies.count,
originals.count
))
.caused_by(trc::location!()));
}
trc::event!(
Server(trc::ServerEvent::Startup),
Details = format!(
"Copied the {} ({}) to its own subspace",
prefix.what,
printable(&prefix.from)
),
Total = originals.count,
);
Ok(originals.count)
}
struct Rows {
count: u64,
/// The keys as they are in the feature's own subspace, when compared.
values: Vec<(Vec<u8>, Vec<u8>)>,
}
async fn rows(
store: &Store,
subspace: u8,
from: &[u8],
to: &[u8],
with_values: bool,
) -> trc::Result<Rows> {
let mut rows = Rows {
count: 0,
values: Vec::new(),
};
// In `_`, the feature's letter comes first
let skip = usize::from(subspace == SUBSPACE_INBUXA);
store
.iterate(
IterateParams::new(
AnyKey {
subspace,
key: from.to_vec(),
},
AnyKey {
subspace,
key: to.to_vec(),
},
)
.set_values(with_values),
|key, value| {
if key.starts_with(from) {
rows.count += 1;
if with_values {
rows.values.push((key[skip..].to_vec(), value.to_vec()));
}
}
Ok(true)
},
)
.await
.caused_by(trc::location!())?;
Ok(rows)
}
fn inbuxa_key(key: &[u8]) -> ValueKey<ValueClass> {
ValueKey::from(inbuxa_class(key))
}
fn inbuxa_class(key: &[u8]) -> ValueClass {
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key: key.to_vec(),
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn every_owner_is_listed_once() {
for (i, (byte, _)) in OWNERS.iter().enumerate() {
assert!(
OWNERS[i + 1..].iter().all(|(other, _)| other != byte),
"{} is listed twice",
*byte as char
);
}
}
#[test]
fn claims() {
assert!(matches!(claim(b"Vs"), Claim::Stays));
assert!(matches!(claim(b"Ls"), Claim::Stays));
assert!(matches!(claim(b"Jj"), Claim::Stays));
assert!(matches!(claim(b"Lz"), Claim::Nobody));
assert!(matches!(claim(b"L"), Claim::Nobody));
assert!(matches!(claim(b"Zz"), Claim::Nobody));
assert!(matches!(claim(b"\x00"), Claim::Nobody));
let Claim::Moves(prefix) = claim(b"Le") else {
panic!()
};
assert_eq!(prefix.dest, SUBSPACE_AUDIT);
assert_eq!(prefix.to, b"Lf");
assert!(!prefix.compared);
let Claim::Moves(prefix) = claim(b"Jh") else {
panic!()
};
assert_eq!(prefix.dest, SUBSPACE_JOURNAL);
assert!(prefix.compared);
let Claim::Moves(prefix) = claim(b"Ug") else {
panic!()
};
assert_eq!(prefix.dest, SUBSPACE_UNDELETE);
}
// The two-byte prefixes on prod's `_` on 2026-11-11, all claimed
#[test]
fn prod_prefixes_are_claimed() {
for prefix in [
"Al", "Ci", "Dr", "Gr", "Gs", "Kd", "Kl", "Le", "Lh", "Lo", "Lt", "Mi", "Pc", "Pg",
"Pu", "Sf", "Sr", "Vs", "Wl", "Wn",
] {
assert!(
!matches!(claim(prefix.as_bytes()), Claim::Nobody),
"{prefix}"
);
}
}
#[test]
fn next_prefixes() {
assert_eq!(next_prefix(b"Le"), Some(b"Lf".to_vec()));
assert_eq!(next_prefix(&[b'L', 0xff]), Some(b"M".to_vec()));
assert_eq!(next_prefix(&[0xff, 0xff]), None);
}
}
+16 -5
View File
@@ -46,15 +46,26 @@ struct Step {
what: &'static str,
}
const STEPS: &[Step] = &[Step {
from: 1,
rolling_safe: true,
what: "in-memory counters on SQL keep their own expiry",
}];
const STEPS: &[Step] = &[
Step {
from: 1,
rolling_safe: true,
what: "in-memory counters on SQL keep their own expiry",
},
Step {
from: 2,
// SL-18: old nodes would keep writing the moved data to `_`, and an
// old binary can't open a RocksDB store with column families it
// doesn't know
rolling_safe: false,
what: "the audit log, journal, holds and undelete notes move to subspaces of their own",
},
];
async fn run_step(server: &Server, from: u32) -> trc::Result<u64> {
match from {
1 => counters_keep_expiry(server).await,
2 => crate::fork_subspaces::fork_data_own_subspaces(server.store()).await,
_ => unreachable!("no store layout step from version {from}"),
}
}
+1
View File
@@ -18,6 +18,7 @@ use store::{
use trc::AddContext;
pub mod destroy;
pub mod fork_subspaces;
pub mod layout;
pub mod v016;
+10 -5
View File
@@ -113,6 +113,11 @@ impl MysqlStore {
SUBSPACE_DELETED_ITEMS,
SUBSPACE_SPAM_SAMPLES,
crate::SUBSPACE_INBUXA, // inbuxa: masked email
// inbuxa: SL-16: the fork's large data, from layout version 3
crate::SUBSPACE_AUDIT,
crate::SUBSPACE_JOURNAL,
crate::SUBSPACE_HOLDS,
crate::SUBSPACE_UNDELETE,
SUBSPACE_BLOB_LINK,
SUBSPACE_IN_MEMORY_VALUE,
SUBSPACE_PROPERTY,
@@ -127,7 +132,7 @@ impl MysqlStore {
SUBSPACE_TELEMETRY_SPAN,
SUBSPACE_TELEMETRY_METRIC,
] {
let table = char::from(table);
let table = crate::subspace_name(table);
conn.query_drop(format!(
"CREATE TABLE IF NOT EXISTS {table} (
k VARBINARY(255) NOT NULL,
@@ -145,13 +150,13 @@ impl MysqlStore {
v LONGBLOB NOT NULL,
PRIMARY KEY (k)
) ENGINE=InnoDB",
char::from(SUBSPACE_BLOBS),
crate::subspace_name(SUBSPACE_BLOBS),
))
.await
.map_err(into_error)?;
for table in [SUBSPACE_INDEXES, SUBSPACE_REGISTRY_IDX] {
let table = char::from(table);
let table = crate::subspace_name(table);
conn.query_drop(format!(
"CREATE TABLE IF NOT EXISTS {table} (
k BLOB,
@@ -169,7 +174,7 @@ impl MysqlStore {
v BIGINT NOT NULL DEFAULT 0,
PRIMARY KEY (k)
) ENGINE=InnoDB",
char::from(table)
crate::subspace_name(table)
))
.await
.map_err(into_error)?;
@@ -179,7 +184,7 @@ impl MysqlStore {
// row. MySQL has no ADD COLUMN IF NOT EXISTS, so look first, and
// take "duplicate column" from a node starting at the same time
// as done.
let table = char::from(SUBSPACE_IN_MEMORY_COUNTER).to_string();
let table = crate::subspace_name(SUBSPACE_IN_MEMORY_COUNTER).to_string();
let has_expiry = conn
.exec_first::<u8, _, _>(
concat!(
+4 -4
View File
@@ -22,7 +22,7 @@ impl MysqlStore {
let s = conn
.prep(format!(
"SELECT v FROM {} WHERE k = ?",
char::from(key.subspace())
crate::subspace_name(key.subspace())
))
.await
.map_err(into_error)?;
@@ -49,7 +49,7 @@ impl MysqlStore {
let s = conn
.prep(format!(
"SELECT 1 FROM {} WHERE k = ?",
char::from(key.subspace())
crate::subspace_name(key.subspace())
))
.await
.map_err(into_error)?;
@@ -69,7 +69,7 @@ impl MysqlStore {
mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result<bool> + Sync + Send,
) -> trc::Result<()> {
let mut conn = self.conn().await?;
let table = char::from(params.begin.subspace());
let table = crate::subspace_name(params.begin.subspace());
let begin = params.begin.serialize(0);
let end = params.end.serialize(0);
let keys = if params.values { "k, v" } else { "k" };
@@ -193,7 +193,7 @@ impl MysqlStore {
key: impl Into<ValueKey<ValueClass>> + Sync + Send,
) -> trc::Result<i64> {
let key = key.into();
let table = char::from(key.subspace());
let table = crate::subspace_name(key.subspace());
let key = key.serialize(0);
let mut conn = self.conn().await?;
let limit = self.timeouts.query;
+38 -6
View File
@@ -137,7 +137,7 @@ impl MysqlStore {
Operation::Value { class, op } => {
let key = class.serialize(account_id, collection, document_id, 0);
let subspace = class.subspace(collection);
let table = char::from(subspace);
let table = crate::subspace_name(subspace);
match op {
ValueOp::Set(value) => {
@@ -393,7 +393,7 @@ impl MysqlStore {
assert_value,
} => {
let key = class.serialize(account_id, collection, document_id, 0);
let table = char::from(class.subspace(collection));
let table = crate::subspace_name(class.subspace(collection));
let s = trx
.prep(format!("SELECT v FROM {} WHERE k = ? FOR UPDATE", table))
@@ -423,7 +423,7 @@ impl MysqlStore {
let limit = self.timeouts.maintenance;
let result = tokio::time::timeout(limit, async {
for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] {
purge_table(&mut conn, char::from(subspace)).await?;
purge_table(&mut conn, crate::subspace_name(subspace)).await?;
}
Ok(())
@@ -440,7 +440,7 @@ impl MysqlStore {
conn.exec_drop(
format!(
"DELETE FROM {} WHERE e <= ?",
char::from(SUBSPACE_IN_MEMORY_COUNTER)
crate::subspace_name(SUBSPACE_IN_MEMORY_COUNTER)
),
(now as i64,),
)
@@ -452,11 +452,43 @@ impl MysqlStore {
bounded(conn, result, limit)
}
/// inbuxa: SL-17: copies the rows from `from` up to `to` into subspace
/// `dest`, without their first `strip` bytes, in one statement on the
/// server. A row already there is overwritten, so a copy can run again.
pub(crate) async fn copy_range(
&self,
from: impl Key,
to: impl Key,
dest: u8,
strip: usize,
) -> trc::Result<u64> {
let mut conn = self.conn().await?;
let limit = self.timeouts.maintenance;
let result = tokio::time::timeout(limit, async {
let table = crate::subspace_name(from.subspace());
let dest = crate::subspace_name(dest);
let from = from.serialize(0);
let to = to.serialize(0);
let s = conn
.prep(format!(
"INSERT INTO {dest} (k, v) SELECT SUBSTRING(k, {}), v FROM {table} \
WHERE k >= ? AND k < ? ON DUPLICATE KEY UPDATE v = VALUES(v)",
strip + 1
))
.await
.map_err(into_error)?;
conn.exec_drop(&s, (&from, &to)).await.map_err(into_error)?;
Ok(conn.affected_rows())
})
.await;
bounded(conn, result, limit)
}
pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> {
let mut conn = self.conn().await?;
let limit = self.timeouts.maintenance;
let result = tokio::time::timeout(limit, async {
let table = char::from(from.subspace());
let table = crate::subspace_name(from.subspace());
let mut from = from.serialize(0);
let to = to.serialize(0);
@@ -518,7 +550,7 @@ impl MysqlStore {
}
}
async fn purge_table(conn: &mut Conn, table: char) -> trc::Result<()> {
async fn purge_table(conn: &mut Conn, table: &str) -> trc::Result<()> {
let s = conn
.prep(format!("DELETE FROM {table} WHERE v = 0"))
.await
+9 -4
View File
@@ -161,6 +161,11 @@ impl PostgresStore {
SUBSPACE_DELETED_ITEMS,
SUBSPACE_SPAM_SAMPLES,
crate::SUBSPACE_INBUXA, // inbuxa: masked email
// inbuxa: SL-16: the fork's large data, from layout version 3
crate::SUBSPACE_AUDIT,
crate::SUBSPACE_JOURNAL,
crate::SUBSPACE_HOLDS,
crate::SUBSPACE_UNDELETE,
SUBSPACE_BLOB_LINK,
SUBSPACE_IN_MEMORY_VALUE,
SUBSPACE_PROPERTY,
@@ -176,7 +181,7 @@ impl PostgresStore {
SUBSPACE_TELEMETRY_SPAN,
SUBSPACE_TELEMETRY_METRIC,
] {
let table = char::from(table);
let table = crate::subspace_name(table);
conn.execute(
&format!(
"CREATE TABLE IF NOT EXISTS {table} (
@@ -191,7 +196,7 @@ impl PostgresStore {
}
for table in [SUBSPACE_INDEXES, SUBSPACE_REGISTRY_IDX] {
let table = char::from(table);
let table = crate::subspace_name(table);
conn.execute(
&format!(
"CREATE TABLE IF NOT EXISTS {table} (
@@ -211,7 +216,7 @@ impl PostgresStore {
k BYTEA PRIMARY KEY,
v BIGINT NOT NULL DEFAULT 0
)",
char::from(table)
crate::subspace_name(table)
),
&[],
)
@@ -221,7 +226,7 @@ impl PostgresStore {
// inbuxa: SL-10: in-memory counters keep their expiry in their
// row. Added only when missing, so a start doesn't lock the table.
let table = char::from(SUBSPACE_IN_MEMORY_COUNTER).to_string();
let table = crate::subspace_name(SUBSPACE_IN_MEMORY_COUNTER).to_string();
let has_expiry = conn
.query_opt(
concat!(
+5 -5
View File
@@ -25,7 +25,7 @@ impl PostgresStore {
let s = conn
.prepare_cached(&format!(
"SELECT v FROM {} WHERE k = $1",
char::from(key.subspace())
crate::subspace_name(key.subspace())
))
.await
.map_err(into_error)?;
@@ -61,7 +61,7 @@ impl PostgresStore {
let s = conn
.prepare_cached(&format!(
"SELECT k, v FROM {} WHERE k = ANY($1)",
char::from(subspace)
crate::subspace_name(subspace)
))
.await
.map_err(into_error)?;
@@ -90,7 +90,7 @@ impl PostgresStore {
let s = conn
.prepare_cached(&format!(
"SELECT 1 FROM {} WHERE k = $1",
char::from(key.subspace())
crate::subspace_name(key.subspace())
))
.await
.map_err(into_error)?;
@@ -110,7 +110,7 @@ impl PostgresStore {
mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result<bool> + Sync + Send,
) -> trc::Result<()> {
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
let table = char::from(params.begin.subspace());
let table = crate::subspace_name(params.begin.subspace());
let begin = params.begin.serialize(0);
let end = params.end.serialize(0);
let keys = if params.values { "k, v" } else { "k" };
@@ -230,7 +230,7 @@ impl PostgresStore {
key: impl Into<ValueKey<ValueClass>> + Sync + Send,
) -> trc::Result<i64> {
let key = key.into();
let table = char::from(key.subspace());
let table = crate::subspace_name(key.subspace());
let key = key.serialize(0);
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
+37 -6
View File
@@ -236,7 +236,7 @@ impl PostgresStore {
Operation::Value { class, op } => {
let key = class.serialize(account_id, collection, document_id, 0);
let subspace = class.subspace(collection);
let table = char::from(subspace);
let table = crate::subspace_name(subspace);
match op {
ValueOp::Set(value) => {
@@ -491,7 +491,7 @@ impl PostgresStore {
} => {
pipeline.flush().await?;
let key = class.serialize(account_id, collection, document_id, 0);
let table = char::from(class.subspace(collection));
let table = crate::subspace_name(class.subspace(collection));
let s = trx
.prepare_cached(&format!("SELECT v FROM {} WHERE k = $1 FOR UPDATE", table))
@@ -526,7 +526,7 @@ impl PostgresStore {
let limit = self.timeouts.maintenance;
let result = tokio::time::timeout(limit, async {
for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] {
purge_table(&conn, char::from(subspace)).await?;
purge_table(&conn, crate::subspace_name(subspace)).await?;
}
Ok(())
@@ -543,7 +543,7 @@ impl PostgresStore {
let s = conn
.prepare_cached(&format!(
"DELETE FROM {} WHERE e <= $1",
char::from(SUBSPACE_IN_MEMORY_COUNTER)
crate::subspace_name(SUBSPACE_IN_MEMORY_COUNTER)
))
.await
.map_err(into_error)?;
@@ -553,11 +553,42 @@ impl PostgresStore {
bounded(conn, result, limit)
}
/// inbuxa: SL-17: copies the rows from `from` up to `to` into subspace
/// `dest`, without their first `strip` bytes, in one statement on the
/// server. A row already there is overwritten, so a copy can run again.
pub(crate) async fn copy_range(
&self,
from: impl Key,
to: impl Key,
dest: u8,
strip: usize,
) -> trc::Result<u64> {
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
let limit = self.timeouts.maintenance;
let result = tokio::time::timeout(limit, async {
let table = crate::subspace_name(from.subspace());
let dest = crate::subspace_name(dest);
let from = from.serialize(0);
let to = to.serialize(0);
let s = conn
.prepare_cached(&format!(
"INSERT INTO {dest} (k, v) SELECT substring(k from {}), v FROM {table} \
WHERE k >= $1 AND k < $2 ON CONFLICT (k) DO UPDATE SET v = EXCLUDED.v",
strip + 1
))
.await
.map_err(into_error)?;
conn.execute(&s, &[&from, &to]).await.map_err(into_error)
})
.await;
bounded(conn, result, limit)
}
pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> {
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
let limit = self.timeouts.maintenance;
let result = tokio::time::timeout(limit, async {
let table = char::from(from.subspace());
let table = crate::subspace_name(from.subspace());
let mut from = from.serialize(0);
let to = to.serialize(0);
@@ -619,7 +650,7 @@ impl PostgresStore {
}
}
async fn purge_table(conn: &Object, table: char) -> trc::Result<()> {
async fn purge_table(conn: &Object, table: &str) -> trc::Result<()> {
let s = conn
.prepare_cached(&format!("DELETE FROM {table} WHERE v = 0"))
.await
+84 -4
View File
@@ -65,7 +65,7 @@ impl RocksDbStore {
// Counters
for subspace in [SUBSPACE_COUNTER, SUBSPACE_QUOTA, SUBSPACE_IN_MEMORY_COUNTER] {
cfs.push(ColumnFamilyDescriptor::new(
std::str::from_utf8(&[subspace]).unwrap(),
crate::subspace_name(subspace),
cf_options(CfProfile::Counter, &cache, write_buffer_size),
));
}
@@ -98,18 +98,35 @@ impl RocksDbStore {
(SUBSPACE_SEARCH_INDEX, CfProfile::Scan),
(SUBSPACE_SPAM_SAMPLES, CfProfile::Churn),
(crate::SUBSPACE_INBUXA, CfProfile::PointLookup), // inbuxa: masked email
// inbuxa: SL-16: the fork's large data, from layout version 3
(crate::SUBSPACE_AUDIT, CfProfile::Scan),
(crate::SUBSPACE_JOURNAL, CfProfile::Scan),
(crate::SUBSPACE_HOLDS, CfProfile::PointLookup),
(crate::SUBSPACE_UNDELETE, CfProfile::Churn),
(SUBSPACE_REGISTRY_IDX, CfProfile::Scan),
(SUBSPACE_REGISTRY_PK, CfProfile::PointLookup),
(SUBSPACE_DIRECTORY, CfProfile::PointLookup),
(LEGACY_SUBSPACE_BITMAP_TEXT, CfProfile::Scan),
(LEGACY_SUBSPACE_BITMAP_TAG, CfProfile::Scan),
] {
cfs.push(ColumnFamilyDescriptor::new(
std::str::from_utf8(&[subspace]).unwrap(),
crate::subspace_name(subspace),
cf_options(profile, &cache, write_buffer_size),
));
}
// inbuxa: SL-20: the legacy `v` and `c` are opened only where they
// still exist, for store layout step 2 -> 3 to drop. RocksDB won't
// open a database without every column family it has.
let existing = rocksdb::DB::list_cf(&Options::default(), &idx_path).unwrap_or_default();
for subspace in [LEGACY_SUBSPACE_BITMAP_TEXT, LEGACY_SUBSPACE_BITMAP_TAG] {
let name = crate::subspace_name(subspace);
if existing.iter().any(|cf| cf == name) {
cfs.push(ColumnFamilyDescriptor::new(
name,
cf_options(CfProfile::Scan, &cache, write_buffer_size),
));
}
}
let mut db_opts = Options::default();
db_opts.create_missing_column_families(true);
db_opts.create_if_missing(true);
@@ -140,6 +157,24 @@ impl RocksDbStore {
})))
}
/// inbuxa: SL-20: drops the legacy `v` and `c` column families where
/// they exist, and returns how many it dropped.
pub(crate) async fn drop_legacy_subspaces(&self) -> trc::Result<u64> {
let db = self.db.clone();
self.spawn_worker(move || {
let mut dropped = 0;
for subspace in [LEGACY_SUBSPACE_BITMAP_TEXT, LEGACY_SUBSPACE_BITMAP_TAG] {
let name = crate::subspace_name(subspace);
if db.cf_handle(name).is_some() {
db.drop_cf(name).map_err(super::into_error)?;
dropped += 1;
}
}
Ok(dropped)
})
.await
}
pub async fn spawn_worker<U, V>(&self, mut f: U) -> trc::Result<V>
where
U: FnMut() -> trc::Result<V> + Send,
@@ -234,3 +269,48 @@ fn cf_options(profile: CfProfile, cache: &Cache, write_buffer_size: usize) -> Op
opts
}
#[cfg(test)]
mod tests {
use super::*;
// inbuxa: SL-20: a store from before step 2 -> 3 has `v` and `c`. It
// still opens, the step drops them, and it opens again without them.
#[tokio::test]
async fn legacy_column_families_are_dropped() {
let dir = std::env::temp_dir().join(format!("inbuxa-legacy-cf-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
{
let mut opts = Options::default();
opts.create_if_missing(true);
opts.create_missing_column_families(true);
rocksdb::DB::open_cf(&opts, &dir, ["v", "c", "p"]).unwrap();
}
let config = || structs::RocksDbStore {
path: dir.to_string_lossy().into_owned(),
..Default::default()
};
let has = |name: &str| {
rocksdb::DB::list_cf(&Options::default(), &dir)
.unwrap()
.iter()
.any(|cf| cf == name)
};
let Store::RocksDb(store) = RocksDbStore::open(config()).await.unwrap() else {
unreachable!()
};
assert_eq!(store.drop_legacy_subspaces().await.unwrap(), 2);
assert_eq!(store.drop_legacy_subspaces().await.unwrap(), 0);
drop(store);
assert!(!has("v") && !has("c"));
assert!(has(crate::subspace_name(crate::SUBSPACE_AUDIT)));
let Store::RocksDb(store) = RocksDbStore::open(config()).await.unwrap() else {
unreachable!()
};
drop(store);
assert!(!has("v") && !has("c"));
let _ = std::fs::remove_dir_all(&dir);
}
}
+4 -6
View File
@@ -18,9 +18,9 @@ pub mod main;
pub mod read;
pub mod write;
static CF_LOGS: &str = unsafe { std::str::from_utf8_unchecked(&[SUBSPACE_LOGS]) };
static CF_INDEXES: &str = unsafe { std::str::from_utf8_unchecked(&[SUBSPACE_INDEXES]) };
static CF_BLOBS: &str = unsafe { std::str::from_utf8_unchecked(&[SUBSPACE_BLOBS]) };
static CF_LOGS: &str = crate::subspace_name(SUBSPACE_LOGS);
static CF_INDEXES: &str = crate::subspace_name(SUBSPACE_INDEXES);
static CF_BLOBS: &str = crate::subspace_name(SUBSPACE_BLOBS);
pub(crate) trait CfHandle {
fn subspace_handle(&self, subspace: u8) -> Arc<BoundColumnFamily<'_>>;
@@ -29,9 +29,7 @@ pub(crate) trait CfHandle {
impl CfHandle for OptimisticTransactionDB<MultiThreaded> {
#[inline(always)]
fn subspace_handle(&self, subspace: u8) -> Arc<BoundColumnFamily<'_>> {
let subspace = &[subspace];
self.cf_handle(unsafe { std::str::from_utf8_unchecked(subspace) })
.unwrap()
self.cf_handle(crate::subspace_name(subspace)).unwrap()
}
}
+16 -22
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use super::{RocksDbStore, into_error};
@@ -17,21 +19,17 @@ impl RocksDbStore {
{
let db = self.db.clone();
self.spawn_worker(move || {
let subspace = &[key.subspace()];
let subspace = crate::subspace_name(key.subspace());
let key = key.serialize(0);
db.get_pinned_cf(
&db.cf_handle(unsafe { std::str::from_utf8_unchecked(subspace.as_slice()) })
.unwrap(),
&key,
)
.map_err(into_error)
.and_then(|value| {
if let Some(value) = value {
U::deserialize_with_key(&key, &value).map(Some)
} else {
Ok(None)
}
})
db.get_pinned_cf(&db.cf_handle(subspace).unwrap(), &key)
.map_err(into_error)
.and_then(|value| {
if let Some(value) = value {
U::deserialize_with_key(&key, &value).map(Some)
} else {
Ok(None)
}
})
})
.await
}
@@ -39,15 +37,11 @@ impl RocksDbStore {
pub(crate) async fn key_exists(&self, key: impl Key) -> trc::Result<bool> {
let db = self.db.clone();
self.spawn_worker(move || {
let subspace = &[key.subspace()];
let subspace = crate::subspace_name(key.subspace());
let key = key.serialize(0);
db.get_pinned_cf(
&db.cf_handle(unsafe { std::str::from_utf8_unchecked(subspace.as_slice()) })
.unwrap(),
&key,
)
.map_err(into_error)
.map(|value| value.is_some())
db.get_pinned_cf(&db.cf_handle(subspace).unwrap(), &key)
.map_err(into_error)
.map(|value| value.is_some())
})
.await
}
+6 -6
View File
@@ -74,8 +74,7 @@ impl RocksDbStore {
let db = self.db.clone();
self.spawn_worker(move || {
db.delete_range_cf(
&db.cf_handle(std::str::from_utf8(&[from.subspace()]).unwrap())
.unwrap(),
&db.cf_handle(crate::subspace_name(from.subspace())).unwrap(),
from.serialize(0),
to.serialize(0),
)
@@ -88,9 +87,7 @@ impl RocksDbStore {
let db = self.db.clone();
self.spawn_worker(move || {
for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] {
let cf = db
.cf_handle(std::str::from_utf8(&[subspace]).unwrap())
.unwrap();
let cf = db.cf_handle(crate::subspace_name(subspace)).unwrap();
let mut delete_keys = Vec::new();
@@ -174,7 +171,10 @@ impl RocksDBTransaction<'_, '_> {
.transaction_opt(&WriteOptions::default(), &self.txn_opts);
if has_changes {
let cf = self.db.cf_handle("n").unwrap();
let cf = self
.db
.cf_handle(crate::subspace_name(crate::SUBSPACE_COUNTER))
.unwrap();
for &account_id in self.batch.changes.keys() {
let key = ValueClass::ChangeId.serialize(account_id, 0, 0, 0);
let change_id = txn
+9 -4
View File
@@ -69,6 +69,11 @@ impl SqliteStore {
SUBSPACE_DELETED_ITEMS,
SUBSPACE_SPAM_SAMPLES,
crate::SUBSPACE_INBUXA, // inbuxa: masked email
// inbuxa: SL-16: the fork's large data, from layout version 3
crate::SUBSPACE_AUDIT,
crate::SUBSPACE_JOURNAL,
crate::SUBSPACE_HOLDS,
crate::SUBSPACE_UNDELETE,
SUBSPACE_BLOB_LINK,
SUBSPACE_IN_MEMORY_VALUE,
SUBSPACE_PROPERTY,
@@ -85,7 +90,7 @@ impl SqliteStore {
SUBSPACE_SEARCH_INDEX,
SUBSPACE_DIRECTORY,
] {
let table = char::from(table);
let table = crate::subspace_name(table);
conn.execute(
&format!(
"CREATE TABLE IF NOT EXISTS {table} (
@@ -99,7 +104,7 @@ impl SqliteStore {
}
for table in [SUBSPACE_INDEXES, SUBSPACE_REGISTRY_IDX] {
let table = char::from(table);
let table = crate::subspace_name(table);
conn.execute(
&format!(
"CREATE TABLE IF NOT EXISTS {table} (
@@ -118,7 +123,7 @@ impl SqliteStore {
k BLOB PRIMARY KEY,
v INTEGER NOT NULL DEFAULT 0
)",
char::from(table)
crate::subspace_name(table)
),
[],
)
@@ -126,7 +131,7 @@ impl SqliteStore {
}
// inbuxa: SL-10: in-memory counters keep their expiry in their row
let table = char::from(SUBSPACE_IN_MEMORY_COUNTER);
let table = crate::subspace_name(SUBSPACE_IN_MEMORY_COUNTER);
let has_expiry = conn
.prepare(&format!(
"SELECT 1 FROM pragma_table_info('{table}') WHERE name = 'e'"
+6 -4
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use super::{SqliteStore, into_error};
@@ -19,7 +21,7 @@ impl SqliteStore {
let mut result = conn
.prepare_cached(&format!(
"SELECT v FROM {} WHERE k = ?",
char::from(key.subspace())
crate::subspace_name(key.subspace())
))
.map_err(into_error)?;
let key = key.serialize(0);
@@ -41,7 +43,7 @@ impl SqliteStore {
let mut result = conn
.prepare_cached(&format!(
"SELECT 1 FROM {} WHERE k = ?",
char::from(key.subspace())
crate::subspace_name(key.subspace())
))
.map_err(into_error)?;
let key = key.serialize(0);
@@ -62,7 +64,7 @@ impl SqliteStore {
let manager = self.conn_pool.clone();
self.spawn_worker(move || {
let conn = manager.get().map_err(into_error)?;
let table = char::from(params.begin.subspace());
let table = crate::subspace_name(params.begin.subspace());
let begin = params.begin.serialize(0);
let end = params.end.serialize(0);
let keys = if params.values { "k, v" } else { "k" };
@@ -132,7 +134,7 @@ impl SqliteStore {
key: impl Into<ValueKey<ValueClass>> + Sync + Send,
) -> trc::Result<i64> {
let key = key.into();
let table = char::from(key.subspace());
let table = crate::subspace_name(key.subspace());
let key = key.serialize(0);
let manager = self.conn_pool.clone();
self.spawn_worker(move || {
+42 -10
View File
@@ -73,7 +73,7 @@ impl SqliteStore {
Operation::Value { class, op } => {
let key = class.serialize(account_id, collection, document_id, 0);
let subspace = class.subspace(collection);
let table = char::from(subspace);
let table = crate::subspace_name(subspace);
match op {
ValueOp::Set(value) => {
@@ -271,7 +271,7 @@ impl SqliteStore {
assert_value,
} => {
let key = class.serialize(account_id, collection, document_id, 0);
let table = char::from(class.subspace(collection));
let table = crate::subspace_name(class.subspace(collection));
let matches = trx
.prepare_cached(&format!("SELECT v FROM {} WHERE k = ?", table))
@@ -306,12 +306,15 @@ impl SqliteStore {
self.spawn_worker(move || {
let conn = manager.get().map_err(into_error)?;
for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] {
conn.prepare_cached(&format!("DELETE FROM {} WHERE v = 0", char::from(subspace),))
.map_err(into_error)
.caused_by(trc::location!())?
.execute([])
.map_err(into_error)
.caused_by(trc::location!())?;
conn.prepare_cached(&format!(
"DELETE FROM {} WHERE v = 0",
crate::subspace_name(subspace),
))
.map_err(into_error)
.caused_by(trc::location!())?
.execute([])
.map_err(into_error)
.caused_by(trc::location!())?;
}
Ok(())
@@ -326,7 +329,7 @@ impl SqliteStore {
let conn = manager.get().map_err(into_error)?;
conn.prepare_cached(&format!(
"DELETE FROM {} WHERE e <= ?",
char::from(SUBSPACE_IN_MEMORY_COUNTER)
crate::subspace_name(SUBSPACE_IN_MEMORY_COUNTER)
))
.map_err(into_error)
.caused_by(trc::location!())?
@@ -338,6 +341,35 @@ impl SqliteStore {
.await
}
/// inbuxa: SL-17: copies the rows from `from` up to `to` into subspace
/// `dest`, without their first `strip` bytes, in one statement. A row
/// already there is overwritten, so a copy can run again.
pub(crate) async fn copy_range(
&self,
from: impl Key,
to: impl Key,
dest: u8,
strip: usize,
) -> trc::Result<u64> {
let manager = self.conn_pool.clone();
self.spawn_worker(move || {
let conn = manager.get().map_err(into_error)?;
conn.prepare_cached(&format!(
"INSERT OR REPLACE INTO {} (k, v) SELECT substr(k, {}), v FROM {} WHERE k >= ? AND k < ?",
crate::subspace_name(dest),
strip + 1,
crate::subspace_name(from.subspace()),
))
.map_err(into_error)
.caused_by(trc::location!())?
.execute([from.serialize(0), to.serialize(0)])
.map(|rows| rows as u64)
.map_err(into_error)
.caused_by(trc::location!())
})
.await
}
pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> {
let manager = self.conn_pool.clone();
self.spawn_worker(move || {
@@ -345,7 +377,7 @@ impl SqliteStore {
conn.prepare_cached(&format!(
"DELETE FROM {} WHERE k >= ? AND k < ?",
char::from(from.subspace()),
crate::subspace_name(from.subspace()),
))
.map_err(into_error)
.caused_by(trc::location!())?
+41
View File
@@ -378,6 +378,47 @@ impl Store {
.caused_by(trc::location!())
}
/// inbuxa: SL-17: on SQL, copies the rows from `from` up to `to` into
/// subspace `dest` without their first `strip` bytes, in one statement on
/// the server, and returns how many it wrote. None where the backend has
/// no such statement: the caller copies them itself.
#[allow(unused_variables)]
pub async fn copy_range(
&self,
from: impl Key,
to: impl Key,
dest: u8,
strip: usize,
) -> trc::Result<Option<u64>> {
match self {
#[cfg(feature = "sqlite")]
Self::SQLite(store) => store.copy_range(from, to, dest, strip).await.map(Some),
#[cfg(feature = "postgres")]
Self::PostgreSQL(store) => store.copy_range(from, to, dest, strip).await.map(Some),
#[cfg(feature = "mysql")]
Self::MySQL(store) => store.copy_range(from, to, dest, strip).await.map(Some),
Self::Replicated(store) => crate::sql_backend!(
&store.primary,
db => db.copy_range(from, to, dest, strip).await
)
.map(Some),
_ => Ok(None),
}
.caused_by(trc::location!())
}
/// inbuxa: SL-20: drops the legacy `v` and `c` column families on
/// RocksDB, which nothing writes. Elsewhere they don't exist. Returns
/// how many it dropped.
pub async fn drop_legacy_subspaces(&self) -> trc::Result<u64> {
match self {
#[cfg(feature = "rocks")]
Self::RocksDb(store) => store.drop_legacy_subspaces().await,
_ => Ok(0),
}
.caused_by(trc::location!())
}
pub async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> {
match self {
#[cfg(feature = "sqlite")]
+106
View File
@@ -142,6 +142,49 @@ pub const SUBSPACE_INBUXA: u8 = b'_';
pub const LEGACY_SUBSPACE_BITMAP_TEXT: u8 = b'v';
pub const LEGACY_SUBSPACE_BITMAP_TAG: u8 = b'c';
// inbuxa: SL-15: every lowercase letter is taken, so later subspaces take id
// bytes from 0x80 up, which are never letters, and a name of their own.
pub const SUBSPACE_NAMED_FIRST: u8 = 0x80;
// inbuxa: SL-16: the fork's large data, out of `_` (layout version 3)
pub const SUBSPACE_AUDIT: u8 = 0x80;
pub const SUBSPACE_JOURNAL: u8 = 0x81;
pub const SUBSPACE_HOLDS: u8 = 0x82;
pub const SUBSPACE_UNDELETE: u8 = 0x83;
/// One-byte names for the subspaces up to 0x7f, which are ASCII.
const ASCII_NAMES: &[u8; 128] = &{
let mut names = [0u8; 128];
let mut byte = 0;
while byte < 128 {
names[byte] = byte as u8;
byte += 1;
}
names
};
/// inbuxa: SL-15: the name a backend gives a subspace: its SQL table and its
/// RocksDB column family. A subspace below [`SUBSPACE_NAMED_FIRST`] keeps its
/// one-byte name, so nothing changes for upstream's subspaces or for `_`.
/// One from 0x80 up is named `ib_…`; one with no name is a bug.
pub const fn subspace_name(subspace: u8) -> &'static str {
if subspace < SUBSPACE_NAMED_FIRST {
let (_, rest) = ASCII_NAMES.as_slice().split_at(subspace as usize);
let (name, _) = rest.split_at(1);
match std::str::from_utf8(name) {
Ok(name) => name,
Err(_) => unreachable!(),
}
} else {
match subspace {
SUBSPACE_AUDIT => "ib_audit",
SUBSPACE_JOURNAL => "ib_journal",
SUBSPACE_HOLDS => "ib_holds",
SUBSPACE_UNDELETE => "ib_undelete",
_ => panic!("a subspace from 0x80 up has no name"),
}
}
}
#[derive(Clone)]
pub struct IterateParams<T: Key> {
begin: T,
@@ -764,3 +807,66 @@ impl From<Value<'static>> for () {
unreachable!()
}
}
#[cfg(test)]
mod subspace_names {
use super::*;
// inbuxa: SL-15: renaming a subspace would orphan its table or column
// family, so every one below 0x80 keeps the name it always had.
#[test]
fn one_byte_subspaces_keep_their_names() {
for subspace in [
SUBSPACE_ACL,
SUBSPACE_TASK_QUEUE,
SUBSPACE_INDEXES,
SUBSPACE_BLOB_LINK,
SUBSPACE_BLOBS,
SUBSPACE_LOGS,
SUBSPACE_COUNTER,
SUBSPACE_IN_MEMORY_VALUE,
SUBSPACE_IN_MEMORY_COUNTER,
SUBSPACE_PROPERTY,
SUBSPACE_REGISTRY,
SUBSPACE_REGISTRY_IDX,
SUBSPACE_REGISTRY_PK,
SUBSPACE_DIRECTORY,
SUBSPACE_QUEUE_MESSAGE,
SUBSPACE_QUEUE_EVENT,
SUBSPACE_QUOTA,
SUBSPACE_REPORT_OUT,
SUBSPACE_REPORT_IN,
SUBSPACE_TELEMETRY_SPAN,
SUBSPACE_TELEMETRY_METRIC,
SUBSPACE_SEARCH_INDEX,
SUBSPACE_DELETED_ITEMS,
SUBSPACE_SPAM_SAMPLES,
SUBSPACE_INBUXA,
LEGACY_SUBSPACE_BITMAP_TEXT,
LEGACY_SUBSPACE_BITMAP_TAG,
] {
assert_eq!(subspace_name(subspace).as_bytes(), [subspace]);
}
assert_eq!(subspace_name(b'_'), "_");
}
#[test]
fn names_work_in_const_context() {
const NAME: &str = subspace_name(SUBSPACE_LOGS);
assert_eq!(NAME, "l");
}
#[test]
fn named_subspaces() {
assert_eq!(subspace_name(SUBSPACE_AUDIT), "ib_audit");
assert_eq!(subspace_name(SUBSPACE_JOURNAL), "ib_journal");
assert_eq!(subspace_name(SUBSPACE_HOLDS), "ib_holds");
assert_eq!(subspace_name(SUBSPACE_UNDELETE), "ib_undelete");
}
#[test]
#[should_panic]
fn an_unnamed_high_byte_is_a_bug() {
subspace_name(0xff);
}
}
+47 -2
View File
@@ -127,12 +127,57 @@ before January, else 27.7. Release candidates may carry it before then.
PostgreSQL the copy is done server-side (`INSERT … SELECT`).
- **SL-18.** Not rolling-safe (SL-6). On RocksDB an old binary can't open a
store with column families it doesn't know.
- **SL-19.** The moved data travels in exports in a family of its own,
included by default, so a configuration export can leave it out. Importing
- **SL-19.** The moved data travels in exports in a family of its own
(`compliance`), included by default, so a configuration export can leave
it out. Importing
an export taken at version 1 or 2 runs the steps after the import.
- **SL-20.** The legacy `v` and `c` column families, which nothing writes,
are dropped in the same step.
#### The keys in `_` and their owners (SL-16)
Every key in `_` starts with a byte naming its feature. The step lists the
two-byte prefixes in `_`, reading one key per prefix, and stops on any it
can't place, before it writes anything. The table is `OWNERS` in
`crates/migration/src/fork_subspaces.rs`.
| First byte | Owner | At step 2 → 3 |
|---|---|---|
| `A` | AI limits | stays |
| `B` | first boot defaults | stays |
| `C` | account snapshots | stays |
| `D` | deliverability reports | stays |
| `E` | inbuxa relay | stays |
| `F` | front ends | stays |
| `G` | scheduled reports | stays |
| `H` | legal holds (`h`, `o`, `e`) | moves to `ib_holds` (0x82) |
| `I` | migrations | stays |
| `J` | journal: entries (`e`, `c`, `p`, `t`, `x`, `h`, `f`), archive (`o`, `w`) | moves to `ib_journal` (0x81); the journals themselves (`j`) are settings and stay |
| `K` | account locks | stays |
| `L` | audit log (`e`, `t`, `o`, `h`, `f`) | moves to `ib_audit` (0x80); its settings (`s`) stay |
| `M` | masked email | stays |
| `P` | permissions and protocol policies | stays |
| `Q` | acceptance records | stays |
| `R` | mail flow rules | stays |
| `S` | spam rules, scale-out storage, scheduled reports from 2026.10.6.2 | stays |
| `T` | log files | stays |
| `U` | undelete notes (`n`, `x`, `b`, `r`, `c`, `k`, `a`, `g`) | moves to `ib_undelete` (0x83) |
| `V` | store checks, the layout mark (`Vs`), lease (`Vm`) and step progress (`Vt`) | stays |
| `W` | blob store changes, the sharing policy | stays |
A key moves without its first byte, so `Le` + node + seq becomes `e` + node
+ seq in `ib_audit`. A feature's settings stay in `_`, where a configuration
export (`registry`) carries them. On prod on 2026-11-11, `_` held 9,629
rows (3.2 MB), of which 6,813 were the audit log and none were journal,
hold or undelete rows.
The step copies each moving prefix (on SQL with one `INSERT … SELECT`), then
checks every copy: the row count, and for the audit log's and journal's
chain heads and floors the rows themselves. Only when every check passes
does it write `Vt`, then delete the originals and drop `v` and `c`. A run
that stopped while deleting copies what's left again and deletes it
without checking again; a run that stopped before `Vt` starts over.
### Imports and checks
- **SL-21.** A Stalwart 0.16 store has no mark, so it goes through every step.
+33 -6
View File
@@ -199,13 +199,28 @@ pub async fn test(test: &TestServer) {
batch.set(
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key: [b'U', b'x']
key: [b'M', b'm']
.into_iter()
.chain(item_id.to_be_bytes())
.collect(),
}),
random_bytes(32),
);
// inbuxa: SL-19: the fork's large data, in the compliance family
for subspace in [
store::SUBSPACE_AUDIT,
store::SUBSPACE_JOURNAL,
store::SUBSPACE_HOLDS,
store::SUBSPACE_UNDELETE,
] {
batch.set(
ValueClass::Any(AnyClass {
subspace,
key: [b'x'].into_iter().chain(item_id.to_be_bytes()).collect(),
}),
random_bytes(32),
);
}
}
db.write(batch.build_all()).await.unwrap();
@@ -243,11 +258,15 @@ pub async fn test(test: &TestServer) {
SUBSPACE_DELETED_ITEMS,
SUBSPACE_SPAM_SAMPLES,
SUBSPACE_INBUXA,
store::SUBSPACE_AUDIT,
store::SUBSPACE_JOURNAL,
store::SUBSPACE_HOLDS,
store::SUBSPACE_UNDELETE,
] {
assert!(
snapshot.keys.iter().any(|k| k.subspace == subspace),
"No test data in subspace {}",
char::from(subspace)
store::subspace_name(subspace)
);
}
@@ -286,11 +305,15 @@ pub async fn test(test: &TestServer) {
SUBSPACE_DELETED_ITEMS,
SUBSPACE_SPAM_SAMPLES,
SUBSPACE_INBUXA,
store::SUBSPACE_AUDIT,
store::SUBSPACE_JOURNAL,
store::SUBSPACE_HOLDS,
store::SUBSPACE_UNDELETE,
] {
assert!(
imported.contains(&subspace),
"Subspace {} was not exported",
char::from(subspace)
store::subspace_name(subspace)
);
}
@@ -383,7 +406,7 @@ async fn cross_backend(
source.get_counter(class()).await.unwrap(),
target.get_counter(class()).await.unwrap(),
"Counter mismatch in {} for {key:?}",
char::from(subspace)
store::subspace_name(subspace)
);
}
}
@@ -510,6 +533,10 @@ impl Snapshot {
(SUBSPACE_REPORT_IN, true),
(SUBSPACE_DIRECTORY, true),
(SUBSPACE_INBUXA, true),
(store::SUBSPACE_AUDIT, true),
(store::SUBSPACE_JOURNAL, true),
(store::SUBSPACE_HOLDS, true),
(store::SUBSPACE_UNDELETE, true),
] {
if subspace == SUBSPACE_BLOBS && !with_blobs {
continue;
@@ -548,7 +575,7 @@ impl Snapshot {
if !other.keys.contains(key) {
println!(
"Subspace {}, Key {:?} not found in restored snapshot",
char::from(key.subspace),
store::subspace_name(key.subspace),
key.key,
);
is_err = true;
@@ -558,7 +585,7 @@ impl Snapshot {
if !self.keys.contains(key) {
println!(
"Subspace {}, Key {:?} not found in original snapshot",
char::from(key.subspace),
store::subspace_name(key.subspace),
key.key,
);
is_err = true;
+11 -6
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use crate::utils::{server::TestServer, storage::build_data_store};
@@ -284,7 +286,7 @@ impl Backend {
}
fn populate_query(&self) -> String {
let table = char::from(SUBSPACE_PROPERTY);
let table = store::subspace_name(SUBSPACE_PROPERTY);
let prefix = key_prefix();
let padding = VALUE_SIZE - U32_LEN;
let last = NUM_KEYS - 1;
@@ -308,7 +310,7 @@ impl Backend {
}
fn populate_counters_query(&self) -> String {
let table = char::from(SUBSPACE_COUNTER);
let table = store::subspace_name(SUBSPACE_COUNTER);
let prefix = key_prefix();
let last = NUM_KEYS - 1;
@@ -328,11 +330,14 @@ impl Backend {
}
fn purge_query(&self) -> String {
format!("DELETE FROM {} WHERE v = 0", char::from(SUBSPACE_COUNTER))
format!(
"DELETE FROM {} WHERE v = 0",
store::subspace_name(SUBSPACE_COUNTER)
)
}
fn count_counters_query(&self) -> String {
let table = char::from(SUBSPACE_COUNTER);
let table = store::subspace_name(SUBSPACE_COUNTER);
match self {
#[cfg(feature = "postgres")]
Backend::Postgres => format!("SELECT COUNT(*) FROM {table} WHERE k >= $1 AND k <= $2"),
@@ -342,7 +347,7 @@ impl Backend {
}
fn scan_query(&self) -> String {
let table = char::from(SUBSPACE_PROPERTY);
let table = store::subspace_name(SUBSPACE_PROPERTY);
match self {
#[cfg(feature = "postgres")]
Backend::Postgres => {
@@ -356,7 +361,7 @@ impl Backend {
}
fn delete_query(&self) -> String {
let table = char::from(SUBSPACE_PROPERTY);
let table = store::subspace_name(SUBSPACE_PROPERTY);
match self {
#[cfg(feature = "postgres")]
Backend::Postgres => format!("DELETE FROM {table} WHERE k >= $1 AND k <= $2"),
+364 -2
View File
@@ -9,7 +9,8 @@
//! refused before anything is written to it. A store opened for the first
//! time has upstream's web interface switched off (docs/spec/import-0.16.md).
//! A store at an older layout version is brought up to this one, by one node
//! at a time (docs/spec/features/store-layout.md).
//! at a time (docs/spec/features/store-layout.md). Step 2 -> 3 moves the
//! audit log, journal, holds and undelete notes out of `_`.
use crate::utils::server::TestServerBuilder;
use common::{
@@ -17,13 +18,15 @@ use common::{
manager::first_party::disable_imported_web_interface,
};
use futures::FutureExt;
use inbuxa_features::audit::{Action, Actor, AuditLog, Outcome, Record, Target, log};
use registry::{
schema::{prelude::ObjectInner, structs::Application},
types::map::Map,
};
use std::{panic::AssertUnwindSafe, time::Duration};
use store::{
InMemoryStore, IterateParams, SUBSPACE_INBUXA, SerializeInfallible, Store, ValueKey,
Deserialize, InMemoryStore, IterateParams, SUBSPACE_AUDIT, SUBSPACE_HOLDS, SUBSPACE_INBUXA,
SUBSPACE_JOURNAL, SUBSPACE_UNDELETE, SerializeInfallible, Store, ValueKey,
dispatch::lookup::KeyValue,
registry::{bootstrap::Bootstrap, write::RegistryWrite},
write::{AnyClass, AnyKey, BatchBuilder, InMemoryClass, Operation, ValueClass, ValueOp, now},
@@ -91,6 +94,7 @@ async fn store_version() {
layout_step_1_2(server).await;
layout_lease(server).await;
layout_step_2_3(server).await;
}
/// A counter the way a version 1 store holds it: a row in `y` and an expiry
@@ -386,3 +390,361 @@ async fn imported_web_interface() {
"a marked store's web interface is left as the operator set it"
);
}
struct Raw(Vec<u8>);
impl Deserialize for Raw {
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
Ok(Raw(bytes.to_vec()))
}
}
/// Every row of a subspace, keys as stored.
async fn rows(store: &Store, subspace: u8) -> Vec<(Vec<u8>, Vec<u8>)> {
let mut rows = Vec::new();
store
.iterate(
IterateParams::new(
AnyKey {
subspace,
key: vec![0u8],
},
AnyKey {
subspace,
key: vec![u8::MAX; 64],
},
),
|key, value| {
rows.push((key.to_vec(), value.to_vec()));
Ok(true)
},
)
.await
.unwrap();
rows
}
async fn set_raw(store: &Store, subspace: u8, key: &[u8], value: &[u8]) {
let mut batch = BatchBuilder::new();
batch.set(
ValueClass::Any(AnyClass {
subspace,
key: key.to_vec(),
}),
value.to_vec(),
);
store.write(batch.build_all()).await.unwrap();
}
async fn clear_raw(store: &Store, subspace: u8, key: &[u8]) {
let mut batch = BatchBuilder::new();
batch.clear(ValueClass::Any(AnyClass {
subspace,
key: key.to_vec(),
}));
store.write(batch.build_all()).await.unwrap();
}
async fn get_raw(store: &Store, subspace: u8, key: &[u8]) -> Option<Vec<u8>> {
store
.get_value::<Raw>(AnyKey {
subspace,
key: key.to_vec(),
})
.await
.unwrap()
.map(|Raw(bytes)| bytes)
}
/// The moved subspaces, and the letter each had in `_`.
const MOVED: [(u8, u8); 4] = [
(SUBSPACE_AUDIT, b'L'),
(SUBSPACE_JOURNAL, b'J'),
(SUBSPACE_HOLDS, b'H'),
(SUBSPACE_UNDELETE, b'U'),
];
/// Puts a version 3 store's moved data back the way version 2 kept it,
/// under its letter in `_`.
async fn back_to_v2(store: &Store) {
for (subspace, letter) in MOVED {
for (key, value) in rows(store, subspace).await {
let mut old = vec![letter];
old.extend_from_slice(&key);
set_raw(store, SUBSPACE_INBUXA, &old, &value).await;
clear_raw(store, subspace, &key).await;
}
}
}
/// The rows left in `_` under the moved letters, but for the settings that
/// stay.
async fn left_in_inbuxa(store: &Store) -> Vec<Vec<u8>> {
rows(store, SUBSPACE_INBUXA)
.await
.into_iter()
.map(|(key, _)| key)
.filter(|key| {
MOVED.iter().any(|(_, letter)| key.first() == Some(letter))
&& key.get(..2) != Some(b"Ls")
&& key.get(..2) != Some(b"Jj")
})
.collect()
}
fn audit_record(n: u64) -> Record {
Record {
at: 1_000 + n,
actor: Actor::account(7, "[email protected]", None),
via: None,
remote_ip: None,
action: Action::Update,
target: Target {
kind: "x:Domain".into(),
id: Some(format!("d{n}")),
name: Some("example.org".into()),
..Default::default()
},
changes: vec![],
details: None,
reason: None,
outcome: Outcome::success(),
}
}
// SL-16, SL-17, SL-20, SL-22: a version 2 store's audit log, journal,
// holds and undelete notes move to their own subspaces; settings and other
// features stay in `_`; a second run changes nothing; a run that stopped
// while deleting finishes; an unclaimed key or a copy that doesn't match
// stops the step with nothing deleted.
async fn layout_step_2_3(server: &Server) {
let store = server.store();
let node = 3;
// Version 3 data, written the way this build writes it
let audit = AuditLog::new();
let mut ids = Vec::new();
for n in 0..3 {
ids.push(audit.append(store, node, &audit_record(n)).await.unwrap());
}
set_raw(
store,
SUBSPACE_JOURNAL,
b"c\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x01",
b"{\"journal\":1}",
)
.await;
set_raw(store, SUBSPACE_HOLDS, b"h\x00\x00\x00\x09", b"{\"hold\":9}").await;
set_raw(
store,
SUBSPACE_UNDELETE,
b"g\x01\x00\x00\x00\x05\x00\x00\x00\x06",
b"{\"note\":6}",
)
.await;
set_raw(
store,
SUBSPACE_UNDELETE,
b"k\x00\x00\x00\x05",
b"{\"kept\":5}",
)
.await;
// Settings and another feature's key, which stay in `_`
set_raw(store, SUBSPACE_INBUXA, b"Ls", b"{\"keepFor\":100}").await;
set_raw(
store,
SUBSPACE_INBUXA,
b"Jj\x00\x00\x00\x01",
b"{\"journal\":\"def\"}",
)
.await;
set_raw(store, SUBSPACE_INBUXA, b"Kl\x00\x00\x00\x05", b"lock").await;
let audit_rows = rows(store, SUBSPACE_AUDIT).await;
let mut expected = Vec::new();
for (subspace, _) in MOVED {
expected.push(rows(store, subspace).await);
}
assert_eq!(audit_rows.len(), expected[0].len());
assert!(!audit_rows.is_empty());
let moved_rows = |store: Store| async move {
let mut all = Vec::new();
for (subspace, _) in MOVED {
all.push(rows(&store, subspace).await);
}
all
};
// Brought back to version 2, then up again
back_to_v2(store).await;
assert!(rows(store, SUBSPACE_AUDIT).await.is_empty());
assert_eq!(
left_in_inbuxa(store).await.len(),
expected.iter().map(Vec::len).sum::<usize>()
);
write_mark(server, Some(2)).await;
migration::try_migrate(server).await.unwrap();
assert_eq!(read_mark(server).await, Some(INBUXA_STORE_VERSION));
assert_eq!(moved_rows(store.clone()).await, expected);
assert!(left_in_inbuxa(store).await.is_empty(), "left in _");
assert!(
get_raw(
store,
SUBSPACE_INBUXA,
migration::fork_subspaces::CHECKED_KEY
)
.await
.is_none()
);
assert_eq!(
get_raw(store, SUBSPACE_INBUXA, b"Ls").await.as_deref(),
Some(&b"{\"keepFor\":100}"[..])
);
assert_eq!(
get_raw(store, SUBSPACE_INBUXA, b"Jj\x00\x00\x00\x01")
.await
.as_deref(),
Some(&b"{\"journal\":\"def\"}"[..])
);
assert_eq!(
get_raw(store, SUBSPACE_INBUXA, b"Kl\x00\x00\x00\x05")
.await
.as_deref(),
Some(&b"lock"[..])
);
// The audit log reads back, its chain whole, and goes on from its head
for (n, id) in ids.iter().enumerate() {
assert_eq!(
log::get(store, *id).await.unwrap(),
Some(audit_record(n as u64))
);
}
let next = audit.append(store, node, &audit_record(3)).await.unwrap();
assert_eq!(next.seq, ids[2].seq + 1);
let report = log::verify(store).await.unwrap();
let chain = report.iter().find(|chain| chain.node == node).unwrap();
assert_eq!(chain.broken_at, None, "{chain:?}");
assert_eq!(chain.entries, 4);
let mut expected = moved_rows(store.clone()).await;
// A second run changes nothing
write_mark(server, Some(2)).await;
migration::layout::run_steps(server, server.registry().node_id())
.await
.unwrap();
assert_eq!(read_mark(server).await, Some(INBUXA_STORE_VERSION));
assert_eq!(moved_rows(store.clone()).await, expected);
// A run that stopped while deleting: copies checked, some originals left
for (key, value) in &expected[2] {
let mut old = vec![b'H'];
old.extend_from_slice(key);
set_raw(store, SUBSPACE_INBUXA, &old, value).await;
}
set_raw(
store,
SUBSPACE_INBUXA,
migration::fork_subspaces::CHECKED_KEY,
&[1],
)
.await;
write_mark(server, Some(2)).await;
migration::layout::run_steps(server, server.registry().node_id())
.await
.unwrap();
assert_eq!(moved_rows(store.clone()).await, expected);
assert!(left_in_inbuxa(store).await.is_empty());
assert!(
get_raw(
store,
SUBSPACE_INBUXA,
migration::fork_subspaces::CHECKED_KEY
)
.await
.is_none()
);
// A key nobody claims stops the step before it changes anything
back_to_v2(store).await;
write_mark(server, Some(2)).await;
let before = rows(store, SUBSPACE_INBUXA).await;
set_raw(store, SUBSPACE_INBUXA, b"Zz", b"?").await;
let err = migration::layout::run_steps(server, server.registry().node_id())
.await
.unwrap_err();
assert!(format!("{err:?}").contains("Zz"), "{err:?}");
assert_eq!(read_mark(server).await, Some(2));
clear_raw(store, SUBSPACE_INBUXA, b"Zz").await;
assert_eq!(rows(store, SUBSPACE_INBUXA).await, before);
assert!(rows(store, SUBSPACE_AUDIT).await.is_empty());
// A copy that doesn't match its originals stops it with nothing deleted
let (stray, _) = &expected[0][0];
let mut stray = stray.clone();
stray.push(0xff);
set_raw(store, SUBSPACE_AUDIT, &stray, b"stray").await;
let err = migration::layout::run_steps(server, server.registry().node_id())
.await
.unwrap_err();
assert!(format!("{err:?}").contains("audit log"), "{err:?}");
assert_eq!(read_mark(server).await, Some(2));
assert_eq!(
rows(store, SUBSPACE_INBUXA).await,
before,
"deleted after a mismatch"
);
// Put right, it goes through
clear_raw(store, SUBSPACE_AUDIT, &stray).await;
migration::layout::run_steps(server, server.registry().node_id())
.await
.unwrap();
assert_eq!(read_mark(server).await, Some(INBUXA_STORE_VERSION));
expected.iter_mut().for_each(|rows| rows.sort());
let mut after = moved_rows(store.clone()).await;
after.iter_mut().for_each(|rows| rows.sort());
assert_eq!(after, expected);
// More rows than the step copies at a time, where it copies them itself
let many = 10_050u32;
let mut batch = BatchBuilder::new();
for n in 0..many {
let mut key = b"Un".to_vec();
key.extend_from_slice(&n.to_be_bytes());
batch.set(
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key,
}),
n.to_be_bytes().to_vec(),
);
if batch.is_large_batch() {
store.write(batch.build_all()).await.unwrap();
batch = BatchBuilder::new();
}
}
store.write(batch.build_all()).await.unwrap();
write_mark(server, Some(2)).await;
migration::layout::run_steps(server, server.registry().node_id())
.await
.unwrap();
let notes = rows(store, SUBSPACE_UNDELETE)
.await
.into_iter()
.filter(|(key, _)| key.len() == 5 && key[0] == b'n')
.collect::<Vec<_>>();
assert_eq!(notes.len(), many as usize);
assert!(notes.iter().all(|(key, value)| key[1..] == value[..]));
assert!(left_in_inbuxa(store).await.is_empty());
// Clean up
for (subspace, _) in MOVED {
for (key, _) in rows(store, subspace).await {
clear_raw(store, subspace, &key).await;
}
}
for key in [&b"Ls"[..], b"Jj\x00\x00\x00\x01", b"Kl\x00\x00\x00\x05"] {
clear_raw(store, SUBSPACE_INBUXA, key).await;
}
}
+6 -1
View File
@@ -28,6 +28,11 @@ pub async fn store_destroy(store: &Store) {
SUBSPACE_DELETED_ITEMS,
SUBSPACE_SPAM_SAMPLES,
store::SUBSPACE_INBUXA, // inbuxa: masked email
// inbuxa: SL-16: the fork's large data
store::SUBSPACE_AUDIT,
store::SUBSPACE_JOURNAL,
store::SUBSPACE_HOLDS,
store::SUBSPACE_UNDELETE,
SUBSPACE_BLOB_LINK,
SUBSPACE_LOGS,
SUBSPACE_IN_MEMORY_COUNTER,
@@ -362,7 +367,7 @@ pub async fn store_assert_is_empty(store: &Store, blob_store: BlobStore, include
_ => {
println!(
"Found key in {:?}: {:?} ({:?}) = {:?} ({:?})",
char::from(subspace),
store::subspace_name(subspace),
key,
String::from_utf8_lossy(key),
value,