From d71058478f25fe7f04d65f80272d158ba7cb4416 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Tue, 10 Nov 2026 21:57:08 -0700 Subject: [PATCH 1/2] Name subspaces through one function, so later ones can have names (SL-15) Every backend built its table or column family name from the subspace byte, with char::from or an unchecked UTF-8 cast. Every lowercase letter is taken, so step 2 -> 3 needs subspaces from 0x80 up, which have no one-byte name and would have made the RocksDB cast undefined. store::subspace_name now gives each subspace its name. Below 0x80 it is the byte, as before, so no table or column family changes. From 0x80 up there are none yet; step 2 -> 3 adds them as ib_ names. --- crates/common/src/manager/backup.rs | 4 +- crates/store/src/backend/mysql/main.rs | 10 +-- crates/store/src/backend/mysql/read.rs | 8 +- crates/store/src/backend/mysql/write.rs | 12 +-- crates/store/src/backend/postgres/main.rs | 8 +- crates/store/src/backend/postgres/read.rs | 10 +-- crates/store/src/backend/postgres/write.rs | 12 +-- crates/store/src/backend/rocksdb/main.rs | 4 +- crates/store/src/backend/rocksdb/mod.rs | 10 +-- crates/store/src/backend/rocksdb/read.rs | 38 ++++------ crates/store/src/backend/rocksdb/write.rs | 12 +-- crates/store/src/backend/sqlite/main.rs | 8 +- crates/store/src/backend/sqlite/read.rs | 10 ++- crates/store/src/backend/sqlite/write.rs | 23 +++--- crates/store/src/lib.rs | 87 ++++++++++++++++++++++ tests/src/store/import_export.rs | 10 +-- tests/src/store/sql_timeout.rs | 17 +++-- tests/src/utils/cleanup.rs | 2 +- 18 files changed, 187 insertions(+), 98 deletions(-) diff --git a/crates/common/src/manager/backup.rs b/crates/common/src/manager/backup.rs index 4c38bc0a..d32d9cda 100644 --- a/crates/common/src/manager/backup.rs +++ b/crates/common/src/manager/backup.rs @@ -110,7 +110,7 @@ impl Core { let store = self.storage.data.clone(); let blob_store = self.storage.blob.clone(); let (handle, writer) = spawn_writer( - dest.join(format!("subspace_{}", char::from(subspace))), + dest.join(format!("subspace_{}", store::subspace_name(subspace))), subspace, schema_version, ); @@ -175,7 +175,7 @@ impl Core { fn backup_subspace(&self, dest: &Path, subspace: u8, schema_version: u32) -> TaskHandle { let store = self.storage.data.clone(); let (handle, writer) = spawn_writer( - dest.join(format!("subspace_{}", char::from(subspace))), + dest.join(format!("subspace_{}", store::subspace_name(subspace))), subspace, schema_version, ); diff --git a/crates/store/src/backend/mysql/main.rs b/crates/store/src/backend/mysql/main.rs index d33c04c4..5282a3e3 100644 --- a/crates/store/src/backend/mysql/main.rs +++ b/crates/store/src/backend/mysql/main.rs @@ -127,7 +127,7 @@ impl MysqlStore { SUBSPACE_TELEMETRY_SPAN, SUBSPACE_TELEMETRY_METRIC, ] { - let table = char::from(table); + let table = crate::subspace_name(table); conn.query_drop(format!( "CREATE TABLE IF NOT EXISTS {table} ( k VARBINARY(255) NOT NULL, @@ -145,13 +145,13 @@ impl MysqlStore { v LONGBLOB NOT NULL, PRIMARY KEY (k) ) ENGINE=InnoDB", - char::from(SUBSPACE_BLOBS), + crate::subspace_name(SUBSPACE_BLOBS), )) .await .map_err(into_error)?; for table in [SUBSPACE_INDEXES, SUBSPACE_REGISTRY_IDX] { - let table = char::from(table); + let table = crate::subspace_name(table); conn.query_drop(format!( "CREATE TABLE IF NOT EXISTS {table} ( k BLOB, @@ -169,7 +169,7 @@ impl MysqlStore { v BIGINT NOT NULL DEFAULT 0, PRIMARY KEY (k) ) ENGINE=InnoDB", - char::from(table) + crate::subspace_name(table) )) .await .map_err(into_error)?; @@ -179,7 +179,7 @@ impl MysqlStore { // row. MySQL has no ADD COLUMN IF NOT EXISTS, so look first, and // take "duplicate column" from a node starting at the same time // as done. - let table = char::from(SUBSPACE_IN_MEMORY_COUNTER).to_string(); + let table = crate::subspace_name(SUBSPACE_IN_MEMORY_COUNTER).to_string(); let has_expiry = conn .exec_first::( concat!( diff --git a/crates/store/src/backend/mysql/read.rs b/crates/store/src/backend/mysql/read.rs index ce42baca..40d26cb6 100644 --- a/crates/store/src/backend/mysql/read.rs +++ b/crates/store/src/backend/mysql/read.rs @@ -22,7 +22,7 @@ impl MysqlStore { let s = conn .prep(format!( "SELECT v FROM {} WHERE k = ?", - char::from(key.subspace()) + crate::subspace_name(key.subspace()) )) .await .map_err(into_error)?; @@ -49,7 +49,7 @@ impl MysqlStore { let s = conn .prep(format!( "SELECT 1 FROM {} WHERE k = ?", - char::from(key.subspace()) + crate::subspace_name(key.subspace()) )) .await .map_err(into_error)?; @@ -69,7 +69,7 @@ impl MysqlStore { mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, ) -> trc::Result<()> { let mut conn = self.conn().await?; - let table = char::from(params.begin.subspace()); + let table = crate::subspace_name(params.begin.subspace()); let begin = params.begin.serialize(0); let end = params.end.serialize(0); let keys = if params.values { "k, v" } else { "k" }; @@ -193,7 +193,7 @@ impl MysqlStore { key: impl Into> + Sync + Send, ) -> trc::Result { let key = key.into(); - let table = char::from(key.subspace()); + let table = crate::subspace_name(key.subspace()); let key = key.serialize(0); let mut conn = self.conn().await?; let limit = self.timeouts.query; diff --git a/crates/store/src/backend/mysql/write.rs b/crates/store/src/backend/mysql/write.rs index 102d3563..009e9b6a 100644 --- a/crates/store/src/backend/mysql/write.rs +++ b/crates/store/src/backend/mysql/write.rs @@ -137,7 +137,7 @@ impl MysqlStore { Operation::Value { class, op } => { let key = class.serialize(account_id, collection, document_id, 0); let subspace = class.subspace(collection); - let table = char::from(subspace); + let table = crate::subspace_name(subspace); match op { ValueOp::Set(value) => { @@ -393,7 +393,7 @@ impl MysqlStore { assert_value, } => { let key = class.serialize(account_id, collection, document_id, 0); - let table = char::from(class.subspace(collection)); + let table = crate::subspace_name(class.subspace(collection)); let s = trx .prep(format!("SELECT v FROM {} WHERE k = ? FOR UPDATE", table)) @@ -423,7 +423,7 @@ impl MysqlStore { let limit = self.timeouts.maintenance; let result = tokio::time::timeout(limit, async { for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] { - purge_table(&mut conn, char::from(subspace)).await?; + purge_table(&mut conn, crate::subspace_name(subspace)).await?; } Ok(()) @@ -440,7 +440,7 @@ impl MysqlStore { conn.exec_drop( format!( "DELETE FROM {} WHERE e <= ?", - char::from(SUBSPACE_IN_MEMORY_COUNTER) + crate::subspace_name(SUBSPACE_IN_MEMORY_COUNTER) ), (now as i64,), ) @@ -456,7 +456,7 @@ impl MysqlStore { let mut conn = self.conn().await?; let limit = self.timeouts.maintenance; let result = tokio::time::timeout(limit, async { - let table = char::from(from.subspace()); + let table = crate::subspace_name(from.subspace()); let mut from = from.serialize(0); let to = to.serialize(0); @@ -518,7 +518,7 @@ impl MysqlStore { } } -async fn purge_table(conn: &mut Conn, table: char) -> trc::Result<()> { +async fn purge_table(conn: &mut Conn, table: &str) -> trc::Result<()> { let s = conn .prep(format!("DELETE FROM {table} WHERE v = 0")) .await diff --git a/crates/store/src/backend/postgres/main.rs b/crates/store/src/backend/postgres/main.rs index 22458bd7..6913747d 100644 --- a/crates/store/src/backend/postgres/main.rs +++ b/crates/store/src/backend/postgres/main.rs @@ -176,7 +176,7 @@ impl PostgresStore { SUBSPACE_TELEMETRY_SPAN, SUBSPACE_TELEMETRY_METRIC, ] { - let table = char::from(table); + let table = crate::subspace_name(table); conn.execute( &format!( "CREATE TABLE IF NOT EXISTS {table} ( @@ -191,7 +191,7 @@ impl PostgresStore { } for table in [SUBSPACE_INDEXES, SUBSPACE_REGISTRY_IDX] { - let table = char::from(table); + let table = crate::subspace_name(table); conn.execute( &format!( "CREATE TABLE IF NOT EXISTS {table} ( @@ -211,7 +211,7 @@ impl PostgresStore { k BYTEA PRIMARY KEY, v BIGINT NOT NULL DEFAULT 0 )", - char::from(table) + crate::subspace_name(table) ), &[], ) @@ -221,7 +221,7 @@ impl PostgresStore { // inbuxa: SL-10: in-memory counters keep their expiry in their // row. Added only when missing, so a start doesn't lock the table. - let table = char::from(SUBSPACE_IN_MEMORY_COUNTER).to_string(); + let table = crate::subspace_name(SUBSPACE_IN_MEMORY_COUNTER).to_string(); let has_expiry = conn .query_opt( concat!( diff --git a/crates/store/src/backend/postgres/read.rs b/crates/store/src/backend/postgres/read.rs index ca7a6cb3..7629f070 100644 --- a/crates/store/src/backend/postgres/read.rs +++ b/crates/store/src/backend/postgres/read.rs @@ -25,7 +25,7 @@ impl PostgresStore { let s = conn .prepare_cached(&format!( "SELECT v FROM {} WHERE k = $1", - char::from(key.subspace()) + crate::subspace_name(key.subspace()) )) .await .map_err(into_error)?; @@ -61,7 +61,7 @@ impl PostgresStore { let s = conn .prepare_cached(&format!( "SELECT k, v FROM {} WHERE k = ANY($1)", - char::from(subspace) + crate::subspace_name(subspace) )) .await .map_err(into_error)?; @@ -90,7 +90,7 @@ impl PostgresStore { let s = conn .prepare_cached(&format!( "SELECT 1 FROM {} WHERE k = $1", - char::from(key.subspace()) + crate::subspace_name(key.subspace()) )) .await .map_err(into_error)?; @@ -110,7 +110,7 @@ impl PostgresStore { mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result + Sync + Send, ) -> trc::Result<()> { let conn = self.conn_pool.get().await.map_err(into_pool_error)?; - let table = char::from(params.begin.subspace()); + let table = crate::subspace_name(params.begin.subspace()); let begin = params.begin.serialize(0); let end = params.end.serialize(0); let keys = if params.values { "k, v" } else { "k" }; @@ -230,7 +230,7 @@ impl PostgresStore { key: impl Into> + Sync + Send, ) -> trc::Result { let key = key.into(); - let table = char::from(key.subspace()); + let table = crate::subspace_name(key.subspace()); let key = key.serialize(0); let conn = self.conn_pool.get().await.map_err(into_pool_error)?; diff --git a/crates/store/src/backend/postgres/write.rs b/crates/store/src/backend/postgres/write.rs index a0a7d05b..0ab0ece2 100644 --- a/crates/store/src/backend/postgres/write.rs +++ b/crates/store/src/backend/postgres/write.rs @@ -236,7 +236,7 @@ impl PostgresStore { Operation::Value { class, op } => { let key = class.serialize(account_id, collection, document_id, 0); let subspace = class.subspace(collection); - let table = char::from(subspace); + let table = crate::subspace_name(subspace); match op { ValueOp::Set(value) => { @@ -491,7 +491,7 @@ impl PostgresStore { } => { pipeline.flush().await?; let key = class.serialize(account_id, collection, document_id, 0); - let table = char::from(class.subspace(collection)); + let table = crate::subspace_name(class.subspace(collection)); let s = trx .prepare_cached(&format!("SELECT v FROM {} WHERE k = $1 FOR UPDATE", table)) @@ -526,7 +526,7 @@ impl PostgresStore { let limit = self.timeouts.maintenance; let result = tokio::time::timeout(limit, async { for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] { - purge_table(&conn, char::from(subspace)).await?; + purge_table(&conn, crate::subspace_name(subspace)).await?; } Ok(()) @@ -543,7 +543,7 @@ impl PostgresStore { let s = conn .prepare_cached(&format!( "DELETE FROM {} WHERE e <= $1", - char::from(SUBSPACE_IN_MEMORY_COUNTER) + crate::subspace_name(SUBSPACE_IN_MEMORY_COUNTER) )) .await .map_err(into_error)?; @@ -557,7 +557,7 @@ impl PostgresStore { let conn = self.conn_pool.get().await.map_err(into_pool_error)?; let limit = self.timeouts.maintenance; let result = tokio::time::timeout(limit, async { - let table = char::from(from.subspace()); + let table = crate::subspace_name(from.subspace()); let mut from = from.serialize(0); let to = to.serialize(0); @@ -619,7 +619,7 @@ impl PostgresStore { } } -async fn purge_table(conn: &Object, table: char) -> trc::Result<()> { +async fn purge_table(conn: &Object, table: &str) -> trc::Result<()> { let s = conn .prepare_cached(&format!("DELETE FROM {table} WHERE v = 0")) .await diff --git a/crates/store/src/backend/rocksdb/main.rs b/crates/store/src/backend/rocksdb/main.rs index e48b809b..e15c5e55 100644 --- a/crates/store/src/backend/rocksdb/main.rs +++ b/crates/store/src/backend/rocksdb/main.rs @@ -65,7 +65,7 @@ impl RocksDbStore { // Counters for subspace in [SUBSPACE_COUNTER, SUBSPACE_QUOTA, SUBSPACE_IN_MEMORY_COUNTER] { cfs.push(ColumnFamilyDescriptor::new( - std::str::from_utf8(&[subspace]).unwrap(), + crate::subspace_name(subspace), cf_options(CfProfile::Counter, &cache, write_buffer_size), )); } @@ -105,7 +105,7 @@ impl RocksDbStore { (LEGACY_SUBSPACE_BITMAP_TAG, CfProfile::Scan), ] { cfs.push(ColumnFamilyDescriptor::new( - std::str::from_utf8(&[subspace]).unwrap(), + crate::subspace_name(subspace), cf_options(profile, &cache, write_buffer_size), )); } diff --git a/crates/store/src/backend/rocksdb/mod.rs b/crates/store/src/backend/rocksdb/mod.rs index 3840f3c8..6d82bb92 100644 --- a/crates/store/src/backend/rocksdb/mod.rs +++ b/crates/store/src/backend/rocksdb/mod.rs @@ -18,9 +18,9 @@ pub mod main; pub mod read; pub mod write; -static CF_LOGS: &str = unsafe { std::str::from_utf8_unchecked(&[SUBSPACE_LOGS]) }; -static CF_INDEXES: &str = unsafe { std::str::from_utf8_unchecked(&[SUBSPACE_INDEXES]) }; -static CF_BLOBS: &str = unsafe { std::str::from_utf8_unchecked(&[SUBSPACE_BLOBS]) }; +static CF_LOGS: &str = crate::subspace_name(SUBSPACE_LOGS); +static CF_INDEXES: &str = crate::subspace_name(SUBSPACE_INDEXES); +static CF_BLOBS: &str = crate::subspace_name(SUBSPACE_BLOBS); pub(crate) trait CfHandle { fn subspace_handle(&self, subspace: u8) -> Arc>; @@ -29,9 +29,7 @@ pub(crate) trait CfHandle { impl CfHandle for OptimisticTransactionDB { #[inline(always)] fn subspace_handle(&self, subspace: u8) -> Arc> { - let subspace = &[subspace]; - self.cf_handle(unsafe { std::str::from_utf8_unchecked(subspace) }) - .unwrap() + self.cf_handle(crate::subspace_name(subspace)).unwrap() } } diff --git a/crates/store/src/backend/rocksdb/read.rs b/crates/store/src/backend/rocksdb/read.rs index 4f1c1fe2..fb99c02d 100644 --- a/crates/store/src/backend/rocksdb/read.rs +++ b/crates/store/src/backend/rocksdb/read.rs @@ -2,6 +2,8 @@ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ use super::{RocksDbStore, into_error}; @@ -17,21 +19,17 @@ impl RocksDbStore { { let db = self.db.clone(); self.spawn_worker(move || { - let subspace = &[key.subspace()]; + let subspace = crate::subspace_name(key.subspace()); let key = key.serialize(0); - db.get_pinned_cf( - &db.cf_handle(unsafe { std::str::from_utf8_unchecked(subspace.as_slice()) }) - .unwrap(), - &key, - ) - .map_err(into_error) - .and_then(|value| { - if let Some(value) = value { - U::deserialize_with_key(&key, &value).map(Some) - } else { - Ok(None) - } - }) + db.get_pinned_cf(&db.cf_handle(subspace).unwrap(), &key) + .map_err(into_error) + .and_then(|value| { + if let Some(value) = value { + U::deserialize_with_key(&key, &value).map(Some) + } else { + Ok(None) + } + }) }) .await } @@ -39,15 +37,11 @@ impl RocksDbStore { pub(crate) async fn key_exists(&self, key: impl Key) -> trc::Result { let db = self.db.clone(); self.spawn_worker(move || { - let subspace = &[key.subspace()]; + let subspace = crate::subspace_name(key.subspace()); let key = key.serialize(0); - db.get_pinned_cf( - &db.cf_handle(unsafe { std::str::from_utf8_unchecked(subspace.as_slice()) }) - .unwrap(), - &key, - ) - .map_err(into_error) - .map(|value| value.is_some()) + db.get_pinned_cf(&db.cf_handle(subspace).unwrap(), &key) + .map_err(into_error) + .map(|value| value.is_some()) }) .await } diff --git a/crates/store/src/backend/rocksdb/write.rs b/crates/store/src/backend/rocksdb/write.rs index 29cd2fc4..3481fae5 100644 --- a/crates/store/src/backend/rocksdb/write.rs +++ b/crates/store/src/backend/rocksdb/write.rs @@ -74,8 +74,7 @@ impl RocksDbStore { let db = self.db.clone(); self.spawn_worker(move || { db.delete_range_cf( - &db.cf_handle(std::str::from_utf8(&[from.subspace()]).unwrap()) - .unwrap(), + &db.cf_handle(crate::subspace_name(from.subspace())).unwrap(), from.serialize(0), to.serialize(0), ) @@ -88,9 +87,7 @@ impl RocksDbStore { let db = self.db.clone(); self.spawn_worker(move || { for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] { - let cf = db - .cf_handle(std::str::from_utf8(&[subspace]).unwrap()) - .unwrap(); + let cf = db.cf_handle(crate::subspace_name(subspace)).unwrap(); let mut delete_keys = Vec::new(); @@ -174,7 +171,10 @@ impl RocksDBTransaction<'_, '_> { .transaction_opt(&WriteOptions::default(), &self.txn_opts); if has_changes { - let cf = self.db.cf_handle("n").unwrap(); + let cf = self + .db + .cf_handle(crate::subspace_name(crate::SUBSPACE_COUNTER)) + .unwrap(); for &account_id in self.batch.changes.keys() { let key = ValueClass::ChangeId.serialize(account_id, 0, 0, 0); let change_id = txn diff --git a/crates/store/src/backend/sqlite/main.rs b/crates/store/src/backend/sqlite/main.rs index 0c510bed..6bce693d 100644 --- a/crates/store/src/backend/sqlite/main.rs +++ b/crates/store/src/backend/sqlite/main.rs @@ -85,7 +85,7 @@ impl SqliteStore { SUBSPACE_SEARCH_INDEX, SUBSPACE_DIRECTORY, ] { - let table = char::from(table); + let table = crate::subspace_name(table); conn.execute( &format!( "CREATE TABLE IF NOT EXISTS {table} ( @@ -99,7 +99,7 @@ impl SqliteStore { } for table in [SUBSPACE_INDEXES, SUBSPACE_REGISTRY_IDX] { - let table = char::from(table); + let table = crate::subspace_name(table); conn.execute( &format!( "CREATE TABLE IF NOT EXISTS {table} ( @@ -118,7 +118,7 @@ impl SqliteStore { k BLOB PRIMARY KEY, v INTEGER NOT NULL DEFAULT 0 )", - char::from(table) + crate::subspace_name(table) ), [], ) @@ -126,7 +126,7 @@ impl SqliteStore { } // inbuxa: SL-10: in-memory counters keep their expiry in their row - let table = char::from(SUBSPACE_IN_MEMORY_COUNTER); + let table = crate::subspace_name(SUBSPACE_IN_MEMORY_COUNTER); let has_expiry = conn .prepare(&format!( "SELECT 1 FROM pragma_table_info('{table}') WHERE name = 'e'" diff --git a/crates/store/src/backend/sqlite/read.rs b/crates/store/src/backend/sqlite/read.rs index c4fe10df..afd014b6 100644 --- a/crates/store/src/backend/sqlite/read.rs +++ b/crates/store/src/backend/sqlite/read.rs @@ -2,6 +2,8 @@ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ use super::{SqliteStore, into_error}; @@ -19,7 +21,7 @@ impl SqliteStore { let mut result = conn .prepare_cached(&format!( "SELECT v FROM {} WHERE k = ?", - char::from(key.subspace()) + crate::subspace_name(key.subspace()) )) .map_err(into_error)?; let key = key.serialize(0); @@ -41,7 +43,7 @@ impl SqliteStore { let mut result = conn .prepare_cached(&format!( "SELECT 1 FROM {} WHERE k = ?", - char::from(key.subspace()) + crate::subspace_name(key.subspace()) )) .map_err(into_error)?; let key = key.serialize(0); @@ -62,7 +64,7 @@ impl SqliteStore { let manager = self.conn_pool.clone(); self.spawn_worker(move || { let conn = manager.get().map_err(into_error)?; - let table = char::from(params.begin.subspace()); + let table = crate::subspace_name(params.begin.subspace()); let begin = params.begin.serialize(0); let end = params.end.serialize(0); let keys = if params.values { "k, v" } else { "k" }; @@ -132,7 +134,7 @@ impl SqliteStore { key: impl Into> + Sync + Send, ) -> trc::Result { let key = key.into(); - let table = char::from(key.subspace()); + let table = crate::subspace_name(key.subspace()); let key = key.serialize(0); let manager = self.conn_pool.clone(); self.spawn_worker(move || { diff --git a/crates/store/src/backend/sqlite/write.rs b/crates/store/src/backend/sqlite/write.rs index d5b3b82b..b5d04121 100644 --- a/crates/store/src/backend/sqlite/write.rs +++ b/crates/store/src/backend/sqlite/write.rs @@ -73,7 +73,7 @@ impl SqliteStore { Operation::Value { class, op } => { let key = class.serialize(account_id, collection, document_id, 0); let subspace = class.subspace(collection); - let table = char::from(subspace); + let table = crate::subspace_name(subspace); match op { ValueOp::Set(value) => { @@ -271,7 +271,7 @@ impl SqliteStore { assert_value, } => { let key = class.serialize(account_id, collection, document_id, 0); - let table = char::from(class.subspace(collection)); + let table = crate::subspace_name(class.subspace(collection)); let matches = trx .prepare_cached(&format!("SELECT v FROM {} WHERE k = ?", table)) @@ -306,12 +306,15 @@ impl SqliteStore { self.spawn_worker(move || { let conn = manager.get().map_err(into_error)?; for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] { - conn.prepare_cached(&format!("DELETE FROM {} WHERE v = 0", char::from(subspace),)) - .map_err(into_error) - .caused_by(trc::location!())? - .execute([]) - .map_err(into_error) - .caused_by(trc::location!())?; + conn.prepare_cached(&format!( + "DELETE FROM {} WHERE v = 0", + crate::subspace_name(subspace), + )) + .map_err(into_error) + .caused_by(trc::location!())? + .execute([]) + .map_err(into_error) + .caused_by(trc::location!())?; } Ok(()) @@ -326,7 +329,7 @@ impl SqliteStore { let conn = manager.get().map_err(into_error)?; conn.prepare_cached(&format!( "DELETE FROM {} WHERE e <= ?", - char::from(SUBSPACE_IN_MEMORY_COUNTER) + crate::subspace_name(SUBSPACE_IN_MEMORY_COUNTER) )) .map_err(into_error) .caused_by(trc::location!())? @@ -345,7 +348,7 @@ impl SqliteStore { conn.prepare_cached(&format!( "DELETE FROM {} WHERE k >= ? AND k < ?", - char::from(from.subspace()), + crate::subspace_name(from.subspace()), )) .map_err(into_error) .caused_by(trc::location!())? diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index fa6ea1f8..51f71f1c 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -142,6 +142,38 @@ pub const SUBSPACE_INBUXA: u8 = b'_'; pub const LEGACY_SUBSPACE_BITMAP_TEXT: u8 = b'v'; pub const LEGACY_SUBSPACE_BITMAP_TAG: u8 = b'c'; +// inbuxa: SL-15: every lowercase letter is taken, so later subspaces take id +// bytes from 0x80 up, which are never letters, and a name of their own. +pub const SUBSPACE_NAMED_FIRST: u8 = 0x80; + +/// One-byte names for the subspaces up to 0x7f, which are ASCII. +const ASCII_NAMES: &[u8; 128] = &{ + let mut names = [0u8; 128]; + let mut byte = 0; + while byte < 128 { + names[byte] = byte as u8; + byte += 1; + } + names +}; + +/// inbuxa: SL-15: the name a backend gives a subspace: its SQL table and its +/// RocksDB column family. A subspace below [`SUBSPACE_NAMED_FIRST`] keeps its +/// one-byte name, so nothing changes for upstream's subspaces or for `_`. +/// One from 0x80 up is named `ib_…`; one with no name is a bug. +pub const fn subspace_name(subspace: u8) -> &'static str { + if subspace < SUBSPACE_NAMED_FIRST { + let (_, rest) = ASCII_NAMES.as_slice().split_at(subspace as usize); + let (name, _) = rest.split_at(1); + match std::str::from_utf8(name) { + Ok(name) => name, + Err(_) => unreachable!(), + } + } else { + panic!("a subspace from 0x80 up has no name") + } +} + #[derive(Clone)] pub struct IterateParams { begin: T, @@ -764,3 +796,58 @@ impl From> for () { unreachable!() } } + +#[cfg(test)] +mod subspace_names { + use super::*; + + // inbuxa: SL-15: renaming a subspace would orphan its table or column + // family, so every one below 0x80 keeps the name it always had. + #[test] + fn one_byte_subspaces_keep_their_names() { + for subspace in [ + SUBSPACE_ACL, + SUBSPACE_TASK_QUEUE, + SUBSPACE_INDEXES, + SUBSPACE_BLOB_LINK, + SUBSPACE_BLOBS, + SUBSPACE_LOGS, + SUBSPACE_COUNTER, + SUBSPACE_IN_MEMORY_VALUE, + SUBSPACE_IN_MEMORY_COUNTER, + SUBSPACE_PROPERTY, + SUBSPACE_REGISTRY, + SUBSPACE_REGISTRY_IDX, + SUBSPACE_REGISTRY_PK, + SUBSPACE_DIRECTORY, + SUBSPACE_QUEUE_MESSAGE, + SUBSPACE_QUEUE_EVENT, + SUBSPACE_QUOTA, + SUBSPACE_REPORT_OUT, + SUBSPACE_REPORT_IN, + SUBSPACE_TELEMETRY_SPAN, + SUBSPACE_TELEMETRY_METRIC, + SUBSPACE_SEARCH_INDEX, + SUBSPACE_DELETED_ITEMS, + SUBSPACE_SPAM_SAMPLES, + SUBSPACE_INBUXA, + LEGACY_SUBSPACE_BITMAP_TEXT, + LEGACY_SUBSPACE_BITMAP_TAG, + ] { + assert_eq!(subspace_name(subspace).as_bytes(), [subspace]); + } + assert_eq!(subspace_name(b'_'), "_"); + } + + #[test] + fn names_work_in_const_context() { + const NAME: &str = subspace_name(SUBSPACE_LOGS); + assert_eq!(NAME, "l"); + } + + #[test] + #[should_panic] + fn an_unnamed_high_byte_is_a_bug() { + subspace_name(SUBSPACE_NAMED_FIRST); + } +} diff --git a/tests/src/store/import_export.rs b/tests/src/store/import_export.rs index 3e95472e..d4a91458 100644 --- a/tests/src/store/import_export.rs +++ b/tests/src/store/import_export.rs @@ -247,7 +247,7 @@ pub async fn test(test: &TestServer) { assert!( snapshot.keys.iter().any(|k| k.subspace == subspace), "No test data in subspace {}", - char::from(subspace) + store::subspace_name(subspace) ); } @@ -290,7 +290,7 @@ pub async fn test(test: &TestServer) { assert!( imported.contains(&subspace), "Subspace {} was not exported", - char::from(subspace) + store::subspace_name(subspace) ); } @@ -383,7 +383,7 @@ async fn cross_backend( source.get_counter(class()).await.unwrap(), target.get_counter(class()).await.unwrap(), "Counter mismatch in {} for {key:?}", - char::from(subspace) + store::subspace_name(subspace) ); } } @@ -548,7 +548,7 @@ impl Snapshot { if !other.keys.contains(key) { println!( "Subspace {}, Key {:?} not found in restored snapshot", - char::from(key.subspace), + store::subspace_name(key.subspace), key.key, ); is_err = true; @@ -558,7 +558,7 @@ impl Snapshot { if !self.keys.contains(key) { println!( "Subspace {}, Key {:?} not found in original snapshot", - char::from(key.subspace), + store::subspace_name(key.subspace), key.key, ); is_err = true; diff --git a/tests/src/store/sql_timeout.rs b/tests/src/store/sql_timeout.rs index d69f0a4b..7ec79e33 100644 --- a/tests/src/store/sql_timeout.rs +++ b/tests/src/store/sql_timeout.rs @@ -2,6 +2,8 @@ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ use crate::utils::{server::TestServer, storage::build_data_store}; @@ -284,7 +286,7 @@ impl Backend { } fn populate_query(&self) -> String { - let table = char::from(SUBSPACE_PROPERTY); + let table = store::subspace_name(SUBSPACE_PROPERTY); let prefix = key_prefix(); let padding = VALUE_SIZE - U32_LEN; let last = NUM_KEYS - 1; @@ -308,7 +310,7 @@ impl Backend { } fn populate_counters_query(&self) -> String { - let table = char::from(SUBSPACE_COUNTER); + let table = store::subspace_name(SUBSPACE_COUNTER); let prefix = key_prefix(); let last = NUM_KEYS - 1; @@ -328,11 +330,14 @@ impl Backend { } fn purge_query(&self) -> String { - format!("DELETE FROM {} WHERE v = 0", char::from(SUBSPACE_COUNTER)) + format!( + "DELETE FROM {} WHERE v = 0", + store::subspace_name(SUBSPACE_COUNTER) + ) } fn count_counters_query(&self) -> String { - let table = char::from(SUBSPACE_COUNTER); + let table = store::subspace_name(SUBSPACE_COUNTER); match self { #[cfg(feature = "postgres")] Backend::Postgres => format!("SELECT COUNT(*) FROM {table} WHERE k >= $1 AND k <= $2"), @@ -342,7 +347,7 @@ impl Backend { } fn scan_query(&self) -> String { - let table = char::from(SUBSPACE_PROPERTY); + let table = store::subspace_name(SUBSPACE_PROPERTY); match self { #[cfg(feature = "postgres")] Backend::Postgres => { @@ -356,7 +361,7 @@ impl Backend { } fn delete_query(&self) -> String { - let table = char::from(SUBSPACE_PROPERTY); + let table = store::subspace_name(SUBSPACE_PROPERTY); match self { #[cfg(feature = "postgres")] Backend::Postgres => format!("DELETE FROM {table} WHERE k >= $1 AND k <= $2"), diff --git a/tests/src/utils/cleanup.rs b/tests/src/utils/cleanup.rs index 4e1d5e41..d206a090 100644 --- a/tests/src/utils/cleanup.rs +++ b/tests/src/utils/cleanup.rs @@ -362,7 +362,7 @@ pub async fn store_assert_is_empty(store: &Store, blob_store: BlobStore, include _ => { println!( "Found key in {:?}: {:?} ({:?}) = {:?} ({:?})", - char::from(subspace), + store::subspace_name(subspace), key, String::from_utf8_lossy(key), value, -- 2.54.0 From e492880fa231032dcb6f2cf7089ff49061fa9f1d Mon Sep 17 00:00:00 2001 From: John Coffey Date: Tue, 10 Nov 2026 22:47:02 -0700 Subject: [PATCH 2/2] Store layout step 2 -> 3: the fork's large data gets subspaces of its own The audit log, the journal's entries and archive, legal holds and undelete's notes move out of _ into ib_audit, ib_journal, ib_holds and ib_undelete, each key without the letter that named its feature. The audit settings and the journals themselves stay in _, where a configuration export carries them. Exports carry the moved data in a compliance family of its own, on by default (SL-19). The step lists the two-byte prefixes in _ and stops before writing anything on a key no feature claims (SL-16). It copies each moving prefix, on SQL with one INSERT ... SELECT, checks counts and the audit and journal chain heads and floors, and only then deletes the originals and drops RocksDB's legacy v and c (SL-17, SL-20). It isn't rolling-safe, so it waits for every other node to stop (SL-18). INBUXA_STORE_VERSION is 3. --- crates/common/src/lib.rs | 4 +- crates/common/src/manager/backup.rs | 14 +- crates/features/src/audit/log.rs | 33 +- crates/features/src/hold/mod.rs | 23 +- crates/features/src/journal/archive.rs | 12 +- crates/features/src/journal/entries.rs | 21 +- crates/features/src/journal/mod.rs | 8 +- crates/features/src/undelete/data.rs | 26 +- crates/features/src/undelete/groupware.rs | 6 +- crates/migration/src/fork_subspaces.rs | 547 +++++++++++++++++++++ crates/migration/src/layout.rs | 21 +- crates/migration/src/lib.rs | 1 + crates/store/src/backend/mysql/main.rs | 5 + crates/store/src/backend/mysql/write.rs | 32 ++ crates/store/src/backend/postgres/main.rs | 5 + crates/store/src/backend/postgres/write.rs | 31 ++ crates/store/src/backend/rocksdb/main.rs | 84 +++- crates/store/src/backend/sqlite/main.rs | 5 + crates/store/src/backend/sqlite/write.rs | 29 ++ crates/store/src/dispatch/store.rs | 41 ++ crates/store/src/lib.rs | 23 +- docs/spec/features/store-layout.md | 49 +- tests/src/store/import_export.rs | 29 +- tests/src/store_version.rs | 366 +++++++++++++- tests/src/utils/cleanup.rs | 5 + 25 files changed, 1340 insertions(+), 80 deletions(-) create mode 100644 crates/migration/src/fork_subspaces.rs diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 303b0d61..de1e8ba6 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -116,7 +116,9 @@ pub const DATABASE_SCHEMA_VERSION: u32 = 6; // 1 - the 0.16 layout plus the fork's own subspace (2026.10.x, 27.1-rc.16) // 2 - in-memory counters on SQL keep their own expiry (store-layout spec // SL-10); the steps are in crates/migration/src/layout.rs -pub const INBUXA_STORE_VERSION: u32 = 2; +// 3 - the audit log, journal, holds and undelete notes have subspaces of +// their own (SL-16), and RocksDB's legacy `v` and `c` are gone (SL-20) +pub const INBUXA_STORE_VERSION: u32 = 3; pub const INBUXA_STORE_VERSION_KEY: &[u8] = b"Vs"; pub const LONG_1D_SLUMBER: Duration = Duration::from_secs(60 * 60 * 24); diff --git a/crates/common/src/manager/backup.rs b/crates/common/src/manager/backup.rs index d32d9cda..c7bdddc4 100644 --- a/crates/common/src/manager/backup.rs +++ b/crates/common/src/manager/backup.rs @@ -40,6 +40,8 @@ pub(super) enum Family { Report = 6, Telemetry = 7, Tasks = 8, + // inbuxa: SL-19: the fork's large data (store layout version 3) + Compliance = 9, } type TaskHandle = (tokio::task::JoinHandle<()>, std::thread::JoinHandle<()>); @@ -81,6 +83,7 @@ impl Core { Family::Report, Family::Telemetry, Family::Tasks, + Family::Compliance, ] .into_iter() .collect(); @@ -342,13 +345,21 @@ impl Family { // travel in this family and in `data`, so they ride along. SUBSPACE_DELETED_ITEMS, SUBSPACE_SPAM_SAMPLES, - store::SUBSPACE_INBUXA, // inbuxa: the fork's own data (masked email, undelete, policies) + store::SUBSPACE_INBUXA, // inbuxa: the fork's own settings and small records (masked email, policies) ], Family::Changelog => &[SUBSPACE_LOGS], Family::Queue => &[SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUEUE_EVENT], Family::Report => &[SUBSPACE_REPORT_OUT, SUBSPACE_REPORT_IN], Family::Telemetry => &[SUBSPACE_TELEMETRY_SPAN, SUBSPACE_TELEMETRY_METRIC], Family::Tasks => &[SUBSPACE_TASK_QUEUE], + // inbuxa: SL-19: in a family of its own, so a configuration + // export (`registry`) can leave out the audit log and journal + Family::Compliance => &[ + store::SUBSPACE_AUDIT, + store::SUBSPACE_JOURNAL, + store::SUBSPACE_HOLDS, + store::SUBSPACE_UNDELETE, + ], } } @@ -362,6 +373,7 @@ impl Family { "report" => Ok(Family::Report), "telemetry" => Ok(Family::Telemetry), "tasks" => Ok(Family::Tasks), + "compliance" => Ok(Family::Compliance), _ => Err(format!("Unknown family {}", family)), } } diff --git a/crates/features/src/audit/log.rs b/crates/features/src/audit/log.rs index 67e1e2d9..e113964f 100644 --- a/crates/features/src/audit/log.rs +++ b/crates/features/src/audit/log.rs @@ -4,9 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -//! The audit log's storage (AU-2, AU-3, AU-6, AU-7), in the fork's own -//! subspace (`store::SUBSPACE_INBUXA`). Every key starts with `L`, then one -//! byte for the kind: +//! The audit log's storage (AU-2, AU-3, AU-6, AU-7), in a subspace of its +//! own (`store::SUBSPACE_AUDIT`, store layout version 3). Every key starts +//! with one byte for the kind: //! //! - `e` + node + seq: one entry of that node's chain, as JSON. An entry is //! an event, or the outcome of an event written before its change was @@ -18,7 +18,8 @@ //! both add the same seq. //! - `f` + node: where the chain starts after purging, and the hash the //! first kept entry names. -//! - `s`: the settings (`keepFor`). +//! - `s`: the settings (`keepFor`). These stay in the fork's `_` subspace, +//! as `L` + `s`, so a configuration export carries them. //! //! Numbers are big-endian, so keys sort in time and chain order. Each node //! writes only its own chain, so nodes never contend for a key; nothing about @@ -31,7 +32,7 @@ use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize}; use sha2::{Digest, Sha256}; use std::{fmt, net::IpAddr, str::FromStr}; use store::{ - Deserialize, IterateParams, SUBSPACE_INBUXA, Serialize, Store, ValueKey, + Deserialize, IterateParams, SUBSPACE_AUDIT, SUBSPACE_INBUXA, Serialize, Store, ValueKey, write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue}, }; use tokio::sync::Mutex; @@ -229,16 +230,17 @@ impl Deserialize for U64 { } fn class(kind: u8, parts: &[u64]) -> ValueClass { - let mut key = Vec::with_capacity(2 + parts.len() * 8); - key.push(FEATURE); + // SL-16: the settings stay in `_`; the log has a subspace of its own + let (subspace, mut key) = if kind == KIND_SETTINGS { + (SUBSPACE_INBUXA, vec![FEATURE]) + } else { + (SUBSPACE_AUDIT, Vec::with_capacity(1 + parts.len() * 8)) + }; key.push(kind); for part in parts { key.extend_from_slice(&part.to_be_bytes()); } - ValueClass::Any(AnyClass { - subspace: SUBSPACE_INBUXA, - key, - }) + ValueClass::Any(AnyClass { subspace, key }) } fn key(kind: u8, parts: &[u64]) -> ValueKey { @@ -259,11 +261,11 @@ pub fn head_key(node: u64) -> ValueKey { /// The numbers after the kind byte, read from the key's tail: the iterator /// may or may not hand back the subspace byte. fn parse_key(key: &[u8], kind: u8, parts: usize) -> Option> { - let len = 2 + parts * 8; + let len = 1 + parts * 8; let tail = key.get(key.len().checked_sub(len)?..)?; - (tail[0] == FEATURE && tail[1] == kind).then_some(())?; + (tail[0] == kind).then_some(())?; Some( - tail[2..] + tail[1..] .chunks_exact(8) .map(|chunk| u64::from_be_bytes(chunk.try_into().unwrap())) .collect(), @@ -875,7 +877,8 @@ mod tests { panic!() }; assert_eq!(parse_key(&any.key, KIND_TIME, 3), Some(vec![5, 3, 9])); - let mut with_subspace = vec![SUBSPACE_INBUXA]; + assert_eq!(any.subspace, SUBSPACE_AUDIT); + let mut with_subspace = vec![SUBSPACE_AUDIT]; with_subspace.extend_from_slice(&any.key); assert_eq!(parse_key(&with_subspace, KIND_TIME, 3), Some(vec![5, 3, 9])); assert_eq!(parse_key(&any.key, KIND_ENTRY, 3), None); diff --git a/crates/features/src/hold/mod.rs b/crates/features/src/hold/mod.rs index bbfecf44..7cab687d 100644 --- a/crates/features/src/hold/mod.rs +++ b/crates/features/src/hold/mod.rs @@ -11,8 +11,9 @@ //! While any active hold covers an item, nothing may destroy it. A hold is //! never deleted: releasing it keeps it, read-only, for the audit trail. //! -//! Kept in the fork's subspace (`store::SUBSPACE_INBUXA`). Every key starts -//! with `H`, then one byte for the kind: +//! Kept in a subspace of their own (`store::SUBSPACE_HOLDS`, store layout +//! version 3; before it, in `_` under `H`). Every key starts with one byte +//! for the kind: //! //! - `h` + hold id (u32): the hold, as JSON. //! @@ -21,7 +22,7 @@ use registry::schema::{prelude::ObjectInner, structs::Account}; use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize}; use store::{ - Deserialize, IterateParams, SUBSPACE_INBUXA, Serialize, Store, ValueKey, + Deserialize, IterateParams, SUBSPACE_HOLDS, Serialize, Store, ValueKey, write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue}, }; use trc::AddContext; @@ -103,7 +104,6 @@ impl Keeping { } } -const FEATURE: u8 = b'H'; const KIND_HOLD: u8 = b'h'; const KIND_ORIGINAL: u8 = b'o'; const KIND_EXPORT: u8 = b'e'; @@ -401,12 +401,11 @@ impl Deserialize for Json { } fn class(id: u32) -> ValueClass { - let mut key = Vec::with_capacity(6); - key.push(FEATURE); + let mut key = Vec::with_capacity(5); key.push(KIND_HOLD); key.extend_from_slice(&id.to_be_bytes()); ValueClass::Any(AnyClass { - subspace: SUBSPACE_INBUXA, + subspace: SUBSPACE_HOLDS, key, }) } @@ -416,12 +415,11 @@ fn key(id: u32) -> ValueKey { } fn original_class(item_id: u64) -> ValueClass { - let mut key = Vec::with_capacity(10); - key.push(FEATURE); + let mut key = Vec::with_capacity(9); key.push(KIND_ORIGINAL); key.extend_from_slice(&item_id.to_be_bytes()); ValueClass::Any(AnyClass { - subspace: SUBSPACE_INBUXA, + subspace: SUBSPACE_HOLDS, key, }) } @@ -454,12 +452,11 @@ pub async fn set_original_deadline( } fn export_class(id: u32) -> ValueClass { - let mut key = Vec::with_capacity(6); - key.push(FEATURE); + let mut key = Vec::with_capacity(5); key.push(KIND_EXPORT); key.extend_from_slice(&id.to_be_bytes()); ValueClass::Any(AnyClass { - subspace: SUBSPACE_INBUXA, + subspace: SUBSPACE_HOLDS, key, }) } diff --git a/crates/features/src/journal/archive.rs b/crates/features/src/journal/archive.rs index 08a95f7b..c18de5d0 100644 --- a/crates/features/src/journal/archive.rs +++ b/crates/features/src/journal/archive.rs @@ -4,7 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -//! Reports on their way to an outside archive (JR-7). Keys, after `J`: +//! Reports on their way to an outside archive (JR-7), in the journal's +//! subspace (`store::SUBSPACE_JOURNAL`). Keys: //! //! - `o` + the report's queue id: what goes into the built-in journal if //! the archive never takes the report, as JSON. Cleared once it's @@ -12,10 +13,10 @@ //! - `w` + journal id (u32): how often that journal's archive didn't take a //! report, and the last time and reason, for the console's warning. -use super::{FEATURE, Json, entries::Entry}; +use super::{Json, entries::Entry}; use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize}; use store::{ - SUBSPACE_INBUXA, Serialize, Store, ValueKey, + SUBSPACE_JOURNAL, Serialize, Store, ValueKey, write::{AnyClass, BatchBuilder, ValueClass}, }; use trc::AddContext; @@ -44,12 +45,11 @@ pub struct Failures { } fn class(kind: u8, id: &[u8]) -> ValueClass { - let mut key = Vec::with_capacity(2 + id.len()); - key.push(FEATURE); + let mut key = Vec::with_capacity(1 + id.len()); key.push(kind); key.extend_from_slice(id); ValueClass::Any(AnyClass { - subspace: SUBSPACE_INBUXA, + subspace: SUBSPACE_JOURNAL, key, }) } diff --git a/crates/features/src/journal/entries.rs b/crates/features/src/journal/entries.rs index 0683ba20..157c4436 100644 --- a/crates/features/src/journal/entries.rs +++ b/crates/features/src/journal/entries.rs @@ -4,7 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -//! The built-in journal (JR-5, JR-6, JR-13). Keys, after `J`: +//! The built-in journal (JR-5, JR-6, JR-13), in a subspace of its own +//! (`store::SUBSPACE_JOURNAL`, store layout version 3). Keys: //! //! - `e` + node + seq: a chain link: its seq, the hash of the link before //! it, and the SHA-256 of its entry. One chain per node, as the audit log @@ -26,13 +27,13 @@ //! its entry is purged. Nothing here changes or removes an entry before //! its time; nothing in JMAP can. -use super::{Direction, FEATURE, Json}; +use super::{Direction, Json}; use crate::hold::HELD_UNTIL; use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize}; use sha2::{Digest, Sha256}; use std::fmt; use store::{ - BlobStore, Deserialize, IterateParams, SUBSPACE_INBUXA, Serialize, Store, ValueKey, + BlobStore, Deserialize, IterateParams, SUBSPACE_JOURNAL, Serialize, Store, ValueKey, write::{AnyClass, BatchBuilder, BlobLink, BlobOp, ValueClass, assert::AssertValue}, }; use tokio::sync::Mutex; @@ -164,14 +165,13 @@ impl Deserialize for Raw { } fn class(kind: u8, parts: &[u64]) -> ValueClass { - let mut key = Vec::with_capacity(2 + parts.len() * 8); - key.push(FEATURE); + let mut key = Vec::with_capacity(1 + parts.len() * 8); key.push(kind); for part in parts { key.extend_from_slice(&part.to_be_bytes()); } ValueClass::Any(AnyClass { - subspace: SUBSPACE_INBUXA, + subspace: SUBSPACE_JOURNAL, key, }) } @@ -187,11 +187,11 @@ pub fn content_key(id: EntryId) -> ValueKey { /// The numbers after the kind byte, from the key's tail. fn parse_key(key: &[u8], kind: u8, parts: usize) -> Option> { - let len = 2 + parts * 8; + let len = 1 + parts * 8; let tail = key.get(key.len().checked_sub(len)?..)?; - (tail[0] == FEATURE && tail[1] == kind).then_some(())?; + (tail[0] == kind).then_some(())?; Some( - tail[2..] + tail[1..] .chunks_exact(8) .map(|chunk| u64::from_be_bytes(chunk.try_into().unwrap())) .collect(), @@ -777,7 +777,8 @@ mod tests { panic!() }; assert_eq!(parse_key(&any.key, KIND_EXPIRY, 3), Some(vec![5, 3, 9])); - let mut with_subspace = vec![SUBSPACE_INBUXA]; + assert_eq!(any.subspace, SUBSPACE_JOURNAL); + let mut with_subspace = vec![SUBSPACE_JOURNAL]; with_subspace.extend_from_slice(&any.key); assert_eq!( parse_key(&with_subspace, KIND_EXPIRY, 3), diff --git a/crates/features/src/journal/mod.rs b/crates/features/src/journal/mod.rs index 6e5bd283..2084290f 100644 --- a/crates/features/src/journal/mod.rs +++ b/crates/features/src/journal/mod.rs @@ -12,9 +12,11 @@ //! - [`report`]: the journal report around the untouched message (JR-3); //! - [`entries`]: the built-in journal and its chain (JR-5, JR-6, JR-13). //! -//! Kept in the fork's subspace (`store::SUBSPACE_INBUXA`). Every key starts -//! with `J`; journals are `j` + id (u32), as JSON. There are few, so they're -//! read whole. +//! Journals are settings, so they're kept in the fork's subspace +//! (`store::SUBSPACE_INBUXA`), where a configuration export carries them: +//! `J` + `j` + id (u32), as JSON. There are few, so they're read whole. What +//! they journal has a subspace of its own (`store::SUBSPACE_JOURNAL`; see +//! [`entries`] and [`archive`]). pub mod archive; pub mod entries; diff --git a/crates/features/src/undelete/data.rs b/crates/features/src/undelete/data.rs index 102aab9a..26b608f2 100644 --- a/crates/features/src/undelete/data.rs +++ b/crates/features/src/undelete/data.rs @@ -4,8 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -//! What undelete keeps in the fork's own subspace (`store::SUBSPACE_INBUXA`). -//! Every key starts with `U`, then one byte for the kind: +//! What undelete keeps, in a subspace of its own (`store::SUBSPACE_UNDELETE`, +//! store layout version 3; before it, in `_` under `U`). Every key starts +//! with one byte for the kind: //! //! - `n` + account + document: a deleted email waiting for its archive //! record, with what only the deletion knows (mailboxes, keywords, size) @@ -19,18 +20,19 @@ //! `/changes`. //! - `k` + account id: a deleted account kept for its period (UD-15a). //! - `a` + address: an address a kept account holds reserved (UD-16). +//! - `g` + kind + account + document: a deleted calendar event, contact or +//! file ([`super::groupware`]). //! //! Values are JSON, so they read back across versions of the fork. use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize, de::DeserializeOwned}; use store::{ - Deserialize, IterateParams, SUBSPACE_INBUXA, Serialize, Store, ValueKey, + Deserialize, IterateParams, SUBSPACE_UNDELETE, Serialize, Store, ValueKey, write::{AnyClass, BatchBuilder, ValueClass}, }; use trc::AddContext; use types::id::Id; -const FEATURE: u8 = b'U'; const KIND_NOTE: u8 = b'n'; const KIND_EXTRA: u8 = b'x'; const KIND_BLOB: u8 = b'b'; @@ -40,12 +42,11 @@ const KIND_KEPT: u8 = b'k'; const KIND_RESERVED: u8 = b'a'; fn class(kind: u8, rest: &[u8]) -> ValueClass { - let mut key = Vec::with_capacity(2 + rest.len()); - key.push(FEATURE); + let mut key = Vec::with_capacity(1 + rest.len()); key.push(kind); key.extend_from_slice(rest); ValueClass::Any(AnyClass { - subspace: SUBSPACE_INBUXA, + subspace: SUBSPACE_UNDELETE, key, }) } @@ -416,7 +417,7 @@ pub async fn clear_account(data: &Store, account_id: u32) -> trc::Result<()> { (vec![KIND_BLOB], vec![KIND_BLOB]), (vec![KIND_CHANGE], vec![KIND_CHANGE]), ]; - // The groupware notes, `Ug` + kind + account + document + // The groupware notes, `g` + kind + account + document for kind in 0u8..3 { ranges.push((vec![b'g', kind], vec![b'g', kind])); } @@ -433,14 +434,11 @@ pub async fn clear_account(data: &Store, account_id: u32) -> trc::Result<()> { Ok(()) } -/// A key under `U` spelled out in full (the groupware notes, `Ug`). +/// A key spelled out in full (the groupware notes, `g`). fn class_raw(rest: &[u8]) -> ValueClass { - let mut key = Vec::with_capacity(1 + rest.len()); - key.push(FEATURE); - key.extend_from_slice(rest); ValueClass::Any(AnyClass { - subspace: SUBSPACE_INBUXA, - key, + subspace: SUBSPACE_UNDELETE, + key: rest.to_vec(), }) } diff --git a/crates/features/src/undelete/groupware.rs b/crates/features/src/undelete/groupware.rs index a61ccded..6a8ead02 100644 --- a/crates/features/src/undelete/groupware.rs +++ b/crates/features/src/undelete/groupware.rs @@ -50,13 +50,13 @@ pub struct Note { } fn note_class(kind: Kind, account_id: u32, document_id: u32) -> ValueClass { - let mut key = Vec::with_capacity(11); - key.extend_from_slice(b"Ug"); + let mut key = Vec::with_capacity(10); + key.push(b'g'); key.push(kind as u8); key.extend_from_slice(&account_id.to_be_bytes()); key.extend_from_slice(&document_id.to_be_bytes()); ValueClass::Any(AnyClass { - subspace: store::SUBSPACE_INBUXA, + subspace: store::SUBSPACE_UNDELETE, key, }) } diff --git a/crates/migration/src/fork_subspaces.rs b/crates/migration/src/fork_subspaces.rs new file mode 100644 index 00000000..138a38d3 --- /dev/null +++ b/crates/migration/src/fork_subspaces.rs @@ -0,0 +1,547 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! inbuxa: store layout step 2 -> 3 (store-layout spec SL-16, SL-17, SL-20). +//! The audit log, the journal, holds and undelete notes move out of the +//! fork's `_` subspace into subspaces of their own, each key without the +//! letter that named its feature. Their settings stay in `_`, where a +//! configuration export carries them. RocksDB's legacy `v` and `c` go. +//! +//! Every key in `_` belongs to a feature, by its first byte, listed in +//! [`OWNERS`]. A key nobody claims stops the step before it changes +//! anything: it would mean a feature this step doesn't know about. + +use store::{ + IterateParams, SUBSPACE_AUDIT, SUBSPACE_HOLDS, SUBSPACE_INBUXA, SUBSPACE_JOURNAL, + SUBSPACE_UNDELETE, Store, ValueKey, + write::{AnyClass, AnyKey, BatchBuilder, ValueClass}, +}; +use trc::AddContext; + +/// Set once every copy has been checked, before the first original is +/// deleted, so a run that stops while deleting doesn't check again against +/// originals it already deleted. Cleared at the end. +pub const CHECKED_KEY: &[u8] = b"Vt"; + +/// Rows copied at a time where the backend can't copy on the server. +const COPY_CHUNK: usize = 10_000; + +/// What happens to the keys under one first byte of `_`. +pub enum Owner { + /// They stay in `_`. + Stays(&'static str), + /// They move to `to`, without the first byte. `kinds` are the second + /// bytes that move and `stays` those that stay; any other stops the step. + Moves { + what: &'static str, + to: u8, + kinds: &'static [u8], + stays: &'static [u8], + /// Kinds whose values are compared row for row after the copy, not + /// only counted: the chain heads and floors. + compared: &'static [u8], + }, +} + +/// SL-16: every first byte in use in `_`, and its owner. +pub const OWNERS: &[(u8, Owner)] = &[ + (b'A', Owner::Stays("AI limits")), + (b'B', Owner::Stays("first boot defaults")), + (b'C', Owner::Stays("account snapshots")), + (b'D', Owner::Stays("deliverability reports")), + (b'E', Owner::Stays("inbuxa relay")), + (b'F', Owner::Stays("front ends")), + (b'G', Owner::Stays("scheduled reports")), + ( + b'H', + Owner::Moves { + what: "legal holds", + to: SUBSPACE_HOLDS, + kinds: b"hoe", + stays: b"", + compared: b"h", + }, + ), + (b'I', Owner::Stays("migrations")), + ( + b'J', + Owner::Moves { + what: "journal", + to: SUBSPACE_JOURNAL, + kinds: b"ecptxhfow", + stays: b"j", + compared: b"hf", + }, + ), + (b'K', Owner::Stays("account locks")), + ( + b'L', + Owner::Moves { + what: "audit log", + to: SUBSPACE_AUDIT, + kinds: b"etohf", + stays: b"s", + compared: b"hf", + }, + ), + (b'M', Owner::Stays("masked email")), + (b'P', Owner::Stays("permissions and protocol policies")), + (b'Q', Owner::Stays("acceptance records")), + (b'R', Owner::Stays("mail flow rules")), + ( + b'S', + Owner::Stays("spam rules, scale-out storage and earlier scheduled reports"), + ), + (b'T', Owner::Stays("log files")), + ( + b'U', + Owner::Moves { + what: "undelete notes", + to: SUBSPACE_UNDELETE, + kinds: b"nxbrckag", + stays: b"", + compared: b"", + }, + ), + (b'V', Owner::Stays("store checks and the store layout")), + ( + b'W', + Owner::Stays("blob store changes and the sharing policy"), + ), +]; + +/// Step 2 -> 3. Returns how many rows moved. +pub async fn fork_data_own_subspaces(store: &Store) -> trc::Result { + let moving = moving_prefixes(store).await?; + + // Always copy, even after a run that stopped while deleting: whatever + // is left of the originals is copied again before it goes + for prefix in &moving { + copy(store, prefix).await?; + } + + let checked = store + .key_exists(inbuxa_key(CHECKED_KEY)) + .await + .caused_by(trc::location!())?; + let mut moved = 0; + if !checked { + for prefix in &moving { + moved += check(store, prefix).await?; + } + let mut batch = BatchBuilder::new(); + batch.set(inbuxa_class(CHECKED_KEY), vec![1]); + store + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + } + + for prefix in &moving { + store + .delete_range( + AnyKey { + subspace: SUBSPACE_INBUXA, + key: prefix.from.clone(), + }, + AnyKey { + subspace: SUBSPACE_INBUXA, + key: prefix.to.clone(), + }, + ) + .await + .caused_by(trc::location!())?; + } + + let dropped = store + .drop_legacy_subspaces() + .await + .caused_by(trc::location!())?; + if dropped > 0 { + trc::event!( + Server(trc::ServerEvent::Startup), + Details = "Dropped the legacy v and c column families", + Total = dropped, + ); + } + + let mut batch = BatchBuilder::new(); + batch.clear(inbuxa_class(CHECKED_KEY)); + store + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + + Ok(moved) +} + +/// One second byte under a moving feature: its range in `_` and where it +/// goes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Prefix { + what: &'static str, + /// The two bytes in `_`. + from: Vec, + /// The first key after them in `_`. + to: Vec, + dest: u8, + compared: bool, +} + +/// SL-16: lists the two-byte prefixes in `_` by skipping from one to the +/// next, so it reads one key per prefix, not every key. Stops on any key +/// nobody claims, before anything is written. +pub async fn moving_prefixes(store: &Store) -> trc::Result> { + let mut begin = Vec::new(); + let mut unclaimed = Vec::new(); + let mut moving = Vec::new(); + + loop { + let mut first = None; + store + .iterate( + IterateParams::new(inbuxa_key(&begin), inbuxa_key(&[u8::MAX; 64])) + .no_values() + .only_first(), + |key, _| { + first = Some(key.to_vec()); + Ok(false) + }, + ) + .await + .caused_by(trc::location!())?; + let Some(key) = first else { + break; + }; + + let prefix = &key[..key.len().min(2)]; + match claim(prefix) { + Claim::Stays => {} + Claim::Moves(found) => moving.push(found), + Claim::Nobody => unclaimed.push(printable(prefix)), + } + + // A one-byte key is followed by the longer keys under it + begin = if key.len() == 1 { + vec![key[0], 0] + } else { + match next_prefix(prefix) { + Some(next) => next, + None => break, + } + }; + } + + if unclaimed.is_empty() { + Ok(moving) + } else { + Err(trc::StoreEvent::UnexpectedError + .into_err() + .details(format!( + "Store layout step 2 to 3 found keys in the _ subspace that no feature claims ({}). Nothing was changed", + unclaimed.join(", ") + )) + .caused_by(trc::location!())) + } +} + +enum Claim { + Stays, + Moves(Prefix), + Nobody, +} + +fn claim(prefix: &[u8]) -> Claim { + let Some((_, owner)) = OWNERS.iter().find(|(byte, _)| Some(byte) == prefix.first()) else { + return Claim::Nobody; + }; + match owner { + Owner::Stays(_) => Claim::Stays, + Owner::Moves { + what, + to, + kinds, + stays, + compared, + } => match prefix.get(1) { + Some(kind) if stays.contains(kind) => Claim::Stays, + Some(kind) if kinds.contains(kind) => Claim::Moves(Prefix { + what, + from: prefix.to_vec(), + to: next_prefix(prefix).unwrap_or_else(|| vec![u8::MAX; 64]), + dest: *to, + compared: compared.contains(kind), + }), + _ => Claim::Nobody, + }, + } +} + +/// The first byte string after every key that starts with `prefix`. +fn next_prefix(prefix: &[u8]) -> Option> { + let mut next = prefix.to_vec(); + while let Some(last) = next.pop() { + if last < u8::MAX { + next.push(last + 1); + return Some(next); + } + } + None +} + +fn printable(prefix: &[u8]) -> String { + prefix + .iter() + .map(|&byte| { + if byte.is_ascii_graphic() { + (byte as char).to_string() + } else { + format!("\\x{byte:02x}") + } + }) + .collect() +} + +/// SL-17: on SQL in one statement on the server, elsewhere a row at a +/// time. A row already copied is written again, unchanged. +async fn copy(store: &Store, prefix: &Prefix) -> trc::Result<()> { + let copied = store + .copy_range( + AnyKey { + subspace: SUBSPACE_INBUXA, + key: prefix.from.clone(), + }, + AnyKey { + subspace: SUBSPACE_INBUXA, + key: prefix.to.clone(), + }, + prefix.dest, + 1, + ) + .await + .caused_by(trc::location!())?; + if copied.is_some() { + return Ok(()); + } + + // A chunk at a time, so a large journal isn't held in memory + let mut begin = prefix.from.clone(); + loop { + let mut rows = Vec::new(); + store + .iterate( + IterateParams::new(inbuxa_key(&begin), inbuxa_key(&prefix.to)), + |key, value| { + if key.starts_with(&prefix.from) { + rows.push((key.to_vec(), value.to_vec())); + } + Ok(rows.len() < COPY_CHUNK) + }, + ) + .await + .caused_by(trc::location!())?; + let Some((last, _)) = rows.last() else { + return Ok(()); + }; + let full = rows.len() == COPY_CHUNK; + // The next chunk starts right after the last key of this one + begin = last.clone(); + begin.push(0); + + let mut batch = BatchBuilder::new(); + for (key, value) in rows { + batch.set( + ValueClass::Any(AnyClass { + subspace: prefix.dest, + key: key[1..].to_vec(), + }), + value, + ); + if batch.is_large_batch() { + store + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + batch = BatchBuilder::new(); + } + } + if !batch.is_empty() { + store + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + } + if !full { + return Ok(()); + } + } +} + +/// SL-17: the copy has as many rows as the originals, and the chain heads +/// and floors are the same, byte for byte. Returns the rows checked. +async fn check(store: &Store, prefix: &Prefix) -> trc::Result { + let dest_from = prefix.from[1..].to_vec(); + let dest_to = next_prefix(&dest_from).unwrap_or_else(|| vec![u8::MAX; 64]); + let originals = rows( + store, + SUBSPACE_INBUXA, + &prefix.from, + &prefix.to, + prefix.compared, + ) + .await?; + let copies = rows(store, prefix.dest, &dest_from, &dest_to, prefix.compared).await?; + + let matches = if prefix.compared { + originals.values == copies.values + } else { + originals.count == copies.count + }; + if !matches { + return Err(trc::StoreEvent::UnexpectedError + .into_err() + .details(format!( + "Store layout step 2 to 3: the copy of the {} ({}) doesn't match its originals: {} rows were copied of {}. Nothing was deleted", + prefix.what, + printable(&prefix.from), + copies.count, + originals.count + )) + .caused_by(trc::location!())); + } + + trc::event!( + Server(trc::ServerEvent::Startup), + Details = format!( + "Copied the {} ({}) to its own subspace", + prefix.what, + printable(&prefix.from) + ), + Total = originals.count, + ); + Ok(originals.count) +} + +struct Rows { + count: u64, + /// The keys as they are in the feature's own subspace, when compared. + values: Vec<(Vec, Vec)>, +} + +async fn rows( + store: &Store, + subspace: u8, + from: &[u8], + to: &[u8], + with_values: bool, +) -> trc::Result { + let mut rows = Rows { + count: 0, + values: Vec::new(), + }; + // In `_`, the feature's letter comes first + let skip = usize::from(subspace == SUBSPACE_INBUXA); + store + .iterate( + IterateParams::new( + AnyKey { + subspace, + key: from.to_vec(), + }, + AnyKey { + subspace, + key: to.to_vec(), + }, + ) + .set_values(with_values), + |key, value| { + if key.starts_with(from) { + rows.count += 1; + if with_values { + rows.values.push((key[skip..].to_vec(), value.to_vec())); + } + } + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + Ok(rows) +} + +fn inbuxa_key(key: &[u8]) -> ValueKey { + ValueKey::from(inbuxa_class(key)) +} + +fn inbuxa_class(key: &[u8]) -> ValueClass { + ValueClass::Any(AnyClass { + subspace: SUBSPACE_INBUXA, + key: key.to_vec(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_owner_is_listed_once() { + for (i, (byte, _)) in OWNERS.iter().enumerate() { + assert!( + OWNERS[i + 1..].iter().all(|(other, _)| other != byte), + "{} is listed twice", + *byte as char + ); + } + } + + #[test] + fn claims() { + assert!(matches!(claim(b"Vs"), Claim::Stays)); + assert!(matches!(claim(b"Ls"), Claim::Stays)); + assert!(matches!(claim(b"Jj"), Claim::Stays)); + assert!(matches!(claim(b"Lz"), Claim::Nobody)); + assert!(matches!(claim(b"L"), Claim::Nobody)); + assert!(matches!(claim(b"Zz"), Claim::Nobody)); + assert!(matches!(claim(b"\x00"), Claim::Nobody)); + let Claim::Moves(prefix) = claim(b"Le") else { + panic!() + }; + assert_eq!(prefix.dest, SUBSPACE_AUDIT); + assert_eq!(prefix.to, b"Lf"); + assert!(!prefix.compared); + let Claim::Moves(prefix) = claim(b"Jh") else { + panic!() + }; + assert_eq!(prefix.dest, SUBSPACE_JOURNAL); + assert!(prefix.compared); + let Claim::Moves(prefix) = claim(b"Ug") else { + panic!() + }; + assert_eq!(prefix.dest, SUBSPACE_UNDELETE); + } + + // The two-byte prefixes on prod's `_` on 2026-11-11, all claimed + #[test] + fn prod_prefixes_are_claimed() { + for prefix in [ + "Al", "Ci", "Dr", "Gr", "Gs", "Kd", "Kl", "Le", "Lh", "Lo", "Lt", "Mi", "Pc", "Pg", + "Pu", "Sf", "Sr", "Vs", "Wl", "Wn", + ] { + assert!( + !matches!(claim(prefix.as_bytes()), Claim::Nobody), + "{prefix}" + ); + } + } + + #[test] + fn next_prefixes() { + assert_eq!(next_prefix(b"Le"), Some(b"Lf".to_vec())); + assert_eq!(next_prefix(&[b'L', 0xff]), Some(b"M".to_vec())); + assert_eq!(next_prefix(&[0xff, 0xff]), None); + } +} diff --git a/crates/migration/src/layout.rs b/crates/migration/src/layout.rs index be6ff5b2..f64c45e2 100644 --- a/crates/migration/src/layout.rs +++ b/crates/migration/src/layout.rs @@ -46,15 +46,26 @@ struct Step { what: &'static str, } -const STEPS: &[Step] = &[Step { - from: 1, - rolling_safe: true, - what: "in-memory counters on SQL keep their own expiry", -}]; +const STEPS: &[Step] = &[ + Step { + from: 1, + rolling_safe: true, + what: "in-memory counters on SQL keep their own expiry", + }, + Step { + from: 2, + // SL-18: old nodes would keep writing the moved data to `_`, and an + // old binary can't open a RocksDB store with column families it + // doesn't know + rolling_safe: false, + what: "the audit log, journal, holds and undelete notes move to subspaces of their own", + }, +]; async fn run_step(server: &Server, from: u32) -> trc::Result { match from { 1 => counters_keep_expiry(server).await, + 2 => crate::fork_subspaces::fork_data_own_subspaces(server.store()).await, _ => unreachable!("no store layout step from version {from}"), } } diff --git a/crates/migration/src/lib.rs b/crates/migration/src/lib.rs index 48d4934a..f8b54a75 100644 --- a/crates/migration/src/lib.rs +++ b/crates/migration/src/lib.rs @@ -18,6 +18,7 @@ use store::{ use trc::AddContext; pub mod destroy; +pub mod fork_subspaces; pub mod layout; pub mod v016; diff --git a/crates/store/src/backend/mysql/main.rs b/crates/store/src/backend/mysql/main.rs index 5282a3e3..ae8490d1 100644 --- a/crates/store/src/backend/mysql/main.rs +++ b/crates/store/src/backend/mysql/main.rs @@ -113,6 +113,11 @@ impl MysqlStore { SUBSPACE_DELETED_ITEMS, SUBSPACE_SPAM_SAMPLES, crate::SUBSPACE_INBUXA, // inbuxa: masked email + // inbuxa: SL-16: the fork's large data, from layout version 3 + crate::SUBSPACE_AUDIT, + crate::SUBSPACE_JOURNAL, + crate::SUBSPACE_HOLDS, + crate::SUBSPACE_UNDELETE, SUBSPACE_BLOB_LINK, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_PROPERTY, diff --git a/crates/store/src/backend/mysql/write.rs b/crates/store/src/backend/mysql/write.rs index 009e9b6a..784d9d84 100644 --- a/crates/store/src/backend/mysql/write.rs +++ b/crates/store/src/backend/mysql/write.rs @@ -452,6 +452,38 @@ impl MysqlStore { bounded(conn, result, limit) } + /// inbuxa: SL-17: copies the rows from `from` up to `to` into subspace + /// `dest`, without their first `strip` bytes, in one statement on the + /// server. A row already there is overwritten, so a copy can run again. + pub(crate) async fn copy_range( + &self, + from: impl Key, + to: impl Key, + dest: u8, + strip: usize, + ) -> trc::Result { + let mut conn = self.conn().await?; + let limit = self.timeouts.maintenance; + let result = tokio::time::timeout(limit, async { + let table = crate::subspace_name(from.subspace()); + let dest = crate::subspace_name(dest); + let from = from.serialize(0); + let to = to.serialize(0); + let s = conn + .prep(format!( + "INSERT INTO {dest} (k, v) SELECT SUBSTRING(k, {}), v FROM {table} \ + WHERE k >= ? AND k < ? ON DUPLICATE KEY UPDATE v = VALUES(v)", + strip + 1 + )) + .await + .map_err(into_error)?; + conn.exec_drop(&s, (&from, &to)).await.map_err(into_error)?; + Ok(conn.affected_rows()) + }) + .await; + bounded(conn, result, limit) + } + pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { let mut conn = self.conn().await?; let limit = self.timeouts.maintenance; diff --git a/crates/store/src/backend/postgres/main.rs b/crates/store/src/backend/postgres/main.rs index 6913747d..acfc33dd 100644 --- a/crates/store/src/backend/postgres/main.rs +++ b/crates/store/src/backend/postgres/main.rs @@ -161,6 +161,11 @@ impl PostgresStore { SUBSPACE_DELETED_ITEMS, SUBSPACE_SPAM_SAMPLES, crate::SUBSPACE_INBUXA, // inbuxa: masked email + // inbuxa: SL-16: the fork's large data, from layout version 3 + crate::SUBSPACE_AUDIT, + crate::SUBSPACE_JOURNAL, + crate::SUBSPACE_HOLDS, + crate::SUBSPACE_UNDELETE, SUBSPACE_BLOB_LINK, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_PROPERTY, diff --git a/crates/store/src/backend/postgres/write.rs b/crates/store/src/backend/postgres/write.rs index 0ab0ece2..1a40919d 100644 --- a/crates/store/src/backend/postgres/write.rs +++ b/crates/store/src/backend/postgres/write.rs @@ -553,6 +553,37 @@ impl PostgresStore { bounded(conn, result, limit) } + /// inbuxa: SL-17: copies the rows from `from` up to `to` into subspace + /// `dest`, without their first `strip` bytes, in one statement on the + /// server. A row already there is overwritten, so a copy can run again. + pub(crate) async fn copy_range( + &self, + from: impl Key, + to: impl Key, + dest: u8, + strip: usize, + ) -> trc::Result { + let conn = self.conn_pool.get().await.map_err(into_pool_error)?; + let limit = self.timeouts.maintenance; + let result = tokio::time::timeout(limit, async { + let table = crate::subspace_name(from.subspace()); + let dest = crate::subspace_name(dest); + let from = from.serialize(0); + let to = to.serialize(0); + let s = conn + .prepare_cached(&format!( + "INSERT INTO {dest} (k, v) SELECT substring(k from {}), v FROM {table} \ + WHERE k >= $1 AND k < $2 ON CONFLICT (k) DO UPDATE SET v = EXCLUDED.v", + strip + 1 + )) + .await + .map_err(into_error)?; + conn.execute(&s, &[&from, &to]).await.map_err(into_error) + }) + .await; + bounded(conn, result, limit) + } + pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { let conn = self.conn_pool.get().await.map_err(into_pool_error)?; let limit = self.timeouts.maintenance; diff --git a/crates/store/src/backend/rocksdb/main.rs b/crates/store/src/backend/rocksdb/main.rs index e15c5e55..c1abd4fb 100644 --- a/crates/store/src/backend/rocksdb/main.rs +++ b/crates/store/src/backend/rocksdb/main.rs @@ -98,11 +98,14 @@ impl RocksDbStore { (SUBSPACE_SEARCH_INDEX, CfProfile::Scan), (SUBSPACE_SPAM_SAMPLES, CfProfile::Churn), (crate::SUBSPACE_INBUXA, CfProfile::PointLookup), // inbuxa: masked email + // inbuxa: SL-16: the fork's large data, from layout version 3 + (crate::SUBSPACE_AUDIT, CfProfile::Scan), + (crate::SUBSPACE_JOURNAL, CfProfile::Scan), + (crate::SUBSPACE_HOLDS, CfProfile::PointLookup), + (crate::SUBSPACE_UNDELETE, CfProfile::Churn), (SUBSPACE_REGISTRY_IDX, CfProfile::Scan), (SUBSPACE_REGISTRY_PK, CfProfile::PointLookup), (SUBSPACE_DIRECTORY, CfProfile::PointLookup), - (LEGACY_SUBSPACE_BITMAP_TEXT, CfProfile::Scan), - (LEGACY_SUBSPACE_BITMAP_TAG, CfProfile::Scan), ] { cfs.push(ColumnFamilyDescriptor::new( crate::subspace_name(subspace), @@ -110,6 +113,20 @@ impl RocksDbStore { )); } + // inbuxa: SL-20: the legacy `v` and `c` are opened only where they + // still exist, for store layout step 2 -> 3 to drop. RocksDB won't + // open a database without every column family it has. + let existing = rocksdb::DB::list_cf(&Options::default(), &idx_path).unwrap_or_default(); + for subspace in [LEGACY_SUBSPACE_BITMAP_TEXT, LEGACY_SUBSPACE_BITMAP_TAG] { + let name = crate::subspace_name(subspace); + if existing.iter().any(|cf| cf == name) { + cfs.push(ColumnFamilyDescriptor::new( + name, + cf_options(CfProfile::Scan, &cache, write_buffer_size), + )); + } + } + let mut db_opts = Options::default(); db_opts.create_missing_column_families(true); db_opts.create_if_missing(true); @@ -140,6 +157,24 @@ impl RocksDbStore { }))) } + /// inbuxa: SL-20: drops the legacy `v` and `c` column families where + /// they exist, and returns how many it dropped. + pub(crate) async fn drop_legacy_subspaces(&self) -> trc::Result { + let db = self.db.clone(); + self.spawn_worker(move || { + let mut dropped = 0; + for subspace in [LEGACY_SUBSPACE_BITMAP_TEXT, LEGACY_SUBSPACE_BITMAP_TAG] { + let name = crate::subspace_name(subspace); + if db.cf_handle(name).is_some() { + db.drop_cf(name).map_err(super::into_error)?; + dropped += 1; + } + } + Ok(dropped) + }) + .await + } + pub async fn spawn_worker(&self, mut f: U) -> trc::Result where U: FnMut() -> trc::Result + Send, @@ -234,3 +269,48 @@ fn cf_options(profile: CfProfile, cache: &Cache, write_buffer_size: usize) -> Op opts } + +#[cfg(test)] +mod tests { + use super::*; + + // inbuxa: SL-20: a store from before step 2 -> 3 has `v` and `c`. It + // still opens, the step drops them, and it opens again without them. + #[tokio::test] + async fn legacy_column_families_are_dropped() { + let dir = std::env::temp_dir().join(format!("inbuxa-legacy-cf-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + { + let mut opts = Options::default(); + opts.create_if_missing(true); + opts.create_missing_column_families(true); + rocksdb::DB::open_cf(&opts, &dir, ["v", "c", "p"]).unwrap(); + } + let config = || structs::RocksDbStore { + path: dir.to_string_lossy().into_owned(), + ..Default::default() + }; + let has = |name: &str| { + rocksdb::DB::list_cf(&Options::default(), &dir) + .unwrap() + .iter() + .any(|cf| cf == name) + }; + + let Store::RocksDb(store) = RocksDbStore::open(config()).await.unwrap() else { + unreachable!() + }; + assert_eq!(store.drop_legacy_subspaces().await.unwrap(), 2); + assert_eq!(store.drop_legacy_subspaces().await.unwrap(), 0); + drop(store); + assert!(!has("v") && !has("c")); + assert!(has(crate::subspace_name(crate::SUBSPACE_AUDIT))); + + let Store::RocksDb(store) = RocksDbStore::open(config()).await.unwrap() else { + unreachable!() + }; + drop(store); + assert!(!has("v") && !has("c")); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/store/src/backend/sqlite/main.rs b/crates/store/src/backend/sqlite/main.rs index 6bce693d..63a5a9e4 100644 --- a/crates/store/src/backend/sqlite/main.rs +++ b/crates/store/src/backend/sqlite/main.rs @@ -69,6 +69,11 @@ impl SqliteStore { SUBSPACE_DELETED_ITEMS, SUBSPACE_SPAM_SAMPLES, crate::SUBSPACE_INBUXA, // inbuxa: masked email + // inbuxa: SL-16: the fork's large data, from layout version 3 + crate::SUBSPACE_AUDIT, + crate::SUBSPACE_JOURNAL, + crate::SUBSPACE_HOLDS, + crate::SUBSPACE_UNDELETE, SUBSPACE_BLOB_LINK, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_PROPERTY, diff --git a/crates/store/src/backend/sqlite/write.rs b/crates/store/src/backend/sqlite/write.rs index b5d04121..9a877556 100644 --- a/crates/store/src/backend/sqlite/write.rs +++ b/crates/store/src/backend/sqlite/write.rs @@ -341,6 +341,35 @@ impl SqliteStore { .await } + /// inbuxa: SL-17: copies the rows from `from` up to `to` into subspace + /// `dest`, without their first `strip` bytes, in one statement. A row + /// already there is overwritten, so a copy can run again. + pub(crate) async fn copy_range( + &self, + from: impl Key, + to: impl Key, + dest: u8, + strip: usize, + ) -> trc::Result { + let manager = self.conn_pool.clone(); + self.spawn_worker(move || { + let conn = manager.get().map_err(into_error)?; + conn.prepare_cached(&format!( + "INSERT OR REPLACE INTO {} (k, v) SELECT substr(k, {}), v FROM {} WHERE k >= ? AND k < ?", + crate::subspace_name(dest), + strip + 1, + crate::subspace_name(from.subspace()), + )) + .map_err(into_error) + .caused_by(trc::location!())? + .execute([from.serialize(0), to.serialize(0)]) + .map(|rows| rows as u64) + .map_err(into_error) + .caused_by(trc::location!()) + }) + .await + } + pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { let manager = self.conn_pool.clone(); self.spawn_worker(move || { diff --git a/crates/store/src/dispatch/store.rs b/crates/store/src/dispatch/store.rs index 9cfe6563..ff7682da 100644 --- a/crates/store/src/dispatch/store.rs +++ b/crates/store/src/dispatch/store.rs @@ -378,6 +378,47 @@ impl Store { .caused_by(trc::location!()) } + /// inbuxa: SL-17: on SQL, copies the rows from `from` up to `to` into + /// subspace `dest` without their first `strip` bytes, in one statement on + /// the server, and returns how many it wrote. None where the backend has + /// no such statement: the caller copies them itself. + #[allow(unused_variables)] + pub async fn copy_range( + &self, + from: impl Key, + to: impl Key, + dest: u8, + strip: usize, + ) -> trc::Result> { + match self { + #[cfg(feature = "sqlite")] + Self::SQLite(store) => store.copy_range(from, to, dest, strip).await.map(Some), + #[cfg(feature = "postgres")] + Self::PostgreSQL(store) => store.copy_range(from, to, dest, strip).await.map(Some), + #[cfg(feature = "mysql")] + Self::MySQL(store) => store.copy_range(from, to, dest, strip).await.map(Some), + Self::Replicated(store) => crate::sql_backend!( + &store.primary, + db => db.copy_range(from, to, dest, strip).await + ) + .map(Some), + _ => Ok(None), + } + .caused_by(trc::location!()) + } + + /// inbuxa: SL-20: drops the legacy `v` and `c` column families on + /// RocksDB, which nothing writes. Elsewhere they don't exist. Returns + /// how many it dropped. + pub async fn drop_legacy_subspaces(&self) -> trc::Result { + match self { + #[cfg(feature = "rocks")] + Self::RocksDb(store) => store.drop_legacy_subspaces().await, + _ => Ok(0), + } + .caused_by(trc::location!()) + } + pub async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> { match self { #[cfg(feature = "sqlite")] diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 51f71f1c..7771b45f 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -145,6 +145,11 @@ pub const LEGACY_SUBSPACE_BITMAP_TAG: u8 = b'c'; // inbuxa: SL-15: every lowercase letter is taken, so later subspaces take id // bytes from 0x80 up, which are never letters, and a name of their own. pub const SUBSPACE_NAMED_FIRST: u8 = 0x80; +// inbuxa: SL-16: the fork's large data, out of `_` (layout version 3) +pub const SUBSPACE_AUDIT: u8 = 0x80; +pub const SUBSPACE_JOURNAL: u8 = 0x81; +pub const SUBSPACE_HOLDS: u8 = 0x82; +pub const SUBSPACE_UNDELETE: u8 = 0x83; /// One-byte names for the subspaces up to 0x7f, which are ASCII. const ASCII_NAMES: &[u8; 128] = &{ @@ -170,7 +175,13 @@ pub const fn subspace_name(subspace: u8) -> &'static str { Err(_) => unreachable!(), } } else { - panic!("a subspace from 0x80 up has no name") + match subspace { + SUBSPACE_AUDIT => "ib_audit", + SUBSPACE_JOURNAL => "ib_journal", + SUBSPACE_HOLDS => "ib_holds", + SUBSPACE_UNDELETE => "ib_undelete", + _ => panic!("a subspace from 0x80 up has no name"), + } } } @@ -845,9 +856,17 @@ mod subspace_names { assert_eq!(NAME, "l"); } + #[test] + fn named_subspaces() { + assert_eq!(subspace_name(SUBSPACE_AUDIT), "ib_audit"); + assert_eq!(subspace_name(SUBSPACE_JOURNAL), "ib_journal"); + assert_eq!(subspace_name(SUBSPACE_HOLDS), "ib_holds"); + assert_eq!(subspace_name(SUBSPACE_UNDELETE), "ib_undelete"); + } + #[test] #[should_panic] fn an_unnamed_high_byte_is_a_bug() { - subspace_name(SUBSPACE_NAMED_FIRST); + subspace_name(0xff); } } diff --git a/docs/spec/features/store-layout.md b/docs/spec/features/store-layout.md index c0145d47..3ce7e88d 100644 --- a/docs/spec/features/store-layout.md +++ b/docs/spec/features/store-layout.md @@ -127,12 +127,57 @@ before January, else 27.7. Release candidates may carry it before then. PostgreSQL the copy is done server-side (`INSERT … SELECT`). - **SL-18.** Not rolling-safe (SL-6). On RocksDB an old binary can't open a store with column families it doesn't know. -- **SL-19.** The moved data travels in exports in a family of its own, - included by default, so a configuration export can leave it out. Importing +- **SL-19.** The moved data travels in exports in a family of its own + (`compliance`), included by default, so a configuration export can leave + it out. Importing an export taken at version 1 or 2 runs the steps after the import. - **SL-20.** The legacy `v` and `c` column families, which nothing writes, are dropped in the same step. +#### The keys in `_` and their owners (SL-16) + +Every key in `_` starts with a byte naming its feature. The step lists the +two-byte prefixes in `_`, reading one key per prefix, and stops on any it +can't place, before it writes anything. The table is `OWNERS` in +`crates/migration/src/fork_subspaces.rs`. + +| First byte | Owner | At step 2 → 3 | +|---|---|---| +| `A` | AI limits | stays | +| `B` | first boot defaults | stays | +| `C` | account snapshots | stays | +| `D` | deliverability reports | stays | +| `E` | inbuxa relay | stays | +| `F` | front ends | stays | +| `G` | scheduled reports | stays | +| `H` | legal holds (`h`, `o`, `e`) | moves to `ib_holds` (0x82) | +| `I` | migrations | stays | +| `J` | journal: entries (`e`, `c`, `p`, `t`, `x`, `h`, `f`), archive (`o`, `w`) | moves to `ib_journal` (0x81); the journals themselves (`j`) are settings and stay | +| `K` | account locks | stays | +| `L` | audit log (`e`, `t`, `o`, `h`, `f`) | moves to `ib_audit` (0x80); its settings (`s`) stay | +| `M` | masked email | stays | +| `P` | permissions and protocol policies | stays | +| `Q` | acceptance records | stays | +| `R` | mail flow rules | stays | +| `S` | spam rules, scale-out storage, scheduled reports from 2026.10.6.2 | stays | +| `T` | log files | stays | +| `U` | undelete notes (`n`, `x`, `b`, `r`, `c`, `k`, `a`, `g`) | moves to `ib_undelete` (0x83) | +| `V` | store checks, the layout mark (`Vs`), lease (`Vm`) and step progress (`Vt`) | stays | +| `W` | blob store changes, the sharing policy | stays | + +A key moves without its first byte, so `Le` + node + seq becomes `e` + node ++ seq in `ib_audit`. A feature's settings stay in `_`, where a configuration +export (`registry`) carries them. On prod on 2026-11-11, `_` held 9,629 +rows (3.2 MB), of which 6,813 were the audit log and none were journal, +hold or undelete rows. + +The step copies each moving prefix (on SQL with one `INSERT … SELECT`), then +checks every copy: the row count, and for the audit log's and journal's +chain heads and floors the rows themselves. Only when every check passes +does it write `Vt`, then delete the originals and drop `v` and `c`. A run +that stopped while deleting copies what's left again and deletes it +without checking again; a run that stopped before `Vt` starts over. + ### Imports and checks - **SL-21.** A Stalwart 0.16 store has no mark, so it goes through every step. diff --git a/tests/src/store/import_export.rs b/tests/src/store/import_export.rs index d4a91458..20d72aa3 100644 --- a/tests/src/store/import_export.rs +++ b/tests/src/store/import_export.rs @@ -199,13 +199,28 @@ pub async fn test(test: &TestServer) { batch.set( ValueClass::Any(AnyClass { subspace: SUBSPACE_INBUXA, - key: [b'U', b'x'] + key: [b'M', b'm'] .into_iter() .chain(item_id.to_be_bytes()) .collect(), }), random_bytes(32), ); + // inbuxa: SL-19: the fork's large data, in the compliance family + for subspace in [ + store::SUBSPACE_AUDIT, + store::SUBSPACE_JOURNAL, + store::SUBSPACE_HOLDS, + store::SUBSPACE_UNDELETE, + ] { + batch.set( + ValueClass::Any(AnyClass { + subspace, + key: [b'x'].into_iter().chain(item_id.to_be_bytes()).collect(), + }), + random_bytes(32), + ); + } } db.write(batch.build_all()).await.unwrap(); @@ -243,6 +258,10 @@ pub async fn test(test: &TestServer) { SUBSPACE_DELETED_ITEMS, SUBSPACE_SPAM_SAMPLES, SUBSPACE_INBUXA, + store::SUBSPACE_AUDIT, + store::SUBSPACE_JOURNAL, + store::SUBSPACE_HOLDS, + store::SUBSPACE_UNDELETE, ] { assert!( snapshot.keys.iter().any(|k| k.subspace == subspace), @@ -286,6 +305,10 @@ pub async fn test(test: &TestServer) { SUBSPACE_DELETED_ITEMS, SUBSPACE_SPAM_SAMPLES, SUBSPACE_INBUXA, + store::SUBSPACE_AUDIT, + store::SUBSPACE_JOURNAL, + store::SUBSPACE_HOLDS, + store::SUBSPACE_UNDELETE, ] { assert!( imported.contains(&subspace), @@ -510,6 +533,10 @@ impl Snapshot { (SUBSPACE_REPORT_IN, true), (SUBSPACE_DIRECTORY, true), (SUBSPACE_INBUXA, true), + (store::SUBSPACE_AUDIT, true), + (store::SUBSPACE_JOURNAL, true), + (store::SUBSPACE_HOLDS, true), + (store::SUBSPACE_UNDELETE, true), ] { if subspace == SUBSPACE_BLOBS && !with_blobs { continue; diff --git a/tests/src/store_version.rs b/tests/src/store_version.rs index e27a0f3b..c429e13f 100644 --- a/tests/src/store_version.rs +++ b/tests/src/store_version.rs @@ -9,7 +9,8 @@ //! refused before anything is written to it. A store opened for the first //! time has upstream's web interface switched off (docs/spec/import-0.16.md). //! A store at an older layout version is brought up to this one, by one node -//! at a time (docs/spec/features/store-layout.md). +//! at a time (docs/spec/features/store-layout.md). Step 2 -> 3 moves the +//! audit log, journal, holds and undelete notes out of `_`. use crate::utils::server::TestServerBuilder; use common::{ @@ -17,13 +18,15 @@ use common::{ manager::first_party::disable_imported_web_interface, }; use futures::FutureExt; +use inbuxa_features::audit::{Action, Actor, AuditLog, Outcome, Record, Target, log}; use registry::{ schema::{prelude::ObjectInner, structs::Application}, types::map::Map, }; use std::{panic::AssertUnwindSafe, time::Duration}; use store::{ - InMemoryStore, IterateParams, SUBSPACE_INBUXA, SerializeInfallible, Store, ValueKey, + Deserialize, InMemoryStore, IterateParams, SUBSPACE_AUDIT, SUBSPACE_HOLDS, SUBSPACE_INBUXA, + SUBSPACE_JOURNAL, SUBSPACE_UNDELETE, SerializeInfallible, Store, ValueKey, dispatch::lookup::KeyValue, registry::{bootstrap::Bootstrap, write::RegistryWrite}, write::{AnyClass, AnyKey, BatchBuilder, InMemoryClass, Operation, ValueClass, ValueOp, now}, @@ -91,6 +94,7 @@ async fn store_version() { layout_step_1_2(server).await; layout_lease(server).await; + layout_step_2_3(server).await; } /// A counter the way a version 1 store holds it: a row in `y` and an expiry @@ -386,3 +390,361 @@ async fn imported_web_interface() { "a marked store's web interface is left as the operator set it" ); } + +struct Raw(Vec); + +impl Deserialize for Raw { + fn deserialize(bytes: &[u8]) -> trc::Result { + Ok(Raw(bytes.to_vec())) + } +} + +/// Every row of a subspace, keys as stored. +async fn rows(store: &Store, subspace: u8) -> Vec<(Vec, Vec)> { + let mut rows = Vec::new(); + store + .iterate( + IterateParams::new( + AnyKey { + subspace, + key: vec![0u8], + }, + AnyKey { + subspace, + key: vec![u8::MAX; 64], + }, + ), + |key, value| { + rows.push((key.to_vec(), value.to_vec())); + Ok(true) + }, + ) + .await + .unwrap(); + rows +} + +async fn set_raw(store: &Store, subspace: u8, key: &[u8], value: &[u8]) { + let mut batch = BatchBuilder::new(); + batch.set( + ValueClass::Any(AnyClass { + subspace, + key: key.to_vec(), + }), + value.to_vec(), + ); + store.write(batch.build_all()).await.unwrap(); +} + +async fn clear_raw(store: &Store, subspace: u8, key: &[u8]) { + let mut batch = BatchBuilder::new(); + batch.clear(ValueClass::Any(AnyClass { + subspace, + key: key.to_vec(), + })); + store.write(batch.build_all()).await.unwrap(); +} + +async fn get_raw(store: &Store, subspace: u8, key: &[u8]) -> Option> { + store + .get_value::(AnyKey { + subspace, + key: key.to_vec(), + }) + .await + .unwrap() + .map(|Raw(bytes)| bytes) +} + +/// The moved subspaces, and the letter each had in `_`. +const MOVED: [(u8, u8); 4] = [ + (SUBSPACE_AUDIT, b'L'), + (SUBSPACE_JOURNAL, b'J'), + (SUBSPACE_HOLDS, b'H'), + (SUBSPACE_UNDELETE, b'U'), +]; + +/// Puts a version 3 store's moved data back the way version 2 kept it, +/// under its letter in `_`. +async fn back_to_v2(store: &Store) { + for (subspace, letter) in MOVED { + for (key, value) in rows(store, subspace).await { + let mut old = vec![letter]; + old.extend_from_slice(&key); + set_raw(store, SUBSPACE_INBUXA, &old, &value).await; + clear_raw(store, subspace, &key).await; + } + } +} + +/// The rows left in `_` under the moved letters, but for the settings that +/// stay. +async fn left_in_inbuxa(store: &Store) -> Vec> { + rows(store, SUBSPACE_INBUXA) + .await + .into_iter() + .map(|(key, _)| key) + .filter(|key| { + MOVED.iter().any(|(_, letter)| key.first() == Some(letter)) + && key.get(..2) != Some(b"Ls") + && key.get(..2) != Some(b"Jj") + }) + .collect() +} + +fn audit_record(n: u64) -> Record { + Record { + at: 1_000 + n, + actor: Actor::account(7, "admin@example.org", None), + via: None, + remote_ip: None, + action: Action::Update, + target: Target { + kind: "x:Domain".into(), + id: Some(format!("d{n}")), + name: Some("example.org".into()), + ..Default::default() + }, + changes: vec![], + details: None, + reason: None, + outcome: Outcome::success(), + } +} + +// SL-16, SL-17, SL-20, SL-22: a version 2 store's audit log, journal, +// holds and undelete notes move to their own subspaces; settings and other +// features stay in `_`; a second run changes nothing; a run that stopped +// while deleting finishes; an unclaimed key or a copy that doesn't match +// stops the step with nothing deleted. +async fn layout_step_2_3(server: &Server) { + let store = server.store(); + let node = 3; + + // Version 3 data, written the way this build writes it + let audit = AuditLog::new(); + let mut ids = Vec::new(); + for n in 0..3 { + ids.push(audit.append(store, node, &audit_record(n)).await.unwrap()); + } + set_raw( + store, + SUBSPACE_JOURNAL, + b"c\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x01", + b"{\"journal\":1}", + ) + .await; + set_raw(store, SUBSPACE_HOLDS, b"h\x00\x00\x00\x09", b"{\"hold\":9}").await; + set_raw( + store, + SUBSPACE_UNDELETE, + b"g\x01\x00\x00\x00\x05\x00\x00\x00\x06", + b"{\"note\":6}", + ) + .await; + set_raw( + store, + SUBSPACE_UNDELETE, + b"k\x00\x00\x00\x05", + b"{\"kept\":5}", + ) + .await; + // Settings and another feature's key, which stay in `_` + set_raw(store, SUBSPACE_INBUXA, b"Ls", b"{\"keepFor\":100}").await; + set_raw( + store, + SUBSPACE_INBUXA, + b"Jj\x00\x00\x00\x01", + b"{\"journal\":\"def\"}", + ) + .await; + set_raw(store, SUBSPACE_INBUXA, b"Kl\x00\x00\x00\x05", b"lock").await; + let audit_rows = rows(store, SUBSPACE_AUDIT).await; + let mut expected = Vec::new(); + for (subspace, _) in MOVED { + expected.push(rows(store, subspace).await); + } + assert_eq!(audit_rows.len(), expected[0].len()); + assert!(!audit_rows.is_empty()); + + let moved_rows = |store: Store| async move { + let mut all = Vec::new(); + for (subspace, _) in MOVED { + all.push(rows(&store, subspace).await); + } + all + }; + + // Brought back to version 2, then up again + back_to_v2(store).await; + assert!(rows(store, SUBSPACE_AUDIT).await.is_empty()); + assert_eq!( + left_in_inbuxa(store).await.len(), + expected.iter().map(Vec::len).sum::() + ); + write_mark(server, Some(2)).await; + migration::try_migrate(server).await.unwrap(); + assert_eq!(read_mark(server).await, Some(INBUXA_STORE_VERSION)); + assert_eq!(moved_rows(store.clone()).await, expected); + assert!(left_in_inbuxa(store).await.is_empty(), "left in _"); + assert!( + get_raw( + store, + SUBSPACE_INBUXA, + migration::fork_subspaces::CHECKED_KEY + ) + .await + .is_none() + ); + assert_eq!( + get_raw(store, SUBSPACE_INBUXA, b"Ls").await.as_deref(), + Some(&b"{\"keepFor\":100}"[..]) + ); + assert_eq!( + get_raw(store, SUBSPACE_INBUXA, b"Jj\x00\x00\x00\x01") + .await + .as_deref(), + Some(&b"{\"journal\":\"def\"}"[..]) + ); + assert_eq!( + get_raw(store, SUBSPACE_INBUXA, b"Kl\x00\x00\x00\x05") + .await + .as_deref(), + Some(&b"lock"[..]) + ); + + // The audit log reads back, its chain whole, and goes on from its head + for (n, id) in ids.iter().enumerate() { + assert_eq!( + log::get(store, *id).await.unwrap(), + Some(audit_record(n as u64)) + ); + } + let next = audit.append(store, node, &audit_record(3)).await.unwrap(); + assert_eq!(next.seq, ids[2].seq + 1); + let report = log::verify(store).await.unwrap(); + let chain = report.iter().find(|chain| chain.node == node).unwrap(); + assert_eq!(chain.broken_at, None, "{chain:?}"); + assert_eq!(chain.entries, 4); + let mut expected = moved_rows(store.clone()).await; + + // A second run changes nothing + write_mark(server, Some(2)).await; + migration::layout::run_steps(server, server.registry().node_id()) + .await + .unwrap(); + assert_eq!(read_mark(server).await, Some(INBUXA_STORE_VERSION)); + assert_eq!(moved_rows(store.clone()).await, expected); + + // A run that stopped while deleting: copies checked, some originals left + for (key, value) in &expected[2] { + let mut old = vec![b'H']; + old.extend_from_slice(key); + set_raw(store, SUBSPACE_INBUXA, &old, value).await; + } + set_raw( + store, + SUBSPACE_INBUXA, + migration::fork_subspaces::CHECKED_KEY, + &[1], + ) + .await; + write_mark(server, Some(2)).await; + migration::layout::run_steps(server, server.registry().node_id()) + .await + .unwrap(); + assert_eq!(moved_rows(store.clone()).await, expected); + assert!(left_in_inbuxa(store).await.is_empty()); + assert!( + get_raw( + store, + SUBSPACE_INBUXA, + migration::fork_subspaces::CHECKED_KEY + ) + .await + .is_none() + ); + + // A key nobody claims stops the step before it changes anything + back_to_v2(store).await; + write_mark(server, Some(2)).await; + let before = rows(store, SUBSPACE_INBUXA).await; + set_raw(store, SUBSPACE_INBUXA, b"Zz", b"?").await; + let err = migration::layout::run_steps(server, server.registry().node_id()) + .await + .unwrap_err(); + assert!(format!("{err:?}").contains("Zz"), "{err:?}"); + assert_eq!(read_mark(server).await, Some(2)); + clear_raw(store, SUBSPACE_INBUXA, b"Zz").await; + assert_eq!(rows(store, SUBSPACE_INBUXA).await, before); + assert!(rows(store, SUBSPACE_AUDIT).await.is_empty()); + + // A copy that doesn't match its originals stops it with nothing deleted + let (stray, _) = &expected[0][0]; + let mut stray = stray.clone(); + stray.push(0xff); + set_raw(store, SUBSPACE_AUDIT, &stray, b"stray").await; + let err = migration::layout::run_steps(server, server.registry().node_id()) + .await + .unwrap_err(); + assert!(format!("{err:?}").contains("audit log"), "{err:?}"); + assert_eq!(read_mark(server).await, Some(2)); + assert_eq!( + rows(store, SUBSPACE_INBUXA).await, + before, + "deleted after a mismatch" + ); + + // Put right, it goes through + clear_raw(store, SUBSPACE_AUDIT, &stray).await; + migration::layout::run_steps(server, server.registry().node_id()) + .await + .unwrap(); + assert_eq!(read_mark(server).await, Some(INBUXA_STORE_VERSION)); + expected.iter_mut().for_each(|rows| rows.sort()); + let mut after = moved_rows(store.clone()).await; + after.iter_mut().for_each(|rows| rows.sort()); + assert_eq!(after, expected); + + // More rows than the step copies at a time, where it copies them itself + let many = 10_050u32; + let mut batch = BatchBuilder::new(); + for n in 0..many { + let mut key = b"Un".to_vec(); + key.extend_from_slice(&n.to_be_bytes()); + batch.set( + ValueClass::Any(AnyClass { + subspace: SUBSPACE_INBUXA, + key, + }), + n.to_be_bytes().to_vec(), + ); + if batch.is_large_batch() { + store.write(batch.build_all()).await.unwrap(); + batch = BatchBuilder::new(); + } + } + store.write(batch.build_all()).await.unwrap(); + write_mark(server, Some(2)).await; + migration::layout::run_steps(server, server.registry().node_id()) + .await + .unwrap(); + let notes = rows(store, SUBSPACE_UNDELETE) + .await + .into_iter() + .filter(|(key, _)| key.len() == 5 && key[0] == b'n') + .collect::>(); + assert_eq!(notes.len(), many as usize); + assert!(notes.iter().all(|(key, value)| key[1..] == value[..])); + assert!(left_in_inbuxa(store).await.is_empty()); + + // Clean up + for (subspace, _) in MOVED { + for (key, _) in rows(store, subspace).await { + clear_raw(store, subspace, &key).await; + } + } + for key in [&b"Ls"[..], b"Jj\x00\x00\x00\x01", b"Kl\x00\x00\x00\x05"] { + clear_raw(store, SUBSPACE_INBUXA, key).await; + } +} diff --git a/tests/src/utils/cleanup.rs b/tests/src/utils/cleanup.rs index d206a090..a71cfb81 100644 --- a/tests/src/utils/cleanup.rs +++ b/tests/src/utils/cleanup.rs @@ -28,6 +28,11 @@ pub async fn store_destroy(store: &Store) { SUBSPACE_DELETED_ITEMS, SUBSPACE_SPAM_SAMPLES, store::SUBSPACE_INBUXA, // inbuxa: masked email + // inbuxa: SL-16: the fork's large data + store::SUBSPACE_AUDIT, + store::SUBSPACE_JOURNAL, + store::SUBSPACE_HOLDS, + store::SUBSPACE_UNDELETE, SUBSPACE_BLOB_LINK, SUBSPACE_LOGS, SUBSPACE_IN_MEMORY_COUNTER, -- 2.54.0