inbuxa/inbuxa-server
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 2 Wiki Activity

Scale-out storage: sharded blob, in-memory and lookup stores; configured read replicas reported (ST-1 to ST-4, ST-16 to ST-30)

Browse Source 
A Sharded blob store places each blob on xxh3(key) mod N, over the whole
key; reads fall back to the other members, so blobs placed under an
earlier member list stay readable, and deletes find them wherever they
are. The member list is recorded in the data store (secrets left out):
added or reordered members are a warning, a missing one refuses to open.
Blobs are compressed and marked before they reach a member. A Sharded
in-memory or lookup store sends each key to its home Redis member, and
prefix deletes and purges to all; a node whose member list differs from
the recorded one logs an error and runs on. Members are checked for
duplicates and must all open.

Until read-replica routing is built, each configured replica is reported
at startup instead of being silently ignored, and nothing connects to it.
The new scaleout_blob_tests covers tests 2 to 7, and the existing blob
suite passes against three FileSystem members (BLOB_STORE=Sharded).
This commit is contained in:
jcoffey-dev
2026-09-19 10:11:30 -07:00
parent e7efa91cbc
commit e5e326de6b
16 changed files with 1144 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/store/src/backend/mod.rs
+2
View File
@@ -26,6 +26,8 @@ pub mod rocksdb;
pub mod s3;
#[cfg(feature = "sqlite")]
pub mod sqlite;
// inbuxa: scale-out storage (sharded stores)
pub mod scaleout;
pub const MAX_TOKEN_LENGTH: usize = (u8::MAX >> 1) as usize;
crates/store/src/backend/mysql/main.rs
+8 -10
View File
@@ -51,18 +51,16 @@ impl MysqlStore {
PoolOpts::default().with_constraints(PoolConstraints::new(pool_min, pool_max).unwrap()),
);
let mut replicas = vec![];
// inbuxa: ST-2: replicas aren't used yet (the scale-out decision), so
// each one is reported rather than silently ignored
for replica in config.read_replicas {
replicas.push(Store::MySQL(Arc::new(MysqlStore {
conn_pool: Pool::new(
opts.clone()
.ip_or_hostname(replica.host)
.user(replica.auth_username)
.pass(replica.auth_secret.secret().await?.map(|v| v.into_owned()))
.db_name(Some(replica.database))
.tcp_port(replica.port as u16),
trc::event!(
Store(trc::StoreEvent::MysqlError),
Details = format!(
"Read replica {}:{} {} isn't used yet: every operation goes to the primary",
replica.host, replica.port, replica.database
),
})))
);
}
let primary = Store::MySQL(Arc::new(MysqlStore {
crates/store/src/backend/postgres/main.rs
+9 -20
View File
@@ -57,27 +57,16 @@ impl PostgresStore {
.map_err(|e| format!("Failed to create connection pool: {e}"))?;
let ts_configs = discover_ts_configs(&primary_pool).await;
let mut replicas = vec![];
// inbuxa: ST-2: replicas aren't used yet (the scale-out decision), so
// each one is reported rather than silently ignored
for replica in config.read_replicas {
let mut cfg = cfg.clone();
cfg.dbname = replica.database.into();
cfg.host = replica.host.into();
cfg.user = replica.auth_username;
cfg.password = replica.auth_secret.secret().await?.map(|v| v.into_owned());
cfg.port = (replica.port as u16).into();
cfg.options = replica.options;
replicas.push(Store::PostgreSQL(Arc::new(PostgresStore {
conn_pool: if config.use_tls {
cfg.create_pool(
Some(Runtime::Tokio1),
MakeRustlsConnect::new(rustls_client_config(config.allow_invalid_certs)?),
)
} else {
cfg.create_pool(Some(Runtime::Tokio1), NoTls)
}
.map_err(|e| format!("Failed to create connection pool: {e}"))?,
ts_configs: ts_configs.clone(),
})));
trc::event!(
Store(trc::StoreEvent::PostgresqlError),
Details = format!(
"Read replica {}:{} {} isn't used yet: every operation goes to the primary",
replica.host, replica.port, replica.database
),
);
}
let primary = Store::PostgreSQL(Arc::new(PostgresStore {
crates/store/src/backend/scaleout/blob.rs
+176
View File
@@ -0,0 +1,176 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! The sharded blob store (ST-16 to ST-22). A blob lives on its home
//! member; reads fall back to the others so blobs placed under an earlier
//! member list stay readable. Blobs are stored compressed and marked, as
//! on any blob store, before they reach a member.
use super::{home, layout};
use crate::{BlobStore, Store, backend::fs::FsStore};
use registry::schema::structs::{self, BlobStoreBase};
pub struct ShardedBlobStore {
pub members: Vec<BlobStore>,
pub locations: Vec<String>,
}
/// A member's kind and location, never its secrets (ST-20, ST-22).
pub fn location(member: &BlobStoreBase) -> String {
match member {
BlobStoreBase::S3(s3) => format!(
"S3 {:?} bucket {} prefix {}",
s3.region,
s3.bucket,
s3.key_prefix.as_deref().unwrap_or_default()
),
BlobStoreBase::Azure(azure) => format!(
"Azure account {} container {} prefix {}",
azure.storage_account,
azure.container,
azure.key_prefix.as_deref().unwrap_or_default()
),
BlobStoreBase::FileSystem(fs) => {
format!("FileSystem {}", fs.path.trim_end_matches('/'))
}
BlobStoreBase::FoundationDb(fdb) => format!(
"FoundationDb {}",
fdb.cluster_file.as_deref().unwrap_or("default")
),
BlobStoreBase::PostgreSql(pg) => {
format!("PostgreSql {}:{} {}", pg.host, pg.port, pg.database)
}
BlobStoreBase::MySql(my) => format!("MySql {}:{} {}", my.host, my.port, my.database),
}
}
#[allow(unreachable_patterns, unused_variables)]
async fn open_member(member: BlobStoreBase) -> Result<BlobStore, String> {
match member {
#[cfg(feature = "foundation")]
BlobStoreBase::FoundationDb(config) => crate::backend::foundationdb::FdbStore::open(config)
.await
.map(BlobStore::Store),
#[cfg(feature = "postgres")]
BlobStoreBase::PostgreSql(config) => crate::backend::postgres::PostgresStore::open(config)
.await
.map(BlobStore::Store),
#[cfg(feature = "mysql")]
BlobStoreBase::MySql(config) => crate::backend::mysql::MysqlStore::open(config)
.await
.map(BlobStore::Store),
#[cfg(feature = "s3")]
BlobStoreBase::S3(config) => crate::backend::s3::S3Store::open(config).await,
#[cfg(feature = "azure")]
BlobStoreBase::Azure(config) => crate::backend::azure::AzureStore::open(config).await,
BlobStoreBase::FileSystem(config) => FsStore::open(config).await,
_ => Err("Binary was not compiled with this member's blob store backend".to_string()),
}
}
impl ShardedBlobStore {
/// Opens every member, checks the list (ST-22), and compares it with
/// the recorded one (ST-20). Warnings are returned for the build log.
pub async fn open(
config: structs::ShardedBlobStore,
data: &Store,
warnings: &mut Vec<String>,
) -> Result<BlobStore, String> {
let members = config.stores.into_iter().collect::<Vec<_>>();
if members.len() < 2 {
return Err("A sharded blob store needs at least two members".to_string());
}
let locations = members.iter().map(location).collect::<Vec<_>>();
for (index, location) in locations.iter().enumerate() {
if let Some(first) = locations[..index].iter().position(|l| l == location) {
return Err(format!(
"Members {} and {} are the same place: {location}",
first + 1,
index + 1
));
}
}
let mut opened = Vec::with_capacity(members.len());
for (index, member) in members.into_iter().enumerate() {
opened.push(
open_member(member)
.await
.map_err(|err| format!("Member {}: {err}", index + 1))?,
);
}
match layout::check(data, layout::key(b'b', ""), &locations, true)
.await
.map_err(|err| format!("Failed to read the recorded member list: {err}"))?
{
layout::Comparison::Unchanged => {}
layout::Comparison::Changed(change) => warnings.push(format!(
"The sharded blob store's member list changed ({change}); blobs whose home \
moved are found by searching the other members"
)),
layout::Comparison::Missing(missing) => {
return Err(format!(
"Members recorded for the sharded blob store are missing, and blobs on \
them would be unreachable: {}",
missing.join("; ")
));
}
}
Ok(BlobStore::Sharded(std::sync::Arc::new(ShardedBlobStore {
members: opened,
locations,
})))
}
fn home(&self, key: &[u8]) -> usize {
home(key, self.members.len())
}
/// The home member, then the others in order (ST-17). An error from
/// the home member is returned without searching (ST-21).
pub async fn get(&self, key: &[u8]) -> trc::Result<Option<Vec<u8>>> {
let home = self.home(key);
if let Some(data) = Box::pin(self.members[home].raw_get(key)).await? {
return Ok(Some(data));
}
for (index, member) in self.members.iter().enumerate() {
if index == home {
continue;
}
if let Ok(Some(data)) = Box::pin(member.raw_get(key)).await {
trc::event!(
Store(trc::StoreEvent::UnexpectedError),
Key = key,
Details = format!(
"Misplaced blob: found on member {}, its home is member {}",
index + 1,
home + 1
),
);
return Ok(Some(data));
}
}
Ok(None)
}
/// Writes go to the home member only (ST-18, ST-21).
pub async fn put(&self, key: &[u8], data: &[u8]) -> trc::Result<()> {
Box::pin(self.members[self.home(key)].raw_put(key, data)).await
}
/// The home member first, then the others until one had it (ST-18).
pub async fn delete(&self, key: &[u8]) -> trc::Result<bool> {
let home = self.home(key);
if Box::pin(self.members[home].raw_delete(key)).await? {
return Ok(true);
}
for (index, member) in self.members.iter().enumerate() {
if index != home && Box::pin(member.raw_delete(key)).await? {
return Ok(true);
}
}
Ok(false)
}
}
crates/store/src/backend/scaleout/layout.rs
+133
View File
@@ -0,0 +1,133 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! The member lists of sharded stores, recorded in the data store (ST-20,
//! ST-26): each member's kind and location, never its secrets.
use crate::{
Deserialize, SUBSPACE_INBUXA, Store, ValueKey,
write::{AnyClass, BatchBuilder, ValueClass},
};
/// The fork's feature byte for scale-out storage.
const FEATURE: u8 = b'S';
/// Where a member list is recorded: `b` the blob store, `m` the in-memory
/// store, `l` a lookup store by namespace.
pub fn key(kind: u8, name: &str) -> ValueClass {
let mut key = vec![FEATURE, kind];
key.extend_from_slice(name.as_bytes());
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key,
})
}
struct Recorded(Vec<String>);
impl Deserialize for Recorded {
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
serde_json::from_slice(bytes)
.map(Recorded)
.map_err(|err| trc::StoreEvent::DeserializeError.reason(err))
}
}
/// How the configured list compares with the recorded one.
#[derive(Debug, PartialEq, Eq)]
pub enum Comparison {
/// No record yet, or the same list.
Unchanged,
/// Members added or reordered: the record was updated.
Changed(String),
/// A recorded member is gone.
Missing(Vec<String>),
}
pub fn compare(recorded: &[String], current: &[String]) -> Comparison {
let missing = recorded
.iter()
.filter(|member| !current.contains(member))
.cloned()
.collect::<Vec<_>>();
if !missing.is_empty() {
Comparison::Missing(missing)
} else if recorded == current {
Comparison::Unchanged
} else {
let added = current
.iter()
.filter(|member| !recorded.contains(member))
.cloned()
.collect::<Vec<_>>();
Comparison::Changed(if added.is_empty() {
format!("members reordered, was {recorded:?}, now {current:?}")
} else {
format!("members added: {added:?}")
})
}
}
/// Compares `current` with the record, then records `current` unless a
/// member is missing and `keep_on_missing` is set.
pub async fn check(
data: &Store,
class: ValueClass,
current: &[String],
keep_on_missing: bool,
) -> trc::Result<Comparison> {
if data.is_none() {
return Ok(Comparison::Unchanged);
}
let recorded = data
.get_value::<Recorded>(ValueKey::from(class.clone()))
.await?
.map(|r| r.0);
let comparison = match &recorded {
Some(recorded) => compare(recorded, current),
None => Comparison::Unchanged,
};
let write = match &comparison {
Comparison::Unchanged => recorded.is_none(),
Comparison::Changed(_) => true,
Comparison::Missing(_) => !keep_on_missing,
};
if write {
let mut batch = BatchBuilder::new();
batch.set(class, serde_json::to_vec(current).unwrap_or_default());
data.write(batch.build_all()).await?;
}
Ok(comparison)
}
#[cfg(test)]
mod tests {
use super::*;
fn list(items: &[&str]) -> Vec<String> {
items.iter().map(|s| s.to_string()).collect()
}
#[test]
fn compares_lists() {
assert_eq!(
compare(&list(&["a", "b"]), &list(&["a", "b"])),
Comparison::Unchanged
);
assert!(matches!(
compare(&list(&["a", "b"]), &list(&["a", "b", "c"])),
Comparison::Changed(_)
));
assert!(matches!(
compare(&list(&["a", "b"]), &list(&["b", "a"])),
Comparison::Changed(_)
));
assert_eq!(
compare(&list(&["a", "b", "c"]), &list(&["a", "c"])),
Comparison::Missing(list(&["b"]))
);
}
}
crates/store/src/backend/scaleout/memory.rs
+128
View File
@@ -0,0 +1,128 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! The sharded in-memory and lookup store (ST-23 to ST-29). Every
//! single-key operation goes to the key's home member, with no fallback;
//! operations over many keys go to every member.
use super::{home, layout, without_credentials};
use crate::{InMemoryStore, Store};
use registry::schema::structs::{self, InMemoryStoreBase};
#[derive(Debug)]
pub struct ShardedInMemoryStore {
pub members: Vec<InMemoryStore>,
pub locations: Vec<String>,
}
/// A member's kind and location, without credentials (ST-26, ST-29).
pub fn location(member: &InMemoryStoreBase) -> String {
let urls = |urls: &registry::types::map::Map<String>| {
let mut urls = urls
.iter()
.map(|url| without_credentials(url))
.collect::<Vec<_>>();
urls.sort();
urls.join(",")
};
match member {
InMemoryStoreBase::Redis(redis) => format!("Redis {}", without_credentials(&redis.url)),
InMemoryStoreBase::RedisCluster(cluster) => {
format!("RedisCluster {}", urls(&cluster.urls))
}
InMemoryStoreBase::RedisSentinel(sentinel) => format!(
"RedisSentinel {} {}",
urls(&sentinel.urls),
sentinel.service_name
),
}
}
#[allow(unreachable_patterns, unused_variables)]
async fn open_member(member: InMemoryStoreBase) -> Result<InMemoryStore, String> {
match member {
#[cfg(feature = "redis")]
InMemoryStoreBase::Redis(config) => {
crate::backend::redis::RedisStore::open_single(config).await
}
#[cfg(feature = "redis")]
InMemoryStoreBase::RedisCluster(config) => {
crate::backend::redis::RedisStore::open_cluster(config).await
}
#[cfg(feature = "redis")]
InMemoryStoreBase::RedisSentinel(config) => {
crate::backend::redis::RedisStore::open_sentinel(config).await
}
_ => Err("Binary was not compiled with this member's in-memory backend".to_string()),
}
}
impl ShardedInMemoryStore {
/// Opens every member and checks the list (ST-29), then compares it with
/// the recorded one (ST-26). `name` tells lookup stores apart.
pub async fn open(
config: structs::ShardedInMemoryStore,
name: &str,
data: &Store,
warnings: &mut Vec<String>,
) -> Result<InMemoryStore, String> {
let members = config.stores.into_iter().collect::<Vec<_>>();
if members.len() < 2 {
return Err("A sharded in-memory store needs at least two members".to_string());
}
let locations = members.iter().map(location).collect::<Vec<_>>();
for (index, location) in locations.iter().enumerate() {
if let Some(first) = locations[..index].iter().position(|l| l == location) {
return Err(format!(
"Members {} and {} are the same server: {location}",
first + 1,
index + 1
));
}
}
let mut opened = Vec::with_capacity(members.len());
for (index, member) in members.into_iter().enumerate() {
opened.push(
open_member(member)
.await
.map_err(|err| format!("Member {}: {err}", index + 1))?,
);
}
// ST-26: a different list is an error to fix, not a reason to refuse
let kind = if name.is_empty() { b'm' } else { b'l' };
let difference = match layout::check(data, layout::key(kind, name), &locations, false).await
{
Ok(layout::Comparison::Unchanged) => None,
Ok(layout::Comparison::Changed(change)) => Some(change),
Ok(layout::Comparison::Missing(missing)) => {
Some(format!("members gone: {}", missing.join("; ")))
}
Err(err) => Some(format!("the recorded list couldn't be read: {err}")),
};
if let Some(difference) = difference {
let message = format!(
"The sharded in-memory store's member list differs from the one recorded \
({difference}); every node must use the same list"
);
trc::event!(
Store(trc::StoreEvent::RedisError),
Details = message.clone()
);
warnings.push(message);
}
Ok(InMemoryStore::Sharded(std::sync::Arc::new(
ShardedInMemoryStore {
members: opened,
locations,
},
)))
}
/// The member a key lives on (ST-23).
pub fn member(&self, key: &[u8]) -> &InMemoryStore {
&self.members[home(key, self.members.len())]
}
}
crates/store/src/backend/scaleout/mod.rs
+68
View File
@@ -0,0 +1,68 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! Scale-out storage (`docs/spec/features/scale-out-storage.md`): sharded
//! blob stores (ST-16 to ST-22) and sharded in-memory and lookup stores
//! (ST-23 to ST-29). Each is one more variant of the store enums, whose
//! members are ordinary stores.
pub mod blob;
pub mod layout;
pub mod memory;
pub use blob::ShardedBlobStore;
pub use memory::ShardedInMemoryStore;
/// A key's home: `xxh3_64(key) mod N`, seed 0, over the whole key (ST-16).
/// Fixed forever once shipped.
pub fn home(key: &[u8], members: usize) -> usize {
(xxhash_rust::xxh3::xxh3_64(key) % members.max(1) as u64) as usize
}
/// A URL without its user information, for records and logs.
pub fn without_credentials(url: &str) -> String {
match url.split_once("://") {
Some((scheme, rest)) => {
let rest = match rest.split_once('/') {
Some((authority, path)) => {
let host = authority.rsplit_once('@').map_or(authority, |(_, h)| h);
format!("{host}/{path}")
}
None => rest.rsplit_once('@').map_or(rest, |(_, h)| h).to_string(),
};
format!("{scheme}://{rest}")
}
None => url.rsplit_once('@').map_or(url, |(_, h)| h).to_string(),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn places_and_hides_credentials() {
// Stable placement: these values must never change (ST-16)
assert_eq!(home(b"", 3), (xxhash_rust::xxh3::xxh3_64(b"") % 3) as usize);
let spread = (0u32..3000)
.map(|n| home(&n.to_be_bytes(), 3))
.fold([0; 3], |mut acc, h| {
acc[h] += 1;
acc
});
assert!(spread.iter().all(|n| *n > 800), "{spread:?}");
assert_eq!(
without_credentials("redis://user:secret@host:6379/0"),
"redis://host:6379/0"
);
assert_eq!(without_credentials("rediss://:pw@host"), "rediss://host");
assert_eq!(
without_credentials("redis://host:6379"),
"redis://host:6379"
);
}
}
crates/store/src/build/blob.rs
+14
View File
@@ -39,6 +39,20 @@ impl BlobStore {
structs::BlobStore::FileSystem(file_system_store) => {
FsStore::open(file_system_store).await
}
// inbuxa: ST-16 to ST-22
structs::BlobStore::Sharded(sharded) => {
let mut warnings = Vec::new();
let result = crate::backend::scaleout::ShardedBlobStore::open(
sharded,
&bp.data_store,
&mut warnings,
)
.await;
for warning in warnings {
bp.build_warning(ObjectType::BlobStore.singleton(), warning);
}
result
}
_ => Err("Binary was not compiled with the selected blob store backend".to_string()),
};
crates/store/src/build/lookup.rs
+15
View File
@@ -49,6 +49,21 @@ impl LookupStores {
LookupStore::RedisCluster(redis_cluster_store) => {
crate::backend::redis::RedisStore::open_cluster(redis_cluster_store).await
}
// inbuxa: ST-28
LookupStore::Sharded(sharded) => {
let mut warnings = Vec::new();
let result = crate::backend::scaleout::ShardedInMemoryStore::open(
sharded,
store.namespace.as_str(),
&bp.data_store,
&mut warnings,
)
.await;
for warning in warnings {
bp.build_warning(id, warning);
}
result
}
_ => Err(
"Binary was not compiled with the selected lookup store backend".to_string(),
),
crates/store/src/build/memory.rs
+15
View File
@@ -26,6 +26,21 @@ impl InMemoryStore {
structs::InMemoryStore::RedisSentinel(redis_sentinel_store) => {
crate::backend::redis::RedisStore::open_sentinel(redis_sentinel_store).await
}
// inbuxa: ST-23 to ST-29
structs::InMemoryStore::Sharded(sharded) => {
let mut warnings = Vec::new();
let result = crate::backend::scaleout::ShardedInMemoryStore::open(
sharded,
"",
&bp.data_store,
&mut warnings,
)
.await;
for warning in warnings {
bp.build_warning(ObjectType::InMemoryStore.singleton(), warning);
}
result
}
_ => Err("Binary was not compiled with the selected in-memory backend".to_string()),
};
crates/store/src/dispatch/blob.rs
+70 -54
View File
@@ -16,28 +16,7 @@ const NONE_MARKER: u8 = 0x00;
impl BlobStore {
pub async fn get_blob(&self, key: &[u8], range: Range<usize>) -> trc::Result<Option<Vec<u8>>> {
let start_time = Instant::now();
let result = match &self {
BlobStore::Store(store) => match store {
#[cfg(feature = "sqlite")]
Store::SQLite(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "foundation")]
Store::FoundationDb(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "postgres")]
Store::PostgreSQL(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "mysql")]
Store::MySQL(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "rocks")]
Store::RocksDb(store) => store.get_blob(key, 0..usize::MAX).await,
Store::Ephemeral(store) => store.get_blob(key, 0..usize::MAX).await,
Store::None => Err(trc::StoreEvent::NotConfigured.into()),
},
BlobStore::Fs(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "s3")]
BlobStore::S3(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "azure")]
BlobStore::Azure(store) => store.get_blob(key, 0..usize::MAX).await,
}
.caused_by(trc::location!())?;
let result = self.raw_get(key).await.caused_by(trc::location!())?;
trc::event!(
Store(StoreEvent::BlobRead),
@@ -126,28 +105,7 @@ impl BlobStore {
};
let start_time = Instant::now();
let result = match &self {
BlobStore::Store(store) => match store {
#[cfg(feature = "sqlite")]
Store::SQLite(store) => store.put_blob(key, &data).await,
#[cfg(feature = "foundation")]
Store::FoundationDb(store) => store.put_blob(key, &data).await,
#[cfg(feature = "postgres")]
Store::PostgreSQL(store) => store.put_blob(key, &data).await,
#[cfg(feature = "mysql")]
Store::MySQL(store) => store.put_blob(key, &data).await,
#[cfg(feature = "rocks")]
Store::RocksDb(store) => store.put_blob(key, &data).await,
Store::Ephemeral(store) => store.put_blob(key, &data).await,
Store::None => Err(trc::StoreEvent::NotConfigured.into()),
},
BlobStore::Fs(store) => store.put_blob(key, &data).await,
#[cfg(feature = "s3")]
BlobStore::S3(store) => store.put_blob(key, &data).await,
#[cfg(feature = "azure")]
BlobStore::Azure(store) => store.put_blob(key, &data).await,
}
.caused_by(trc::location!());
let result = self.raw_put(key, &data).await.caused_by(trc::location!());
trc::event!(
Store(StoreEvent::BlobWrite),
@@ -161,7 +119,72 @@ impl BlobStore {
pub async fn delete_blob(&self, key: &[u8]) -> trc::Result<bool> {
let start_time = Instant::now();
let result = match &self {
let result = self.raw_delete(key).await.caused_by(trc::location!());
trc::event!(
Store(StoreEvent::BlobWrite),
Key = key,
Elapsed = start_time.elapsed(),
);
result
}
/// A stored blob as it is on the backend, compression marker included.
pub(crate) async fn raw_get(&self, key: &[u8]) -> trc::Result<Option<Vec<u8>>> {
match &self {
BlobStore::Store(store) => match store {
#[cfg(feature = "sqlite")]
Store::SQLite(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "foundation")]
Store::FoundationDb(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "postgres")]
Store::PostgreSQL(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "mysql")]
Store::MySQL(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "rocks")]
Store::RocksDb(store) => store.get_blob(key, 0..usize::MAX).await,
Store::Ephemeral(store) => store.get_blob(key, 0..usize::MAX).await,
Store::None => Err(trc::StoreEvent::NotConfigured.into()),
},
BlobStore::Fs(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "s3")]
BlobStore::S3(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "azure")]
BlobStore::Azure(store) => store.get_blob(key, 0..usize::MAX).await,
// inbuxa: ST-17
BlobStore::Sharded(store) => store.get(key).await,
}
}
pub(crate) async fn raw_put(&self, key: &[u8], data: &[u8]) -> trc::Result<()> {
match &self {
BlobStore::Store(store) => match store {
#[cfg(feature = "sqlite")]
Store::SQLite(store) => store.put_blob(key, data).await,
#[cfg(feature = "foundation")]
Store::FoundationDb(store) => store.put_blob(key, data).await,
#[cfg(feature = "postgres")]
Store::PostgreSQL(store) => store.put_blob(key, data).await,
#[cfg(feature = "mysql")]
Store::MySQL(store) => store.put_blob(key, data).await,
#[cfg(feature = "rocks")]
Store::RocksDb(store) => store.put_blob(key, data).await,
Store::Ephemeral(store) => store.put_blob(key, data).await,
Store::None => Err(trc::StoreEvent::NotConfigured.into()),
},
BlobStore::Fs(store) => store.put_blob(key, data).await,
#[cfg(feature = "s3")]
BlobStore::S3(store) => store.put_blob(key, data).await,
#[cfg(feature = "azure")]
BlobStore::Azure(store) => store.put_blob(key, data).await,
// inbuxa: ST-18
BlobStore::Sharded(store) => store.put(key, data).await,
}
}
pub(crate) async fn raw_delete(&self, key: &[u8]) -> trc::Result<bool> {
match &self {
BlobStore::Store(store) => match store {
#[cfg(feature = "sqlite")]
Store::SQLite(store) => store.delete_blob(key).await,
@@ -181,15 +204,8 @@ impl BlobStore {
BlobStore::S3(store) => store.delete_blob(key).await,
#[cfg(feature = "azure")]
BlobStore::Azure(store) => store.delete_blob(key).await,
// inbuxa: ST-18
BlobStore::Sharded(store) => store.delete(key).await,
}
.caused_by(trc::location!());
trc::event!(
Store(StoreEvent::BlobWrite),
Key = key,
Elapsed = start_time.elapsed(),
);
result
}
}
crates/store/src/dispatch/lookup.rs
+63
View File
@@ -45,6 +45,11 @@ impl InMemoryStore {
});
store.write(batch.build_all()).await.map(|_| ())
}
// inbuxa: ST-23
InMemoryStore::Sharded(store) => {
let member = store.member(&kv.key);
Box::pin(member.key_set(kv)).await
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(store) => store.key_set(&kv.key, &kv.value, kv.expires).await,
InMemoryStore::Static(_) | InMemoryStore::Http(_) => {
@@ -90,6 +95,11 @@ impl InMemoryStore {
store.write(batch.build_all()).await.map(|_| 0)
}
}
// inbuxa: ST-23
InMemoryStore::Sharded(store) => {
let member = store.member(&kv.key);
Box::pin(member.counter_incr(kv, return_value)).await
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(store) => store.key_incr(&kv.key, kv.value, kv.expires).await,
InMemoryStore::Static(_) | InMemoryStore::Http(_) => {
@@ -109,6 +119,11 @@ impl InMemoryStore {
});
store.write(batch.build_all()).await.map(|_| ())
}
// inbuxa: ST-23
InMemoryStore::Sharded(store) => {
let key = key.into().into_bytes();
Box::pin(store.member(&key).key_delete(key.clone())).await
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(store) => store.key_delete(key.into().as_bytes()).await,
InMemoryStore::Static(_) | InMemoryStore::Http(_) => {
@@ -128,6 +143,11 @@ impl InMemoryStore {
});
store.write(batch.build_all()).await.map(|_| ())
}
// inbuxa: ST-23
InMemoryStore::Sharded(store) => {
let key = key.into().into_bytes();
Box::pin(store.member(&key).counter_delete(key.clone())).await
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(store) => store.key_delete(key.into().as_bytes()).await,
InMemoryStore::Static(_) | InMemoryStore::Http(_) => {
@@ -167,6 +187,15 @@ impl InMemoryStore {
)
.await
}
// inbuxa: ST-24: every member
InMemoryStore::Sharded(store) => {
for (index, member) in store.members.iter().enumerate() {
Box::pin(member.key_delete_prefix(prefix))
.await
.map_err(|err| err.details(format!("Member {}", index + 1)))?;
}
Ok(())
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(store) => store.key_delete_prefix(prefix).await,
InMemoryStore::Static(_) | InMemoryStore::Http(_) => {
@@ -187,6 +216,11 @@ impl InMemoryStore {
)))
.await
.map(|value| value.and_then(|v| v.into())),
// inbuxa: ST-23
InMemoryStore::Sharded(store) => {
let key = key.into().into_bytes();
Box::pin(store.member(&key).key_get::<T>(key.clone())).await
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(store) => store.key_get(key.into().as_bytes()).await,
InMemoryStore::Static(store) => Ok(match store.as_ref() {
@@ -217,6 +251,11 @@ impl InMemoryStore {
)))
.await
}
// inbuxa: ST-23
InMemoryStore::Sharded(store) => {
let key = key.into().into_bytes();
Box::pin(store.member(&key).counter_get(key.clone())).await
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(store) => store.counter_get(key.into().as_bytes()).await,
InMemoryStore::Static(_) | InMemoryStore::Http(_) => {
@@ -234,6 +273,11 @@ impl InMemoryStore {
)))
.await
.map(|value| matches!(value, Some(LookupValue::Value(Empty)))),
// inbuxa: ST-23
InMemoryStore::Sharded(store) => {
let key = key.into().into_bytes();
Box::pin(store.member(&key).key_exists(key.clone())).await
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(store) => store.key_exists(key.into().as_bytes()).await,
InMemoryStore::Static(store) => Ok(match store.as_ref() {
@@ -334,6 +378,15 @@ impl InMemoryStore {
.caused_by(trc::location!())),
}
}
// inbuxa: ST-23: a lock and its release meet on one member
InMemoryStore::Sharded(store) => {
Box::pin(
store
.member(&KeyValue::<()>::build_key(prefix, key))
.try_lock(prefix, key, duration),
)
.await
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(store) => {
store
@@ -431,6 +484,14 @@ impl InMemoryStore {
}
}
}
// inbuxa: ST-24: every member
InMemoryStore::Sharded(store) => {
for (index, member) in store.members.iter().enumerate() {
Box::pin(member.purge_in_memory_store())
.await
.map_err(|err| err.details(format!("Member {}", index + 1)))?;
}
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(_) => {}
InMemoryStore::Static(_) | InMemoryStore::Http(_) => {}
@@ -450,6 +511,8 @@ impl InMemoryStore {
match self {
#[cfg(feature = "redis")]
InMemoryStore::Redis(_) => true,
// inbuxa: ST-3: as its members are
InMemoryStore::Sharded(store) => store.members.iter().all(|m| m.is_redis()),
InMemoryStore::Static(_) => false,
_ => false,
}
crates/store/src/lib.rs
+4
View File
@@ -171,6 +171,8 @@ pub enum BlobStore {
S3(Arc<backend::s3::S3Store>),
#[cfg(feature = "azure")]
Azure(Arc<backend::azure::AzureStore>),
// inbuxa: ST-16 to ST-22
Sharded(Arc<backend::scaleout::ShardedBlobStore>),
}
#[derive(Clone)]
@@ -187,6 +189,8 @@ pub enum InMemoryStore {
Redis(Arc<backend::redis::RedisStore>),
Http(Arc<HttpStore>),
Static(Arc<StaticMemoryStore>),
// inbuxa: ST-23 to ST-29
Sharded(Arc<backend::scaleout::ShardedInMemoryStore>),
}
#[derive(Clone)]