inbuxa/inbuxa-server
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 3 Wiki Activity

Scale-out storage: sharded blob, in-memory and lookup stores; configured read replicas reported (ST-1 to ST-4, ST-16 to ST-30)

Browse Source 
A Sharded blob store places each blob on xxh3(key) mod N, over the whole
key; reads fall back to the other members, so blobs placed under an
earlier member list stay readable, and deletes find them wherever they
are. The member list is recorded in the data store (secrets left out):
added or reordered members are a warning, a missing one refuses to open.
Blobs are compressed and marked before they reach a member. A Sharded
in-memory or lookup store sends each key to its home Redis member, and
prefix deletes and purges to all; a node whose member list differs from
the recorded one logs an error and runs on. Members are checked for
duplicates and must all open.

Until read-replica routing is built, each configured replica is reported
at startup instead of being silently ignored, and nothing connects to it.
The new scaleout_blob_tests covers tests 2 to 7, and the existing blob
suite passes against three FileSystem members (BLOB_STORE=Sharded).
This commit is contained in:
jcoffey-dev
2026-09-19 10:11:30 -07:00
parent e7efa91cbc
commit e5e326de6b
16 changed files with 1144 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/store/src/dispatch/blob.rs
+70 -54
View File
@@ -16,28 +16,7 @@ const NONE_MARKER: u8 = 0x00;
impl BlobStore {
pub async fn get_blob(&self, key: &[u8], range: Range<usize>) -> trc::Result<Option<Vec<u8>>> {
let start_time = Instant::now();
let result = match &self {
BlobStore::Store(store) => match store {
#[cfg(feature = "sqlite")]
Store::SQLite(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "foundation")]
Store::FoundationDb(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "postgres")]
Store::PostgreSQL(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "mysql")]
Store::MySQL(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "rocks")]
Store::RocksDb(store) => store.get_blob(key, 0..usize::MAX).await,
Store::Ephemeral(store) => store.get_blob(key, 0..usize::MAX).await,
Store::None => Err(trc::StoreEvent::NotConfigured.into()),
},
BlobStore::Fs(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "s3")]
BlobStore::S3(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "azure")]
BlobStore::Azure(store) => store.get_blob(key, 0..usize::MAX).await,
}
.caused_by(trc::location!())?;
let result = self.raw_get(key).await.caused_by(trc::location!())?;
trc::event!(
Store(StoreEvent::BlobRead),
@@ -126,28 +105,7 @@ impl BlobStore {
};
let start_time = Instant::now();
let result = match &self {
BlobStore::Store(store) => match store {
#[cfg(feature = "sqlite")]
Store::SQLite(store) => store.put_blob(key, &data).await,
#[cfg(feature = "foundation")]
Store::FoundationDb(store) => store.put_blob(key, &data).await,
#[cfg(feature = "postgres")]
Store::PostgreSQL(store) => store.put_blob(key, &data).await,
#[cfg(feature = "mysql")]
Store::MySQL(store) => store.put_blob(key, &data).await,
#[cfg(feature = "rocks")]
Store::RocksDb(store) => store.put_blob(key, &data).await,
Store::Ephemeral(store) => store.put_blob(key, &data).await,
Store::None => Err(trc::StoreEvent::NotConfigured.into()),
},
BlobStore::Fs(store) => store.put_blob(key, &data).await,
#[cfg(feature = "s3")]
BlobStore::S3(store) => store.put_blob(key, &data).await,
#[cfg(feature = "azure")]
BlobStore::Azure(store) => store.put_blob(key, &data).await,
}
.caused_by(trc::location!());
let result = self.raw_put(key, &data).await.caused_by(trc::location!());
trc::event!(
Store(StoreEvent::BlobWrite),
@@ -161,7 +119,72 @@ impl BlobStore {
pub async fn delete_blob(&self, key: &[u8]) -> trc::Result<bool> {
let start_time = Instant::now();
let result = match &self {
let result = self.raw_delete(key).await.caused_by(trc::location!());
trc::event!(
Store(StoreEvent::BlobWrite),
Key = key,
Elapsed = start_time.elapsed(),
);
result
}
/// A stored blob as it is on the backend, compression marker included.
pub(crate) async fn raw_get(&self, key: &[u8]) -> trc::Result<Option<Vec<u8>>> {
match &self {
BlobStore::Store(store) => match store {
#[cfg(feature = "sqlite")]
Store::SQLite(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "foundation")]
Store::FoundationDb(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "postgres")]
Store::PostgreSQL(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "mysql")]
Store::MySQL(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "rocks")]
Store::RocksDb(store) => store.get_blob(key, 0..usize::MAX).await,
Store::Ephemeral(store) => store.get_blob(key, 0..usize::MAX).await,
Store::None => Err(trc::StoreEvent::NotConfigured.into()),
},
BlobStore::Fs(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "s3")]
BlobStore::S3(store) => store.get_blob(key, 0..usize::MAX).await,
#[cfg(feature = "azure")]
BlobStore::Azure(store) => store.get_blob(key, 0..usize::MAX).await,
// inbuxa: ST-17
BlobStore::Sharded(store) => store.get(key).await,
}
}
pub(crate) async fn raw_put(&self, key: &[u8], data: &[u8]) -> trc::Result<()> {
match &self {
BlobStore::Store(store) => match store {
#[cfg(feature = "sqlite")]
Store::SQLite(store) => store.put_blob(key, data).await,
#[cfg(feature = "foundation")]
Store::FoundationDb(store) => store.put_blob(key, data).await,
#[cfg(feature = "postgres")]
Store::PostgreSQL(store) => store.put_blob(key, data).await,
#[cfg(feature = "mysql")]
Store::MySQL(store) => store.put_blob(key, data).await,
#[cfg(feature = "rocks")]
Store::RocksDb(store) => store.put_blob(key, data).await,
Store::Ephemeral(store) => store.put_blob(key, data).await,
Store::None => Err(trc::StoreEvent::NotConfigured.into()),
},
BlobStore::Fs(store) => store.put_blob(key, data).await,
#[cfg(feature = "s3")]
BlobStore::S3(store) => store.put_blob(key, data).await,
#[cfg(feature = "azure")]
BlobStore::Azure(store) => store.put_blob(key, data).await,
// inbuxa: ST-18
BlobStore::Sharded(store) => store.put(key, data).await,
}
}
pub(crate) async fn raw_delete(&self, key: &[u8]) -> trc::Result<bool> {
match &self {
BlobStore::Store(store) => match store {
#[cfg(feature = "sqlite")]
Store::SQLite(store) => store.delete_blob(key).await,
@@ -181,15 +204,8 @@ impl BlobStore {
BlobStore::S3(store) => store.delete_blob(key).await,
#[cfg(feature = "azure")]
BlobStore::Azure(store) => store.delete_blob(key).await,
// inbuxa: ST-18
BlobStore::Sharded(store) => store.delete(key).await,
}
.caused_by(trc::location!());
trc::event!(
Store(StoreEvent::BlobWrite),
Key = key,
Elapsed = start_time.elapsed(),
);
result
}
}
crates/store/src/dispatch/lookup.rs
+63
View File
@@ -45,6 +45,11 @@ impl InMemoryStore {
});
store.write(batch.build_all()).await.map(|_| ())
}
// inbuxa: ST-23
InMemoryStore::Sharded(store) => {
let member = store.member(&kv.key);
Box::pin(member.key_set(kv)).await
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(store) => store.key_set(&kv.key, &kv.value, kv.expires).await,
InMemoryStore::Static(_) | InMemoryStore::Http(_) => {
@@ -90,6 +95,11 @@ impl InMemoryStore {
store.write(batch.build_all()).await.map(|_| 0)
}
}
// inbuxa: ST-23
InMemoryStore::Sharded(store) => {
let member = store.member(&kv.key);
Box::pin(member.counter_incr(kv, return_value)).await
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(store) => store.key_incr(&kv.key, kv.value, kv.expires).await,
InMemoryStore::Static(_) | InMemoryStore::Http(_) => {
@@ -109,6 +119,11 @@ impl InMemoryStore {
});
store.write(batch.build_all()).await.map(|_| ())
}
// inbuxa: ST-23
InMemoryStore::Sharded(store) => {
let key = key.into().into_bytes();
Box::pin(store.member(&key).key_delete(key.clone())).await
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(store) => store.key_delete(key.into().as_bytes()).await,
InMemoryStore::Static(_) | InMemoryStore::Http(_) => {
@@ -128,6 +143,11 @@ impl InMemoryStore {
});
store.write(batch.build_all()).await.map(|_| ())
}
// inbuxa: ST-23
InMemoryStore::Sharded(store) => {
let key = key.into().into_bytes();
Box::pin(store.member(&key).counter_delete(key.clone())).await
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(store) => store.key_delete(key.into().as_bytes()).await,
InMemoryStore::Static(_) | InMemoryStore::Http(_) => {
@@ -167,6 +187,15 @@ impl InMemoryStore {
)
.await
}
// inbuxa: ST-24: every member
InMemoryStore::Sharded(store) => {
for (index, member) in store.members.iter().enumerate() {
Box::pin(member.key_delete_prefix(prefix))
.await
.map_err(|err| err.details(format!("Member {}", index + 1)))?;
}
Ok(())
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(store) => store.key_delete_prefix(prefix).await,
InMemoryStore::Static(_) | InMemoryStore::Http(_) => {
@@ -187,6 +216,11 @@ impl InMemoryStore {
)))
.await
.map(|value| value.and_then(|v| v.into())),
// inbuxa: ST-23
InMemoryStore::Sharded(store) => {
let key = key.into().into_bytes();
Box::pin(store.member(&key).key_get::<T>(key.clone())).await
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(store) => store.key_get(key.into().as_bytes()).await,
InMemoryStore::Static(store) => Ok(match store.as_ref() {
@@ -217,6 +251,11 @@ impl InMemoryStore {
)))
.await
}
// inbuxa: ST-23
InMemoryStore::Sharded(store) => {
let key = key.into().into_bytes();
Box::pin(store.member(&key).counter_get(key.clone())).await
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(store) => store.counter_get(key.into().as_bytes()).await,
InMemoryStore::Static(_) | InMemoryStore::Http(_) => {
@@ -234,6 +273,11 @@ impl InMemoryStore {
)))
.await
.map(|value| matches!(value, Some(LookupValue::Value(Empty)))),
// inbuxa: ST-23
InMemoryStore::Sharded(store) => {
let key = key.into().into_bytes();
Box::pin(store.member(&key).key_exists(key.clone())).await
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(store) => store.key_exists(key.into().as_bytes()).await,
InMemoryStore::Static(store) => Ok(match store.as_ref() {
@@ -334,6 +378,15 @@ impl InMemoryStore {
.caused_by(trc::location!())),
}
}
// inbuxa: ST-23: a lock and its release meet on one member
InMemoryStore::Sharded(store) => {
Box::pin(
store
.member(&KeyValue::<()>::build_key(prefix, key))
.try_lock(prefix, key, duration),
)
.await
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(store) => {
store
@@ -431,6 +484,14 @@ impl InMemoryStore {
}
}
}
// inbuxa: ST-24: every member
InMemoryStore::Sharded(store) => {
for (index, member) in store.members.iter().enumerate() {
Box::pin(member.purge_in_memory_store())
.await
.map_err(|err| err.details(format!("Member {}", index + 1)))?;
}
}
#[cfg(feature = "redis")]
InMemoryStore::Redis(_) => {}
InMemoryStore::Static(_) | InMemoryStore::Http(_) => {}
@@ -450,6 +511,8 @@ impl InMemoryStore {
match self {
#[cfg(feature = "redis")]
InMemoryStore::Redis(_) => true,
// inbuxa: ST-3: as its members are
InMemoryStore::Sharded(store) => store.members.iter().all(|m| m.is_redis()),
InMemoryStore::Static(_) => false,
_ => false,
}