inbuxa/inbuxa-server
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 3 Wiki Activity

Scale-out storage: PostgreSQL and MySQL read replicas (ST-5 to ST-15)

Browse Source 
A data store with readReplicas becomes a replicated store. Writes,
operator-written SQL and everything outside a read scope go to the
primary. JMAP reads before a request's first write, IMAP LIST, STATUS,
SEARCH, SORT and FETCH, POP3 RETR and TOP, DAV GET, PROPFIND and REPORT,
and blob downloads run in a read scope. Only account data (properties,
indexes, change logs, counters, ACLs, blobs, the search index) is read
from a replica; the registry, in-memory values, the task queue and the
rest stay on the primary.

In a scope, the first read picks a replica round-robin among those up
and under the lag limit, and only if it has every change this node has
written or heard of for the scope's accounts: marks come from write
results, the cluster's state-change broadcasts, a sinceState the client
presents, and, with more than one node, Redis. A write inside the scope
sends the rest of it to the primary. A miss on a replica is looked up on
the primary, and a replica error retries the read there and marks the
replica down.

Each node samples lag every second (WAL positions on PostgreSQL; GTID
sets or Seconds_Behind_Source on MySQL), stops reading from a replica
over 5 s and starts again under 2.5 s, and probes a down replica every
10 s. At startup a replica is left out if it's the primary, isn't
read-only, applies out of commit order, or doesn't show a marker written
to the primary within six tries.

replica_tests (postgres, STORE=PostgreSqlReplicated) runs a primary and a
streaming hot standby in containers: tests 9, 10, 12, 13, 14 and 15 pass.
This commit is contained in:
jcoffey-dev
2026-09-19 14:05:59 -07:00
parent a635b490ec
commit 1518c69033
24 changed files with 1722 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/store/src/dispatch/store.rs
+103
View File
@@ -35,6 +35,23 @@ impl Store {
#[cfg(feature = "rocks")]
Self::RocksDb(store) => store.get_value(key).await,
Self::Ephemeral(store) => store.get_value(key).await,
// inbuxa: ST-6 to ST-8, ST-12
Self::Replicated(store) => match store.read_target(key.subspace()).await {
Some(index) => {
match crate::sql_backend!(&store.replicas[index].store, db => db.get_value::<U>(key.clone()).await) {
Ok(Some(value)) => {
store.served(index);
Ok(Some(value))
}
Ok(None) => crate::sql_backend!(&store.primary, db => db.get_value(key).await),
Err(err) => {
store.failed(index, err);
crate::sql_backend!(&store.primary, db => db.get_value(key).await)
}
}
}
None => crate::sql_backend!(&store.primary, db => db.get_value(key).await),
},
Self::None => Err(trc::StoreEvent::NotConfigured.into()),
}
.caused_by(trc::location!())
@@ -53,6 +70,21 @@ impl Store {
#[cfg(feature = "rocks")]
Self::RocksDb(store) => store.key_exists(key).await,
Self::Ephemeral(store) => store.key_exists(key).await,
// inbuxa: ST-6 to ST-8, ST-12
Self::Replicated(store) => match store.read_target(key.subspace()).await {
Some(index) => match crate::sql_backend!(&store.replicas[index].store, db => db.key_exists(key.clone()).await) {
Ok(true) => {
store.served(index);
Ok(true)
}
Ok(false) => crate::sql_backend!(&store.primary, db => db.key_exists(key).await),
Err(err) => {
store.failed(index, err);
crate::sql_backend!(&store.primary, db => db.key_exists(key).await)
}
},
None => crate::sql_backend!(&store.primary, db => db.key_exists(key).await),
},
Self::None => Err(trc::StoreEvent::NotConfigured.into()),
}
.caused_by(trc::location!())
@@ -76,6 +108,38 @@ impl Store {
#[cfg(feature = "rocks")]
Self::RocksDb(store) => store.iterate(params, cb).await,
Self::Ephemeral(store) => store.iterate(params, cb).await,
// inbuxa: ST-6, ST-12: a failed replica iteration is repeated on
// the primary when nothing was handed to the callback yet
#[allow(unused_mut, unused_variables)]
Self::Replicated(store) => {
let mut cb = cb;
match store.read_target(params.begin.subspace()).await {
Some(index) => {
let mut called = false;
let result = crate::sql_backend!(&store.replicas[index].store, db => db
.iterate(params.clone(), |key, value| {
called = true;
cb(key, value)
})
.await);
match result {
Ok(()) => {
store.served(index);
Ok(())
}
Err(err) if !called => {
store.failed(index, err);
crate::sql_backend!(&store.primary, db => db.iterate(params, cb).await)
}
Err(err) => {
store.failed(index, err.clone());
Err(err)
}
}
}
None => crate::sql_backend!(&store.primary, db => db.iterate(params, cb).await),
}
}
Self::None => Err(trc::StoreEvent::NotConfigured.into()),
}
.caused_by(trc::location!());
@@ -104,6 +168,25 @@ impl Store {
#[cfg(feature = "rocks")]
Self::RocksDb(store) => store.get_counter(key).await,
Self::Ephemeral(store) => store.get_counter(key).await,
// inbuxa: ST-6, ST-12
Self::Replicated(store) => {
let key: ValueKey<ValueClass> = key.into();
match store.read_target(crate::Key::subspace(&key)).await {
Some(index) => {
match crate::sql_backend!(&store.replicas[index].store, db => db.get_counter(key.clone()).await) {
Ok(value) => {
store.served(index);
Ok(value)
}
Err(err) => {
store.failed(index, err);
crate::sql_backend!(&store.primary, db => db.get_counter(key).await)
}
}
}
None => crate::sql_backend!(&store.primary, db => db.get_counter(key).await),
}
}
Self::None => Err(trc::StoreEvent::NotConfigured.into()),
}
.caused_by(trc::location!())
@@ -123,6 +206,10 @@ impl Store {
Self::PostgreSQL(store) => store.sql_query(query, &params).await,
#[cfg(feature = "mysql")]
Self::MySQL(store) => store.sql_query(query, &params).await,
// inbuxa: ST-5: operator-written statements always go to the primary
Self::Replicated(store) => {
crate::sql_backend!(&store.primary, db => db.sql_query(query, &params).await)
}
_ => Err(trc::StoreEvent::NotSupported.into_err()),
};
@@ -152,6 +239,15 @@ impl Store {
#[cfg(feature = "rocks")]
Self::RocksDb(store) => store.write(batch).await,
Self::Ephemeral(store) => store.write(batch).await,
// inbuxa: ST-5, ST-7: writes go to the primary, and record marks
Self::Replicated(store) => {
crate::backend::scaleout::replica::note_scope_write();
let result = crate::sql_backend!(&store.primary, db => db.write(batch).await);
if let Ok(ids) = &result {
store.note_write(ids).await;
}
result
}
Self::None => Err(trc::StoreEvent::NotConfigured.into()),
};
@@ -197,6 +293,7 @@ impl Store {
#[cfg(feature = "rocks")]
Self::RocksDb(store) => store.purge_store().await,
Self::Ephemeral(store) => store.purge_store().await,
Self::Replicated(store) => crate::sql_backend!(&store.primary, db => db.purge_store().await),
Self::None => Err(trc::StoreEvent::NotConfigured.into()),
}
.caused_by(trc::location!())
@@ -215,6 +312,9 @@ impl Store {
#[cfg(feature = "rocks")]
Self::RocksDb(store) => store.delete_range(from, to).await,
Self::Ephemeral(store) => store.delete_range(from, to).await,
Self::Replicated(store) => {
crate::sql_backend!(&store.primary, db => db.delete_range(from, to).await)
}
Self::None => Err(trc::StoreEvent::NotConfigured.into()),
}
.caused_by(trc::location!())
@@ -360,6 +460,9 @@ impl Store {
Self::PostgreSQL(store) => store.create_storage_tables().await,
#[cfg(feature = "mysql")]
Self::MySQL(store) => store.create_storage_tables().await,
Self::Replicated(store) => {
crate::sql_backend!(&store.primary, db => db.create_storage_tables().await)
}
_ => Ok(()),
}
}