Scale-out storage: PostgreSQL and MySQL read replicas (ST-5 to ST-15)

A data store with readReplicas becomes a replicated store. Writes,
operator-written SQL and everything outside a read scope go to the
primary. JMAP reads before a request's first write, IMAP LIST, STATUS,
SEARCH, SORT and FETCH, POP3 RETR and TOP, DAV GET, PROPFIND and REPORT,
and blob downloads run in a read scope. Only account data (properties,
indexes, change logs, counters, ACLs, blobs, the search index) is read
from a replica; the registry, in-memory values, the task queue and the
rest stay on the primary.

In a scope, the first read picks a replica round-robin among those up
and under the lag limit, and only if it has every change this node has
written or heard of for the scope's accounts: marks come from write
results, the cluster's state-change broadcasts, a sinceState the client
presents, and, with more than one node, Redis. A write inside the scope
sends the rest of it to the primary. A miss on a replica is looked up on
the primary, and a replica error retries the read there and marks the
replica down.

Each node samples lag every second (WAL positions on PostgreSQL; GTID
sets or Seconds_Behind_Source on MySQL), stops reading from a replica
over 5 s and starts again under 2.5 s, and probes a down replica every
10 s. At startup a replica is left out if it's the primary, isn't
read-only, applies out of commit order, or doesn't show a marker written
to the primary within six tries.

replica_tests (postgres, STORE=PostgreSqlReplicated) runs a primary and a
streaming hot standby in containers: tests 9, 10, 12, 13, 14 and 15 pass.

crates/store/src/dispatch/blob.rs

@@ -145,6 +145,33 @@ impl BlobStore {
                 #[cfg(feature = "rocks")]
                 Store::RocksDb(store) => store.get_blob(key, 0..usize::MAX).await,
                 Store::Ephemeral(store) => store.get_blob(key, 0..usize::MAX).await,
+                // inbuxa: ST-9: a replica, then the primary for what it lacks
+                Store::Replicated(store) => match store.read_target(crate::SUBSPACE_BLOBS).await {
+                    Some(index) => match crate::sql_backend!(
+                        &store.replicas[index].store,
+                        db => db.get_blob(key, 0..usize::MAX).await
+                    ) {
+                        Ok(Some(data)) => {
+                            store.served(index);
+                            Ok(Some(data))
+                        }
+                        Ok(None) => crate::sql_backend!(
+                            &store.primary,
+                            db => db.get_blob(key, 0..usize::MAX).await
+                        ),
+                        Err(err) => {
+                            store.failed(index, err);
+                            crate::sql_backend!(
+                                &store.primary,
+                                db => db.get_blob(key, 0..usize::MAX).await
+                            )
+                        }
+                    },
+                    None => crate::sql_backend!(
+                        &store.primary,
+                        db => db.get_blob(key, 0..usize::MAX).await
+                    ),
+                },
                 Store::None => Err(trc::StoreEvent::NotConfigured.into()),
             },
             BlobStore::Fs(store) => store.get_blob(key, 0..usize::MAX).await,
@@ -171,6 +198,9 @@ impl BlobStore {
                 #[cfg(feature = "rocks")]
                 Store::RocksDb(store) => store.put_blob(key, data).await,
                 Store::Ephemeral(store) => store.put_blob(key, data).await,
+                Store::Replicated(store) => {
+                    crate::sql_backend!(&store.primary, db => db.put_blob(key, data).await)
+                }
                 Store::None => Err(trc::StoreEvent::NotConfigured.into()),
             },
             BlobStore::Fs(store) => store.put_blob(key, data).await,
@@ -197,6 +227,9 @@ impl BlobStore {
                 #[cfg(feature = "rocks")]
                 Store::RocksDb(store) => store.delete_blob(key).await,
                 Store::Ephemeral(store) => store.delete_blob(key).await,
+                Store::Replicated(store) => {
+                    crate::sql_backend!(&store.primary, db => db.delete_blob(key).await)
+                }
                 Store::None => Err(trc::StoreEvent::NotConfigured.into()),
             },
             BlobStore::Fs(store) => store.delete_blob(key).await,

crates/store/src/dispatch/mod.rs

@@ -26,6 +26,8 @@ impl Store {
             #[cfg(feature = "rocks")]
             Self::RocksDb(_) => "rocksdb",
             Self::Ephemeral(_) => "ephemeral",
+            // inbuxa: ST-3: as its primary
+            Self::Replicated(store) => store.primary.id(),
             Self::None => "none",
         }
     }

crates/store/src/dispatch/search.rs

@@ -201,6 +201,25 @@ impl SearchStore {
         filters: &[SearchFilter],
         sort: &[SearchComparator],
     ) -> trc::Result<Vec<u32>> {
+        // inbuxa: ST-6, ST-12: a replica answers search queries in a read scope
+        if let SearchStore::Store(Store::Replicated(store)) = self {
+            return match store.read_target(crate::SUBSPACE_SEARCH_INDEX).await {
+                Some(replica) => match crate::sql_backend!(
+                    &store.replicas[replica].store,
+                    db => db.query(index, filters, sort).await
+                ) {
+                    Ok(ids) => {
+                        store.served(replica);
+                        Ok(ids)
+                    }
+                    Err(err) => {
+                        store.failed(replica, err);
+                        crate::sql_backend!(&store.primary, db => db.query(index, filters, sort).await)
+                    }
+                },
+                None => crate::sql_backend!(&store.primary, db => db.query(index, filters, sort).await),
+            };
+        }
         match self {
             SearchStore::Store(store) => match store {
                 #[cfg(feature = "postgres")]
@@ -215,6 +234,13 @@ impl SearchStore {
     }
 
     pub async fn query_global(&self, query: SearchQuery) -> trc::Result<Vec<u64>> {
+        // inbuxa: ST-5: global queries are maintenance, on the primary
+        if let SearchStore::Store(Store::Replicated(store)) = self {
+            return crate::sql_backend!(
+                &store.primary,
+                db => db.query(query.index, &query.filters, &query.comparators).await
+            );
+        }
         match self {
             SearchStore::Store(store) => match store {
                 #[cfg(feature = "postgres")]
@@ -245,6 +271,9 @@ impl SearchStore {
     }
 
     pub async fn index(&self, documents: Vec<IndexDocument>) -> trc::Result<()> {
+        if let SearchStore::Store(Store::Replicated(store)) = self {
+            return crate::sql_backend!(&store.primary, db => db.index(documents).await);
+        }
         match self {
             SearchStore::Store(store) => match store {
                 #[cfg(feature = "postgres")]
@@ -259,6 +288,9 @@ impl SearchStore {
     }
 
     pub async fn unindex(&self, query: SearchQuery) -> trc::Result<u64> {
+        if let SearchStore::Store(Store::Replicated(store)) = self {
+            return crate::sql_backend!(&store.primary, db => db.unindex(query).await);
+        }
         match self {
             SearchStore::Store(store) => match store {
                 #[cfg(feature = "postgres")]
@@ -279,6 +311,8 @@ impl SearchStore {
                 Store::PostgreSQL(_) => None,
                 #[cfg(feature = "mysql")]
                 Store::MySQL(_) => None,
+                // inbuxa: ST-3: as its primary
+                Store::Replicated(replicated) if replicated.primary.is_pg_or_mysql() => None,
                 store => Some(store),
             },
             _ => None,
@@ -289,6 +323,10 @@ impl SearchStore {
         match self {
             #[cfg(feature = "mysql")]
             SearchStore::Store(Store::MySQL(_)) => true,
+            #[cfg(feature = "mysql")]
+            SearchStore::Store(Store::Replicated(store)) => {
+                matches!(store.primary, Store::MySQL(_))
+            }
             _ => false,
         }
     }
@@ -297,6 +335,10 @@ impl SearchStore {
         match self {
             #[cfg(feature = "postgres")]
             SearchStore::Store(Store::PostgreSQL(_)) => true,
+            #[cfg(feature = "postgres")]
+            SearchStore::Store(Store::Replicated(store)) => {
+                matches!(store.primary, Store::PostgreSQL(_))
+            }
             _ => false,
         }
     }
@@ -316,6 +358,9 @@ impl SearchStore {
                 Store::PostgreSQL(store) => store.create_search_tables().await,
                 #[cfg(feature = "mysql")]
                 Store::MySQL(store) => store.create_search_tables().await,
+                Store::Replicated(store) => {
+                    crate::sql_backend!(&store.primary, db => db.create_search_tables().await)
+                }
                 _ => Ok(()),
             },
             SearchStore::ElasticSearch(store) => store.create_indexes().await,

crates/store/src/dispatch/store.rs

@@ -35,6 +35,23 @@ impl Store {
             #[cfg(feature = "rocks")]
             Self::RocksDb(store) => store.get_value(key).await,
             Self::Ephemeral(store) => store.get_value(key).await,
+            // inbuxa: ST-6 to ST-8, ST-12
+            Self::Replicated(store) => match store.read_target(key.subspace()).await {
+                Some(index) => {
+                    match crate::sql_backend!(&store.replicas[index].store, db => db.get_value::<U>(key.clone()).await) {
+                        Ok(Some(value)) => {
+                            store.served(index);
+                            Ok(Some(value))
+                        }
+                        Ok(None) => crate::sql_backend!(&store.primary, db => db.get_value(key).await),
+                        Err(err) => {
+                            store.failed(index, err);
+                            crate::sql_backend!(&store.primary, db => db.get_value(key).await)
+                        }
+                    }
+                }
+                None => crate::sql_backend!(&store.primary, db => db.get_value(key).await),
+            },
             Self::None => Err(trc::StoreEvent::NotConfigured.into()),
         }
         .caused_by(trc::location!())
@@ -53,6 +70,21 @@ impl Store {
             #[cfg(feature = "rocks")]
             Self::RocksDb(store) => store.key_exists(key).await,
             Self::Ephemeral(store) => store.key_exists(key).await,
+            // inbuxa: ST-6 to ST-8, ST-12
+            Self::Replicated(store) => match store.read_target(key.subspace()).await {
+                Some(index) => match crate::sql_backend!(&store.replicas[index].store, db => db.key_exists(key.clone()).await) {
+                    Ok(true) => {
+                        store.served(index);
+                        Ok(true)
+                    }
+                    Ok(false) => crate::sql_backend!(&store.primary, db => db.key_exists(key).await),
+                    Err(err) => {
+                        store.failed(index, err);
+                        crate::sql_backend!(&store.primary, db => db.key_exists(key).await)
+                    }
+                },
+                None => crate::sql_backend!(&store.primary, db => db.key_exists(key).await),
+            },
             Self::None => Err(trc::StoreEvent::NotConfigured.into()),
         }
         .caused_by(trc::location!())
@@ -76,6 +108,38 @@ impl Store {
             #[cfg(feature = "rocks")]
             Self::RocksDb(store) => store.iterate(params, cb).await,
             Self::Ephemeral(store) => store.iterate(params, cb).await,
+            // inbuxa: ST-6, ST-12: a failed replica iteration is repeated on
+            // the primary when nothing was handed to the callback yet
+            #[allow(unused_mut, unused_variables)]
+            Self::Replicated(store) => {
+                let mut cb = cb;
+                match store.read_target(params.begin.subspace()).await {
+                    Some(index) => {
+                        let mut called = false;
+                        let result = crate::sql_backend!(&store.replicas[index].store, db => db
+                            .iterate(params.clone(), |key, value| {
+                                called = true;
+                                cb(key, value)
+                            })
+                            .await);
+                        match result {
+                            Ok(()) => {
+                                store.served(index);
+                                Ok(())
+                            }
+                            Err(err) if !called => {
+                                store.failed(index, err);
+                                crate::sql_backend!(&store.primary, db => db.iterate(params, cb).await)
+                            }
+                            Err(err) => {
+                                store.failed(index, err.clone());
+                                Err(err)
+                            }
+                        }
+                    }
+                    None => crate::sql_backend!(&store.primary, db => db.iterate(params, cb).await),
+                }
+            }
             Self::None => Err(trc::StoreEvent::NotConfigured.into()),
         }
         .caused_by(trc::location!());
@@ -104,6 +168,25 @@ impl Store {
             #[cfg(feature = "rocks")]
             Self::RocksDb(store) => store.get_counter(key).await,
             Self::Ephemeral(store) => store.get_counter(key).await,
+            // inbuxa: ST-6, ST-12
+            Self::Replicated(store) => {
+                let key: ValueKey<ValueClass> = key.into();
+                match store.read_target(crate::Key::subspace(&key)).await {
+                    Some(index) => {
+                        match crate::sql_backend!(&store.replicas[index].store, db => db.get_counter(key.clone()).await) {
+                            Ok(value) => {
+                                store.served(index);
+                                Ok(value)
+                            }
+                            Err(err) => {
+                                store.failed(index, err);
+                                crate::sql_backend!(&store.primary, db => db.get_counter(key).await)
+                            }
+                        }
+                    }
+                    None => crate::sql_backend!(&store.primary, db => db.get_counter(key).await),
+                }
+            }
             Self::None => Err(trc::StoreEvent::NotConfigured.into()),
         }
         .caused_by(trc::location!())
@@ -123,6 +206,10 @@ impl Store {
             Self::PostgreSQL(store) => store.sql_query(query, &params).await,
             #[cfg(feature = "mysql")]
             Self::MySQL(store) => store.sql_query(query, &params).await,
+            // inbuxa: ST-5: operator-written statements always go to the primary
+            Self::Replicated(store) => {
+                crate::sql_backend!(&store.primary, db => db.sql_query(query, &params).await)
+            }
             _ => Err(trc::StoreEvent::NotSupported.into_err()),
         };
 
@@ -152,6 +239,15 @@ impl Store {
             #[cfg(feature = "rocks")]
             Self::RocksDb(store) => store.write(batch).await,
             Self::Ephemeral(store) => store.write(batch).await,
+            // inbuxa: ST-5, ST-7: writes go to the primary, and record marks
+            Self::Replicated(store) => {
+                crate::backend::scaleout::replica::note_scope_write();
+                let result = crate::sql_backend!(&store.primary, db => db.write(batch).await);
+                if let Ok(ids) = &result {
+                    store.note_write(ids).await;
+                }
+                result
+            }
             Self::None => Err(trc::StoreEvent::NotConfigured.into()),
         };
 
@@ -197,6 +293,7 @@ impl Store {
             #[cfg(feature = "rocks")]
             Self::RocksDb(store) => store.purge_store().await,
             Self::Ephemeral(store) => store.purge_store().await,
+            Self::Replicated(store) => crate::sql_backend!(&store.primary, db => db.purge_store().await),
             Self::None => Err(trc::StoreEvent::NotConfigured.into()),
         }
         .caused_by(trc::location!())
@@ -215,6 +312,9 @@ impl Store {
             #[cfg(feature = "rocks")]
             Self::RocksDb(store) => store.delete_range(from, to).await,
             Self::Ephemeral(store) => store.delete_range(from, to).await,
+            Self::Replicated(store) => {
+                crate::sql_backend!(&store.primary, db => db.delete_range(from, to).await)
+            }
             Self::None => Err(trc::StoreEvent::NotConfigured.into()),
         }
         .caused_by(trc::location!())
@@ -360,6 +460,9 @@ impl Store {
             Self::PostgreSQL(store) => store.create_storage_tables().await,
             #[cfg(feature = "mysql")]
             Self::MySQL(store) => store.create_storage_tables().await,
+            Self::Replicated(store) => {
+                crate::sql_backend!(&store.primary, db => db.create_storage_tables().await)
+            }
             _ => Ok(()),
         }
     }