jcoffey-dev
6e50ba25a9
A 3-node rehearsal (PostgreSQL + NATS + Garage) found two ways a crash leaves work stuck: Pool hangs. The PostgreSQL pool (deadpool) was built with no timeouts, so a request waited for a free connection, and for one to be opened or recycled, for as long as it took: forever when the server stopped answering. MySQL's pool (mysql_async) has no wait timeout at all. - PostgreSQL: wait 30 s (or the store's timeout if longer), create the store's timeout or 15 s (it bounds the whole handshake, where tokio-postgres's connect_timeout covers only the TCP connect), recycle 10 s. The pool config is now always set, not only with poolMaxConnections. - MySQL: every connection is taken through MysqlStore::conn(), which gives up after 30 s. - Both: TCP keepalive after 60 s idle, so a server that vanished without closing the connection is noticed in minutes rather than the two-hour system default. The DataStore schema has no pool timeout settings, so these are fixed defaults; the store's own timeout bounds connecting on PostgreSQL. Task locks. A task lock lasted an hour, so after a hard crash the dead node's tasks waited up to an hour and five minutes. The lock is now a five-minute lease: while this node runs a task, the task manager renews its lock every third of the lifetime (InMemoryStore::renew_lock, a compare-and-set on the store backends and SET XX EX on Redis, which leaves a lock that already expired alone). A killed node's tasks run elsewhere within about five minutes plus the claim recheck. A task this node holds isn't handed to a worker again by the scan. store::pool_timeout (new): a local listener that accepts connections and never answers plays a hung server; a PostgreSQL store with a 2 s timeout returns an error in about 4 s, and a MySQL store in 30 s. Without the timeouts both wait for good. store::task_locks gains a task held for 1.5 lock lifetimes: its lease is still held, and released when the task ends.
172 lines
5.8 KiB
Rust
172 lines
5.8 KiB
Rust
/*
|
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
|
*
|
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
|
*
|
|
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
*/
|
|
|
|
use super::{MysqlStore, into_error, is_timeout_error};
|
|
use crate::{Deserialize, IterateParams, Key, ValueKey, write::ValueClass};
|
|
use futures::TryStreamExt;
|
|
use mysql_async::{Row, prelude::Queryable};
|
|
|
|
impl MysqlStore {
|
|
pub(crate) async fn get_value<U>(&self, key: impl Key) -> trc::Result<Option<U>>
|
|
where
|
|
U: Deserialize + 'static,
|
|
{
|
|
let mut conn = self.conn().await?;
|
|
let s = conn
|
|
.prep(format!(
|
|
"SELECT v FROM {} WHERE k = ?",
|
|
char::from(key.subspace())
|
|
))
|
|
.await
|
|
.map_err(into_error)?;
|
|
let key = key.serialize(0);
|
|
conn.exec_first::<Vec<u8>, _, _>(&s, (&key,))
|
|
.await
|
|
.map_err(into_error)
|
|
.and_then(|r| {
|
|
if let Some(r) = r {
|
|
Ok(Some(U::deserialize_owned_with_key(&key, r)?))
|
|
} else {
|
|
Ok(None)
|
|
}
|
|
})
|
|
}
|
|
|
|
pub(crate) async fn key_exists(&self, key: impl Key) -> trc::Result<bool> {
|
|
let mut conn = self.conn().await?;
|
|
let s = conn
|
|
.prep(format!(
|
|
"SELECT 1 FROM {} WHERE k = ?",
|
|
char::from(key.subspace())
|
|
))
|
|
.await
|
|
.map_err(into_error)?;
|
|
let key = key.serialize(0);
|
|
conn.exec_first::<u8, _, _>(&s, (&key,))
|
|
.await
|
|
.map_err(into_error)
|
|
.map(|r| r.is_some())
|
|
}
|
|
|
|
pub(crate) async fn iterate<T: Key>(
|
|
&self,
|
|
params: IterateParams<T>,
|
|
mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result<bool> + Sync + Send,
|
|
) -> trc::Result<()> {
|
|
let mut conn = self.conn().await?;
|
|
let table = char::from(params.begin.subspace());
|
|
let begin = params.begin.serialize(0);
|
|
let end = params.end.serialize(0);
|
|
let keys = if params.values { "k, v" } else { "k" };
|
|
|
|
let s = conn
|
|
.prep(&match (params.first, params.ascending) {
|
|
(true, true) => {
|
|
format!(
|
|
"SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC LIMIT 1"
|
|
)
|
|
}
|
|
(true, false) => {
|
|
format!(
|
|
"SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC LIMIT 1"
|
|
)
|
|
}
|
|
(false, true) => {
|
|
format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC")
|
|
}
|
|
(false, false) => {
|
|
format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC")
|
|
}
|
|
})
|
|
.await
|
|
.map_err(into_error)?;
|
|
let mut from = begin;
|
|
let mut to = end;
|
|
let mut resume_key = None;
|
|
|
|
loop {
|
|
let mut last_key = None;
|
|
let mut timed_out = false;
|
|
|
|
{
|
|
let mut rows = conn
|
|
.exec_stream::<Row, _, _>(&s, (from.clone(), to.clone()))
|
|
.await
|
|
.map_err(into_error)?;
|
|
|
|
loop {
|
|
match rows.try_next().await {
|
|
Ok(Some(mut row)) => {
|
|
let value = if params.values {
|
|
row.take_opt::<Vec<u8>, _>(1)
|
|
.unwrap_or_else(|| Ok(vec![]))
|
|
.map_err(into_error)?
|
|
} else {
|
|
vec![]
|
|
};
|
|
let key = row
|
|
.take_opt::<Vec<u8>, _>(0)
|
|
.unwrap_or_else(|| Ok(vec![]))
|
|
.map_err(into_error)?;
|
|
|
|
if resume_key.take().is_some_and(|resumed| resumed == key) {
|
|
continue;
|
|
}
|
|
|
|
if !cb(&key, &value)? {
|
|
return Ok(());
|
|
}
|
|
|
|
last_key = Some(key);
|
|
}
|
|
Ok(None) => break,
|
|
Err(err) => {
|
|
if params.first || last_key.is_none() || !is_timeout_error(&err) {
|
|
return Err(into_error(err));
|
|
}
|
|
timed_out = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
match last_key {
|
|
Some(last_key) if timed_out => {
|
|
if params.ascending {
|
|
from.clone_from(&last_key);
|
|
} else {
|
|
to.clone_from(&last_key);
|
|
}
|
|
resume_key = Some(last_key);
|
|
}
|
|
_ => return Ok(()),
|
|
}
|
|
}
|
|
}
|
|
|
|
pub(crate) async fn get_counter(
|
|
&self,
|
|
key: impl Into<ValueKey<ValueClass>> + Sync + Send,
|
|
) -> trc::Result<i64> {
|
|
let key = key.into();
|
|
let table = char::from(key.subspace());
|
|
let key = key.serialize(0);
|
|
let mut conn = self.conn().await?;
|
|
let s = conn
|
|
.prep(format!("SELECT v FROM {table} WHERE k = ?"))
|
|
.await
|
|
.map_err(into_error)?;
|
|
match conn.exec_first::<i64, _, _>(&s, (key,)).await {
|
|
Ok(Some(num)) => Ok(num),
|
|
Ok(None) => Ok(0),
|
|
Err(e) => Err(into_error(e)),
|
|
}
|
|
}
|
|
}
|