Import upstream v0.16.22, stripped
Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f Enterprise-only files removed or emptied: 63 Enterprise-only snippets removed: 117 in 50 files Dangling module declarations removed: 5 Cargo edits turning enterprise off: 14 Verification: clean Enterprise feature gates left for rebuilt features: 19 in 18 files Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.
This commit is contained in:
@@ -0,0 +1,64 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use std::ops::Range;
|
||||
|
||||
use mysql_async::prelude::Queryable;
|
||||
|
||||
use super::{MysqlStore, into_error};
|
||||
|
||||
impl MysqlStore {
|
||||
pub(crate) async fn get_blob(
|
||||
&self,
|
||||
key: &[u8],
|
||||
range: Range<usize>,
|
||||
) -> trc::Result<Option<Vec<u8>>> {
|
||||
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||
let s = conn
|
||||
.prep("SELECT v FROM t WHERE k = ?")
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
conn.exec_first::<Vec<u8>, _, _>(&s, (key,))
|
||||
.await
|
||||
.map(|bytes| {
|
||||
if range.start == 0 && range.end == usize::MAX {
|
||||
bytes
|
||||
} else {
|
||||
bytes.map(|bytes| {
|
||||
bytes
|
||||
.get(range.start..std::cmp::min(bytes.len(), range.end))
|
||||
.unwrap_or_default()
|
||||
.to_vec()
|
||||
})
|
||||
}
|
||||
})
|
||||
.map_err(into_error)
|
||||
}
|
||||
|
||||
pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> {
|
||||
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||
let s = conn
|
||||
.prep("INSERT INTO t (k, v) VALUES (?, ?) ON DUPLICATE KEY UPDATE v = VALUES(v)")
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
conn.exec_drop(&s, (key, data))
|
||||
.await
|
||||
.map_err(into_error)
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result<bool> {
|
||||
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||
let s = conn
|
||||
.prep("DELETE FROM t WHERE k = ?")
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
conn.exec_iter(&s, (key,))
|
||||
.await
|
||||
.map_err(into_error)
|
||||
.map(|hits| hits.affected_rows() > 0)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use mysql_async::{Params, Row, prelude::Queryable};
|
||||
|
||||
use crate::{IntoRows, QueryResult, QueryType, Value};
|
||||
|
||||
use super::{MysqlStore, into_error};
|
||||
|
||||
impl MysqlStore {
|
||||
pub(crate) async fn sql_query<T: QueryResult>(
|
||||
&self,
|
||||
query: &str,
|
||||
params: &[Value<'_>],
|
||||
) -> trc::Result<T> {
|
||||
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||
let s = conn.prep(query).await.map_err(into_error)?;
|
||||
let params = Params::Positional(params.iter().map(Into::into).collect());
|
||||
|
||||
match T::query_type() {
|
||||
QueryType::Execute => conn.exec_drop(s, params).await.map_or_else(
|
||||
|e| Err(into_error(e)),
|
||||
|_| Ok(T::from_exec(conn.affected_rows() as usize)),
|
||||
),
|
||||
QueryType::Exists => conn
|
||||
.exec_first::<Row, _, _>(s, params)
|
||||
.await
|
||||
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_exists(r.is_some()))),
|
||||
QueryType::QueryOne => conn
|
||||
.exec_first::<Row, _, _>(s, params)
|
||||
.await
|
||||
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_query_one(r))),
|
||||
QueryType::QueryAll => conn
|
||||
.exec::<Row, _, _>(s, params)
|
||||
.await
|
||||
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_query_all(r))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<crate::Value<'_>> for mysql_async::Value {
|
||||
fn from(value: crate::Value) -> Self {
|
||||
match value {
|
||||
crate::Value::Integer(i) => mysql_async::Value::Int(i),
|
||||
crate::Value::Bool(b) => mysql_async::Value::Int(b as i64),
|
||||
crate::Value::Float(f) => mysql_async::Value::Double(f),
|
||||
crate::Value::Text(t) => mysql_async::Value::Bytes(t.into_owned().into_bytes()),
|
||||
crate::Value::Blob(b) => mysql_async::Value::Bytes(b.into_owned()),
|
||||
crate::Value::Null => mysql_async::Value::NULL,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<mysql_async::Value> for crate::Value<'static> {
|
||||
fn from(value: mysql_async::Value) -> Self {
|
||||
match value {
|
||||
mysql_async::Value::Int(i) => Self::Integer(i),
|
||||
mysql_async::Value::UInt(i) => Self::Integer(i as i64),
|
||||
mysql_async::Value::Double(f) => Self::Float(f),
|
||||
mysql_async::Value::Bytes(b) => String::from_utf8(b).map_or_else(
|
||||
|e| Self::Blob(e.into_bytes().into()),
|
||||
|s| Self::Text(s.into()),
|
||||
),
|
||||
mysql_async::Value::NULL => Self::Null,
|
||||
mysql_async::Value::Float(f) => Self::Float(f as f64),
|
||||
mysql_async::Value::Date(_, _, _, _, _, _, _)
|
||||
| mysql_async::Value::Time(_, _, _, _, _, _) => Self::Text(value.as_sql(true).into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl IntoRows for Vec<mysql_async::Row> {
|
||||
fn into_rows(self) -> crate::Rows {
|
||||
crate::Rows {
|
||||
rows: self
|
||||
.into_iter()
|
||||
.map(|r| crate::Row {
|
||||
values: r
|
||||
.unwrap_raw()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(Into::into)
|
||||
.collect(),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn into_named_rows(self) -> crate::NamedRows {
|
||||
crate::NamedRows {
|
||||
names: self
|
||||
.first()
|
||||
.map(|r| r.columns().iter().map(|c| c.name_str().into()).collect())
|
||||
.unwrap_or_default(),
|
||||
rows: self
|
||||
.into_iter()
|
||||
.map(|r| crate::Row {
|
||||
values: r
|
||||
.unwrap_raw()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(Into::into)
|
||||
.collect(),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn into_row(self) -> Option<crate::Row> {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
|
||||
impl IntoRows for Option<mysql_async::Row> {
|
||||
fn into_row(self) -> Option<crate::Row> {
|
||||
self.map(|row| crate::Row {
|
||||
values: row
|
||||
.unwrap_raw()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(Into::into)
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
fn into_rows(self) -> crate::Rows {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn into_named_rows(self) -> crate::NamedRows {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use super::{MysqlStore, into_error};
|
||||
use crate::{
|
||||
backend::mysql::MysqlSearchField,
|
||||
search::{
|
||||
CalendarSearchField, ContactSearchField, EmailSearchField, SearchableField,
|
||||
TracingSearchField,
|
||||
},
|
||||
*,
|
||||
};
|
||||
use ::registry::schema::structs;
|
||||
use mysql_async::{
|
||||
Conn, OptsBuilder, Pool, PoolConstraints, PoolOpts, SslOpts, prelude::Queryable,
|
||||
};
|
||||
|
||||
impl MysqlStore {
|
||||
pub async fn open(config: structs::MySqlStore) -> Result<Store, String> {
|
||||
let mut opts = OptsBuilder::default()
|
||||
.ip_or_hostname(config.host)
|
||||
.user(config.auth_username)
|
||||
.pass(config.auth_secret.secret().await?.map(|v| v.into_owned()))
|
||||
.db_name(Some(config.database))
|
||||
.max_allowed_packet(config.max_allowed_packet.map(|v| v as usize))
|
||||
.wait_timeout(config.timeout.map(|t| t.as_secs() as usize))
|
||||
.client_found_rows(true)
|
||||
.tcp_port(config.port as u16);
|
||||
|
||||
if config.use_tls {
|
||||
opts = opts.ssl_opts(Some(
|
||||
SslOpts::default()
|
||||
.with_danger_accept_invalid_certs(config.allow_invalid_certs)
|
||||
.with_danger_skip_domain_validation(config.allow_invalid_certs),
|
||||
));
|
||||
}
|
||||
|
||||
// Configure connection pool
|
||||
let mut pool_min = PoolConstraints::default().min();
|
||||
let mut pool_max = PoolConstraints::default().max();
|
||||
if let Some(n_size) = config.pool_min_connections {
|
||||
pool_min = n_size as usize;
|
||||
}
|
||||
if let Some(n_size) = config.pool_max_connections {
|
||||
pool_max = n_size as usize;
|
||||
}
|
||||
opts = opts.pool_opts(
|
||||
PoolOpts::default().with_constraints(PoolConstraints::new(pool_min, pool_max).unwrap()),
|
||||
);
|
||||
|
||||
let mut replicas = vec![];
|
||||
for replica in config.read_replicas {
|
||||
replicas.push(Store::MySQL(Arc::new(MysqlStore {
|
||||
conn_pool: Pool::new(
|
||||
opts.clone()
|
||||
.ip_or_hostname(replica.host)
|
||||
.user(replica.auth_username)
|
||||
.pass(replica.auth_secret.secret().await?.map(|v| v.into_owned()))
|
||||
.db_name(Some(replica.database))
|
||||
.tcp_port(replica.port as u16),
|
||||
),
|
||||
})))
|
||||
}
|
||||
|
||||
let primary = Store::MySQL(Arc::new(MysqlStore {
|
||||
conn_pool: Pool::new(opts),
|
||||
}));
|
||||
|
||||
|
||||
Ok(primary)
|
||||
}
|
||||
|
||||
pub(crate) async fn create_storage_tables(&self) -> trc::Result<()> {
|
||||
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||
|
||||
for table in [
|
||||
SUBSPACE_ACL,
|
||||
SUBSPACE_TASK_QUEUE,
|
||||
SUBSPACE_DELETED_ITEMS,
|
||||
SUBSPACE_SPAM_SAMPLES,
|
||||
SUBSPACE_BLOB_LINK,
|
||||
SUBSPACE_IN_MEMORY_VALUE,
|
||||
SUBSPACE_PROPERTY,
|
||||
SUBSPACE_REGISTRY,
|
||||
SUBSPACE_REGISTRY_PK,
|
||||
SUBSPACE_DIRECTORY,
|
||||
SUBSPACE_QUEUE_MESSAGE,
|
||||
SUBSPACE_QUEUE_EVENT,
|
||||
SUBSPACE_REPORT_OUT,
|
||||
SUBSPACE_REPORT_IN,
|
||||
SUBSPACE_LOGS,
|
||||
SUBSPACE_TELEMETRY_SPAN,
|
||||
SUBSPACE_TELEMETRY_METRIC,
|
||||
] {
|
||||
let table = char::from(table);
|
||||
conn.query_drop(format!(
|
||||
"CREATE TABLE IF NOT EXISTS {table} (
|
||||
k VARBINARY(255) NOT NULL,
|
||||
v MEDIUMBLOB NOT NULL,
|
||||
PRIMARY KEY (k)
|
||||
) ENGINE=InnoDB"
|
||||
))
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
}
|
||||
|
||||
conn.query_drop(format!(
|
||||
"CREATE TABLE IF NOT EXISTS {} (
|
||||
k VARBINARY(255) NOT NULL,
|
||||
v LONGBLOB NOT NULL,
|
||||
PRIMARY KEY (k)
|
||||
) ENGINE=InnoDB",
|
||||
char::from(SUBSPACE_BLOBS),
|
||||
))
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
|
||||
for table in [SUBSPACE_INDEXES, SUBSPACE_REGISTRY_IDX] {
|
||||
let table = char::from(table);
|
||||
conn.query_drop(format!(
|
||||
"CREATE TABLE IF NOT EXISTS {table} (
|
||||
k BLOB,
|
||||
PRIMARY KEY (k(400))
|
||||
) ENGINE=InnoDB"
|
||||
))
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
}
|
||||
|
||||
for table in [SUBSPACE_COUNTER, SUBSPACE_QUOTA, SUBSPACE_IN_MEMORY_COUNTER] {
|
||||
conn.query_drop(format!(
|
||||
"CREATE TABLE IF NOT EXISTS {} (
|
||||
k VARBINARY(255) NOT NULL,
|
||||
v BIGINT NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (k)
|
||||
) ENGINE=InnoDB",
|
||||
char::from(table)
|
||||
))
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn create_search_tables(&self) -> trc::Result<()> {
|
||||
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||
|
||||
create_search_tables::<EmailSearchField>(&mut conn).await?;
|
||||
create_search_tables::<CalendarSearchField>(&mut conn).await?;
|
||||
create_search_tables::<ContactSearchField>(&mut conn).await?;
|
||||
//create_search_tables::<FileSearchField>(&mut conn).await?;
|
||||
create_search_tables::<TracingSearchField>(&mut conn).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_search_tables<T: SearchableField + MysqlSearchField + 'static>(
|
||||
conn: &mut Conn,
|
||||
) -> trc::Result<()> {
|
||||
let table_name = T::index().mysql_table();
|
||||
let mut query = format!("CREATE TABLE IF NOT EXISTS {} (", table_name);
|
||||
|
||||
// Add primary key columns
|
||||
let pkeys = T::primary_keys();
|
||||
for pkey in pkeys {
|
||||
query.push_str(&format!("{} {}, ", pkey.column(), pkey.column_type()));
|
||||
}
|
||||
|
||||
// Add other columns
|
||||
for field in T::all_fields() {
|
||||
query.push_str(&format!("{} {}, ", field.column(), field.column_type()));
|
||||
}
|
||||
|
||||
// Add primary key constraint
|
||||
query.push_str("PRIMARY KEY (");
|
||||
for (i, pkey) in pkeys.iter().enumerate() {
|
||||
if i > 0 {
|
||||
query.push_str(", ");
|
||||
}
|
||||
query.push_str(pkey.column());
|
||||
}
|
||||
query.push_str(")) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci");
|
||||
|
||||
conn.query_drop(&query).await.map_err(into_error)?;
|
||||
|
||||
// Create indexes
|
||||
for field in T::all_fields() {
|
||||
if field.is_text() {
|
||||
let column_name = field.column();
|
||||
let create_index_query = format!(
|
||||
"CREATE FULLTEXT INDEX fts_{table_name}_{column_name} ON {table_name}({column_name})",
|
||||
);
|
||||
|
||||
let _ = conn.query_drop(&create_index_query).await;
|
||||
}
|
||||
|
||||
if field.is_indexed() {
|
||||
let column_name = field.column();
|
||||
let create_index_query = format!(
|
||||
"CREATE INDEX idx_{table_name}_{column_name} ON {table_name}({column_name})",
|
||||
);
|
||||
let _ = conn.query_drop(&create_index_query).await;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use crate::{
|
||||
search::{
|
||||
CalendarSearchField, ContactSearchField, EmailSearchField, FileSearchField, SearchField,
|
||||
TracingSearchField,
|
||||
},
|
||||
write::SearchIndex,
|
||||
};
|
||||
use mysql_async::Pool;
|
||||
use std::fmt::Display;
|
||||
|
||||
pub mod blob;
|
||||
pub mod lookup;
|
||||
pub mod main;
|
||||
pub mod read;
|
||||
pub mod search;
|
||||
pub mod write;
|
||||
|
||||
pub struct MysqlStore {
|
||||
pub(crate) conn_pool: Pool,
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn into_error(err: impl Display) -> trc::Error {
|
||||
trc::StoreEvent::MysqlError.reason(err)
|
||||
}
|
||||
|
||||
const ER_LOCK_WAIT_TIMEOUT: u16 = 1205;
|
||||
const ER_STATEMENT_TIMEOUT: u16 = 1969;
|
||||
const ER_QUERY_TIMEOUT: u16 = 3024;
|
||||
|
||||
pub(crate) const DELETE_CHUNK_SIZE: usize = 1000;
|
||||
pub(crate) const MIN_DELETE_CHUNK_SIZE: usize = 10;
|
||||
|
||||
#[inline(always)]
|
||||
pub(crate) fn is_timeout_error(err: &mysql_async::Error) -> bool {
|
||||
matches!(err, mysql_async::Error::Server(err)
|
||||
if matches!(
|
||||
err.code,
|
||||
ER_LOCK_WAIT_TIMEOUT | ER_STATEMENT_TIMEOUT | ER_QUERY_TIMEOUT
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
impl SearchIndex {
|
||||
pub fn mysql_table(&self) -> &'static str {
|
||||
match self {
|
||||
SearchIndex::Email => "s_email",
|
||||
SearchIndex::Calendar => "s_cal",
|
||||
SearchIndex::Contacts => "s_card",
|
||||
SearchIndex::File => "s_file",
|
||||
SearchIndex::Tracing => "s_trace",
|
||||
SearchIndex::InMemory => "",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
trait MysqlSearchField {
|
||||
fn column(&self) -> &'static str;
|
||||
fn column_type(&self) -> &'static str;
|
||||
}
|
||||
|
||||
impl MysqlSearchField for EmailSearchField {
|
||||
fn column(&self) -> &'static str {
|
||||
match self {
|
||||
EmailSearchField::From => "fadr",
|
||||
EmailSearchField::To => "tadr",
|
||||
EmailSearchField::Cc => "cc",
|
||||
EmailSearchField::Bcc => "bcc",
|
||||
EmailSearchField::Subject => "subj",
|
||||
EmailSearchField::Body => "body",
|
||||
EmailSearchField::Attachment => "atta",
|
||||
EmailSearchField::ReceivedAt => "rcvd",
|
||||
EmailSearchField::SentAt => "sent",
|
||||
EmailSearchField::Size => "size",
|
||||
EmailSearchField::HasAttachment => "hatt",
|
||||
EmailSearchField::Headers => "hdrs",
|
||||
}
|
||||
}
|
||||
|
||||
fn column_type(&self) -> &'static str {
|
||||
match self {
|
||||
EmailSearchField::ReceivedAt | EmailSearchField::SentAt => "BIGINT",
|
||||
EmailSearchField::Size => "INT",
|
||||
EmailSearchField::HasAttachment => "BOOLEAN",
|
||||
EmailSearchField::Headers => "JSON",
|
||||
EmailSearchField::From => "TEXT",
|
||||
EmailSearchField::To => "TEXT",
|
||||
EmailSearchField::Cc => "TEXT",
|
||||
EmailSearchField::Bcc => "TEXT",
|
||||
EmailSearchField::Subject => "TEXT",
|
||||
EmailSearchField::Body => "MEDIUMTEXT",
|
||||
EmailSearchField::Attachment => "MEDIUMTEXT",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MysqlSearchField for CalendarSearchField {
|
||||
fn column(&self) -> &'static str {
|
||||
match self {
|
||||
CalendarSearchField::Title => "titl",
|
||||
CalendarSearchField::Description => "dscd",
|
||||
CalendarSearchField::Location => "locn",
|
||||
CalendarSearchField::Owner => "ownr",
|
||||
CalendarSearchField::Attendee => "atnd",
|
||||
CalendarSearchField::Start => "strt",
|
||||
CalendarSearchField::Uid => "uid",
|
||||
}
|
||||
}
|
||||
|
||||
fn column_type(&self) -> &'static str {
|
||||
match self {
|
||||
CalendarSearchField::Start => "BIGINT NOT NULL",
|
||||
_ => "TEXT",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MysqlSearchField for ContactSearchField {
|
||||
fn column(&self) -> &'static str {
|
||||
match self {
|
||||
ContactSearchField::Member => "mmbr",
|
||||
ContactSearchField::Name => "name",
|
||||
ContactSearchField::Nickname => "nick",
|
||||
ContactSearchField::Organization => "orgn",
|
||||
ContactSearchField::Email => "eml",
|
||||
ContactSearchField::Phone => "phon",
|
||||
ContactSearchField::OnlineService => "olsv",
|
||||
ContactSearchField::Address => "addr",
|
||||
ContactSearchField::Note => "note",
|
||||
ContactSearchField::Kind => "kind",
|
||||
ContactSearchField::Uid => "uid",
|
||||
}
|
||||
}
|
||||
|
||||
fn column_type(&self) -> &'static str {
|
||||
match self {
|
||||
ContactSearchField::Kind | ContactSearchField::Uid => "TEXT",
|
||||
_ => "TEXT",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MysqlSearchField for FileSearchField {
|
||||
fn column(&self) -> &'static str {
|
||||
match self {
|
||||
FileSearchField::Name => "name",
|
||||
FileSearchField::Content => "body",
|
||||
}
|
||||
}
|
||||
|
||||
fn column_type(&self) -> &'static str {
|
||||
match self {
|
||||
FileSearchField::Name => "TEXT",
|
||||
FileSearchField::Content => "MEDIUMTEXT",
|
||||
}
|
||||
}
|
||||
}
|
||||
impl MysqlSearchField for TracingSearchField {
|
||||
fn column(&self) -> &'static str {
|
||||
match self {
|
||||
TracingSearchField::QueueId => "qid",
|
||||
TracingSearchField::EventType => "etyp",
|
||||
TracingSearchField::Keywords => "kwds",
|
||||
}
|
||||
}
|
||||
|
||||
fn column_type(&self) -> &'static str {
|
||||
match self {
|
||||
TracingSearchField::EventType => "BIGINT",
|
||||
TracingSearchField::QueueId => "BIGINT",
|
||||
TracingSearchField::Keywords => "TEXT",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MysqlSearchField for SearchField {
|
||||
fn column(&self) -> &'static str {
|
||||
match self {
|
||||
SearchField::AccountId => "accid",
|
||||
SearchField::DocumentId => "docid",
|
||||
SearchField::Id => "id",
|
||||
SearchField::Email(field) => field.column(),
|
||||
SearchField::Calendar(field) => field.column(),
|
||||
SearchField::Contact(field) => field.column(),
|
||||
SearchField::File(field) => field.column(),
|
||||
SearchField::Tracing(field) => field.column(),
|
||||
}
|
||||
}
|
||||
|
||||
fn column_type(&self) -> &'static str {
|
||||
match self {
|
||||
SearchField::AccountId => "INT NOT NULL",
|
||||
SearchField::DocumentId => "INT NOT NULL",
|
||||
SearchField::Id => "BIGINT NOT NULL",
|
||||
SearchField::Email(field) => field.column_type(),
|
||||
SearchField::Calendar(field) => field.column_type(),
|
||||
SearchField::Contact(field) => field.column_type(),
|
||||
SearchField::File(field) => field.column_type(),
|
||||
SearchField::Tracing(field) => field.column_type(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use super::{MysqlStore, into_error, is_timeout_error};
|
||||
use crate::{Deserialize, IterateParams, Key, ValueKey, write::ValueClass};
|
||||
use futures::TryStreamExt;
|
||||
use mysql_async::{Row, prelude::Queryable};
|
||||
|
||||
impl MysqlStore {
|
||||
pub(crate) async fn get_value<U>(&self, key: impl Key) -> trc::Result<Option<U>>
|
||||
where
|
||||
U: Deserialize + 'static,
|
||||
{
|
||||
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||
let s = conn
|
||||
.prep(format!(
|
||||
"SELECT v FROM {} WHERE k = ?",
|
||||
char::from(key.subspace())
|
||||
))
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
let key = key.serialize(0);
|
||||
conn.exec_first::<Vec<u8>, _, _>(&s, (&key,))
|
||||
.await
|
||||
.map_err(into_error)
|
||||
.and_then(|r| {
|
||||
if let Some(r) = r {
|
||||
Ok(Some(U::deserialize_owned_with_key(&key, r)?))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn key_exists(&self, key: impl Key) -> trc::Result<bool> {
|
||||
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||
let s = conn
|
||||
.prep(format!(
|
||||
"SELECT 1 FROM {} WHERE k = ?",
|
||||
char::from(key.subspace())
|
||||
))
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
let key = key.serialize(0);
|
||||
conn.exec_first::<u8, _, _>(&s, (&key,))
|
||||
.await
|
||||
.map_err(into_error)
|
||||
.map(|r| r.is_some())
|
||||
}
|
||||
|
||||
pub(crate) async fn iterate<T: Key>(
|
||||
&self,
|
||||
params: IterateParams<T>,
|
||||
mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result<bool> + Sync + Send,
|
||||
) -> trc::Result<()> {
|
||||
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||
let table = char::from(params.begin.subspace());
|
||||
let begin = params.begin.serialize(0);
|
||||
let end = params.end.serialize(0);
|
||||
let keys = if params.values { "k, v" } else { "k" };
|
||||
|
||||
let s = conn
|
||||
.prep(&match (params.first, params.ascending) {
|
||||
(true, true) => {
|
||||
format!(
|
||||
"SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC LIMIT 1"
|
||||
)
|
||||
}
|
||||
(true, false) => {
|
||||
format!(
|
||||
"SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC LIMIT 1"
|
||||
)
|
||||
}
|
||||
(false, true) => {
|
||||
format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC")
|
||||
}
|
||||
(false, false) => {
|
||||
format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC")
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
let mut from = begin;
|
||||
let mut to = end;
|
||||
let mut resume_key = None;
|
||||
|
||||
loop {
|
||||
let mut last_key = None;
|
||||
let mut timed_out = false;
|
||||
|
||||
{
|
||||
let mut rows = conn
|
||||
.exec_stream::<Row, _, _>(&s, (from.clone(), to.clone()))
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
|
||||
loop {
|
||||
match rows.try_next().await {
|
||||
Ok(Some(mut row)) => {
|
||||
let value = if params.values {
|
||||
row.take_opt::<Vec<u8>, _>(1)
|
||||
.unwrap_or_else(|| Ok(vec![]))
|
||||
.map_err(into_error)?
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
let key = row
|
||||
.take_opt::<Vec<u8>, _>(0)
|
||||
.unwrap_or_else(|| Ok(vec![]))
|
||||
.map_err(into_error)?;
|
||||
|
||||
if resume_key.take().is_some_and(|resumed| resumed == key) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if !cb(&key, &value)? {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
last_key = Some(key);
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(err) => {
|
||||
if params.first || last_key.is_none() || !is_timeout_error(&err) {
|
||||
return Err(into_error(err));
|
||||
}
|
||||
timed_out = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match last_key {
|
||||
Some(last_key) if timed_out => {
|
||||
if params.ascending {
|
||||
from.clone_from(&last_key);
|
||||
} else {
|
||||
to.clone_from(&last_key);
|
||||
}
|
||||
resume_key = Some(last_key);
|
||||
}
|
||||
_ => return Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn get_counter(
|
||||
&self,
|
||||
key: impl Into<ValueKey<ValueClass>> + Sync + Send,
|
||||
) -> trc::Result<i64> {
|
||||
let key = key.into();
|
||||
let table = char::from(key.subspace());
|
||||
let key = key.serialize(0);
|
||||
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||
let s = conn
|
||||
.prep(format!("SELECT v FROM {table} WHERE k = ?"))
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
match conn.exec_first::<i64, _, _>(&s, (key,)).await {
|
||||
Ok(Some(num)) => Ok(num),
|
||||
Ok(None) => Ok(0),
|
||||
Err(e) => Err(into_error(e)),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use crate::{
|
||||
backend::{
|
||||
MAX_TOKEN_LENGTH,
|
||||
mysql::{
|
||||
DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, MysqlSearchField, MysqlStore, into_error,
|
||||
is_timeout_error,
|
||||
},
|
||||
},
|
||||
search::{
|
||||
IndexDocument, SearchComparator, SearchDocumentId, SearchFilter, SearchOperator,
|
||||
SearchQuery, SearchValue,
|
||||
},
|
||||
write::SearchIndex,
|
||||
};
|
||||
use mysql_async::{IsolationLevel, TxOpts, Value, prelude::Queryable};
|
||||
use nlp::tokenizers::word::WordTokenizer;
|
||||
use std::fmt::Write;
|
||||
|
||||
impl MysqlStore {
|
||||
pub async fn index(&self, documents: Vec<IndexDocument>) -> trc::Result<()> {
|
||||
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||
let mut tx_opts = TxOpts::default();
|
||||
tx_opts
|
||||
.with_consistent_snapshot(false)
|
||||
.with_isolation_level(IsolationLevel::ReadCommitted);
|
||||
let mut trx = conn.start_transaction(tx_opts).await.map_err(into_error)?;
|
||||
|
||||
for document in documents {
|
||||
let index = document.index;
|
||||
let primary_keys = index.primary_keys();
|
||||
let all_fields = index.all_fields();
|
||||
let mut fields = document.fields;
|
||||
let mut values = Vec::with_capacity(fields.len() + 2);
|
||||
let mut query = format!("INSERT INTO {} (", index.mysql_table());
|
||||
|
||||
for (i, field) in primary_keys.iter().chain(all_fields).enumerate() {
|
||||
if i > 0 {
|
||||
query.push(',');
|
||||
}
|
||||
query.push_str(field.column());
|
||||
}
|
||||
|
||||
query.push_str(") VALUES (");
|
||||
|
||||
for (i, field) in primary_keys.iter().chain(all_fields).enumerate() {
|
||||
if i > 0 {
|
||||
query.push(',');
|
||||
}
|
||||
|
||||
if let Some(value) = fields.remove(field) {
|
||||
query.push('?');
|
||||
values.push(value);
|
||||
} else {
|
||||
query.push_str("NULL");
|
||||
}
|
||||
}
|
||||
|
||||
query.push_str(") ON DUPLICATE KEY UPDATE ");
|
||||
for (i, field) in all_fields.iter().enumerate() {
|
||||
if i > 0 {
|
||||
query.push(',');
|
||||
}
|
||||
let column = field.column();
|
||||
let _ = write!(&mut query, "{column} = VALUES({column})");
|
||||
}
|
||||
|
||||
let s = trx.prep(&query).await.map_err(into_error)?;
|
||||
|
||||
trx.exec_drop(&s, values).await.map_err(into_error)?;
|
||||
}
|
||||
|
||||
trx.commit().await.map_err(into_error)
|
||||
}
|
||||
|
||||
pub async fn query<R: SearchDocumentId>(
|
||||
&self,
|
||||
index: SearchIndex,
|
||||
filters: &[SearchFilter],
|
||||
sort: &[SearchComparator],
|
||||
) -> trc::Result<Vec<R>> {
|
||||
let mut query = format!(
|
||||
"SELECT {} FROM {}",
|
||||
R::field().column(),
|
||||
index.mysql_table()
|
||||
);
|
||||
let params = build_filter(&mut query, filters);
|
||||
if !sort.is_empty() {
|
||||
build_sort(&mut query, sort);
|
||||
}
|
||||
|
||||
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||
let s = conn.prep(query).await.map_err(into_error)?;
|
||||
|
||||
conn.exec::<i64, _, _>(s, params)
|
||||
.await
|
||||
.map(|r| r.into_iter().map(|r| R::from_u64(r as u64)).collect())
|
||||
.map_err(into_error)
|
||||
}
|
||||
|
||||
pub async fn unindex(&self, filter: SearchQuery) -> trc::Result<u64> {
|
||||
let table = filter.index.mysql_table();
|
||||
let mut query = format!("DELETE FROM {table} ");
|
||||
let params = build_filter(&mut query, &filter.filters);
|
||||
|
||||
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||
let s = conn.prep(&query).await.map_err(into_error)?;
|
||||
|
||||
match conn.exec_drop(s, params.clone()).await {
|
||||
Ok(_) => return Ok(conn.affected_rows()),
|
||||
Err(err) if is_timeout_error(&err) => (),
|
||||
Err(err) => return Err(into_error(err)),
|
||||
}
|
||||
|
||||
let mut chunk_size = DELETE_CHUNK_SIZE;
|
||||
let mut deleted = 0;
|
||||
|
||||
loop {
|
||||
let s = conn
|
||||
.prep(format!("{query} LIMIT {chunk_size}"))
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
|
||||
loop {
|
||||
match conn.exec_drop(&s, params.clone()).await {
|
||||
Ok(_) => {
|
||||
let affected = conn.affected_rows();
|
||||
if affected == 0 {
|
||||
return Ok(deleted);
|
||||
}
|
||||
deleted += affected;
|
||||
}
|
||||
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
|
||||
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
|
||||
break;
|
||||
}
|
||||
Err(err) => return Err(into_error(err)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_filter(query: &mut String, filters: &[SearchFilter]) -> Vec<Value> {
|
||||
if filters.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
query.push_str(" WHERE ");
|
||||
let mut operator_stack = Vec::new();
|
||||
let mut operator = &SearchFilter::And;
|
||||
let mut is_first = true;
|
||||
let mut values: Vec<Value> = Vec::new();
|
||||
|
||||
for filter in filters {
|
||||
match filter {
|
||||
SearchFilter::Operator { field, op, value } => {
|
||||
if !is_first {
|
||||
match operator {
|
||||
SearchFilter::And => query.push_str(" AND "),
|
||||
SearchFilter::Or => query.push_str(" OR "),
|
||||
_ => (),
|
||||
}
|
||||
} else {
|
||||
is_first = false;
|
||||
}
|
||||
|
||||
if field.is_text() && matches!(op, SearchOperator::Equal | SearchOperator::Contains)
|
||||
{
|
||||
let (value, mode) = match (value, op) {
|
||||
(SearchValue::Text { value, .. }, SearchOperator::Equal) => {
|
||||
(Value::Bytes(format!("{value:?}").into_bytes()), "BOOLEAN")
|
||||
}
|
||||
(SearchValue::Text { value, .. }, ..) => {
|
||||
let mut text_query = String::with_capacity(value.len() + 1);
|
||||
|
||||
for item in WordTokenizer::new(value, MAX_TOKEN_LENGTH) {
|
||||
if !text_query.is_empty() {
|
||||
text_query.push(' ');
|
||||
}
|
||||
text_query.push('+');
|
||||
text_query.push_str(&item.word);
|
||||
}
|
||||
|
||||
(Value::Bytes(text_query.into_bytes()), "BOOLEAN")
|
||||
}
|
||||
_ => {
|
||||
debug_assert!(false, "Invalid search value for text field");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let _ = write!(query, "MATCH({}) AGAINST(? IN {mode} MODE)", field.column());
|
||||
values.push(value);
|
||||
} else if let SearchValue::KeyValues(kv) = value {
|
||||
let (key, value) = kv.iter().next().unwrap();
|
||||
|
||||
values.push(Value::Bytes(format!("$.{key:?}").into_bytes()));
|
||||
|
||||
if !value.is_empty() {
|
||||
if op == &SearchOperator::Equal {
|
||||
let _ = write!(query, "JSON_EXTRACT({}, ?) = ?", field.column());
|
||||
values.push(Value::Bytes(value.as_bytes().to_vec()));
|
||||
} else {
|
||||
let _ = write!(query, "JSON_EXTRACT({}, ?) LIKE ?", field.column(),);
|
||||
values.push(Value::Bytes(format!("%{value}%").into_bytes()));
|
||||
}
|
||||
} else {
|
||||
let _ = write!(query, "JSON_CONTAINS_PATH({}, 'one', ?)", field.column(),);
|
||||
}
|
||||
} else {
|
||||
query.push_str(field.column());
|
||||
query.push(' ');
|
||||
op.write_mysql(query);
|
||||
values.push(to_mysql(value));
|
||||
}
|
||||
}
|
||||
SearchFilter::And | SearchFilter::Or => {
|
||||
if !is_first {
|
||||
match operator {
|
||||
SearchFilter::And => query.push_str(" AND "),
|
||||
SearchFilter::Or => query.push_str(" OR "),
|
||||
_ => (),
|
||||
}
|
||||
} else {
|
||||
is_first = false;
|
||||
}
|
||||
|
||||
operator_stack.push((operator, is_first));
|
||||
operator = filter;
|
||||
is_first = true;
|
||||
query.push('(');
|
||||
}
|
||||
SearchFilter::Not => {
|
||||
if !is_first {
|
||||
match operator {
|
||||
SearchFilter::And => query.push_str(" AND "),
|
||||
SearchFilter::Or => query.push_str(" OR "),
|
||||
_ => (),
|
||||
}
|
||||
} else {
|
||||
is_first = false;
|
||||
}
|
||||
|
||||
operator_stack.push((operator, is_first));
|
||||
operator = &SearchFilter::And;
|
||||
is_first = true;
|
||||
query.push_str("NOT (");
|
||||
}
|
||||
SearchFilter::End => {
|
||||
let p = operator_stack.pop().unwrap_or((&SearchFilter::And, true));
|
||||
operator = p.0;
|
||||
is_first = p.1;
|
||||
query.push(')');
|
||||
}
|
||||
SearchFilter::DocumentSet(_) => {
|
||||
debug_assert!(
|
||||
false,
|
||||
"DocumentSet filters are not supported in Postgres backend"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
values
|
||||
}
|
||||
|
||||
fn build_sort(query: &mut String, sort: &[SearchComparator]) {
|
||||
query.push_str(" ORDER BY ");
|
||||
for (i, comparator) in sort.iter().enumerate() {
|
||||
if i > 0 {
|
||||
query.push_str(", ");
|
||||
}
|
||||
match comparator {
|
||||
SearchComparator::Field { field, ascending } => {
|
||||
query.push_str(field.column());
|
||||
if *ascending {
|
||||
query.push_str(" ASC");
|
||||
} else {
|
||||
query.push_str(" DESC");
|
||||
}
|
||||
}
|
||||
SearchComparator::DocumentSet { .. } | SearchComparator::SortedSet { .. } => {
|
||||
debug_assert!(
|
||||
false,
|
||||
"DocumentSet and SortedSet comparators are not supported "
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SearchOperator {
|
||||
fn write_mysql(&self, query: &mut String) {
|
||||
match self {
|
||||
SearchOperator::LowerThan => {
|
||||
let _ = write!(query, "< ?");
|
||||
}
|
||||
SearchOperator::LowerEqualThan => {
|
||||
let _ = write!(query, "<= ?");
|
||||
}
|
||||
SearchOperator::GreaterThan => {
|
||||
let _ = write!(query, "> ?");
|
||||
}
|
||||
SearchOperator::GreaterEqualThan => {
|
||||
let _ = write!(query, ">= ?");
|
||||
}
|
||||
SearchOperator::Equal => {
|
||||
let _ = write!(query, "= ?");
|
||||
}
|
||||
SearchOperator::Contains => {
|
||||
let _ = write!(query, "LIKE '%' CONCAT('%', ?, '%')");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SearchValue> for Value {
|
||||
fn from(value: SearchValue) -> Self {
|
||||
match value {
|
||||
SearchValue::Text { mut value, .. } => {
|
||||
// Truncate values larger than 16MB to avoid MySQL errors
|
||||
if value.len() > 16_777_214 {
|
||||
let pos = value.floor_char_boundary(16_777_214);
|
||||
value.truncate(pos);
|
||||
}
|
||||
|
||||
Value::Bytes(value.into_bytes())
|
||||
}
|
||||
SearchValue::KeyValues(vec_map) => serde_json::to_string(&vec_map)
|
||||
.map(|v| Value::Bytes(v.into_bytes()))
|
||||
.unwrap_or(Value::NULL),
|
||||
SearchValue::Int(i) => Value::Int(i),
|
||||
SearchValue::Uint(i) => Value::Int(i as i64),
|
||||
SearchValue::Boolean(b) => Value::Int(b as i64),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn to_mysql(value: &SearchValue) -> Value {
|
||||
match value {
|
||||
SearchValue::Text { value, .. } => Value::Bytes(value.as_bytes().to_vec()),
|
||||
SearchValue::KeyValues(vec_map) => serde_json::to_string(&vec_map)
|
||||
.map(|v| Value::Bytes(v.into_bytes()))
|
||||
.unwrap_or(Value::NULL),
|
||||
SearchValue::Int(i) => Value::Int(*i),
|
||||
SearchValue::Uint(i) => Value::Int(*i as i64),
|
||||
SearchValue::Boolean(b) => Value::Int(*b as i64),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,529 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use super::{DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, MysqlStore, into_error, is_timeout_error};
|
||||
use crate::{
|
||||
IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA,
|
||||
SUBSPACE_REGISTRY_IDX,
|
||||
write::{
|
||||
AssignedIds, Batch, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, MergeResult, Operation,
|
||||
ValueClass, ValueOp,
|
||||
},
|
||||
};
|
||||
use ahash::AHashMap;
|
||||
use mysql_async::{Conn, Error, IsolationLevel, TxOpts, params, prelude::Queryable};
|
||||
use rand::RngExt;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
#[derive(Debug)]
|
||||
enum CommitError {
|
||||
Mysql(mysql_async::Error),
|
||||
Internal(trc::Error),
|
||||
//Retry,
|
||||
}
|
||||
|
||||
impl MysqlStore {
|
||||
pub(crate) async fn write(&self, mut batch: Batch<'_>) -> trc::Result<AssignedIds> {
|
||||
let start = Instant::now();
|
||||
let mut retry_count = 0;
|
||||
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||
|
||||
loop {
|
||||
let err = match self.write_trx(&mut conn, &mut batch).await {
|
||||
Ok(result) => {
|
||||
return Ok(result);
|
||||
}
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
let _ = conn.query_drop("ROLLBACK;").await;
|
||||
|
||||
match err {
|
||||
CommitError::Mysql(Error::Server(err))
|
||||
if [1062, 1213].contains(&err.code)
|
||||
&& retry_count < MAX_COMMIT_ATTEMPTS
|
||||
&& start.elapsed() < MAX_COMMIT_TIME => {}
|
||||
/*CommitError::Retry => {
|
||||
if retry_count > MAX_COMMIT_ATTEMPTS || start.elapsed() > MAX_COMMIT_TIME {
|
||||
return Err(trc::StoreEvent::AssertValueFailed
|
||||
.into_err()
|
||||
.caused_by(trc::location!()));
|
||||
}
|
||||
}*/
|
||||
CommitError::Mysql(err) => {
|
||||
return Err(into_error(err));
|
||||
}
|
||||
CommitError::Internal(err) => {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
|
||||
let backoff = rand::rng().random_range(50..=300);
|
||||
tokio::time::sleep(Duration::from_millis(backoff)).await;
|
||||
retry_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_trx(
|
||||
&self,
|
||||
conn: &mut Conn,
|
||||
batch: &mut Batch<'_>,
|
||||
) -> Result<AssignedIds, CommitError> {
|
||||
let has_changes = !batch.changes.is_empty();
|
||||
let mut account_id = u32::MAX;
|
||||
let mut collection = u8::MAX;
|
||||
let mut document_id = u32::MAX;
|
||||
let mut change_id = 0u64;
|
||||
let mut asserted_values = AHashMap::new();
|
||||
let mut tx_opts = TxOpts::default();
|
||||
tx_opts
|
||||
.with_consistent_snapshot(false)
|
||||
.with_isolation_level(IsolationLevel::ReadCommitted);
|
||||
let mut trx = conn.start_transaction(tx_opts).await?;
|
||||
let mut result = AssignedIds::default();
|
||||
|
||||
if has_changes {
|
||||
for &account_id in batch.changes.keys() {
|
||||
let key = ValueClass::ChangeId.serialize(account_id, 0, 0, 0);
|
||||
let s = trx
|
||||
.prep(concat!(
|
||||
"INSERT INTO n (k, v) VALUES (:k, LAST_INSERT_ID(1)) ",
|
||||
"ON DUPLICATE KEY UPDATE v = LAST_INSERT_ID(v + 1)"
|
||||
))
|
||||
.await?;
|
||||
trx.exec_drop(&s, params! {"k" => key}).await?;
|
||||
let s = trx.prep("SELECT LAST_INSERT_ID()").await?;
|
||||
let change_id = trx.exec_first::<i64, _, _>(&s, ()).await?.ok_or_else(|| {
|
||||
mysql_async::Error::Io(mysql_async::IoError::Io(std::io::Error::other(
|
||||
"LAST_INSERT_ID() did not return a value",
|
||||
)))
|
||||
})?;
|
||||
result.push_change_id(account_id, change_id as u64);
|
||||
}
|
||||
}
|
||||
|
||||
for op in batch.ops.iter_mut() {
|
||||
match op {
|
||||
Operation::AccountId {
|
||||
account_id: account_id_,
|
||||
} => {
|
||||
account_id = *account_id_;
|
||||
if has_changes {
|
||||
change_id = result.set_current_change_id(account_id)?;
|
||||
}
|
||||
}
|
||||
Operation::Collection {
|
||||
collection: collection_,
|
||||
} => {
|
||||
collection = u8::from(*collection_);
|
||||
}
|
||||
Operation::DocumentId {
|
||||
document_id: document_id_,
|
||||
} => {
|
||||
document_id = *document_id_;
|
||||
}
|
||||
Operation::Value { class, op } => {
|
||||
let key = class.serialize(account_id, collection, document_id, 0);
|
||||
let subspace = class.subspace(collection);
|
||||
let table = char::from(subspace);
|
||||
|
||||
match op {
|
||||
ValueOp::Set(value) => {
|
||||
if subspace != SUBSPACE_REGISTRY_IDX {
|
||||
let exists = asserted_values.get(&key);
|
||||
let s = if let Some(exists) = exists {
|
||||
if *exists {
|
||||
trx.prep(format!(
|
||||
"UPDATE {} SET v = :v WHERE k = :k",
|
||||
table
|
||||
))
|
||||
.await?
|
||||
} else {
|
||||
trx.prep(format!(
|
||||
"INSERT INTO {} (k, v) VALUES (:k, :v)",
|
||||
table
|
||||
))
|
||||
.await?
|
||||
}
|
||||
} else {
|
||||
trx
|
||||
.prep(
|
||||
format!("INSERT INTO {} (k, v) VALUES (:k, :v) ON DUPLICATE KEY UPDATE v = VALUES(v)", table),
|
||||
)
|
||||
.await?
|
||||
};
|
||||
|
||||
match trx
|
||||
.exec_drop(&s, params! {"k" => key, "v" => &*value})
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
if trx.affected_rows() == 0 {
|
||||
trx.rollback().await?;
|
||||
return Err(trc::StoreEvent::AssertValueFailed
|
||||
.into_err()
|
||||
.caused_by(trc::location!())
|
||||
.into());
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
trx.rollback().await?;
|
||||
return Err(err.into());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let s = trx.prep("INSERT IGNORE INTO b (k) VALUES (?)").await?;
|
||||
trx.exec_drop(&s, (key,)).await?;
|
||||
}
|
||||
}
|
||||
ValueOp::SetFnc(set_op) => {
|
||||
let value = (set_op.fnc)(&set_op.params, &result)?;
|
||||
let exists = asserted_values.get(&key);
|
||||
let s = if let Some(exists) = exists {
|
||||
if *exists {
|
||||
trx.prep(format!("UPDATE {} SET v = :v WHERE k = :k", table))
|
||||
.await?
|
||||
} else {
|
||||
trx.prep(format!(
|
||||
"INSERT INTO {} (k, v) VALUES (:k, :v)",
|
||||
table
|
||||
))
|
||||
.await?
|
||||
}
|
||||
} else {
|
||||
trx
|
||||
.prep(
|
||||
format!("INSERT INTO {} (k, v) VALUES (:k, :v) ON DUPLICATE KEY UPDATE v = VALUES(v)", table),
|
||||
)
|
||||
.await?
|
||||
};
|
||||
|
||||
match trx.exec_drop(&s, params! {"k" => key, "v" => &value}).await {
|
||||
Ok(_) => {
|
||||
if trx.affected_rows() == 0 {
|
||||
trx.rollback().await?;
|
||||
return Err(trc::StoreEvent::AssertValueFailed
|
||||
.into_err()
|
||||
.caused_by(trc::location!())
|
||||
.into());
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
trx.rollback().await?;
|
||||
return Err(err.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
ValueOp::MergeFnc(merge_op) => {
|
||||
let s = trx
|
||||
.prep(format!("SELECT v FROM {} WHERE k = ? FOR UPDATE", table))
|
||||
.await?;
|
||||
let (exists, merge_result) = trx
|
||||
.exec_first::<Vec<u8>, _, _>(&s, (&key,))
|
||||
.await?
|
||||
.map(|bytes| {
|
||||
(merge_op.fnc)(&merge_op.params, &result, Some(bytes.as_ref()))
|
||||
.map(|v| (true, v))
|
||||
.map_err(CommitError::from)
|
||||
})
|
||||
.unwrap_or_else(|| {
|
||||
(merge_op.fnc)(&merge_op.params, &result, None)
|
||||
.map(|v| (false, v))
|
||||
.map_err(CommitError::from)
|
||||
})?;
|
||||
|
||||
let s = if exists {
|
||||
trx.prep(format!("UPDATE {} SET v = :v WHERE k = :k", table))
|
||||
.await?
|
||||
} else {
|
||||
trx.prep(format!("INSERT INTO {} (k, v) VALUES (:k, :v)", table))
|
||||
.await?
|
||||
};
|
||||
|
||||
match merge_result {
|
||||
MergeResult::Update(value) => {
|
||||
if let Err(err) =
|
||||
trx.exec_drop(&s, params! {"k" => key, "v" => &value}).await
|
||||
{
|
||||
trx.rollback().await?;
|
||||
return Err(err.into());
|
||||
}
|
||||
}
|
||||
MergeResult::Delete if exists => {
|
||||
// Update asserted value
|
||||
if let Some(exists) = asserted_values.get_mut(&key) {
|
||||
*exists = false;
|
||||
}
|
||||
|
||||
let s = trx
|
||||
.prep(format!("DELETE FROM {} WHERE k = ?", table))
|
||||
.await?;
|
||||
trx.exec_drop(&s, (key,)).await?;
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
ValueOp::AtomicAdd(by) => {
|
||||
if *by >= 0 {
|
||||
let s = trx
|
||||
.prep(format!(
|
||||
concat!(
|
||||
"INSERT INTO {} (k, v) VALUES (?, ?) ",
|
||||
"ON DUPLICATE KEY UPDATE v = v + VALUES(v)"
|
||||
),
|
||||
table
|
||||
))
|
||||
.await?;
|
||||
trx.exec_drop(&s, (key, &*by)).await?;
|
||||
} else {
|
||||
let s = trx
|
||||
.prep(format!("UPDATE {table} SET v = v + ? WHERE k = ?"))
|
||||
.await?;
|
||||
trx.exec_drop(&s, (&*by, key)).await?;
|
||||
}
|
||||
}
|
||||
ValueOp::AddAndGet(by) => {
|
||||
let s = trx
|
||||
.prep(format!(
|
||||
concat!(
|
||||
"INSERT INTO {} (k, v) VALUES (:k, LAST_INSERT_ID(:v)) ",
|
||||
"ON DUPLICATE KEY UPDATE v = LAST_INSERT_ID(v + :v)"
|
||||
),
|
||||
table
|
||||
))
|
||||
.await?;
|
||||
trx.exec_drop(&s, params! {"k" => key, "v" => &*by}).await?;
|
||||
let s = trx.prep("SELECT LAST_INSERT_ID()").await?;
|
||||
result.push_counter_id(
|
||||
trx.exec_first::<i64, _, _>(&s, ()).await?.ok_or_else(|| {
|
||||
mysql_async::Error::Io(mysql_async::IoError::Io(
|
||||
std::io::Error::other(
|
||||
"LAST_INSERT_ID() did not return a value",
|
||||
),
|
||||
))
|
||||
})?,
|
||||
);
|
||||
}
|
||||
ValueOp::Clear => {
|
||||
// Update asserted value
|
||||
if let Some(exists) = asserted_values.get_mut(&key) {
|
||||
*exists = false;
|
||||
}
|
||||
|
||||
let s = trx
|
||||
.prep(format!("DELETE FROM {} WHERE k = ?", table))
|
||||
.await?;
|
||||
trx.exec_drop(&s, (key,)).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Operation::Index { field, key, set } => {
|
||||
let key = IndexKey {
|
||||
account_id,
|
||||
collection,
|
||||
document_id,
|
||||
field: *field,
|
||||
key: &*key,
|
||||
}
|
||||
.serialize(0);
|
||||
|
||||
let s = if *set {
|
||||
trx.prep("INSERT IGNORE INTO i (k) VALUES (?)").await?
|
||||
} else {
|
||||
trx.prep("DELETE FROM i WHERE k = ?").await?
|
||||
};
|
||||
trx.exec_drop(&s, (key,)).await?;
|
||||
}
|
||||
Operation::Log { collection, set } => {
|
||||
let key = LogKey {
|
||||
account_id,
|
||||
collection: u8::from(*collection),
|
||||
change_id,
|
||||
}
|
||||
.serialize(0);
|
||||
|
||||
let s = trx
|
||||
.prep("INSERT INTO l (k, v) VALUES (?, ?) ON DUPLICATE KEY UPDATE v = VALUES(v)")
|
||||
.await?;
|
||||
|
||||
trx.exec_drop(&s, (key, &*set)).await?;
|
||||
}
|
||||
Operation::AssertValue {
|
||||
class,
|
||||
assert_value,
|
||||
} => {
|
||||
let key = class.serialize(account_id, collection, document_id, 0);
|
||||
let table = char::from(class.subspace(collection));
|
||||
|
||||
let s = trx
|
||||
.prep(format!("SELECT v FROM {} WHERE k = ? FOR UPDATE", table))
|
||||
.await?;
|
||||
let (exists, matches) = trx
|
||||
.exec_first::<Vec<u8>, _, _>(&s, (&key,))
|
||||
.await?
|
||||
.map(|bytes| (true, assert_value.matches(&bytes)))
|
||||
.unwrap_or_else(|| (false, assert_value.is_none()));
|
||||
if !matches {
|
||||
trx.rollback().await?;
|
||||
return Err(trc::StoreEvent::AssertValueFailed
|
||||
.into_err()
|
||||
.caused_by(trc::location!())
|
||||
.into());
|
||||
}
|
||||
asserted_values.insert(key, exists);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
trx.commit().await.map(|_| result).map_err(Into::into)
|
||||
}
|
||||
|
||||
pub(crate) async fn purge_store(&self) -> trc::Result<()> {
|
||||
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||
for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] {
|
||||
purge_table(&mut conn, char::from(subspace)).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> {
|
||||
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||
let table = char::from(from.subspace());
|
||||
let mut from = from.serialize(0);
|
||||
let to = to.serialize(0);
|
||||
|
||||
let delete = conn
|
||||
.prep(format!("DELETE FROM {table} WHERE k >= ? AND k < ?"))
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
|
||||
match conn.exec_drop(&delete, (&from, &to)).await {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(err) if is_timeout_error(&err) => (),
|
||||
Err(err) => return Err(into_error(err)),
|
||||
}
|
||||
|
||||
let mut chunk_size = DELETE_CHUNK_SIZE;
|
||||
|
||||
loop {
|
||||
let boundary = conn
|
||||
.prep(format!(
|
||||
"SELECT k FROM {table} WHERE k >= ? AND k < ? ORDER BY k ASC LIMIT 1 OFFSET {chunk_size}"
|
||||
))
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
|
||||
loop {
|
||||
let next = match conn
|
||||
.exec_first::<Vec<u8>, _, _>(&boundary, (&from, &to))
|
||||
.await
|
||||
{
|
||||
Ok(next) => next,
|
||||
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
|
||||
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
|
||||
break;
|
||||
}
|
||||
Err(err) => return Err(into_error(err)),
|
||||
};
|
||||
|
||||
match conn
|
||||
.exec_drop(&delete, (&from, next.as_ref().unwrap_or(&to)))
|
||||
.await
|
||||
{
|
||||
Ok(_) => (),
|
||||
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
|
||||
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
|
||||
break;
|
||||
}
|
||||
Err(err) => return Err(into_error(err)),
|
||||
}
|
||||
|
||||
match next {
|
||||
Some(next) => from = next,
|
||||
None => return Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn purge_table(conn: &mut Conn, table: char) -> trc::Result<()> {
|
||||
let s = conn
|
||||
.prep(format!("DELETE FROM {table} WHERE v = 0"))
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
|
||||
match conn.exec_drop(&s, ()).await {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(err) if is_timeout_error(&err) => (),
|
||||
Err(err) => return Err(into_error(err)),
|
||||
}
|
||||
|
||||
let purge = conn
|
||||
.prep(format!(
|
||||
"DELETE FROM {table} WHERE v = 0 AND k >= ? AND k < ?"
|
||||
))
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
let purge_last = conn
|
||||
.prep(format!("DELETE FROM {table} WHERE v = 0 AND k >= ?"))
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
let mut chunk_size = DELETE_CHUNK_SIZE;
|
||||
let mut from = Vec::new();
|
||||
|
||||
loop {
|
||||
let boundary = conn
|
||||
.prep(format!(
|
||||
"SELECT k FROM {table} WHERE k >= ? ORDER BY k ASC LIMIT 1 OFFSET {chunk_size}"
|
||||
))
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
|
||||
loop {
|
||||
let next = match conn.exec_first::<Vec<u8>, _, _>(&boundary, (&from,)).await {
|
||||
Ok(next) => next,
|
||||
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
|
||||
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
|
||||
break;
|
||||
}
|
||||
Err(err) => return Err(into_error(err)),
|
||||
};
|
||||
|
||||
let result = match &next {
|
||||
Some(next) => conn.exec_drop(&purge, (&from, next)).await,
|
||||
None => conn.exec_drop(&purge_last, (&from,)).await,
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(_) => (),
|
||||
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
|
||||
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
|
||||
break;
|
||||
}
|
||||
Err(err) => return Err(into_error(err)),
|
||||
}
|
||||
|
||||
match next {
|
||||
Some(next) => from = next,
|
||||
None => return Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<trc::Error> for CommitError {
|
||||
fn from(err: trc::Error) -> Self {
|
||||
CommitError::Internal(err)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<mysql_async::Error> for CommitError {
|
||||
fn from(err: mysql_async::Error) -> Self {
|
||||
CommitError::Mysql(err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user