inbuxa/inbuxa-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 23 Wiki Activity

Import upstream v0.16.24, stripped

Browse Source 
Upstream commit: af37a234981722493b74623a983581691d2b70b6
Enterprise-only files removed or emptied: 63
Enterprise-only snippets removed: 118 in 50 files
Dangling module declarations removed: 5
Edits turning enterprise off: 25
Third-party code: 14 files, 0 not in THIRD-PARTY.md
Renamed identifiers: 62 in 18 files
Verification: clean

The same Enterprise footprint as v0.16.23. The build check fails only on
tests/src/directory/issuer.rs, unchanged since v0.16.23: it calls a helper
from upstream's Enterprise-only OIDC test, and tests issuer-based directory
routing, an Enterprise feature. main has never carried it.
This commit is contained in:
jcoffey-dev
2026-09-28 06:29:38 -07:00
parent 3a272096c0
commit f59b084ce5
98 changed files with 2851 additions and 1068 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/store/src/backend/http/lookup.rs
+142 -25
View File
@@ -4,21 +4,19 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use super::{HttpStore, HttpStoreConfig};
use crate::{Value, backend::http::HttpStoreFormat, write::now};
use ahash::AHashMap;
use compact_str::ToCompactString;
use rand::seq::IndexedRandom;
use std::{
borrow::Cow,
io::{BufRead, BufReader},
sync::{Arc, atomic::Ordering},
time::Instant,
};
use ahash::AHashMap;
use compact_str::ToCompactString;
use rand::seq::IndexedRandom;
use utils::HttpLimitResponse;
use crate::{Value, backend::http::HttpStoreFormat, write::now};
use super::HttpStore;
const BROWSER_USER_AGENTS: [&str; 5] = [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/120.0.0.0 Safari/537.36",
@@ -36,7 +34,7 @@ pub(crate) trait HttpStoreGet {
impl HttpStoreGet for Arc<HttpStore> {
fn get(&self, key: &str) -> Option<Value<'static>> {
self.refresh();
self.entries.load().get(key).cloned()
self.entries.load().get(lookup_key(key).as_ref()).cloned()
}
fn contains(&self, key: &str) -> bool {
@@ -59,7 +57,7 @@ impl HttpStoreGet for Arc<HttpStore> {
}
self.refresh();
self.entries.load().contains_key(key)
self.entries.load().contains_key(lookup_key(key).as_ref())
}
fn refresh(&self) {
@@ -142,9 +140,10 @@ impl HttpStore {
Box::new(&bytes[..])
};
let mut entries = AHashMap::new();
for (pos, line) in BufReader::new(reader).lines().enumerate() {
let line_ = line.map_err(|err| {
let entries = self
.config
.parse_entries(BufReader::new(reader))
.map_err(|err| {
trc::StoreEvent::HttpStoreError
.into_err()
.reason(err)
@@ -153,11 +152,31 @@ impl HttpStore {
.details("Failed to read line")
})?;
match &self.config.format {
trc::event!(
Store(trc::StoreEvent::HttpStoreFetch),
Url = self.config.url.to_compact_string(),
Total = entries.len(),
Elapsed = time.elapsed(),
);
Ok(entries)
}
}
impl HttpStoreConfig {
fn parse_entries(
&self,
reader: impl BufRead,
) -> std::io::Result<AHashMap<String, Value<'static>>> {
let mut entries = AHashMap::new();
for (pos, line) in reader.lines().enumerate() {
let line_ = line?;
match &self.format {
HttpStoreFormat::List => {
let line = line_.trim();
if !line.is_empty() {
entries.insert(line.to_string(), Value::Integer(1));
entries.insert(lookup_key(line).into_owned(), Value::Integer(1));
}
}
HttpStoreFormat::Csv {
@@ -188,12 +207,12 @@ impl HttpStore {
}
} else if col_num == *index_key {
entry_key.push(ch);
if entry_key.len() > self.config.max_entry_size {
if entry_key.len() > self.max_entry_size {
break;
}
} else if index_value.is_some_and(|v| col_num == v) {
entry_value.push(ch);
if entry_value.len() > self.config.max_entry_size {
if entry_value.len() > self.max_entry_size {
break;
}
}
@@ -209,24 +228,122 @@ impl HttpStore {
} else {
Value::Integer(1)
};
let entry_key = match lookup_key(&entry_key) {
Cow::Owned(key) => key,
Cow::Borrowed(_) => entry_key,
};
entries.insert(entry_key, entry_value);
}
}
_ => (),
}
if entries.len() == self.config.max_entries {
if entries.len() == self.max_entries {
break;
}
}
trc::event!(
Store(trc::StoreEvent::HttpStoreFetch),
Url = self.config.url.to_compact_string(),
Total = entries.len(),
Elapsed = time.elapsed(),
);
Ok(entries)
}
}
fn lookup_key(key: &str) -> Cow<'_, str> {
if key.bytes().any(|b| !b.is_ascii() || b.is_ascii_uppercase()) {
Cow::Owned(key.to_lowercase())
} else {
Cow::Borrowed(key)
}
}
#[cfg(test)]
mod tests {
use super::*;
use arc_swap::ArcSwap;
use reqwest::Client;
use std::{
sync::atomic::{AtomicBool, AtomicU64},
time::Duration,
};
fn http_store(format: HttpStoreFormat, feed: &str) -> Arc<HttpStore> {
let config = HttpStoreConfig {
id: "test".into(),
url: "https://lists.example.org/feed".into(),
retry: 0,
refresh: 0,
timeout: Duration::from_secs(1),
gzipped: false,
max_size: 1024 * 1024,
max_entries: 100,
max_entry_size: 512,
format,
};
let entries = config
.parse_entries(feed.as_bytes())
.expect("feed is readable");
Arc::new(HttpStore {
entries: ArcSwap::from_pointee(entries),
expires: AtomicU64::new(u64::MAX),
in_flight: AtomicBool::new(false),
config,
client: Client::new(),
})
}
#[test]
fn list_keys_ignore_case() {
let store = http_store(
HttpStoreFormat::List,
"https://phish.example.org/Account/Verify?Token=AbC123\n\
https://PHISH.example.net/lower\n",
);
assert!(store.contains("https://phish.example.org/account/verify?token=abc123"));
assert!(store.contains("https://phish.example.org/Account/Verify?Token=AbC123"));
assert!(store.contains("https://phish.example.net/lower"));
assert!(store.contains("HTTPS://PHISH.EXAMPLE.NET/LOWER"));
assert!(!store.contains("https://phish.example.org/account/verify"));
}
#[test]
fn csv_keys_ignore_case() {
let store = http_store(
HttpStoreFormat::Csv {
index_key: 1,
index_value: None,
separator: ',',
skip_first: true,
},
"phish_id,url,phish_detail_url\n\
1,\"https://phish.example.org/Login.PHP?Id=Xy\",https://phishtank.example/1\n",
);
assert!(store.contains("https://phish.example.org/login.php?id=xy"));
assert!(store.contains("https://phish.example.org/Login.PHP?Id=Xy"));
assert!(!store.contains("phish_id"));
assert!(!store.contains("url"));
}
#[test]
fn csv_values_keep_case() {
let store = http_store(
HttpStoreFormat::Csv {
index_key: 0,
index_value: Some(1),
separator: ',',
skip_first: false,
},
"Example.ORG,Some Value\n",
);
assert_eq!(
store.get("example.org"),
Some(Value::Text("Some Value".into()))
);
assert_eq!(
store.get("EXAMPLE.org"),
Some(Value::Text("Some Value".into()))
);
}
}
crates/store/src/backend/mysql/mod.rs
+14
View File
@@ -30,7 +30,10 @@ fn into_error(err: impl Display) -> trc::Error {
trc::StoreEvent::MysqlError.reason(err)
}
const ER_UNKNOWN_ERROR: u16 = 1105;
const ER_TRANS_CACHE_FULL: u16 = 1197;
const ER_LOCK_WAIT_TIMEOUT: u16 = 1205;
const ER_LOCK_TABLE_FULL: u16 = 1206;
const ER_STATEMENT_TIMEOUT: u16 = 1969;
const ER_QUERY_TIMEOUT: u16 = 3024;
@@ -47,6 +50,17 @@ pub(crate) fn is_timeout_error(err: &mysql_async::Error) -> bool {
)
}
#[inline(always)]
pub(crate) fn is_chunk_too_large_error(err: &mysql_async::Error) -> bool {
is_timeout_error(err)
|| matches!(err, mysql_async::Error::Server(err)
if matches!(
err.code,
ER_UNKNOWN_ERROR | ER_TRANS_CACHE_FULL | ER_LOCK_TABLE_FULL
)
)
}
impl SearchIndex {
pub fn mysql_table(&self) -> &'static str {
match self {
crates/store/src/backend/mysql/search.rs
+6 -12
View File
@@ -9,7 +9,7 @@ use crate::{
MAX_TOKEN_LENGTH,
mysql::{
DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, MysqlSearchField, MysqlStore, into_error,
is_timeout_error,
is_chunk_too_large_error,
},
},
search::{
@@ -109,14 +109,6 @@ impl MysqlStore {
let params = build_filter(&mut query, &filter.filters);
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
let s = conn.prep(&query).await.map_err(into_error)?;
match conn.exec_drop(s, params.clone()).await {
Ok(_) => return Ok(conn.affected_rows()),
Err(err) if is_timeout_error(&err) => (),
Err(err) => return Err(into_error(err)),
}
let mut chunk_size = DELETE_CHUNK_SIZE;
let mut deleted = 0;
@@ -130,12 +122,14 @@ impl MysqlStore {
match conn.exec_drop(&s, params.clone()).await {
Ok(_) => {
let affected = conn.affected_rows();
if affected == 0 {
deleted += affected;
if affected < chunk_size as u64 {
return Ok(deleted);
}
deleted += affected;
}
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
Err(err)
if is_chunk_too_large_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE =>
{
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
break;
}
crates/store/src/backend/mysql/write.rs
+16 -13
View File
@@ -4,7 +4,9 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use super::{DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, MysqlStore, into_error, is_timeout_error};
use super::{
DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, MysqlStore, into_error, is_chunk_too_large_error,
};
use crate::{
IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA,
SUBSPACE_REGISTRY_IDX,
@@ -400,13 +402,6 @@ impl MysqlStore {
.prep(format!("DELETE FROM {table} WHERE k >= ? AND k < ?"))
.await
.map_err(into_error)?;
match conn.exec_drop(&delete, (&from, &to)).await {
Ok(_) => return Ok(()),
Err(err) if is_timeout_error(&err) => (),
Err(err) => return Err(into_error(err)),
}
let mut chunk_size = DELETE_CHUNK_SIZE;
loop {
@@ -423,7 +418,9 @@ impl MysqlStore {
.await
{
Ok(next) => next,
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
Err(err)
if is_chunk_too_large_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE =>
{
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
break;
}
@@ -435,7 +432,9 @@ impl MysqlStore {
.await
{
Ok(_) => (),
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
Err(err)
if is_chunk_too_large_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE =>
{
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
break;
}
@@ -459,7 +458,7 @@ async fn purge_table(conn: &mut Conn, table: char) -> trc::Result<()> {
match conn.exec_drop(&s, ()).await {
Ok(_) => return Ok(()),
Err(err) if is_timeout_error(&err) => (),
Err(err) if is_chunk_too_large_error(&err) => (),
Err(err) => return Err(into_error(err)),
}
@@ -487,7 +486,9 @@ async fn purge_table(conn: &mut Conn, table: char) -> trc::Result<()> {
loop {
let next = match conn.exec_first::<Vec<u8>, _, _>(&boundary, (&from,)).await {
Ok(next) => next,
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
Err(err)
if is_chunk_too_large_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE =>
{
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
break;
}
@@ -501,7 +502,9 @@ async fn purge_table(conn: &mut Conn, table: char) -> trc::Result<()> {
match result {
Ok(_) => (),
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
Err(err)
if is_chunk_too_large_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE =>
{
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
break;
}
crates/store/src/backend/redis/lookup.rs
+6 -14
View File
@@ -7,16 +7,7 @@
use super::{RedisPool, RedisStore, into_error};
use crate::{Deserialize, write::now};
use deadpool::managed::{Manager, Object, Pool};
use redis::{AsyncCommands, RedisError, RedisResult, RetryMethod, Script};
use std::sync::LazyLock;
static INCR_EXPIRE: LazyLock<Script> = LazyLock::new(|| {
Script::new(
"redis.call('INCRBY', KEYS[1], ARGV[1])
redis.call('EXPIRE', KEYS[1], ARGV[2])
return redis.call('GET', KEYS[1])",
)
});
use redis::{AsyncCommands, RedisError, RedisResult, RetryMethod};
impl RedisStore {
pub async fn key_set(&self, key: &[u8], value: &[u8], expires: Option<u64>) -> trc::Result<()> {
@@ -46,19 +37,19 @@ impl RedisStore {
match &self.pool {
RedisPool::Single(pool) => {
with_conn(pool, async |conn| {
Self::key_incr_(conn, key, value, expires).await
self.key_incr_(conn, key, value, expires).await
})
.await
}
RedisPool::Cluster(pool) => {
with_conn(pool, async |conn| {
Self::key_incr_(conn, key, value, expires).await
self.key_incr_(conn, key, value, expires).await
})
.await
}
RedisPool::Sentinel(pool) => {
with_conn(pool, async |conn| {
Self::key_incr_(conn, key, value, expires).await
self.key_incr_(conn, key, value, expires).await
})
.await
}
@@ -193,13 +184,14 @@ impl RedisStore {
}
async fn key_incr_(
&self,
conn: &mut impl AsyncCommands,
key: &[u8],
value: i64,
expires: Option<u64>,
) -> RedisResult<i64> {
if let Some(expires) = expires {
INCR_EXPIRE
self.incr_expire
.key(key)
.arg(value)
.arg(expires as i64)
crates/store/src/backend/redis/mod.rs
+22 -10
View File
@@ -10,7 +10,7 @@ use deadpool::{
managed::{Manager, Pool},
};
use redis::{
Client, ConnectionAddr, IntoConnectionInfo, ProtocolVersion, TlsMode,
Client, ConnectionAddr, IntoConnectionInfo, ProtocolVersion, Script, TlsMode,
cluster::{ClusterClient, ClusterClientBuilder},
cluster_read_routing::RandomReplicaStrategy,
sentinel::{SentinelClient, SentinelClientBuilder, SentinelServerType},
@@ -27,6 +27,7 @@ pub mod pool;
#[derive(Debug)]
pub struct RedisStore {
pub pool: RedisPool,
incr_expire: Script,
}
pub struct RedisConnectionManager {
@@ -51,9 +52,20 @@ pub enum RedisPool {
}
impl RedisStore {
fn new(pool: RedisPool) -> Self {
RedisStore {
pool,
incr_expire: Script::new(
"redis.call('INCRBY', KEYS[1], ARGV[1])
redis.call('EXPIRE', KEYS[1], ARGV[2])
return redis.call('GET', KEYS[1])",
),
}
}
pub async fn open_single(config: structs::RedisStore) -> Result<InMemoryStore, String> {
Ok(InMemoryStore::Redis(Arc::new(RedisStore {
pool: RedisPool::Single(build_pool(
Ok(InMemoryStore::Redis(Arc::new(RedisStore::new(
RedisPool::Single(build_pool(
RedisConnectionManager {
client: Client::open(config.url)
.map_err(|err| format!("Failed to open Redis client: {err:?}"))?,
@@ -64,7 +76,7 @@ impl RedisStore {
config.pool_timeout_wait,
config.pool_timeout_recycle,
)?),
})))
))))
}
pub async fn open_cluster(config: structs::RedisClusterStore) -> Result<InMemoryStore, String> {
@@ -95,8 +107,8 @@ impl RedisStore {
.build()
.map_err(|err| format!("Failed to open Redis client: {err:?}"))?;
Ok(InMemoryStore::Redis(Arc::new(RedisStore {
pool: RedisPool::Cluster(build_pool(
Ok(InMemoryStore::Redis(Arc::new(RedisStore::new(
RedisPool::Cluster(build_pool(
RedisClusterConnectionManager {
client,
timeout: config.timeout.into_inner(),
@@ -106,7 +118,7 @@ impl RedisStore {
config.pool_timeout_wait,
config.pool_timeout_recycle,
)?),
})))
))))
}
pub async fn open_sentinel(
@@ -167,8 +179,8 @@ impl RedisStore {
.build()
.map_err(|err| format!("Failed to open Redis Sentinel client: {err:?}"))?;
Ok(InMemoryStore::Redis(Arc::new(RedisStore {
pool: RedisPool::Sentinel(build_pool(
Ok(InMemoryStore::Redis(Arc::new(RedisStore::new(
RedisPool::Sentinel(build_pool(
RedisSentinelConnectionManager {
client: tokio::sync::Mutex::new(client),
timeout: config.timeout.into_inner(),
@@ -178,7 +190,7 @@ impl RedisStore {
config.pool_timeout_wait,
config.pool_timeout_recycle,
)?),
})))
))))
}
}