Cache message blobs on each node #175
No files matched your search
@@ -197,6 +197,7 @@ impl Caches {
|
||||
((std::mem::size_of::<Ipv4Addr>() + 255) * 2) as u64,
|
||||
),
|
||||
negative_cache_ttl: cache.negative_ttl.into_inner(),
|
||||
blobs: crate::storage::blob_cache::BlobCache::from_env(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -216,6 +216,9 @@ pub struct Caches {
|
||||
|
||||
pub dkim_signers: Cache<u32, Arc<DkimSigners>>,
|
||||
|
||||
/// inbuxa: message blobs this node wrote or read recently
|
||||
pub blobs: Option<storage::blob_cache::BlobCache>,
|
||||
|
||||
pub dns_txt: CacheWithTtl<Box<str>, Txt>,
|
||||
pub dns_mx: CacheWithTtl<Box<str>, RecordSet<MX>>,
|
||||
pub dns_ptr: CacheWithTtl<IpAddr, RecordSet<Box<str>>>,
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use crate::{KV_QUOTA_BLOB, Server};
|
||||
@@ -9,6 +11,8 @@ use mail_parser::{
|
||||
Encoding,
|
||||
decoders::{base64::base64_decode, quoted_printable::quoted_printable_decode},
|
||||
};
|
||||
use std::ops::Range;
|
||||
use std::sync::Arc;
|
||||
use store::{
|
||||
U32_LEN, U64_LEN,
|
||||
dispatch::lookup::KeyValue,
|
||||
@@ -114,6 +118,7 @@ impl Server {
|
||||
.put_blob(hash.as_ref(), data, self.core.email.compression)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
self.cache_message_blob(hash.as_ref(), data);
|
||||
|
||||
// Commit blob
|
||||
let mut batch = BatchBuilder::new();
|
||||
@@ -177,6 +182,7 @@ impl Server {
|
||||
.put_blob(hash.as_ref(), data, self.core.email.compression)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
self.cache_message_blob(hash.as_ref(), data);
|
||||
|
||||
// Commit blob
|
||||
let mut batch = BatchBuilder::new();
|
||||
@@ -198,14 +204,55 @@ impl Server {
|
||||
))
|
||||
}
|
||||
|
||||
/// inbuxa: a message blob, from this node's cache when it has it. Only
|
||||
/// for content-hash keys; a key that is rewritten in place must go to
|
||||
/// `blob_store().get_blob()`.
|
||||
pub async fn get_message_blob(
|
||||
&self,
|
||||
hash: &[u8],
|
||||
range: Range<usize>,
|
||||
) -> trc::Result<Option<Vec<u8>>> {
|
||||
let Some(cache) = &self.inner.cache.blobs else {
|
||||
return self.blob_store().get_blob(hash, range).await;
|
||||
};
|
||||
let blob = match cache.get(hash) {
|
||||
Some(blob) => blob,
|
||||
None => {
|
||||
// The whole blob, so the next range is served from memory:
|
||||
// the store never reads a range anyway
|
||||
let Some(blob) = self.blob_store().get_blob(hash, 0..usize::MAX).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
cache.insert(hash, &blob);
|
||||
if range.start == 0 && range.end >= blob.len() {
|
||||
return Ok(Some(blob));
|
||||
}
|
||||
Arc::from(blob)
|
||||
}
|
||||
};
|
||||
Ok(Some(
|
||||
blob.get(range.start..range.end.min(blob.len()))
|
||||
.unwrap_or_default()
|
||||
.to_vec(),
|
||||
))
|
||||
}
|
||||
|
||||
/// inbuxa: remembers a blob this node just wrote, so a read that
|
||||
/// follows (delivery, indexing, the client fetching what it sent) is
|
||||
/// served from memory.
|
||||
pub fn cache_message_blob(&self, hash: &[u8], data: &[u8]) {
|
||||
if let Some(cache) = &self.inner.cache.blobs {
|
||||
cache.insert(hash, data);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_blob_section(
|
||||
&self,
|
||||
hash: &BlobHash,
|
||||
section: &BlobSection,
|
||||
) -> trc::Result<Option<Vec<u8>>> {
|
||||
Ok(self
|
||||
.blob_store()
|
||||
.get_blob(
|
||||
.get_message_blob(
|
||||
hash.as_slice(),
|
||||
(section.offset_start)..(section.offset_start.saturating_add(section.size)),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! inbuxa: a per-node cache of message blobs, keyed by content hash.
|
||||
//!
|
||||
//! Upstream fetches a message's blob from the blob store on every read: the
|
||||
//! spool blob straight back for delivery, the same message for an IMAP
|
||||
//! BODYSTRUCTURE and then its BODY[], every outbound retry. On the cluster
|
||||
//! each of those is a round trip to Garage. The cache holds the blobs this
|
||||
//! node wrote or read recently, capped in size; only content-hash keys go
|
||||
//! through it (a hash never changes meaning), so named keys that are
|
||||
//! rewritten in place never do.
|
||||
//!
|
||||
//! `INBUXA_BLOB_CACHE` sets the capacity (`64M` by default, `0` turns it
|
||||
//! off). A blob above a sixteenth of the capacity is not cached.
|
||||
|
||||
use std::sync::Arc;
|
||||
use types::blob_hash::BLOB_HASH_LEN;
|
||||
use utils::cache::{Cache, CacheItemWeight};
|
||||
|
||||
pub const BLOB_CACHE_ENV: &str = "INBUXA_BLOB_CACHE";
|
||||
const DEFAULT_CAPACITY: u64 = 64 * 1024 * 1024;
|
||||
|
||||
#[derive(Clone, PartialEq, Eq, Hash)]
|
||||
pub struct BlobKey([u8; BLOB_HASH_LEN]);
|
||||
|
||||
impl CacheItemWeight for BlobKey {
|
||||
fn weight(&self) -> u64 {
|
||||
BLOB_HASH_LEN as u64
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct CachedBlob(Arc<[u8]>);
|
||||
|
||||
impl CacheItemWeight for CachedBlob {
|
||||
fn weight(&self) -> u64 {
|
||||
self.0.len() as u64 + 64
|
||||
}
|
||||
}
|
||||
|
||||
pub struct BlobCache {
|
||||
cache: Cache<BlobKey, CachedBlob>,
|
||||
max_item: u64,
|
||||
}
|
||||
|
||||
impl BlobCache {
|
||||
/// The cache the environment asks for, or `None` when it is turned off.
|
||||
pub fn from_env() -> Option<Self> {
|
||||
let capacity = match std::env::var(BLOB_CACHE_ENV) {
|
||||
Ok(value) => parse_size(&value).unwrap_or_else(|| {
|
||||
trc::event!(
|
||||
Server(trc::ServerEvent::Startup),
|
||||
Details =
|
||||
format!("{BLOB_CACHE_ENV}={value:?} is not a size; using the default"),
|
||||
);
|
||||
DEFAULT_CAPACITY
|
||||
}),
|
||||
Err(_) => DEFAULT_CAPACITY,
|
||||
};
|
||||
Self::with_capacity(capacity)
|
||||
}
|
||||
|
||||
pub fn with_capacity(capacity: u64) -> Option<Self> {
|
||||
if capacity < 1024 * 1024 {
|
||||
return None;
|
||||
}
|
||||
let max_item = capacity / 16;
|
||||
Some(Self {
|
||||
cache: Cache::new(capacity, max_item / 4),
|
||||
max_item,
|
||||
})
|
||||
}
|
||||
|
||||
fn key(hash: &[u8]) -> Option<BlobKey> {
|
||||
hash.try_into().ok().map(BlobKey)
|
||||
}
|
||||
|
||||
pub fn get(&self, hash: &[u8]) -> Option<Arc<[u8]>> {
|
||||
self.cache.get(&Self::key(hash)?).map(|blob| blob.0)
|
||||
}
|
||||
|
||||
/// Keeps `data` under its content hash when it fits.
|
||||
pub fn insert(&self, hash: &[u8], data: &[u8]) {
|
||||
if data.len() as u64 <= self.max_item
|
||||
&& let Some(key) = Self::key(hash)
|
||||
{
|
||||
self.cache.insert(key, CachedBlob(Arc::from(data)));
|
||||
}
|
||||
}
|
||||
|
||||
pub fn capacity(&self) -> u64 {
|
||||
self.cache.weight_capacity()
|
||||
}
|
||||
}
|
||||
|
||||
/// `64M`, `512K`, `1G` or plain bytes.
|
||||
fn parse_size(value: &str) -> Option<u64> {
|
||||
let value = value.trim();
|
||||
let (digits, unit) = value
|
||||
.find(|c: char| !c.is_ascii_digit())
|
||||
.map(|at| value.split_at(at))
|
||||
.unwrap_or((value, ""));
|
||||
let number = digits.parse::<u64>().ok()?;
|
||||
let multiplier = match unit.trim().to_ascii_lowercase().as_str() {
|
||||
"" | "b" => 1,
|
||||
"k" | "kb" | "kib" => 1024,
|
||||
"m" | "mb" | "mib" => 1024 * 1024,
|
||||
"g" | "gb" | "gib" => 1024 * 1024 * 1024,
|
||||
_ => return None,
|
||||
};
|
||||
number.checked_mul(multiplier)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn sizes_parse() {
|
||||
assert_eq!(parse_size("64M"), Some(64 * 1024 * 1024));
|
||||
assert_eq!(parse_size(" 512 KiB "), Some(512 * 1024));
|
||||
assert_eq!(parse_size("1g"), Some(1 << 30));
|
||||
assert_eq!(parse_size("4096"), Some(4096));
|
||||
assert_eq!(parse_size("0"), Some(0));
|
||||
assert_eq!(parse_size("lots"), None);
|
||||
assert_eq!(parse_size("1t"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn off_below_a_megabyte() {
|
||||
assert!(BlobCache::with_capacity(0).is_none());
|
||||
assert!(BlobCache::with_capacity(1024 * 1024).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_hash_keys_and_fitting_blobs() {
|
||||
let cache = BlobCache::with_capacity(16 * 1024 * 1024).unwrap();
|
||||
let hash = [7u8; BLOB_HASH_LEN];
|
||||
cache.insert(&hash, b"hello");
|
||||
assert_eq!(cache.get(&hash).as_deref(), Some(&b"hello"[..]));
|
||||
cache.insert(b"INBUXA_SPAM_CLASSIFIER_MODEL", b"x");
|
||||
assert!(cache.get(b"INBUXA_SPAM_CLASSIFIER_MODEL").is_none());
|
||||
let big = vec![0u8; 1024 * 1024 + 1];
|
||||
let other = [9u8; BLOB_HASH_LEN];
|
||||
cache.insert(&other, &big);
|
||||
assert!(
|
||||
cache.get(&other).is_none(),
|
||||
"above a sixteenth of the capacity"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ use store::{BlobStore, InMemoryStore, RegistryStore, SearchStore, Store};
|
||||
|
||||
pub mod archive;
|
||||
pub mod blob;
|
||||
pub mod blob_cache; // inbuxa: per-node message blob cache
|
||||
pub mod branding; // inbuxa: branding BT-1, BT-2
|
||||
pub mod dav;
|
||||
pub mod document;
|
||||
|
||||
@@ -437,8 +437,7 @@ impl<T: SessionStream> SessionData<T> {
|
||||
// Retrieve raw message if needed
|
||||
raw_body = self
|
||||
.server
|
||||
.blob_store()
|
||||
.get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX)
|
||||
.get_message_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX)
|
||||
.await
|
||||
.imap_ctx(&arguments.tag, trc::location!())?;
|
||||
|
||||
|
||||
@@ -204,8 +204,7 @@ impl EmailGet for Server {
|
||||
let mut raw_message = ChainedBytes::new(metadata.raw_headers.as_ref());
|
||||
if needs_body {
|
||||
raw_body = self
|
||||
.blob_store()
|
||||
.get_blob(blob_hash.as_slice(), 0..usize::MAX)
|
||||
.get_message_blob(blob_hash.as_slice(), 0..usize::MAX)
|
||||
.await?;
|
||||
|
||||
if let Some(raw_body) = &raw_body {
|
||||
|
||||
@@ -75,8 +75,7 @@ async fn preview(server: &Server, queue_id: u64) -> trc::Result<Option<String>>
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(raw) = server
|
||||
.blob_store()
|
||||
.get_blob(message.message.blob_hash.as_slice(), 0..usize::MAX)
|
||||
.get_message_blob(message.message.blob_hash.as_slice(), 0..usize::MAX)
|
||||
.await?
|
||||
else {
|
||||
return Ok(None);
|
||||
|
||||
@@ -271,7 +271,7 @@ async fn accounts(server: &Server, hold: &Hold, asked: &[u32]) -> trc::Result<Ve
|
||||
}
|
||||
|
||||
async fn blob(server: &Server, hash: &[u8]) -> trc::Result<Option<Vec<u8>>> {
|
||||
server.blob_store().get_blob(hash, 0..usize::MAX).await
|
||||
server.get_message_blob(hash, 0..usize::MAX).await
|
||||
}
|
||||
|
||||
/// Collects `accounts` under `hold` into a ZIP. Returns its bytes and item
|
||||
|
||||
@@ -188,8 +188,7 @@ async fn report_bytes(server: &Server, entry: &Entry) -> trc::Result<Option<Vec<
|
||||
match entry.blob_hash() {
|
||||
Some(hash) => {
|
||||
server
|
||||
.blob_store()
|
||||
.get_blob(hash.as_slice(), 0..usize::MAX)
|
||||
.get_message_blob(hash.as_slice(), 0..usize::MAX)
|
||||
.await
|
||||
}
|
||||
None => Ok(None),
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use crate::{Session, protocol::response::Response};
|
||||
@@ -44,8 +46,7 @@ impl<T: SessionStream> Session<T> {
|
||||
.caused_by(trc::location!())?;
|
||||
if let Some(bytes) = self
|
||||
.server
|
||||
.blob_store()
|
||||
.get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX)
|
||||
.get_message_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX)
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
{
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use aho_corasick::AhoCorasick;
|
||||
@@ -124,8 +126,7 @@ pub async fn build_email_push_object(
|
||||
let mut raw_message = ChainedBytes::new(metadata.raw_headers.as_ref());
|
||||
if needs_body {
|
||||
let Some(blob) = server
|
||||
.blob_store()
|
||||
.get_blob(blob_hash.as_slice(), 0..usize::MAX)
|
||||
.get_message_blob(blob_hash.as_slice(), 0..usize::MAX)
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
else {
|
||||
|
||||
@@ -494,8 +494,7 @@ async fn build_email_document(
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
let raw_message = server
|
||||
.blob_store()
|
||||
.get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX)
|
||||
.get_message_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX)
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.ok_or_else(|| {
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use super::session::SessionParams;
|
||||
@@ -177,8 +179,7 @@ impl<T: AsyncRead + AsyncWrite + Unpin> SmtpClient<T> {
|
||||
) -> Result<(), Status<HostResponse<Box<str>>, ErrorDetails>> {
|
||||
match params
|
||||
.server
|
||||
.blob_store()
|
||||
.get_blob(message.message.blob_hash.as_slice(), 0..usize::MAX)
|
||||
.get_message_blob(message.message.blob_hash.as_slice(), 0..usize::MAX)
|
||||
.await
|
||||
{
|
||||
Ok(Some(raw_message)) => {
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use super::spool::SmtpSpool;
|
||||
@@ -294,8 +296,7 @@ impl MessageWrapper {
|
||||
|
||||
// Fetch up to MAX_HEADER_SIZE bytes of message headers
|
||||
let headers = match server
|
||||
.blob_store()
|
||||
.get_blob(self.message.blob_hash.as_slice(), 0..MAX_HEADER_SIZE)
|
||||
.get_message_blob(self.message.blob_hash.as_slice(), 0..MAX_HEADER_SIZE)
|
||||
.await
|
||||
{
|
||||
Ok(Some(mut buf)) => {
|
||||
|
||||
@@ -457,6 +457,8 @@ impl MessageWrapper {
|
||||
|
||||
return false;
|
||||
}
|
||||
// inbuxa: delivery and indexing on this node read it straight back
|
||||
server.cache_message_blob(self.message.blob_hash.as_slice(), message.as_ref());
|
||||
|
||||
// inbuxa: journaling, JR-1: the copy is taken before the message is
|
||||
// queued; if it can't be, the message isn't queued either
|
||||
|
||||
Reference in new issue
Block a user