Import upstream v0.16.22, stripped

Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f
Enterprise-only files removed or emptied: 63
Enterprise-only snippets removed: 117 in 50 files
Dangling module declarations removed: 5
Cargo edits turning enterprise off: 14
Verification: clean
Enterprise feature gates left for rebuilt features: 19 in 18 files

Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.
This commit is contained in:
2026-09-18 10:21:56 -07:00
commit 7dae9b29fd
1650 changed files with 485521 additions and 0 deletions
@@ -0,0 +1,78 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use std::sync::Arc;
use crate::Coordinator;
use rdkafka::{
ClientConfig, ClientContext, TopicPartitionList,
consumer::{BaseConsumer, ConsumerContext, Rebalance, StreamConsumer},
error::KafkaResult,
producer::FutureProducer,
};
use registry::schema::structs::KafkaCoordinator;
pub mod pubsub;
pub(super) type LoggingConsumer = StreamConsumer<CustomContext>;
pub struct KafkaPubSub {
consumer_builder: ClientConfig,
producer: FutureProducer,
}
impl KafkaPubSub {
pub async fn open(config: KafkaCoordinator) -> Result<Coordinator, String> {
if config.brokers.is_empty() {
return Err("No Kafka brokers specified".to_string());
}
let brokers = config.brokers.into_inner().join(",");
let mut consumer_builder = ClientConfig::new();
consumer_builder
.set("group.id", config.group_id)
.set("bootstrap.servers", &brokers)
.set("enable.partition.eof", "false")
.set(
"session.timeout.ms",
config.timeout_session.as_millis().to_string(),
)
.set("enable.auto.commit", "true");
let producer = ClientConfig::new()
.set("bootstrap.servers", brokers)
.set(
"message.timeout.ms",
config.timeout_message.as_millis().to_string(),
)
.create()
.map_err(|err| format!("Failed to create Kafka producer: {}", err))?;
Ok(Coordinator::Kafka(Arc::new(KafkaPubSub {
consumer_builder,
producer,
})))
}
}
impl std::fmt::Debug for KafkaPubSub {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("KafkaPubSub").finish()
}
}
pub(super) struct CustomContext;
impl ClientContext for CustomContext {}
impl ConsumerContext for CustomContext {
fn pre_rebalance(&self, _: &BaseConsumer<Self>, _: &Rebalance) {}
fn post_rebalance(&self, _: &BaseConsumer<Self>, _: &Rebalance) {}
fn commit_callback(&self, _: KafkaResult<()>, _: &TopicPartitionList) {}
}
@@ -0,0 +1,56 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use super::{CustomContext, KafkaPubSub, LoggingConsumer};
use crate::{Msg, PubSubStream};
use rdkafka::{
Message,
consumer::{CommitMode, Consumer, StreamConsumer},
producer::FutureRecord,
};
use std::time::Duration;
use trc::{ClusterEvent, Error, EventType};
pub struct KafkaPubSubStream {
subs: LoggingConsumer,
}
impl KafkaPubSub {
pub async fn publish(&self, topic: &'static str, message: Vec<u8>) -> trc::Result<()> {
self.producer
.send(
FutureRecord::<(), [u8]>::to(topic).payload(message.as_slice()),
Duration::from_secs(0),
)
.await
.map(|_| ())
.map_err(|(err, _)| {
Error::new(EventType::Cluster(ClusterEvent::PublisherError)).reason(err)
})
}
pub async fn subscribe(&self, topic: &'static str) -> trc::Result<PubSubStream> {
let subs: StreamConsumer<CustomContext> = self
.consumer_builder
.create_with_context(CustomContext)
.map_err(|err| {
Error::new(EventType::Cluster(ClusterEvent::SubscriberError)).reason(err)
})?;
subs.subscribe(&[topic]).map_err(|err| {
Error::new(EventType::Cluster(ClusterEvent::SubscriberError)).reason(err)
})?;
Ok(PubSubStream::Kafka(KafkaPubSubStream { subs }))
}
}
impl KafkaPubSubStream {
pub async fn next(&mut self) -> Option<Msg> {
let msg = self.subs.recv().await.ok()?;
let _ = self.subs.commit_message(&msg, CommitMode::Async);
Msg::Kafka(msg.payload().unwrap_or_default().to_vec()).into()
}
}
+14
View File
@@ -0,0 +1,14 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
#[cfg(feature = "kafka")]
pub mod kafka;
#[cfg(feature = "nats")]
pub mod nats;
#[cfg(feature = "redis")]
pub mod redis;
#[cfg(feature = "zenoh")]
pub mod zenoh;
@@ -0,0 +1,55 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use std::sync::Arc;
use crate::Coordinator;
use async_nats::Client;
use registry::schema::structs::NatsCoordinator;
pub mod pubsub;
#[derive(Debug)]
pub struct NatsPubSub {
client: Client,
}
impl NatsPubSub {
pub async fn open(config: NatsCoordinator) -> Result<Coordinator, String> {
if config.addresses.is_empty() {
return Err("No Nats addresses specified".to_string());
}
let mut opts = async_nats::ConnectOptions::new()
.max_reconnects(config.max_reconnects.map(|v| v as usize))
.connection_timeout(config.timeout_connection.into_inner())
.request_timeout(config.timeout_request.into_inner().into())
.ping_interval(config.ping_interval.into_inner())
.client_capacity(config.capacity_client as usize)
.subscription_capacity(config.capacity_subscription as usize)
.read_buffer_capacity(config.capacity_read_buffer as u16)
.require_tls(config.use_tls);
if config.no_echo {
opts = opts.no_echo();
}
if let (Some(user), Some(pass)) = (
config.auth_username,
config.auth_secret.secret().await?.map(|v| v.into_owned()),
) {
opts = opts.user_and_password(user.to_string(), pass.to_string());
} else if let Some(credentials) = config.credentials.secret().await?.map(|v| v.into_owned())
{
opts = opts.token(credentials);
}
async_nats::connect_with_options(config.addresses.into_inner(), opts)
.await
.map(|client| Coordinator::Nats(Arc::new(NatsPubSub { client })))
.map_err(|err| format!("Failed to connect to Nats: {}", err))
}
}
@@ -0,0 +1,39 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use super::NatsPubSub;
use crate::{Msg, PubSubStream};
use futures::StreamExt;
use trc::{ClusterEvent, Error, EventType};
pub struct NatsPubSubStream {
subs: async_nats::Subscriber,
}
impl NatsPubSub {
pub async fn publish(&self, topic: &'static str, message: Vec<u8>) -> trc::Result<()> {
self.client
.publish(topic, message.into())
.await
.map_err(|err| Error::new(EventType::Cluster(ClusterEvent::PublisherError)).reason(err))
}
pub async fn subscribe(&self, topic: &'static str) -> trc::Result<PubSubStream> {
self.client
.subscribe(topic)
.await
.map(|subs| PubSubStream::Nats(NatsPubSubStream { subs }))
.map_err(|err| {
Error::new(EventType::Cluster(ClusterEvent::SubscriberError)).reason(err)
})
}
}
impl NatsPubSubStream {
pub async fn next(&mut self) -> Option<Msg> {
self.subs.next().await.map(Msg::Nats)
}
}
@@ -0,0 +1,7 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
pub mod pubsub;
@@ -0,0 +1,130 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::{Msg, PubSubStream};
use futures::StreamExt;
use redis::{AsyncCommands, PushInfo, cluster::ClusterConfig, cluster_async::ClusterConnection};
use std::fmt::Display;
use store::backend::redis::{RedisPool, RedisStore};
use tokio::sync::mpsc::UnboundedReceiver;
pub struct RedisPubSubStream {
stream: redis::aio::PubSubStream,
}
pub struct RedisClusterPubSubStream {
_conn: ClusterConnection,
rx: UnboundedReceiver<PushInfo>,
}
pub(crate) async fn redis_publish(
redis: &RedisStore,
topic: &'static str,
message: Vec<u8>,
) -> trc::Result<()> {
match &redis.pool {
RedisPool::Single(pool) => pool
.get()
.await
.map_err(into_error)?
.as_mut()
.publish(topic, message)
.await
.map_err(into_error),
RedisPool::Cluster(pool) => pool
.get()
.await
.map_err(into_error)?
.as_mut()
.publish(topic, message)
.await
.map_err(into_error),
RedisPool::Sentinel(pool) => pool
.get()
.await
.map_err(into_error)?
.as_mut()
.publish(topic, message)
.await
.map_err(into_error),
}
}
pub(crate) async fn redis_subscribe(
redis: &RedisStore,
topic: &'static str,
) -> trc::Result<PubSubStream> {
match &redis.pool {
RedisPool::Single(pool) => {
let mut pubsub = pool
.manager()
.client
.get_async_pubsub()
.await
.map_err(into_error)?;
pubsub.subscribe(topic).await.map_err(into_error)?;
Ok(PubSubStream::Redis(RedisPubSubStream {
stream: pubsub.into_on_message(),
}))
}
RedisPool::Cluster(pool) => {
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
let mut _conn = pool
.manager()
.client
.get_async_connection_with_config(ClusterConfig::default().set_push_sender(tx))
.await
.map_err(into_error)?;
_conn.subscribe(topic).await.map_err(into_error)?;
Ok(PubSubStream::RedisCluster(RedisClusterPubSubStream {
_conn,
rx,
}))
}
RedisPool::Sentinel(pool) => {
let client = pool
.manager()
.client
.lock()
.await
.async_get_client()
.await
.map_err(into_error)?;
let mut pubsub = client.get_async_pubsub().await.map_err(into_error)?;
pubsub.subscribe(topic).await.map_err(into_error)?;
Ok(PubSubStream::Redis(RedisPubSubStream {
stream: pubsub.into_on_message(),
}))
}
}
}
impl RedisPubSubStream {
pub async fn next(&mut self) -> Option<Msg> {
self.stream.next().await.map(Msg::Redis)
}
}
impl RedisClusterPubSubStream {
pub async fn next(&mut self) -> Option<Msg> {
loop {
if let Some(msg) = redis::Msg::from_push_info(self.rx.recv().await?) {
return Some(Msg::Redis(msg));
}
}
}
}
#[inline(always)]
fn into_error(err: impl Display) -> trc::Error {
trc::StoreEvent::RedisError.reason(err)
}
@@ -0,0 +1,27 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use registry::schema::structs::ZenohCoordinator;
use crate::Coordinator;
pub mod pubsub;
#[derive(Debug)]
pub struct ZenohPubSub {
session: zenoh::Session,
}
impl ZenohPubSub {
pub async fn open(config: ZenohCoordinator) -> Result<Coordinator, String> {
let zenoh_config = zenoh::Config::from_json5(&config.config)
.map_err(|err| format!("Invalid Zenoh config: {}", err))?;
zenoh::open(zenoh_config)
.await
.map_err(|err| format!("Failed to create Zenoh session: {}", err))
.map(|session| ZenohPubSub { session })
.map(|store| Coordinator::Zenoh(std::sync::Arc::new(store)))
}
}
@@ -0,0 +1,47 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use super::ZenohPubSub;
use crate::{Msg, PubSubStream};
use trc::{ClusterEvent, Error, EventType};
pub struct ZenohPubSubStream {
subs: zenoh::pubsub::Subscriber<zenoh::handlers::FifoChannelHandler<zenoh::sample::Sample>>,
}
impl ZenohPubSub {
pub async fn publish(&self, topic: &'static str, message: Vec<u8>) -> trc::Result<()> {
self.session
.declare_publisher(topic)
.await
.map_err(|err| {
Error::new(EventType::Cluster(ClusterEvent::PublisherError)).reason(err)
})?
.put(message)
.await
.map_err(|err| Error::new(EventType::Cluster(ClusterEvent::PublisherError)).reason(err))
}
pub async fn subscribe(&self, topic: &'static str) -> trc::Result<PubSubStream> {
self.session
.declare_subscriber(topic)
.await
.map(|subs| PubSubStream::Zenoh(ZenohPubSubStream { subs }))
.map_err(|err| {
Error::new(EventType::Cluster(ClusterEvent::SubscriberError)).reason(err)
})
}
}
impl ZenohPubSubStream {
pub async fn next(&mut self) -> Option<Msg> {
self.subs
.recv_async()
.await
.map(|sample| Msg::Zenoh(sample.payload().to_bytes().into_owned()))
.ok()
}
}
+81
View File
@@ -0,0 +1,81 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::Coordinator;
use registry::schema::{prelude::ObjectType, structs};
use store::{InMemoryStore, registry::bootstrap::Bootstrap};
#[allow(unreachable_patterns)]
impl Coordinator {
pub async fn build(bp: &mut Bootstrap, _in_memory: &InMemoryStore) -> Option<Self> {
let result = match bp.setting_infallible::<structs::Coordinator>().await {
structs::Coordinator::Disabled => Ok(Coordinator::None),
#[cfg(feature = "redis")]
structs::Coordinator::Default => {
if let InMemoryStore::Redis(redis) = &_in_memory {
Ok(Coordinator::Redis(redis.clone()))
} else {
Err(
"Default coordinator requires Redis or Redis Cluster in-memory backend"
.to_string(),
)
}
}
#[cfg(feature = "kafka")]
structs::Coordinator::Kafka(kafka_coordinator) => {
crate::backend::kafka::KafkaPubSub::open(kafka_coordinator).await
}
#[cfg(feature = "nats")]
structs::Coordinator::Nats(nats_coordinator) => {
crate::backend::nats::NatsPubSub::open(nats_coordinator).await
}
#[cfg(feature = "zenoh")]
structs::Coordinator::Zenoh(zenoh_coordinator) => {
crate::backend::zenoh::ZenohPubSub::open(zenoh_coordinator).await
}
#[cfg(feature = "redis")]
structs::Coordinator::Redis(redis_store) => {
store::backend::redis::RedisStore::open_single(redis_store)
.await
.map(unwrap_redis)
}
#[cfg(feature = "redis")]
structs::Coordinator::RedisCluster(redis_cluster_store) => {
store::backend::redis::RedisStore::open_cluster(redis_cluster_store)
.await
.map(unwrap_redis)
}
#[cfg(feature = "redis")]
structs::Coordinator::RedisSentinel(redis_sentinel_store) => {
store::backend::redis::RedisStore::open_sentinel(redis_sentinel_store)
.await
.map(unwrap_redis)
}
_ => Err("Binary was not compiled with the selected coordinator backend".to_string()),
};
match result {
Ok(store) => Some(store),
Err(err) => {
bp.build_error(ObjectType::Coordinator.singleton(), err);
None
}
}
}
pub fn is_enabled(&self) -> bool {
!matches!(self, Coordinator::None)
}
}
#[cfg(feature = "redis")]
fn unwrap_redis(store: InMemoryStore) -> Coordinator {
if let InMemoryStore::Redis(redis) = store {
Coordinator::Redis(redis)
} else {
unreachable!()
}
}
+94
View File
@@ -0,0 +1,94 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::{Coordinator, Msg, PubSubStream};
#[allow(unused_variables)]
impl Coordinator {
pub async fn publish(&self, topic: &'static str, message: Vec<u8>) -> trc::Result<()> {
match self {
#[cfg(feature = "redis")]
Coordinator::Redis(store) => {
crate::backend::redis::pubsub::redis_publish(store, topic, message).await
}
#[cfg(feature = "nats")]
Coordinator::Nats(store) => store.publish(topic, message).await,
#[cfg(feature = "zenoh")]
Coordinator::Zenoh(store) => store.publish(topic, message).await,
#[cfg(feature = "kafka")]
Coordinator::Kafka(store) => store.publish(topic, message).await,
Coordinator::None => Err(trc::StoreEvent::NotSupported.into_err()),
}
}
pub async fn subscribe(&self, topic: &'static str) -> trc::Result<PubSubStream> {
match self {
#[cfg(feature = "redis")]
Coordinator::Redis(store) => {
crate::backend::redis::pubsub::redis_subscribe(store, topic).await
}
#[cfg(feature = "nats")]
Coordinator::Nats(store) => store.subscribe(topic).await,
#[cfg(feature = "zenoh")]
Coordinator::Zenoh(store) => store.subscribe(topic).await,
#[cfg(feature = "kafka")]
Coordinator::Kafka(store) => store.subscribe(topic).await,
Coordinator::None => Err(trc::StoreEvent::NotSupported.into_err()),
}
}
pub fn is_none(&self) -> bool {
matches!(self, Coordinator::None)
}
}
impl PubSubStream {
pub async fn next(&mut self) -> Option<Msg> {
match self {
#[cfg(feature = "redis")]
PubSubStream::Redis(stream) => stream.next().await,
#[cfg(feature = "redis")]
PubSubStream::RedisCluster(stream) => stream.next().await,
#[cfg(feature = "nats")]
PubSubStream::Nats(stream) => stream.next().await,
#[cfg(feature = "zenoh")]
PubSubStream::Zenoh(stream) => stream.next().await,
#[cfg(feature = "kafka")]
PubSubStream::Kafka(stream) => stream.next().await,
PubSubStream::Unimplemented => None,
}
}
}
impl Msg {
pub fn payload(&self) -> &[u8] {
match self {
#[cfg(feature = "redis")]
Msg::Redis(msg) => msg.get_payload_bytes(),
#[cfg(feature = "nats")]
Msg::Nats(msg) => msg.payload.as_ref(),
#[cfg(feature = "zenoh")]
Msg::Zenoh(msg) => msg.as_slice(),
#[cfg(feature = "kafka")]
Msg::Kafka(msg) => msg.as_slice(),
Msg::Unimplemented => &[],
}
}
pub fn topic(&self) -> &str {
match self {
#[cfg(feature = "redis")]
Msg::Redis(msg) => msg.get_channel_name(),
#[cfg(feature = "nats")]
Msg::Nats(msg) => msg.subject.as_str(),
#[cfg(feature = "zenoh")]
Msg::Zenoh(_) => "",
#[cfg(feature = "kafka")]
Msg::Kafka(_) => "",
Msg::Unimplemented => "",
}
}
}
+54
View File
@@ -0,0 +1,54 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
#![warn(clippy::large_futures)]
#[allow(unused_imports)]
use std::sync::Arc;
pub mod backend;
pub mod bootstrap;
pub mod dispatch;
#[derive(Clone, Default)]
pub enum Coordinator {
#[cfg(feature = "redis")]
Redis(Arc<store::backend::redis::RedisStore>),
#[cfg(feature = "nats")]
Nats(Arc<backend::nats::NatsPubSub>),
#[cfg(feature = "zenoh")]
Zenoh(Arc<backend::zenoh::ZenohPubSub>),
#[cfg(feature = "kafka")]
Kafka(Arc<backend::kafka::KafkaPubSub>),
#[default]
None,
}
pub enum PubSubStream {
#[cfg(feature = "redis")]
Redis(crate::backend::redis::pubsub::RedisPubSubStream),
#[cfg(feature = "redis")]
RedisCluster(crate::backend::redis::pubsub::RedisClusterPubSubStream),
#[cfg(feature = "nats")]
Nats(crate::backend::nats::pubsub::NatsPubSubStream),
#[cfg(feature = "zenoh")]
Zenoh(crate::backend::zenoh::pubsub::ZenohPubSubStream),
#[cfg(feature = "kafka")]
Kafka(crate::backend::kafka::pubsub::KafkaPubSubStream),
Unimplemented,
}
pub enum Msg {
#[cfg(feature = "redis")]
Redis(redis::Msg),
#[cfg(feature = "nats")]
Nats(async_nats::Message),
#[cfg(feature = "zenoh")]
Zenoh(Vec<u8>),
#[cfg(feature = "kafka")]
Kafka(Vec<u8>),
Unimplemented,
}