Import upstream v0.16.22, stripped

Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f
Enterprise-only files removed or emptied: 63
Enterprise-only snippets removed: 117 in 50 files
Dangling module declarations removed: 5
Cargo edits turning enterprise off: 14
Verification: clean
Enterprise feature gates left for rebuilt features: 19 in 18 files

Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.

crates/coordinator/src/backend/kafka/mod.rs

@@ -0,0 +1,78 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::sync::Arc;
+
+use crate::Coordinator;
+use rdkafka::{
+    ClientConfig, ClientContext, TopicPartitionList,
+    consumer::{BaseConsumer, ConsumerContext, Rebalance, StreamConsumer},
+    error::KafkaResult,
+    producer::FutureProducer,
+};
+use registry::schema::structs::KafkaCoordinator;
+
+pub mod pubsub;
+
+pub(super) type LoggingConsumer = StreamConsumer<CustomContext>;
+
+pub struct KafkaPubSub {
+    consumer_builder: ClientConfig,
+    producer: FutureProducer,
+}
+
+impl KafkaPubSub {
+    pub async fn open(config: KafkaCoordinator) -> Result<Coordinator, String> {
+        if config.brokers.is_empty() {
+            return Err("No Kafka brokers specified".to_string());
+        }
+
+        let brokers = config.brokers.into_inner().join(",");
+        let mut consumer_builder = ClientConfig::new();
+
+        consumer_builder
+            .set("group.id", config.group_id)
+            .set("bootstrap.servers", &brokers)
+            .set("enable.partition.eof", "false")
+            .set(
+                "session.timeout.ms",
+                config.timeout_session.as_millis().to_string(),
+            )
+            .set("enable.auto.commit", "true");
+
+        let producer = ClientConfig::new()
+            .set("bootstrap.servers", brokers)
+            .set(
+                "message.timeout.ms",
+                config.timeout_message.as_millis().to_string(),
+            )
+            .create()
+            .map_err(|err| format!("Failed to create Kafka producer: {}", err))?;
+
+        Ok(Coordinator::Kafka(Arc::new(KafkaPubSub {
+            consumer_builder,
+            producer,
+        })))
+    }
+}
+
+impl std::fmt::Debug for KafkaPubSub {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("KafkaPubSub").finish()
+    }
+}
+
+pub(super) struct CustomContext;
+
+impl ClientContext for CustomContext {}
+
+impl ConsumerContext for CustomContext {
+    fn pre_rebalance(&self, _: &BaseConsumer<Self>, _: &Rebalance) {}
+
+    fn post_rebalance(&self, _: &BaseConsumer<Self>, _: &Rebalance) {}
+
+    fn commit_callback(&self, _: KafkaResult<()>, _: &TopicPartitionList) {}
+}

crates/coordinator/src/backend/kafka/pubsub.rs

@@ -0,0 +1,56 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::{CustomContext, KafkaPubSub, LoggingConsumer};
+use crate::{Msg, PubSubStream};
+use rdkafka::{
+    Message,
+    consumer::{CommitMode, Consumer, StreamConsumer},
+    producer::FutureRecord,
+};
+use std::time::Duration;
+use trc::{ClusterEvent, Error, EventType};
+
+pub struct KafkaPubSubStream {
+    subs: LoggingConsumer,
+}
+
+impl KafkaPubSub {
+    pub async fn publish(&self, topic: &'static str, message: Vec<u8>) -> trc::Result<()> {
+        self.producer
+            .send(
+                FutureRecord::<(), [u8]>::to(topic).payload(message.as_slice()),
+                Duration::from_secs(0),
+            )
+            .await
+            .map(|_| ())
+            .map_err(|(err, _)| {
+                Error::new(EventType::Cluster(ClusterEvent::PublisherError)).reason(err)
+            })
+    }
+
+    pub async fn subscribe(&self, topic: &'static str) -> trc::Result<PubSubStream> {
+        let subs: StreamConsumer<CustomContext> = self
+            .consumer_builder
+            .create_with_context(CustomContext)
+            .map_err(|err| {
+                Error::new(EventType::Cluster(ClusterEvent::SubscriberError)).reason(err)
+            })?;
+        subs.subscribe(&[topic]).map_err(|err| {
+            Error::new(EventType::Cluster(ClusterEvent::SubscriberError)).reason(err)
+        })?;
+
+        Ok(PubSubStream::Kafka(KafkaPubSubStream { subs }))
+    }
+}
+
+impl KafkaPubSubStream {
+    pub async fn next(&mut self) -> Option<Msg> {
+        let msg = self.subs.recv().await.ok()?;
+        let _ = self.subs.commit_message(&msg, CommitMode::Async);
+        Msg::Kafka(msg.payload().unwrap_or_default().to_vec()).into()
+    }
+}