Import upstream v0.16.22, stripped

Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f
Enterprise-only files removed or emptied: 63
Enterprise-only snippets removed: 117 in 50 files
Dangling module declarations removed: 5
Cargo edits turning enterprise off: 14
Verification: clean
Enterprise feature gates left for rebuilt features: 19 in 18 files

Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.
This commit is contained in:
2026-09-18 10:21:56 -07:00
commit 7dae9b29fd
1650 changed files with 485521 additions and 0 deletions
+287
View File
@@ -0,0 +1,287 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use common::ipc::{
BroadcastEvent, CacheInvalidation, CalendarAlert, EmailPush, PushNotification, RegistryChange,
};
use registry::{
schema::prelude::ObjectType,
types::{EnumImpl, id::ObjectId},
};
use std::{borrow::Borrow, io::Write};
use types::{id::Id, type_state::StateChange};
use utils::{
codec::leb128::{Leb128Iterator, Leb128Writer},
map::bitmap::Bitmap,
};
pub mod publisher;
pub mod subscriber;
#[derive(Debug)]
pub(crate) struct BroadcastBatch<T> {
messages: T,
}
const MAX_BATCH_SIZE: usize = 100;
pub(crate) const BROADCAST_TOPIC: &str = "stwt.agora";
impl BroadcastBatch<Vec<BroadcastEvent>> {
pub fn init() -> Self {
Self {
messages: Vec::with_capacity(MAX_BATCH_SIZE),
}
}
pub fn insert(&mut self, message: BroadcastEvent) -> bool {
self.messages.push(message);
self.messages.len() < MAX_BATCH_SIZE
}
pub fn serialize(&self, node_id: u16) -> Vec<u8> {
let mut serialized =
Vec::with_capacity((self.messages.len() * 10) + std::mem::size_of::<u16>());
let _ = serialized.write_leb128(node_id);
for message in &self.messages {
match message {
BroadcastEvent::PushNotification(notification) => match notification {
PushNotification::StateChange(state_change) => {
serialized.push(0u8);
let _ = serialized.write_leb128(state_change.change_id);
let _ = serialized.write_leb128(*state_change.types.as_ref());
let _ = serialized.write_leb128(state_change.account_id);
}
PushNotification::CalendarAlert(calendar_alert) => {
serialized.push(1u8);
let _ = serialized.write_leb128(calendar_alert.account_id);
let _ = serialized.write_leb128(calendar_alert.event_id);
let _ = serialized
.write_leb128(calendar_alert.recurrence_id.unwrap_or_default() as u64);
let _ = serialized.write_leb128(calendar_alert.uid.len());
let _ = serialized.write(calendar_alert.uid.as_bytes());
let _ = serialized.write_leb128(calendar_alert.alert_id.len());
let _ = serialized.write(calendar_alert.alert_id.as_bytes());
}
PushNotification::EmailPush(email_push) => {
serialized.push(2u8);
let _ = serialized.write_leb128(email_push.account_id);
let _ = serialized.write_leb128(email_push.email_id);
let _ = serialized.write_leb128(email_push.change_id);
}
},
BroadcastEvent::PushServerUpdate(account_id) => {
serialized.push(3u8);
let _ = serialized.write_leb128(*account_id);
}
BroadcastEvent::RegistryChange(items) => match items {
RegistryChange::Insert(id) => {
serialized.push(4u8);
let _ = serialized.write_leb128(id.object().to_id());
let _ = serialized.write_leb128(id.id().id());
}
RegistryChange::Delete(id) => {
serialized.push(5u8);
let _ = serialized.write_leb128(id.object().to_id());
let _ = serialized.write_leb128(id.id().id());
}
RegistryChange::Reload(object) => {
serialized.push(6u8);
let _ = serialized.write_leb128(object.to_id());
}
},
BroadcastEvent::CacheInvalidate(items) => {
serialized.push(7u8);
let _ = serialized.write_leb128(items.len());
for item in items {
let (marker, id) = match item {
CacheInvalidation::AccessToken(id) => (0u8, *id),
CacheInvalidation::DavResources(id) => (1u8, *id),
CacheInvalidation::Domain(id) => (2u8, *id),
CacheInvalidation::Account(id) => (3u8, *id),
CacheInvalidation::DkimSignature(id) => (4u8, *id),
CacheInvalidation::Tenant(id) => (5u8, *id),
CacheInvalidation::Role(id) => (6u8, *id),
CacheInvalidation::List(id) => (7u8, *id),
CacheInvalidation::DomainLogo(id) => (8u8, *id),
CacheInvalidation::TenantLogo(id) => (9u8, *id),
CacheInvalidation::EmailNegative {
domain_id,
local_part_hash,
} => {
serialized.push(10u8);
let _ = serialized.write_leb128(*domain_id);
let _ = serialized.write_leb128(*local_part_hash);
continue;
}
CacheInvalidation::DomainNegative => (11u8, 0),
};
serialized.push(marker);
let _ = serialized.write_leb128(id);
}
}
BroadcastEvent::CacheInvalidateAll => {
serialized.push(8u8);
}
BroadcastEvent::CacheInvalidateNegative => {
serialized.push(9u8);
}
BroadcastEvent::MtaQueueStatus { is_running } => {
if *is_running {
serialized.push(10u8);
} else {
serialized.push(11u8);
}
}
BroadcastEvent::QueueRefresh => {
serialized.push(12u8);
}
}
}
serialized
}
pub fn clear(&mut self) {
self.messages.clear();
}
}
impl<T, I> BroadcastBatch<T>
where
T: Iterator<Item = I> + Leb128Iterator<I>,
I: Borrow<u8>,
{
pub fn node_id(&mut self) -> Option<u16> {
self.messages.next_leb128::<u16>()
}
pub fn next_event(&mut self) -> Result<Option<BroadcastEvent>, ()> {
if let Some(id) = self.messages.next() {
match id.borrow() {
0 => Ok(Some(BroadcastEvent::PushNotification(
PushNotification::StateChange(StateChange {
change_id: self.messages.next_leb128().ok_or(())?,
types: Bitmap::from(self.messages.next_leb128::<u64>().ok_or(())?),
account_id: self.messages.next_leb128().ok_or(())?,
}),
))),
1 => {
let account_id = self.messages.next_leb128().ok_or(())?;
let event_id = self.messages.next_leb128().ok_or(())?;
let recurrence_id = self.messages.next_leb128::<u64>().ok_or(())? as i64;
let uid_len = self.messages.next_leb128::<usize>().ok_or(())?;
let mut uid_bytes = vec![0u8; uid_len];
for byte in uid_bytes.iter_mut() {
*byte = self.messages.next().ok_or(())?.borrow().to_owned();
}
let uid = String::from_utf8(uid_bytes).map_err(|_| ())?;
let alert_id_len = self.messages.next_leb128::<usize>().ok_or(())?;
let mut alert_id_bytes = vec![0u8; alert_id_len];
for byte in alert_id_bytes.iter_mut() {
*byte = self.messages.next().ok_or(())?.borrow().to_owned();
}
let alert_id = String::from_utf8(alert_id_bytes).map_err(|_| ())?;
Ok(Some(BroadcastEvent::PushNotification(
PushNotification::CalendarAlert(CalendarAlert {
account_id,
event_id,
recurrence_id: if recurrence_id == 0 {
None
} else {
Some(recurrence_id)
},
uid,
alert_id,
}),
)))
}
2 => Ok(Some(BroadcastEvent::PushNotification(
PushNotification::EmailPush(EmailPush {
account_id: self.messages.next_leb128().ok_or(())?,
email_id: self.messages.next_leb128().ok_or(())?,
change_id: self.messages.next_leb128().ok_or(())?,
}),
))),
3 => {
let account_id = self.messages.next_leb128().ok_or(())?;
Ok(Some(BroadcastEvent::PushServerUpdate(account_id)))
}
4 => {
let object_id = self.messages.next_leb128().ok_or(())?;
let id = self.messages.next_leb128::<u64>().ok_or(())?;
Ok(Some(BroadcastEvent::RegistryChange(
RegistryChange::Insert(ObjectId::new(
ObjectType::from_id(object_id).ok_or(())?,
Id::new(id),
)),
)))
}
5 => {
let object_id = self.messages.next_leb128().ok_or(())?;
let id = self.messages.next_leb128::<u64>().ok_or(())?;
Ok(Some(BroadcastEvent::RegistryChange(
RegistryChange::Delete(ObjectId::new(
ObjectType::from_id(object_id).ok_or(())?,
Id::new(id),
)),
)))
}
6 => {
let object_id = self.messages.next_leb128().ok_or(())?;
Ok(Some(BroadcastEvent::RegistryChange(
RegistryChange::Reload(ObjectType::from_id(object_id).ok_or(())?),
)))
}
7 => {
let count = self.messages.next_leb128::<usize>().ok_or(())?;
let mut items = Vec::with_capacity(count);
for _ in 0..count {
let marker = self.messages.next().ok_or(())?.borrow().to_owned();
let id = self.messages.next_leb128::<u32>().ok_or(())?;
items.push(match marker {
0 => CacheInvalidation::AccessToken(id),
1 => CacheInvalidation::DavResources(id),
2 => CacheInvalidation::Domain(id),
3 => CacheInvalidation::Account(id),
4 => CacheInvalidation::DkimSignature(id),
5 => CacheInvalidation::Tenant(id),
6 => CacheInvalidation::Role(id),
7 => CacheInvalidation::List(id),
8 => CacheInvalidation::DomainLogo(id),
9 => CacheInvalidation::TenantLogo(id),
10 => {
let local_part_hash =
self.messages.next_leb128::<u32>().ok_or(())?;
CacheInvalidation::EmailNegative {
domain_id: id,
local_part_hash,
}
}
11 => CacheInvalidation::DomainNegative,
_ => return Err(()),
});
}
Ok(Some(BroadcastEvent::CacheInvalidate(items)))
}
8 => Ok(Some(BroadcastEvent::CacheInvalidateAll)),
9 => Ok(Some(BroadcastEvent::CacheInvalidateNegative)),
10 => Ok(Some(BroadcastEvent::MtaQueueStatus { is_running: true })),
11 => Ok(Some(BroadcastEvent::MtaQueueStatus { is_running: false })),
12 => Ok(Some(BroadcastEvent::QueueRefresh)),
_ => Err(()),
}
} else {
Ok(None)
}
}
}
impl<T> BroadcastBatch<T> {
pub fn new(messages: T) -> Self {
Self { messages }
}
}
@@ -0,0 +1,55 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use std::sync::Arc;
use common::{Inner, ipc::BroadcastEvent};
use tokio::sync::mpsc;
use trc::ClusterEvent;
use super::{BROADCAST_TOPIC, BroadcastBatch};
pub fn spawn_broadcast_publisher(inner: Arc<Inner>, mut event_rx: mpsc::Receiver<BroadcastEvent>) {
let (coordinator, this_node_id) = {
let _core = inner.shared_core.load();
let coordinator = inner.shared_core.load().storage.coordinator.clone();
if coordinator.is_none() {
return;
}
(coordinator, _core.network.node_id as u16)
};
tokio::spawn(async move {
let mut batch = BroadcastBatch::init();
trc::event!(Cluster(ClusterEvent::PublisherStart));
while let Some(event) = event_rx.recv().await {
batch.insert(event);
while let Ok(event) = event_rx.try_recv() {
if !batch.insert(event) {
break;
}
}
match coordinator
.publish(BROADCAST_TOPIC, batch.serialize(this_node_id))
.await
{
Ok(_) => {
batch.clear();
}
Err(err) => {
batch.clear();
trc::event!(Cluster(ClusterEvent::PublisherError), CausedBy = err);
}
}
}
trc::event!(Cluster(ClusterEvent::PublisherStop));
});
}
+276
View File
@@ -0,0 +1,276 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::broadcast::{BROADCAST_TOPIC, BroadcastBatch};
use common::{
BuildServer, Inner,
ipc::{BroadcastEvent, PushEvent, PushNotification, QueueEvent, RegistryChange},
};
use registry::types::EnumImpl;
use std::{sync::Arc, time::Duration};
use tokio::sync::watch;
use trc::{ClusterEvent, ServerEvent};
pub fn spawn_broadcast_subscriber(inner: Arc<Inner>, mut shutdown_rx: watch::Receiver<bool>) {
let this_node_id = {
let _core = inner.shared_core.load();
if _core.storage.coordinator.is_none() || _core.storage.registry.is_recovery_mode() {
return;
}
_core.network.node_id as u16
};
tokio::spawn(async move {
let mut retry_count = 0;
trc::event!(Cluster(ClusterEvent::SubscriberStart));
'subscribe: loop {
let coordinator = inner.shared_core.load().storage.coordinator.clone();
if coordinator.is_none() {
trc::event!(
Cluster(ClusterEvent::SubscriberError),
Details = "Coordinator is no longer configured"
);
break;
}
let mut stream = match coordinator.subscribe(BROADCAST_TOPIC).await {
Ok(stream) => {
retry_count = 0;
stream
}
Err(err) => {
trc::event!(
Cluster(ClusterEvent::SubscriberError),
CausedBy = err,
Details = "Failed to subscribe to channel"
);
match tokio::time::timeout(
Duration::from_secs(1 << retry_count.max(6)),
shutdown_rx.changed(),
)
.await
{
Ok(_) => {
break;
}
Err(_) => {
retry_count += 1;
continue;
}
}
}
};
loop {
tokio::select! {
message = stream.next() => {
match message {
Some(message) => {
let mut batch = BroadcastBatch::new(message.payload().iter());
let node_id = match batch.node_id() {
Some(node_id) => {
if node_id != this_node_id {
node_id
} else {
trc::event!(
Cluster(ClusterEvent::MessageSkipped),
Details = message.payload()
);
continue;
}
}
None => {
trc::event!(
Cluster(ClusterEvent::MessageInvalid),
Details = message.payload()
);
continue;
}
};
loop {
match batch.next_event() {
Ok(Some(event)) => {
trc::event!(
Cluster(ClusterEvent::MessageReceived),
From = node_id,
To = this_node_id,
Details = log_event(&event),
);
match event {
BroadcastEvent::PushNotification(notification) => {
if inner
.ipc
.push_tx
.send(PushEvent::Publish {
notification,
broadcast: false,
})
.await
.is_err()
{
trc::event!(
Server(ServerEvent::ThreadError),
Details = "Error sending push notification.",
CausedBy = trc::location!()
);
}
}
BroadcastEvent::PushServerUpdate(account_id) => {
if inner
.ipc
.push_tx
.send(PushEvent::PushServerUpdate { account_id, broadcast: false })
.await
.is_err()
{
trc::event!(
Server(ServerEvent::ThreadError),
Details = "Error sending reload request.",
CausedBy = trc::location!()
);
}
}
BroadcastEvent::CacheInvalidate(changes) => {
inner.build_server().invalidate_local_caches(&changes).await;
}
BroadcastEvent::CacheInvalidateAll => {
inner.build_server().invalidate_all_local_caches();
}
BroadcastEvent::CacheInvalidateNegative => {
inner.build_server().invalidate_all_local_negative_caches();
}
BroadcastEvent::MtaQueueStatus { is_running } => {
let _ = inner
.ipc
.queue_tx
.send(QueueEvent::Paused(!is_running))
.await;
}
BroadcastEvent::QueueRefresh => {
let core = inner.shared_core.load_full();
if core.network.roles.outbound_mta {
core.storage.data.invalidate_read_snapshot();
let _ = inner
.ipc
.queue_tx
.send(QueueEvent::Refresh)
.await;
}
}
BroadcastEvent::RegistryChange(change) => {
let server = inner.build_server();
server.store().invalidate_read_snapshot();
match Box::pin(server.reload_registry(change)).await {
Ok(result) => {
result.log();
}
Err(err) => {
trc::error!(
err.details("Failed to reload settings")
.caused_by(trc::location!())
);
}
}
}
}
}
Ok(None) => break,
Err(_) => {
trc::event!(
Cluster(ClusterEvent::MessageInvalid),
Details = message.payload()
);
break;
}
}
}
}
None => {
trc::event!(
Cluster(ClusterEvent::SubscriberDisconnected),
);
break;
}
}
},
_ = shutdown_rx.changed() => {
break 'subscribe;
}
};
}
}
trc::event!(Cluster(ClusterEvent::SubscriberStop));
});
}
fn log_event(event: &BroadcastEvent) -> trc::Value {
match event {
BroadcastEvent::PushNotification(notification) => match notification {
PushNotification::StateChange(state_change) => trc::Value::Array(vec![
"StateChange".into(),
state_change.account_id.into(),
state_change.change_id.into(),
(*state_change.types.as_ref()).into(),
]),
PushNotification::CalendarAlert(calendar_alert) => trc::Value::Array(vec![
"CalendarAlert".into(),
calendar_alert.account_id.into(),
calendar_alert.event_id.into(),
calendar_alert.recurrence_id.into(),
calendar_alert.uid.clone().into(),
calendar_alert.alert_id.clone().into(),
]),
PushNotification::EmailPush(email_push) => trc::Value::Array(vec![
"EmailPush".into(),
email_push.account_id.into(),
email_push.email_id.into(),
email_push.change_id.into(),
]),
},
BroadcastEvent::PushServerUpdate(account_id) => {
trc::Value::Array(vec!["PushServerUpdate".into(), (*account_id).into()])
}
BroadcastEvent::RegistryChange(change) => match change {
RegistryChange::Insert(id) => trc::Value::Array(vec![
"RegistryInsert".into(),
id.object().as_str().into(),
id.id().id().into(),
]),
RegistryChange::Delete(id) => trc::Value::Array(vec![
"RegistryDelete".into(),
id.object().as_str().into(),
id.id().id().into(),
]),
RegistryChange::Reload(object) => {
trc::Value::Array(vec!["RegistryReload".into(), object.as_str().into()])
}
},
BroadcastEvent::CacheInvalidate(items) => {
let mut array = Vec::with_capacity(items.len() + 1);
array.push("CacheInvalidation".into());
for item in items {
array.push(format!("{:?}", item).into());
}
trc::Value::Array(array)
}
BroadcastEvent::CacheInvalidateAll => "CacheInvalidateAll".into(),
BroadcastEvent::CacheInvalidateNegative => "CacheInvalidateNegative".into(),
BroadcastEvent::MtaQueueStatus { is_running } => {
if *is_running {
"MtaQueueRunning".into()
} else {
"MtaQueuePaused".into()
}
}
BroadcastEvent::QueueRefresh => "QueueRefresh".into(),
}
}