inbuxa/inbuxa-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 23 Wiki Activity

Import upstream v0.16.24, stripped

Browse Source 
Upstream commit: af37a234981722493b74623a983581691d2b70b6
Enterprise-only files removed or emptied: 63
Enterprise-only snippets removed: 118 in 50 files
Dangling module declarations removed: 5
Edits turning enterprise off: 25
Third-party code: 14 files, 0 not in THIRD-PARTY.md
Renamed identifiers: 62 in 18 files
Verification: clean

The same Enterprise footprint as v0.16.23. The build check fails only on
tests/src/directory/issuer.rs, unchanged since v0.16.23: it calls a helper
from upstream's Enterprise-only OIDC test, and tests issuer-based directory
routing, an Enterprise feature. main has never carried it.
This commit is contained in:
jcoffey-dev
2026-09-28 06:29:38 -07:00
parent 3a272096c0
commit f59b084ce5
98 changed files with 2851 additions and 1068 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/services/src/state_manager/http.rs
+71 -57
View File
@@ -9,7 +9,7 @@ use super::{
ece::{ECE_WEBPUSH_MAX_PLAINTEXT_SIZE, WEBPUSH_MAX_BODY_SIZE, ece_encrypt},
email_push::build_email_push_object,
};
use crate::state_manager::PushRegistration;
use crate::state_manager::{PushBatch, PushRegistration};
use calcard::jscalendar::JSCalendarDateTime;
use common::{Server, ipc::PushNotification, network::webpush::Vapid};
use email::push::{PushSubscription, Urgency};
@@ -29,7 +29,7 @@ use std::time::{Duration, Instant};
use store::write::now;
use tokio::sync::mpsc;
use trc::PushSubscriptionEvent;
use types::{id::Id, type_state::DataType};
use types::id::Id;
use utils::map::vec_map::VecMap;
const MAX_ERROR_RESPONSE_LEN: usize = 1024;
@@ -48,34 +48,29 @@ impl PushRegistration {
pub fn send(
&mut self,
id: Id,
push_client: &Client,
push_tx: mpsc::Sender<Event>,
push_timeout: Duration,
server: Server,
) {
let subscription = self.server.clone();
let push_client = self.client.clone();
let notifications = std::mem::take(&mut self.notifications);
let push_client = push_client.clone();
let batch = std::mem::take(&mut self.pending);
self.in_flight = true;
self.last_request = Instant::now();
tokio::spawn(async move {
let mut changed: VecMap<Id, VecMap<DataType, State>> = VecMap::new();
let vapid = server.core.jmap.vapid.as_ref();
let mut email_pushes: VecMap<Id, EmailPushObject> = VecMap::new();
let mut failed_state_change = false;
let mut failed = PushBatch::default();
let mut failed_email_pushes = Vec::new();
let mut failed_calendar_alerts = Vec::new();
for notification in &notifications {
for notification in &batch.notifications {
match notification {
PushNotification::StateChange(state_change) => {
for type_state in state_change.types {
changed
.get_mut_or_insert(state_change.account_id.into())
.set(type_state, State::Exact(state_change.change_id));
}
}
PushNotification::StateChange(_) => {}
PushNotification::CalendarAlert(calendar_alert) => {
let payload = PushObject::CalendarAlert {
account_id: calendar_alert.account_id.into(),
@@ -86,12 +81,12 @@ impl PushRegistration {
}),
alert_id: calendar_alert.alert_id.clone(),
};
if !http_request(
if !post_object(
&push_client,
&subscription,
serde_json::to_string(&payload).unwrap().into_bytes(),
&payload,
push_timeout,
server.core.jmap.vapid.as_ref(),
vapid,
Urgency::Normal,
)
.await
@@ -155,18 +150,23 @@ impl PushRegistration {
}
}
if !changed.is_empty() {
failed_state_change = !http_request(
if !batch.state_changes.is_empty() {
let payload = PushObject::StateChange {
changed: batch.state_changes,
};
if !post_object(
&push_client,
&subscription,
serde_json::to_string(&PushObject::StateChange { changed })
.unwrap()
.into_bytes(),
&payload,
push_timeout,
server.core.jmap.vapid.as_ref(),
vapid,
Urgency::Normal,
)
.await;
.await
&& let PushObject::StateChange { changed } = payload
{
failed.state_changes = changed;
}
}
for (account_id, email_push) in email_pushes {
@@ -180,12 +180,12 @@ impl PushRegistration {
state: email_push.change_id.map(State::Exact),
};
if !http_request(
if !post_object(
&push_client,
&subscription,
serde_json::to_string(&payload).unwrap().into_bytes(),
&payload,
push_timeout,
server.core.jmap.vapid.as_ref(),
vapid,
email_push.urgency,
)
.await
@@ -194,44 +194,26 @@ impl PushRegistration {
}
}
let result = if !failed_state_change
let result = if failed.state_changes.is_empty()
&& failed_email_pushes.is_empty()
&& failed_calendar_alerts.is_empty()
{
Event::DeliverySuccess { id }
} else {
let mut failed_notifications = Vec::with_capacity(
failed_state_change as usize
+ failed_email_pushes.len()
+ failed_calendar_alerts.len(),
);
for notification in notifications {
match &notification {
PushNotification::StateChange(_) => {
if failed_state_change {
failed_notifications.push(notification);
}
}
failed.notifications = batch
.notifications
.into_iter()
.filter(|notification| match notification {
PushNotification::StateChange(_) => false,
PushNotification::EmailPush(email_push) => {
if failed_email_pushes.contains(&email_push.account_id) {
failed_notifications.push(notification);
}
failed_email_pushes.contains(&email_push.account_id)
}
PushNotification::CalendarAlert(calendar_alert) => {
if failed_calendar_alerts
.contains(&(calendar_alert.account_id, calendar_alert.event_id))
{
failed_notifications.push(notification);
}
}
}
}
PushNotification::CalendarAlert(calendar_alert) => failed_calendar_alerts
.contains(&(calendar_alert.account_id, calendar_alert.event_id)),
})
.collect();
Event::DeliveryFailure {
id,
notifications: failed_notifications,
}
Event::DeliveryFailure { id, failed }
};
push_tx.send(result).await.ok();
@@ -239,6 +221,38 @@ impl PushRegistration {
}
}
async fn post_object(
push_client: &Client,
subscription: &PushSubscription,
object: &PushObject,
push_timeout: Duration,
vapid: Option<&Vapid>,
urgency: Urgency,
) -> bool {
match serde_json::to_vec(object) {
Ok(body) => {
http_request(
push_client,
subscription,
body,
push_timeout,
vapid,
urgency,
)
.await
}
Err(err) => {
trc::event!(
PushSubscription(PushSubscriptionEvent::Error),
Details = "Failed to serialize push object",
Url = subscription.url.to_string(),
Reason = err.to_string()
);
true
}
}
}
pub(crate) fn build_push_client() -> Client {
utils::http::http_client_builder(cfg!(feature = "test_mode"))
.redirect(Policy::custom(|attempt| match attempt.previous().last() {
crates/services/src/state_manager/manager.rs
+26 -22
View File
@@ -12,7 +12,7 @@ use common::{
};
use std::{sync::Arc, time::Instant};
use store::ahash::AHashMap;
use tokio::sync::mpsc;
use tokio::sync::mpsc::{self, error::TrySendError};
use trc::ServerEvent;
#[derive(Default)]
@@ -84,7 +84,7 @@ pub fn spawn_push_router(inner: Arc<Inner>, mut change_rx: mpsc::Receiver<PushEv
} => {
// Publish event to cluster
if broadcast
&& let Some(broadcast_tx) = &inner.ipc.broadcast_tx.clone()
&& let Some(broadcast_tx) = inner.ipc.broadcast_tx.as_ref()
&& broadcast_tx
.send(BroadcastEvent::PushNotification(notification.clone()))
.await
@@ -102,26 +102,30 @@ pub fn spawn_push_router(inner: Arc<Inner>, mut change_rx: mpsc::Receiver<PushEv
for subscriber in &subscribers.ipc {
if let Some(notification) = notification.filter_types(&subscriber.types)
{
if subscriber.is_valid() {
let subscriber_tx = subscriber.tx.clone();
match subscriber.tx.try_send(notification) {
Ok(()) => {}
Err(TrySendError::Full(notification)) => {
let subscriber_tx = subscriber.tx.clone();
tokio::spawn(async move {
// Timeout after 500ms in case there is a blocked client
if subscriber_tx
.send_timeout(notification, SEND_TIMEOUT)
.await
.is_err()
{
trc::event!(
Server(ServerEvent::ThreadError),
Details =
"Error sending state change to subscriber.",
CausedBy = trc::location!()
);
}
});
} else {
purge_needed = true;
tokio::spawn(async move {
// Timeout after 500ms in case there is a blocked client
if subscriber_tx
.send_timeout(notification, SEND_TIMEOUT)
.await
.is_err()
{
trc::event!(
Server(ServerEvent::ThreadError),
Details =
"Error sending state change to subscriber.",
CausedBy = trc::location!()
);
}
});
}
Err(TrySendError::Closed(_)) => {
purge_needed = true;
}
}
}
}
@@ -144,7 +148,7 @@ pub fn spawn_push_router(inner: Arc<Inner>, mut change_rx: mpsc::Receiver<PushEv
} => {
// Publish event to cluster
if broadcast
&& let Some(broadcast_tx) = &inner.ipc.broadcast_tx.clone()
&& let Some(broadcast_tx) = inner.ipc.broadcast_tx.as_ref()
&& broadcast_tx
.send(BroadcastEvent::PushServerUpdate(account_id))
.await
crates/services/src/state_manager/mod.rs
+140 -17
View File
@@ -12,14 +12,14 @@ pub mod push;
use common::ipc::PushNotification;
use email::push::PushSubscription;
use reqwest::Client;
use jmap_proto::types::state::State;
use std::{
sync::Arc,
time::{Duration, Instant},
};
use tokio::sync::mpsc;
use types::{id::Id, type_state::DataType};
use utils::map::bitmap::Bitmap;
use utils::map::{bitmap::Bitmap, vec_map::VecMap};
const PURGE_EVERY: Duration = Duration::from_secs(3600);
const SEND_TIMEOUT: Duration = Duration::from_millis(500);
@@ -36,26 +36,22 @@ pub struct PushRegistration {
member_account_ids: Vec<u32>,
num_attempts: u32,
last_request: Instant,
notifications: Vec<PushNotification>,
pending: PushBatch,
in_flight: bool,
client: Client,
}
#[derive(Debug, Default)]
pub struct PushBatch {
state_changes: VecMap<Id, VecMap<DataType, State>>,
notifications: Vec<PushNotification>,
}
#[derive(Debug)]
pub enum Event {
Push {
notification: PushNotification,
},
Update {
account_id: u32,
},
DeliverySuccess {
id: Id,
},
DeliveryFailure {
id: Id,
notifications: Vec<PushNotification>,
},
Push { notification: PushNotification },
Update { account_id: u32 },
DeliverySuccess { id: Id },
DeliveryFailure { id: Id, failed: PushBatch },
Reset,
}
@@ -64,3 +60,130 @@ impl IpcSubscriber {
!self.tx.is_closed()
}
}
impl PushBatch {
pub fn push(&mut self, notification: PushNotification) {
match notification {
PushNotification::StateChange(state_change) => {
if !state_change.types.is_empty() {
let states = self
.state_changes
.get_mut_or_insert(Id::from(state_change.account_id));
for data_type in state_change.types {
merge_state(states, data_type, state_change.change_id);
}
}
}
notification => self.notifications.push(notification),
}
}
pub fn merge_failed(&mut self, failed: PushBatch) {
for (account_id, failed_states) in failed.state_changes {
let states = self.state_changes.get_mut_or_insert(account_id);
for (data_type, state) in failed_states {
if let State::Exact(change_id) = state {
merge_state(states, data_type, change_id);
}
}
}
if !failed.notifications.is_empty() {
let mut notifications = failed.notifications;
notifications.append(&mut self.notifications);
self.notifications = notifications;
}
}
pub fn is_empty(&self) -> bool {
self.state_changes.is_empty() && self.notifications.is_empty()
}
pub fn clear(&mut self) {
self.state_changes.clear();
self.notifications.clear();
}
}
fn merge_state(states: &mut VecMap<DataType, State>, data_type: DataType, change_id: u64) {
match states.get_mut(&data_type) {
Some(State::Exact(current)) if *current >= change_id => {}
Some(state) => *state = State::Exact(change_id),
None => states.append(data_type, State::Exact(change_id)),
}
}
#[cfg(test)]
mod tests {
use super::PushBatch;
use common::ipc::{EmailPush, PushNotification};
use jmap_proto::types::state::State;
use types::{
id::Id,
type_state::{DataType, StateChange},
};
use utils::map::bitmap::Bitmap;
fn state_change<const N: usize>(change_id: u64, types: [DataType; N]) -> PushNotification {
PushNotification::StateChange(StateChange {
account_id: 1,
change_id,
types: Bitmap::from_iter(types),
})
}
fn email_push(email_id: u32) -> PushNotification {
PushNotification::EmailPush(EmailPush {
account_id: 1,
email_id,
change_id: email_id.into(),
})
}
#[test]
fn batch_keeps_newest_state_per_type() {
let mut batch = PushBatch::default();
batch.push(state_change(5, [DataType::Email]));
batch.push(state_change(7, [DataType::Email, DataType::Mailbox]));
batch.push(state_change(6, [DataType::Mailbox]));
let mut failed = PushBatch::default();
failed.push(state_change(
4,
[DataType::Email, DataType::Mailbox, DataType::Thread],
));
batch.merge_failed(failed);
let states = batch.state_changes.get(&Id::from(1u32)).unwrap();
assert_eq!(states.len(), 3);
assert_eq!(states.get(&DataType::Email), Some(&State::Exact(7)));
assert_eq!(states.get(&DataType::Mailbox), Some(&State::Exact(7)));
assert_eq!(states.get(&DataType::Thread), Some(&State::Exact(4)));
assert!(!batch.is_empty());
batch.clear();
assert!(batch.is_empty());
}
#[test]
fn failed_notifications_are_retried_first() {
let mut batch = PushBatch::default();
batch.push(email_push(3));
let mut failed = PushBatch::default();
failed.push(email_push(1));
failed.push(email_push(2));
batch.merge_failed(failed);
let email_ids = batch
.notifications
.iter()
.filter_map(|notification| match notification {
PushNotification::EmailPush(email_push) => Some(email_push.email_id),
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(email_ids, [1, 2, 3]);
assert!(batch.state_changes.is_empty());
}
}
crates/services/src/state_manager/push.rs
+157 -89
View File
@@ -8,13 +8,14 @@ use super::{
Event,
http::{build_push_client, http_request},
};
use crate::state_manager::PushRegistration;
use crate::state_manager::{PushBatch, PushRegistration};
use common::{
BuildServer, IPC_CHANNEL_BUFFER, Inner, LONG_1Y_SLUMBER, Server,
auth::BuildAccessToken,
ipc::{PushEvent, PushNotification},
};
use email::push::{PushSubscription, PushSubscriptions, Urgency};
use reqwest::Client;
use std::{
collections::hash_map::Entry,
sync::Arc,
@@ -36,7 +37,10 @@ pub fn spawn_push_manager(inner: Arc<Inner>) -> mpsc::Sender<Event> {
tokio::spawn(async move {
let mut push_servers: AHashMap<Id, PushRegistration> = AHashMap::default();
let mut account_push_ids: AHashMap<u32, AHashSet<Id>> = AHashMap::default();
let mut last_verify: AHashMap<u32, Instant> = AHashMap::default();
let mut last_verify: AHashMap<u32, (Instant, u32)> = AHashMap::default();
let mut pending_verify: AHashMap<u32, (Instant, Arc<PushSubscription>)> =
AHashMap::default();
let mut next_verify: Option<Instant> = None;
let mut last_retry = Instant::now();
let mut retry_timeout = LONG_1Y_SLUMBER;
let mut retry_ids = AHashSet::default();
@@ -90,10 +94,9 @@ pub fn spawn_push_manager(inner: Arc<Inner>) -> mpsc::Sender<Event> {
last_request: Instant::now()
- (server.core.jmap.push_throttle
+ Duration::from_millis(1)),
notifications: Vec::new(),
pending: PushBatch::default(),
server: subscription.clone(),
in_flight: false,
client: push_client.clone(),
},
);
}
@@ -129,8 +132,39 @@ pub fn spawn_push_manager(inner: Arc<Inner>) -> mpsc::Sender<Event> {
}
loop {
if let Some(verify_due) = next_verify {
let current_instant = Instant::now();
if verify_due <= current_instant {
let server = inner.build_server();
let push_timeout = server.core.jmap.push_timeout;
let current_time = now();
next_verify = None;
pending_verify.retain(|account_id, (verify_due, subscription)| {
if *verify_due > current_instant {
next_verify =
Some(next_verify.map_or(*verify_due, |next| next.min(*verify_due)));
true
} else {
if subscription.expires > current_time {
last_verify.insert(*account_id, (current_instant, subscription.id));
send_verification(
&push_client,
subscription.clone(),
&server,
push_timeout,
);
}
false
}
});
}
}
// Wait for the next event or timeout
let event_or_timeout = tokio::time::timeout(retry_timeout, push_rx.recv()).await;
let wait_timeout = next_verify.map_or(retry_timeout, |verify_due| {
retry_timeout.min(verify_due.saturating_duration_since(Instant::now()))
});
let event_or_timeout = tokio::time::timeout(wait_timeout, push_rx.recv()).await;
// Load settings
let server = inner.build_server();
@@ -193,10 +227,9 @@ pub fn spawn_push_manager(inner: Arc<Inner>) -> mpsc::Sender<Event> {
num_attempts: 0,
last_request: Instant::now()
- (push_throttle + Duration::from_millis(1)),
notifications: Vec::new(),
pending: PushBatch::default(),
server: subscription.clone(),
in_flight: false,
client: push_client.clone(),
});
}
}
@@ -213,52 +246,56 @@ pub fn spawn_push_manager(inner: Arc<Inner>) -> mpsc::Sender<Event> {
#[cfg(feature = "test_mode")]
if subscription.url.contains("skip_checks") {
last_verify.insert(
account_id,
current_time - (push_verify_timeout + Duration::from_millis(1)),
);
last_verify.remove(&account_id);
}
if last_verify
match last_verify
.get(&account_id)
.map(|last_verify| {
current_time - *last_verify > push_verify_timeout
.map(|(verified_at, verified_id)| {
(*verified_at + push_verify_timeout, *verified_id)
})
.unwrap_or(true)
.filter(|(verify_due, _)| *verify_due >= current_time)
{
let core = server.core.clone();
let push_client = push_client.clone();
tokio::spawn(async move {
http_request(
None => {
last_verify.retain(|_, (verified_at, _)| {
current_time.duration_since(*verified_at)
<= push_verify_timeout
});
last_verify.insert(account_id, (current_time, subscription.id));
pending_verify.remove(&account_id);
send_verification(
&push_client,
&subscription,
format!(
concat!(
"{{\"@type\":\"PushVerification\",",
"\"pushSubscriptionId\":\"{}\",",
"\"verificationCode\":\"{}\"}}"
),
Id::from(subscription.id),
subscription.verification_code
)
.into_bytes(),
subscription,
&server,
push_timeout,
core.jmap.vapid.as_ref(),
Urgency::Normal,
)
.await;
});
last_verify.insert(account_id, current_time);
} else {
trc::event!(
PushSubscription(PushSubscriptionEvent::Error),
Details = "Failed to verify push subscription",
Url = subscription.url.clone(),
AccountId = account_id,
Reason = "Too many requests"
);
);
}
Some((_, verified_id)) if verified_id == subscription.id => {
trc::event!(
PushSubscription(PushSubscriptionEvent::Error),
Details = "Failed to verify push subscription",
Url = subscription.url.clone(),
AccountId = account_id,
Reason = "Too many requests"
);
pending_verify.remove(&account_id);
}
Some((verify_due, _)) => {
trc::event!(
PushSubscription(PushSubscriptionEvent::Error),
Details = "Push subscription verification deferred",
Url = subscription.url.clone(),
AccountId = account_id,
Reason = "Too many requests"
);
next_verify = Some(
next_verify.map_or(verify_due, |next| next.min(verify_due)),
);
pending_verify.insert(account_id, (verify_due, subscription));
}
}
} else {
pending_verify.remove(&account_id);
}
// Update subscriptions
@@ -342,7 +379,7 @@ pub fn spawn_push_manager(inner: Arc<Inner>) -> mpsc::Sender<Event> {
);
}
subscription.notifications.push(notification);
subscription.pending.push(notification);
let last_request = subscription.last_request.elapsed();
if !subscription.in_flight
@@ -354,6 +391,7 @@ pub fn spawn_push_manager(inner: Arc<Inner>) -> mpsc::Sender<Event> {
{
subscription.send(
*id,
&push_client,
push_tx.clone(),
push_timeout,
server.clone(),
@@ -402,19 +440,25 @@ pub fn spawn_push_manager(inner: Arc<Inner>) -> mpsc::Sender<Event> {
Event::Reset => {
push_servers.clear();
account_push_ids.clear();
pending_verify.clear();
next_verify = None;
}
Event::DeliverySuccess { id } => {
if let Some(subscription) = push_servers.get_mut(&id) {
subscription.num_attempts = 0;
subscription.in_flight = false;
retry_ids.remove(&id);
if subscription.pending.is_empty() {
retry_ids.remove(&id);
} else {
retry_ids.insert(id);
}
}
}
Event::DeliveryFailure { id, notifications } => {
Event::DeliveryFailure { id, failed } => {
if let Some(subscription) = push_servers.get_mut(&id) {
subscription.last_request = Instant::now();
subscription.num_attempts += 1;
subscription.notifications.extend(notifications);
subscription.pending.merge_failed(failed);
subscription.in_flight = false;
retry_ids.insert(id);
}
@@ -430,52 +474,46 @@ pub fn spawn_push_manager(inner: Arc<Inner>) -> mpsc::Sender<Event> {
let last_retry_elapsed = last_retry.elapsed();
if last_retry_elapsed >= push_retry_interval {
let mut remove_ids = Vec::with_capacity(retry_ids.len());
retry_ids.retain(|retry_id| {
let Some(subscription) = push_servers.get_mut(retry_id) else {
return false;
};
let last_request = subscription.last_request.elapsed();
let is_due = !subscription.in_flight
&& ((subscription.num_attempts == 0 && last_request >= push_throttle)
|| (subscription.num_attempts > 0
&& last_request >= push_attempt_interval));
if !is_due {
return true;
}
for retry_id in &retry_ids {
if let Some(subscription) = push_servers.get_mut(retry_id) {
let last_request = subscription.last_request.elapsed();
if !subscription.in_flight
&& ((subscription.num_attempts == 0
&& last_request >= push_throttle)
|| (subscription.num_attempts > 0
&& last_request >= push_attempt_interval))
{
if subscription.num_attempts < push_attempts_max {
subscription.send(
*retry_id,
push_tx.clone(),
push_timeout,
server.clone(),
);
} else {
trc::event!(
PushSubscription(PushSubscriptionEvent::Error),
Details = "Failed to deliver push subscription",
Url = subscription.server.url.clone(),
Reason = "Too many failed attempts"
);
subscription.notifications.clear();
subscription.num_attempts = 0;
}
remove_ids.push(*retry_id);
}
if subscription.num_attempts < push_attempts_max {
subscription.send(
*retry_id,
&push_client,
push_tx.clone(),
push_timeout,
server.clone(),
);
} else {
remove_ids.push(*retry_id);
}
}
trc::event!(
PushSubscription(PushSubscriptionEvent::Error),
Details = "Failed to deliver push subscription",
Url = subscription.server.url.clone(),
Reason = "Too many failed attempts"
);
if remove_ids.len() < retry_ids.len() {
for remove_id in remove_ids {
retry_ids.remove(&remove_id);
subscription.pending.clear();
subscription.num_attempts = 0;
}
false
});
if retry_ids.is_empty() {
LONG_1Y_SLUMBER
} else {
last_retry = Instant::now();
push_retry_interval
} else {
retry_ids.clear();
LONG_1Y_SLUMBER
}
} else {
push_retry_interval - last_retry_elapsed
@@ -489,6 +527,36 @@ pub fn spawn_push_manager(inner: Arc<Inner>) -> mpsc::Sender<Event> {
push_tx_
}
fn send_verification(
push_client: &Client,
subscription: Arc<PushSubscription>,
server: &Server,
push_timeout: Duration,
) {
let core = server.core.clone();
let push_client = push_client.clone();
tokio::spawn(async move {
http_request(
&push_client,
&subscription,
format!(
concat!(
"{{\"@type\":\"PushVerification\",",
"\"pushSubscriptionId\":\"{}\",",
"\"verificationCode\":\"{}\"}}"
),
Id::from(subscription.id),
subscription.verification_code
)
.into_bytes(),
push_timeout,
core.jmap.vapid.as_ref(),
Urgency::Normal,
)
.await;
});
}
async fn load_push_subscriptions(
server: &Server,
account_id: u32,
crates/services/src/task_manager/spam_classifier.rs
+192 -162
View File
@@ -13,17 +13,18 @@ use common::{
use registry::{
schema::{
enums::TaskSpamFilterMaintenanceType,
prelude::ObjectType,
prelude::{Object, ObjectInner, ObjectType},
structs::{
HttpLookup, MemoryLookupKey, SpamDnsblServer, SpamFileExtension, SpamRule, SpamTag,
TaskSpamFilterMaintenance,
},
},
types::EnumImpl,
types::{EnumImpl, ObjectImpl},
};
use spam_filter::modules::classifier::SpamClassifier;
use std::time::{Duration, Instant};
use store::{
RegistryStore,
ahash::AHashMap,
registry::write::{RegistryWrite, RegistryWriteResult},
};
@@ -97,13 +98,67 @@ struct Rules {
file_exts: Vec<SpamFileExtension>,
}
#[derive(Default)]
trait UpstreamObject: ObjectImpl + PartialEq + From<Object> + Into<ObjectInner> {
fn replacement_for(self, _local: &Self) -> Option<Self> {
Some(self)
}
}
impl UpstreamObject for SpamRule {
fn replacement_for(mut self, local: &Self) -> Option<Self> {
self.set_enable(local.enable());
Some(self)
}
}
impl UpstreamObject for SpamDnsblServer {
fn replacement_for(mut self, local: &Self) -> Option<Self> {
self.set_enable(local.enable());
Some(self)
}
}
impl UpstreamObject for HttpLookup {
fn replacement_for(mut self, local: &Self) -> Option<Self> {
self.enable = local.enable;
Some(self)
}
}
impl UpstreamObject for SpamTag {
fn replacement_for(self, _local: &Self) -> Option<Self> {
None
}
}
impl UpstreamObject for MemoryLookupKey {}
impl UpstreamObject for SpamFileExtension {}
struct RuleUpdateResult {
success: usize,
already_exists: usize,
object_type: ObjectType,
added: usize,
updated: usize,
unchanged: usize,
failed: usize,
}
impl RuleUpdateResult {
fn new(object_type: ObjectType) -> Self {
RuleUpdateResult {
object_type,
added: 0,
updated: 0,
unchanged: 0,
failed: 0,
}
}
fn has_changes(&self) -> bool {
self.added + self.updated > 0
}
}
async fn update_spam_rules(server: &Server) -> trc::Result<TaskResult> {
let started = Instant::now();
let rules = match fetch_spam_rules(server).await {
@@ -118,170 +173,52 @@ async fn update_spam_rules(server: &Server) -> trc::Result<TaskResult> {
};
let registry = server.registry();
let mut stats: AHashMap<ObjectType, RuleUpdateResult> = AHashMap::new();
let settings = [
apply_upstream(registry, rules.rules).await?,
apply_upstream(registry, rules.dnsbls).await?,
apply_upstream(registry, rules.tags).await?,
apply_upstream(registry, rules.file_exts).await?,
];
let lookups = [
apply_upstream(registry, rules.http_lookups).await?,
apply_upstream(registry, rules.key_lookups).await?,
];
let mut reload_settings = false;
let mut reload_lookups = false;
for rule in rules.rules {
match registry.write(RegistryWrite::insert(&rule.into())).await? {
RegistryWriteResult::Success(_) => {
stats.entry(ObjectType::SpamRule).or_default().success += 1;
reload_settings = true;
}
RegistryWriteResult::PrimaryKeyConflict { .. } => {
stats
.entry(ObjectType::SpamRule)
.or_default()
.already_exists += 1;
}
_ => {
stats.entry(ObjectType::SpamRule).or_default().failed += 1;
}
let mut reload_errors = Vec::new();
for object in [
settings
.iter()
.any(RuleUpdateResult::has_changes)
.then_some(ObjectType::SpamRule),
lookups
.iter()
.any(RuleUpdateResult::has_changes)
.then_some(ObjectType::MemoryLookupKey),
]
.into_iter()
.flatten()
{
if let Err(reason) = reload_and_broadcast(server, object).await {
reload_errors.push(reason);
}
}
for dnsbl in rules.dnsbls {
match registry.write(RegistryWrite::insert(&dnsbl.into())).await? {
RegistryWriteResult::Success(_) => {
stats
.entry(ObjectType::SpamDnsblServer)
.or_default()
.success += 1;
reload_settings = true;
}
RegistryWriteResult::PrimaryKeyConflict { .. } => {
stats
.entry(ObjectType::SpamDnsblServer)
.or_default()
.already_exists += 1;
}
_ => {
stats.entry(ObjectType::SpamDnsblServer).or_default().failed += 1;
}
}
}
for tag in rules.tags {
match registry.write(RegistryWrite::insert(&tag.into())).await? {
RegistryWriteResult::Success(_) => {
stats.entry(ObjectType::SpamTag).or_default().success += 1;
reload_settings = true;
}
RegistryWriteResult::PrimaryKeyConflict { .. } => {
stats.entry(ObjectType::SpamTag).or_default().already_exists += 1;
}
_ => {
stats.entry(ObjectType::SpamTag).or_default().failed += 1;
}
}
}
for lookup in rules.http_lookups {
match registry
.write(RegistryWrite::insert(&lookup.into()))
.await?
{
RegistryWriteResult::Success(_) => {
stats.entry(ObjectType::HttpLookup).or_default().success += 1;
reload_lookups = true;
}
RegistryWriteResult::PrimaryKeyConflict { .. } => {
stats
.entry(ObjectType::HttpLookup)
.or_default()
.already_exists += 1;
}
_ => {
stats.entry(ObjectType::HttpLookup).or_default().failed += 1;
}
}
}
for key_lookup in rules.key_lookups {
match registry
.write(RegistryWrite::insert(&key_lookup.into()))
.await?
{
RegistryWriteResult::Success(_) => {
stats
.entry(ObjectType::MemoryLookupKey)
.or_default()
.success += 1;
reload_lookups = true;
}
RegistryWriteResult::PrimaryKeyConflict { .. } => {
stats
.entry(ObjectType::MemoryLookupKey)
.or_default()
.already_exists += 1;
}
_ => {
stats.entry(ObjectType::MemoryLookupKey).or_default().failed += 1;
}
}
}
for ext in rules.file_exts {
match registry.write(RegistryWrite::insert(&ext.into())).await? {
RegistryWriteResult::Success(_) => {
stats
.entry(ObjectType::SpamFileExtension)
.or_default()
.success += 1;
reload_settings = true;
}
RegistryWriteResult::PrimaryKeyConflict { .. } => {
stats
.entry(ObjectType::SpamFileExtension)
.or_default()
.already_exists += 1;
}
_ => {
stats
.entry(ObjectType::SpamFileExtension)
.or_default()
.failed += 1;
}
}
}
if reload_settings {
if let Err(err) =
Box::pin(server.reload_registry(RegistryChange::Reload(ObjectType::SpamRule))).await
{
trc::error!(err.details("Failed to reload registry after updating spam rules"));
}
server
.cluster_broadcast(BroadcastEvent::RegistryChange(RegistryChange::Reload(
ObjectType::SpamRule,
)))
.await;
}
if reload_lookups {
if let Err(err) =
Box::pin(server.reload_registry(RegistryChange::Reload(ObjectType::MemoryLookupKey)))
.await
{
trc::error!(err.details("Failed to reload registry after updating spam rules"));
}
server
.cluster_broadcast(BroadcastEvent::RegistryChange(RegistryChange::Reload(
ObjectType::MemoryLookupKey,
)))
.await;
}
let failed: usize = settings
.iter()
.chain(&lookups)
.map(|result| result.failed)
.sum();
trc::event!(
Spam(SpamEvent::RulesUpdated),
Details = stats
Details = settings
.into_iter()
.map(|(object_type, result)| {
.chain(lookups)
.map(|result| {
Value::Array(vec![
Value::String(object_type.as_str().into()),
Value::from(result.success),
Value::from(result.already_exists),
Value::String(result.object_type.as_str().into()),
Value::from(result.added),
Value::from(result.updated),
Value::from(result.unchanged),
Value::from(result.failed),
])
})
@@ -289,7 +226,100 @@ async fn update_spam_rules(server: &Server) -> trc::Result<TaskResult> {
Elapsed = started.elapsed(),
);
Ok(TaskResult::Success(vec![]))
if !reload_errors.is_empty() {
Ok(TaskResult::permanent(format!(
"Spam rules were stored but not activated ({}); fix the logged errors and run Reload settings",
reload_errors.join("; ")
)))
} else if failed > 0 {
Ok(TaskResult::permanent(format!(
"{failed} spam filter objects failed to import or update"
)))
} else {
Ok(TaskResult::Success(vec![]))
}
}
async fn reload_and_broadcast(server: &Server, object: ObjectType) -> Result<(), String> {
match Box::pin(server.reload_registry(RegistryChange::Reload(object))).await {
Ok(result) => {
result.log();
if result.has_errors() {
return Err(format!("{} configuration errors", result.errors.len()));
}
server
.cluster_broadcast(BroadcastEvent::RegistryChange(RegistryChange::Reload(
object,
)))
.await;
Ok(())
}
Err(err) => {
let reason = err.to_string();
trc::error!(err.details("Failed to reload registry after updating spam rules"));
Err(reason)
}
}
}
async fn apply_upstream<T: UpstreamObject>(
registry: &RegistryStore,
objects: Vec<T>,
) -> trc::Result<RuleUpdateResult> {
let mut result = RuleUpdateResult::new(T::OBJECT);
for upstream in objects {
let upstream = Object::from(upstream);
let existing_id = match registry.write(RegistryWrite::insert(&upstream)).await? {
RegistryWriteResult::Success(_) => {
result.added += 1;
continue;
}
RegistryWriteResult::PrimaryKeyConflict { existing_id, .. }
if existing_id.object() == T::OBJECT =>
{
existing_id
}
RegistryWriteResult::PrimaryKeyConflict { .. } => {
result.unchanged += 1;
continue;
}
_ => {
result.failed += 1;
continue;
}
};
let Some(local) = registry.get(existing_id).await? else {
result.failed += 1;
continue;
};
let revision = local.revision;
let local = T::from(local);
let Some(replacement) = T::from(upstream)
.replacement_for(&local)
.filter(|replacement| replacement != &local)
else {
result.unchanged += 1;
continue;
};
let replacement = Object::from(replacement);
let local = Object::with_revision(local.into(), revision);
match registry
.write(RegistryWrite::update(
existing_id.id(),
&replacement,
&local,
))
.await?
{
RegistryWriteResult::Success(_) => result.updated += 1,
_ => result.failed += 1,
}
}
Ok(result)
}
async fn fetch_spam_rules(server: &Server) -> Result<Rules, RuleUpdateError> {