Merge main (upstream v0.16.24) into feature/compliance-roles
The schema conflicted as a binary file: taken from main and the one edit here re-applied (sysComplianceGet after sysLegalHoldExport). The import kept the permission count at 673, so the new id stays 673. Retested on the merged tree in its own target directory: the compliance and system suites pass. One earlier system run failed in purge.rs (an imported blob not found) and didn't recur.
This commit is contained in:
@@ -16,6 +16,9 @@ pub mod per_domain; // inbuxa: per-domain directories
|
||||
pub mod sql;
|
||||
pub mod synchronization;
|
||||
pub mod unavailable;
|
||||
// inbuxa: upstream's issuer.rs (since v0.16.23) is left out. It tests routing a
|
||||
// token that names no address by its issuer, which upstream ships in
|
||||
// Enterprise; here such a token gets the server's default directory (DIR-2).
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
pub async fn directory_tests() {
|
||||
|
||||
@@ -129,6 +129,21 @@ pub async fn test(test: &TestServer) {
|
||||
assert_eq!(samples.iter().filter(|x| !x.1.is_spam).count(), 11);
|
||||
assert_eq!(samples.iter().filter(|x| x.1.is_spam).count(), 11);
|
||||
|
||||
let last_id = samples.iter().map(|(id, _)| id.id()).max().unwrap();
|
||||
for _ in 0..2 {
|
||||
admin
|
||||
.registry_create_object(Task::SpamFilterMaintenance(TaskSpamFilterMaintenance {
|
||||
maintenance_type: TaskSpamFilterMaintenanceType::Train,
|
||||
status: TaskStatus::now(),
|
||||
}))
|
||||
.await;
|
||||
test.wait_for_tasks().await;
|
||||
let model = spam_classifier_model(&test.server).await;
|
||||
assert_eq!(model.reservoir.ham.total_seen, 11);
|
||||
assert_eq!(model.reservoir.spam.total_seen, 11);
|
||||
assert_eq!(model.last_id, last_id);
|
||||
}
|
||||
|
||||
// Global spam samples should not appear in the account
|
||||
let samples = account.spam_training_samples().await;
|
||||
assert_eq!(samples.iter().filter(|x| !x.1.is_spam).count(), 11);
|
||||
|
||||
@@ -174,4 +174,53 @@ pub async fn test(imap: &mut ImapConnection, imap_check: &mut ImapConnection) {
|
||||
imap.assert_read(Type::Tagged, ResponseType::Ok)
|
||||
.await
|
||||
.assert_contains("COPYUID");
|
||||
|
||||
move_store_race(imap, imap_check).await;
|
||||
}
|
||||
|
||||
async fn move_store_race(imap: &mut ImapConnection, imap_check: &mut ImapConnection) {
|
||||
const RACE_MESSAGES: usize = 10;
|
||||
const RACE_ROUNDS: usize = 20;
|
||||
|
||||
// A MOVE that loses a race with a STORE on the same messages in another
|
||||
// session is retried instead of failing with CONTACTADMIN
|
||||
imap.send_ok("CREATE \"Race Left\"").await;
|
||||
imap.send_ok("CREATE \"Race Right\"").await;
|
||||
for i in 0..RACE_MESSAGES {
|
||||
imap.append(
|
||||
"Race Left",
|
||||
&format!("From: [email protected]\r\nSubject: Move race {i}\r\n\r\nrace\r\n"),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
for round in 0..RACE_ROUNDS {
|
||||
let (src, dest, store) = if round % 2 == 0 {
|
||||
("Race Left", "Race Right", "UID STORE 1:* +FLAGS (\\Seen)")
|
||||
} else {
|
||||
("Race Right", "Race Left", "UID STORE 1:* -FLAGS (\\Seen)")
|
||||
};
|
||||
imap.send_ok(&format!("SELECT \"{src}\"")).await;
|
||||
imap_check.send_ok(&format!("SELECT \"{src}\"")).await;
|
||||
|
||||
// The STORE may lose the race instead, so only the MOVE is checked
|
||||
imap.send(&format!("UID MOVE 1:* \"{dest}\"")).await;
|
||||
imap_check.send(store).await;
|
||||
let (moved, _) = tokio::join!(
|
||||
imap.assert_read(Type::Tagged, ResponseType::Ok),
|
||||
imap_check.read(Type::Tagged)
|
||||
);
|
||||
moved.assert_contains("COPYUID");
|
||||
|
||||
imap.send(&format!("STATUS \"{dest}\" (MESSAGES)")).await;
|
||||
imap.assert_read(Type::Tagged, ResponseType::Ok)
|
||||
.await
|
||||
.assert_contains(&format!("(MESSAGES {RACE_MESSAGES})"));
|
||||
}
|
||||
|
||||
// Restore the state the following tests expect
|
||||
imap.send_ok("SELECT \"Burrata al Tartufo\"").await;
|
||||
imap_check.send_ok("SELECT \"Burrata al Tartufo\"").await;
|
||||
imap.send_ok("DELETE \"Race Left\"").await;
|
||||
imap.send_ok("DELETE \"Race Right\"").await;
|
||||
}
|
||||
|
||||
@@ -367,6 +367,82 @@ pub async fn test(test: &TestServer) {
|
||||
}
|
||||
test.blob_expire_all().await;
|
||||
|
||||
let inbox_id = Id::from(INBOX_ID).to_string();
|
||||
let response = account
|
||||
.jmap_method_calls(json!([
|
||||
[
|
||||
"Blob/upload",
|
||||
{
|
||||
"accountId": account.id_string(),
|
||||
"create": {
|
||||
"m0": {
|
||||
"data": [
|
||||
{
|
||||
"data:asText": concat!(
|
||||
"From: [email protected]\r\n",
|
||||
"To: [email protected]\r\n",
|
||||
"Subject: Blob reference import\r\n",
|
||||
"\r\n",
|
||||
"Imported through a Blob/upload creation id."
|
||||
)
|
||||
}
|
||||
],
|
||||
"type": "message/rfc822"
|
||||
}
|
||||
}
|
||||
},
|
||||
"U0"
|
||||
],
|
||||
[
|
||||
"Email/import",
|
||||
{
|
||||
"accountId": account.id_string(),
|
||||
"emails": {
|
||||
"i0": {
|
||||
"blobId": "#m0",
|
||||
"mailboxIds": { (inbox_id.as_str()): true }
|
||||
}
|
||||
}
|
||||
},
|
||||
"I0"
|
||||
],
|
||||
[
|
||||
"Email/import",
|
||||
{
|
||||
"accountId": account.id_string(),
|
||||
"emails": {
|
||||
"i1": {
|
||||
"blobId": "#missing",
|
||||
"mailboxIds": { (inbox_id.as_str()): true }
|
||||
}
|
||||
}
|
||||
},
|
||||
"I1"
|
||||
]
|
||||
]))
|
||||
.await;
|
||||
|
||||
assert_eq!(response.name_at(1), "Email/import", "{response:?}");
|
||||
assert!(
|
||||
response
|
||||
.pointer("/methodResponses/1/1/created/i0/id")
|
||||
.and_then(|v| v.as_str())
|
||||
.is_some(),
|
||||
"{response:?}"
|
||||
);
|
||||
assert!(
|
||||
response
|
||||
.pointer("/methodResponses/1/1/notCreated")
|
||||
.is_none(),
|
||||
"{response:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
response.error_type_at(2),
|
||||
Some("invalidResultReference"),
|
||||
"{response:?}"
|
||||
);
|
||||
test.blob_expire_all().await;
|
||||
|
||||
// Blob/lookup
|
||||
let client = account.jmap_client().await;
|
||||
let blob_id = client
|
||||
|
||||
@@ -188,13 +188,37 @@ pub async fn test(test: &TestServer) {
|
||||
client.push_subscription_destroy(&push_id).await.unwrap();
|
||||
|
||||
// Only one verification per minute is allowed
|
||||
let push_id = client
|
||||
.push_subscription_create("invalid", "https://127.0.0.1:19000/push", None)
|
||||
let first_id = client
|
||||
.push_subscription_create(
|
||||
"first",
|
||||
"https://127.0.0.1:19000/push?skip_checks=true",
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.take_id();
|
||||
let verification = expect_push(&mut event_rx).await.unwrap_verification();
|
||||
assert_eq!(verification.push_subscription_id, first_id);
|
||||
let deferred_id = client
|
||||
.push_subscription_create("deferred", "https://127.0.0.1:19000/push", None)
|
||||
.await
|
||||
.unwrap()
|
||||
.take_id();
|
||||
expect_nothing(&mut event_rx).await;
|
||||
client.push_subscription_destroy(&push_id).await.unwrap();
|
||||
let verification = expect_push_within(&mut event_rx, Duration::from_secs(8))
|
||||
.await
|
||||
.unwrap_verification();
|
||||
assert_eq!(verification.push_subscription_id, deferred_id);
|
||||
account
|
||||
.jmap_request(
|
||||
&["urn:ietf:params:jmap:core"],
|
||||
json!([[
|
||||
"PushSubscription/set",
|
||||
{ "destroy": [first_id, deferred_id] },
|
||||
"0"
|
||||
]]),
|
||||
)
|
||||
.await;
|
||||
|
||||
// Register push notification (with encryption)
|
||||
let push_id = client
|
||||
@@ -731,7 +755,14 @@ fn assert_vapid_authorization(header: &str, expected_key: &str, expected_origin:
|
||||
}
|
||||
|
||||
async fn expect_push(event_rx: &mut mpsc::Receiver<PushMessage>) -> PushMessage {
|
||||
match tokio::time::timeout(Duration::from_millis(1500), event_rx.recv()).await {
|
||||
expect_push_within(event_rx, Duration::from_millis(1500)).await
|
||||
}
|
||||
|
||||
async fn expect_push_within(
|
||||
event_rx: &mut mpsc::Receiver<PushMessage>,
|
||||
wait: Duration,
|
||||
) -> PushMessage {
|
||||
match tokio::time::timeout(wait, event_rx.recv()).await {
|
||||
Ok(Some(push)) => {
|
||||
//println!("Push received: {:?}", push);
|
||||
push
|
||||
|
||||
@@ -8,7 +8,7 @@ use crate::{
|
||||
jmap::{find_values, replace_blob_ids, replace_boundaries, replace_values},
|
||||
utils::server::TestServer,
|
||||
};
|
||||
use ::email::mailbox::INBOX_ID;
|
||||
use ::email::{mailbox::INBOX_ID, message::metadata::MessageData};
|
||||
use ahash::AHashSet;
|
||||
use jmap_client::{
|
||||
Error, Set,
|
||||
@@ -18,8 +18,12 @@ use jmap_client::{
|
||||
mailbox::Role,
|
||||
};
|
||||
use registry::schema::prelude::ObjectType;
|
||||
use std::{fs, path::PathBuf};
|
||||
use types::id::Id;
|
||||
use std::{fs, path::PathBuf, str::FromStr};
|
||||
use store::{
|
||||
ValueKey,
|
||||
write::{AlignedBytes, Archive},
|
||||
};
|
||||
use types::{collection::Collection, id::Id};
|
||||
|
||||
pub async fn test(test: &TestServer) {
|
||||
println!("Running Email Set tests...");
|
||||
@@ -29,6 +33,7 @@ pub async fn test(test: &TestServer) {
|
||||
|
||||
create(&client, &mailbox_id).await;
|
||||
update(&client, &mailbox_id).await;
|
||||
update_preserves_uids(test, &client, account.id().document_id(), &mailbox_id).await;
|
||||
|
||||
test.destroy_all_mailboxes(account).await;
|
||||
test.account("[email protected]")
|
||||
@@ -296,6 +301,94 @@ async fn update(client: &Client, root_mailbox_id: &str) {
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn update_preserves_uids(
|
||||
test: &TestServer,
|
||||
client: &Client,
|
||||
account_id: u32,
|
||||
root_mailbox_id: &str,
|
||||
) {
|
||||
let email_id = client
|
||||
.email_query(
|
||||
email::query::Filter::in_mailbox(root_mailbox_id).into(),
|
||||
None::<Vec<_>>,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.take_ids()
|
||||
.pop()
|
||||
.unwrap();
|
||||
let document_id = Id::from_str(&email_id).unwrap().document_id();
|
||||
let test_mailbox_id = client
|
||||
.mailbox_create("UID Test", None::<String>, Role::None)
|
||||
.await
|
||||
.unwrap()
|
||||
.take_id();
|
||||
let test_mailbox_document_id = Id::from_str(&test_mailbox_id).unwrap().document_id();
|
||||
let uids = message_uids(test, account_id, document_id).await;
|
||||
let inbox_uid = uids[&INBOX_ID];
|
||||
assert_ne!(inbox_uid, 0);
|
||||
|
||||
// Full mailboxIds identical to the current ones plus a keyword change must keep the UID
|
||||
let mut request = client.build();
|
||||
request
|
||||
.set_email()
|
||||
.update(&email_id)
|
||||
.mailbox_ids([root_mailbox_id])
|
||||
.keywords(["uid-test"]);
|
||||
request
|
||||
.send_set_email()
|
||||
.await
|
||||
.unwrap()
|
||||
.updated(&email_id)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
message_uids(test, account_id, document_id).await,
|
||||
[(INBOX_ID, inbox_uid)].into_iter().collect()
|
||||
);
|
||||
|
||||
// Full mailboxIds that keeps a mailbox and adds another must only assign a UID to the new one
|
||||
let mut request = client.build();
|
||||
request
|
||||
.set_email()
|
||||
.update(&email_id)
|
||||
.mailbox_ids([root_mailbox_id, test_mailbox_id.as_str()]);
|
||||
request
|
||||
.send_set_email()
|
||||
.await
|
||||
.unwrap()
|
||||
.updated(&email_id)
|
||||
.unwrap();
|
||||
let uids = message_uids(test, account_id, document_id).await;
|
||||
assert_eq!(uids.len(), 2);
|
||||
assert_eq!(uids[&INBOX_ID], inbox_uid);
|
||||
assert_ne!(uids[&test_mailbox_document_id], 0);
|
||||
|
||||
client.mailbox_destroy(&test_mailbox_id, true).await.unwrap();
|
||||
}
|
||||
|
||||
async fn message_uids(
|
||||
test: &TestServer,
|
||||
account_id: u32,
|
||||
document_id: u32,
|
||||
) -> std::collections::BTreeMap<u32, u32> {
|
||||
test.server
|
||||
.store()
|
||||
.get_value::<Archive<AlignedBytes>>(ValueKey::archive(
|
||||
account_id,
|
||||
Collection::Email,
|
||||
document_id,
|
||||
))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.deserialize::<MessageData>()
|
||||
.unwrap()
|
||||
.mailboxes
|
||||
.iter()
|
||||
.map(|m| (m.mailbox_id, m.uid))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub async fn assert_email_properties(
|
||||
client: &Client,
|
||||
message_id: &str,
|
||||
|
||||
@@ -99,6 +99,7 @@ pub async fn jmap_tests() {
|
||||
push_throttle: 500u64.into(),
|
||||
websocket_throttle: 500u64.into(),
|
||||
push_attempt_wait: 500u64.into(),
|
||||
push_verify_timeout: 5000u64.into(),
|
||||
..Default::default()
|
||||
},
|
||||
&[
|
||||
@@ -108,6 +109,7 @@ pub async fn jmap_tests() {
|
||||
Property::PushThrottle,
|
||||
Property::WebsocketThrottle,
|
||||
Property::PushAttemptWait,
|
||||
Property::PushVerifyTimeout,
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -254,7 +254,7 @@ pub async fn test(test: &TestServer) {
|
||||
// inbuxa: MT-22, the logo that applies to the account, and
|
||||
// LP-19, whether the legacy protocols are open to it, and
|
||||
// ai-explain EX-1, whether Explain can be offered
|
||||
"urn:inbuxa:jmap": { "logo": null, "legacyProtocols": "enabled", "aiExplain": false },
|
||||
"urn:inbuxa:jmap": { "logo": null, "legacyProtocols": "enabled", "legacyAllowed": ["imap", "pop3", "manageSieve", "submission"], "aiExplain": false },
|
||||
"https://www.fastmail.com/dev/maskedemail": {}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,9 +28,12 @@ use registry::{
|
||||
schema::{
|
||||
enums::{self, MtaStage},
|
||||
prelude::{ObjectType, Property},
|
||||
structs::{Expression, MtaHook, MtaMilter, MtaStageRcpt},
|
||||
structs::{
|
||||
Expression, ExpressionMatch, MtaHook, MtaMilter, MtaStageData, MtaStageRcpt,
|
||||
SieveSystemScript,
|
||||
},
|
||||
},
|
||||
types::map::Map,
|
||||
types::{list::List, map::Map},
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use smtp::{
|
||||
@@ -238,6 +241,34 @@ async fn mta_hook_session() {
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
admin
|
||||
.registry_create_object(MtaStageData {
|
||||
script: Expression {
|
||||
match_: List::from_iter([ExpressionMatch {
|
||||
if_: "sender = '[email protected]'".into(),
|
||||
then: "'tag_quarantine'".into(),
|
||||
}]),
|
||||
else_: "false".into(),
|
||||
},
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
admin
|
||||
.registry_create_object(SieveSystemScript {
|
||||
contents: concat!(
|
||||
"require [\"editheader\"];\n",
|
||||
"addheader \"X-Sieve\" \"Seen\";\n",
|
||||
"if exists \"X-Quarantine\" {\n",
|
||||
" deleteheader \"Subject\";\n",
|
||||
" addheader \"Subject\" \"INFECTED\";\n",
|
||||
"}\n"
|
||||
)
|
||||
.into(),
|
||||
description: None,
|
||||
is_active: true,
|
||||
name: "tag_quarantine".into(),
|
||||
})
|
||||
.await;
|
||||
admin.reload_settings().await;
|
||||
test.reload_core();
|
||||
test.expect_reload_settings().await;
|
||||
@@ -355,6 +386,41 @@ async fn mta_hook_session() {
|
||||
.await
|
||||
.assert_contains("X-Spam: Yes")
|
||||
.assert_contains("123456");
|
||||
|
||||
// Test quarantine
|
||||
session
|
||||
.send_message(
|
||||
"[email protected]",
|
||||
&["[email protected]"],
|
||||
"test:no_dkim",
|
||||
"250 2.0.0",
|
||||
)
|
||||
.await;
|
||||
test.expect_message()
|
||||
.await
|
||||
.read_lines(&test)
|
||||
.await
|
||||
.assert_contains("X-Quarantine: true")
|
||||
.assert_contains("Subject: Is dinner ready?")
|
||||
.assert_contains("Are you hungry yet?");
|
||||
|
||||
// Test that a DATA stage Sieve script sees and preserves hook modifications
|
||||
session
|
||||
.send_message(
|
||||
"[email protected]",
|
||||
&["[email protected]"],
|
||||
"test:no_dkim",
|
||||
"250 2.0.0",
|
||||
)
|
||||
.await;
|
||||
test.expect_message()
|
||||
.await
|
||||
.read_lines(&test)
|
||||
.await
|
||||
.assert_contains("X-Quarantine: true")
|
||||
.assert_contains("X-Sieve: Seen")
|
||||
.assert_contains("Subject: INFECTED")
|
||||
.assert_contains("Are you hungry yet?");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -865,6 +931,11 @@ fn handle_mta_hook(request: Request, tests: Arc<Vec<HeaderTest>>) -> hooks::Resp
|
||||
response: None,
|
||||
modifications: vec![],
|
||||
},
|
||||
"quarantine" | "quarantine_only" => hooks::Response {
|
||||
action: hooks::Action::Quarantine,
|
||||
response: None,
|
||||
modifications: vec![],
|
||||
},
|
||||
"temp_fail" => hooks::Response {
|
||||
action: hooks::Action::Reject,
|
||||
response: SmtpResponse {
|
||||
|
||||
@@ -36,6 +36,8 @@ pub mod rcpt;
|
||||
pub mod rewrite;
|
||||
pub mod scripts;
|
||||
pub mod sign;
|
||||
pub mod spam_rules;
|
||||
pub mod spam_rules_kept; // inbuxa: spam rules updates keep admin edits
|
||||
pub mod throttle;
|
||||
pub mod vrfy;
|
||||
|
||||
|
||||
@@ -0,0 +1,364 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use crate::utils::server::{TestServer, TestServerBuilder};
|
||||
use registry::{
|
||||
schema::{
|
||||
enums::TaskSpamFilterMaintenanceType,
|
||||
prelude::{Object, ObjectType},
|
||||
structs::{
|
||||
HttpLookup, MemoryLookupKey, MemoryLookupKeyValue, SpamDnsblServer, SpamFileExtension,
|
||||
SpamRule, SpamSettings, SpamTag, SpamTagScore, Task, TaskSpamFilterMaintenance,
|
||||
TaskStatus,
|
||||
},
|
||||
},
|
||||
types::{ObjectImpl, float::Float},
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use std::fs;
|
||||
use store::registry::RegistryObject;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn spam_rules_update() {
|
||||
let test = TestServerBuilder::new("spam_rules_update_test")
|
||||
.await
|
||||
.with_http_listener(19057)
|
||||
.await
|
||||
.build()
|
||||
.await;
|
||||
let admin = test.account("admin");
|
||||
let rules_path = test.temp_dir.path.join("spam-filter-rules.json");
|
||||
admin
|
||||
.registry_create_object(SpamSettings {
|
||||
spam_filter_rules_url: format!("file://{}", rules_path.display()).into(),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
admin
|
||||
.registry_create_object(MemoryLookupKeyValue {
|
||||
namespace: "test-keys".into(),
|
||||
key: "conflict.org".into(),
|
||||
value: "local".into(),
|
||||
is_glob_pattern: false,
|
||||
})
|
||||
.await;
|
||||
admin.reload_settings().await;
|
||||
|
||||
update_rules(&test, &release_1()).await;
|
||||
|
||||
for name in ["STWT_TEST_RULE", "STWT_TEST_DISABLED"] {
|
||||
assert!(
|
||||
stored::<SpamRule>(&test, "name", name)
|
||||
.await
|
||||
.unwrap()
|
||||
.object
|
||||
.enable()
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
stored::<SpamDnsblServer>(&test, "name", "STWT_TEST_DNSBL")
|
||||
.await
|
||||
.unwrap()
|
||||
.object
|
||||
.enable()
|
||||
);
|
||||
assert!(
|
||||
stored::<MemoryLookupKey>(&test, "key", "conflict.org")
|
||||
.await
|
||||
.is_none()
|
||||
);
|
||||
|
||||
let disabled_rule = stored::<SpamRule>(&test, "name", "STWT_TEST_DISABLED")
|
||||
.await
|
||||
.unwrap();
|
||||
admin
|
||||
.registry_update_object(
|
||||
ObjectType::SpamRule,
|
||||
disabled_rule.id.id(),
|
||||
json!({"enable": false}),
|
||||
)
|
||||
.await;
|
||||
let local_tag = stored::<SpamTag>(&test, "tag", "TEST_TAG_LOCAL")
|
||||
.await
|
||||
.unwrap();
|
||||
admin
|
||||
.registry_update_object(
|
||||
ObjectType::SpamTag,
|
||||
local_tag.id.id(),
|
||||
json!({"score": 2.0}),
|
||||
)
|
||||
.await;
|
||||
let feed = stored::<HttpLookup>(&test, "namespace", "stwt_test_feed")
|
||||
.await
|
||||
.unwrap();
|
||||
admin
|
||||
.registry_update_object(
|
||||
ObjectType::HttpLookup,
|
||||
feed.id.id(),
|
||||
json!({"enable": false}),
|
||||
)
|
||||
.await;
|
||||
|
||||
let release_2 = release_2();
|
||||
for _ in 0..2 {
|
||||
update_rules(&test, &release_2).await;
|
||||
|
||||
let rule = stored::<SpamRule>(&test, "name", "STWT_TEST_RULE")
|
||||
.await
|
||||
.unwrap()
|
||||
.object;
|
||||
assert!(rule.enable());
|
||||
assert_eq!(object_json(&rule)["priority"], 400);
|
||||
|
||||
let rule = stored::<SpamRule>(&test, "name", "STWT_TEST_DISABLED")
|
||||
.await
|
||||
.unwrap()
|
||||
.object;
|
||||
assert!(!rule.enable());
|
||||
assert_eq!(object_json(&rule)["priority"], 450);
|
||||
|
||||
assert!(
|
||||
stored::<SpamRule>(&test, "name", "STWT_TEST_NEW")
|
||||
.await
|
||||
.is_some()
|
||||
);
|
||||
|
||||
for upstream in release_2["SpamRule"].as_array().unwrap() {
|
||||
let mut upstream: SpamRule = serde_json::from_value(upstream.clone()).unwrap();
|
||||
let local = stored::<SpamRule>(
|
||||
&test,
|
||||
"name",
|
||||
object_json(&upstream)["name"].as_str().unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.object;
|
||||
upstream.set_enable(local.enable());
|
||||
assert_eq!(local, upstream);
|
||||
}
|
||||
for upstream in release_2["SpamDnsblServer"].as_array().unwrap() {
|
||||
let upstream: SpamDnsblServer = serde_json::from_value(upstream.clone()).unwrap();
|
||||
let local = stored::<SpamDnsblServer>(
|
||||
&test,
|
||||
"name",
|
||||
object_json(&upstream)["name"].as_str().unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.object;
|
||||
assert_eq!(local, upstream);
|
||||
}
|
||||
|
||||
for (tag, score) in [("TEST_TAG", 6.5), ("TEST_TAG_LOCAL", 2.0)] {
|
||||
assert_eq!(
|
||||
stored::<SpamTag>(&test, "tag", tag).await.unwrap().object,
|
||||
SpamTag::Score(SpamTagScore {
|
||||
tag: tag.into(),
|
||||
score: Float::new(score),
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
let feed = stored::<HttpLookup>(&test, "namespace", "stwt_test_feed")
|
||||
.await
|
||||
.unwrap()
|
||||
.object;
|
||||
assert!(!feed.enable);
|
||||
assert_eq!(feed.max_entries, 2000);
|
||||
|
||||
assert!(
|
||||
stored::<MemoryLookupKey>(&test, "key", "example.org")
|
||||
.await
|
||||
.unwrap()
|
||||
.object
|
||||
.is_glob_pattern
|
||||
);
|
||||
assert!(
|
||||
stored::<MemoryLookupKey>(&test, "key", "conflict.org")
|
||||
.await
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(
|
||||
stored::<MemoryLookupKeyValue>(&test, "key", "conflict.org")
|
||||
.await
|
||||
.unwrap()
|
||||
.object
|
||||
.value,
|
||||
"local"
|
||||
);
|
||||
|
||||
assert!(
|
||||
stored::<SpamFileExtension>(&test, "extension", "tst")
|
||||
.await
|
||||
.unwrap()
|
||||
.object
|
||||
.is_bad
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn update_rules(test: &TestServer, rules: &Value) {
|
||||
fs::write(
|
||||
test.temp_dir.path.join("spam-filter-rules.json"),
|
||||
rules.to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
test.account("admin")
|
||||
.registry_create_object(Task::SpamFilterMaintenance(TaskSpamFilterMaintenance {
|
||||
maintenance_type: TaskSpamFilterMaintenanceType::UpdateRules,
|
||||
status: TaskStatus::now(),
|
||||
}))
|
||||
.await;
|
||||
test.wait_for_tasks().await;
|
||||
}
|
||||
|
||||
async fn stored<T: ObjectImpl + From<Object>>(
|
||||
test: &TestServer,
|
||||
property: &str,
|
||||
value: &str,
|
||||
) -> Option<RegistryObject<T>> {
|
||||
test.server
|
||||
.registry()
|
||||
.list::<T>()
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find(|item| object_json(&item.object)[property] == value)
|
||||
}
|
||||
|
||||
fn object_json<T: ObjectImpl>(object: &T) -> Value {
|
||||
serde_json::to_value(object).unwrap()
|
||||
}
|
||||
|
||||
fn release_1() -> Value {
|
||||
json!({
|
||||
"SpamRule": [
|
||||
{
|
||||
"@type": "Any",
|
||||
"name": "STWT_TEST_RULE",
|
||||
"enable": true,
|
||||
"priority": 500,
|
||||
"condition": {"else": "false", "match": {"0": {"if": "$MISSING_ESSENTIAL_HEADERS && $SINGLE_SHORT_PART", "then": "'SHORT_PART_BAD_HEADERS'"}}}
|
||||
},
|
||||
{
|
||||
"@type": "Any",
|
||||
"name": "STWT_TEST_DISABLED",
|
||||
"enable": true,
|
||||
"priority": 500,
|
||||
"condition": {"else": "false", "match": {"0": {"if": "$MISSING_ESSENTIAL_HEADERS && $SINGLE_SHORT_PART", "then": "'SHORT_PART_BAD_HEADERS'"}}}
|
||||
}
|
||||
],
|
||||
"SpamDnsblServer": [
|
||||
{
|
||||
"@type": "Domain",
|
||||
"name": "STWT_TEST_DNSBL",
|
||||
"enable": true,
|
||||
"tag": {"else": "false", "match": {
|
||||
"0": {"if": "octets[3] == 1", "then": "'URIBL_BLOCKED'"},
|
||||
"1": {"if": "octets[3] == 2", "then": "'URIBL_BLACK'"},
|
||||
"2": {"if": "octets[3] == 4", "then": "'URIBL_GREY'"},
|
||||
"3": {"if": "octets[3] == 8", "then": "'URIBL_RED'"}
|
||||
}},
|
||||
"zone": {"else": "value + '.multi.uribl.com'", "match": {}}
|
||||
}
|
||||
],
|
||||
"SpamTag": [
|
||||
{"@type": "Score", "tag": "TEST_TAG", "score": 6.5},
|
||||
{"@type": "Score", "tag": "TEST_TAG_LOCAL", "score": 1.0}
|
||||
],
|
||||
"HttpLookup": [
|
||||
{
|
||||
"namespace": "stwt_test_feed",
|
||||
"url": "http://127.0.0.1:1/feed.txt",
|
||||
"format": {"@type": "List"},
|
||||
"enable": true,
|
||||
"isGzipped": false,
|
||||
"maxSize": 1048576,
|
||||
"maxEntries": 1000,
|
||||
"maxEntrySize": 512,
|
||||
"refresh": 43200000,
|
||||
"retry": 3600000,
|
||||
"timeout": 30000
|
||||
}
|
||||
],
|
||||
"MemoryLookupKey": [
|
||||
{"namespace": "test-keys", "key": "example.org", "isGlobPattern": false},
|
||||
{"namespace": "test-keys", "key": "conflict.org", "isGlobPattern": false}
|
||||
],
|
||||
"SpamFileExtension": [
|
||||
{"extension": "tst", "contentTypes": {}, "isArchive": false, "isBad": false, "isNz": false}
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
fn release_2() -> Value {
|
||||
json!({
|
||||
"SpamRule": [
|
||||
{
|
||||
"@type": "Any",
|
||||
"name": "STWT_TEST_RULE",
|
||||
"enable": true,
|
||||
"priority": 400,
|
||||
"condition": {"else": "false", "match": {"0": {"if": "$MISSING_ESSENTIAL_HEADERS && $SINGLE_SHORT_PART", "then": "'SHORT_PART_BAD_HEADERS'"}}}
|
||||
},
|
||||
{
|
||||
"@type": "Any",
|
||||
"name": "STWT_TEST_DISABLED",
|
||||
"enable": true,
|
||||
"priority": 450,
|
||||
"condition": {"else": "false", "match": {"0": {"if": "$MISSING_ESSENTIAL_HEADERS && $SINGLE_SHORT_PART", "then": "'SHORT_PART_BAD_HEADERS'"}}}
|
||||
},
|
||||
{
|
||||
"@type": "Any",
|
||||
"name": "STWT_TEST_NEW",
|
||||
"enable": true,
|
||||
"priority": 500,
|
||||
"condition": {"else": "false", "match": {"0": {"if": "$MISSING_ESSENTIAL_HEADERS && $SINGLE_SHORT_PART", "then": "'SHORT_PART_BAD_HEADERS'"}}}
|
||||
}
|
||||
],
|
||||
"SpamDnsblServer": [
|
||||
{
|
||||
"@type": "Domain",
|
||||
"name": "STWT_TEST_DNSBL",
|
||||
"enable": true,
|
||||
"tag": {"else": "false", "match": {
|
||||
"0": {"if": "octets[0] != 127", "then": "false"},
|
||||
"1": {"if": "octets[3] == 1", "then": "'URIBL_BLOCKED'"},
|
||||
"2": {"if": "bit_and(octets[3], 2) != 0", "then": "'URIBL_BLACK'"},
|
||||
"3": {"if": "bit_and(octets[3], 4) != 0", "then": "'URIBL_GREY'"},
|
||||
"4": {"if": "bit_and(octets[3], 8) != 0", "then": "'URIBL_RED'"}
|
||||
}},
|
||||
"zone": {"else": "value + '.multi.uribl.com'", "match": {}}
|
||||
}
|
||||
],
|
||||
"SpamTag": [
|
||||
{"@type": "Score", "tag": "TEST_TAG", "score": 4.5},
|
||||
{"@type": "Score", "tag": "TEST_TAG_LOCAL", "score": 1.0}
|
||||
],
|
||||
"HttpLookup": [
|
||||
{
|
||||
"namespace": "stwt_test_feed",
|
||||
"url": "http://127.0.0.1:1/feed.txt",
|
||||
"format": {"@type": "List"},
|
||||
"enable": true,
|
||||
"isGzipped": false,
|
||||
"maxSize": 1048576,
|
||||
"maxEntries": 2000,
|
||||
"maxEntrySize": 512,
|
||||
"refresh": 43200000,
|
||||
"retry": 3600000,
|
||||
"timeout": 30000
|
||||
}
|
||||
],
|
||||
"MemoryLookupKey": [
|
||||
{"namespace": "test-keys", "key": "example.org", "isGlobPattern": true},
|
||||
{"namespace": "test-keys", "key": "conflict.org", "isGlobPattern": false}
|
||||
],
|
||||
"SpamFileExtension": [
|
||||
{"extension": "tst", "contentTypes": {}, "isArchive": false, "isBad": true, "isNz": false}
|
||||
]
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! A spam rules update brings unedited rules up to date and leaves an admin's
|
||||
//! edits alone (common::manager::spam_rules), including on an install whose
|
||||
//! rules were loaded before updates fingerprinted what they wrote. Each
|
||||
//! update records what it replaced and what it kept in one audit record
|
||||
//! (AU-1.10).
|
||||
|
||||
use crate::utils::server::{TestServer, TestServerBuilder};
|
||||
use registry::{
|
||||
schema::{
|
||||
enums::TaskSpamFilterMaintenanceType,
|
||||
prelude::ObjectType,
|
||||
structs::{SpamRule, SpamSettings, Task, TaskSpamFilterMaintenance, TaskStatus},
|
||||
},
|
||||
types::ObjectImpl,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use std::fs;
|
||||
|
||||
const USING: &[&str] = &["urn:ietf:params:jmap:core", "urn:inbuxa:jmap"];
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn spam_rules_keep_admin_edits() {
|
||||
let test = TestServerBuilder::new("spam_rules_kept_test")
|
||||
.await
|
||||
.with_http_listener(19059)
|
||||
.await
|
||||
.build()
|
||||
.await;
|
||||
let admin = test.account("admin");
|
||||
let rules_path = test.temp_dir.path.join("spam-filter-rules.json");
|
||||
admin
|
||||
.registry_create_object(SpamSettings {
|
||||
spam_filter_rules_url: format!("file://{}", rules_path.display()).into(),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
|
||||
// Two rules an admin made before any update ran: one exactly as the
|
||||
// release has it, as an install from before fingerprints would, and one
|
||||
// different from it, as an edited one would be.
|
||||
admin.registry_create_object(rule("INBX_PRESET", 500)).await;
|
||||
admin
|
||||
.registry_create_object(rule("INBX_PRESET_EDITED", 300))
|
||||
.await;
|
||||
admin.reload_settings().await;
|
||||
|
||||
update_rules(&test, &release(500)).await;
|
||||
assert_eq!(priority(&test, "INBX_UNTOUCHED").await, 500);
|
||||
assert_eq!(priority(&test, "INBX_PRESET_EDITED").await, 300);
|
||||
|
||||
// An admin edits one rule the update wrote, and switches another off.
|
||||
let edited = id_of(&test, "INBX_EDITED").await;
|
||||
admin
|
||||
.registry_update_object(ObjectType::SpamRule, edited, json!({"priority": 300}))
|
||||
.await;
|
||||
let switched_off = id_of(&test, "INBX_SWITCHED_OFF").await;
|
||||
admin
|
||||
.registry_update_object(ObjectType::SpamRule, switched_off, json!({"enable": false}))
|
||||
.await;
|
||||
|
||||
for run in 0..2 {
|
||||
update_rules(&test, &release(400)).await;
|
||||
|
||||
for (name, expected) in [
|
||||
("INBX_UNTOUCHED", 400),
|
||||
("INBX_SWITCHED_OFF", 400),
|
||||
("INBX_PRESET", 400),
|
||||
("INBX_EDITED", 300),
|
||||
("INBX_PRESET_EDITED", 300),
|
||||
] {
|
||||
assert_eq!(priority(&test, name).await, expected, "{name}, run {run}");
|
||||
}
|
||||
assert!(
|
||||
!stored(&test, "INBX_SWITCHED_OFF").await.enable(),
|
||||
"run {run}: switching a rule off was undone"
|
||||
);
|
||||
}
|
||||
|
||||
// Two updates changed something: the first and the first of release 400.
|
||||
// The second run of 400 changed nothing, so it isn't recorded.
|
||||
let records = audit(&test, json!({"targetKind": "x:SpamRule"})).await;
|
||||
let details = records
|
||||
.iter()
|
||||
.filter_map(|record| record["details"].as_str())
|
||||
.filter(|details| details.starts_with("Rules update"))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(details.len(), 2, "{details:?}");
|
||||
assert!(
|
||||
details[0].contains("replaced 3 SpamRule")
|
||||
&& details[0].contains("kept as edited locally 2 SpamRule"),
|
||||
"{}",
|
||||
details[0]
|
||||
);
|
||||
assert!(details[1].contains("added 3 SpamRule"), "{}", details[1]);
|
||||
}
|
||||
|
||||
fn rule(name: &str, priority: i64) -> SpamRule {
|
||||
serde_json::from_value(rule_json(name, priority)).unwrap()
|
||||
}
|
||||
|
||||
fn rule_json(name: &str, priority: i64) -> Value {
|
||||
json!({
|
||||
"@type": "Any",
|
||||
"name": name,
|
||||
"enable": true,
|
||||
"priority": priority,
|
||||
"condition": {"else": "false", "match": {"0": {"if": "$MISSING_ESSENTIAL_HEADERS && $SINGLE_SHORT_PART", "then": "'SHORT_PART_BAD_HEADERS'"}}}
|
||||
})
|
||||
}
|
||||
|
||||
fn release(priority: i64) -> Value {
|
||||
let rules = [
|
||||
"INBX_UNTOUCHED",
|
||||
"INBX_EDITED",
|
||||
"INBX_SWITCHED_OFF",
|
||||
"INBX_PRESET",
|
||||
"INBX_PRESET_EDITED",
|
||||
]
|
||||
.into_iter()
|
||||
.map(|name| rule_json(name, priority))
|
||||
.collect::<Vec<_>>();
|
||||
json!({ "SpamRule": rules })
|
||||
}
|
||||
|
||||
async fn update_rules(test: &TestServer, rules: &Value) {
|
||||
fs::write(
|
||||
test.temp_dir.path.join("spam-filter-rules.json"),
|
||||
rules.to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
test.account("admin")
|
||||
.registry_create_object(Task::SpamFilterMaintenance(TaskSpamFilterMaintenance {
|
||||
maintenance_type: TaskSpamFilterMaintenanceType::UpdateRules,
|
||||
status: TaskStatus::now(),
|
||||
}))
|
||||
.await;
|
||||
test.wait_for_tasks().await;
|
||||
}
|
||||
|
||||
async fn stored(test: &TestServer, name: &str) -> SpamRule {
|
||||
test.server
|
||||
.registry()
|
||||
.list::<SpamRule>()
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find(|item| object_json(&item.object)["name"] == name)
|
||||
.unwrap_or_else(|| panic!("no rule {name}"))
|
||||
.object
|
||||
}
|
||||
|
||||
async fn id_of(test: &TestServer, name: &str) -> types::id::Id {
|
||||
test.server
|
||||
.registry()
|
||||
.list::<SpamRule>()
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find(|item| object_json(&item.object)["name"] == name)
|
||||
.unwrap_or_else(|| panic!("no rule {name}"))
|
||||
.id
|
||||
.id()
|
||||
}
|
||||
|
||||
async fn priority(test: &TestServer, name: &str) -> i64 {
|
||||
object_json(&stored(test, name).await)["priority"]
|
||||
.as_i64()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn object_json<T: ObjectImpl>(object: &T) -> Value {
|
||||
serde_json::to_value(object).unwrap()
|
||||
}
|
||||
|
||||
/// Audit records matching `filter`, newest first.
|
||||
async fn audit(test: &TestServer, filter: Value) -> Vec<Value> {
|
||||
let admin = test.account("admin");
|
||||
let response = admin
|
||||
.jmap_request(
|
||||
USING,
|
||||
json!([
|
||||
["inbuxa:AuditEvent/query", {
|
||||
"accountId": admin.id_string(), "filter": filter, "limit": 100
|
||||
}, "q"],
|
||||
["inbuxa:AuditEvent/get", {
|
||||
"accountId": admin.id_string(),
|
||||
"#ids": {"resultOf": "q", "name": "inbuxa:AuditEvent/query", "path": "/ids"}
|
||||
}, "g"]
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
response
|
||||
.0
|
||||
.pointer("/methodResponses/1/1/list")
|
||||
.and_then(Value::as_array)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| panic!("audit query failed: {}", response.0))
|
||||
}
|
||||
@@ -51,7 +51,7 @@ use utils::snowflake::SnowflakeIdGenerator;
|
||||
async fn report_reschedule() {
|
||||
let mut test = TestServerBuilder::new("smtp_report_reschedule")
|
||||
.await
|
||||
.with_http_listener(19057)
|
||||
.with_http_listener(19058)
|
||||
.await
|
||||
.capture_queue()
|
||||
.build()
|
||||
|
||||
+43
-19
@@ -6,7 +6,7 @@
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use crate::utils::{account::Account, server::TestServer};
|
||||
use crate::utils::{account::Account, jmap::JmapUtils, server::TestServer};
|
||||
use registry::{
|
||||
schema::{
|
||||
enums::TaskStoreMaintenanceType,
|
||||
@@ -301,14 +301,29 @@ impl Account {
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn tasks(&self) -> Vec<TaskId> {
|
||||
pub async fn tasks(&self) -> Option<Vec<TaskId>> {
|
||||
let ids = self.task_ids().await;
|
||||
let mut results = Vec::with_capacity(ids.len());
|
||||
for id in ids {
|
||||
let sample = self.registry_get::<Task>(id).await;
|
||||
results.push(TaskId { id, task: sample });
|
||||
if ids.is_empty() {
|
||||
return Some(Vec::new());
|
||||
}
|
||||
results
|
||||
|
||||
let response = self.registry_get_many(ObjectType::Task, &ids).await;
|
||||
if response.not_found().next().is_some() {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(
|
||||
response
|
||||
.list()
|
||||
.iter()
|
||||
.map(|item| TaskId {
|
||||
id: item.object_id(),
|
||||
task: serde_json::from_str(&item.to_string()).unwrap_or_else(|err| {
|
||||
panic!("Failed to deserialize {item}: {err}");
|
||||
}),
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
async fn assert_no_tasks(&self) {
|
||||
@@ -344,19 +359,28 @@ impl Account {
|
||||
) -> Vec<TaskId> {
|
||||
let mut attempt = 0;
|
||||
loop {
|
||||
let tasks = self.tasks().await;
|
||||
if tasks.len() == count && tasks.iter().all(&is_expected) {
|
||||
return tasks;
|
||||
match self.tasks().await {
|
||||
Some(tasks) if tasks.len() == count && tasks.iter().all(&is_expected) => {
|
||||
return tasks;
|
||||
}
|
||||
Some(tasks) => {
|
||||
attempt += 1;
|
||||
assert!(
|
||||
attempt < TASK_WAIT_ATTEMPTS,
|
||||
"Expected {} tasks, found {}: {:?}",
|
||||
count,
|
||||
tasks.len(),
|
||||
tasks
|
||||
);
|
||||
}
|
||||
None => {
|
||||
attempt += 1;
|
||||
assert!(
|
||||
attempt < TASK_WAIT_ATTEMPTS,
|
||||
"Task/query keeps returning ids that Task/get reports as not found"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
attempt += 1;
|
||||
assert!(
|
||||
attempt < TASK_WAIT_ATTEMPTS,
|
||||
"Expected {} tasks, found {}: {:?}",
|
||||
count,
|
||||
tasks.len(),
|
||||
tasks
|
||||
);
|
||||
tokio::time::sleep(TASK_WAIT_INTERVAL).await;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user