Journaling: outside archives, and Journal it in mail flow rules

Phase 3 of the journaling spec.

- A journal's destination: builtIn (true for journals stored before) and
  archiveAddress, at least one. Reports to an archive are queued from the
  empty sender, one per address, flagged so they're never journaled.
- A pending record per report. When the queue lets go of one without
  delivering it (refused, expired, deleted), it becomes its own entry in
  the built-in journal under the sending journals' retention, the
  journal's archiveFailures (count, last time, reason) goes up, and the
  audit log records it; if that can't be written it stays queued.
- Journal it: a rule action naming a journal, on mail flow rules and
  beside a DLP rule's block, warn or hold. A journal whose scope chooses
  nobody takes only what rules send it.
- The report lists recipients a rule added or redirected to under
  "Added by rule", by rule name.
- A rule's route is cleared between messages in one SMTP session, with the
  new journal marks; a second message used to keep the first one's route.

tests/src/system/journal.rs: destination validation, a rule-only journal
fed by a rule that also adds a recipient, an unreachable archive's report
kept in the built-in journal with the failure counted, a report delivered
to an archive here and not journaled itself.

tests/src/system/journal.rs

@@ -20,6 +20,7 @@ use inbuxa_features::journal::{
 };
 use registry::schema::structs::{Expression, MtaStageAuth};
 use serde_json::{Value, json};
+use std::str::FromStr;
 use store::{Deserialize, write::BatchBuilder};
 
 const USING: &[&str] = &[
@@ -27,6 +28,7 @@ const USING: &[&str] = &[
     "urn:ietf:params:jmap:mail",
     "urn:ietf:params:jmap:submission",
     "urn:inbuxa:jmap",
+    "urn:inbuxa:jmap:registry",
 ];
 
 async fn call(account: &Account, method: &str, mut arguments: Value) -> (String, Value) {
@@ -171,8 +173,11 @@ pub async fn test(test: &mut TestServer) {
             "both": {"name": "Both", "enabled": true, "direction": "any",
                      "scope": {"everyone": true, "accounts": [sender.id_string()]},
                      "retentionDays": 365},
-            "none": {"name": "None", "enabled": true, "direction": "any",
-                     "scope": {}, "retentionDays": 365},
+            "none": {"name": "Nowhere", "enabled": true, "direction": "any",
+                     "scope": {"everyone": true}, "retentionDays": 365, "builtIn": false},
+            "badaddr": {"name": "Bad archive", "enabled": true, "direction": "any",
+                        "scope": {"everyone": true}, "retentionDays": 365,
+                        "archiveAddress": "not an address"},
             "server": {"name": "Mine", "enabled": true, "direction": "any",
                        "scope": {"everyone": true}, "retentionDays": 365,
                        "createdBy": "me"},
@@ -183,7 +188,7 @@ pub async fn test(test: &mut TestServer) {
         }}),
     )
     .await;
-    for refused in ["short", "both", "none", "server"] {
+    for refused in ["short", "both", "none", "badaddr", "server"] {
         assert_eq!(
             response["notCreated"][refused]["type"], "invalidProperties",
             "{refused}: {response}"
@@ -197,6 +202,14 @@ pub async fn test(test: &mut TestServer) {
         response["notCreated"]["both"]["properties"],
         json!(["scope"])
     );
+    assert_eq!(
+        response["notCreated"]["none"]["properties"],
+        json!(["builtIn"])
+    );
+    assert_eq!(
+        response["notCreated"]["badaddr"]["properties"],
+        json!(["archiveAddress"])
+    );
     let everything = response["created"]["all"]["id"]
         .as_str()
         .unwrap_or_else(|| panic!("{response}"))
@@ -445,6 +458,230 @@ pub async fn test(test: &mut TestServer) {
     );
 }
 
+/// Phase 3: journals only rules send mail to, recipients a rule added,
+/// reports sent to an outside archive, and what happens when the archive
+/// doesn't take one.
+pub async fn archive(test: &mut TestServer) {
+    println!("Running journal archive tests...");
+    let admin = test.account("[email protected]");
+    let sender = admin
+        .create_user_account(
+            "[email protected]",
+            "archive-sender-secret-7201",
+            "Archive sender",
+            &[],
+            vec![],
+        )
+        .await;
+    let vault = admin
+        .create_user_account(
+            "[email protected]",
+            "journal-vault-secret-7202",
+            "Journal vault",
+            &[],
+            vec![],
+        )
+        .await;
+    let (_, response) = call(
+        &sender,
+        "Identity/set",
+        json!({"create": {"i": {"name": "Sender", "email": "[email protected]"}}}),
+    )
+    .await;
+    let identity = response["created"]["i"]["id"].as_str().unwrap().to_string();
+    let (_, response) = call(
+        &sender,
+        "Mailbox/set",
+        json!({"create": {"m": {"name": "Archive drafts"}}}),
+    )
+    .await;
+    let mailbox = response["created"]["m"]["id"].as_str().unwrap().to_string();
+
+    let (_, response) = call(
+        &admin,
+        "inbuxa:Journal/set",
+        json!({"create": {
+            "rules": {"name": "Only what rules send", "enabled": true, "direction": "any",
+                      "scope": {}, "retentionDays": 30},
+            "local": {"name": "To the vault", "enabled": true, "direction": "outgoing",
+                      "scope": {"accounts": [sender.id_string()]}, "retentionDays": 30,
+                      "builtIn": false, "archiveAddress": "[email protected]"},
+            "remote": {"name": "To an outside archive", "enabled": true, "direction": "internal",
+                       "scope": {"accounts": [sender.id_string()]}, "retentionDays": 30,
+                       "builtIn": false, "archiveAddress": "[email protected]"}
+        }}),
+    )
+    .await;
+    let id = |name: &str| {
+        response["created"][name]["id"]
+            .as_str()
+            .unwrap_or_else(|| panic!("{name}: {response}"))
+            .to_string()
+    };
+    let (rules_only, local, remote) = (id("rules"), id("local"), id("remote"));
+    let number = |id: &str| types::id::Id::from_str(id).unwrap().document_id();
+    let (_, response) = call(
+        &admin,
+        "inbuxa:MailRule/set",
+        json!({"create": {"r": {
+            "name": "Copy and journal", "kind": "transport", "direction": "outgoing",
+            "conditions": [{"type": "words", "words": ["journal-me"]}],
+            "actions": [
+                {"type": "addRecipient", "address": "[email protected]"},
+                {"type": "journal", "journal": rules_only.clone()}
+            ]
+        }}}),
+    )
+    .await;
+    let rule = response["created"]["r"]["id"]
+        .as_str()
+        .unwrap_or_else(|| panic!("{response}"))
+        .to_string();
+    inbuxa_features::journal::invalidate();
+
+    // A rule sends it to a journal whose scope takes nobody, and says who
+    // it added
+    let response = send(
+        &sender,
+        &identity,
+        &mailbox,
+        &["[email protected]"],
+        &["[email protected]"],
+        "Marked journal-me",
+    )
+    .await;
+    assert!(response["created"].get("s").is_some(), "{response}");
+    let entry = all_entries(test)
+        .await
+        .into_iter()
+        .map(|(_, e)| e)
+        .find(|e| e.subject == "Marked journal-me" && e.journals.contains(&number(&rules_only)))
+        .expect("journaled by the rule");
+    assert_eq!(entry.journals, vec![number(&rules_only)], "{entry:?}");
+    let text = String::from_utf8_lossy(&report_of(test, &entry).await).into_owned();
+    assert!(
+        text.contains("Added by rule: Copy and journal -> [email protected]\r\n"),
+        "{text}"
+    );
+    assert!(!text.contains("Bcc:"), "{text}");
+
+    // The same message went to the outside archive, which can't be reached
+    // from here: once it leaves the queue (given up on, or deleted), it's
+    // kept in the built-in journal
+    let fallback = |entries: &[(EntryId, Entry)]| {
+        entries
+            .iter()
+            .any(|(_, e)| e.subject == "Marked journal-me" && e.journals == vec![number(&remote)])
+    };
+    let mut deleted = false;
+    for _ in 0..100 {
+        if fallback(&all_entries(test).await) {
+            break;
+        }
+        let (_, response) = call(&admin, "x:QueuedMessage/get", json!({"ids": null})).await;
+        if let Some(queued) = response["list"]
+            .as_array()
+            .unwrap()
+            .iter()
+            .find(|m| m.to_string().contains("[email protected]"))
+        {
+            let queued_id = queued["id"].as_str().unwrap().to_string();
+            let (_, response) = call(
+                &admin,
+                "x:QueuedMessage/set",
+                json!({"destroy": [queued_id.clone()]}),
+            )
+            .await;
+            deleted = response["destroyed"] == json!([queued_id]);
+        }
+        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
+    }
+    let kept: Vec<Entry> = all_entries(test)
+        .await
+        .into_iter()
+        .map(|(_, e)| e)
+        .filter(|e| e.subject == "Marked journal-me")
+        .collect();
+    assert_eq!(kept.len(), 2, "{kept:?}");
+    assert!(kept.iter().any(|e| e.journals == vec![number(&remote)]));
+    let (_, response) = call(
+        &admin,
+        "inbuxa:Journal/get",
+        json!({"ids": [remote.clone()]}),
+    )
+    .await;
+    let failures = &response["list"][0]["archiveFailures"];
+    assert_eq!(failures["count"], 1, "{response}");
+    assert_eq!(
+        failures["lastReason"],
+        if deleted {
+            "it wasn't delivered before leaving the queue"
+        } else {
+            "the archive refused it"
+        },
+        "{response}"
+    );
+    let (_, response) = call(
+        &admin,
+        "inbuxa:Journal/get",
+        json!({"ids": [local.clone()]}),
+    )
+    .await;
+    assert_eq!(response["list"][0]["archiveFailures"]["count"], 0);
+
+    // Delivered to an archive here: the report arrives, and nothing goes
+    // into the built-in journal for that journal
+    let response = send(
+        &sender,
+        &identity,
+        &mailbox,
+        &["[email protected]"],
+        &["[email protected]"],
+        "To the vault",
+    )
+    .await;
+    assert!(response["created"].get("s").is_some(), "{response}");
+    let mut arrived = Vec::new();
+    for _ in 0..100 {
+        let (_, response) = call(
+            &vault,
+            "Email/query",
+            json!({"filter": {"subject": "Journal report: To the vault"}}),
+        )
+        .await;
+        arrived = response["ids"].as_array().cloned().unwrap_or_default();
+        if !arrived.is_empty() {
+            break;
+        }
+        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
+    }
+    assert_eq!(arrived.len(), 1, "the report arrived");
+    assert!(entry_for(test, "To the vault").await.is_none());
+    assert!(
+        all_entries(test)
+            .await
+            .iter()
+            .all(|(_, e)| !e.subject.starts_with("Journal report")),
+        "reports aren't journaled"
+    );
+    let (_, response) = call(
+        &admin,
+        "inbuxa:Journal/get",
+        json!({"ids": [local.clone()]}),
+    )
+    .await;
+    assert_eq!(response["list"][0]["archiveFailures"]["count"], 0);
+
+    call(&admin, "inbuxa:MailRule/set", json!({"destroy": [rule]})).await;
+    call(
+        &admin,
+        "inbuxa:Journal/set",
+        json!({"destroy": [rules_only, local, remote]}),
+    )
+    .await;
+    inbuxa_features::journal::invalidate();
+}
+
 struct Raw(Vec<u8>);
 
 impl Deserialize for Raw {
@@ -465,6 +702,7 @@ pub async fn journal_tests() {
     let admin = test.create_admin_account("[email protected]").await;
     test.insert_account(admin);
     self::test(&mut test).await;
+    self::archive(&mut test).await;
     if test.is_reset() {
         test.temp_dir.delete();
     }