Journaling: outside archives, and Journal it in mail flow rules

Phase 3 of the journaling spec.

- A journal's destination: builtIn (true for journals stored before) and
  archiveAddress, at least one. Reports to an archive are queued from the
  empty sender, one per address, flagged so they're never journaled.
- A pending record per report. When the queue lets go of one without
  delivering it (refused, expired, deleted), it becomes its own entry in
  the built-in journal under the sending journals' retention, the
  journal's archiveFailures (count, last time, reason) goes up, and the
  audit log records it; if that can't be written it stays queued.
- Journal it: a rule action naming a journal, on mail flow rules and
  beside a DLP rule's block, warn or hold. A journal whose scope chooses
  nobody takes only what rules send it.
- The report lists recipients a rule added or redirected to under
  "Added by rule", by rule name.
- A rule's route is cleared between messages in one SMTP session, with the
  new journal marks; a second message used to keep the first one's route.

tests/src/system/journal.rs: destination validation, a rule-only journal
fed by a rule that also adds a recipient, an unreachable archive's report
kept in the built-in journal with the failure counted, a report delivered
to an archive here and not journaled itself.

crates/features/src/journal/archive.rs

@@ -0,0 +1,120 @@
+/*
+ * SPDX-FileCopyrightText: 2026 Coffey Labs
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+//! Reports on their way to an outside archive (JR-7). Keys, after `J`:
+//!
+//! - `o` + the report's queue id: what goes into the built-in journal if
+//!   the archive never takes the report, as JSON. Cleared once it's
+//!   delivered or kept.
+//! - `w` + journal id (u32): how often that journal's archive didn't take a
+//!   report, and the last time and reason, for the console's warning.
+
+use super::{FEATURE, Json, entries::Entry};
+use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize};
+use store::{
+    SUBSPACE_INBUXA, Serialize, Store, ValueKey,
+    write::{AnyClass, BatchBuilder, ValueClass},
+};
+use trc::AddContext;
+
+const KIND_PENDING: u8 = b'o';
+const KIND_FAILURES: u8 = b'w';
+
+/// A report queued to an archive.
+#[derive(Debug, Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Pending {
+    pub address: String,
+    /// The entry, should the archive not take it: its own, with the
+    /// sending journals' retention, whatever else the built-in journal has.
+    pub entry: Entry,
+}
+
+/// How a journal's archive has been taking its reports.
+#[derive(Debug, Clone, Default, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Failures {
+    pub count: u64,
+    /// Seconds.
+    pub last_at: u64,
+    pub last_reason: String,
+}
+
+fn class(kind: u8, id: &[u8]) -> ValueClass {
+    let mut key = Vec::with_capacity(2 + id.len());
+    key.push(FEATURE);
+    key.push(kind);
+    key.extend_from_slice(id);
+    ValueClass::Any(AnyClass {
+        subspace: SUBSPACE_INBUXA,
+        key,
+    })
+}
+
+pub async fn set_pending(data: &Store, queue_id: u64, pending: &Pending) -> trc::Result<()> {
+    let mut batch = BatchBuilder::new();
+    batch.set(
+        class(KIND_PENDING, &queue_id.to_be_bytes()),
+        Json(pending).serialize()?,
+    );
+    data.write(batch.build_all())
+        .await
+        .caused_by(trc::location!())?;
+    Ok(())
+}
+
+pub async fn pending(data: &Store, queue_id: u64) -> trc::Result<Option<Pending>> {
+    Ok(data
+        .get_value::<Json<Pending>>(ValueKey::from(class(KIND_PENDING, &queue_id.to_be_bytes())))
+        .await
+        .caused_by(trc::location!())?
+        .map(|Json(pending)| pending))
+}
+
+pub async fn clear_pending(data: &Store, queue_id: u64) -> trc::Result<()> {
+    let mut batch = BatchBuilder::new();
+    batch.clear(class(KIND_PENDING, &queue_id.to_be_bytes()));
+    data.write(batch.build_all())
+        .await
+        .caused_by(trc::location!())?;
+    Ok(())
+}
+
+pub async fn failures(data: &Store, journal_id: u32) -> trc::Result<Failures> {
+    Ok(data
+        .get_value::<Json<Failures>>(ValueKey::from(class(
+            KIND_FAILURES,
+            &journal_id.to_be_bytes(),
+        )))
+        .await
+        .caused_by(trc::location!())?
+        .map(|Json(failures)| failures)
+        .unwrap_or_default())
+}
+
+/// Counts one report an archive didn't take, for each of `journals`.
+pub async fn record_failure(
+    data: &Store,
+    journals: &[u32],
+    at: u64,
+    reason: &str,
+) -> trc::Result<()> {
+    for journal_id in journals {
+        let mut failures = failures(data, *journal_id).await?;
+        failures.count += 1;
+        failures.last_at = at;
+        failures.last_reason = reason.chars().take(500).collect();
+        let mut batch = BatchBuilder::new();
+        batch.set(
+            class(KIND_FAILURES, &journal_id.to_be_bytes()),
+            Json(&failures).serialize()?,
+        );
+        data.write(batch.build_all())
+            .await
+            .caused_by(trc::location!())?;
+    }
+    Ok(())
+}

crates/features/src/journal/mod.rs

@@ -16,6 +16,7 @@
 //! with `J`; journals are `j` + id (u32), as JSON. There are few, so they're
 //! read whole.
 
+pub mod archive;
 pub mod entries;
 pub mod report;
 
@@ -128,6 +129,12 @@ pub struct Journal {
     /// How long an entry this journal writes is kept. An entry keeps the
     /// retention it was written with (JR-12).
     pub retention_days: u32,
+    /// Whether entries go into the built-in journal (JR-5).
+    #[serde(default = "yes")]
+    pub built_in: bool,
+    /// An outside archive's journal address, sent each report (JR-7).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub archive_address: Option<String>,
     #[serde(default)]
     pub created_by: String,
     #[serde(default)]
@@ -164,13 +171,28 @@ impl Journal {
                 format!("Keep entries between {MIN_RETENTION_DAYS} and {MAX_RETENTION_DAYS} days."),
             );
         }
+        // Neither is a journal only rules send mail to (JR-10)
         let chosen = self.scope.lists().iter().any(|list| !list.is_empty());
-        if self.scope.everyone == chosen {
+        if self.scope.everyone && chosen {
             return invalid(
                 "scope",
                 "Journal everyone, or choose accounts, groups, domains or tenants; not both.",
             );
         }
+        if !self.built_in && self.archive_address.is_none() {
+            return invalid(
+                "builtIn",
+                "Keep entries in the built-in journal, send them to an archive, or both.",
+            );
+        }
+        if let Some(address) = &self.archive_address
+            && !is_address(address)
+        {
+            return invalid(
+                "archiveAddress",
+                format!("\"{address}\" isn't an email address."),
+            );
+        }
         if self.scope.lists().iter().any(|list| list.len() > MAX_LIST) {
             return invalid("scope", format!("Choose at most {MAX_LIST} of each."));
         }
@@ -179,6 +201,11 @@ impl Journal {
 
     /// Whether this journal takes a message going `direction` with these
     /// people here on either side.
+    /// Whether only rules send this journal mail (JR-10).
+    pub fn rules_only(&self) -> bool {
+        !self.scope.everyone && self.scope.lists().iter().all(|list| list.is_empty())
+    }
+
     pub fn takes(&self, direction: Direction, members: &[Member]) -> bool {
         self.enabled
             && self.direction.includes(direction)
@@ -186,6 +213,22 @@ impl Journal {
     }
 }
 
+fn yes() -> bool {
+    true
+}
+
+/// An address an archive can be sent to: one `@`, something either side,
+/// nothing that would break an envelope.
+fn is_address(address: &str) -> bool {
+    address.len() <= 320
+        && address.split_once('@').is_some_and(|(local, domain)| {
+            !local.is_empty() && domain.contains('.') && !domain.contains('@')
+        })
+        && !address
+            .chars()
+            .any(|c| c.is_whitespace() || c.is_control() || matches!(c, '<' | '>' | ',' | ';'))
+}
+
 /// A value stored as JSON.
 pub(crate) struct Json<T>(pub T);
 
@@ -347,6 +390,8 @@ mod tests {
             direction: Direction::Any,
             scope,
             retention_days: 365,
+            built_in: true,
+            archive_address: None,
             created_by: String::new(),
             created_at: 0,
             updated_at: 0,
@@ -372,7 +417,11 @@ mod tests {
             .validate()
             .is_ok()
         );
-        assert!(journal(Scope::default()).validate().is_err());
+        // Nobody chosen: only rules send it mail
+        let rules_only = journal(Scope::default());
+        assert!(rules_only.validate().is_ok());
+        assert!(rules_only.rules_only());
+        assert!(!rules_only.takes(Direction::Any, &[member(3, vec![7])]));
         let both = Scope {
             everyone: true,
             groups: vec![4],
@@ -381,6 +430,38 @@ mod tests {
         assert_eq!(journal(both).validate().unwrap_err().property, "scope");
     }
 
+    #[test]
+    fn destinations() {
+        let mut j = journal(Scope {
+            everyone: true,
+            ..Default::default()
+        });
+        j.built_in = false;
+        assert_eq!(j.validate().unwrap_err().property, "builtIn");
+        j.archive_address = Some("[email protected]".into());
+        assert!(j.validate().is_ok());
+        for bad in [
+            "archive",
+            "a@b",
+            "a [email protected]",
+            "<[email protected]>",
+            "a@[email protected]",
+        ] {
+            j.archive_address = Some(bad.into());
+            assert_eq!(
+                j.validate().unwrap_err().property,
+                "archiveAddress",
+                "{bad}"
+            );
+        }
+        // Stored before destinations existed: the built-in journal
+        let old: Journal = serde_json::from_str(
+            r#"{"name":"Old","direction":"any","scope":{"everyone":true},"retentionDays":30}"#,
+        )
+        .unwrap();
+        assert!(old.built_in && old.archive_address.is_none());
+    }
+
     #[test]
     fn retention_has_bounds() {
         let mut j = journal(Scope {

crates/features/src/journal/report.rs

@@ -20,6 +20,8 @@ use sha2::{Digest, Sha256};
 pub struct Recipient {
     pub address: String,
     pub orcpt: Option<String>,
+    /// The mail flow rule that added or redirected to it.
+    pub added_by: Option<String>,
 }
 
 /// What the queue knows about a message.
@@ -46,6 +48,8 @@ pub struct Fields {
     pub bcc: Vec<String>,
     /// A list's address, and its members among the recipients.
     pub expanded: Vec<(String, Vec<String>)>,
+    /// A rule's name, and the recipients it added.
+    pub added: Vec<(String, Vec<String>)>,
 }
 
 /// One line's worth of a value: no line breaks, no control characters.
@@ -106,7 +110,12 @@ pub fn fields(envelope: &Envelope<'_>, original: &[u8]) -> Fields {
             .as_deref()
             .map(orcpt_address)
             .filter(|via| !via.is_empty() && *via != address);
-        if header_to.contains(&address) {
+        if let Some(rule) = &rcpt.added_by {
+            match fields.added.iter_mut().find(|(name, _)| name == rule) {
+                Some((_, added)) => added.push(line(&rcpt.address)),
+                None => fields.added.push((line(rule), vec![line(&rcpt.address)])),
+            }
+        } else if header_to.contains(&address) {
             fields.to.push(line(&rcpt.address));
         } else if header_cc.contains(&address) {
             fields.cc.push(line(&rcpt.address));
@@ -159,6 +168,9 @@ pub fn text(envelope: &Envelope<'_>, fields: &Fields) -> String {
     for (list, members) in &fields.expanded {
         field("Expanded", &format!("{list} -> {}", members.join(", ")));
     }
+    for (rule, added) in &fields.added {
+        field("Added by rule", &format!("{rule} -> {}", added.join(", ")));
+    }
     if envelope.held {
         field("Held for review", "yes");
     }
@@ -265,6 +277,7 @@ The figures.\r\n";
         Recipient {
             address: address.into(),
             orcpt: orcpt.map(Into::into),
+            added_by: None,
         }
     }
 
@@ -337,6 +350,19 @@ The figures.\r\n";
         assert_eq!(original(&report), Some(unterminated));
     }
 
+    #[test]
+    fn rule_added_recipients_say_so() {
+        let mut copied = rcpt("[email protected]", None);
+        copied.added_by = Some("Copy finance".into());
+        let recipients = [rcpt("[email protected]", None), copied];
+        let env = envelope(&recipients);
+        let fields = fields(&env, ORIGINAL);
+        assert!(fields.bcc.is_empty(), "{fields:?}");
+        assert!(
+            text(&env, &fields).contains("Added by rule: Copy finance -> [email protected]\r\n")
+        );
+    }
+
     #[test]
     fn values_stay_on_one_line() {
         let recipients = [rcpt("[email protected]", None)];

crates/features/src/mailflow/rules.rs

@@ -95,6 +95,33 @@ pub(crate) mod jmap_ids {
     }
 }
 
+/// One id in the same form.
+pub(crate) mod jmap_id {
+    use serde::{Deserialize, Deserializer, Serializer, de::Error};
+    use std::str::FromStr;
+    use types::id::Id;
+
+    pub fn serialize<S: Serializer>(id: &u32, serializer: S) -> Result<S::Ok, S::Error> {
+        serializer.serialize_str(&Id::from(*id).to_string())
+    }
+
+    #[derive(Deserialize)]
+    #[serde(untagged)]
+    enum Either {
+        Text(String),
+        Number(u32),
+    }
+
+    pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result<u32, D::Error> {
+        match Either::deserialize(deserializer)? {
+            Either::Number(n) => Ok(n),
+            Either::Text(text) => Id::from_str(&text)
+                .map(|id| id.document_id())
+                .map_err(|_| D::Error::custom(format!("\"{text}\" isn't an id"))),
+        }
+    }
+}
+
 /// A detector and the least it must find.
 #[derive(Debug, Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)]
 #[serde(rename_all = "camelCase")]
@@ -222,6 +249,11 @@ pub enum Action {
     Route {
         queue: String,
     },
+    /// Journaling spec, JR-10: a copy into this journal, whatever its scope.
+    Journal {
+        #[serde(with = "jmap_id")]
+        journal: u32,
+    },
     // DLP actions
     Block {
         notice: String,
@@ -311,10 +343,16 @@ impl Rule {
                 if self.direction != Direction::Outgoing {
                     return Err(invalid("direction", "DLP rules check outgoing mail only."));
                 }
-                if dlp_actions != 1 || self.actions.len() != 1 {
+                // One of block, warn or hold; journaling may go with it
+                if dlp_actions != 1
+                    || self
+                        .actions
+                        .iter()
+                        .any(|a| !a.is_dlp() && !matches!(a, Action::Journal { .. }))
+                {
                     return Err(invalid(
                         "actions",
-                        "A DLP rule has exactly one action: block, warn or hold.",
+                        "A DLP rule has exactly one action: block, warn or hold, and may also journal the message.",
                     ));
                 }
             }
@@ -478,6 +516,7 @@ fn validate_action(action: &Action) -> Result<(), String> {
         }
         Action::Refuse { text: t } => text(t, "refusal text"),
         Action::Route { queue } => text(queue, "queue"),
+        Action::Journal { .. } => Ok(()),
         Action::Block { notice } | Action::Warn { notice } | Action::Hold { notice, .. } => {
             text(notice, "notice")
         }
@@ -619,6 +658,38 @@ mod tests {
         }
     }
 
+    #[test]
+    fn journal_action_goes_with_either_kind() {
+        let hold = Action::Hold {
+            notice: "Held.".into(),
+            notify_sender: false,
+        };
+        let journal = Action::Journal { journal: 3 };
+        assert!(
+            rule(Kind::Dlp, vec![hold.clone(), journal.clone()])
+                .validate()
+                .is_ok()
+        );
+        assert!(rule(Kind::Dlp, vec![journal.clone()]).validate().is_err());
+        assert!(
+            rule(
+                Kind::Dlp,
+                vec![hold, Action::PrefixSubject { text: "x".into() }]
+            )
+            .validate()
+            .is_err()
+        );
+        assert!(
+            rule(Kind::Transport, vec![journal.clone()])
+                .validate()
+                .is_ok()
+        );
+        let json = serde_json::to_value(&journal).unwrap();
+        assert_eq!(json, serde_json::json!({"type": "journal", "journal": "d"}));
+        let back: Action = serde_json::from_value(json).unwrap();
+        assert_eq!(back, journal);
+    }
+
     #[test]
     fn wire_format() {
         let json = r#"{"name":"Cards","kind":"dlp","direction":"outgoing",