From e26dea3665ff1ca17369a4ded3ae7104414abc0b Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 30 Oct 2026 19:51:03 -0700 Subject: [PATCH] Relay: the relay keys wrapped under a key file (RL-35) With INBUXA_RELAY_KEY_FILE naming a file of 32 random bytes in base64, the link CA key (and the next one's, during a replacement), the sealing key and each relay's recipient key are stored wrapped with AES-256-GCM, as {"wrapped": ...} where each was a string. A copy of the store or a backup alone then can't issue node certificates or open held mail. The link service wraps any key still in the clear at its next scan. Without the file, with one that can't be read, or with another key, a node starts no relay link, and each link's lastError says why; while the setting names a file that can't be read, no key is written in the clear. Without the setting, keys are stored as before. inbuxa:Relay shows keysWrapped. Tested: unit tests of the wrapping, the stored forms and the key file; the server's relay link test against the real relay binary (RL-T27: no key in the clear in the stored bytes once wrapped, still linked; with the file taken away, no link and the reason; with it back, linked again and the mail held meanwhile delivered), and relay_tests. --- crates/features/src/relay/link.rs | 20 +- crates/features/src/relay/mod.rs | 118 ++++++- crates/features/src/relay/wrap.rs | 348 +++++++++++++++++++ crates/jmap-proto/src/object/inbuxa_relay.rs | 5 + crates/jmap/src/inbuxa/relay.rs | 7 +- crates/services/src/inbuxa_relay_link.rs | 22 +- docs/spec/features/relay.md | 15 +- tests/src/system/relay.rs | 90 ++++- 8 files changed, 594 insertions(+), 31 deletions(-) create mode 100644 crates/features/src/relay/wrap.rs diff --git a/crates/features/src/relay/link.rs b/crates/features/src/relay/link.rs index bdb5b7f2..e3a1afdf 100644 --- a/crates/features/src/relay/link.rs +++ b/crates/features/src/relay/link.rs @@ -11,6 +11,7 @@ use super::{ DownstreamKeys, FEATURE, Json, KIND_INGESTED, KIND_STATUS, Relay, get_versioned, replace, + wrap::KeyError, }; use inbuxa_relay_proto::{ pin::{Pin, pins_match}, @@ -57,7 +58,7 @@ pub fn issue_node_certificate( node_id: u64, now: u64, ) -> trc::Result { - let ca_key = rcgen::KeyPair::from_pem(&keys.link_ca_key_pem).map_err(cert_error)?; + let ca_key = rcgen::KeyPair::from_pem(&keys.link_ca_key_pem.open()?).map_err(cert_error)?; let issuer = rcgen::Issuer::from_ca_cert_pem(&keys.link_ca_cert_pem, ca_key).map_err(cert_error)?; let key = rcgen::KeyPair::generate_for(&rcgen::PKCS_ECDSA_P256_SHA256).map_err(cert_error)?; @@ -221,16 +222,17 @@ impl RecipientPlan { /// What the relay is sent with `Relay/set`: its settings as an /// administrator made them, with the served domains, the keys and the -/// STARTTLS certificate added. +/// STARTTLS certificate added. Fails only when the recipient key is +/// wrapped and can't be opened (RL-35). pub fn pushed_settings( relay: &Relay, keys: &DownstreamKeys, plan: &RecipientPlan, -) -> RelaySettings { +) -> Result { let mut settings = relay.settings.clone(); settings.domains = plan.domains.keys().cloned().collect(); settings.sealing_key = Some(keys.sealing_public); - settings.recipient_key = Some(relay.recipient_key); + settings.recipient_key = Some(relay.recipient_key.open()?); // While the link CA is replaced, the relay trusts the next one too (RL-34) settings.next_link_ca_pin = keys.next_link_ca.as_ref().map(|next| next.pin); settings.smtp_certificate = match (&relay.smtp_certificate, &relay.smtp_private_key) { @@ -240,7 +242,7 @@ pub fn pushed_settings( }), _ => None, }; - settings + Ok(settings) } /// A node's link to a relay, as that node sees it. @@ -502,7 +504,7 @@ pub async fn forget_ingested(data: &Store, relay_id: u32, before: u64) -> trc::R #[cfg(test)] mod tests { use super::*; - use crate::relay::{Join, new_downstream_keys}; + use crate::relay::{Join, new_downstream_keys, wrap::Secret}; use inbuxa_relay_proto::time; fn relay(domains: &[&str]) -> Relay { @@ -523,7 +525,7 @@ mod tests { }, smtp_certificate: None, smtp_private_key: None, - recipient_key: [9; 32], + recipient_key: Secret::new([9; 32]), join: Join::default(), created_by: String::new(), created_at: 0, @@ -575,7 +577,7 @@ mod tests { let hashes = one.hashes(&[9; 32]); assert_eq!(hashes, vec![recipient::hash(&[9; 32], "jane@example.com")]); let keys = new_downstream_keys(time::now()).unwrap(); - let settings = pushed_settings(&relay(&["example.com"]), &keys, &one); + let settings = pushed_settings(&relay(&["example.com"]), &keys, &one).unwrap(); assert_eq!(settings.domains, ["example.com"]); assert_eq!(settings.sealing_key, Some(keys.sealing_public)); assert_eq!(settings.recipient_key, Some([9; 32])); @@ -609,7 +611,7 @@ mod tests { let keys = new_downstream_keys(time::now()).unwrap(); let mut replacing = keys.clone(); replacing.next_link_ca = Some(super::super::NextLinkCa { - key_pem: String::new(), + key_pem: Secret::new(String::new()), cert_pem: String::new(), pin: [7; 32], started_at: 1, diff --git a/crates/features/src/relay/mod.rs b/crates/features/src/relay/mod.rs index fab07d45..fff0ed34 100644 --- a/crates/features/src/relay/mod.rs +++ b/crates/features/src/relay/mod.rs @@ -26,12 +26,15 @@ //! ([`dns::dns_change`]). //! //! Numbers are big-endian. There are few relays, so they're read whole. +//! The secrets in `k` and `r` are wrapped under the key file +//! `INBUXA_RELAY_KEY_FILE` names, when it names one ([`wrap`], RL-35). pub mod certificate; pub mod dns; pub mod link; pub mod messages; pub mod outbound; +pub mod wrap; use inbuxa_relay_proto::{ b64, @@ -47,6 +50,7 @@ use store::{ }; use trc::AddContext; use types::id::Id; +use wrap::Secret; /// Wakes the link service on this node when a relay is added, changed or /// removed here. Other nodes see it at their next scan. @@ -77,13 +81,12 @@ const MAX_DOMAINS: usize = 5_000; #[derive(Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)] #[serde(rename_all = "camelCase")] pub struct DownstreamKeys { - pub link_ca_key_pem: String, + pub link_ca_key_pem: Secret, pub link_ca_cert_pem: String, /// The pin of the link CA's key, which every relay keeps. #[serde(with = "b64")] pub link_ca_pin: Pin, - #[serde(with = "b64")] - pub sealing_secret: [u8; 32], + pub sealing_secret: Secret<[u8; 32]>, #[serde(with = "b64")] pub sealing_public: [u8; 32], pub created_at: u64, @@ -98,7 +101,7 @@ pub struct DownstreamKeys { #[derive(Clone, PartialEq, Eq, SerdeSerialize, SerdeDeserialize)] #[serde(rename_all = "camelCase")] pub struct NextLinkCa { - pub key_pem: String, + pub key_pem: Secret, pub cert_pem: String, #[serde(with = "b64")] pub pin: Pin, @@ -125,6 +128,16 @@ impl LinkCaChange { } impl DownstreamKeys { + /// Some secret is in the clear while a key file is named (RL-35). + pub fn needs_wrapping(&self) -> bool { + self.link_ca_key_pem.needs_wrapping() + || self.sealing_secret.needs_wrapping() + || self + .next_link_ca + .as_ref() + .is_some_and(|next| next.key_pem.needs_wrapping()) + } + /// The keys with the next link CA as the link CA, if there is one. pub fn finished(&self) -> Option { let next = self.next_link_ca.as_ref()?; @@ -223,8 +236,8 @@ pub struct Relay { #[serde(default, skip_serializing_if = "Option::is_none")] pub smtp_private_key: Option, /// The key this relay's recipient hashes are made with (RL-11). - #[serde(with = "b64", default)] - pub recipient_key: [u8; 32], + #[serde(default)] + pub recipient_key: Secret<[u8; 32]>, #[serde(default)] pub join: Join, #[serde(default)] @@ -401,7 +414,7 @@ impl Relay { /// to show the administrator once. pub fn start_join(&mut self, keys: &DownstreamKeys, now: u64) -> String { let token = JoinToken::issue(self.relay_id(), keys.link_ca_pin, now); - self.recipient_key = random_key(); + self.recipient_key = Secret::new(random_key()); self.join = Join { secret: Some(token.secret), expires_at: token.expires_at, @@ -429,9 +442,9 @@ pub fn new_downstream_keys(now: u64) -> trc::Result { let (sealing_secret, sealing_public) = seal::generate_keypair(); Ok(DownstreamKeys { link_ca_pin: pin, - link_ca_key_pem: key_pem, + link_ca_key_pem: Secret::new(key_pem), link_ca_cert_pem: cert_pem, - sealing_secret: *sealing_secret, + sealing_secret: Secret::new(*sealing_secret), sealing_public, created_at: now, next_link_ca: None, @@ -635,7 +648,7 @@ pub async fn change_link_ca(data: &Store, change: LinkCaChange, now: u64) -> trc let (key_pem, cert_pem, pin) = new_link_ca()?; DownstreamKeys { next_link_ca: Some(NextLinkCa { - key_pem, + key_pem: Secret::new(key_pem), cert_pem, pin, started_at: now, @@ -711,6 +724,79 @@ pub async fn replace(data: &Store, relay: &Relay, version: u64) -> trc::Result trc::Result { + if !wrap::turned_on() { + return Ok(false); + } + let mut wrapped = false; + while let Some(VersionedKeys(keys, version)) = data + .get_value::(ValueKey::from(class(KIND_KEYS, None))) + .await + .caused_by(trc::location!())? + .filter(|VersionedKeys(keys, _)| keys.needs_wrapping()) + { + let mut batch = BatchBuilder::new(); + batch.assert_value(class(KIND_KEYS, None), AssertValue::Hash(version)); + batch.set(class(KIND_KEYS, None), Json(&keys).serialize()?); + match data.write(batch.build_all()).await { + Ok(_) => { + wrapped = true; + break; + } + Err(err) + if matches!( + err.as_ref(), + trc::EventType::Store(trc::StoreEvent::AssertValueFailed) + ) => {} + Err(err) => return Err(err.caused_by(trc::location!())), + } + } + for relay in all(data).await? { + while let Some((relay, version)) = get_versioned(data, relay.id) + .await? + .filter(|(relay, _)| relay.recipient_key.needs_wrapping()) + { + if replace(data, &relay, version).await? { + wrapped = true; + break; + } + } + } + Ok(wrapped) +} + +/// The downstream keys and every relay as the store holds them: what a copy +/// of the store gives away (RL-T27). +pub async fn stored_bytes(data: &Store) -> trc::Result>> { + struct Raw(Vec); + impl Deserialize for Raw { + fn deserialize(bytes: &[u8]) -> trc::Result { + Ok(Raw(bytes.to_vec())) + } + } + let mut out = Vec::new(); + if let Some(Raw(bytes)) = data + .get_value::(ValueKey::from(class(KIND_KEYS, None))) + .await + .caused_by(trc::location!())? + { + out.push(bytes); + } + for relay in all(data).await? { + if let Some(Raw(bytes)) = data + .get_value::(relay_key(relay.id)) + .await + .caused_by(trc::location!())? + { + out.push(bytes); + } + } + Ok(out) +} + /// Removes a relay, and every node's record of its link. pub async fn delete(data: &Store, id: u32) -> trc::Result<()> { let nodes = link::statuses(data, id).await?; @@ -747,7 +833,7 @@ mod tests { }, smtp_certificate: None, smtp_private_key: None, - recipient_key: [0; 32], + recipient_key: Secret::new([0; 32]), join: Join::default(), created_by: String::new(), created_at: 0, @@ -809,16 +895,16 @@ mod tests { assert_eq!(Some(decoded.secret), r.join.secret); assert_eq!(r.join.state(now), JoinState::Waiting); assert_eq!(r.join.state(decoded.expires_at), JoinState::Expired); - assert_ne!(r.recipient_key, [0; 32]); + assert!(r.recipient_key != Secret::new([0; 32])); // A new join forgets the pinned key and the old recipient key r.join.relay_pin = Some([7; 32]); assert_eq!(r.join.state(decoded.expires_at), JoinState::Joined); - let old_key = r.recipient_key; + let old_key = r.recipient_key.clone(); let second = r.start_join(&keys, now); assert_ne!(second, token); assert_eq!(r.join.relay_pin, None); - assert_ne!(r.recipient_key, old_key); + assert!(r.recipient_key != old_key); // The pin is of the key inside the CA certificate the relay sees use base64::Engine; @@ -830,7 +916,7 @@ mod tests { .collect::(), ) .unwrap(); - let ca_key = rcgen::KeyPair::from_pem(&keys.link_ca_key_pem).unwrap(); + let ca_key = rcgen::KeyPair::from_pem(&keys.link_ca_key_pem.open().unwrap()).unwrap(); let spki = rcgen::PublicKeyData::subject_public_key_info(&ca_key); assert!(der.windows(spki.len()).any(|w| w == spki)); assert_eq!(spki_pin(&spki), keys.link_ca_pin); @@ -838,7 +924,7 @@ mod tests { // The sealing key opens what's sealed to its public half let sealed = seal::seal(&keys.sealing_public, b"aad", b"hello").unwrap(); assert_eq!( - seal::open(&keys.sealing_secret, b"aad", &sealed).unwrap(), + seal::open(&keys.sealing_secret.open().unwrap(), b"aad", &sealed).unwrap(), b"hello" ); } diff --git a/crates/features/src/relay/wrap.rs b/crates/features/src/relay/wrap.rs new file mode 100644 index 00000000..7ff212e6 --- /dev/null +++ b/crates/features/src/relay/wrap.rs @@ -0,0 +1,348 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! The downstream's relay keys, wrapped (RL-35): the link CA key, the +//! sealing key and each relay's recipient key are stored encrypted under a +//! key the store doesn't hold, from the file `INBUXA_RELAY_KEY_FILE` names. +//! +//! A [`Secret`] is stored as before, a string, while no file is named, and +//! as `{"wrapped": nonce || ciphertext}` once one is. A secret read in the +//! clear is wrapped the next time it is written, which [`wrap_stored`] does +//! for every one. Opening a wrapped secret without the file fails, saying +//! so: the link service then starts no link, and the node shows why. + +use std::{ + path::{Path, PathBuf}, + sync::{Arc, LazyLock, RwLock}, +}; + +use aws_lc_rs::aead::{AES_256_GCM, Aad, NONCE_LEN, Nonce, RandomizedNonceKey}; +use base64::{ + Engine, + engine::general_purpose::{STANDARD, URL_SAFE}, +}; +use inbuxa_relay_proto::b64; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de, ser}; + +/// Names the key file. +pub const ENV: &str = "INBUXA_RELAY_KEY_FILE"; + +/// Bound to every wrapped secret, so nothing else under the key opens as one. +const AAD: &[u8] = b"inbuxa relay key v1"; + +/// The key from the file, none when no file is named, or why it can't be +/// read. +type KeyFile = Result, String>; + +/// Read from [`ENV`] when first needed. +static KEY_FILE: LazyLock>>> = LazyLock::new(|| RwLock::new(None)); + +fn key_file() -> Arc { + if let Some(loaded) = KEY_FILE.read().unwrap_or_else(|e| e.into_inner()).as_ref() { + return loaded.clone(); + } + let path = std::env::var_os(ENV) + .filter(|path| !path.is_empty()) + .map(PathBuf::from); + set_key_file(path.as_deref()) +} + +/// Reads the key from `path`, or none, in place of what [`ENV`] names. The +/// server reads [`ENV`] itself; the tests use this to turn the file on and +/// off. +pub fn set_key_file(path: Option<&Path>) -> Arc { + let loaded = Arc::new(path.map_or(Ok(None), |path| load(path).map(Some))); + if let Err(why) = loaded.as_ref() { + trc::error!(trc::Error::from(KeyError(why.clone()))); + } + *KEY_FILE.write().unwrap_or_else(|e| e.into_inner()) = Some(loaded.clone()); + loaded +} + +/// 32 random bytes in base64, as `openssl rand -base64 32` writes them. +fn load(path: &Path) -> Result { + let text = std::fs::read_to_string(path) + .map_err(|err| format!("{ENV}: can't read {}: {err}", path.display()))?; + let text = text.trim(); + let bytes = STANDARD + .decode(text) + .or_else(|_| URL_SAFE.decode(text)) + .or_else(|_| b64::decode(text).ok_or(())) + .ok() + .filter(|bytes| bytes.len() == 32) + .ok_or_else(|| { + format!( + "{ENV}: {} doesn't hold 32 bytes in base64, as `openssl rand -base64 32` writes them.", + path.display() + ) + })?; + key(&bytes) +} + +fn key(bytes: &[u8]) -> Result { + RandomizedNonceKey::new(&AES_256_GCM, bytes).map_err(|_| format!("{ENV}: not a key.")) +} + +/// Whether a key file is named, and reads. +pub fn turned_on() -> bool { + matches!(key_file().as_ref(), Ok(Some(_))) +} + +/// A value that can be a [`Secret`]: how it is stored in the clear, and as +/// bytes to wrap. +pub trait SecretValue: Sized + Clone { + fn to_bytes(&self) -> Vec; + fn from_bytes(bytes: Vec) -> Option; + fn to_text(&self) -> String; + fn from_text(text: String) -> Option; +} + +/// A PEM private key. +impl SecretValue for String { + fn to_bytes(&self) -> Vec { + self.as_bytes().to_vec() + } + + fn from_bytes(bytes: Vec) -> Option { + String::from_utf8(bytes).ok() + } + + fn to_text(&self) -> String { + self.clone() + } + + fn from_text(text: String) -> Option { + Some(text) + } +} + +/// A key, in base64url. +impl SecretValue for [u8; 32] { + fn to_bytes(&self) -> Vec { + self.to_vec() + } + + fn from_bytes(bytes: Vec) -> Option { + bytes.try_into().ok() + } + + fn to_text(&self) -> String { + b64::encode(self) + } + + fn from_text(text: String) -> Option { + b64::decode(&text).and_then(|bytes| bytes.try_into().ok()) + } +} + +/// A secret as the store holds it. +#[derive(Clone, PartialEq, Eq)] +pub enum Secret { + /// In the clear: no key file was named when it was written. + Open(T), + /// The nonce, then the ciphertext and its tag. + Wrapped(Vec), +} + +impl Secret { + pub fn new(value: T) -> Self { + Secret::Open(value) + } + + /// The secret itself, or why it can't be opened. + pub fn open(&self) -> Result { + match self { + Secret::Open(value) => Ok(value.clone()), + Secret::Wrapped(wrapped) => match key_file().as_ref() { + Ok(Some(key)) => unwrap_with(key, wrapped).ok_or_else(|| { + KeyError(format!( + "The relay keys were wrapped with another key than the one {ENV} names." + )) + }), + Ok(None) => Err(KeyError(format!( + "The relay keys are wrapped: set {ENV} to the key file they were wrapped with." + ))), + Err(why) => Err(KeyError(why.clone())), + }, + } + } + + /// In the clear, while a key file is named: written again, it is + /// wrapped. + pub fn needs_wrapping(&self) -> bool { + matches!(self, Secret::Open(_)) && turned_on() + } + + pub fn is_wrapped(&self) -> bool { + matches!(self, Secret::Wrapped(_)) + } +} + +impl Default for Secret { + fn default() -> Self { + Secret::Open(T::default()) + } +} + +/// Why a relay key can't be opened, or the key file read: in words for +/// the node's link status. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct KeyError(pub String); + +impl std::fmt::Display for KeyError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl From for trc::Error { + fn from(err: KeyError) -> Self { + trc::EventType::Server(trc::ServerEvent::StartupError) + .into_err() + .details(err.0) + } +} + +fn wrap_with(key: &RandomizedNonceKey, value: &[u8]) -> Option> { + let mut data = value.to_vec(); + let nonce = key + .seal_in_place_append_tag(Aad::from(AAD), &mut data) + .ok()?; + let mut wrapped = nonce.as_ref().to_vec(); + wrapped.append(&mut data); + Some(wrapped) +} + +fn unwrap_with(key: &RandomizedNonceKey, wrapped: &[u8]) -> Option { + let (nonce, data) = wrapped.split_at_checked(NONCE_LEN)?; + let nonce = Nonce::try_assume_unique_for_key(nonce).ok()?; + let mut data = data.to_vec(); + let len = key + .open_in_place(nonce, Aad::from(AAD), &mut data) + .ok()? + .len(); + data.truncate(len); + T::from_bytes(data) +} + +#[derive(Serialize, Deserialize)] +#[serde(untagged)] +enum Stored { + Open(String), + Wrapped { wrapped: String }, +} + +impl Serialize for Secret { + fn serialize(&self, serializer: S) -> Result { + let wrapped = match self { + Secret::Wrapped(wrapped) => wrapped.clone(), + Secret::Open(value) => match key_file().as_ref() { + Ok(None) => return Stored::Open(value.to_text()).serialize(serializer), + Ok(Some(key)) => wrap_with(key, &value.to_bytes()) + .ok_or_else(|| ser::Error::custom("can't wrap a relay key"))?, + // Never in the clear while a file is named + Err(why) => return Err(ser::Error::custom(why)), + }, + }; + Stored::Wrapped { + wrapped: b64::encode(&wrapped), + } + .serialize(serializer) + } +} + +impl<'de, T: SecretValue> Deserialize<'de> for Secret { + fn deserialize>(deserializer: D) -> Result { + match Stored::deserialize(deserializer)? { + Stored::Open(text) => T::from_text(text) + .map(Secret::Open) + .ok_or_else(|| de::Error::custom("not a relay key")), + Stored::Wrapped { wrapped } => b64::decode(&wrapped) + .map(Secret::Wrapped) + .ok_or_else(|| de::Error::custom("not a wrapped relay key")), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wraps_and_opens() { + let key = key(&[7; 32]).unwrap(); + let wrapped = wrap_with(&key, b"-----BEGIN PRIVATE KEY-----").unwrap(); + assert!(!wrapped.windows(5).any(|w| w == b"BEGIN")); + assert_eq!( + unwrap_with::(&key, &wrapped).as_deref(), + Some("-----BEGIN PRIVATE KEY-----") + ); + // A new nonce every time + assert_ne!( + wrap_with(&key, b"same").unwrap(), + wrap_with(&key, b"same").unwrap() + ); + // Another key, or a changed byte, opens nothing + assert_eq!( + unwrap_with::(&self::key(&[8; 32]).unwrap(), &wrapped), + None + ); + let mut changed = wrapped.clone(); + *changed.last_mut().unwrap() ^= 1; + assert_eq!(unwrap_with::(&key, &changed), None); + assert_eq!(unwrap_with::(&key, &wrapped[..4]), None); + // A 32-byte key comes back as one + let wrapped = wrap_with(&key, &[3; 32]).unwrap(); + assert_eq!(unwrap_with::<[u8; 32]>(&key, &wrapped), Some([3; 32])); + } + + #[test] + fn stored_forms() { + // In the clear, as before RL-35 + let open: Secret<[u8; 32]> = + serde_json::from_str(&format!("\"{}\"", b64::encode(&[5; 32]))).unwrap(); + assert!(open == Secret::Open([5; 32])); + let open: Secret = serde_json::from_str("\"-----BEGIN\"").unwrap(); + assert!(open == Secret::Open("-----BEGIN".to_string())); + // Wrapped stays as it was, whatever the key file + let wrapped: Secret = serde_json::from_str("{\"wrapped\":\"AAEC\"}").unwrap(); + assert!(wrapped == Secret::Wrapped(vec![0, 1, 2])); + assert_eq!( + serde_json::to_string(&wrapped).unwrap(), + "{\"wrapped\":\"AAEC\"}" + ); + assert!(serde_json::from_str::>("\"short\"").is_err()); + } + + #[test] + fn key_files() { + let dir = std::env::temp_dir().join(format!("inbuxa-relay-key-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("relay.key"); + for good in [ + STANDARD.encode([1; 32]) + "\n", + URL_SAFE.encode([250; 32]), + b64::encode(&[250; 32]), + ] { + std::fs::write(&path, good).unwrap(); + assert!(load(&path).is_ok()); + } + for bad in [ + STANDARD.encode([1; 16]), + "not base64".to_string(), + String::new(), + ] { + std::fs::write(&path, bad).unwrap(); + assert!(load(&path).unwrap_err().contains("32 bytes")); + } + assert!( + load(&dir.join("missing")) + .unwrap_err() + .contains("can't read") + ); + std::fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/crates/jmap-proto/src/object/inbuxa_relay.rs b/crates/jmap-proto/src/object/inbuxa_relay.rs index c6589c88..f34e40a9 100644 --- a/crates/jmap-proto/src/object/inbuxa_relay.rs +++ b/crates/jmap-proto/src/object/inbuxa_relay.rs @@ -65,6 +65,9 @@ pub enum RelayProperty { LinkCa, /// When a replacement of the link CA began, while one is under way. LinkCaReplacing, + /// Whether the relay keys are wrapped under the key file + /// `INBUXA_RELAY_KEY_FILE` names (RL-35). + KeysWrapped, /// In a create or rejoin response only. JoinToken, /// Each node's link to the relay, and what the relay last said. @@ -114,6 +117,7 @@ impl Property for RelayProperty { RelayProperty::ReplaceLinkCa => "replaceLinkCa", RelayProperty::LinkCa => "linkCa", RelayProperty::LinkCaReplacing => "linkCaReplacing", + RelayProperty::KeysWrapped => "keysWrapped", RelayProperty::JoinToken => "joinToken", RelayProperty::Links => "links", RelayProperty::CreatedBy => "createdBy", @@ -151,6 +155,7 @@ impl RelayProperty { b"replaceLinkCa" => RelayProperty::ReplaceLinkCa, b"linkCa" => RelayProperty::LinkCa, b"linkCaReplacing" => RelayProperty::LinkCaReplacing, + b"keysWrapped" => RelayProperty::KeysWrapped, b"joinToken" => RelayProperty::JoinToken, b"links" => RelayProperty::Links, b"createdBy" => RelayProperty::CreatedBy, diff --git a/crates/jmap/src/inbuxa/relay.rs b/crates/jmap/src/inbuxa/relay.rs index fa3fff1c..19381aea 100644 --- a/crates/jmap/src/inbuxa/relay.rs +++ b/crates/jmap/src/inbuxa/relay.rs @@ -69,6 +69,7 @@ const ALL: &[P] = &[ P::OlderThanServer, P::LinkCa, P::LinkCaReplacing, + P::KeysWrapped, P::CreatedBy, P::CreatedAt, P::UpdatedAt, @@ -91,6 +92,7 @@ const SERVER_SET: &[P] = &[ P::OlderThanServer, P::LinkCa, P::LinkCaReplacing, + P::KeysWrapped, P::CreatedBy, P::CreatedAt, P::UpdatedAt, @@ -220,6 +222,9 @@ fn to_value( P::LinkCaReplacing => keys .and_then(|keys| keys.next_link_ca.as_ref()) .map_or(Value::Null, |next| date(next.started_at)), + P::KeysWrapped => { + Value::Bool(keys.is_some_and(|keys| keys.link_ca_key_pem.is_wrapped())) + } P::CreatedBy => text(Some(&relay.created_by)), P::CreatedAt => date(relay.created_at), P::UpdatedAt => date(relay.updated_at), @@ -413,7 +418,7 @@ fn apply(current: &StoredRelay, sent: &Sent) -> Result> let next = parse(json)?; Ok(StoredRelay { id: current.id, - recipient_key: current.recipient_key, + recipient_key: current.recipient_key.clone(), join: current.join.clone(), created_by: current.created_by.clone(), created_at: current.created_at, diff --git a/crates/services/src/inbuxa_relay_link.rs b/crates/services/src/inbuxa_relay_link.rs index c26203c3..291dfa50 100644 --- a/crates/services/src/inbuxa_relay_link.rs +++ b/crates/services/src/inbuxa_relay_link.rs @@ -50,6 +50,7 @@ use inbuxa_features::{ record_ingested, record_join, set_status, was_ingested, }, outbound::{self, Call, HandOver, Handed, Linked, Refused, Sender}, + wrap::KeyError, }, }; use inbuxa_relay_proto::{ @@ -184,6 +185,8 @@ async fn reconcile( links: &mut HashMap, ) -> trc::Result<()> { let data = server.store(); + // Keys stored in the clear, wrapped once a key file is named (RL-35) + relays::wrap_stored(data).await?; let (keys, all) = match relays::keys(data).await? { Some(keys) => (Some(keys), relays::all(data).await?), None => (None, Vec::new()), @@ -301,6 +304,13 @@ impl LinkError { } } +/// A relay key that can't be opened (RL-35): the node says why, in words. +impl From for LinkError { + fn from(err: KeyError) -> Self { + LinkError::down(err.0) + } +} + impl From for LinkError { fn from(err: trc::Error) -> Self { LinkError::down(format!("This server failed: {err}")) @@ -401,6 +411,9 @@ async fn link_keys( let keys = relays::keys(server.store()) .await? .ok_or_else(|| LinkError::down("This server has no relay keys yet."))?; + // Wrapped, and no key file to open them with (RL-35): no link starts + keys.sealing_secret.open()?; + keys.link_ca_key_pem.open()?; let cert = certs.get(&keys, server.core.network.node_id)?; Ok((keys, cert)) } @@ -1010,7 +1023,7 @@ async fn session( dialer: dialer.clone(), relay_id: relay.id, relay_name: relay.name.clone(), - secret: keys.sealing_secret, + secret: keys.sealing_secret.open()?, taking: HashSet::new(), answers: answers_tx, }; @@ -1807,7 +1820,7 @@ async fn sync( } // Settings, whole, when they changed - let settings = pushed_settings(relay, keys, &plan); + let settings = pushed_settings(relay, keys, &plan)?; if sent.settings.as_ref() != Some(&settings) { let mut patch = serde_json::to_value(&settings).unwrap_or_default(); // The relay merges what it is sent: a setting left out stays as it @@ -1872,7 +1885,10 @@ async fn sync( // Recipients: in full on a new link or after many changes, else as // changes - let hashes: BTreeSet = plan.hashes(&relay.recipient_key).into_iter().collect(); + let hashes: BTreeSet = plan + .hashes(&relay.recipient_key.open()?) + .into_iter() + .collect(); let added: Vec<&RecipientHash> = hashes.difference(&sent.hashes).collect(); let removed: Vec<&RecipientHash> = sent.hashes.difference(&hashes).collect(); if !sent.started || added.len() + removed.len() > MAX_DELTA { diff --git a/docs/spec/features/relay.md b/docs/spec/features/relay.md index f6468c31..cd40a410 100644 --- a/docs/spec/features/relay.md +++ b/docs/spec/features/relay.md @@ -205,6 +205,19 @@ the protocol the rest of the suite already speaks. turned on. Without the setting, keys are stored as today, as registry secrets. + How it goes: + - The file holds 32 random bytes in base64, as `openssl rand -base64 32` + writes them. Each key is wrapped with AES-256-GCM and a random nonce, + stored as `{"wrapped": …}` where it was a string. + - Once the setting names a file, the link service wraps every key still + stored in the clear, within a scan. Keys stay wrapped: taking the + setting away later doesn't unwrap them. + - A node that can't open them, because the setting is missing, the file + can't be read, or it holds another key, starts no link. Each of its + links shows why in `lastError`. While the setting names a file that + can't be read, no key is written in the clear. + - `inbuxa:Relay` shows `keysWrapped`. + ### Joining and reset 1. In inbuxa-admin, an administrator adds a relay: a name and the address the @@ -537,7 +550,7 @@ When no link is up, or after RL-13: | Object | What it holds | |---|---| -| `inbuxa:Relay` | One per relay: `name`, `address`, `linkPort`, `enabled`, the settings pushed to the relay, `sendThrough` (route outbound mail through relays), `inboundOnly` (takes mail in but sends none, e.g. while its IP isn't warmed up or isn't in SPF yet), `mxPreference` (its MX preference, 10 by default, RL-25), `certificateNotAfter` and `certificateError` (when its certificate runs out, and why the last order failed, RL-27), `olderThanServer` (a link last heard an older release than this server's, RL-28), `linkCa` and `linkCaReplacing` (RL-34), and read-only status mirrored from the relay: link state per node, version, queue counts, `lastSeen`. Create, or an update with `rejoin: true`, returns `joinToken` once. An update with `upgrade: {blobId, signature}`, sent on its own, hands the relay a release over the node's link (RL-29). An update with `replaceLinkCa`, sent on its own, starts, finishes or cancels a replacement of the shared link CA (RL-34) | +| `inbuxa:Relay` | One per relay: `name`, `address`, `linkPort`, `enabled`, the settings pushed to the relay, `sendThrough` (route outbound mail through relays), `inboundOnly` (takes mail in but sends none, e.g. while its IP isn't warmed up or isn't in SPF yet), `mxPreference` (its MX preference, 10 by default, RL-25), `certificateNotAfter` and `certificateError` (when its certificate runs out, and why the last order failed, RL-27), `olderThanServer` (a link last heard an older release than this server's, RL-28), `linkCa` and `linkCaReplacing` (RL-34), `keysWrapped` (RL-35), and read-only status mirrored from the relay: link state per node, version, queue counts, `lastSeen`. Create, or an update with `rejoin: true`, returns `joinToken` once. An update with `upgrade: {blobId, signature}`, sent on its own, hands the relay a release over the node's link (RL-29). An update with `replaceLinkCa`, sent on its own, starts, finishes or cancels a replacement of the shared link CA (RL-34) | | `inbuxa:RelayMessage` | Read-only view of a relay's `Inbound` and `Outbound` queues, fetched over the link when asked. Never message content. `/get`, `/query` (filter `relayId`, `direction`; newest first) and `/set` destroy only: discards held mail, or cancels an outbound message, whose recipients fail here with no DSN as a delivery cancelled in the queue does. Each id is the relay's id and a hash of the relay's own message id, so nothing is stored | Permissions: `sysRelayGet`, `sysRelayUpdate`, `sysRelayQueueGet`, diff --git a/tests/src/system/relay.rs b/tests/src/system/relay.rs index 9331a352..b7805502 100644 --- a/tests/src/system/relay.rs +++ b/tests/src/system/relay.rs @@ -24,7 +24,7 @@ use crate::utils::{ dns::DnsCache, server::{TestServer, TestServerBuilder}, }; -use inbuxa_relay_proto::{time, token::JoinToken}; +use inbuxa_relay_proto::{b64, time, token::JoinToken}; use mail_auth::{common::parse::TxtRecordParser, mta_sts::TlsRpt}; use registry::{ schema::{ @@ -279,6 +279,8 @@ pub async fn test(test: &mut TestServer) { ); // A relay still joining has said nothing of its CA assert_eq!(get(admin, &id).await["linkCa"], Value::Null); + // No key file: the keys are stored as before RL-35 + assert_eq!(get(admin, &id).await["keysWrapped"], false); let (_, response) = call( admin, "inbuxa:Relay/set", @@ -1453,6 +1455,92 @@ pub async fn link(test: &mut TestServer) { assert_eq!(identity["caPin"], new_pin, "the relay keeps the new CA"); assert_eq!(relay["joinState"], "joined", "without a new join"); + // RL-T27: with a key file, a copy of the store holds none of the relay + // keys in the clear... + use inbuxa_features::relay as relays; + let data = test.server.store().clone(); + let keys = relays::keys(&data).await.unwrap().unwrap(); + let mut clear = vec![ + keys.link_ca_key_pem + .open() + .unwrap() + .lines() + .nth(1) + .unwrap() + .to_string(), + b64::encode(&keys.sealing_secret.open().unwrap()), + ]; + for relay in relays::all(&data).await.unwrap() { + clear.push(b64::encode(&relay.recipient_key.open().unwrap())); + } + let in_the_clear = async || { + relays::stored_bytes(&data) + .await + .unwrap() + .iter() + .any(|bytes| { + clear + .iter() + .any(|secret| bytes.windows(secret.len()).any(|w| w == secret.as_bytes())) + }) + }; + assert!(in_the_clear().await, "in the clear without one, as before"); + let key_file = test.temp_dir.path.join("relay.key"); + std::fs::write(&key_file, b64::encode(&relays::random_key())).unwrap(); + relays::wrap::set_key_file(Some(&key_file)); + relays::changed(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while in_the_clear().await { + assert!( + std::time::Instant::now() < deadline, + "the relay keys were never wrapped" + ); + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + } + wait_for( + admin, + &id, + "the keys wrapped, and still linked", + 30, + |relay| relay["keysWrapped"] == true && relay["links"][0]["state"] == "linked", + ) + .await; + // ...and a node without the file starts no relay link, and says why + call( + admin, + "inbuxa:Relay/set", + json!({"update": {&id: {"enabled": false}}}), + ) + .await; + wait_for(admin, &id, "the link ended", 30, |relay| { + relay["links"][0]["state"] == "down" + }) + .await; + relays::wrap::set_key_file(None); + let reply = send_mail(smtp_port, "jane@example.com", &message("Wrapped one", "")).await; + assert!(reply.starts_with("250"), "held: {reply}"); + call( + admin, + "inbuxa:Relay/set", + json!({"update": {&id: {"enabled": true}}}), + ) + .await; + wait_for(admin, &id, "no link without the key file", 30, |relay| { + relay["links"][0]["state"] == "down" + && relay["links"][0]["lastError"] + .as_str() + .is_some_and(|e| e.contains("INBUXA_RELAY_KEY_FILE")) + }) + .await; + // With it again, the link is back, and the mail held meanwhile opens + relays::wrap::set_key_file(Some(&key_file)); + relays::changed(); + wait_for_mail(&jane, "Wrapped one", 1).await; + wait_for(admin, &id, "linked again", 30, |relay| { + relay["links"][0]["state"] == "linked" + }) + .await; + // The relay's events were taken as this server's relay.* trace events, // and acknowledged, so the relay forgot them let events = state.join("events.jsonl"); -- 2.54.0