From 34c5ee53aaf863d42eaddd8917208f96c8ed7c33 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 19 Nov 2026 07:15:16 -0700 Subject: [PATCH] Certificates from mounted directories, reread when renewed In Kubernetes, cert-manager keeps a certificate in a TLS Secret and renews it in place. The server read certificates only at start and when a Certificate changed in the store, so a renewal waited for a restart, and a stored Certificate pointing at the files had to be created by hand on every new store. INBUXA_TLS_CERTIFICATE_DIRS names directories, each holding tls.crt and tls.key as a TLS Secret mounts them. Their certificates are used for the names in them and aren't stored. Every node checks the files of these and of file-backed stored certificates once a minute, and reloads its certificates when they change. A directory that can't be read, or a certificate not valid now, is a build error and left out. K-C8. --- crates/common/src/config/server/tls.rs | 96 +++++++++++++++++- .../services/src/inbuxa_certificate_files.rs | 98 +++++++++++++++++++ crates/services/src/lib.rs | 4 + 3 files changed, 196 insertions(+), 2 deletions(-) create mode 100644 crates/services/src/inbuxa_certificate_files.rs diff --git a/crates/common/src/config/server/tls.rs b/crates/common/src/config/server/tls.rs index a59189e7..58a26046 100644 --- a/crates/common/src/config/server/tls.rs +++ b/crates/common/src/config/server/tls.rs @@ -11,7 +11,7 @@ use ahash::{AHashMap, AHashSet}; use rcgen::generate_simple_self_signed; use registry::{ schema::{ - prelude::Object, + prelude::{Object, ObjectType}, structs::{Certificate, PublicText, SecretText, SystemSettings}, }, types::{datetime::UTCDateTime, map::Map}, @@ -24,12 +24,32 @@ use rustls::{ }; use rustls_pemfile::{Item, certs, read_all}; use rustls_pki_types::PrivateKeyDer; -use std::{io::Cursor, sync::Arc}; +use std::{io::Cursor, path::PathBuf, sync::Arc}; use store::{ registry::{bootstrap::Bootstrap, write::RegistryWrite}, write::now, }; +/// inbuxa: K-C8. Directories, each holding tls.crt and tls.key as a +/// Kubernetes TLS Secret mounts them, whose certificates the server uses +/// without storing them: cert-manager renews them in place, and every node +/// reads its own copy. Separated by commas or whitespace. +pub const CERTIFICATE_DIRS_VAR: &str = "TLS_CERTIFICATE_DIRS"; + +/// The files of each directory in INBUXA_TLS_CERTIFICATE_DIRS: (certificate, +/// private key) +pub fn certificate_dir_files() -> Vec<(PathBuf, PathBuf)> { + types::branding::env_var(CERTIFICATE_DIRS_VAR) + .unwrap_or_default() + .split(|c: char| c == ',' || c.is_whitespace()) + .filter(|dir| !dir.is_empty()) + .map(|dir| { + let dir = PathBuf::from(dir); + (dir.join("tls.crt"), dir.join("tls.key")) + }) + .collect() +} + pub static TLS13_VERSION: &[&SupportedProtocolVersion] = &[&TLS13]; pub static TLS12_VERSION: &[&SupportedProtocolVersion] = &[&TLS12]; @@ -196,6 +216,78 @@ pub(crate) async fn parse_certificates( } } + // inbuxa: K-C8, certificates kept in mounted directories, not in the store + for (cert_path, key_path) in certificate_dir_files() { + let read = async { + let public = tokio::fs::read(&cert_path) + .await + .map_err(|err| format!("{}: {err}", cert_path.display()))?; + let secret = tokio::fs::read(&key_path) + .await + .map_err(|err| format!("{}: {err}", key_path.display()))?; + let parsed = ParsedCert::parse(&public) + .map_err(|err| format!("{}: {err}", cert_path.display()))?; + Ok::<_, String>((public, secret, parsed)) + }; + let (public, secret, parsed) = match read.await { + Ok(read) => read, + Err(err) => { + bp.build_error( + ObjectType::Certificate.singleton(), + format!( + "Failed to read the certificate in INBUXA_{CERTIFICATE_DIRS_VAR}: {err}" + ), + ); + continue; + } + }; + let not_valid_after = parsed.valid_not_after.timestamp(); + if not_valid_after <= now || parsed.valid_not_before.timestamp() > now { + bp.build_error( + ObjectType::Certificate.singleton(), + format!( + "The certificate in {} is not valid now (from {} to {})", + cert_path.display(), + parsed.valid_not_before, + parsed.valid_not_after + ), + ); + continue; + } + let mut names = parsed.sans; + names.retain(|name| { + if certs_expirations + .get(name) + .is_none_or(|expires| *expires < not_valid_after) + { + certs_expirations.insert(name.clone(), not_valid_after); + true + } else { + false + } + }); + match build_certified_key(public, secret) { + Ok(key) => { + let key = Arc::new(key); + for name in names { + subject_names.insert(name.as_str().into()); + certificates.insert( + name.strip_prefix("*.") + .map(Into::into) + .unwrap_or_else(|| name.into_boxed_str()), + key.clone(), + ); + } + } + Err(err) => { + bp.build_error( + ObjectType::Certificate.singleton(), + format!("Invalid certificate in {}: {err}", cert_path.display()), + ); + } + } + } + // No default certificate set: pick one now, once per load, so handshakes // without SNI or for an unknown name get a predictable certificate. if !certificates.contains_key("*") && certificates.len() > 1 { diff --git a/crates/services/src/inbuxa_certificate_files.rs b/crates/services/src/inbuxa_certificate_files.rs new file mode 100644 index 00000000..938c9040 --- /dev/null +++ b/crates/services/src/inbuxa_certificate_files.rs @@ -0,0 +1,98 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! Rereads certificates kept in files when the files change: stored +//! certificates whose certificate or key is a File, and the directories in +//! INBUXA_TLS_CERTIFICATE_DIRS. Something outside the server renews them +//! (cert-manager, in Kubernetes, rewrites a mounted Secret), and +//! certificates are otherwise read only at start and when a Certificate +//! changes in the store. Files are local, so every node +//! watches its own: once a minute, it reads the files of every file-backed +//! certificate and reloads the certificates when their contents differ from +//! the last look. A reload that finds new dates writes them to the store, +//! which tells the other nodes to reload too. + +use common::{ + BuildServer, Inner, Server, config::server::tls::certificate_dir_files, ipc::RegistryChange, +}; +use registry::schema::{ + prelude::ObjectType, + structs::{Certificate, PublicText, SecretText}, +}; +use sha2::{Digest, Sha256}; +use std::{sync::Arc, time::Duration}; + +const EVERY: Duration = Duration::from_secs(60); + +pub fn spawn_certificate_files(inner: Arc) { + tokio::spawn(async move { + let mut last = None; + loop { + let server = inner.build_server(); + match fingerprint(&server).await { + Ok(seen) => { + // The first look is the baseline: the certificates were + // read when the server started + if last.as_ref().is_some_and(|last| *last != seen) { + reload(&server).await; + } + last = Some(seen); + } + Err(err) => { + trc::error!(err.details("Failed to list file-backed TLS certificates")); + } + } + tokio::time::sleep(EVERY).await; + } + }); +} + +/// The contents of every file a certificate is kept in, hashed together. +/// None when no certificate is kept in a file. A file that can't be read +/// counts as empty: when it comes back, it's a change. +async fn fingerprint(server: &Server) -> trc::Result> { + let mut paths = Vec::new(); + for (cert, key) in certificate_dir_files() { + paths.push(cert.display().to_string()); + paths.push(key.display().to_string()); + } + for cert in server.registry().list::().await? { + if let PublicText::File(file) = &cert.object.certificate { + paths.push(file.file_path.clone()); + } + if let SecretText::File(file) = &cert.object.private_key { + paths.push(file.file_path.clone()); + } + } + if paths.is_empty() { + return Ok(None); + } + paths.sort(); + paths.dedup(); + let mut hash = Sha256::new(); + for path in paths { + let contents = tokio::fs::read(&path).await.unwrap_or_default(); + hash.update((path.len() as u64).to_be_bytes()); + hash.update(path.as_bytes()); + hash.update((contents.len() as u64).to_be_bytes()); + hash.update(&contents); + } + Ok(Some(hash.finalize().into())) +} + +async fn reload(server: &Server) { + match Box::pin(server.reload_registry(RegistryChange::Reload(ObjectType::Certificate))).await { + // A certificate the files no longer give is logged as a build error; + // a renewed one shows by its new dates in the store + Ok(result) => result.log(), + Err(err) => { + trc::error!( + err.details("Certificate files changed, but reloading them failed") + .caused_by(trc::location!()) + ); + } + } +} diff --git a/crates/services/src/lib.rs b/crates/services/src/lib.rs index af7b3252..445da7d9 100644 --- a/crates/services/src/lib.rs +++ b/crates/services/src/lib.rs @@ -25,6 +25,7 @@ pub mod broadcast; // inbuxa: AL-5, delegations end at their date pub mod inbuxa_blob_cleanup; // inbuxa: cleaning up after a blob store change pub mod inbuxa_blob_move; // inbuxa: changing the blob store +pub mod inbuxa_certificate_files; // inbuxa: certificates renewed in their files pub mod inbuxa_deliverability; // inbuxa: the deliverability check pub mod inbuxa_lock_expiry; pub mod inbuxa_log_retention; // inbuxa: personal-data catalog, D1 @@ -79,6 +80,9 @@ impl SpawnServices for IpcReceivers { // inbuxa: AL-5, end delegations at their `until` inbuxa_lock_expiry::spawn_lock_expiry(inner.clone()); + // inbuxa: K-C8, certificates renewed in their files (cert-manager) + inbuxa_certificate_files::spawn_certificate_files(inner.clone()); + // inbuxa: personal-data catalog, D1: old log files go, per node inbuxa_log_retention::spawn_log_retention(inner.clone()); -- 2.54.0