Certificates from mounted directories, reread when renewed #477

Merged
jcoffey-dev merged 1 commits from feat/certificate-file-reload into main 2026-11-19 15:12:23 +00:00
3 changed files with 196 additions and 2 deletions

No files matched your search

+94 -2
View File
@@ -11,7 +11,7 @@ use ahash::{AHashMap, AHashSet};
use rcgen::generate_simple_self_signed;
use registry::{
schema::{
prelude::Object,
prelude::{Object, ObjectType},
structs::{Certificate, PublicText, SecretText, SystemSettings},
},
types::{datetime::UTCDateTime, map::Map},
@@ -24,12 +24,32 @@ use rustls::{
};
use rustls_pemfile::{Item, certs, read_all};
use rustls_pki_types::PrivateKeyDer;
use std::{io::Cursor, sync::Arc};
use std::{io::Cursor, path::PathBuf, sync::Arc};
use store::{
registry::{bootstrap::Bootstrap, write::RegistryWrite},
write::now,
};
/// inbuxa: K-C8. Directories, each holding tls.crt and tls.key as a
/// Kubernetes TLS Secret mounts them, whose certificates the server uses
/// without storing them: cert-manager renews them in place, and every node
/// reads its own copy. Separated by commas or whitespace.
pub const CERTIFICATE_DIRS_VAR: &str = "TLS_CERTIFICATE_DIRS";
/// The files of each directory in INBUXA_TLS_CERTIFICATE_DIRS: (certificate,
/// private key)
pub fn certificate_dir_files() -> Vec<(PathBuf, PathBuf)> {
types::branding::env_var(CERTIFICATE_DIRS_VAR)
.unwrap_or_default()
.split(|c: char| c == ',' || c.is_whitespace())
.filter(|dir| !dir.is_empty())
.map(|dir| {
let dir = PathBuf::from(dir);
(dir.join("tls.crt"), dir.join("tls.key"))
})
.collect()
}
pub static TLS13_VERSION: &[&SupportedProtocolVersion] = &[&TLS13];
pub static TLS12_VERSION: &[&SupportedProtocolVersion] = &[&TLS12];
@@ -196,6 +216,78 @@ pub(crate) async fn parse_certificates(
}
}
// inbuxa: K-C8, certificates kept in mounted directories, not in the store
for (cert_path, key_path) in certificate_dir_files() {
let read = async {
let public = tokio::fs::read(&cert_path)
.await
.map_err(|err| format!("{}: {err}", cert_path.display()))?;
let secret = tokio::fs::read(&key_path)
.await
.map_err(|err| format!("{}: {err}", key_path.display()))?;
let parsed = ParsedCert::parse(&public)
.map_err(|err| format!("{}: {err}", cert_path.display()))?;
Ok::<_, String>((public, secret, parsed))
};
let (public, secret, parsed) = match read.await {
Ok(read) => read,
Err(err) => {
bp.build_error(
ObjectType::Certificate.singleton(),
format!(
"Failed to read the certificate in INBUXA_{CERTIFICATE_DIRS_VAR}: {err}"
),
);
continue;
}
};
let not_valid_after = parsed.valid_not_after.timestamp();
if not_valid_after <= now || parsed.valid_not_before.timestamp() > now {
bp.build_error(
ObjectType::Certificate.singleton(),
format!(
"The certificate in {} is not valid now (from {} to {})",
cert_path.display(),
parsed.valid_not_before,
parsed.valid_not_after
),
);
continue;
}
let mut names = parsed.sans;
names.retain(|name| {
if certs_expirations
.get(name)
.is_none_or(|expires| *expires < not_valid_after)
{
certs_expirations.insert(name.clone(), not_valid_after);
true
} else {
false
}
});
match build_certified_key(public, secret) {
Ok(key) => {
let key = Arc::new(key);
for name in names {
subject_names.insert(name.as_str().into());
certificates.insert(
name.strip_prefix("*.")
.map(Into::into)
.unwrap_or_else(|| name.into_boxed_str()),
key.clone(),
);
}
}
Err(err) => {
bp.build_error(
ObjectType::Certificate.singleton(),
format!("Invalid certificate in {}: {err}", cert_path.display()),
);
}
}
}
// No default certificate set: pick one now, once per load, so handshakes
// without SNI or for an unknown name get a predictable certificate.
if !certificates.contains_key("*") && certificates.len() > 1 {
@@ -0,0 +1,98 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! Rereads certificates kept in files when the files change: stored
//! certificates whose certificate or key is a File, and the directories in
//! INBUXA_TLS_CERTIFICATE_DIRS. Something outside the server renews them
//! (cert-manager, in Kubernetes, rewrites a mounted Secret), and
//! certificates are otherwise read only at start and when a Certificate
//! changes in the store. Files are local, so every node
//! watches its own: once a minute, it reads the files of every file-backed
//! certificate and reloads the certificates when their contents differ from
//! the last look. A reload that finds new dates writes them to the store,
//! which tells the other nodes to reload too.
use common::{
BuildServer, Inner, Server, config::server::tls::certificate_dir_files, ipc::RegistryChange,
};
use registry::schema::{
prelude::ObjectType,
structs::{Certificate, PublicText, SecretText},
};
use sha2::{Digest, Sha256};
use std::{sync::Arc, time::Duration};
const EVERY: Duration = Duration::from_secs(60);
pub fn spawn_certificate_files(inner: Arc<Inner>) {
tokio::spawn(async move {
let mut last = None;
loop {
let server = inner.build_server();
match fingerprint(&server).await {
Ok(seen) => {
// The first look is the baseline: the certificates were
// read when the server started
if last.as_ref().is_some_and(|last| *last != seen) {
reload(&server).await;
}
last = Some(seen);
}
Err(err) => {
trc::error!(err.details("Failed to list file-backed TLS certificates"));
}
}
tokio::time::sleep(EVERY).await;
}
});
}
/// The contents of every file a certificate is kept in, hashed together.
/// None when no certificate is kept in a file. A file that can't be read
/// counts as empty: when it comes back, it's a change.
async fn fingerprint(server: &Server) -> trc::Result<Option<[u8; 32]>> {
let mut paths = Vec::new();
for (cert, key) in certificate_dir_files() {
paths.push(cert.display().to_string());
paths.push(key.display().to_string());
}
for cert in server.registry().list::<Certificate>().await? {
if let PublicText::File(file) = &cert.object.certificate {
paths.push(file.file_path.clone());
}
if let SecretText::File(file) = &cert.object.private_key {
paths.push(file.file_path.clone());
}
}
if paths.is_empty() {
return Ok(None);
}
paths.sort();
paths.dedup();
let mut hash = Sha256::new();
for path in paths {
let contents = tokio::fs::read(&path).await.unwrap_or_default();
hash.update((path.len() as u64).to_be_bytes());
hash.update(path.as_bytes());
hash.update((contents.len() as u64).to_be_bytes());
hash.update(&contents);
}
Ok(Some(hash.finalize().into()))
}
async fn reload(server: &Server) {
match Box::pin(server.reload_registry(RegistryChange::Reload(ObjectType::Certificate))).await {
// A certificate the files no longer give is logged as a build error;
// a renewed one shows by its new dates in the store
Ok(result) => result.log(),
Err(err) => {
trc::error!(
err.details("Certificate files changed, but reloading them failed")
.caused_by(trc::location!())
);
}
}
}
+4
View File
@@ -25,6 +25,7 @@ pub mod broadcast;
// inbuxa: AL-5, delegations end at their date
pub mod inbuxa_blob_cleanup; // inbuxa: cleaning up after a blob store change
pub mod inbuxa_blob_move; // inbuxa: changing the blob store
pub mod inbuxa_certificate_files; // inbuxa: certificates renewed in their files
pub mod inbuxa_deliverability; // inbuxa: the deliverability check
pub mod inbuxa_lock_expiry;
pub mod inbuxa_log_retention; // inbuxa: personal-data catalog, D1
@@ -79,6 +80,9 @@ impl SpawnServices for IpcReceivers {
// inbuxa: AL-5, end delegations at their `until`
inbuxa_lock_expiry::spawn_lock_expiry(inner.clone());
// inbuxa: K-C8, certificates renewed in their files (cert-manager)
inbuxa_certificate_files::spawn_certificate_files(inner.clone());
// inbuxa: personal-data catalog, D1: old log files go, per node
inbuxa_log_retention::spawn_log_retention(inner.clone());