Certificates from mounted directories, reread when renewed #477
No files matched your search
@@ -11,7 +11,7 @@ use ahash::{AHashMap, AHashSet};
|
||||
use rcgen::generate_simple_self_signed;
|
||||
use registry::{
|
||||
schema::{
|
||||
prelude::Object,
|
||||
prelude::{Object, ObjectType},
|
||||
structs::{Certificate, PublicText, SecretText, SystemSettings},
|
||||
},
|
||||
types::{datetime::UTCDateTime, map::Map},
|
||||
@@ -24,12 +24,32 @@ use rustls::{
|
||||
};
|
||||
use rustls_pemfile::{Item, certs, read_all};
|
||||
use rustls_pki_types::PrivateKeyDer;
|
||||
use std::{io::Cursor, sync::Arc};
|
||||
use std::{io::Cursor, path::PathBuf, sync::Arc};
|
||||
use store::{
|
||||
registry::{bootstrap::Bootstrap, write::RegistryWrite},
|
||||
write::now,
|
||||
};
|
||||
|
||||
/// inbuxa: K-C8. Directories, each holding tls.crt and tls.key as a
|
||||
/// Kubernetes TLS Secret mounts them, whose certificates the server uses
|
||||
/// without storing them: cert-manager renews them in place, and every node
|
||||
/// reads its own copy. Separated by commas or whitespace.
|
||||
pub const CERTIFICATE_DIRS_VAR: &str = "TLS_CERTIFICATE_DIRS";
|
||||
|
||||
/// The files of each directory in INBUXA_TLS_CERTIFICATE_DIRS: (certificate,
|
||||
/// private key)
|
||||
pub fn certificate_dir_files() -> Vec<(PathBuf, PathBuf)> {
|
||||
types::branding::env_var(CERTIFICATE_DIRS_VAR)
|
||||
.unwrap_or_default()
|
||||
.split(|c: char| c == ',' || c.is_whitespace())
|
||||
.filter(|dir| !dir.is_empty())
|
||||
.map(|dir| {
|
||||
let dir = PathBuf::from(dir);
|
||||
(dir.join("tls.crt"), dir.join("tls.key"))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub static TLS13_VERSION: &[&SupportedProtocolVersion] = &[&TLS13];
|
||||
pub static TLS12_VERSION: &[&SupportedProtocolVersion] = &[&TLS12];
|
||||
|
||||
@@ -196,6 +216,78 @@ pub(crate) async fn parse_certificates(
|
||||
}
|
||||
}
|
||||
|
||||
// inbuxa: K-C8, certificates kept in mounted directories, not in the store
|
||||
for (cert_path, key_path) in certificate_dir_files() {
|
||||
let read = async {
|
||||
let public = tokio::fs::read(&cert_path)
|
||||
.await
|
||||
.map_err(|err| format!("{}: {err}", cert_path.display()))?;
|
||||
let secret = tokio::fs::read(&key_path)
|
||||
.await
|
||||
.map_err(|err| format!("{}: {err}", key_path.display()))?;
|
||||
let parsed = ParsedCert::parse(&public)
|
||||
.map_err(|err| format!("{}: {err}", cert_path.display()))?;
|
||||
Ok::<_, String>((public, secret, parsed))
|
||||
};
|
||||
let (public, secret, parsed) = match read.await {
|
||||
Ok(read) => read,
|
||||
Err(err) => {
|
||||
bp.build_error(
|
||||
ObjectType::Certificate.singleton(),
|
||||
format!(
|
||||
"Failed to read the certificate in INBUXA_{CERTIFICATE_DIRS_VAR}: {err}"
|
||||
),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let not_valid_after = parsed.valid_not_after.timestamp();
|
||||
if not_valid_after <= now || parsed.valid_not_before.timestamp() > now {
|
||||
bp.build_error(
|
||||
ObjectType::Certificate.singleton(),
|
||||
format!(
|
||||
"The certificate in {} is not valid now (from {} to {})",
|
||||
cert_path.display(),
|
||||
parsed.valid_not_before,
|
||||
parsed.valid_not_after
|
||||
),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let mut names = parsed.sans;
|
||||
names.retain(|name| {
|
||||
if certs_expirations
|
||||
.get(name)
|
||||
.is_none_or(|expires| *expires < not_valid_after)
|
||||
{
|
||||
certs_expirations.insert(name.clone(), not_valid_after);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
});
|
||||
match build_certified_key(public, secret) {
|
||||
Ok(key) => {
|
||||
let key = Arc::new(key);
|
||||
for name in names {
|
||||
subject_names.insert(name.as_str().into());
|
||||
certificates.insert(
|
||||
name.strip_prefix("*.")
|
||||
.map(Into::into)
|
||||
.unwrap_or_else(|| name.into_boxed_str()),
|
||||
key.clone(),
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
bp.build_error(
|
||||
ObjectType::Certificate.singleton(),
|
||||
format!("Invalid certificate in {}: {err}", cert_path.display()),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// No default certificate set: pick one now, once per load, so handshakes
|
||||
// without SNI or for an unknown name get a predictable certificate.
|
||||
if !certificates.contains_key("*") && certificates.len() > 1 {
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Rereads certificates kept in files when the files change: stored
|
||||
//! certificates whose certificate or key is a File, and the directories in
|
||||
//! INBUXA_TLS_CERTIFICATE_DIRS. Something outside the server renews them
|
||||
//! (cert-manager, in Kubernetes, rewrites a mounted Secret), and
|
||||
//! certificates are otherwise read only at start and when a Certificate
|
||||
//! changes in the store. Files are local, so every node
|
||||
//! watches its own: once a minute, it reads the files of every file-backed
|
||||
//! certificate and reloads the certificates when their contents differ from
|
||||
//! the last look. A reload that finds new dates writes them to the store,
|
||||
//! which tells the other nodes to reload too.
|
||||
|
||||
use common::{
|
||||
BuildServer, Inner, Server, config::server::tls::certificate_dir_files, ipc::RegistryChange,
|
||||
};
|
||||
use registry::schema::{
|
||||
prelude::ObjectType,
|
||||
structs::{Certificate, PublicText, SecretText},
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
const EVERY: Duration = Duration::from_secs(60);
|
||||
|
||||
pub fn spawn_certificate_files(inner: Arc<Inner>) {
|
||||
tokio::spawn(async move {
|
||||
let mut last = None;
|
||||
loop {
|
||||
let server = inner.build_server();
|
||||
match fingerprint(&server).await {
|
||||
Ok(seen) => {
|
||||
// The first look is the baseline: the certificates were
|
||||
// read when the server started
|
||||
if last.as_ref().is_some_and(|last| *last != seen) {
|
||||
reload(&server).await;
|
||||
}
|
||||
last = Some(seen);
|
||||
}
|
||||
Err(err) => {
|
||||
trc::error!(err.details("Failed to list file-backed TLS certificates"));
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(EVERY).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// The contents of every file a certificate is kept in, hashed together.
|
||||
/// None when no certificate is kept in a file. A file that can't be read
|
||||
/// counts as empty: when it comes back, it's a change.
|
||||
async fn fingerprint(server: &Server) -> trc::Result<Option<[u8; 32]>> {
|
||||
let mut paths = Vec::new();
|
||||
for (cert, key) in certificate_dir_files() {
|
||||
paths.push(cert.display().to_string());
|
||||
paths.push(key.display().to_string());
|
||||
}
|
||||
for cert in server.registry().list::<Certificate>().await? {
|
||||
if let PublicText::File(file) = &cert.object.certificate {
|
||||
paths.push(file.file_path.clone());
|
||||
}
|
||||
if let SecretText::File(file) = &cert.object.private_key {
|
||||
paths.push(file.file_path.clone());
|
||||
}
|
||||
}
|
||||
if paths.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
paths.sort();
|
||||
paths.dedup();
|
||||
let mut hash = Sha256::new();
|
||||
for path in paths {
|
||||
let contents = tokio::fs::read(&path).await.unwrap_or_default();
|
||||
hash.update((path.len() as u64).to_be_bytes());
|
||||
hash.update(path.as_bytes());
|
||||
hash.update((contents.len() as u64).to_be_bytes());
|
||||
hash.update(&contents);
|
||||
}
|
||||
Ok(Some(hash.finalize().into()))
|
||||
}
|
||||
|
||||
async fn reload(server: &Server) {
|
||||
match Box::pin(server.reload_registry(RegistryChange::Reload(ObjectType::Certificate))).await {
|
||||
// A certificate the files no longer give is logged as a build error;
|
||||
// a renewed one shows by its new dates in the store
|
||||
Ok(result) => result.log(),
|
||||
Err(err) => {
|
||||
trc::error!(
|
||||
err.details("Certificate files changed, but reloading them failed")
|
||||
.caused_by(trc::location!())
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,7 @@ pub mod broadcast;
|
||||
// inbuxa: AL-5, delegations end at their date
|
||||
pub mod inbuxa_blob_cleanup; // inbuxa: cleaning up after a blob store change
|
||||
pub mod inbuxa_blob_move; // inbuxa: changing the blob store
|
||||
pub mod inbuxa_certificate_files; // inbuxa: certificates renewed in their files
|
||||
pub mod inbuxa_deliverability; // inbuxa: the deliverability check
|
||||
pub mod inbuxa_lock_expiry;
|
||||
pub mod inbuxa_log_retention; // inbuxa: personal-data catalog, D1
|
||||
@@ -79,6 +80,9 @@ impl SpawnServices for IpcReceivers {
|
||||
// inbuxa: AL-5, end delegations at their `until`
|
||||
inbuxa_lock_expiry::spawn_lock_expiry(inner.clone());
|
||||
|
||||
// inbuxa: K-C8, certificates renewed in their files (cert-manager)
|
||||
inbuxa_certificate_files::spawn_certificate_files(inner.clone());
|
||||
|
||||
// inbuxa: personal-data catalog, D1: old log files go, per node
|
||||
inbuxa_log_retention::spawn_log_retention(inner.clone());
|
||||
|
||||
|
||||
Reference in new issue
Block a user