Listeners that can't open: retry, then report through readiness #487

Merged
jcoffey-dev merged 1 commits from feat/485-listener-bind-failure into main 2026-11-19 17:52:18 +00:00
15 changed files with 506 additions and 30 deletions

No files matched your search

+265
View File
@@ -0,0 +1,265 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! inbuxa: listeners that can't open (#485).
//!
//! A listener whose port is taken is retried for a while: after a restart
//! the old process often still holds it for a few seconds. Any other bind
//! error is reported at once. A listener that stays shut is:
//!
//! - logged as an error (`network.listen-error`), as before;
//! - kept in [`ListenerFailures`], which `/healthz/ready` reads, so a probe
//! takes the node out of service without restarting it in a loop;
//! - written to the store under this node's id, so `x:ClusterNode` shows it
//! in the console whichever node the console talks to.
//!
//! The process stays up: one blocked port shouldn't take every other
//! protocol down with it. `INBUXA_LISTENER_FAILURE=exit` makes a listener
//! that can't open at startup end the process instead (exit code 1), for
//! setups that would rather have their supervisor restart it. Listeners put
//! back later by the legacy-protocols switch (LP-5) only ever report.
use crate::Server;
use parking_lot::Mutex;
use std::{
collections::BTreeMap,
sync::atomic::{AtomicBool, Ordering},
time::{Duration, Instant},
};
use store::{
Deserialize, SUBSPACE_INBUXA, Store, ValueKey,
write::{AnyClass, BatchBuilder, ValueClass},
};
use trc::AddContext;
/// How long a port in use is retried.
pub const RETRY_FOR: Duration = Duration::from_secs(30);
const FIRST_BACKOFF: Duration = Duration::from_millis(500);
const MAX_BACKOFF: Duration = Duration::from_secs(5);
/// Set once the listeners configured at startup have been spawned. Anything
/// spawned after that (LP-5) never ends the process.
pub static STARTED: AtomicBool = AtomicBool::new(false);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FailureMode {
Report,
Exit,
}
impl FailureMode {
pub fn parse(value: Option<&str>) -> Result<Self, String> {
match value.map(str::trim).unwrap_or_default() {
"" | "report" => Ok(FailureMode::Report),
"exit" => Ok(FailureMode::Exit),
other => Err(format!(
"INBUXA_LISTENER_FAILURE is '{other}'; it takes 'report' (the default) or 'exit'"
)),
}
}
pub fn from_env() -> Self {
Self::parse(types::branding::env_var("LISTENER_FAILURE").ok().as_deref()).unwrap_or_else(
|err| {
trc::event!(Server(trc::ServerEvent::StartupError), Details = err,);
FailureMode::Report
},
)
}
}
/// Listeners of this node that aren't open, keyed "listener-id address".
#[derive(Default)]
pub struct ListenerFailures {
failed: Mutex<BTreeMap<String, String>>,
}
impl ListenerFailures {
/// Records why a listener isn't open. True when that changed anything.
pub fn set(&self, key: &str, reason: &str) -> bool {
self.failed
.lock()
.insert(key.to_string(), reason.to_string())
.is_none_or(|before| before != reason)
}
/// The listener opened, or went away. True when it had been recorded.
pub fn clear(&self, key: &str) -> bool {
self.failed.lock().remove(key).is_some()
}
/// Forgets every address of a listener, when it's stopped on purpose.
pub fn clear_listener(&self, id: &str) -> bool {
let prefix = format!("{id} ");
let mut failed = self.failed.lock();
let before = failed.len();
failed.retain(|key, _| !key.starts_with(&prefix));
failed.len() != before
}
pub fn is_empty(&self) -> bool {
self.failed.lock().is_empty()
}
/// One line per listener: "id address: reason".
pub fn lines(&self) -> Vec<String> {
self.failed
.lock()
.iter()
.map(|(key, reason)| format!("{key}: {reason}"))
.collect()
}
}
/// Under `O`, listener status (store-layout SL-16), by node id.
fn status_class(node_id: u16) -> ValueClass {
let mut key = b"Ol".to_vec();
key.extend_from_slice(&node_id.to_be_bytes());
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key,
})
}
struct Lines(Vec<String>);
impl Deserialize for Lines {
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
Ok(Lines(
String::from_utf8_lossy(bytes)
.lines()
.filter(|line| !line.is_empty())
.map(str::to_string)
.collect(),
))
}
}
/// Writes this node's unbound listeners where every node can read them;
/// none clears the record.
pub async fn publish(store: &Store, node_id: u16, lines: &[String]) -> trc::Result<()> {
let mut batch = BatchBuilder::new();
if lines.is_empty() {
batch.clear(status_class(node_id));
} else {
batch.set(status_class(node_id), lines.join("\n").into_bytes());
}
store
.write(batch.build_all())
.await
.caused_by(trc::location!())
.map(|_| ())
}
/// A node's unbound listeners, as it last wrote them.
pub async fn read(store: &Store, node_id: u16) -> trc::Result<Vec<String>> {
Ok(store
.get_value::<Lines>(ValueKey::from(status_class(node_id)))
.await
.caused_by(trc::location!())?
.map(|Lines(lines)| lines)
.unwrap_or_default())
}
impl Server {
/// Writes this node's current failures to the store. A store that's
/// down is logged and left: readiness reads memory, not the store.
pub async fn publish_listener_failures(&self) {
let Ok(node_id) = u16::try_from(self.core.network.node_id) else {
return;
};
let lines = self.inner.data.listener_control.failures().lines();
if let Err(err) = publish(&self.core.storage.data, node_id, &lines).await {
trc::error!(err.details("Failed to record this node's listener status"));
}
}
}
/// How long to wait before the next try at a port in use, or `None` once
/// `retry_for` has passed since `started`.
pub fn next_backoff(
started: Instant,
retry_for: Duration,
previous: Option<Duration>,
) -> Option<Duration> {
let next = previous.map_or(FIRST_BACKOFF, |d| (d * 2).min(MAX_BACKOFF));
let left = retry_for.checked_sub(started.elapsed())?;
(!left.is_zero()).then(|| next.min(left))
}
/// True for the bind errors worth waiting out.
pub fn is_retryable(err: &std::io::Error) -> bool {
err.kind() == std::io::ErrorKind::AddrInUse
}
/// Whether startup is over, for [`STARTED`].
pub fn startup_done() {
STARTED.store(true, Ordering::Release);
}
pub fn is_startup() -> bool {
!STARTED.load(Ordering::Acquire)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn failure_mode_from_its_setting() {
assert_eq!(FailureMode::parse(None), Ok(FailureMode::Report));
assert_eq!(FailureMode::parse(Some("report")), Ok(FailureMode::Report));
assert_eq!(FailureMode::parse(Some(" exit ")), Ok(FailureMode::Exit));
assert!(FailureMode::parse(Some("crash")).is_err());
}
#[test]
fn failures_by_listener() {
let failures = ListenerFailures::default();
assert!(failures.is_empty());
assert!(failures.set("imaps [::]:993", "address in use"));
assert!(!failures.set("imaps [::]:993", "address in use"));
assert!(failures.set("imaps [::]:993", "permission denied"));
assert!(failures.set("imaps-alt [::]:1993", "address in use"));
assert_eq!(
failures.lines(),
vec![
"imaps [::]:993: permission denied".to_string(),
"imaps-alt [::]:1993: address in use".to_string()
]
);
// Stopping "imaps" leaves "imaps-alt" alone
assert!(failures.clear_listener("imaps"));
assert_eq!(failures.lines().len(), 1);
assert!(failures.clear("imaps-alt [::]:1993"));
assert!(failures.is_empty());
}
#[test]
fn backoff_doubles_and_stops() {
let now = Instant::now();
let first = next_backoff(now, RETRY_FOR, None).unwrap();
assert_eq!(first, FIRST_BACKOFF);
assert_eq!(
next_backoff(now, RETRY_FOR, Some(first)),
Some(FIRST_BACKOFF * 2)
);
assert_eq!(
next_backoff(now, RETRY_FOR, Some(MAX_BACKOFF)),
Some(MAX_BACKOFF)
);
let long_ago = Instant::now() - RETRY_FOR - Duration::from_secs(1);
assert_eq!(next_backoff(long_ago, RETRY_FOR, Some(first)), None);
}
#[test]
fn only_address_in_use_is_retried() {
use std::io::{Error, ErrorKind};
assert!(is_retryable(&Error::from(ErrorKind::AddrInUse)));
assert!(!is_retryable(&Error::from(ErrorKind::PermissionDenied)));
assert!(!is_retryable(&Error::from(ErrorKind::AddrNotAvailable)));
}
}
+9
View File
@@ -55,6 +55,8 @@ pub struct ListenerInfo {
pub struct ListenerControl {
running: RwLock<AHashMap<String, Running>>,
spawner: OnceLock<SpawnListener>,
// inbuxa: #485, listeners that couldn't open
failures: super::bind_health::ListenerFailures,
}
impl ListenerControl {
@@ -105,9 +107,16 @@ impl ListenerControl {
true
}
/// Listeners that couldn't open (#485).
pub fn failures(&self) -> &super::bind_health::ListenerFailures {
&self.failures
}
/// Stops one listener by id. Returns what was stopped, or `None` when no
/// listener of that id is running.
pub fn stop(&self, id: &str) -> Option<ListenerInfo> {
// inbuxa: #485: a listener stopped on purpose isn't a failure
self.failures.clear_listener(id);
let running = self.running.write().remove(id).map(|running| {
let _ = running.shutdown_tx.send(true);
ListenerInfo {
+179 -26
View File
@@ -63,32 +63,11 @@ impl Listener {
ttl: listener.ttl,
};
// Bind socket
let listener = match listener.listen() {
Ok(listener) => {
trc::event!(
Network(trc::NetworkEvent::ListenStart),
ListenerId = instance.id.clone(),
LocalIp = local_addr.ip(),
LocalPort = local_addr.port(),
Tls = is_tls,
);
listener
}
Err(err) => {
trc::event!(
Network(trc::NetworkEvent::ListenError),
ListenerId = instance.id.clone(),
LocalIp = local_addr.ip(),
LocalPort = local_addr.port(),
Tls = is_tls,
Reason = err,
);
continue;
}
};
// inbuxa: #485: a listener opens in its own task, so a port still
// held by the previous process can be waited out without holding
// up the rest. Only listeners configured at startup may end the
// process, and only when INBUXA_LISTENER_FAILURE=exit.
let at_startup = super::bind_health::is_startup();
// Spawn listener
let mut shutdown_rx = instance.shutdown_rx.clone();
@@ -96,6 +75,56 @@ impl Listener {
let instance = instance.clone();
let inner = inner.clone();
tokio::spawn(async move {
let key = format!("{} {}", instance.id, local_addr);
let listener = match open(listener, &inner, &key).await {
Ok(listener) => {
trc::event!(
Network(trc::NetworkEvent::ListenStart),
ListenerId = instance.id.clone(),
LocalIp = local_addr.ip(),
LocalPort = local_addr.port(),
Tls = is_tls,
);
if inner.data.listener_control.failures().clear(&key) {
inner.build_server().publish_listener_failures().await;
}
listener
}
Err(err) => {
trc::event!(
Network(trc::NetworkEvent::ListenError),
ListenerId = instance.id.clone(),
LocalIp = local_addr.ip(),
LocalPort = local_addr.port(),
Tls = is_tls,
Reason = err.clone(),
);
if inner.data.listener_control.failures().set(&key, &err) {
inner.build_server().publish_listener_failures().await;
}
if at_startup
&& super::bind_health::FailureMode::from_env()
== super::bind_health::FailureMode::Exit
{
trc::event!(
Server(trc::ServerEvent::StartupError),
Details = "A listener couldn't open and INBUXA_LISTENER_FAILURE=exit: stopping",
ListenerId = instance.id.clone(),
Reason = err.clone(),
);
eprintln!(
"Listener {key} couldn't open ({err}) and INBUXA_LISTENER_FAILURE=exit: stopping."
);
// Give the log a moment to write the events above
tokio::time::sleep(Duration::from_millis(500)).await;
std::process::exit(1);
}
return;
}
};
let (span_start, span_end) = match self.protocol {
ServerProtocol::Smtp | ServerProtocol::Lmtp => (
EventType::Smtp(SmtpEvent::ConnectionStart),
@@ -430,7 +459,74 @@ impl Listeners {
}
}
/// inbuxa: #485: listens on a socket, binding it again if the bind at
/// startup failed. A port in use is retried for `RETRY_FOR`, and recorded
/// as a failure meanwhile, so the node reports not ready while it waits.
async fn open(
listener: TcpListener,
inner: &Arc<Inner>,
key: &str,
) -> Result<tokio::net::TcpListener, String> {
let inner_ = inner.clone();
let key_ = key.to_string();
open_retrying(listener, super::bind_health::RETRY_FOR, move |retrying| {
if inner_
.data
.listener_control
.failures()
.set(&key_, &retrying)
{
let inner = inner_.clone();
tokio::spawn(async move {
inner.build_server().publish_listener_failures().await;
});
}
})
.await
}
/// Binds (if the bind at startup failed) and listens. A port in use is
/// retried with backoff for `retry_for`, calling `on_retry` once with the
/// reason when the waiting starts; any other bind error fails at once.
async fn open_retrying(
listener: TcpListener,
retry_for: Duration,
mut on_retry: impl FnMut(String),
) -> Result<tokio::net::TcpListener, String> {
use super::bind_health::{is_retryable, next_backoff};
let started = std::time::Instant::now();
let mut backoff = None;
while !listener.is_bound() {
let Err(err) = listener.socket.bind(listener.addr) else {
break;
};
let reason = format!("Failed to bind to {}: {}", listener.addr, err);
if !is_retryable(&err) {
return Err(reason);
}
let Some(wait) = next_backoff(started, retry_for, backoff) else {
return Err(format!(
"{reason} (still in use after {} s)",
retry_for.as_secs()
));
};
if backoff.is_none() {
on_retry(format!("{reason}; retrying"));
}
backoff = Some(wait);
tokio::time::sleep(wait).await;
}
listener.listen()
}
impl TcpListener {
fn is_bound(&self) -> bool {
self.socket
.local_addr()
.is_ok_and(|bound| bound.port() != 0)
}
pub fn listen(self) -> Result<tokio::net::TcpListener, String> {
// inbuxa: a socket whose bind failed is still unbound, and listen()
// on it makes the kernel pick a random port on every interface
@@ -508,7 +604,9 @@ impl ServerInstance {
#[cfg(test)]
mod tests {
use super::open_retrying;
use crate::config::server::TcpListener;
use std::time::Duration;
use tokio::net::TcpSocket;
fn listener(socket: TcpSocket, addr: &str) -> TcpListener {
@@ -529,6 +627,61 @@ mod tests {
assert!(err.contains("isn't bound"), "{err}");
}
// inbuxa: #485
#[tokio::test]
async fn a_port_in_use_is_waited_for() {
let held = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = held.local_addr().unwrap().to_string();
let socket = TcpSocket::new_v4().unwrap();
let mut retried = Vec::new();
let opening = tokio::spawn(async move {
let result =
open_retrying(listener(socket, &addr), Duration::from_secs(20), |reason| {
retried.push(reason)
})
.await;
(result, retried)
});
tokio::time::sleep(Duration::from_millis(1500)).await;
drop(held);
let (result, retried) = opening.await.unwrap();
assert!(result.is_ok(), "{result:?}");
assert_eq!(retried.len(), 1, "{retried:?}");
assert!(retried[0].contains("retrying"), "{retried:?}");
}
#[tokio::test]
async fn a_port_held_too_long_fails() {
let held = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = held.local_addr().unwrap().to_string();
let socket = TcpSocket::new_v4().unwrap();
let started = std::time::Instant::now();
let err = open_retrying(listener(socket, &addr), Duration::from_secs(2), |_| {})
.await
.unwrap_err();
assert!(err.contains("still in use after 2 s"), "{err}");
assert!(started.elapsed() < Duration::from_secs(5));
drop(held);
}
#[tokio::test]
async fn other_bind_errors_are_not_retried() {
// An address this host doesn't have
let socket = TcpSocket::new_v4().unwrap();
let started = std::time::Instant::now();
let mut retried = 0;
let err = open_retrying(
listener(socket, "192.0.2.1:2525"),
Duration::from_secs(20),
|_| retried += 1,
)
.await
.unwrap_err();
assert!(err.contains("Failed to bind"), "{err}");
assert_eq!(retried, 0);
assert!(started.elapsed() < Duration::from_secs(2));
}
#[tokio::test]
async fn a_bound_socket_listens_even_on_port_zero() {
let socket = TcpSocket::new_v4().unwrap();
+1
View File
@@ -35,6 +35,7 @@ use utils::snowflake::SnowflakeIdGenerator;
pub mod acme;
pub mod asn;
pub mod autoconfig;
pub mod bind_health; // inbuxa: #485, listeners that can't open
pub mod control;
pub mod dkim;
pub mod dns;
+11
View File
@@ -606,6 +606,17 @@ impl ParseHttp for Server {
return Ok(JsonProblemResponse(StatusCode::OK).into_http_response());
}
"ready" => {
// inbuxa: #485: not ready while a configured
// listener isn't open, naming each one
let unbound = self.inner.data.listener_control.failures().lines();
if !unbound.is_empty() {
return Ok(http_proto::JsonResponse::with_status(
StatusCode::SERVICE_UNAVAILABLE,
serde_json::json!({ "unboundListeners": unbound }),
)
.no_cache()
.into_http_response());
}
// inbuxa: ready only while the data store answers
// (a cached, time-limited read); liveness stays 200
return Ok(JsonProblemResponse({
+11 -2
View File
@@ -2,10 +2,12 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use jmap_proto::{object::registry::RegistryComparator, types::state::State};
use registry::{jmap::IntoValue, schema::prelude::Property};
use registry::{jmap::IntoValue, schema::prelude::Property, types::map::Map};
use store::ahash::AHashSet;
use crate::{
@@ -25,8 +27,15 @@ pub(crate) async fn cluster_node_get(
.map(|id| id.id())
.collect::<AHashSet<_>>();
for node in nodes {
for mut node in nodes {
if ids.is_empty() || ids.remove(&node.node_id) {
// inbuxa: #485: what each node last said it couldn't open
if let Ok(node_id) = u16::try_from(node.node_id) {
node.unbound_listeners = Map::new(
common::network::bind_health::read(&get.server.core.storage.data, node_id)
.await?,
);
}
get.insert(node.node_id.into(), node.into_value());
}
}
+8
View File
@@ -121,6 +121,14 @@ async fn main() -> std::io::Result<()> {
spawn_listener(&spawn_inner, server, acceptor, shutdown_rx);
});
// inbuxa: #485: listeners spawned from here on (LP-5) never end the
// process; and a node starts with no failures left from its last run
common::network::bind_health::startup_done();
{
let server = init.inner.build_server();
tokio::spawn(async move { server.publish_listener_failures().await });
}
// Leave behind how to spawn a listener, so putting one back opens its port
// without a restart (LP-5). Only this file knows the session manager for a
// protocol, so only this file can say.
+1
View File
@@ -89,6 +89,7 @@ pub const OWNERS: &[(u8, Owner)] = &[
),
(b'M', Owner::Stays("masked email")),
(b'N', Owner::Stays("passkeys")),
(b'O', Owner::Stays("listener status")),
(b'P', Owner::Stays("permissions and protocol policies")),
(b'Q', Owner::Stays("acceptance records")),
(b'R', Owner::Stays("mail flow rules")),
+1
View File
@@ -1174,6 +1174,7 @@ pub enum Property {
TrustReplies = 774,
TsigAlgorithm = 338,
Ttl = 310,
UnboundListeners = 947,
UnpackDirectory = 54,
UpdateRecords = 812,
UploadQuota = 445,
@@ -1322,6 +1322,7 @@ impl EnumImpl for Property {
b"tsigAlgorithm" => Property::TsigAlgorithm,
b"ttl" => Property::Ttl,
b"unpackDirectory" => Property::UnpackDirectory,
b"unboundListeners" => Property::UnboundListeners,
b"updateRecords" => Property::UpdateRecords,
b"uploadQuota" => Property::UploadQuota,
b"uploadTtl" => Property::UploadTtl,
@@ -2274,6 +2275,7 @@ impl EnumImpl for Property {
Property::TsigAlgorithm => "tsigAlgorithm",
Property::Ttl => "ttl",
Property::UnpackDirectory => "unpackDirectory",
Property::UnboundListeners => "unboundListeners",
Property::UpdateRecords => "updateRecords",
Property::UploadQuota => "uploadQuota",
Property::UploadTtl => "uploadTtl",
@@ -3057,6 +3059,7 @@ impl EnumImpl for Property {
944 => Some(Property::PasskeysAllowed),
945 => Some(Property::PasskeyRpId),
946 => Some(Property::MaxPasskeys),
947 => Some(Property::UnboundListeners),
615 => Some(Property::RequireClientRegistration),
608 => Some(Property::RequireScopes),
525 => Some(Property::RequireTls),
@@ -3268,7 +3271,7 @@ impl EnumImpl for Property {
}
}
const COUNT: usize = 947;
const COUNT: usize = 948;
}
impl serde::Serialize for Property {
+3
View File
@@ -760,6 +760,9 @@ pub struct ClusterNode {
pub last_renewal: UTCDateTime,
#[serde(rename = "status")]
pub status: ClusterNodeStatus,
// inbuxa: #485, listeners this node couldn't bind, as "id address: reason"
#[serde(rename = "unboundListeners")]
pub unbound_listeners: Map<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -5455,6 +5455,7 @@ impl Pickle for ClusterNode {
self.hostname.pickle(out);
self.last_renewal.pickle(out);
self.status.pickle(out);
self.unbound_listeners.pickle(out);
}
fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option<Self> {
@@ -5463,6 +5464,7 @@ impl Pickle for ClusterNode {
this.hostname = Pickle::unpickle(stream)?;
this.last_renewal = Pickle::unpickle(stream)?;
this.status = Pickle::unpickle(stream)?;
this.unbound_listeners = Pickle::unpickle(stream)?;
Some(this)
}
}
@@ -5474,6 +5476,7 @@ impl Default for ClusterNode {
hostname: Default::default(),
last_renewal: Default::default(),
status: Default::default(),
unbound_listeners: Default::default(),
}
}
}
@@ -5485,6 +5488,10 @@ impl IntoValue for ClusterNode {
map.insert_unchecked(Property::Hostname, self.hostname.into_value());
map.insert_unchecked(Property::LastRenewal, self.last_renewal.into_value());
map.insert_unchecked(Property::Status, self.status.into_value());
map.insert_unchecked(
Property::UnboundListeners,
self.unbound_listeners.into_value(),
);
JmapValue::Object(map)
}
}
@@ -5500,6 +5507,9 @@ impl RegistryJsonPropertyPatch for ClusterNode {
Some(Property::Hostname) => self.hostname.patch(pointer, value),
Some(Property::LastRenewal) => self.last_renewal.patch(pointer, value),
Some(Property::Status) => self.status.patch(pointer, value),
Some(Property::UnboundListeners) => self
.unbound_listeners
.patch(pointer.assert_read_only()?, value),
Some(Property::Type) => Ok(MaybeUnpatched::Unpatched {
property: Property::Type,
value,
+2
View File
@@ -418,6 +418,8 @@ impl From<NodeSlot> for ClusterNode {
last_renewal: UTCDateTime::from_timestamp(slot.last_renewal.cast_signed()),
node_id: slot.node_id as u64,
hostname: slot.hostname,
// INBUXA: #485, filled in from the store when the node is read
unbound_listeners: Default::default(),
}
}
}
Binary file not shown.
+1 -1
View File
@@ -1 +1 @@
KXw6nEosTA5QQYzIzA68OlHDImM7fjTIYCHRdaLTO8w
s1j6j79sdVPo9onPNL8iJOJRz_V5NLJAaU6MHps8z54