Author SHA1 Message Date
jcoffey-dev 22d8ad8572 Publish amd64 first, then arm64, on a builder that keeps its cache
ci / fork-checks (pull_request) Successful in 49s
ci / build (pull_request) Successful in 4m30s
Two release builds side by side on one machine each take twice as long,
and production only needs amd64. publish-amd64 now pushes :<version> as
soon as the amd64 build is done; publish-arm64 builds arm64 afterwards,
then replaces :<version> with the two-platform index and moves :latest.

Both jobs use one named BuildKit builder whose container outlives the
job, so the dependency layer (cargo chef cook) is reused until the
dependencies change. The release is created after amd64; the binaries
are attached once arm64 is in.
2026-09-24 08:04:53 -07:00
11 changed files with 75 additions and 357 deletions
+69 -15
View File
@@ -3,11 +3,28 @@
# whether a person pushed it or weekly-release.yml created it through the # whether a person pushed it or weekly-release.yml created it through the
# releases API. # releases API.
# #
# The image is multi-arch (linux/amd64, linux/arm64) as before, but built in # The image is multi-arch (linux/amd64, linux/arm64), built by two jobs on
# one buildx run on host1 instead of one native runner per architecture: the # the image-build runner rather than one buildx run for both. The Dockerfile's
# Dockerfile's builder stage runs on the build platform and cross-compiles # builder stage runs on the build platform and cross-compiles with an aarch64
# with an aarch64 linker, so only the small final stage (apt, setcap) goes # linker, so only the small final stage (apt, setcap) goes through QEMU for
# through QEMU for arm64. No digest-joining job is needed. # arm64 -- but two release builds (LTO, one codegen unit) side by side on one
# machine each take twice as long. Production runs amd64, so amd64 goes first
# and on its own:
# * publish-amd64 pushes :<version>-amd64 and :<version>, a plain amd64
# image, as soon as its build is done. A deploy can start from it.
# * publish-arm64 then builds arm64, pushes :<version>-arm64, and replaces
# :<version> with the two-platform index. :latest moves only here, so it
# never names an image without arm64.
#
# Both jobs use one BuildKit builder, `gitea-builder`, whose container
# (buildx_buildkit_gitea-builder0) and state volume stay on the runner's host
# between jobs: a job container's `buildx create` finds the existing container
# and reuses it and its cache. The dependency build (`cargo chef cook`) is
# keyed on the recipe, which only a dependency change alters, so a release
# normally compiles just the workspace. Removing that container or its volume
# costs the next release a cold build, nothing more. The planner and dependency
# layers for the build platform are shared, so arm64 also reuses what amd64
# just did where it can.
# #
# Two guards before anything is pushed: # Two guards before anything is pushed:
# * the tag must be v<brand_version!>. The version is a string in # * the tag must be v<brand_version!>. The version is a string in
@@ -62,7 +79,7 @@ jobs:
echo "version=$V" >> "$GITHUB_OUTPUT" echo "version=$V" >> "$GITHUB_OUTPUT"
echo "version $V" echo "version $V"
publish: publish-amd64:
needs: [version] needs: [version]
runs-on: docker runs-on: docker
container: container:
@@ -81,16 +98,15 @@ jobs:
test -n "$REGISTRY" && test -n "$VERSION" test -n "$REGISTRY" && test -n "$VERSION"
test -n "$PACKAGE_TOKEN" || { echo "PACKAGE_TOKEN secret is not set on this repository" >&2; exit 1; } test -n "$PACKAGE_TOKEN" || { echo "PACKAGE_TOKEN secret is not set on this repository" >&2; exit 1; }
echo "$PACKAGE_TOKEN" | docker login -u jcoffey-dev --password-stdin "$REGISTRY" echo "$PACKAGE_TOKEN" | docker login -u jcoffey-dev --password-stdin "$REGISTRY"
docker run --privileged --rm tonistiigi/binfmt --install arm64
docker buildx create --use --name gitea-builder --driver docker-container || docker buildx use gitea-builder docker buildx create --use --name gitea-builder --driver docker-container || docker buildx use gitea-builder
# Attestations off, as before: they add manifests of their own to the # Attestations off, as before: they add manifests of their own, and the
# index, and the index should hold the two images and nothing else. # index should hold the two images and nothing else.
- run: | - run: |
docker buildx build \ docker buildx build \
--platform linux/amd64,linux/arm64 \ --platform linux/amd64 \
--provenance=false --sbom=false \ --provenance=false --sbom=false \
--tag "$IMAGE:$VERSION-amd64" \
--tag "$IMAGE:$VERSION" \ --tag "$IMAGE:$VERSION" \
--tag "$IMAGE:latest" \
--push . --push .
docker buildx imagetools inspect "$IMAGE:$VERSION" docker buildx imagetools inspect "$IMAGE:$VERSION"
# Gitea keeps a container package on its owner; linking it shows it on # Gitea keeps a container package on its owner; linking it shows it on
@@ -103,11 +119,47 @@ jobs:
- if: always() - if: always()
run: docker logout "$REGISTRY" || true run: docker logout "$REGISTRY" || true
publish-arm64:
needs: [version, publish-amd64]
runs-on: docker
container:
image: docker:28-cli@sha256:625d9431a9f54c5a2bc90f24f0e1c3d55b1349fd857dd85035f98c2c9acbdd4d # 28-cli
volumes:
- /var/run/docker.sock:/var/run/docker.sock
env:
DOCKER_BUILDKIT: "1"
REGISTRY: ${{ vars.REGISTRY }}
IMAGE: ${{ vars.REGISTRY }}/${{ github.repository }}
VERSION: ${{ needs.version.outputs.version }}
PACKAGE_TOKEN: ${{ secrets.PACKAGE_TOKEN }}
steps:
- uses: coffey-labs/actions/checkout@fab0c4d45e0162963965f1555df27b7bed5e20ec
- run: |
echo "$PACKAGE_TOKEN" | docker login -u jcoffey-dev --password-stdin "$REGISTRY"
docker run --privileged --rm tonistiigi/binfmt --install arm64
docker buildx create --use --name gitea-builder --driver docker-container || docker buildx use gitea-builder
# The index is built from the two per-architecture tags rather than from
# :<version>, which by now is the amd64 image and would be read as such.
- run: |
docker buildx build \
--platform linux/arm64 \
--provenance=false --sbom=false \
--tag "$IMAGE:$VERSION-arm64" \
--push .
docker buildx imagetools create \
--tag "$IMAGE:$VERSION" \
--tag "$IMAGE:latest" \
"$IMAGE:$VERSION-amd64" "$IMAGE:$VERSION-arm64"
docker buildx imagetools inspect "$IMAGE:$VERSION"
- if: always()
run: docker logout "$REGISTRY" || true
# The weekly release creates its Release (and so the tag) first; a tag # The weekly release creates its Release (and so the tag) first; a tag
# pushed by hand has none. Either way the tag ends up with exactly one # pushed by hand has none. Either way the tag ends up with exactly one
# Release, created after the image exists so its pull instructions work. # Release, created once the amd64 image exists so its pull instructions
# work; arm64 and the binaries follow.
release: release:
needs: [version, publish] needs: [version, publish-amd64]
runs-on: light runs-on: light
container: container:
image: python:3.13-slim@sha256:8d9d0b8bcf6506481eae4907c18f5e3e7902e629f5f6d684f9e7c32e85e3ddf0 # 3.13-slim image: python:3.13-slim@sha256:8d9d0b8bcf6506481eae4907c18f5e3e7902e629f5f6d684f9e7c32e85e3ddf0 # 3.13-slim
@@ -131,7 +183,9 @@ jobs:
except urllib.error.HTTPError as e: except urllib.error.HTTPError as e:
if e.code != 404: raise if e.code != 404: raise
image = f"{os.environ['REGISTRY']}/{os.environ['REPO']}:{version}" image = f"{os.environ['REGISTRY']}/{os.environ['REPO']}:{version}"
body = (f"Container image: `{image}` (linux/amd64, linux/arm64); also `:latest`.\n\n" body = (f"Container image: `{image}` (linux/amd64, linux/arm64); also `:latest`. "
"amd64 is published first; arm64 is added to the same tag when its build "
"finishes, and `:latest` moves then.\n\n"
"Binaries for a host install are attached: `inbuxa-linux-amd64.tar.gz` and " "Binaries for a host install are attached: `inbuxa-linux-amd64.tar.gz` and "
"`inbuxa-linux-arm64.tar.gz`, with `SHA256SUMS`. Each is the binary out of this " "`inbuxa-linux-arm64.tar.gz`, with `SHA256SUMS`. Each is the binary out of this "
"release's image for that architecture, so it is the same build. The image " "release's image for that architecture, so it is the same build. The image "
@@ -154,7 +208,7 @@ jobs:
# `docker create` does not start anything, so pulling an arm64 image on an # `docker create` does not start anything, so pulling an arm64 image on an
# amd64 runner and copying a file out of it needs no emulation. # amd64 runner and copying a file out of it needs no emulation.
binaries: binaries:
needs: [version, publish, release] needs: [version, publish-arm64, release]
runs-on: docker runs-on: docker
container: container:
image: docker:28-cli@sha256:625d9431a9f54c5a2bc90f24f0e1c3d55b1349fd857dd85035f98c2c9acbdd4d # 28-cli image: docker:28-cli@sha256:625d9431a9f54c5a2bc90f24f0e1c3d55b1349fd857dd85035f98c2c9acbdd4d # 28-cli
+1 -1
View File
@@ -8,7 +8,7 @@ store = { path = "../store" }
registry = { path = "../registry" } registry = { path = "../registry" }
trc = { path = "../trc" } trc = { path = "../trc" }
futures = { version = "0.3", optional = true } futures = { version = "0.3", optional = true }
tokio = { version = "1.53", features = ["sync", "fs", "io-util", "rt", "time"] } tokio = { version = "1.53", features = ["sync", "fs", "io-util"] }
async-nats = { version = "0.50", default-features = false, features = ["server_2_10", "server_2_11", "aws-lc-rs"], optional = true } async-nats = { version = "0.50", default-features = false, features = ["server_2_10", "server_2_11", "aws-lc-rs"], optional = true }
zenoh = { version = "1.10.0", default-features = false, features = ["auth_pubkey", "transport_multilink", "transport_compression", "transport_quic", "transport_tcp", "transport_tls", "transport_udp"], optional = true } zenoh = { version = "1.10.0", default-features = false, features = ["auth_pubkey", "transport_multilink", "transport_compression", "transport_quic", "transport_tcp", "transport_tls", "transport_udp"], optional = true }
rdkafka = { version = "0.39", features = ["cmake-build"], optional = true } rdkafka = { version = "0.39", features = ["cmake-build"], optional = true }
+2 -118
View File
@@ -2,22 +2,13 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]> * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
* *
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/ */
use std::{ use std::sync::Arc;
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
time::Duration,
};
use crate::Coordinator; use crate::Coordinator;
use async_nats::Client; use async_nats::Client;
use registry::schema::structs::NatsCoordinator; use registry::schema::structs::NatsCoordinator;
use trc::ClusterEvent;
pub mod pubsub; pub mod pubsub;
@@ -56,116 +47,9 @@ impl NatsPubSub {
opts = opts.token(credentials); opts = opts.token(credentials);
} }
// inbuxa: connect in the background and keep trying, so a node that
// starts while NATS is down still joins the cluster once NATS is
// back, instead of running without a coordinator until restarted;
// and report the connection going and coming back
let reporter = Arc::new(Reporter::default());
opts = opts.retry_on_initial_connect().event_callback({
let reporter = reporter.clone();
move |event| {
let reporter = reporter.clone();
async move { reporter.report(event) }
}
});
let connection_timeout = config.timeout_connection.into_inner();
async_nats::connect_with_options(config.addresses.into_inner(), opts) async_nats::connect_with_options(config.addresses.into_inner(), opts)
.await .await
.map(|client| { .map(|client| Coordinator::Nats(Arc::new(NatsPubSub { client })))
reporter.watch_first_connection(client.clone(), connection_timeout);
Coordinator::Nats(Arc::new(NatsPubSub { client }))
})
.map_err(|err| format!("Failed to connect to Nats: {}", err)) .map_err(|err| format!("Failed to connect to Nats: {}", err))
} }
/// inbuxa: whether the client is connected to a NATS server right now.
pub fn is_connected(&self) -> bool {
matches!(
self.client.connection_state(),
async_nats::connection::State::Connected
)
}
}
/// inbuxa: reports the client's connection events as the server's own.
#[derive(Default)]
struct Reporter {
connected_once: AtomicBool,
// A failed attempt raises an error each time the client retries, every
// few seconds while NATS is down: report the first after each change
error_reported: AtomicBool,
}
impl Reporter {
fn report(&self, event: async_nats::Event) {
match event {
async_nats::Event::Connected => {
self.connected_once.store(true, Ordering::Relaxed);
self.error_reported.store(false, Ordering::Relaxed);
trc::event!(Cluster(ClusterEvent::CoordinatorConnected), Type = "nats");
}
async_nats::Event::Disconnected => {
self.error_reported.store(false, Ordering::Relaxed);
trc::event!(
Cluster(ClusterEvent::CoordinatorDisconnected),
Type = "nats",
Details = "Connection lost; reconnecting in the background",
);
}
async_nats::Event::Closed => {
trc::event!(
Cluster(ClusterEvent::CoordinatorDisconnected),
Type = "nats",
Details = "Connection closed; no further attempts will be made",
);
}
async_nats::Event::ClientError(async_nats::ClientError::MaxReconnects) => {
trc::event!(
Cluster(ClusterEvent::CoordinatorDisconnected),
Type = "nats",
Details = "Gave up reconnecting (maxReconnects reached)",
);
}
async_nats::Event::ClientError(err) => {
if !self.error_reported.swap(true, Ordering::Relaxed) {
trc::event!(
Cluster(ClusterEvent::CoordinatorError),
Type = "nats",
Details = "Connection attempt failed; retrying",
Reason = err.to_string(),
);
}
}
event => {
trc::event!(
Cluster(ClusterEvent::CoordinatorError),
Type = "nats",
Details = event.to_string(),
);
}
}
}
/// The first connection is made in the background, so say so when it
/// hasn't been made within the connection timeout. The client keeps
/// trying, and reports the connection when it comes.
fn watch_first_connection(self: &Arc<Self>, client: Client, timeout: Duration) {
let reporter = self.clone();
tokio::spawn(async move {
tokio::time::sleep(timeout).await;
if !reporter.connected_once.load(Ordering::Relaxed)
&& !matches!(
client.connection_state(),
async_nats::connection::State::Connected
)
{
trc::event!(
Cluster(ClusterEvent::CoordinatorDisconnected),
Type = "nats",
Details = "Not connected at startup; retrying in the background",
);
}
});
}
} }
-13
View File
@@ -2,8 +2,6 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]> * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
* *
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/ */
use crate::{Coordinator, Msg, PubSubStream}; use crate::{Coordinator, Msg, PubSubStream};
@@ -45,17 +43,6 @@ impl Coordinator {
pub fn is_none(&self) -> bool { pub fn is_none(&self) -> bool {
matches!(self, Coordinator::None) matches!(self, Coordinator::None)
} }
/// inbuxa: whether the coordinator is connected right now, for the
/// backends that track it (NATS); `None` for the others and when no
/// coordinator is configured.
pub fn is_connected(&self) -> Option<bool> {
match self {
#[cfg(feature = "nats")]
Coordinator::Nats(store) => Some(store.is_connected()),
_ => None,
}
}
} }
impl PubSubStream { impl PubSubStream {
-21
View File
@@ -562,27 +562,6 @@ impl ParseHttp for Server {
}) })
.into_http_response()); .into_http_response());
} }
// inbuxa: the cluster coordinator's connection, for
// monitoring. It stays out of live and ready on purpose:
// a node without its coordinator still serves mail, and
// failing those would have an orchestrator restart, or
// take out of service, every node at once when the
// coordinator goes down
"cluster" => {
let coordinator = &self.core.storage.coordinator;
let (status, state) = match coordinator.is_connected() {
Some(true) => (StatusCode::OK, "connected"),
Some(false) => (StatusCode::SERVICE_UNAVAILABLE, "disconnected"),
None if coordinator.is_none() => (StatusCode::OK, "none"),
None => (StatusCode::OK, "unknown"),
};
return Ok(http_proto::JsonResponse::with_status(
status,
serde_json::json!({ "coordinator": state }),
)
.no_cache()
.into_http_response());
}
_ => (), _ => (),
} }
} }
+2 -7
View File
@@ -10,9 +10,8 @@
// inbuxa: 637 to 641 are the fork's SCIM events (SCIM-54); 642 is // inbuxa: 637 to 641 are the fork's SCIM events (SCIM-54); 642 is
// auth.legacy-protocol-refused (legacy-protocols LP-6); 643 is // auth.legacy-protocol-refused (legacy-protocols LP-6); 643 is
// security.legacy-protocols-changed (LP-8); 644 to 646 are the cluster // security.legacy-protocols-changed (LP-8)
// coordinator's connection events pub const TOTAL_EVENT_COUNT: usize = 644;
pub const TOTAL_EVENT_COUNT: usize = 647;
pub const TOTAL_METRIC_COUNT: usize = 369; pub const TOTAL_METRIC_COUNT: usize = 369;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -151,10 +150,6 @@ pub enum ClusterEvent {
MessageSkipped = 47, MessageSkipped = 47,
MessageInvalid = 49, MessageInvalid = 49,
NodeIdRenewed = 275, NodeIdRenewed = 275,
// inbuxa: the coordinator's connection
CoordinatorConnected = 644,
CoordinatorDisconnected = 645,
CoordinatorError = 646,
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-32
View File
@@ -81,10 +81,6 @@ impl EventType {
b"cluster.message-skipped" => EventType::Cluster(ClusterEvent::MessageSkipped), b"cluster.message-skipped" => EventType::Cluster(ClusterEvent::MessageSkipped),
b"cluster.message-invalid" => EventType::Cluster(ClusterEvent::MessageInvalid), b"cluster.message-invalid" => EventType::Cluster(ClusterEvent::MessageInvalid),
b"cluster.node-id-renewed" => EventType::Cluster(ClusterEvent::NodeIdRenewed), b"cluster.node-id-renewed" => EventType::Cluster(ClusterEvent::NodeIdRenewed),
// inbuxa: coordinator connection
b"cluster.coordinator-connected" => EventType::Cluster(ClusterEvent::CoordinatorConnected),
b"cluster.coordinator-disconnected" => EventType::Cluster(ClusterEvent::CoordinatorDisconnected),
b"cluster.coordinator-error" => EventType::Cluster(ClusterEvent::CoordinatorError),
b"dane.authentication-success" => EventType::Dane(DaneEvent::AuthenticationSuccess), b"dane.authentication-success" => EventType::Dane(DaneEvent::AuthenticationSuccess),
b"dane.authentication-failure" => EventType::Dane(DaneEvent::AuthenticationFailure), b"dane.authentication-failure" => EventType::Dane(DaneEvent::AuthenticationFailure),
b"dane.no-certificates-found" => EventType::Dane(DaneEvent::NoCertificatesFound), b"dane.no-certificates-found" => EventType::Dane(DaneEvent::NoCertificatesFound),
@@ -746,14 +742,6 @@ impl EventType {
EventType::Cluster(ClusterEvent::MessageSkipped) => "cluster.message-skipped", EventType::Cluster(ClusterEvent::MessageSkipped) => "cluster.message-skipped",
EventType::Cluster(ClusterEvent::MessageInvalid) => "cluster.message-invalid", EventType::Cluster(ClusterEvent::MessageInvalid) => "cluster.message-invalid",
EventType::Cluster(ClusterEvent::NodeIdRenewed) => "cluster.node-id-renewed", EventType::Cluster(ClusterEvent::NodeIdRenewed) => "cluster.node-id-renewed",
// inbuxa: coordinator connection
EventType::Cluster(ClusterEvent::CoordinatorConnected) => {
"cluster.coordinator-connected"
}
EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => {
"cluster.coordinator-disconnected"
}
EventType::Cluster(ClusterEvent::CoordinatorError) => "cluster.coordinator-error",
EventType::Dane(DaneEvent::AuthenticationSuccess) => "dane.authentication-success", EventType::Dane(DaneEvent::AuthenticationSuccess) => "dane.authentication-success",
EventType::Dane(DaneEvent::AuthenticationFailure) => "dane.authentication-failure", EventType::Dane(DaneEvent::AuthenticationFailure) => "dane.authentication-failure",
EventType::Dane(DaneEvent::NoCertificatesFound) => "dane.no-certificates-found", EventType::Dane(DaneEvent::NoCertificatesFound) => "dane.no-certificates-found",
@@ -1536,10 +1524,6 @@ impl EventType {
EventType::Cluster(ClusterEvent::MessageSkipped) => 47, EventType::Cluster(ClusterEvent::MessageSkipped) => 47,
EventType::Cluster(ClusterEvent::MessageInvalid) => 49, EventType::Cluster(ClusterEvent::MessageInvalid) => 49,
EventType::Cluster(ClusterEvent::NodeIdRenewed) => 275, EventType::Cluster(ClusterEvent::NodeIdRenewed) => 275,
// inbuxa: coordinator connection
EventType::Cluster(ClusterEvent::CoordinatorConnected) => 644,
EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => 645,
EventType::Cluster(ClusterEvent::CoordinatorError) => 646,
EventType::Dane(DaneEvent::AuthenticationSuccess) => 67, EventType::Dane(DaneEvent::AuthenticationSuccess) => 67,
EventType::Dane(DaneEvent::AuthenticationFailure) => 66, EventType::Dane(DaneEvent::AuthenticationFailure) => 66,
EventType::Dane(DaneEvent::NoCertificatesFound) => 69, EventType::Dane(DaneEvent::NoCertificatesFound) => 69,
@@ -2192,10 +2176,6 @@ impl EventType {
47 => Some(EventType::Cluster(ClusterEvent::MessageSkipped)), 47 => Some(EventType::Cluster(ClusterEvent::MessageSkipped)),
49 => Some(EventType::Cluster(ClusterEvent::MessageInvalid)), 49 => Some(EventType::Cluster(ClusterEvent::MessageInvalid)),
275 => Some(EventType::Cluster(ClusterEvent::NodeIdRenewed)), 275 => Some(EventType::Cluster(ClusterEvent::NodeIdRenewed)),
// inbuxa: coordinator connection
644 => Some(EventType::Cluster(ClusterEvent::CoordinatorConnected)),
645 => Some(EventType::Cluster(ClusterEvent::CoordinatorDisconnected)),
646 => Some(EventType::Cluster(ClusterEvent::CoordinatorError)),
67 => Some(EventType::Dane(DaneEvent::AuthenticationSuccess)), 67 => Some(EventType::Dane(DaneEvent::AuthenticationSuccess)),
66 => Some(EventType::Dane(DaneEvent::AuthenticationFailure)), 66 => Some(EventType::Dane(DaneEvent::AuthenticationFailure)),
69 => Some(EventType::Dane(DaneEvent::NoCertificatesFound)), 69 => Some(EventType::Dane(DaneEvent::NoCertificatesFound)),
@@ -3134,10 +3114,6 @@ impl EventType {
EventType::Auth(AuthEvent::TooManyAttempts) => Level::Warn, EventType::Auth(AuthEvent::TooManyAttempts) => Level::Warn,
EventType::Calendar(CalendarEvent::AlarmFailed) => Level::Warn, EventType::Calendar(CalendarEvent::AlarmFailed) => Level::Warn,
EventType::Cluster(ClusterEvent::SubscriberDisconnected) => Level::Warn, EventType::Cluster(ClusterEvent::SubscriberDisconnected) => Level::Warn,
// inbuxa: coordinator connection
EventType::Cluster(ClusterEvent::CoordinatorConnected) => Level::Info,
EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => Level::Warn,
EventType::Cluster(ClusterEvent::CoordinatorError) => Level::Warn,
EventType::Delivery(DeliveryEvent::MissingOutboundHostname) => Level::Warn, EventType::Delivery(DeliveryEvent::MissingOutboundHostname) => Level::Warn,
EventType::Delivery(DeliveryEvent::ConcurrencyLimitExceeded) => Level::Warn, EventType::Delivery(DeliveryEvent::ConcurrencyLimitExceeded) => Level::Warn,
EventType::Delivery(DeliveryEvent::RateLimitExceeded) => Level::Warn, EventType::Delivery(DeliveryEvent::RateLimitExceeded) => Level::Warn,
@@ -3268,10 +3244,6 @@ impl EventType {
EventType::Cluster(ClusterEvent::MessageSkipped) => "PubSub message skipped", EventType::Cluster(ClusterEvent::MessageSkipped) => "PubSub message skipped",
EventType::Cluster(ClusterEvent::MessageInvalid) => "Invalid PubSub message", EventType::Cluster(ClusterEvent::MessageInvalid) => "Invalid PubSub message",
EventType::Cluster(ClusterEvent::NodeIdRenewed) => "Node ID renewed", EventType::Cluster(ClusterEvent::NodeIdRenewed) => "Node ID renewed",
// inbuxa: coordinator connection
EventType::Cluster(ClusterEvent::CoordinatorConnected) => "Coordinator connected",
EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => "Coordinator unavailable",
EventType::Cluster(ClusterEvent::CoordinatorError) => "Coordinator error",
EventType::Dane(DaneEvent::AuthenticationSuccess) => "DANE authentication successful", EventType::Dane(DaneEvent::AuthenticationSuccess) => "DANE authentication successful",
EventType::Dane(DaneEvent::AuthenticationFailure) => "DANE authentication failed", EventType::Dane(DaneEvent::AuthenticationFailure) => "DANE authentication failed",
EventType::Dane(DaneEvent::NoCertificatesFound) => "No certificates found for DANE", EventType::Dane(DaneEvent::NoCertificatesFound) => "No certificates found for DANE",
@@ -4350,10 +4322,6 @@ impl EventType {
EventType::Cluster(ClusterEvent::MessageSkipped), EventType::Cluster(ClusterEvent::MessageSkipped),
EventType::Cluster(ClusterEvent::MessageInvalid), EventType::Cluster(ClusterEvent::MessageInvalid),
EventType::Cluster(ClusterEvent::NodeIdRenewed), EventType::Cluster(ClusterEvent::NodeIdRenewed),
// inbuxa: coordinator connection
EventType::Cluster(ClusterEvent::CoordinatorConnected),
EventType::Cluster(ClusterEvent::CoordinatorDisconnected),
EventType::Cluster(ClusterEvent::CoordinatorError),
EventType::Dane(DaneEvent::AuthenticationSuccess), EventType::Dane(DaneEvent::AuthenticationSuccess),
EventType::Dane(DaneEvent::AuthenticationFailure), EventType::Dane(DaneEvent::AuthenticationFailure),
EventType::Dane(DaneEvent::NoCertificatesFound), EventType::Dane(DaneEvent::NoCertificatesFound),
Binary file not shown.
+1 -1
View File
@@ -1 +1 @@
XFI3xuKC_rH1KZyaVBF0uTIiRDXRqyYboijquiGz2eg VbnFuwCOTBh0s2T-NuRhb2JaJr8Jl5s3LgXv4Pv2sTg
-145
View File
@@ -1,145 +0,0 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! A node that starts while its NATS coordinator is down joins the cluster
//! once NATS comes up, without a restart, and reports the coordinator's
//! connection on `/healthz/cluster` as it goes and comes back.
use crate::utils::server::TestServerBuilder;
use coordinator::Coordinator;
use registry::{
schema::{
enums::NetworkListenerProtocol,
structs::{Coordinator as CoordinatorSetting, NatsCoordinator},
},
types::map::Map,
};
use serde_json::{Value, json};
use std::time::{Duration, Instant};
use testcontainers::{
GenericImage, ImageExt, core::IntoContainerPort, core::WaitFor, runners::AsyncRunner,
};
const HTTP_PORT: u16 = 11_310;
const TOPIC: &str = "inbuxa-coordinator-test";
#[tokio::test(flavor = "multi_thread")]
pub async fn coordinator_reconnect_tests() {
println!("Running coordinator reconnect tests...");
// A port with no NATS server behind it, yet
let nats_port = std::net::TcpListener::bind("127.0.0.1:0")
.unwrap()
.local_addr()
.unwrap()
.port();
let config = NatsCoordinator {
addresses: Map::new(vec![format!("127.0.0.1:{nats_port}")]),
use_tls: false,
timeout_connection: 1_000u64.into(),
..Default::default()
};
// 1. The node starts, without a build error, while NATS is down, and
// says so
let test = TestServerBuilder::new("coordinator_reconnect_tests")
.await
.with_object(CoordinatorSetting::Nats(config.clone()))
.await
.with_listener(NetworkListenerProtocol::Http, "http", HTTP_PORT, true)
.await
.build()
.await;
let coordinator = test.server.core.storage.coordinator.clone();
assert!(
coordinator.is_enabled(),
"a coordinator, though not connected"
);
assert_eq!(coordinator.is_connected(), Some(false));
assert_eq!(
cluster_health().await,
(503, json!({"coordinator": "disconnected"}))
);
// A subscription made now, as the broadcast subscriber makes it at
// startup, has to work once NATS is up
let mut stream = coordinator.subscribe(TOPIC).await.unwrap();
// 2. NATS comes up: the node connects on its own
let nats = GenericImage::new("nats", "latest")
.with_wait_for(WaitFor::message_on_stderr("Server is ready"))
.with_mapped_port(nats_port, 4222.tcp())
.start()
.await
.expect("Failed to start NATS container");
wait_for_health(200, "connected").await;
let other_node = coordinator::backend::nats::NatsPubSub::open(config.clone())
.await
.unwrap();
wait_until_connected(&other_node).await;
round_trip(&other_node, &mut stream, b"after startup").await;
// 3. NATS goes away: the node reports it; and it comes back: the node
// reconnects and the same subscription carries on
nats.stop().await.unwrap();
wait_for_health(503, "disconnected").await;
nats.start().await.unwrap();
wait_for_health(200, "connected").await;
wait_until_connected(&other_node).await;
round_trip(&other_node, &mut stream, b"after reconnect").await;
drop(nats);
if test.is_reset() {
test.temp_dir.delete();
}
}
async fn cluster_health() -> (u16, Value) {
let response = reqwest::Client::builder()
.danger_accept_invalid_certs(true)
.timeout(Duration::from_secs(5))
.build()
.unwrap()
.get(format!("https://127.0.0.1:{HTTP_PORT}/healthz/cluster"))
.send()
.await
.unwrap();
let status = response.status().as_u16();
(status, response.json().await.unwrap())
}
async fn wait_for_health(status: u16, state: &str) {
let started = Instant::now();
loop {
let health = cluster_health().await;
if health == (status, json!({"coordinator": state})) {
return;
}
assert!(
started.elapsed() < Duration::from_secs(30),
"expected {status} {state}, still {health:?}"
);
tokio::time::sleep(Duration::from_millis(250)).await;
}
}
async fn wait_until_connected(coordinator: &Coordinator) {
let started = Instant::now();
while coordinator.is_connected() != Some(true) {
assert!(started.elapsed() < Duration::from_secs(30), "not connected");
tokio::time::sleep(Duration::from_millis(100)).await;
}
}
/// Another node publishes; this one's subscription receives it.
async fn round_trip(from: &Coordinator, stream: &mut coordinator::PubSubStream, payload: &[u8]) {
from.publish(TOPIC, payload.to_vec()).await.unwrap();
let message = tokio::time::timeout(Duration::from_secs(10), stream.next())
.await
.expect("no message within 10 seconds")
.expect("subscription ended");
assert_eq!(message.payload(), payload);
}
-4
View File
@@ -2,11 +2,7 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]> * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
* *
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/ */
pub mod broadcast; pub mod broadcast;
#[cfg(feature = "nats")]
pub mod coordinator; // inbuxa: coordinator reconnects
pub mod stress; pub mod stress;