Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
22d8ad8572 |
@@ -3,11 +3,28 @@
|
|||||||
# whether a person pushed it or weekly-release.yml created it through the
|
# whether a person pushed it or weekly-release.yml created it through the
|
||||||
# releases API.
|
# releases API.
|
||||||
#
|
#
|
||||||
# The image is multi-arch (linux/amd64, linux/arm64) as before, but built in
|
# The image is multi-arch (linux/amd64, linux/arm64), built by two jobs on
|
||||||
# one buildx run on host1 instead of one native runner per architecture: the
|
# the image-build runner rather than one buildx run for both. The Dockerfile's
|
||||||
# Dockerfile's builder stage runs on the build platform and cross-compiles
|
# builder stage runs on the build platform and cross-compiles with an aarch64
|
||||||
# with an aarch64 linker, so only the small final stage (apt, setcap) goes
|
# linker, so only the small final stage (apt, setcap) goes through QEMU for
|
||||||
# through QEMU for arm64. No digest-joining job is needed.
|
# arm64 -- but two release builds (LTO, one codegen unit) side by side on one
|
||||||
|
# machine each take twice as long. Production runs amd64, so amd64 goes first
|
||||||
|
# and on its own:
|
||||||
|
# * publish-amd64 pushes :<version>-amd64 and :<version>, a plain amd64
|
||||||
|
# image, as soon as its build is done. A deploy can start from it.
|
||||||
|
# * publish-arm64 then builds arm64, pushes :<version>-arm64, and replaces
|
||||||
|
# :<version> with the two-platform index. :latest moves only here, so it
|
||||||
|
# never names an image without arm64.
|
||||||
|
#
|
||||||
|
# Both jobs use one BuildKit builder, `gitea-builder`, whose container
|
||||||
|
# (buildx_buildkit_gitea-builder0) and state volume stay on the runner's host
|
||||||
|
# between jobs: a job container's `buildx create` finds the existing container
|
||||||
|
# and reuses it and its cache. The dependency build (`cargo chef cook`) is
|
||||||
|
# keyed on the recipe, which only a dependency change alters, so a release
|
||||||
|
# normally compiles just the workspace. Removing that container or its volume
|
||||||
|
# costs the next release a cold build, nothing more. The planner and dependency
|
||||||
|
# layers for the build platform are shared, so arm64 also reuses what amd64
|
||||||
|
# just did where it can.
|
||||||
#
|
#
|
||||||
# Two guards before anything is pushed:
|
# Two guards before anything is pushed:
|
||||||
# * the tag must be v<brand_version!>. The version is a string in
|
# * the tag must be v<brand_version!>. The version is a string in
|
||||||
@@ -62,7 +79,7 @@ jobs:
|
|||||||
echo "version=$V" >> "$GITHUB_OUTPUT"
|
echo "version=$V" >> "$GITHUB_OUTPUT"
|
||||||
echo "version $V"
|
echo "version $V"
|
||||||
|
|
||||||
publish:
|
publish-amd64:
|
||||||
needs: [version]
|
needs: [version]
|
||||||
runs-on: docker
|
runs-on: docker
|
||||||
container:
|
container:
|
||||||
@@ -81,16 +98,15 @@ jobs:
|
|||||||
test -n "$REGISTRY" && test -n "$VERSION"
|
test -n "$REGISTRY" && test -n "$VERSION"
|
||||||
test -n "$PACKAGE_TOKEN" || { echo "PACKAGE_TOKEN secret is not set on this repository" >&2; exit 1; }
|
test -n "$PACKAGE_TOKEN" || { echo "PACKAGE_TOKEN secret is not set on this repository" >&2; exit 1; }
|
||||||
echo "$PACKAGE_TOKEN" | docker login -u jcoffey-dev --password-stdin "$REGISTRY"
|
echo "$PACKAGE_TOKEN" | docker login -u jcoffey-dev --password-stdin "$REGISTRY"
|
||||||
docker run --privileged --rm tonistiigi/binfmt --install arm64
|
|
||||||
docker buildx create --use --name gitea-builder --driver docker-container || docker buildx use gitea-builder
|
docker buildx create --use --name gitea-builder --driver docker-container || docker buildx use gitea-builder
|
||||||
# Attestations off, as before: they add manifests of their own to the
|
# Attestations off, as before: they add manifests of their own, and the
|
||||||
# index, and the index should hold the two images and nothing else.
|
# index should hold the two images and nothing else.
|
||||||
- run: |
|
- run: |
|
||||||
docker buildx build \
|
docker buildx build \
|
||||||
--platform linux/amd64,linux/arm64 \
|
--platform linux/amd64 \
|
||||||
--provenance=false --sbom=false \
|
--provenance=false --sbom=false \
|
||||||
|
--tag "$IMAGE:$VERSION-amd64" \
|
||||||
--tag "$IMAGE:$VERSION" \
|
--tag "$IMAGE:$VERSION" \
|
||||||
--tag "$IMAGE:latest" \
|
|
||||||
--push .
|
--push .
|
||||||
docker buildx imagetools inspect "$IMAGE:$VERSION"
|
docker buildx imagetools inspect "$IMAGE:$VERSION"
|
||||||
# Gitea keeps a container package on its owner; linking it shows it on
|
# Gitea keeps a container package on its owner; linking it shows it on
|
||||||
@@ -103,11 +119,47 @@ jobs:
|
|||||||
- if: always()
|
- if: always()
|
||||||
run: docker logout "$REGISTRY" || true
|
run: docker logout "$REGISTRY" || true
|
||||||
|
|
||||||
|
publish-arm64:
|
||||||
|
needs: [version, publish-amd64]
|
||||||
|
runs-on: docker
|
||||||
|
container:
|
||||||
|
image: docker:28-cli@sha256:625d9431a9f54c5a2bc90f24f0e1c3d55b1349fd857dd85035f98c2c9acbdd4d # 28-cli
|
||||||
|
volumes:
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock
|
||||||
|
env:
|
||||||
|
DOCKER_BUILDKIT: "1"
|
||||||
|
REGISTRY: ${{ vars.REGISTRY }}
|
||||||
|
IMAGE: ${{ vars.REGISTRY }}/${{ github.repository }}
|
||||||
|
VERSION: ${{ needs.version.outputs.version }}
|
||||||
|
PACKAGE_TOKEN: ${{ secrets.PACKAGE_TOKEN }}
|
||||||
|
steps:
|
||||||
|
- uses: coffey-labs/actions/checkout@fab0c4d45e0162963965f1555df27b7bed5e20ec
|
||||||
|
- run: |
|
||||||
|
echo "$PACKAGE_TOKEN" | docker login -u jcoffey-dev --password-stdin "$REGISTRY"
|
||||||
|
docker run --privileged --rm tonistiigi/binfmt --install arm64
|
||||||
|
docker buildx create --use --name gitea-builder --driver docker-container || docker buildx use gitea-builder
|
||||||
|
# The index is built from the two per-architecture tags rather than from
|
||||||
|
# :<version>, which by now is the amd64 image and would be read as such.
|
||||||
|
- run: |
|
||||||
|
docker buildx build \
|
||||||
|
--platform linux/arm64 \
|
||||||
|
--provenance=false --sbom=false \
|
||||||
|
--tag "$IMAGE:$VERSION-arm64" \
|
||||||
|
--push .
|
||||||
|
docker buildx imagetools create \
|
||||||
|
--tag "$IMAGE:$VERSION" \
|
||||||
|
--tag "$IMAGE:latest" \
|
||||||
|
"$IMAGE:$VERSION-amd64" "$IMAGE:$VERSION-arm64"
|
||||||
|
docker buildx imagetools inspect "$IMAGE:$VERSION"
|
||||||
|
- if: always()
|
||||||
|
run: docker logout "$REGISTRY" || true
|
||||||
|
|
||||||
# The weekly release creates its Release (and so the tag) first; a tag
|
# The weekly release creates its Release (and so the tag) first; a tag
|
||||||
# pushed by hand has none. Either way the tag ends up with exactly one
|
# pushed by hand has none. Either way the tag ends up with exactly one
|
||||||
# Release, created after the image exists so its pull instructions work.
|
# Release, created once the amd64 image exists so its pull instructions
|
||||||
|
# work; arm64 and the binaries follow.
|
||||||
release:
|
release:
|
||||||
needs: [version, publish]
|
needs: [version, publish-amd64]
|
||||||
runs-on: light
|
runs-on: light
|
||||||
container:
|
container:
|
||||||
image: python:3.13-slim@sha256:8d9d0b8bcf6506481eae4907c18f5e3e7902e629f5f6d684f9e7c32e85e3ddf0 # 3.13-slim
|
image: python:3.13-slim@sha256:8d9d0b8bcf6506481eae4907c18f5e3e7902e629f5f6d684f9e7c32e85e3ddf0 # 3.13-slim
|
||||||
@@ -131,7 +183,9 @@ jobs:
|
|||||||
except urllib.error.HTTPError as e:
|
except urllib.error.HTTPError as e:
|
||||||
if e.code != 404: raise
|
if e.code != 404: raise
|
||||||
image = f"{os.environ['REGISTRY']}/{os.environ['REPO']}:{version}"
|
image = f"{os.environ['REGISTRY']}/{os.environ['REPO']}:{version}"
|
||||||
body = (f"Container image: `{image}` (linux/amd64, linux/arm64); also `:latest`.\n\n"
|
body = (f"Container image: `{image}` (linux/amd64, linux/arm64); also `:latest`. "
|
||||||
|
"amd64 is published first; arm64 is added to the same tag when its build "
|
||||||
|
"finishes, and `:latest` moves then.\n\n"
|
||||||
"Binaries for a host install are attached: `inbuxa-linux-amd64.tar.gz` and "
|
"Binaries for a host install are attached: `inbuxa-linux-amd64.tar.gz` and "
|
||||||
"`inbuxa-linux-arm64.tar.gz`, with `SHA256SUMS`. Each is the binary out of this "
|
"`inbuxa-linux-arm64.tar.gz`, with `SHA256SUMS`. Each is the binary out of this "
|
||||||
"release's image for that architecture, so it is the same build. The image "
|
"release's image for that architecture, so it is the same build. The image "
|
||||||
@@ -154,7 +208,7 @@ jobs:
|
|||||||
# `docker create` does not start anything, so pulling an arm64 image on an
|
# `docker create` does not start anything, so pulling an arm64 image on an
|
||||||
# amd64 runner and copying a file out of it needs no emulation.
|
# amd64 runner and copying a file out of it needs no emulation.
|
||||||
binaries:
|
binaries:
|
||||||
needs: [version, publish, release]
|
needs: [version, publish-arm64, release]
|
||||||
runs-on: docker
|
runs-on: docker
|
||||||
container:
|
container:
|
||||||
image: docker:28-cli@sha256:625d9431a9f54c5a2bc90f24f0e1c3d55b1349fd857dd85035f98c2c9acbdd4d # 28-cli
|
image: docker:28-cli@sha256:625d9431a9f54c5a2bc90f24f0e1c3d55b1349fd857dd85035f98c2c9acbdd4d # 28-cli
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ store = { path = "../store" }
|
|||||||
registry = { path = "../registry" }
|
registry = { path = "../registry" }
|
||||||
trc = { path = "../trc" }
|
trc = { path = "../trc" }
|
||||||
futures = { version = "0.3", optional = true }
|
futures = { version = "0.3", optional = true }
|
||||||
tokio = { version = "1.53", features = ["sync", "fs", "io-util", "rt", "time"] }
|
tokio = { version = "1.53", features = ["sync", "fs", "io-util"] }
|
||||||
async-nats = { version = "0.50", default-features = false, features = ["server_2_10", "server_2_11", "aws-lc-rs"], optional = true }
|
async-nats = { version = "0.50", default-features = false, features = ["server_2_10", "server_2_11", "aws-lc-rs"], optional = true }
|
||||||
zenoh = { version = "1.10.0", default-features = false, features = ["auth_pubkey", "transport_multilink", "transport_compression", "transport_quic", "transport_tcp", "transport_tls", "transport_udp"], optional = true }
|
zenoh = { version = "1.10.0", default-features = false, features = ["auth_pubkey", "transport_multilink", "transport_compression", "transport_quic", "transport_tcp", "transport_tls", "transport_udp"], optional = true }
|
||||||
rdkafka = { version = "0.39", features = ["cmake-build"], optional = true }
|
rdkafka = { version = "0.39", features = ["cmake-build"], optional = true }
|
||||||
|
|||||||
@@ -2,22 +2,13 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use std::{
|
use std::sync::Arc;
|
||||||
sync::{
|
|
||||||
Arc,
|
|
||||||
atomic::{AtomicBool, Ordering},
|
|
||||||
},
|
|
||||||
time::Duration,
|
|
||||||
};
|
|
||||||
|
|
||||||
use crate::Coordinator;
|
use crate::Coordinator;
|
||||||
use async_nats::Client;
|
use async_nats::Client;
|
||||||
use registry::schema::structs::NatsCoordinator;
|
use registry::schema::structs::NatsCoordinator;
|
||||||
use trc::ClusterEvent;
|
|
||||||
|
|
||||||
pub mod pubsub;
|
pub mod pubsub;
|
||||||
|
|
||||||
@@ -56,116 +47,9 @@ impl NatsPubSub {
|
|||||||
opts = opts.token(credentials);
|
opts = opts.token(credentials);
|
||||||
}
|
}
|
||||||
|
|
||||||
// inbuxa: connect in the background and keep trying, so a node that
|
|
||||||
// starts while NATS is down still joins the cluster once NATS is
|
|
||||||
// back, instead of running without a coordinator until restarted;
|
|
||||||
// and report the connection going and coming back
|
|
||||||
let reporter = Arc::new(Reporter::default());
|
|
||||||
opts = opts.retry_on_initial_connect().event_callback({
|
|
||||||
let reporter = reporter.clone();
|
|
||||||
move |event| {
|
|
||||||
let reporter = reporter.clone();
|
|
||||||
async move { reporter.report(event) }
|
|
||||||
}
|
|
||||||
});
|
|
||||||
let connection_timeout = config.timeout_connection.into_inner();
|
|
||||||
|
|
||||||
async_nats::connect_with_options(config.addresses.into_inner(), opts)
|
async_nats::connect_with_options(config.addresses.into_inner(), opts)
|
||||||
.await
|
.await
|
||||||
.map(|client| {
|
.map(|client| Coordinator::Nats(Arc::new(NatsPubSub { client })))
|
||||||
reporter.watch_first_connection(client.clone(), connection_timeout);
|
|
||||||
Coordinator::Nats(Arc::new(NatsPubSub { client }))
|
|
||||||
})
|
|
||||||
.map_err(|err| format!("Failed to connect to Nats: {}", err))
|
.map_err(|err| format!("Failed to connect to Nats: {}", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// inbuxa: whether the client is connected to a NATS server right now.
|
|
||||||
pub fn is_connected(&self) -> bool {
|
|
||||||
matches!(
|
|
||||||
self.client.connection_state(),
|
|
||||||
async_nats::connection::State::Connected
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: reports the client's connection events as the server's own.
|
|
||||||
#[derive(Default)]
|
|
||||||
struct Reporter {
|
|
||||||
connected_once: AtomicBool,
|
|
||||||
// A failed attempt raises an error each time the client retries, every
|
|
||||||
// few seconds while NATS is down: report the first after each change
|
|
||||||
error_reported: AtomicBool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Reporter {
|
|
||||||
fn report(&self, event: async_nats::Event) {
|
|
||||||
match event {
|
|
||||||
async_nats::Event::Connected => {
|
|
||||||
self.connected_once.store(true, Ordering::Relaxed);
|
|
||||||
self.error_reported.store(false, Ordering::Relaxed);
|
|
||||||
trc::event!(Cluster(ClusterEvent::CoordinatorConnected), Type = "nats");
|
|
||||||
}
|
|
||||||
async_nats::Event::Disconnected => {
|
|
||||||
self.error_reported.store(false, Ordering::Relaxed);
|
|
||||||
trc::event!(
|
|
||||||
Cluster(ClusterEvent::CoordinatorDisconnected),
|
|
||||||
Type = "nats",
|
|
||||||
Details = "Connection lost; reconnecting in the background",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
async_nats::Event::Closed => {
|
|
||||||
trc::event!(
|
|
||||||
Cluster(ClusterEvent::CoordinatorDisconnected),
|
|
||||||
Type = "nats",
|
|
||||||
Details = "Connection closed; no further attempts will be made",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
async_nats::Event::ClientError(async_nats::ClientError::MaxReconnects) => {
|
|
||||||
trc::event!(
|
|
||||||
Cluster(ClusterEvent::CoordinatorDisconnected),
|
|
||||||
Type = "nats",
|
|
||||||
Details = "Gave up reconnecting (maxReconnects reached)",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
async_nats::Event::ClientError(err) => {
|
|
||||||
if !self.error_reported.swap(true, Ordering::Relaxed) {
|
|
||||||
trc::event!(
|
|
||||||
Cluster(ClusterEvent::CoordinatorError),
|
|
||||||
Type = "nats",
|
|
||||||
Details = "Connection attempt failed; retrying",
|
|
||||||
Reason = err.to_string(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
event => {
|
|
||||||
trc::event!(
|
|
||||||
Cluster(ClusterEvent::CoordinatorError),
|
|
||||||
Type = "nats",
|
|
||||||
Details = event.to_string(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The first connection is made in the background, so say so when it
|
|
||||||
/// hasn't been made within the connection timeout. The client keeps
|
|
||||||
/// trying, and reports the connection when it comes.
|
|
||||||
fn watch_first_connection(self: &Arc<Self>, client: Client, timeout: Duration) {
|
|
||||||
let reporter = self.clone();
|
|
||||||
tokio::spawn(async move {
|
|
||||||
tokio::time::sleep(timeout).await;
|
|
||||||
if !reporter.connected_once.load(Ordering::Relaxed)
|
|
||||||
&& !matches!(
|
|
||||||
client.connection_state(),
|
|
||||||
async_nats::connection::State::Connected
|
|
||||||
)
|
|
||||||
{
|
|
||||||
trc::event!(
|
|
||||||
Cluster(ClusterEvent::CoordinatorDisconnected),
|
|
||||||
Type = "nats",
|
|
||||||
Details = "Not connected at startup; retrying in the background",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::{Coordinator, Msg, PubSubStream};
|
use crate::{Coordinator, Msg, PubSubStream};
|
||||||
@@ -45,17 +43,6 @@ impl Coordinator {
|
|||||||
pub fn is_none(&self) -> bool {
|
pub fn is_none(&self) -> bool {
|
||||||
matches!(self, Coordinator::None)
|
matches!(self, Coordinator::None)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// inbuxa: whether the coordinator is connected right now, for the
|
|
||||||
/// backends that track it (NATS); `None` for the others and when no
|
|
||||||
/// coordinator is configured.
|
|
||||||
pub fn is_connected(&self) -> Option<bool> {
|
|
||||||
match self {
|
|
||||||
#[cfg(feature = "nats")]
|
|
||||||
Coordinator::Nats(store) => Some(store.is_connected()),
|
|
||||||
_ => None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PubSubStream {
|
impl PubSubStream {
|
||||||
|
|||||||
@@ -562,27 +562,6 @@ impl ParseHttp for Server {
|
|||||||
})
|
})
|
||||||
.into_http_response());
|
.into_http_response());
|
||||||
}
|
}
|
||||||
// inbuxa: the cluster coordinator's connection, for
|
|
||||||
// monitoring. It stays out of live and ready on purpose:
|
|
||||||
// a node without its coordinator still serves mail, and
|
|
||||||
// failing those would have an orchestrator restart, or
|
|
||||||
// take out of service, every node at once when the
|
|
||||||
// coordinator goes down
|
|
||||||
"cluster" => {
|
|
||||||
let coordinator = &self.core.storage.coordinator;
|
|
||||||
let (status, state) = match coordinator.is_connected() {
|
|
||||||
Some(true) => (StatusCode::OK, "connected"),
|
|
||||||
Some(false) => (StatusCode::SERVICE_UNAVAILABLE, "disconnected"),
|
|
||||||
None if coordinator.is_none() => (StatusCode::OK, "none"),
|
|
||||||
None => (StatusCode::OK, "unknown"),
|
|
||||||
};
|
|
||||||
return Ok(http_proto::JsonResponse::with_status(
|
|
||||||
status,
|
|
||||||
serde_json::json!({ "coordinator": state }),
|
|
||||||
)
|
|
||||||
.no_cache()
|
|
||||||
.into_http_response());
|
|
||||||
}
|
|
||||||
_ => (),
|
_ => (),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,9 +10,8 @@
|
|||||||
|
|
||||||
// inbuxa: 637 to 641 are the fork's SCIM events (SCIM-54); 642 is
|
// inbuxa: 637 to 641 are the fork's SCIM events (SCIM-54); 642 is
|
||||||
// auth.legacy-protocol-refused (legacy-protocols LP-6); 643 is
|
// auth.legacy-protocol-refused (legacy-protocols LP-6); 643 is
|
||||||
// security.legacy-protocols-changed (LP-8); 644 to 646 are the cluster
|
// security.legacy-protocols-changed (LP-8)
|
||||||
// coordinator's connection events
|
pub const TOTAL_EVENT_COUNT: usize = 644;
|
||||||
pub const TOTAL_EVENT_COUNT: usize = 647;
|
|
||||||
pub const TOTAL_METRIC_COUNT: usize = 369;
|
pub const TOTAL_METRIC_COUNT: usize = 369;
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||||
@@ -151,10 +150,6 @@ pub enum ClusterEvent {
|
|||||||
MessageSkipped = 47,
|
MessageSkipped = 47,
|
||||||
MessageInvalid = 49,
|
MessageInvalid = 49,
|
||||||
NodeIdRenewed = 275,
|
NodeIdRenewed = 275,
|
||||||
// inbuxa: the coordinator's connection
|
|
||||||
CoordinatorConnected = 644,
|
|
||||||
CoordinatorDisconnected = 645,
|
|
||||||
CoordinatorError = 646,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||||
|
|||||||
@@ -81,10 +81,6 @@ impl EventType {
|
|||||||
b"cluster.message-skipped" => EventType::Cluster(ClusterEvent::MessageSkipped),
|
b"cluster.message-skipped" => EventType::Cluster(ClusterEvent::MessageSkipped),
|
||||||
b"cluster.message-invalid" => EventType::Cluster(ClusterEvent::MessageInvalid),
|
b"cluster.message-invalid" => EventType::Cluster(ClusterEvent::MessageInvalid),
|
||||||
b"cluster.node-id-renewed" => EventType::Cluster(ClusterEvent::NodeIdRenewed),
|
b"cluster.node-id-renewed" => EventType::Cluster(ClusterEvent::NodeIdRenewed),
|
||||||
// inbuxa: coordinator connection
|
|
||||||
b"cluster.coordinator-connected" => EventType::Cluster(ClusterEvent::CoordinatorConnected),
|
|
||||||
b"cluster.coordinator-disconnected" => EventType::Cluster(ClusterEvent::CoordinatorDisconnected),
|
|
||||||
b"cluster.coordinator-error" => EventType::Cluster(ClusterEvent::CoordinatorError),
|
|
||||||
b"dane.authentication-success" => EventType::Dane(DaneEvent::AuthenticationSuccess),
|
b"dane.authentication-success" => EventType::Dane(DaneEvent::AuthenticationSuccess),
|
||||||
b"dane.authentication-failure" => EventType::Dane(DaneEvent::AuthenticationFailure),
|
b"dane.authentication-failure" => EventType::Dane(DaneEvent::AuthenticationFailure),
|
||||||
b"dane.no-certificates-found" => EventType::Dane(DaneEvent::NoCertificatesFound),
|
b"dane.no-certificates-found" => EventType::Dane(DaneEvent::NoCertificatesFound),
|
||||||
@@ -746,14 +742,6 @@ impl EventType {
|
|||||||
EventType::Cluster(ClusterEvent::MessageSkipped) => "cluster.message-skipped",
|
EventType::Cluster(ClusterEvent::MessageSkipped) => "cluster.message-skipped",
|
||||||
EventType::Cluster(ClusterEvent::MessageInvalid) => "cluster.message-invalid",
|
EventType::Cluster(ClusterEvent::MessageInvalid) => "cluster.message-invalid",
|
||||||
EventType::Cluster(ClusterEvent::NodeIdRenewed) => "cluster.node-id-renewed",
|
EventType::Cluster(ClusterEvent::NodeIdRenewed) => "cluster.node-id-renewed",
|
||||||
// inbuxa: coordinator connection
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorConnected) => {
|
|
||||||
"cluster.coordinator-connected"
|
|
||||||
}
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => {
|
|
||||||
"cluster.coordinator-disconnected"
|
|
||||||
}
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorError) => "cluster.coordinator-error",
|
|
||||||
EventType::Dane(DaneEvent::AuthenticationSuccess) => "dane.authentication-success",
|
EventType::Dane(DaneEvent::AuthenticationSuccess) => "dane.authentication-success",
|
||||||
EventType::Dane(DaneEvent::AuthenticationFailure) => "dane.authentication-failure",
|
EventType::Dane(DaneEvent::AuthenticationFailure) => "dane.authentication-failure",
|
||||||
EventType::Dane(DaneEvent::NoCertificatesFound) => "dane.no-certificates-found",
|
EventType::Dane(DaneEvent::NoCertificatesFound) => "dane.no-certificates-found",
|
||||||
@@ -1536,10 +1524,6 @@ impl EventType {
|
|||||||
EventType::Cluster(ClusterEvent::MessageSkipped) => 47,
|
EventType::Cluster(ClusterEvent::MessageSkipped) => 47,
|
||||||
EventType::Cluster(ClusterEvent::MessageInvalid) => 49,
|
EventType::Cluster(ClusterEvent::MessageInvalid) => 49,
|
||||||
EventType::Cluster(ClusterEvent::NodeIdRenewed) => 275,
|
EventType::Cluster(ClusterEvent::NodeIdRenewed) => 275,
|
||||||
// inbuxa: coordinator connection
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorConnected) => 644,
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => 645,
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorError) => 646,
|
|
||||||
EventType::Dane(DaneEvent::AuthenticationSuccess) => 67,
|
EventType::Dane(DaneEvent::AuthenticationSuccess) => 67,
|
||||||
EventType::Dane(DaneEvent::AuthenticationFailure) => 66,
|
EventType::Dane(DaneEvent::AuthenticationFailure) => 66,
|
||||||
EventType::Dane(DaneEvent::NoCertificatesFound) => 69,
|
EventType::Dane(DaneEvent::NoCertificatesFound) => 69,
|
||||||
@@ -2192,10 +2176,6 @@ impl EventType {
|
|||||||
47 => Some(EventType::Cluster(ClusterEvent::MessageSkipped)),
|
47 => Some(EventType::Cluster(ClusterEvent::MessageSkipped)),
|
||||||
49 => Some(EventType::Cluster(ClusterEvent::MessageInvalid)),
|
49 => Some(EventType::Cluster(ClusterEvent::MessageInvalid)),
|
||||||
275 => Some(EventType::Cluster(ClusterEvent::NodeIdRenewed)),
|
275 => Some(EventType::Cluster(ClusterEvent::NodeIdRenewed)),
|
||||||
// inbuxa: coordinator connection
|
|
||||||
644 => Some(EventType::Cluster(ClusterEvent::CoordinatorConnected)),
|
|
||||||
645 => Some(EventType::Cluster(ClusterEvent::CoordinatorDisconnected)),
|
|
||||||
646 => Some(EventType::Cluster(ClusterEvent::CoordinatorError)),
|
|
||||||
67 => Some(EventType::Dane(DaneEvent::AuthenticationSuccess)),
|
67 => Some(EventType::Dane(DaneEvent::AuthenticationSuccess)),
|
||||||
66 => Some(EventType::Dane(DaneEvent::AuthenticationFailure)),
|
66 => Some(EventType::Dane(DaneEvent::AuthenticationFailure)),
|
||||||
69 => Some(EventType::Dane(DaneEvent::NoCertificatesFound)),
|
69 => Some(EventType::Dane(DaneEvent::NoCertificatesFound)),
|
||||||
@@ -3134,10 +3114,6 @@ impl EventType {
|
|||||||
EventType::Auth(AuthEvent::TooManyAttempts) => Level::Warn,
|
EventType::Auth(AuthEvent::TooManyAttempts) => Level::Warn,
|
||||||
EventType::Calendar(CalendarEvent::AlarmFailed) => Level::Warn,
|
EventType::Calendar(CalendarEvent::AlarmFailed) => Level::Warn,
|
||||||
EventType::Cluster(ClusterEvent::SubscriberDisconnected) => Level::Warn,
|
EventType::Cluster(ClusterEvent::SubscriberDisconnected) => Level::Warn,
|
||||||
// inbuxa: coordinator connection
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorConnected) => Level::Info,
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => Level::Warn,
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorError) => Level::Warn,
|
|
||||||
EventType::Delivery(DeliveryEvent::MissingOutboundHostname) => Level::Warn,
|
EventType::Delivery(DeliveryEvent::MissingOutboundHostname) => Level::Warn,
|
||||||
EventType::Delivery(DeliveryEvent::ConcurrencyLimitExceeded) => Level::Warn,
|
EventType::Delivery(DeliveryEvent::ConcurrencyLimitExceeded) => Level::Warn,
|
||||||
EventType::Delivery(DeliveryEvent::RateLimitExceeded) => Level::Warn,
|
EventType::Delivery(DeliveryEvent::RateLimitExceeded) => Level::Warn,
|
||||||
@@ -3268,10 +3244,6 @@ impl EventType {
|
|||||||
EventType::Cluster(ClusterEvent::MessageSkipped) => "PubSub message skipped",
|
EventType::Cluster(ClusterEvent::MessageSkipped) => "PubSub message skipped",
|
||||||
EventType::Cluster(ClusterEvent::MessageInvalid) => "Invalid PubSub message",
|
EventType::Cluster(ClusterEvent::MessageInvalid) => "Invalid PubSub message",
|
||||||
EventType::Cluster(ClusterEvent::NodeIdRenewed) => "Node ID renewed",
|
EventType::Cluster(ClusterEvent::NodeIdRenewed) => "Node ID renewed",
|
||||||
// inbuxa: coordinator connection
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorConnected) => "Coordinator connected",
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => "Coordinator unavailable",
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorError) => "Coordinator error",
|
|
||||||
EventType::Dane(DaneEvent::AuthenticationSuccess) => "DANE authentication successful",
|
EventType::Dane(DaneEvent::AuthenticationSuccess) => "DANE authentication successful",
|
||||||
EventType::Dane(DaneEvent::AuthenticationFailure) => "DANE authentication failed",
|
EventType::Dane(DaneEvent::AuthenticationFailure) => "DANE authentication failed",
|
||||||
EventType::Dane(DaneEvent::NoCertificatesFound) => "No certificates found for DANE",
|
EventType::Dane(DaneEvent::NoCertificatesFound) => "No certificates found for DANE",
|
||||||
@@ -4350,10 +4322,6 @@ impl EventType {
|
|||||||
EventType::Cluster(ClusterEvent::MessageSkipped),
|
EventType::Cluster(ClusterEvent::MessageSkipped),
|
||||||
EventType::Cluster(ClusterEvent::MessageInvalid),
|
EventType::Cluster(ClusterEvent::MessageInvalid),
|
||||||
EventType::Cluster(ClusterEvent::NodeIdRenewed),
|
EventType::Cluster(ClusterEvent::NodeIdRenewed),
|
||||||
// inbuxa: coordinator connection
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorConnected),
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorDisconnected),
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorError),
|
|
||||||
EventType::Dane(DaneEvent::AuthenticationSuccess),
|
EventType::Dane(DaneEvent::AuthenticationSuccess),
|
||||||
EventType::Dane(DaneEvent::AuthenticationFailure),
|
EventType::Dane(DaneEvent::AuthenticationFailure),
|
||||||
EventType::Dane(DaneEvent::NoCertificatesFound),
|
EventType::Dane(DaneEvent::NoCertificatesFound),
|
||||||
|
|||||||
Binary file not shown.
@@ -1 +1 @@
|
|||||||
XFI3xuKC_rH1KZyaVBF0uTIiRDXRqyYboijquiGz2eg
|
VbnFuwCOTBh0s2T-NuRhb2JaJr8Jl5s3LgXv4Pv2sTg
|
||||||
@@ -1,145 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
//! A node that starts while its NATS coordinator is down joins the cluster
|
|
||||||
//! once NATS comes up, without a restart, and reports the coordinator's
|
|
||||||
//! connection on `/healthz/cluster` as it goes and comes back.
|
|
||||||
|
|
||||||
use crate::utils::server::TestServerBuilder;
|
|
||||||
use coordinator::Coordinator;
|
|
||||||
use registry::{
|
|
||||||
schema::{
|
|
||||||
enums::NetworkListenerProtocol,
|
|
||||||
structs::{Coordinator as CoordinatorSetting, NatsCoordinator},
|
|
||||||
},
|
|
||||||
types::map::Map,
|
|
||||||
};
|
|
||||||
use serde_json::{Value, json};
|
|
||||||
use std::time::{Duration, Instant};
|
|
||||||
use testcontainers::{
|
|
||||||
GenericImage, ImageExt, core::IntoContainerPort, core::WaitFor, runners::AsyncRunner,
|
|
||||||
};
|
|
||||||
|
|
||||||
const HTTP_PORT: u16 = 11_310;
|
|
||||||
const TOPIC: &str = "inbuxa-coordinator-test";
|
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn coordinator_reconnect_tests() {
|
|
||||||
println!("Running coordinator reconnect tests...");
|
|
||||||
|
|
||||||
// A port with no NATS server behind it, yet
|
|
||||||
let nats_port = std::net::TcpListener::bind("127.0.0.1:0")
|
|
||||||
.unwrap()
|
|
||||||
.local_addr()
|
|
||||||
.unwrap()
|
|
||||||
.port();
|
|
||||||
let config = NatsCoordinator {
|
|
||||||
addresses: Map::new(vec![format!("127.0.0.1:{nats_port}")]),
|
|
||||||
use_tls: false,
|
|
||||||
timeout_connection: 1_000u64.into(),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
|
|
||||||
// 1. The node starts, without a build error, while NATS is down, and
|
|
||||||
// says so
|
|
||||||
let test = TestServerBuilder::new("coordinator_reconnect_tests")
|
|
||||||
.await
|
|
||||||
.with_object(CoordinatorSetting::Nats(config.clone()))
|
|
||||||
.await
|
|
||||||
.with_listener(NetworkListenerProtocol::Http, "http", HTTP_PORT, true)
|
|
||||||
.await
|
|
||||||
.build()
|
|
||||||
.await;
|
|
||||||
let coordinator = test.server.core.storage.coordinator.clone();
|
|
||||||
assert!(
|
|
||||||
coordinator.is_enabled(),
|
|
||||||
"a coordinator, though not connected"
|
|
||||||
);
|
|
||||||
assert_eq!(coordinator.is_connected(), Some(false));
|
|
||||||
assert_eq!(
|
|
||||||
cluster_health().await,
|
|
||||||
(503, json!({"coordinator": "disconnected"}))
|
|
||||||
);
|
|
||||||
|
|
||||||
// A subscription made now, as the broadcast subscriber makes it at
|
|
||||||
// startup, has to work once NATS is up
|
|
||||||
let mut stream = coordinator.subscribe(TOPIC).await.unwrap();
|
|
||||||
|
|
||||||
// 2. NATS comes up: the node connects on its own
|
|
||||||
let nats = GenericImage::new("nats", "latest")
|
|
||||||
.with_wait_for(WaitFor::message_on_stderr("Server is ready"))
|
|
||||||
.with_mapped_port(nats_port, 4222.tcp())
|
|
||||||
.start()
|
|
||||||
.await
|
|
||||||
.expect("Failed to start NATS container");
|
|
||||||
wait_for_health(200, "connected").await;
|
|
||||||
let other_node = coordinator::backend::nats::NatsPubSub::open(config.clone())
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
wait_until_connected(&other_node).await;
|
|
||||||
round_trip(&other_node, &mut stream, b"after startup").await;
|
|
||||||
|
|
||||||
// 3. NATS goes away: the node reports it; and it comes back: the node
|
|
||||||
// reconnects and the same subscription carries on
|
|
||||||
nats.stop().await.unwrap();
|
|
||||||
wait_for_health(503, "disconnected").await;
|
|
||||||
nats.start().await.unwrap();
|
|
||||||
wait_for_health(200, "connected").await;
|
|
||||||
wait_until_connected(&other_node).await;
|
|
||||||
round_trip(&other_node, &mut stream, b"after reconnect").await;
|
|
||||||
|
|
||||||
drop(nats);
|
|
||||||
if test.is_reset() {
|
|
||||||
test.temp_dir.delete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn cluster_health() -> (u16, Value) {
|
|
||||||
let response = reqwest::Client::builder()
|
|
||||||
.danger_accept_invalid_certs(true)
|
|
||||||
.timeout(Duration::from_secs(5))
|
|
||||||
.build()
|
|
||||||
.unwrap()
|
|
||||||
.get(format!("https://127.0.0.1:{HTTP_PORT}/healthz/cluster"))
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
let status = response.status().as_u16();
|
|
||||||
(status, response.json().await.unwrap())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn wait_for_health(status: u16, state: &str) {
|
|
||||||
let started = Instant::now();
|
|
||||||
loop {
|
|
||||||
let health = cluster_health().await;
|
|
||||||
if health == (status, json!({"coordinator": state})) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
assert!(
|
|
||||||
started.elapsed() < Duration::from_secs(30),
|
|
||||||
"expected {status} {state}, still {health:?}"
|
|
||||||
);
|
|
||||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn wait_until_connected(coordinator: &Coordinator) {
|
|
||||||
let started = Instant::now();
|
|
||||||
while coordinator.is_connected() != Some(true) {
|
|
||||||
assert!(started.elapsed() < Duration::from_secs(30), "not connected");
|
|
||||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Another node publishes; this one's subscription receives it.
|
|
||||||
async fn round_trip(from: &Coordinator, stream: &mut coordinator::PubSubStream, payload: &[u8]) {
|
|
||||||
from.publish(TOPIC, payload.to_vec()).await.unwrap();
|
|
||||||
let message = tokio::time::timeout(Duration::from_secs(10), stream.next())
|
|
||||||
.await
|
|
||||||
.expect("no message within 10 seconds")
|
|
||||||
.expect("subscription ended");
|
|
||||||
assert_eq!(message.payload(), payload);
|
|
||||||
}
|
|
||||||
@@ -2,11 +2,7 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
pub mod broadcast;
|
pub mod broadcast;
|
||||||
#[cfg(feature = "nats")]
|
|
||||||
pub mod coordinator; // inbuxa: coordinator reconnects
|
|
||||||
pub mod stress;
|
pub mod stress;
|
||||||
|
|||||||
Reference in New Issue
Block a user