--seed: set up a new store from a file, then exit (K-S5) #451

Merged
jcoffey-dev merged 1 commits from feat/store-seed into main 2026-11-18 19:37:21 +00:00
7 changed files with 525 additions and 18 deletions

No files matched your search

+49 -12
View File
@@ -33,6 +33,12 @@ pub struct BootManager {
/// inbuxa: K-S3: started with `--upgrade-store`: no listeners, no
/// services; main upgrades the store's layout and exits
pub upgrade_store: bool,
/// inbuxa: K-S5: started with `--seed <PATH>`: like `--upgrade-store`,
/// and then main applies the seed and exits
pub seed: Option<PathBuf>,
/// inbuxa: K-S5: whether the store was new when this process started,
/// before the defaults went in
pub store_was_new: bool,
}
pub struct IpcReceivers {
@@ -56,6 +62,7 @@ Options:
-i, --import <PATH> Import store data from a specific path
-o, --console Open the store console
-u, --upgrade-store Upgrade the store's layout, then exit
--seed <PATH> Set up a new store from a file, then exit
-h, --help Print help
-V, --version Print version
@@ -71,6 +78,12 @@ exits: 0 when the store is at this release's layout (whether it upgraded it
or found it there), 1 when the upgrade failed, 2 when it refused because
another node holds the upgrade lease, or still runs when a step needs every
other node stopped. It opens no ports and starts no services.
--seed sets up a new store from a file: a JMAP request body, its calls run
once as the system administrator, where {"$file": "<PATH>"} stands for that
file's contents. It upgrades the store's layout first, as --upgrade-store
does, and exits: 0 when the seed was applied (or was before, to this store),
1 when it failed, 2 when it refused because the store was set up already.
"#
);
@@ -81,6 +94,8 @@ enum StoreOp {
Console,
// inbuxa: K-S3
UpgradeStore,
// inbuxa: K-S5
Seed(PathBuf),
None,
}
@@ -106,7 +121,7 @@ impl BootManager {
};
let takes_value = matches!(
key.as_str(),
"config" | "c" | "export" | "e" | "import" | "i"
"config" | "c" | "export" | "e" | "import" | "i" | "seed"
);
let value = if takes_value {
inline.or_else(|| args.next())
@@ -138,6 +153,9 @@ impl BootManager {
("upgrade-store" | "u", None) => {
import_export = StoreOp::UpgradeStore;
}
("seed", Some(value)) => {
import_export = StoreOp::Seed(value.into());
}
(_, None) if takes_value => failed(&format!(
"Missing value for argument '{key}', try '--help'."
)),
@@ -159,28 +177,41 @@ impl BootManager {
// Initialize registry
let upgrade_store = import_export == StoreOp::UpgradeStore;
let seed = match &import_export {
StoreOp::Seed(path) => Some(path.clone()),
_ => None,
};
// inbuxa: K-S5: a seed run boots as --upgrade-store does
let one_shot = upgrade_store || seed.is_some();
let registry = RegistryStore::init(
PathBuf::from(config_path.unwrap()),
import_export == StoreOp::None || upgrade_store,
import_export == StoreOp::None || one_shot,
)
.await
.failed("⚠️ Startup failed");
if upgrade_store && (registry.is_bootstrap_mode() || registry.is_recovery_mode()) {
if one_shot && (registry.is_bootstrap_mode() || registry.is_recovery_mode()) {
failed(
"--upgrade-store needs a configured server: there is no data store to \
upgrade in bootstrap mode, and recovery mode is for a person at the console.",
"--upgrade-store and --seed need a configured server: there is no data \
store in bootstrap mode, and recovery mode is for a person at the console.",
);
}
let mut bootstrap = Bootstrap::new(registry).await;
// inbuxa: K-S5: a seed is only for a new store, which the defaults
// below stop being
let store_was_new = seed.is_some()
&& super::defaults::is_new_install(&bootstrap)
.await
.failed("⚠️ Startup failed");
// Add safe defaults if missing
if import_export == StoreOp::None {
if import_export == StoreOp::None || seed.is_some() {
bootstrap.insert_safe_defaults().await;
}
// Start listeners
// inbuxa: K-S3: not for --upgrade-store, which serves nothing
let mut servers = if upgrade_store {
let mut servers = if one_shot {
Listeners::default()
} else {
let servers = Listeners::parse(&mut bootstrap).await;
@@ -195,7 +226,7 @@ impl BootManager {
let telemetry = Telemetry::parse(&mut bootstrap, &storage).await;
match import_export {
StoreOp::None | StoreOp::UpgradeStore => {
StoreOp::None | StoreOp::UpgradeStore | StoreOp::Seed(_) => {
// Parse components
let core: Box<Core> =
Box::new(Box::pin(Core::parse(&mut bootstrap, storage)).await);
@@ -206,10 +237,14 @@ impl BootManager {
telemetry.enable();
if upgrade_store {
if one_shot {
trc::event!(
Server(trc::ServerEvent::Startup),
Details = "Started with --upgrade-store: upgrading the store layout, then exiting",
Details = if upgrade_store {
"Started with --upgrade-store: upgrading the store layout, then exiting"
} else {
"Started with --seed: setting up a new store from a file, then exiting"
},
Hostname = bootstrap.registry.local_hostname().to_string(),
Version = types::brand_version!(),
);
@@ -262,7 +297,7 @@ impl BootManager {
cache,
});
if !bootstrap.registry.is_recovery_mode() && !upgrade_store {
if !bootstrap.registry.is_recovery_mode() && !one_shot {
// Load spam model
if let Err(err) = inner.build_server().spam_model_reload().await {
trc::error!(
@@ -281,7 +316,7 @@ impl BootManager {
}
// Parse TCP acceptors
if !upgrade_store {
if !one_shot {
servers
.parse_tcp_acceptors(&mut bootstrap, inner.clone())
.await;
@@ -300,6 +335,8 @@ impl BootManager {
servers,
ipc_rxs,
upgrade_store,
seed,
store_was_new,
}
}
StoreOp::Export(path) => {
+7 -2
View File
@@ -96,8 +96,7 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> {
return Ok(());
}
let is_new_install =
bp.registry.count_object(ObjectType::Role).await? == 0 || first_boot_pending(bp).await?;
let is_new_install = is_new_install(bp).await?;
// inbuxa: a store brought over from Stalwart doesn't serve upstream's web
// interface (docs/spec/import-0.16.md), before the applications load
@@ -643,6 +642,12 @@ fn first_boot_key() -> ValueClass {
})
}
/// inbuxa: K-S5: a store no start has set up yet: no roles, or a first boot
/// that hasn't finished. The same test the defaults use.
pub async fn is_new_install(bp: &Bootstrap) -> trc::Result<bool> {
Ok(bp.registry.count_object(ObjectType::Role).await? == 0 || first_boot_pending(bp).await?)
}
async fn first_boot_pending(bp: &Bootstrap) -> trc::Result<bool> {
bp.data_store
.get_value::<String>(ValueKey::from(first_boot_key()))
+1 -1
View File
@@ -33,7 +33,7 @@ serde_json = "1.0"
hyper = { version = "1.11.1", features = ["server", "http1", "http2"] }
hyper-util = { version = "0.1.20", features = ["tokio"] }
http-body-util = "0.1.5"
tokio = { version = "1.53", features = ["rt"] }
tokio = { version = "1.53", features = ["rt", "sync"] }
futures-util = "0.3.34"
async-stream = "0.3.6"
base64 = "0.23"
+1
View File
@@ -38,6 +38,7 @@ pub mod relay; // inbuxa: inbuxa Relay
pub mod relay_message; // inbuxa: a relay's queues
pub mod scheduled_reports; // inbuxa: scheduled reports and the weekly digest
pub mod security_acceptance;
pub mod seed; // inbuxa: K-S5: a new store's setup from a file (--seed)
pub mod sharing_policy;
pub mod store_check; // inbuxa: checking the store's blobs
pub mod store_snapshot; // inbuxa: a snapshot of the store
+322
View File
@@ -0,0 +1,322 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! inbuxa: K-S5: a new store's setup from a file (`--seed`), for installs
//! that are declared rather than clicked through, such as the Helm chart
//! (docs/spec/store-seed.md).
//!
//! The file is a JMAP request body, the same `methodCalls` an administrator
//! would send to `/jmap`, with back-references between calls. It runs once,
//! through the same request handler, as the system administrator. A value
//! written `{"$file": "/path"}` is replaced by that file's contents, so
//! secrets stay in mounted files and out of the seed.
use common::{
Server,
auth::AccessToken,
config::server::{DEFAULT_TLS_TIMEOUT, ServerProtocol},
network::{ServerInstance, TcpAcceptor, limiter::ConcurrencyLimiter},
};
use http_proto::HttpSessionData;
use jmap_proto::request::Request;
use serde_json::Value;
use sha2::{Digest, Sha256};
use std::{
net::{IpAddr, Ipv4Addr},
path::Path,
sync::Arc,
};
use store::{
SUBSPACE_INBUXA, ValueKey,
write::{AnyClass, BatchBuilder, ValueClass},
};
use tokio::sync::watch;
use utils::snowflake::SnowflakeIdGenerator;
use crate::api::request::RequestHandler;
/// Where the store notes that a seed ran: `started <sha256>` while it runs,
/// `applied <sha256>` once every call succeeded.
pub const SEED_MARKER_KEY: &[u8] = b"Ks";
/// A seed has a few dozen calls, not the thousands a client may never send
const MAX_CALLS: usize = 1_000;
#[derive(Debug, PartialEq, Eq)]
pub enum SeedOutcome {
/// Every call succeeded; `created` objects were created or updated
Applied { calls: usize, created: usize },
/// This store was seeded before. `changed`: from a different file, which
/// isn't applied: after the first run the registry is what counts.
AlreadyApplied { changed: bool },
/// Not applied, and nothing written
Refused(String),
}
pub async fn apply_seed(
server: &Server,
path: &Path,
store_was_new: bool,
) -> Result<SeedOutcome, String> {
let raw = std::fs::read(path)
.map_err(|err| format!("Can't read the seed {}: {err}", path.display()))?;
let digest = hex(&Sha256::digest(&raw));
let mut request: Value = serde_json::from_slice(&raw)
.map_err(|err| format!("The seed {} isn't JSON: {err}", path.display()))?;
resolve_files(&mut request, &mut |file| {
std::fs::read_to_string(file)
.map(|contents| contents.trim_end_matches(['\r', '\n']).to_string())
.map_err(|err| format!("Can't read {file}, named by the seed: {err}"))
})?;
let body = serde_json::to_vec(&request).map_err(|err| err.to_string())?;
let store = server.store();
let marker = store
.get_value::<String>(ValueKey::from(marker_class()))
.await
.map_err(|err| format!("Can't read the store: {err}"))?;
match marker.as_deref().and_then(|m| m.split_once(' ')) {
Some(("applied", previous)) => {
return Ok(SeedOutcome::AlreadyApplied {
changed: previous != digest,
});
}
Some(_) => {
return Ok(SeedOutcome::Refused(
"An earlier seed run on this store stopped partway, so part of the seed may \
already be there. Finish the setup in the console, or start again from an \
empty store."
.to_string(),
));
}
None if !store_was_new => {
return Ok(SeedOutcome::Refused(
"The store is already set up, so the seed isn't applied: a seed is only for \
a new store. Make changes in the console or through the API."
.to_string(),
));
}
None => {}
}
let request = Request::parse(&body, MAX_CALLS, body.len() + 1)
.map_err(|err| format!("The seed isn't a JMAP request: {err}"))?;
let calls = request.method_calls.len();
set_marker(server, &format!("started {digest}")).await?;
let response = server
.handle_jmap_request(request, &AccessToken::new_admin(), &local_session())
.await;
let response = serde_json::to_value(&response).map_err(|err| err.to_string())?;
let (created, failed) = summarize(&response);
if !failed.is_empty() {
return Err(format!(
"The seed failed, and the store may hold part of it:\n {}",
failed.join("\n ")
));
}
set_marker(server, &format!("applied {digest}")).await?;
Ok(SeedOutcome::Applied { calls, created })
}
/// Replaces each `{"$file": "/path"}` with what `read` returns for the path
pub fn resolve_files(
value: &mut Value,
read: &mut impl FnMut(&str) -> Result<String, String>,
) -> Result<(), String> {
match value {
Value::Object(map) => {
if map.len() == 1
&& let Some(file) = map.get("$file")
{
let Value::String(file) = file else {
return Err("\"$file\" takes a path, as a string".to_string());
};
*value = Value::String(read(file)?);
return Ok(());
}
map.values_mut().try_for_each(|v| resolve_files(v, read))
}
Value::Array(items) => items.iter_mut().try_for_each(|v| resolve_files(v, read)),
_ => Ok(()),
}
}
/// How many objects the response created or updated, and every failure in
/// it: a method error, or anything not created, updated or destroyed. Only
/// error types and descriptions go into the failures, never the values sent,
/// which may be secrets.
pub fn summarize(response: &Value) -> (usize, Vec<String>) {
let mut created = 0;
let mut failed = Vec::new();
let calls = response
.get("methodResponses")
.and_then(Value::as_array)
.map(Vec::as_slice)
.unwrap_or_default();
for call in calls {
let (Some(name), Some(args), Some(id)) = (
call.get(0).and_then(Value::as_str),
call.get(1),
call.get(2).and_then(Value::as_str),
) else {
continue;
};
if name == "error" {
failed.push(format!("call \"{id}\": {}", describe(args)));
continue;
}
for key in ["created", "updated"] {
created += args
.get(key)
.and_then(Value::as_object)
.map_or(0, |m| m.len());
}
for key in ["notCreated", "notUpdated", "notDestroyed"] {
if let Some(errors) = args.get(key).and_then(Value::as_object) {
for (object, error) in errors {
failed.push(format!(
"call \"{id}\" ({name}) {key} {object}: {}",
describe(error)
));
}
}
}
}
(created, failed)
}
fn describe(error: &Value) -> String {
let kind = error.get("type").and_then(Value::as_str).unwrap_or("error");
let mut text = kind.to_string();
if let Some(description) = error.get("description").and_then(Value::as_str) {
text.push_str(": ");
text.push_str(description);
}
if let Some(properties) = error.get("properties").and_then(Value::as_array) {
let names = properties
.iter()
.filter_map(Value::as_str)
.collect::<Vec<_>>()
.join(", ");
if !names.is_empty() {
text.push_str(&format!(" ({names})"));
}
}
text
}
fn marker_class() -> ValueClass {
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key: SEED_MARKER_KEY.to_vec(),
})
}
async fn set_marker(server: &Server, value: &str) -> Result<(), String> {
let mut batch = BatchBuilder::new();
batch.set(marker_class(), value.as_bytes().to_vec());
server
.store()
.write(batch.build_all())
.await
.map(|_| ())
.map_err(|err| format!("Can't write the seed's note to the store: {err}"))
}
/// The seed is applied in-process: no connection, so a session of its own,
/// as the task manager makes one for alarms. Marked TLS because nothing
/// crosses a network.
fn local_session() -> HttpSessionData {
HttpSessionData {
instance: Arc::new(ServerInstance {
id: "_seed".to_string(),
protocol: ServerProtocol::Http,
acceptor: TcpAcceptor::Plain,
limiter: ConcurrencyLimiter::new(1),
proxy_networks: vec![],
tls_timeout: DEFAULT_TLS_TIMEOUT,
shutdown_rx: watch::channel(false).1,
span_id_gen: Arc::new(SnowflakeIdGenerator::new()),
}),
local_ip: IpAddr::V4(Ipv4Addr::LOCALHOST),
local_port: 0,
remote_ip: IpAddr::V4(Ipv4Addr::LOCALHOST),
remote_port: 0,
is_tls: true,
session_id: 0,
}
}
fn hex(bytes: &[u8]) -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn reads_secrets_from_files() {
let mut seed = json!({"methodCalls": [["x:Account/set", {"create": {"a": {
"name": "admin",
"credentials": {"0": {"@type": "Password", "secret": {"$file": "/s/admin"}}}
}}}, "c"]]});
resolve_files(&mut seed, &mut |path| Ok(format!("pw-from-{path}"))).unwrap();
assert_eq!(
seed.pointer("/methodCalls/0/1/create/a/credentials/0/secret"),
Some(&json!("pw-from-/s/admin"))
);
}
#[test]
fn leaves_other_objects_alone() {
let mut seed = json!({"a": {"$file": "/x", "other": 1}, "b": [{"filePath": "/y"}]});
let before = seed.clone();
resolve_files(&mut seed, &mut |_| Err("not read".to_string())).unwrap();
assert_eq!(seed, before);
}
#[test]
fn stops_on_a_file_it_cannot_read() {
let mut seed = json!([{"$file": "/missing"}]);
assert_eq!(
resolve_files(&mut seed, &mut |_| Err("gone".to_string())),
Err("gone".to_string())
);
let mut seed = json!({"$file": 3});
assert!(resolve_files(&mut seed, &mut |_| Ok(String::new())).is_err());
}
#[test]
fn counts_changes_and_reports_failures_without_values() {
let response = json!({"methodResponses": [
["x:Domain/set", {"created": {"d": {"id": "a1"}}}, "d"],
["x:Account/set", {"created": {}, "notCreated": {"a": {
"type": "invalidProperties", "description": "Bad secret", "properties": ["credentials"]
}}}, "a"],
["x:SystemSettings/set", {"updated": {"singleton": null}}, "s"],
["error", {"type": "unknownMethod"}, "x"]
]});
let (created, failed) = summarize(&response);
assert_eq!(created, 2);
assert_eq!(
failed,
vec![
"call \"a\" (x:Account/set) notCreated a: invalidProperties: Bad secret (credentials)",
"call \"x\": unknownMethod",
]
);
}
}
+66 -3
View File
@@ -68,6 +68,12 @@ async fn main() -> std::io::Result<()> {
std::process::exit(Box::pin(upgrade_store_and_exit(&server)).await);
}
// inbuxa: K-S5: --seed sets up a new store from a file and exits
if let Some(path) = init.seed.clone() {
let server = init.inner.build_server();
std::process::exit(Box::pin(seed_and_exit(&server, &path, init.store_was_new)).await);
}
// Migrate database
if let Err(err) = migration::try_migrate(&init.inner.build_server()).await {
trc::event!(
@@ -199,9 +205,65 @@ fn spawn_listener(
/// this build's layout (upgraded now or already), 1 when the upgrade failed,
/// 2 when it was refused because another node is in the way.
async fn upgrade_store_and_exit(server: &common::Server) -> i32 {
let (code, message) = upgrade_store_only(server).await;
eprintln!("{message}");
flush_and_exit_code(code).await
}
/// inbuxa: K-S5: `--seed`: the layout upgrade, then the seed
async fn seed_and_exit(
server: &common::Server,
path: &std::path::Path,
store_was_new: bool,
) -> i32 {
use jmap::inbuxa::seed::{SeedOutcome, apply_seed};
let (code, message) = upgrade_store_only(server).await;
eprintln!("{message}");
if code != 0 {
return flush_and_exit_code(code).await;
}
let (code, message) = match apply_seed(server, path, store_was_new).await {
Ok(SeedOutcome::Applied { calls, created }) => (
0,
format!("Applied the seed: {calls} calls, {created} objects created or updated."),
),
Ok(SeedOutcome::AlreadyApplied { changed: false }) => (
0,
"This store was seeded before: nothing to do.".to_string(),
),
Ok(SeedOutcome::AlreadyApplied { changed: true }) => (
0,
"This store was seeded before, from a different file. The seed is only \
applied once: change the setup in the console or through the API."
.to_string(),
),
Ok(SeedOutcome::Refused(reason)) => {
trc::event!(
Server(trc::ServerEvent::StartupError),
Details = "Refused to apply the seed",
Reason = reason.clone(),
);
(2, format!("Refused to apply the seed: {reason}"))
}
Err(reason) => {
trc::event!(
Server(trc::ServerEvent::StartupError),
Details = "Failed to apply the seed",
Reason = reason.clone(),
);
(1, reason)
}
};
eprintln!("{message}");
flush_and_exit_code(code).await
}
async fn upgrade_store_only(server: &common::Server) -> (i32, String) {
use migration::layout::Outcome;
let (code, message) = match migration::upgrade_store(server).await {
match migration::upgrade_store(server).await {
Ok(Outcome::Current) => (
0,
format!(
@@ -233,9 +295,10 @@ async fn upgrade_store_and_exit(server: &common::Server) -> i32 {
);
(1, message)
}
};
eprintln!("{message}");
}
}
async fn flush_and_exit_code(code: i32) -> i32 {
// Let the log reach its collector
Collector::shutdown();
tokio::time::sleep(Duration::from_secs(1)).await;
+79
View File
@@ -0,0 +1,79 @@
# Feature spec: setting up a new store from a file (`--seed`)
Status: approved direction, 2026-11-18 (John chose "a setup file the server
reads" over a chart job that drives recovery mode). Kubernetes item K-S5.
Not a rebuild (SPEC.md §4): inbuxa's own design.
## Provenance
| Source | Used for |
|---|---|
| The Kubernetes recipe's chart work (inbuxa-charts K-C2) | The gap: a chart can name the data store, but nothing declarative creates the registry objects a cluster needs |
| This server's `--upgrade-store` (K-S3, `crates/main/src/main.rs`) | The one-shot shape: boot without listeners, do one thing, exit with 0, 1 or 2 |
| This server's JMAP request handler and `x:*/set` (`crates/jmap`) | How the seed's calls run: the same validation, password hashing and follow-up tasks as the API |
| RFC 8620 §3.3 and §5.3 | The request body the seed is, and its creation-id references (`#d`) |
Nothing here comes from Enterprise code or from upstream.
## What it is
A server reads only `config.json`, which holds the data store, plus some
environment variables. Everything else lives in the registry, inside the
store, and is set through the API or the console. An install that is
declared rather than clicked through has to put the first objects there: the
domain, the administrator, the blob store, the coordinator and the cluster
roles. A Helm chart is that kind of install.
`inbuxa --seed <PATH>` does it. The file is a JMAP request body. The server
applies it once, to a new store, and exits.
## Requirements
- **SD-1.** `--seed <PATH>` boots as `--upgrade-store` does: no listeners, no
services, no spam model. It also refuses bootstrap mode and recovery mode.
It upgrades the store's layout first. If that doesn't end at this
release's layout, it exits with `--upgrade-store`'s code and applies
nothing.
- **SD-2.** The safe defaults a first start writes (roles, listeners, queue
settings and the rest) go in before the seed. So the seed run leaves the
store as a first start would, plus the seed. Pods started after it find
their cluster role, and don't race each other to write the defaults of an
empty store.
- **SD-3.** The file is a JMAP request body: `using` plus `methodCalls`, the
body an administrator would POST to `/jmap`. Its calls run through the
server's request handler as the system administrator, in order, with
creation-id and result references. A seed may hold up to 1,000 calls.
- **SD-4.** A value written `{"$file": "<PATH>"}`, an object with that one
key, is replaced by the file's contents, with trailing line breaks taken
off. This happens before the calls run. Secrets stay in mounted files, and
the seed can live in a ConfigMap or in git. A `$file` that can't be read
fails the run before anything is written.
- **SD-5.** A seed applies once, to a new store. "New" is the defaults'
test: no roles yet, or a first start that never finished. The store keeps
a note under key `Ks` in the fork's subspace:
- `started <sha256 of the file>` is written before the calls run;
- `applied <sha256>` once every call succeeded.
A later run finds `applied` and does nothing. If the file changed since,
it says so: after the first run, the registry is the source of truth.
- **SD-6.** Exit codes:
| Code | When |
|---|---|
| 0 | Applied, or applied before to this store |
| 1 | Failed: a call returned an error, or something wasn't created, updated or destroyed. The store may hold part of the seed |
| 2 | Refused: the store was set up another way, or an earlier seed run stopped partway (a `started` note). Nothing written |
The message names each failed call and error type, never the values sent.
- **SD-7.** A partial seed isn't retried: running it again would create
objects twice. The operator finishes the setup in the console or starts
again from an empty store. A Kubernetes Job runs the seed with
`backoffLimit: 0` for the same reason.
## Out of scope
- Applying a seed again to bring the registry back in line with the file
(reconciliation). A later spec can add that, if operators ask for it.
- Serialising the defaults of an empty store between nodes that all start
at once without a seed. With the seed (SD-2) they don't start on an empty
store.