From 4568f9abbfc07acadc16b8ad8658d14b7173deaf Mon Sep 17 00:00:00 2001 From: John Coffey Date: Sun, 23 Aug 2026 18:51:19 -0700 Subject: [PATCH] Capture the pre-migration snapshot from 0.15.x, and stop claiming counts match when none were compared Found by running preflight against a real Stalwart 0.15.5 in a VM. Two defects, the second worse than the first. 1. AccountSnapshot could not read the version this tool migrates FROM. 0.15.5 advertises no urn:stalwart:jmap capability and POST /api returns 404 - the JMAP management API and x:Account are 0.16 features. 0.15.x exposes a REST API at GET /api/principal instead. So preflight's account-snapshot check warned and moved on, and every run against a real source instance had no "before" data at all. AccountSnapshot now dispatches on the capability the session document advertises - a positive signal, not an inference from a failed call - and internal/stalwartapi/principal.go implements the 0.15.x REST path, including its 1-based page/limit pagination so an install larger than one page isn't silently truncated. 2. With no "before" counts, the content-integrity comparison iterated an empty map, checked nothing, and reported "all message counts match". That is the strongest claim this tool makes - ARCHITECTURE 4.7 calls it the actual no-data-loss guarantee - made vacuously, and it would have passed on a migration that lost every message. The comparison now derives its account set from whatever the source could report, verifies every account and domain survived either way, and carries MessageCountsCompared so the report says plainly "MESSAGE COUNTS NOT COMPARED ... no-data-loss is NOT verified here" rather than implying otherwise. What can and cannot be checked across the 0.15/0.16 boundary, now that a real server has answered: 0.15.x has no per-mailbox message count at any endpoint, and the impersonation login 0.16 offers returns 401 there, so before/after message counts are impossible for the boundary migration this tool exists for. Both versions do report per-account used quota (usedQuota in 0.15's REST list, usedDiskQuota on 0.16's x:Account), so that is captured on both sides. It is recorded and reported, not asserted on: 4.5 notes the 0.16 migration resets quotas to zero pending recalculation, so comparing those bytes across the boundary would be a false alarm generator. Test servers across preflight, validate and stalwartapi now advertise urn:stalwart:jmap, since they stand in for 0.16 instances and that capability is what says so. Verified end to end against the smoke VM: all nine preflight checks pass, and the checkpoint records 2 accounts, 1 domain and per-account used quota where it previously recorded nothing. --- internal/checkpoint/types.go | 20 +- internal/preflight/checks.go | 11 +- internal/preflight/checks_test.go | 7 +- internal/stalwartapi/client.go | 6 + internal/stalwartapi/mailbox.go | 4 + internal/stalwartapi/management.go | 30 ++- internal/stalwartapi/management_test.go | 30 +++ internal/stalwartapi/principal.go | 197 ++++++++++++++++++++ internal/stalwartapi/principal_test.go | 183 ++++++++++++++++++ internal/validate/content_integrity.go | 109 ++++++++++- internal/validate/content_integrity_test.go | 76 ++++++++ internal/validate/main_test.go | 5 +- 12 files changed, 656 insertions(+), 22 deletions(-) create mode 100644 internal/stalwartapi/principal.go create mode 100644 internal/stalwartapi/principal_test.go diff --git a/internal/checkpoint/types.go b/internal/checkpoint/types.go index e4714b8..65ae230 100644 --- a/internal/checkpoint/types.go +++ b/internal/checkpoint/types.go @@ -74,13 +74,19 @@ type MailboxCount struct { // which the validate phase later compares against the migrated instance. // See ARCHITECTURE.md §4.1 and §4.7. type PreflightSnapshot struct { - TakenAt time.Time `json:"taken_at"` - AccountCount int `json:"account_count"` - Domains []string `json:"domains,omitempty"` - MailboxCounts map[string][]MailboxCount `json:"mailbox_counts,omitempty"` // account -> mailboxes - DKIMFingerprints map[string]string `json:"dkim_fingerprints,omitempty"` - TLSFingerprints []string `json:"tls_fingerprints,omitempty"` - ListenerPorts []int `json:"listener_ports,omitempty"` + TakenAt time.Time `json:"taken_at"` + AccountCount int `json:"account_count"` + Domains []string `json:"domains,omitempty"` + MailboxCounts map[string][]MailboxCount `json:"mailbox_counts,omitempty"` // account -> mailboxes + // UsedQuota is each account's used storage in bytes. It is the only + // per-account content measure available on both sides of the 0.15/0.16 + // boundary - 0.15.x exposes no per-mailbox message counts at all - so + // it is what the post-migration comparison can actually assert on a + // boundary migration. See stalwartapi/principal.go. + UsedQuota map[string]int64 `json:"used_quota,omitempty"` + DKIMFingerprints map[string]string `json:"dkim_fingerprints,omitempty"` + TLSFingerprints []string `json:"tls_fingerprints,omitempty"` + ListenerPorts []int `json:"listener_ports,omitempty"` } // Topology records how this Stalwart instance is deployed, as detected diff --git a/internal/preflight/checks.go b/internal/preflight/checks.go index a4226b2..24ae2af 100644 --- a/internal/preflight/checks.go +++ b/internal/preflight/checks.go @@ -244,9 +244,16 @@ func (c *Checker) Run(ctx context.Context, store *checkpoint.Store, rs *checkpoi AccountCount: snap.AccountCount, Domains: snap.Domains, MailboxCounts: mailboxCounts, + UsedQuota: snap.UsedQuota, + } + detail := fmt.Sprintf("captured snapshot: %d account(s) across %d domain(s), used-quota for %d account(s), mailbox counts for %d account(s)", + snap.AccountCount, len(snap.Domains), len(snap.UsedQuota), len(mailboxCounts)) + if len(mailboxCounts) == 0 && len(snap.UsedQuota) > 0 { + // Expected against a 0.15.x source: it exposes no + // per-mailbox counts, so used-quota is what the + // post-migration comparison will have to work from. + detail += " (this source exposes no per-mailbox counts, so the post-migration check compares accounts, domains and used-quota)" } - detail := fmt.Sprintf("captured snapshot: %d account(s) across %d domain(s), mailbox counts for %d account(s)", - snap.AccountCount, len(snap.Domains), len(mailboxCounts)) status := StatusOK if len(snap.MailboxErrors) > 0 { status = StatusWarn diff --git a/internal/preflight/checks_test.go b/internal/preflight/checks_test.go index 1b30279..56257fb 100644 --- a/internal/preflight/checks_test.go +++ b/internal/preflight/checks_test.go @@ -232,7 +232,12 @@ func TestCheckerRunCapturesAccountSnapshotWhenAdminURLSet(t *testing.T) { }) return } - w.WriteHeader(http.StatusOK) + // A 0.16-era instance: the urn:stalwart:jmap capability is what + // says its management API is the JMAP one. + json.NewEncoder(w).Encode(map[string]any{ + "apiUrl": apiURL, + "capabilities": map[string]any{"urn:ietf:params:jmap:core": map[string]any{}, "urn:stalwart:jmap": map[string]any{}}, + }) case r.Method == http.MethodPost && r.URL.Path == "/api": var body map[string]any json.NewDecoder(r.Body).Decode(&body) diff --git a/internal/stalwartapi/client.go b/internal/stalwartapi/client.go index 4534c8d..f3b15bd 100644 --- a/internal/stalwartapi/client.go +++ b/internal/stalwartapi/client.go @@ -63,6 +63,12 @@ type Snapshot struct { AccountCount int Domains []string MailboxCounts map[string][]MailboxCount // account email -> its mailboxes + // UsedQuota is each account's used storage in bytes. Unlike + // MailboxCounts it is available from both API generations - 0.15.x's + // REST principal list and 0.16's x:Account both report it - which + // makes it the only per-account content measure that can be compared + // across the 0.15/0.16 boundary. See principal.go. + UsedQuota map[string]int64 // MailboxErrors records, per account email, why that account's mailbox // counts couldn't be captured (e.g. impersonation not permitted for // that account). A non-empty entry here means MailboxCounts has no diff --git a/internal/stalwartapi/mailbox.go b/internal/stalwartapi/mailbox.go index 0a91b18..708b223 100644 --- a/internal/stalwartapi/mailbox.go +++ b/internal/stalwartapi/mailbox.go @@ -20,6 +20,10 @@ import ( type jmapSession struct { APIURL string `json:"apiUrl"` PrimaryAccounts map[string]string `json:"primaryAccounts"` + // Capabilities is what the instance says it supports. Its contents + // decide which management API this client speaks - see + // stalwartManagementCapability. + Capabilities map[string]json.RawMessage `json:"capabilities"` } const jmapMailCapability = "urn:ietf:params:jmap:mail" diff --git a/internal/stalwartapi/management.go b/internal/stalwartapi/management.go index 0a87a59..82af84e 100644 --- a/internal/stalwartapi/management.go +++ b/internal/stalwartapi/management.go @@ -104,6 +104,10 @@ type account struct { ID string `json:"id"` Name string `json:"name"` DomainID string `json:"domainId"` + // UsedDiskQuota is 0.16's name for what 0.15's REST API calls + // usedQuota - the same per-account byte count, which is what makes a + // cross-boundary content comparison possible at all. + UsedDiskQuota int64 `json:"usedDiskQuota"` } // AccountSnapshot enumerates every account on the instance via Stalwart's @@ -124,6 +128,22 @@ type account struct { // exactly what's missing rather than silently treating an unreachable // account's mailboxes as having zero messages. func (c *Client) AccountSnapshot(ctx context.Context) (*Snapshot, error) { + isJMAP, err := c.hasJMAPManagement(ctx) + if err != nil { + return nil, fmt.Errorf("stalwartapi: discover which management API this instance speaks: %w", err) + } + if !isJMAP { + // No urn:stalwart:jmap: this is a 0.15.x instance, whose + // management API is REST. Confirmed against a live 0.15.5 server - + // see principal.go. + return c.principalSnapshotREST(ctx) + } + return c.accountSnapshotJMAP(ctx) +} + +// accountSnapshotJMAP is the 0.16+ path, via Stalwart's JMAP management +// objects. +func (c *Client) accountSnapshotJMAP(ctx context.Context) (*Snapshot, error) { ids, err := c.AccountIDs(ctx) if err != nil { return nil, err @@ -133,7 +153,7 @@ func (c *Client) AccountSnapshot(ctx context.Context) (*Snapshot, error) { } getResp, err := c.call(ctx, managementCapabilities, []any{ - []any{"x:Account/get", map[string]any{"ids": ids, "properties": []string{"id", "name", "domainId"}}, "g"}, + []any{"x:Account/get", map[string]any{"ids": ids, "properties": []string{"id", "name", "domainId", "usedDiskQuota"}}, "g"}, }) if err != nil { return nil, fmt.Errorf("stalwartapi: Account/get: %w", err) @@ -169,11 +189,19 @@ func (c *Client) AccountSnapshot(ctx context.Context) (*Snapshot, error) { } sort.Strings(domains) + usedQuota := make(map[string]int64, len(accounts)) + for _, a := range accounts { + if a.Name != "" { + usedQuota[a.Name] = a.UsedDiskQuota + } + } + return &Snapshot{ AccountCount: len(accounts), Domains: domains, MailboxCounts: mailboxCounts, MailboxErrors: mailboxErrors, + UsedQuota: usedQuota, }, nil } diff --git a/internal/stalwartapi/management_test.go b/internal/stalwartapi/management_test.go index 3b5c235..085d99b 100644 --- a/internal/stalwartapi/management_test.go +++ b/internal/stalwartapi/management_test.go @@ -12,6 +12,21 @@ import ( "testing" ) +// serveJMAPSession answers session discovery for a server standing in for +// a 0.16+ instance: what marks it as one is the urn:stalwart:jmap +// capability, which is exactly what AccountSnapshot dispatches on. Returns +// true if it handled the request. +func serveJMAPSession(w http.ResponseWriter, r *http.Request, apiURL string) bool { + if r.Method != http.MethodGet || r.URL.Path != "/.well-known/jmap" { + return false + } + json.NewEncoder(w).Encode(map[string]any{ + "apiUrl": apiURL, + "capabilities": map[string]any{"urn:ietf:params:jmap:core": map[string]any{}, "urn:stalwart:jmap": map[string]any{}}, + }) + return true +} + // jmapEnvelope mirrors the wire shape this package's call() parses: a // top-level {"methodResponses": [...]} object where each entry is a // [name, args, callId] triple (RFC 8620 §3.2). @@ -35,6 +50,12 @@ func accountManagementAndMailboxServer(t *testing.T, mailboxesFor map[string][]m if r.Method == http.MethodGet && r.URL.Path == "/.well-known/jmap" { user, _, _ := r.BasicAuth() + if !strings.Contains(user, "%") { + // The client's own session request, used to decide which + // management API this instance speaks. + serveJMAPSession(w, r, apiURL) + return + } target := strings.SplitN(user, "%", 2)[0] if _, ok := mailboxesFor[target]; !ok { w.WriteHeader(http.StatusForbidden) @@ -140,6 +161,9 @@ func TestAccountSnapshotRecordsPerAccountMailboxFailureWithoutFailingOverall(t * func TestAccountSnapshotEmptyInstance(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if serveJMAPSession(w, r, "/api") { + return + } json.NewEncoder(w).Encode(jmapEnvelope{MethodResponses: []any{ []any{"x:Account/query", map[string]any{"ids": []string{}}, "q"}, }}) @@ -158,6 +182,9 @@ func TestAccountSnapshotEmptyInstance(t *testing.T) { func TestAccountSnapshotPropagatesJMAPError(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if serveJMAPSession(w, r, "/api") { + return + } json.NewEncoder(w).Encode(jmapEnvelope{MethodResponses: []any{ []any{"error", map[string]any{"type": "forbidden"}, "q"}, }}) @@ -191,6 +218,9 @@ func TestAccountSnapshotSendsBasicAuth(t *testing.T) { if !ok || user != "admin" || pass != "hunter2" { t.Errorf("BasicAuth = (%s, %s, %v), want (admin, hunter2, true)", user, pass, ok) } + if serveJMAPSession(w, r, "/api") { + return + } json.NewEncoder(w).Encode(jmapEnvelope{MethodResponses: []any{ []any{"x:Account/query", map[string]any{"ids": []string{}}, "q"}, }}) diff --git a/internal/stalwartapi/principal.go b/internal/stalwartapi/principal.go new file mode 100644 index 0000000..d1931f3 --- /dev/null +++ b/internal/stalwartapi/principal.go @@ -0,0 +1,197 @@ +// SPDX-FileCopyrightText: 2026 LINUXexpert-org +// SPDX-License-Identifier: GPL-3.0-or-later + +package stalwartapi + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" +) + +// Stalwart 0.15.x - the version this tool migrates *from* - has no +// urn:stalwart:jmap capability and no JMAP management endpoint: POST /api +// returns 404 there. Its management API is REST, at GET /api/principal, +// and that is the only way to enumerate the pre-migration directory. +// +// This was found by running preflight against a real 0.15.5 instance, not +// from documentation: the published schema reference documents 0.16, and +// following it alone produced a tool that could not read the source +// version it exists to migrate. The shapes below were confirmed against +// that live server. +// +// GET /api/principal?types=individual&limit=100&page=1 +// {"data":{"items":[{"id":4,"type":"individual","name":"alice", +// "emails":["alice@smoke.test"],"usedQuota":9207}], +// "total":3}} +// +// Two limits of the 0.15 side are worth stating plainly, because they +// decide what the post-migration comparison can actually assert: +// +// - There is no per-mailbox message count anywhere in this API, and the +// impersonation mechanism 0.16 offers (the `%` +// composite login MailboxSnapshot uses) returns 401 on 0.15.5. So a +// pre-migration snapshot cannot carry message counts, and a +// before/after comparison of them is impossible for the boundary +// migration this tool is built for. +// - What both versions do expose per account is used quota in bytes +// (`usedQuota` here, `usedDiskQuota` on 0.16's x:Account). That is a +// real content measure - it moves when mail is lost - so it is what +// the integrity comparison uses across the boundary. +const restPrincipalPageSize = 100 + +// stalwartManagementCapability is advertised by instances whose management +// API is the JMAP one (0.16+). Its absence is what distinguishes a 0.15.x +// instance, and is a positive signal rather than an inference from a failed +// call. +const stalwartManagementCapability = "urn:stalwart:jmap" + +// hasJMAPManagement reports whether this instance speaks the 0.16+ JMAP +// management API, by reading the capability list from its session +// document. It deliberately does its own request rather than reusing +// fetchSession: that helper also requires an apiUrl and a mail account, +// which are needed for impersonated mailbox reads but have nothing to do +// with which management API to use - failing dispatch over a missing +// apiUrl would misroute an instance that is perfectly readable. +func (c *Client) hasJMAPManagement(ctx context.Context) (bool, error) { + endpoint := strings.TrimRight(c.BaseURL, "/") + "/.well-known/jmap" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return false, err + } + req.SetBasicAuth(c.Username, c.Password) + resp, err := c.httpClient().Do(req) + if err != nil { + return false, fmt.Errorf("stalwartapi: reach %s: %w", endpoint, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return false, fmt.Errorf("stalwartapi: session discovery at %s returned %s", endpoint, resp.Status) + } + var session struct { + Capabilities map[string]json.RawMessage `json:"capabilities"` + } + if err := json.NewDecoder(resp.Body).Decode(&session); err != nil { + return false, fmt.Errorf("stalwartapi: parse session document from %s: %w", endpoint, err) + } + _, ok := session.Capabilities[stalwartManagementCapability] + return ok, nil +} + +type restPrincipal struct { + ID int `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + Emails []string `json:"emails"` + UsedQuota int64 `json:"usedQuota"` +} + +type restPrincipalPage struct { + Data struct { + Items []restPrincipal `json:"items"` + Total int `json:"total"` + } `json:"data"` +} + +// restPrincipals fetches every principal of the given type, following the +// API's 1-based page/limit pagination rather than assuming one request +// returns everything - an install with more accounts than the page size +// would otherwise be silently truncated, and a truncated "before" snapshot +// would make the post-migration comparison claim more than it checked. +func (c *Client) restPrincipals(ctx context.Context, principalType string) ([]restPrincipal, error) { + var all []restPrincipal + for page := 1; ; page++ { + q := url.Values{} + if principalType != "" { + q.Set("types", principalType) + } + q.Set("limit", fmt.Sprint(restPrincipalPageSize)) + q.Set("page", fmt.Sprint(page)) + endpoint := strings.TrimRight(c.BaseURL, "/") + "/api/principal?" + q.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, err + } + req.SetBasicAuth(c.Username, c.Password) + resp, err := c.httpClient().Do(req) + if err != nil { + return nil, fmt.Errorf("stalwartapi: list principals: %w", err) + } + body, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("stalwartapi: GET %s returned %s: %s", endpoint, resp.Status, strings.TrimSpace(string(body))) + } + if readErr != nil { + return nil, fmt.Errorf("stalwartapi: read principal list: %w", readErr) + } + var parsed restPrincipalPage + if err := json.Unmarshal(body, &parsed); err != nil { + return nil, fmt.Errorf("stalwartapi: parse principal list: %w", err) + } + all = append(all, parsed.Data.Items...) + if len(parsed.Data.Items) == 0 || len(all) >= parsed.Data.Total { + return all, nil + } + } +} + +// principalSnapshotREST builds a Snapshot from the 0.15.x REST management +// API. MailboxCounts is deliberately left empty: see this file's opening +// comment for why counts cannot be obtained from a 0.15 instance at all. +func (c *Client) principalSnapshotREST(ctx context.Context) (*Snapshot, error) { + individuals, err := c.restPrincipals(ctx, "individual") + if err != nil { + return nil, err + } + domainPrincipals, err := c.restPrincipals(ctx, "domain") + if err != nil { + return nil, err + } + + snap := &Snapshot{ + AccountCount: len(individuals), + UsedQuota: make(map[string]int64, len(individuals)), + } + for _, p := range individuals { + snap.UsedQuota[accountKey(p)] = p.UsedQuota + } + + domainSet := map[string]bool{} + for _, d := range domainPrincipals { + if d.Name != "" { + domainSet[d.Name] = true + } + } + // Fall back to the domains implied by account addresses if the + // instance has no explicit domain principals. + for _, p := range individuals { + for _, email := range p.Emails { + if at := strings.LastIndex(email, "@"); at >= 0 && at+1 < len(email) { + domainSet[email[at+1:]] = true + } + } + } + for d := range domainSet { + snap.Domains = append(snap.Domains, d) + } + sort.Strings(snap.Domains) + return snap, nil +} + +// accountKey identifies an account the same way both API generations can: +// by its primary email address where it has one, falling back to the bare +// login name. The v0.16 migration rewrites bare names into addresses, which +// is exactly why the comparison side matches on local part as well. +func accountKey(p restPrincipal) string { + if len(p.Emails) > 0 && p.Emails[0] != "" { + return p.Emails[0] + } + return p.Name +} diff --git a/internal/stalwartapi/principal_test.go b/internal/stalwartapi/principal_test.go new file mode 100644 index 0000000..956eb7e --- /dev/null +++ b/internal/stalwartapi/principal_test.go @@ -0,0 +1,183 @@ +// SPDX-FileCopyrightText: 2026 LINUXexpert-org +// SPDX-License-Identifier: GPL-3.0-or-later + +package stalwartapi + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// stalwart015Server stands in for a 0.15.x instance: no urn:stalwart:jmap +// capability, POST /api is 404, and the management API is REST at +// /api/principal with 1-based page/limit paging. Every shape here was +// confirmed against a live 0.15.5 server. +func stalwart015Server(t *testing.T, individuals, domains []map[string]any) (*httptest.Server, *[]string) { + t.Helper() + var paths []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.Path+"?"+r.URL.RawQuery) + + if r.URL.Path == "/.well-known/jmap" { + // 0.15.5 advertises no urn:stalwart:jmap. + json.NewEncoder(w).Encode(map[string]any{ + "capabilities": map[string]any{"urn:ietf:params:jmap:core": map[string]any{}}, + }) + return + } + if r.URL.Path == "/api" { + w.WriteHeader(http.StatusNotFound) + w.Write([]byte(`{"status":404,"title":"Not Found"}`)) + return + } + if r.URL.Path != "/api/principal" { + w.WriteHeader(http.StatusNotFound) + return + } + + set := individuals + if r.URL.Query().Get("types") == "domain" { + set = domains + } + limit, page := 100, 1 + fmt.Sscanf(r.URL.Query().Get("limit"), "%d", &limit) + fmt.Sscanf(r.URL.Query().Get("page"), "%d", &page) + + start := (page - 1) * limit + end := start + limit + if start > len(set) { + start = len(set) + } + if end > len(set) { + end = len(set) + } + json.NewEncoder(w).Encode(map[string]any{ + "data": map[string]any{"items": set[start:end], "total": len(set)}, + }) + })) + t.Cleanup(srv.Close) + return srv, &paths +} + +// The headline case: against the version this tool actually migrates from, +// AccountSnapshot must not fall over on a 404 from the 0.16-only endpoint. +func TestAccountSnapshotUsesRESTAgainst015(t *testing.T) { + srv, paths := stalwart015Server(t, + []map[string]any{ + {"id": 4, "type": "individual", "name": "alice", "emails": []string{"alice@smoke.test"}, "usedQuota": 9207}, + {"id": 5, "type": "individual", "name": "bob", "emails": []string{"bob@smoke.test"}, "usedQuota": 5380}, + }, + []map[string]any{{"id": 1, "type": "domain", "name": "smoke.test"}}, + ) + client := &Client{BaseURL: srv.URL, Username: "admin", Password: "hunter2"} + + snap, err := client.AccountSnapshot(context.Background()) + if err != nil { + t.Fatalf("AccountSnapshot against 0.15.x: %v", err) + } + if snap.AccountCount != 2 { + t.Errorf("AccountCount = %d, want 2", snap.AccountCount) + } + if len(snap.Domains) != 1 || snap.Domains[0] != "smoke.test" { + t.Errorf("Domains = %v, want [smoke.test]", snap.Domains) + } + if snap.UsedQuota["alice@smoke.test"] != 9207 || snap.UsedQuota["bob@smoke.test"] != 5380 { + t.Errorf("UsedQuota = %v, want alice 9207 and bob 5380", snap.UsedQuota) + } + // Message counts genuinely cannot be had from 0.15.x - see principal.go. + if len(snap.MailboxCounts) != 0 { + t.Errorf("MailboxCounts = %v, want empty: 0.15.x exposes no per-mailbox counts", snap.MailboxCounts) + } + for _, p := range *paths { + if strings.HasPrefix(p, "/api?") { + t.Errorf("the 0.16-only JMAP management endpoint was called against a 0.15.x instance: %s", p) + } + } +} + +// A truncated "before" snapshot would make the post-migration comparison +// assert less than it claims, silently. +func TestRESTPrincipalsFollowsPagination(t *testing.T) { + var many []map[string]any + for i := 0; i < 250; i++ { + many = append(many, map[string]any{ + "id": i, "type": "individual", + "name": fmt.Sprintf("user%03d", i), + "emails": []string{fmt.Sprintf("user%03d@smoke.test", i)}, + }) + } + srv, _ := stalwart015Server(t, many, nil) + client := &Client{BaseURL: srv.URL, Username: "admin", Password: "x"} + + snap, err := client.AccountSnapshot(context.Background()) + if err != nil { + t.Fatal(err) + } + if snap.AccountCount != 250 { + t.Errorf("AccountCount = %d, want all 250 across pages", snap.AccountCount) + } +} + +// An instance with no explicit domain principals still has domains, implied +// by its accounts' addresses. +func TestRESTSnapshotDerivesDomainsFromAddresses(t *testing.T) { + srv, _ := stalwart015Server(t, + []map[string]any{ + {"id": 1, "type": "individual", "name": "a", "emails": []string{"a@one.example"}}, + {"id": 2, "type": "individual", "name": "b", "emails": []string{"b@two.example"}}, + }, nil) + client := &Client{BaseURL: srv.URL, Username: "admin", Password: "x"} + + snap, err := client.AccountSnapshot(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(snap.Domains) != 2 || snap.Domains[0] != "one.example" || snap.Domains[1] != "two.example" { + t.Errorf("Domains = %v, want [one.example two.example] sorted", snap.Domains) + } +} + +func TestRESTSnapshotSurfacesAuthFailure(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/jmap" { + json.NewEncoder(w).Encode(map[string]any{"capabilities": map[string]any{}}) + return + } + w.WriteHeader(http.StatusUnauthorized) + w.Write([]byte("invalid credentials")) + })) + defer srv.Close() + + client := &Client{BaseURL: srv.URL, Username: "admin", Password: "wrong"} + _, err := client.AccountSnapshot(context.Background()) + if err == nil { + t.Fatal("want an error when the principal list rejects the credentials") + } + if !strings.Contains(err.Error(), "401") { + t.Errorf("error %q should carry the status it got", err) + } +} + +// 0.16 reports the same per-account measure under a different name; both +// have to land in the same field or the comparison can't span the boundary. +func TestJMAPSnapshotCapturesUsedDiskQuota(t *testing.T) { + srv, _ := accountManagementAndMailboxServer(t, map[string][]map[string]any{ + "alice@example.com": {{"name": "Inbox", "totalEmails": 10}}, + "bob@example.org": {{"name": "Inbox", "totalEmails": 3}}, + }) + defer srv.Close() + + client := &Client{BaseURL: srv.URL, Username: "admin", Password: "hunter2"} + snap, err := client.AccountSnapshot(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(snap.UsedQuota) != 2 { + t.Errorf("UsedQuota = %v, want an entry per account", snap.UsedQuota) + } +} diff --git a/internal/validate/content_integrity.go b/internal/validate/content_integrity.go index d56ae51..2787c17 100644 --- a/internal/validate/content_integrity.go +++ b/internal/validate/content_integrity.go @@ -23,30 +23,54 @@ type MailboxDelta struct { } // ContentIntegrityResult is the outcome of comparing a pre-migration -// snapshot against a freshly captured post-migration one - the actual +// snapshot against a freshly captured post-migration one - the // no-data-loss check described in ARCHITECTURE.md §4.7. +// +// MessageCountsCompared is the field that decides how much this result is +// worth, and it is not always true. Stalwart 0.15.x exposes no per-mailbox +// message counts at any endpoint, and its impersonation login (which 0.16 +// offers) returns 401 - so on the 0.15/0.16 boundary migration there are no +// "before" counts to compare and this check can only assert that every +// account and domain survived. Saying so is the whole point: an earlier +// version of this comparison iterated the before-counts map, found it +// empty, and reported "all message counts match" having checked nothing. type ContentIntegrityResult struct { AccountsChecked int MailboxesChecked int MissingAccounts []string // present before, not found after (even accounting for the email-address rewrite) MessageCountMismatches []MailboxDelta // present both before and after, but with a different message count + MissingDomains []string // present before, absent after + MessageCountsCompared bool // false when the source version could not report counts } -// OK reports whether every account and mailbox the pre-migration snapshot -// knew about was found afterward with an identical message count. +// OK reports whether everything this comparison was able to check matched. +// Read it together with MessageCountsCompared: OK with that false means +// "the directory survived", not "no mail was lost". func (r ContentIntegrityResult) OK() bool { - return len(r.MissingAccounts) == 0 && len(r.MessageCountMismatches) == 0 + return len(r.MissingAccounts) == 0 && len(r.MessageCountMismatches) == 0 && len(r.MissingDomains) == 0 } func (r ContentIntegrityResult) String() string { - if r.OK() { - return fmt.Sprintf("content integrity: %d account(s), %d mailbox(es) checked, all message counts match", r.AccountsChecked, r.MailboxesChecked) - } var b strings.Builder - fmt.Fprintf(&b, "content integrity: %d account(s), %d mailbox(es) checked", r.AccountsChecked, r.MailboxesChecked) + if r.MessageCountsCompared { + fmt.Fprintf(&b, "content integrity: %d account(s), %d mailbox(es) checked", r.AccountsChecked, r.MailboxesChecked) + } else { + fmt.Fprintf(&b, "content integrity: %d account(s) and their domains checked; MESSAGE COUNTS NOT COMPARED "+ + "(this migration's source version reports no per-mailbox counts, so no-data-loss is NOT verified here - "+ + "only that every account and domain survived)", r.AccountsChecked) + } + if r.OK() { + if r.MessageCountsCompared { + b.WriteString(", all message counts match") + } + return b.String() + } for _, a := range r.MissingAccounts { fmt.Fprintf(&b, "; MISSING ACCOUNT %s", a) } + for _, d := range r.MissingDomains { + fmt.Fprintf(&b, "; MISSING DOMAIN %s", d) + } for _, d := range r.MessageCountMismatches { fmt.Fprintf(&b, "; MESSAGE COUNT MISMATCH %s/%s: %d before, %d after", d.Account, d.Mailbox, d.Before, d.After) } @@ -66,16 +90,54 @@ func compareContentIntegrity(ctx context.Context, client *stalwartapi.Client, be return nil, fmt.Errorf("capture post-migration snapshot: %w", err) } - result := &ContentIntegrityResult{} + result := &ContentIntegrityResult{MessageCountsCompared: len(before.MailboxCounts) > 0} - beforeAccounts := make([]string, 0, len(before.MailboxCounts)) + // The set of accounts to verify comes from whichever pre-migration + // facts the source version was able to report: mailbox counts when it + // had them (0.16+), otherwise the used-quota map, which the 0.15.x REST + // principal list does populate. Deriving it from MailboxCounts alone + // would silently check nothing on a 0.15 source. + beforeAccountSet := map[string]bool{} for a := range before.MailboxCounts { + beforeAccountSet[a] = true + } + for a := range before.UsedQuota { + beforeAccountSet[a] = true + } + beforeAccounts := make([]string, 0, len(beforeAccountSet)) + for a := range beforeAccountSet { beforeAccounts = append(beforeAccounts, a) } sort.Strings(beforeAccounts) + // Likewise for the "after" side: an account that exists but whose + // mailboxes couldn't be read still counts as present. + afterAccounts := map[string]bool{} + for a := range after.MailboxCounts { + afterAccounts[a] = true + } + for a := range after.UsedQuota { + afterAccounts[a] = true + } + for a := range after.MailboxErrors { + afterAccounts[a] = true + } + + for _, d := range before.Domains { + if !containsDomain(after.Domains, d) { + result.MissingDomains = append(result.MissingDomains, d) + } + } + for _, beforeAccount := range beforeAccounts { result.AccountsChecked++ + if !accountPresent(afterAccounts, beforeAccount) { + result.MissingAccounts = append(result.MissingAccounts, beforeAccount) + continue + } + if !result.MessageCountsCompared { + continue // nothing to compare counts against; presence is all this source could give + } afterMailboxes, found := after.MailboxCounts[beforeAccount] if !found { afterMailboxes, found = findByLocalPart(after.MailboxCounts, beforeAccount) @@ -105,6 +167,33 @@ func compareContentIntegrity(ctx context.Context, client *stalwartapi.Client, be return result, nil } +// accountPresent matches an account against the post-migration set the +// same way findByLocalPart does, so the v0.16 rewrite of bare usernames +// into full addresses doesn't read as every account having vanished. +func accountPresent(afterAccounts map[string]bool, beforeAccount string) bool { + if afterAccounts[beforeAccount] { + return true + } + local := strings.SplitN(beforeAccount, "@", 2)[0] + for a := range afterAccounts { + if strings.SplitN(a, "@", 2)[0] == local { + return true + } + } + return false +} + +// containsDomain matches domains exactly; unlike account names, the +// migration does not rewrite them. +func containsDomain(domains []string, want string) bool { + for _, d := range domains { + if d == want { + return true + } + } + return false +} + func findByLocalPart(mailboxCounts map[string][]stalwartapi.MailboxCount, beforeAccount string) ([]stalwartapi.MailboxCount, bool) { local := strings.SplitN(beforeAccount, "@", 2)[0] for afterAccount, mb := range mailboxCounts { diff --git a/internal/validate/content_integrity_test.go b/internal/validate/content_integrity_test.go index 4bcde0c..7039f4a 100644 --- a/internal/validate/content_integrity_test.go +++ b/internal/validate/content_integrity_test.go @@ -29,6 +29,15 @@ func fakeManagementServer(t *testing.T, accounts []map[string]any, mailboxesByEm srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodGet && r.URL.Path == "/.well-known/jmap" { user, _, _ := r.BasicAuth() + if !strings.Contains(user, "%") { + // This instance is a migrated 0.16 one, which is what the + // urn:stalwart:jmap capability says. + json.NewEncoder(w).Encode(map[string]any{ + "apiUrl": apiURL, + "capabilities": map[string]any{"urn:ietf:params:jmap:core": map[string]any{}, "urn:stalwart:jmap": map[string]any{}}, + }) + return + } target := strings.SplitN(user, "%", 2)[0] if _, ok := mailboxesByEmail[target]; !ok { w.WriteHeader(http.StatusForbidden) @@ -166,3 +175,70 @@ func TestCompareContentIntegrityMultipleMailboxesPerAccount(t *testing.T) { t.Errorf("mismatch = %+v, want Archive 199->200", m) } } + +// The bug this guards against was found by running preflight against a real +// Stalwart 0.15.5: it reports no per-mailbox counts, so the "before" +// snapshot has none, and the comparison used to iterate that empty map, +// check nothing, and report "all message counts match" - the strongest +// claim this tool makes, made vacuously. +func TestCompareContentIntegrityDoesNotClaimCountsMatchWhenSourceHadNone(t *testing.T) { + srv := fakeManagementServer(t, + []map[string]any{{"id": "a1", "name": "alice@smoke.test", "domainId": "smoke.test"}}, + map[string][]map[string]any{"alice@smoke.test": {{"name": "Inbox", "totalEmails": 3}}}, + ) + defer srv.Close() + + // A 0.15.x-shaped snapshot: accounts and used-quota, no mailbox counts. + before := &checkpoint.PreflightSnapshot{ + AccountCount: 1, + Domains: []string{"smoke.test"}, + UsedQuota: map[string]int64{"alice@smoke.test": 9207}, + } + client := &stalwartapi.Client{BaseURL: srv.URL, Username: "admin", Password: "x"} + result, err := compareContentIntegrity(context.Background(), client, before) + if err != nil { + t.Fatal(err) + } + if result.MessageCountsCompared { + t.Error("MessageCountsCompared = true, but the source snapshot had no counts") + } + if result.AccountsChecked != 1 { + t.Errorf("AccountsChecked = %d, want 1 - the account set must still be verified", result.AccountsChecked) + } + if strings.Contains(result.String(), "all message counts match") { + t.Errorf("report claims counts match when none were compared:\n%s", result) + } + if !strings.Contains(result.String(), "MESSAGE COUNTS NOT COMPARED") { + t.Errorf("report must say plainly that no-data-loss was not verified:\n%s", result) + } +} + +// Presence checking still has to work on that path, or it would be no +// better than the vacuous pass it replaced. +func TestCompareContentIntegrityDetectsLostAccountWithoutCounts(t *testing.T) { + srv := fakeManagementServer(t, + []map[string]any{{"id": "a1", "name": "alice@smoke.test", "domainId": "smoke.test"}}, + map[string][]map[string]any{"alice@smoke.test": {{"name": "Inbox", "totalEmails": 3}}}, + ) + defer srv.Close() + + before := &checkpoint.PreflightSnapshot{ + AccountCount: 2, + Domains: []string{"smoke.test", "gone.example"}, + UsedQuota: map[string]int64{"alice@smoke.test": 9207, "bob@smoke.test": 5380}, + } + client := &stalwartapi.Client{BaseURL: srv.URL, Username: "admin", Password: "x"} + result, err := compareContentIntegrity(context.Background(), client, before) + if err != nil { + t.Fatal(err) + } + if result.OK() { + t.Fatalf("want a failing result when an account and a domain vanished:\n%s", result) + } + if len(result.MissingAccounts) != 1 || result.MissingAccounts[0] != "bob@smoke.test" { + t.Errorf("MissingAccounts = %v, want [bob@smoke.test]", result.MissingAccounts) + } + if len(result.MissingDomains) != 1 || result.MissingDomains[0] != "gone.example" { + t.Errorf("MissingDomains = %v, want [gone.example]", result.MissingDomains) + } +} diff --git a/internal/validate/main_test.go b/internal/validate/main_test.go index 121c04d..540de91 100644 --- a/internal/validate/main_test.go +++ b/internal/validate/main_test.go @@ -60,7 +60,10 @@ func runFakeStalwartServer() { }) return } - w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(map[string]any{ + "apiUrl": "http://127.0.0.1:" + port + "/api", + "capabilities": map[string]any{"urn:ietf:params:jmap:core": map[string]any{}, "urn:stalwart:jmap": map[string]any{}}, + }) case r.Method == http.MethodPost && r.URL.Path == "/api": var body map[string]any json.NewDecoder(r.Body).Decode(&body)