diff --git a/internal/checkpoint/types.go b/internal/checkpoint/types.go index e4714b8..65ae230 100644 --- a/internal/checkpoint/types.go +++ b/internal/checkpoint/types.go @@ -74,13 +74,19 @@ type MailboxCount struct { // which the validate phase later compares against the migrated instance. // See ARCHITECTURE.md §4.1 and §4.7. type PreflightSnapshot struct { - TakenAt time.Time `json:"taken_at"` - AccountCount int `json:"account_count"` - Domains []string `json:"domains,omitempty"` - MailboxCounts map[string][]MailboxCount `json:"mailbox_counts,omitempty"` // account -> mailboxes - DKIMFingerprints map[string]string `json:"dkim_fingerprints,omitempty"` - TLSFingerprints []string `json:"tls_fingerprints,omitempty"` - ListenerPorts []int `json:"listener_ports,omitempty"` + TakenAt time.Time `json:"taken_at"` + AccountCount int `json:"account_count"` + Domains []string `json:"domains,omitempty"` + MailboxCounts map[string][]MailboxCount `json:"mailbox_counts,omitempty"` // account -> mailboxes + // UsedQuota is each account's used storage in bytes. It is the only + // per-account content measure available on both sides of the 0.15/0.16 + // boundary - 0.15.x exposes no per-mailbox message counts at all - so + // it is what the post-migration comparison can actually assert on a + // boundary migration. See stalwartapi/principal.go. + UsedQuota map[string]int64 `json:"used_quota,omitempty"` + DKIMFingerprints map[string]string `json:"dkim_fingerprints,omitempty"` + TLSFingerprints []string `json:"tls_fingerprints,omitempty"` + ListenerPorts []int `json:"listener_ports,omitempty"` } // Topology records how this Stalwart instance is deployed, as detected diff --git a/internal/preflight/checks.go b/internal/preflight/checks.go index a4226b2..24ae2af 100644 --- a/internal/preflight/checks.go +++ b/internal/preflight/checks.go @@ -244,9 +244,16 @@ func (c *Checker) Run(ctx context.Context, store *checkpoint.Store, rs *checkpoi AccountCount: snap.AccountCount, Domains: snap.Domains, MailboxCounts: mailboxCounts, + UsedQuota: snap.UsedQuota, + } + detail := fmt.Sprintf("captured snapshot: %d account(s) across %d domain(s), used-quota for %d account(s), mailbox counts for %d account(s)", + snap.AccountCount, len(snap.Domains), len(snap.UsedQuota), len(mailboxCounts)) + if len(mailboxCounts) == 0 && len(snap.UsedQuota) > 0 { + // Expected against a 0.15.x source: it exposes no + // per-mailbox counts, so used-quota is what the + // post-migration comparison will have to work from. + detail += " (this source exposes no per-mailbox counts, so the post-migration check compares accounts, domains and used-quota)" } - detail := fmt.Sprintf("captured snapshot: %d account(s) across %d domain(s), mailbox counts for %d account(s)", - snap.AccountCount, len(snap.Domains), len(mailboxCounts)) status := StatusOK if len(snap.MailboxErrors) > 0 { status = StatusWarn diff --git a/internal/preflight/checks_test.go b/internal/preflight/checks_test.go index 1b30279..56257fb 100644 --- a/internal/preflight/checks_test.go +++ b/internal/preflight/checks_test.go @@ -232,7 +232,12 @@ func TestCheckerRunCapturesAccountSnapshotWhenAdminURLSet(t *testing.T) { }) return } - w.WriteHeader(http.StatusOK) + // A 0.16-era instance: the urn:stalwart:jmap capability is what + // says its management API is the JMAP one. + json.NewEncoder(w).Encode(map[string]any{ + "apiUrl": apiURL, + "capabilities": map[string]any{"urn:ietf:params:jmap:core": map[string]any{}, "urn:stalwart:jmap": map[string]any{}}, + }) case r.Method == http.MethodPost && r.URL.Path == "/api": var body map[string]any json.NewDecoder(r.Body).Decode(&body) diff --git a/internal/stalwartapi/client.go b/internal/stalwartapi/client.go index 4534c8d..f3b15bd 100644 --- a/internal/stalwartapi/client.go +++ b/internal/stalwartapi/client.go @@ -63,6 +63,12 @@ type Snapshot struct { AccountCount int Domains []string MailboxCounts map[string][]MailboxCount // account email -> its mailboxes + // UsedQuota is each account's used storage in bytes. Unlike + // MailboxCounts it is available from both API generations - 0.15.x's + // REST principal list and 0.16's x:Account both report it - which + // makes it the only per-account content measure that can be compared + // across the 0.15/0.16 boundary. See principal.go. + UsedQuota map[string]int64 // MailboxErrors records, per account email, why that account's mailbox // counts couldn't be captured (e.g. impersonation not permitted for // that account). A non-empty entry here means MailboxCounts has no diff --git a/internal/stalwartapi/mailbox.go b/internal/stalwartapi/mailbox.go index 0a91b18..708b223 100644 --- a/internal/stalwartapi/mailbox.go +++ b/internal/stalwartapi/mailbox.go @@ -20,6 +20,10 @@ import ( type jmapSession struct { APIURL string `json:"apiUrl"` PrimaryAccounts map[string]string `json:"primaryAccounts"` + // Capabilities is what the instance says it supports. Its contents + // decide which management API this client speaks - see + // stalwartManagementCapability. + Capabilities map[string]json.RawMessage `json:"capabilities"` } const jmapMailCapability = "urn:ietf:params:jmap:mail" diff --git a/internal/stalwartapi/management.go b/internal/stalwartapi/management.go index 0a87a59..82af84e 100644 --- a/internal/stalwartapi/management.go +++ b/internal/stalwartapi/management.go @@ -104,6 +104,10 @@ type account struct { ID string `json:"id"` Name string `json:"name"` DomainID string `json:"domainId"` + // UsedDiskQuota is 0.16's name for what 0.15's REST API calls + // usedQuota - the same per-account byte count, which is what makes a + // cross-boundary content comparison possible at all. + UsedDiskQuota int64 `json:"usedDiskQuota"` } // AccountSnapshot enumerates every account on the instance via Stalwart's @@ -124,6 +128,22 @@ type account struct { // exactly what's missing rather than silently treating an unreachable // account's mailboxes as having zero messages. func (c *Client) AccountSnapshot(ctx context.Context) (*Snapshot, error) { + isJMAP, err := c.hasJMAPManagement(ctx) + if err != nil { + return nil, fmt.Errorf("stalwartapi: discover which management API this instance speaks: %w", err) + } + if !isJMAP { + // No urn:stalwart:jmap: this is a 0.15.x instance, whose + // management API is REST. Confirmed against a live 0.15.5 server - + // see principal.go. + return c.principalSnapshotREST(ctx) + } + return c.accountSnapshotJMAP(ctx) +} + +// accountSnapshotJMAP is the 0.16+ path, via Stalwart's JMAP management +// objects. +func (c *Client) accountSnapshotJMAP(ctx context.Context) (*Snapshot, error) { ids, err := c.AccountIDs(ctx) if err != nil { return nil, err @@ -133,7 +153,7 @@ func (c *Client) AccountSnapshot(ctx context.Context) (*Snapshot, error) { } getResp, err := c.call(ctx, managementCapabilities, []any{ - []any{"x:Account/get", map[string]any{"ids": ids, "properties": []string{"id", "name", "domainId"}}, "g"}, + []any{"x:Account/get", map[string]any{"ids": ids, "properties": []string{"id", "name", "domainId", "usedDiskQuota"}}, "g"}, }) if err != nil { return nil, fmt.Errorf("stalwartapi: Account/get: %w", err) @@ -169,11 +189,19 @@ func (c *Client) AccountSnapshot(ctx context.Context) (*Snapshot, error) { } sort.Strings(domains) + usedQuota := make(map[string]int64, len(accounts)) + for _, a := range accounts { + if a.Name != "" { + usedQuota[a.Name] = a.UsedDiskQuota + } + } + return &Snapshot{ AccountCount: len(accounts), Domains: domains, MailboxCounts: mailboxCounts, MailboxErrors: mailboxErrors, + UsedQuota: usedQuota, }, nil } diff --git a/internal/stalwartapi/management_test.go b/internal/stalwartapi/management_test.go index 3b5c235..085d99b 100644 --- a/internal/stalwartapi/management_test.go +++ b/internal/stalwartapi/management_test.go @@ -12,6 +12,21 @@ import ( "testing" ) +// serveJMAPSession answers session discovery for a server standing in for +// a 0.16+ instance: what marks it as one is the urn:stalwart:jmap +// capability, which is exactly what AccountSnapshot dispatches on. Returns +// true if it handled the request. +func serveJMAPSession(w http.ResponseWriter, r *http.Request, apiURL string) bool { + if r.Method != http.MethodGet || r.URL.Path != "/.well-known/jmap" { + return false + } + json.NewEncoder(w).Encode(map[string]any{ + "apiUrl": apiURL, + "capabilities": map[string]any{"urn:ietf:params:jmap:core": map[string]any{}, "urn:stalwart:jmap": map[string]any{}}, + }) + return true +} + // jmapEnvelope mirrors the wire shape this package's call() parses: a // top-level {"methodResponses": [...]} object where each entry is a // [name, args, callId] triple (RFC 8620 §3.2). @@ -35,6 +50,12 @@ func accountManagementAndMailboxServer(t *testing.T, mailboxesFor map[string][]m if r.Method == http.MethodGet && r.URL.Path == "/.well-known/jmap" { user, _, _ := r.BasicAuth() + if !strings.Contains(user, "%") { + // The client's own session request, used to decide which + // management API this instance speaks. + serveJMAPSession(w, r, apiURL) + return + } target := strings.SplitN(user, "%", 2)[0] if _, ok := mailboxesFor[target]; !ok { w.WriteHeader(http.StatusForbidden) @@ -140,6 +161,9 @@ func TestAccountSnapshotRecordsPerAccountMailboxFailureWithoutFailingOverall(t * func TestAccountSnapshotEmptyInstance(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if serveJMAPSession(w, r, "/api") { + return + } json.NewEncoder(w).Encode(jmapEnvelope{MethodResponses: []any{ []any{"x:Account/query", map[string]any{"ids": []string{}}, "q"}, }}) @@ -158,6 +182,9 @@ func TestAccountSnapshotEmptyInstance(t *testing.T) { func TestAccountSnapshotPropagatesJMAPError(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if serveJMAPSession(w, r, "/api") { + return + } json.NewEncoder(w).Encode(jmapEnvelope{MethodResponses: []any{ []any{"error", map[string]any{"type": "forbidden"}, "q"}, }}) @@ -191,6 +218,9 @@ func TestAccountSnapshotSendsBasicAuth(t *testing.T) { if !ok || user != "admin" || pass != "hunter2" { t.Errorf("BasicAuth = (%s, %s, %v), want (admin, hunter2, true)", user, pass, ok) } + if serveJMAPSession(w, r, "/api") { + return + } json.NewEncoder(w).Encode(jmapEnvelope{MethodResponses: []any{ []any{"x:Account/query", map[string]any{"ids": []string{}}, "q"}, }}) diff --git a/internal/stalwartapi/principal.go b/internal/stalwartapi/principal.go new file mode 100644 index 0000000..d1931f3 --- /dev/null +++ b/internal/stalwartapi/principal.go @@ -0,0 +1,197 @@ +// SPDX-FileCopyrightText: 2026 LINUXexpert-org +// SPDX-License-Identifier: GPL-3.0-or-later + +package stalwartapi + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" +) + +// Stalwart 0.15.x - the version this tool migrates *from* - has no +// urn:stalwart:jmap capability and no JMAP management endpoint: POST /api +// returns 404 there. Its management API is REST, at GET /api/principal, +// and that is the only way to enumerate the pre-migration directory. +// +// This was found by running preflight against a real 0.15.5 instance, not +// from documentation: the published schema reference documents 0.16, and +// following it alone produced a tool that could not read the source +// version it exists to migrate. The shapes below were confirmed against +// that live server. +// +// GET /api/principal?types=individual&limit=100&page=1 +// {"data":{"items":[{"id":4,"type":"individual","name":"alice", +// "emails":["alice@smoke.test"],"usedQuota":9207}], +// "total":3}} +// +// Two limits of the 0.15 side are worth stating plainly, because they +// decide what the post-migration comparison can actually assert: +// +// - There is no per-mailbox message count anywhere in this API, and the +// impersonation mechanism 0.16 offers (the `%` +// composite login MailboxSnapshot uses) returns 401 on 0.15.5. So a +// pre-migration snapshot cannot carry message counts, and a +// before/after comparison of them is impossible for the boundary +// migration this tool is built for. +// - What both versions do expose per account is used quota in bytes +// (`usedQuota` here, `usedDiskQuota` on 0.16's x:Account). That is a +// real content measure - it moves when mail is lost - so it is what +// the integrity comparison uses across the boundary. +const restPrincipalPageSize = 100 + +// stalwartManagementCapability is advertised by instances whose management +// API is the JMAP one (0.16+). Its absence is what distinguishes a 0.15.x +// instance, and is a positive signal rather than an inference from a failed +// call. +const stalwartManagementCapability = "urn:stalwart:jmap" + +// hasJMAPManagement reports whether this instance speaks the 0.16+ JMAP +// management API, by reading the capability list from its session +// document. It deliberately does its own request rather than reusing +// fetchSession: that helper also requires an apiUrl and a mail account, +// which are needed for impersonated mailbox reads but have nothing to do +// with which management API to use - failing dispatch over a missing +// apiUrl would misroute an instance that is perfectly readable. +func (c *Client) hasJMAPManagement(ctx context.Context) (bool, error) { + endpoint := strings.TrimRight(c.BaseURL, "/") + "/.well-known/jmap" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return false, err + } + req.SetBasicAuth(c.Username, c.Password) + resp, err := c.httpClient().Do(req) + if err != nil { + return false, fmt.Errorf("stalwartapi: reach %s: %w", endpoint, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return false, fmt.Errorf("stalwartapi: session discovery at %s returned %s", endpoint, resp.Status) + } + var session struct { + Capabilities map[string]json.RawMessage `json:"capabilities"` + } + if err := json.NewDecoder(resp.Body).Decode(&session); err != nil { + return false, fmt.Errorf("stalwartapi: parse session document from %s: %w", endpoint, err) + } + _, ok := session.Capabilities[stalwartManagementCapability] + return ok, nil +} + +type restPrincipal struct { + ID int `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + Emails []string `json:"emails"` + UsedQuota int64 `json:"usedQuota"` +} + +type restPrincipalPage struct { + Data struct { + Items []restPrincipal `json:"items"` + Total int `json:"total"` + } `json:"data"` +} + +// restPrincipals fetches every principal of the given type, following the +// API's 1-based page/limit pagination rather than assuming one request +// returns everything - an install with more accounts than the page size +// would otherwise be silently truncated, and a truncated "before" snapshot +// would make the post-migration comparison claim more than it checked. +func (c *Client) restPrincipals(ctx context.Context, principalType string) ([]restPrincipal, error) { + var all []restPrincipal + for page := 1; ; page++ { + q := url.Values{} + if principalType != "" { + q.Set("types", principalType) + } + q.Set("limit", fmt.Sprint(restPrincipalPageSize)) + q.Set("page", fmt.Sprint(page)) + endpoint := strings.TrimRight(c.BaseURL, "/") + "/api/principal?" + q.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, err + } + req.SetBasicAuth(c.Username, c.Password) + resp, err := c.httpClient().Do(req) + if err != nil { + return nil, fmt.Errorf("stalwartapi: list principals: %w", err) + } + body, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("stalwartapi: GET %s returned %s: %s", endpoint, resp.Status, strings.TrimSpace(string(body))) + } + if readErr != nil { + return nil, fmt.Errorf("stalwartapi: read principal list: %w", readErr) + } + var parsed restPrincipalPage + if err := json.Unmarshal(body, &parsed); err != nil { + return nil, fmt.Errorf("stalwartapi: parse principal list: %w", err) + } + all = append(all, parsed.Data.Items...) + if len(parsed.Data.Items) == 0 || len(all) >= parsed.Data.Total { + return all, nil + } + } +} + +// principalSnapshotREST builds a Snapshot from the 0.15.x REST management +// API. MailboxCounts is deliberately left empty: see this file's opening +// comment for why counts cannot be obtained from a 0.15 instance at all. +func (c *Client) principalSnapshotREST(ctx context.Context) (*Snapshot, error) { + individuals, err := c.restPrincipals(ctx, "individual") + if err != nil { + return nil, err + } + domainPrincipals, err := c.restPrincipals(ctx, "domain") + if err != nil { + return nil, err + } + + snap := &Snapshot{ + AccountCount: len(individuals), + UsedQuota: make(map[string]int64, len(individuals)), + } + for _, p := range individuals { + snap.UsedQuota[accountKey(p)] = p.UsedQuota + } + + domainSet := map[string]bool{} + for _, d := range domainPrincipals { + if d.Name != "" { + domainSet[d.Name] = true + } + } + // Fall back to the domains implied by account addresses if the + // instance has no explicit domain principals. + for _, p := range individuals { + for _, email := range p.Emails { + if at := strings.LastIndex(email, "@"); at >= 0 && at+1 < len(email) { + domainSet[email[at+1:]] = true + } + } + } + for d := range domainSet { + snap.Domains = append(snap.Domains, d) + } + sort.Strings(snap.Domains) + return snap, nil +} + +// accountKey identifies an account the same way both API generations can: +// by its primary email address where it has one, falling back to the bare +// login name. The v0.16 migration rewrites bare names into addresses, which +// is exactly why the comparison side matches on local part as well. +func accountKey(p restPrincipal) string { + if len(p.Emails) > 0 && p.Emails[0] != "" { + return p.Emails[0] + } + return p.Name +} diff --git a/internal/stalwartapi/principal_test.go b/internal/stalwartapi/principal_test.go new file mode 100644 index 0000000..956eb7e --- /dev/null +++ b/internal/stalwartapi/principal_test.go @@ -0,0 +1,183 @@ +// SPDX-FileCopyrightText: 2026 LINUXexpert-org +// SPDX-License-Identifier: GPL-3.0-or-later + +package stalwartapi + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// stalwart015Server stands in for a 0.15.x instance: no urn:stalwart:jmap +// capability, POST /api is 404, and the management API is REST at +// /api/principal with 1-based page/limit paging. Every shape here was +// confirmed against a live 0.15.5 server. +func stalwart015Server(t *testing.T, individuals, domains []map[string]any) (*httptest.Server, *[]string) { + t.Helper() + var paths []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.Path+"?"+r.URL.RawQuery) + + if r.URL.Path == "/.well-known/jmap" { + // 0.15.5 advertises no urn:stalwart:jmap. + json.NewEncoder(w).Encode(map[string]any{ + "capabilities": map[string]any{"urn:ietf:params:jmap:core": map[string]any{}}, + }) + return + } + if r.URL.Path == "/api" { + w.WriteHeader(http.StatusNotFound) + w.Write([]byte(`{"status":404,"title":"Not Found"}`)) + return + } + if r.URL.Path != "/api/principal" { + w.WriteHeader(http.StatusNotFound) + return + } + + set := individuals + if r.URL.Query().Get("types") == "domain" { + set = domains + } + limit, page := 100, 1 + fmt.Sscanf(r.URL.Query().Get("limit"), "%d", &limit) + fmt.Sscanf(r.URL.Query().Get("page"), "%d", &page) + + start := (page - 1) * limit + end := start + limit + if start > len(set) { + start = len(set) + } + if end > len(set) { + end = len(set) + } + json.NewEncoder(w).Encode(map[string]any{ + "data": map[string]any{"items": set[start:end], "total": len(set)}, + }) + })) + t.Cleanup(srv.Close) + return srv, &paths +} + +// The headline case: against the version this tool actually migrates from, +// AccountSnapshot must not fall over on a 404 from the 0.16-only endpoint. +func TestAccountSnapshotUsesRESTAgainst015(t *testing.T) { + srv, paths := stalwart015Server(t, + []map[string]any{ + {"id": 4, "type": "individual", "name": "alice", "emails": []string{"alice@smoke.test"}, "usedQuota": 9207}, + {"id": 5, "type": "individual", "name": "bob", "emails": []string{"bob@smoke.test"}, "usedQuota": 5380}, + }, + []map[string]any{{"id": 1, "type": "domain", "name": "smoke.test"}}, + ) + client := &Client{BaseURL: srv.URL, Username: "admin", Password: "hunter2"} + + snap, err := client.AccountSnapshot(context.Background()) + if err != nil { + t.Fatalf("AccountSnapshot against 0.15.x: %v", err) + } + if snap.AccountCount != 2 { + t.Errorf("AccountCount = %d, want 2", snap.AccountCount) + } + if len(snap.Domains) != 1 || snap.Domains[0] != "smoke.test" { + t.Errorf("Domains = %v, want [smoke.test]", snap.Domains) + } + if snap.UsedQuota["alice@smoke.test"] != 9207 || snap.UsedQuota["bob@smoke.test"] != 5380 { + t.Errorf("UsedQuota = %v, want alice 9207 and bob 5380", snap.UsedQuota) + } + // Message counts genuinely cannot be had from 0.15.x - see principal.go. + if len(snap.MailboxCounts) != 0 { + t.Errorf("MailboxCounts = %v, want empty: 0.15.x exposes no per-mailbox counts", snap.MailboxCounts) + } + for _, p := range *paths { + if strings.HasPrefix(p, "/api?") { + t.Errorf("the 0.16-only JMAP management endpoint was called against a 0.15.x instance: %s", p) + } + } +} + +// A truncated "before" snapshot would make the post-migration comparison +// assert less than it claims, silently. +func TestRESTPrincipalsFollowsPagination(t *testing.T) { + var many []map[string]any + for i := 0; i < 250; i++ { + many = append(many, map[string]any{ + "id": i, "type": "individual", + "name": fmt.Sprintf("user%03d", i), + "emails": []string{fmt.Sprintf("user%03d@smoke.test", i)}, + }) + } + srv, _ := stalwart015Server(t, many, nil) + client := &Client{BaseURL: srv.URL, Username: "admin", Password: "x"} + + snap, err := client.AccountSnapshot(context.Background()) + if err != nil { + t.Fatal(err) + } + if snap.AccountCount != 250 { + t.Errorf("AccountCount = %d, want all 250 across pages", snap.AccountCount) + } +} + +// An instance with no explicit domain principals still has domains, implied +// by its accounts' addresses. +func TestRESTSnapshotDerivesDomainsFromAddresses(t *testing.T) { + srv, _ := stalwart015Server(t, + []map[string]any{ + {"id": 1, "type": "individual", "name": "a", "emails": []string{"a@one.example"}}, + {"id": 2, "type": "individual", "name": "b", "emails": []string{"b@two.example"}}, + }, nil) + client := &Client{BaseURL: srv.URL, Username: "admin", Password: "x"} + + snap, err := client.AccountSnapshot(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(snap.Domains) != 2 || snap.Domains[0] != "one.example" || snap.Domains[1] != "two.example" { + t.Errorf("Domains = %v, want [one.example two.example] sorted", snap.Domains) + } +} + +func TestRESTSnapshotSurfacesAuthFailure(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/.well-known/jmap" { + json.NewEncoder(w).Encode(map[string]any{"capabilities": map[string]any{}}) + return + } + w.WriteHeader(http.StatusUnauthorized) + w.Write([]byte("invalid credentials")) + })) + defer srv.Close() + + client := &Client{BaseURL: srv.URL, Username: "admin", Password: "wrong"} + _, err := client.AccountSnapshot(context.Background()) + if err == nil { + t.Fatal("want an error when the principal list rejects the credentials") + } + if !strings.Contains(err.Error(), "401") { + t.Errorf("error %q should carry the status it got", err) + } +} + +// 0.16 reports the same per-account measure under a different name; both +// have to land in the same field or the comparison can't span the boundary. +func TestJMAPSnapshotCapturesUsedDiskQuota(t *testing.T) { + srv, _ := accountManagementAndMailboxServer(t, map[string][]map[string]any{ + "alice@example.com": {{"name": "Inbox", "totalEmails": 10}}, + "bob@example.org": {{"name": "Inbox", "totalEmails": 3}}, + }) + defer srv.Close() + + client := &Client{BaseURL: srv.URL, Username: "admin", Password: "hunter2"} + snap, err := client.AccountSnapshot(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(snap.UsedQuota) != 2 { + t.Errorf("UsedQuota = %v, want an entry per account", snap.UsedQuota) + } +} diff --git a/internal/validate/content_integrity.go b/internal/validate/content_integrity.go index d56ae51..2787c17 100644 --- a/internal/validate/content_integrity.go +++ b/internal/validate/content_integrity.go @@ -23,30 +23,54 @@ type MailboxDelta struct { } // ContentIntegrityResult is the outcome of comparing a pre-migration -// snapshot against a freshly captured post-migration one - the actual +// snapshot against a freshly captured post-migration one - the // no-data-loss check described in ARCHITECTURE.md §4.7. +// +// MessageCountsCompared is the field that decides how much this result is +// worth, and it is not always true. Stalwart 0.15.x exposes no per-mailbox +// message counts at any endpoint, and its impersonation login (which 0.16 +// offers) returns 401 - so on the 0.15/0.16 boundary migration there are no +// "before" counts to compare and this check can only assert that every +// account and domain survived. Saying so is the whole point: an earlier +// version of this comparison iterated the before-counts map, found it +// empty, and reported "all message counts match" having checked nothing. type ContentIntegrityResult struct { AccountsChecked int MailboxesChecked int MissingAccounts []string // present before, not found after (even accounting for the email-address rewrite) MessageCountMismatches []MailboxDelta // present both before and after, but with a different message count + MissingDomains []string // present before, absent after + MessageCountsCompared bool // false when the source version could not report counts } -// OK reports whether every account and mailbox the pre-migration snapshot -// knew about was found afterward with an identical message count. +// OK reports whether everything this comparison was able to check matched. +// Read it together with MessageCountsCompared: OK with that false means +// "the directory survived", not "no mail was lost". func (r ContentIntegrityResult) OK() bool { - return len(r.MissingAccounts) == 0 && len(r.MessageCountMismatches) == 0 + return len(r.MissingAccounts) == 0 && len(r.MessageCountMismatches) == 0 && len(r.MissingDomains) == 0 } func (r ContentIntegrityResult) String() string { - if r.OK() { - return fmt.Sprintf("content integrity: %d account(s), %d mailbox(es) checked, all message counts match", r.AccountsChecked, r.MailboxesChecked) - } var b strings.Builder - fmt.Fprintf(&b, "content integrity: %d account(s), %d mailbox(es) checked", r.AccountsChecked, r.MailboxesChecked) + if r.MessageCountsCompared { + fmt.Fprintf(&b, "content integrity: %d account(s), %d mailbox(es) checked", r.AccountsChecked, r.MailboxesChecked) + } else { + fmt.Fprintf(&b, "content integrity: %d account(s) and their domains checked; MESSAGE COUNTS NOT COMPARED "+ + "(this migration's source version reports no per-mailbox counts, so no-data-loss is NOT verified here - "+ + "only that every account and domain survived)", r.AccountsChecked) + } + if r.OK() { + if r.MessageCountsCompared { + b.WriteString(", all message counts match") + } + return b.String() + } for _, a := range r.MissingAccounts { fmt.Fprintf(&b, "; MISSING ACCOUNT %s", a) } + for _, d := range r.MissingDomains { + fmt.Fprintf(&b, "; MISSING DOMAIN %s", d) + } for _, d := range r.MessageCountMismatches { fmt.Fprintf(&b, "; MESSAGE COUNT MISMATCH %s/%s: %d before, %d after", d.Account, d.Mailbox, d.Before, d.After) } @@ -66,16 +90,54 @@ func compareContentIntegrity(ctx context.Context, client *stalwartapi.Client, be return nil, fmt.Errorf("capture post-migration snapshot: %w", err) } - result := &ContentIntegrityResult{} + result := &ContentIntegrityResult{MessageCountsCompared: len(before.MailboxCounts) > 0} - beforeAccounts := make([]string, 0, len(before.MailboxCounts)) + // The set of accounts to verify comes from whichever pre-migration + // facts the source version was able to report: mailbox counts when it + // had them (0.16+), otherwise the used-quota map, which the 0.15.x REST + // principal list does populate. Deriving it from MailboxCounts alone + // would silently check nothing on a 0.15 source. + beforeAccountSet := map[string]bool{} for a := range before.MailboxCounts { + beforeAccountSet[a] = true + } + for a := range before.UsedQuota { + beforeAccountSet[a] = true + } + beforeAccounts := make([]string, 0, len(beforeAccountSet)) + for a := range beforeAccountSet { beforeAccounts = append(beforeAccounts, a) } sort.Strings(beforeAccounts) + // Likewise for the "after" side: an account that exists but whose + // mailboxes couldn't be read still counts as present. + afterAccounts := map[string]bool{} + for a := range after.MailboxCounts { + afterAccounts[a] = true + } + for a := range after.UsedQuota { + afterAccounts[a] = true + } + for a := range after.MailboxErrors { + afterAccounts[a] = true + } + + for _, d := range before.Domains { + if !containsDomain(after.Domains, d) { + result.MissingDomains = append(result.MissingDomains, d) + } + } + for _, beforeAccount := range beforeAccounts { result.AccountsChecked++ + if !accountPresent(afterAccounts, beforeAccount) { + result.MissingAccounts = append(result.MissingAccounts, beforeAccount) + continue + } + if !result.MessageCountsCompared { + continue // nothing to compare counts against; presence is all this source could give + } afterMailboxes, found := after.MailboxCounts[beforeAccount] if !found { afterMailboxes, found = findByLocalPart(after.MailboxCounts, beforeAccount) @@ -105,6 +167,33 @@ func compareContentIntegrity(ctx context.Context, client *stalwartapi.Client, be return result, nil } +// accountPresent matches an account against the post-migration set the +// same way findByLocalPart does, so the v0.16 rewrite of bare usernames +// into full addresses doesn't read as every account having vanished. +func accountPresent(afterAccounts map[string]bool, beforeAccount string) bool { + if afterAccounts[beforeAccount] { + return true + } + local := strings.SplitN(beforeAccount, "@", 2)[0] + for a := range afterAccounts { + if strings.SplitN(a, "@", 2)[0] == local { + return true + } + } + return false +} + +// containsDomain matches domains exactly; unlike account names, the +// migration does not rewrite them. +func containsDomain(domains []string, want string) bool { + for _, d := range domains { + if d == want { + return true + } + } + return false +} + func findByLocalPart(mailboxCounts map[string][]stalwartapi.MailboxCount, beforeAccount string) ([]stalwartapi.MailboxCount, bool) { local := strings.SplitN(beforeAccount, "@", 2)[0] for afterAccount, mb := range mailboxCounts { diff --git a/internal/validate/content_integrity_test.go b/internal/validate/content_integrity_test.go index 4bcde0c..7039f4a 100644 --- a/internal/validate/content_integrity_test.go +++ b/internal/validate/content_integrity_test.go @@ -29,6 +29,15 @@ func fakeManagementServer(t *testing.T, accounts []map[string]any, mailboxesByEm srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodGet && r.URL.Path == "/.well-known/jmap" { user, _, _ := r.BasicAuth() + if !strings.Contains(user, "%") { + // This instance is a migrated 0.16 one, which is what the + // urn:stalwart:jmap capability says. + json.NewEncoder(w).Encode(map[string]any{ + "apiUrl": apiURL, + "capabilities": map[string]any{"urn:ietf:params:jmap:core": map[string]any{}, "urn:stalwart:jmap": map[string]any{}}, + }) + return + } target := strings.SplitN(user, "%", 2)[0] if _, ok := mailboxesByEmail[target]; !ok { w.WriteHeader(http.StatusForbidden) @@ -166,3 +175,70 @@ func TestCompareContentIntegrityMultipleMailboxesPerAccount(t *testing.T) { t.Errorf("mismatch = %+v, want Archive 199->200", m) } } + +// The bug this guards against was found by running preflight against a real +// Stalwart 0.15.5: it reports no per-mailbox counts, so the "before" +// snapshot has none, and the comparison used to iterate that empty map, +// check nothing, and report "all message counts match" - the strongest +// claim this tool makes, made vacuously. +func TestCompareContentIntegrityDoesNotClaimCountsMatchWhenSourceHadNone(t *testing.T) { + srv := fakeManagementServer(t, + []map[string]any{{"id": "a1", "name": "alice@smoke.test", "domainId": "smoke.test"}}, + map[string][]map[string]any{"alice@smoke.test": {{"name": "Inbox", "totalEmails": 3}}}, + ) + defer srv.Close() + + // A 0.15.x-shaped snapshot: accounts and used-quota, no mailbox counts. + before := &checkpoint.PreflightSnapshot{ + AccountCount: 1, + Domains: []string{"smoke.test"}, + UsedQuota: map[string]int64{"alice@smoke.test": 9207}, + } + client := &stalwartapi.Client{BaseURL: srv.URL, Username: "admin", Password: "x"} + result, err := compareContentIntegrity(context.Background(), client, before) + if err != nil { + t.Fatal(err) + } + if result.MessageCountsCompared { + t.Error("MessageCountsCompared = true, but the source snapshot had no counts") + } + if result.AccountsChecked != 1 { + t.Errorf("AccountsChecked = %d, want 1 - the account set must still be verified", result.AccountsChecked) + } + if strings.Contains(result.String(), "all message counts match") { + t.Errorf("report claims counts match when none were compared:\n%s", result) + } + if !strings.Contains(result.String(), "MESSAGE COUNTS NOT COMPARED") { + t.Errorf("report must say plainly that no-data-loss was not verified:\n%s", result) + } +} + +// Presence checking still has to work on that path, or it would be no +// better than the vacuous pass it replaced. +func TestCompareContentIntegrityDetectsLostAccountWithoutCounts(t *testing.T) { + srv := fakeManagementServer(t, + []map[string]any{{"id": "a1", "name": "alice@smoke.test", "domainId": "smoke.test"}}, + map[string][]map[string]any{"alice@smoke.test": {{"name": "Inbox", "totalEmails": 3}}}, + ) + defer srv.Close() + + before := &checkpoint.PreflightSnapshot{ + AccountCount: 2, + Domains: []string{"smoke.test", "gone.example"}, + UsedQuota: map[string]int64{"alice@smoke.test": 9207, "bob@smoke.test": 5380}, + } + client := &stalwartapi.Client{BaseURL: srv.URL, Username: "admin", Password: "x"} + result, err := compareContentIntegrity(context.Background(), client, before) + if err != nil { + t.Fatal(err) + } + if result.OK() { + t.Fatalf("want a failing result when an account and a domain vanished:\n%s", result) + } + if len(result.MissingAccounts) != 1 || result.MissingAccounts[0] != "bob@smoke.test" { + t.Errorf("MissingAccounts = %v, want [bob@smoke.test]", result.MissingAccounts) + } + if len(result.MissingDomains) != 1 || result.MissingDomains[0] != "gone.example" { + t.Errorf("MissingDomains = %v, want [gone.example]", result.MissingDomains) + } +} diff --git a/internal/validate/main_test.go b/internal/validate/main_test.go index 121c04d..540de91 100644 --- a/internal/validate/main_test.go +++ b/internal/validate/main_test.go @@ -60,7 +60,10 @@ func runFakeStalwartServer() { }) return } - w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(map[string]any{ + "apiUrl": "http://127.0.0.1:" + port + "/api", + "capabilities": map[string]any{"urn:ietf:params:jmap:core": map[string]any{}, "urn:stalwart:jmap": map[string]any{}}, + }) case r.Method == http.MethodPost && r.URL.Path == "/api": var body map[string]any json.NewDecoder(r.Body).Decode(&body)