Fix three defects a full VM migration exposed

Ran a complete 0.15.5 -> 0.16.14 migration of the smoke VM, driving the
phases in the order the real pipeline will. It worked - all mail intact and
readable afterwards, all ten listeners up, cutover executed for the first
time ever and checkpoint resume exercised - and it exposed three defects.

1. The converted config was installed root-owned while the service runs as
   its own user. Stalwart crash-looped 28 times on "Failed to read data
   store settings: Permission denied", minutes after the mistake and
   nowhere near it. This is the same ownership trap that retired the
   rollback implementation, in a new place: writing files as root is the
   natural thing for a tool running as root to do, and it is wrong every
   time the service is not root.

   Cutover now installs the config itself, copying ownership and mode from
   the config being replaced.

2. v0.16.14 does not serve /api - the endpoint stalwartapi assumed.
   Confirmed against a fully migrated, fully configured, serving instance
   rather than a sandbox: /api, /api/principal and /jmap/ all 404. The JMAP
   endpoint is the one the session document advertises, which is what RFC
   8620 discovery is for.

   The client now discovers it, re-basing the advertised path onto the
   operator's host: a real instance advertises its canonical public URL
   ("https://mail.smoke.test/jmap/") which frequently isn't reachable from
   where this tool runs. The session is authoritative about the path; the
   operator is authoritative about the host.

3. Dispatching on the urn:stalwart:jmap capability was wrong, because
   NEITHER version advertises it - not 0.15.5, and not a fully migrated
   0.16.14. That sent 0.16 instances down the 0.15 REST path where every
   call 404s. The client probes what the instance actually serves instead.
   Less elegant than a declared capability, with the advantage of being
   true.

Also: a JMAP "forbidden" now explains itself. An account holding the admin
role before the migration was refused x:Account/query afterwards, and a
bare "forbidden" gives an operator nowhere to start. Whether the role
failed to carry or v0.16 wants different permissions was not isolated, and
that question is recorded as open - it gates quota recalculation and any
post-migration validation.

Verified against both live instances: the 0.15.5 reports 3 accounts and its
domain over REST, and the migrated 0.16.14 routes to JMAP, finds the right
endpoint, and returns the explained refusal.

internal/stalwartapi/client.go

@@ -7,7 +7,9 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -24,6 +26,66 @@ type Client struct {
 	Username   string
 	Password   string
 	HTTPClient *http.Client
+
+	mu       sync.Mutex
+	endpoint string // resolved JMAP endpoint; see apiEndpoint
+}
+
+// apiEndpoint returns the URL to POST JMAP method calls to, discovered from
+// the instance's own session document rather than assumed.
+//
+// This used to be hardcoded as BaseURL + "/api", which is wrong for the
+// version this tool migrates *to*: a fully configured, serving 0.16.14
+// returns 404 for /api, and advertises its JMAP endpoint through the
+// session document's apiUrl instead (RFC 8620 §2 - discovery is how a
+// client is *supposed* to find it).
+//
+// The path is taken from apiUrl but re-based onto BaseURL's scheme and
+// host. A real instance advertises its canonical public URL - observed:
+// "https://mail.smoke.test/jmap/" - which frequently isn't reachable from
+// where this tool runs, over a hostname that may not resolve or a
+// certificate that may not validate. The operator told us how to reach
+// this server when they passed --admin-url; the session is only authoritative
+// about *where on it* the API lives.
+func (c *Client) apiEndpoint(ctx context.Context) (string, error) {
+	c.mu.Lock()
+	cached := c.endpoint
+	c.mu.Unlock()
+	if cached != "" {
+		return cached, nil
+	}
+
+	session, err := c.fetchSession(ctx, c.Username, c.Password)
+	if err != nil {
+		return "", fmt.Errorf("stalwartapi: discover the JMAP endpoint: %w", err)
+	}
+	resolved, err := c.rebaseOntoBaseURL(session.APIURL)
+	if err != nil {
+		return "", err
+	}
+
+	c.mu.Lock()
+	c.endpoint = resolved
+	c.mu.Unlock()
+	return resolved, nil
+}
+
+// rebaseOntoBaseURL keeps the advertised path but the operator's host.
+func (c *Client) rebaseOntoBaseURL(apiURL string) (string, error) {
+	base, err := url.Parse(strings.TrimRight(c.BaseURL, "/"))
+	if err != nil {
+		return "", fmt.Errorf("stalwartapi: parse base URL %q: %w", c.BaseURL, err)
+	}
+	if apiURL == "" {
+		return "", fmt.Errorf("stalwartapi: the instance's session document advertises no apiUrl, so there is no JMAP endpoint to call")
+	}
+	advertised, err := url.Parse(apiURL)
+	if err != nil {
+		return "", fmt.Errorf("stalwartapi: parse advertised apiUrl %q: %w", apiURL, err)
+	}
+	base.Path = advertised.Path
+	base.RawQuery = advertised.RawQuery
+	return base.String(), nil
 }
 
 func (c *Client) httpClient() *http.Client {

internal/stalwartapi/management.go

@@ -39,12 +39,16 @@ type methodResponse struct {
 	CallID string
 }
 
-// call POSTs one JMAP-style request to the management API (Stalwart's /api
-// endpoint - see docs/ref/object/account.md, which is distinct from /jmap)
-// using this Client's own credentials, and returns its parsed method
-// responses in order.
+// call POSTs one JMAP-style request to the management API using this
+// Client's own credentials, and returns its parsed method responses in
+// order. The endpoint is discovered from the instance's session document -
+// see apiEndpoint for why it is not simply BaseURL + "/api".
 func (c *Client) call(ctx context.Context, using []string, methodCalls []any) ([]methodResponse, error) {
-	return c.callAs(ctx, c.Username, c.Password, strings.TrimRight(c.BaseURL, "/")+"/api", using, methodCalls)
+	endpoint, err := c.apiEndpoint(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return c.callAs(ctx, c.Username, c.Password, endpoint, using, methodCalls)
 }
 
 // callAs is call's underlying primitive: it accepts an explicit
@@ -128,14 +132,12 @@ type account struct {
 // exactly what's missing rather than silently treating an unreachable
 // account's mailboxes as having zero messages.
 func (c *Client) AccountSnapshot(ctx context.Context) (*Snapshot, error) {
-	isJMAP, err := c.hasJMAPManagement(ctx)
+	isREST, err := c.hasRESTManagement(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("stalwartapi: discover which management API this instance speaks: %w", err)
 	}
-	if !isJMAP {
-		// No urn:stalwart:jmap: this is a 0.15.x instance, whose
-		// management API is REST. Confirmed against a live 0.15.5 server -
-		// see principal.go.
+	if isREST {
+		// v0.15.x: REST at /api/principal. See principal.go.
 		return c.principalSnapshotREST(ctx)
 	}
 	return c.accountSnapshotJMAP(ctx)
@@ -205,13 +207,43 @@ func (c *Client) accountSnapshotJMAP(ctx context.Context) (*Snapshot, error) {
 	}, nil
 }
 
+// describeJMAPError turns a JMAP method-level error into something an
+// operator can act on.
+//
+// "forbidden" gets special handling because of where it shows up: against a
+// freshly migrated instance, an account that held the admin role before the
+// migration was refused x:Account/query afterwards. Whether the role failed
+// to carry over or v0.16 requires different permissions was not isolated,
+// but the operator's situation is the same either way - they have an admin
+// account that can no longer administer - and a bare "forbidden" gives them
+// nothing to go on.
+func describeJMAPError(method string, args json.RawMessage) error {
+	var parsed struct {
+		Type        string `json:"type"`
+		Description string `json:"description"`
+	}
+	if err := json.Unmarshal(args, &parsed); err != nil || parsed.Type == "" {
+		return fmt.Errorf("stalwartapi: %s error: %s", method, args)
+	}
+	if parsed.Type == "forbidden" {
+		return fmt.Errorf("stalwartapi: %s was refused (forbidden): %s - this account is authenticated but not "+
+			"permitted to perform management operations. After a v0.16 migration this is worth checking first: an "+
+			"account that held the admin role beforehand may not have it afterwards",
+			method, parsed.Description)
+	}
+	if parsed.Description != "" {
+		return fmt.Errorf("stalwartapi: %s error (%s): %s", method, parsed.Type, parsed.Description)
+	}
+	return fmt.Errorf("stalwartapi: %s error: %s", method, parsed.Type)
+}
+
 func accountQueryIDs(responses []methodResponse) ([]string, error) {
 	if len(responses) == 0 {
 		return nil, fmt.Errorf("stalwartapi: Account/query returned no method responses")
 	}
 	r := responses[0]
 	if r.Name == "error" {
-		return nil, fmt.Errorf("stalwartapi: Account/query error: %s", r.Args)
+		return nil, describeJMAPError("Account/query", r.Args)
 	}
 	var result struct {
 		IDs []string `json:"ids"`
@@ -228,7 +260,7 @@ func accountGetList(responses []methodResponse) ([]account, error) {
 	}
 	r := responses[0]
 	if r.Name == "error" {
-		return nil, fmt.Errorf("stalwartapi: Account/get error: %s", r.Args)
+		return nil, describeJMAPError("Account/get", r.Args)
 	}
 	var result struct {
 		List []account `json:"list"`

internal/stalwartapi/management_test.go

@@ -46,6 +46,10 @@ func accountManagementAndMailboxServer(t *testing.T, mailboxesFor map[string][]m
 	var apiURL string
 
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/api/principal" {
+			w.WriteHeader(http.StatusNotFound) // v0.16 shape: no REST management API
+			return
+		}
 		gotPaths = append(gotPaths, r.URL.Path)
 
 		if r.Method == http.MethodGet && r.URL.Path == "/.well-known/jmap" {
@@ -182,6 +186,10 @@ func TestAccountSnapshotEmptyInstance(t *testing.T) {
 
 func TestAccountSnapshotPropagatesJMAPError(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/api/principal" {
+			w.WriteHeader(http.StatusNotFound) // v0.16 shape: no REST management API
+			return
+		}
 		if serveJMAPSession(w, r, "/api") {
 			return
 		}

internal/stalwartapi/principal.go

@@ -51,15 +51,22 @@ const restPrincipalPageSize = 100
 // call.
 const stalwartManagementCapability = "urn:stalwart:jmap"
 
-// hasJMAPManagement reports whether this instance speaks the 0.16+ JMAP
-// management API, by reading the capability list from its session
-// document. It deliberately does its own request rather than reusing
-// fetchSession: that helper also requires an apiUrl and a mail account,
-// which are needed for impersonated mailbox reads but have nothing to do
-// with which management API to use - failing dispatch over a missing
-// apiUrl would misroute an instance that is perfectly readable.
-func (c *Client) hasJMAPManagement(ctx context.Context) (bool, error) {
-	endpoint := strings.TrimRight(c.BaseURL, "/") + "/.well-known/jmap"
+// hasRESTManagement reports whether this instance serves the v0.15.x REST
+// management API, by asking it for a single principal.
+//
+// This replaced a capability check, which cannot work: *neither* version
+// advertises urn:stalwart:jmap. A real 0.15.5 doesn't, and a fully
+// migrated, fully configured 0.16.14 doesn't either - verified against
+// both. Dispatching on the capability sent 0.16 instances down the 0.15
+// REST path, where every call 404s.
+//
+// So the client asks what the instance actually serves instead. 0.15.x
+// answers GET /api/principal with a principal list; 0.16.14 returns 404
+// for that path and serves JMAP management objects at the endpoint its
+// session document advertises. A cheap probe is less elegant than a
+// declared capability and has the considerable advantage of being true.
+func (c *Client) hasRESTManagement(ctx context.Context) (bool, error) {
+	endpoint := strings.TrimRight(c.BaseURL, "/") + "/api/principal?limit=1"
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
 	if err != nil {
 		return false, err
@@ -67,20 +74,24 @@ func (c *Client) hasJMAPManagement(ctx context.Context) (bool, error) {
 	req.SetBasicAuth(c.Username, c.Password)
 	resp, err := c.httpClient().Do(req)
 	if err != nil {
-		return false, fmt.Errorf("stalwartapi: reach %s: %w", endpoint, err)
+		return false, fmt.Errorf("stalwartapi: probe %s: %w", endpoint, err)
 	}
 	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		return false, fmt.Errorf("stalwartapi: session discovery at %s returned %s", endpoint, resp.Status)
+	io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16))
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return true, nil
+	case http.StatusNotFound:
+		return false, nil
+	case http.StatusUnauthorized, http.StatusForbidden:
+		// The path exists but these credentials can't use it. Say so here
+		// rather than falling through to the other API and reporting a
+		// confusing error from there instead.
+		return false, fmt.Errorf("stalwartapi: %s returned %s - the credentials are not accepted for management operations", endpoint, resp.Status)
+	default:
+		return false, nil
 	}
-	var session struct {
-		Capabilities map[string]json.RawMessage `json:"capabilities"`
-	}
-	if err := json.NewDecoder(resp.Body).Decode(&session); err != nil {
-		return false, fmt.Errorf("stalwartapi: parse session document from %s: %w", endpoint, err)
-	}
-	_, ok := session.Capabilities[stalwartManagementCapability]
-	return ok, nil
 }
 
 type restPrincipal struct {

internal/stalwartapi/principal_test.go

@@ -181,3 +181,98 @@ func TestJMAPSnapshotCapturesUsedDiskQuota(t *testing.T) {
 		t.Errorf("UsedQuota = %v, want an entry per account", snap.UsedQuota)
 	}
 }
+
+// A fully configured, serving Stalwart 0.16.14 returns 404 for /api - the
+// endpoint this client used to assume. It advertises its JMAP endpoint in
+// the session document instead. This test pins the discovery so the
+// assumption can't creep back.
+func TestCallDiscoversTheEndpointRatherThanAssumingSlashApi(t *testing.T) {
+	var posted []string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodGet && r.URL.Path == "/.well-known/jmap" {
+			json.NewEncoder(w).Encode(map[string]any{
+				"apiUrl":       "/jmap/",
+				"capabilities": map[string]any{"urn:stalwart:jmap": map[string]any{}},
+			})
+			return
+		}
+		posted = append(posted, r.URL.Path)
+		if r.URL.Path != "/jmap/" {
+			// Stand in for 0.16.14, which 404s anything else.
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		json.NewEncoder(w).Encode(jmapEnvelope{MethodResponses: []any{
+			[]any{"x:Account/query", map[string]any{"ids": []string{}}, "q"},
+		}})
+	}))
+	defer srv.Close()
+
+	client := &Client{BaseURL: srv.URL, Username: "admin", Password: "x"}
+	if _, err := client.AccountIDs(context.Background()); err != nil {
+		t.Fatalf("AccountIDs against an instance that only serves /jmap/: %v", err)
+	}
+	for _, p := range posted {
+		if p == "/api" {
+			t.Error("posted to /api, which 0.16.14 does not serve")
+		}
+	}
+	if len(posted) == 0 || posted[0] != "/jmap/" {
+		t.Errorf("posted to %v, want the advertised /jmap/", posted)
+	}
+}
+
+// A real instance advertises its canonical public URL, which routinely
+// isn't reachable from where this tool runs - a hostname that may not
+// resolve, over TLS that may not validate. Observed on a migrated
+// instance: "https://mail.smoke.test/jmap/" while the operator reached it
+// as http://127.0.0.1:8090. The path is the session's to dictate; the host
+// is the operator's.
+func TestEndpointKeepsTheOperatorsHostAndTheSessionsPath(t *testing.T) {
+	client := &Client{BaseURL: "http://127.0.0.1:8090"}
+	got, err := client.rebaseOntoBaseURL("https://mail.smoke.test/jmap/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != "http://127.0.0.1:8090/jmap/" {
+		t.Errorf("endpoint = %q, want the advertised path on the operator's host", got)
+	}
+}
+
+func TestEndpointRefusesASessionWithNoAPIURL(t *testing.T) {
+	client := &Client{BaseURL: "http://127.0.0.1:8090"}
+	if _, err := client.rebaseOntoBaseURL(""); err == nil {
+		t.Fatal("want an error when the instance advertises no apiUrl")
+	}
+}
+
+// Observed on a freshly migrated 0.16.14: an account that held the admin
+// role before the migration was refused x:Account/query afterwards. A bare
+// "forbidden" leaves an operator with nowhere to start.
+func TestForbiddenExplainsThePostMigrationPermissionTrap(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/api/principal" {
+			w.WriteHeader(http.StatusNotFound) // v0.16 shape: no REST management API
+			return
+		}
+		if r.Method == http.MethodGet && r.URL.Path == "/.well-known/jmap" {
+			json.NewEncoder(w).Encode(map[string]any{"apiUrl": "/jmap/"})
+			return
+		}
+		json.NewEncoder(w).Encode(jmapEnvelope{MethodResponses: []any{
+			[]any{"error", map[string]any{"type": "forbidden", "description": "You are not authorized to perform this action"}, "q"},
+		}})
+	}))
+	defer srv.Close()
+
+	client := &Client{BaseURL: srv.URL, Username: "sysadmin", Password: "x"}
+	_, err := client.AccountSnapshot(context.Background())
+	if err == nil {
+		t.Fatal("want an error for a forbidden management call")
+	}
+	for _, want := range []string{"forbidden", "admin role", "not permitted"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("error %q should mention %q", err, want)
+		}
+	}
+}

internal/stalwartapi/task.go

@@ -132,7 +132,7 @@ func (c *Client) setTasks(ctx context.Context, create map[string]any, subjectFor
 	}
 	r := responses[0]
 	if r.Name == "error" {
-		return nil, fmt.Errorf("stalwartapi: Task/set error: %s", r.Args)
+		return nil, describeJMAPError("Task/set", r.Args)
 	}
 	var result struct {
 		Created map[string]struct {
@@ -245,7 +245,7 @@ func (c *Client) taskStatuses(ctx context.Context, ids []string) (map[string]tas
 	}
 	r := responses[0]
 	if r.Name == "error" {
-		return nil, fmt.Errorf("stalwartapi: Task/get error: %s", r.Args)
+		return nil, describeJMAPError("Task/get", r.Args)
 	}
 	var result struct {
 		List []struct {

internal/stalwartapi/task_test.go

@@ -28,6 +28,18 @@ func newTaskServer(t *testing.T) (*taskServer, *httptest.Server) {
 	t.Helper()
 	ts := &taskServer{queue: map[string]string{}}
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/api/principal" {
+			w.WriteHeader(http.StatusNotFound) // v0.16 shape: no REST management API
+			return
+		}
+		if r.Method == http.MethodGet && r.URL.Path == "/.well-known/jmap" {
+			// The endpoint is discovered, not assumed - see apiEndpoint.
+			json.NewEncoder(w).Encode(map[string]any{
+				"apiUrl":       "/api",
+				"capabilities": map[string]any{"urn:stalwart:jmap": map[string]any{}},
+			})
+			return
+		}
 		var body map[string]any
 		json.NewDecoder(r.Body).Decode(&body)
 		call := body["methodCalls"].([]any)[0].([]any)