Fix the domain/tenant mismatch that failed the second live migration

The second production attempt failed during recovery-mode apply, with the
mail server already stopped and the store already at schema v6:

    create Account restore-13: invalidForeignKey | Object id: Domain#d

v0.16 requires a tenant-scoped Account to sit on a Domain owned by that
same tenant, for its primary domain and for every alias. v0.15 imposed no
such rule, and migrate_v016.py carries the two facts over independently:
_build_domains sets a domain's memberTenantId only for domains declared as
their own `domain` principal with a `tenant`, while _build_user sets the
account's from the account's own record. A domain that exists only inside
an email address is inferred, gets no tenant, and every tenant-scoped
account using it is then rejected.

Established by reproduction rather than inference: a synthetic v0.15
principal dump, run through the unpatched upstream converter and applied to
a real 0.16.14 in recovery mode, reproduces the error character for
character - the `#d` is the server's own object id for the offending
domain, not a plan client-id. The same harness establishes which directions
are constrained: a tenant-scoped account on a tenant-less domain or on
another tenant's domain is rejected; a global account on a tenant-owned
domain is accepted.

  - applyplan.ReconcileDomainTenants repairs the plan between convert and
    apply. Where a tenant-less domain is used only by accounts of one
    tenant, the domain adopts that tenant - the sole assignment that both
    applies and keeps every account. Where accounts genuinely disagree it
    changes nothing and reports why, because forcing such a plan through
    would mean dropping mailboxes.
  - stalwartapi.FetchTenantLayout maps tenant membership over the 0.15 REST
    API and predicts the outcome with the same rule the server enforces, so
    preflight either warns about the domains that will adopt a tenant or
    fails - while the service is still running.
  - The plan is parsed generically rather than through the typed Operation.
    A real export.json mixes shapes: `create` maps a client-id to an object,
    `update` carries a flat one. The typed form failed on the first `update`
    line, found by running against actual converter output. Numbers decode
    as json.Number so a 10 GiB quota is not rewritten as 1.073741824e+10.

Corrects the record: the previous commit claimed the converter emits every
Account with `tenantId: null` and made preflight refuse every multi-tenant
install on that basis. The field is memberTenantId, the converter does
populate it, and the export had been inspected for a key no version of the
script ever writes. The refusal is now narrowed to what v0.16 genuinely
cannot represent.

The same fix has been prepared for migrate_v016.py upstream. The tool
downloads that script rather than vendoring it, so the repair stays here
until a released version carries it, and is a no-op on a consistent plan.

internal/stalwartapi/principal.go

@@ -209,12 +209,15 @@ func accountKey(p restPrincipal) string {
 
 // TenantNames returns the tenant principals on a v0.15.x instance.
 //
-// Multi-tenancy has to be detected before a migration starts, because
-// Stalwart's own converter does not survive it: it emits the Tenant and the
-// Domains correctly and then every Account with `tenantId: null`, so the
-// account references a tenant-owned domain while belonging to no tenant and
-// the apply is rejected with `invalidForeignKey`. Observed on a real
-// migration, at the point where the mail server was already stopped.
+// Multi-tenancy has to be established before a migration starts. v0.16
+// requires a tenant-scoped account to sit on a domain owned by that same
+// tenant; v0.15 did not, and Stalwart's converter carries the two facts
+// over independently, so an install that is valid today can convert into a
+// plan the new server rejects with `invalidForeignKey` on the Domain
+// reference - during the recovery-mode apply, with the mail server already
+// stopped and the store already at schema v6. See FetchTenantLayout, which
+// builds on this to predict that outcome, and applyplan.ReconcileDomainTenants,
+// which repairs the plan.
 func (c *Client) TenantNames(ctx context.Context) ([]string, error) {
 	tenants, err := c.restPrincipals(ctx, "tenant")
 	if err != nil {

internal/stalwartapi/tenantlayout.go

@@ -0,0 +1,291 @@
+// SPDX-FileCopyrightText: 2026 LINUXexpert-org
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package stalwartapi
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"sort"
+	"strings"
+)
+
+// TenantLayout is who-belongs-to-which-tenant on a v0.15.x instance, in
+// enough detail to predict whether the v0.16 conversion will produce a plan
+// the new server accepts.
+//
+// v0.16 requires a tenant-scoped account to sit on a domain owned by that
+// same tenant, for its primary domain and for every alias. v0.15 imposed no
+// such rule, so an install can be perfectly valid today and unconvertible
+// tomorrow. Establishing that here - while the server is still running and
+// nothing has been stopped - is the whole point: the alternative is finding
+// out during the recovery-mode apply, which is the one moment in the run
+// with no way forward and no way back.
+type TenantLayout struct {
+	// Tenants is every tenant principal's name.
+	Tenants []string
+	// DomainTenant maps a declared domain name to its tenant name. Domains
+	// that exist only inside an email address are absent, which mirrors the
+	// converter: it infers those domains and gives them no tenant.
+	DomainTenant map[string]string
+	// Principals is every account, group and mailing list, with the tenant
+	// it belongs to and the domains it touches.
+	Principals []PrincipalTenancy
+}
+
+// PrincipalTenancy is one account's tenant and the domains it references.
+type PrincipalTenancy struct {
+	Name    string
+	Type    string
+	Tenant  string   // "" for a global principal
+	Domains []string // primary plus alias domains, lowercased
+}
+
+// flexString reads a field that a v0.15 instance may return either as a
+// plain string or wrapped - migrate_v016.py's own pv_string tolerates the
+// same shapes, and a preflight that only understood one of them would
+// silently see every account as global.
+func flexString(raw json.RawMessage) string {
+	if len(raw) == 0 {
+		return ""
+	}
+	var s string
+	if err := json.Unmarshal(raw, &s); err == nil {
+		return s
+	}
+	var obj map[string]json.RawMessage
+	if err := json.Unmarshal(raw, &obj); err == nil {
+		for _, key := range []string{"string", "name", "id"} {
+			if v, ok := obj[key]; ok {
+				var inner string
+				if json.Unmarshal(v, &inner) == nil && inner != "" {
+					return inner
+				}
+			}
+		}
+		return ""
+	}
+	var list []json.RawMessage
+	if err := json.Unmarshal(raw, &list); err == nil && len(list) > 0 {
+		return flexString(list[0])
+	}
+	return ""
+}
+
+// detailedPrincipal is the per-principal view, which carries the tenant the
+// paginated list does not reliably include.
+type detailedPrincipal struct {
+	Type   json.RawMessage `json:"type"`
+	Name   json.RawMessage `json:"name"`
+	Tenant json.RawMessage `json:"tenant"`
+	Emails []string        `json:"emails"`
+}
+
+// principalDetail fetches one principal by name. The response may or may
+// not be wrapped in a "data" envelope depending on the point release, so
+// both are accepted.
+func (c *Client) principalDetail(ctx context.Context, name string) (detailedPrincipal, error) {
+	var out detailedPrincipal
+	endpoint := strings.TrimRight(c.BaseURL, "/") + "/api/principal/" + url.PathEscape(name)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+	if err != nil {
+		return out, err
+	}
+	req.SetBasicAuth(c.Username, c.Password)
+	resp, err := c.httpClient().Do(req)
+	if err != nil {
+		return out, fmt.Errorf("stalwartapi: fetch principal %q: %w", name, err)
+	}
+	body, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return out, fmt.Errorf("stalwartapi: GET %s returned %s", endpoint, resp.Status)
+	}
+	if readErr != nil {
+		return out, fmt.Errorf("stalwartapi: read principal %q: %w", name, readErr)
+	}
+
+	var envelope struct {
+		Data json.RawMessage `json:"data"`
+	}
+	payload := body
+	if json.Unmarshal(body, &envelope) == nil && len(envelope.Data) > 0 {
+		payload = envelope.Data
+	}
+	if err := json.Unmarshal(payload, &out); err != nil {
+		return out, fmt.Errorf("stalwartapi: parse principal %q: %w", name, err)
+	}
+	return out, nil
+}
+
+// domainsOf returns every domain a principal touches: the domain in its
+// name, if it is an address, plus one per email.
+func domainsOf(name string, emails []string) []string {
+	seen := map[string]bool{}
+	var out []string
+	add := func(addr string) {
+		at := strings.LastIndex(addr, "@")
+		if at < 0 || at == len(addr)-1 {
+			return
+		}
+		d := strings.ToLower(strings.TrimSpace(addr[at+1:]))
+		if d == "" || seen[d] {
+			return
+		}
+		seen[d] = true
+		out = append(out, d)
+	}
+	add(name)
+	for _, e := range emails {
+		add(e)
+	}
+	sort.Strings(out)
+	return out
+}
+
+// FetchTenantLayout builds a TenantLayout from a v0.15.x instance.
+//
+// It returns an empty layout, and no error, on a single-tenant install:
+// there is nothing that can mismatch, and that is the common case.
+func (c *Client) FetchTenantLayout(ctx context.Context) (*TenantLayout, error) {
+	tenants, err := c.TenantNames(ctx)
+	if err != nil {
+		return nil, err
+	}
+	layout := &TenantLayout{Tenants: tenants, DomainTenant: map[string]string{}}
+	if len(tenants) == 0 {
+		return layout, nil
+	}
+
+	domains, err := c.restPrincipals(ctx, "domain")
+	if err != nil {
+		return nil, err
+	}
+	for _, d := range domains {
+		if d.Name == "" {
+			continue
+		}
+		detail, err := c.principalDetail(ctx, d.Name)
+		if err != nil {
+			return nil, err
+		}
+		layout.DomainTenant[strings.ToLower(d.Name)] = flexString(detail.Tenant)
+	}
+
+	for _, pType := range []string{"individual", "group", "list"} {
+		principals, err := c.restPrincipals(ctx, pType)
+		if err != nil {
+			return nil, err
+		}
+		for _, p := range principals {
+			if p.Name == "" {
+				continue
+			}
+			detail, err := c.principalDetail(ctx, p.Name)
+			if err != nil {
+				return nil, err
+			}
+			emails := p.Emails
+			if len(detail.Emails) > 0 {
+				emails = detail.Emails
+			}
+			layout.Principals = append(layout.Principals, PrincipalTenancy{
+				Name:    p.Name,
+				Type:    pType,
+				Tenant:  flexString(detail.Tenant),
+				Domains: domainsOf(p.Name, emails),
+			})
+		}
+	}
+	return layout, nil
+}
+
+// TenancyProblem is one domain whose users cannot all be represented in
+// v0.16.
+type TenancyProblem struct {
+	Domain string
+	Detail string
+}
+
+// TenancyPlan is what the conversion will have to do to this layout.
+type TenancyPlan struct {
+	// Adoptions are domains with no tenant of their own that will be given
+	// one, because the only tenant-scoped accounts using them agree.
+	Adoptions []string
+	// Problems are the domains v0.16 cannot represent at all.
+	Problems []TenancyProblem
+}
+
+// Analyze predicts whether this layout converts cleanly, applying exactly
+// the rule the server enforces and the same repair applyplan performs.
+func (l *TenantLayout) Analyze() TenancyPlan {
+	var plan TenancyPlan
+	if len(l.Tenants) == 0 {
+		return plan
+	}
+
+	// domain -> tenant -> an example principal requiring it
+	required := map[string]map[string]string{}
+	for _, p := range l.Principals {
+		if p.Tenant == "" {
+			continue // a global principal constrains nothing
+		}
+		for _, d := range p.Domains {
+			if required[d] == nil {
+				required[d] = map[string]string{}
+			}
+			if _, ok := required[d][p.Tenant]; !ok {
+				required[d][p.Tenant] = p.Name
+			}
+		}
+	}
+
+	domains := make([]string, 0, len(required))
+	for d := range required {
+		domains = append(domains, d)
+	}
+	sort.Strings(domains)
+
+	for _, d := range domains {
+		wanted := required[d]
+		names := make([]string, 0, len(wanted))
+		for t := range wanted {
+			names = append(names, t)
+		}
+		sort.Strings(names)
+
+		if len(names) > 1 {
+			var parts []string
+			for _, t := range names {
+				parts = append(parts, fmt.Sprintf("%s (e.g. %s)", t, wanted[t]))
+			}
+			plan.Problems = append(plan.Problems, TenancyProblem{
+				Domain: d,
+				Detail: fmt.Sprintf("used by accounts from more than one tenant: %s - v0.16 allows a domain "+
+					"to belong to at most one tenant, and requires each tenant-scoped account to sit on its "+
+					"own tenant's domain", strings.Join(parts, ", ")),
+			})
+			continue
+		}
+
+		want := names[0]
+		switch declared, isDeclared := l.DomainTenant[d]; {
+		case !isDeclared || declared == "":
+			// Either inferred from an address, or declared with no tenant.
+			// Either way the conversion gives it no tenant and the account
+			// is rejected - this is the case that broke production.
+			plan.Adoptions = append(plan.Adoptions, d)
+		case declared != want:
+			plan.Problems = append(plan.Problems, TenancyProblem{
+				Domain: d,
+				Detail: fmt.Sprintf("belongs to tenant %s, but %s (tenant %s) uses it - v0.16 rejects an "+
+					"account whose tenant differs from its domain's", declared, wanted[want], want),
+			})
+		}
+	}
+	return plan
+}

internal/stalwartapi/tenantlayout_test.go

@@ -0,0 +1,138 @@
+// SPDX-FileCopyrightText: 2026 LINUXexpert-org
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package stalwartapi
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+)
+
+func TestFlexStringAcceptsEveryShapeAV015InstanceReturns(t *testing.T) {
+	cases := map[string]string{
+		`"acme"`:              "acme",
+		`{"string":"acme"}`:   "acme",
+		`{"name":"acme"}`:     "acme",
+		`["acme","other"]`:    "acme",
+		`null`:                "",
+		`{}`:                  "",
+		`[]`:                  "",
+		`{"other":"ignored"}`: "",
+	}
+	for raw, want := range cases {
+		if got := flexString(json.RawMessage(raw)); got != want {
+			t.Errorf("flexString(%s) = %q, want %q", raw, got, want)
+		}
+	}
+	if got := flexString(nil); got != "" {
+		t.Errorf("flexString(nil) = %q", got)
+	}
+}
+
+func TestDomainsOfCollectsNameAndAliasDomains(t *testing.T) {
+	got := domainsOf("[email protected]", []string{"[email protected]", "[email protected]", "malformed", "trailing@"})
+	want := []string{"alias.net", "example.com"}
+	if strings.Join(got, ",") != strings.Join(want, ",") {
+		t.Errorf("domainsOf = %v, want %v", got, want)
+	}
+}
+
+func TestAnalyzeIsEmptyForSingleTenant(t *testing.T) {
+	l := &TenantLayout{DomainTenant: map[string]string{}}
+	plan := l.Analyze()
+	if len(plan.Adoptions) != 0 || len(plan.Problems) != 0 {
+		t.Errorf("single-tenant layout produced %+v", plan)
+	}
+}
+
+func TestAnalyzeFlagsUndeclaredDomainForAdoption(t *testing.T) {
+	// The production failure: a tenant account with an address on a domain
+	// that was never declared, so the conversion gives it no tenant.
+	l := &TenantLayout{
+		Tenants:      []string{"acme"},
+		DomainTenant: map[string]string{"acme-corp.test": "acme"},
+		Principals: []PrincipalTenancy{
+			{Name: "bob", Tenant: "acme", Domains: []string{"acme-corp.test", "inferred.test"}},
+		},
+	}
+	plan := l.Analyze()
+	if len(plan.Problems) != 0 {
+		t.Fatalf("unexpected problems: %+v", plan.Problems)
+	}
+	if len(plan.Adoptions) != 1 || plan.Adoptions[0] != "inferred.test" {
+		t.Errorf("adoptions = %v, want [inferred.test]", plan.Adoptions)
+	}
+}
+
+func TestAnalyzeIgnoresGlobalAccountsOnTenantDomains(t *testing.T) {
+	// Verified against 0.16.14: this direction applies cleanly, so it must
+	// not be reported as anything.
+	l := &TenantLayout{
+		Tenants:      []string{"acme"},
+		DomainTenant: map[string]string{"acme-corp.test": "acme"},
+		Principals: []PrincipalTenancy{
+			{Name: "admin", Tenant: "", Domains: []string{"acme-corp.test"}},
+		},
+	}
+	plan := l.Analyze()
+	if len(plan.Adoptions) != 0 || len(plan.Problems) != 0 {
+		t.Errorf("global account on a tenant domain produced %+v", plan)
+	}
+}
+
+func TestAnalyzeReportsDomainSharedByTwoTenants(t *testing.T) {
+	l := &TenantLayout{
+		Tenants:      []string{"alpha", "beta"},
+		DomainTenant: map[string]string{},
+		Principals: []PrincipalTenancy{
+			{Name: "u1", Tenant: "alpha", Domains: []string{"shared.test"}},
+			{Name: "u2", Tenant: "beta", Domains: []string{"shared.test"}},
+		},
+	}
+	plan := l.Analyze()
+	if len(plan.Problems) != 1 {
+		t.Fatalf("problems = %+v, want one", plan.Problems)
+	}
+	if len(plan.Adoptions) != 0 {
+		t.Errorf("a conflicting domain was also queued for adoption: %v", plan.Adoptions)
+	}
+	for _, want := range []string{"alpha", "beta", "u1", "u2"} {
+		if !strings.Contains(plan.Problems[0].Detail, want) {
+			t.Errorf("detail %q omits %q", plan.Problems[0].Detail, want)
+		}
+	}
+}
+
+func TestAnalyzeReportsAccountOnAnotherTenantsDomain(t *testing.T) {
+	l := &TenantLayout{
+		Tenants:      []string{"alpha", "beta"},
+		DomainTenant: map[string]string{"owned.test": "alpha"},
+		Principals: []PrincipalTenancy{
+			{Name: "u2", Tenant: "beta", Domains: []string{"owned.test"}},
+		},
+	}
+	plan := l.Analyze()
+	if len(plan.Problems) != 1 || plan.Problems[0].Domain != "owned.test" {
+		t.Fatalf("problems = %+v", plan.Problems)
+	}
+}
+
+func TestAnalyzeAcceptsAConsistentMultiTenantLayout(t *testing.T) {
+	l := &TenantLayout{
+		Tenants: []string{"alpha", "beta"},
+		DomainTenant: map[string]string{
+			"alpha.test": "alpha",
+			"beta.test":  "beta",
+		},
+		Principals: []PrincipalTenancy{
+			{Name: "u1", Tenant: "alpha", Domains: []string{"alpha.test"}},
+			{Name: "u2", Tenant: "beta", Domains: []string{"beta.test"}},
+			{Name: "admin", Tenant: "", Domains: []string{"alpha.test"}},
+		},
+	}
+	plan := l.Analyze()
+	if len(plan.Adoptions) != 0 || len(plan.Problems) != 0 {
+		t.Errorf("a already-consistent multi-tenant layout produced %+v", plan)
+	}
+}